import time
import requests
import pandas as pd
import json
import ast
import datetime
import calendar
from dateutil.relativedelta import relativedelta, FR
from bs4 import BeautifulSoup


class Database:

    def __init__(self,authToken):
        self.authToken = authToken

    def get_hk_stock_ohlc(self, code, start_date, end_date, freq, price_adj=False, vol_adj=False):

        check_bool_dict = self.check_hk_stock_ohlc_args(code, start_date, end_date, freq)

        if False not in list(check_bool_dict.values()):
            link_url = 'http://www.hkfdb.net/data_api?'
            token_str = 'token=' + str(self.authToken)
            database_str = 'database=' + 'hk_stock_ohlc'
            code_str = 'code=' + code
            start_date_str = 'start_date=' + str(start_date)
            end_date_str = 'end_date=' + str(end_date)
            freq_str = 'freq=' + freq
            price_adj_str = 'price_adj=0' if price_adj == False else 'price_adj=1'
            vol_adj_str = 'vol_adj=0' if vol_adj == False else 'vol_adj=1'
            link_str = link_url + code_str + '&' + token_str + '&' + database_str + '&' + start_date_str + '&' + \
                       end_date_str + '&' + freq_str + '&' + price_adj_str + '&' + vol_adj_str

            response = requests.get(link_str)
            response_ok = response_check(response)
            if response_ok == True:

                if price_adj == False:
                    json1 = json.loads(response.content).replace("'", "\"").replace('False','0').replace('True','1').replace('nan','NaN')
                    ohlc_result = json.loads(json1)
                else:
                    json1 = json.loads(response.content).replace("'", "\"").replace('[(', '\"[(').replace(')]',')]\"').replace('False','0').replace('True','1').replace('nan','NaN')

                    result = json.loads(json1)
                    ohlc_result = result[0]

                    qfq_result = result[1].replace('[','').replace(']','').replace(' ','')
                    qfq_result = ast.literal_eval(qfq_result)
                    qfq_df = pd.DataFrame(qfq_result, columns=['date', 'qfq_factor'])

                df_list = []

                for item in ohlc_result:
                    date_int = item['_id']
                    df = pd.DataFrame(item['content'])

                    if len(df) > 0:
                        df['date'] = str(date_int)
                        df_list.append(df)

                df = pd.concat(df_list)

                if 'T' in freq:
                    df['time'] = df['time'].astype(str)
                    df['datetime'] = df['date'] + ' ' + df['time']
                    df['datetime'] = pd.to_datetime(df['datetime'], format='%Y%m%d %H%M%S')
                    cols = ['datetime', 'date', 'time', 'open', 'high', 'low', 'close', 'volume']

                elif freq == '1D':
                    df['datetime'] = df['date']
                    df['datetime'] = pd.to_datetime(df['datetime'], format='%Y%m%d')
                    cols = ['datetime', 'date', 'open', 'high', 'low', 'close', 'volume']

                elif freq == '1DW':
                    df['datetime'] = df['date']
                    df['datetime'] = pd.to_datetime(df['datetime'], format='%Y%m%d')
                    cols = ['datetime','date','open','high','low','close','volume','susp','auc_close',
                            'adj_close','gross_volume','turnover','VWAP']
                    df['susp'] = df['susp'].astype(bool)

                df = df[cols]
                df = df.set_index(keys='datetime')
                df = df.sort_index()

                if price_adj == True:

                    qfq_df = qfq_df[qfq_df['date'] > 20100000]

                    qfq_df = qfq_df.iloc[::-1].reset_index(drop=True)
                    qfq_df['date'] = qfq_df['date'].astype(str)
                    first_date = qfq_df.loc[qfq_df.index[0], 'date']
                    min_date_of_ohlc_df = df['date'].min()
                    if int(first_date) > int(min_date_of_ohlc_df):
                        first_date = min_date_of_ohlc_df

                    last_date = str(max([int(qfq_df.loc[qfq_df.index[-1], 'date']), int(df.loc[df.index[-1], 'date'])]))

                    if '-' in qfq_df.at[0, 'date']:
                        qfq_df['date'] = pd.to_datetime(qfq_df['date'], format='%Y-%m-%d')
                    else:
                        qfq_df['date'] = pd.to_datetime(qfq_df['date'], format='%Y%m%d')

                    qfq_df = qfq_df.set_index(keys='date')
                    t_index = pd.DatetimeIndex(pd.date_range(start=first_date, end=last_date, freq='1D'))

                    qfq_df = qfq_df.reindex(t_index)
                    qfq_df = qfq_df.fillna(method='bfill')
                    qfq_df = qfq_df.fillna(method='ffill')
                    qfq_df = qfq_df.reset_index()
                    qfq_df['index'] = qfq_df['index'].astype(str)
                    qfq_df['index'] = qfq_df['index'].str.replace('-', '', regex=False)
                    df = pd.merge(left=df, left_on='date', right=qfq_df, right_on='index')

                    if df['qfq_factor'].isnull().values.any():
                        df['qfq_factor'] = df['qfq_factor'].fillna(method='ffill')

                    if 'T' in freq:
                        df['datetime'] = df['date'] + ' ' + df['time']
                        df['datetime'] = pd.to_datetime(df['datetime'], format='%Y%m%d %H%M%S')
                    else:
                        df['datetime'] = df['date']
                        df['datetime'] = pd.to_datetime(df['datetime'], format='%Y%m%d')

                    df = df.set_index(keys='datetime')

                    if vol_adj == True:
                        if freq == '1DW':
                            col_list = ['open', 'high', 'low', 'close', 'volume','gross_volume']
                        else:
                            col_list = ['open', 'high', 'low', 'close', 'volume']
                    else:
                        col_list = ['open', 'high', 'low', 'close']

                    for col in col_list:
                        df[col] = df[col] * df['qfq_factor']
                        if 'volume' not in col:
                            df[col] = df[col].map(lambda x: round(x, 3))
                        else:
                            df[col] = df[col].map(lambda x: round(x, 0))
                            df[col] = df[col].astype(int)

                    if 'T' in freq:
                        df = df[['date', 'time', 'open', 'high', 'low', 'close', 'volume']]
                    else:
                        if freq == '1DW':
                            df = df[['date','open','high','low','close','volume','susp',
                                     'auc_close','gross_volume','turnover','VWAP']]
                        elif freq == '1D':
                            df = df[['date','open','high','low','close','volume']]

                df['date'] = df['date'].astype(int)
                if 'T' in freq:
                    df['time'] = df['time'].astype(int)
                else:
                    df['time'] = 0

                df = df[['date', 'time', 'open', 'high', 'low', 'close', 'volume']]

                return df

    def get_us_stock_ohlc(self, code, start_date, end_date):

        check_bool_dict = self.check_us_stock_ohlc_args(code, start_date, end_date)

        if False not in list(check_bool_dict.values()):
            link_url = 'http://www.hkfdb.net/data_api?'
            token_str = 'token=' + str(self.authToken)
            database_str = 'database=' + 'spx_stock_ohlc'
            code_str = 'code=' + code
            start_date_str = 'start_date=' + str(start_date)
            end_date_str = 'end_date=' + str(end_date)
            link_str = link_url + code_str + '&' + token_str + '&' + database_str + '&' + start_date_str + '&' + \
                       end_date_str

            response = requests.get(link_str)
            response_ok = response_check(response)
            if response_ok == True:

                result = json.loads(json.loads(response.content).replace("'", "\""))

                df_list = []
                for item in result:
                    date_int = item['_id']
                    df = pd.DataFrame(item['content'])
                    if len(df) > 0:
                        cols = ['datetime', 'date'] + list(df.columns)
                        df['time'] = df['time'].astype(str)
                        df['date'] = str(date_int)
                        df['datetime'] = df['date'] + ' ' + df['time'].str.zfill(6)
                        df['datetime'] = pd.to_datetime(df['datetime'], format='%Y%m%d %H%M%S')

                        df = df[cols]
                        df = df.set_index(keys='datetime')

                        df_list.append(df)

                df = pd.concat(df_list)
                df = df.sort_index()

                return df

        else:
            err_msg = 'Error in: '
            for error in check_bool_dict:
                if check_bool_dict[error] == False:
                    err_msg += error + ','
            print(err_msg)


    ## Futures
    expiry_dict = {}
    rolling_day = 0
    rolling_time = 0

    def apply_rolling_day(self, row):
        rolling_day = self.rolling_day
        rolling_time = self.rolling_time

        if row.current_month_expiry_date_diff < rolling_day:  # less than rolling_day, keep next month
            if not row.current_month:  # next month, keep it
                return True
            else:
                return False

        elif row.current_month_expiry_date_diff > rolling_day:  # more than rolling_dat, keep current
            if row.current_month:  # current month, keep it
                return True
            else:
                return False

        else:  # current_month_expiry_date_diff == rolling_day, need to consider time
            if row.time >= rolling_time:  # need to keep next month
                if not row.current_month:  # next month, keep it
                    return True
                else:
                    return False

            else:  # need to keep current month
                if row.current_month:  # next month, keep it
                    return True
                else:
                    return False

    def apply_rolling_1D(self, row):
        rolling_day = self.rolling_day

        if row.current_month_expiry_date_diff <= rolling_day:  # less than rolling_day, keep next month
            if not row.current_month:  # next month, keep it
                return True
            else:
                return False

        elif row.current_month_expiry_date_diff > rolling_day:  # more than rolling_dat, keep current
            if row.current_month:  # current month, keep it
                return True
            else:
                return False


    def apply_RTH_ATH(self, row):
        if (row.time > 90000 and row.time < 163000):
            return True
        else:
            return False


    def apply_roll_diff(self, row):
        rolling_day = self.rolling_day
        rolling_time = self.rolling_time
        roll_diff = self.roll_diff
        if row.current_month_expiry_date_diff == rolling_day and row.time == rolling_time:
            return roll_diff
        else:
            return 0

    def get_hk_futures_ohlc(self, index, start_date, end_date, freq, rolling_day, rolling_time=0, rth_only=False):

        check_bool_dict = self.check_hk_fut_ohlc_args(index, freq, start_date, end_date, rolling_day, rolling_time)

        self.rolling_day = rolling_day
        self.rolling_time = rolling_time


        if False not in list(check_bool_dict.values()):

            link_url = 'http://www.hkfdb.net/data_api?'
            token = str(self.authToken)
            start_date_str = str(start_date)
            end_date_str = str(end_date)
            link_str = f'{link_url}token={token}&database=hk_futures_ohlc&index={index}&freq={freq}&start_date={start_date_str}&end_date={end_date_str}'
            # print(link_str)

            response = requests.get(link_str)
            response_ok = response_check(response)


            if response_ok == True:

                result_list = json.loads(json.loads(response.content).replace("'", "\"").replace("True", "true").replace("False", "false") )

                columns = ['datetime', 'open', 'high', 'low', 'close', 'volume',
                           'trade_date_mask', 'expiry_date', 'current_month',
                           'current_month_expiry_date', 'current_month_expiry_date_diff']


                if freq == '1D':
                    dfs = []
                    for item in list(reversed(result_list)):
                        df_single = pd.DataFrame(item['content'], columns=columns)
                        df_single['datetime'] = pd.to_datetime(df_single['datetime'])
                        df_single['date'] = pd.to_datetime(df_single['datetime']).dt.strftime('%Y%m%d').astype(int)
                        df_single['time'] = pd.to_datetime(df_single['datetime']).dt.strftime('%H%M%S').astype(int)
                        df_single['RTH'] = df_single.apply(self.apply_RTH_ATH, axis=1)

                        if rth_only:
                            df_single = df_single[df_single['RTH'] == True]

                        df_all = []

                        for current_month in [True, False]:
                            df_current = df_single[df_single['current_month'] == current_month].sort_values(
                                by=['datetime']).reset_index(drop=True)
                            df = df_current.head(1).copy()

                            df['open'] = df_current.at[0, 'open']
                            df['high'] = df_current['high'].max()
                            df['low'] = df_current['low'].min()
                            df['close'] = df_current.at[len(df) - 1, 'close']
                            df['volume'] = df_current['volume'].sum()

                            df_all.append(df.copy())

                        dfs.append(pd.concat(df_all))

                    df = pd.concat(dfs)
                    df['datetime'] = pd.to_datetime(df['datetime'])

                    ## Find the valid expiry date to processed , ie within date range
                    expiry_dates = list(df['expiry_date'].unique())
                    expiry_dates.sort()

                    # # Calculate roll diff
                    roll_diff_dict = {}

                    for expiry_date in expiry_dates:
                        try:
                            exact_rolling_date = df[(df['expiry_date'] == expiry_date) &
                                                    (df['current_month_expiry_date_diff'] == rolling_day) &
                                                    (df['current_month'] == True)].reset_index(drop=True).sort_values(
                                by=['datetime'], ascending=False).at[0, 'date']

                            df_roll = df[(df['date'] == exact_rolling_date)].copy()
                            df_roll = df_roll.sort_values(by=['datetime']).reset_index(drop=True)

                            roll_diff_dict[expiry_date] = {}

                            roll_diff = df_roll.at[0, 'close'] - df_roll.at[1, 'close']
                            roll_diff_dict[expiry_date]['rolling_day'] = exact_rolling_date
                            roll_diff_dict[expiry_date]['roll_diff'] = roll_diff
                        except:
                            pass

                    df = df.sort_values(by=['datetime']).reset_index(drop=True)

                    df['roll_diff'] = 0

                    for expiry_date in roll_diff_dict.keys():
                        d = roll_diff_dict[expiry_date]['rolling_day']
                        diff = roll_diff_dict[expiry_date]['roll_diff']

                        condition = (df['date'] == d) & (df['current_month'] == False)
                        indices = df[condition].index[0]
                        df.at[indices, 'roll_diff'] = diff

                    df['keep'] = df.apply(self.apply_rolling_1D, axis=1)
                    df = df[df['keep'] == True]
                    selected_columns = ['datetime', 'date', 'time', 'open', 'high', 'low', 'close', 'volume',
                                        'expiry_date', 'roll_diff']
                    df = df[selected_columns]
                    df = df.sort_values(by=['datetime']).set_index('datetime', drop=True)

                    df.index = df.index.strftime('%Y-%m-%d')
                    df['time'] = 0
                    df['open'] = df['open'].astype(int)
                    df['high'] = df['high'].astype(int)
                    df['low'] = df['low'].astype(int)
                    df['close'] = df['close'].astype(int)

                    return df


                else:  # 1T, 5T, 15T
                    df_list = []
                    for item in result_list:
                        date_int = item['_id']
                        df = pd.DataFrame(item['content'], columns=columns)
                        if len(df) > 0:
                            df['date'] = str(date_int)
                            df_list.append(df)
                    df = pd.concat(df_list, )

                    df['datetime'] = pd.to_datetime(df['datetime'])

                    df = df.copy()
                    df['date'] = pd.to_datetime(df['datetime']).dt.strftime('%Y%m%d').astype(int)
                    df['time'] = pd.to_datetime(df['datetime']).dt.strftime('%H%M%S').astype(int)
                    condition = df['time'] == 120000  # remove 120000 time
                    df = df[~condition]
                    df['RTH'] = df.apply(self.apply_RTH_ATH, axis=1)  # Determine RTH and ATH

                    ## Find the valid expiry date to processed , ie within date range
                    expiry_dates = list(df['expiry_date'].unique())
                    expiry_dates.sort()
                    # print(expiry_dates)

                    # Calculate roll diff
                    roll_diff_dict = {}

                    for expiry_date in expiry_dates:

                        try:
                            exact_rolling_date = df[(df['expiry_date'] == expiry_date) &
                                                    (df['current_month_expiry_date_diff'] == rolling_day) &
                                                    (df['current_month'] == True)].sort_values(by=['datetime'],
                                                                                               ascending=False).reset_index(
                                                                            drop=True).reset_index(drop=True).at[0, 'date']
                                # Sorting Reverse because get later date to make sure it is RTH trading date



                            df_roll = df[(df['date'] == exact_rolling_date)].copy()
                            df_roll = df_roll[df_roll['time'] == rolling_time].sort_values(by=['expiry_date']).reset_index(drop=True)

                            roll_diff_dict[expiry_date] = {}


                            if len(df_roll) == 2:
                                roll_diff = df_roll.at[0, 'close'] - df_roll.at[1, 'close']
                                roll_diff_dict[expiry_date]['rolling_day'] = exact_rolling_date
                                roll_diff_dict[expiry_date]['roll_diff'] = roll_diff
                                roll_diff_dict[expiry_date]['rolling_time'] = rolling_time

                            else:
                                # Cannot find rolling time, assume roll at RTH last trade
                                df_roll = df[(df['date'] == exact_rolling_date) & (df['RTH'] == True)].copy()
                                df_roll = df_roll.sort_values(by=['time', 'expiry_date']).reset_index(drop=True)
                                df_roll = df_roll.tail(2).sort_values(by=['expiry_date']).reset_index(drop=True)
                                roll_diff = df_roll.at[0, 'close'] - df_roll.at[1, 'close']
                                roll_diff_dict[expiry_date]['rolling_day'] = exact_rolling_date
                                roll_diff_dict[expiry_date]['roll_diff'] = roll_diff
                                roll_diff_dict[expiry_date]['rolling_time'] = df_roll.at[0, 'time']

                        except:
                            pass


                    df = df.reset_index(drop=True)
                    df['roll_diff'] = 0


                    for expiry_date in roll_diff_dict.keys():
                        d = roll_diff_dict[expiry_date]['rolling_day']
                        t = roll_diff_dict[expiry_date]['rolling_time']
                        diff = roll_diff_dict[expiry_date]['roll_diff']

                        condition = (df['date'] == d) & (df['time'] == t) & (df['current_month'] == False)
                        indices = df[condition].index[0]
                        df.at[indices, 'roll_diff'] = diff

                    ## Exclude ATH if needed
                    if rth_only:
                        df = df[df['RTH'] == True]

                    ## Calculate rolling day and determine keep or not
                    df['keep'] = df.apply(self.apply_rolling_day, axis=1)
                    df = df[df['keep'] == True]
                    selected_columns = ['datetime', 'date', 'time', 'open', 'high', 'low', 'close', 'volume',
                                        'expiry_date', 'RTH', 'roll_diff']
                    df = df[selected_columns]
                    df = df.sort_values(by=['datetime']).set_index('datetime', drop=True)

                    df['time'] = df['time'].astype(str).str.zfill(6)
                    df['open'] = df['open'].astype(int)
                    df['high'] = df['high'].astype(int)
                    df['low'] = df['low'].astype(int)
                    df['close'] = df['close'].astype(int)


                    return df


        else:
            err_msg = 'Error in: '
            for error in check_bool_dict:
                if check_bool_dict[error] == False:
                    err_msg += error + ','
            print(err_msg)


    def get_hk_fut_ohlc(self, index, start_date, end_date, freq, rolling_day, rolling_time=0, rth_only=False):

        check_bool_dict = self.check_hk_fut_ohlc_args(index, freq, start_date, end_date, rolling_day, rolling_time)

        if False not in list(check_bool_dict.values()):

            if freq == '1D':
                rolling_time = 0
                if rth_only == True:
                    freq = freq + '_rth'
                elif rth_only == False:
                    freq = freq + '_all'

            start_date_dt = datetime.datetime.strptime(str(start_date), '%Y%m%d').date()
            end_date_dt = datetime.datetime.strptime(str(end_date), '%Y%m%d').date()

            if start_date_dt.month > 1:
                start_date_year = start_date_dt.year
            else:
                start_date_year = start_date_dt.year - 1
                start_date_dt = start_date_dt - relativedelta(months=1)

            end_date_year = end_date_dt.year if end_date_dt.month < 11 else end_date_dt.year + 1

            holiday_dict = get_hk_holiday_and_expiry_date(start_date_year, end_date_year, 'datetime')
            expiry_date_list = holiday_dict['expiry_date']

            expiry_date_dict = {}
            for expiry_date in expiry_date_list:
                expiry_year_month_str = str(expiry_date.year - 2000) + str(expiry_date.month).zfill(2)
                expiry_date_dict[expiry_year_month_str] = expiry_date

            day_diff = (end_date_dt - start_date_dt).days

            year_month_involved_list = []
            for i in range(day_diff + 1):
                i_date = start_date_dt + datetime.timedelta(days=i)
                year_month = i_date.strftime('%y%m')
                expiry_date = expiry_date_dict[year_month]
                if i_date > expiry_date:
                    year_month = (i_date + relativedelta(months=1)).strftime('%y%m')
                if year_month not in year_month_involved_list:
                    year_month_involved_list.append(year_month)

            year_month_involved_list.append(
                (datetime.datetime.strptime(year_month_involved_list[-1], '%y%m') + relativedelta(months=1)).strftime(
                    '%y%m'))
            year_month_involved_list_str = json.dumps(str(year_month_involved_list))

            year_month_involved_list_str = year_month_involved_list_str.replace('[', '')
            year_month_involved_list_str = year_month_involved_list_str.replace(']', '')
            year_month_involved_list_str = year_month_involved_list_str.replace('\'', '')
            year_month_involved_list_str = year_month_involved_list_str.replace(',', '')
            year_month_involved_list_str = year_month_involved_list_str.replace(' ', '_')
            year_month_involved_list_str = year_month_involved_list_str.replace('\"', '')

            link_url = 'http://www.hkfdb.net/data_api?'
            token_str = 'token=' + str(self.authToken)
            database_str = 'database=' + 'hk_fut_ohlc'
            index_str = 'index=' + index
            freq_str = 'freq=' + freq
            start_date_str = 'start_date=' + str(start_date)
            end_date_str = 'end_date=' + str(end_date)
            rolling_day_str = 'rolling_day=' + str(rolling_day)
            rolling_time_str = 'rolling_time=' + str(rolling_time)
            year_month_involved_list_str = 'year_month_involved_list=' + year_month_involved_list_str
            link_str = link_url + index_str + '&' + freq_str + '&' + rolling_day_str + '&' + rolling_time_str + \
                       '&' + year_month_involved_list_str + '&' \
                       + token_str + '&' + database_str + '&' \
                       + start_date_str + '&' + end_date_str

            response = requests.get(link_str)
            response_ok = response_check(response)
            if response_ok == True:

                result_list = json.loads(json.loads(response.content).replace("'", "\""))

                df_list = []
                date_time_list_list = []
                date_time_intersect_list = []
                for result in result_list:
                    sub_df_list = []
                    for item in result:
                        df = pd.DataFrame(item['content'])
                        if len(df) > 0:
                            sub_df_list.append(df)

                    if len(sub_df_list) > 0:
                        df = pd.concat(sub_df_list)
                        df = df.reset_index(drop=True)
                        temp_date_list = list(df['date'].unique())
                        temp_date_list.sort()

                        date_list = []
                        for temp_date in temp_date_list:
                            if datetime.datetime.strptime(str(temp_date), '%Y%m%d').weekday() < 5:
                                date_list.append(temp_date)

                        front_year_month = str(date_list[0])[2:6]
                        back_year_month = str(date_list[-1])[2:6]
                        front_expiry_date = int(expiry_date_dict[front_year_month].strftime('%Y%m%d'))
                        back_expiry_date = int(expiry_date_dict[back_year_month].strftime('%Y%m%d'))
                        for i in range(len(date_list)):
                            date_item = date_list[i]

                            if i + rolling_day <= len(date_list) - 1:
                                if date_list[i + rolling_day] == front_expiry_date:
                                    front_cut_off_date = date_item
                                elif date_list[i + rolling_day] == back_expiry_date:
                                    back_cut_off_date = date_item
                                    break
                            else:
                                back_cut_off_date = max(date_list)

                        if df.loc[0, 'expiry_date'] == '20130530':
                            front_cut_off_date = 20130501

                        if (back_cut_off_date != end_date) or (
                                back_cut_off_date == end_date and back_cut_off_date not in expiry_date_list):
                            if '1D' not in freq:
                                df_front = df[(df['date'] == front_cut_off_date) & (df['time'] == rolling_time)]
                                df_back  = df[(df['date'] == back_cut_off_date) & (df['time'] == rolling_time)]

                                df = df[(df['date'] > front_cut_off_date) | (
                                            (df['date'] == front_cut_off_date) & (df['time'] > rolling_time))]
                                df = df[(df['date'] < back_cut_off_date) | (
                                            (df['date'] == back_cut_off_date) & (df['time'] < rolling_time))]

                                df = pd.concat([df, df_front, df_back])

                            else:
                                df = df[(df['date'] > front_cut_off_date) | (df['date'] == front_cut_off_date)]
                                df = df[(df['date'] < back_cut_off_date) | (df['date'] == back_cut_off_date)]

                        df = df[df['date'] >= start_date]
                        df = df[df['date'] <= end_date]
                        df['date'] = df['date'].astype(str)
                        if '1D' not in freq:
                            df['time'] = df['time'].astype(str)
                            df['time'] = df['time'].str.zfill(6)
                            df['datetime'] = df['date'] + ' ' + df['time']
                        else:
                            df['datetime'] = df['date']

                        df = df.reset_index(drop=True)
                        date_time_list = df['datetime'].to_list()

                        df = df.sort_values(by='datetime')

                        if '1D' not in freq:
                            # if len(date_time_list_list) > 0:
                            #     date_time_intersect = (set(date_time_list_list).intersection(date_time_list))
                            #     if len(date_time_intersect) == 0:
                            #         df_list.append(df)
                            # else:
                            #     df_list.append(df)
                            df_list.append(df)

                        else:
                            date_time_intersect = (set(date_time_list_list).intersection(date_time_list))
                            date_time_intersect = list(date_time_intersect)
                            if len(date_time_intersect) > 0:
                                date_time_intersect_list.append(date_time_intersect[0])
                            df_list.append(df)

                        date_time_list_list += date_time_list

                df = pd.concat(df_list)

                if '1D' not in freq:
                    cols = ['datetime', 'date', 'time', 'open', 'high', 'low', 'close', 'volume', 'expiry_date', 'RTH']

                else:
                    cols = ['datetime', 'date', 'time', 'open', 'high', 'low', 'close', 'volume', 'expiry_date']
                    df['time'] = 0

                if '1D' not in freq:
                    df['datetime'] = pd.to_datetime(df['datetime'], format='%Y%m%d %H%M%S')
                    df['RTH'] = df['RTH'].astype(bool)
                else:
                    df['datetime'] = pd.to_datetime(df['datetime'], format='%Y%m%d')

                df = df[cols]
                df['roll_diff'] = 0

                if '1D' in freq:
                    drop_index_list = []
                    df = df.reset_index()
                    for date_time_intersect in date_time_intersect_list:
                        check_drop_index_row = df[df['date'] == date_time_intersect]
                        if len(check_drop_index_row) > 1:
                            index_0 = check_drop_index_row.index[0]
                            index_1 = check_drop_index_row.index[1]
                            expiry_date_0 = df.loc[index_0, 'expiry_date']
                            expiry_date_1 = df.loc[index_1, 'expiry_date']
                            close_price_0 = df.loc[index_0, 'close']
                            close_price_1 = df.loc[index_1, 'close']
                            if expiry_date_0 > expiry_date_1:
                                drop_index_list.append(index_1)
                                roll_diff = close_price_0 - close_price_1
                                df.at[index_0,'roll_diff'] = roll_diff
                            elif expiry_date_0 < expiry_date_1:
                                drop_index_list.append(index_0)
                                roll_diff = close_price_1 - close_price_0
                                roll_diff = close_price_0 - close_price_1
                                df.at[index_1,'roll_diff'] = roll_diff
                    if len(drop_index_list) > 0:
                        df = df.drop(drop_index_list, axis=0)
                    df = df.drop('index', axis=1)

                else:
                    df = df.reset_index()

                    duplicates = df[df.duplicated(subset=['datetime'], keep=False)]
                    min_expiry_row_index_list = []

                    for dt in duplicates['datetime'].unique():
                        df2 = duplicates[duplicates['datetime'] == dt]
                        min_expiry = df2['expiry_date'].min()
                        max_expiry = df2['expiry_date'].max()
                        min_expiry_row = df2.loc[df2['expiry_date'] == min_expiry]
                        max_expiry_row = df2.loc[df2['expiry_date'] == max_expiry]
                        min_expiry_row_index = min_expiry_row.index[0]
                        max_expiry_row_index  = max_expiry_row.index[0]
                        min_expiry_close = min_expiry_row['close'].values[0]
                        max_expiry_close = max_expiry_row['close'].values[0]

                        roll_diff = min_expiry_close - max_expiry_close

                        min_expiry_row_index_list.append(min_expiry_row_index)
                        df.at[max_expiry_row_index, 'roll_diff'] = roll_diff

                    if len(min_expiry_row_index_list) > 0:
                        df = df.drop(min_expiry_row_index_list, axis=0)
                    df = df.drop('index', axis=1)

                df = df.set_index(keys='datetime')
                df = df.sort_index(ascending=True)

                if '1D' not in freq and rth_only == True:
                    df = df[df['RTH'] == True]

                #df = df.drop_duplicates(keep='first')

                return df

        else:
            err_msg = 'Error in: '
            for error in check_bool_dict:
                if check_bool_dict[error] == False:
                    err_msg += error + ','
            print(err_msg)

    def get_hk_market_cap_hist_by_date(self, start_date, end_date):

        check_bool_dict = self.check_start_end_date(start_date, end_date)

        if False not in list(check_bool_dict.values()):
            link_url = 'http://www.hkfdb.net/data_api?'
            token_str = 'token=' + str(self.authToken)
            database_str = 'database=' + 'hk_market_cap_hist_by_date'
            start_date_str = 'start_date=' + str(start_date)
            end_date_str = 'end_date=' + str(end_date)
            link_str = link_url + '&' + token_str + '&' + database_str + '&' + start_date_str + '&' + end_date_str

            response = requests.get(link_str)
            response_ok = response_check(response)
            if response_ok == True:

                json_content = json.loads(response.content)
                json_content = json_content.replace(' nan', '\" nan\"')

                result = json.loads(json_content.replace("'", "\""))

                df_list = []
                for item in result:
                    date_int = item['_id']
                    df = pd.DataFrame(item['content'])
                    if len(df) > 0:
                        df['date'] = str(date_int)
                        df_list.append(df)
                df = pd.concat(df_list)

                df = df.sort_values(['date', 'market_cap_mil'], ascending=[True, False])
                df = df.reset_index(drop=True)

                if '-' in df.at[0, 'date']:
                    df['date'] = pd.to_datetime(df['date'], format='%Y-%m-%d')
                else:
                    df['date'] = pd.to_datetime(df['date'], format='%Y%m%d')

                df = df[['date','code','issued_share_mil','market_cap_mil','cumulative_market_cap_mil']]
                return df

        else:
            err_msg = 'Error in: '
            for error in check_bool_dict:
                if check_bool_dict[error] == False:
                    err_msg += error + ','
            print(err_msg)

    def get_hk_market_cap_hist_by_code(self, code):
        check_bool_dict = self.check_hk_code_args(code)

        if False not in list(check_bool_dict.values()):
            link_url = 'http://www.hkfdb.net/data_api?'
            token_str = 'token=' + str(self.authToken)
            database_str = 'database=' + 'hk_market_cap_hist_by_code'
            code_str = 'code=' + str(code)
            link_str = link_url + '&' + token_str + '&' + database_str + '&' + code_str

            response = requests.get(link_str)
            response_ok = response_check(response)
            if response_ok == True:

                json_content = json.loads(response.content)
                json_content = json_content.replace(' nan', '\" nan\"')
                result = json.loads(json_content.replace("'", "\""))

                df_list = []
                for item in result:
                    date_int = item['_id']
                    df = pd.DataFrame(item['content'])
                    if len(df) > 0:
                        df['date'] = str(date_int)
                        df_list.append(df)
                df = pd.concat(df_list)

                df = df.sort_values(by='date')
                df = df.reset_index(drop=True)
                if '-' in df.at[0, 'date']:
                    df['date'] = pd.to_datetime(df['date'], format='%Y-%m-%d')
                else:
                    df['date'] = pd.to_datetime(df['date'], format='%Y%m%d')
                df = df[['date', 'code', 'issued_share_mil', 'market_cap_mil', 'cumulative_market_cap_mil']]
                df = df.set_index('date')
                df = df[~df.index.duplicated(keep='first')]

                df2 = df[df.index >= '2023-02-20']
                df = df[df.index < '2023-02-20']

                index_end_date = datetime.datetime.now() + datetime.timedelta(days=1)
                index_end_year = index_end_date.year
                date_index = pd.date_range(start='2023-02-20', end=index_end_date.strftime('%Y-%m-%d'), freq='1D')
                df2 = df2.reindex(date_index)
                df2 = df2.shift(-1)
                df = pd.concat([df, df2])
                df = df.dropna()

                # holiday_list = get_hk_holiday_and_expiry_date(2023, index_end_year, format='dt')['public_holiday']
                # df = df[~df.index.isin(holiday_list)]

                return df

        else:
            err_msg = 'Error in: '
            for error in check_bool_dict:
                if check_bool_dict[error] == False:
                    err_msg += error + ','
            print(err_msg)

    def get_hk_buyback_by_code(self, code, agg_value = False):

        link_url = 'http://www.hkfdb.net/data_api?'
        token_str = 'token=' + str(self.authToken)
        database_str = 'database=' + 'hk_buyback_by_code'
        code_str = 'code=' + code
        link_str = link_url + code_str + '&' + token_str + '&' + database_str

        response = requests.get(link_str)
        response_ok = response_check(response)
        if response_ok == True:

            result = json.loads(json.loads(response.content).replace("'", "\""))

            df = pd.DataFrame(result)
            df['date'] = pd.to_datetime(df['date'], format = '%Y%m%d')
            df = df.set_index(keys='date')
            df = df.sort_index()

            if agg_value == True:
                df = df[['value']]
                df = df.groupby(df.index).agg({'value': sum})

            return df

    def get_hk_buyback_by_date(self, start_date, end_date):

        check_bool_dict = self.check_start_end_date(start_date, end_date)

        if False not in list(check_bool_dict.values()):

            link_url = 'http://www.hkfdb.net/data_api?'
            token_str = 'token=' + str(self.authToken)
            database_str = 'database=' + 'hk_buyback_by_date'
            start_date_str = 'start_date=' + str(start_date)
            end_date_str = 'end_date=' + str(end_date)
            link_str = link_url + '&' + token_str + '&' + database_str + '&' + start_date_str + '&' + end_date_str

            response = requests.get(link_str)
            response_ok = response_check(response)
            if response_ok == True:

                json_content = json.loads(response.content)
                json_content = json_content.replace(' nan', '\" nan\"')

                result = ast.literal_eval(json_content)

                df_list = []
                for item in result:
                    date_int = item['_id']
                    df = pd.DataFrame(item['content'])
                    if len(df) > 0:
                        df['date'] = str(date_int)
                        df_list.append(df)
                df = pd.concat(df_list)


                df['date'] = pd.to_datetime(df['date'], format='%Y%m%d')
                df = df.set_index(keys='date')
                df = df.sort_index()

                return df
        else:
            err_msg = 'Error in: '
            for error in check_bool_dict:
                if check_bool_dict[error] == False:
                    err_msg += error + ','
            print(err_msg)

    def get_hk_shortsell_by_code(self, code):

        link_url = 'http://www.hkfdb.net/data_api?'
        token_str = 'token=' + str(self.authToken)
        database_str = 'database=' + 'hk_shortsell_by_code'
        code_str = 'code=' + code
        link_str = link_url + code_str + '&' + token_str + '&' + database_str

        response = requests.get(link_str)
        response_ok = response_check(response)
        if response_ok == True:
            result = json.loads(json.loads(response.content).replace("'", "\""))

            df = pd.DataFrame(result)
            df['date'] = pd.to_datetime(df['date'], format = '%Y%m%d')
            df = df.set_index(keys='date')
            df = df.sort_index()

            df['ss_ratio'] = df['ss_ratio'].astype(float)
            df['ss_shares'] = df['ss_shares'].astype(float)
            df['ss_dollars'] = df['ss_dollars'].astype(float)

            return df

    def get_hk_shortsell_by_date(self, start_date, end_date):

        check_bool_dict = self.check_start_end_date(start_date, end_date)

        if False not in list(check_bool_dict.values()):

            link_url = 'http://www.hkfdb.net/data_api?'
            token_str = 'token=' + str(self.authToken)
            database_str = 'database=' + 'hk_shortsell_by_date'
            start_date_str = 'start_date=' + str(start_date)
            end_date_str = 'end_date=' + str(end_date)
            link_str = link_url + '&' + token_str + '&' + database_str + '&' + start_date_str + '&' + end_date_str

            response = requests.get(link_str)
            response_ok = response_check(response)
            if response_ok == True:

                json_content = json.loads(response.content)
                json_content = json_content.replace(' nan', '\" nan\"')

                result = ast.literal_eval(json_content)

                df_list = []
                for item in result:
                    date_int = item['_id']
                    df = pd.DataFrame(item['content'])
                    if len(df) > 0:
                        df['date'] = str(date_int)
                        df_list.append(df)
                df = pd.concat(df_list)

                df['date'] = pd.to_datetime(df['date'], format='%Y%m%d')
                df = df.set_index(keys='date')
                df = df.sort_index()

                return df
        else:
            err_msg = 'Error in: '
            for error in check_bool_dict:
                if check_bool_dict[error] == False:
                    err_msg += error + ','
            print(err_msg)

    def get_hk_earning_calendar_by_code(self, code):

        link_url = 'http://www.hkfdb.net/data_api?'
        token_str = 'token=' + str(self.authToken)
        database_str = 'database=' + 'hk_earning_calendar_by_code'
        code_str = 'code=' + code
        link_str = link_url + code_str + '&' + token_str + '&' + database_str

        response = requests.get(link_str)
        response_ok = response_check(response)
        if response_ok == True:

            result = json.loads(json.loads(response.content).replace("'", "\""))

            df = pd.DataFrame(result)
            df['datetime'] = pd.to_datetime(df['datetime'], format = '%Y-%m-%d %H:%M:%S')
            df = df.set_index(keys='datetime')
            df = df.sort_index()

            return df

    def get_hk_earning_calendar_by_date(self, start_date, end_date):

        check_bool_dict = self.check_start_end_date(start_date, end_date)

        if False not in list(check_bool_dict.values()):
            link_url = 'http://www.hkfdb.net/data_api?'
            token_str = 'token=' + str(self.authToken)
            database_str = 'database=' + 'hk_earning_calendar_by_date'
            start_date_str = 'start_date=' + str(start_date)
            end_date_str = 'end_date=' + str(end_date)
            link_str = link_url + '&' + token_str + '&' + database_str + '&' + start_date_str + '&' + end_date_str

            response = requests.get(link_str)
            response_ok = response_check(response)
            if response_ok == True:

                result = ast.literal_eval(json.loads(response.content))

                df_list = []
                for item in result:
                    date_int = item['_id']
                    df = pd.DataFrame(item['content'])
                    if len(df) > 0:
                        df['date'] = str(date_int)
                        df_list.append(df)
                df = pd.concat(df_list)

                df['datetime'] = pd.to_datetime(df['datetime'], format='%Y%m%d%H%M%S')

                df = df.set_index(keys='datetime')
                df = df[['code','name','result']]
                df = df.sort_index()
                return df

        else:
            err_msg = 'Error in: '
            for error in check_bool_dict:
                if check_bool_dict[error] == False:
                    err_msg += error + ','
            print(err_msg)

    def get_us_earning_calendar_by_code(self, code):

        link_url = 'http://www.hkfdb.net/data_api?'
        token_str = 'token=' + str(self.authToken)
        database_str = 'database=' + 'us_earning_calendar_by_code'
        code_str = 'code=' + code
        link_str = link_url + code_str + '&' + token_str + '&' + database_str

        response = requests.get(link_str)
        response_ok = response_check(response)
        if response_ok == True:

            result = ast.literal_eval(json.loads(response.content))

            temp_list = list(result)
            date_list = []
            for date in temp_list:
                date_list.append(datetime.datetime.strptime(date,'%Y-%m-%d').date())

            return date_list

    def get_us_earning_calendar_by_date(self, start_date, end_date):

        check_bool_dict = self.check_start_end_date(start_date, end_date)

        if False not in list(check_bool_dict.values()):
            link_url = 'http://www.hkfdb.net/data_api?'
            token_str = 'token=' + str(self.authToken)
            database_str = 'database=' + 'us_earning_calendar_by_date'
            start_date_str = 'start_date=' + str(start_date)
            end_date_str = 'end_date=' + str(end_date)
            link_str = link_url + '&' + token_str + '&' + database_str + '&' + start_date_str + '&' + end_date_str

            response = requests.get(link_str)
            response_ok = response_check(response)
            if response_ok == True:

                result = json.loads(json.loads(response.content).replace("'", "\""))

                df_list = []
                for item in result:
                    date_int = item['_id']
                    df = pd.DataFrame(item['content'])
                    if len(df) > 0:
                        df['date'] = str(date_int)
                        df_list.append(df)
                df = pd.concat(df_list)
                df = df[['date', 'code']]

                df['date'] = pd.to_datetime(df['date'], format='%Y%m%d')

                df = df.set_index(keys='date')
                df = df.sort_index()
                return df

        else:
            err_msg = 'Error in: '
            for error in check_bool_dict:
                if check_bool_dict[error] == False:
                    err_msg += error + ','
            print(err_msg)

    def get_ccass_by_code(self, code, start_date, end_date):

        check_bool_dict = self.check_ccass_by_code_args(code, start_date, end_date)

        if False not in list(check_bool_dict.values()):
            link_url = 'http://www.hkfdb.net/data_api?'
            token_str = 'token=' + str(self.authToken)
            database_str = 'database=' + 'ccass_by_code'
            code_str = 'code=' + code
            start_date_str = 'start_date=' + str(start_date)
            end_date_str = 'end_date=' + str(end_date)
            link_str = link_url + code_str + '&' + token_str + '&' + database_str + '&' + start_date_str + '&' + end_date_str

            response = requests.get(link_str)
            response_ok = response_check(response)
            if response_ok == True:

                result = json.loads(json.loads(response.content).replace("'", "\""))

                df_list = []
                for item in result:
                    date_int = item['_id']
                    df = pd.DataFrame(item['content'])
                    df['date'] = str(date_int)
                    df_list.append(df)

                df = pd.concat(df_list)
                df['date'] = pd.to_datetime(df['date'], format='%Y%m%d')
                df = df.set_index('date')
                df = df.sort_index()

                for col in df.columns:
                    if 'participants_participant_' in col:
                        df = df.rename(columns={col:col.replace('participants_participant_','participants_')})

                df['total_non_ccass_participants_shares'] = df['total_issued_shares'] - df['total_ccass_participants_shares']

                return df

        else:
            err_msg = 'Error in: '
            for error in check_bool_dict:
                if check_bool_dict[error] == False:
                    err_msg += error + ','
            print(err_msg)

    def get_ccass_holding_rank(self, code, start_date, end_date):

        check_bool_dict = self.check_ccass_holding_rank_args(code, start_date, end_date)

        if False not in list(check_bool_dict.values()):
            link_url = 'http://www.hkfdb.net/data_api?'
            token_str = 'token=' + str(self.authToken)
            database_str = 'database=' + 'ccass_holding_rank'
            code_str = 'code=' + code
            start_date_str = 'start_date=' + str(start_date)
            end_date_str = 'end_date=' + str(end_date)
            link_str = link_url + code_str + '&' + token_str + '&' + database_str + '&' + start_date_str + '&' + end_date_str

            response = requests.get(link_str)
            response_ok = response_check(response)
            if response_ok == True:

                result = json.loads(json.loads(response.content).replace("'", "\""))

                df_list = []
                for item in result:
                    date_int = item['_id']
                    df = pd.DataFrame(item['content'])
                    if len(df) > 0:
                        df['date'] = date_int
                        df_list.append(df)

                df = pd.concat(df_list)

                df = df.sort_values(['date', 'share'], ascending=False)
                df = df[['date', 'ccass_id', 'name', 'share', 'percent']]
                df = df.reset_index(drop=True)
                df['date'] = pd.to_datetime(df['date'],format ='%Y%m%d')

                return df

        else:
            err_msg = 'Error in: '
            for error in check_bool_dict:
                if check_bool_dict[error] == False:
                    err_msg += error + ','
            print(err_msg)

    def get_ccass_all_id(self):

        link_url = 'http://www.hkfdb.net/data_api?'
        token_str = 'token=' + str(self.authToken)
        database_str = 'database=' + 'ccass_all_id'
        link_str = link_url + '&' + token_str + '&' + database_str

        response = requests.get(link_str)
        response_ok = response_check(response)
        if response_ok == True:

            result = ast.literal_eval(json.loads(response.content))

            df = pd.DataFrame(result)

            return df

    def get_ccass_by_id(self, ccass_id, start_date, end_date):

        check_bool_dict = self.check_ccass_by_id_args(ccass_id, start_date, end_date)

        if False not in list(check_bool_dict.values()):
            link_url = 'http://www.hkfdb.net/data_api?'
            token_str = 'token=' + str(self.authToken)
            database_str = 'database=' + 'ccass_by_id'
            ccass_id_str = 'ccass_id=' + ccass_id
            start_date_str = 'start_date=' + str(start_date)
            end_date_str = 'end_date=' + str(end_date)
            link_str = link_url + ccass_id_str + '&' + token_str + '&' + database_str + '&' + start_date_str + '&' + \
                       end_date_str

            response = requests.get(link_str)
            response_ok = response_check(response)
            if response_ok == True:

                result = json.loads(json.loads(response.content).replace("'", "\""))

                df_list = []
                for item in result:
                    date_int = item['_id']
                    df = pd.DataFrame(item['content'])
                    if len(df) > 0:
                        df['date'] = str(date_int)
                        df_list.append(df)

                df = pd.concat(df_list)

                df['date'] = pd.to_datetime(df['date'], format='%Y%m%d')

                df = df[['date', 'code', 'percent', 'share']]
                df = df.set_index(keys='date')
                df = df.sort_index()

                return df

        else:
            err_msg = 'Error in: '
            for error in check_bool_dict:
                if check_bool_dict[error] == False:
                    err_msg += error + ','
            print(err_msg)

    def get_ccass_by_id_change(self, ccass_id, start_date, end_date):

        check_bool_dict = self.check_ccass_by_id_args(ccass_id, start_date, end_date)

        if False not in list(check_bool_dict.values()):
            start_date_dt = datetime.datetime.strptime(str(start_date), '%Y%m%d')
            end_date_dt = datetime.datetime.strptime(str(end_date), '%Y%m%d')
            if start_date_dt.weekday() > 4:
                start_date_dt = start_date_dt + relativedelta(weekday=FR(-1))
                start_date = int(start_date_dt.strftime('%Y%m%d'))
            if end_date_dt.weekday() > 4:
                end_date_dt = end_date_dt + relativedelta(weekday=FR(-1))
                end_date = int(end_date_dt.strftime('%Y%m%d'))

            link_url = 'http://www.hkfdb.net/data_api?'
            token_str = 'token=' + str(self.authToken)
            database_str = 'database=' + 'ccass_by_id'
            ccass_id_str = 'ccass_id=' + ccass_id
            start_date_str = 'start_date=' + str(start_date)
            end_date_str = 'end_date=' + str(start_date)
            link_str = link_url + ccass_id_str + '&' + token_str + '&' + database_str + '&' + start_date_str + '&' + \
                       end_date_str

            response = requests.get(link_str)

            response_ok = response_check(response)
            if response_ok == True:

                result = json.loads(json.loads(response.content).replace("'", "\""))
                df_list = []
                for item in result:
                    date_int = item['_id']
                    df = pd.DataFrame(item['content'])
                    if len(df) > 0:
                        df['date'] = str(date_int)
                        df_list.append(df)

                df_first = pd.concat(df_list)

                time.sleep(1)

                start_date_str = 'start_date=' + str(end_date)
                end_date_str = 'end_date=' + str(end_date)
                link_str = link_url + ccass_id_str + '&' + token_str + '&' + database_str + '&' + start_date_str + '&' + \
                           end_date_str

                response = requests.get(link_str)
                response_ok = response_check(response)
                if response_ok == True:

                    result = json.loads(json.loads(response.content).replace("'", "\""))

                    df_list = []
                    for item in result:
                        date_int = item['_id']
                        df = pd.DataFrame(item['content'])
                        if len(df) > 0:
                            df['date'] = str(date_int)
                            df_list.append(df)

                    df_last = pd.concat(df_list)

                    df_first = df_first.set_index(keys='code')
                    df_last = df_last.set_index(keys='code')

                    for col in df_first:
                        df_first = df_first.rename(columns={col: col + '_first'})
                        df_last = df_last.rename(columns={col: col + '_last'})

                    df_first['date_first'] = pd.to_datetime(df_first['date_first'], format='%Y%m%d')
                    df_last['date_last'] = pd.to_datetime(df_last['date_last'], format='%Y%m%d')

                    df = pd.concat([df_first, df_last], axis=1)
                    df['percent_chg'] = df['percent_last'] - df['percent_first']
                    df['share_chg'] = df['share_last'] - df['share_first']
                    df['date_diff'] = df['date_last'] - df['date_first']
                    df = df.dropna()

                    for col in df.columns:
                        if 'share' in col:
                            df[col] = df[col].astype(int)
                    return df

        else:
            err_msg = 'Error in: '
            for error in check_bool_dict:
                if check_bool_dict[error] == False:
                    err_msg += error + ','
            print(err_msg)

    def get_spx_index_const(self, hist=True):

        link_url = 'http://www.hkfdb.net/data_api?'
        token_str = 'token=' + str(self.authToken)

        if hist == False:
            database_str = 'database=' + 'spx_index_const'
            link_str = link_url + '&' + token_str + '&' + database_str

            response = requests.get(link_str)
            response_ok = response_check(response)
            if response_ok == True:

                result = ast.literal_eval(json.loads(response.content))

                df = pd.DataFrame(result)

        else:
            database_str = 'database=' + 'spx_const_hist'
            link_str = link_url + '&' + token_str + '&' + database_str

            response = requests.get(link_str)
            result = ast.literal_eval(json.loads(response.content))

            df = pd.DataFrame(result)
            mode = df['end_date'].mode()[0]
            df['is_active'] = df['is_active'].astype(bool)
            df['is_delisted'] = df['is_delisted'].astype(bool)

            for i in range(len(df)):
                end_date = df.loc[i,'end_date']
                if end_date == mode:
                    df.at[i,'end_date'] = datetime.datetime.now().strftime('%Y-%m-%d')
            for col in df.columns:
                if col == 'start_date' or col == 'end_date':
                    if '-' in df.at[0, col]:
                        df[col] = pd.to_datetime(df[col], format='%Y-%m-%d')
                    else:
                        df[col] = pd.to_datetime(df[col], format='%Y%m%d')


        return df

    def get_hk_index_const(self, index_name, sort_mkt_cap = False):

        if len(index_name) > 0:

            link_url = 'http://www.hkfdb.net/data_api?'
            token_str = 'token=' + str(self.authToken)
            database_str = 'database=' + 'hk_index_const'
            index_name_str = 'index_name=' + index_name
            link_str = link_url + index_name_str + '&' + token_str + '&' + database_str

            response = requests.get(link_str)
            response_ok = response_check(response)

            if response_ok == True:

                result = ast.literal_eval(json.loads(response.content))

                df = pd.DataFrame(result)
                df['code'] = df['code'].str.zfill(5)

                for col in df.columns:
                    if col == 'start_date' or col == 'end_date':
                        if '-' in df.at[0, col]:
                            df[col] = pd.to_datetime(df[col], format='%Y-%m-%d')
                        else:
                            df[col] = pd.to_datetime(df[col], format='%Y%m%d')

                if index_name == 'hsi_const_hist':

                    df['is_alive'] = df['is_alive'].astype(bool)
                    df = df.rename(columns={'is_alive':'is_active'})

                    for i in range(len(df)):
                        end_date = df.loc[i, 'end_date']
                        is_active = df.loc[i, 'is_active']
                        if is_active == True:

                            df.at[i, 'end_date'] = datetime.datetime.now().strftime('%Y-%m-%d')

                    for col in df.columns:
                        if col == 'start_date' or col == 'end_date':
                            if '-' in df.at[0, col]:
                                df[col] = pd.to_datetime(df[col], format='%Y-%m-%d')
                            else:
                                df[col] = pd.to_datetime(df[col], format='%Y%m%d')

                    df = df[['code','name','start_date','end_date','is_active']]

                if sort_mkt_cap:
                    info = self.get_basic_hk_stock_info()
                    info = info[['code','market_capital']]
                    df = pd.merge(df, info, on='code', how='inner')
                    df = df.sort_values(by='market_capital', ascending=False)

                return df

        else:
            err_msg =  'index_name missing'
            print(err_msg)

    def get_hk_stock_plate_const(self, plate_name, sort_mkt_cap=False):
        if len(plate_name) > 0:

            link_url = 'http://www.hkfdb.net/data_api?'
            token_str = 'token=' + str(self.authToken)
            database_str = 'database=' + 'hk_stock_plate_const'
            plate_name_str = 'plate_name=' + plate_name
            link_str = link_url + plate_name_str + '&' + token_str + '&' + database_str

            response = requests.get(link_str)
            response_ok = response_check(response)
            if response_ok == True:

                result = json.loads(json.loads(response.content).replace("'", "\""))

                df = pd.DataFrame(result)
                df['code'] = df['code'].astype(str)
                df['code'] = df['code'].str.zfill(5)

                if sort_mkt_cap:
                    info = self.get_basic_hk_stock_info()
                    info = info[['code', 'market_capital']]
                    df = pd.merge(df, info, on='code', how='inner')
                    df = df.sort_values(by='market_capital', ascending=False)

                return df

        else:
            err_msg = 'index_name missing'
            print(err_msg)

    def get_all_hk_index_name(self):

        link_url = 'http://www.hkfdb.net/data_api?'
        token_str = 'token=' + str(self.authToken)
        database_str = 'database=' + 'all_hk_index_name'
        link_str = link_url + '&' + token_str + '&' + database_str

        response = requests.get(link_str)
        response_ok = response_check(response)
        if response_ok == True:

            result = ast.literal_eval(json.loads(response.content))

            name_list = list(result)
            if 'hk_full_market' in name_list:
                name_list.remove('hk_full_market')

            return name_list

    def get_all_hk_stock_plate_name(self):

        link_url = 'http://www.hkfdb.net/data_api?'
        token_str = 'token=' + str(self.authToken)
        database_str = 'database=' + 'all_hk_stock_plate_name'
        link_str = link_url + '&' + token_str + '&' + database_str

        response = requests.get(link_str)
        response_ok = response_check(response)

        if response_ok == True:

            result = ast.literal_eval(json.loads(response.content))

            name_list = list(result)

            return name_list

    def get_basic_hk_stock_info(self):

        link_url = 'http://www.hkfdb.net/data_api?'
        token_str = 'token=' + str(self.authToken)
        database_str = 'database=' + 'basic_hk_stock_info'
        link_str = link_url + '&' + token_str + '&' + database_str

        response = requests.get(link_str)
        response_ok = response_check(response)
        if response_ok == True:

            r_content = response.content
            json1 = json.loads(r_content).replace("'", "\"")
            json1 = json1.replace('nan','NaN')
            json1 = json1.replace('INT\"L','INTL')
            json1 = json1.replace('L\"OCCITANE','LOCCITANE')
            json1 = json1.replace('True','\"True\"')
            json1 = json1.replace('False','\"False\"')
            json1 = json1.replace('\"\"True\"\"','\"True\"')
            json1 = json1.replace('\"\"False\"\"','\"False\"')

            result = json.loads(json1)

            df = pd.DataFrame(result)

            if '-' in df.at[0, 'ipo_date']:
                df['ipo_date'] = pd.to_datetime(df['ipo_date'], format='%Y-%m-%d')
            else:
                df['ipo_date'] = pd.to_datetime(df['ipo_date'], format='%Y%m%d')

            df['share_issued'] = df['share_issued'].astype(str)
            df['share_issued'] = df['share_issued'].str.replace('-','', regex=False)
            df['share_issued'] = df['share_issued'].astype('int64')

            df['volume'] = df['volume'].str.replace('.0','', regex=False)
            df['turnover'] = df['turnover'].str.replace('.0','', regex=False)
            df['market_capital'] = df['market_capital'].str.replace('.0','', regex=False)

            df = df.sort_values(by='code')

            df['market_capital'] = df['market_capital'].astype('int64')
            df['is_stop_trading'] = df['is_stop_trading'].map({'True': True, 'False': False})
            df = df.rename(columns={'is_stop_trading':'stop_trading'})

            float_col_list = ['price','PE','EPS','dividend']
            int_col_list = ['volume','turnover']
            for col in float_col_list:
                df[col] = df[col].str.replace('-', '', regex=False)
                df[col] = df[col].str.replace('.000', '', regex=False)
                df[col] = df[col].str.replace(',', '', regex=False)
                df[col] = df[col].astype(float)

            for col in int_col_list:
                df[col] = df[col].astype('int64')
            df['lot_size'] = df['lot_size'].astype(int)

            df = df[['code','stock_name','price','PE','EPS','dividend','volume','turnover','share_issued',
                     'lot_size','market_capital','ipo_date','year_end_date', 'stop_trading','category']]
            return df

    def get_hk_ipo_hist(self):

        link_url = 'http://www.hkfdb.net/data_api?'
        token_str = 'token=' + str(self.authToken)
        database_str = 'database=' + 'hk_ipo_hist'
        link_str = link_url + '&' + token_str + '&' + database_str

        response = requests.get(link_str)
        response_ok = response_check(response)
        if response_ok == True:

            json_content = json.loads(response.content)
            json_content = json_content.replace(' nan', '\" nan\"')
            json_content = json_content.replace('N/A\\n\\n', '0')
            json_content = json_content.replace('N/A\\n', '0')
            json_content = json_content.replace('0\\n', '0')


            result = ast.literal_eval(json_content)

            df = pd.DataFrame(result)
            col_list = ['name','sponsors','accountants','valuers']

            for col in col_list:
                df[col] = df[col].str.replace('\n',' ', regex= False)
            for col in col_list:
                for i in range(len(df)):
                    content = df.loc[i,col]
                    if content[-1] == ' ':
                        df.at[i,col] = content[0:-1]
                    if 'Appraisaland' in content:
                        df.at[i,col] = content.replace('Appraisaland','Appraisal and')

            df['prospectus_date'] = pd.to_datetime(df['prospectus_date'], format='%d/%m/%Y')
            df['listing_date'] = pd.to_datetime(df['listing_date'], format='%d/%m/%Y')

            return df

    def get_market_highlight(self, start_date, end_date):

        market = 'hk_main_board'

        check_bool_dict = self.check_market_highlight_args(market, start_date, end_date)

        if False not in list(check_bool_dict.values()):
            link_url = 'http://www.hkfdb.net/data_api?'
            token_str = 'token=' + str(self.authToken)
            database_str = 'database=' + 'market_highlight'
            market_str = 'market=' + market
            start_date_str = 'start_date=' + str(start_date)
            end_date_str = 'end_date=' + str(end_date)
            link_str = link_url + market_str + '&' + token_str + '&' + database_str + '&' + start_date_str + '&' + \
                       end_date_str

            response = requests.get(link_str)
            response_ok = response_check(response)
            if response_ok == True:

                result = json.loads(json.loads(response.content).replace("'", "\""))

                df_list = []
                for item in result:
                    date_int = item['_id']
                    df = pd.DataFrame(item['content'])
                    if len(df) > 0:
                        df['date'] = str(date_int)
                        df_list.append(df)
                df = pd.concat(df_list)

                df['date'] = pd.to_datetime(df['date'], format='%Y%m%d')
                df = df.drop_duplicates(subset='date',keep='last')
                df = df.set_index(keys='date')
                df = df.sort_index()

                df = df.rename(columns={'average_pe_ratio_times':'average_pe_ratio'})

                return df

        else:
            err_msg = 'Error in: '
            for error in check_bool_dict:
                if check_bool_dict[error] == False:
                    err_msg += error + ','
            print(err_msg)

    def get_north_water(self, start_date, end_date):

        check_bool_dict = self.check_start_end_date(start_date, end_date)

        if False not in list(check_bool_dict.values()):
            link_url = 'http://www.hkfdb.net/data_api?'
            token_str = 'token=' + str(self.authToken)
            database_str = 'database=' + 'north_water'
            start_date_str = 'start_date=' + str(start_date)
            end_date_str = 'end_date=' + str(end_date)
            link_str = link_url + '&' + token_str + '&' + database_str + '&' + start_date_str + '&' + \
                       end_date_str

            response = requests.get(link_str)
            response_ok = response_check(response)
            if response_ok == True:

                json_content = json.loads(response.content)
                if 'nan' in json_content:
                    result = json.loads(json_content.replace("'", "\"").replace("nan", "0"))
                else:
                    result = json.loads(json_content.replace("'", "\""))

                df_list = []
                for item in result:
                    date_int = item['_id']
                    df = pd.DataFrame(item['content'])
                    if len(df) > 0:
                        df['date'] = str(date_int)
                        df_list.append(df)
                df = pd.concat(df_list)

                df['date'] = pd.to_datetime(df['date'], format='%Y%m%d')

                df = df.set_index(keys='date')
                df = df.sort_index()
                for col in df.columns:
                    df[col] = df[col].astype(int)

                return df

        else:
            err_msg = 'Error in: '
            for error in check_bool_dict:
                if check_bool_dict[error] == False:
                    err_msg += error + ','
            print(err_msg)

    def get_hk_deri_daily_market_report(self, deri, code ,start_date, end_date, exp = 'current'):

        check_bool_dict = self.check_hk_deri_daily_market_report_args(deri, code ,start_date, end_date, exp)

        if False not in list(check_bool_dict.values()):
            link_url = 'http://www.hkfdb.net/data_api?'
            token_str = 'token=' + str(self.authToken)
            deri_str = 'deri=' + deri
            code_str = 'code=' + code
            exp_str = 'exp=' + exp
            database_str = 'database=' + 'hk_deri_daily_market_report'
            start_date_str = 'start_date=' + str(start_date)
            end_date_str = 'end_date=' + str(end_date)
            link_str = link_url + '&' + token_str + '&' + database_str \
                       + '&' + start_date_str + '&' + end_date_str\
                       + '&' + deri_str  + '&' + code_str  + '&' + exp_str

            response = requests.get(link_str)
            response_ok = response_check(response)
            if response_ok == True:

                result = json.loads(json.loads(response.content).replace("'", "\""))

                df_list = []

                for item in result:
                    date_int = item['_id']
                    df = pd.DataFrame(item['content'])
                    if len(df) > 0:
                        df['date'] = str(date_int)
                        df_list.append(df)
                df = pd.concat(df_list)

                df['date'] = pd.to_datetime(df['date'], format='%Y%m%d')

                if deri == 'opt':
                    df = df.rename(columns={'contract_month':'year_month'})
                    df['year_month'] = df['year_month'].str.replace('-','', regex=False)

                    if 'wo' in code:
                        df['year_month'] = pd.to_datetime(df['year_month'], format='%d%b%y')
                        df = df.sort_values(by=['date','year_month'])
                        df['year_month'] = df['year_month'].dt.strftime('%d%b%y')
                    else:
                        df['year_month'] = pd.to_datetime(df['year_month'], format='%b%y')
                        df = df.sort_values(by=['date', 'year_month'])
                        df['year_month'] = df['year_month'].dt.strftime('%b-%y')

                    if code.isdigit() == True:
                        cols = list(df.columns)
                        cols = [cols[-1]] + cols[:-1]
                        df = df[cols]
                    df = df.reset_index(drop=True)

                elif deri == 'fut':
                    df = df.sort_values(by='date')
                    df = df.set_index('date')

                if 'strike_price' in df.columns:
                    df = df.rename(columns={'strike_price': 'strike'})
                if 'change_in_settlement' in df.columns:
                    df = df.rename(columns={'change_in_settlement': 'close_change'})
                if 'year_month' in df.columns:
                    df = df.rename(columns={'year_month': 'contract_month'})
                if 'volumne' in df.columns:
                    df = df.rename(columns={'volumne': 'volume'})

                return df

        else:
            err_msg = 'Error in: '
            for error in check_bool_dict:
                if check_bool_dict[error] == False:
                    err_msg += error + ','
            print(err_msg)

    def get_hk_cbbc(self, code, start_date, end_date):

        check_bool_dict = self.check_hk_cbbc(code, start_date, end_date)

        if False not in list(check_bool_dict.values()):
            link_url = 'http://www.hkfdb.net/data_api?'
            token_str = 'token=' + str(self.authToken)
            code_str = 'code=' + code
            database_str = 'database=' + 'hk_cbbc'
            start_date_str = 'start_date=' + str(start_date)
            end_date_str = 'end_date=' + str(end_date)
            link_str = link_url + '&' + token_str + '&' + database_str \
                       + '&' + start_date_str + '&' + end_date_str + '&' + code_str

            response = requests.get(link_str)
            response_ok = response_check(response)
            if response_ok == True:

                result_list = json.loads(json.loads(response.content).replace("'", "\""))

                columns = ['unique_id', 'cbbc_code', 'cbbc_name', 'date', 'num_of_cbbc_bought',
                           'avg_price_per_cbbc_bought',
                           'num_of_cbbc_sold', 'avg_price_per_cbbc_sold', 'num_of_cbbc_in_mkt', 'total_issued_size',
                           'currency', 'issuer', 'underlying', 'bull_bear', 'listing_date', 'strike_or_call_currency',
                           'strike_level', 'call_level', 'entitlement_ratio']

                df_list = []
                for result in result_list:
                    for item, content in result.items():
                        if type(content) is list:
                            df = pd.DataFrame(content, columns=columns)
                            df['date'] = pd.to_datetime(df['date'], format='%Y%m%d')
                            df['listing_date'] = pd.to_datetime(df['listing_date'], format='%Y%m%d')
                            df_list.append(df)

                df = pd.concat(df_list)
                df = df.sort_values('date')
                df['num_of_cbbc_sold'] = df['num_of_cbbc_sold'].abs()
                df['avg_price_per_cbbc_sold'] = df['avg_price_per_cbbc_sold'].abs()
                df['bull_bear'] = df['bull_bear'].str.replace(' ','')
                df = df.set_index('unique_id')
                # df = df.drop('index', axis=1)

                return df

        else:
            err_msg = 'Error in: '
            for error in check_bool_dict:
                if check_bool_dict[error] == False:
                    err_msg += error + ','
            print(err_msg)

    #########################################################################################

    def check_hk_stock_ohlc_args(self, code, start_date, end_date, freq):
        freq_list = ['1T', '5T', '15T', '30T', '1D','1DW']
        freq_valid = True if freq in freq_list else False

        try:
            code_length = len(code) == 5
        except:
            code_length = False
        try:
            code_isdigit = code.isdigit() == True
        except:
            code_isdigit = False
        try:
            start_date_is_int = isinstance(start_date, int)
        except:
            start_date_is_int = False
        try:
            start_date_length = len(str(start_date)) == 8
        except:
            start_date_length = False
        try:
            start_date_future = datetime.datetime.strptime(str(start_date),
                                                           '%Y%m%d').date() <= datetime.datetime.now().date()
        except:
            start_date_future = False
        try:
            end_date_is_int = isinstance(end_date, int)
        except:
            end_date_is_int = False
        try:
            end_date_length = len(str(end_date)) == 8
        except:
            end_date_length = False

        # try:
        #     end_date_future = datetime.datetime.strptime(str(end_date),
        #                                                  '%Y%m%d').date() <= datetime.datetime.now().date()
        # except:
        #     end_date_future = False

        try:
            end_after_start_date = datetime.datetime.strptime(str(end_date),'%Y%m%d').date() >= \
                                   datetime.datetime.strptime(str(start_date), '%Y%m%d').date()
        except:
            end_after_start_date = False

        check_bool_dict = {'freq_valid': freq_valid,
                           'code_isdigit': code_isdigit,
                           'code_length': code_length,
                           'start_date_length': start_date_length,
                           'start_date_is_int': start_date_is_int,
                           'start_date_future': start_date_future,
                           'end_date_is_int': end_date_is_int,
                           'end_date_length': end_date_length,
                           #'end_date_future': end_date_future,
                           'end_after_start_date': end_after_start_date}

        return check_bool_dict
    def check_us_stock_ohlc_args(self, code, start_date, end_date):

        try:
            code_length = len(code) > 0
        except:
            code_length = False

        try:
            start_date_is_int = isinstance(start_date, int)
        except:
            start_date_is_int = False
        try:
            start_date_length = len(str(start_date)) == 8
        except:
            start_date_length = False
        try:
            start_date_future = datetime.datetime.strptime(str(start_date),
                                                           '%Y%m%d').date() <= datetime.datetime.now().date()
        except:
            start_date_future = False
        try:
            end_date_is_int = isinstance(end_date, int)
        except:
            end_date_is_int = False
        try:
            end_date_length = len(str(end_date)) == 8
        except:
            end_date_length = False
        # try:
        #     end_date_future = datetime.datetime.strptime(str(end_date),
        #                                                  '%Y%m%d').date() <= datetime.datetime.now().date()
        # except:
        #     end_date_future = False
        try:
            end_after_start_date = datetime.datetime.strptime(str(end_date),
                                                              '%Y%m%d').date() >= datetime.datetime.strptime(
                str(start_date), '%Y%m%d').date()
        except:
            end_after_start_date = False


        check_bool_dict = {'code_length': code_length,
                           'start_date_length': start_date_length,
                           'start_date_is_int': start_date_is_int,
                           'start_date_future': start_date_future,
                           'end_date_is_int': end_date_is_int,
                           'end_date_length': end_date_length,
                           #'end_date_future': end_date_future,
                           'end_after_start_date': end_after_start_date}

        return check_bool_dict

    def check_hk_fut_ohlc_args(self, index, freq, start_date, end_date, rolling_day, rolling_time):

        freq_list = ['1T', '5T', '15T', '1D']
        freq_valid = True if freq in freq_list else False

        try:
            index_name = (index == 'HSI') or (index == 'HHI') or (index == 'HTI')
        except:
            index_name = False
        try:
            start_date_is_int = isinstance(start_date, int)
        except:
            start_date_is_int = False
        try:
            start_date_length = len(str(start_date)) == 8
        except:
            start_date_length = False
        try:
            start_date_future = datetime.datetime.strptime(str(start_date),
                                                           '%Y%m%d').date() <= datetime.datetime.now().date()
        except:
            start_date_future = False
        try:
            end_date_is_int = isinstance(end_date, int)
        except:
            end_date_is_int = False
        try:
            end_date_length = len(str(end_date)) == 8
        except:
            end_date_length = False
        # try:
        #     end_date_future = datetime.datetime.strptime(str(end_date),
        #                                                  '%Y%m%d').date() <= datetime.datetime.now().date()
        # except:
        #     end_date_future = False
        try:
            end_after_start_date = datetime.datetime.strptime(str(end_date),
                                                              '%Y%m%d').date() >= datetime.datetime.strptime(
                str(start_date), '%Y%m%d').date()
        except:
            end_after_start_date = False

        try:
            rolling_day_int = isinstance(rolling_day, int)
        except:
            rolling_day_int = False
        try:
            rolling_day_length = rolling_day <= 5
        except:
            rolling_day_length = False
        try:
            rolling_time_int = isinstance(rolling_time, int)
        except:
            rolling_time_int = False

        check_bool_dict = {'index_name': index_name,
                           'freq_valid': freq_valid,
                           'start_date_length': start_date_length,
                           'start_date_is_int': start_date_is_int,
                           'start_date_future': start_date_future,
                           'end_date_is_int': end_date_is_int,
                           'end_date_length': end_date_length,
                           #'end_date_future': end_date_future,
                           'end_after_start_date': end_after_start_date,
                           'rolling_day_int': rolling_day_int,
                           'rolling_day_length': rolling_day_length,
                           'rolling_time_int': rolling_time_int}

        return check_bool_dict

    def check_market_highlight_args(self, market, start_date, end_date):
        try:
            market_length = len(market) > 0
        except:
            market_length = False
        try:
            start_date_is_int = isinstance(start_date, int)
        except:
            start_date_is_int = False
        try:
            start_date_length = len(str(start_date)) == 8
        except:
            start_date_length = False
        try:
            start_date_future = datetime.datetime.strptime(str(start_date),
                                                           '%Y%m%d').date() <= datetime.datetime.now().date()
        except:
            start_date_future = False
        try:
            end_date_is_int = isinstance(end_date, int)
        except:
            end_date_is_int = False
        try:
            end_date_length = len(str(end_date)) == 8
        except:
            end_date_length = False
        # try:
        #     end_date_future = datetime.datetime.strptime(str(end_date),
        #                                                  '%Y%m%d').date() <= datetime.datetime.now().date()
        # except:
        #     end_date_future = False
        try:
            end_after_start_date = datetime.datetime.strptime(str(end_date),
                                                              '%Y%m%d').date() >= datetime.datetime.strptime(
                str(start_date), '%Y%m%d').date()
        except:
            end_after_start_date = False

        check_bool_dict = {'market_length': market_length,
                           'start_date_length': start_date_length,
                           'start_date_is_int': start_date_is_int,
                           'start_date_future': start_date_future,
                           'end_date_is_int': end_date_is_int,
                           'end_date_length': end_date_length,
                           #'end_date_future': end_date_future,
                           'end_after_start_date': end_after_start_date}
        return check_bool_dict

    def check_start_end_date(self, start_date, end_date):

        try:
            start_date_is_int = isinstance(start_date, int)
        except:
            start_date_is_int = False
        try:
            start_date_length = len(str(start_date)) == 8
        except:
            start_date_length = False
        try:
            start_date_future = datetime.datetime.strptime(str(start_date),'%Y%m%d').date() <= datetime.datetime.now().date()
        except:
            start_date_future = False
        try:
            end_date_is_int = isinstance(end_date, int)
        except:
            end_date_is_int = False
        try:
            end_date_length = len(str(end_date)) == 8
        except:
            end_date_length = False
        # try:
        #     end_date_future = datetime.datetime.strptime(str(end_date),
        #                                                  '%Y%m%d').date() <= datetime.datetime.now().date()
        # except:
        #     end_date_future = False
        try:
            end_after_start_date = datetime.datetime.strptime(str(end_date),
                                                              '%Y%m%d').date() >= datetime.datetime.strptime(
                str(start_date), '%Y%m%d').date()
        except:
            end_after_start_date = False

        check_bool_dict = {
            'start_date_length': start_date_length,
            'start_date_is_int': start_date_is_int,
            'start_date_future': start_date_future,
            'end_date_is_int': end_date_is_int,
            'end_date_length': end_date_length,
            #'end_date_future': end_date_future,
            'end_after_start_date': end_after_start_date}

        return check_bool_dict

    def check_ccass_by_id_args(self, ccass_id, start_date, end_date):
        if len(ccass_id) > 0:
            ccass_id_len = True
        else:
            ccass_id_len = False
        try:
            start_date_is_int = isinstance(start_date, int)
        except:
            start_date_is_int = False
        try:
            start_date_length = len(str(start_date)) == 8
        except:
            start_date_length = False
        try:
            start_date_future = datetime.datetime.strptime(str(start_date),
                                                           '%Y%m%d').date() <= datetime.datetime.now().date()
        except:
            start_date_future = False
        try:
            end_date_is_int = isinstance(end_date, int)
        except:
            end_date_is_int = False
        try:
            end_date_length = len(str(end_date)) == 8
        except:
            end_date_length = False
        # try:
        #     end_date_future = datetime.datetime.strptime(str(end_date), '%Y%m%d').date() <= datetime.datetime.now().date()
        # except:
        #     end_date_future = False
        try:
            end_after_start_date = datetime.datetime.strptime(str(end_date), '%Y%m%d').date() >= datetime.datetime.strptime(
                str(start_date), '%Y%m%d').date()
        except:
            end_after_start_date = False


        check_bool_dict = {'ccass_id_len' : ccass_id_len,
                           'start_date_length': start_date_length,
                           'start_date_is_int': start_date_is_int,
                           'start_date_future': start_date_future,
                           'end_date_is_int': end_date_is_int,
                           'end_date_length': end_date_length,
                           #'end_date_future': end_date_future,
                           'end_after_start_date': end_after_start_date}
        return check_bool_dict

    def check_ccass_by_code_args(self, code, start_date, end_date):

        try:
            code_length = len(code) == 5
        except:
            code_length = False
        try:
            code_isdigit = code.isdigit() == True
        except:
            code_isdigit = False
        try:
            start_date_is_int = isinstance(start_date, int)
        except:
            start_date_is_int = False
        try:
            start_date_length = len(str(start_date)) == 8
        except:
            start_date_length = False
        try:
            start_date_future = datetime.datetime.strptime(str(start_date),
                                                           '%Y%m%d').date() <= datetime.datetime.now().date()
        except:
            start_date_future = False
        try:
            end_date_is_int = isinstance(end_date, int)
        except:
            end_date_is_int = False
        try:
            end_date_length = len(str(end_date)) == 8
        except:
            end_date_length = False
        # try:
        #     end_date_future = datetime.datetime.strptime(str(end_date),
        #                                                  '%Y%m%d').date() <= datetime.datetime.now().date()
        # except:
        #     end_date_future = False
        try:
            end_after_start_date = datetime.datetime.strptime(str(end_date),
                                                              '%Y%m%d').date() >= datetime.datetime.strptime(
                str(start_date), '%Y%m%d').date()
        except:
            end_after_start_date = False


        check_bool_dict = {
            'code_isdigit': code_isdigit,
            'code_length': code_length,
            'start_date_length': start_date_length,
            'start_date_is_int': start_date_is_int,
            'start_date_future': start_date_future,
            'end_date_is_int': end_date_is_int,
            'end_date_length': end_date_length,
            #'end_date_future': end_date_future,
            'end_after_start_date': end_after_start_date}

        return check_bool_dict
    def check_ccass_holding_rank_args(self, code, start_date, end_date):

        try:
            code_length = len(code) == 5
        except:
            code_length = False
        try:
            code_isdigit = code.isdigit() == True
        except:
            code_isdigit = False
        try:
            start_date_is_int = isinstance(start_date, int)
        except:
            start_date_is_int = False
        try:
            start_date_length = len(str(start_date)) == 8
        except:
            start_date_length = False
        try:
            start_date_future = datetime.datetime.strptime(str(start_date),
                                                           '%Y%m%d').date() <= datetime.datetime.now().date()
        except:
            start_date_future = False

        try:
            end_date_is_int = isinstance(end_date, int)
        except:
            end_date_is_int = False
        try:
            end_date_length = len(str(end_date)) == 8
        except:
            end_date_length = False
        # try:
        #     end_date_future = datetime.datetime.strptime(str(end_date),
        #                                                  '%Y%m%d').date() <= datetime.datetime.now().date()
        # except:
        #     end_date_future = False
        try:
            end_after_start_date = datetime.datetime.strptime(str(end_date),
                                                              '%Y%m%d').date() >= datetime.datetime.strptime(
                str(start_date), '%Y%m%d').date()
        except:
            end_after_start_date = False


        check_bool_dict = {
            'code_isdigit': code_isdigit,
            'code_length': code_length,
            'start_date_length': start_date_length,
            'start_date_is_int': start_date_is_int,
            'start_date_future': start_date_future,
            'end_date_is_int': end_date_is_int,
            'end_date_length': end_date_length,
            #'end_date_future': end_date_future,
            'end_after_start_date': end_after_start_date}

        return check_bool_dict

    def check_hk_deri_daily_market_report_args(self, deri, code, start_date, end_date, exp):

        if deri == 'fut':
            deri_type = True
        elif deri == 'opt':
            deri_type = True
        else:
            deri_type = False

        if exp == 'current':
            exp_type = True
        elif exp == 'next':
            exp_type = True
        else:
            exp_type = False

        try:
            code_length = len(code) > 0
        except:
            code_length = False
        try:
            start_date_is_int = isinstance(start_date, int)
        except:
            start_date_is_int = False
        try:
            start_date_length = len(str(start_date)) == 8
        except:
            start_date_length = False
        try:
            start_date_future = datetime.datetime.strptime(str(start_date),
                                                           '%Y%m%d').date() <= datetime.datetime.now().date()
        except:
            start_date_future = False
        try:
            end_date_is_int = isinstance(end_date, int)
        except:
            end_date_is_int = False
        try:
            end_date_length = len(str(end_date)) == 8
        except:
            end_date_length = False
        # try:
        #     end_date_future = datetime.datetime.strptime(str(end_date),
        #                                                  '%Y%m%d').date() <= datetime.datetime.now().date()
        # except:
        #     end_date_future = False
        try:
            end_after_start_date = datetime.datetime.strptime(str(end_date),
                                                              '%Y%m%d').date() >= datetime.datetime.strptime(
                str(start_date), '%Y%m%d').date()
        except:
            end_after_start_date = False


        check_bool_dict = {'code_length': code_length,
                           'start_date_length': start_date_length,
                           'start_date_is_int': start_date_is_int,
                           'start_date_future': start_date_future,
                           'end_date_is_int': end_date_is_int,
                           'end_date_length': end_date_length,
                           #'end_date_future': end_date_future,
                           'end_after_start_date': end_after_start_date,
                           'deri_type': deri_type,
                           'exp_type': exp_type}

        return check_bool_dict

    def check_hk_cbbc(self, code, start_date, end_date):

        try:
            code_length = len(code) > 0
        except:
            code_length = False
        try:
            start_date_is_int = isinstance(start_date, int)
        except:
            start_date_is_int = False
        try:
            start_date_length = len(str(start_date)) == 8
        except:
            start_date_length = False
        try:
            end_date_is_int = isinstance(end_date, int)
        except:
            end_date_is_int = False
        try:
            end_date_length = len(str(end_date)) == 8
        except:
            end_date_length = False


        check_bool_dict = {'code_length': code_length,
                           'start_date_length': start_date_length,
                           'start_date_is_int': start_date_is_int,
                           'end_date_is_int': end_date_is_int,
                           'end_date_length': end_date_length,}

        return check_bool_dict


    def check_hk_code_args(self, code):

        try:
            code_length = len(code) > 0
        except:
            code_length = False
        try:
            code_isdigit = code.isdigit()
        except:
            code_isdigit = False

        check_bool_dict = {'code_length': code_length,
                           'code_isdigit': code_isdigit}

        return check_bool_dict


################################ other tools ################################ 

def get_holiday_from_gov(year):
    r = requests.get('https://www.gov.hk/en/about/abouthk/holiday/' + year + '.htm')
    soup = BeautifulSoup(r.content.decode('utf-8'), 'lxml')
    items = soup.find('table').find_all('tr')

    holiday_date_list = []
    half_day_mkt_date_list = []

    for item in items[1:]:
        tds = item.find_all('td')
        holiday_date = tds[1].text + ' ' + year
        holiday_date = datetime.datetime.strptime(holiday_date, '%d %B %Y').date()
        holiday_name = tds[0].text.lower()
        holiday_date_list.append(holiday_date)
        if 'lunar new year' in holiday_name and 'the' not in holiday_name:
            lin30_date = holiday_date - datetime.timedelta(days=1)
            half_day_mkt_date_list.append(lin30_date)

    xmax_eva_date = datetime.date(int(year), 12, 24)
    if xmax_eva_date.weekday() < 5:
        half_day_mkt_date_list.append(xmax_eva_date)

    xmax_eva_date = datetime.date(int(year), 12, 24)
    if xmax_eva_date.weekday() < 5:
        half_day_mkt_date_list.append(xmax_eva_date)
    year_eva_date = datetime.date(int(year), 12, 31)
    if year_eva_date.weekday() < 5:
        half_day_mkt_date_list.append(year_eva_date)

    return holiday_date_list, half_day_mkt_date_list

def get_hk_holiday_and_expiry_date(start_year, end_year, format='int', str_format = '%Y%m%d'):
    holiday_date_list = []
    half_day_mkt_date_list = []

    for i in range(end_year - start_year + 1):
        this_year = end_year - i
        this_year_str = str(this_year)
        new_holiday_date_list, new_half_day_mkt_date_list = get_holiday_from_gov(this_year_str)
        holiday_date_list = holiday_date_list + new_holiday_date_list
        half_day_mkt_date_list = half_day_mkt_date_list + new_half_day_mkt_date_list

    start_date = datetime.date(start_year, 1, 1)
    end_date = datetime.date(end_year, 12, 31)
    date_diff = (end_date - start_date).days

    expiry_date_list = []
    for i in range(date_diff):
        date = end_date - datetime.timedelta(days=i)
        last_date = date + datetime.timedelta(days=1)
        if last_date.day == 1:
            trading_days = 0
            for j in range(7):
                test_date = date - datetime.timedelta(days=j)
                if test_date.weekday() < 5 and test_date not in holiday_date_list:
                    trading_days += 1
                if trading_days == 2:
                    if format == 'str' or format == 'string':
                        expiry_date = test_date.strftime(str_format)
                    elif format == 'int' or format == 'integer':
                        expiry_date = int(test_date.strftime('%Y%m%d'))
                    elif format == 'dt' or format == 'datetime':
                        expiry_date = test_date
                    expiry_date_list.append(expiry_date)
                    break

    holiday_list = []
    for day in holiday_date_list:
        if format == 'str' or format == 'string':
            holiday = day.strftime(str_format)
        elif format == 'int' or format == 'integer':
            holiday = int(day.strftime('%Y%m%d'))
        elif format == 'dt' or format == 'datetime':
            holiday = day
        holiday_list.append(holiday)

    half_day_mkt_list = []
    for day in half_day_mkt_date_list:
        if format == 'str' or format == 'string':
            holiday = day.strftime(str_format)
        elif format == 'int' or format == 'integer':
            holiday = int(day.strftime('%Y%m%d'))
        elif format == 'dt' or format == 'datetime':
            holiday = day
        half_day_mkt_list.append(holiday)

    dict1 = {'expiry_date': expiry_date_list, 'public_holiday': holiday_list, 'half_day_mkt': half_day_mkt_list}
    return dict1


def get_current_us_holiday(format='int', str_format='%Y%m%d'):
    url = 'https://www.nasdaqtrader.com/Trader.aspx?id=Calendar'
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) \
        Chrome/86.0.4240.193 Safari/537.36'}
    r = requests.get(url, headers=headers)
    r.raise_for_status()
    r.encoding = r.apparent_encoding
    text = r.text

    soup = BeautifulSoup(text, 'lxml')

    ths = soup.find('table').find_all('th')
    cols = []
    for th in ths:
        text = th.text
        cols.append(text)

    data = []
    trs = soup.find('table').find_all('tr')
    for tr in trs:
        info = []
        tds = tr.find_all('td')
        for td in tds:
            text = td.text
            info.append(text)
        if len(info) > 0:
            data.append(info)
    df = pd.DataFrame(data)
    df.columns = ['date', 'holiday', 'status']
    df = df.loc[df['status'] == 'Closed']

    us_holiday_list = []
    date_list = df['date'].to_list()
    for holiday in date_list:
        if format == 'dt':
            us_holiday_list.append(datetime.datetime.strptime(holiday, '%B %d, %Y').date())
        elif format == 'str':
            us_holiday_list.append(datetime.datetime.strptime(holiday, '%B %d, %Y').strftime(str_format))
        elif format == 'int':
            us_holiday_list.append(int(datetime.datetime.strptime(holiday, '%B %d, %Y').strftime(str_format)))

    return us_holiday_list


def response_check(response):
    response_ok = True

    if response.status_code == 200:
        response_string = response.content.decode('utf-8')
        response_dict = json.loads(response_string)
        if 'Content' in response_dict:
            if response_dict['Content'] == 'Token Error':
                print('Token Error, please check for token or payment issues')
                response_ok = False
            if response_dict['State'] == 400:
                response_ok = False
    else:
        print('Connection Error, please check pc internet setting')
        response_ok = False

    return response_ok

def get_stock_tick_size(market, price):
    if market == 'HK':
        if price >= 0.01 and price < 0.25:
            tick_size = 0.001
        elif price >= 0.25 and price < 0.5:
            tick_size = 0.005
        elif price >= 0.5 and price < 10:
            tick_size = 0.01
        elif price >= 10 and price < 20:
            tick_size = 0.02
        elif price >= 20 and price < 100:
            tick_size = 0.05
        elif price >= 100 and price < 200:
            tick_size = 0.1
        elif price >= 200 and price < 500:
            tick_size = 0.2
        elif price >= 500 and price < 1000:
            tick_size = 0.5
        elif price >= 1000 and price < 2000:
            tick_size = 1
        elif price >= 2000 and price < 5000:
            tick_size = 2
        elif price >= 5000 and price < 9995:
            tick_size = 5

    elif market == 'US':
        if price < 1:
            tick_size = 0.0001
        else:
            tick_size = 0.001

    return tick_size