#!/usr/bin/env python
# coding: utf-8

import numpy as np
import pandas as pd
from typing import List, Set, Tuple, Dict, Optional
from operator import itemgetter
from collections import defaultdict
from .EMQuantAPI_Python.python3.EmQuantAPI import *
import datetime
import time
import json
import os
import traceback
import sys
from sqlalchemy import distinct
from sqlalchemy.orm import sessionmaker
from ...wrapper.mysql import RawDatabaseConnector
from ...view.raw_models import *
from ...api.basic import BasicDataApi
from ...api.raw import RawDataApi
from ....constant import SectorType, IndexPriceSource, CodeFeeMode, INDEX_PRICE_EXTRA_TRADE_DAYS
from .raw_data_helper import RawDataHelper, ALL_CSI_INDEX_INNER_NAME


class EmRawDataDownloader:

    _EM_RETRY_TIMES = 10
    _EM_RETRY_INTERVAL_S = 10
    _MACROECONOMIC_MAP = {
        'EMM00166216': 'RA0000.IR',
        'EMM00166217': 'RA0001.IR',
        'EMM00166218': 'RA0002.IR',
        'EMM00166219': 'RA0003.IR',
        'EMM00166220': 'RA0004.IR',
        'EMM00166221': 'RA0005.IR',
        'EMM00166222': 'RA0006.IR',
        'EMM00166228': 'RA0007.IR',
        'EMM00072301': 'CPI',
        'EMM00073348': 'PPI',
        'EMM00087084': 'M1',
        'EMM00087086': 'M2',
        'EMM00634721': 'AFtoRE',
        'EMM00008445': 'IVA',
        'EMM00121996': 'PMI',
        'EMM00122023': 'CX_PMI',
        'EMI00135328': 'SOC_RG',
        'EMM00588703': 'TB_1Y',
        'EMM00588705': 'TB_3Y',
        'EMM00588707': 'TB_5Y',
        'EMM00588712': 'TB_10Y',
        'EMM00166255': 'SHIBOR_1M',
        'EMM00166256': 'SHIBOR_3M',
        'E1002041': 'CB_AAA_1Y',
    }
    _MACROECONOMIC_MAP_I = {v: k for k, v in _MACROECONOMIC_MAP.items()}

    def __init__(self, data_helper):
        self._data_helper = data_helper
        # ForceLogin
        # 取值0，当线上已存在该账户时，不强制登录
        # 取值1，当线上已存在该账户时，强制登录，将前一位在线用户踢下线
        options = 'TestLatency=1,ForceLogin=1'
        loginResult = c.start(options, mainCallBack=self._main_callback)
        if loginResult.ErrorCode != 0:
            print('EM login failed')
            sys.exit()

        self._quarter_update_dates = ['0331', '0630', '0930', '1231']
        self._semi_update_dates = ['0630', '1231']
        self._index_block_list: Optional[Set[str]] = None

    def __del__(self):
        c.stop()

    def _main_callback(self, quantdata):
        '''
        mainCallback 是主回调函数，可捕捉连接错误
        该函数只有一个为c.EmQuantData类型的参数quantdata
        :param quantdata: c.EmQuantData
        '''
        print(f'MainCallback: {quantdata}')

    def _get_date_list(self, start_date, end_date):
        start_datetime = datetime.datetime.strptime(start_date, '%Y%m%d')
        end_datetime = datetime.datetime.strptime(end_date, '%Y%m%d')
        date_list = []
        while start_datetime <= end_datetime:
            date_list.append(start_datetime.strftime('%Y%m%d'))
            start_datetime += datetime.timedelta(days=1)
        return date_list

    # 使用em wrapper系列的函数可以在失败时自动重试
    def _em_tradedates_wrapper(self, start_date: str, end_date: str, options: str):
        for i in range(self._EM_RETRY_TIMES + 1):
            tradedates_result = c.tradedates(start_date, end_date, options)
            if tradedates_result.ErrorCode == 0:
                return tradedates_result.Data
            print(f'Failed to get tradedates({i}): {tradedates_result.ErrorMsg} (err_code){tradedates_result.ErrorCode}')
            time.sleep(self._EM_RETRY_INTERVAL_S)
        return

    def _em_sector_wrapper(self, sector_code: str, end_date: str):
        for i in range(self._EM_RETRY_TIMES + 1):
            ids_result = c.sector(sector_code, end_date)
            if ids_result.ErrorCode == 0:
                return ids_result.Codes
            print(f'Failed to get sector list({i}): {ids_result.ErrorMsg} (err_code){ids_result.ErrorCode}')
            time.sleep(self._EM_RETRY_INTERVAL_S)
        return

    def _em_css_wrapper(self, codes, indicators: str, options: str):
        if 'ispandas=1' not in options.lower():
            if options:
                options += ','
            options += 'Ispandas=1'
        for i in range(self._EM_RETRY_TIMES + 1):
            data = c.css(codes, indicators, options)
            if isinstance(data, pd.DataFrame):
                return data
            if data.ErrorCode != 0:
                print(f'failed to get css data({i}): {data.ErrorMsg} (err_code){data.ErrorCode}')
            else:
                print('weird, no error but not a df returned')
            time.sleep(self._EM_RETRY_INTERVAL_S)
        return

    def _em_csd_wrapper(self, codes, indicators: str, start_date: str, end_date: str, options: str):
        if 'ispandas=1' not in options.lower():
            if options:
                options += ','
            options += 'Ispandas=1'
        for i in range(self._EM_RETRY_TIMES + 1):
            data = c.csd(codes, indicators, start_date, end_date, options)
            if isinstance(data, pd.DataFrame):
                return data
            if data.ErrorCode != 0:
                print(f'failed to get csd data({i}): {data.ErrorMsg} (err_code){data.ErrorCode}')
            else:
                print('weird, no error but not a df returned')
            time.sleep(self._EM_RETRY_INTERVAL_S)
        return

    def _em_ctr_wrapper(self, ctr_name: str, indicators: str, options: str):
        for i in range(self._EM_RETRY_TIMES + 1):
            data = c.ctr(ctr_name, indicators, options)
            if data.ErrorCode == 0:
                return data.Data
            elif data.ErrorCode == 10000009:
                print(f'no data when get ctr data({i}), return an empty dict')
                return {}
            print(f'failed to get ctr data({i}): {data.ErrorMsg} (err_code){data.ErrorCode}')
            time.sleep(self._EM_RETRY_INTERVAL_S)
        return

    def _em_getdate_wrapper(self, date: str, offday: str):
        for i in range(self._EM_RETRY_TIMES + 1):
            data = c.getdate(date, offday)
            if data.ErrorCode == 0:
                return data.Data
            print(f'Failed to get date data({i}): {data.ErrorMsg} (err_code){data.ErrorCode}')
            time.sleep(self._EM_RETRY_INTERVAL_S)
        return

    def _em_csqsnapshot_wrapper(self, codes: str, indicators: str, options: str):
        if 'ispandas=1' not in options.lower():
            if options:
                options += ','
            options += 'Ispandas=1'
        for i in range(self._EM_RETRY_TIMES + 1):
            data = c.csqsnapshot(codes, indicators, options)
            if isinstance(data, pd.DataFrame):
                return data
            if data.ErrorCode != 0:
                print(f'failed to get csqsnapshot data({i}): {data.ErrorMsg} (err_code){data.ErrorCode}')
            else:
                print('weird, no error but not a df returned')
            time.sleep(self._EM_RETRY_INTERVAL_S)
        return

    def _em_edb_wrapper(self, codes: str, options: str):
        if 'ispandas=1' not in options.lower():
            if options:
                options += ','
            options += 'Ispandas=1'
        for i in range(self._EM_RETRY_TIMES + 1):
            data = c.edb(codes, options)
            if isinstance(data, pd.DataFrame):
                return data
            if data.ErrorCode != 0:
                print(f'failed to get edb data({i}): {data.ErrorMsg} (err_code){data.ErrorCode}')
            else:
                print('weird, no error but not a df returned')
            time.sleep(self._EM_RETRY_INTERVAL_S)
        return

    def em_fund_nav_history(self, nav_file_dir):
        try:
            index = 1
            for filename in os.listdir(nav_file_dir):
                if filename.endswith(".xls"):
                    print(f'{index}: {filename}')
                    index += 1
                    fund_nav = pd.concat(pd.read_excel(os.path.join(nav_file_dir, filename), sheet_name=None),
                                         ignore_index=True)
                    fund_nav = fund_nav.drop(['简称'], axis=1)
                    fund_nav = fund_nav.rename(columns={
                        '代码': 'CODES',
                        '时间': 'DATES',
                        '单位净值(元)': 'ORIGINALUNIT',
                        '累计净值(元)': 'ORIGINALNAVACCUM',
                        '复权净值(元)': 'ADJUSTEDNAV',
                        '万份基金单位收益(元)': 'UNITYIELD10K',
                        '7日年化收益率(%)': 'YIELDOF7DAYS'
                    })

                    fund_nav = fund_nav[(fund_nav['DATES'] != '--')
                                        & (~fund_nav['DATES'].isnull())]
                    fund_nav = fund_nav.replace('--', np.nan)
                    self._data_helper._upload_raw(fund_nav, EmFundNav.__table__.name)
                else:
                    continue
            return True
        except Exception as e:
            print(e)
            traceback.print_exc()
            return False

    def em_index_price_history(self, em_ids, start_date, end_date):
        em_id_str = ','.join(em_ids) if isinstance(em_ids, list) else em_ids
        df = c.csd(em_id_str, "OPEN,HIGH,LOW,VOLUME,CLOSE", start_date, end_date,
                   "period=1,adjustflag=1,curtype=1,order=1,market=0,Ispandas=1")
        if not isinstance(df, pd.DataFrame):
            if df.ErrorCode != 0:
                print(f'failed to get index price history: {df.ErrorMsg} (index_ids){em_ids}')
            return

        df = df.reset_index().rename(columns={
            'CODES': 'em_id',
            'DATES': 'datetime',
            'CLOSE': 'close',
            'OPEN': 'open',
            'HIGH': 'high',
            'LOW': 'low',
            'VOLUME': 'volume'
        })
        df = df[df['close'].notna()]
        self._data_helper._upload_raw(df, EmIndexPrice.__table__.name)

    def em_fund_scale_history(self):
        with RawDatabaseConnector().managed_session() as db_session:
            try:
                dates = db_session.query(
                    distinct(EmFundNav.DATES)
                ).all()

                dates = [d for d, in dates]
                dates.sort()
                start_date = dates[0]
                for i in range(1, 100):
                    dates.insert(0, start_date - datetime.timedelta(days=i))

                for d in dates:
                    month_day = d.strftime('%m%d')
                    if month_day not in self._quarter_update_dates:
                        continue

                    print(d)

                    fund_ids = db_session.query(
                        distinct(EmFundNav.CODES)
                    ).filter(
                        EmFundNav.DATES >= d,
                        EmFundNav.DATES <= d + datetime.timedelta(days=100)
                    ).all()

                    fund_ids = [f for f, in fund_ids]
                    fund_id_str = ','.join(fund_ids)

                    df = c.css(fund_id_str, "FUNDSCALE",
                               f"EndDate={d},Ispandas=1").reset_index()
                    df['DATES'] = d

                    self._data_helper._upload_raw(df, EmFundScale.__table__.name)
            except Exception as e:
                print('Failed to get data <err_msg> {}'.format(e))
                return None

    def em_fund_holding_rate_history(self, end_date: str):
        fund_list = RawDataHelper.get_all_fund_list(end_date)
        if fund_list is None:
            print('[em_fund_holding_rate_history] failed to get fund list')
            return False
        y_list = [str(y) for y in range(2020, 2021, 1)]
        d_list = []
        for y in y_list:
            for d in self._semi_update_dates:
                d_list.append(y + d)
        for d in d_list:
            data = c.css(fund_list, "HOLDPERSONALHOLDINGPCT,HOLDINSTIHOLDINGPCT,HOLDNUM", f"ReportDate={d},Ispandas=1")
            data = data[data.drop(columns='DATES').notna().any(axis=1)].reset_index()
            data['DATES'] = d
            self._data_helper._upload_raw(data, EmFundHoldingRate.__table__.name)
            print(f'finish {d}')
        return True

    def em_fund_nav(self, start_date, end_date, fund_list: Tuple[str] = ()):
        try:
            # TODO: 应该使用每一天当天的成分列表, 可能所有的sector都需要这样
            # Get all fund ids, 功能函数-板块成分
            # http://quantapi.eastmoney.com/Cmd/Sector?from=web
            start_date = datetime.datetime.strptime(start_date, '%Y%m%d') - datetime.timedelta(days=15)
            start_date = start_date.strftime('%Y%m%d')
            trade_date_df = RawDataApi().get_em_tradedates(start_date, end_date)
            if trade_date_df is None:
                print('[em_fund_nav] failed to get em trade dates for em_fund_nav')
                return False
            if not fund_list:
                fund_list = RawDataHelper.get_all_live_fund_list(end_date)
                if fund_list is None:
                    print('[em_fund_nav] failed to get fund list')
                    return False
            df_list: List[pd.DataFrame] = []
            for date in trade_date_df.TRADEDATES:
                # 基金 万份基金单位收益 7日年化收益率
                data1 = self._em_css_wrapper(fund_list, "10KUNITYIELD,YIELDOF7DAYS", f"EndDate={date},Ispandas=1")
                if data1 is None:
                    print('[em_fund_nav] failed to get fund nav data1')
                    return False
                # 基金 单位净值(原始披露) 累计单位净值(原始披露) 复权单位净值
                data2 = self._em_css_wrapper(fund_list, "ORIGINALUNIT,ORIGINALNAVACCUM,NAVADJ", f"TradeDate={date},Ispandas=1")
                if data2 is None:
                    print('[em_fund_nav] failed to get fund nava data2')
                    return False
                df = data2.drop(columns='DATES').join(data1.drop(columns='DATES'), how='outer')
                df = df[df[['ORIGINALUNIT', 'ORIGINALNAVACCUM', 'NAVADJ', '10KUNITYIELD', 'YIELDOF7DAYS']].notna().any(axis=1)]
                df = df.rename(columns={'10KUNITYIELD': 'UNITYIELD10K', 'NAVADJ': 'ADJUSTEDNAV'}).reset_index()
                df['DATES'] = pd.to_datetime(date).date().strftime('%Y-%m-%d')
                df_list.append(df)

            df_total = pd.concat(df_list)
            # 这里删掉之前几天的数据，然后再重刷一遍
            # TODO: 最好可以原子提交下边两步操作
            self.delete_em_fund_nav(start_date, end_date)
            self._data_helper._upload_raw(df_total, EmFundNav.__table__.name)
            return True
        except Exception as e:
            print(e)
            traceback.print_exc()
            return False

    def delete_em_fund_nav(self, start_date, end_date):
        try:
            RawDataApi().delete_em_fund_nav(start_date, end_date)
            return True
        except Exception as e:
            print(e)
            traceback.print_exc()
            return False

    def em_fund_status(self, end_date: str, is_history: bool = True):
        try:
            if not is_history:
                fund_list = RawDataHelper.get_all_live_fund_list(end_date)
            else:
                # 获取全量基金列表
                fund_list = self._em_sector_wrapper('507010', end_date)
            if fund_list is None:
                print('[em_fund_status] failed to get fund list')
                return False

            data1 = self._em_css_wrapper(fund_list, 'FUNDSCALE', f'EndDate={end_date},Ispandas=1')
            if data1 is None:
                print('failed to get fund status data1')
                return False
            data2 = self._em_css_wrapper(fund_list, 'PURCHSTATUS,REDEMSTATUS,LPLIMIT,OTCLPLIMITJG,TRADESTATUS', f'TradeDate={end_date},SHAREOBJECT=1,Ispandas=1')
            if data2 is None:
                print('failed to get fund status data2')
                return False
            data3 = self._em_css_wrapper(fund_list, 'OTCLPLIMITJG', f'TradeDate={end_date},SHAREOBJECT=2,Ispandas=1')
            if data3 is None:
                print('failed to get fund status data3')
                return False

            df = data1.drop(columns='DATES').join(data2.drop(columns='DATES'), how='outer').join(data3.drop(columns='DATES').rename(columns={'OTCLPLIMITJG': 'OTCLPLIMITGR'}), how='outer')
            df = df[df.notna().any(axis=1)]
            if df.empty:
                return True
            df = df.reset_index()
            df['DATES'] = end_date
            if not is_history:
                df = df.astype({'FUNDSCALE': 'float64', 'LPLIMIT': 'float64', 'OTCLPLIMITJG': 'float64', 'OTCLPLIMITGR': 'float64'})
                # 获取当前的fund status
                now_df = RawDataApi().get_em_fund_status(end_date=end_date)
                if now_df is not None:
                    # 过滤出来新增基金status以及旧基金的status发生变化的情况
                    # 这里由于存在未采用当前这种更新方式的历史数据，所以最后的validate需要是one_to_many
                    df = df.merge(now_df.drop(columns=['_update_time']), how='left', on=['CODES', 'FUNDSCALE', 'LPLIMIT', 'OTCLPLIMITJG', 'OTCLPLIMITGR', 'PURCHSTATUS', 'REDEMSTATUS', 'TRADESTATUS'], indicator=True, validate='one_to_many')
                    df = df[df._merge == 'left_only'].drop(columns=['_merge', 'DATES_y']).rename(columns={'DATES_x': 'DATES'})
            self._data_helper._upload_raw(df, EmFundScale.__table__.name)
            return True
        except Exception as e:
            print(e)
            traceback.print_exc()
            return False

    def em_index_price(self, em_ids: Set[str], start_date, end_date):
        try:
            # 每次重刷15个交易日的数据
            tradedate = self._em_getdate_wrapper(start_date, -INDEX_PRICE_EXTRA_TRADE_DAYS)
            if tradedate is None:
                print(f'[em_index_price] failed to get trade dates (date){start_date}')
                return False
            start_date = pd.to_datetime(tradedate[0]).strftime('%Y%m%d')

            # 不支持跨品种的index同时请求，所以需要拆出来分开去做
            df_list = []
            # 利率部分
            ir_index = []
            for one in ('LIGBP1M.IR', 'HIBOR3M.IR'):
                try:
                    em_ids.remove(one)
                except KeyError:
                    continue
                ir_index.append(one)
            if ir_index:
                data = self._em_csd_wrapper(','.join(ir_index), "CLOSE", start_date, end_date, "period=1,adjustflag=1,curtype=1,order=1,market=CNSESH,Ispandas=1")
                if data is None:
                    print(f'[em_index_price] failed to get index price (index_id){ir_index}')
                    return False
                df_list.append(data)

            # 期货部分
            future_index = []
            for one in ('AU9999.SGE', 'AU0.SHF'):
                try:
                    em_ids.remove(one)
                except KeyError:
                    continue
                future_index.append(one)
            if future_index:
                data = self._em_csd_wrapper(','.join(future_index), "OPEN,HIGH,LOW,VOLUME,CLOSE", start_date, end_date, "period=1,adjustflag=1,curtype=1,order=1,market=CNSESH,Ispandas=1")
                if data is None:
                    print(f'[em_index_price] failed to get index price (index_id){future_index}')
                    return False
                df_list.append(data)

            # 常规指数部分
            data = self._em_csd_wrapper(','.join(em_ids), "OPEN,HIGH,LOW,VOLUME,CLOSE", start_date, end_date, "period=1,adjustflag=1,curtype=1,order=1,market=CNSESH,Ispandas=1")
            if data is None:
                print(f'[em_index_price] failed to get index price (index_id){em_ids}')
                return False

            df = pd.concat(df_list + [data]).reset_index().rename(columns={
                'CODES': 'em_id',
                'DATES': 'datetime',
                'CLOSE': 'close',
                'OPEN': 'open',
                'HIGH': 'high',
                'LOW': 'low',
                'VOLUME': 'volume'
            })
            df = df[df['close'].notna()]
            # TODO: 最好可以原子提交下边两步操作
            RawDataApi().delete_em_index_price(list(em_ids)+ir_index+future_index, start_date, end_date)
            self._data_helper._upload_raw(df, EmIndexPrice.__table__.name)
            return True
        except Exception as e:
            print(e)
            traceback.print_exc()
            return False

    def em_index_info(self, index_ids: List[str]):
        try:
            data = self._em_css_wrapper(','.join(index_ids), 'SHORTNAME,NAME,PUBLISHDATE,MAKERNAME,INDEXPROFILE', 'Ispandas=1')
            if data is None:
                print('failed to get index info')
                return False

            df = data.drop(columns='DATES').reset_index().rename(columns=lambda x: EmIndexInfo.__getattribute__(EmIndexInfo, x).name)
            # 统一设置一个默认值，之后可以按需手工修改
            df['price_source'] = IndexPriceSource.default.name
            # 更新到db
            self._data_helper._upload_raw(df, EmIndexInfo.__table__.name)
            return True
        except Exception as e:
            print(e)
            traceback.print_exc()
            return False

    def em_macroeconomic_price(self, start_date, end_date):
        try:
            index_info_df = RawDataApi().get_em_index_info()
            if index_info_df is None:
                print('failed to get index info for macroeconomic price')
                return False
            index_info_df = index_info_df[index_info_df.price_source == IndexPriceSource.macroeconomic]
            macroeconomic_list = [self._MACROECONOMIC_MAP_I[one] for one in index_info_df.em_id.to_list() if one in self._MACROECONOMIC_MAP_I]
            data = c.edb(','.join(macroeconomic_list), f'StartDate={start_date},EndDate={end_date},IsLatest=0,RowIndex=2,Ispandas=1')
            if not isinstance(data, pd.DataFrame):
                if data.ErrorCode != 0:
                    if data.ErrorCode == 10000009:
                        print(f'no macroeconomic price data during {start_date} and {end_date}')
                        return True
                    print(f'failed to get macroeconomic price: {data.ErrorMsg}')
                return False

            df = data.reset_index().rename(columns={
                'CODES': 'em_id',
                'DATES': 'datetime',
                'RESULT': 'close'
            })
            df = df[df.close.notna()]
            df['em_id'] = df['em_id'].map(self._MACROECONOMIC_MAP)
            # 更新到db
            self._data_helper._upload_raw(df, EmIndexPrice.__table__.name)
            return True
        except Exception as e:
            print(e)
            traceback.print_exc()
            return False

    def em_realtime_index(self, em_ids: Set[str], data_start_time, to_save):
        try:
            # 不支持跨品种的index同时请求，所以需要拆出来分开去做
            df_list = []

            # 利率部分
            ir_index = []
            for one in ('LIGBP1M.IR', 'HIBOR3M.IR'):
                try:
                    em_ids.remove(one)
                except KeyError:
                    continue
                ir_index.append(one)

            # 期货部分
            future_index = []
            for one in ('AU9999.SGE', 'AU0.SHF'):
                try:
                    em_ids.remove(one)
                except KeyError:
                    continue
                future_index.append(one)

            # 常规指数部分
            data = self._em_csqsnapshot_wrapper(','.join(em_ids), 'Date,Time,Now', 'Ispandas=1')
            if data is None:
                print(f'[em_index_price] failed to get index price (index_id){em_ids}')
                return None
            df_list.append(data)

            # # 每两次 csqsnapshot 请求之间必须大于 3000 ms
            # # 暂不实时更新利率指数（需要更新时取消注释）
            # time.sleep(3.1)
            # if ir_index:
            #     data = c.csqsnapshot(','.join(ir_index), 'Date,Time,Now', 'Ispandas=1')
            #     if not isinstance(data, pd.DataFrame):
            #         if data.ErrorCode != 0:
            #             print(f'[em_index_price] failed to get index price: {data.ErrorMsg} (index_id){ir_index}')
            #         return False
            #     df_list.append(data)

            # # 暂不实时更新期货指数（需要更新时取消注释）
            # time.sleep(3.1)
            # if future_index:
            #     data = c.csqsnapshot(','.join(future_index), 'Date,Time,Now', 'Ispandas=1')
            #     if not isinstance(data, pd.DataFrame):
            #         if data.ErrorCode != 0:
            #             print(f'[em_index_price] failed to get index price: {data.ErrorMsg} (index_id){future_index}')
            #         return False
            #     df_list.append(data)

            df = pd.concat(df_list)
            df = df[df['DATE'] != 0]
            df = df[df['TIME'] != 0]
            df['datetime'] = df.apply(
                lambda x: datetime.datetime.strptime(f'{x["DATE"]}{x["TIME"]}', '%Y%m%d%H%M%S'),
                axis=1
            )
            if data_start_time:
                df = df[df['datetime'] > data_start_time]

            df = df.drop(columns=['DATES', 'DATE', 'TIME'])
            df = df.reset_index().rename(columns={
                'CODES': 'em_id',
                'NOW': 'price'
            })
            if to_save:
                try:
                    self._data_helper._upload_raw(df, EmRealtimeIndexPrice.__table__.name)
                except Exception as e:
                    print(e)
                    traceback.print_exc()
            return df

        except Exception as e:
            print(e)
            traceback.print_exc()
            return None

    # 从em客户端导出数据后读取并处理
    def em_stock_price_history(self, stock_price_file_dir: str):
        for index, filename in enumerate(os.listdir(stock_price_file_dir)):
            if not filename.endswith(".xls"):
                continue
            index += 1
            print(f'{index}: {filename}')
            stock_price = pd.concat(pd.read_excel(os.path.join(stock_price_file_dir, filename), sheet_name=None, na_values='--'),
                                    ignore_index=True)
            # 将列名字改为从API获取到数据时的列名字，便于后边统一处理
            stock_price = stock_price.drop(['简称'], axis=1).rename(columns={
                '代码': 'CODES',
                '时间': 'DATES',
                '开盘价(元)': 'OPEN',
                '收盘价(元)': 'CLOSE',
                '最高价(元)': 'HIGH',
                '最低价(元)': 'LOW',
                '前收盘价(元)': 'PRECLOSE',
                '均价(元)': 'AVERAGE',
                '成交量(股)': 'VOLUME',
                '成交金额(元)': 'AMOUNT',
                '换手率(%)': 'TURN',
                '交易状态': 'TRADESTATUS',
                '内盘成交量': 'BUYVOL',
                '外盘成交量': 'SELLVOL',
            })

            # 过滤掉个股未上市时的数据，以及日期为nan的行（表示整行为无效数据）
            stock_price = stock_price.loc[stock_price['TRADESTATUS'] != '未上市'].loc[~stock_price['DATES'].isna()].rename(columns=lambda x: EmStockPrice.__getattribute__(EmStockPrice, x).name)
            # 更新到db
            self._data_helper._upload_raw(stock_price, EmStockPrice.__table__.name)

    def em_stock_price(self, start_date: str, end_date: str, stock_list: Tuple[str] = ()):
        try:
            # 获取区间内所有交易日
            tradedates = self._em_tradedates_wrapper(start_date, end_date, "period=1,order=1,market=CNSESH")
            if tradedates is None:
                print('[em_stock_price] failed to get trade dates')
                return False
            if len(tradedates) == 0:
                print(f'[em_stock_price] no trade dates, return immediately (start_date){start_date} (end_date){end_date}')
                return False

            if not stock_list:
                # 获取所有A股股票的ID, 功能函数-板块成分
                # http://quantapi.eastmoney.com/Cmd/Sector?from=web
                # 全部A股股票
                stock_list = self._em_sector_wrapper('001004', end_date)
                if stock_list is None:
                    print('[em_stock_price] failed to get stock id list')
                    return False

            df_list = []
            for date in tradedates:
                # 获取个股股价信息
                temp_df = self._em_csd_wrapper(stock_list,
                                               'OPEN,CLOSE,HIGH,LOW,PRECLOSE,AVERAGE,VOLUME,AMOUNT,TURN,TRADESTATUS,TNUM,BUYVOL,SELLVOL',
                                               date, date, 'period=1,adjustflag=1,curtype=1,order=1,market=0,Ispandas=1')
                if temp_df is None:
                    print(f'failed to get stock price info (date){date}')
                    return False
                # 过滤掉个股未上市时的数据，以及日期为nan的行（表示整行为无效数据）
                df_list.append(temp_df.loc[temp_df['TRADESTATUS'] != '未上市'].loc[~temp_df['DATES'].isna()])

            df = pd.concat(df_list).reset_index().rename(columns=lambda x: EmStockPrice.__getattribute__(EmStockPrice, x).name)
            # 更新到db
            self._data_helper._upload_raw(df, EmStockPrice.__table__.name)
            return True
        except Exception as e:
            print(e)
            traceback.print_exc()
            return False

    # 从em客户端导出数据后读取并处理
    def em_stock_post_price_history(self, stock_post_price_file_dir: str):
        for index, filename in enumerate(os.listdir(stock_post_price_file_dir)):
            if not filename.endswith(".xls"):
                continue
            index += 1
            print(f'{index}: {filename}')
            stock_price = pd.concat(pd.read_excel(os.path.join(stock_post_price_file_dir, filename), sheet_name=None, na_values='--'),
                                    ignore_index=True)
            # 将列名字改为从API获取到数据时的列名字，便于后边统一处理
            stock_price = stock_price.drop(['简称'], axis=1).rename(columns={
                '代码': 'CODES',
                '时间': 'DATES',
                '开盘价(元)': 'OPEN',
                '收盘价(元)': 'CLOSE',
                '最高价(元)': 'HIGH',
                '最低价(元)': 'LOW',
                '前收盘价(元)': 'PRECLOSE',
                '均价(元)': 'AVERAGE',
                '成交量(股)': 'VOLUME',
                '成交金额(元)': 'AMOUNT',
                '交易状态': 'TRADESTATUS',
            })
            # 过滤掉个股未上市时的数据，以及日期为nan的行（表示整行为无效数据）
            stock_price = stock_price.loc[stock_price['TRADESTATUS'] != '未上市'].loc[~stock_price['DATES'].isna()].rename(columns=lambda x: EmStockPostPrice.__getattribute__(EmStockPostPrice, x).name)
            # 更新到db
            self._data_helper._upload_raw(stock_price, EmStockPostPrice.__table__.name)

    def em_stock_post_price(self, start_date: str, end_date: str, stock_list: Tuple[str] = ()):
        try:
            # 获取区间内所有交易日
            tradedates = self._em_tradedates_wrapper(start_date, end_date, "period=1,order=1,market=CNSESH")
            if tradedates is None:
                print('[em_stock_post_price] failed to get trade dates')
                return False
            if len(tradedates) == 0:
                print(f'[em_stock_post_price] no trade dates, return immediately (start_date){start_date} (end_date){end_date}')
                return False

            if not stock_list:
                # 获取所有A股股票的ID, 功能函数-板块成分
                # http://quantapi.eastmoney.com/Cmd/Sector?from=web
                # 全部A股股票
                stock_list = self._em_sector_wrapper('001004', end_date)
                if stock_list is None:
                    print('[em_stock_post_price] failed to get stock id list')
                    return False

            df_list = []
            for date in tradedates:
                # 获取个股股价信息
                temp_df = self._em_csd_wrapper(stock_list,
                                               'OPEN,CLOSE,HIGH,LOW,PRECLOSE,AVERAGE,VOLUME,AMOUNT,TRADESTATUS,TAFACTOR',
                                               date, date, 'period=1,adjustflag=2,curtype=1,order=1,market=0,Ispandas=1')
                if temp_df is None:
                    print(f'failed to get stock price info (date){date}')
                    return False
                # 过滤掉个股未上市时的数据，以及日期为nan的行（表示整行为无效数据）
                df_list.append(temp_df.loc[temp_df['TRADESTATUS'] != '未上市'].loc[~temp_df['DATES'].isna()])

            df = pd.concat(df_list).reset_index().rename(columns=lambda x: EmStockPostPrice.__getattribute__(EmStockPostPrice, x).name)
            # 更新到db
            self._data_helper._upload_raw(df, EmStockPostPrice.__table__.name)
            return True
        except Exception as e:
            print(e)
            traceback.print_exc()
            return False

    def em_stock_info(self, end_date: str):
        try:
            # 获取所有A股股票的ID, 功能函数-板块成分
            # http://quantapi.eastmoney.com/Cmd/Sector?from=web
            # 全部A股股票
            stock_list = self._em_sector_wrapper('001004', end_date)
            if stock_list is None:
                print('[em_stock_info] failed to get stock id list')
                return False

            # 去掉我们已经有的股票ID
            stock_info_df = RawDataApi().get_em_stock_info()
            new_stock_code = list(set(stock_list).difference(set(stock_info_df.stock_id.to_list())))
            if not new_stock_code:
                # 直接返回成功
                return True

            df_list: List[pd.DataFrame] = []
            # 获取无参数指标
            df_no_param = self._em_css_wrapper(new_stock_code,
                                               'NAME,LISTDATE,FINPURCHORNOT,FINSELLORNOT,STOHSTOCKCONNECTEDORNOT,SHENGUTONGTARGET,ENGNAME,COMPNAME,COMPNAMEENG',
                                               'Ispandas=1')
            if df_no_param is None:
                print('failed to get stock info(no param)')
                return False
            # 将其中'是'/'否'的结果转换为True/False
            columns = ['FINPURCHORNOT', 'FINSELLORNOT', 'STOHSTOCKCONNECTEDORNOT', 'SHENGUTONGTARGET']
            df_no_param.loc[:, columns] = df_no_param.loc[:, columns].apply(lambda x: x.apply(lambda x: True if x == '是' else False))
            df_list.append(df_no_param)

            # 获取行业分类共有三级的指标
            df_lv4 = self._em_css_wrapper(new_stock_code, "BLEMINDCODE,BLSWSINDCODE,SW2014CODE,EMINDCODE2016,CITICCODE2020", "ClassiFication=4,Ispandas=1")
            if df_lv4 is None:
                print('failed to get stock info(lv4)')
                return False
            df_list.append(df_lv4)

            # 获取行业分类共有二级的指标
            df_lv3 = self._em_css_wrapper(new_stock_code, "BLCSRCINDCODE,CSRCCODENEW", "ClassiFication=3,Ispandas=1")
            if df_lv3 is None:
                print('failed to get stock info(lv3)')
                return False
            df_list.append(df_lv3)

            # 获取行业分类共有四级的指标
            df_lv5 = self._em_css_wrapper(new_stock_code, "CSINDCODE2016,GICSCODE", "ClassiFication=5,Ispandas=1")
            if df_lv5 is None:
                print('failed to get stock info(lv5)')
                return False
            df_list.append(df_lv5)

            if df_list:
                # 将每部分按列粘到一起
                df = pd.concat(df_list, axis=1)
                # 不需要存日期的列，去除索引并做列名转换
                df = df.drop(columns='DATES').reset_index().rename(columns=lambda x: EmStockInfo.__getattribute__(EmStockInfo, x).name)
                # 更新到db
                self._data_helper._upload_raw(df, EmStockInfo.__table__.name)
            return True
        except Exception as e:
            print(e)
            traceback.print_exc()
            return False

    def add_columns_to_fin_fac(self, end_date: str):
        # 获取所有A股股票的ID, 功能函数-板块成分
        # http://quantapi.eastmoney.com/Cmd/Sector?from=web
        # 全部A股股票
        stock_id_result = c.sector('001004', end_date)
        if stock_id_result.ErrorCode != 0:
            print(f'failed to get stock id list: {stock_id_result.ErrorMsg}')
            return

        # 获取主营收入构成(按产品)
        df = c.css(stock_id_result.Codes, "BALANCESTATEMENT_195", f'ReportDate={end_date},type=1,Ispandas=1')
        if not isinstance(df, pd.DataFrame) and df.ErrorCode != 0:
            print(f'failed to get stock financial factors: {df.ErrorMsg}')
            return

        count = 0
        Session = sessionmaker(RawDatabaseConnector().get_engine())
        db_session = Session()
        for row in db_session.query(EmStockFinFac):
            if row.DATES != datetime.datetime.strptime(end_date, '%Y%m%d').date():
                continue
            if row.CODES in df.index:
                row.BALANCESTATEMENT_195 = df.at[row.CODES, 'BALANCESTATEMENT_195']
                count += 1
        db_session.commit()
        db_session.close()
        print(f'append {count} value(s) to fin fac table')

    def em_stock_fin_fac(self, end_date: str):
        # 获取所有A股股票的ID, 功能函数-板块成分
        # http://quantapi.eastmoney.com/Cmd/Sector?from=web
        # 全部A股股票
        stock_list = self._em_sector_wrapper('001004', end_date)
        if stock_list is None:
            print('[em_stock_fin_fac] failed to get stock id list')
            return False

        df_list: List[pd.DataFrame] = []
        # 获取无参数指标
        df_no_param = self._em_sector_wrapper(stock_list,
                                              'EBIT,EBITDA,EXTRAORDINARY,LOWERDIANDNI,GROSSMARGIN,OPERATEINCOME,INVESTINCOME,EBITDRIVE,TOTALCAPITAL,WORKINGCAPITAL,\
                                              NETWORKINGCAPITAL,TANGIBLEASSET,RETAINED,INTERESTLIBILITY,NETLIBILITY,EXINTERESTCL,EXINTERESTNCL,FCFF,FCFE,DA,FCFFDRIVE,\
                                              PERFORMANCEEXPRESSPARENTNI,MBSALESCONS,GPMARGIN,NPMARGIN,EXPENSETOOR,INVTURNRATIO,ARTURNRATIO,ROEAVG,ROEWA,EPSBASIC,EPSDILUTED,\
                                              BPS,MBREVENUE,MBCOST,BALANCESTATEMENT_25,BALANCESTATEMENT_46,BALANCESTATEMENT_93,BALANCESTATEMENT_103,BALANCESTATEMENT_141,BALANCESTATEMENT_195,\
                                              BALANCESTATEMENT_140,INCOMESTATEMENT_9,INCOMESTATEMENT_48,INCOMESTATEMENT_60,INCOMESTATEMENT_61,INCOMESTATEMENT_83,INCOMESTATEMENT_85,INCOMESTATEMENT_127,\
                                              INCOMESTATEMENT_14,CASHFLOWSTATEMENT_39,CASHFLOWSTATEMENT_59,CASHFLOWSTATEMENT_77,CASHFLOWSTATEMENT_82,CASHFLOWSTATEMENT_86',
                                              f'ReportDate={end_date},type=1,Ispandas=1')
        if df_no_param is None:
            print('failed to get stock financial factors')
        else:
            df_list.append(df_no_param)

        # 获取主营收入构成(按产品)
        df_mb_sales_cons = self._em_sector_wrapper(stock_list, "MBSALESCONS", f'ReportDate={end_date},type=2,Ispandas=1')
        if df_mb_sales_cons is None:
            print('failed to get stock financial factors')
        else:
            df_list.append(df_mb_sales_cons.rename(columns={'MBSALESCONS': 'MBSALESCONS_P'}))

        # 获取归属于上市公司股东的扣除非经常性损益后的净利润(调整前)
        df_with_param_1 = self._em_sector_wrapper(stock_list, "DEDUCTEDINCOME", f'ReportDate={end_date},DataAdjustType=1,Ispandas=1')
        if df_with_param_1 is None:
            print('failed to get stock financial factors')
        else:
            df_list.append(df_with_param_1.rename(columns={'DEDUCTEDINCOME': 'DEDUCTEDINCOME_BA'}))

        # 获取归属于上市公司股东的扣除非经常性损益后的净利润(调整后)
        df_with_param_2 = self._em_sector_wrapper(stock_list, "DEDUCTEDINCOME", f'ReportDate={end_date},DataAdjustType=2,Ispandas=1')
        if df_with_param_2 is None:
            print('failed to get stock financial factors')
        else:
            df_list.append(df_with_param_2.rename(columns={'DEDUCTEDINCOME': 'DEDUCTEDINCOME_AA'}))

        # 获取TTM指标
        df_ttm = self._em_sector_wrapper(stock_list, 'GRTTMR,GCTTMR,ORTTMR,OCTTMR,EXPENSETTMR,GROSSMARGINTTMR,OPERATEEXPENSETTMR,ADMINEXPENSETTMR,FINAEXPENSETTMR,IMPAIRMENTTTMR,\
                                         OPERATEINCOMETTMR,INVESTINCOMETTMR,OPTTMR,NONOPERATEPROFITTTMR,EBITTTMR,EBTTTMR,TAXTTMR,PNITTMR,KCFJCXSYJLRTTMR,\
                                         NPTTMRP,FVVPALRP,IRTTMRP,IITTMFJVAJVRP,BTAATTMRP,SALESCASHINTTMR,CFOTTMR,CFITTMR,CFFTTMR,CFTTMR,CAPEXR',
                                         f'ReportDate={end_date},Ispandas=1')
        if df_ttm is None:
            print('failed to get stock financial factors')
        else:
            df_list.append(df_ttm)

        if len(df_list) > 0:
            # 将每部分按列粘到一起
            df = pd.concat(df_list, axis=1)
            # 日期列修改为end_date
            df.loc[:, 'DATES'] = end_date
            df = df.reset_index().rename(columns=lambda x: EmStockFinFac.__getattribute__(EmStockFinFac, x).name)
            # 更新到db
            self._data_helper._upload_raw(df, EmStockFinFac.__table__.name)

    # 从em客户端导出数据后读取并处理
    def em_stock_daily_info_history(self, stock_daily_info_file_dir: str):
        for index, filename in enumerate(os.listdir(stock_daily_info_file_dir)):
            if not filename.endswith(".xls"):
                continue
            index += 1
            print(f'{index}: {filename}')
            stock_daily_info = pd.concat(pd.read_excel(os.path.join(stock_daily_info_file_dir, filename), sheet_name=None, na_values='--'),
                                         ignore_index=True)
            # 将列名字改为从API获取到数据时的列名字，便于后边统一处理
            stock_daily_info = stock_daily_info.drop(['简称'], axis=1).rename(columns={
                '代码': 'CODES',
                '时间': 'DATES',
                '总股本(股)': 'TOTALSHARE',
            })

            stock_daily_info = stock_daily_info.loc[~stock_daily_info['DATES'].isna()].rename(columns=lambda x: EmStockDailyInfo.__getattribute__(EmStockDailyInfo, x).name)

            # 更新到db
            self._data_helper._upload_raw(stock_daily_info, EmStockDailyInfo.__table__.name)

    def em_stock_daily_info(self, start_date: str, end_date: str, predict_year: int = 0):
        try:
            # 获取区间内所有交易日
            tradedates = self._em_tradedates_wrapper(start_date, end_date, "period=1,order=1,market=CNSESH")
            if tradedates is None:
                print('[em_stock_daily_info] failed to get trade dates')
                return False
            if len(tradedates) == 0:
                print(f'[em_stock_daily_info] no trade dates, return immediately (start_date){start_date} (end_date){end_date}')
                return False

            # 获取所有A股股票的ID, 功能函数-板块成分
            # http://quantapi.eastmoney.com/Cmd/Sector?from=web
            # 全部A股股票
            stock_list = self._em_sector_wrapper('001004', end_date)
            if stock_list is None:
                print('[em_stock_daily_info] failed to get stock id list')
                return False

            if predict_year == 0:
                predict_year = datetime.datetime.strptime(end_date, '%Y%m%d').date().year

            df_list = []
            for date in tradedates:
                temp_df = self._em_css_wrapper(stock_list,
                                               'TOTALSHARE,HOLDFROZENAMTACCUMRATIO,PETTMDEDUCTED,PBLYRN,PSTTM,AHOLDER,ESTPEG',
                                               f'EndDate={date},TradeDate={date},PREDICTYEAR={predict_year},Ispandas=1')
                if temp_df is None:
                    print(f'failed to get stock daily info (date){date}')
                    return False
                # 日期列修改为end_date
                temp_df.loc[:, 'DATES'] = date
                df_list.append(temp_df)

            df = pd.concat(df_list).reset_index().rename(columns=lambda x: EmStockDailyInfo.__getattribute__(EmStockDailyInfo, x).name)
            # 更新到db
            self._data_helper._upload_raw(df, EmStockDailyInfo.__table__.name)
            return True
        except Exception as e:
            print(e)
            traceback.print_exc()
            return False

    def em_stock_refinancing(self, start_date: str, end_date: str):
        try:
            data = c.ctr("AdditionPlanInfo",
                         "SECURITYCODE,APPROVENOTICEDATE,PLANNOTICEDDATE,NEWPROGRESS,SUMFINA_T,ATTACHNAME,ADDPURPOSE",
                         f"DateType=2,StartDate={start_date},EndDate={end_date},Process=0")
            if data.ErrorCode != 0:
                print(f'failed to get stock refinancing info: {data.ErrorMsg}')
                return False

            infos: List[List] = []
            for v in data.Data.values():
                # if v[3] in ('董事会批准', '董事会修改', '董事会终止', '股东大会批准', '股东大会修改'):
                #    continue
                infos.append(v)
            df = pd.DataFrame(infos, columns=['SECURITYCODE', 'APPROVENOTICEDATE', 'PLANNOTICEDDATE',
                                              'NEWPROGRESS', 'SUMFINA_T', 'ATTACHNAME', 'ADDPURPOSE'])
            self._data_helper._upload_raw(df.rename(columns=lambda x: EmStockRefinancing.__getattribute__(EmStockRefinancing, x).name), EmStockRefinancing.__table__.name)
            return True
        except Exception as e:
            print(e)
            traceback.print_exc()
            return False

    def em_stock_estimate_fac(self, start_date: str, end_date: str, predict_year: int = 0):
        try:
            # 获取区间内所有交易日
            df = RawDataApi().get_em_tradedates(start_date, end_date)
            if df is None or df.shape[0] == 0:
                print(f'[em_stock_estimate_fac] no trade dates, return immediately (start_date){start_date} (end_date){end_date}')
                return False

            # 获取所有A股股票的ID, 功能函数-板块成分
            # http://quantapi.eastmoney.com/Cmd/Sector?from=web
            # 全部A股股票
            stock_list = self._em_sector_wrapper('001004', end_date)
            if stock_list is None:
                print('[em_stock_estimate_fac] failed to get stock id list')
                return False

            if predict_year == 0:
                predict_year = datetime.datetime.strptime(end_date, '%Y%m%d').date().year + 1

            df_list = []
            for date in df.TRADEDATES:
                temp_df = self._em_css_wrapper(stock_list,
                                               'ESTGR,ESTNI',
                                               f'EndDate={date},PREDICTYEAR={predict_year},Ispandas=1')
                if temp_df is None:
                    print(f'failed to get stock estimate fac (date){date}')
                    return False
                # 日期列修改为end_date
                temp_df.loc[:, 'DATES'] = date
                df_list.append(temp_df)

            df = pd.concat(df_list).reset_index().rename(columns=lambda x: EmStockEstimateFac.__getattribute__(EmStockEstimateFac, x).name)
            df = df[df.notna().all(axis=1)]
            df['predict_year'] = predict_year
            # 更新到db
            self._data_helper._upload_raw(df, EmStockEstimateFac.__table__.name)
            return True
        except Exception as e:
            print(e)
            traceback.print_exc()
            return False

    def em_index_val_history(self, start_date: str, end_date: str, index_list: List[str]):
        # 获取区间内所有交易日
        trade_day_df = RawDataApi().get_em_tradedates(start_date, end_date)
        if trade_day_df is None or trade_day_df.shape[0] == 0:
            print(f'[em_index_val_history] no trade dates, return immediately (start_date){start_date} (end_date){end_date}')
            return

        df_list: List[pd.DataFrame] = []
        # 一天一天地获取
        for date in trade_day_df.TRADEDATES:
            data = c.csd(index_list,
                         "PETTM,PBMRQ,PSTTM,PCFTTM,ROE,EPSTTM,DIVIDENDYIELD",
                         date, date,
                         "DelType=1,period=1,adjustflag=1,curtype=1,order=1,market=0,Ispandas=1")
            if not isinstance(data, pd.DataFrame):
                if data.ErrorCode != 0:
                    print(f'failed to get index val info: {data.ErrorMsg} (date){date}')
                break

            _y = pd.to_datetime(date).date().year + 1
            another_data = c.css(index_list, "ESTPEG", f"TradeDate={date},PREDICTYEAR={_y},Ispandas=1")
            if not isinstance(another_data, pd.DataFrame):
                if another_data.ErrorCode != 0:
                    print(f'failed to get index val info part 2: {another_data.ErrorMsg} (date){date}')
                break
            df_list.append(data.join(another_data.drop(columns='DATES'), how='outer'))
            print(f'{date} done')

        if df_list:
            if len(df_list) == 1:
                df = df_list[0]
            else:
                df = pd.concat(df_list)
            df = df[df.drop(columns=['DATES']).notna().any(axis=1)]
            df = df.reset_index().rename(columns={'DIVIDENDYIELD': 'DIVIDENDRATETTM'}).rename(columns=lambda x: EmIndexVal.__getattribute__(EmIndexVal, x).name)
            self._data_helper._upload_raw(df, EmIndexVal.__table__.name)

    def em_index_val(self, start_date: str, end_date: str):
        try:
            # 获取区间内所有交易日
            trade_day_df = RawDataApi().get_em_tradedates(start_date, end_date)
            if trade_day_df is None or trade_day_df.shape[0] == 0:
                print(f'[em_index_val] no trade dates, return immediately (start_date){start_date} (end_date){end_date}')
                return False

            # 获取basic table中index_info信息
            index_info_df = BasicDataApi().get_index_info()
            if index_info_df is None:
                print(f'[em_index_val] get index info failed')
                return False
            em_indexes = index_info_df[index_info_df.tag_method.isin(['PE百分位', 'PB百分位', 'PS百分位'])].em_id
            # 过滤掉几个特殊指数
            em_indexes = em_indexes[em_indexes.notna() & ~em_indexes.isin(['LIGBP1M.IR', 'HIBOR3M.IR', 'AU9999.SGE', 'AU0.SHF'])]
            index_list = em_indexes.to_list()
            if self._index_block_list is not None:
                index_list = list(set(index_list) - self._index_block_list)
            if not index_list:
                print(f'[em_index_val] empty index list, return immediately')
                return False

            index_split_point = len(index_list) // 2
            df_list: List[pd.DataFrame] = []
            # 一天一天地获取
            for date in trade_day_df.TRADEDATES:
                # 分成两部分去获取，以降低可能因为单条请求数据量较大导致出现超时错误
                for index_list_part in (index_list[:index_split_point], index_list[index_split_point:]):
                    if not index_list_part:
                        continue
                    data = self._em_csd_wrapper(index_list_part,
                                                "PETTM,PBMRQ,PSTTM,PCFTTM,ROE,EPSTTM",
                                                date, date,
                                                "DelType=1,period=1,adjustflag=1,curtype=1,order=1,market=0,Ispandas=1")
                    if data is None:
                        print(f'failed to get index val info (date){date}')
                        break

                    # 实际上更容易出现超时错误的是下边这个css请求
                    # Caution!! 股息率补历史的话需要使用csd中的DIVIDENDYIELD，这里的DIVIDENDRATETTM只能用作每日更新
                    _y = pd.to_datetime(date).date().year + 1
                    another_data = self._em_css_wrapper(index_list_part, "DIVIDENDRATETTM,ESTPEG", f"TradeDate={date},PREDICTYEAR={_y},Ispandas=1")
                    if another_data is None:
                        print(f'failed to get index val info part 2 (date){date}')
                        break
                    # 这里除以100与股息率历史数据保持单位一致
                    another_data['DIVIDENDRATETTM'] /= 100
                    df_list.append(data.join(another_data.drop(columns='DATES'), how='outer'))

            if df_list:
                if len(df_list) == 1:
                    df = df_list[0]
                else:
                    df = pd.concat(df_list)
                df = df[df.drop(columns='DATES').notna().any(axis=1)]
                df = df.reset_index().rename(columns=lambda x: EmIndexVal.__getattribute__(EmIndexVal, x).name)
                self._data_helper._upload_raw(df, EmIndexVal.__table__.name)
                return True
            else:
                return False
        except Exception as e:
            print(e)
            traceback.print_exc()
            return False

    def em_tradedates(self, start_date, end_date, is_automatic_update=True):
        try:
            if is_automatic_update:
                # 如果是每日自动更新，我们需要去存T+1日的数据，这里保险起见多取30天的
                end_date_dt: datetime.datetime = datetime.datetime.strptime(end_date, '%Y%m%d')
                extra_end_date = (end_date_dt + datetime.timedelta(days=30)).strftime('%Y%m%d')
            else:
                extra_end_date = end_date
            # 这里确保返回结果按日期升序排序
            tradedates = self._em_tradedates_wrapper(start_date, extra_end_date, "period=1,order=1,market=CNSESH")
            if tradedates is None:
                print('[em_tradedates] failed to get trade dates')
                return False

            if is_automatic_update:
                # 如果是每日自动更新，看一下返回的第一个日期是否是start_date，如果不是，表明今天不是交易日
                start_date_dt: datetime.datetime = pd.to_datetime(start_date, infer_datetime_format=True)
                if start_date_dt != pd.to_datetime(tradedates[0], infer_datetime_format=True):
                    print(f'today {start_date} is not a trading day')
                    return False

            df = pd.DataFrame(tradedates, columns=['TRADEDATES'])
            if is_automatic_update:
                # 如果是每日自动更新，向DB中insert T+1日的日期
                df = df.iloc[[1], :]
            self._data_helper._upload_raw(df, EmTradeDates.__table__.name)
            return True
        except Exception as e:
            print(e)
            traceback.print_exc()
            return False

    def em_index_component(self, start_date: str, end_date: str):
        try:
            index_info = BasicDataApi().get_index_info()

            date_list = self._get_date_list(start_date, end_date)
            s_list: List[pd.Series] = []
            for row in index_info[~index_info['em_plate_id'].isna()].loc[:, ['index_id', 'em_id', 'em_plate_id']].itertuples(index=False):
                for date in date_list:
                    stock_list = self._em_sector_wrapper(row.em_plate_id, end_date)
                    if stock_list is None:
                        print('[em_index_component] failed to get stock id list')
                        continue
                    if len(stock_list) != 0:
                        s_list.append(pd.Series(row._asdict()).append(pd.Series({'datetime': date, 'stock_list': ','.join(stock_list)})))
            # 更新到db
            self._data_helper._upload_raw(pd.DataFrame(s_list), EmIndexComponent.__table__.name)

            # 在这里也插一条中证指数的成分数据
            em_id = '905009'
            csi_index_list = self._em_sector_wrapper(em_id, end_date)
            if csi_index_list is None:
                print('failed to get data of sector csi index')
                return False
            prev_index_comp_df = RawDataApi().get_em_index_component(index_list=[ALL_CSI_INDEX_INNER_NAME])
            if prev_index_comp_df is not None:
                prev_index_list = set(prev_index_comp_df.stock_list.array[-1].split(','))
                # 上一天的数据 - 今天的数据 = 失效的指数，但是我们这里只处理了中证指数
                self._index_block_list = prev_index_list - set(csi_index_list)
                if self._index_block_list:
                    print(f'[WARNING]!! these indexes become invalid: {self._index_block_list}')
            self._data_helper._upload_raw(pd.DataFrame([{'index_id': ALL_CSI_INDEX_INNER_NAME, 'datetime': end_date,
                                                         'em_id': em_id, 'em_plate_id': em_id, 'stock_list': ','.join(csi_index_list)}]), EmIndexComponent.__table__.name)
            return True
        except Exception as e:
            print(e)
            traceback.print_exc()
            return False

    @staticmethod
    def _lambda_func_index_component_1(x: pd.Series):
        value: List[Dict[str, str]] = json.loads(x)
        new_value = []
        for one in value:
            one = (*one.keys(), float(*one.values()))
            new_value.append(one)
        new_value.sort(key=itemgetter(1), reverse=True)
        return json.dumps(new_value)

    @staticmethod
    def _lambda_func_index_component_2(em_id_map: Dict[str, str], x: pd.Series):
        try:
            return em_id_map[x]
        except KeyError:
            return x

    @staticmethod
    def _lambda_func_index_component_3(x: pd.Series):
        value: List[Dict[str, str]] = json.loads(x)
        new_value = []
        for one in value:
            one = [*one.keys(), float(*one.values())]
            if one[0][0] in ('0', '3'):
                one[0] += '.SZ'
            elif one[0][0] in ('6'):
                one[0] += '.SH'
            new_value.append(one)
        new_value.sort(key=itemgetter(1), reverse=True)
        return json.dumps(new_value)

    @staticmethod
    def _lambda_func_index_component_4(x: pd.Series):
        if pd.isnull(x):
            return x
        return json.dumps([one + '.OF' for one in x.array])

    def cs_index_component(self, date: str):
        try:
            em_id_list = BasicDataApi().get_index_info().em_id.to_list()

            # 中证指数爬下来的部分
            columns_list = ['id', 'name', 'num', 'id_cat', 'sector', 'top10', 'related_funds']
            df_list: List[pd.DataFrame] = []
            for cat in ('industry', 'topic', 'strat', 'cap'):
                raw_df = pd.read_excel(f'./data/Index/{cat}_data.xlsx', na_values='--')
                df_list.append(raw_df[columns_list])

            df = pd.concat(df_list)
            df['id'] = df['id'].transform(lambda x: EmRawDataDownloader._lambda_func_index_component_2({em_id.split('.')[0]: em_id for em_id in em_id_list if em_id is not None}, x))
            df['id_cat'] = df['id_cat'].map({'行业': SectorType.industry.name, '主题': SectorType.topic.name, '策略': SectorType.strategy.name, '规模': SectorType.scale.name})
            df['sector'] = df['sector'].transform(EmRawDataDownloader._lambda_func_index_component_1)
            df['top10'] = df['top10'].transform(EmRawDataDownloader._lambda_func_index_component_3)
            df = df.rename(columns={'id': 'index_id'})
            # Choice中指数的全部成分及权重
            # 该表主要提供指定日期的指数成分股代码及权重等信息
            choice_data = defaultdict(list)
            for em_id in df['index_id'].to_list():
                data = self._em_ctr_wrapper("INDEXCOMPOSITION", "SECUCODE,NAME,WEIGHT", f"IndexCode={em_id},EndDate={date}")
                if data is None:
                    print(f'[cs_index_component] failed to get data (em_id){em_id}')
                    continue
                if data:
                    for constituent in data.values():
                        choice_data[em_id].append((constituent[0], round(constituent[2] * 100, 2)))
                    choice_data[em_id].sort(key=itemgetter(1), reverse=True)
                    choice_data[em_id] = json.dumps(choice_data[em_id])

            df['datetime'] = date
            df = df.set_index('index_id').assign(all_constitu=pd.Series(choice_data))
            # 更新到db
            self._data_helper._upload_raw(df.drop(columns='name').reset_index(), CSIndexComponent.__table__.name)
            return True
        except Exception as e:
            print(e)
            traceback.print_exc()
            return

    def em_fund_list(self, start_date: str, end_date: str):
        try:
            # 这两个成分只能取到最新的，所以直接用end_date即可，没必要遍历
            # 获取所有基金列表（不包含未成立、已到期）
            live_fund_list = self._em_sector_wrapper('507013', end_date)
            if live_fund_list is None:
                print('[em_fund_list] failed to get live fund list')
                return False

            # 获取已摘牌基金列表
            delisted_fund_list = self._em_sector_wrapper('507022', end_date)
            if delisted_fund_list is None:
                print('[em_fund_list] failed to get delisted fund list')
                return False

            df = pd.DataFrame({'datetime': [end_date], 'all_live_fund_list': [','.join(live_fund_list)],
                               'delisted_fund_list': [','.join(delisted_fund_list)]})
            # 更新到db
            self._data_helper._upload_raw(df, EmFundList.__table__.name)
            return True
        except Exception as e:
            print(e)
            traceback.print_exc()
            return

    def _em_get_all_fund_list(self, date: str):
        # 获取全部基金
        all_fund = c.sector('507013', date)
        if all_fund.ErrorCode != 0:
            print(f'failed to get all fund list: {all_fund.ErrorMsg}')
            return

        # 获取全部基金列表（含未成立、已到期）
        all_fund_extra_full = c.sector('507010', date)
        if all_fund_extra_full.ErrorCode != 0:
            print(f'failed to get all fund list: {all_fund_extra_full.ErrorMsg}')
            return

        # 获取全部基金列表（含未成立）
        all_fund_extra = c.sector('507009', date)
        if all_fund_extra.ErrorCode != 0:
            print(f'failed to get all fund list: {all_fund_extra.ErrorMsg}')
            return

        # 不应该保存未成立基金的数据，这里我们需要(全部 + (含未成立、已到期 - 含未成立))
        all_fund_list = set(all_fund.Codes)
        all_fund_extra_full_list = set(all_fund_extra_full.Codes)
        all_fund_extra_list = set(all_fund_extra.Codes)
        all_fund_list.update(all_fund_extra_full_list.difference(all_fund_extra_list))
        return all_fund_list

    def em_fund_info_history(self, date: str):
        fund_list = self._em_get_all_fund_list(date)
        if fund_list is None:
            return

        # 更新wind_fund_info中没有的数据
        wind_df = RawDataApi().get_wind_fund_info()
        self.em_fund_info(fund_list.difference(wind_df.wind_id))

    def em_fund_info(self, fund_id_list: Set[str]):
        if not fund_id_list:
            return True
        try:
            data = self._em_css_wrapper(','.join(fund_id_list), 'NAME,FNAME,FUNDTYPE,FOUNDDATE,MATURITYDATE,FIRSTINVESTTYPE,SECONDINVESTTYPE,TRADECUR,STRUCFUNDORNOT,GFCODEM,PREDFUNDMANAGER,MGRCOMP,'
                                                                'CUSTODIANBANK,FOREIGNCUSTODIAN,FRONTENDFEECODE,BACKENDFEECODE,ARCODEIN', 'Ispandas=1')
            if data is None:
                print('failed to get fund info')
                return False

            data['FUNDTYPE'] = data['FUNDTYPE'].map(lambda x: True if x == '契约型开放式' else False)
            data['STRUCFUNDORNOT'] = data['STRUCFUNDORNOT'].map(lambda x: True if x == '是' else False)
            df = data.drop(columns='DATES').reset_index().rename(columns=lambda x: EmFundInfo.__getattribute__(EmFundInfo, x).name)
            # 更新到db，先删再添，确保数据不会冲突，且有一个更新的机会
            # TODO: 最好原子提交下边两个操作
            RawDataApi().delete_em_fund_info(df.em_id.to_list())
            self._data_helper._upload_raw(df, EmFundInfo.__table__.name)
            return True
        except Exception as e:
            print(e)
            traceback.print_exc()
            return False

    def em_fund_fee_history(self, date: str):
        fund_list = self._em_get_all_fund_list(date)
        if fund_list is None:
            return
        self.em_fund_fee(fund_list)

    def _lambda_fund_fee_1(self, x):
        if x.CODEFEEMODE.array[0] == CodeFeeMode.none:
            fee_type = 1
        else:
            fee_type = x.CODEFEEMODE.array[0].value
        data = self._em_css_wrapper(','.join(x.index.to_list()), 'MANAGFEERATIO,CUSTODIANFEERATIO,PURCHFEERATIO,REDEMFEERATIO,SERVICEFEERATIO,SUBSCRFEERATIO', f'FeeType={fee_type},Ispandas=1')
        if data is None:
            print('failed to get fund code fee')
            return
        return data

    def em_fund_fee(self, fund_id_list: Set[str]):
        if not fund_id_list:
            return True
        try:
            data = self._em_css_wrapper(','.join(fund_id_list), 'CODEFEEMODE', 'Ispandas=1')
            if data is None:
                print('failed to get fund code fee mode')
                return False
            data['CODEFEEMODE'] = data['CODEFEEMODE'].map({'前端': CodeFeeMode.a_type, '后端': CodeFeeMode.b_type, None: CodeFeeMode.none})
            data = data.groupby(by='CODEFEEMODE', sort=False).apply(self._lambda_fund_fee_1).reset_index()
            data['CODEFEEMODE'] = data['CODEFEEMODE'].transform(lambda x: CodeFeeMode(x).name)
            df = data.drop(columns='DATES').rename(columns=lambda x: EmFundFee.__getattribute__(EmFundFee, x).name)
            # 更新到db，先删再添，确保数据不会冲突，且有一个更新的机会
            # TODO: 最好原子提交下边两个操作
            RawDataApi().delete_em_fund_fee(df.em_id.to_list())
            self._data_helper._upload_raw(df, EmFundFee.__table__.name)
            return True
        except Exception as e:
            print(e)
            traceback.print_exc()
            return False

    def em_fund_benchmark_history(self, start_date: str, end_date: str):
        # 获取所有基金列表（含未成立、已到期）
        fund_list_result = c.sector('507010', end_date)
        if fund_list_result.ErrorCode != 0:
            print(f'failed to get all fund list: {fund_list_result.ErrorMsg}')
            return

        df = RawDataApi().get_em_tradedates(start_date, end_date)
        for row in df.itertuples(index=False):
            if self.em_fund_benchmark(row.TRADEDATES, fund_list_result.Codes):
                print(f'[em_fund_benchmark_history] {row.TRADEDATES} done')
            else:
                print(f'[em_fund_benchmark_history] {row.TRADEDATES} failed')
                break

    def em_fund_benchmark(self, end_date: str, fund_list=()):
        try:
            if not fund_list:
                fund_list = RawDataHelper.get_all_live_fund_list(end_date)
                if fund_list is None:
                    print(f'[em_fund_benchmark] failed to get fund list')
                    return False
            data = c.css(fund_list, 'BENCHMARK', f'EndDate={end_date},Ispandas=1')
            if not isinstance(data, pd.DataFrame):
                if data.ErrorCode != 0:
                    print(f'failed to get fund benchmark: {data.ErrorMsg}')
                return False
            data = data[data.notna().all(axis=1)]
            if data.empty:
                return True

            df = data.reset_index().rename(columns=lambda x: EmFundBenchmark.__getattribute__(EmFundBenchmark, x).name)
            df['datetime'] = end_date
            df['benchmark'] = df.benchmark.str.rstrip()
            # 获取当前的benchmark
            now_df = RawDataApi().get_em_fund_benchmark(end_date)
            if now_df is not None:
                # 过滤出来新增基金benchmark以及旧基金的benchmark发生变化的情况
                df = df.merge(now_df.drop(columns=['_update_time']), how='left', on=['em_id', 'benchmark'], indicator=True, validate='one_to_one')
                df = df[df._merge == 'left_only'].drop(columns=['_merge', 'datetime_y']).rename(columns={'datetime_x': 'datetime'})
            # 更新到db
            self._data_helper._upload_raw(df, EmFundBenchmark.__table__.name)
            return True
        except Exception as e:
            print(e)
            traceback.print_exc()
            return False

    def em_fund_rate(self, end_date: str):
        try:
            fund_list = RawDataHelper.get_all_live_fund_list(end_date)
            if fund_list is None:
                print('[em_fund_rate] failed to get fund list')
                return False
            df = self._em_css_wrapper(fund_list, "MKTCOMPRE3YRAT,MORNSTAR3YRAT,MORNSTAR5YRAT,MERCHANTS3YRAT,TXCOMRATRAT,SHSTOCKSTAR3YCOMRAT,SHSTOCKSTAR3YSTOCKRAT,SHSTOCKSTAR3YTIMERAT,SHSTOCKSTAR3YSHARPERAT,SHSTOCKSTAR5YCOMRATRAT,SHSTOCKSTAR5YSTOCKRAT,SHSTOCKSTAR5YTIMERAT,SHSTOCKSTAR5YSHARPRAT,JAJXCOMRAT,JAJXEARNINGPOWERRAT,JAJXACHIEVESTABILITYRAT,JAJXANTIRISKRAT,JAJXSTOCKSELECTIONRAT,JAJXTIMESELECTIONRAT,JAJXBENCHMARKTRACKINGRAT,JAJXEXCESSEARNINGSRAT,JAJXTOTALFEERAT",
                                      f"EndDate={end_date},Ispandas=1")
            if df is None:
                print('[em_fund_rate] failed to get data')
                return False
            df = df[df.drop(columns='DATES').notna().any(axis=1)]
            df['DATES'] = datetime.datetime.strptime(end_date, '%Y%m%d').date()
            df = df.reset_index()
            self._data_helper._upload_raw(df, EMFundRate.__table__.name)
            return True
        except Exception as e:
            print(e)
            traceback.print_exc()
            return False

    def em_fund_asset(self, end_date: str):
        fund_list = RawDataHelper.get_all_fund_list(end_date)
        if fund_list is None:
            print('[em_fund_asset] failed to get fund list')
            return False

        start_year = 2010
        end_year = 2014
        res: List[pd.DataFrame] = []
        for year in range(start_year, end_year+1):
            for date in self._quarter_update_dates:
                dt = str(year) + date
                print(f'doing {dt}')
                data = c.css(fund_list, "PRTSTOCKTONAV,PRTBONDTONAV,PRTCASHTONAV,PRTOTHERTONAV,MMFFIRSTREPOTONAV,MMFAVGPTM", f"ReportDate={dt},Ispandas=1")
                data['DATES'] = dt
                res.append(data)
                print(f'{dt} done')
        df = pd.concat(res).reset_index()
        df = df[df.drop(columns=['CODES', 'DATES']).notna().any(axis=1)]
        self._data_helper._upload_raw(df, EMFundHoldAsset.__table__.name)

    def em_fund_industry(self, end_date: str):
        fund_list = RawDataHelper.get_all_fund_list(end_date)
        if fund_list is None:
            print('[em_fund_industry] failed to get fund list')
            return False

        res = []
        start_year = 2015
        end_year = 2019
        rank_list = [1, 2, 3]
        for year in range(start_year, end_year+1):
            for date in self._quarter_update_dates:
                dt = str(year) + date
                print(f'doing {dt}')
                _res = []
                for rank in rank_list:
                    data = c.css(fund_list, "PRTKEYINDNAME,PRTKEYINDTOASSET", f"ReportDate={dt},Rank={rank},IndustryCode=A,NewOldType=1,Ispandas=1")
                    data = data.rename(columns={'PRTKEYINDNAME': f'rank{rank}_indname', 'PRTKEYINDTOASSET': f'rank{rank}_indweight'})
                    data = data.drop(columns='DATES')
                    _res.append(data)
                df = pd.concat(_res, axis=1)
                df['DATES'] = dt
                res.append(df)
                print(f'{dt} done')
        df = pd.concat(res).reset_index()
        df = df[df.drop(columns=['CODES', 'DATES']).notna().any(axis=1)]
        self._data_helper._upload_raw(df, EMFundHoldIndustry.__table__.name)

    def em_fund_stock_history(self, fund_stock_path: str, report_date: str):
        df: pd.DataFrame = pd.read_excel(fund_stock_path, na_values='——')
        df = df[df['证券名称'].notna()]
        df = df.drop(columns='证券名称')

        columns_name = ['CODES'] + ['rank'+str(i+1)+'_stock'+suffix for suffix in ('', '_code', 'val', 'weight') for i in range(10)]
        assert len(columns_name) == df.shape[1], 'len of columns name should be equal with columns number of df'
        df = df.set_axis(columns_name, axis=1)
        df['DATES'] = report_date
        em._data_helper._upload_raw(df, EMFundHoldStock.__table__.name)

    def em_fund_stock(self, end_date: str):
        fund_list = RawDataHelper.get_all_live_fund_list(end_date)
        if fund_list is None:
            print('[em_fund_stock] failed to get fund list')
            return False
        dt_list = ['20191231', '20200331', '20200630']
        res = []
        rank_list = [i for i in range(1, 11, 1)]
        for dt in dt_list:
            _res = []
            for rank in rank_list:
                data = c.css(fund_list,"PRTKEYSTOCKNAME,PRTKEYSTOCKCODE,PRTKEYSTOCKVALUE,PRTKEYSTOCKTONAV",f"ReportDate={dt},Rank={rank},Ispandas=1")
                data = data.rename(columns={'PRTKEYSTOCKNAME': f'rank{rank}_stock',
                                            'PRTKEYSTOCKCODE': f'rank{rank}_stock_code',
                                            'PRTKEYSTOCKVALUE': f'rank{rank}_stockval',
                                            'PRTKEYSTOCKTONAV': f'rank{rank}_stockweight',
                                            })
                data = data.drop('DATES',axis=1)
                _res.append(data)
                print(f'[em_fund_stock] {dt} rank {rank} done')
            df = pd.concat(_res,axis=1)
            df['DATES'] = dt
            res.append(df)
        df = pd.concat(res).reset_index()
        self._data_helper._upload_raw(df, 'em_fund_stock')
        return True

    def em_fund_bond_history(self, fund_bond_path: str, report_date: str):
        df: pd.DataFrame = pd.read_excel(fund_bond_path, na_values='——')
        df = df[df['证券名称'].notna()]
        df = df.drop(columns='证券名称')

        columns_name = ['CODES'] + ['rank'+str(i+1)+'_bond'+suffix for suffix in ('', '_code', 'val', 'weight') for i in range(10)]
        assert len(columns_name) == df.shape[1], 'len of columns name should be equal with columns number of df'
        df = df.set_axis(columns_name, axis=1)
        df['DATES'] = report_date
        em._data_helper._upload_raw(df, EMFundHoldBond.__table__.name)

    def em_mng_info_history(self,path):
        col_dic = {'基金经理':'mng_name',
            '代码': 'code',
            '名称':'desc_name',
            '在管基金规模(亿元)':'fund_size',
            '任职日期':'start_date',
            '任职天数':'work_days',
            '任职以来回报(%)':'his_ret',
            '任职以来年化回报(%)':'his_annual_ret',
            '业绩比较基准':'benchmark',
            '基金类型':'fund_type',
            '管理公司':'company_name',
            '共同任职经理':'coworkers',
            '基金经理年限':'mng_work_year',
            '任职基金数':'worked_funds_num',
            '任职基金公司数':'worked_funds_num',
            '任职基金几何总回报(%)':'fund_total_geo_ret',
            '几何平均年化收益率(%)':'geo_annual_ret',
            '算术平均年化收益率(%)':'cal_annual_ret',
            '性别':'gender',
            '出生年份':'birth_year',
            '年龄':'age',
            '学历':'education',
            '国籍':'nationality',
            '简历':'resume',
            '最早任职基金经理日期':'start_mng_date'}
        df = pd.read_excel(path, header=1).dropna(axis=0)
        df = df.rename(columns = col_dic)
        df = df.replace('——',np.NaN)
        em._data_helper._upload_raw(df,EMMngInfo.__table__.name)

    def em_fund_mng_change_history(self, path):
        col_dic = {
            '代码':'code',
            '名称':'desc_name',
            '现任基金经理':'mng_now',
            '现任经理最新任职日期':'mng_now_start_date',
            '最早任职基金经理':'mng_begin',
            '最早任职基金经理日期':'mng_begin_date',
            '现任基金经理年限':'mng_now_work_year',
            '同类型基金现任基金经理年限均值':'same_type_fund_work_year',
            '已离任基金经理':'resign_mngs',
            '历任基金经理人数':'total_mng_num',
            '历任基金经理人均任职年限':'total_mng_avg_work_year',
            '投资类型':'fund_type',
            '管理公司':'company',
            '监管辖区':'region'
        }
        df = pd.read_excel(path, header=0).dropna(axis=0)
        df = df.rename(columns = col_dic)
        df = df.replace('——',np.NaN)
        df = df.drop_duplicates('code') # 发现有重复的数据 比如 codes = 000063.OF
        em._data_helper._upload_raw(df,EMFundMngChange.__table__.name)

    def em_fund_comp_mng_change_history(self, path):
        col_dic = {
            '基金公司':'company',
            '基金经理数':'total_mng_num',
            '基金经理平均年限':'mng_avg_year',
            '基金经理最大年限':'mng_max_year',
            '团队稳定性':'team_stability',
            '新聘基金经理数':'new_mng_num',
            '离职基金经理数':'resign_mng_num',
            '基金经理变动率(%)':'mng_turnover_rate',
            '1年以内':'exp_less_than_1',
            '1-2年':'exp_1_to_2',
            '2-3年':'exp_2_to_3',
            '3-4年':'exp_3_to_4',
            '4年以上':'exp_more_than_4',
        }
        df = pd.read_excel(path, header=1).dropna(axis=0)
        df = df.rename(columns = col_dic)
        df = df.replace('——',np.NaN)
        em._data_helper._upload_raw(df,EMFundCompMngChange.__table__.name)

    def em_fund_com_core_mng_history(self, path):
        col_dic = {
            '基金代码':'code',
            '基金名称':'desc_name',
            '本基金离职基金经理人数':'fund_resign_mng_num',
            '所属基金公司':'company',
            '基金公司基金经理数':'com_mng_num',
            '基金公司离职基金经理数':'com_resign_mng_num',
            '基金公司基金经理离职率(%)':'com_resign_mng_rate',
            '基金公司核心人员人数':'com_core_mng_num',
            '基金公司核心人员离职人数':'com_core_mng_resign_num',
            '基金公司核心人员离职率(%)':'com_core_mng_resign_rate',
            '投资类型':'fund_type',
            '监管区域':'region'
        }
        df = pd.read_excel(path, header=1).dropna(axis=0)
        df = df.rename(columns = col_dic)
        df = df.replace('——',np.NaN)
        df = df.drop_duplicates('code') # 发现有重复的数据 比如 codes = 000424.OF
        em._data_helper._upload_raw(df,EMFundCompCoreMng.__table__.name)

    def em_fund_bond(self, end_date: str):
        fund_list = RawDataHelper.get_all_live_fund_list(end_date)
        if fund_list is None:
            print('[em_fund_bond] failed to get fund list')
            return False
        dt_list = ['20191231','20200331','20200630']
        res = []
        rank_list = [i for i in range(1,11,1)]
        for dt in dt_list:
            print(f'doing {dt}')
            _res = []
            for rank in rank_list:
                data=c.css(fund_list,"PRTKEYBONDNAME,PRTKEYBONDCODE,PRTKEYBONDVALUE,PRTKEYBONDTONAV",f"ReportDate={dt},Rank={rank},Ispandas=1")
                data = data.rename(columns={'PRTKEYBONDNAME':f'rank{rank}_bond','PRTKEYBONDCODE':f'rank{rank}_bond_code','PRTKEYBONDVALUE':f'rank{rank}_bondval','PRTKEYBONDTONAV':f'rank{rank}_bondweight'})
                data = data.drop('DATES',axis=1)
                _res.append(data)
            df = pd.concat(_res,axis=1)
            df['DATES'] = dt
            df = df.dropna(subset=['rank1_bond'])
            res.append(df)
            print(f'{dt} done')
        df = pd.concat(res).reset_index()
        em._data_helper._upload_raw(df, EMFundHoldBond.__table__.name)
        return True

    def em_macroeconomc_manually(self, file_name: str):
        df = pd.read_csv(f'./{file_name}')
        df = df[df.value.notna()]
        df['codes'] = df.codes.map(self._MACROECONOMIC_MAP)
        self._data_helper._upload_raw(df, EmMacroEconomicDaily.__table__.name)

    def em_macroeconomic_daily_history(self, end_date: str):
        data1 = self._em_edb_wrapper("EMM00588707,EMM00588705",
                                    f"IsLatest=0,StartDate=1990-01-01,EndDate={end_date},Ispanads=1")
        if data1 is None:
            print('[em_macroeconomic_daily_history] failed to get data1')
            return
        data2 = self._em_edb_wrapper("E1002041",
                                    f"IsLatest=0,StartDate=1990-01-01,EndDate={end_date},Ispanads=1")
        if data2 is None:
            print('[em_macroeconomic_daily_history] failed to get data2')
            return
        df = pd.concat([data1.reset_index(), data2.reset_index()])
        df = df.rename(columns={
            'CODES': 'codes',
            'DATES': 'datetime',
            'RESULT': 'value',
        })
        df = df[df.value.notna()]
        df['codes'] = df.codes.map(self._MACROECONOMIC_MAP)
        self._data_helper._upload_raw(df, EmMacroEconomicDaily.__table__.name)

    def em_macroeconomic_daily(self):
        try:
            data1 = self._em_edb_wrapper("EMM00588712,EMM00588707,EMM00588705,EMM00588703",
                                        f"IsLatest=1,Ispanads=1")
            if data1 is None:
                print('[em_macroeconomic_daily] failed to get data1')
                return False
            data2 = self._em_edb_wrapper("E1002041",
                                        f"IsLatest=1,Ispanads=1")
            if data2 is None:
                print('[em_macroeconomic_daily] failed to get data2')
                return False
            df = pd.concat([data1.reset_index(), data2.reset_index()])
            df = df.rename(columns={
                'CODES': 'codes',
                'DATES': 'datetime',
                'RESULT': 'value',
            })
            df = df[df.value.notna()]
            df['codes'] = df.codes.map(self._MACROECONOMIC_MAP)

            now_df = RawDataApi().get_em_macroeconomic_daily()
            if now_df is not None:
                df['datetime'] = pd.to_datetime(df.datetime, infer_datetime_format=True).dt.date
                df = df.merge(now_df.drop(columns='_update_time'), how='left', on=['codes', 'datetime', 'value'], indicator=True, validate='one_to_one')
                df = df[df._merge == 'left_only'].drop(columns=['_merge'])
            self._data_helper._upload_raw(df, EmMacroEconomicDaily.__table__.name)
            return True
        except Exception as e:
            print(e)
            traceback.print_exc()
            return False

    def em_macroeconomic_monthly_history(self, end_date: str):
        data = self._em_edb_wrapper("EMM00072301,EMM00073348,EMM00087084,EMM00087086,EMM00634721,EMM00008445,EMM00121996,EMM00122023,EMI00135328",
                                   f"IsLatest=0,StartDate=1990-01-01,EndDate={end_date},Ispanads=1")
        if data is None:
            print('[em_macroeconomic_monthly_history] failed to get data')
            return
        df = data.reset_index().rename(columns={
            'CODES': 'codes',
            'DATES': 'datetime',
            'RESULT': 'value',
        })
        df = df[df.value.notna()]
        df['codes'] = df.codes.map(self._MACROECONOMIC_MAP)
        self._data_helper._upload_raw(df, EmMacroEconomicMonthly.__table__.name)

    def em_macroeconomic_monthly(self):
        try:
            data = self._em_edb_wrapper("EMM00072301,EMM00073348,EMM00087084,EMM00087086,EMM00634721,EMM00008445,EMM00121996,EMM00122023,EMI00135328",
                                        "IsLatest=1,Ispanads=1")
            if data is None:
                print('[em_macroeconomic_monthly] failed to get data')
                return False
            df = data.reset_index().rename(columns={
                'CODES': 'codes',
                'DATES': 'datetime',
                'RESULT': 'value',
            })
            df = df[df.value.notna()]
            df['codes'] = df.codes.map(self._MACROECONOMIC_MAP)

            now_df = RawDataApi().get_em_macroeconomic_monthly()
            if now_df is not None:
                df['datetime'] = pd.to_datetime(df.datetime, infer_datetime_format=True).dt.date
                df = df.merge(now_df.drop(columns='_update_time'), how='left', on=['codes', 'datetime', 'value'], indicator=True, validate='one_to_one')
                df = df[df._merge == 'left_only'].drop(columns=['_merge'])
            self._data_helper._upload_raw(df, EmMacroEconomicMonthly.__table__.name)
            return True
        except Exception as e:
            print(e)
            traceback.print_exc()
            return False

    def em_fund_ipo_stats(self, csv_path):
        df_i = pd.read_excel(csv_path, encoding='utf_8_sig').dropna(subset=['基金全称'])
        df_i = df_i.drop(['打新年化收益率(%)','最新净值','净值日期','基金类型','基金全称','基金所属公司'], axis=1)
        b_d_str = i.split('.')[0].split('_')[1]
        e_d_str = i.split('.')[0].split('_')[2]
        start_date = datetime.datetime.strptime(b_d_str,'%Y%m%d').date()
        end_date = datetime.datetime.strptime(e_d_str,'%Y%m%d').date()
        col_name = {
            '基金代码':'em_id',
            '打中新股数量':'ipo_allocation_num',
            '累计获配股数(万股)':'ipo_allocation_share_num',
            '累计获配金额(万元)':'ipo_allocation_amount',
            '打新收益率(%)':'ipo_allocation_ret',
        }
        df_i = df_i.rename(columns=col_name)
        df_i.loc[:,'start_date'] = start_date
        df_i.loc[:,'end_date'] = end_date
        df_i = df_i.replace('——',np.nan)
        self._data_helper._upload_raw(df_i, EmFundIPOStats.__table__.name)

    def em_fund_conr_info(self, csv_path):
        df = pd.read_excel(csv_path, encoding='utf_8_sig', na_values='——').dropna(subset=['转债名称'])
        col_dic = {
            '代码':'em_id',
            '名称':'desc_name',
            '转债代码':'conv_bond_id',
            '转债名称':'bond_name',
            '发行方式':'ipo_method',
            '发行规模(亿元)':'ipo_size',
            '发行期限(年)':'duration',
            '信用评级':'rating',
            '票面利率(%)':'coupon_rate',
            '补偿利率(%)':'compensation_rate',
            '利率类型':'rate_type',
            '最新余额(万元)':'latest_balance',
            '付息频率':'payment_frequency',
            '付息说明':'payment_doc',
            '发行日期':'issue_date',
            '上市日期':'listing_date',
            '起息日期':'value_date',
            '到期日期':'date_of_expiry',
            '转股起始日':'conv_start_date',
            '初始转股价':'init_conv_price',
            '最新转股价':'latest_conv_price',
            '利率说明':'rate_doc',
            '赎回条款':'redemption_clause',
            '回售条款':'resale_clause',
            '特别向下修正条款':'downward_amendment',
            '转股条款':'conv_clause',
            '担保人':'warrent',
            '担保方式':'warrent_method',
            '担保范围':'warrent_scope',
            '信用等级':'credit_rating',
            '评估机构':'evaluation_agency',
            '主承销商':'lead_underwriter',
            '证监会行业(2012)':'industry',
            '东财行业':'em_industry'
            }
        df = df.rename(columns=col_dic)
        df = df.drop_duplicates(subset=['issue_date','conv_bond_id','bond_name'])
        self._data_helper._upload_raw(df_i, EmFundConvInfo.__table__.name)

    def em_fund_ipo_detail(self, csv_path, em_id):
        df_i = pd.read_excel(csv_path, encoding='utf_8_sig', na_values='——').dropna(subset=['名称'])
        df_i['em_id'] = em_id
        dic = {'代码':'stock_id',
            '名称':'desc_name',
            '配售日期':'placedate',
            '有效申购数量(万股)':'effective_volume',
            '实际配售数量(万股)':'actual_volume',
            '获配比例(%)':'place_rate',
            '获配金额(万元)':'place_amt',
            '锁定期限':'lock_up_period',
            '发行价格(元)':'issue_price',
            '首个开板日涨跌幅(%)':'first_open_d_ret',
            '上市以来最高涨幅(%)':'historcal_ret',
            '打新收益率(%)':'ipo_allocation_ret',
            '年化打新收益率(%)':'annual_allocation_ret',
            '证监会行业(2012)':'industry',
            '东财行业':'em_industry'
            }
        df_i = df_i.rename(columns = dic)
        self._data_helper._upload_raw(df_i, EmFundIPODetail.__table__.name)

    def em_stock_industrial_capital(self, end_date: str, is_history: bool = True) -> bool:
        def transform_columns_name(x: str) -> str:
            if x in ('EMM00593994', 'EMM00594026', 'EMM00594058', 'EMM00594090'):
                return 'inc_amt_all'
            elif x in ('EMM00594001', 'EMM00594033', 'EMM00594065', 'EMM00594097'):
                return 'inc_amt_gem'
            elif x in ('EMM00594010', 'EMM00594042', 'EMM00594074', 'EMM00594106'):
                return 'dec_amt_all'
            elif x in ('EMM00594017', 'EMM00594049', 'EMM00594081', 'EMM00594113'):
                return 'dec_amt_gem'
            assert False, f'got an unknown column name {x}'

        try:
            if not is_history:
                now_df = RawDataApi().get_em_stock_industrial_capital()
                options = 'IsLatest=1,Ispanads=1'
            else:
                options = f'IsLatest=0,StartDate=1990-01-01,EndDate={end_date},Ispanads=1'

            data = self._em_edb_wrapper("EMM00593994,EMM00594001,EMM00594010,EMM00594017,EMM00594026,EMM00594033,EMM00594042,EMM00594049,EMM00594058,EMM00594065,EMM00594074,EMM00594081,EMM00594090,EMM00594097,EMM00594106,EMM00594113", options)
            if data is None:
                print('[em_stock_industrial_capital] failed to get data')
                return False
            df = data.reset_index().rename(columns={
                'CODES': 'codes',
                'DATES': 'datetime',
                'RESULT': 'value',
            })
            df = df[df.value.notna()]
            df_dict: Dict[str, pd.DataFrame] = {}
            df_dict['D'] = df[df.codes.isin(['EMM00593994','EMM00594001','EMM00594010','EMM00594017'])]
            df_dict['W'] = df[df.codes.isin(['EMM00594026','EMM00594033','EMM00594042','EMM00594049'])]
            df_dict['M'] = df[df.codes.isin(['EMM00594058','EMM00594065','EMM00594074','EMM00594081'])]
            df_dict['Y'] = df[df.codes.isin(['EMM00594090','EMM00594097','EMM00594106','EMM00594113'])]
            df_list: List[pd.DataFrame] = []
            for period, one in df_dict.items():
                pivoted_one = one.pivot(index='datetime', columns='codes', values='value')
                pivoted_one = pivoted_one.rename(columns=transform_columns_name)
                pivoted_one['period'] = period
                df_list.append(pivoted_one.reset_index())

            df_all = pd.concat(df_list)
            if not is_history and now_df is not None:
                df_all['datetime'] = pd.to_datetime(df_all.datetime, infer_datetime_format=True).dt.date
                df_all = df_all.merge(now_df.drop(columns='_update_time'), how='left', on=['datetime', 'period', 'inc_amt_all', 'inc_amt_gem', 'dec_amt_all', 'dec_amt_gem'], indicator=True, validate='one_to_one')
                df_all = df_all[df_all._merge == 'left_only'].drop(columns=['_merge'])

                # 删掉旧的之后再插新的
                df_all.apply(lambda x: RawDataApi().delete_em_stock_industrial_capital(x.datetime, x.period), axis=1)
            self._data_helper._upload_raw(df_all, EmStockIndustrialCapital.__table__.name)
            return True
        except Exception as e:
            print(e)
            traceback.print_exc()
            return False

    def em_stock_to_hk_connect(self, end_date: str) -> bool:
        ret = True
        try:
            data_list: List[pd.Series] = []
            for ctr_name in (('SH', 'ActiveStockTOP10SHInfo'), ('SZ', 'ActiveStockTOP10SZInfo')):
                data = self._em_ctr_wrapper(ctr_name[1], "TRADEDATE,DEC_RANK,MSECUCODE,BMONEY,SMONEY,TVAL,ZTVAL,ZSZ,CNT,STR_PUBLISHNAMEDC3",f"period=1,TradeDate={end_date}")
                if data is None:
                    ret = False
                    print(f'[em_stock_to_hk_connect] failed to get data of {ctr_name[0]}')
                    continue
                for v in data.values():
                    data_list.append(pd.Series(v + [ctr_name[0]], index=['datetime', 'dec_rank', 'msecu_code', 'bmoney', 'smoney', 'tval', 'ztval', 'zsz', 'cnt', 'publish_name_dc3', 'market']))
            if data_list:
                df = pd.DataFrame(data_list)
                df['period'] = 'D'
                self._data_helper._upload_raw(df, EmSHSZToHKStockConnect.__table__.name)
        except Exception as e:
            print(e)
            traceback.print_exc()
        return ret

    def em_stock_to_hk_connect_history(self, start_date: str, end_date: str):
        trading_day_df = BasicDataApi().get_trading_day_list(start_date, end_date)
        for date in trading_day_df.datetime:
            print(f'do {date}')
            self.em_stock_to_hk_connect(date.strftime('%Y%m%d'))
            print(f'{date} done')

    def em_stock_industrial_capital_trade_detail(self, start_date: str, end_date: str) -> bool:
        try:
            data_list: List[pd.Series] = []
            data = self._em_ctr_wrapper('HoldTradeDetailInfo', 'SECURITYCODE,NOTICEDATE,SHAREHDNAME,SHAREHDTYPE,IS_controller,POSITION1,FX,CHANGENUM,BDSLZLTB,TGDZJYPTZR,'
                                                               'BDHCYLTGSL,BDHCYLTSLZLTGB,BDHCGZS,BDHCGBL,JYPJJ,NEW,DEC_JYPJJ_RATE,BDQJGPJJ,BDBFCKSZ,BDQSRQ,BDJZRQ,'
                                                               'CLB_REMARK,STR_PUBLISHNAMEDC3,TYPE',
                                                               f'StartDate={start_date},EndDate={end_date},secucode=,HoldType=0')
            if data is None:
                print(f'[em_stock_industrial_capital_trade_detail] failed to get data')
                return False
            for v in data.values():
                data_list.append(pd.Series(v, index=['sec_code', 'notice_date', 'share_hd_name', 'share_hd_type', 'is_ctrl', 'position1', 'fx', 'change_num', 'bdsl_zltb',
                                                     'tg_dzjy_ptzr', 'bdh_cyltgsl', 'bdh_cyltslzltgb', 'bdh_cgzs', 'bdh_cgbl', 'jyp_jj', 'latest_price', 'dec_jypjj_rate',
                                                     'bdqj_gpjj', 'bdbf_cksz', 'bd_qsrq', 'bd_jzrq', 'clb_remark', 'publish_name_dc3', 'trade_type']))
            if data_list:
                df = pd.DataFrame(data_list)
                self._data_helper._upload_raw(df, EmStockIndustrialCapitalTradeDetail.__table__.name)
            return True
        except Exception as e:
            print(e)
            traceback.print_exc()
            return False

    def em_stock_research_info(self, end_date: str) -> bool:
        try:
            stock_list = self._em_sector_wrapper('001004', end_date)
            if stock_list is None:
                print('[em_stock_price] failed to get stock id list')
                return False
            data = self._em_css_wrapper(stock_list, "RESERCHNUM,RESERCHOTHERNUM,RESERCHINSTITUTENUM,RESERCHSECUNUM,RESERCHSECUNAME,RESERCHSECUFREQUENCY1", f"StartDate={end_date},EndDate={end_date}")
            if data is None:
                print(f'[em_stock_research_info] failed to get data')
                return False
            data['DATES'] = end_date
            df = data[data.drop(columns='DATES').notna().any(axis=1)]
            df = df.reset_index().rename(columns=lambda x: EmStockResearchInfo.__getattribute__(EmStockResearchInfo, x).name)
            self._data_helper._upload_raw(df, EmStockResearchInfo.__table__.name)
            return True
        except Exception as e:
            print(e)
            traceback.print_exc()
            return False

    def em_stock_research_info_history(self, start_date: str, end_date: str):
        trading_day_df = BasicDataApi().get_trading_day_list(start_date, end_date)
        for date in trading_day_df.datetime:
            print(f'do {date}')
            self.em_stock_research_info(date.strftime('%Y%m%d'))
            print(f'{date} done')

    def download_all(self, start_date, end_date):
        failed_tasks = []
        # If em_tradedates fails, there is no trading day between start_date and end_date
        # Stop and return
        if not self.em_tradedates(start_date, end_date):
            failed_tasks.append('em_tradedates')
            return failed_tasks

        # 这里先更新，更新完之后可以看一下中证指数的列表
        if not self.em_index_component(start_date, end_date):
            failed_tasks.append('em_index_component')

        # could only update the end_date's data
        if not self.em_fund_status(end_date, is_history=False):
            failed_tasks.append('em_fund_status')

        if not self.em_fund_nav(start_date, end_date):
            failed_tasks.append('em_fund_nav')

        index_info_df = BasicDataApi().get_index_info()
        if index_info_df is None:
            failed_tasks.append('retrieve data for em_index_price')
        else:
            index_info_df = index_info_df[index_info_df.price_source == IndexPriceSource.default]
            index_ids = set(index_info_df.em_id.to_list())
            index_ids.discard(None)
            if self._index_block_list is not None:
                index_ids -= self._index_block_list
            if not self.em_index_price(index_ids, start_date, end_date):
                failed_tasks.append('em_index_price')

        if not self.em_stock_info(end_date):
            failed_tasks.append('em_stock_info')

        if not self.em_stock_price(start_date, end_date):
            failed_tasks.append('em_stock_price')

        if not self.em_stock_post_price(start_date, end_date):
            failed_tasks.append('em_stock_post_price')

        if not self.em_index_val(start_date, end_date):
            failed_tasks.append('em_index_val')

        if not self.em_stock_daily_info(start_date, end_date):
            failed_tasks.append('em_stock_daily_info')

        if not self.em_fund_list(start_date, end_date):
            failed_tasks.append('em_fund_list')
        else:
            new_fund_list, _ = RawDataHelper.get_new_and_delisted_fund_list(end_date)
            if new_fund_list is None:
                failed_tasks.append('get new_fund_list')
            else:
                if not self.em_fund_info(new_fund_list):
                    failed_tasks.append('em_fund_info')
                if not self.em_fund_fee(new_fund_list):
                    failed_tasks.append('em_fund_fee')

        if not self.em_macroeconomic_price(start_date, end_date):
            failed_tasks.append('em_macroeconomic_price')

        if not self.em_fund_rate(end_date):
            failed_tasks.append('em_fund_rate')

        if not self.em_fund_benchmark(end_date):
            failed_task.append('em_fund_benchmark')

        if not self.em_macroeconomic_monthly():
            failed_tasks.append('em_macroeconomic_monthly')

        if not self.em_macroeconomic_daily():
            failed_tasks.append('em_macroeconomic_daily')

        if not self.em_stock_industrial_capital(end_date, is_history=False):
            failed_tasks.append('em_stock_industrial_capital')

        if not self.em_stock_to_hk_connect(end_date):
            failed_tasks.append('em_stock_to_hk_connect')

        if not self.em_stock_industrial_capital_trade_detail(start_date, end_date):
            failed_tasks.append('em_stock_industrial_capital_trade_detail')

        if not self.em_stock_research_info(end_date):
            failed_tasks.append('em_stock_research_info')

        return failed_tasks


if __name__ == '__main__':
    em = EmRawDataDownloader(RawDataHelper())
    # em.em_tradedates('20000101', '20091231')
    # em.em_index_val('20200428', '20200428')
    # em.em_fund_nav('20200415', '20200427')
    # em.em_index_price_history('H00140.SH', '20050101', '20200419')
    # em.em_fund_scale_history()
    # em.em_stock_fin_fac('20161231')
    # em.em_stock_info('20200424')
    # em.em_stock_price('20041231', '20041231')
    # em.em_stock_post_price('20041231', '20041231')
    # em.em_stock_daily_info('20041231', '20041231')
    # em.em_stock_price_history('./')
    # em.em_stock_post_price_history('./')
    # em.em_stock_daily_info_history('./')
    # em.em_stock_refinancing('20170101', '20181231')
    # em.add_columns_to_fin_fac('20190930')
    # em.em_fund_status('20200513')
    # em.em_index_component('20200528', '20200529')
    # em.em_fund_list('20200702', '20200702')
    # em.em_fund_info_history('20200707')
    # em.em_fund_benchmark_history('20200929', '20201004')
    # em.em_index_info(['399972.SZ'])
    # em.em_fund_info({'165807.SZ'})
    # em.em_stock_estimate_fac('20200101', '20200709', 2020)
    # em.cs_index_component('20200713')
    # em.em_macroeconomic_price('20200714', '20200714')
    # em.em_fund_fee_history('20200715')
    # em.em_fund_asset('20200819')
    # em.em_fund_holding_rate_history('20200819')
    # em.em_macroeconomic_monthly_history('20200914')
    # em.em_macroeconomic_monthly()
    # em.em_macroeconomic_daily_history('20200914')
    # em.em_macroeconomic_daily()
