# coding:utf-8
import sys
from tools_hjh import DBConn
from tools_hjh import Log
from tools_hjh import ThreadPool
from tools_hjh import Tools
from tools_hjh import HTTPTools
import json
import time
import datetime
from dateutil.relativedelta import relativedelta
import math
from tools_hjh.Tools import str_exception
from bs4 import BeautifulSoup

stop_flag = False
page_num = None

date = Tools.locatdate()
log = Log('E:/MyFiles/MyPy/log/fund/' + date + '.log')

host = 'https://'

chrome_path = r'D:\MyApps\CentBrowser\App\chrome.exe'
chromedriver_path = r'D:\MyApps\CentBrowser\chromedriver.exe'

headers = {
    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
    'accept-encoding': 'text',
    'accept-language': 'zh-CN,zh;q=0.9',
    'cache-control': 'no-cache',
    'dnt': '1',
    'pragma': 'no-cache',
    'sec-ch-ua': 'x86;v=99, Windows;v=10, Surface Laptop Studio;v=1',
    'sec-ch-ua-mobile': '?0',
    'sec-ch-ua-platform': '"Windows"',
    'Referer':'http://fundf10.eastmoney.com/',
    'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36',
    'cookie':'qgqp_b_id=83a199356104c7aa279534a8ad26f63a; emshistory=%5B%22%E7%BE%8E%E5%85%83%E5%85%91%E7%A6%BB%E5%B2%B8%E4%BA%BA%E6%B0%91%E5%B8%81%22%5D; st_nvi=CcZVuMTz5IhmsufyNCKse50b5; nid=00e15026f1062a9c810d5874360ccee6; nid_create_time=1753455327603; gvi=VkSGuR16GXiIJZI_ZREui577f; gvi_create_time=1753455327603; nid_id=148913046; websitepoptg_api_time=1754570340785; ASP.NET_SessionId=b3ymoksfubguqfwo5c14g0j1; st_si=75005746448136; st_asi=delete; searchbar_code=007229_007228_008566_006210_006212_009018_001247_019533_486002_510080; EMFUND0=08-07%2022%3A06%3A59@%23%24%u534E%u5B89%u4E2D%u503A7-10%u5E74%u56FD%u5F00%u503AC@%23%24007229; EMFUND1=08-07%2022%3A07%3A21@%23%24%u534E%u5B89%u4E2D%u503A7-10%u5E74%u56FD%u5F00%u503AA@%23%24007228; EMFUND2=08-07%2022%3A08%3A35@%23%24%u8702%u5DE2%u6DFB%u76C8%u7EAF%u503AA@%23%24008566; EMFUND3=08-07%2022%3A25%3A37@%23%24%u4E1C%u65B9%u81FB%u5B9D%u7EAF%u503A%u503A%u5238A@%23%24006210; EMFUND4=08-07%2022%3A33%3A33@%23%24%u897F%u90E8%u5229%u5F97%u805A%u6CF018%u4E2A%u6708%u5B9A%u5F00%u503AA@%23%24009018; EMFUND5=08-08%2000%3A34%3A43@%23%24%u534E%u6CF0%u67CF%u745E%u65B0%u5229%u6DF7%u5408A@%23%24001247; EMFUND6=08-08%2012%3A04%3A47@%23%24%u946B%u5143%u6CFD%u5229C@%23%24019533; EMFUND9=08-08%2022%3A12%3A17@%23%24%u957F%u76DB%u5168%u503A%u6307%u6570%u589E%u5F3A%u503A%u5238A@%23%24510080; fullscreengg=1; fullscreengg2=1; EMFUND8=08-09%2001%3A08%3A39@%23%24%u5DE5%u94F6%u5168%u7403%u7CBE%u9009%u80A1%u7968%28QDII%29@%23%24486002; EMFUND7=08-09 01:12:00@#$%u5DE5%u94F6%u5168%u7403%u80A1%u7968%28QDII%29%u4EBA%u6C11%u5E01@%23%24486001; st_pvi=49894389850834; st_sp=2025-07-17%2004%3A08%3A29; st_inirUrl=https%3A%2F%2Fquote.eastmoney.com%2Fgb%2FzsNDX100.html; st_sn=49; st_psi=2025080901120046-112200305282-5761806720'
}

try:
    sys_argv_1 = sys.argv[1]
except:
    sys_argv_1 = 'download'
try:
    sys_argv_2 = int(sys.argv[2])
except:
    sys_argv_2 = -1  # 获取条数
try:
    sys_argv_3 = sys.argv[3]
except:
    sys_argv_3 = 2
    
thread_num = 64


def main(): 
    if sys_argv_3 == 1:
        db = DBConn('pgsql', host='127.0.0.1', port=5432, db='fund', username='postgres', password='postgres', poolsize=8)
        try:
            createDatabase(db, rebuild=False)
            get_fund_main(db, HTTPTools)
            get_ndx(db)
            get_ndx_pe(db)
            # get_fund_dv(db, HTTPTools, begin_code='000000', code=None, name_like='%国泰纳斯达克100指数%')
            # get_fund_dv(db, HTTPTools, begin_code='000000', code=None, name_like='%纳斯达克%')
            # get_fund_dv(db, HTTPTools, begin_code='000000', code=None, name_like='%纳指%')
            get_fund_dv(db, HTTPTools, begin_code='000000', code=None, name_like='%%')
            get_zf(db)
            get_val(db)
            get_hc(db)
            get_no_new_high_days(db)
            get_ret_ann(db)
            # get_excel()
            # backtesting(db)
        finally:
            db.close()
            pass
    else:
        get_excel()


def createDatabase(db, rebuild=False):
    if rebuild:
        db.run('drop table if exists t_m')
        db.run('drop table if exists t_e')
        db.run('drop table if exists t_e_imf')

    t_m = '''
        create table if not exists t_m(
            code varchar(9),
            name varchar(255),
            type varchar(255),
            primary key(code)
        )
    '''
    
    t_e = '''
        create table if not exists t_e(
            code varchar(9),
            date date,
            dwjz real,
            ljjz real,
            zf real,
            fh varchar(255),
            val real,
            hc real,
            ret_1y_ann real,
            ret_3y_ann real,
            ret_5y_ann real,
            ret_6m real,
            ret_3m real,
            ret_1m real,
            primary key(code, date)
        )
    '''
    
    # 货币基金的每日滚动7日年化
    t_e_imf = '''
        create table if not exists t_e_imf(
            code varchar(9),
            date date,
            ret_7d_ann real,
            primary key(code, date)
        )
    '''
        
    idx_e_code_date_ret_1y_ann = '''
        CREATE INDEX IF NOT EXISTS idx_e_code_date_syl
        ON t_e (code, date)
        INCLUDE (ret_1y_ann, ret_3y_ann, ret_5y_ann, hc, zf)
    '''
    
    tmp_code_mindate_1 = '''
        insert into tmp_code_mindate select code,min(date) from t_e t where t.code not in (select code from tmp_code_mindate) group by code
    '''
    tmp_code_mindate_2 = '''
        insert into tmp_code_mindate select code,min(date) from t_e_imf t where t.code not in (select code from tmp_code_mindate) group by code
    '''
    
    db.run(t_m)
    db.run(t_e)
    db.run(t_e_imf)
    db.run(idx_e_code_date_ret_1y_ann)
    db.run(tmp_code_mindate_1)
    db.run(tmp_code_mindate_2)


def get_fund_main(db, rep):
    sql = 'insert into t_m(code,name,type) values(?,?,?) on conflict(code) do update set name = EXCLUDED.name, type = EXCLUDED.type'
    page = rep.get('http://fund.eastmoney.com/js/fundcode_search.js')
    page = page.encode('utf8')[3:].decode('utf8')
    page = page.replace('var r = ', '').replace(';', '')
    fund_list = json.loads(page)
    params = []
    for fund in fund_list:
        code = fund[0]
        name = fund[2]
        type_ = fund[3]
        params.append((code, name, type_))
    num = db.run(sql, params)
    db.run("update t_m set type = '货币型-浮动净值' where code in('511880','003816')")
    log.info('get_fund_main', num)

    
def get_fund_top10_holdings(db, rep):
    pass


# 从库中最大的date开始，获取至今的数据，如果中间缺了不会补
def get_fund_dv(db, rep, begin_code='000000', code=None, name_like=''):
            
    sql = "insert into t_e(code,date,dwjz,ljjz,fh) values(?,?,?,?,?) on conflict(code,date) do update set dwjz = EXCLUDED.dwjz, ljjz = EXCLUDED.ljjz, fh = EXCLUDED.fh"
    sql_imf = "insert into t_e_imf(code,date,ret_7d_ann) values(?,?,?) on conflict(code,date) do update set ret_7d_ann = EXCLUDED.ret_7d_ann"
    vals_all = []
    imfs_all = []

    def get_fund_dv_one(code, type_, rep, times=1):
        try:
            page_size = 20
            timestamp = str(int(time.time() * 1000))
            end_date = Tools.locatdate()
            start_date = db.run("select to_char(max(date),'YYYY-MM-DD') from t_e where code = ?", (code,)).get_rows()[0][0]
            if start_date is None:
                start_date = '1949-10-01'
            if end_date == start_date:
                log.info('get_fund_dv', code, 0, len(vals_all), times)
                return
            
            all_size = (datetime.datetime.strptime(end_date, "%Y-%m-%d").date() - datetime.datetime.strptime(start_date, "%Y-%m-%d").date()).days
            # all_size = 230
            page_num = math.ceil(all_size / page_size)
            
            rss_num = 0
            for page_idx in range(1, page_num + 1):
                url = 'http://api.fund.eastmoney.com/f10/lsjz?callback=jQuery18306743973867400965_1722217638986&fundCode=' + code + '&pageIndex=' + str(page_idx) + '&pageSize=' + str(page_size) + '&startDate=' + start_date + '&endDate=' + end_date + '&_=' + timestamp
                page = rep.get(url, headers=headers)
                # Tools.echo(page, 'E:\page.txt')
                page = page.replace('jQuery18306743973867400965_1722217638986', '').strip()
                page = page.split('"LSJZList":')[1].split(',"FundType"')[0]
                rss = json.loads(page)
                rss_num = rss_num + len(rss)
                
                if len(rss) == 0:
                    break
                
                for rs in reversed(rss):
                    date_ = rs['FSRQ']
                    dwjz = rs['DWJZ']
                    if dwjz == '':
                        dwjz = None
                    ljjz = rs['LJJZ']
                    if ljjz == '':
                        ljjz = None
                    zf = rs['JZZZL']
                    if zf == '':
                        zf = 0
                    fh = rs['FHSP']
                    if fh == '':
                        fh = None
         
                    if type_ == '货币型-普通货币':
                        imfs_all.append((code, date_, ljjz))
                    else:
                        vals_all.append((code, date_, dwjz, ljjz, fh))

            log.info('get_fund_dv', code, rss_num, len(vals_all) + len(imfs_all), times)
                    
        except Exception as _:
            if times <= 99:
                log.warning('get_fund_dv', code, times, str_exception(_))
                time.sleep(1)
                times = times + 1
                get_fund_dv_one(code, rep, times)
            else:
                log.error('get_fund_dv', code, times, str_exception(_))
    
    if code is not None:
        get_code_sql = '''
            select code,type from t_m 
            where code = ?
        '''
        funds = db.run(get_code_sql, (code,)).get_rows()
    else:
        get_code_sql = '''
            select code,type from t_m 
            where 1=1
            -- and type not in('货币型-普通货币')
            and code >= ?
            and name like ?
            order by code
        '''
        funds = db.run(get_code_sql, (begin_code, name_like)).get_rows()
    
    tp = ThreadPool(thread_num)
    for fund in funds:
        if len(vals_all) >= 1000000 or len(imfs_all) >= 1000000:
            tp.wait()
            if len(vals_all) >= 1000000:
                num = db.run(sql, vals_all)
                log.info('get_fund_dv', num)
                vals_all.clear()
            if len(imfs_all) >= 1000000:
                num = db.run(sql_imf, imfs_all)
                log.info('get_fund_dv', num)
                imfs_all.clear()
        else:
            code = fund[0]
            type_ = fund[1]
            tp.run_wait(get_fund_dv_one, (code, type_, rep))
            
    tp.wait()
    num1 = db.run(sql, vals_all)
    num2 = db.run(sql_imf, imfs_all)
    num = num1 + num2
    log.info('get_fund_dv', num)
    vals_all.clear()


def get_zf(db):

    def get_zf_one(db, code):
        sql = 'update t_e set zf = ? where code = ? and date = ?'
        params = []
        rss = db.run('select date,dwjz,fh from t_e where code = ? and zf is null order by date', (code,)).get_rows()
        for rs in rss:
            date_ = rs[0]
            dwjz = float(rs[1])
            fh = rs[2]
            try:
                previous_dwjz = float(db.run('select dwjz from t_e where code = ? and date < ? order by date desc limit 1', (code, date_)).get_rows()[0][0])
            except:
                zf = 0
                params.append((zf, code, date_))
                continue
            
            if '每份基金份额分拆' in str(fh):
                fh_cf = float(fh.replace('每份基金份额分拆', '').replace('份', ''))
                zf = ((dwjz * fh_cf) - previous_dwjz) / previous_dwjz * 100
            elif '每份派现金' in str(fh):
                fh_xj = float(fh.replace('每份派现金', '').replace('元', ''))
                zf = ((dwjz + fh_xj) - previous_dwjz) / previous_dwjz * 100
            elif '每份基金份额折算' in str(fh):
                fh_zs = float(fh.replace('每份基金份额折算', '').replace('份', ''))
                zf = (dwjz - (previous_dwjz / fh_zs)) / (previous_dwjz / fh_zs) * 100
            else:
                zf = (dwjz - previous_dwjz) / previous_dwjz * 100
            
            params.append((zf, code, date_))
        try:
            num = db.run(sql, params)
            if num > 0:
                log.info('get_zf', code, num, len(params))
        except Exception as _:
            log.error('get_zf', code, str(_))

    tp = ThreadPool(64)
    log.info('get_zf', 'begin')
    funds = db.run("select code from t_m order by code").get_rows()
    
    for fund in funds:
        num = db.run('select count(1) from t_e where code = ? and dwjz is null', (fund[0],)).get_rows()[0][0]
        if num == 0:
            tp.run(get_zf_one, (db, fund[0]))
    tp.wait()

    
def get_val(db):

    def get_val_one(db, code):
        sql = 'update t_e set val = ? where code = ? and date = ?'
        params = []   
        rss = db.run('select date,zf from t_e where code = ? and val is null order by date', (code,)).get_rows()
        next_val = 1 
        for rs in rss:
            try:
                date_ = rs[0]
                zf = rs[1]
                try:
                    next_val_db = None
                    next_val_db = db.run('select val from t_e where code = ? and date < ? order by date desc limit 1', (code, date_)).get_rows()[0][0]
                except:
                    next_val_db = None
                if next_val_db is not None:
                    next_val = next_val_db
                val = float(next_val) * (1 + float(zf) / 100)
                params.append((val, code, date_))
                next_val = val
            except Exception as _:
                print(_, code, date_, next_val, zf)
        try:
            num = db.run(sql, params)
            if num > 0:
                log.info('get_val', code, num, len(params))
        except Exception as _:
            log.error('get_val', code, str(_))

    tp = ThreadPool(64)
    log.info('get_val', 'begin')
    funds = db.run("select code from t_m order by code").get_rows()
    
    for fund in funds:
        num = db.run('select count(1) from t_e where code = ? and zf is null', (fund[0],)).get_rows()[0][0]
        if num == 0:
            tp.run(get_val_one, (db, fund[0]))
    tp.wait()


def get_hc(db):

    def get_hc_one(db, code):
        params = []      
        sql = 'update t_e set hc = ? where code = ? and date = ?'
        rows = db.run('select date,val from t_e where hc is null and code = ? order by date', (code,)).get_rows()
        for row in rows:
            date_now = row[0]
            date_before = date_now - relativedelta(years=1)
            val = row[1]
            max_val = db.run('select max(val) from t_e where date <= ? and date >= ? and code = ?', (date_now, date_before, code)).get_rows()[0][0]
            val = float(val)
            max_val = float(max_val)
            hc = round((max_val - val) / max_val * 100, 3)
            params.append((hc, code, date_now))
        try:
            num = db.run(sql, params)
            if num > 0:
                log.info('get_hc', code, num, len(params))
        except Exception as _:
            log.error('get_hc', code, str(_))
    
    tp = ThreadPool(64)
    log.info('get_hc', 'begin')
    funds = db.run("select code from t_m order by code").get_rows()
    
    for fund in funds:
        num = db.run('select count(1) from t_e where code = ? and val is null', (fund[0],)).get_rows()[0][0]
        if num == 0:
            tp.run(get_hc_one, (db, fund[0]))
    tp.wait()

    
def get_no_new_high_days(db):

    def get_no_new_high_days_one(db, code):
        params = []      
        sql = 'update t_e set no_new_high_days = ? where code = ? and date = ?'
        rows = db.run('select date,val from t_e where no_new_high_days is null and code = ? order by date', (code,)).get_rows()
        for row in rows:
            date_now = row[0]
            val = row[1]
            date_before, max_val = db.run('select date,val from t_e where date <= ? and code = ? order by val desc limit 1', (date_now, code)).get_rows()[0]
            if max_val <= val:
                no_new_high_days = 0
            else:
                no_new_high_days = (date_now - date_before).days
            params.append((no_new_high_days, code, date_now))
        try:
            num = db.run(sql, params)
            if num > 0:
                log.info('get_no_new_high_days', code, num, len(params))
        except Exception as _:
            log.error('get_no_new_high_days', code, str(_))
    
    tp = ThreadPool(64)
    log.info('get_no_new_high_days', 'begin')
    funds = db.run("select code from t_m order by code").get_rows()
    
    for fund in funds:
        num = db.run('select count(1) from t_e where code = ? and val is null', (fund[0],)).get_rows()[0][0]
        if num == 0:
            tp.run(get_no_new_high_days_one, (db, fund[0]))
    tp.wait()


def get_ret_ann(db):

    def get_ret_1y_ann_one(db, code):
        ret_1y_ann_params = []
        ret_1y_ann_sql = 'update t_e set ret_1y_ann = ? where code = ? and date = ?'
        rows = db.run('select date,val from t_e where ret_1y_ann is null and code = ? order by date', (code,)).get_rows()
        for row in rows:
            date_now = row[0]
            date_before = date_now - relativedelta(years=1)
            val = row[1]
            val = float(val)
            try:
                val_before = db.run('select val from t_e where date >= ? and code = ? order by date limit 1', (date_before, code)).get_rows()[0][0]
                val_before = float(val_before)
                ret_1y_ann = round((val - val_before) / val_before * 100, 3)
            except Exception as _:
                ret_1y_ann = None
            if ret_1y_ann is None:
                break
            ret_1y_ann_params.append((ret_1y_ann, code, date_now))
        try:
            num = db.run(ret_1y_ann_sql, ret_1y_ann_params)
            if num > 0:
                log.info('get_ret_1y_ann_one', code, num, len(ret_1y_ann_params))
        except Exception as _:
            log.error('get_ret_1y_ann_one', code, str(_))
        
    def get_ret_3y_ann_one(db, code):
        ret_3y_ann_params = []
        ret_3y_ann_sql = 'update t_e set ret_3y_ann = ? where code = ? and date = ?'
        rows = db.run('select date,val from t_e where ret_3y_ann is null and code = ? order by date', (code,)).get_rows()
        for row in rows:
            date_now = row[0]
            date_before = date_now - relativedelta(years=3)
            val = row[1]
            val = float(val)
            try:
                val_before = db.run('select val from t_e where date >= ? and code = ? order by date limit 1', (date_before, code)).get_rows()[0][0]
                val_before = float(val_before)
                ret_3y_ann = round((val - val_before) / val_before / 3 * 100, 3)
            except Exception as _:
                ret_3y_ann = None
            if ret_3y_ann is None:
                break
            ret_3y_ann_params.append((ret_3y_ann, code, date_now))
        try:
            num = db.run(ret_3y_ann_sql, ret_3y_ann_params)
            if num > 0:
                log.info('get_ret_3y_ann_one', code, num, len(ret_3y_ann_params))
        except Exception as _:
            log.error('get_ret_3y_ann_one', code, str(_))
        
    def get_ret_5y_ann_one(db, code):
        ret_5y_ann_params = []
        ret_5y_ann_sql = 'update t_e set ret_5y_ann = ? where code = ? and date = ?'
        rows = db.run('select date,val from t_e where ret_5y_ann is null and code = ? order by date', (code,)).get_rows()
        for row in rows:
            date_now = row[0]
            date_before = date_now - relativedelta(years=5)
            val = row[1]
            val = float(val)
            try:
                val_before = db.run('select val from t_e where date >= ? and code = ? order by date limit 1', (date_before, code)).get_rows()[0][0]
                val_before = float(val_before)
                ret_5y_ann = round((val - val_before) / val_before / 5 * 100, 3)
            except:
                ret_5y_ann = None
            if ret_5y_ann is None:
                break
            ret_5y_ann_params.append((ret_5y_ann, code, date_now))
        try:
            num = db.run(ret_5y_ann_sql, ret_5y_ann_params)
            if num > 0:
                log.info('get_ret_5y_ann_one', code, num, len(ret_5y_ann_params))
        except Exception as _:
            log.error('get_ret_5y_ann_one', code, str(_))
            
    def get_ret_6m_one(db, code):
        ret_6m_params = []
        ret_6m_sql = 'update t_e set ret_6m = ? where code = ? and date = ?'
        rows = db.run('select date,val from t_e where ret_6m is null and code = ? order by date', (code,)).get_rows()
        for row in rows:
            date_now = row[0]
            date_before = date_now - relativedelta(months=6)
            val = row[1]
            val = float(val)
            try:
                val_before = db.run('select val from t_e where date >= ? and code = ? order by date limit 1', (date_before, code)).get_rows()[0][0]
                val_before = float(val_before)
                ret_6m = round((val - val_before) / val_before * 100, 3)
            except:
                ret_6m = None
            if ret_6m is None:
                break
            ret_6m_params.append((ret_6m, code, date_now))
        try:
            num = db.run(ret_6m_sql, ret_6m_params)
            if num > 0:
                log.info('get_ret_6m_one', code, num, len(ret_6m_params))
        except Exception as _:
            log.error('get_ret_6m_one', code, str(_))

    def get_ret_3m_one(db, code):
        ret_3m_params = []
        ret_3m_sql = 'update t_e set ret_3m = ? where code = ? and date = ?'
        rows = db.run('select date,val from t_e where ret_3m is null and code = ? order by date', (code,)).get_rows()
        for row in rows:
            date_now = row[0]
            date_before = date_now - relativedelta(months=3)
            val = row[1]
            val = float(val)
            try:
                val_before = db.run('select val from t_e where date >= ? and code = ? order by date limit 1', (date_before, code)).get_rows()[0][0]
                val_before = float(val_before)
                ret_3m = round((val - val_before) / val_before * 100, 3)
            except:
                ret_3m = None
            if ret_3m is None:
                break
            ret_3m_params.append((ret_3m, code, date_now))
        try:
            num = db.run(ret_3m_sql, ret_3m_params)
            if num > 0:
                log.info('get_ret_3m_one', code, num, len(ret_3m_params))
        except Exception as _:
            log.error('get_ret_3m_one', code, str(_))

    def get_ret_1m_one(db, code):
        ret_1m_params = []
        ret_1m_sql = 'update t_e set ret_1m = ? where code = ? and date = ?'
        rows = db.run('select date,val from t_e where ret_1m is null and code = ? order by date', (code,)).get_rows()
        for row in rows:
            date_now = row[0]
            date_before = date_now - relativedelta(months=6)
            val = row[1]
            val = float(val)
            try:
                val_before = db.run('select val from t_e where date >= ? and code = ? order by date limit 1', (date_before, code)).get_rows()[0][0]
                val_before = float(val_before)
                ret_1m = round((val - val_before) / val_before * 100, 3)
            except:
                ret_1m = None
            if ret_1m is None:
                break
            ret_1m_params.append((ret_1m, code, date_now))
        try:
            num = db.run(ret_1m_sql, ret_1m_params)
            if num > 0:
                log.info('get_ret_1m_one', code, num, len(ret_1m_params))
        except Exception as _:
            log.error('get_ret_1m_one', code, str(_))
    
    tp = ThreadPool(64)
    log.info('get_ret_ann', 'begin')
    funds = db.run("select code from t_m order by code").get_rows()
    for fund in funds:
        num = db.run('select count(1) from t_e where code = ? and val is null', (fund[0],)).get_rows()[0][0]
        if num == 0:
            tp.run(get_ret_1y_ann_one, (db, fund[0]))
            tp.run(get_ret_3y_ann_one, (db, fund[0]))
            tp.run(get_ret_5y_ann_one, (db, fund[0]))
            tp.run(get_ret_6m_one, (db, fund[0]))
            tp.run(get_ret_3m_one, (db, fund[0]))
            tp.run(get_ret_1m_one, (db, fund[0]))
    tp.wait()


def get_excel():
    codes = []
    codes.append('000218')
    codes.append('000436')
    codes.append('000834')
    codes.append('005561')
    codes.append('015299')
    codes.append('016055')
    codes.append('016452')
    codes.append('016532')
    codes.append('017091')
    codes.append('018043')
    codes.append('018966')
    codes.append('019118')
    codes.append('019172')
    codes.append('019441')
    codes.append('019524')
    codes.append('019547')
    codes.append('019736')
    codes.append('040046')
    codes.append('160213')
    codes.append('161130')
    codes.append('270042')
    codes.append('539001')
    codes.sort()
    for code in codes:
        time.sleep(0.1)
        url = 'https://fund.eastmoney.com/' + code + '.html'
        text = HTTPTools.get(url, headers)
        n1 = text.split('近1年：')[1].split('ui-font-middle')[1].split('ui-num">')[1].split('<')[0]
        # n3 = text.split('近3年：')[1].split('ui-font-middle')[1].split('ui-num">')[1].split('<')[0]
        y1 = text.split('近1月：')[1].split('ui-font-middle')[1].split('ui-num">')[1].split('<')[0]
        y3 = text.split('近3月：')[1].split('ui-font-middle')[1].split('ui-num">')[1].split('<')[0]
        y6 = text.split('近6月：')[1].split('ui-font-middle')[1].split('ui-num">')[1].split('<')[0]
        if '暂停申购' in text:
            xe = '0'
        else:
            try:
                xe = text.split('单日累计购买上限')[1].split('元<')[0]
            except:
                xe = '/'
            if str(xe).endswith('万'):
                xe = int(float(xe.replace('万', '')) * 10000)
        
        try:
            clr = text.split('成 立 日</span>：')[1].split('</td>')[0]
        except:
            clr = ''
        try:
            nhgzwc = text.split('年化跟踪误差：</a>')[1].split('%')[0] + '%'
        except:
            nhgzwc = ''
            
        # 夏普比率
        url2 = 'http://fundf10.eastmoney.com/tsdata_' + code + '.html'
        text2 = HTTPTools.get(url2, headers)
        try:
            xp1 = text2.split("夏普比率</td><td class='num'>")[1].split("</td>")[0].replace("<td class='num'>", '')
        except:
            xp1 = ''
        # try:
            # xp2 = text2.split("夏普比率</td><td class='num'>")[1].split("</td>")[1].replace("<td class='num'>", '')
        # except:
            # xp2 = ''
        # try:
            # xp3 = text2.split("夏普比率</td><td class='num'>")[1].split("</td>")[2].replace("<td class='num'>", '')
        # except:
            # xp3 = ''
            
        # 年度季度涨幅
        url3 = 'http://fundf10.eastmoney.com/FundArchivesDatas.aspx?type=jdndzf&code=' + code + '&rt=0.6097534654587713'
        text3 = HTTPTools.get(url3, headers)
        bs = BeautifulSoup(text3, features="lxml")
        trs = bs.find('table', class_='w782 comm jndxq').find('tbody').find_all('tr')
        n2025q1 = float(trs[0].find_all('td')[1].text.replace('%', '').replace('---', '0')) / 100
        n2025q2 = float(trs[0].find_all('td')[2].text.replace('%', '').replace('---', '0')) / 100
        n2025q3 = float(trs[0].find_all('td')[3].text.replace('%', '').replace('---', '0')) / 100
        n2025q4 = float(trs[0].find_all('td')[4].text.replace('%', '').replace('---', '0')) / 100
        n2025 = round(n2025q1 + n2025q2 + n2025q3 + n2025q4, 3)
        n2024q1 = float(trs[1].find_all('td')[1].text.replace('%', '').replace('---', '0')) / 100
        n2024q2 = float(trs[1].find_all('td')[2].text.replace('%', '').replace('---', '0')) / 100
        n2024q3 = float(trs[1].find_all('td')[3].text.replace('%', '').replace('---', '0')) / 100
        n2024q4 = float(trs[1].find_all('td')[4].text.replace('%', '').replace('---', '0')) / 100
        n2024 = round(n2024q1 + n2024q2 + n2024q3 + n2024q4, 3)
            
        print(str(n2024) + '\t' + str(y1) + '\t' + str(y3) + '\t' + str(y6) + '\t' + str(n1) + '\t' + str(xp1) + '\t' + str(xe))
        
        
def get_ndx(db):
    code = 'NDX'
    sql_m = 'insert into t_m(code,name,type) values(?,?,?) on conflict(code) do update set name = EXCLUDED.name, type = EXCLUDED.type'
    db.run(sql_m, ('NDX', '纳斯达克100指数', '指数'))    
    sql = 'insert into t_e(code,date,dwjz,ljjz) select ?,?,?,? where not exists(select 1 from t_e where code = ? and date = ?)'
    params = []
    start_date = db.run('select max(date) from t_e where code = ?', (code,)).get_rows()[0][0]
    end_date = Tools.locatdate()
    limit = datetime.date.today() - start_date
    url = 'https://api.nasdaq.com/api/quote/NDX/historical?assetclass=index&fromdate=' + str(start_date) + '&limit=' + str(limit.days) + '&todate=' + str(end_date) + '&lang=zh'
    text = HTTPTools.get(url, headers=headers)
    data = json.loads(text)
    rows = data['data']['tradesTable']['rows']
    for row in rows:
        day = datetime.datetime.strptime(row['date'], '%m/%d/%Y').date()
        val = float(row['close'].replace(',', ''))
        params.append((code, day, val, val, code, day))
    params.sort()
    rows = db.run(sql, params)
    log.info('get_fund_dv', code, rows, len(params))

    
def get_ndx_pe(db):
    code = 'NDX_PE'
    sql_m = 'insert into t_m(code,name,type) values(?,?,?) on conflict(code) do update set name = EXCLUDED.name, type = EXCLUDED.type'
    db.run(sql_m, (code, '纳斯达克100指数市盈率', '指数市盈率'))
    headers['cookie'] = 'acw_tc=1a0c63d917526443351605559e0085796b71f21cc037870343a4e7e1e9055c; channel=; h5_channel=; device_id=web_Sygwxj2EUgg; xq_a_token=a84b412749a831734f9e3e966ee495a5e2d4ae57; Hm_lvt_d8a99640d3ba3fdec41370651ce9b2ac=1752644342; Hm_lpvt_d8a99640d3ba3fdec41370651ce9b2ac=1752644342; HMACCOUNT=AE2B08950A2DD0B7; timestamp=1752644427208'
    text = HTTPTools.get(url='https://danjuanfunds.com/djapi/index_eva/pe_history/NDX?day=all', headers=headers)
    kvs = json.loads(text)['data']['index_eva_pe_growths']
    sql = 'insert into t_e(code,date,dwjz,ljjz) select ?,?,?,? where not exists(select 1 from t_e where code = ? and date = ?)'
    params = []
    for kv in kvs:
        date = kv['ts'] / 1000
        date = time.localtime(date)
        date = time.strftime("%Y%m%d", date)
        dwjz = kv['pe']
        params.append((code, date, dwjz, dwjz, code, date))
    params.sort()
    rows = db.run(sql, params)
    log.info('get_fund_dv', code, rows, len(params))


def backtesting(db):
    import pandas as pd
    code = ''
    yq_syl = '3.5'
    sql = '''
        select 名称
        ,(select val from t_e t2 where t2.code = t.编码 and t2.date >= ? order by date limit 1) val1
        ,(select val from t_e t2 where t2.code = t.编码 and t2.date <= ? order by date desc limit 1) val2
        ,t.* 
        from f_fund_summary(?,?,[--yq_syl--],'ret_1y_ann') t 
        where 1=1 
        and 最大单日涨幅 <= 25
        and 最大回撤 >= -10
        and 修复天数 <= 365
        and 平均收益 >= [--yq_syl--]
        and 索提诺比率 >= 1
        and 欧米茄比率 >= 1
        and 名称 not like ? 
        and 名称 not like ?     
        and 名称 not like ? 
        and 名称 not like ? 
        and 名称 not like ?
        and 名称 not like ?
        order by 索提诺比率 desc
        limit 3
    '''.replace('[--yq_syl--]', yq_syl)
    sql2 = '''
        select (select name from t_m where code = '[--code--]') name
        ,(select val from t_e where code = '[--code--]' and date >= ? order by date limit 1) val1
        ,(select val from t_e where code = '[--code--]' and date <= ? order by date desc limit 1) val2
    '''.replace('[--code--]', code)
    report = []
    all_money = 10000
    start = pd.Timestamp('2018-08-22')
    end = pd.Timestamp('2025-08-22')
    date_list = [start + pd.DateOffset(months=i) for i in range((end.year - start.year) * 12 + end.month - start.month + 1)]
    date_list = [pd.Timestamp(d) for d in date_list]
    report.append((date_list[0], all_money))
    for i in range(0, len(date_list) - 1):
        buy_day = date_list[i]
        sell_day = date_list[i + 1]
        begin_day = buy_day - relativedelta(years=6)
        end_day = buy_day
        param = (
            buy_day.strftime('%Y-%m-%d')
            , sell_day.strftime('%Y-%m-%d')
            , begin_day.strftime('%Y-%m-%d')
            , end_day.strftime('%Y-%m-%d')
            , '%美元%'
            , '%C%'
            , '%B%'
            , '%ETF'
            , '%商品%'
            , '%油%'
            # , '%指数%'
        )
        print(param)
        if code == '':
            rows = db.run(sql, param).get_rows()
        else:
            rows = db.run(sql2, (buy_day.strftime('%Y-%m-%d'), sell_day.strftime('%Y-%m-%d'))).get_rows()
        if len(rows) > 0:
            bug_money = all_money / len(rows)
            old_all_money = all_money
            all_money = 0
            for row in rows:
                # code = row[3]
                name = row[0]
                buy_price = row[1]
                sell_price = row[2]
                sell_money = bug_money / buy_price * sell_price
                all_money = all_money + sell_money
                lll = str(round((sell_money - bug_money) / bug_money * 100, 2)) + '%'
                print(buy_day.strftime('%Y-%m-%d'), sell_day.strftime('%Y-%m-%d'), name, lll, str(int(bug_money)) + '->' + str(int(sell_money)))
            lll_all_dq = str(round((all_money - old_all_money) / old_all_money * 100, 2)) + '%'
        else:
            lll_all_dq = str(round(0.02 / 12, 2)) + '%'
            old_all_money = all_money
            all_money = old_all_money * (1 + 0.02 / 12)
        lll_all = str(round((all_money - 10000) / 10000 * 100, 2)) + '%'
        print(buy_day.strftime('%Y-%m-%d'), sell_day.strftime('%Y-%m-%d'), lll_all_dq, lll_all, str(int(old_all_money)) + '->' + str(int(all_money)))
        print()
        report.append((sell_day, all_money))
        
    for r in report:
        print(r[0].strftime('%Y/%m/%d') + '\t' + str(r[1]))


if __name__ == '__main__':
    main()

