# coding:utf-8
import sys
from tools_hjh import DBConn
from tools_hjh import Log
from tools_hjh import ThreadPool
from tools_hjh import Tools
from bs4 import BeautifulSoup
from tools_hjh import ChromePool
from tools_hjh import HTTPTools
import os
import time
import signal

proxies = {
    'http': 'http://117.42.94.68:22165',
    'https': 'https://117.42.94.68:22165'
}

proxies = None

stop_flag = False
page_num = None

date = Tools.locatdate()
log = Log('U:/MyFiles/MyPy/log/avbebe_benzi/' + date + '.log')

host = 'https://avbebe.com/'

chrome_path = r'U:\MyApps\CentBrowser\App\chrome.exe'
chromedriver_path = r'U:\MyApps\CentBrowser\chromedriver.exe'

headers = {
    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
    'accept-language': 'zh-CN,zh;q=0.9',
    'cache-control': 'no-cache',
    'dnt': '1',
    'pragma': 'no-cache',
    'sec-ch-ua': 'x86;v=99, Windows;v=10, Surface Laptop Studio;v=1',
    'sec-ch-ua-mobile': '?0',
    'sec-ch-ua-platform': '"Windows"',
    'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36',
    'Referer': 'https://avbebe.com/'
}

try:
    sys_argv_1 = sys.argv[1]
except:
    sys_argv_1 = 'find'
try:
    sys_argv_2 = sys.argv[2]
except:
    sys_argv_2 = None
try:
    sys_argv_3 = int(sys.argv[3])
except:
    sys_argv_3 = 16
    
pic_path = r'd:/avbebe_pic'
pdf_path = r'u:/avbebe_pdf'


def main(): 
    db = DBConn('sqlite', db='U:/MyFiles/MyPy/data/avbebe_benzi.db')
    createDatabase(db, rebuild=False)

    if sys_argv_1 == 'find':
        chrome = ChromePool(4, chrome_path, chromedriver_path, is_display_picture=False, is_hidden=False)
        getM(db, chrome, '/archives/category/%e6%88%90%e4%ba%bah%e6%bc%ab%e7%95%ab/page/PAGEIDX')
        getE(db, chrome)
    elif sys_argv_1 == 'download':
        download_pic(db, re=False)
    elif sys_argv_1 == '2pdf':
        to_pdf(db)
    elif sys_argv_1 == 'awr_ai':
        test(db)

    
def createDatabase(db, rebuild=False):
    if rebuild:
        db.run('drop table if exists t_m')
        db.run('drop table if exists t_e')
        
    t_m = '''
        create table if not exists t_m(
            url text, 
            name_ text,
            uploader text,
            upload_time text,
            status text
        )
    '''
    
    t_e = '''
        create table if not exists t_e(
            url text,
            picture_url text,
            size real,
            status real
        )
    '''

    db.run(t_m)
    db.run('create index if not exists idx_m_u on t_m(url)')

    db.run(t_e)
    db.run('create index if not exists idx_e_pu on t_e(picture_url)')
    db.run('create index if not exists idx_e_u on t_e(url)')
    db.run('create index if not exists idx_e_size on t_e(size)')
    db.run('create index if not exists idx_e_status on t_e(status)')
    
    # db.run('update t_e set status = null')
    # db.run('update t_m set status = null')

    
def getM(db, chrome, firstPageUrl):

    def getOne(idx, db):
        try:
            pageurl = host + firstPageUrl.replace('PAGEIDX', str(idx))
            page = chrome.get(pageurl)
            bs = BeautifulSoup(page, features="lxml")
            ones = bs.find_all('article', class_='jeg_post')
            insert_params = []
            for one in ones:
                url = one.find('h3', class_='jeg_post_title').find('a')['href'].strip()
                name_ = one.find('h3', class_='jeg_post_title').find('a').text.strip()
                try:
                    uploader = one.find('div', class_='jeg_meta_author').find('a').text.strip()
                except:
                    uploader = None
                upload_time = one.find('div', class_='jeg_meta_date').find('a').text.strip()
                upload_time = upload_time.replace('月, ', '')
                upload_times = upload_time.split(' ')
                r = upload_times[0]
                y = upload_times[1]
                n = upload_times[2]
                if len(r) == 1:
                    r = '0' + r
                if len(y) == 1:
                    y = '0' + y
                upload_time = n + '-' + y + '-' + r
                
                insert_param = (url, name_, uploader, upload_time, url)
                insert_params.append(insert_param)
            
            insert_sql = 'insert into t_m(url,name_,uploader,upload_time) select ?,?,?,? where not exists(select 1 from t_m t where t.url = ?)'
            insert_num = db.run(insert_sql, insert_params)
            
            if insert_num == 0:
                global stop_flag
                stop_flag = True
                    
            log.info('getM', str(idx), insert_num, len(insert_params))
        except Exception as _:
            log.error('getM', str(idx), str(_))
        
    tp = ThreadPool(4)
    global page_num
    global stop_flag
    stop_flag = False
    page = chrome.get(host + firstPageUrl.replace('PAGEIDX', '1'))
    bs = BeautifulSoup(page, features="lxml")
    page_num = bs.find('div', class_='wp-pagenavi').find('a', class_='last').text
    for idx in range(1, int(page_num) + 1):
        if stop_flag:
            break
        else:
            tp.run(getOne, (idx, db))
    tp.wait()

    
def getE(db, chrome):
    
    def getOne(url, db, is_fl, m_url, p_num, count_num):
        params = []
        page = chrome.get(url)
        bs = BeautifulSoup(page, features="lxml")
        try:
            # 分流
            div_fl = bs.find('div', class_='nav_link')
            if div_fl is not None and is_fl == False:
                div_fl = bs.find('div', class_='nav_link')
                fl_max_num = int(div_fl.find_all('a')[-2].text.replace('分流', ''))
                for idx in range(1, int(fl_max_num) + 1):
                    fx_url = url + '/' + str(idx)
                    getOne(fx_url, db, True, m_url, p_num, count_num)
            else:
                div = bs.find('div', class_='content-inner')
                imgs = div.find_all('img')
                for img in imgs:
                    if 'akabebe' in img['src']:
                        picture_url = img['src']
                        params.append((m_url, picture_url, picture_url, m_url))
                    elif 'akabebe' in img['data-src']:
                        picture_url = img['data-src']
                        params.append((m_url, picture_url, picture_url, m_url))
                num = db.run("insert into t_e(url, picture_url) select ?,? where not exists(select 1 from t_e where picture_url = ? and url = ?)", params)
                num = int(num) + int(count_num)
                log.info('getE', url, num, len(params), 'p_num=' + str(p_num), p_num == num)
        except Exception as _:
            log.error('getE', url, str(_))

    tp = ThreadPool(4)
    sql = '''
        select m.url,m.name_,count(1) from t_m m left join t_e e
        on m.url = e.url
        group by m.url,m.name_
        order by cast(replace(m.url,'https://avbebe.com/archives/','') as integer) desc
    '''
    rss = db.run(sql).get_rows()
    urls = []
    for rs in rss:
        url = rs[0]
        name = rs[1]
        if name.startswith('【'):
            name = name.replace(name.split('】')[0] + '】', '')
        name = Tools.merge_spaces(name)
        name = name.replace('] [', '][')
        name = name.replace('/', '-').replace('\\', '-').replace(':', '：').replace('*', '·').replace('?', '？').replace('"', '\'').replace('<', '[').replace('>', ']').replace('|', ' ')
        count_num = int(rs[2])
        try:
            p_num = int(name.split('[')[-1].replace('P]', ''))
            if p_num != count_num:
                # log.info('getE', url, 'p_num != count_num', p_num, count_num)
                # time.sleep(1)
                urls.append((url, p_num, count_num))
        except:
            # log.info('getE', url, 'p_num != count_num', 'except')
            # time.sleep(1)
            p_num = -1
            urls.append((url, p_num, count_num))
    
    for url_list in urls:
        url = url_list[0]
        p_num = url_list[1]
        count_num = url_list[2]
        # print(url)
        tp.run(getOne, (url, db, False, url, p_num, count_num))
    tp.wait()


def download_pic(db, re=False):
    params = []
    sizes = []
    update_sql = '''
        update t_e set status = ?, size = ? where picture_url = ? and url = ?
    '''

    def get_one(url, pic_url, pic_path_folder, pic_name, retry_time, db, tp):
        try:
            retry_time = retry_time + 1
            status_code = HTTPTools.get_status_code(pic_url, headers=headers, proxies=proxies)
            
            # 文件已下载情况
            true_size = HTTPTools.get_size(pic_url, headers=headers, proxies=proxies)
            if os.path.exists(pic_path_folder + '/' + pic_name):
                exists_file_size = os.path.getsize(pic_path_folder + '/' + pic_name)
                if status_code == 200:
                    if exists_file_size == true_size:
                        if exists_file_size == 0:
                            Tools.rm(pic_path_folder + '/' + pic_name)
                        true_size = round(true_size / 1024, 2)
                        exists_file_size = round(exists_file_size / 1024, 2)
                        param = (exists_file_size, true_size, pic_url, url)
                        params.append(param)
                        log.info('download_pic', url.split('/')[-1], pic_url, len(params), 0, str(exists_file_size) + 'KB', str(round(sum(sizes) / 1024, 2)) + 'MB', tp.get_running_num())
                        return
                else:
                    if exists_file_size == 4515:
                        exists_file_size = round(exists_file_size / 1024, 2)
                        Tools.rm(pic_path_folder + '/' + pic_name)
                        log.error('download_pic', 'rm', url.split('/')[-1], pic_url, len(params), 0, str(exists_file_size) + 'KB', str(round(sum(sizes) / 1024, 2)) + 'MB', tp.get_running_num())
                        return
            
            # 封禁状态
            if status_code != 200:
                mess = HTTPTools.get(pic_url, headers=headers, timeout=5, proxies=proxies)
                if 'IP' in mess and 'blocked' in mess:
                    Tools.rm(pic_path_folder + '/' + pic_name)
                    log.info('download_pic', url.split('/')[-1], pic_url, len(params), retry_time, status_code, '检查到IP封锁，已终止程序', tp.get_running_num())
                    print(mess)
                    os.kill(os.getpid(), signal.SIGTERM)
            
            size_ = HTTPTools.download(pic_url, filepath=pic_path_folder + '/' + pic_name, headers=headers, replace=True, timeout=5, proxies=proxies)
            # 完美情况
            if size_ > 0 and status_code == 200 and size_ == true_size:
                true_size = round(true_size / 1024, 2)
                size_ = round(size_ / 1024, 2)
                sizes.append(size_)
                param = (size_, true_size, pic_url, url)
                params.append(param)
                log.info('download_pic', url.split('/')[-1], pic_url, len(params), retry_time, status_code, str(size_) + 'KB', str(round(sum(sizes) / 1024, 2)) + 'MB', tp.get_running_num())
            
            # 文件缺失情况
            elif size_ == 0 and status_code == 404 and size_ == true_size:
                Tools.rm(pic_path_folder + '/' + pic_name)
                Tools.get_pic_by_text(pic_path_folder + '/' + pic_name, pic_name)
                true_size = round(true_size / 1024, 2)
                size_ = round(size_ / 1024, 2)
                sizes.append(size_)
                if not re:
                    param = (size_, true_size, pic_url, url)
                    params.append(param)
                log.info('download_pic', url.split('/')[-1], pic_url, len(params), retry_time, status_code, str(size_) + 'KB', str(round(sum(sizes) / 1024, 2)) + 'MB', tp.get_running_num())
            
            # 其他情况重试
            else:
                if retry_time <= 4:
                    get_one(url, pic_url, pic_path_folder, pic_name, retry_time, db, tp)
                else:
                    Tools.rm(pic_path_folder + '/' + pic_name)
                    log.error('download_pic', url.split('/')[-1], pic_url, retry_time, status_code)
                
        except Exception as _:
            if retry_time <= 4:
                get_one(url, pic_url, pic_path_folder, pic_name, retry_time, db, tp)
            # 另一种文件缺失情况
            elif retry_time >= 5 and 'content-length' in str(_):
                Tools.get_pic_by_text(pic_path_folder + '/' + pic_name, pic_name)
                if not re:
                    param = (-1, -1, pic_url, url)
                    params.append(param)
                log.error('download_pic', url.split('/')[-1], pic_url, retry_time, status_code, str(_))
            else:
                log.error('download_pic', url.split('/')[-1], pic_url, retry_time, status_code, str(_))

    tp = ThreadPool(sys_argv_3)
    # tp2 = ThreadPool(1)
    # tp2.run(to_pdf, (db,))
    boos_sql = '''
        select m.url, m.name_ from t_m m
        where (m.status != 1 or m.status is null)
        and exists(select 1 from t_e e where e.url = m.url)
        order by cast(replace(url,'https://avbebe.com/archives/','') as int) desc
    '''
    if re:
        boos_sql = '''
            select t.url, t.name_ from t_m t
            where exists(select 1 from t_e t2 where t2.url = t.url and t2.status <= 0)
        '''
    get_count_sql = '''
        select count(1) from t_e where url = ?
    '''
    pic_sql = '''
        select picture_url from t_e where url = ?
        --and (status is null or status = 4.41)
    '''
    if re:
        pic_sql = '''
            select picture_url from t_e where url = ?
        '''
    
    rss = db.run(boos_sql).get_rows()
    for rs in rss:
        url = rs[0]
        name = rs[1]
        if name.startswith('【'):
            name = name.replace(name.split('】')[0] + '】', '')
        name = Tools.merge_spaces(name)
        name = name.replace('] [', '][')
        name = name.replace('/', '-').replace('\\', '-').replace(':', '：').replace('*', '·').replace('?', '？').replace('"', '\'').replace('<', '[').replace('>', ']').replace('|', ' ')
        name = name + '[' + url.split('/')[-1] + ']'
        pic_path_folder = pic_path + '/' + name
        try:
            p_num = int(rs[1].split('[')[-1].replace('P]', ''))
        except:
            p_num = -1
        c_num = int(db.run(get_count_sql, (url,)).get_rows()[0][0])
        if p_num == c_num:
            Tools.mkdir(pic_path_folder)
            pics = db.run(pic_sql, (rs[0],)).get_rows()
            for pic in pics:
                pic_url = pic[0]
                pic_name = pic_url.split('/')[-1]
                tp.run(get_one, (url, pic_url, pic_path_folder, pic_name, 0, db, tp), time_out=60)
                # log.info('download_pic_begin', pic_url, tp.get_running_num())

        if len(params) >= 500:
            tp.wait()
            log.info('download_pic', 'begin run update')
            _ = db.run(update_sql, params)
            log.info('download_pic', _, len(params))
            params.clear()
            sizes.clear()
            # if tp2.get_running_num() < 1:
            #    tp2.run(to_pdf, (db,))
                
    tp.wait()
    log.info('download_pic', 'begin run update last')
    _ = db.run(update_sql, params)
    log.info('download_pic', _, len(params))
    params.clear()
    # tp2.wait()
    # tp2.run(to_pdf, (db,))
    # tp2.wait()

    
def to_pdf(db):
    while(1):
        boos_sql = '''
            select m.name_,m.url from t_m m
            where not exists(select 1 from t_e e where e.url = m.url and (e.status is null or e.status = '' or e.status = 4.41))
            and exists(select 1 from t_e e2 where e2.url = m.url)
        '''
        if_qy_sql = '''
            select count(1) from t_e where url = ? and status <= 0
        '''
        rss = db.run(boos_sql).get_rows()
        
        for rs in rss:
            try:
                url = rs[1]
                name = rs[0]
                p_num = int(rs[0].split('[')[-1].replace('P]', ''))
                if name.startswith('【'):
                    name = name.replace(name.split('】')[0] + '】', '')
                name = Tools.merge_spaces(name)
                name = name.replace('] [', '][')
                name = name.replace('/', '-').replace('\\', '-').replace(':', '：').replace('*', '·').replace('?', '？').replace('"', '\'').replace('<', '[').replace('>', ']').replace('|', ' ')
                name = name + '[' + url.split('/')[-1] + ']'
                download_folder = pic_path + '/' + name
                
                if os.path.exists(download_folder) and os.path.isdir(download_folder) and len(os.listdir(download_folder)) == p_num:
                    
                    if_qy = int(db.run(if_qy_sql, (rs[1],)).get_rows()[0][0])
                    if if_qy > 0:
                        target_pdf = pdf_path + '/[缺页]' + name + '.pdf'
                    else:
                        target_pdf = pdf_path + '/' + name + '.pdf'
                    
                    # 目标pdf文件已存在的情况（缺页情况）
                    if os.path.exists(pdf_path + '/[缺页]' + name + '.pdf'):
                        # 以前缺页，现在不缺页，删除缺页的，正常处理
                        if if_qy == 0:
                            Tools.rm(pdf_path + '/[缺页]' + name + '.pdf')
                        # 以前缺页，现在也缺页，删除缺页的，正常处理
                        elif if_qy > 0:
                            Tools.rm(pdf_path + '/[缺页]' + name + '.pdf')
                            
                    # 目标pdf文件已存在的情况（不缺页情况）    
                    # 以前不缺页，现在缺页，不处理
                    if os.path.exists(pdf_path + '/' + name + '.pdf') and if_qy > 0:
                        continue
                    # 以前不缺页，现在不缺页，可能是上个任务没有处理完，正常处理
                    elif os.path.exists(pdf_path + '/' + name + '.pdf') and if_qy == 0:
                        pass
                        
                    Tools.pictures_folder_2_pdf(download_folder, target_pdf)
                    db.run("update t_m set status = '1' where url = ?", (url,))
                    Tools.rm(download_folder)
                    log.info('to_pdf', url.split('/')[-1], if_qy)
                    
            except Exception as _:
                    Tools.rm(target_pdf)
                    log.error('to_pdf', url.split('/')[-1], str(_))
        time.sleep(10)


def test(db):
    names = []
    sql = "select name_ from t_m where status = '1'"
    rss = db.run(sql).get_rows()
    for rs in rss:
        name = rs[0]
        if name.startswith('【'):
            name = name.replace(name.split('】')[0] + '】', '')
        name = Tools.merge_spaces(name)
        name = name.replace('] [', '][')
        name = name.replace('/', '-').replace('\\', '-').replace(':', '：').replace('*', '·').replace('?', '？').replace('"', '\'').replace('<', '[').replace('>', ']').replace('|', ' ')
        names.append(name)
    
    i = 0
    for filee in os.listdir(r'D:\avbebe_books2'):
        if filee in names:
            i = i + 1
            print(filee)
            Tools.rm('D:/avbebe_books2/' + filee)


if __name__ == '__main__':
    main()
    
