'''
Author: GanJianWen
Date: 2021-02-22 17:26:05
LastEditors: GanJianWen
LastEditTime: 2021-03-04 20:00:17
QQ: 1727949032
GitHub: https://github.com/1727949032a/
Gitee: https://gitee.com/gan_jian_wen_main
'''

import requests
import json
from lxml import etree
from py3db import mysql, log
from pprint import pprint
from bs4 import BeautifulSoup
from os.path import exists, abspath
from os import makedirs, listdir
from datetime import datetime


class YuLinZhanYeScrapy(object):

    def __init__(self) -> None:
        self.DEFAULT_REQUEST_HEADERS = {
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
            'Accept-Encoding': 'gzip, deflate',
            'Accept-Language': 'zh-CN,zh;q=0.9',
            'Cache-Control': 'max-age = 0',
            'Connection': 'keep-alive',
            'Host': 'www.yulinzhanye.la',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36'
        }
        self.BOOK_LISTS = [
            "http://www.yulinzhanye.la/shuku/0-lastupdate-0-%d.html" % i for i in range(0, 1054)]
        self.mysql_config = self.get_mysql_config()
        self.ip = self.mysql_config["ip"]
        self.user = self.mysql_config["user_name"]
        self.password = self.mysql_config["password"]
        self.database = self.mysql_config["database"]
        self.mysql_config_path = None
        self.mysql = mysql.MySql(
            self.ip, self.user, self.password, self.database)
        self.log = log.Log("./book.log")
        if self.mysql_config["first"]:
            self.create_database()
            self.create_table()
            self.mysql_config["first"] = False
            self.save_mysql_config(self.mysql_config)

    @classmethod
    def get_mysql_config(cls):
        mysql_config_path = "%s/../config/mysql.json" % abspath(__file__)
        return json.load(fp=open(
            mysql_config_path,
            "r",
            encoding="utf-8"
        ))

    @classmethod
    def set_mysql_config(cls, ip=None, user_name=None, password=None, database=None):
        mysql_config = cls.get_mysql_config()
        config_dict = {
            "ip": ip,
            "user_name": user_name,
            "password": password,
            "database": database
        }
        for key, value in config_dict.items():
            if value:
                mysql_config[key] = value
        cls.save_mysql_config(cls, mysql_config)

    def create_database(self):
        database_sql = "create database sex"
        self.mysql.operation_database(database_sql)

    def create_table(self):
        book_sql_file = "%s/../sql/book.sql" % abspath(__file__)
        chapter_sql_file = "%s/../sql/chapter.sql" % abspath(__file__)
        with open(book_sql_file, "r", encoding="utf-8") as fp:
            book_sql_list = fp.read().split(";")
        with open(chapter_sql_file, "r", encoding="utf-8") as fk:
            chapter_sql_list = fk.read().split(";")
            try:
                for book_sql in book_sql_list:
                    self.mysql.operation_database(book_sql)
                self.log.info("成功创建book表")
            except Exception as e:
                self.log.error(e)
            try:
                for chapter_sql in chapter_sql_list:
                    self.mysql.operation_database(chapter_sql)
                self.log.info("成功创建chapter表")
            except Exception as e:
                self.log.error(e)

    def save_mysql_config(self, mysql_config):
        mysql_config_path = "%s/../config/mysql.json" % abspath(__file__)
        with open(mysql_config_path, "w", encoding="utf-8") as fp:
            json.dump(mysql_config, fp, indent=4)

    def aks_url(self, url):
        while True:
            try:
                response = requests.get(
                    url, self.DEFAULT_REQUEST_HEADERS, timeout=3)
                break
            except:
                continue
        return response.text

    def mysql_query(self, sql):
        self.mysql.create_connect()
        self.mysql.operation_database(sql)
        self.mysql.close_connect()

    def book_scrapyed(self, book_name, word_number):
        self.mysql.create_connect()
        select_sql = "select book_name,word_number from book where book_name='%s' and word_number=%d;" % (
            book_name, word_number)
        cursor = self.mysql.db.cursor()
        cursor.execute(select_sql)
        book_list = list(cursor.fetchall())
        self.mysql.close_connect()
        if len(book_list) > 0:
            return True
        else:
            return False

    def visit_book_one_lists(self, url):
        print("list_url:", url)
        book_list = list()
        html = self.aks_url(url)
        html = etree.HTML(html)

        li_list = html.xpath('//div[@class="bd"]/ul/li')
        for li in li_list:
            book_name = li.xpath(
                'div[@class="right"]/a/text()')[0].replace(':', " ").replace('?', ' ').replace('*', 'x')
            book_link = "http://www.yulinzhanye.la" + \
                        li.xpath('div[@class="right"]/a/@href')[0]
            author = li.xpath(
                'div[@class="right"]//p[@class="info"]/a/text()')[0]
            word_numbers = li.xpath(
                'div[@class="right"]//p[@class="info"]/span/text()')[0].replace('字数：', '')
            print(book_name)
            try:
                update_date = li.xpath(
                    'div[@class="right"]//p[@class="info"]/font/text()')[0]
            except:
                # print("book_name=", book_name)
                update_date = li.xpath(
                    'div[@class="right"]//p[3]/text()')[0].replace('\n更新：', '').strip()
                # print("update_date=", update_date)

            if self.book_scrapyed(book_name, int(word_numbers)):
                continue
            item = dict()
            item['book_name'] = book_name
            item['author'] = author
            item['book_link'] = book_link
            item['word_numbers'] = word_numbers
            item['update_date'] = update_date
            book_list.append(item)

        pprint(book_list)
        self.visit_book_detail(book_list)

    def book_exist_and_not_updated(self, book_name, chapter_number):
        select = "select * from book where book_name='%s';" % book_name
        self.db = self.mysql.create_connect()
        cursor = self.mysql.db.cursor()
        cursor.execute(select)
        book_list = list(cursor.fetchall())
        self.mysql.close_connect()
        if len(book_list) > 0 and book_list[0][-2] < chapter_number:
            return True
        else:
            return False

    def update_book_message(self, book_name, word_number, popularity, update_time, status, chapter_number):
        if self.book_exist_and_not_updated(book_name, chapter_number):
            update_sql = "update book set word_number=%d,popularity=%d,update_time='%s',status='%s',chapter_number=%d,latest=0 where book_name='%s';" % (
                int(word_number), int(popularity), update_time, status, int(chapter_number), book_name)
            print("update_sql:", update_sql)
            self.mysql_query(update_sql)

    def visit_book_detail(self, book_list):
        for book in book_list:
            begin, count = 1, 0
            print("开始获取%s的详细信息" % book['book_name'])
            intro = str()
            print("对应链接为%s" % book['book_link'])
            html = self.aks_url(book['book_link'])
            html = etree.HTML(html)
            content = html.xpath(
                '//div[@class="right"]//p[@class="info"]/text()')
            status = html.xpath(
                '//div[@class="right"]/span/text()')[0]
            # print(content)
            type = content[1].replace('\n类型：', '')
            pupular = content[-1].replace('\n', '').replace('人气：', '').strip()
            print(book["book_name"])
            try:
                intro = html.xpath(
                    '//div[@class="mod book-intro"]/div/text()')[0]
            except:
                pass

            try:
                end_page = int(html.xpath(
                    '//div[@class="pagelistbox"]//a[@class="endPage"]/@href')[0].split('/')[-2].split('_')[-1])
            except:
                continue
            print("type=", type)
            print("popular=", pupular)
            print("status=", status)
            print("intro=", intro)
            page_list = [book['book_link'][:-1] + "_%d/" %
                         i for i in range(1, end_page + 1)]
            # pprint(page_list)
            chapter_number = 0
            file_path = "小说/%s" % book['book_name']
            if exists(file_path):
                begin = len(listdir(file_path)) + 1

            for page in page_list:
                html_page = self.aks_url(page)
                html_page = etree.HTML(html_page)
                li_list = html_page.xpath(
                    '//div[@class="mod block update chapter-list"]')[1].xpath('div[@class="bd"]/ul//li')
                chapter_number += len(li_list)
                self.update_book_message(book['book_name'], book['word_numbers'], int(
                    pupular), update_time=book['update_date'], status=status, chapter_number=chapter_number)
                for li in li_list:
                    count += 1
                    if count < begin:
                        continue
                    chapter_header = li.xpath('a/text()')[0]
                    chapter_link = "http://www.yulinzhanye.la" + \
                                   li.xpath('a/@href')[0]
                    print('chapter_header=', chapter_header)
                    print('chapter_link=', chapter_link)
                    self.visit_chapter(
                        book['book_name'], chapter_header, chapter_link)

            insert = "insert into book values(NULL,'%s','%s','%s',%d,%d,'%s','%s','%s',%d,0);" % (
                book['book_name'], book['author'], type, int(book['word_numbers']), int(pupular), book['update_date'],
                intro, status, chapter_number)
            self.mysql_query(insert)
            print("插入完成")

    def visit_chapter(self, book_name, chapter_name, chapter_link):
        contents = str()
        num = 1
        file_path = "小说/%s" % (book_name)
        if not exists(file_path):
            makedirs(file_path)
        chapter_pre = chapter_link.replace('.html', '')
        while True:
            try:
                chapter_link = chapter_pre + "_%d.html" % num
                print('链接；', chapter_link)
                html = self.aks_url(chapter_link)
                bs = BeautifulSoup(html, 'lxml')
                if num == 1:
                    try:
                        content = bs.find(
                            attrs={'class': 'page-content font-large'}).find('div').find('div')
                        content.find(
                            "div", {'class': 'slide-baidu'}).decompose()
                        content.find(
                            "div", {'class': 'mod page-control'}).decompose()
                        content.find_all('div')[0].decompose()
                    except:
                        content = bs.find('p')

                    if bs.find('center', {'class': 'chapterPages'}) is None:
                        contents = str(content).replace('    ', '')
                        break
                else:
                    content = bs.find('p')
                if content.get_text() == "" and num > 1:
                    break
                else:
                    contents += str(content).replace('    ', '')
                    pprint(contents)

                num += 1
            except Exception as e:
                print(str(e))
                return
        chapter_name = chapter_name.replace(
            '?', '').replace('2u2u2u', '').replace('/', '-').replace('*', 'x').replace(':', '：').replace('|', '')
        with open("%s/%s.html" % (file_path, chapter_name), "w", encoding='utf-8') as fp:
            fp.write(contents)

    def main(self):
        self.log.info(
            "{}\tbegin to scrapy book".format(
                datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
        )
        url_list = [
            "http://www.yulinzhanye.la/shuku/0-lastupdate-0-%d.html" % i for i in range(1, 1058)]
        for url in url_list:
            self.visit_book_one_lists(url)


if __name__ == "__main__":
    spider = YuLinZhanYeScrapy()
    spider.main()
