#!/usr/bin/env python3

import requests
import re,os
from multiprocessing.pool import ThreadPool
from os import path
from hashlib import md5
from bs4 import BeautifulSoup 
import sys
import shelve
import time
import copy
from IPython import embed
__version__='0.1.2'
__author__="Blacksong"
class Wget:
    def __init__(self,url,**kargs):
        url0=url
        url=url.split('?')[0]
        self.n=0
        dbname=md5(url.encode()).hexdigest()[:7]+'.pydb'
        record_db=shelve.open(dbname)
        self.filetype=kargs.get('filetype','.jpg')
        self.headers=self.setheaders()
        self.srcpro=None
        self.rule=kargs.get('rule',list())
        self.re_rule=kargs.get('re_rule',list())
        self.max_download=8
        self.num_download = 0
        self.asyncThread=ThreadPool(self.max_download)
        self.htm=url.split('/')[2]
        print(self.htm,'htm')
        dirname = kargs.get('dirname',None)
        if dirname is None:
            dirname = self.htm
        if not path.isdir(dirname): 
            os.makedirs(dirname)
        self.dirname = dirname
        self.auto=kargs.get('auto',True)
        print(self.htm)

        self.rule_list=[re.sub('[^A-Za-z]','', url)]
        [self.rule_list.append(re.sub('[^A-Za-z]','', i)) for i in self.rule]
        self.rule_list=list(set(self.rule_list))
        self.rule_dir=[path.dirname(url)]
        [self.rule_dir.append(path.dirname(i)) for i in self.rule]
        self.rule_dir=list(set(self.rule_dir))

        self.re_rule=[re.compile(i) for i in self.re_rule]
        url=url0
        print(self.re_rule,'\n',self.rule_dir,'\n',self.rule_list)
        self.autofind(url)
        try:
            halt=record_db.get('halt',False)
            if halt == True:
                self.href=record_db.get('href',[(url,{})])
                self.pagedb = record_db.get('pagedb',set())
                self.srcdb = record_db.get('srcdb',set())
                record_db['halt']=False
            else:
                self.href=[(url,{})]
                self.pagedb=set()
                self.srcdb=set()
            self.main()
            record_db.close()
            if path.isfile(dbname): os.remove(dbname)
        except:
            print('the program is halted!')
            record_db['halt']=True
            record_db['srcdb']=self.srcdb
            record_db['pagedb']=self.pagedb
            record_db['href']=[i for i in self.href if i!=None]
        self.asyncThread.close()
        self.asyncThread.join()
    def my_hash(self,x):
        return int(md5(x.encode()).hexdigest()[:8],16)
    def setheaders(self):
        headers='''
User-Agent: Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:48.0) Gecko/20100101 Firefox/48.0
DNT: 1
Connection: keep-alive
Upgrade-Insecure-Requests: 1
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
Accept-Encoding: gzip, deflate
Accept-Language: zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2
Host: i.meizitu.net
Referer: http://www.mzitu.com/131504/5
'''
        headers=headers.split('\n')
        d=dict()
        for i in headers:
            n=i.find(':')
            if n ==-1:continue
            d[i[:n].strip()]=i[n+1:].strip()
        return d
    def getHost(self,url):
        return url.split('/')[2]
    def autofind(self,url):
        print('Autofind the label of the target picture')
        volume=0
        x,y=self.Analyze(url)
        
        r=set()
        for i in y:
            s=str(i[1])
            print(i,'ssssssss')
            if s in r:continue
            r.add(s)
            try:
                headers = self.headers
                headers['Host']=self.getHost(i[0])
                headers['Referer']=url
                f=requests.get(i[0],timeout=30,headers = headers).content
                t=len(f)
                open(time.ctime(time.time())+'.jpg','wb').write(f)
            except:
                t=0
            print(t,i[0])
            if volume<t:
                volume=t
                tt=i[1]
        
        self.srcpro=tuple(tt.items())
        print(self.srcpro)
        input('autofind')
    def main(self):
        '''主循环函数，控制寻找页面以及查找页面资源链接，主要起控制作用'''
        def run(u):
            print('Analyse the html ',u)
            hr,src=self.Analyze(u)

            for i in src:
                if self.UsefulSrc0(*i):
                    self.num_download+=1
                    self.asyncThread.apply_async(self.Download,(i[0],u))
                    while self.num_download>self.max_download+4:
                        time.sleep(1)
                        print('Downloading !!')
                    # self.Download(i[0],u)
            self.pagedb.add(self.my_hash(u))
            for i in hr:
                if self.UsefulHtml0(*i):
                    self.href.append(i)
        while True:
            ii=0
            n=len(self.href)
            while ii<n:
                if self.UsefulHtml0(*self.href[ii]):
                    run(self.href[ii][0])
                self.href[ii]=None
                ii+=1
            self.href=[i for i in self.href if i!=None]
            if len(self.href)==0:break
    def DivSplit(self,s):
        '''将一个html页面分成多个div块，主要通过寻找div标签的位置，返回一个记录了div块所在位置以及各个块的名字的链表'''
        a=[]
        [a.append((-1,i.span())) for i in re.finditer('< *div[^><]*>', s)]
        b=[]
        for i,j in a:
            if i==1:
                b.append((i,j[0]))
            else:
                t=s[j[0]:j[1]]
                n=re.findall('id *= *"[^"]*"|class *= *"[^"]*"', t)
                d=dict([i.replace('"','').split('=') for i in n])
                b.append((i,j[0],d))
        b.sort(key=lambda x:x[1])
        return b
    def DivSplit2(self,s):
        '''将一个html页面分成多个div块，主要通过寻找div标签的位置，返回一个记录了div块所在位置以及各个块的名字的链表'''
        a=[(-1,0,{'id':'mystart'})]
        for i in re.finditer('(id|class) *=["\' ]*[^"\']*', s):
            j=re.sub('["\' ]', '',i.group())
            n=j.find('=')
            d={j[:n]:j[n+1:]}
            a.append((-1,i.span()[1],d))
        a.sort(key=lambda x:x[1])
        return a
    def Download(self,url,purl,nn=[0]):
        '''下载url'''
        
        tf=re.sub('\W','', purl)
        filename=self.dirname+'/'+tf[-min(len(tf),10):]+md5(url.encode()).hexdigest()[:5]+self.filetype
        if not path.isfile(filename):
            nn[0]+=1
            if nn[0]%50==0:
                print('Downloading ',url)
            headers = self.headers
            headers['Host']=self.getHost(url)
            headers['Referer']=purl
            x=requests.get(url,timeout=30,headers = headers)
            t=open(filename,'wb').write(x.content)
        else:
            print('file already exist')
        self.srcdb.add(self.my_hash(url))
        self.num_download -= 1
        return filename
    def UsefulHtml0(self,url,pro):
        '''判断一个页面的url是否是有用的'''
        if self.my_hash(url) in self.pagedb:return False
        if self.re_rule:
            for i in self.re_rule_list:
                if i.search(url):return True
        if not self.rule:
            if not re.search(self.htm, url):return False
        t= re.sub('[^A-Za-z]','', url)
        d=path.dirname(url)
        if t in self.rule_list:return True
        if d in self.rule_dir:return True
        if self.auto:
            return False
        return self.UsefulHtml(url,pro)
    def UsefulHtml(self,url,pro):
        '''判断一个页面的url是否是 有用的，这个函数可以在不同的环境中重写，其中pro是该链接所在div的属性'''
        return True
    def UsefulSrc0(self,url,pro):
        if self.my_hash(url) in self.srcdb:return False
        if self.auto:
            for k,v in self.srcpro:
                if v.isdigit():continue
                if pro.get(k)!=v:return False
        return self.UsefulSrc(url,pro)
    def UsefulSrc(self,url,pro):
        return True
    def correct_url(self,s,website,webdir):
        if s[0]=='/':return website+s
        elif s=='#':return ''
        elif s.find('http')!=-1:return s
        else: return webdir+s
    def Analyze(self,url):
        '''返回 href 和 src的链接，返回值为一个二元tuple'''
        headers = self.headers
        headers['Host']=self.getHost(url)
        s=requests.get(url,timeout=30,headers = headers).text
        divs=self.DivSplit(s)
        href=[]
        src=[]
        split_url=url.split('/')
        website='/'.join(split_url[:3])
        webdir='/'.join(split_url[:-1])+'/'
        for i in re.finditer(' *(href|src) *=["\' ]*[^ )("\';\+>}]+', s):
            div=self.FindDiv(divs, i.span()[0])
            j=i.group()
            j=re.sub('["\' \\\\]', '', j) #针对某些网站将url写在javascript中，用到了转义符\
            if j[0]=='h':
                j=j.replace('href=', '')
                j=self.correct_url(j,website,webdir)
                if len(j)==0:continue
                href.append((j,div))
            if j[0]=='s':
                j=j.replace('src=', '')
                if j.find(self.filetype)==-1:continue
                div=self.FindDiv(divs, i.span()[0])
                j=self.correct_url(j,website,webdir)
                if len(j)==0:continue
                src.append((j,div))
        return href,src
    def FindDiv(self,divs,pos):
        a,b=0,len(divs)
        if b==0:
            return {'id':'nodivs'}
        if pos>divs[-1][1]:return divs[-1][2]
        while b-a>1:
            t=int((a+b)/2)
            p0=divs[t][1]
            if pos>p0:a=t
            else:b=t
        return divs[a][2]
class Wget_novel:
    website = ['https://www.x83zw.com/', 'https://www.zhuaji.org/','https://www.ybdu.com/',
    'https://www.23us.so']
    trans=str.maketrans('一二三四五六七八九','123456789')
    trans2=str.maketrans(dict(zip(['零','十','百','千','万','亿'],['','0 ','00 ','000 ',' 10000 ',' 100000000 '])))
    trans.update(trans2)
    def __init__(self,url,novel_name=None,author = None,chapter_attrs=None,content_attrs=None,next_attrs=None):
        content = requests.get(url)
        bs=BeautifulSoup(content.content)

        self.re_br = re.compile('<[^>]+>')
        self.re_n = re.compile('[\n]+')
        self.re_chapter = re.compile('(第[\d]+[章节回]|第[一二三四五六七八九]+[章节回])')
        self.re_chinese_number = re.compile('[一二三四五六七八九十零百千万\d]+')

        self.author = author
        self.chapter_attrs = chapter_attrs
        self.content_attrs = content_attrs 
        self.next_attrs = next_attrs
        
        title,author = self.__get_author(bs)
        self.__auto_select(bs)
        print(self.chapter_attrs,self.content_attrs,self.next_attrs)
        print(title,author)
        if novel_name is None:fname=title+'.txt'
        else:fname=novel_name+'.txt'
        self.fp=open(fname,'w',encoding = 'utf8')
        print(fname)
        self.fp.write(title+'\n'+author+'\n')
        self.chapter=[]
        try:
            self.__start(url)
        except Exception as e:
            print(e)
        self.fp.close()
        x=set(self.chapter)
        y=set(range(1,max(self.chapter)+1))
        print(y-x)
    def div_generate(self,bs,div_list=None):
        if div_list is None:
            div_list = list()
        if bs.div is None:
            return div_list
        for i in bs.find_all('div'):
            div_list.append(i)
            self.div_generate(i,div_list)
        return div_list
    def __auto_select(self,bs):#自动识别 章 div标签 正文div标签 和下一章div标签
        div_list = self.div_generate(bs)
        if self.chapter_attrs is None:
            for i in div_list:
                t = i.text.lstrip()
                if len(t)>50:
                    continue
                if self.re_chapter.match(t):
                    self.chapter_attrs = i.attrs
                    break
            else:
                try:
                    t = self.__get_chapter(bs)
                    if not self.re_chapter.match( t[0]):
                        assert False
                except:
                    title = bs.title.text
                    t = self.re_chapter.search( title)
                    chapter = t.group()
                    t = title.split(chapter)
                    self.chapter_attrs = len(t)

        if self.content_attrs is None:
            length = 0
            for i in div_list:
                div = copy.deepcopy(i)
                for j in div.find_all('div'):
                    j.decompose()
                l = len(re.sub('\s','',div.text))
                if length<l:
                    length = l
                    self.content_attrs = div.attrs
        if self.next_attrs is None:
            for i in div_list:
                if i.div:continue
                t = re.sub('\s', '', i.text)
                if len(t)>50:
                    continue
                if t.find('下一') !=-1 and t.find('上一') != -1 and i.a is not None:
                    self.next_attrs = i.attrs
                    break

    def __start(self,url):#主循环函数
        self.url = url
        re_sub_title = re.compile('^[^\n]+\n')
        while True:
            print(url,'url')
            content = requests.get(url)
            bs = BeautifulSoup(content.content)
            title_info=self.__get_chapter(bs)
            self.chapter.append(title_info[-1])
            print(title_info)
            #必须先找下一页才能找内容，查找内容会破坏bs数据
            next_info=self.__get_next(bs)
            content_info=self.__get_content(bs)
            
            if title_info[0] is not None:
                titles = '\n\n'+title_info[0]+' '+title_info[1]+'\n\n'
                if content_info.startswith(title_info[0]):
                    content_info = re_sub_title.sub('',content_info)
                self.fp.write(titles)
                self.fp.write(content_info)
            if next_info is False:break
            url=next_info
    def __get_author(self,bs):
        title = bs.title.text.split()
        title = ''.join(title)
        return title,'作者：无名氏'
    def __get_content(self,bs):
        x = bs.find(attrs=self.content_attrs)
        if x.div:
            for i in x.find_all('div'):
                i.decompose()
        sx = str(x)
        # print(sx)
        # t = ''.join(sx.split())
        # embed()
        a = x.find_all('a')
        for i in a:
            sx = sx.replace(str(i),'')
        sx = self.re_br.sub('\n',sx)
        sx = self.re_n.sub('\n',sx)
        return sx.strip()
    def __get_next(self,bs,html=None):
        c=bs.find(attrs=self.next_attrs)
        t=c.find_all('a')
        for i in t:
            if i.text.find('下')!=-1:
                href=i['href']
        if len(href)==0:return False
        if href[0]!='/':
            url=self.url.split('/')
            url[-1]=href
            return '/'.join(url)
        else:
            url=self.url.split('/')
            url = url[:3]
            url.append(href)
            return '/'.join(url)
    def __get_chapter(self,bs,html=None):
        if isinstance(self.chapter_attrs,int):
            title = bs.title.text
            t = self.re_chapter.search( title)
            chapter = t.group()
            tt = title.split(chapter)
            n = self.__ChineseNumber_to_number(self.re_chinese_number.search(chapter).group())
            tt = chapter,tt[-1],n
            return tt
            
        else:
            title=bs.find(attrs=self.chapter_attrs).h1.text
            t=self.re_chinese_number.search(title)
            if not t:return None,None,-1
            t=t.group()
            n=self.__ChineseNumber_to_number(t)
            m=title.find(t)+len(t)
            name=title[m+1:].strip()
            if name:
                if name[0]=='章' or name[0]=='节':name=name[1:]
            else:
                name = ''
            name=re.sub(':|：|,|，','',name)
            s='第'+t+'章',name,n
        return s
    def __ChineseNumber_to_number(self,s):
        t=self.trans
        s=s.translate(t)
        l=s.rstrip().split()
        n,m=0,0
        for i in l:
            j=int(i)
            if j<10000:
                if j==0:j=10
                n+=j
            else:
                n*=j
                m+=n
                n=0
        m+=n
        return m
def main(*d,**karg):

    url = sys.argv[1]
    Wget(url)
if __name__=='__main__':
    pass
    # random_name('./www.35uo.com')
    # rule=['http://www.35uo.com/p05/list_37.html','http://www.35uo.com/p05/index.html']
    # Wget('https://m.733.so/mh/26798/443604.html?page=3')
    # Wget('http://www.uutu.me/index.php?c=Article&id=2527&p=2',re_rule=['id=2527&p=.*'])
    # Wget('http://m.7kk.com/picture/2918797.html')
    # Wget('http://www.umei.cc/meinvtupian/nayimeinv/27322.htm')
    # url='https://image.baidu.com/search/detail?ct=503316480&z=0&ipn=false&word=angelababy&hs=0&pn=-1&spn=0&di=baikeimg&pi=&rn=1&tn=baiduimagedetail&is=&istype=&ie=utf-8&oe=utf-8&in=&cl=2&lm=-1&st=&lpn=0&ln=undefined&fr=ala&fmq=undefined&fm=undefined&ic=&s=&se=&sme=&tab=&width=&height=&face=&cg=star&bdtype=0&oriquery=&objurl=http%3A%2F%2Fimgsrc.baidu.com%2Fbaike%2Fpic%2Fitem%2Fa1ec08fa513d2697d2379f9f50fbb2fb4216d808.jpg&fromurl=http%3A%2F%2Fbaike.baidu.com%2Fview%2F1513794.htm&gsm='
    # x=requests.get(url)
    # t=re.finditer('(href|src) *=["\' ]*(http:|https:|/)[^ )("\';>}]+', x.text)
    # for i in t:
    #     j=i.group()
    #     print(j)
    # Wget('http://www.mm131.com/xinggan/2216.html')
    # Wget('http://www.mm131.com/chemo/2043.html')
    # Wget('http://pic.yesky.com/235/108576735.shtml')
    # Wget('http://www.ycgkja.com/siwameinv/13125.html')
    # Wget('http://www.tu11.com/meituisiwatupian/2017/7933.html')
    # Wget('http://www.4493.com/siwameitui/115401/1.htm',rule=['http://www.4493.com/siwameitui/115401/1.htm','http://www.4493.com/gaoqingmeinv/115538/1.htm'])
    # Wget('http://www.522yw.mobi/article/44140.html')
    Wget_novel('https://www.x83zw.com/book/13/13860/6646835.html')
