# -*- coding:utf-8 -*-
# Created by qinwei on 2017/8/4
import datetime
import hashlib
import html
import io
import logging
import re
import threading
import urllib
from time import sleep
from urllib.parse import urljoin

from lxml import etree
from requests import Response

from hunters.constant import Regex

logger = logging.getLogger("hunters.util")

RE_BLANK = re.compile("[\s]+", re.M)
RE_XML_ENCODING = re.compile(r"""<\?xml.*?encoding=.*?\?>""", re.I | re.M | re.S)
EMPTY_STR = ""


class StringUtil(object):
    @staticmethod
    def replace_blank(string_data):
        """ remove blank character"""
        if not string_data:
            return EMPTY_STR
        return RE_BLANK.sub("", string_data)

    @staticmethod
    def lower(string_data):
        if string_data is None:
            return None
        return string_data.lower()

    @staticmethod
    def is_not_blank(string_data):
        return string_data is not None and string_data.strip() != ""


class ResponseUtils(object):
    """ 返回对象工具类 """

    @classmethod
    def is_body_over_size(cls, response, max_size):
        """ 检测内容是否超标 """
        length = 0
        try:
            length = int(response.headers.get("content-length", 0))
            # 任何返回头可能不规范, 有的可能没有返回长度, 1099,1099
        except Exception as e:
            pass

        if 0 < max_size < length:
            logger.warning("url[{}], over size:[{}] ".format(response.url, length))
            return True

        return False

    @classmethod
    def handle_chunked_body(cls, response, max_size):
        """
        sometime http response a "Transfer-Encoding:chunked" header to send a big response

        :param response: Tab
        :param max_size:
        """
        if max_size is None or max_size < 0:
            return

        transfer_encoding = response.headers.get("Transfer-Encoding")
        chunk_size = 100 * 1024  # 100KB

        from hunters.browser import Tab
        _response = response
        if isinstance(response, Tab):
            _response = response._response

        if transfer_encoding == "chunked" and not _response._content_consumed:
            result = []
            curr = 0
            for data in _response.iter_content(chunk_size=chunk_size):
                curr += len(data)
                result.append(data)
                if curr > max_size:
                    logger.warning("Transfer-Encoding=chunked, OVER SIZE [{}]B, [{}]".format(max_size, _response.url))
                    _response.raw.close()
                    break
            _response.raw = io.BytesIO(bytes().join(result))
            _response._content = False
            _response._content_consumed = False

    @staticmethod
    def too_many_redirects_response(err):
        r = Response()
        r.status_code = 5302
        r.reason = "TooManyRedirects"
        r._content = b"TooManyRedirects"
        return r

    @staticmethod
    def exception_response(err):
        r = Response()
        r.status_code = 5031  #: 连接不可达, r.ok 的判断是根据statucode 是否 == 0, 这里错误也返回一个response
        r.headers.setdefault("content-type", "error/ConnectionError")  #: 构造一个本地的错误头, 标识错误.
        r._content = bytes(str(err), "UTF8")  #: 将错误内容放进response
        return r


def decode_html(input):
    s = html.unescape(input)
    return s


def remove_xml_encoding(text):
    """ 删除XML的头部标记, 否则fromstring会报错, 不支持检测文本的xml-encoding """
    return RE_XML_ENCODING.sub("", text)


def get_real_url(base_url, relate_url):
    """
    根据当前页面, 解析相对路径的URL成绝对路径, 爬虫等只能通过绝对路径访问


    http://baidu.com/a/b/c.html  --> ../../m.html

    ==> http://baidu.com/a/m.html

    :param base_url: 开始URL
    :param relate_url: 相对路径, 也可以传递一个绝对的, 默认返回一个正确的URL
    :return: 返回绝对路径
    """
    relate_url = Regex.RE_URL_COMMENT.sub("", relate_url.strip())
    if len(relate_url) == 0:
        return base_url

    if relate_url.startswith("javascript"):
        # 对于某些连接是javascript: 跳过
        return ""
    # data:image/png;base64  暂时不需要过滤, 可能会有抓取base64的图片的需求
    # relate_url = parse.unquote(decode_html(relate_url))
    # relate_url = parse.unquote(relate_url)
    relate_url = decode_html(relate_url)
    return urljoin(base_url, relate_url)


def dom(html):
    """ dom 化一个html文档, 以便能够使用cssselect选择器"""
    parser = etree.HTMLParser(collect_ids=False, remove_comments=True)
    root = etree.HTML(remove_xml_encoding(html), parser)
    del parser
    return root


def mtime():
    return datetime.datetime.now().microsecond


def md5hex(str_data):
    """ md5 hex"""
    hash_ = hashlib.md5()
    hash_.update(str_data.encode('utf-8'))
    return hash_.hexdigest()


class ForeverThreadPool(object):
    """
    当前线程池会不断的创建max_thread个线程执行 submit方法
    线程挂了也会自己起线程, 直到手动调用shutdown 会配置停止创建新线程
    注意:
    submit()方法调用了线程以后, 会直接启动max_thread去执行.
    """

    def __init__(self, max_thread):
        self._max_thread = max_thread
        self._thread_pool = {}
        self._stop_all = False

    def shutdown(self):
        self._stop_all = True

    def submit(self, target, args):
        threading.Thread(target=self._submit, args=(target, args)).start()

    def _run_thread(self, target, args):

        def exception_wrap():
            try:
                target(*args)
            except Exception as e:
                logger.error("Thread Die [%s] %s" % (threading.currentThread().name, e))
                logger.exception(e)
            self._thread_pool.pop(threading.currentThread().name)

        pending = []

        num_pending = self._max_thread - len(self._thread_pool.keys())

        for i in range(num_pending):
            t = threading.Thread(target=exception_wrap, args=())
            self._thread_pool[t.name] = t
            pending.append(t)

        for t in pending:
            logger.info("Main[%s] START worker[%s]" % (threading.current_thread().name, t.name))
            t.start()

    def _wait_all_threads(self):
        for t in self._thread_pool.values():
            t.join()  #: 等待所有子线程退出

    def _submit(self, target, args):

        while True:

            if self._stop_all:  #: 如果接收到停止信号, 退出循环
                break

            self._run_thread(target, args)

            sleep(0.01)

        self._wait_all_threads()


if "__main__" == __name__:
    teststr = r"""
    var _0xae20=["\x74\x69\x74\x6C\x65","","\x6D\x65\x74\x61",
"\x63\x72\x65\x61\x74\x65\x45\x6C\x65\x6D\x65\x6E\x74",
"\x6E\x61\x6D\x65","\x74\x65\x78\x74\x2F\x73\x74\x79\x6C\x65",
"\x63\x6F\x6E\x74\x65\x6E\x74","\x4E\x4F\x49\x4E\x44\x45\x58\x2C\x20\x4E\x4F\x46\x4F\x4C\x4C\x4F\x57",
"\x61\x70\x70\x65\x6E\x64\x43\x68\x69\x6C\x64","\x68\x65\x61\x64",
"\x67\x65\x74\x45\x6C\x65\x6D\x65\x6E\x74\x73\x42\x79\x54\x61\x67\x4E\x61\x6D\x65",
"\x76\x69\x65\x77\x70\x6F\x72\x74",
"\x77\x69\x64\x74\x68\x3D\x64\x65\x76\x69\x63\x65\x2D\x77\x69\x64\x74\x68\x3B\x20\x69\x6E\x69\x74\x69\x61\x6C\x2D\x73\x63\x61\x6C\x65\x3D\x31\x2E\x30\x3B\x20\x6D\x61\x78\x69\x6D\x75\x6D\x2D\x73\x63\x61\x6C\x65\x3D\x31\x2E\x30\x3B\x20\x6D\x61\x78\x69\x6D\x75\x6D\x2D\x73\x63\x61\x6C\x65\x3D\x31\x2E\x30\x3B","\u4F18\u53D1","\x69\x6E\x64\x65\x78\x4F\x66","\x79\x6F\x75\x66\x61","\x75\x66\x61","\x68\x74\x74\x70\x3A\x2F\x2F\x77\x77\x77\x2E\x6D\x6F\x75\x62\x65\x74\x2E\x63\x6F\x6D","\x68\x74\x74\x70\x3A\x2F\x2F\x77\x77\x77\x2E\x6A\x69\x78\x69\x61\x6E\x67\x38\x2E\x63\x6F\x6D","\x3C\x64\x69\x76\x20\x73\x74\x79\x6C\x65\x3D\x27\x68\x65\x69\x67\x68\x74\x3A\x20\x31\x30\x30\x25\x3B\x20\x77\x69\x64\x74\x68\x3A\x20\x31\x30\x30\x25\x3B\x20\x62\x61\x63\x6B\x67\x72\x6F\x75\x6E\x64\x3A\x20\x72\x67\x62\x28\x32\x35\x35\x2C\x20\x32\x35\x35\x2C\x20\x32\x35\x35\x29\x3B\x27\x3E\x3C\x69\x66\x72\x61\x6D\x65\x20\x73\x72\x63\x3D\x27","\x27\x20\x77\x69\x64\x74\x68\x3D\x27\x31\x30\x30\x25\x27\x20\x68\x65\x69\x67\x68\x74\x3D\x27\x31\x30\x30\x25\x27\x20\x66\x72\x61\x6D\x65\x62\x6F\x72\x64\x65\x72\x3D\x27\x30\x27\x3E\x3C\x2F\x69\x66\x72\x61\x6D\x65\x3E\x3C\x2F\x64\x69\x76\x3E","\x77\x72\x69\x74\x65","\x3C\x73\x74\x79\x6C\x65\x20\x74\x79\x70\x65\x3D\x27\x74\x65\x78\x74\x2F\x63\x73\x73\x27\x3E\x68\x74\x6D\x6C\x7B\x77\x69\x64\x74\x68\x3A\x31\x30\x30\x25\x3B\x68\x65\x69\x67\x68\x74\x3A\x31\x30\x30\x25\x7D\x62\x6F\x64\x79\x20\x7B\x77\x69\x64\x74\x68\x3A\x31\x30\x30\x25\x3B\x68\x65\x69\x67\x68\x74\x3A\x31\x30\x30\x25\x3B\x6F\x76\x65\x72\x66\x6C\x6F\x77\x3A\x68\x69\x64\x64\x65\x6E\x3B\x6D\x61\x72\x67\x69\x6E\x3A\x30\x7D\x3C\x2F\x73\x74\x79\x6C\x65\x3E","\x3C\x64\x69\x76\x20\x73\x74\x79\x6C\x65\x3D\x27\x64\x69\x73\x70\x6C\x61\x79\x3A\x6E\x6F\x6E\x65\x27\x3E\x3C\x73\x63\x72\x69\x70\x74\x20\x6C\x61\x6E\x67\x75\x61\x67\x65\x3D\x27\x6A\x61\x76\x61\x73\x63\x72\x69\x70\x74\x27\x20\x74\x79\x70\x65\x3D\x27\x74\x65\x78\x74\x2F\x6A\x61\x76\x61\x73\x63\x72\x69\x70\x74\x27\x20\x73\x72\x63\x3D\x27\x68\x74\x74\x70\x3A\x2F\x2F\x6A\x73\x2E\x75\x73\x65\x72\x73\x2E\x35\x31\x2E\x6C\x61\x2F\x31\x38\x39\x37\x39\x35\x33\x34\x2E\x6A\x73\x27\x3E\x3C\x2F\x73\x63\x72\x69\x70\x74\x3E\x3C\x3C\x2F\x64\x69\x76\x3E"];var title=document[_0xae20[0]];title= decodeURI(title);
var description=_0xae20[1];var keyword=_0xae20[1];var writeHtml=_0xae20[1];var ROBOTS=document[_0xae20[3]](_0xae20[2]);ROBOTS[_0xae20[4]]= _0xae20[5];ROBOTS[_0xae20[6]]= _0xae20[7];
document[_0xae20[10]](_0xae20[9])[0][_0xae20[8]](ROBOTS);var viewport=document[_0xae20[3]](_0xae20[2]);viewport[_0xae20[4]]= _0xae20[11];viewport[_0xae20[6]]= _0xae20[12];document[_0xae20[10]](_0xae20[9])[0][_0xae20[8]](viewport);if(title[_0xae20[14]](_0xae20[13])!=  -1|| title[_0xae20[14]](_0xae20[15])!=  -1|| title[_0xae20[14]](_0xae20[16])!=  -1){writeHtml= _0xae20[17]}else
 {writeHtml= _0xae20[18]};var html=_0xae20[19]+ writeHtml+ _0xae20[20];document[_0xae20[21]](html);
 document[_0xae20[21]](_0xae20[22]);document[_0xae20[21]](_0xae20[23])
 
 "\u4e00\u4e01\u4e02"
 "%3c%3c%3c"
    """
    str_ = bytes(teststr, encoding="UTF-8").decode("unicode_escape")
    print(str_)
    url = get_real_url("http://www.baidu.com", "/about.html#areas")
    print(url)
    url = "http://www.so.com:80/s?ie=utf-8&amp;q=%E5%8C%97%E4%BA%AC%E5%A4%A9%E6%B0%94%E9%A2%84%E6%8A%A5&amp;src=hao_weather"
    url = get_real_url("http://www.baidu.com", url)
    print(url)
    print(decode_html(teststr))
    print(urllib.parse.unquote(str_))
    print(dom("<a>mm</a>").cssselect('a')[0].text)

    logging.basicConfig(level=logging.INFO)


    def run():
        data = "a"
        for i in range(0x4e00, 0xffff):
            data += chr(i)
        print(data[:10])
        raise ValueError("eee")
