# -*- coding: utf-8 -*-
"""
Created on 2020/4/22 10:41 PM
---------
@summary:
---------
@author: Boris
@email: boris_liu@foxmail.com
"""

import feapder
from feapder import Item, UpdateItem


class SpiderDataItem(Item):
    """
    This class was generated by feapder.
    command: feapder create -i spider_data.
    """

    def __init__(self, *args, **kwargs):
        # self.id = None
        self.title = None

class TestAirSpider(feapder.AirSpider):
    __custom_setting__ = dict(
        ITEM_PIPELINES=[
            "pipelines.Pipeline",

        ],
        EXPORT_DATA_MAX_FAILED_TIMES=1,  # 导出数据时最大的失败次数，包括保存和更新，超过这个次数报警
        EXPORT_DATA_MAX_RETRY_TIMES=1  # 导出数据时最大的重试次数，包括保存和更新，超过这个次数则放弃重试
    )

    def start_callback(self):
        print("爬虫开始")

    def end_callback(self):
        print("爬虫结束")

    def start_requests(self, *args, **kws):
        for i in range(2):
            yield feapder.Request(f"https://www.baidu.com#{i}")

    def download_midware(self, request):
        # request.headers = {'User-Agent': ""}
        # request.proxies = {"https":"https://12.12.12.12:6666"}
        # request.cookies = {}
        return request

    def validate(self, request, response):
        if response.status_code != 200:
            raise Exception("response code not 200") # 重试

        # if "哈哈" not in response.text:
        #     return False # 抛弃当前请求


    def parse(self, request, response):
        print(response.bs4().title)
        print(response.xpath("//title").extract_first())

        title = response.xpath("//title/text()").extract_first()  # 取标题
        item = SpiderDataItem()  # 声明一个item
        item.title = title  # 给item属性赋值
        yield item  # 返回item， item会自动批量入库


if __name__ == "__main__":
    TestAirSpider().start()
