# This package will contain the spiders of your Scrapy project
#
# Please refer to the documentation for information on how to create and manage
# your spiders.

import scrapy

from lich_linkextractor.items import ExampleItem


class ExampleSpider(scrapy.Spider):
    """ ExampleSpider
    Auto generated by os-scrapy-cookiecuter

    Run:
        scrapy crawl example
    """

    name = "example"

    def start_requests(self):
        yield scrapy.Request(
            url="https://www.example.com/",
            headers={"Referer":"http://www.123.com"},
            meta={
                # "depth": 0,
                "extractor.depth_limit": 1,
                # "extractor.redirect.switch": True,
                "extractor.rules": [
                    {
                        
                        "same_domain_only": False,
                        "allow": [],
                        "allow_domains": [],
                        "deny": [],
                        "deny_domains": [],
                        "restrict_xpaths": [],
                        "type": "lxml",
                    }
                ],
                # "extractor2.template": {
                #     "isRoot": True,
                #     "method": "table",
                #     "listData": [
                #         {
                #             "id": "92147549961982",
                #             "listName": "xingming",
                #             "method": "text",
                #             "rawSelector": ".neiye > .Survey > .Search_box > .Expert_xq > .Expert_xq11",
                #             "required": True,
                #             "selector": ".Expert_xq11",
                #             "xpath": './/*[contains(concat(" ",normalize-space(@class)," ")," Expert_xq11 ")]',
                #         },
                #         {
                #             "id": "92148240689730",
                #             "listName": "jianjie",
                #             "method": "text",
                #             "rawSelector": ".neiye > .Survey > .Search_box > .Expert_xq > ul > li > span > span",
                #             "required": True,
                #             "selector": "ul > li > span > span",
                #             "xpath": ".//ul/li/span/span",
                #         },
                #     ],
                #     "selector": ".neiye > .Survey > .Search_box > .Expert_xq",
                #     "xpath": './/*[contains(concat(" ",normalize-space(@class)," ")," neiye ")]/*[contains(concat(" ",normalize-space(@class)," ")," Survey ")]/*[contains(concat(" ",normalize-space(@class)," ")," Search_box ")]/*[contains(concat(" ",normalize-space(@class)," ")," Expert_xq ")]',
                # },
                "render_proxy.api": "http://render-proxy.thanos.sogou/RenderService/Render",
                "render_proxy.disableCacheserver": False,
                "render_proxy.downloadCss": False,
                "render_proxy.downloadPic": False,
                "render_proxy.enableJs": False,
                "render_proxy.loadIframe": False,
                "render_proxy.loadOtherResource": False,
                "render_proxy.needPosition": False,
                "render_proxy.needScreenshot": False,
            },
        )

    def parse(self, response):
        yield ExampleItem(
            url=response.url,
            request_headers=response.request.headers,
            response_headers=response.headers,
            status=response.status,
            meta=response.meta,
            body=response.body,
        )
