# This package will contain the spiders of your Scrapy project
#
# Please refer to the documentation for information on how to create and manage
# your spiders.

import scrapy

from lich_linkextractor.items import ExampleItem


class ExampleSpider(scrapy.Spider):
    """ ExampleSpider
    Auto generated by os-scrapy-cookiecuter

    Run:
        scrapy crawl example
    """

    name = "example"

    def start_requests(self):
        yield scrapy.Request(
            url="http://www.example.com",
            meta={
                "extractor.depth_limit":2,
                "extractor.rules": [
                    {
                        "allow": [],
                        "deny": [],
                        "allow_domains": [],
                        "deny_domains": [],
                        "restrict_xpaths": [],
                        "restrict_css": [],
                        
                    }
                ],
                "render_proxy.api": "http://render-proxy.thanos.sogou/RenderService/Render",
                "render_proxy.disableCacheserver": False,
                "render_proxy.downloadCss": False,
                "render_proxy.downloadPic": False,
                "render_proxy.enableJs": False,
                "render_proxy.loadIframe": False,
                "render_proxy.loadOtherResource": False,
                "render_proxy.needPosition": False,
                "render_proxy.needScreenshot": False,
            },
        )

    def parse(self, response):
        yield ExampleItem(
            url=response.url,
            request_headers=response.request.headers,
            response_headers=response.headers,
            status=response.status,
            meta=response.meta,
            body=response.body,
        )
