#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
@File    :  AyuRequest.py
@Time    :  2022/8/30 14:52
@Author  :  Ayuge
@Version :  1.0
@Contact :  ayuge.s@qq.com
@License :  (c)Copyright 2022-2023
@Desc    :  None
"""
import copy
from scrapy import Request, FormRequest
from parsel.selector import create_root_node
from w3lib.html import strip_html5_whitespace
from scrapy.utils.response import get_base_url
from ayugespidertools.common.Params import Param
from scrapy.http.response.text import TextResponse
from scrapy.utils.python import to_bytes, is_listlike
from urllib.parse import urljoin, urlencode, urlsplit, urlunsplit
from lxml.html import FormElement, HtmlElement, HTMLParser, SelectElement
from typing import Callable, Iterable, List, Optional, Tuple, Type, TypeVar, Union


__all__ = [
    "AiohttpRequest",
    "AioFormRequest",
]


FormRequestTypeVar = TypeVar("FormRequestTypeVar", bound="FormRequest")
FormdataType = Optional[Union[dict, List[Tuple[str, str]]]]


class AiohttpRequest(Request):
    """
    为 scrapy 的 Request 对象添加额外的参数
    """

    def __init__(
        self,
        url: str,
        callback: Optional[Callable] = None,
        method: str = "GET",
        headers: Optional[dict] = None,
        body: Optional[Union[bytes, str]] = None,
        cookies: Optional[Union[dict, List[dict]]] = None,
        meta: Optional[dict] = None,
        *args,
        **kwargs
    ) -> None:
        # 用 meta 缓存 scrapy meta 的参数
        meta = copy.deepcopy(meta) or {}
        aiohttp_meta = meta.get("aiohttp_args") or {}

        # TODO: 可添加和修改默认参数的值，后续可以在此完善功能
        self.proxy = aiohttp_meta.get("proxy", {"http:": "http://10.10.10.10:10000"})
        self.timeout = aiohttp_meta.get("timeout", Param.aiohttp_req_timeout)
        self.sleep = aiohttp_meta.get("sleep", None)

        aiohttp_meta = meta.setdefault("aiohttp_args", {})
        aiohttp_meta["proxy"] = self.proxy
        aiohttp_meta["timeout"] = self.timeout
        super(AiohttpRequest, self).__init__(
            url,
            callback,
            method=method,
            headers=headers,
            body=body,
            cookies=cookies,
            meta=meta,
            *args,
            **kwargs
        )


class AioFormRequestBackup(AiohttpRequest, FormRequest):
    """
    这里只是用于测试，后期会删除此部分
    """

    def __init__(
        self,
        url: str,
        callback: Optional[Callable] = None,
        method: str = "GET",
        headers: Optional[dict] = None,
        body: Optional[Union[bytes, str]] = None,
        cookies: Optional[Union[dict, List[dict]]] = None,
        meta: Optional[dict] = None,
        formdata: FormdataType = None,
        *args,
        **kwargs
    ) -> None:
        super().__init__(
            url,
            callback,
            method,
            headers,
            body,
            cookies,
            meta,
            formdata,
            *args,
            **kwargs
        )

    @classmethod
    def from_response(
            cls: Type[FormRequestTypeVar],
            response: TextResponse,
            formname: Optional[str] = None,
            formid: Optional[str] = None,
            formnumber: Optional[int] = 0,
            formdata: FormdataType = None,
            clickdata: Optional[dict] = None,
            dont_click: bool = False,
            formxpath: Optional[str] = None,
            formcss: Optional[str] = None,
            **kwargs,
    ) -> FormRequestTypeVar:
        return super().from_response(
            response,
            formname,
            formid,
            formnumber,
            formdata,
            clickdata,
            dont_click,
            formxpath,
            formcss,
            **kwargs
        )


class AioFormRequest(AiohttpRequest):
    """
    为 scrapy 的 FormRequest 对象添加额外的参数
    (这里直接 copy 的 scrapy 的 FormRequest 内容)
    """

    valid_form_methods = ['GET', 'POST']

    def __init__(self, *args, formdata: FormdataType = None, **kwargs) -> None:
        if formdata and kwargs.get('method') is None:
            kwargs['method'] = 'POST'

        super().__init__(*args, **kwargs)

        if formdata:
            items = formdata.items() if isinstance(formdata, dict) else formdata
            form_query_str = _urlencode(items, self.encoding)
            if self.method == 'POST':
                self.headers.setdefault(b'Content-Type', b'application/x-www-form-urlencoded')
                self._set_body(form_query_str)
            else:
                self._set_url(urlunsplit(urlsplit(self.url)._replace(query=form_query_str)))

    @classmethod
    def from_response(
            cls: Type[FormRequestTypeVar],
            response: TextResponse,
            formname: Optional[str] = None,
            formid: Optional[str] = None,
            formnumber: Optional[int] = 0,
            formdata: FormdataType = None,
            clickdata: Optional[dict] = None,
            dont_click: bool = False,
            formxpath: Optional[str] = None,
            formcss: Optional[str] = None,
            **kwargs,
    ) -> FormRequestTypeVar:
        kwargs.setdefault('encoding', response.encoding)

        if formcss is not None:
            from parsel.csstranslator import HTMLTranslator
            formxpath = HTMLTranslator().css_to_xpath(formcss)

        form = _get_form(response, formname, formid, formnumber, formxpath)
        formdata = _get_inputs(form, formdata, dont_click, clickdata)
        url = _get_form_url(form, kwargs.pop('url', None))

        method = kwargs.pop('method', form.method)
        if method is not None:
            method = method.upper()
            if method not in cls.valid_form_methods:
                method = 'GET'

        return cls(url=url, method=method, formdata=formdata, **kwargs)


def _get_form_url(form: FormElement, url: Optional[str]) -> str:
    if url is None:
        action = form.get('action')
        if action is None:
            return form.base_url
        return urljoin(form.base_url, strip_html5_whitespace(action))
    return urljoin(form.base_url, url)


def _urlencode(seq: Iterable, enc: str) -> str:
    values = [(to_bytes(k, enc), to_bytes(v, enc))
              for k, vs in seq
              for v in (vs if is_listlike(vs) else [vs])]
    return urlencode(values, doseq=True)


def _get_form(
    response: TextResponse,
    formname: Optional[str],
    formid: Optional[str],
    formnumber: Optional[int],
    formxpath: Optional[str],
) -> FormElement:
    """Find the wanted form element within the given response."""
    root = create_root_node(response.text, HTMLParser, base_url=get_base_url(response))
    forms = root.xpath('//form')
    if not forms:
        raise ValueError(f"No <form> element found in {response}")

    if formname is not None:
        f = root.xpath(f'//form[@name="{formname}"]')
        if f:
            return f[0]

    if formid is not None:
        f = root.xpath(f'//form[@id="{formid}"]')
        if f:
            return f[0]

    # Get form element from xpath, if not found, go up
    if formxpath is not None:
        nodes = root.xpath(formxpath)
        if nodes:
            el = nodes[0]
            while True:
                if el.tag == 'form':
                    return el
                el = el.getparent()
                if el is None:
                    break
        raise ValueError(f'No <form> element found with {formxpath}')

    # If we get here, it means that either formname was None or invalid
    if formnumber is not None:
        try:
            form = forms[formnumber]
        except IndexError:
            raise IndexError(f"Form number {formnumber} not found in {response}")
        else:
            return form


def _get_inputs(
    form: FormElement,
    formdata: FormdataType,
    dont_click: bool,
    clickdata: Optional[dict],
) -> List[Tuple[str, str]]:
    """Return a list of key-value pairs for the inputs found in the given form."""
    try:
        formdata_keys = dict(formdata or ()).keys()
    except (ValueError, TypeError):
        raise ValueError('formdata should be a dict or iterable of tuples')

    if not formdata:
        formdata = []
    inputs = form.xpath('descendant::textarea'
                        '|descendant::select'
                        '|descendant::input[not(@type) or @type['
                        ' not(re:test(., "^(?:submit|image|reset)$", "i"))'
                        ' and (../@checked or'
                        '  not(re:test(., "^(?:checkbox|radio)$", "i")))]]',
                        namespaces={"re": "http://exslt.org/regular-expressions"})
    values = [
        (k, '' if v is None else v)
        for k, v in (_value(e) for e in inputs)
        if k and k not in formdata_keys
    ]

    if not dont_click:
        clickable = _get_clickable(clickdata, form)
        if clickable and clickable[0] not in formdata and not clickable[0] is None:
            values.append(clickable)

    if isinstance(formdata, dict):
        formdata = formdata.items()  # type: ignore[assignment]

    values.extend((k, v) for k, v in formdata if v is not None)
    return values


def _value(ele: HtmlElement):
    n = ele.name
    v = ele.value
    if ele.tag == 'select':
        return _select_value(ele, n, v)
    return n, v


def _select_value(ele: SelectElement, n: str, v: str):
    multiple = ele.multiple
    if v is None and not multiple:
        # Match browser behaviour on simple select tag without options selected
        # And for select tags without options
        o = ele.value_options
        return (n, o[0]) if o else (None, None)
    elif v is not None and multiple:
        # This is a workround to bug in lxml fixed 2.3.1
        # fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139
        selected_options = ele.xpath('.//option[@selected]')
        values = [(o.get('value') or o.text or '').strip() for o in selected_options]
        return n, values
    return n, v


def _get_clickable(clickdata: Optional[dict], form: FormElement) -> Optional[Tuple[str, str]]:
    """
    Returns the clickable element specified in clickdata,
    if the latter is given. If not, it returns the first
    clickable element found
    """
    clickables = list(form.xpath(
        'descendant::input[re:test(@type, "^(submit|image)$", "i")]'
        '|descendant::button[not(@type) or re:test(@type, "^submit$", "i")]',
        namespaces={"re": "http://exslt.org/regular-expressions"}
    ))
    if not clickables:
        return None

    # If we don't have clickdata, we just use the first clickable element
    if clickdata is None:
        el = clickables[0]
        return (el.get('name'), el.get('value') or '')

    # If clickdata is given, we compare it to the clickable elements to find a
    # match. We first look to see if the number is specified in clickdata,
    # because that uniquely identifies the element
    nr = clickdata.get('nr', None)
    if nr is not None:
        try:
            el = list(form.inputs)[nr]
        except IndexError:
            pass
        else:
            return (el.get('name'), el.get('value') or '')

    # We didn't find it, so now we build an XPath expression out of the other
    # arguments, because they can be used as such
    xpath = './/*' + ''.join(f'[@{k}="{v}"]' for k, v in clickdata.items())
    el = form.xpath(xpath)
    if len(el) == 1:
        return (el[0].get('name'), el[0].get('value') or '')
    elif len(el) > 1:
        raise ValueError(f"Multiple elements found ({el!r}) matching the "
                         f"criteria in clickdata: {clickdata!r}")
    else:
        raise ValueError(f'No clickable element matching clickdata: {clickdata!r}')
