"""
Spider Middleware Manager
爬虫中间件管理器

This module provides the SpiderMiddlewareManager class, which manages the execution
of spider middleware components. Spider middlewares process the input and output of
spiders, allowing modification of requests and responses, as well as handling of
exceptions that occur during spider processing.
此模块提供了SpiderMiddlewareManager类，用于管理爬虫中间件组件的执行。爬虫中间件
处理爬虫的输入和输出，允许修改请求和响应，以及处理爬虫处理过程中发生的异常。

Spider middlewares are loaded from the SPIDER_MIDDLEWARES setting and are executed
in the order specified by their priority values. They can intercept and modify the
requests generated by spiders, the responses passed to spiders, and the items and
requests yielded by spiders.
爬虫中间件从SPIDER_MIDDLEWARES设置加载，并按照其优先级值指定的顺序执行。它们可以
拦截和修改由爬虫生成的请求、传递给爬虫的响应，以及由爬虫产生的项目和请求。

See documentation in docs/topics/spider-middleware.rst
参见文档 docs/topics/spider-middleware.rst
"""
from itertools import islice
from typing import AsyncIterable, Iterable, Callable, Union

from aioscrapy.exceptions import _InvalidOutput
from aioscrapy.utils.conf import build_component_list

from aioscrapy.http import Request, Response
from aioscrapy import Spider
from aioscrapy.settings import Settings
from aioscrapy.utils.tools import async_generator_wrapper
from aioscrapy.utils.tools import call_helper
from aioscrapy.middleware.absmanager import AbsMiddlewareManager


def _fname(f):
    """
    Get the full name of a method.
    获取方法的完整名称。

    This helper function returns the full name of a method in the format
    "ClassName.method_name". It is used for error messages to identify which
    middleware method raised an exception or returned an invalid output.
    此辅助函数返回格式为"ClassName.method_name"的方法的完整名称。它用于
    错误消息，以识别哪个中间件方法引发了异常或返回了无效输出。

    Args:
        f: The method to get the name of.
           要获取名称的方法。

    Returns:
        str: The full name of the method in the format "ClassName.method_name".
             格式为"ClassName.method_name"的方法的完整名称。
    """
    # Get the class name and method name and format them
    # 获取类名和方法名并格式化它们
    return "{}.{}".format(
        f.__self__.__class__.__name__,  # Class name 类名
        f.__func__.__name__             # Method name 方法名
    )


class SpiderMiddlewareManager(AbsMiddlewareManager):
    """
    Manager for spider middleware components.
    爬虫中间件组件的管理器。

    This class manages the execution of spider middleware components, which process
    the input and output of spiders. It inherits from AbsMiddlewareManager and
    implements the specific behavior for spider middlewares.
    此类管理爬虫中间件组件的执行，这些组件处理爬虫的输入和输出。它继承自
    AbsMiddlewareManager，并实现了爬虫中间件的特定行为。

    Spider middlewares can intercept and modify the requests generated by spiders,
    the responses passed to spiders, and the items and requests yielded by spiders.
    They can also handle exceptions that occur during spider processing.
    爬虫中间件可以拦截和修改由爬虫生成的请求、传递给爬虫的响应，以及由爬虫产生的
    项目和请求。它们还可以处理爬虫处理过程中发生的异常。
    """

    # Name of the component
    # 组件的名称
    component_name = 'spider middleware'

    @classmethod
    def _get_mwlist_from_settings(cls, settings: Settings):
        """
        Get the list of spider middleware classes from settings.
        从设置中获取爬虫中间件类列表。

        This method implements the abstract method from AbsMiddlewareManager.
        It retrieves the list of spider middleware classes from the
        SPIDER_MIDDLEWARES setting.
        此方法实现了AbsMiddlewareManager中的抽象方法。它从SPIDER_MIDDLEWARES
        设置中检索爬虫中间件类列表。

        Args:
            settings: The settings object.
                     设置对象。

        Returns:
            list: A list of spider middleware class paths.
                 爬虫中间件类路径列表。
        """
        # Build component list from SPIDER_MIDDLEWARES setting
        # 从SPIDER_MIDDLEWARES设置构建组件列表
        return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES'))

    def _add_middleware(self, mw):
        """
        Add a middleware instance to the manager.
        将中间件实例添加到管理器。

        This method overrides the method from AbsMiddlewareManager to register
        the specific methods of spider middlewares: process_spider_input,
        process_start_requests, process_spider_output, and process_spider_exception.
        此方法覆盖了AbsMiddlewareManager中的方法，以注册爬虫中间件的特定方法：
        process_spider_input、process_start_requests、process_spider_output和
        process_spider_exception。

        Note that process_spider_input and process_start_requests methods are called
        in the order they are registered, while process_spider_output and
        process_spider_exception methods are called in reverse order.
        请注意，process_spider_input和process_start_requests方法按照它们注册的
        顺序调用，而process_spider_output和process_spider_exception方法按照
        相反的顺序调用。

        Args:
            mw: The middleware instance to add.
                要添加的中间件实例。
        """
        # Call parent method to register open_spider and close_spider methods
        # 调用父方法来注册open_spider和close_spider方法
        super(SpiderMiddlewareManager, self)._add_middleware(mw)

        # Register process_spider_input method if it exists
        # 如果存在，则注册process_spider_input方法
        if hasattr(mw, 'process_spider_input'):
            self.methods['process_spider_input'].append(mw.process_spider_input)

        # Register process_start_requests method if it exists (added to the left for reverse order)
        # 如果存在，则注册process_start_requests方法（添加到左侧以便逆序执行）
        if hasattr(mw, 'process_start_requests'):
            self.methods['process_start_requests'].appendleft(mw.process_start_requests)

        # Get process_spider_output method if it exists
        # 如果存在，则获取process_spider_output方法
        process_spider_output = getattr(mw, 'process_spider_output', None)

        # Register process_spider_output method (added to the left for reverse order)
        # 注册process_spider_output方法（添加到左侧以便逆序执行）
        self.methods['process_spider_output'].appendleft(process_spider_output)

        # Get process_spider_exception method if it exists
        # 如果存在，则获取process_spider_exception方法
        process_spider_exception = getattr(mw, 'process_spider_exception', None)

        # Register process_spider_exception method (added to the left for reverse order)
        # 注册process_spider_exception方法（添加到左侧以便逆序执行）
        self.methods['process_spider_exception'].appendleft(process_spider_exception)

    async def scrape_response(self, scrape_func: Callable, response: Response, request: Request, spider: Spider):
        """
        Process a response through the spider middleware chain.
        通过爬虫中间件链处理响应。

        This method is the core of the spider middleware processing. It handles the
        entire process of passing a response through the middleware chain, calling
        the spider's callback function, and processing the output through the
        middleware chain again.
        此方法是爬虫中间件处理的核心。它处理通过中间件链传递响应、调用爬虫的回调
        函数以及再次通过中间件链处理输出的整个过程。

        Args:
            scrape_func: The spider's callback function to process the response.
                        处理响应的爬虫回调函数。
            response: The response to process.
                     要处理的响应。
            request: The request that generated the response.
                    生成响应的请求。
            spider: The spider that made the request.
                   发出请求的爬虫。

        Returns:
            The result of processing the response through the middleware chain.
            通过中间件链处理响应的结果。
        """
        async def process_spider_input(response_) -> Union[AsyncIterable, Iterable]:
            """
            Process a response through all registered process_spider_input methods.
            通过所有已注册的process_spider_input方法处理响应。

            This function calls each middleware's process_spider_input method in the
            order they were registered. If any middleware raises an exception, it is
            handled by the scrape_func.
            此函数按照它们注册的顺序调用每个中间件的process_spider_input方法。
            如果任何中间件引发异常，则由scrape_func处理。

            Args:
                response_: The response to process.
                          要处理的响应。

            Returns:
                Union[AsyncIterable, Iterable]: The result of processing the response.
                                              处理响应的结果。
            """
            # Process the response through all registered process_spider_input methods
            # 通过所有已注册的process_spider_input方法处理响应
            for method in self.methods['process_spider_input']:
                try:
                    # Call the method with the response
                    # 使用响应调用方法
                    result = await call_helper(method, response=response_, spider=spider)

                    # Validate the return value
                    # 验证返回值
                    if result is not None:
                        raise _InvalidOutput(
                            f"Middleware {_fname(method)} must return None or raise an exception, got {type(result)}"
                        )
                except _InvalidOutput:
                    # Re-raise _InvalidOutput exceptions
                    # 重新引发_InvalidOutput异常
                    raise
                except BaseException as exception:
                    # Handle other exceptions by calling the scrape_func with the exception
                    # 通过使用异常调用scrape_func来处理其他异常
                    iterable_or_exception = await call_helper(scrape_func, exception, request)

                    # If the scrape_func returned the exception, re-raise it
                    # 如果scrape_func返回了异常，则重新引发它
                    if iterable_or_exception is exception:
                        raise iterable_or_exception

                    # Otherwise, return the result
                    # 否则，返回结果
                    return iterable_or_exception

            # If all middleware methods passed, call the scrape_func with the response
            # 如果所有中间件方法都通过，则使用响应调用scrape_func
            return await call_helper(scrape_func, response_, request)

        async def _evaluate_iterable(result: Union[AsyncIterable, Iterable], exception_processor_index):
            """
            Evaluate an iterable and handle any exceptions that occur.
            评估可迭代对象并处理发生的任何异常。

            This function converts any iterable to an async generator and yields
            its items. If an exception occurs during iteration, it is handled by
            the process_spider_exception function.
            此函数将任何可迭代对象转换为异步生成器并产生其项目。如果在迭代期间
            发生异常，则由process_spider_exception函数处理。

            Args:
                result: The iterable to evaluate.
                       要评估的可迭代对象。
                exception_processor_index: The index to start processing exceptions from.
                                          开始处理异常的索引。

            Yields:
                The items from the iterable.
                可迭代对象中的项目。
            """
            try:
                # Convert all non-AsyncGeneratorType to AsyncGeneratorType objects
                # 将所有非AsyncGeneratorType变成AsyncGeneratorType对象
                async for r in await async_generator_wrapper(result):
                    yield r
            except BaseException as ex:
                # Handle exceptions by calling process_spider_exception
                # 通过调用process_spider_exception处理异常
                exception_result = await process_spider_exception(ex, exception_processor_index)

                # If the result is an exception, re-raise it
                # 如果结果是异常，则重新引发它
                if isinstance(exception_result, BaseException):
                    raise exception_result

                # Otherwise, recursively evaluate the result
                # 否则，递归评估结果
                async for r in _evaluate_iterable(exception_result, exception_processor_index):
                    yield r

        async def process_spider_exception(exception, start_index=0):
            """
            Process an exception through all registered process_spider_exception methods.
            通过所有已注册的process_spider_exception方法处理异常。

            This function calls each middleware's process_spider_exception method in
            reverse order from how they were registered. If any middleware returns an
            iterable, the exception handling stops and the iterable is processed by
            the process_spider_output function.
            此函数按照与它们注册的相反的顺序调用每个中间件的process_spider_exception方法。
            如果任何中间件返回可迭代对象，则异常处理停止，并且可迭代对象由
            process_spider_output函数处理。

            Args:
                exception: The exception to process.
                          要处理的异常。
                start_index: The index to start processing from.
                            开始处理的索引。

            Returns:
                The result of processing the exception, or the exception itself if
                no middleware handled it.
                处理异常的结果，如果没有中间件处理它，则为异常本身。
            """
            # Don't handle _InvalidOutput exceptions
            # 不处理_InvalidOutput异常
            if isinstance(exception, _InvalidOutput):
                raise exception

            # Get the list of methods to call, starting from start_index
            # 获取要调用的方法列表，从start_index开始
            method_list = islice(self.methods['process_spider_exception'], start_index, None)

            # Process each method
            # 处理每个方法
            for method_index, method in enumerate(method_list, start=start_index):
                # Skip None methods
                # 跳过None方法
                if method is None:
                    continue

                # Call the method with the exception
                # 使用异常调用方法
                result = await call_helper(method, response=response, exception=exception, spider=spider)

                # If the result is an iterable, stop exception handling and process the output
                # 如果结果是可迭代对象，则停止异常处理并处理输出
                if isinstance(result, AsyncIterable):
                    # Stop exception handling by handing control over to the
                    # process_spider_output chain if an iterable has been returned
                    # 如果返回了可迭代对象，则通过将控制权交给process_spider_output链来停止异常处理
                    return await process_spider_output(result, method_index + 1)
                elif result is None:
                    # If the result is None, continue to the next method
                    # 如果结果为None，则继续下一个方法
                    continue
                else:
                    # If the result is neither an iterable nor None, raise an error
                    # 如果结果既不是可迭代对象也不是None，则引发错误
                    raise _InvalidOutput(
                            f"Middleware {_fname(method)}  must return None or an iterable, got {type(result)}"
                        )

            # If no middleware handled the exception, re-raise it
            # 如果没有中间件处理异常，则重新引发它
            raise exception

        async def process_spider_output(result, start_index=0):
            """
            Process an iterable through all registered process_spider_output methods.
            通过所有已注册的process_spider_output方法处理可迭代对象。

            This function calls each middleware's process_spider_output method in
            reverse order from how they were registered. Each middleware can modify
            the iterable or return a new one.
            此函数按照与它们注册的相反的顺序调用每个中间件的process_spider_output方法。
            每个中间件可以修改可迭代对象或返回一个新的可迭代对象。

            Args:
                result: The iterable to process.
                       要处理的可迭代对象。
                start_index: The index to start processing from.
                            开始处理的索引。

            Returns:
                The processed iterable.
                处理后的可迭代对象。
            """
            # Items in this iterable do not need to go through the process_spider_output
            # chain, they went through it already from the process_spider_exception method
            # 此可迭代对象中的项目不需要通过process_spider_output链，
            # 它们已经通过process_spider_exception方法通过了它

            # Get the list of methods to call, starting from start_index
            # 获取要调用的方法列表，从start_index开始
            method_list = islice(self.methods['process_spider_output'], start_index, None)

            # Process each method
            # 处理每个方法
            for method_index, method in enumerate(method_list, start=start_index):
                # Skip None methods
                # 跳过None方法
                if method is None:
                    continue

                try:
                    # Call the method with the result
                    # 使用结果调用方法
                    # Might fail directly if the output value is not a generator
                    # 如果输出值不是生成器，可能会直接失败
                    result = await call_helper(method, response=response, result=result, spider=spider)
                except BaseException as ex:
                    # Handle exceptions by calling process_spider_exception
                    # 通过调用process_spider_exception处理异常
                    exception_result = await process_spider_exception(ex, method_index + 1)

                    # If the result is an exception, re-raise it
                    # 如果结果是异常，则重新引发它
                    if isinstance(exception_result, BaseException):
                        raise

                    # Otherwise, return the result
                    # 否则，返回结果
                    return exception_result

                # Validate the return value
                # 验证返回值
                if isinstance(result, AsyncIterable):
                    # If the result is an iterable, evaluate it
                    # 如果结果是可迭代对象，则评估它
                    result = _evaluate_iterable(result, method_index + 1)
                else:
                    # If the result is not an iterable, raise an error
                    # 如果结果不是可迭代对象，则引发错误
                    raise _InvalidOutput(f"Middleware {_fname(method)} must return an iterable, got {type(result)}")

            # Return the final result
            # 返回最终结果
            return result

        async def process_callback_output(result: Union[AsyncIterable, Iterable]):
            """
            Process the output of the spider's callback function.
            处理爬虫回调函数的输出。

            This function evaluates the iterable returned by the spider's callback
            function and processes it through the process_spider_output chain.
            此函数评估爬虫回调函数返回的可迭代对象，并通过process_spider_output链处理它。

            Args:
                result: The iterable returned by the spider's callback function.
                       爬虫回调函数返回的可迭代对象。

            Returns:
                The processed iterable.
                处理后的可迭代对象。
            """
            # Evaluate the iterable
            # 评估可迭代对象
            result: AsyncIterable = _evaluate_iterable(result, 0)

            # Process the result through the process_spider_output chain
            # 通过process_spider_output链处理结果
            return await process_spider_output(result)

        try:
            # Process the response through the process_spider_input chain
            # 通过process_spider_input链处理响应
            _iterable = await process_spider_input(response)
        except BaseException as exc:
            # If an exception occurs, process it through the process_spider_exception chain
            # 如果发生异常，则通过process_spider_exception链处理它
            return await process_spider_exception(exc)
        else:
            # If no exception occurs, process the output through the process_callback_output chain
            # 如果没有发生异常，则通过process_callback_output链处理输出
            return await process_callback_output(_iterable)

    async def process_start_requests(self, start_requests, spider):
        """
        Process start requests through all registered process_start_requests methods.
        通过所有已注册的process_start_requests方法处理起始请求。

        This method calls each middleware's process_start_requests method in the
        order they were registered. Each middleware can modify the start requests
        or return a new iterable of requests.
        此方法按照它们注册的顺序调用每个中间件的process_start_requests方法。
        每个中间件可以修改起始请求或返回一个新的请求可迭代对象。

        Args:
            start_requests: The iterable of start requests from the spider.
                           来自爬虫的起始请求可迭代对象。
            spider: The spider that generated the start requests.
                   生成起始请求的爬虫。

        Returns:
            The processed iterable of start requests.
            处理后的起始请求可迭代对象。
        """
        # Process the start requests through the chain of process_start_requests methods
        # 通过process_start_requests方法链处理起始请求
        return await self._process_chain('process_start_requests', start_requests, spider)
