# This file was auto-generated by Fern from our API Definition.

from ..core.unchecked_base_model import UncheckedBaseModel
import typing
from .multi_page_crawler_settings_output_scraper import MultiPageCrawlerSettingsOutputScraper
from .multi_page_crawler_settings_output_text_extractor import MultiPageCrawlerSettingsOutputTextExtractor
import pydantic
from ..core.pydantic_utilities import IS_PYDANTIC_V2


class MultiPageCrawlerSettingsOutput(UncheckedBaseModel):
    scraper: typing.Optional[MultiPageCrawlerSettingsOutputScraper] = None
    text_extractor: typing.Optional[MultiPageCrawlerSettingsOutputTextExtractor] = None
    allow: typing.Optional[str] = pydantic.Field(default=None)
    """
    Comma-separated list of URL patterns to include. Only URLs matching these patterns will be crawled.
    """

    deny: typing.Optional[str] = pydantic.Field(default=None)
    """
    Comma-separated list of URL patterns to exclude. URLs matching these patterns will be skipped.
    """

    exclude_patterns: typing.Optional[str] = pydantic.Field(default=None)
    """
    Comma-separated list of regex patterns to exclude. URLs matching any of these patterns will be skipped. For example: '/private/, login\.html$' will skip URLs containing '/private/' and ending with 'login.html'
    """

    strip: typing.Optional[str] = pydantic.Field(default=None)
    """
    Comma-separated list of html tags to strip from the content.
    """

    strip_urls: typing.Optional[bool] = pydantic.Field(default=None)
    """
    Removes query parameters, fragments, or trailing slashes, to simplify or standardize urls.
    """

    allowed_domains: typing.Optional[str] = pydantic.Field(default=None)
    """
    Comma-separated list of domains to include. Only URLs from these domains will be crawled. Defaults to the domain of the startingURL.
    """

    max_depth: typing.Optional[int] = pydantic.Field(default=None)
    """
    Maximum number of link levels to follow from the starting page
    """

    max_page_count: typing.Optional[int] = pydantic.Field(default=None)
    """
    Maximum number of pages to crawl.
    """

    if IS_PYDANTIC_V2:
        model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True)  # type: ignore # Pydantic v2
    else:

        class Config:
            frozen = True
            smart_union = True
            extra = pydantic.Extra.allow
