# This file was auto-generated by Fern from our API Definition.

import typing

import pydantic
from ..core.pydantic_utilities import IS_PYDANTIC_V2
from ..core.unchecked_base_model import UncheckedBaseModel
from .sitemap_crawler_settings_output_scraper import SitemapCrawlerSettingsOutputScraper
from .sitemap_crawler_settings_output_text_extractor import SitemapCrawlerSettingsOutputTextExtractor


class SitemapCrawlerSettingsOutput(UncheckedBaseModel):
    scraper: typing.Optional[SitemapCrawlerSettingsOutputScraper] = None
    text_extractor: typing.Optional[SitemapCrawlerSettingsOutputTextExtractor] = None
    allow: typing.Optional[str] = pydantic.Field(default=None)
    """
    Comma-separated list of URL patterns to include. Only URLs matching these patterns will be crawled.
    """

    deny: typing.Optional[str] = pydantic.Field(default=None)
    """
    Comma-separated list of URL patterns to exclude. URLs matching these patterns will be skipped. Supports both simple substrings (like '/admin') and regex patterns (like 'login\.html$').
    """

    strip: typing.Optional[str] = pydantic.Field(default=None)
    """
    Comma-separated list of html tags to strip from the content.
    """

    strip_urls: typing.Optional[bool] = pydantic.Field(default=None)
    """
    Removes query parameters, fragments, or trailing slashes, to simplify or standardize urls.
    """

    allowed_domains: typing.Optional[str] = pydantic.Field(default=None)
    """
    Comma-separated list of domains to include. Only URLs from these domains will be crawled. Defaults to the domain of the startingURL.
    """

    user_agent_header: typing.Optional[str] = pydantic.Field(default=None)
    """
    User agent header to use for the crawl. By default, crawler will rotate user agents.
    """

    max_depth: typing.Optional[int] = pydantic.Field(default=None)
    """
    Maximum number of link levels to follow from the starting page
    """

    max_page_count: typing.Optional[int] = pydantic.Field(default=None)
    """
    Maximum number of pages to crawl.
    """

    exclude_before_lastmod_date: typing.Optional[str] = pydantic.Field(default=None)
    """
    Exclude pages last modified before this date.
    """

    if IS_PYDANTIC_V2:
        model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True)  # type: ignore # Pydantic v2
    else:

        class Config:
            frozen = True
            smart_union = True
            extra = pydantic.Extra.allow
