# This file was auto-generated by Fern from our API Definition.

import typing

import pydantic
from ..core.pydantic_utilities import IS_PYDANTIC_V2
from ..core.unchecked_base_model import UncheckedBaseModel
from .source_sync_website_settings_splitter import SourceSyncWebsiteSettingsSplitter
from .text_extractor import TextExtractor


class SourceSyncWebsiteSettings(UncheckedBaseModel):
    selectors: typing.Optional[typing.List[str]] = None
    exclude_selectors: typing.Optional[typing.List[str]] = None
    strip: typing.Optional[typing.List[str]] = None
    strip_urls: typing.Optional[bool] = None
    allowed_domains: typing.Optional[typing.List[str]] = None
    deny: typing.Optional[typing.List[str]] = None
    job_id: typing.Optional[str] = None
    exclude_before_lastmod_date: typing.Optional[str] = None
    splitter: typing.Optional[SourceSyncWebsiteSettingsSplitter] = None
    wait_for_load_state: typing.Optional[str] = None
    scraper: typing.Optional[str] = None
    text_extractor: typing.Optional[TextExtractor] = None
    text_extractor_config: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = None
    max_depth: typing.Optional[int] = None
    max_page_count: typing.Optional[int] = None
    allow: typing.Optional[typing.List[str]] = None
    exclude_patterns: typing.Optional[typing.List[str]] = pydantic.Field(default=None)
    """
    A comma-separated list of classes, ids, or tags to exclude
    """

    sitemap_urls: typing.Optional[typing.List[str]] = None
    start_urls: typing.Optional[typing.List[str]] = None
    source_archetype_id: typing.Literal["com.scoutos.website"] = "com.scoutos.website"

    if IS_PYDANTIC_V2:
        model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True)  # type: ignore # Pydantic v2
    else:

        class Config:
            frozen = True
            smart_union = True
            extra = pydantic.Extra.allow
