# This file was auto-generated by Fern from our API Definition.

import typing

import pydantic
from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
from .date_filter import DateFilter
from .engine_type import EngineType
from .llm_mode import LlmMode


class ScrapeOptions(UniversalBaseModel):
    """
    Options for scraping operations.
    """

    wait_for_content: typing.Optional[bool] = pydantic.Field(default=None)
    """
    Wait for dynamic content to load
    """

    extract_images: typing.Optional[bool] = pydantic.Field(default=None)
    """
    Extract image URLs
    """

    follow_pagination: typing.Optional[bool] = pydantic.Field(default=None)
    """
    Follow pagination links
    """

    max_items: typing.Optional[int] = pydantic.Field(default=None)
    """
    Maximum items to extract
    """

    timeout: typing.Optional[int] = pydantic.Field(default=None)
    """
    Overall job timeout in seconds (default 5 min, max 30 min). Job will be marked as failed if it exceeds this time.
    """

    sync: typing.Optional[bool] = pydantic.Field(default=None)
    """
    Wait for results vs return job ID
    """

    date_filter: typing.Optional[DateFilter] = pydantic.Field(default=None)
    """
    Filter items by date
    """

    llm_mode: typing.Optional[LlmMode] = pydantic.Field(default=None)
    """
    LLM processing mode
    """

    engine_type: typing.Optional[EngineType] = pydantic.Field(default=None)
    """
    Force specific engine type
    """

    html_mode: typing.Optional[str] = pydantic.Field(default=None)
    """
    HTML processing mode: 'text' (clean text only), 'semantic' (preserve semantic HTML structure for e-commerce), 'full' (raw HTML), 'auto' (intelligent fallback based on content quality)
    """

    content_mode: typing.Optional[str] = pydantic.Field(default=None)
    """
    Content mode for LLM processing: 'text' (clean text), 'semantic_html' (preserve semantic structure), 'full_html' (minimal cleaning)
    """

    respect_rate_limits: typing.Optional[bool] = pydantic.Field(default=None)
    """
    Enable intelligent rate limit avoidance
    """

    delay_between_requests: typing.Optional[float] = pydantic.Field(default=None)
    """
    Delay between requests in seconds (auto-calculated if None)
    """

    max_retries: typing.Optional[int] = pydantic.Field(default=None)
    """
    Maximum retry attempts on rate limit errors
    """

    exponential_backoff: typing.Optional[bool] = pydantic.Field(default=None)
    """
    Use exponential backoff for retries
    """

    enable_pagination: typing.Optional[bool] = pydantic.Field(default=None)
    """
    Automatically follow pagination to scrape multiple pages
    """

    max_pages: typing.Optional[int] = pydantic.Field(default=None)
    """
    Maximum pages to scrape (None = unlimited, respecting max_properties)
    """

    max_properties: typing.Optional[int] = pydantic.Field(default=None)
    """
    Stop after this many properties (None = unlimited, respecting max_pages)
    """

    depth: typing.Optional[int] = pydantic.Field(default=None)
    """
    Scraping depth: 0=current page only, 1=follow links from listing, 2=go 2 levels deep
    """

    link_extraction_field: typing.Optional[str] = pydantic.Field(default=None)
    """
    Schema field containing URLs to follow (e.g., 'similar_listings[].url'). Auto-detected if None.
    """

    link_pattern: typing.Optional[str] = pydantic.Field(default=None)
    """
    Regex pattern to filter links for depth scraping (e.g., '/oferta/'). Auto-detected for common sites if None.
    """

    same_domain_only: typing.Optional[bool] = pydantic.Field(default=None)
    """
    Only follow links from the same domain when depth > 0
    """

    exclude_patterns: typing.Optional[typing.List[str]] = pydantic.Field(default=None)
    """
    List of regex patterns to exclude from link extraction (e.g., ['/login', '/cart'])
    """

    max_links_per_page: typing.Optional[int] = pydantic.Field(default=None)
    """
    Maximum links to follow per page when depth > 0
    """

    parallel_depth_scraping: typing.Optional[bool] = pydantic.Field(default=None)
    """
    Scrape depth links in parallel (faster) vs sequential (safer)
    """

    listing_page_schema: typing.Optional[typing.Dict[str, typing.Any]] = pydantic.Field(default=None)
    """
    Optional simplified schema for listing pages (depth=0). If provided, main schema is used for detail pages (depth=1+). Should extract URLs for depth scraping.
    """

    if IS_PYDANTIC_V2:
        model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True)  # type: ignore # Pydantic v2
    else:

        class Config:
            frozen = True
            smart_union = True
            extra = pydantic.Extra.allow
