# coding: utf-8

"""
    Mixpeek API

    This is the Mixpeek API, providing access to various endpoints for data processing and retrieval.

    The version of the OpenAPI document: 1.3.0
    Contact: info@mixpeek.com
    Generated by OpenAPI Generator (https://openapi-generator.tech)

    Do not edit the class manually.
"""  # noqa: E501


from __future__ import annotations
import pprint
import re  # noqa: F401
import json

from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictStr, field_validator
from typing import Any, ClassVar, Dict, List, Optional, Union
from typing_extensions import Annotated
from mixpeek.models.chunk_strategy import ChunkStrategy
from mixpeek.models.crawl_mode import CrawlMode
from mixpeek.models.document_id_strategy import DocumentIdStrategy
from mixpeek.models.render_strategy import RenderStrategy
from mixpeek.models.response_shape3 import ResponseShape3
from typing import Optional, Set
from typing_extensions import Self

class WebScraperExtractorParams(BaseModel):
    """
    Parameters for the web scraper extractor.  The web scraper extractor crawls websites and extracts content with three types of embeddings for comprehensive multimodal search:  **Embedding Types:** - Text (E5-Large): 1024D embeddings for page content - Code (Jina Code): 768D embeddings for code blocks - Images (SigLIP): 768D embeddings for figures/screenshots  **Crawl Modes:** - DETERMINISTIC: BFS following all links (default, predictable) - SEMANTIC: LLM-guided, prioritizes pages matching crawl_goal  **Rendering Strategies:** - STATIC: Fast HTTP fetch (default, works for most sites) - JAVASCRIPT: Playwright browser for SPAs (React/Vue/Angular) - AUTO: Tries static, falls back to JS if content too short  **Use Cases:** - Documentation freshness: Crawl docs, compare against course content - Job board ingestion: Extract job listings with structured data - Knowledge base building: Convert websites to searchable collections - Code example indexing: Find API usage patterns across docs
    """ # noqa: E501
    extractor_type: Optional[StrictStr] = Field(default='web_scraper', description="Discriminator field for parameter type identification.")
    max_depth: Optional[Annotated[int, Field(le=1000, strict=True, ge=0)]] = Field(default=2, description="Maximum link depth to crawl. 0=seed page only, 1=seed+direct links, etc. Default: 2. Increase for comprehensive crawls, decrease for targeted extraction.")
    max_pages: Optional[Annotated[int, Field(le=1000000, strict=True, ge=1)]] = Field(default=50, description="Maximum pages to crawl. Default: 50. Set higher (1000+) for large documentation sites. Max: 1,000,000.")
    crawl_timeout: Optional[Annotated[int, Field(le=3600, strict=True, ge=10)]] = Field(default=300, description="Maximum total time for crawling in seconds. Default: 300 (5 minutes). Increase for large sites with many pages. Max: 3600 (1 hour).")
    crawl_mode: Optional[CrawlMode] = Field(default=None, description="Crawl strategy. DETERMINISTIC: BFS all links (predictable). SEMANTIC: LLM-guided, prioritizes relevant pages (requires crawl_goal).")
    crawl_goal: Optional[StrictStr] = Field(default=None, description="Goal for semantic crawling. Only used when crawl_mode=SEMANTIC. Example: 'Find all S3 API documentation and examples'")
    render_strategy: Optional[RenderStrategy] = Field(default=None, description="How to render pages. AUTO (default): tries static, falls back to JS. STATIC: fast HTTP fetch. JAVASCRIPT: Playwright browser for SPAs.")
    include_patterns: Optional[List[StrictStr]] = Field(default=None, description="Regex patterns for URLs to include. Example: ['/docs/', '/api/']")
    exclude_patterns: Optional[List[StrictStr]] = Field(default=None, description="Regex patterns for URLs to exclude. Example: ['/blog/', '\\.pdf$']")
    chunk_strategy: Optional[ChunkStrategy] = Field(default=None, description="How to split page content. NONE: one chunk per page. SENTENCES/PARAGRAPHS: semantic boundaries. WORDS/CHARACTERS: fixed size chunks.")
    chunk_size: Optional[Annotated[int, Field(le=10000, strict=True, ge=1)]] = Field(default=500, description="Target size for each chunk (in units of chunk_strategy).")
    chunk_overlap: Optional[Annotated[int, Field(le=5000, strict=True, ge=0)]] = Field(default=50, description="Overlap between chunks to preserve context.")
    document_id_strategy: Optional[DocumentIdStrategy] = Field(default=None, description="How to generate document IDs. URL (default): stable across re-crawls. POSITION: order-based. CONTENT: deduplicates identical content.")
    generate_text_embeddings: Optional[StrictBool] = Field(default=True, description="Generate E5 embeddings for text content.")
    generate_code_embeddings: Optional[StrictBool] = Field(default=True, description="Generate Jina code embeddings for code blocks.")
    generate_image_embeddings: Optional[StrictBool] = Field(default=True, description="Generate SigLIP embeddings for images/figures.")
    response_shape: Optional[ResponseShape3] = None
    llm_provider: Optional[StrictStr] = Field(default=None, description="LLM provider for structured extraction: openai, google, anthropic")
    llm_model: Optional[StrictStr] = Field(default=None, description="LLM model for structured extraction.")
    llm_api_key: Optional[StrictStr] = Field(default=None, description="API key for LLM operations (BYOK - Bring Your Own Key). Supports: - Direct key: 'sk-proj-abc123...' - Secret reference: '{{SECRET.openai_api_key}}'  When using secret reference, the key is loaded from your organization's secrets vault at runtime. Store secrets via POST /v1/organizations/secrets.  If not provided, uses Mixpeek's default API keys.")
    max_retries: Optional[Annotated[int, Field(le=10, strict=True, ge=0)]] = Field(default=3, description="Maximum retry attempts for failed HTTP requests. Uses exponential backoff with jitter. Default: 3.")
    retry_base_delay: Optional[Union[Annotated[float, Field(le=30.0, strict=True, ge=0.1)], Annotated[int, Field(le=30, strict=True, ge=1)]]] = Field(default=1.0, description="Base delay in seconds for retry backoff. Actual delay = base * 2^attempt + jitter. Default: 1.0.")
    retry_max_delay: Optional[Union[Annotated[float, Field(le=300.0, strict=True, ge=1.0)], Annotated[int, Field(le=300, strict=True, ge=1)]]] = Field(default=30.0, description="Maximum delay in seconds between retries. Default: 30.")
    respect_retry_after: Optional[StrictBool] = Field(default=True, description="Respect Retry-After header from 429/503 responses. If False, uses exponential backoff instead. Default: True.")
    proxies: Optional[List[StrictStr]] = Field(default=None, description="List of proxy URLs for rotation. Supports formats: 'http://host:port', 'http://user:pass@host:port', 'socks5://host:port'. Proxies rotate on errors or every N requests.")
    rotate_proxy_on_error: Optional[StrictBool] = Field(default=True, description="Rotate to next proxy when request fails. Default: True.")
    rotate_proxy_every_n_requests: Optional[Annotated[int, Field(le=1000, strict=True, ge=0)]] = Field(default=0, description="Rotate proxy every N requests (0 = disabled). Useful for avoiding IP-based rate limits. Default: 0 (disabled).")
    captcha_service_provider: Optional[StrictStr] = Field(default=None, description="Captcha solving service provider: '2captcha', 'anti-captcha', 'capsolver'. If not set, captcha pages are skipped gracefully.")
    captcha_service_api_key: Optional[StrictStr] = Field(default=None, description="API key for captcha solving service. Supports secret reference: '{{SECRET.captcha_api_key}}'. Required if captcha_service_provider is set.")
    detect_captcha: Optional[StrictBool] = Field(default=True, description="Detect captcha challenges (Cloudflare, reCAPTCHA, hCaptcha). If detected and no solver configured, page is skipped. Default: True.")
    persist_cookies: Optional[StrictBool] = Field(default=True, description="Persist cookies across requests within a crawl session. Useful for sites requiring authentication. Default: True.")
    custom_headers: Optional[Dict[str, StrictStr]] = Field(default=None, description="Custom HTTP headers to include in all requests. Example: {'Authorization': 'Bearer token', 'X-Custom': 'value'}")
    delay_between_requests: Optional[Union[Annotated[float, Field(le=60.0, strict=True, ge=0.0)], Annotated[int, Field(le=60, strict=True, ge=0)]]] = Field(default=0.0, description="Delay in seconds between consecutive requests. Useful for polite crawling and avoiding rate limits. Default: 0 (no delay).")
    __properties: ClassVar[List[str]] = ["extractor_type", "max_depth", "max_pages", "crawl_timeout", "crawl_mode", "crawl_goal", "render_strategy", "include_patterns", "exclude_patterns", "chunk_strategy", "chunk_size", "chunk_overlap", "document_id_strategy", "generate_text_embeddings", "generate_code_embeddings", "generate_image_embeddings", "response_shape", "llm_provider", "llm_model", "llm_api_key", "max_retries", "retry_base_delay", "retry_max_delay", "respect_retry_after", "proxies", "rotate_proxy_on_error", "rotate_proxy_every_n_requests", "captcha_service_provider", "captcha_service_api_key", "detect_captcha", "persist_cookies", "custom_headers", "delay_between_requests"]

    @field_validator('extractor_type')
    def extractor_type_validate_enum(cls, value):
        """Validates the enum"""
        if value is None:
            return value

        if value not in set(['web_scraper']):
            raise ValueError("must be one of enum values ('web_scraper')")
        return value

    model_config = ConfigDict(
        populate_by_name=True,
        validate_assignment=True,
        protected_namespaces=(),
    )


    def to_str(self) -> str:
        """Returns the string representation of the model using alias"""
        return pprint.pformat(self.model_dump(by_alias=True))

    def to_json(self) -> str:
        """Returns the JSON representation of the model using alias"""
        # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
        return json.dumps(self.to_dict())

    @classmethod
    def from_json(cls, json_str: str) -> Optional[Self]:
        """Create an instance of WebScraperExtractorParams from a JSON string"""
        return cls.from_dict(json.loads(json_str))

    def to_dict(self) -> Dict[str, Any]:
        """Return the dictionary representation of the model using alias.

        This has the following differences from calling pydantic's
        `self.model_dump(by_alias=True)`:

        * `None` is only added to the output dict for nullable fields that
          were set at model initialization. Other fields with value `None`
          are ignored.
        """
        excluded_fields: Set[str] = set([
        ])

        _dict = self.model_dump(
            by_alias=True,
            exclude=excluded_fields,
            exclude_none=True,
        )
        # override the default output from pydantic by calling `to_dict()` of response_shape
        if self.response_shape:
            _dict['response_shape'] = self.response_shape.to_dict()
        return _dict

    @classmethod
    def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
        """Create an instance of WebScraperExtractorParams from a dict"""
        if obj is None:
            return None

        if not isinstance(obj, dict):
            return cls.model_validate(obj)

        _obj = cls.model_validate({
            "extractor_type": obj.get("extractor_type") if obj.get("extractor_type") is not None else 'web_scraper',
            "max_depth": obj.get("max_depth") if obj.get("max_depth") is not None else 2,
            "max_pages": obj.get("max_pages") if obj.get("max_pages") is not None else 50,
            "crawl_timeout": obj.get("crawl_timeout") if obj.get("crawl_timeout") is not None else 300,
            "crawl_mode": obj.get("crawl_mode"),
            "crawl_goal": obj.get("crawl_goal"),
            "render_strategy": obj.get("render_strategy"),
            "include_patterns": obj.get("include_patterns"),
            "exclude_patterns": obj.get("exclude_patterns"),
            "chunk_strategy": obj.get("chunk_strategy"),
            "chunk_size": obj.get("chunk_size") if obj.get("chunk_size") is not None else 500,
            "chunk_overlap": obj.get("chunk_overlap") if obj.get("chunk_overlap") is not None else 50,
            "document_id_strategy": obj.get("document_id_strategy"),
            "generate_text_embeddings": obj.get("generate_text_embeddings") if obj.get("generate_text_embeddings") is not None else True,
            "generate_code_embeddings": obj.get("generate_code_embeddings") if obj.get("generate_code_embeddings") is not None else True,
            "generate_image_embeddings": obj.get("generate_image_embeddings") if obj.get("generate_image_embeddings") is not None else True,
            "response_shape": ResponseShape3.from_dict(obj["response_shape"]) if obj.get("response_shape") is not None else None,
            "llm_provider": obj.get("llm_provider"),
            "llm_model": obj.get("llm_model"),
            "llm_api_key": obj.get("llm_api_key"),
            "max_retries": obj.get("max_retries") if obj.get("max_retries") is not None else 3,
            "retry_base_delay": obj.get("retry_base_delay") if obj.get("retry_base_delay") is not None else 1.0,
            "retry_max_delay": obj.get("retry_max_delay") if obj.get("retry_max_delay") is not None else 30.0,
            "respect_retry_after": obj.get("respect_retry_after") if obj.get("respect_retry_after") is not None else True,
            "proxies": obj.get("proxies"),
            "rotate_proxy_on_error": obj.get("rotate_proxy_on_error") if obj.get("rotate_proxy_on_error") is not None else True,
            "rotate_proxy_every_n_requests": obj.get("rotate_proxy_every_n_requests") if obj.get("rotate_proxy_every_n_requests") is not None else 0,
            "captcha_service_provider": obj.get("captcha_service_provider"),
            "captcha_service_api_key": obj.get("captcha_service_api_key"),
            "detect_captcha": obj.get("detect_captcha") if obj.get("detect_captcha") is not None else True,
            "persist_cookies": obj.get("persist_cookies") if obj.get("persist_cookies") is not None else True,
            "custom_headers": obj.get("custom_headers"),
            "delay_between_requests": obj.get("delay_between_requests") if obj.get("delay_between_requests") is not None else 0.0
        })
        return _obj


