# coding: utf-8

"""
    Mixpeek API

    This is the Mixpeek API, providing access to various endpoints for data processing and retrieval.

    The version of the OpenAPI document: 1.3.0
    Contact: info@mixpeek.com
    Generated by OpenAPI Generator (https://openapi-generator.tech)

    Do not edit the class manually.
"""  # noqa: E501


from __future__ import annotations
import pprint
import re  # noqa: F401
import json

from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictStr
from typing import Any, ClassVar, Dict, List, Optional, Union
from typing_extensions import Annotated
from mixpeek.models.error_handling import ErrorHandling
from mixpeek.models.logical_operator import LogicalOperator
from mixpeek.models.url_patterns import URLPatterns
from typing import Optional, Set
from typing_extensions import Self

class StageParamsWebScrape(BaseModel):
    """
    Configuration for web scrape (content fetching) stage.  **Stage Category**: ENRICH (1-1 Enrichment)  **Transformation**: N documents → N documents (same count, expanded schema)  **Purpose**: Fetches full web page content from URLs found in document fields. Uses the Engine's Playwright service to extract clean text, metadata, and HTML from web pages. Enriches documents with complete content beyond the snippets provided by search APIs like Exa.  **When to Use**:     - After web_search stage to fetch full content from search results     - When documents contain URLs that need content extraction     - For article/blog content aggregation     - To get full page text beyond API snippets     - For content analysis requiring complete text     - Web scraping as part of retrieval pipeline  **When NOT to Use**:     - When URL snippets/titles are sufficient (web_search provides these)     - For very large numbers of URLs (slow, browser-intensive)     - When content is behind authentication (Playwright can't authenticate)     - For dynamic content requiring complex interactions     - When speed is critical (browser rendering is slow: 2-5s per page)  **Operational Behavior**:     - Enriches each input document (1-1 operation)     - Maintains document count: N in → N out (or fewer with REMOVE error handling)     - Expands schema: adds web content fields to each document     - Makes HTTP requests to Engine Playwright service     - Slow operation: 100ms-5s per document depending on strategy     - Supports concurrent requests with configurable batch size     - Browser-intensive for JavaScript strategy  **Common Pipeline Position**:     - web_search → web_scrape (get snippets then full content)     - semantic_search → web_scrape (if docs have URLs)     - filter → web_scrape → llm_enrich (fetch then analyze)  **Cost & Performance**:     - Expensive: Browser rendering is resource-intensive     - Slow: 2-5s per page for JavaScript, 100-500ms for static     - Use STATIC strategy when possible for speed     - Use conditional enrichment (when parameter) to reduce load     - Consider batch_size for balancing speed vs resource usage  **Conditional Enrichment**: Supports `when` parameter to only fetch content for specific documents. CRITICAL FOR PERFORMANCE - web scraping is slow!     - Only fetch for high-scoring results     - Only fetch specific content types     - Only fetch when URL field is present  Requirements:     - url_field: REQUIRED, document field containing URL to fetch     - output_field: REQUIRED, where to store fetched content     - strategy: OPTIONAL, scraping strategy (static/javascript/auto, default: auto)     - timeout: OPTIONAL, request timeout in seconds (default: 10)     - include_html: OPTIONAL, include raw HTML in response (default: False)     - min_content_length: OPTIONAL, minimum content length for auto fallback (default: 500)     - when: OPTIONAL but RECOMMENDED for performance     - on_error: OPTIONAL, error handling strategy (skip/remove/raise, default: skip)     - batch_size: OPTIONAL, concurrent requests (default: 5)  Use Cases:     - Content aggregation: Fetch full articles after web search     - Research pipelines: Get complete documents for analysis     - Content monitoring: Scrape pages for change detection     - Data enrichment: Add full page text to search results     - Competitive analysis: Fetch competitor content     - News aggregation: Get full articles from headlines  Examples:     Basic web scrape after search:         ```json         {             \"url_field\": \"metadata.url\",             \"output_field\": \"metadata.full_content\",             \"strategy\": \"auto\"         }         ```      Fast static scraping:         ```json         {             \"url_field\": \"metadata.url\",             \"output_field\": \"metadata.content\",             \"strategy\": \"static\",             \"timeout\": 5,             \"on_error\": \"skip\"         }         ```      JavaScript SPA scraping:         ```json         {             \"url_field\": \"metadata.url\",             \"output_field\": \"metadata.rendered_content\",             \"strategy\": \"javascript\",             \"timeout\": 30,             \"include_html\": true         }         ```      Conditional scrape (only high scores):         ```json         {             \"url_field\": \"metadata.url\",             \"output_field\": \"metadata.full_text\",             \"strategy\": \"auto\",             \"when\": {                 \"field\": \"score\",                 \"operator\": \"gte\",                 \"value\": 0.8             },             \"batch_size\": 3         }         ```
    """ # noqa: E501
    url: Optional[StrictStr] = Field(default='null', description="Direct URL to fetch (supports templates like '{{inputs.url}}'). Use this when you want to scrape a URL from inputs without needing existing documents. Creates a new document with the scraped content. Either 'url' or 'url_field' must be provided, but not both.")
    url_field: Optional[StrictStr] = Field(default='metadata.url', description="Dot-path to document field containing URL to fetch. Use this when enriching existing documents that have URL fields. Supports nested paths like 'metadata.url' or 'data.source.link'. The field must contain a valid HTTP/HTTPS URL. If field is missing or URL is invalid, behavior depends on on_error setting. Either 'url' or 'url_field' must be provided, but not both.")
    output_field: Optional[StrictStr] = Field(default='metadata.web_content', description="Dot-path where fetched content should be stored. Creates nested structure if path doesn't exist. Stores object with: text, title, url, final_url, content_length, metadata, strategy_used. Example: 'metadata.web_content' creates doc.metadata.web_content.text, doc.metadata.web_content.title, etc.")
    strategy: Optional[StrictStr] = Field(default='auto', description="OPTIONAL. Scraping strategy to use. 'static': Fast HTML parsing, no JavaScript (best for articles/blogs). 'javascript': Full browser rendering (slow, required for SPAs). 'auto': Try static first, fallback to JavaScript if needed. Default is 'auto' for best balance of speed and completeness.")
    timeout: Optional[Annotated[int, Field(le=60, strict=True, ge=1)]] = Field(default=10, description="OPTIONAL. Request timeout in seconds. Range: 1-60 seconds. Default is 10. JavaScript rendering may need higher timeouts (20-30s). Shorter timeouts fail faster but may miss slow-loading pages.")
    include_html: Optional[StrictBool] = Field(default=False, description="OPTIONAL. Include raw HTML in output. Default is False. Set to True if you need HTML for custom parsing. Warning: Significantly increases response size and memory usage.")
    min_content_length: Optional[Annotated[int, Field(strict=True, ge=100)]] = Field(default=500, description="OPTIONAL. Minimum content length (characters) for auto strategy. If static parsing yields less than this, fallback to JavaScript. Default is 500 characters. Higher values = more likely to use JavaScript. Range: 100+.")
    when: Optional[LogicalOperator] = Field(default=None, description="OPTIONAL. Conditional filter that documents must satisfy to be enriched with web content. Uses LogicalOperator (AND/OR/NOT) for complex boolean logic, or simple field/operator/value for single conditions. Documents NOT matching this condition will SKIP enrichment (pass-through unchanged). CRITICAL FOR PERFORMANCE - web scraping is slow! Only fetch content for documents that need it. When NOT specified, ALL documents are enriched unconditionally (may be very slow).   Use cases: - Only fetch for high-scoring results (score >= 0.8) - Only fetch specific content types (category = 'article') - Only fetch when URL field is present   Simple condition example: {\"field\": \"score\", \"operator\": \"gte\", \"value\": 0.8} Boolean AND example: {\"AND\": [{\"field\": \"metadata.category\", \"operator\": \"eq\", \"value\": \"article\"}, ...]} ")
    on_error: Optional[ErrorHandling] = Field(default=None, description="OPTIONAL. How to handle fetch errors. 'skip': Skip failed documents, pass them through unchanged (default). 'remove': Remove failed documents from results entirely. 'raise': Raise exception on first failure, halt pipeline. Default is 'skip' for fault tolerance.")
    batch_size: Optional[Annotated[int, Field(le=20, strict=True, ge=1)]] = Field(default=5, description="OPTIONAL. Number of concurrent fetch requests. Range: 1-20. Default is 5. Higher values = faster but more resource intensive. Lower values = slower but safer for rate limits. Consider Engine resource capacity when setting.")
    max_depth: Optional[Annotated[int, Field(le=5, strict=True, ge=0)]] = Field(default=null, description="OPTIONAL. Maximum link depth to follow from seed URL. None = single URL mode (default behavior). 0 = only seed page. 1 = seed + direct links from seed. 2 = seed + links + links from those pages. Higher values exponentially increase crawl time. Recommended: 1-2 for most use cases.")
    max_pages: Optional[Annotated[int, Field(le=100, strict=True, ge=1)]] = Field(default=null, description="OPTIONAL. Maximum total pages to crawl. None = single URL mode (default behavior). Crawling stops when this limit is reached regardless of depth. Acts as a safety limit to prevent runaway crawls. Range: 1-100.")
    same_domain_only: Optional[StrictBool] = Field(default=True, description="OPTIONAL. Only follow links to the same domain as seed URL. Default is True (strongly recommended). Set to False only for intentional cross-domain crawling. Prevents crawl explosion to external sites.")
    url_patterns: Optional[URLPatterns] = Field(default=None, description="OPTIONAL. Include/exclude regex patterns for filtering discovered URLs. Include patterns: URLs must match at least one (if provided). Exclude patterns: URLs matching any are skipped. Applied after same_domain_only filter. Example: {'include': ['/docs/.*'], 'exclude': ['/login', '/admin/.*']}")
    delay_between_requests: Optional[Union[Annotated[float, Field(le=5.0, strict=True, ge=0.0)], Annotated[int, Field(le=5, strict=True, ge=0)]]] = Field(default=0.5, description="OPTIONAL. Delay in seconds between starting new requests (politeness). Helps avoid overwhelming target servers. 0 = no delay (use with caution). Default is 0.5 seconds. Recommended: 0.5-1.0 for most sites.")
    __properties: ClassVar[List[str]] = ["url", "url_field", "output_field", "strategy", "timeout", "include_html", "min_content_length", "when", "on_error", "batch_size", "max_depth", "max_pages", "same_domain_only", "url_patterns", "delay_between_requests"]

    model_config = ConfigDict(
        populate_by_name=True,
        validate_assignment=True,
        protected_namespaces=(),
    )


    def to_str(self) -> str:
        """Returns the string representation of the model using alias"""
        return pprint.pformat(self.model_dump(by_alias=True))

    def to_json(self) -> str:
        """Returns the JSON representation of the model using alias"""
        # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
        return json.dumps(self.to_dict())

    @classmethod
    def from_json(cls, json_str: str) -> Optional[Self]:
        """Create an instance of StageParamsWebScrape from a JSON string"""
        return cls.from_dict(json.loads(json_str))

    def to_dict(self) -> Dict[str, Any]:
        """Return the dictionary representation of the model using alias.

        This has the following differences from calling pydantic's
        `self.model_dump(by_alias=True)`:

        * `None` is only added to the output dict for nullable fields that
          were set at model initialization. Other fields with value `None`
          are ignored.
        """
        excluded_fields: Set[str] = set([
        ])

        _dict = self.model_dump(
            by_alias=True,
            exclude=excluded_fields,
            exclude_none=True,
        )
        # override the default output from pydantic by calling `to_dict()` of when
        if self.when:
            _dict['when'] = self.when.to_dict()
        # override the default output from pydantic by calling `to_dict()` of url_patterns
        if self.url_patterns:
            _dict['url_patterns'] = self.url_patterns.to_dict()
        return _dict

    @classmethod
    def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
        """Create an instance of StageParamsWebScrape from a dict"""
        if obj is None:
            return None

        if not isinstance(obj, dict):
            return cls.model_validate(obj)

        _obj = cls.model_validate({
            "url": obj.get("url") if obj.get("url") is not None else 'null',
            "url_field": obj.get("url_field") if obj.get("url_field") is not None else 'metadata.url',
            "output_field": obj.get("output_field") if obj.get("output_field") is not None else 'metadata.web_content',
            "strategy": obj.get("strategy") if obj.get("strategy") is not None else 'auto',
            "timeout": obj.get("timeout") if obj.get("timeout") is not None else 10,
            "include_html": obj.get("include_html") if obj.get("include_html") is not None else False,
            "min_content_length": obj.get("min_content_length") if obj.get("min_content_length") is not None else 500,
            "when": LogicalOperator.from_dict(obj["when"]) if obj.get("when") is not None else None,
            "on_error": obj.get("on_error"),
            "batch_size": obj.get("batch_size") if obj.get("batch_size") is not None else 5,
            "max_depth": obj.get("max_depth") if obj.get("max_depth") is not None else null,
            "max_pages": obj.get("max_pages") if obj.get("max_pages") is not None else null,
            "same_domain_only": obj.get("same_domain_only") if obj.get("same_domain_only") is not None else True,
            "url_patterns": URLPatterns.from_dict(obj["url_patterns"]) if obj.get("url_patterns") is not None else None,
            "delay_between_requests": obj.get("delay_between_requests") if obj.get("delay_between_requests") is not None else 0.5
        })
        return _obj


