"""Web scraping, crawling, and structured extraction via Firecrawl API.

Provides single URL scraping, batch scraping, URL discovery, web search,
multi-page crawling, and LLM-powered data extraction.

API docs: https://docs.firecrawl.dev/api-reference
Python SDK: https://pypi.org/project/firecrawl/
"""

from __future__ import annotations

# Pack for dot notation: firecrawl.scrape(), firecrawl.crawl(), etc.
pack = "firecrawl"

__all__ = [
    "crawl",
    "crawl_status",
    "deep_research",
    "extract",
    "map_urls",
    "scrape",
    "scrape_batch",
    "search",
]

# Dependency declarations for CLI validation
__ot_requires__ = {
    "lib": [("firecrawl", "pip install firecrawl")],
    "secrets": ["FIRECRAWL_API_KEY"],
}

from typing import Any, Literal
from urllib.parse import urlparse

from firecrawl import Firecrawl
from pydantic import BaseModel, Field

from ot.config import get_secret, get_tool_config
from ot.logging import LogSpan
from ot.utils import batch_execute, lazy_client, normalize_items


def _validate_url(url: str) -> str | None:
    """Return error message if URL is invalid, None otherwise."""
    if not url or not url.strip():
        return "URL is required and cannot be empty"
    try:
        parsed = urlparse(url)
        if not parsed.scheme or not parsed.netloc:
            return f"Invalid URL: {url} (missing scheme or host)"
        if parsed.scheme not in ("http", "https"):
            return f"Invalid URL scheme: {parsed.scheme} (expected http/https)"
    except Exception as e:
        return f"Invalid URL: {e}"
    return None


class Config(BaseModel):
    """Pack configuration - discovered by registry."""

    api_url: str | None = Field(
        default=None,
        description="Custom API URL for self-hosted Firecrawl instances",
    )


def _get_config() -> Config:
    """Get firecrawl pack configuration."""
    return get_tool_config("firecrawl", Config)


def _create_client() -> Firecrawl | None:
    """Create Firecrawl client with API key."""
    api_key = get_secret("FIRECRAWL_API_KEY")
    if not api_key:
        return None

    api_url = _get_config().api_url
    if api_url:
        return Firecrawl(api_key=api_key, api_url=api_url)
    return Firecrawl(api_key=api_key)


# Thread-safe lazy client using SDK utility
_get_client = lazy_client(_create_client)


def _to_dict(obj: Any) -> dict[str, Any]:
    """Convert SDK response objects to dicts for JSON serialization."""
    if isinstance(obj, dict):
        return obj
    # Pydantic v2
    if hasattr(obj, "model_dump"):
        return obj.model_dump()
    # Pydantic v1
    if hasattr(obj, "dict"):
        return obj.dict()
    # Fallback for dataclasses or other objects
    if hasattr(obj, "__dict__"):
        return {k: v for k, v in obj.__dict__.items() if not k.startswith("_")}
    return {"value": str(obj)}


def scrape(
    *,
    url: str,
    formats: list[
        Literal[
            "markdown", "html", "rawHtml", "links", "screenshot", "screenshot@fullPage"
        ]
    ]
    | None = None,
    only_main_content: bool = True,
    include_tags: list[str] | None = None,
    exclude_tags: list[str] | None = None,
    wait_for: int | None = None,
    timeout: int | None = None,
    mobile: bool = False,
    skip_tls_verification: bool = False,
    remove_base64_images: bool = True,
    location: dict[str, Any] | None = None,
) -> dict[str, Any] | str:
    """Scrape content from a single URL.

    Extracts content in various formats with configurable filtering.

    Args:
        url: The URL to scrape
        formats: Output formats to include. Options:
            - "markdown": Clean markdown text (default)
            - "html": Cleaned HTML
            - "rawHtml": Original HTML
            - "links": All hyperlinks on the page
            - "screenshot": Screenshot image (base64)
            - "screenshot@fullPage": Full page screenshot
        only_main_content: Extract only main content, excluding nav/footer (default: True)
        include_tags: HTML tags to include (e.g., ["article", "main"])
        exclude_tags: HTML tags to exclude (e.g., ["nav", "footer"])
        wait_for: Milliseconds to wait for dynamic content
        timeout: Request timeout in milliseconds
        mobile: Use mobile user agent
        skip_tls_verification: Skip TLS certificate validation
        remove_base64_images: Remove base64 images from markdown (default: True)
        location: Geolocation for request (e.g., {"country": "US", "languages": ["en"]})

    Returns:
        Dict with scraped content in requested formats, or error message

    Example:
        # Basic scrape
        firecrawl.scrape(url="https://example.com")

        # Get markdown and links
        firecrawl.scrape(url="https://example.com", formats=["markdown", "links"])

        # Scrape with geolocation
        firecrawl.scrape(url="https://example.com", location={"country": "US"})
    """
    # Validate URL
    if url_error := _validate_url(url):
        return f"Error: {url_error}"

    with LogSpan(span="firecrawl.scrape", url=url) as span:
        client = _get_client()
        if client is None:
            return "Error: FIRECRAWL_API_KEY secret not configured"

        try:
            # Build kwargs for v2 API
            kwargs: dict[str, Any] = {}

            if formats:
                kwargs["formats"] = formats
            if not only_main_content:
                kwargs["only_main_content"] = False
            if include_tags:
                kwargs["include_tags"] = include_tags
            if exclude_tags:
                kwargs["exclude_tags"] = exclude_tags
            if wait_for is not None:
                kwargs["wait_for"] = wait_for
            if timeout is not None:
                kwargs["timeout"] = timeout
            if mobile:
                kwargs["mobile"] = True
            if skip_tls_verification:
                kwargs["skip_tls_verification"] = True
            if not remove_base64_images:
                kwargs["remove_base64_images"] = False
            if location:
                kwargs["location"] = location

            result = client.scrape(url, **kwargs)

            span.add(success=True)
            result_dict = _to_dict(result)
            if isinstance(result_dict, dict):
                span.add(formats=list(result_dict.keys()))
            return result_dict

        except Exception as e:
            error_msg = f"Scrape failed: {e}"
            span.add(error=str(e))
            return error_msg


def scrape_batch(
    *,
    urls: list[str] | list[tuple[str, str]],
    formats: list[
        Literal[
            "markdown", "html", "rawHtml", "links", "screenshot", "screenshot@fullPage"
        ]
    ]
    | None = None,
    only_main_content: bool = True,
    max_workers: int = 5,
) -> dict[str, dict[str, Any] | str]:
    """Scrape multiple URLs concurrently.

    Uses ThreadPoolExecutor for parallel execution with error isolation.

    Args:
        urls: List of URLs to scrape. Each item can be:
            - A string (URL used as key)
            - A tuple of (url, label) for custom labeling
        formats: Output formats (see scrape() for options)
        only_main_content: Extract only main content (default: True)
        max_workers: Maximum concurrent scrapes (default: 5)

    Returns:
        Dict mapping URL/label to scraped content or error message

    Example:
        # Simple list
        firecrawl.scrape_batch(urls=[
            "https://docs.python.org/3/library/asyncio.html",
            "https://docs.python.org/3/library/threading.html",
        ])

        # With labels
        firecrawl.scrape_batch(urls=[
            ("https://example.com/page1", "Page 1"),
            ("https://example.com/page2", "Page 2"),
        ])
    """
    normalized = normalize_items(urls)

    with LogSpan(span="firecrawl.scrape_batch", url_count=len(normalized)) as span:

        def _scrape_one(url: str, label: str) -> tuple[str, dict[str, Any] | str]:
            result = scrape(
                url=url,
                formats=formats,
                only_main_content=only_main_content,
            )
            return label, result

        results = batch_execute(_scrape_one, normalized, max_workers=max_workers)
        span.add(success_count=sum(1 for r in results.values() if isinstance(r, dict)))
        return results


def map_urls(
    *,
    url: str,
    search: str | None = None,
    ignore_sitemap: bool = False,
    sitemap_only: bool = False,
    include_subdomains: bool = False,
    limit: int | None = None,
) -> list[str] | str:
    """Discover URLs from a website.

    Maps all accessible URLs from a site via sitemap and crawling.

    Args:
        url: The starting URL to map
        search: Optional search term to filter URLs
        ignore_sitemap: Skip sitemap discovery, only crawl (default: False)
        sitemap_only: Only use sitemap, no crawling (default: False)
        include_subdomains: Include URLs from subdomains (default: False)
        limit: Maximum number of URLs to return

    Returns:
        List of discovered URLs, or error message

    Example:
        # Map entire site
        firecrawl.map_urls(url="https://docs.python.org")

        # Search for specific pages
        firecrawl.map_urls(url="https://docs.python.org", search="asyncio")

        # Limit results
        firecrawl.map_urls(url="https://example.com", limit=100)
    """
    # Validate URL
    if url_error := _validate_url(url):
        return f"Error: {url_error}"

    with LogSpan(span="firecrawl.map_urls", url=url, search=search) as span:
        client = _get_client()
        if client is None:
            return "Error: FIRECRAWL_API_KEY secret not configured"

        try:
            # Build kwargs for v2 API
            kwargs: dict[str, Any] = {}

            if search:
                kwargs["search"] = search
            if ignore_sitemap:
                kwargs["sitemap"] = "skip"
            if sitemap_only:
                kwargs["sitemap"] = "only"
            if include_subdomains:
                kwargs["include_subdomains"] = True
            if limit:
                kwargs["limit"] = limit

            result = client.map(url, **kwargs)

            # Extract links list from result
            if isinstance(result, list):
                links = result
            else:
                # Handle MapResponse object
                links = getattr(result, "links", None) or []

            # Convert LinkResult objects to URL strings
            urls = []
            for link in links:
                if isinstance(link, str):
                    urls.append(link)
                elif hasattr(link, "url"):
                    urls.append(link.url)
                elif hasattr(link, "model_dump"):
                    urls.append(link.model_dump().get("url", str(link)))
                else:
                    urls.append(str(link))

            span.add(url_count=len(urls))
            return urls

        except Exception as e:
            error_msg = f"Map failed: {e}"
            span.add(error=str(e))
            return error_msg


def search(
    *,
    query: str,
    limit: int = 5,
    lang: str | None = None,
    country: str | None = None,
    scrape_options: dict[str, Any] | None = None,
) -> list[dict[str, Any]] | str:
    """Search the web and optionally scrape results.

    Performs web search with optional content retrieval for each result.

    Args:
        query: Search query string
        limit: Maximum number of results (default: 5)
        lang: Language code for results (e.g., "en")
        country: Country code for results (e.g., "US")
        scrape_options: Options for scraping result pages (see scrape() params)

    Returns:
        List of search results with optional scraped content, or error message

    Example:
        # Basic search
        firecrawl.search(query="Python async best practices")

        # Search with scraping
        firecrawl.search(
            query="machine learning tutorials",
            limit=3,
            scrape_options={"formats": ["markdown"]}
        )
    """
    with LogSpan(span="firecrawl.search", query=query, limit=limit) as span:
        client = _get_client()
        if client is None:
            return "Error: FIRECRAWL_API_KEY secret not configured"

        try:
            # Build kwargs for v2 API
            kwargs: dict[str, Any] = {"limit": limit}

            if lang:
                kwargs["lang"] = lang
            if country:
                kwargs["location"] = country
            if scrape_options:
                kwargs["scrape_options"] = scrape_options

            result = client.search(query, **kwargs)

            if isinstance(result, list):
                span.add(result_count=len(result))
                return [_to_dict(item) for item in result]
            # Handle SearchData object (v2 API returns .web, .news, .images)
            data = getattr(result, "web", None) or getattr(result, "data", None) or []
            span.add(result_count=len(data))
            return [_to_dict(item) for item in data]

        except Exception as e:
            error_msg = f"Search failed: {e}"
            span.add(error=str(e))
            return error_msg


def crawl(
    *,
    url: str,
    max_depth: int | None = None,
    limit: int | None = None,
    include_paths: list[str] | None = None,
    exclude_paths: list[str] | None = None,
    ignore_sitemap: bool = False,
    scrape_options: dict[str, Any] | None = None,
    webhook: str | None = None,
) -> dict[str, Any] | str:
    """Start an asynchronous multi-page crawl job.

    Crawls a website starting from the given URL. Returns immediately with
    a job ID. Use crawl_status() to poll for results.

    Args:
        url: The starting URL to crawl
        max_depth: Maximum link depth to crawl
        limit: Maximum number of pages to crawl
        include_paths: URL patterns to include (glob syntax)
        exclude_paths: URL patterns to exclude (glob syntax)
        ignore_sitemap: Skip sitemap discovery (default: False)
        scrape_options: Options for scraping pages (see scrape() params)
        webhook: URL to receive completion notification

    Returns:
        Dict with job ID and status URL, or error message

    Example:
        # Start a crawl
        job = firecrawl.crawl(url="https://docs.python.org", max_depth=2, limit=100)

        # Check status
        firecrawl.crawl_status(id=job["id"])
    """
    # Validate URL
    if url_error := _validate_url(url):
        return f"Error: {url_error}"

    with LogSpan(span="firecrawl.crawl", url=url, max_depth=max_depth, limit=limit) as span:
        client = _get_client()
        if client is None:
            return "Error: FIRECRAWL_API_KEY secret not configured"

        try:
            # Build kwargs for v2 API
            kwargs: dict[str, Any] = {}

            if max_depth is not None:
                kwargs["max_discovery_depth"] = max_depth
            if limit is not None:
                kwargs["limit"] = limit
            if include_paths:
                kwargs["include_paths"] = include_paths
            if exclude_paths:
                kwargs["exclude_paths"] = exclude_paths
            if ignore_sitemap:
                kwargs["ignore_sitemap"] = True
            if scrape_options:
                kwargs["scrape_options"] = scrape_options
            if webhook:
                kwargs["webhook"] = webhook

            result = client.crawl(url, **kwargs)

            # Convert to dict for consistent handling
            result_dict = _to_dict(result)

            # Extract job ID with multiple fallbacks
            job_id = (
                result_dict.get("id")
                or result_dict.get("jobId")
                or result_dict.get("job_id")
            )

            span.add(job_id=job_id)

            # If result already has data (sync completion), return as-is
            if result_dict.get("data"):
                return result_dict

            # Return normalized response with job info
            return {
                "id": job_id,
                "status": result_dict.get("status", "started"),
                "url": url,
            }

        except Exception as e:
            error_msg = f"Crawl failed: {e}"
            span.add(error=str(e))
            return error_msg


def crawl_status(
    *,
    id: str,
) -> dict[str, Any] | str:
    """Check the status of a crawl job.

    Polls the crawl job for current progress and results.

    Args:
        id: The crawl job ID returned by crawl()

    Returns:
        Dict with status, progress, and results (if complete), or error message

    Example:
        # Check crawl progress
        status = firecrawl.crawl_status(id="abc123")

        if status["status"] == "completed":
            for page in status["data"]:
                print(page["url"])
    """
    # Validate job ID
    if not id or not id.strip():
        return "Error: Job ID is required and cannot be empty"

    with LogSpan(span="firecrawl.crawl_status", job_id=id) as span:
        client = _get_client()
        if client is None:
            return "Error: FIRECRAWL_API_KEY secret not configured"

        try:
            result = client.get_crawl_status(id)

            if isinstance(result, dict):
                span.add(status=result.get("status"))
                return result

            # Handle CrawlStatusResponse object
            status = getattr(result, "status", "unknown")
            span.add(status=status)

            response: dict[str, Any] = {
                "id": id,
                "status": status,
            }

            # Add optional fields if present
            if hasattr(result, "completed"):
                response["completed"] = result.completed
            if hasattr(result, "total"):
                response["total"] = result.total
            if hasattr(result, "data"):
                response["data"] = result.data

            return response

        except Exception as e:
            error_msg = f"Status check failed: {e}"
            span.add(error=str(e))
            return error_msg


def extract(
    *,
    urls: list[str],
    prompt: str | None = None,
    schema: dict[str, Any] | None = None,
    system_prompt: str | None = None,
    allow_external_links: bool = False,
) -> dict[str, Any] | str:
    """Extract structured data from URLs using LLM.

    Uses an LLM to extract data matching a JSON schema from web pages.

    Args:
        urls: URLs to extract data from
        prompt: Natural language description of what to extract
        schema: JSON schema defining the structure of extracted data
            (OpenAI JSON schema format)
        system_prompt: Custom system prompt for the LLM
        allow_external_links: Follow external links during extraction (default: False)

    Returns:
        Dict with extracted data matching schema, or error message

    Example:
        # Extract with prompt
        firecrawl.extract(
            urls=["https://example.com/products"],
            prompt="Extract product names and prices"
        )

        # Extract with schema
        firecrawl.extract(
            urls=["https://example.com/team"],
            schema={
                "type": "object",
                "properties": {
                    "team_members": {
                        "type": "array",
                        "items": {
                            "type": "object",
                            "properties": {
                                "name": {"type": "string"},
                                "role": {"type": "string"}
                            }
                        }
                    }
                }
            }
        )
    """
    with LogSpan(span="firecrawl.extract", url_count=len(urls)) as span:
        client = _get_client()
        if client is None:
            return "Error: FIRECRAWL_API_KEY secret not configured"

        if not prompt and not schema:
            return "Error: Either prompt or schema is required"

        try:
            # Build kwargs for v2 API
            kwargs: dict[str, Any] = {}

            if prompt:
                kwargs["prompt"] = prompt
            if schema:
                kwargs["schema"] = schema
            if system_prompt:
                kwargs["system_prompt"] = system_prompt
            if allow_external_links:
                kwargs["allow_external_links"] = True

            result = client.extract(urls, **kwargs)

            if isinstance(result, dict):
                span.add(success=True)
                return result

            # Handle ExtractResponse object
            data = getattr(result, "data", None)
            span.add(success=True)
            return {"data": data} if data else result

        except Exception as e:
            error_msg = f"Extract failed: {e}"
            span.add(error=str(e))
            return error_msg


def deep_research(
    *,
    prompt: str,
    urls: list[str] | None = None,
    timeout: int | None = None,
    max_credits: int | None = None,
) -> dict[str, Any] | str:
    """Run autonomous deep research on a topic.

    Launches an AI agent that autonomously researches a topic by
    searching, crawling, and synthesizing information from the web.

    Args:
        prompt: Research question or topic
        urls: Starting URLs to research (optional, will search if not provided)
        timeout: Time limit in seconds for the research
        max_credits: Maximum credits to spend on research

    Returns:
        Dict with research results and sources, or error message

    Example:
        # Research a topic
        firecrawl.deep_research(
            prompt="What are the latest developments in quantum computing?",
            timeout=300
        )

        # Research from specific sources
        firecrawl.deep_research(
            prompt="Compare pricing models",
            urls=["https://company1.com/pricing", "https://company2.com/pricing"]
        )
    """
    with LogSpan(span="firecrawl.deep_research", prompt=prompt[:100]) as span:
        client = _get_client()
        if client is None:
            return "Error: FIRECRAWL_API_KEY secret not configured"

        try:
            # Build kwargs for v2 API (uses 'agent' method)
            kwargs: dict[str, Any] = {"prompt": prompt}

            if urls:
                kwargs["urls"] = urls
            if timeout is not None:
                kwargs["timeout"] = timeout
            if max_credits is not None:
                kwargs["max_credits"] = max_credits

            # The SDK's agent method corresponds to deep research
            result = client.agent(**kwargs)

            if isinstance(result, dict):
                span.add(success=True)
                return result

            # Handle response object
            data = getattr(result, "data", None)
            sources = getattr(result, "sources", None)
            span.add(success=True, source_count=len(sources) if sources else 0)
            return {
                "data": data,
                "sources": sources,
            }

        except Exception as e:
            error_msg = f"Deep research failed: {e}"
            span.add(error=str(e))
            return error_msg
