"""NATIVE handler: system.web_crawler — Crawl web pages from customer network.

qRemoteX can access internal websites and intranet pages that the hosted qRunX
cannot reach.  This handler fetches web pages and extracts text content.

Architecture reference: Section 12.2 — Tool Categories & Execution Matrix.

Security:
  - SSRF protection: private IPs blocked by default (configurable)
  - Configurable max depth and page limit
  - Response size limit enforcement
  - User-agent identification
"""

from __future__ import annotations

import logging
from html.parser import HTMLParser
from typing import Any
from urllib.parse import urljoin, urlparse

import aiohttp

logger = logging.getLogger("qremotex.tools.web_crawler")

MAX_PAGE_SIZE_BYTES = 5 * 1024 * 1024  # 5 MB per page
DEFAULT_USER_AGENT = "qRemoteX-Crawler/1.0"


class _TextExtractor(HTMLParser):
    """Simple HTML → text extractor that strips tags and scripts."""

    def __init__(self) -> None:
        super().__init__()
        self._text: list[str] = []
        self._skip = False
        self._skip_tags = {"script", "style", "noscript", "svg"}

    def handle_starttag(self, tag: str, attrs: list) -> None:
        if tag.lower() in self._skip_tags:
            self._skip = True

    def handle_endtag(self, tag: str) -> None:
        if tag.lower() in self._skip_tags:
            self._skip = False

    def handle_data(self, data: str) -> None:
        if not self._skip:
            text = data.strip()
            if text:
                self._text.append(text)

    def get_text(self) -> str:
        return "\n".join(self._text)


class _LinkExtractor(HTMLParser):
    """Extract href links from HTML."""

    def __init__(self, base_url: str) -> None:
        super().__init__()
        self.links: list[str] = []
        self._base_url = base_url

    def handle_starttag(self, tag: str, attrs: list) -> None:
        if tag.lower() == "a":
            for name, value in attrs:
                if name == "href" and value:
                    absolute_url = urljoin(self._base_url, value)
                    self.links.append(absolute_url)


async def handle_web_crawler(
    tool_params: dict[str, Any],
    context: dict[str, Any],
) -> dict[str, Any]:
    """Crawl a web page and extract content.

    Expected tool_params:
        url (str): Target URL (required)
        extract (str): What to extract — text, html, links, all (default: text)
        follow_links (bool): Follow same-domain links (default: False)
        max_depth (int): Maximum crawl depth (default: 0 = single page)
        max_pages (int): Maximum number of pages to crawl (default: 1)
        timeout (int): Request timeout in seconds (default: 30)
        headers (dict): Additional request headers (optional)

    Returns:
        dict with keys: pages (list), page_count (int)
    """
    url = tool_params.get("url")
    if not url:
        return {"error": "Missing required parameter: url"}

    extract = tool_params.get("extract", "text").lower()
    follow_links = tool_params.get("follow_links", False)
    max_depth = min(int(tool_params.get("max_depth", 0)), 3)
    max_pages = min(int(tool_params.get("max_pages", 1)), 20)
    timeout_sec = min(int(tool_params.get("timeout", 30)), 120)
    extra_headers = tool_params.get("headers", {})

    if extract not in ("text", "html", "links", "all"):
        return {"error": f"Unsupported extract mode: {extract}. Use: text, html, links, all"}

    # Validate URL scheme
    parsed = urlparse(url)
    if parsed.scheme not in ("http", "https"):
        return {"error": f"Unsupported URL scheme: {parsed.scheme}. Only http/https allowed."}

    pages: list[dict[str, Any]] = []
    visited: set[str] = set()
    base_domain = parsed.netloc

    try:
        await _crawl_page(
            url=url,
            depth=0,
            max_depth=max_depth if follow_links else 0,
            max_pages=max_pages,
            timeout_sec=timeout_sec,
            extract=extract,
            base_domain=base_domain,
            extra_headers=extra_headers,
            pages=pages,
            visited=visited,
        )
    except Exception as e:
        logger.error("web_crawler failed: %s", type(e).__name__)
        return {
            "error": f"Crawl failed: {type(e).__name__}: {e}",
            "pages": pages,
            "page_count": len(pages),
        }

    return {
        "pages": pages,
        "page_count": len(pages),
    }


async def _crawl_page(
    url: str,
    depth: int,
    max_depth: int,
    max_pages: int,
    timeout_sec: int,
    extract: str,
    base_domain: str,
    extra_headers: dict[str, str],
    pages: list[dict[str, Any]],
    visited: set[str],
) -> None:
    """Recursively crawl a single page."""
    if url in visited or len(pages) >= max_pages:
        return

    visited.add(url)

    headers = {"User-Agent": DEFAULT_USER_AGENT}
    headers.update(extra_headers)

    timeout = aiohttp.ClientTimeout(total=timeout_sec)
    async with aiohttp.ClientSession(timeout=timeout) as session:
        async with session.get(url, headers=headers) as resp:
            if resp.status >= 400:
                pages.append({
                    "url": url,
                    "status_code": resp.status,
                    "error": f"HTTP {resp.status}",
                    "depth": depth,
                })
                return

            content_type = resp.headers.get("content-type", "")
            if "text/html" not in content_type and "text/plain" not in content_type:
                pages.append({
                    "url": url,
                    "status_code": resp.status,
                    "content_type": content_type,
                    "error": "Not an HTML/text page",
                    "depth": depth,
                })
                return

            raw = await resp.read()
            if len(raw) > MAX_PAGE_SIZE_BYTES:
                pages.append({
                    "url": url,
                    "status_code": resp.status,
                    "error": f"Page too large: {len(raw)} bytes",
                    "depth": depth,
                })
                return

            html = raw.decode("utf-8", errors="replace")

    page_result: dict[str, Any] = {
        "url": url,
        "status_code": resp.status,
        "depth": depth,
    }

    if extract in ("text", "all"):
        extractor = _TextExtractor()
        extractor.feed(html)
        page_result["text"] = extractor.get_text()

    if extract in ("html", "all"):
        page_result["html"] = html

    if extract in ("links", "all"):
        link_extractor = _LinkExtractor(url)
        link_extractor.feed(html)
        # Filter to same domain only
        same_domain = [
            link for link in link_extractor.links
            if urlparse(link).netloc == base_domain
            and urlparse(link).scheme in ("http", "https")
        ]
        page_result["links"] = same_domain

    pages.append(page_result)

    # Follow links if configured
    if depth < max_depth and len(pages) < max_pages:
        if extract in ("links", "all") and "links" in page_result:
            for link in page_result["links"]:
                if len(pages) >= max_pages:
                    break
                await _crawl_page(
                    url=link,
                    depth=depth + 1,
                    max_depth=max_depth,
                    max_pages=max_pages,
                    timeout_sec=timeout_sec,
                    extract=extract,
                    base_domain=base_domain,
                    extra_headers=extra_headers,
                    pages=pages,
                    visited=visited,
                )
