"""Web crawl tool: multi-page recursive web crawling with depth control."""

from __future__ import annotations

import asyncio
import json
import re
import time
from typing import Any
from urllib.parse import urljoin, urlparse

import httpx
from bs4 import BeautifulSoup
from loguru import logger

from madpod_agent.agent.tools.base import Tool

_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7_2) AppleWebKit/537.36"
_DEFAULT_MAX_PAGES = 20
_ABSOLUTE_MAX_PAGES = 100
_DEFAULT_DEPTH = 2
_MAX_DEPTH = 5
_REQUEST_TIMEOUT = 15.0
_DELAY_BETWEEN_REQUESTS = 0.5  # Rate limiting
_MAX_CONTENT_PER_PAGE = 10_000  # chars per page summary
_MAX_TOTAL_CHARS = 200_000


def _same_domain(url1: str, url2: str) -> bool:
    """Check if two URLs share the same domain."""
    return urlparse(url1).netloc == urlparse(url2).netloc


def _normalize_url(url: str) -> str:
    """Normalize URL by removing fragments and trailing slashes."""
    parsed = urlparse(url)
    path = parsed.path.rstrip("/") or "/"
    return f"{parsed.scheme}://{parsed.netloc}{path}"
    # Intentionally drop query params for dedup — can revisit if needed


def _extract_links(html: str, base_url: str) -> list[str]:
    """Extract all href links from HTML, resolved to absolute URLs."""
    soup = BeautifulSoup(html, "html.parser")
    links = []
    for a_tag in soup.find_all("a", href=True):
        href = a_tag["href"].strip()
        if href.startswith(("#", "javascript:", "mailto:", "tel:")):
            continue
        absolute = urljoin(base_url, href)
        links.append(absolute)
    return links


def _extract_text(html: str, max_chars: int = _MAX_CONTENT_PER_PAGE) -> str:
    """Extract readable text from HTML."""
    soup = BeautifulSoup(html, "html.parser")
    for tag in soup(["script", "style", "nav", "footer", "header", "aside"]):
        tag.decompose()
    text = soup.get_text(separator="\n", strip=True)
    # Collapse multiple newlines
    text = re.sub(r"\n{3,}", "\n\n", text)
    if len(text) > max_chars:
        text = text[:max_chars] + "\n... [truncated]"
    return text


def _is_ssl_verify_failure(exc: Exception) -> bool:
    return "CERTIFICATE_VERIFY_FAILED" in str(exc)


class WebCrawlTool(Tool):
    """Crawl multiple pages of a website following links."""

    def __init__(self, proxy: str | None = None):
        self._proxy = proxy

    @property
    def name(self) -> str:
        return "web_crawl"

    @property
    def description(self) -> str:
        return (
            "Crawl a website by following links from a starting URL. "
            "Extracts text content from each page. Use for sitemapping, "
            "multi-page scraping, and comprehensive site analysis. "
            "Respects same-domain restriction and rate limiting."
        )

    @property
    def parameters(self) -> dict[str, Any]:
        return {
            "type": "object",
            "properties": {
                "url": {"type": "string", "description": "Starting URL to crawl"},
                "max_pages": {
                    "type": "integer",
                    "description": f"Maximum pages to crawl (default: {_DEFAULT_MAX_PAGES}, max: {_ABSOLUTE_MAX_PAGES})",
                },
                "max_depth": {
                    "type": "integer",
                    "description": f"Maximum link-follow depth (default: {_DEFAULT_DEPTH}, max: {_MAX_DEPTH})",
                },
                "same_domain": {
                    "type": "boolean",
                    "description": "Only follow links on the same domain (default: true)",
                },
                "url_pattern": {
                    "type": "string",
                    "description": "Regex pattern — only crawl URLs matching this pattern",
                },
                "extract_text": {
                    "type": "boolean",
                    "description": "Extract text content from pages (default: true)",
                },
            },
            "required": ["url"],
        }

    async def execute(
        self,
        url: str,
        max_pages: int | None = None,
        max_depth: int | None = None,
        same_domain: bool = True,
        url_pattern: str | None = None,
        extract_text: bool = True,
        **kwargs: Any,
    ) -> str:
        parsed = urlparse(url)
        if parsed.scheme not in ("http", "https"):
            return json.dumps({"error": f"Only http/https URLs supported, got '{parsed.scheme}'"})

        resolved_max_pages = min(max_pages or _DEFAULT_MAX_PAGES, _ABSOLUTE_MAX_PAGES)
        resolved_max_depth = min(max_depth or _DEFAULT_DEPTH, _MAX_DEPTH)

        pattern_re = None
        if url_pattern:
            try:
                pattern_re = re.compile(url_pattern)
            except re.error as exc:
                return json.dumps({"error": f"Invalid regex pattern: {exc}"})

        visited: set[str] = set()
        pages: list[dict[str, Any]] = []
        total_chars = 0

        # BFS queue: (url, depth)
        queue: list[tuple[str, int]] = [(_normalize_url(url), 0)]

        start_time = time.monotonic()

        async with httpx.AsyncClient(
            follow_redirects=True,
            timeout=_REQUEST_TIMEOUT,
            proxy=self._proxy,
            headers={"User-Agent": _USER_AGENT},
        ) as client:
            while queue and len(pages) < resolved_max_pages:
                current_url, depth = queue.pop(0)
                normalized = _normalize_url(current_url)

                if normalized in visited:
                    continue
                visited.add(normalized)

                if same_domain and not _same_domain(url, current_url):
                    continue
                if pattern_re and not pattern_re.search(current_url):
                    continue

                try:
                    try:
                        response = await client.get(current_url)
                    except Exception as exc:
                        if not _is_ssl_verify_failure(exc):
                            raise
                        logger.warning(
                            "SSL certificate verification failed for {}; retrying with verify=False",
                            current_url,
                        )
                        async with httpx.AsyncClient(
                            follow_redirects=True,
                            timeout=_REQUEST_TIMEOUT,
                            proxy=self._proxy,
                            verify=False,
                            headers={"User-Agent": _USER_AGENT},
                        ) as insecure_client:
                            response = await insecure_client.get(current_url)
                    content_type = response.headers.get("content-type", "")
                    if "text/html" not in content_type and "text/xml" not in content_type:
                        continue

                    html = response.text
                    page_info: dict[str, Any] = {
                        "url": str(response.url),
                        "status": response.status_code,
                        "depth": depth,
                        "title": "",
                    }

                    if extract_text:
                        remaining = _MAX_TOTAL_CHARS - total_chars
                        if remaining <= 0:
                            page_info["text"] = "[total content limit reached]"
                        else:
                            text = _extract_text(html, min(_MAX_CONTENT_PER_PAGE, remaining))
                            page_info["text"] = text
                            total_chars += len(text)

                    # Extract title
                    soup = BeautifulSoup(html, "html.parser")
                    title_tag = soup.find("title")
                    if title_tag:
                        page_info["title"] = title_tag.get_text(strip=True)

                    # Extract links for next depth
                    if depth < resolved_max_depth:
                        links = _extract_links(html, str(response.url))
                        for link in links:
                            link_normalized = _normalize_url(link)
                            if link_normalized not in visited:
                                queue.append((link, depth + 1))

                    page_info["links_found"] = len(_extract_links(html, str(response.url)))
                    pages.append(page_info)

                except Exception as exc:
                    pages.append({
                        "url": current_url,
                        "error": str(exc),
                        "depth": depth,
                        "title": "",
                    })

                # Rate limiting
                if queue:
                    await asyncio.sleep(_DELAY_BETWEEN_REQUESTS)

        elapsed = time.monotonic() - start_time
        errors_count = sum(1 for p in pages if "error" in p)
        success_count = len(pages) - errors_count
        logger.info("Crawled {} pages from {} in {:.1f}s ({} errors)", len(pages), url, elapsed, errors_count)

        return json.dumps({
            "ok": success_count > 0,
            "start_url": url,
            "pages_crawled": len(pages),
            "pages_succeeded": success_count,
            "errors_count": errors_count,
            "urls_discovered": len(visited),
            "max_depth_reached": max((p.get("depth", 0) for p in pages), default=0),
            "elapsed_seconds": round(elapsed, 1),
            "pages": pages,
        }, default=str)
