"""
Web Page Fetch Tool
Fetches and extracts clean readable text from web pages using httpx + BeautifulSoup.
No browser needed — fast, lightweight, undetectable.

Use FETCH_PAGE for static/server-rendered pages.
Use BROWSER for JavaScript-heavy SPAs that need rendering.
"""

import logging
import re
from typing import Dict, Any, List
from urllib.parse import urljoin, urlparse

from ..base import (
    BaseTool, ToolMetadata, ToolParameter, ToolExecutionContext,
    ToolResult, ToolPermission
)

logger = logging.getLogger(__name__)

# Elements to strip from the page (noise)
STRIP_TAGS = [
    'script', 'style', 'noscript', 'iframe', 'svg',
    'nav', 'footer', 'header',  # navigation/chrome
    'aside',  # sidebars
    '[role="navigation"]', '[role="banner"]', '[role="contentinfo"]',
]

# Realistic browser headers to avoid basic bot detection
DEFAULT_HEADERS = {
    'User-Agent': (
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
        'AppleWebKit/537.36 (KHTML, like Gecko) '
        'Chrome/120.0.0.0 Safari/537.36'
    ),
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'en-US,en;q=0.9',
    'Accept-Encoding': 'gzip, deflate, br',
    'DNT': '1',
    'Connection': 'keep-alive',
    'Upgrade-Insecure-Requests': '1',
}


class FetchPageTool(BaseTool):
    """
    Fetch a web page and extract clean, readable text content.
    
    Uses httpx for HTTP requests and BeautifulSoup for HTML parsing.
    Strips scripts, styles, navigation, and other noise.
    Returns clean text, optionally with links.
    
    For JavaScript-heavy pages, use the BROWSER tool instead.
    """

    def get_metadata(self) -> ToolMetadata:
        return ToolMetadata(
            name="FETCH_PAGE",
            category="web",
            description=(
                "Fetch a web page and extract clean readable text. "
                "Returns the page title, meta description, and main text content. "
                "Strips navigation, ads, scripts, and styling for clean output. "
                "Use the optional 'selector' parameter to extract specific page sections. "
                "For JavaScript-heavy pages (SPAs), use the BROWSER tool instead. "
                "Set extract_links=true to include all links on the page."
            ),
            version="1.0",
            parameters=[
                ToolParameter(
                    name="url",
                    type="string",
                    required=True,
                    description="The URL to fetch"
                ),
                ToolParameter(
                    name="selector",
                    type="string",
                    required=False,
                    default=None,
                    description="Optional CSS selector to extract specific content (e.g. 'article', 'main', '.content')"
                ),
                ToolParameter(
                    name="max_length",
                    type="int",
                    required=False,
                    default=4000,
                    description="Maximum characters of text content to return (default 4000)"
                ),
                ToolParameter(
                    name="extract_links",
                    type="bool",
                    required=False,
                    default=False,
                    description="If true, include all links found on the page"
                ),
            ],
            permissions=[ToolPermission.NETWORK],
            examples=[
                '```json\n{"name": "FETCH_PAGE", "parameters": {"url": "https://example.com/article"}}\n```',
                '```json\n{"name": "FETCH_PAGE", "parameters": {"url": "https://example.com/pricing", "selector": ".pricing-table", "extract_links": true}}\n```',
            ],
            tags=["web", "scrape", "fetch", "research"],
        )

    def execute(self, params: Dict[str, Any], context: ToolExecutionContext) -> ToolResult:
        url = params.get("url", "").strip()
        if not url:
            return ToolResult(success=False, output="", error="URL is required")

        # Validate URL
        parsed = urlparse(url)
        if parsed.scheme not in ('http', 'https'):
            if not parsed.scheme:
                url = f"https://{url}"
            else:
                return ToolResult(success=False, output="", error=f"Invalid URL scheme: {parsed.scheme}")

        # Block local/internal URLs
        hostname = urlparse(url).hostname or ''
        if hostname in ('localhost', '127.0.0.1', '0.0.0.0') or hostname.startswith('192.168.') or hostname.startswith('10.'):
            return ToolResult(success=False, output="", error="Cannot fetch internal/local URLs")

        selector = params.get("selector")
        max_length = min(int(params.get("max_length", 4000)), 8000)
        extract_links = params.get("extract_links", False)
        if isinstance(extract_links, str):
            extract_links = extract_links.lower() in ('true', '1', 'yes')

        try:
            import httpx
            from bs4 import BeautifulSoup

            # Fetch page
            logger.info(f"[FETCH_PAGE] Fetching: {url}")
            response = httpx.get(
                url,
                headers=DEFAULT_HEADERS,
                follow_redirects=True,
                timeout=20,
            )
            response.raise_for_status()

            content_type = response.headers.get('content-type', '')
            if 'text/html' not in content_type and 'application/xhtml' not in content_type:
                # Non-HTML content — return raw text (truncated)
                text = response.text[:max_length]
                return ToolResult(
                    success=True,
                    output=f"Content-Type: {content_type}\n\n{text}",
                    metadata={"url": str(response.url), "content_type": content_type}
                )

            # Parse HTML
            soup = BeautifulSoup(response.text, 'lxml')

            # Extract metadata
            title = soup.title.get_text(strip=True) if soup.title else ''
            meta_desc = ''
            meta_tag = soup.find('meta', attrs={'name': 'description'})
            if meta_tag:
                meta_desc = meta_tag.get('content', '')

            # Strip noise elements
            for tag_selector in STRIP_TAGS:
                if tag_selector.startswith('['):
                    # Attribute selector
                    for el in soup.select(tag_selector):
                        el.decompose()
                else:
                    for el in soup.find_all(tag_selector):
                        el.decompose()

            # Extract target content
            if selector:
                target = soup.select_one(selector)
                if not target:
                    # Try broader match
                    targets = soup.select(selector)
                    if targets:
                        text_parts = [t.get_text(separator='\n', strip=True) for t in targets]
                        text = '\n\n'.join(text_parts)
                    else:
                        text = soup.body.get_text(separator='\n', strip=True) if soup.body else soup.get_text(separator='\n', strip=True)
                        text = f"[Selector '{selector}' not found, returning full page]\n\n{text}"
                else:
                    text = target.get_text(separator='\n', strip=True)
            else:
                # Use body, or fall back to full document
                body = soup.body or soup
                text = body.get_text(separator='\n', strip=True)

            # Clean up text: collapse multiple newlines, strip empty lines
            text = re.sub(r'\n{3,}', '\n\n', text)
            text = '\n'.join(line for line in text.split('\n') if line.strip())

            # Truncate
            truncated = False
            if len(text) > max_length:
                text = text[:max_length]
                text += f"\n\n... [truncated at {max_length} chars]"
                truncated = True

            # Build output
            output_parts = []
            output_parts.append(f"Page: {response.url}")
            output_parts.append(f"Title: {title}")
            if meta_desc:
                output_parts.append(f"Description: {meta_desc}")
            output_parts.append(f"Content length: {len(text)} chars")
            output_parts.append("")
            output_parts.append(text)

            # Extract links if requested
            links = []
            if extract_links:
                output_parts.append("")
                output_parts.append("── Links ──")
                # Re-parse original HTML for links (we stripped nav above)
                link_soup = BeautifulSoup(response.text, 'lxml')
                for a in link_soup.find_all('a', href=True):
                    href = a['href']
                    link_text = a.get_text(strip=True)[:80]
                    if href.startswith(('#', 'javascript:', 'mailto:')):
                        continue
                    full_url = urljoin(str(response.url), href)
                    links.append({"text": link_text, "url": full_url})
                    output_parts.append(f"  [{len(links)}] {link_text} → {full_url}")

                if not links:
                    output_parts.append("  (no links found)")

            return ToolResult(
                success=True,
                output="\n".join(output_parts),
                metadata={
                    "url": str(response.url),
                    "title": title,
                    "description": meta_desc,
                    "content_length": len(text),
                    "truncated": truncated,
                    "links_count": len(links),
                }
            )

        except Exception as e:
            logger.error(f"[FETCH_PAGE] Failed to fetch {url}: {e}", exc_info=True)
            error_msg = str(e)
            if 'ConnectError' in error_msg or 'ConnectTimeout' in error_msg:
                error_msg = f"Could not connect to {url}. The site may be down or blocking requests."
            elif 'ReadTimeout' in error_msg:
                error_msg = f"Timed out reading {url}. Try again or use the BROWSER tool for complex pages."
            return ToolResult(
                success=False, output="",
                error=f"Failed to fetch page: {error_msg}"
            )


def get_all_tools():
    """Return all web fetch tool classes for registry."""
    return [FetchPageTool]
