"""Web tools: smart web_search and shared web_fetch."""

from __future__ import annotations

import asyncio
import contextlib
import html
import json
import os
import re
import shutil
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from urllib.parse import parse_qs, urlparse, urlsplit, urlunsplit

import httpx
from bs4 import BeautifulSoup
from loguru import logger
from readability import Document

from madpod_agent.agent.investigation import classify_request
from madpod_agent.agent.tools.base import Tool
from madpod_agent.config.schema import WebSearchConfig
from madpod_agent.providers.base import LLMProvider

# Shared constants
USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7_2) AppleWebKit/537.36"
ACCEPT_LANGUAGE = "en-US,en;q=0.9"
MAX_REDIRECTS = 5
MIN_BROWSER_PROMOTION_CHARS = 450
TRACKING_QUERY_PARAMS = {
    "fbclid",
    "gclid",
    "mc_cid",
    "mc_eid",
    "ref",
    "ref_src",
    "utm_campaign",
    "utm_content",
    "utm_medium",
    "utm_source",
    "utm_term",
}
CHALLENGE_MARKERS = (
    "captcha",
    "cf-chl",
    "cloudflare",
    "verify you are human",
    "why did this happen",
    "press and hold",
)
JS_REQUIRED_MARKERS = (
    "enable javascript",
    "javascript is disabled",
    "__next_data__",
    "__nuxt",
    "window.__initial_state__",
    "window.__apollo_state__",
)
PROXY_ERROR_MARKERS = (
    "err_proxy_connection_failed",
    "err_tunnel_connection_failed",
    "proxy error",
    "proxy authentication",
    "proxy connection failed",
    "proxyconnect",
    "tunnel connection failed",
    "502 bad gateway",
    "407 proxy authentication required",
    "x-brd-error",
)
DIRECT_ONLY_PROXIES: set[str] = set()
LOW_MEMORY_DEEP_SEARCH_LIMIT_BYTES = 3 * 1024 * 1024 * 1024
LOW_MEMORY_SYSTEM_RAM_FALLBACK_BYTES = 4 * 1024 * 1024 * 1024
LOW_MEMORY_DEEP_MAX_PAGES = 4
LOW_MEMORY_DEEP_MAX_DEPTH = 1
LOW_MEMORY_DEEP_MAX_SEEDS = 1


def _strip_tags(text: str) -> str:
    """Remove HTML tags and decode entities."""
    text = re.sub(r"<script[\s\S]*?</script>", "", text, flags=re.I)
    text = re.sub(r"<style[\s\S]*?</style>", "", text, flags=re.I)
    text = re.sub(r"<[^>]+>", "", text)
    return html.unescape(text).strip()


def _normalize(text: str) -> str:
    """Normalize whitespace."""
    text = re.sub(r"[ \t]+", " ", text)
    return re.sub(r"\n{3,}", "\n\n", text).strip()


def _truncate(text: str, limit: int) -> str:
    """Trim text to a character limit without splitting awkwardly."""
    if len(text) <= limit:
        return text
    clipped = text[: max(limit - 1, 1)].rstrip()
    return clipped + "…"


def _extract_snippet(text: str, limit: int = 280) -> str:
    """Return a short single-paragraph excerpt."""
    compact = _normalize(text).replace("\n", " ")
    return _truncate(compact, limit)


def _markdown_to_text(markdown: str) -> str:
    """Convert a markdown-ish string into plain text for summarization."""
    if not markdown:
        return ""
    text = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", markdown)
    text = re.sub(r"`{1,3}", "", text)
    text = re.sub(r"^[#>\-\*\s]+", "", text, flags=re.M)
    return _normalize(text)


def _html_to_markdown(html_text: str) -> str:
    """Convert HTML to markdown."""
    text = re.sub(
        r'<a\s+[^>]*href=["\']([^"\']+)["\'][^>]*>([\s\S]*?)</a>',
        lambda m: f"[{_strip_tags(m[2])}]({m[1]})",
        html_text,
        flags=re.I,
    )
    text = re.sub(
        r"<h([1-6])[^>]*>([\s\S]*?)</h\1>",
        lambda m: f'\n{"#" * int(m[1])} {_strip_tags(m[2])}\n',
        text,
        flags=re.I,
    )
    text = re.sub(
        r"<li[^>]*>([\s\S]*?)</li>",
        lambda m: f"\n- {_strip_tags(m[1])}",
        text,
        flags=re.I,
    )
    text = re.sub(r"</(p|div|section|article)>", "\n\n", text, flags=re.I)
    text = re.sub(r"<(br|hr)\s*/?>", "\n", text, flags=re.I)
    return _normalize(_strip_tags(text))


def _validate_url(url: str) -> tuple[bool, str]:
    """Validate URL: must be http(s) with valid domain."""
    try:
        parsed = urlparse(url)
        if parsed.scheme not in ("http", "https"):
            return False, f"Only http/https allowed, got '{parsed.scheme or 'none'}'"
        if not parsed.netloc:
            return False, "Missing domain"
        return True, ""
    except Exception as exc:
        return False, str(exc)


def _normalize_domain(domain: str) -> str:
    """Normalize a user-supplied domain filter."""
    value = domain.strip().lower()
    if not value:
        return ""
    if "://" in value:
        value = urlparse(value).netloc.lower()
    value = value.split("/", 1)[0]
    return value[4:] if value.startswith("www.") else value


def _canonicalize_url(url: str) -> str:
    """Canonicalize a URL for dedupe while preserving search-relevant params."""
    parsed = urlsplit(url.strip())
    query_pairs = []
    for piece in filter(None, parsed.query.split("&")):
        key = piece.split("=", 1)[0].lower()
        if key in TRACKING_QUERY_PARAMS or key.startswith("utm_"):
            continue
        query_pairs.append(piece)
    netloc = parsed.netloc.lower()
    path = parsed.path or "/"
    return urlunsplit((parsed.scheme.lower(), netloc, path, "&".join(query_pairs), ""))


def _host_matches(url: str, domains: list[str]) -> bool:
    """Return True when a URL host is inside the allowed domain set."""
    if not domains:
        return True
    host = _normalize_domain(urlparse(url).netloc)
    return any(host == domain or host.endswith(f".{domain}") for domain in domains)


def _looks_like_serp_challenge(status_code: int, body: str) -> bool:
    """Detect a likely challenge page."""
    lowered = body.lower()
    if status_code in {401, 403, 429, 503}:
        return True
    return any(marker in lowered for marker in CHALLENGE_MARKERS)


def _should_promote_to_browser(
    *,
    status_code: int,
    content_type: str,
    text: str,
    body: str,
) -> bool:
    """Heuristics for escalating a fetch from HTTP to the browser crawler."""
    lowered = body.lower()
    if _looks_like_serp_challenge(status_code, body):
        return True
    if "text/html" not in content_type and not lowered.startswith(("<!doctype", "<html")):
        return False
    if len(text) < MIN_BROWSER_PROMOTION_CHARS:
        return True
    return any(marker in lowered for marker in JS_REQUIRED_MARKERS)


def _is_proxy_related_error(exc: Exception | str) -> bool:
    """Heuristically detect proxy and tunnel failures worth retrying directly."""
    lowered = str(exc or "").lower()
    return any(marker in lowered for marker in PROXY_ERROR_MARKERS)


def _looks_like_proxy_failure_response(response: httpx.Response) -> bool:
    """Detect responses that likely came from proxy infrastructure instead of the target site."""
    if response.status_code not in {407, 502, 503, 504}:
        return False
    header_names = {key.lower() for key in response.headers.keys()}
    body_preview = ""
    with contextlib.suppress(Exception):
        body_preview = (response.text or "")[:256].lower()
    if "x-brd-error" in header_names or "proxy-authenticate" in header_names:
        return True
    return "proxy" in body_preview or "bad gateway" in body_preview


def _google_result_url(raw_url: str) -> str:
    """Extract a direct destination URL from Google result links."""
    if raw_url.startswith("/url?"):
        query = parse_qs(urlsplit(raw_url).query)
        return (query.get("q") or query.get("url") or [""])[0]
    return raw_url


def _normalized_proxy(proxy: str | None) -> str | None:
    """Normalize proxy identifiers for sticky in-process routing decisions."""
    value = str(proxy or "").strip()
    return value or None


def _proxy_prefers_direct(proxy: str | None) -> bool:
    """Return whether this proxy has already been marked unhealthy in-process."""
    normalized = _normalized_proxy(proxy)
    return bool(normalized and normalized in DIRECT_ONLY_PROXIES)


def _mark_proxy_direct_only(proxy: str | None) -> None:
    """Remember that a proxy should be bypassed for the rest of the process lifetime."""
    normalized = _normalized_proxy(proxy)
    if normalized:
        DIRECT_ONLY_PROXIES.add(normalized)


def _current_cgroup_memory_limit_candidates() -> list[Path]:
    """Return likely memory limit files for the current process cgroup."""
    candidates: list[Path] = []
    seen: set[str] = set()

    def add(path: Path) -> None:
        normalized = str(path)
        if normalized not in seen:
            seen.add(normalized)
            candidates.append(path)

    try:
        raw_cgroup = Path("/proc/self/cgroup").read_text(encoding="utf-8")
    except OSError:
        raw_cgroup = ""

    base = Path("/sys/fs/cgroup")
    for line in raw_cgroup.splitlines():
        parts = line.strip().split(":", 2)
        if len(parts) != 3:
            continue
        _, controllers, raw_path = parts
        cgroup_path = raw_path.strip().lstrip("/")
        controller_names = {item for item in controllers.split(",") if item and item != "0"}
        if cgroup_path:
            add(base / cgroup_path / "memory.max")
            add(base / cgroup_path / "memory.limit_in_bytes")
            if "memory" in controller_names:
                add(base / "memory" / cgroup_path / "memory.limit_in_bytes")

    add(base / "memory.max")
    add(base / "memory" / "memory.limit_in_bytes")
    return candidates


def _runtime_memory_limit_bytes() -> int | None:
    """Best-effort detection of the current runtime memory limit."""
    for candidate in _current_cgroup_memory_limit_candidates():
        try:
            raw = candidate.read_text(encoding="utf-8").strip()
        except OSError:
            continue
        if not raw or raw == "max":
            continue
        with contextlib.suppress(ValueError):
            value = int(raw)
            if value > 0:
                return value
    return None


def _total_physical_memory_bytes() -> int | None:
    """Best-effort detection of the host RAM available to the runtime."""
    with contextlib.suppress(OSError, ValueError):
        page_size = os.sysconf("SC_PAGE_SIZE")
        page_count = os.sysconf("SC_PHYS_PAGES")
        if page_size > 0 and page_count > 0:
            return int(page_size) * int(page_count)

    try:
        for line in Path("/proc/meminfo").read_text(encoding="utf-8").splitlines():
            if not line.startswith("MemTotal:"):
                continue
            parts = line.split()
            if len(parts) >= 2:
                return int(parts[1]) * 1024
    except OSError:
        pass
    return None


def _should_use_memory_saver_for_deep_search() -> bool:
    """Enable lighter deep-search budgets on low-memory workers."""
    limit = _runtime_memory_limit_bytes()
    if limit and limit <= LOW_MEMORY_DEEP_SEARCH_LIMIT_BYTES:
        return True
    total_memory = _total_physical_memory_bytes()
    return bool(total_memory and total_memory <= LOW_MEMORY_SYSTEM_RAM_FALLBACK_BYTES)


@dataclass(slots=True)
class SearchResult:
    """A normalized search engine result."""

    title: str
    url: str
    snippet: str
    engine: str


@dataclass(slots=True)
class FetchedDocument:
    """Structured representation of fetched page content."""

    url: str
    final_url: str
    status_code: int | None
    title: str
    markdown: str
    text: str
    snippet: str
    extractor: str
    fetch_method: str
    errors: list[str] = field(default_factory=list)


@dataclass(slots=True)
class EnrichedResult:
    """A search result enriched with fetched page content."""

    rank: int
    title: str
    url: str
    final_url: str
    snippet: str
    excerpt: str
    engine: str
    extractor: str | None
    fetch_method: str | None
    status_code: int | None
    markdown: str
    text: str
    errors: list[str] = field(default_factory=list)


@dataclass(slots=True)
class CircuitBreakerState:
    """Small in-memory circuit breaker per engine."""

    consecutive_failures: int = 0
    open_until: float = 0.0


class WebContentPipeline:
    """Shared HTTP-first, browser-second fetch pipeline."""

    def __init__(
        self,
        *,
        workspace: Path | None,
        proxy: str | None,
        search_config: WebSearchConfig,
    ) -> None:
        self.workspace = workspace
        self.proxy = proxy
        self.search_config = search_config
        self._client: httpx.AsyncClient | None = None
        self._direct_client: httpx.AsyncClient | None = None
        self._crawler: Any = None
        self._crawler_proxy: str | None = None
        self._prefer_direct = _proxy_prefers_direct(proxy)
        self._browser_lock = asyncio.Lock()

    async def __aenter__(self) -> WebContentPipeline:
        self._client = self._build_http_client(self.proxy)
        return self

    async def __aexit__(self, *_exc_info: object) -> None:
        if self._client is not None:
            await self._client.aclose()
            self._client = None
        if self._direct_client is not None:
            await self._direct_client.aclose()
            self._direct_client = None
        await self._close_browser()

    @property
    def prefer_direct(self) -> bool:
        """Expose when the pipeline had to abandon the configured proxy."""
        return self._prefer_direct

    def _build_http_client(self, proxy: str | None) -> httpx.AsyncClient:
        return httpx.AsyncClient(
            follow_redirects=True,
            max_redirects=MAX_REDIRECTS,
            timeout=float(self.search_config.http_timeout_s),
            proxy=proxy,
            headers={
                "User-Agent": USER_AGENT,
                "Accept-Language": ACCEPT_LANGUAGE,
            },
        )

    async def _close_browser(self) -> None:
        if self._crawler is None:
            return
        await self._crawler.__aexit__(None, None, None)
        self._crawler = None
        self._crawler_proxy = None

    async def release_browser(self) -> None:
        """Allow callers to proactively drop browser state between heavy crawls."""
        await self._close_browser()

    def _proxy_attempt_order(self) -> list[bool]:
        if not self.proxy:
            return [False]
        if self._prefer_direct:
            return [False]
        return [True, False]

    async def _http_client_for(self, *, use_proxy: bool) -> httpx.AsyncClient:
        if self._client is None:
            raise RuntimeError("WebContentPipeline must be used as an async context manager")
        if use_proxy or not self.proxy:
            return self._client
        if self._direct_client is None:
            self._direct_client = self._build_http_client(None)
        return self._direct_client

    async def fetch_document(self, url: str, *, max_chars: int = 50_000) -> FetchedDocument:
        """Fetch a single document, escalating to the browser only when needed."""
        is_valid, error_msg = _validate_url(url)
        if not is_valid:
            raise ValueError(f"URL validation failed: {error_msg}")

        response, errors = await self._fetch_http_with_fallback(url)
        body = response.text
        content_type = response.headers.get("content-type", "")

        if "application/json" in content_type:
            try:
                payload = json.dumps(response.json(), indent=2, ensure_ascii=False)
            except Exception:
                payload = response.text
            return FetchedDocument(
                url=url,
                final_url=str(response.url),
                status_code=response.status_code,
                title="JSON response",
                markdown=payload,
                text=payload,
                snippet=_extract_snippet(payload),
                extractor="json",
                fetch_method="http",
                errors=list(errors),
            )

        http_doc = self._document_from_html_or_text(
            url=url,
            final_url=str(response.url),
            status_code=response.status_code,
            raw_body=body,
            content_type=content_type,
            extractor="readability",
            fetch_method="http",
        )
        http_doc.errors = [*errors, *http_doc.errors]

        if not self.search_config.browser_fallback_enabled:
            if response.status_code >= 400:
                raise httpx.HTTPStatusError(
                    f"HTTP {response.status_code}",
                    request=response.request,
                    response=response,
                )
            return self._truncate_document(http_doc, max_chars)

        if _should_promote_to_browser(
            status_code=response.status_code,
            content_type=content_type,
            text=http_doc.text,
            body=body,
        ):
            errors.append(
                f"promoted_to_browser status={response.status_code} body_chars={len(http_doc.text)}"
            )
            try:
                browser_doc = await self._fetch_with_browser(url)
                browser_doc.errors = [*errors, *browser_doc.errors]
                return self._truncate_document(browser_doc, max_chars)
            except Exception as exc:
                errors.append(f"browser_fallback_failed: {exc}")
                logger.warning("Web fetch browser fallback failed for {}: {}", url, exc)

        if response.status_code >= 400 and not http_doc.text:
            raise httpx.HTTPStatusError(
                f"HTTP {response.status_code}",
                request=response.request,
                response=response,
            )

        return self._truncate_document(http_doc, max_chars)

    async def deep_crawl(
        self,
        seed_url: str,
        *,
        max_depth: int,
        max_pages: int,
        allowed_domains: list[str],
    ) -> list[FetchedDocument]:
        """Run a bounded deep crawl from a seed URL."""
        results = await self._deep_crawl_with_fallback(
            seed_url,
            max_depth=max_depth,
            max_pages=max_pages,
        )

        documents: list[FetchedDocument] = []
        containers = results if isinstance(results, list) else [results]
        for container in containers:
            document = self._document_from_crawl_result(container)
            if allowed_domains and not _host_matches(document.final_url, allowed_domains):
                continue
            documents.append(document)
        return documents

    async def _fetch_http(self, url: str) -> httpx.Response:
        """Fetch raw content over HTTP with proxy fallback when necessary."""
        response, _errors = await self._fetch_http_with_fallback(url)
        return response

    async def _fetch_http_with_fallback(self, url: str) -> tuple[httpx.Response, list[str]]:
        errors: list[str] = []
        last_exc: Exception | None = None
        for use_proxy in self._proxy_attempt_order():
            try:
                response = await self._fetch_http_once(url, use_proxy=use_proxy)
            except Exception as exc:
                last_exc = exc
                if use_proxy and self.proxy and _is_proxy_related_error(exc):
                    self._prefer_direct = True
                    _mark_proxy_direct_only(self.proxy)
                    errors.append(f"proxy_http_failed: {exc}")
                    logger.warning("WebFetch HTTP proxy failed for {}: {}. Retrying direct.", url, exc)
                    continue
                raise
            if use_proxy and self.proxy and _looks_like_proxy_failure_response(response):
                self._prefer_direct = True
                _mark_proxy_direct_only(self.proxy)
                errors.append(f"proxy_http_failed: HTTP {response.status_code}")
                logger.warning(
                    "WebFetch HTTP got proxy-like response for {} (status={}); retrying direct.",
                    url,
                    response.status_code,
                )
                continue
            return response, errors
        if last_exc is not None:
            raise last_exc
        raise RuntimeError(f"Unable to fetch {url}")

    async def _fetch_http_once(self, url: str, *, use_proxy: bool) -> httpx.Response:
        client = await self._http_client_for(use_proxy=use_proxy)
        logger.debug("WebFetch HTTP url={} proxy={}", url, bool(use_proxy and self.proxy))
        return await client.get(url)

    async def _fetch_with_browser(self, url: str) -> FetchedDocument:
        """Fetch a page with Crawl4AI."""
        errors: list[str] = []
        last_exc: Exception | None = None
        for use_proxy in self._proxy_attempt_order():
            try:
                document = await self._fetch_with_browser_once(url, use_proxy=use_proxy)
            except Exception as exc:
                last_exc = exc
                if use_proxy and self.proxy and _is_proxy_related_error(exc):
                    self._prefer_direct = True
                    _mark_proxy_direct_only(self.proxy)
                    errors.append(f"proxy_browser_failed: {exc}")
                    logger.warning("WebFetch browser proxy failed for {}: {}. Retrying direct.", url, exc)
                    continue
                raise
            if errors:
                document.errors = [*errors, *document.errors]
            return document
        if last_exc is not None:
            raise last_exc
        raise RuntimeError(f"Unable to browser-fetch {url}")

    async def _fetch_with_browser_once(self, url: str, *, use_proxy: bool) -> FetchedDocument:
        crawler = await self._ensure_browser(use_proxy=use_proxy)
        from crawl4ai import CacheMode, CrawlerRunConfig

        run_config = CrawlerRunConfig(
            cache_mode=CacheMode.BYPASS,
            page_timeout=int(self.search_config.browser_timeout_s * 1000),
            wait_until="domcontentloaded",
            delay_before_return_html=0.2,
            verbose=False,
        )

        async with self._browser_lock:
            try:
                result = await self._await_browser_operation(
                    crawler.arun(url, config=run_config),
                    action="fetch",
                    target=url,
                )
            except Exception:
                with contextlib.suppress(Exception):
                    await self._close_browser()
                raise

        return self._document_from_crawl_result(result)

    async def _deep_crawl_with_fallback(
        self,
        seed_url: str,
        *,
        max_depth: int,
        max_pages: int,
    ) -> Any:
        last_exc: Exception | None = None
        for use_proxy in self._proxy_attempt_order():
            try:
                return await self._deep_crawl_once(
                    seed_url,
                    max_depth=max_depth,
                    max_pages=max_pages,
                    use_proxy=use_proxy,
                )
            except Exception as exc:
                last_exc = exc
                if use_proxy and self.proxy and _is_proxy_related_error(exc):
                    self._prefer_direct = True
                    _mark_proxy_direct_only(self.proxy)
                    logger.warning("WebSearch deep crawl proxy failed for {}: {}. Retrying direct.", seed_url, exc)
                    continue
                raise
        if last_exc is not None:
            raise last_exc
        raise RuntimeError(f"Unable to deep crawl {seed_url}")

    async def _deep_crawl_once(
        self,
        seed_url: str,
        *,
        max_depth: int,
        max_pages: int,
        use_proxy: bool,
    ) -> Any:
        crawler = await self._ensure_browser(use_proxy=use_proxy)
        from crawl4ai import CacheMode, CrawlerRunConfig
        from crawl4ai.deep_crawling import BFSDeepCrawlStrategy

        run_config = CrawlerRunConfig(
            cache_mode=CacheMode.BYPASS,
            page_timeout=int(self.search_config.browser_timeout_s * 1000),
            wait_until="domcontentloaded",
            delay_before_return_html=0.2,
            verbose=False,
            stream=False,
            deep_crawl_strategy=BFSDeepCrawlStrategy(
                max_depth=max_depth,
                max_pages=max_pages,
                include_external=False,
            ),
        )

        async with self._browser_lock:
            try:
                return await self._await_browser_operation(
                    crawler.arun(seed_url, config=run_config),
                    action="crawl",
                    target=seed_url,
                )
            except Exception:
                with contextlib.suppress(Exception):
                    await self._close_browser()
                raise

    async def _ensure_browser(self, *, use_proxy: bool) -> Any:
        """Start the Crawl4AI browser on first use."""
        requested_proxy = self.proxy if use_proxy and self.proxy else None
        if self._crawler is not None and self._crawler_proxy == requested_proxy:
            return self._crawler
        if self._crawler is not None:
            await self._close_browser()

        from crawl4ai import AsyncWebCrawler, BrowserConfig

        channel = "chrome" if self._system_chrome_available() else "chromium"
        geteuid = getattr(os, "geteuid", None)
        is_root = bool(geteuid and geteuid() == 0)
        extra_args = ["--no-sandbox"] if is_root else []
        profile_dir = self._browser_profile_dir()
        logger.debug("WebFetch browser init channel={} proxy={}", channel, bool(requested_proxy))
        crawler = AsyncWebCrawler(
            config=BrowserConfig(
                headless=True,
                verbose=False,
                channel=channel,
                chrome_channel=channel,
                user_data_dir=str(profile_dir) if profile_dir else None,
                proxy=requested_proxy,
                viewport_width=1280,
                viewport_height=800,
                extra_args=extra_args or None,
            )
        )
        try:
            entered = await self._await_browser_operation(
                crawler.__aenter__(),
                action="startup",
                target=None,
            )
        except Exception:
            with contextlib.suppress(Exception):
                await crawler.__aexit__(None, None, None)
            raise
        self._crawler = entered or crawler
        self._crawler_proxy = requested_proxy
        return self._crawler

    def _browser_operation_timeout_s(self) -> float:
        return max(float(self.search_config.browser_timeout_s), 1.0)

    async def _await_browser_operation(
        self,
        awaitable: Any,
        *,
        action: str,
        target: str | None,
    ) -> Any:
        timeout_s = self._browser_operation_timeout_s()
        try:
            return await asyncio.wait_for(awaitable, timeout=timeout_s)
        except asyncio.TimeoutError as exc:
            suffix = f" for {target}" if target else ""
            raise TimeoutError(
                f"Browser {action} timed out after {timeout_s:.1f}s{suffix}"
            ) from exc

    def _browser_profile_dir(self) -> Path | None:
        """Resolve a dedicated browser profile directory for search crawling."""
        if self.workspace is None:
            return None
        base = self.workspace.parent if self.workspace.name == "workspace" else self.workspace
        profile_dir = base / "chrome" / "smart-search"
        profile_dir.mkdir(parents=True, exist_ok=True)
        return profile_dir

    @staticmethod
    def _system_chrome_available() -> bool:
        """Detect whether the runtime has a system Chrome binary."""
        return any(
            shutil.which(candidate)
            for candidate in ("google-chrome", "google-chrome-stable", "chrome")
        )

    @staticmethod
    def _truncate_document(document: FetchedDocument, max_chars: int) -> FetchedDocument:
        """Cap large payloads while keeping snippets useful."""
        if len(document.markdown) <= max_chars and len(document.text) <= max_chars:
            return document
        return FetchedDocument(
            url=document.url,
            final_url=document.final_url,
            status_code=document.status_code,
            title=document.title,
            markdown=_truncate(document.markdown, max_chars),
            text=_truncate(document.text, max_chars),
            snippet=document.snippet,
            extractor=document.extractor,
            fetch_method=document.fetch_method,
            errors=list(document.errors),
        )

    def _document_from_html_or_text(
        self,
        *,
        url: str,
        final_url: str,
        status_code: int | None,
        raw_body: str,
        content_type: str,
        extractor: str,
        fetch_method: str,
    ) -> FetchedDocument:
        """Extract title, markdown, and text from HTTP content."""
        errors: list[str] = []
        body = raw_body or ""
        lowered = body[:512].lower()

        if "text/html" in content_type or lowered.startswith(("<!doctype", "<html")):
            title = ""
            markdown = ""
            text = ""
            try:
                doc = Document(body)
                summary_html = doc.summary()
                title = doc.title() or ""
                markdown = _html_to_markdown(summary_html)
                text = _normalize(_strip_tags(summary_html))
            except Exception as exc:
                errors.append(f"readability_failed: {exc}")

            if len(text) < MIN_BROWSER_PROMOTION_CHARS:
                soup = BeautifulSoup(body, "html.parser")
                title = title or (soup.title.get_text(" ", strip=True) if soup.title else "")
                body_text = _normalize(soup.get_text("\n", strip=True))
                markdown = markdown or _truncate(body_text, 50_000)
                text = text or body_text
                extractor = "html"

            if title and markdown:
                markdown = f"# {title}\n\n{markdown}"

            return FetchedDocument(
                url=url,
                final_url=final_url,
                status_code=status_code,
                title=title or final_url,
                markdown=markdown or text,
                text=text,
                snippet=_extract_snippet(text),
                extractor=extractor,
                fetch_method=fetch_method,
                errors=errors,
            )

        text = _normalize(body)
        return FetchedDocument(
            url=url,
            final_url=final_url,
            status_code=status_code,
            title=final_url,
            markdown=text,
            text=text,
            snippet=_extract_snippet(text),
            extractor="raw",
            fetch_method=fetch_method,
            errors=errors,
        )

    def _document_from_crawl_result(self, result: Any) -> FetchedDocument:
        """Convert Crawl4AI output into the shared fetched-document shape."""
        container = result[0] if isinstance(result, list) else result
        success = bool(getattr(container, "success", False))
        if not success:
            raise RuntimeError(getattr(container, "error_message", "Browser crawl failed"))

        markdown_result = getattr(container, "markdown", None)
        markdown = ""
        if markdown_result is not None:
            markdown = (
                getattr(markdown_result, "fit_markdown", None)
                or getattr(markdown_result, "raw_markdown", None)
                or ""
            )

        html_body = getattr(container, "cleaned_html", None) or getattr(container, "html", "") or ""
        title = ""
        if html_body:
            try:
                title = Document(html_body).title() or ""
            except Exception:
                title = ""

        text = _markdown_to_text(markdown)
        if not text and html_body:
            fallback = self._document_from_html_or_text(
                url=getattr(container, "url", ""),
                final_url=getattr(container, "redirected_url", None) or getattr(container, "url", ""),
                status_code=getattr(container, "status_code", None),
                raw_body=html_body,
                content_type="text/html",
                extractor="crawl4ai",
                fetch_method="browser",
            )
            title = title or fallback.title
            markdown = markdown or fallback.markdown
            text = fallback.text

        final_url = getattr(container, "redirected_url", None) or getattr(container, "url", "")
        return FetchedDocument(
            url=getattr(container, "url", ""),
            final_url=final_url,
            status_code=getattr(container, "status_code", None),
            title=title or final_url or getattr(container, "url", ""),
            markdown=markdown or text,
            text=text,
            snippet=_extract_snippet(text),
            extractor="crawl4ai",
            fetch_method="browser",
            errors=[],
        )


class WebSearchTool(Tool):
    """Search the web without requiring a dedicated search API key."""

    name = "web_search"
    description = (
        "Search the web and return structured JSON with a summary, ranked results, sources, engine trace, and errors. "
        "Use this for factual lookups, current information, comparisons, and early-stage research. "
        "If you omit mode, madpod-agent will choose shallow search for simple queries and bounded deep search for investigative ones. "
        "Use explicit deep mode when you know you want a broader crawl, and follow up with web_fetch or browser tools when you need page-level evidence."
    )
    parameters = {
        "type": "object",
        "properties": {
            "query": {"type": "string", "description": "Search query"},
            "count": {
                "type": "integer",
                "description": "Desired ranked results (1-10)",
                "minimum": 1,
                "maximum": 10,
            },
            "mode": {
                "type": "string",
                "enum": ["search", "deep"],
                "description": "Optional explicit mode. Omit it to let madpod-agent choose shallow or deep search based on the query.",
            },
            "maxPages": {
                "type": "integer",
                "description": "Maximum pages to fetch or crawl",
                "minimum": 1,
                "maximum": 20,
            },
            "maxDepth": {
                "type": "integer",
                "description": "Depth limit for deep mode",
                "minimum": 0,
                "maximum": 3,
            },
            "domains": {
                "type": "array",
                "description": "Optional domain allowlist",
                "items": {"type": "string"},
            },
            "summarize": {
                "type": "boolean",
                "description": "Whether to synthesize a summary when configured",
            },
        },
        "required": ["query"],
    }

    def __init__(
        self,
        *,
        api_key: str | None = None,
        max_results: int = 5,
        proxy: str | None = None,
        provider: LLMProvider | None = None,
        workspace: Path | None = None,
        search_config: WebSearchConfig | None = None,
        summary_model: str | None = None,
    ) -> None:
        config = search_config.model_copy(deep=True) if search_config else WebSearchConfig()
        if api_key and not config.api_key:
            config.api_key = api_key
        if search_config is None:
            config.max_results = max_results
        if not (config.summary_model or "").strip() and summary_model:
            config.summary_model = summary_model
        self.search_config = config
        self.proxy = proxy
        self._prefer_direct = _proxy_prefers_direct(proxy)
        self.provider = provider
        self.workspace = workspace
        self._breakers = {engine: CircuitBreakerState() for engine in self.search_config.engines}
        self._memory_saver_deep = _should_use_memory_saver_for_deep_search()

    async def execute(
        self,
        query: str,
        count: int | None = None,
        mode: str | None = None,
        max_pages: int | None = None,
        max_depth: int | None = None,
        domains: list[str] | None = None,
        summarize: bool = True,
        **kwargs: Any,
    ) -> str:
        requested_mode = kwargs.get("mode", mode)
        profile = classify_request(query)
        resolved_mode = requested_mode or (
            "deep" if profile.is_investigative and profile.domain in {"web", "mixed"} else "search"
        )
        if (
            requested_mode is None
            and resolved_mode == "deep"
            and not self.search_config.browser_fallback_enabled
        ):
            resolved_mode = "search"
        max_pages = kwargs.get("maxPages", max_pages)
        max_depth = kwargs.get("maxDepth", max_depth)
        n = min(max(count or self.search_config.max_results, 1), 10)
        allowed_domains = [item for item in (_normalize_domain(d) for d in (domains or [])) if item]
        page_budget = min(max_pages or (n if resolved_mode == "search" else max(n * 2, 6)), 20)
        depth_budget = min(max(max_depth if max_depth is not None else 1, 0), 3)
        if resolved_mode == "deep" and self._memory_saver_deep:
            page_budget = min(page_budget, LOW_MEMORY_DEEP_MAX_PAGES)
            depth_budget = min(depth_budget, LOW_MEMORY_DEEP_MAX_DEPTH)
        engine_trace: list[dict[str, Any]] = []
        errors: list[dict[str, str]] = []

        async with WebContentPipeline(
            workspace=self.workspace,
            proxy=None if self._prefer_direct else self.proxy,
            search_config=self.search_config,
        ) as pipeline:
            seeds = await self._collect_seed_results(
                query=query,
                count=max(n, min(page_budget, 10)),
                domains=allowed_domains,
                engine_trace=engine_trace,
                errors=errors,
            )

            if not seeds:
                return json.dumps(
                    {
                        "query": query,
                        "mode": resolved_mode,
                        "summary": f"No results found for: {query}",
                        "results": [],
                        "sources": [],
                        "engine_trace": engine_trace,
                        "errors": errors,
                    },
                    ensure_ascii=False,
                )

            deep_mode_downgraded = resolved_mode == "deep" and self._memory_saver_deep
            if deep_mode_downgraded:
                errors.append(
                    {
                        "engine": "memory_saver",
                        "message": (
                            "Low-memory runtime downgraded deep crawl to targeted fetch mode "
                            "to avoid browser OOM."
                        ),
                    }
                )
            if resolved_mode == "deep" and not deep_mode_downgraded:
                enriched = await self._run_deep_mode(
                    query=query,
                    seeds=seeds,
                    page_budget=page_budget,
                    depth_budget=depth_budget,
                    allowed_domains=allowed_domains,
                    pipeline=pipeline,
                    errors=errors,
                )
            else:
                enriched = await self._run_search_mode(
                    seeds=seeds,
                    page_budget=page_budget,
                    pipeline=pipeline,
                    errors=errors,
                )
            self._prefer_direct = self._prefer_direct or pipeline.prefer_direct

        summary = await self._build_summary(
            query=query,
            mode=resolved_mode,
            results=enriched,
            use_llm=summarize and self.search_config.summary_enabled,
        )
        payload = {
            "query": query,
            "mode": resolved_mode,
            "summary": summary,
            "results": [self._serialize_result(item) for item in enriched[:n]],
            "sources": [self._serialize_source(idx, item) for idx, item in enumerate(enriched[:n], 1)],
            "engine_trace": engine_trace,
            "errors": errors,
        }
        return json.dumps(payload, ensure_ascii=False)

    async def _collect_seed_results(
        self,
        *,
        query: str,
        count: int,
        domains: list[str],
        engine_trace: list[dict[str, Any]],
        errors: list[dict[str, str]],
    ) -> list[SearchResult]:
        """Collect seed results from the configured engine chain."""
        seen: set[str] = set()
        collected: list[SearchResult] = []
        for engine in self.search_config.engines:
            breaker = self._breakers.setdefault(engine, CircuitBreakerState())
            if breaker.open_until > time.monotonic():
                engine_trace.append(
                    {
                        "engine": engine,
                        "status": "skipped",
                        "reason": "circuit_breaker_open",
                        "open_until": breaker.open_until,
                    }
                )
                continue

            try:
                batch, proxy_enabled, retried_without_proxy = await self._run_engine_with_fallback(
                    engine=engine,
                    query=query,
                    count=min(max(count * 2, count), 10),
                )
                filtered: list[SearchResult] = []
                for item in batch:
                    canonical = _canonicalize_url(item.url)
                    if not canonical or canonical in seen or not _host_matches(item.url, domains):
                        continue
                    seen.add(canonical)
                    filtered.append(item)
                    collected.append(item)
                    if len(collected) >= count:
                        break
                breaker.consecutive_failures = 0
                breaker.open_until = 0.0
                engine_trace.append(
                    {
                        "engine": engine,
                        "status": "success",
                        "results": len(filtered),
                        "proxy_enabled": proxy_enabled,
                        "retried_without_proxy": retried_without_proxy,
                    }
                )
                if len(collected) >= count:
                    break
            except Exception as exc:
                breaker.consecutive_failures += 1
                if breaker.consecutive_failures >= 3:
                    breaker.open_until = time.monotonic() + 120.0
                message = str(exc)
                engine_trace.append(
                    {
                        "engine": engine,
                        "status": "error",
                        "error": message,
                        "proxy_enabled": bool(self.proxy and not self._prefer_direct),
                    }
                )
                errors.append({"engine": engine, "message": message})
                logger.warning("WebSearch engine={} failed: {}", engine, exc)

        return collected[:count]

    async def _run_engine_with_fallback(
        self,
        *,
        engine: str,
        query: str,
        count: int,
    ) -> tuple[list[SearchResult], bool, bool]:
        """Run a search engine request and retry direct on proxy transport failures."""
        use_proxy = bool(self.proxy and not self._prefer_direct and not _proxy_prefers_direct(self.proxy))
        try:
            return await self._run_engine_request(
                engine=engine,
                query=query,
                count=count,
                use_proxy=use_proxy,
            ), use_proxy, False
        except Exception as exc:
            if use_proxy and _is_proxy_related_error(exc):
                self._prefer_direct = True
                _mark_proxy_direct_only(self.proxy)
                logger.warning(
                    "WebSearch engine={} proxy failed: {}. Retrying direct.",
                    engine,
                    exc,
                )
                return await self._run_engine_request(
                    engine=engine,
                    query=query,
                    count=count,
                    use_proxy=False,
                ), False, True
            raise

    async def _run_engine_request(
        self,
        *,
        engine: str,
        query: str,
        count: int,
        use_proxy: bool,
    ) -> list[SearchResult]:
        """Run a search engine adapter with a client configured for the current proxy mode."""
        async with httpx.AsyncClient(
            follow_redirects=True,
            max_redirects=MAX_REDIRECTS,
            timeout=float(self.search_config.http_timeout_s),
            proxy=self.proxy if use_proxy and self.proxy else None,
            headers={
                "User-Agent": USER_AGENT,
                "Accept-Language": ACCEPT_LANGUAGE,
            },
        ) as client:
            return await self._run_engine(
                client=client,
                engine=engine,
                query=query,
                count=count,
            )

    async def _run_engine(
        self,
        *,
        client: httpx.AsyncClient,
        engine: str,
        query: str,
        count: int,
    ) -> list[SearchResult]:
        """Execute a single search provider adapter."""
        if engine == "google_html":
            response = await client.get(
                "https://www.google.com/search",
                params={"q": query, "num": count, "hl": "en", "gbv": "1"},
            )
            response.raise_for_status()
            return self._parse_google_results(response.text)

        if engine == "duckduckgo_html":
            response = await client.post(
                "https://html.duckduckgo.com/html/",
                data={"q": query},
            )
            response.raise_for_status()
            return self._parse_duckduckgo_results(response.text)

        if engine == "bing_html":
            response = await client.get(
                "https://www.bing.com/search",
                params={"q": query, "count": count, "setlang": "en-US"},
            )
            response.raise_for_status()
            return self._parse_bing_results(response.text)

        if engine == "searxng":
            if not self.search_config.searxng_base_url:
                raise ValueError("searxng engine selected but tools.web.search.searxng_base_url is empty")
            base_url = self.search_config.searxng_base_url.rstrip("/")
            response = await client.get(
                f"{base_url}/search",
                params={"q": query, "format": "json"},
            )
            response.raise_for_status()
            payload = response.json()
            return [
                SearchResult(
                    title=item.get("title", "").strip(),
                    url=item.get("url", "").strip(),
                    snippet=item.get("content", "").strip(),
                    engine="searxng",
                )
                for item in payload.get("results", [])
                if item.get("url")
            ]

        if engine == "brave_api":
            api_key = self.search_config.api_key or os.environ.get("BRAVE_API_KEY", "")
            if not api_key:
                raise ValueError("brave_api engine selected but Brave API key is not configured")
            response = await client.get(
                "https://api.search.brave.com/res/v1/web/search",
                params={"q": query, "count": count},
                headers={"Accept": "application/json", "X-Subscription-Token": api_key},
            )
            response.raise_for_status()
            payload = response.json().get("web", {}).get("results", [])
            return [
                SearchResult(
                    title=item.get("title", "").strip(),
                    url=item.get("url", "").strip(),
                    snippet=item.get("description", "").strip(),
                    engine="brave_api",
                )
                for item in payload
                if item.get("url")
            ]

        raise ValueError(f"Unknown search engine '{engine}'")

    def _parse_google_results(self, html_text: str) -> list[SearchResult]:
        """Parse Google HTML results."""
        soup = BeautifulSoup(html_text, "html.parser")
        results: list[SearchResult] = []
        blocks = soup.select("div.g, div.tF2Cxc, div.Gx5Zad")
        for block in blocks:
            link = block.select_one("a[href]")
            title_el = block.select_one("h3")
            if link is None or title_el is None:
                continue
            url = _google_result_url(link.get("href", "").strip())
            if not url or url.startswith("/search?") or "google.com" in urlparse(url).netloc.lower():
                continue
            snippet_el = block.select_one(".VwiC3b, .st, .aCOpRe, div[data-sncf], span[data-sncf]")
            snippet = snippet_el.get_text(" ", strip=True) if snippet_el else ""
            results.append(
                SearchResult(
                    title=title_el.get_text(" ", strip=True),
                    url=url,
                    snippet=snippet,
                    engine="google_html",
                )
            )
        if results:
            return results

        for link in soup.select("a[href]"):
            title_el = link.find("h3")
            if title_el is None:
                continue
            url = _google_result_url(link.get("href", "").strip())
            if not url or "google.com" in urlparse(url).netloc.lower():
                continue
            results.append(
                SearchResult(
                    title=title_el.get_text(" ", strip=True),
                    url=url,
                    snippet="",
                    engine="google_html",
                )
            )
        return results

    @staticmethod
    def _parse_duckduckgo_results(html_text: str) -> list[SearchResult]:
        """Parse DuckDuckGo HTML results."""
        soup = BeautifulSoup(html_text, "html.parser")
        results: list[SearchResult] = []
        for block in soup.select(".result, .web-result"):
            link = block.select_one(".result__title a[href], a.result__a[href]")
            if link is None:
                link = block.select_one(".result-link[href]")
            if link is None:
                continue
            snippet_el = block.select_one(".result__snippet, .result-snippet")
            results.append(
                SearchResult(
                    title=link.get_text(" ", strip=True),
                    url=link.get("href", "").strip(),
                    snippet=snippet_el.get_text(" ", strip=True) if snippet_el else "",
                    engine="duckduckgo_html",
                )
            )
        return results

    @staticmethod
    def _parse_bing_results(html_text: str) -> list[SearchResult]:
        """Parse Bing HTML results."""
        soup = BeautifulSoup(html_text, "html.parser")
        results: list[SearchResult] = []
        for block in soup.select("li.b_algo"):
            link = block.select_one("h2 a[href], a.tilk[href]")
            if link is None:
                continue
            snippet_el = block.select_one(".b_caption p, .b_snippet, .b_algoSlug")
            results.append(
                SearchResult(
                    title=link.get_text(" ", strip=True),
                    url=link.get("href", "").strip(),
                    snippet=snippet_el.get_text(" ", strip=True) if snippet_el else "",
                    engine="bing_html",
                )
            )
        return results

    async def _run_search_mode(
        self,
        *,
        seeds: list[SearchResult],
        page_budget: int,
        pipeline: WebContentPipeline,
        errors: list[dict[str, str]],
    ) -> list[EnrichedResult]:
        """Fetch top search results and enrich them with readable content."""
        budgeted = seeds[:page_budget]
        semaphore = asyncio.Semaphore(max(self.search_config.concurrency, 1))

        async def enrich(rank: int, seed: SearchResult) -> EnrichedResult:
            async with semaphore:
                try:
                    document = await pipeline.fetch_document(seed.url, max_chars=12_000)
                    return EnrichedResult(
                        rank=rank,
                        title=document.title or seed.title,
                        url=seed.url,
                        final_url=document.final_url,
                        snippet=seed.snippet,
                        excerpt=document.snippet,
                        engine=seed.engine,
                        extractor=document.extractor,
                        fetch_method=document.fetch_method,
                        status_code=document.status_code,
                        markdown=document.markdown,
                        text=document.text,
                        errors=document.errors,
                    )
                except Exception as exc:
                    errors.append({"engine": seed.engine, "message": f"{seed.url}: {exc}"})
                    return EnrichedResult(
                        rank=rank,
                        title=seed.title,
                        url=seed.url,
                        final_url=seed.url,
                        snippet=seed.snippet,
                        excerpt=seed.snippet,
                        engine=seed.engine,
                        extractor=None,
                        fetch_method=None,
                        status_code=None,
                        markdown=seed.snippet,
                        text=seed.snippet,
                        errors=[str(exc)],
                    )

        enriched = await asyncio.gather(
            *(enrich(index, seed) for index, seed in enumerate(budgeted, 1))
        )
        return list(enriched)

    async def _run_deep_mode(
        self,
        *,
        query: str,
        seeds: list[SearchResult],
        page_budget: int,
        depth_budget: int,
        allowed_domains: list[str],
        pipeline: WebContentPipeline,
        errors: list[dict[str, str]],
    ) -> list[EnrichedResult]:
        """Perform a bounded deep crawl starting from the best seed URLs."""
        del query
        seed_cap = LOW_MEMORY_DEEP_MAX_SEEDS if self._memory_saver_deep else 2
        seed_count = min(len(seeds), seed_cap)
        remaining_pages = max(page_budget, seed_count)
        seen: set[str] = set()
        enriched: list[EnrichedResult] = []

        for seed_index, seed in enumerate(seeds[:seed_count], 1):
            seeds_left = seed_count - seed_index + 1
            per_seed_budget = max(1, remaining_pages // max(seeds_left, 1))
            try:
                documents = await pipeline.deep_crawl(
                    seed.url,
                    max_depth=depth_budget,
                    max_pages=per_seed_budget,
                    allowed_domains=allowed_domains,
                )
                for document in documents:
                    canonical = _canonicalize_url(document.final_url or document.url)
                    if not canonical or canonical in seen:
                        continue
                    seen.add(canonical)
                    enriched.append(
                        EnrichedResult(
                            rank=len(enriched) + 1,
                            title=document.title or seed.title,
                            url=seed.url,
                            final_url=document.final_url,
                            snippet=seed.snippet,
                            excerpt=document.snippet,
                            engine=seed.engine,
                            extractor=document.extractor,
                            fetch_method=document.fetch_method,
                            status_code=document.status_code,
                            markdown=document.markdown,
                            text=document.text,
                            errors=document.errors,
                        )
                    )
                    if len(enriched) >= page_budget:
                        break
                remaining_pages = max(remaining_pages - per_seed_budget, 0)
                if len(enriched) >= page_budget:
                    break
            except Exception as exc:
                errors.append({"engine": seed.engine, "message": f"deep crawl {seed.url}: {exc}"})
                logger.warning("WebSearch deep crawl seed={} failed: {}", seed.url, exc)
            finally:
                if self._memory_saver_deep:
                    await pipeline.release_browser()

        if not enriched:
            return await self._run_search_mode(
                seeds=seeds,
                page_budget=min(page_budget, len(seeds)),
                pipeline=pipeline,
                errors=errors,
            )
        return enriched[:page_budget]

    async def _build_summary(
        self,
        *,
        query: str,
        mode: str,
        results: list[EnrichedResult],
        use_llm: bool,
    ) -> str:
        """Build either an LLM-backed or extractive summary."""
        if not results:
            return f"No results found for: {query}"

        can_use_llm = bool(
            use_llm and self.provider is not None and (self.search_config.summary_model or "").strip()
        )
        if can_use_llm:
            try:
                return await self._summarize_with_llm(query=query, mode=mode, results=results[:5])
            except Exception as exc:
                logger.warning("WebSearch summarizer fallback for '{}': {}", query, exc)

        if mode == "deep":
            lines = [f"Deep search findings for '{query}':"]
        else:
            lines = [f"Search findings for '{query}':"]
        for idx, item in enumerate(results[:3], 1):
            fragment = item.excerpt or item.snippet or item.title
            lines.append(f"[{idx}] {item.title}: {fragment}")
        return " ".join(lines)

    async def _summarize_with_llm(
        self,
        *,
        query: str,
        mode: str,
        results: list[EnrichedResult],
    ) -> str:
        """Use the configured lightweight model to summarize fetched documents."""
        if self.provider is None:
            raise RuntimeError("No provider configured for search summarization")
        prompt_lines = [
            f"Task: Summarize web search findings for the query '{query}'.",
            f"Mode: {mode}",
            "Write 4-6 factual sentences.",
            "Use inline citation markers like [1], [2].",
            "If the sources disagree or are thin, say so.",
            "",
            "Sources:",
        ]
        for idx, item in enumerate(results, 1):
            prompt_lines.extend(
                [
                    f"[{idx}] {item.title}",
                    f"URL: {item.final_url}",
                    f"Snippet: {item.snippet or item.excerpt}",
                    f"Excerpt: {_truncate(item.text or item.markdown, 900)}",
                    "",
                ]
            )
        response = await self.provider.chat(
            messages=[
                {
                    "role": "system",
                    "content": (
                        "You are a concise web research summarizer. Synthesize the evidence faithfully "
                        "and keep the answer short."
                    ),
                },
                {"role": "user", "content": "\n".join(prompt_lines)},
            ],
            model=self.search_config.summary_model,
            max_tokens=500,
            temperature=0.2,
        )
        content = (response.content or "").strip()
        if not content:
            raise RuntimeError("Summary model returned empty content")
        return content

    @staticmethod
    def _serialize_result(item: EnrichedResult) -> dict[str, Any]:
        """Convert a fetched result into the tool's JSON payload format."""
        return {
            "rank": item.rank,
            "title": item.title,
            "url": item.url,
            "finalUrl": item.final_url,
            "engine": item.engine,
            "snippet": item.snippet,
            "excerpt": item.excerpt,
            "extractor": item.extractor,
            "fetchMethod": item.fetch_method,
            "status": item.status_code,
            "errors": item.errors,
        }

    @staticmethod
    def _serialize_source(idx: int, item: EnrichedResult) -> dict[str, Any]:
        """Convert a result into a compact source entry."""
        return {
            "id": idx,
            "title": item.title,
            "url": item.final_url or item.url,
            "engine": item.engine,
            "fetchMethod": item.fetch_method,
        }


class WebFetchTool(Tool):
    """Fetch and extract content from a URL using the shared web pipeline."""

    name = "web_fetch"
    description = (
        "Fetch a specific URL and extract readable content as markdown or plain text. "
        "Use this after search when you already know which page to inspect, or when you need page-level evidence instead of a broad search summary. "
        "The tool prefers cheap HTTP extraction first and only promotes to a browser-style crawl when the page looks too thin, blocked, or JavaScript-heavy."
    )
    parameters = {
        "type": "object",
        "properties": {
            "url": {"type": "string", "description": "URL to fetch"},
            "extractMode": {"type": "string", "enum": ["markdown", "text"], "default": "markdown"},
            "maxChars": {"type": "integer", "minimum": 100},
        },
        "required": ["url"],
    }

    def __init__(
        self,
        *,
        max_chars: int = 50_000,
        proxy: str | None = None,
        workspace: Path | None = None,
        search_config: WebSearchConfig | None = None,
    ) -> None:
        self.max_chars = max_chars
        self.proxy = proxy
        self._prefer_direct = _proxy_prefers_direct(proxy)
        self.workspace = workspace
        self.search_config = search_config.model_copy(deep=True) if search_config else WebSearchConfig()

    async def execute(
        self,
        url: str,
        extract_mode: str = "markdown",
        max_chars: int | None = None,
        **kwargs: Any,
    ) -> str:
        extract_mode = kwargs.get("extractMode", extract_mode)
        max_chars = kwargs.get("maxChars", max_chars) or self.max_chars
        is_valid, error_msg = _validate_url(url)
        if not is_valid:
            return json.dumps(
                {"error": f"URL validation failed: {error_msg}", "url": url},
                ensure_ascii=False,
            )

        try:
            async with WebContentPipeline(
                workspace=self.workspace,
                proxy=None if self._prefer_direct else self.proxy,
                search_config=self.search_config,
            ) as pipeline:
                document = await pipeline.fetch_document(url, max_chars=max_chars)
                self._prefer_direct = self._prefer_direct or pipeline.prefer_direct

            text = document.markdown if extract_mode == "markdown" else document.text
            truncated = len(text) > max_chars
            if truncated:
                text = _truncate(text, max_chars)

            return json.dumps(
                {
                    "url": url,
                    "finalUrl": document.final_url,
                    "status": document.status_code,
                    "extractor": document.extractor,
                    "fetchMethod": document.fetch_method,
                    "truncated": truncated,
                    "length": len(text),
                    "text": text,
                    "errors": document.errors,
                },
                ensure_ascii=False,
            )
        except httpx.ProxyError as exc:
            logger.error("WebFetch proxy error for {}: {}", url, exc)
            return json.dumps({"error": f"Proxy error: {exc}", "url": url}, ensure_ascii=False)
        except Exception as exc:
            logger.error("WebFetch error for {}: {}", url, exc)
            return json.dumps({"error": str(exc), "url": url}, ensure_ascii=False)
