# Familiar - Self-Hosted AI Companion Platform
# Copyright (c) 2026 George Scott Foley
# ORCID: 0009-0006-4957-0540
# Email: Georgescottfoley@proton.me
# Licensed under the MIT License - see LICENSE file for details

"""
Web Search Skill

Search the internet, retrieve web pages, extract content.
Enables grant research, organization lookup, regulatory checks.

Search backends (in order of preference):
  1. Self-hosted SearXNG (HIPAA-safe, configurable)
  2. DuckDuckGo HTML lite (no API key, queries leave network)

Dependencies:
  - beautifulsoup4 (HTML content extraction) -- optional but recommended
  - urllib (HTTP) -- stdlib, always available
"""

import json
import logging
import re
import urllib.error
import urllib.parse
import urllib.request
from html.parser import HTMLParser
from pathlib import Path

logger = logging.getLogger(__name__)

try:
    from bs4 import BeautifulSoup

    BS4_AVAILABLE = True
except ImportError:
    BS4_AVAILABLE = False
    logger.info("beautifulsoup4 not installed. HTML extraction will use basic parser.")

_CONFIG = {
    "backend": "duckduckgo",
    "searxng_url": "http://localhost:8888",
    "max_results": 8,
    "timeout": 15,
    "user_agent": "Familiar/2.6.3 (AI Agent; +https://familiar.ai)",
}


def _load_config():
    config_file = Path.home() / ".familiar" / "config.yaml"
    if config_file.exists():
        try:
            import yaml

            with open(config_file) as f:
                data = yaml.safe_load(f) or {}
            search_conf = data.get("websearch", {})
            _CONFIG.update({k: v for k, v in search_conf.items() if k in _CONFIG})
        except Exception as e:
            logger.debug("Error suppressed: %s", e)


def _http_get(url: str, timeout: int = None) -> str:
    # SSRF protection: block requests to private/internal networks
    from familiar.core.sanitization import check_ssrf

    is_safe, ssrf_msg = check_ssrf(url)
    if not is_safe:
        raise ValueError(f"Blocked by SSRF protection: {ssrf_msg}")

    timeout = timeout or _CONFIG["timeout"]
    req = urllib.request.Request(
        url,
        headers={
            "User-Agent": _CONFIG["user_agent"],
            "Accept": "text/html,application/xhtml+xml,application/json",
            "Accept-Language": "en-US,en;q=0.9",
        },
    )
    with urllib.request.urlopen(req, timeout=timeout) as resp:
        charset = resp.headers.get_content_charset() or "utf-8"
        return resp.read().decode(charset, errors="replace")


# --- DuckDuckGo HTML Lite Backend ---


class _DDGHTMLParser(HTMLParser):
    def __init__(self):
        super().__init__()
        self.results = []
        self._current = None
        self._in_result_link = False
        self._in_snippet = False
        self._snippet_tag = ""
        self._capture = ""

    def handle_starttag(self, tag, attrs):
        ad = dict(attrs)
        cls = ad.get("class", "")
        if tag == "a" and cls == "result__a":
            self._in_result_link = True
            href = ad.get("href", "")
            if "uddg=" in href:
                parsed = urllib.parse.parse_qs(urllib.parse.urlparse(href).query)
                href = parsed.get("uddg", [href])[0]
            self._current = {"title": "", "url": href, "snippet": ""}
            self._capture = ""
        elif cls == "result__snippet":
            # DDG uses <a> or <td> for snippets depending on version
            self._in_snippet = True
            self._snippet_tag = tag
            self._capture = ""

    def handle_endtag(self, tag):
        if tag == "a" and self._in_result_link and not self._in_snippet:
            self._in_result_link = False
            if self._current:
                self._current["title"] = self._capture.strip()
            self._capture = ""
        elif self._in_snippet and tag == self._snippet_tag:
            self._in_snippet = False
            self._in_result_link = False
            self._snippet_tag = ""
            if self._current:
                self._current["snippet"] = self._capture.strip()
                self.results.append(self._current)
                self._current = None
            self._capture = ""

    def handle_data(self, data):
        if self._in_result_link or self._in_snippet:
            self._capture += data


def _search_duckduckgo(query: str, max_results: int = 8) -> list:
    encoded = urllib.parse.urlencode({"q": query})
    url = f"https://html.duckduckgo.com/html/?{encoded}"
    try:
        html = _http_get(url)
    except Exception as e:
        return [{"title": "Search Error", "url": "", "snippet": str(e)}]
    parser = _DDGHTMLParser()
    parser.feed(html)
    return parser.results[:max_results]


# --- SearXNG Backend ---


def _search_searxng(query: str, max_results: int = 8) -> list:
    base_url = _CONFIG["searxng_url"].rstrip("/")
    params = urllib.parse.urlencode(
        {
            "q": query,
            "format": "json",
            "engines": "google,duckduckgo,brave",
        }
    )
    url = f"{base_url}/search?{params}"
    try:
        body = _http_get(url)
        data = json.loads(body)
        return [
            {"title": r.get("title", ""), "url": r.get("url", ""), "snippet": r.get("content", "")}
            for r in data.get("results", [])[:max_results]
        ]
    except Exception as e:
        logger.warning(f"SearXNG search failed: {e}, falling back to DuckDuckGo")
        return _search_duckduckgo(query, max_results)


# --- Content Extraction ---


def _extract_readable_text(html: str, max_chars: int = 8000) -> str:
    if BS4_AVAILABLE:
        soup = BeautifulSoup(html, "html.parser")
        for tag in soup.find_all(
            ["script", "style", "nav", "footer", "header", "aside", "iframe", "noscript", "form"]
        ):
            tag.decompose()
        article = soup.find("article") or soup.find("main") or soup.find(role="main")
        if article:
            text = article.get_text(separator="\n", strip=True)
        else:
            body = soup.find("body")
            text = (
                body.get_text(separator="\n", strip=True)
                if body
                else soup.get_text(separator="\n", strip=True)
            )
        lines = [line.strip() for line in text.split("\n") if line.strip()]
        return "\n".join(lines)[:max_chars]

    # Fallback: regex
    text = re.sub(r"<script[^>]*>.*?</script>", "", html, flags=re.DOTALL | re.IGNORECASE)
    text = re.sub(r"<style[^>]*>.*?</style>", "", html, flags=re.DOTALL | re.IGNORECASE)
    text = re.sub(r"<[^>]+>", " ", text)
    text = (
        text.replace("&amp;", "&").replace("&lt;", "<").replace("&gt;", ">").replace("&nbsp;", " ")
    )
    text = re.sub(r"\s+", " ", text).strip()
    return text[:max_chars]


# === Tool Handlers ===


def web_search(data: dict) -> str:
    """Search the web and return top results with titles, snippets, URLs."""
    _load_config()
    query = data.get("query", "").strip()
    if not query:
        return "Please provide a search query."

    max_results = min(data.get("max_results", _CONFIG["max_results"]), 15)

    if _CONFIG["backend"] == "searxng":
        results = _search_searxng(query, max_results)
    else:
        results = _search_duckduckgo(query, max_results)

    if not results:
        return f"No results found for: {query}"

    lines = [f"🔍 Search: {query}\n"]
    for i, r in enumerate(results, 1):
        lines.append(f"  {i}. {r['title']}")
        if r.get("url"):
            lines.append(f"     {r['url']}")
        if r.get("snippet"):
            lines.append(f"     {r['snippet'][:200]}")
        lines.append("")
    return "\n".join(lines)


def web_read(data: dict) -> str:
    """Fetch a URL and extract readable text content."""
    url = data.get("url", "").strip()
    if not url:
        return "Please provide a URL to read."
    if not url.startswith(("http://", "https://")):
        url = "https://" + url

    try:
        html = _http_get(url)
    except Exception as e:
        logger.warning(f"Fetch failed for {url}: {e}")
        return f"Failed to fetch {url}: connection error"

    text = _extract_readable_text(html)
    if not text or len(text) < 50:
        return f"Could not extract meaningful content from {url}"

    max_len = int(data.get("max_chars", 6000))
    if len(text) > max_len:
        text = text[:max_len] + "\n\n[Content truncated]"
    return f"📄 Content from {url}:\n\n{text}"


def web_summarize(data: dict) -> str:
    """Fetch a URL and return extracted text for LLM summarization."""
    url = data.get("url", "").strip()
    if not url:
        return "Please provide a URL to summarize."
    if not url.startswith(("http://", "https://")):
        url = "https://" + url

    try:
        html = _http_get(url)
    except Exception as e:
        logger.warning(f"Fetch failed for {url}: {e}")
        return f"Failed to fetch {url}: connection error"

    text = _extract_readable_text(html, max_chars=10000)
    if not text or len(text) < 50:
        return f"Could not extract meaningful content from {url}"

    return f"📄 Extracted content from {url} ({len(text)} chars). Please summarize:\n\n{text}"


def search_grants(data: dict) -> str:
    """Search for grant opportunities using web search."""
    _load_config()
    query = data.get("query", "").strip()
    if not query:
        return "Please provide a grant search query."

    eligibility = data.get("eligibility", "nonprofit")
    web_query = f"grants {query} {eligibility} site:grants.gov OR site:foundationcenter.org"
    web_results = _search_duckduckgo(web_query, max_results=8)

    lines = [f"🔍 Grant Search: {query}\n"]
    if web_results:
        for i, r in enumerate(web_results, 1):
            lines.append(f"  {i}. {r['title']}")
            if r.get("url"):
                lines.append(f"     {r['url']}")
            if r.get("snippet"):
                lines.append(f"     {r['snippet'][:200]}")
            lines.append("")
    else:
        lines.append("  No grant results found. Try broadening your search terms.")
    lines.append("  💡 Tip: Also check foundationcenter.org and your state's grant portal.")
    return "\n".join(lines)


def instant_answer(data: dict) -> str:
    """Get an instant answer from DuckDuckGo's Instant Answer API."""
    query = data.get("query", "").strip()
    if not query:
        return "Please provide a query."

    encoded = urllib.parse.urlencode(
        {"q": query, "format": "json", "no_html": "1", "skip_disambig": "1"}
    )
    url = f"https://api.duckduckgo.com/?{encoded}"

    try:
        body = _http_get(url)
        data = json.loads(body)
    except Exception as e:
        logger.warning("DDG Instant Answer failed: %s", e)
        return f"Could not get instant answer for '{query}'."

    lines = []

    # Abstract (Wikipedia-style summary)
    abstract = data.get("AbstractText", "")
    abstract_src = data.get("AbstractSource", "")
    abstract_url = data.get("AbstractURL", "")
    heading = data.get("Heading", "")

    if heading:
        lines.append(f"**{heading}**")
        lines.append("")

    if abstract:
        lines.append(abstract[:800])
        if abstract_src:
            lines.append(f"_Source: {abstract_src}_")
        if abstract_url:
            lines.append(f"Read more: {abstract_url}")
        lines.append("")

    # Answer (direct answer like calculations, conversions)
    answer = data.get("Answer", "")
    if answer:
        lines.append(f"**Answer:** {answer}")
        lines.append("")

    # Definition
    definition = data.get("Definition", "")
    if definition:
        def_source = data.get("DefinitionSource", "")
        lines.append(f"**Definition:** {definition}")
        if def_source:
            lines.append(f"_Source: {def_source}_")
        lines.append("")

    # Infobox
    infobox = data.get("Infobox", {})
    if infobox and isinstance(infobox, dict):
        content = infobox.get("content", [])
        if content:
            lines.append("**Quick Facts:**")
            for item in content[:10]:
                label = item.get("label", "")
                value = item.get("value", "")
                if label and value:
                    lines.append(f"  - {label}: {value}")
            lines.append("")

    # Related topics
    related = data.get("RelatedTopics", [])
    if related:
        lines.append("**Related:**")
        for topic in related[:8]:
            if isinstance(topic, dict):
                text = topic.get("Text", "")
                first_url = topic.get("FirstURL", "")
                if text:
                    lines.append(f"  - {text[:200]}")
                    if first_url:
                        lines.append(f"    {first_url}")
                # Handle subtopics (grouped results)
                topics = topic.get("Topics", [])
                if topics:
                    name = topic.get("Name", "")
                    if name:
                        lines.append(f"  **{name}:**")
                    for sub in topics[:3]:
                        sub_text = sub.get("Text", "")
                        if sub_text:
                            lines.append(f"    - {sub_text[:150]}")
        lines.append("")

    if not lines:
        return (
            f"No instant answer available for '{query}'. Try using web_search for broader results."
        )

    return "\n".join(lines)


def searxng_search(data: dict) -> str:
    """Search via SearXNG with category support (images, news, files, etc.)."""
    _load_config()
    if _CONFIG["backend"] != "searxng":
        return (
            "SearXNG is not configured as the search backend.\n"
            "Set backend to 'searxng' in config.yaml and provide SEARXNG_URL."
        )

    query = data.get("query", "").strip()
    if not query:
        return "Please provide a search query."

    category = data.get("category", "general").lower()
    max_results = min(int(data.get("max_results", 10)), 20)

    valid_categories = [
        "general",
        "images",
        "news",
        "videos",
        "files",
        "music",
        "it",
        "science",
        "social_media",
    ]
    if category not in valid_categories:
        return f"Invalid category '{category}'. Valid: {', '.join(valid_categories)}"

    base_url = _CONFIG["searxng_url"].rstrip("/")
    params = urllib.parse.urlencode(
        {
            "q": query,
            "format": "json",
            "categories": category,
        }
    )
    url = f"{base_url}/search?{params}"

    try:
        body = _http_get(url)
        data = json.loads(body)
        results = data.get("results", [])[:max_results]
    except Exception as e:
        logger.warning("SearXNG category search failed: %s", e)
        return f"SearXNG search failed: {e}"

    if not results:
        return f"No {category} results for '{query}'."

    lines = [f"**SearXNG {category.title()}: {query}**", ""]

    for i, r in enumerate(results, 1):
        title = r.get("title", "Untitled")
        result_url = r.get("url", "")
        content = r.get("content", "")

        lines.append(f"{i}. **{title}**")
        if result_url:
            lines.append(f"   {result_url}")

        # Category-specific fields
        if category == "images":
            img_src = r.get("img_src", "")
            resolution = r.get("resolution", "")
            if img_src:
                lines.append(f"   Image: {img_src}")
            if resolution:
                lines.append(f"   Resolution: {resolution}")
        elif category == "news":
            published = r.get("publishedDate", "")
            if published:
                lines.append(f"   Published: {published[:16]}")
        elif category == "videos":
            length = r.get("length", "")
            if length:
                lines.append(f"   Duration: {length}")
        elif category == "files":
            filesize = r.get("filesize", "")
            filetype = r.get("filetype", "")
            if filetype:
                lines.append(f"   Type: {filetype}")
            if filesize:
                lines.append(f"   Size: {filesize}")

        if content:
            lines.append(f"   {content[:200]}")
        lines.append("")

    return "\n".join(lines)


def search_nonprofit(data: dict) -> str:
    """Look up an organization by name or EIN on ProPublica Nonprofit Explorer API."""
    query = data.get("query", "").strip()
    ein = data.get("ein", "").strip().replace("-", "")

    if not query and not ein:
        return "Please provide an organization name or EIN."

    results = []

    if ein:
        url = f"https://projects.propublica.org/nonprofits/api/v2/organizations/{ein}.json"
        try:
            body = _http_get(url)
            resp = json.loads(body)
            org = resp.get("organization", {})
            if org:
                results.append(
                    {
                        "name": org.get("name", ""),
                        "ein": org.get("ein", ""),
                        "city": org.get("city", ""),
                        "state": org.get("state", ""),
                        "ntee_code": org.get("ntee_code", ""),
                        "income": org.get("income_amount"),
                        "revenue": org.get("revenue_amount"),
                    }
                )
        except Exception as e:
            logger.warning(f"ProPublica EIN lookup failed: {e}")

    if query and not results:
        encoded = urllib.parse.urlencode({"q": query})
        url = f"https://projects.propublica.org/nonprofits/api/v2/search.json?{encoded}"
        try:
            body = _http_get(url)
            resp = json.loads(body)
            for org in resp.get("organizations", [])[:8]:
                results.append(
                    {
                        "name": org.get("name", ""),
                        "ein": org.get("ein", ""),
                        "city": org.get("city", ""),
                        "state": org.get("state", ""),
                        "ntee_code": org.get("ntee_code", ""),
                        "income": org.get("income_amount"),
                        "revenue": org.get("revenue_amount"),
                    }
                )
        except Exception as e:
            logger.warning(f"ProPublica search failed: {e}")
            web_results = _search_duckduckgo(
                f"nonprofit {query} site:guidestar.org OR site:propublica.org"
            )
            lines = [
                f"🔍 Nonprofit Search: {query}\n",
                "  (ProPublica API unavailable, web results):\n",
            ]
            for i, r in enumerate(web_results[:5], 1):
                lines.append(f"  {i}. {r['title']}")
                if r.get("url"):
                    lines.append(f"     {r['url']}")
            return "\n".join(lines)

    if not results:
        return f"No nonprofit organizations found matching: {query or ein}"

    lines = ["🏛️ Nonprofit Search Results:\n"]
    for org in results:
        lines.append(f"  {org['name']}")
        lines.append(f"    EIN: {org['ein']}")
        location = ", ".join(filter(None, [org.get("city"), org.get("state")]))
        if location:
            lines.append(f"    Location: {location}")
        if org.get("ntee_code"):
            lines.append(f"    NTEE Code: {org['ntee_code']}")
        if org.get("income"):
            lines.append(f"    Income: ${org['income']:,.0f}")
        if org.get("revenue"):
            lines.append(f"    Revenue: ${org['revenue']:,.0f}")
        lines.append("")
    return "\n".join(lines)


# === Tool Definitions ===

TOOLS = [
    {
        "name": "web_search",
        "description": (
            "Search the web for information on any topic using DuckDuckGo or self-hosted SearXNG. "
            "This is the PRIMARY tool for web searches -- always use this instead of browser tools "
            "when the user asks to search, look up, or find information online. "
            "Returns top results with titles, snippets, and URLs."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "query": {"type": "string", "description": "Search query"},
                "max_results": {
                    "type": "integer",
                    "default": 8,
                    "description": "Max results (up to 15)",
                },
            },
            "required": ["query"],
        },
        "handler": web_search,
        "category": "websearch",
    },
    {
        "name": "web_read",
        "description": (
            "Fetch a URL and extract readable text content using article extraction. "
            "Use this to read the text of a specific web page after finding it via web_search. "
            "Strips scripts, navigation, and boilerplate; returns cleaned text truncated to max_chars."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "url": {"type": "string", "description": "URL to fetch and read"},
                "max_chars": {
                    "type": "integer",
                    "default": 6000,
                    "description": "Max characters to return",
                },
            },
            "required": ["url"],
        },
        "handler": web_read,
        "category": "websearch",
    },
    {
        "name": "web_summarize",
        "description": (
            "Fetch a URL and return the extracted text for the LLM to summarize in its response. "
            "Use this when the user wants a summary of a web page rather than the full text. "
            "Extracts up to 10,000 characters of readable content from the page."
        ),
        "input_schema": {
            "type": "object",
            "properties": {"url": {"type": "string", "description": "URL to fetch and summarize"}},
            "required": ["url"],
        },
        "handler": web_summarize,
        "category": "websearch",
    },
    {
        "name": "search_grants",
        "description": (
            "Search for grant opportunities using grants.gov and foundation directory websites. "
            "Use this when a nonprofit user needs to find funding sources or grant programs. "
            "Queries are scoped to grants.gov and foundationcenter.org for relevant results."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "query": {
                    "type": "string",
                    "description": "Grant search terms (topic, program area)",
                },
                "eligibility": {
                    "type": "string",
                    "default": "nonprofit",
                    "description": "Eligibility type",
                },
            },
            "required": ["query"],
        },
        "handler": search_grants,
        "category": "websearch",
    },
    {
        "name": "instant_answer",
        "description": (
            "Get an instant answer from DuckDuckGo for factual queries. "
            "Returns structured data: summaries, definitions, infoboxes, and related topics. "
            "Best for quick facts, definitions, and entity lookups. No API key required."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "query": {"type": "string", "description": "Factual query or entity name"},
            },
            "required": ["query"],
        },
        "handler": instant_answer,
        "category": "websearch",
    },
    {
        "name": "searxng_search",
        "description": (
            "Search via self-hosted SearXNG with category support. "
            "Categories: general, images, news, videos, files, music, it, science, social_media. "
            "Only available when SearXNG is configured as the backend. HIPAA-safe — queries stay on-premises."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "query": {"type": "string", "description": "Search query"},
                "category": {
                    "type": "string",
                    "description": "Search category (default: general)",
                    "default": "general",
                },
                "max_results": {
                    "type": "integer",
                    "default": 10,
                    "description": "Max results (up to 20)",
                },
            },
            "required": ["query"],
        },
        "handler": searxng_search,
        "category": "websearch",
    },
    {
        "name": "search_nonprofit",
        "description": (
            "Look up a nonprofit organization by name or EIN using the ProPublica Nonprofit Explorer API. "
            "Use this to find IRS data on nonprofits including revenue, income, and NTEE classification. "
            "Falls back to web search results if the ProPublica API is unavailable."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "query": {"type": "string", "description": "Organization name to search"},
                "ein": {"type": "string", "description": "EIN (e.g. 12-3456789 or 123456789)"},
            },
        },
        "handler": search_nonprofit,
        "category": "websearch",
    },
]
