import httpx
from bs4 import BeautifulSoup

async def scrape(url: str) -> str:
    """
    Scrape readable text content from a URL.
    Extracts p, h1-h3, li tags. Removes nav, footer, script, style.
    """
    async with httpx.AsyncClient(
        timeout=10.0,
        follow_redirects=True,
        headers={
            "User-Agent": "Mozilla/5.0 (compatible; TopoRAG/0.1; +https://github.com/yourusername/toporag)"
        },
    ) as client:
        response = await client.get(url)
        response.raise_for_status()

    soup = BeautifulSoup(response.text, "html.parser")

    # Remove unwanted elements
    for tag in soup.find_all(["nav", "footer", "script", "style", "noscript", "header"]):
        tag.decompose()

    # Extract text from content tags
    content_tags = soup.find_all(["p", "h1", "h2", "h3", "li", "article", "section"])
    paragraphs = []

    for tag in content_tags:
        text = tag.get_text(strip=True)
        if len(text) >= 20:  # Skip very short fragments
            paragraphs.append(text)

    if not paragraphs:
        # Fallback: get all visible text
        text = soup.get_text(separator="\n\n", strip=True)
        return text

    return "\n\n".join(paragraphs)
