import httpx
from bs4 import BeautifulSoup
from ..models import DocumentContent, TextSegment


async def parse_web(url: str) -> tuple[str, DocumentContent]:
    async with httpx.AsyncClient(follow_redirects=True, timeout=30.0) as client:
        response = await client.get(url)
    response.raise_for_status()

    soup = BeautifulSoup(response.text, "html.parser")

    title_tag = soup.find("title")
    title = title_tag.get_text().strip() if title_tag else url

    for tag in soup(["script", "style", "noscript", "nav", "footer", "header"]):
        tag.decompose()

    text_parts: list[str] = []
    for el in soup.find_all(["p", "h1", "h2", "h3", "h4", "h5", "h6", "li", "td"]):
        text = el.get_text().strip()
        if text:
            text_parts.append(text)

    segments: list[TextSegment] = []
    raw_text = ""
    for text in text_parts:
        segments.append(TextSegment(text=text, charOffset=len(raw_text)))
        raw_text += text + "\n\n"

    return title, DocumentContent(rawText=raw_text.rstrip(), segments=segments)
