from urllib.parse import urljoin

import pandas as pd
import requests
from bs4 import BeautifulSoup

BASE_URL = "https://www.alignmentforum.org"
START_URL = "https://www.alignmentforum.org/w/ai-risk-concrete-stories"

data: list[dict[str, str]] = []


def parse() -> None:
    res = requests.get(START_URL)
    soup = BeautifulSoup(res.text, "html.parser")

    # Replace with your class name from the real structure
    links = soup.select('span[class*="PostsTitle-eaTitleDesktopEllipsis"] a')

    for a_tag in links:
        href = a_tag.get("href", None)
        if href is not None and isinstance(href, str):
            full_url = urljoin(BASE_URL, href)
            print(full_url)
            parse_link(full_url)

    df = pd.DataFrame(data)
    df.to_json("./output.json", orient="records", indent=2)


def parse_link(url: str) -> None:
    res = requests.get(url)
    soup = BeautifulSoup(res.text, "html.parser")
    # Example logic; you'll need to adjust for rendered content
    for p in soup.find_all("p"):
        data.append({"link": url, "text": p.text})


if __name__ == "__main__":
    parse()
