#!/usr/bin/env python3
"""scrape python docs for standard library external links.

with all my heart, 2025-2026, mark joshwel <mark@joshwel.co>
SPDX-License-Identifier: Unlicense OR 0BSD

this script scrapes docs.python.org to build a mapping of standard library
modules and their types to their documentation URLs. the output can be used
to populate meadow's external links configuration.

usage:
    uv run scripts/scrape_stdlib_links.py > stdlib_links.py

    # or with specific python version:
    uv run scripts/scrape_stdlib_links.py --version 3.13 > stdlib_links.py

output format:
    generates a python module with STDLIB_LINKS dictionary:

    STDLIB_LINKS = {
        "pathlib.Path": "https://docs.python.org/3/library/pathlib.html#pathlib.Path",
        "datetime.datetime": "https://docs.python.org/3/library/datetime.html#datetime.datetime",
        # ...
    }
"""

import argparse
import sys
from urllib.parse import urljoin

import requests
from bs4 import BeautifulSoup

BASE_URL = "https://docs.python.org/{version}/library/"

# modules to skip (internal, deprecated, or not relevant)
SKIP_MODULES: frozenset[str] = frozenset(
    [
        "__future__",
        "__main__",
        "__builtin__",
        "builtins",
        "test",
        "tests",
        "idlelib",
    ]
)


def get_module_list(version: str) -> dict[str, str]:
    """get list of standard library modules from the library index.

    arguments:
        `version: str`
            python version (e.g., "3.13")

    returns: `dict[str, str]`
        mapping of module name to documentation URL
    """
    url = BASE_URL.format(version=version)
    response = requests.get(url, timeout=30)
    response.raise_for_status()

    soup = BeautifulSoup(response.text, "html.parser")
    modules: dict[str, str] = {}

    for link in soup.find_all("a", class_="reference internal"):
        href_attr = link.get("href")

        # ensure we have a string, not AttributeValueList or None
        if not isinstance(href_attr, str):
            continue

        href = href_attr

        # module links look like "module_name.html" or "module.name.html"
        if not href.endswith(".html"):
            continue
        if href.startswith(("http", "#", "../")):
            continue

        module_name = href.replace(".html", "").split("#")[0]

        # skip internal/deprecated modules
        if module_name in SKIP_MODULES or module_name.startswith(("test.", "_")):
            continue

        full_url = urljoin(url, href)
        modules[module_name] = full_url

    return modules


def scrape_module_types(module_name: str, module_url: str) -> dict[str, str]:
    """scrape a module page for classes, functions, and exceptions.

    arguments:
        `module_name: str`
            name of the module
        `module_url: str`
            URL to the module documentation

    returns: `dict[str, str]`
        mapping of fully qualified name to anchor URL
    """
    links: dict[str, str] = {}

    try:
        response = requests.get(module_url, timeout=30)
        response.raise_for_status()
    except requests.RequestException:
        return links

    soup = BeautifulSoup(response.text, "html.parser")

    # look for section headers with function/class definitions
    # python docs use dt elements with ids for definitions
    for dt in soup.find_all("dt"):
        dt_tag = dt
        anchor_id_attr = dt_tag.get("id")

        # ensure we have a string id
        if not isinstance(anchor_id_attr, str):
            continue

        anchor_id = anchor_id_attr
        if not anchor_id:
            continue

        # check if this looks like a class/function definition
        # format is usually "module.ClassName" or "module.function_name"
        if not anchor_id.startswith(module_name + "."):
            continue

        # clean up the id - sometimes has extra info
        parts = anchor_id.split(".")
        if len(parts) < 2:
            continue

        clean_id = parts[-1].split("-")[0]
        if not clean_id:
            continue
        if clean_id.startswith("_") or clean_id.startswith("test_"):
            continue

        full_name = f"{module_name}.{clean_id}"
        anchor_url = f"{module_url}#{anchor_id}"
        links[full_name] = anchor_url

    return links


def generate_links_code(links: dict[str, str]) -> str:
    """generate python code with the links dictionary.

    arguments:
        `links: dict[str, str]`
            mapping of names to URLs

    returns: `str`
        formatted python code
    """
    lines: list[str] = [
        '"""standard library external links for meadow.',
        "",
        "this module contains documentation URLs for standard library modules,",
        "classes, and functions. auto-generated by scripts/scrape_stdlib_links.py.",
        '"""',
        "",
        "# fmt: off",
        "STDLIB_LINKS: dict[str, str] = {",
    ]

    # sort by key for consistent output
    for name in sorted(links.keys()):
        url = links[name]
        lines.append(f'    "{name}": "{url}",')

    lines.extend(
        [
            "}",
            "# fmt: on",
            "",
            "",
            "def get_stdlib_link(name: str) -> str | None:",
            '    """get documentation URL for a standard library name.',
            "",
            "    arguments:",
            "        name: fully qualified name (e.g., 'pathlib.Path')",
            "",
            "    returns: documentation URL or None if not found",
            '    """',
            "    return STDLIB_LINKS.get(name)",
            "",
        ]
    )

    return "\n".join(lines)


def main() -> int:
    """main entry point

    returns: `int`
        exit code (0 for success)
    """
    parser = argparse.ArgumentParser(
        description="scrape python docs for standard library links",
    )
    _ = parser.add_argument(
        "--version",
        default="3.13",
        help="python version to scrape (default: 3.13)",
    )
    _ = parser.add_argument(
        "-o",
        "--output",
        help="output file (default: stdout)",
    )
    _ = parser.add_argument(
        "--modules-only",
        action="store_true",
        help="only scrape module-level links, not individual classes/functions",
    )

    args = parser.parse_args()

    version: str = args.version  # pyright: ignore[reportAny]
    output: str | None = args.output  # pyright: ignore[reportAny]
    modules_only: bool = args.modules_only  # pyright: ignore[reportAny]

    print(f"scraping python {version} documentation...", file=sys.stderr)

    # get module list
    modules = get_module_list(version)
    print(f"found {len(modules)} modules", file=sys.stderr)

    all_links: dict[str, str] = {}

    # add module-level links
    for module_name, module_url in modules.items():
        all_links[module_name] = module_url

        if not modules_only:
            # scrape individual types from module pages
            types = scrape_module_types(module_name, module_url)
            all_links.update(types)

    print(f"total links: {len(all_links)}", file=sys.stderr)

    # generate output
    code = generate_links_code(all_links)

    if output:
        with open(output, "w", encoding="utf-8") as f:
            _ = f.write(code)
        print(f"written to {output}", file=sys.stderr)
    else:
        print(code)

    return 0


if __name__ == "__main__":
    sys.exit(main())
