from __future__ import annotations

import argparse
import csv
import json
import os
import re
import sys
import threading
import time
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
from dataclasses import dataclass
from html.parser import HTMLParser
from pathlib import Path
from typing import Iterable
from urllib.error import HTTPError, URLError
from urllib.parse import urlparse
from urllib.request import Request, urlopen

PYPI_SIMPLE_URL = "https://pypi.org/simple/"
PYPI_JSON_URL = "https://pypi.org/pypi/{name}/json"
GITHUB_REPO_API_URL = "https://api.github.com/repos/{owner}/{repo}"
PYPI_SIMPLE_JSON_ACCEPT = "application/vnd.pypi.simple.v1+json"

CSV_HEADERS = [
    "package_name",
    "pypi_url",
    "github_repo",
    "github_url",
    "stars",
    "forks",
    "open_issues",
    "archived",
    "disabled",
    "source_url",
    "project_url",
    "home_page",
    "summary",
    "error",
]

GITHUB_REPO_RE = re.compile(r"^/([^/]+)/([^/]+?)(?:\.git)?/?$")


class _SimpleIndexHTMLParser(HTMLParser):
    def __init__(self) -> None:
        super().__init__()
        self.projects: list[str] = []

    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
        if tag != "a":
            return
        href = dict(attrs).get("href")
        if not href:
            return
        path = urlparse(href).path.strip("/")
        if path:
            self.projects.append(path)


@dataclass
class PackageRecord:
    package_name: str
    pypi_url: str
    github_repo: str = ""
    github_url: str = ""
    stars: str = ""
    forks: str = ""
    open_issues: str = ""
    archived: str = ""
    disabled: str = ""
    source_url: str = ""
    project_url: str = ""
    home_page: str = ""
    summary: str = ""
    error: str = ""

    def as_csv_row(self) -> dict[str, str]:
        return {
            "package_name": self.package_name,
            "pypi_url": self.pypi_url,
            "github_repo": self.github_repo,
            "github_url": self.github_url,
            "stars": self.stars,
            "forks": self.forks,
            "open_issues": self.open_issues,
            "archived": self.archived,
            "disabled": self.disabled,
            "source_url": self.source_url,
            "project_url": self.project_url,
            "home_page": self.home_page,
            "summary": self.summary,
            "error": self.error,
        }


def _json_request(url: str, *, headers: dict[str, str] | None = None, timeout: int = 20) -> dict:
    request_headers = {"User-Agent": "soigia-pypi-catalog/0.1"}
    if headers:
        request_headers.update(headers)
    request = Request(url, headers=request_headers)
    with urlopen(request, timeout=timeout) as response:
        return json.load(response)


def _text_request(url: str, *, headers: dict[str, str] | None = None, timeout: int = 20) -> str:
    request_headers = {"User-Agent": "soigia-pypi-catalog/0.1"}
    if headers:
        request_headers.update(headers)
    request = Request(url, headers=request_headers)
    with urlopen(request, timeout=timeout) as response:
        return response.read().decode("utf-8", errors="replace")


def fetch_pypi_project_names(index_url: str = PYPI_SIMPLE_URL) -> list[str]:
    try:
        payload = _json_request(index_url, headers={"Accept": PYPI_SIMPLE_JSON_ACCEPT})
        projects = payload.get("projects", [])
        names = [project["name"] for project in projects if project.get("name")]
        if names:
            return names
    except Exception:
        pass

    parser = _SimpleIndexHTMLParser()
    parser.feed(_text_request(index_url))
    return parser.projects


def _candidate_urls(info: dict) -> Iterable[tuple[str, str]]:
    project_urls = info.get("project_urls") or {}
    for label, url in project_urls.items():
        if url:
            yield label.lower(), url

    for label in ("project_url", "home_page", "package_url", "download_url"):
        value = info.get(label)
        if value:
            yield label.lower(), value


def _normalize_github_repo(url: str) -> str | None:
    parsed = urlparse(url)
    if parsed.netloc.lower() not in {"github.com", "www.github.com"}:
        return None

    path = parsed.path.rstrip("/")
    match = GITHUB_REPO_RE.match(path)
    if not match:
        return None

    owner, repo = match.groups()
    if not owner or not repo:
        return None

    lowered_repo = repo.lower()
    if lowered_repo in {"issues", "pulls", "actions", "wiki", "releases", "tags", "commits"}:
        return None

    return f"{owner}/{repo}"


def find_github_repo(info: dict) -> tuple[str | None, dict[str, str]]:
    selected: dict[str, str] = {"source_url": "", "project_url": "", "home_page": ""}
    ranked_candidates: list[tuple[int, str, str]] = []

    for label, url in _candidate_urls(info):
        if label == "home_page":
            selected["home_page"] = selected["home_page"] or url
        if label == "project_url":
            selected["project_url"] = selected["project_url"] or url

        repo = _normalize_github_repo(url)
        if not repo:
            continue

        priority = 50
        if "source" in label or "code" in label or "github" in label:
            priority = 0
            selected["source_url"] = selected["source_url"] or url
        elif "home" in label:
            priority = 20
        elif "project" in label:
            priority = 30

        ranked_candidates.append((priority, repo.lower(), repo))

    if not ranked_candidates:
        return None, selected

    ranked_candidates.sort()
    return ranked_candidates[0][2], selected


def fetch_package_record(name: str, github_token: str | None = None) -> PackageRecord:
    pypi_url = f"https://pypi.org/project/{name}/"
    record = PackageRecord(package_name=name, pypi_url=pypi_url)

    try:
        payload = _json_request(PYPI_JSON_URL.format(name=name))
    except HTTPError as exc:
        record.error = f"pypi_http_{exc.code}"
        return record
    except URLError as exc:
        record.error = f"pypi_url_{exc.reason}"
        return record
    except Exception as exc:
        record.error = f"pypi_error_{type(exc).__name__}"
        return record

    info = payload.get("info") or {}
    record.summary = info.get("summary") or ""

    repo, selected_urls = find_github_repo(info)
    record.source_url = selected_urls["source_url"]
    record.project_url = selected_urls["project_url"]
    record.home_page = selected_urls["home_page"]

    if not repo:
        return record

    record.github_repo = repo
    record.github_url = f"https://github.com/{repo}"

    headers: dict[str, str] = {}
    if github_token:
        headers["Authorization"] = f"Bearer {github_token}"
    try:
        repo_payload = _json_request(
            GITHUB_REPO_API_URL.format(owner=repo.split("/", 1)[0], repo=repo.split("/", 1)[1]),
            headers=headers,
        )
        record.stars = str(repo_payload.get("stargazers_count", ""))
        record.forks = str(repo_payload.get("forks_count", ""))
        record.open_issues = str(repo_payload.get("open_issues_count", ""))
        record.archived = str(repo_payload.get("archived", ""))
        record.disabled = str(repo_payload.get("disabled", ""))
    except HTTPError as exc:
        if exc.code == 403:
            record.error = "github_rate_limited_or_forbidden"
        elif exc.code == 404:
            record.error = "github_repo_not_found"
        else:
            record.error = f"github_http_{exc.code}"
    except URLError as exc:
        record.error = f"github_url_{exc.reason}"
    except Exception as exc:
        record.error = f"github_error_{type(exc).__name__}"

    return record


def _load_existing_package_names(output_path: Path) -> set[str]:
    if not output_path.exists():
        return set()

    package_names: set[str] = set()
    with output_path.open("r", encoding="utf-8", newline="") as handle:
        reader = csv.DictReader(handle)
        for row in reader:
            package_name = (row.get("package_name") or "").strip()
            if package_name:
                package_names.add(package_name)
    return package_names


def _write_csv_header_if_needed(output_path: Path) -> None:
    if output_path.exists() and output_path.stat().st_size > 0:
        return
    with output_path.open("w", encoding="utf-8", newline="") as handle:
        writer = csv.DictWriter(handle, fieldnames=CSV_HEADERS)
        writer.writeheader()


def export_catalog(
    output_path: Path,
    *,
    limit: int | None,
    offset: int,
    workers: int,
    github_token: str | None,
    sleep_seconds: float,
    resume: bool,
) -> dict[str, int]:
    project_names = fetch_pypi_project_names()
    selected_names = project_names[offset:]
    if limit is not None:
        selected_names = selected_names[:limit]

    existing_package_names = _load_existing_package_names(output_path) if resume else set()
    pending_names = [name for name in selected_names if name not in existing_package_names]

    _write_csv_header_if_needed(output_path)

    totals = {
        "pypi_projects_seen": len(selected_names),
        "rows_written": 0,
        "github_rows": 0,
        "skipped_existing": len(existing_package_names.intersection(selected_names)),
    }

    lock = threading.Lock()

    def write_record(record: PackageRecord) -> None:
        with lock:
            with output_path.open("a", encoding="utf-8", newline="") as handle:
                writer = csv.DictWriter(handle, fieldnames=CSV_HEADERS)
                writer.writerow(record.as_csv_row())
            totals["rows_written"] += 1
            if record.github_repo:
                totals["github_rows"] += 1

    with ThreadPoolExecutor(max_workers=max(1, workers)) as executor:
        futures = {executor.submit(fetch_package_record, name, github_token): name for name in pending_names[:workers]}
        index = len(futures)

        while futures:
            done, _ = wait(futures, return_when=FIRST_COMPLETED)
            for future in done:
                futures.pop(future)
                record = future.result()
                write_record(record)
                if sleep_seconds > 0:
                    time.sleep(sleep_seconds)
                if index < len(pending_names):
                    next_name = pending_names[index]
                    futures[executor.submit(fetch_package_record, next_name, github_token)] = next_name
                    index += 1

    return totals


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(
        description=(
            "Enumerate PyPI projects, detect GitHub repositories from project metadata, "
            "and export GitHub star counts to CSV."
        )
    )
    parser.add_argument(
        "--output",
        default="pypi_github_stars.csv",
        help="CSV output path. Default: %(default)s",
    )
    parser.add_argument(
        "--limit",
        type=int,
        default=None,
        help="Maximum number of PyPI packages to process after offset.",
    )
    parser.add_argument(
        "--offset",
        type=int,
        default=0,
        help="Skip the first N PyPI packages from the index.",
    )
    parser.add_argument(
        "--workers",
        type=int,
        default=4,
        help="Concurrent workers for PyPI and GitHub requests. Default: %(default)s",
    )
    parser.add_argument(
        "--github-token",
        default=os.environ.get("GITHUB_TOKEN", ""),
        help="GitHub token. Defaults to GITHUB_TOKEN env var when present.",
    )
    parser.add_argument(
        "--sleep",
        type=float,
        default=0.0,
        help="Sleep seconds after each completed package to reduce request pressure.",
    )
    parser.add_argument(
        "--resume",
        action="store_true",
        help="Skip package names already present in the output CSV.",
    )
    return parser


def main(argv: list[str] | None = None) -> int:
    parser = build_parser()
    args = parser.parse_args(argv)

    output_path = Path(args.output).resolve()
    github_token = args.github_token or None

    try:
        totals = export_catalog(
            output_path,
            limit=args.limit,
            offset=args.offset,
            workers=args.workers,
            github_token=github_token,
            sleep_seconds=args.sleep,
            resume=args.resume,
        )
    except KeyboardInterrupt:
        print("Interrupted.", file=sys.stderr)
        return 130

    print(json.dumps({"output": str(output_path), **totals}, ensure_ascii=True, indent=2))
    if not github_token:
        print(
            "Hint: set GITHUB_TOKEN to avoid GitHub unauthenticated rate limits.",
            file=sys.stderr,
        )
    return 0


if __name__ == "__main__":
    raise SystemExit(main())
