"""Utilities for discovering documents across web and filesystem sources."""

from __future__ import annotations

import re
import xml.etree.ElementTree as ET
from collections import deque
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Mapping, Sequence
from urllib.parse import urljoin, urlparse

import httpx
from bs4 import BeautifulSoup
from bs4.element import Tag


URL_RE = re.compile(r"https?://", re.IGNORECASE)
DEFAULT_LOCAL_EXTENSIONS = (".html", ".htm", ".md", ".txt", ".pdf")


@dataclass(frozen=True)
class DiscoveredSource:
    fetch_url: str
    kind: str
    display_url: str | None = None
    metadata: Mapping[str, Any] = field(default_factory=dict)
    mime_type: str | None = None


async def load_index_locations(
    index_location: str, *, timeout: float = 30.0
) -> list[str]:
    """Load URLs from a sitemap, HTML index, or newline-delimited index."""

    payload = await _load_text_resource(index_location, timeout=timeout)
    payload = payload.strip()
    if not payload:
        return []
    if payload.startswith("<"):
        # Try XML sitemap first, fall back to HTML if that fails
        urls = _parse_sitemap_xml(payload)
        if not urls:
            # Might be HTML instead of XML sitemap
            urls = _parse_html_index(payload, base_location=index_location)
        return urls
    return _parse_plain_index(payload)


def crawl_local_root(
    root: Path,
    *,
    url_prefix: str,
    include_exts: Sequence[str] | None = None,
    max_files: int = 500,
) -> list[DiscoveredSource]:
    root = root.expanduser().resolve()
    if not root.exists():
        raise FileNotFoundError(f"Local root '{root}' does not exist.")

    normalized_exts = (
        {ext.lower().lstrip(".") for ext in include_exts}
        if include_exts
        else {ext.lstrip(".") for ext in DEFAULT_LOCAL_EXTENSIONS}
    )

    sources: list[DiscoveredSource] = []
    count = 0
    for file_path in root.rglob("*"):
        if not file_path.is_file():
            continue
        suffix = file_path.suffix.lower().lstrip(".")
        if normalized_exts and suffix not in normalized_exts:
            continue
        relative = file_path.relative_to(root).as_posix()
        display_url = _rewrite_local_path(relative, url_prefix)
        metadata = {
            "source_kind": "local",
            "original_path": str(file_path),
        }
        sources.append(
            DiscoveredSource(
                fetch_url=str(file_path),
                kind="local",
                display_url=display_url,
                metadata=metadata,
            )
        )
        count += 1
        if count >= max_files:
            break
    return sources


async def crawl_entry_points(
    entry_points: Sequence[str],
    *,
    max_pages: int = 50,
    max_depth: int = 2,
    same_domain: bool = True,
    timeout: float = 15.0,
) -> list[str]:
    if not entry_points:
        return []

    allowed_domains = (
        {urlparse(url).netloc for url in entry_points} if same_domain else None
    )
    seen: set[str] = set()
    queue: deque[tuple[str, int]] = deque((url, 0) for url in entry_points)
    discovered: list[str] = []

    async with httpx.AsyncClient(follow_redirects=True, timeout=timeout) as client:
        while queue and len(discovered) < max_pages:
            url, depth = queue.popleft()
            if url in seen:
                continue
            seen.add(url)
            try:
                response = await client.get(url)
                response.raise_for_status()
            except Exception:
                continue

            discovered.append(url)

            if depth >= max_depth:
                continue

            content_type = (response.headers.get("content-type") or "").lower()
            if "html" not in content_type:
                continue

            soup = BeautifulSoup(response.text, "html.parser")
            for anchor in soup.find_all("a", href=True):
                if not isinstance(anchor, Tag):
                    continue
                href_value = anchor.get("href")
                if not isinstance(href_value, str):
                    continue
                href = href_value.strip()
                if not href:
                    continue
                next_url = urljoin(url, href)
                parsed = urlparse(next_url)
                if parsed.scheme not in {"http", "https"}:
                    continue
                if allowed_domains and parsed.netloc not in allowed_domains:
                    continue
                if next_url not in seen:
                    queue.append((next_url, depth + 1))

    return discovered


def _parse_sitemap_xml(payload: str) -> list[str]:
    """Parse XML sitemap and return URLs from <loc> tags."""
    try:
        root = ET.fromstring(payload)
    except ET.ParseError:
        return []

    urls: list[str] = []
    for element in root.findall(".//{*}loc"):
        if element.text:
            urls.append(element.text.strip())
    return urls


def _parse_plain_index(payload: str) -> list[str]:
    urls: list[str] = []
    for line in payload.splitlines():
        stripped = line.strip()
        if not stripped:
            continue
        if stripped.startswith("#"):
            continue
        urls.append(stripped)
    return urls


def _parse_html_index(payload: str, base_location: str) -> list[str]:
    """Parse HTML file and extract all links from anchor tags."""
    soup = BeautifulSoup(payload, "html.parser")
    urls: list[str] = []

    # Determine if base_location is a URL or local path
    is_url = URL_RE.match(base_location)

    # For local files, calculate base directory once
    if is_url:
        base_path: Path | None = None
    else:
        base_path = Path(base_location).expanduser().resolve().parent

    for anchor in soup.find_all("a", href=True):
        if not isinstance(anchor, Tag):
            continue
        href_value = anchor.get("href")
        if not isinstance(href_value, str):
            continue
        href = href_value.strip()
        if not href:
            continue

        # Skip fragment-only links
        if href.startswith("#"):
            continue

        # Extract filename from javascript:call_link('filename') pattern
        if href.startswith("javascript:"):
            # Try to extract filename from common JavaScript patterns
            match = re.search(r"['\"]([^'\"]+\.[a-zA-Z0-9]+)['\"]", href)
            if match:
                href = match.group(1)
            else:
                # Can't extract a useful link, skip it
                continue

        # Resolve relative URLs
        if is_url:
            resolved = urljoin(base_location, href)
        elif href.startswith(("http://", "https://")):
            # Absolute URL in local file
            resolved = href
        else:
            # Relative path - resolve against base directory
            # base_path is guaranteed to be a Path here (not None) because is_url is False
            resolved_path = (base_path / href).resolve()  # type: ignore[operator]
            resolved = str(resolved_path)

        urls.append(resolved)

    return urls


async def _load_text_resource(location: str, *, timeout: float) -> str:
    if URL_RE.match(location):
        async with httpx.AsyncClient(follow_redirects=True, timeout=timeout) as client:
            response = await client.get(location)
            response.raise_for_status()
            response.encoding = response.encoding or "utf-8"
            return response.text
    path = Path(location).expanduser()
    return path.read_text(encoding="utf-8")


def _rewrite_local_path(relative_path: str, url_prefix: str) -> str:
    prefix = url_prefix.rstrip("/")
    return f"{prefix}/{relative_path}" if prefix else relative_path


__all__ = [
    "DiscoveredSource",
    "crawl_entry_points",
    "crawl_local_root",
    "load_index_locations",
]
