"""Web scrape source connector — crawls and parses configured URLs."""

from __future__ import annotations

import hashlib
import logging
import ssl
from collections import deque
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
from typing import TYPE_CHECKING, Any
from urllib.parse import urljoin, urlparse

import httpx
from bs4 import BeautifulSoup
from rich.progress import Progress, TaskID

from redhat_oracle.core.config import OracleConfig
from redhat_oracle.core.vectordb.chunker import Chunk

if TYPE_CHECKING:
    from redhat_oracle.core.vectordb.store import VectorStore

logger = logging.getLogger(__name__)

# File extensions to skip when crawling
_SKIP_EXTENSIONS = frozenset({
    ".png", ".jpg", ".jpeg", ".gif", ".svg", ".ico", ".webp", ".bmp",
    ".pdf", ".zip", ".gz", ".tar", ".bz2", ".xz", ".7z",
    ".mp3", ".mp4", ".avi", ".mov", ".webm", ".ogg",
    ".woff", ".woff2", ".ttf", ".eot", ".otf",
    ".css", ".js", ".json", ".xml", ".rss", ".atom",
    ".exe", ".dmg", ".rpm", ".deb", ".msi",
})


@dataclass
class WebPage:
    """A fetched and parsed web page."""

    url: str
    title: str
    text: str
    fetched_at: str  # ISO 8601


def _generate_chunk_id(prefix: str, content: str) -> str:
    """Generate a deterministic chunk ID."""
    content_hash = hashlib.sha256(content.encode("utf-8")).hexdigest()[:12]
    return f"{prefix}::{content_hash}"


def _get_ssl_context() -> ssl.SSLContext | bool:
    """Get SSL context using system CA bundle if available."""
    for ca_path in (
        "/etc/pki/tls/certs/ca-bundle.crt",
        "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem",
    ):
        if Path(ca_path).exists():
            return ssl.create_default_context(cafile=ca_path)
    return True


def _normalize_url(url: str) -> str:
    """Normalize a URL by removing fragments and trailing slashes."""
    parsed = urlparse(url)
    # Remove fragment, normalize path
    path = parsed.path.rstrip("/") or "/"
    return f"{parsed.scheme}://{parsed.netloc}{path}"


def _is_same_site(url: str, base_url: str) -> bool:
    """Check if a URL belongs to the same site as the base URL."""
    parsed = urlparse(url)
    base_parsed = urlparse(base_url)
    return parsed.netloc == base_parsed.netloc


def _should_skip_url(url: str) -> bool:
    """Check if a URL should be skipped based on extension or pattern."""
    parsed = urlparse(url)
    path_lower = parsed.path.lower()

    # Skip non-HTTP schemes
    if parsed.scheme not in ("http", "https"):
        return True

    # Skip file extensions we don't want
    suffix = Path(path_lower).suffix
    if suffix in _SKIP_EXTENSIONS:
        return True

    # Skip common non-content paths
    skip_paths = ("/auth", "/login", "/logout", "/oauth", "/api/", "/_")
    return bool(any(path_lower.startswith(p) or f"/{p.lstrip('/')}/" in path_lower for p in skip_paths))


def _extract_links(soup: BeautifulSoup, page_url: str) -> list[str]:
    """Extract and resolve all links from a page."""
    links: list[str] = []
    for a_tag in soup.find_all("a", href=True):
        href = str(a_tag["href"])
        # Resolve relative URLs
        absolute = urljoin(page_url, href)
        # Remove fragment
        normalized = _normalize_url(absolute)
        links.append(normalized)
    return links


def _parse_page(html: str, url: str) -> tuple[str, str]:
    """Parse HTML and return (title, clean_text)."""
    soup = BeautifulSoup(html, "html.parser")

    # Remove non-content elements
    for tag in soup(["script", "style", "nav", "footer", "header", "aside",
                     "form", "button", "noscript", "iframe"]):
        tag.decompose()

    title = soup.title.string.strip() if soup.title and soup.title.string else url
    text = soup.get_text(separator="\n", strip=True)
    return title, text


async def _crawl_site(
    client: httpx.AsyncClient,
    start_url: str,
    max_depth: int,
    max_pages: int,
    progress: Progress | None = None,
    task_id: TaskID | None = None,
) -> list[WebPage]:
    """Crawl a site starting from start_url, following same-site links.

    Uses breadth-first traversal up to max_depth levels deep.
    """
    pages: list[WebPage] = []
    visited: set[str] = set()
    queue: deque[tuple[str, int]] = deque()  # (url, depth)

    start_normalized = _normalize_url(start_url)
    queue.append((start_normalized, 0))
    visited.add(start_normalized)

    while queue and len(pages) < max_pages:
        url, depth = queue.popleft()

        try:
            resp = await client.get(url)
            resp.raise_for_status()

            # Skip non-HTML responses
            content_type = resp.headers.get("content-type", "")
            if "text/html" not in content_type:
                continue

            title, text = _parse_page(resp.text, url)

            # Skip pages with very little content (login pages, redirects)
            if len(text.split()) < 20:
                continue

            pages.append(
                WebPage(
                    url=url,
                    title=title,
                    text=text,
                    fetched_at=datetime.now(UTC).isoformat(),
                )
            )

            if progress and task_id is not None:
                progress.update(
                    task_id,
                    description=f"[cyan]Crawling {urlparse(start_url).hostname} ({len(pages)} pages)",
                )

            # Extract and queue links if we haven't hit max depth
            if depth < max_depth:
                soup = BeautifulSoup(resp.text, "html.parser")
                for link in _extract_links(soup, url):
                    if link in visited:
                        continue
                    if not _is_same_site(link, start_url):
                        continue
                    if _should_skip_url(link):
                        continue
                    visited.add(link)
                    queue.append((link, depth + 1))

        except Exception:
            logger.debug("Failed to fetch %s", url)

    return pages


async def sync_web(
    config: OracleConfig,
    store: VectorStore | None = None,
    progress: Progress | None = None,
) -> dict[str, Any]:
    """Crawl all configured URLs, following same-site links.

    Each configured URL is treated as a site root. The crawler follows
    internal links up to max_depth levels deep, collecting up to
    max_pages_per_site pages per site. When store is provided, upserts
    chunks after each site for incremental persistence.

    Returns:
        Dict with stats: pages, chunks, sites_crawled, sites_skipped.
    """
    from redhat_oracle.core.cache import SyncCache

    web_config = config.sources.web

    if not web_config.urls:
        logger.info("No web URLs configured, skipping web sync")
        return {"pages": 0, "chunks": 0, "sites_crawled": 0, "sites_skipped": 0}

    cache = SyncCache(config)
    total_pages = 0
    total_chunks = 0
    sites_crawled = 0
    sites_skipped = 0
    verify = _get_ssl_context()

    task_id: TaskID | None = None
    if progress:
        task_id = progress.add_task(
            "[cyan]Crawling web sites", total=len(web_config.urls)
        )

    async with httpx.AsyncClient(
        timeout=web_config.timeout,
        follow_redirects=True,
        verify=verify,
        headers={"User-Agent": "RedHatOracle/0.1"},
    ) as client:
        for url in web_config.urls:
            # Skip sites already completed in a previous run
            if cache.is_web_site_done(url):
                sites_skipped += 1
                logger.info("Skipping %s (already indexed)", url)
                if progress and task_id is not None:
                    progress.advance(task_id)
                continue

            logger.info("Crawling %s (max_depth=%d, max_pages=%d)",
                        url, web_config.max_depth, web_config.max_pages_per_site)

            site_pages = await _crawl_site(
                client,
                url,
                max_depth=web_config.max_depth,
                max_pages=web_config.max_pages_per_site,
                progress=progress,
                task_id=task_id,
            )

            # Upsert this site's chunks to the store
            site_chunks = chunks_from_web_pages(site_pages)
            if store and site_chunks:
                store.upsert_chunks(site_chunks)

            cache.mark_web_site_done(url, len(site_pages), len(site_chunks))
            total_pages += len(site_pages)
            total_chunks += len(site_chunks)
            sites_crawled += 1
            logger.info("Crawled %d pages, %d chunks from %s",
                        len(site_pages), len(site_chunks), url)

            if progress and task_id is not None:
                progress.advance(task_id)

    logger.info("Web total: %d pages, %d chunks from %d sites (%d skipped)",
                total_pages, total_chunks, sites_crawled, sites_skipped)
    return {
        "pages": total_pages,
        "chunks": total_chunks,
        "sites_crawled": sites_crawled,
        "sites_skipped": sites_skipped,
    }


def chunks_from_web_pages(
    pages: list[WebPage],
    chunk_size: int = 500,
    chunk_overlap: int = 50,
) -> list[Chunk]:
    """Convert web pages into Chunk objects using fixed-size chunking.

    Args:
        pages: List of WebPage objects to chunk.
        chunk_size: Words per chunk.
        chunk_overlap: Overlap words between chunks.

    Returns:
        List of Chunk objects.
    """
    chunks: list[Chunk] = []

    for page in pages:
        words = page.text.split()
        if not words:
            continue

        step = max(1, chunk_size - chunk_overlap)
        i = 0

        while i < len(words):
            chunk_words = words[i : i + chunk_size]
            text = " ".join(chunk_words)

            chunk_id = _generate_chunk_id(f"web/{page.url}", text)
            chunks.append(
                Chunk(
                    chunk_id=chunk_id,
                    text=text,
                    source="web",
                    org="",
                    repo="",
                    path=page.url,
                    content_type="web_page",
                    url=page.url,
                    timestamp=page.fetched_at,
                )
            )

            i += step

    return chunks
