"""HTTP crawl skill for technical SEO audits."""

from __future__ import annotations

import logging
import re
from collections import deque
from dataclasses import dataclass
from typing import Any
from urllib import error as urllib_error
from urllib import request as urllib_request
from urllib.parse import urldefrag, urljoin, urlparse
 
try:
    import requests
except ImportError:  # pragma: no cover - fallback path used in restricted envs
    requests = None

try:
    from bs4 import BeautifulSoup  # type: ignore
except ImportError:  # pragma: no cover - fallback used when bs4 unavailable
    BeautifulSoup = None

logger = logging.getLogger(__name__)


EMAIL_PATTERN = re.compile(r"[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}", re.IGNORECASE)


@dataclass
class CrawlPage:
    """Represents a crawled page summary."""

    url: str
    status_code: int
    title: str
    meta_description: str
    h1_count: int
    internal_links: list[str]
    external_links: list[str]
    emails: list[str]


class CrawlSkill:
    """Crawl websites and produce deterministic technical SEO diagnostics."""

    def __init__(self, config: dict[str, Any]):
        self.config = config
        settings = config.get("agents", {}).get("KRAWL", {}).get("settings", {})
        self.max_pages = int(settings.get("max_pages", 20))
        self.request_timeout = int(settings.get("request_timeout", 10))
        self._headers = {"User-Agent": "KIESSCLAW-KRAWL/0.2 (+https://github.com/monykiss/kiessclaw)"}
        self._request_exception: type[BaseException] = Exception
        if requests is not None:
            self.session = requests.Session()
            self.session.headers.update(self._headers)
            self._request_exception = requests.RequestException
        else:
            self.session = _UrlLibSession(self._headers)
            self._request_exception = _UrlLibRequestError

    def audit_site(self, url: str, max_pages: int | None = None) -> dict[str, Any]:
        """Crawl the target site and return a technical SEO audit payload."""
        root_url = self._normalize_url(url)
        parsed_root = urlparse(root_url)
        domain = parsed_root.netloc.lower()
        page_limit = max_pages or self.max_pages

        queue: deque[str] = deque([root_url])
        visited: set[str] = set()
        pages: list[CrawlPage] = []
        all_internal_links: set[str] = set()
        errors: list[str] = []

        while queue and len(visited) < page_limit:
            current = queue.popleft()
            if current in visited:
                continue
            visited.add(current)

            page = self._fetch_page(current, domain)
            if page is None:
                errors.append(f"Failed to fetch {current}")
                continue

            pages.append(page)
            for link in page.internal_links:
                all_internal_links.add(link)
                if link not in visited and link not in queue and len(visited) + len(queue) < page_limit:
                    queue.append(link)

        broken_links = self._check_broken_links(all_internal_links)
        robots_ok = self._check_path(root_url, "/robots.txt")
        sitemap_ok = self._check_path(root_url, "/sitemap.xml")

        missing_titles = sum(1 for page in pages if not page.title)
        missing_meta = sum(1 for page in pages if not page.meta_description)
        error_pages = sum(1 for page in pages if page.status_code >= 400)
        contact_emails = sorted({email for page in pages for email in page.emails})
        owner_contact = self._choose_owner_contact(contact_emails)
        technical_score = self._compute_technical_score(
            page_count=len(pages),
            missing_titles=missing_titles,
            missing_meta=missing_meta,
            broken_links=len(broken_links),
            checked_links=max(1, len(all_internal_links)),
            error_pages=error_pages,
            robots_ok=robots_ok,
            sitemap_ok=sitemap_ok,
        )

        return {
            "url": root_url,
            "domain": domain,
            "technical_score": technical_score,
            "pages_crawled": len(pages),
            "missing_title_pages": missing_titles,
            "missing_meta_pages": missing_meta,
            "error_pages": error_pages,
            "broken_links": broken_links,
            "robots_ok": robots_ok,
            "sitemap_ok": sitemap_ok,
            "contact_emails": contact_emails,
            "owner_contact": owner_contact,
            "errors": errors,
            "timestamp": self._timestamp(),
        }

    def _fetch_page(self, url: str, root_domain: str) -> CrawlPage | None:
        """Fetch and parse a single HTML page."""
        try:
            response = self.session.get(url, timeout=self.request_timeout)
        except self._request_exception as exc:
            logger.warning("[CrawlSkill] Request failed for %s: %s", url, exc)
            return None

        content_type = response.headers.get("Content-Type", "")
        html = response.text if "html" in content_type.lower() or not content_type else ""
        title, meta_description, links, h1_count, emails = self._parse_html(html, url)

        internal_links: list[str] = []
        external_links: list[str] = []
        for link in links:
            normalized = self._clean_link(urljoin(url, link))
            if not normalized:
                continue
            parsed = urlparse(normalized)
            if parsed.netloc.lower() == root_domain:
                internal_links.append(normalized)
            else:
                external_links.append(normalized)

        return CrawlPage(
            url=url,
            status_code=response.status_code,
            title=title,
            meta_description=meta_description,
            h1_count=h1_count,
            internal_links=sorted(set(internal_links)),
            external_links=sorted(set(external_links)),
            emails=sorted(set(emails)),
        )

    def _parse_html(self, html: str, base_url: str) -> tuple[str, str, list[str], int, list[str]]:
        """Parse HTML for key SEO signals and links."""
        if not html:
            return "", "", [], 0, []

        emails = EMAIL_PATTERN.findall(html)
        if BeautifulSoup is None:
            title_match = re.search(r"<title>(.*?)</title>", html, flags=re.IGNORECASE | re.DOTALL)
            meta_match = re.search(
                r'<meta[^>]+name=["\']description["\'][^>]+content=["\'](.*?)["\']',
                html,
                flags=re.IGNORECASE | re.DOTALL,
            )
            hrefs = re.findall(r'href=["\'](.*?)["\']', html, flags=re.IGNORECASE)
            h1_count = len(re.findall(r"<h1\b", html, flags=re.IGNORECASE))
            return (
                (title_match.group(1).strip() if title_match else ""),
                (meta_match.group(1).strip() if meta_match else ""),
                hrefs,
                h1_count,
                emails,
            )

        soup = BeautifulSoup(html, "html.parser")
        title = soup.title.get_text(strip=True) if soup.title else ""
        meta = soup.find("meta", attrs={"name": re.compile("^description$", re.IGNORECASE)})
        meta_description = meta.get("content", "").strip() if meta else ""
        links = [tag.get("href", "").strip() for tag in soup.find_all("a") if tag.get("href")]
        h1_count = len(soup.find_all("h1"))

        # Also collect mailto: links as direct owner contact hints.
        for tag in soup.find_all("a"):
            href = (tag.get("href") or "").strip()
            if href.startswith("mailto:"):
                emails.append(href.replace("mailto:", "").split("?")[0].strip())

        return title, meta_description, links, h1_count, emails

    def _check_broken_links(self, links: set[str]) -> list[str]:
        """Probe discovered internal links and return links that fail."""
        broken: list[str] = []
        for link in sorted(links)[:50]:
            try:
                response = self.session.get(link, timeout=self.request_timeout, allow_redirects=True)
            except self._request_exception:
                broken.append(link)
                continue
            if response.status_code >= 400:
                broken.append(link)
        return broken

    def _check_path(self, root_url: str, path: str) -> bool:
        """Check whether a required SEO path is reachable."""
        probe_url = urljoin(root_url, path)
        try:
            response = self.session.get(probe_url, timeout=self.request_timeout)
            return response.status_code < 400
        except self._request_exception:
            return False

    def _compute_technical_score(
        self,
        page_count: int,
        missing_titles: int,
        missing_meta: int,
        broken_links: int,
        checked_links: int,
        error_pages: int,
        robots_ok: bool,
        sitemap_ok: bool,
    ) -> int:
        """Compute a bounded SEO technical score from crawl diagnostics."""
        score = 100.0
        if page_count == 0:
            return 0

        score -= min(25.0, (missing_titles / page_count) * 25.0)
        score -= min(20.0, (missing_meta / page_count) * 20.0)
        score -= min(20.0, (error_pages / page_count) * 20.0)
        score -= min(20.0, (broken_links / checked_links) * 20.0)
        if not robots_ok:
            score -= 8.0
        if not sitemap_ok:
            score -= 7.0
        if page_count < 3:
            score -= 10.0

        return max(0, min(100, int(round(score))))

    def _choose_owner_contact(self, emails: list[str]) -> str | None:
        """Select the best owner-like contact email from discovered emails."""
        if not emails:
            return None
        ranked = sorted(
            emails,
            key=lambda value: (
                "noreply" in value.lower() or "no-reply" in value.lower(),
                "support@" in value.lower(),
                "info@" in value.lower(),
                value,
            ),
        )
        return ranked[0]

    def _clean_link(self, link: str) -> str:
        """Normalize links and discard unsupported schemes."""
        if not link:
            return ""
        if link.startswith("#") or link.startswith("javascript:") or link.startswith("mailto:"):
            return ""
        cleaned, _ = urldefrag(link.strip())
        parsed = urlparse(cleaned)
        if parsed.scheme not in {"http", "https"}:
            return ""
        return cleaned.rstrip("/")

    def _normalize_url(self, url: str) -> str:
        """Normalize incoming URLs for crawl start."""
        value = url.strip()
        if not value.startswith(("http://", "https://")):
            value = "https://" + value
        parsed = urlparse(value)
        normalized = f"{parsed.scheme}://{parsed.netloc}{parsed.path or '/'}"
        return normalized.rstrip("/")

    def _timestamp(self) -> str:
        """Return RFC3339 timestamp string."""
        from datetime import datetime

        return datetime.utcnow().isoformat() + "Z"


class _UrlLibRequestError(Exception):
    """Internal fallback request exception type."""


@dataclass
class _UrlLibResponse:
    """Lightweight response wrapper for urllib fallback mode."""

    status_code: int
    text: str
    headers: dict[str, str]


class _UrlLibSession:
    """Minimal session wrapper offering `.get()` compatible with requests usage."""

    def __init__(self, headers: dict[str, str]):
        self.headers = headers

    def get(self, url: str, timeout: int = 10, allow_redirects: bool = True) -> _UrlLibResponse:  # noqa: ARG002
        """Execute GET and return normalized response wrapper."""
        request = urllib_request.Request(url, headers=self.headers, method="GET")
        try:
            with urllib_request.urlopen(request, timeout=timeout) as response:
                body = response.read()
                charset = response.headers.get_content_charset() or "utf-8"
                return _UrlLibResponse(
                    status_code=int(getattr(response, "status", 200)),
                    text=body.decode(charset, errors="replace"),
                    headers={k: v for k, v in response.headers.items()},
                )
        except urllib_error.URLError as exc:
            raise _UrlLibRequestError(str(exc)) from exc
