"""
HTML Content Downloader for web pages.

Downloads and extracts clean text content from HTML web pages.
"""

from typing import Optional, Dict, Any
from urllib.parse import urlparse
from loguru import logger
from bs4 import BeautifulSoup, Tag

from .base import BaseDownloader, ContentType, DownloadResult


class HTMLDownloader(BaseDownloader):
    """Downloader for HTML web pages - extracts clean text content."""

    # Elements to remove (navigation, ads, etc.)
    REMOVE_ELEMENTS = [
        "script",
        "style",
        "nav",
        "header",
        "footer",
        "aside",
        "iframe",
        "noscript",
        "svg",
        "form",
        "button",
        "input",
        "select",
        "textarea",
    ]

    # Classes/IDs that typically contain non-content
    REMOVE_PATTERNS = [
        "nav",
        "menu",
        "sidebar",
        "advertisement",
        "ads",
        "footer",
        "header",
        "comment",
        "share",
        "social",
        "related",
        "recommended",
        "popup",
        "modal",
        "cookie",
        "newsletter",
        "subscribe",
    ]

    def can_handle(self, url: str) -> bool:
        """
        Check if this downloader can handle the given URL.

        Returns True for any HTTP/HTTPS URL (fallback downloader for web content).
        """
        try:
            parsed = urlparse(url)
            return parsed.scheme in ("http", "https")
        except Exception:
            return False

    def download(
        self, url: str, content_type: ContentType = ContentType.TEXT
    ) -> Optional[bytes]:
        """
        Download and extract text content from HTML page.

        Args:
            url: The URL to download
            content_type: Type of content (TEXT for HTML extraction)

        Returns:
            Extracted text as UTF-8 bytes, or None if failed
        """
        if content_type == ContentType.PDF:
            # HTML downloader doesn't handle PDFs
            logger.warning(f"HTML downloader cannot download PDFs: {url}")
            return None

        try:
            html_content = self._fetch_html(url)
            if not html_content:
                return None

            extracted = self._extract_content(html_content, url)
            if extracted:
                # Format as structured text
                text = self._format_extracted_content(extracted)
                return text.encode("utf-8")

            return None

        except Exception:
            logger.exception(f"Failed to download HTML from {url}")
            return None

    def download_with_result(
        self, url: str, content_type: ContentType = ContentType.TEXT
    ) -> DownloadResult:
        """Download content and return detailed result with skip reason."""
        if content_type == ContentType.PDF:
            return DownloadResult(
                skip_reason="HTML downloader does not support PDF downloads"
            )

        try:
            html_content = self._fetch_html(url)
            if not html_content:
                return DownloadResult(
                    skip_reason="Failed to fetch HTML content from URL"
                )

            extracted = self._extract_content(html_content, url)
            if not extracted:
                return DownloadResult(
                    skip_reason="Could not extract meaningful content from page"
                )

            text = self._format_extracted_content(extracted)
            if not text.strip():
                return DownloadResult(skip_reason="Extracted content is empty")

            return DownloadResult(
                content=text.encode("utf-8"),
                is_success=True,
            )

        except Exception as e:
            logger.exception(f"Failed to download HTML from {url}")
            return DownloadResult(skip_reason=f"Error: {str(e)}")

    def _fetch_html(self, url: str) -> Optional[str]:
        """Fetch raw HTML content from URL."""
        domain = urlparse(url).netloc
        engine_type = f"html_download_{domain}"

        wait_time = self.rate_tracker.apply_rate_limit(engine_type)

        try:
            response = self.session.get(
                url,
                timeout=self.timeout,
                allow_redirects=True,
            )

            if response.status_code == 200:
                content_type = response.headers.get("content-type", "").lower()
                if (
                    "text/html" in content_type
                    or "application/xhtml" in content_type
                ):
                    self.rate_tracker.record_outcome(
                        engine_type=engine_type,
                        wait_time=wait_time,
                        success=True,
                        retry_count=1,
                        search_result_count=1,
                    )
                    # Try to get encoding from response
                    return response.text
                logger.warning(
                    f"Unexpected content type for HTML download: {content_type}"
                )
                return None
            logger.warning(f"HTTP {response.status_code} fetching {url}")
            self.rate_tracker.record_outcome(
                engine_type=engine_type,
                wait_time=wait_time,
                success=False,
                retry_count=1,
                error_type=f"HTTP_{response.status_code}",
            )
            return None

        except Exception as e:
            logger.exception(f"Error fetching HTML from {url}")
            self.rate_tracker.record_outcome(
                engine_type=engine_type,
                wait_time=wait_time,
                success=False,
                retry_count=1,
                error_type=type(e).__name__,
            )
            return None

    def _extract_content(self, html: str, url: str) -> Optional[Dict[str, Any]]:
        """
        Extract clean content from HTML.

        Returns dict with title, content, and metadata.
        """
        try:
            soup = BeautifulSoup(html, "html.parser")

            # Extract title
            title = None
            if soup.title and soup.title.string:
                title = soup.title.string.strip()

            # Try to find Open Graph or meta title
            og_title = soup.find("meta", property="og:title")
            if og_title and og_title.get("content"):
                title = str(og_title["content"]).strip()

            # Extract meta description
            description = None
            meta_desc = soup.find("meta", attrs={"name": "description"})
            if meta_desc and meta_desc.get("content"):
                description = str(meta_desc["content"]).strip()

            og_desc = soup.find("meta", property="og:description")
            if og_desc and og_desc.get("content"):
                description = str(og_desc["content"]).strip()

            # Remove unwanted elements
            for element in self.REMOVE_ELEMENTS:
                for tag in soup.find_all(element):
                    tag.decompose()

            # Remove elements with common non-content class/id patterns
            for pattern in self.REMOVE_PATTERNS:
                for tag in soup.find_all(
                    class_=lambda x: x and pattern in str(x).lower()
                ):
                    tag.decompose()
                for tag in soup.find_all(
                    id=lambda x: x and pattern in str(x).lower()
                ):
                    tag.decompose()

            # Try to find main content area
            main_content: Optional[Tag] = None

            # Look for article tag first
            article = soup.find("article")
            if article and isinstance(article, Tag):
                main_content = article

            # Try main tag
            if not main_content:
                main = soup.find("main")
                if main and isinstance(main, Tag):
                    main_content = main

            # Try common content div patterns
            if not main_content:
                for pattern in ["content", "article", "post", "entry", "body"]:
                    content_div = soup.find(
                        "div",
                        class_=lambda x: x and pattern in str(x).lower(),
                    )
                    if content_div and isinstance(content_div, Tag):
                        main_content = content_div
                        break

            # Fall back to body
            if not main_content:
                _body = soup.find("body")
                if _body and isinstance(_body, Tag):
                    main_content = _body

            if not main_content:
                return None

            # Extract text with some structure preservation
            text_parts = []
            for tag_element in main_content.find_all(
                [
                    "p",
                    "h1",
                    "h2",
                    "h3",
                    "h4",
                    "h5",
                    "h6",
                    "li",
                    "blockquote",
                    "pre",
                ]
            ):
                if not isinstance(tag_element, Tag):
                    continue
                text = tag_element.get_text(separator=" ", strip=True)
                if text and len(text) > 10:  # Skip very short fragments
                    if tag_element.name and tag_element.name.startswith("h"):
                        text_parts.append(f"\n## {text}\n")
                    elif tag_element.name == "li":
                        text_parts.append(f"• {text}")
                    elif tag_element.name == "blockquote":
                        text_parts.append(f"> {text}")
                    elif tag_element.name == "pre":
                        text_parts.append(f"```\n{text}\n```")
                    else:
                        text_parts.append(text)

            content = "\n\n".join(text_parts)

            # If we didn't get much content, fall back to simpler extraction
            if len(content) < 100:
                content = main_content.get_text(separator="\n", strip=True)

            # Clean up whitespace
            content = self._clean_text(content)

            if not content or len(content) < 50:
                return None

            return {
                "title": title,
                "description": description,
                "content": content,
                "url": url,
            }

        except Exception:
            logger.exception("Error extracting content from HTML")
            return None

    def _clean_text(self, text: str) -> str:
        """Clean up extracted text."""
        import re

        # Normalize whitespace
        text = re.sub(r"\n{3,}", "\n\n", text)
        text = re.sub(r" {2,}", " ", text)
        text = re.sub(r"\t+", " ", text)

        # Remove lines that are just whitespace
        lines = [line.strip() for line in text.split("\n")]
        lines = [line for line in lines if line]

        return "\n".join(lines)

    def _format_extracted_content(self, extracted: Dict[str, Any]) -> str:
        """Format extracted content as readable text."""
        parts = []

        if extracted.get("title"):
            parts.append(f"# {extracted['title']}")
            parts.append("")

        if extracted.get("description"):
            parts.append(f"*{extracted['description']}*")
            parts.append("")

        if extracted.get("url"):
            parts.append(f"Source: {extracted['url']}")
            parts.append("")

        if extracted.get("content"):
            parts.append(extracted["content"])

        return "\n".join(parts)

    def get_metadata(self, url: str) -> Dict[str, Any]:
        """Get metadata about the page."""
        html_content = self._fetch_html(url)
        if not html_content:
            return {}

        try:
            soup = BeautifulSoup(html_content, "html.parser")

            metadata = {"url": url}

            # Title
            if soup.title and soup.title.string:
                metadata["title"] = soup.title.string.strip()

            # Description
            meta_desc = soup.find("meta", attrs={"name": "description"})
            if meta_desc and meta_desc.get("content"):
                metadata["description"] = str(meta_desc["content"])

            # Author
            author = soup.find("meta", attrs={"name": "author"})
            if author and author.get("content"):
                metadata["author"] = str(author["content"])

            # Published date
            for prop in ["article:published_time", "datePublished"]:
                date_tag = soup.find("meta", property=prop)
                if date_tag and date_tag.get("content"):
                    metadata["published_date"] = str(date_tag["content"])
                    break

            return metadata

        except Exception:
            logger.exception(f"Error extracting metadata from {url}")
            return {"url": url}
