"""
Content Scraper - Scrape content from dark web URLs via Tor proxy

Provides multi-threaded content scraping with text extraction and normalization.

Security Features (SEC-017):
- Explicit SSL/TLS verification for clearnet URLs
- Certificate validation with documented exceptions for .onion domains

Reliability Features (REL-002):
- Specific exception handling instead of bare except
- Proper error logging
- Resource cleanup on failure
"""

import logging
import re
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError, as_completed
from dataclasses import dataclass, field
from datetime import datetime
from typing import Optional
from urllib.parse import urlparse

import requests.exceptions
from bs4 import BeautifulSoup

from groknroll.osint.tor_manager import TorManager


# REL-002: Configure logger for error tracking
logger = logging.getLogger(__name__)


def _should_verify_ssl(url: str) -> bool:
    """
    Determine if SSL should be verified for a URL.

    SEC-017: .onion domains don't use traditional SSL certificates,
    so we skip verification for them but enforce it for clearnet.

    Args:
        url: The URL to check

    Returns:
        True if SSL should be verified, False for .onion domains
    """
    parsed = urlparse(url)
    host = parsed.hostname or ""

    # .onion domains don't have traditional SSL certificates
    if host.endswith(".onion"):
        return False

    # All clearnet URLs should verify SSL
    return True


@dataclass
class ScrapedContent:
    """Represents scraped content from a URL."""

    url: str
    title: str
    content: str
    success: bool = True
    error: Optional[str] = None
    scraped_at: datetime = field(default_factory=datetime.utcnow)
    content_length: int = 0

    def __post_init__(self):
        """Calculate content length after initialization."""
        self.content_length = len(self.content)

    def to_dict(self) -> dict:
        """Convert to dictionary."""
        return {
            "url": self.url,
            "title": self.title,
            "content": self.content,
            "success": self.success,
            "error": self.error,
            "scraped_at": self.scraped_at.isoformat(),
            "content_length": self.content_length,
        }


class ContentScraper:
    """
    Scrape content from dark web URLs via Tor proxy.

    Provides multi-threaded scraping with:
    - Text extraction and normalization
    - Script/style removal
    - Content truncation
    - Fallback behavior on failures

    Usage:
        tor = TorManager()
        scraper = ContentScraper(tor)
        content = scraper.scrape("http://example.onion")
    """

    def __init__(
        self,
        tor_manager: TorManager,
        max_workers: int = 5,
        timeout: int = 45,
        max_content_length: int = 5000,
    ):
        """
        Initialize content scraper.

        Args:
            tor_manager: TorManager instance for proxy connections
            max_workers: Maximum concurrent scraping threads
            timeout: Request timeout in seconds
            max_content_length: Maximum content characters to keep
        """
        self.tor_manager = tor_manager
        self.max_workers = max_workers
        self.timeout = timeout
        self.max_content_length = max_content_length

    def scrape(self, url: str, timeout: Optional[int] = None) -> ScrapedContent:
        """
        Scrape content from a single URL.

        SEC-017: Uses explicit SSL verification for clearnet URLs.

        Args:
            url: URL to scrape (supports .onion and clearweb)
            timeout: Custom timeout (uses default if not specified)

        Returns:
            ScrapedContent object with extracted text
        """
        actual_timeout = timeout or self.timeout

        # SEC-017: Determine SSL verification based on URL type
        verify_ssl = _should_verify_ssl(url)

        # Use Tor for .onion URLs, direct for clearweb
        is_onion = ".onion" in url.lower()
        if is_onion:
            # .onion domains don't use traditional SSL
            session = self.tor_manager.get_session(timeout=actual_timeout, verify_ssl=False)
        else:
            import requests

            session = requests.Session()
            session.timeout = actual_timeout
            session.verify = verify_ssl  # SEC-017: Explicit SSL verification

        headers = self.tor_manager.get_headers()

        try:
            # SEC-017: Pass verify parameter explicitly
            response = session.get(url, headers=headers, timeout=actual_timeout, verify=verify_ssl)

            if response.status_code != 200:
                return ScrapedContent(
                    url=url,
                    title="",
                    content="",
                    success=False,
                    error=f"HTTP {response.status_code}",
                )

            # Parse HTML and extract content
            soup = BeautifulSoup(response.text, "html.parser")

            # Remove scripts and styles
            for tag in soup(["script", "style", "noscript", "iframe"]):
                tag.decompose()

            # Extract title
            title = ""
            if soup.title and soup.title.string:
                title = soup.title.string.strip()[:200]

            # Extract and normalize text
            text = soup.get_text(separator=" ", strip=True)
            text = self._normalize_text(text)

            # Truncate if needed
            if len(text) > self.max_content_length:
                text = text[: self.max_content_length] + "... (truncated)"

            return ScrapedContent(
                url=url,
                title=title,
                content=text,
                success=True,
            )

        # REL-002: Specific exception handling
        except requests.exceptions.Timeout as e:
            logger.debug(f"Timeout scraping {url}: {e}")
            return ScrapedContent(
                url=url,
                title="",
                content="",
                success=False,
                error="Request timed out",
            )
        except requests.exceptions.ConnectionError as e:
            logger.debug(f"Connection error scraping {url}: {e}")
            return ScrapedContent(
                url=url,
                title="",
                content="",
                success=False,
                error="Connection failed",
            )
        except requests.exceptions.RequestException as e:
            logger.debug(f"Request error scraping {url}: {e}")
            return ScrapedContent(
                url=url,
                title="",
                content="",
                success=False,
                error=f"Request error: {str(e)[:100]}",
            )
        except Exception as e:
            # REL-002: Log unexpected exceptions
            logger.warning(f"Unexpected error scraping {url}: {type(e).__name__}: {e}")
            return ScrapedContent(
                url=url,
                title="",
                content="",
                success=False,
                error=f"{type(e).__name__}: {str(e)[:150]}",
            )

    def scrape_batch(
        self,
        urls: list[str],
        timeout: Optional[int] = None,
    ) -> list[ScrapedContent]:
        """
        Scrape content from multiple URLs in parallel.

        Args:
            urls: List of URLs to scrape
            timeout: Custom timeout per request

        Returns:
            List of ScrapedContent objects (same order as input)
        """
        results: dict[str, ScrapedContent] = {}

        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
            future_to_url = {executor.submit(self.scrape, url, timeout): url for url in urls}

            for future in as_completed(future_to_url):
                url = future_to_url[future]
                try:
                    result = future.result(timeout=self.timeout + 10)
                    results[url] = result
                # REL-002: Specific exception handling for batch scraping
                except FuturesTimeoutError:
                    logger.warning(f"Timeout in batch scrape for {url}")
                    results[url] = ScrapedContent(
                        url=url,
                        title="",
                        content="",
                        success=False,
                        error="Batch operation timed out",
                    )
                except Exception as e:
                    logger.warning(f"Error in batch scrape for {url}: {type(e).__name__}: {e}")
                    results[url] = ScrapedContent(
                        url=url,
                        title="",
                        content="",
                        success=False,
                        error=f"{type(e).__name__}: {str(e)[:150]}",
                    )

        # Return in original order
        return [
            results.get(url, ScrapedContent(url=url, title="", content="", success=False))
            for url in urls
        ]

    def _normalize_text(self, text: str) -> str:
        """
        Normalize extracted text.

        - Collapse multiple whitespace
        - Remove excessive newlines
        - Strip leading/trailing whitespace

        Args:
            text: Raw extracted text

        Returns:
            Normalized text string
        """
        # Collapse multiple whitespace to single space
        text = re.sub(r"\s+", " ", text)
        # Remove any remaining control characters
        text = re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]", "", text)
        return text.strip()

    def extract_links(self, url: str) -> list[str]:
        """
        Extract all .onion links from a page.

        SEC-017: Uses explicit SSL verification based on URL type.

        Args:
            url: URL to scrape for links

        Returns:
            List of unique .onion URLs found
        """
        # SEC-017: Determine SSL verification based on URL type
        verify_ssl = _should_verify_ssl(url)
        session = self.tor_manager.get_session(timeout=self.timeout, verify_ssl=verify_ssl)
        headers = self.tor_manager.get_headers()

        try:
            # SEC-017: Pass verify parameter explicitly
            response = session.get(url, headers=headers, timeout=self.timeout, verify=verify_ssl)
            if response.status_code != 200:
                return []

            # Find all onion links in the page
            onion_pattern = r"https?://[a-z0-9.]+\.onion[^\s\"'<>]*"
            links = re.findall(onion_pattern, response.text, re.IGNORECASE)

            # Deduplicate and clean
            seen = set()
            unique_links = []
            for link in links:
                clean_link = link.rstrip("/")
                if clean_link not in seen:
                    seen.add(clean_link)
                    unique_links.append(link)

            return unique_links

        # REL-002: Specific exception handling for link extraction
        except requests.exceptions.Timeout as e:
            logger.debug(f"Timeout extracting links from {url}: {e}")
            return []
        except requests.exceptions.ConnectionError as e:
            logger.debug(f"Connection error extracting links from {url}: {e}")
            return []
        except requests.exceptions.RequestException as e:
            logger.debug(f"Request error extracting links from {url}: {e}")
            return []
        except Exception as e:
            logger.warning(f"Unexpected error extracting links from {url}: {type(e).__name__}: {e}")
            return []
