"""Web crawler for documentation sites."""

import re
import time
from pathlib import Path
from urllib.parse import urljoin, urlparse
from typing import Optional
from collections import deque

import httpx
from bs4 import BeautifulSoup
import html2text


class WebCrawler:
    """Crawl documentation websites and convert to markdown."""
    
    def __init__(
        self,
        output_dir: Path,
        max_depth: int = 3,
        max_pages: int = 100,
        delay: float = 0.5,
        exclude_languages: list[str] | None = None
    ):
        self.output_dir = Path(output_dir)
        self.max_depth = max_depth
        self.max_pages = max_pages
        self.delay = delay
        
        # Default non-English language codes to exclude
        if exclude_languages is None:
            self.exclude_languages = [
                "/de/", "/fr/", "/es/", "/it/", "/ja/", "/zh/", "/ru/", "/pt/", "/ko/",
                "/docs/de/", "/docs/fr/", "/docs/es/", "/docs/it/", "/docs/ja/",
                "/zh-cn/", "/zh-tw/", "/uk/"
            ]
        else:
            self.exclude_languages = exclude_languages
            
        # Track visited URLs
        self.visited: set[str] = set()
        self.user_agent = "DocToSkill/0.1 (+https://github.com/example/doc-to-skill)"
        
        # HTML to Markdown converter
        self.h2t = html2text.HTML2Text()
        self.h2t.ignore_links = False
        self.h2t.ignore_images = False
        self.h2t.ignore_emphasis = False
        self.h2t.body_width = 0  # No wrapping
        self.h2t.unicode_snob = True
        self.h2t.skip_internal_links = False
        
        self.visited: set[str] = set()
        self.pages_saved: list[dict] = []
    
    def crawl(self, start_url: str) -> dict:
        """Start crawling from the given URL."""
        self.output_dir.mkdir(parents=True, exist_ok=True)
        
        # Parse start URL to get base domain
        parsed = urlparse(start_url)
        self.base_domain = parsed.netloc
        self.base_url = f"{parsed.scheme}://{parsed.netloc}"
        
        # BFS queue: (url, depth)
        queue: deque[tuple[str, int]] = deque([(start_url, 0)])
        
        with httpx.Client(
            headers={"User-Agent": self.user_agent},
            follow_redirects=True,
            max_redirects=10,
            timeout=30.0
        ) as client:
            while queue and len(self.visited) < self.max_pages:
                url, depth = queue.popleft()
                
                if url in self.visited:
                    continue
                
                if depth > self.max_depth:
                    continue
                
                try:
                    # Check for excluded languages
                    if any(lang in url for lang in self.exclude_languages):
                        print(f"Skipping excluded language: {url}")
                        self.visited.add(url) # Mark as visited to avoid reprocessing
                        continue

                    # Fetch and process page
                    links = self._process_page(client, url, depth)
                    
                    # Add discovered links to queue
                    if depth < self.max_depth:
                        for link in links:
                            if link not in self.visited:
                                queue.append((link, depth + 1))
                    
                    time.sleep(self.delay)
                    
                except Exception as e:
                    print(f"Error processing {url}: {e}")
                    continue
        
        # Create index file
        self._create_index()
        
        return {
            "pages_count": len(self.pages_saved),
            "output_dir": str(self.output_dir),
            "pages": self.pages_saved
        }
    
    def _process_page(self, client: httpx.Client, url: str, depth: int) -> list[str]:
        """Process a single page and return discovered links."""
        self.visited.add(url)
        
        response = client.get(url)
        response.raise_for_status()
        
        content_type = response.headers.get("content-type", "")
        if "text/html" not in content_type:
            return []
        
        soup = BeautifulSoup(response.text, "lxml")
        
        # Extract main content (try common content containers)
        main_content = self._extract_main_content(soup)
        
        # Convert to markdown
        markdown = self.h2t.handle(str(main_content))
        markdown = self._clean_markdown(markdown)
        
        # Extract title
        title = self._extract_title(soup, url)
        
        # Generate filename from URL
        filename = self._url_to_filename(url)
        
        # Add frontmatter
        full_content = f"---\ntitle: {title}\nurl: {url}\n---\n\n# {title}\n\n{markdown}"
        
        # Save file
        filepath = self.output_dir / filename
        filepath.parent.mkdir(parents=True, exist_ok=True)
        filepath.write_text(full_content, encoding="utf-8")
        
        self.pages_saved.append({
            "url": url,
            "title": title,
            "path": str(filepath),
            "depth": depth
        })
        
        print(f"[{len(self.pages_saved)}/{self.max_pages}] {title}")
        
        # Extract links for further crawling
        links = self._extract_links(soup, url)
        return links
    
    def _extract_main_content(self, soup: BeautifulSoup) -> BeautifulSoup:
        """Try to extract main content from page."""
        # Common content containers
        selectors = [
            "main",
            "article",
            "[role='main']",
            ".content",
            ".main-content",
            ".documentation",
            ".docs-content",
            ".markdown-body",
            "#content",
            "#main-content",
        ]
        
        for selector in selectors:
            content = soup.select_one(selector)
            if content:
                # Remove navigation, sidebars, etc
                for tag in content.select("nav, .sidebar, .toc, .navigation, header, footer"):
                    tag.decompose()
                return content
        
        # Fallback to body
        body = soup.body or soup
        for tag in body.select("nav, .sidebar, .toc, .navigation, header, footer, script, style"):
            tag.decompose()
        return body
    
    def _extract_title(self, soup: BeautifulSoup, url: str) -> str:
        """Extract page title."""
        # Try title tag
        title_tag = soup.find("title")
        if title_tag and title_tag.string:
            title = title_tag.string.strip()
            # Remove common suffixes
            for sep in [" | ", " - ", " :: ", " — "]:
                if sep in title:
                    title = title.split(sep)[0].strip()
            return title
        
        # Try h1
        h1 = soup.find("h1")
        if h1:
            return h1.get_text(strip=True)
        
        # Fallback to URL path
        parsed = urlparse(url)
        return parsed.path.strip("/").replace("/", " - ") or "Index"
    
    def _extract_links(self, soup: BeautifulSoup, current_url: str) -> list[str]:
        """Extract internal links from page."""
        links = []
        
        for a in soup.find_all("a", href=True):
            href = a["href"]
            
            # Skip anchors, javascript, mailto
            if href.startswith(("#", "javascript:", "mailto:", "tel:")):
                continue
            
            # Resolve relative URLs
            full_url = urljoin(current_url, href)
            parsed = urlparse(full_url)
            
            # Only same domain
            if parsed.netloc != self.base_domain:
                continue
            
            # Remove fragment
            clean_url = f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
            
            # Fix Anthropic docs broken links: /docs/en/xxx -> /en/docs/xxx
            if "/docs/en/" in clean_url and "/en/docs/" not in clean_url:
                clean_url = clean_url.replace("/docs/en/", "/en/docs/")
            
            # Skip common non-doc paths
            skip_patterns = ["/search", "/login", "/signup", "/api/v", "?"]
            if any(p in clean_url for p in skip_patterns):
                continue
            
            # Skip non-html extensions
            if parsed.path:
                ext = Path(parsed.path).suffix.lower()
                if ext and ext not in [".html", ".htm", ".md", ""]:
                    continue
            
            links.append(clean_url)
        
        return list(set(links))
    
    def _url_to_filename(self, url: str) -> str:
        """Convert URL to safe filename."""
        parsed = urlparse(url)
        path = parsed.path.strip("/")
        
        if not path or path == "":
            return "index.md"
        
        # Clean path
        path = re.sub(r"[^\w\-/]", "_", path)
        path = re.sub(r"_+", "_", path)
        
        # Handle .html extension
        if path.endswith(".html") or path.endswith(".htm"):
            path = Path(path).stem
        
        return f"{path}.md"
    
    def _clean_markdown(self, markdown: str) -> str:
        """Clean up generated markdown."""
        # Remove excessive newlines
        markdown = re.sub(r"\n{4,}", "\n\n\n", markdown)
        
        # Remove empty links
        markdown = re.sub(r"\[\s*\]\([^)]*\)", "", markdown)
        
        # Clean up list formatting
        markdown = re.sub(r"^\s*\*\s*\n", "", markdown, flags=re.MULTILINE)
        
        return markdown.strip()
    
    def _create_index(self):
        """Create index.md with links to all pages."""
        if not self.pages_saved:
            return
        
        # Group by depth
        by_depth: dict[int, list] = {}
        for page in self.pages_saved:
            depth = page["depth"]
            if depth not in by_depth:
                by_depth[depth] = []
            by_depth[depth].append(page)
        
        lines = [
            "---",
            "title: Documentation Index",
            "---",
            "",
            "# Documentation Index",
            "",
            f"Total pages: {len(self.pages_saved)}",
            "",
        ]
        
        for depth in sorted(by_depth.keys()):
            lines.append(f"## Level {depth}")
            lines.append("")
            for page in sorted(by_depth[depth], key=lambda x: x["title"]):
                rel_path = Path(page["path"]).relative_to(self.output_dir)
                lines.append(f"- [{page['title']}]({rel_path})")
            lines.append("")
        
        index_path = self.output_dir / "_index.md"
        index_path.write_text("\n".join(lines), encoding="utf-8")
