"""GitHub-based crawler for downloading documentation from repositories."""

import re
import tempfile
import shutil
from pathlib import Path
from urllib.parse import urlparse

import httpx
import html2text


class GitHubCrawler:
    """Download documentation directly from GitHub repositories."""
    
    # Common documentation paths in repos
    DOC_PATHS = [
        "docs",
        "documentation", 
        "doc",
        "wiki",
        "README.md",
        "readme.md",
        "README.rst",
    ]
    
    def __init__(
        self,
        output_dir: Path,
        max_pages: int = 100,
    ):
        self.output_dir = Path(output_dir)
        self.max_pages = max_pages
        self.pages_saved: list[dict] = []
        
        # HTML to Markdown converter
        self.h2t = html2text.HTML2Text()
        self.h2t.ignore_links = False
        self.h2t.ignore_images = False
        self.h2t.body_width = 0
    
    def crawl(self, url: str) -> dict:
        """
        Crawl documentation from GitHub.
        
        Supports URLs like:
        - https://github.com/owner/repo
        - https://github.com/owner/repo/tree/main/docs
        - https://owner.github.io/repo (redirects to repo)
        """
        self.output_dir.mkdir(parents=True, exist_ok=True)
        
        # Parse GitHub URL
        repo_info = self._parse_github_url(url)
        if not repo_info:
            print(f"Could not parse GitHub URL: {url}")
            return {"pages_count": 0, "output_dir": str(self.output_dir), "pages": []}
        
        owner, repo, path = repo_info
        
        # Try to find and download docs
        docs_found = False
        if path:
            # Specific path provided
            self._download_path(owner, repo, path)
            docs_found = len(self.pages_saved) > 0
        else:
            # Search for docs in common locations
            for doc_path in self.DOC_PATHS:
                if len(self.pages_saved) >= self.max_pages:
                    break
                
                prev_count = len(self.pages_saved)
                self._download_path(owner, repo, doc_path)
                if len(self.pages_saved) > prev_count:
                    docs_found = True
        
        # If no docs found in common folders, or we need more pages, scan root
        if len(self.pages_saved) < self.max_pages:
             # Scan root for .md and .ipynb files
             self._download_path(owner, repo, "", recursive=False)
             
        # Also try to get README (if not already downloaded)
        if len(self.pages_saved) < self.max_pages:
            self._download_readme(owner, repo)
        
        # Create index
        self._create_index()
        
        return {
            "pages_count": len(self.pages_saved),
            "output_dir": str(self.output_dir),
            "pages": self.pages_saved
        }
    
    def _parse_github_url(self, url: str) -> tuple[str, str, str] | None:
        """Parse GitHub URL to extract owner, repo, and optional path."""
        parsed = urlparse(url)
        
        # Handle github.com URLs
        if "github.com" in parsed.netloc:
            parts = parsed.path.strip("/").split("/")
            if len(parts) >= 2:
                owner = parts[0]
                repo = parts[1]
                
                # Check for path (e.g., /tree/main/docs)
                path = ""
                if len(parts) > 4 and parts[2] in ("tree", "blob"):
                    path = "/".join(parts[4:])
                
                return owner, repo, path
        
        # Handle owner.github.io URLs
        if parsed.netloc.endswith(".github.io"):
            owner = parsed.netloc.replace(".github.io", "")
            repo = parsed.path.strip("/").split("/")[0] if parsed.path.strip("/") else owner + ".github.io"
            return owner, repo, ""
        
        return None
    
    def _download_path(self, owner: str, repo: str, path: str, recursive: bool = True):
        """Download files from a specific path in the repo."""
        # GitHub API URL for contents
        if path:
            api_url = f"https://api.github.com/repos/{owner}/{repo}/contents/{path}"
        else:
            api_url = f"https://api.github.com/repos/{owner}/{repo}/contents"
        
        try:
            with httpx.Client(timeout=30.0) as client:
                response = client.get(api_url, headers={"Accept": "application/vnd.github.v3+json"})
                
                if response.status_code == 404:
                    return
                
                response.raise_for_status()
                data = response.json()
                
                # Single file
                if isinstance(data, dict) and data.get("type") == "file":
                    self._save_file(data, owner, repo)
                    return
                
                # Directory listing
                if isinstance(data, list):
                    for item in data:
                        if len(self.pages_saved) >= self.max_pages:
                            break
                        
                        if item["type"] == "file" and self._is_doc_file(item["name"]):
                            self._save_file(item, owner, repo)
                        elif item["type"] == "dir" and recursive:
                            # Recursively download subdirectories
                            self._download_path(owner, repo, item["path"], recursive=True)
                            
        except Exception as e:
            # Only print error if it's not a root scan (root might have many files/errors)
            if path:
                print(f"Error downloading {path}: {e}")
    
    def _is_doc_file(self, filename: str) -> bool:
        """Check if file is a documentation file."""
        doc_extensions = {".md", ".rst", ".txt", ".html", ".htm", ".ipynb"}
        return Path(filename).suffix.lower() in doc_extensions
    
    def _save_file(self, file_info: dict, owner: str, repo: str):
        """Download and save a single file."""
        try:
            # Get raw content
            download_url = file_info.get("download_url")
            if not download_url:
                return
            
            with httpx.Client(timeout=30.0) as client:
                response = client.get(download_url)
                response.raise_for_status()
                content = response.text
                
                # For binary files (like ipynb sometimes), getting text might fail encoding
                # So we might need to handle content differently if it's not text
            
            filename = file_info["name"]
            
            # Convert notebooks to markdown
            if filename.endswith(".ipynb"):
                content = self._notebook_to_md(content)
                filename = Path(filename).stem + ".md"
            elif filename.endswith((".html", ".htm")):
                content = self.h2t.handle(content)
                filename = Path(filename).stem + ".md"
            elif filename.endswith(".rst"):
                content = self._rst_to_md(content)
                filename = Path(filename).stem + ".md"
            
            # Extract title
            title = self._extract_title(content, filename)
            
            # Create file path preserving directory structure
            rel_path = file_info["path"]
            if not rel_path.endswith(".md"):
                rel_path = str(Path(rel_path).with_suffix(".md"))
            
            filepath = self.output_dir / rel_path
            filepath.parent.mkdir(parents=True, exist_ok=True)
            
            # Add frontmatter
            url = f"https://github.com/{owner}/{repo}/blob/main/{file_info['path']}"
            full_content = f"---\ntitle: {title}\nurl: {url}\nsource: github\n---\n\n{content}"
            
            filepath.write_text(full_content, encoding="utf-8")
            
            self.pages_saved.append({
                "url": url,
                "title": title,
                "path": str(filepath),
                "depth": rel_path.count("/")
            })
            
            print(f"[{len(self.pages_saved)}/{self.max_pages}] {title}")
            
        except Exception as e:
            print(f"Error saving file {file_info.get('name')}: {e}")

    def _notebook_to_md(self, content: str) -> str:
        """Convert Jupyter Notebook content to Markdown."""
        import json
        
        try:
            nb = json.loads(content)
            markdown = []
            
            for cell in nb.get("cells", []):
                cell_type = cell.get("cell_type")
                source = "".join(cell.get("source", []))
                
                if cell_type == "markdown":
                    markdown.append(source)
                elif cell_type == "code":
                    markdown.append("```python")
                    markdown.append(source)
                    markdown.append("```")
            
            return "\n\n".join(markdown)
        except Exception as e:
            print(f"Error converting notebook: {e}")
            return f"Error converting notebook: {e}"
    
    def _download_readme(self, owner: str, repo: str):
        """Download repository README."""
        for readme in ["README.md", "readme.md", "README.rst", "README.txt"]:
            api_url = f"https://api.github.com/repos/{owner}/{repo}/contents/{readme}"
            
            try:
                with httpx.Client(timeout=30.0) as client:
                    response = client.get(api_url)
                    if response.status_code == 200:
                        data = response.json()
                        self._save_file(data, owner, repo)
                        break
            except:
                continue
    
    def _extract_title(self, content: str, filename: str) -> str:
        """Extract title from content."""
        # Try to find h1
        h1_match = re.search(r'^#\s+(.+)$', content, re.MULTILINE)
        if h1_match:
            return h1_match.group(1).strip()
        
        # Use filename
        return Path(filename).stem.replace("-", " ").replace("_", " ").title()
    
    def _rst_to_md(self, content: str) -> str:
        """Basic RST to Markdown conversion."""
        # Convert headers
        content = re.sub(r'^(.+)\n=+$', r'# \1', content, flags=re.MULTILINE)
        content = re.sub(r'^(.+)\n-+$', r'## \1', content, flags=re.MULTILINE)
        
        # Convert code blocks
        content = re.sub(r'::\s*\n\n', '\n```\n', content)
        
        return content
    
    def _create_index(self):
        """Create index.md with links to all pages."""
        if not self.pages_saved:
            return
        
        lines = [
            "---",
            "title: Documentation Index",
            "source: github",
            "---",
            "",
            "# Documentation Index",
            "",
            f"Total pages: {len(self.pages_saved)}",
            "",
        ]
        
        for page in sorted(self.pages_saved, key=lambda x: x["title"]):
            rel_path = Path(page["path"]).relative_to(self.output_dir)
            lines.append(f"- [{page['title']}]({rel_path})")
        
        index_path = self.output_dir / "_index.md"
        index_path.write_text("\n".join(lines), encoding="utf-8")
