"""Firecrawl-based crawler for protected sites."""

import re
from pathlib import Path
from urllib.parse import urlparse

import httpx


class FirecrawlCrawler:
    """Crawl sites using Firecrawl API (local Docker or cloud)."""
    
    def __init__(
        self,
        output_dir: Path,
        max_depth: int = 3,
        max_pages: int = 100,
        api_url: str = "http://localhost:3002",  # Local Docker default
        api_key: str | None = None,  # Only needed for cloud
    ):
        self.output_dir = Path(output_dir)
        self.max_depth = max_depth
        self.max_pages = max_pages
        self.api_url = api_url.rstrip("/")
        self.api_key = api_key
        
        self.pages_saved: list[dict] = []
    
    def crawl(self, start_url: str) -> dict:
        """Crawl using Firecrawl API."""
        self.output_dir.mkdir(parents=True, exist_ok=True)
        
        # Check if Firecrawl is available
        if not self._check_firecrawl():
            print(f"Firecrawl not available at {self.api_url}")
            print("Start Firecrawl with: docker compose up -d")
            print("Or use cloud API: set FIRECRAWL_API_KEY")
            return {"pages_count": 0, "output_dir": str(self.output_dir), "pages": []}
        
        # Use crawl endpoint for multi-page crawl
        try:
            result = self._crawl_site(start_url)
            
            if result:
                for page_data in result:
                    if len(self.pages_saved) >= self.max_pages:
                        break
                    self._save_page(page_data)
            
        except Exception as e:
            print(f"Firecrawl error: {e}")
            # Fallback to single page scrape
            try:
                page_data = self._scrape_page(start_url)
                if page_data:
                    self._save_page(page_data)
            except Exception as e2:
                print(f"Scrape error: {e2}")
        
        self._create_index()
        
        return {
            "pages_count": len(self.pages_saved),
            "output_dir": str(self.output_dir),
            "pages": self.pages_saved
        }
    
    def _check_firecrawl(self) -> bool:
        """Check if Firecrawl API is available."""
        try:
            with httpx.Client(timeout=5.0) as client:
                # Try health endpoint
                response = client.get(f"{self.api_url}/health")
                return response.status_code == 200
        except:
            return False
    
    def _get_headers(self) -> dict:
        """Get API headers."""
        headers = {"Content-Type": "application/json"}
        if self.api_key:
            headers["Authorization"] = f"Bearer {self.api_key}"
        return headers
    
    def _crawl_site(self, url: str) -> list[dict]:
        """Crawl entire site using Firecrawl."""
        with httpx.Client(timeout=120.0) as client:
            response = client.post(
                f"{self.api_url}/v1/crawl",
                headers=self._get_headers(),
                json={
                    "url": url,
                    "limit": self.max_pages,
                    "maxDepth": self.max_depth,
                    "scrapeOptions": {
                        "formats": ["markdown"]
                    }
                }
            )
            response.raise_for_status()
            data = response.json()
            
            # Check if async job
            if "id" in data:
                return self._wait_for_crawl(data["id"])
            
            return data.get("data", [])
    
    def _wait_for_crawl(self, job_id: str, max_wait: int = 300) -> list[dict]:
        """Wait for async crawl job to complete."""
        import time
        
        with httpx.Client(timeout=30.0) as client:
            for _ in range(max_wait // 5):
                response = client.get(
                    f"{self.api_url}/v1/crawl/{job_id}",
                    headers=self._get_headers()
                )
                data = response.json()
                
                status = data.get("status", "")
                if status == "completed":
                    return data.get("data", [])
                elif status == "failed":
                    print(f"Crawl failed: {data.get('error', 'Unknown error')}")
                    return []
                
                # Show progress
                completed = data.get("completed", 0)
                total = data.get("total", 0)
                if total > 0:
                    print(f"Crawling... {completed}/{total} pages")
                
                time.sleep(5)
        
        print("Crawl timed out")
        return []
    
    def _scrape_page(self, url: str) -> dict | None:
        """Scrape a single page."""
        with httpx.Client(timeout=60.0) as client:
            response = client.post(
                f"{self.api_url}/v1/scrape",
                headers=self._get_headers(),
                json={
                    "url": url,
                    "formats": ["markdown"]
                }
            )
            response.raise_for_status()
            data = response.json()
            return data.get("data")
    
    def _save_page(self, page_data: dict):
        """Save a crawled page."""
        try:
            url = page_data.get("metadata", {}).get("sourceURL", "") or page_data.get("url", "")
            markdown = page_data.get("markdown", "")
            
            if not markdown or not url:
                return
            
            # Extract metadata
            metadata = page_data.get("metadata", {})
            title = metadata.get("title", "") or self._extract_title(markdown, url)
            
            # Generate filename
            filename = self._url_to_filename(url)
            
            # Add frontmatter
            full_content = f"---\ntitle: {title}\nurl: {url}\nsource: firecrawl\n---\n\n{markdown}"
            
            # Save
            filepath = self.output_dir / filename
            filepath.parent.mkdir(parents=True, exist_ok=True)
            filepath.write_text(full_content, encoding="utf-8")
            
            self.pages_saved.append({
                "url": url,
                "title": title,
                "path": str(filepath),
                "depth": 0
            })
            
            print(f"[{len(self.pages_saved)}/{self.max_pages}] {title}")
            
        except Exception as e:
            print(f"Error saving page: {e}")
    
    def _extract_title(self, markdown: str, url: str) -> str:
        """Extract title from markdown."""
        h1_match = re.search(r'^#\s+(.+)$', markdown, re.MULTILINE)
        if h1_match:
            return h1_match.group(1).strip()
        
        parsed = urlparse(url)
        return parsed.path.strip("/").replace("/", " - ") or "Index"
    
    def _url_to_filename(self, url: str) -> str:
        """Convert URL to filename."""
        parsed = urlparse(url)
        path = parsed.path.strip("/")
        
        if not path:
            return "index.md"
        
        path = re.sub(r"[^\w\-/]", "_", path)
        path = re.sub(r"_+", "_", path)
        
        if path.endswith(".html") or path.endswith(".htm"):
            path = Path(path).stem
        
        return f"{path}.md"
    
    def _create_index(self):
        """Create index file."""
        if not self.pages_saved:
            return
        
        lines = [
            "---",
            "title: Documentation Index", 
            "source: firecrawl",
            "---",
            "",
            "# Documentation Index",
            "",
            f"Total pages: {len(self.pages_saved)}",
            "",
        ]
        
        for page in sorted(self.pages_saved, key=lambda x: x["title"]):
            rel_path = Path(page["path"]).relative_to(self.output_dir)
            lines.append(f"- [{page['title']}]({rel_path})")
        
        index_path = self.output_dir / "_index.md"
        index_path.write_text("\n".join(lines), encoding="utf-8")
