# ankur_scraper/core/page_scraper.py

import httpx
import time
from ankur_scraper.core.html_extractor import extract_text_sections
from ankur_scraper.logging_config import get_logger
import re

# Lazy import Playwright to avoid errors if not installed
try:
    from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeout
    PLAYWRIGHT_AVAILABLE = True
except ImportError:
    PLAYWRIGHT_AVAILABLE = False
    PlaywrightTimeout = TimeoutError

info_logger = get_logger("info")
error_logger = get_logger("error")


def fetch_static_html(url, timeout=10, headers=None, retries=3, backoff=2):
    """Try to fetch HTML using httpx with retries."""
    attempt = 0
    while attempt < retries:
        try:
            resp = httpx.get(url, timeout=timeout, headers=headers, follow_redirects=True)
            if resp.status_code == 200:
                content_type = resp.headers.get("Content-Type", "").lower()
                if "text/html" in content_type:
                    return resp.text
                else:
                    error_logger.warning(f"Non-HTML content type: {content_type}")
                    return None
            else:
                error_logger.warning(f"HTTP {resp.status_code} for {url}")
                if resp.status_code in [403, 429]:  # Forbidden or rate limited
                    return None  # Don't retry these
        except Exception as e:
            error_logger.error(f"Static fetch error ({attempt+1}/{retries}) for {url}: {e}")
        
        if attempt < retries - 1:
            time.sleep(backoff * (attempt + 1))
        attempt += 1
    
    return None


def detect_dynamic_content_needed(html: str) -> bool:
    """
    Analyze HTML to determine if JavaScript rendering is needed.
    
    Returns True if the page likely needs dynamic rendering.
    """
    if not html:
        return True
    
    # Check 1: Very small HTML (likely skeleton)
    if len(html) < 1000:
        info_logger.debug("HTML too small, likely needs JS rendering")
        return True
    
    # Check 2: Look for common JS framework indicators
    js_frameworks = [
        r'<div[^>]+id=["\']root["\']',  # React
        r'<div[^>]+id=["\']app["\']',   # Vue/React
        r'ng-app',                        # Angular
        r'data-react',
        r'__NEXT_DATA__',                # Next.js
        r'__NUXT__',                     # Nuxt.js
    ]
    
    for pattern in js_frameworks:
        if re.search(pattern, html, re.IGNORECASE):
            info_logger.debug(f"Detected JS framework pattern: {pattern}")
            return True
    
    # Check 3: High script-to-content ratio
    soup_preview = html[:10000]  # Check first 10KB
    script_tags = len(re.findall(r'<script', soup_preview, re.IGNORECASE))
    content_tags = len(re.findall(r'<(p|div|article|section)', soup_preview, re.IGNORECASE))
    
    if script_tags > 5 and content_tags < 5:
        info_logger.debug(f"High script/content ratio: {script_tags} scripts, {content_tags} content tags")
        return True
    
    # Check 4: Look for loading indicators
    loading_indicators = [
        r'loading', r'spinner', r'skeleton',
        r'Please enable JavaScript',
        r'JavaScript is required'
    ]
    
    for indicator in loading_indicators:
        if re.search(indicator, html, re.IGNORECASE):
            info_logger.debug(f"Found loading indicator: {indicator}")
            return True
    
    # Check 5: Extract sample text and see if it's meaningful
    from bs4 import BeautifulSoup
    try:
        soup = BeautifulSoup(html, 'lxml')
        for tag in soup(['script', 'style', 'noscript']):
            tag.decompose()
        
        text = soup.get_text()
        text = re.sub(r'\s+', ' ', text).strip()
        
        # If very little visible text, probably needs JS
        if len(text) < 200:
            info_logger.debug(f"Very little visible text: {len(text)} chars")
            return True
            
    except Exception as e:
        error_logger.debug(f"Error parsing HTML for dynamic detection: {e}")
        return True
    
    return False


class PageScraper:
    def __init__(self, use_dynamic=False, timeout=10, user_agent=None):
        self.use_dynamic = use_dynamic
        self.timeout = timeout
        self.user_agent = user_agent or "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
        self.playwright = None
        self.browser = None
        self.context = None

    def __enter__(self):
        if self.use_dynamic:
            try:
                self.playwright = sync_playwright().start()
                self.browser = self.playwright.chromium.launch(
                    headless=True,
                    args=[
                        '--disable-blink-features=AutomationControlled',
                        '--disable-dev-shm-usage',
                        '--no-sandbox'
                    ]
                )
                self.context = self.browser.new_context(
                    user_agent=self.user_agent,
                    viewport={'width': 1920, 'height': 1080},
                    java_script_enabled=True
                )
                # Set extra headers to appear more like a real browser
                self.context.set_extra_http_headers({
                    'Accept-Language': 'en-US,en;q=0.9',
                    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
                })
            except Exception as e:
                error_logger.error(f"Failed to initialize Playwright: {e}")
                self.playwright = None
                self.browser = None
                self.context = None
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        if self.context:
            try:
                self.context.close()
            except:
                pass
        if self.browser:
            try:
                self.browser.close()
            except:
                pass
        if self.playwright:
            try:
                self.playwright.stop()
            except:
                pass

    def fetch_dynamic_html(self, url):
        """Render page using shared Playwright browser with smart wait strategies."""
        if not self.context:
            error_logger.error("Playwright context not initialized")
            return None
        
        try:
            page = self.context.new_page()
            page.set_default_navigation_timeout(self.timeout * 1000)
            
            # Navigate to page
            page.goto(url, wait_until="domcontentloaded")
            
            # Wait for network to be mostly idle
            try:
                page.wait_for_load_state("networkidle", timeout=self.timeout * 1000)
            except PlaywrightTimeout:
                info_logger.debug("Network didn't become idle, continuing anyway")
            
            # Additional wait for content to render
            time.sleep(1)
            
            # Try to wait for common content containers
            content_selectors = [
                'article', 'main', '[role="main"]',
                '.content', '.post-content', '.entry-content',
                '#content', '#main-content'
            ]
            
            for selector in content_selectors:
                try:
                    page.wait_for_selector(selector, timeout=2000)
                    info_logger.debug(f"Found content selector: {selector}")
                    break
                except PlaywrightTimeout:
                    continue
            
            # Get the rendered HTML
            html = page.content()
            page.close()
            
            return html
            
        except PlaywrightTimeout as e:
            error_logger.error(f"Playwright timeout for {url}: {e}")
            return None
        except Exception as e:
            error_logger.error(f"Dynamic fetch failed for {url}: {e}")
            return None

    def scrape(self, url):
        """
        Scrape a URL with smart static/dynamic detection.
        
        Returns: List of (section_title, content) tuples
        """
        info_logger.info(f"Scraping: {url}")
        headers = {"User-Agent": self.user_agent}
        
        # Step 1: Always try static first (faster)
        html = fetch_static_html(url, timeout=self.timeout, headers=headers)
        
        if not html:
            # Static fetch completely failed
            if self.use_dynamic:
                info_logger.info(f"Static fetch failed, trying dynamic rendering for {url}")
                html = self.fetch_dynamic_html(url)
            else:
                raise Exception("Static fetch failed and dynamic rendering not enabled")
        
        # Step 2: Analyze if we need dynamic rendering
        elif detect_dynamic_content_needed(html):
            if self.use_dynamic:
                info_logger.info(f"Dynamic content detected, re-fetching with browser for {url}")
                dynamic_html = self.fetch_dynamic_html(url)
                if dynamic_html:
                    html = dynamic_html
            else:
                info_logger.warning(f"Page likely needs JS rendering, but dynamic mode not enabled. Use --dynamic flag.")
        else:
            info_logger.debug("Static HTML appears sufficient")
        
        # Step 3: Extract content
        if html:
            sections = extract_text_sections(html)
            if sections:
                return sections
            else:
                # If no sections found, try dynamic as last resort
                if self.use_dynamic and not detect_dynamic_content_needed(html):
                    info_logger.info("No content extracted from static, trying dynamic")
                    dynamic_html = self.fetch_dynamic_html(url)
                    if dynamic_html:
                        sections = extract_text_sections(dynamic_html)
                        if sections:
                            return sections
                
                raise Exception("No content sections extracted from page")
        
        raise Exception("Failed to load HTML content")