# ankur_scraper/core/crawler.py

from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
import requests
import time
from collections import deque
from ankur_scraper.logging_config import get_logger
from ankur_scraper.core.url_utils import (
    normalize_url,
    is_same_domain,
    clean_url_for_comparison,
    should_skip_url
)

info_logger = get_logger("info")
error_logger = get_logger("error")

IGNORED_SCHEMES = ("mailto:", "tel:", "javascript:", "data:", "file:")
IGNORED_EXTENSIONS = (
    ".pdf", ".jpg", ".jpeg", ".png", ".svg", ".gif", ".webp", ".ico",
    ".zip", ".tar", ".gz", ".rar", ".7z",
    ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx",
    ".mp4", ".mp3", ".wav", ".avi", ".mov",
    ".exe", ".dmg", ".pkg"
)


def is_valid_http_url(url: str) -> bool:
    """Check if URL has valid HTTP/HTTPS scheme"""
    try:
        parsed = urlparse(url)
        return parsed.scheme in ("http", "https") and bool(parsed.netloc)
    except:
        return False


def should_ignore_url_extended(url: str) -> bool:
    """Enhanced URL filtering"""
    # Check basic patterns
    if should_skip_url(url):
        return True
    
    # Check schemes
    url_lower = url.lower()
    if any(url_lower.startswith(scheme) for scheme in IGNORED_SCHEMES):
        return True
    
    # Check extensions
    if any(url_lower.endswith(ext) for ext in IGNORED_EXTENSIONS):
        return True
    
    # Check for empty or fragment-only URLs
    if url.strip() in ("#", "", "/"):
        return True
    
    # Check for common non-content paths
    non_content_keywords = [
        'javascript:', 'void(0)', 'mailto:', 'tel:',
        '/cdn-cgi/', '/wp-content/', '/wp-includes/',
        '.min.js', '.min.css'
    ]
    
    if any(keyword in url_lower for keyword in non_content_keywords):
        return True
    
    return False


def normalize_crawl_url(href: str, base_url: str) -> str:
    """Normalize URL for crawling (resolve relative, remove fragments)"""
    try:
        # Join with base URL to handle relative links
        absolute = urljoin(base_url, href)
        # Remove fragment
        return absolute.split("#")[0]
    except:
        return href


def fetch_with_retries(url, headers, retries=3, backoff=2, timeout=10):
    """Fetch a URL with retries and exponential backoff."""
    # Ensure URL has protocol
    if not url.startswith(('http://', 'https://')):
        error_logger.error(f"URL missing protocol: {url}")
        return None
    
    for attempt in range(1, retries + 1):
        try:
            resp = requests.get(
                url,
                headers=headers,
                timeout=timeout,
                allow_redirects=True
            )
            
            # Check if successful
            if resp.status_code == 200:
                content_type = resp.headers.get("Content-Type", "").lower()
                if "text/html" in content_type:
                    return resp.text
                else:
                    error_logger.debug(f"Non-HTML content type ({content_type}) for {url}")
                    return None
            
            # Don't retry certain status codes
            elif resp.status_code in [404, 410, 403, 401]:
                error_logger.debug(f"HTTP {resp.status_code} for {url} (not retrying)")
                return None
            
            else:
                error_logger.debug(f"HTTP {resp.status_code} for {url}")
                
        except requests.exceptions.Timeout:
            error_logger.debug(f"Timeout ({attempt}/{retries}) for {url}")
        except requests.exceptions.SSLError:
            error_logger.debug(f"SSL error for {url}")
            # Try switching to http
            if url.startswith('https://') and attempt == 1:
                url = url.replace('https://', 'http://', 1)
                continue
            return None
        except requests.RequestException as e:
            error_logger.debug(f"Request error ({attempt}/{retries}) for {url}: {str(e)[:50]}")
        
        # Wait before retry
        if attempt < retries:
            sleep_time = backoff * attempt
            time.sleep(sleep_time)
    
    return None


def get_internal_links(
    base_url: str,
    max_depth: int = 1,
    user_agent: str = None,
    max_pages: int = 200,
    delay: float = 0.5,
    retries: int = 3,
    backoff: int = 2
) -> set:
    """
    Crawl site and return internal links using BFS.
    
    Args:
        base_url: Starting URL (should be normalized with protocol)
        max_depth: Maximum depth to crawl
        user_agent: Custom user agent
        max_pages: Maximum number of pages to crawl
        delay: Seconds to wait between requests (rate limiting)
        retries: Retry attempts per request
        backoff: Multiplier for exponential backoff
    
    Returns:
        Set of discovered internal URLs (all with valid protocols)
    """
    # Validate base URL has protocol
    if not base_url.startswith(('http://', 'https://')):
        error_logger.error(f"Base URL missing protocol: {base_url}")
        raise ValueError(f"Base URL must include http:// or https://: {base_url}")
    
    visited = {}  # Map cleaned URL -> original URL
    queue = deque([(base_url, 0)])
    
    # Use realistic user agent if not provided
    if not user_agent:
        user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
    
    headers = {
        "User-Agent": user_agent,
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        "Accept-Language": "en-US,en;q=0.9",
    }
    
    info_logger.info(f"Starting crawl from {base_url} (max_depth={max_depth}, max_pages={max_pages})")
    
    while queue and len(visited) < max_pages:
        current_url, depth = queue.popleft()
        
        # Validate URL has protocol
        if not current_url.startswith(('http://', 'https://')):
            error_logger.warning(f"Skipping URL without protocol: {current_url}")
            continue
        
        # Clean URL for comparison
        try:
            cleaned_current = clean_url_for_comparison(current_url)
        except Exception as e:
            error_logger.debug(f"Failed to clean URL {current_url}: {e}")
            continue
        
        # Skip if already visited or depth exceeded
        if cleaned_current in visited or depth > max_depth:
            continue
        
        visited[cleaned_current] = current_url  # Store original URL
        
        # Fetch the page
        info_logger.debug(f"Crawling [{len(visited)}/{max_pages}] depth={depth}: {current_url}")
        html = fetch_with_retries(current_url, headers, retries=retries, backoff=backoff)
        
        if not html:
            continue
        
        # Parse and extract links
        try:
            soup = BeautifulSoup(html, "lxml")
            
            for link_tag in soup.find_all("a", href=True):
                href = link_tag["href"].strip()
                
                # Skip obviously bad URLs
                if should_ignore_url_extended(href):
                    continue
                
                # Normalize the URL
                try:
                    normalized = normalize_crawl_url(href, current_url)
                except:
                    continue
                
                # Check if it's a valid HTTP URL
                if not is_valid_http_url(normalized):
                    continue
                
                # Check if it's on the same domain
                if not is_same_domain(base_url, normalized):
                    continue
                
                # Clean for comparison
                try:
                    cleaned_normalized = clean_url_for_comparison(normalized)
                except Exception as e:
                    error_logger.debug(f"Failed to clean URL {normalized}: {e}")
                    continue
                
                # Add to queue if not visited
                if cleaned_normalized not in visited:
                    queue.append((normalized, depth + 1))
        
        except Exception as e:
            error_logger.error(f"Parse error at {current_url}: {str(e)[:80]}")
        
        # Rate limiting
        if delay > 0:
            time.sleep(delay)
    
    info_logger.info(f"[bold green]Crawl complete:[/] Found {len(visited)} internal links")
    
    # Return the original URLs (with protocols)
    result_urls = set(visited.values())
    
    # Final validation - ensure all URLs have protocols
    validated_urls = set()
    for url in result_urls:
        if url.startswith(('http://', 'https://')):
            validated_urls.add(url)
        else:
            error_logger.warning(f"Filtering out URL without protocol: {url}")
    
    return validated_urls


# Additional utility for debugging
def validate_url_set(urls: set) -> tuple[set, list]:
    """
    Validate a set of URLs and return valid ones plus errors.
    
    Returns:
        (valid_urls, errors)
    """
    valid = set()
    errors = []
    
    for url in urls:
        if not isinstance(url, str):
            errors.append(f"Non-string URL: {type(url)}")
            continue
        
        if not url.startswith(('http://', 'https://')):
            errors.append(f"Missing protocol: {url}")
            continue
        
        if not is_valid_http_url(url):
            errors.append(f"Invalid URL format: {url}")
            continue
        
        valid.add(url)
    
    return valid, errors