# ankur_scraper/core/html_extractor.py

from bs4 import BeautifulSoup, NavigableString, Comment
import re
from ankur_scraper.logging_config import get_logger

info_logger = get_logger("info")


def clean_text(text):
    """Normalize and clean extracted text."""
    if not text:
        return ""
    # Replace multiple whitespace with single space
    text = re.sub(r'\s+', ' ', text)
    # Remove zero-width characters and other invisible chars
    text = re.sub(r'[\u200b-\u200f\u202a-\u202e\ufeff]', '', text)
    return text.strip()


def is_likely_boilerplate(text: str, tag_name: str = None) -> bool:
    """
    Heuristic to detect boilerplate content (nav, footer, ads, etc.)
    """
    if not text or len(text) < 20:
        return True
    
    text_lower = text.lower()
    
    # Common boilerplate phrases
    boilerplate_indicators = [
        'cookie', 'privacy policy', 'terms of service', 'all rights reserved',
        'copyright ©', '© 20', 'sign up', 'subscribe', 'newsletter',
        'follow us', 'social media', 'share this'
    ]
    
    # Check if text is mostly boilerplate
    matches = sum(1 for indicator in boilerplate_indicators if indicator in text_lower)
    if matches >= 2:
        return True
    
    # Check for excessive links (navigation)
    word_count = len(text.split())
    if word_count < 50 and text.count('|') > 3:  # Pipe-separated nav
        return True
    
    return False


def safe_get_text(tag):
    """Safely extract text from a tag, handling None"""
    if tag is None:
        return ""
    try:
        return tag.get_text() if hasattr(tag, 'get_text') else str(tag)
    except:
        return ""


def safe_find(tag, *args, **kwargs):
    """Safely find a tag, returning None if tag is None"""
    if tag is None:
        return None
    try:
        return tag.find(*args, **kwargs)
    except:
        return None


def safe_find_all(tag, *args, **kwargs):
    """Safely find all tags, returning empty list if tag is None"""
    if tag is None:
        return []
    try:
        return tag.find_all(*args, **kwargs)
    except:
        return []


def extract_from_semantic_tags(soup) -> list:
    """Strategy 1: Extract from semantic HTML5 tags"""
    sections = []
    
    if soup is None:
        return sections
    
    # Priority order of semantic containers
    semantic_tags = ['article', 'main', 'section']
    
    for tag_name in semantic_tags:
        try:
            for tag in safe_find_all(soup, tag_name):
                if tag is None:
                    continue
                    
                # Find heading within this container
                heading = safe_find(tag, ['h1', 'h2', 'h3', 'h4', 'h5', 'h6'])
                title = clean_text(safe_get_text(heading)) if heading else "Content Section"
                
                # Get all text content
                content = clean_text(safe_get_text(tag))
                
                if content and len(content.split()) > 10 and not is_likely_boilerplate(content):
                    sections.append((title, content))
        except Exception as e:
            info_logger.debug(f"Error in semantic tags strategy: {e}")
            continue
    
    return sections


def extract_from_content_divs(soup) -> list:
    """Strategy 2: Extract from divs with content-related classes/ids"""
    sections = []
    
    if soup is None:
        return sections
    
    # Common class/id patterns for main content
    content_patterns = [
        'content', 'main-content', 'post-content', 'entry-content',
        'article-content', 'page-content', 'body-content',
        'post-body', 'entry-body', 'article-body',
        'text', 'description', 'summary'
    ]
    
    try:
        for div in safe_find_all(soup, ['div', 'section']):
            if div is None:
                continue
            
            # Safely get class and id attributes
            try:
                class_list = div.get('class') if div.has_attr('class') else []
                if class_list is None:
                    class_list = []
                class_str = ' '.join(class_list).lower() if class_list else ''
            except:
                class_str = ''
            
            try:
                id_str = div.get('id', '').lower() if div.has_attr('id') else ''
                if id_str is None:
                    id_str = ''
            except:
                id_str = ''
            
            # Check if class or id matches content patterns
            if any(pattern in class_str or pattern in id_str for pattern in content_patterns):
                heading = safe_find(div, ['h1', 'h2', 'h3', 'h4'])
                title = clean_text(safe_get_text(heading)) if heading else "Content Section"
                
                content = clean_text(safe_get_text(div))
                
                if content and len(content.split()) > 10 and not is_likely_boilerplate(content):
                    sections.append((title, content))
    except Exception as e:
        info_logger.debug(f"Error in content divs strategy: {e}")
    
    return sections


def extract_from_headings(soup) -> list:
    """Strategy 3: Extract content following headings"""
    sections = []
    
    if soup is None:
        return sections
    
    try:
        headings = safe_find_all(soup, ['h1', 'h2', 'h3', 'h4'])
        
        for heading in headings:
            if heading is None:
                continue
                
            title = clean_text(safe_get_text(heading))
            
            if not title or is_likely_boilerplate(title):
                continue
            
            content_parts = []
            
            # Collect all content until next heading
            try:
                for sibling in heading.find_next_siblings():
                    if sibling is None:
                        continue
                    
                    if sibling.name in ['h1', 'h2', 'h3', 'h4']:
                        break
                    
                    if sibling.name in ['p', 'div', 'ul', 'ol', 'dl', 'blockquote', 'pre']:
                        text = clean_text(safe_get_text(sibling))
                        if text:
                            content_parts.append(text)
            except:
                pass
            
            content = ' '.join(content_parts)
            
            if content and len(content.split()) > 10 and not is_likely_boilerplate(content):
                sections.append((title, content))
    except Exception as e:
        info_logger.debug(f"Error in headings strategy: {e}")
    
    return sections


def extract_from_paragraphs(soup) -> list:
    """Strategy 4: Extract substantial paragraphs (fallback)"""
    sections = []
    
    if soup is None:
        return sections
    
    try:
        paragraphs = safe_find_all(soup, ['p', 'blockquote'])
        
        # Group consecutive paragraphs
        current_section = []
        
        for p in paragraphs:
            if p is None:
                continue
                
            text = clean_text(safe_get_text(p))
            
            if text and len(text.split()) > 5:
                current_section.append(text)
        
        # Combine into one section if we have content
        if current_section:
            combined = ' '.join(current_section)
            if len(combined.split()) > 20:
                sections.append(("Main Content", combined))
    except Exception as e:
        info_logger.debug(f"Error in paragraphs strategy: {e}")
    
    return sections


def extract_lists_and_tables(soup) -> list:
    """Strategy 5: Extract structured content (lists, tables)"""
    sections = []
    
    if soup is None:
        return sections
    
    try:
        # Extract lists with context
        for list_tag in safe_find_all(soup, ['ul', 'ol']):
            if list_tag is None:
                continue
            
            # Try to find a heading before this list
            prev_heading = None
            try:
                for prev in list_tag.find_all_previous(['h1', 'h2', 'h3', 'h4', 'h5', 'h6']):
                    prev_heading = prev
                    break
            except:
                pass
            
            title = clean_text(safe_get_text(prev_heading)) if prev_heading else "List Content"
            
            # Extract list items
            items = []
            for li in safe_find_all(list_tag, 'li'):
                if li is None:
                    continue
                item_text = clean_text(safe_get_text(li))
                if item_text:
                    items.append(item_text)
            
            if items and len(items) >= 2:
                content = ' | '.join(items)
                if len(content.split()) > 10:
                    sections.append((title, content))
        
        # Extract tables
        for table in safe_find_all(soup, 'table'):
            if table is None:
                continue
                
            rows = []
            for tr in safe_find_all(table, 'tr'):
                if tr is None:
                    continue
                cells = [clean_text(safe_get_text(cell)) for cell in safe_find_all(tr, ['td', 'th'])]
                cells = [c for c in cells if c]  # Remove empty cells
                if any(cells):
                    rows.append(' | '.join(cells))
            
            if rows:
                content = ' // '.join(rows)
                if len(content.split()) > 10:
                    sections.append(("Table Data", content))
    except Exception as e:
        info_logger.debug(f"Error in lists/tables strategy: {e}")
    
    return sections


def remove_unwanted_elements(soup):
    """Remove elements that are definitely not content"""
    if soup is None:
        return
    
    try:
        # Remove scripts, styles, and other non-visible elements
        for tag in safe_find_all(soup, ['script', 'style', 'noscript', 'iframe', 'embed', 'object']):
            if tag:
                tag.decompose()
        
        # Remove comments
        for comment in soup.find_all(text=lambda text: isinstance(text, Comment)):
            if comment:
                comment.extract()
        
        # Remove hidden elements
        for tag in safe_find_all(soup, True):  # Find all tags
            if tag is None:
                continue
            try:
                if tag.has_attr('style'):
                    style = tag.get('style', '').lower()
                    if style and ('display:none' in style or 'display: none' in style or 
                                  'visibility:hidden' in style or 'visibility: hidden' in style):
                        tag.decompose()
            except:
                continue
        
        # Remove elements with classes/ids indicating non-content
        unwanted_patterns = ['nav', 'menu', 'advertisement', 'ad-container']
        
        for tag in safe_find_all(soup, ['div', 'section', 'aside']):
            if tag is None:
                continue
            
            try:
                # Safely get class and id attributes
                class_list = tag.get('class') if tag.has_attr('class') else []
                if class_list is None:
                    class_list = []
                class_str = ' '.join(class_list).lower() if class_list else ''
            except:
                class_str = ''
            
            try:
                id_str = tag.get('id', '').lower() if tag.has_attr('id') else ''
                if id_str is None:
                    id_str = ''
            except:
                id_str = ''
            
            # Only remove if it's clearly navigation/ads, be conservative
            if any(pattern in class_str or pattern in id_str for pattern in unwanted_patterns):
                try:
                    tag.decompose()
                except:
                    pass
    except Exception as e:
        info_logger.debug(f"Error removing unwanted elements: {e}")


def deduplicate_sections(sections: list) -> list:
    """Remove duplicate or highly similar sections"""
    seen = set()
    unique_sections = []
    
    for title, content in sections:
        # Create a fingerprint (first 100 chars of content)
        fingerprint = content[:100].lower()
        
        if fingerprint not in seen:
            seen.add(fingerprint)
            unique_sections.append((title, content))
    
    return unique_sections


def extract_text_sections(html: str) -> list:
    """
    Main function to extract text sections from HTML using multiple strategies.
    
    Returns: List of (title, content) tuples
    """
    if not html:
        info_logger.warning("No HTML provided for extraction")
        return []
    
    try:
        soup = BeautifulSoup(html, "lxml")
    except Exception as e:
        info_logger.error(f"Failed to parse HTML with lxml: {e}")
        try:
            soup = BeautifulSoup(html, "html.parser")
        except Exception as e2:
            info_logger.error(f"Failed to parse HTML with html.parser: {e2}")
            return []
    
    if soup is None:
        info_logger.error("BeautifulSoup returned None")
        return []
    
    # Clean up the soup
    remove_unwanted_elements(soup)
    
    all_sections = []
    
    # Try multiple extraction strategies
    strategies = [
        ("Semantic Tags", extract_from_semantic_tags),
        ("Content Divs", extract_from_content_divs),
        ("Headings", extract_from_headings),
        ("Lists & Tables", extract_lists_and_tables),
        ("Paragraphs", extract_from_paragraphs),  # Fallback
    ]
    
    for strategy_name, strategy_func in strategies:
        try:
            sections = strategy_func(soup)
            if sections:
                info_logger.debug(f"Strategy '{strategy_name}' found {len(sections)} sections")
                all_sections.extend(sections)
        except Exception as e:
            info_logger.error(f"Strategy '{strategy_name}' failed: {e}")
            continue
    
    info_logger.info(f"Extracted {len(all_sections)} unique content sections")
    
    return all_sections