"""
Additional enhanced extraction methods for WebScraper
"""

def extract_headings_enhanced(self, soup):
    """Extract headings with enhanced metadata"""
    headings = []
    
    for level in range(1, 7):
        for heading in soup.find_all(f'h{level}'):
            text = heading.get_text().strip()
            if text and len(text) > 1:
                heading_data = {
                    "level": level,
                    "text": text[:200],
                    "id": heading.get('id', ''),
                    "class": heading.get('class', []),
                    "word_count": len(text.split()),
                    "char_count": len(text),
                    "has_anchor": bool(heading.get('id')),
                    "position": len(headings)  # Position in document
                }
                headings.append(heading_data)
    
    return headings

def extract_buttons_enhanced(self, soup):
    """Extract buttons with enhanced metadata"""
    buttons = []
    
    for button in soup.find_all(['button', 'input'], type='submit'):
        text = button.get_text().strip() or button.get('value', '') or button.get('title', '')
        if text and len(text) > 1:
            button_data = {
                "text": text[:100],
                "type": button.name,
                "class": button.get('class', []),
                "id": button.get('id', ''),
                "disabled": button.get('disabled') is not None,
                "form_action": button.get('formaction', ''),
                "word_count": len(text.split()),
                "urgency": "high" if any(word in text.lower() for word in ['buy', 'now', 'urgent', 'limited']) else "normal"
            }
            buttons.append(button_data)
    
    return buttons

def extract_forms_enhanced(self, soup):
    """Extract forms with enhanced metadata"""
    forms = []
    
    for form in soup.find_all('form'):
        action = form.get('action', '')
        method = form.get('method', 'GET').upper()
        
        form_data = {
            "action": action,
            "method": method,
            "id": form.get('id', ''),
            "class": form.get('class', []),
            "input_count": len(form.find_all(['input', 'textarea', 'select'])),
            "has_password": bool(form.find('input', type='password')),
            "has_email": bool(form.find('input', type='email')),
            "has_file_upload": bool(form.find('input', type='file')),
            "submit_buttons": len(form.find_all(['button', 'input'], type='submit')),
            "purpose": self.classify_form_purpose(form)
        }
        forms.append(form_data)
    
    return forms

def classify_form_purpose(self, form):
    """Classify the purpose of a form"""
    form_text = form.get_text().lower()
    
    if any(keyword in form_text for keyword in ['login', 'signin', 'sign in', 'log in']):
        return "login"
    elif any(keyword in form_text for keyword in ['register', 'signup', 'sign up', 'create account']):
        return "registration"
    elif any(keyword in form_text for keyword in ['search', 'find', 'lookup']):
        return "search"
    elif any(keyword in form_text for keyword in ['contact', 'message', 'email']):
        return "contact"
    elif any(keyword in form_text for keyword in ['newsletter', 'subscribe']):
        return "newsletter"
    elif any(keyword in form_text for keyword in ['buy', 'purchase', 'order', 'checkout']):
        return "ecommerce"
    else:
        return "general"

def extract_lists_enhanced(self, soup):
    """Extract lists with enhanced metadata"""
    lists = []
    
    for list_elem in soup.find_all(['ul', 'ol', 'dl']):
        list_type = list_elem.name
        items = []
        
        if list_type in ['ul', 'ol']:
            for li in list_elem.find_all('li', recursive=False):
                text = li.get_text().strip()
                if text:
                    items.append({
                        "text": text[:200],
                        "word_count": len(text.split()),
                        "has_link": bool(li.find('a')),
                        "has_emphasis": bool(li.find(['strong', 'em', 'b', 'i']))
                    })
        elif list_type == 'dl':
            pairs = []
            dt_elements = list_elem.find_all('dt')
            dd_elements = list_elem.find_all('dd')
            
            for i, (dt, dd) in enumerate(zip(dt_elements, dd_elements)):
                dt_text = dt.get_text().strip()
                dd_text = dd.get_text().strip()
                if dt_text or dd_text:
                    pairs.append({
                        "term": dt_text[:100],
                        "description": dd_text[:200]
                    })
            items = pairs
        
        if items:
            lists.append({
                "type": list_type,
                "items": items,
                "item_count": len(items),
                "class": list_elem.get('class', []),
                "id": list_elem.get('id', '')
            })
    
    return lists

def extract_tables_enhanced(self, soup):
    """Extract tables with enhanced metadata"""
    tables = []
    
    for table in soup.find_all('table'):
        rows = []
        headers = []
        
        # Extract headers
        for th in table.find_all('th'):
            headers.append(th.get_text().strip())
        
        # Extract rows
        for tr in table.find_all('tr'):
            cells = []
            for td in tr.find_all('td'):
                cells.append(td.get_text().strip())
            if cells:
                rows.append(cells)
        
        if rows or headers:
            tables.append({
                "headers": headers,
                "rows": rows,
                "row_count": len(rows),
                "column_count": len(headers) if headers else len(rows[0]) if rows else 0,
                "has_headers": bool(headers),
                "class": table.get('class', []),
                "id": table.get('id', ''),
                "purpose": self.classify_table_purpose(headers, rows)
            })
    
    return tables

def classify_table_purpose(self, headers, rows):
    """Classify the purpose of a table"""
    header_text = ' '.join(headers).lower()
    
    if any(keyword in header_text for keyword in ['price', 'cost', 'amount', '$']):
        return "pricing"
    elif any(keyword in header_text for keyword in ['name', 'product', 'item']):
        return "product_list"
    elif any(keyword in header_text for keyword in ['date', 'time', 'schedule']):
        return "schedule"
    elif any(keyword in header_text for keyword in ['contact', 'email', 'phone']):
        return "contact_list"
    elif any(keyword in header_text for keyword in ['spec', 'feature', 'detail']):
        return "specifications"
    else:
        return "data"

def extract_main_text(self, soup):
    """Extract main text with intelligent content selection"""
    # Remove unwanted elements
    for element in soup(['script', 'style', 'noscript', 'meta', 'link', 'iframe', 'svg']):
        element.decompose()
    
    # Try to find main content area
    main_content = ""
    
    # Priority order for content selection
    content_selectors = [
        'main', 'article', '[role="main"]', '.content', '#content',
        '.main-content', '.page-content', '.post-content', '.entry-content'
    ]
    
    for selector in content_selectors:
        main_elem = soup.select_one(selector)
        if main_elem:
            main_content = main_elem.get_text()
            break
    
    # Fallback to body if no main content found
    if not main_content:
        main_content = soup.get_text()
    
    # Clean and format text
    lines = []
    for line in main_content.split('\n'):
        line = line.strip()
        if line and len(line) > 3:  # Skip very short lines
            lines.append(line)
    
    # Join and limit length
    text = ' '.join(lines[:100])  # First 100 meaningful lines
    
    return text[:5000] if len(text) > 5000 else text  # Limit to 5000 chars

def extract_metadata_enhanced(self, soup):
    """Extract enhanced metadata"""
    metadata = {}
    
    # Standard meta tags
    for meta in soup.find_all('meta'):
        name = meta.get('name') or meta.get('property') or meta.get('http-equiv')
        content = meta.get('content', '')
        if name and content:
            metadata[name] = content[:200]
    
    # Page structure info
    metadata['page_structure'] = {
        "headings_count": len(soup.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6'])),
        "links_count": len(soup.find_all('a')),
        "images_count": len(soup.find_all('img')),
        "forms_count": len(soup.find_all('form')),
        "tables_count": len(soup.find_all('table')),
        "lists_count": len(soup.find_all(['ul', 'ol', 'dl'])),
        "has_navigation": bool(soup.find('nav')),
        "has_header": bool(soup.find('header')),
        "has_footer": bool(soup.find('footer')),
        "has_aside": bool(soup.find('aside'))
    }
    
    # Language info
    html_tag = soup.find('html')
    if html_tag:
        metadata['language'] = html_tag.get('lang', 'unknown')
    
    # Character encoding
    charset_meta = soup.find('meta', charset=True)
    if charset_meta:
        metadata['charset'] = charset_meta.get('charset', 'unknown')
    
    # Viewport
    viewport_meta = soup.find('meta', name='viewport')
    if viewport_meta:
        metadata['viewport'] = viewport_meta.get('content', 'unknown')
    
    return metadata

def calculate_performance_metrics(self, content):
    """Calculate performance and quality metrics"""
    metrics = {
        "content_density": 0,
        "link_density": 0,
        "image_density": 0,
        "heading_structure": 0,
        "semantic_markup": 0,
        "accessibility_score": 0,
        "readability_score": 0
    }
    
    # Calculate densities
    total_text_length = len(content.get('text', ''))
    if total_text_length > 0:
        metrics["content_density"] = total_text_length
        metrics["link_density"] = len(content.get('links', [])) / (total_text_length / 1000)
        metrics["image_density"] = len(content.get('images', [])) / (total_text_length / 1000)
    
    # Heading structure score
    headings = content.get('headings', [])
    if headings:
        h1_count = sum(1 for h in headings if h['level'] == 1)
        has_h1 = h1_count > 0
        proper_hierarchy = all(
            any(h['level'] == i for h in headings) 
            for i in range(1, max(h['level'] for h in headings) + 1)
        )
        metrics["heading_structure"] = 100 if (has_h1 and proper_hierarchy) else 50
    
    # Semantic markup score
    semantic_elements = ['main', 'article', 'section', 'nav', 'aside', 'header', 'footer']
    # This would require access to the original soup, simplified for now
    metrics["semantic_markup"] = 75  # Placeholder
    
    # Accessibility score (basic checks)
    alt_texts = [img for img in content.get('images', []) if img.get('alt')]
    total_images = len(content.get('images', []))
    if total_images > 0:
        metrics["accessibility_score"] = (len(alt_texts) / total_images) * 100
    else:
        metrics["accessibility_score"] = 100
    
    # Readability score (simplified)
    text = content.get('text', '')
    if text:
        sentences = text.split('.')
        words = text.split()
        avg_sentence_length = len(words) / len(sentences) if sentences else 0
        
        # Simple readability calculation
        if avg_sentence_length < 15:
            metrics["readability_score"] = 90
        elif avg_sentence_length < 20:
            metrics["readability_score"] = 75
        elif avg_sentence_length < 25:
            metrics["readability_score"] = 60
        else:
            metrics["readability_score"] = 40
    
    return metrics
