"""
Web Scraper Module
Enhanced web scraping with AI-powered content analysis
"""

import asyncio
import aiohttp
import json
import re
import subprocess
import sys
import time
import random
from urllib.parse import urljoin, urlparse
from bs4 import BeautifulSoup, SoupStrainer
from fake_useragent import UserAgent
from colorama import Fore, Style
import requests
import os
import hashlib

class WebScraper:
    """Enhanced web scraper with AI-powered content analysis"""
    
    def __init__(self):
        self.ua = UserAgent()
        self.session = None
        self.timeout = 45  # Increased timeout
        self.max_retries = 5  # More retries
        
        # AI model configuration
        self.ai_model = os.getenv('AI_MODEL', 'qwen:7b')
        self.ai_endpoint = os.getenv('AI_ENDPOINT', 'http://localhost:11434/api/generate')
        self.ollama_available = None  # None = not checked yet
        
        # Advanced scraping configuration
        self.rate_limit_delay = random.uniform(0.5, 2.0)  # Random delay
        self.session_cache = {}  # Cache for repeated requests
        self.request_count = 0
        self.last_request_time = 0
        
        # Advanced headers for better scraping
        self.base_headers = {
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
            'Accept-Language': 'en-US,en;q=0.9,en-GB;q=0.8,en;q=0.7',
            'Accept-Encoding': 'gzip, deflate, br',
            'DNT': '1',
            'Connection': 'keep-alive',
            'Upgrade-Insecure-Requests': '1',
            'Sec-Fetch-Dest': 'document',
            'Sec-Fetch-Mode': 'navigate',
            'Sec-Fetch-Site': 'none',
            'Sec-Fetch-User': '?1',
            'Cache-Control': 'max-age=0',
            'sec-ch-ua': '"Google Chrome";v="123", "Not:A-Brand";v="8", "Chromium";v="123"',
            'sec-ch-ua-mobile': '?0',
            'sec-ch-ua-platform': '"Windows"'
        }
        
        # Content selectors for better extraction
        self.content_selectors = {
            'main_content': [
                'main', 'article', '[role="main"]', '.content', '#content',
                '.main-content', '.page-content', '.post-content', '.entry-content',
                'section.content', '.container main', '.wrapper main'
            ],
            'navigation': [
                'nav', 'navigation', '.nav', '.navbar', '.menu', '.navigation-menu',
                '[role="navigation"]', '.header-nav', '.top-nav'
            ],
            'sidebar': [
                '.sidebar', '.side-bar', 'aside', '.widget-area', '.secondary',
                '[role="complementary"]', '.left-sidebar', '.right-sidebar'
            ],
            'footer': [
                'footer', '.footer', '.page-footer', '[role="contentinfo"]',
                '.site-footer', '.bottom'
            ],
            'headers': [
                'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
                '.title', '.heading', '.headline', '.page-title'
            ],
            'links': [
                'a[href]', 'link', '.nav-link', '.menu-item', '.breadcrumb a'
            ],
            'images': [
                'img[src]', 'picture img', '.image', '.photo', '.thumbnail'
            ],
            'buttons': [
                'button', 'input[type="submit"]', 'input[type="button"]',
                '.btn', '.button', '.call-to-action', '.cta'
            ],
            'forms': [
                'form', 'input', 'textarea', 'select', '.form', '.contact-form'
            ],
            'lists': [
                'ul', 'ol', 'dl', '.list', '.menu', '.items'
            ],
            'tables': [
                'table', '.table', '.data-table', '.grid'
            ]
        }
    
    def check_ollama_availability(self):
        """Check if Ollama is installed and running"""
        if self.ollama_available is not None:
            return self.ollama_available
        
        try:
            # Check if ollama command exists
            result = subprocess.run(['ollama', '--version'], capture_output=True, text=True, timeout=5)
            if result.returncode != 0:
                self.ollama_available = False
                return False
            
            # Check if ollama is running (synchronous check)
            try:
                import requests
                response = requests.get('http://localhost:11434/api/tags', timeout=3)
                if response.status_code == 200:
                    self.ollama_available = True
                    return True
            except:
                self.ollama_available = False
                return False
                
        except (subprocess.TimeoutExpired, FileNotFoundError):
            self.ollama_available = False
            return False
        
        self.ollama_available = True
        return True
    
    def prompt_ollama_installation(self):
        """Prompt user to install Ollama if not available"""
        print(f"\n{Fore.YELLOW}=== AI ENHANCEMENT ==={Style.RESET_ALL}")
        print(f"{Fore.YELLOW}Ollama is not installed or not running.{Style.RESET_ALL}")
        print(f"{Fore.WHITE}Ollama provides AI-powered content analysis for web scraping.{Style.RESET_ALL}")
        print(f"{Fore.WHITE}Without Ollama, basic content extraction will be used.{Style.RESET_ALL}")
        
        while True:
            response = input(f"\n{Fore.CYAN}Would you like to install Ollama? (y/n): {Style.RESET_ALL}").strip().lower()
            
            if response == 'y' or response == 'yes':
                print(f"{Fore.GREEN}[*] Installing Ollama...{Style.RESET_ALL}")
                
                try:
                    # Try to install Ollama
                    if sys.platform == "win32":
                        # Windows
                        install_cmd = ['powershell', '-Command', 'iwr -useb https://ollama.ai/install.ps1 | iex']
                    else:
                        # Linux/Mac
                        install_cmd = ['curl', '-fsSL', 'https://ollama.ai/install.sh', '|', 'sh']
                    
                    print(f"{Fore.WHITE}Running: {' '.join(install_cmd)}{Style.RESET_ALL}")
                    
                    # For safety, just show the command instead of executing
                    print(f"{Fore.YELLOW}Please run this command manually to install Ollama:{Style.RESET_ALL}")
                    print(f"{Fore.CYAN}{' '.join(install_cmd)}{Style.RESET_ALL}")
                    print(f"\n{Fore.WHITE}After installation, run: {Style.RESET_ALL}{Fore.GREEN}ollama serve{Style.RESET_ALL}")
                    print(f"{Fore.WHITE}Then pull a model: {Style.RESET_ALL}{Fore.GREEN}ollama pull qwen:7b{Style.RESET_ALL}")
                    
                except Exception as e:
                    print(f"{Fore.RED}[!] Installation failed: {str(e)}{Style.RESET_ALL}")
                
                return False
                
            elif response == 'n' or response == 'no':
                print(f"{Fore.YELLOW}[*] Continuing with basic content extraction...{Style.RESET_ALL}")
                return False
                
            else:
                print(f"{Fore.RED}[!] Please enter 'y' or 'n'{Style.RESET_ALL}")
    
    def get_cache_key(self, url):
        """Generate cache key for URL"""
        return hashlib.md5(url.encode()).hexdigest()
    
    def apply_rate_limiting(self):
        """Apply intelligent rate limiting"""
        current_time = time.time()
        time_since_last = current_time - self.last_request_time
        
        if time_since_last < self.rate_limit_delay:
            sleep_time = self.rate_limit_delay - time_since_last
            time.sleep(sleep_time)
        
        self.last_request_time = time.time()
        self.request_count += 1
    
    def get_smart_headers(self, url):
        """Generate smart headers based on URL"""
        headers = self.base_headers.copy()
        headers['User-Agent'] = self.ua.random
        
        # Add URL-specific headers
        parsed_url = urlparse(url)
        domain = parsed_url.netloc.lower()
        
        if 'github.com' in domain:
            headers['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8'
            headers['X-Requested-With'] = 'XMLHttpRequest'
        elif 'linkedin.com' in domain:
            headers['Accept-Language'] = 'en-US,en;q=0.5'
        elif 'twitter.com' in domain or 'x.com' in domain:
            headers['Accept-Language'] = 'en-US,en;q=0.9'
        
        return headers
    
    async def fetch_with_retry(self, session, url, headers):
        """Fetch URL with intelligent retry logic"""
        for attempt in range(self.max_retries):
            try:
                # Apply rate limiting
                self.apply_rate_limiting()
                
                # Check cache first
                cache_key = self.get_cache_key(url)
                if cache_key in self.session_cache:
                    cached_data = self.session_cache[cache_key]
                    if time.time() - cached_data['timestamp'] < 300:  # 5 minutes cache
                        return cached_data['response']
                
                async with session.get(url, headers=headers, timeout=aiohttp.ClientTimeout(total=self.timeout)) as response:
                    if response.status == 200:
                        content = await response.text()
                        
                        # Cache the response
                        self.session_cache[cache_key] = {
                            'response': content,
                            'timestamp': time.time()
                        }
                        
                        return content
                    elif response.status == 429:  # Rate limited
                        retry_after = int(response.headers.get('Retry-After', random.uniform(5, 15)))
                        if verbose:
                            print(f"{Fore.YELLOW}[!] Rate limited, waiting {retry_after}s...{Style.RESET_ALL}")
                        time.sleep(retry_after)
                        continue
                    elif response.status in [500, 502, 503, 504]:  # Server errors
                        if attempt < self.max_retries - 1:
                            wait_time = random.uniform(2, 5) * (attempt + 1)
                            if verbose:
                                print(f"{Fore.YELLOW}[!] Server error {response.status}, retrying in {wait_time:.1f}s...{Style.RESET_ALL}")
                            time.sleep(wait_time)
                            continue
                        else:
                            return None
                    else:
                        return None
                        
            except asyncio.TimeoutError:
                if attempt < self.max_retries - 1:
                    wait_time = random.uniform(3, 8) * (attempt + 1)
                    if verbose:
                        print(f"{Fore.YELLOW}[!] Timeout, retrying in {wait_time:.1f}s...{Style.RESET_ALL}")
                    time.sleep(wait_time)
                    continue
                else:
                    return None
            except Exception as e:
                if attempt < self.max_retries - 1:
                    wait_time = random.uniform(2, 4) * (attempt + 1)
                    if verbose:
                        print(f"{Fore.YELLOW}[!] Error: {str(e)}, retrying in {wait_time:.1f}s...{Style.RESET_ALL}")
                    time.sleep(wait_time)
                    continue
                else:
                    return None
        
        return None
    
    def ensure_ollama_setup(self):
        """Ensure Ollama is available or prompt for installation"""
        if not self.check_ollama_availability():
            return self.prompt_ollama_installation()
        return True
        
    async def scrape_url(self, url, verbose=False):
        """Scrape URL and extract content with advanced techniques"""
        if verbose:
            print(f"Scraping URL: {url}")
        
        try:
            # Create session with advanced configuration
            connector = aiohttp.TCPConnector(
                limit=100,  # Connection pool size
                limit_per_host=30,  # Connections per host
                ttl_dns_cache=300,  # DNS cache TTL
                use_dns_cache=True,
                family=0,  # IPv4+IPv6
                ssl=False  # For problematic SSL certificates
            )
            
            timeout = aiohttp.ClientTimeout(
                total=self.timeout,
                connect=10,
                sock_read=30
            )
            
            async with aiohttp.ClientSession(
                connector=connector,
                timeout=timeout,
                headers=self.get_smart_headers(url)
            ) as session:
                
                # Fetch content with retry logic
                content = await self.fetch_with_retry(session, url, session.headers)
                
                if content is None:
                    return {"error": "Failed to fetch content after retries", "url": url}
                
                # Parse HTML with advanced techniques
                soup = BeautifulSoup(content, 'html.parser')
                
                # Extract enhanced content
                extracted_data = self.extract_content_enhanced(soup, url)
                
                # AI Analysis
                ai_summary = await self.analyze_with_ai(extracted_data, url)
                
                results = {
                    "url": url,
                    "status_code": 200,
                    "title": self.extract_title_enhanced(soup),
                    "content": extracted_data,
                    "ai_summary": ai_summary,
                    "scraping_info": {
                        "request_count": self.request_count,
                        "cache_hits": len(self.session_cache),
                        "content_length": len(content),
                        "extraction_time": time.time()
                    },
                    "timestamp": asyncio.get_event_loop().time()
                }
                
                return results
                        
        except Exception as e:
            return {"error": str(e), "url": url}
    
    def extract_title_enhanced(self, soup):
        """Extract page title with fallbacks"""
        # Try title tag first
        title_tag = soup.find('title')
        if title_tag:
            title = title_tag.get_text().strip()
            if title:
                return title
        
        # Try h1 tag
        h1_tag = soup.find('h1')
        if h1_tag:
            title = h1_tag.get_text().strip()
            if title:
                return title
        
        # Try meta title
        meta_title = soup.find('meta', property='og:title')
        if meta_title:
            title = meta_title.get('content', '').strip()
            if title:
                return title
        
        # Try meta description as fallback
        meta_desc = soup.find('meta', attrs={'name': 'description'})
        if meta_desc:
            desc = meta_desc.get('content', '').strip()
            if desc:
                return desc[:50] + "..." if len(desc) > 50 else desc
        
        return "No title found"
    
    def extract_content_enhanced(self, soup, base_url):
        """Extract content with advanced techniques and semantic understanding - ABSOLUTELY EVERYTHING"""
        content = {
            "links": [],
            "images": [],
            "text": "",
            "metadata": {},
            "headings": [],
            "buttons": [],
            "forms": [],
            "lists": [],
            "tables": [],
            "navigation": [],
            "sidebar": [],
            "footer": [],
            "main_content": "",
            "structured_data": {},
            "performance_metrics": {},
            # NEW: Capture absolutely everything
            "scripts": [],
            "styles": [],
            "meta_tags": [],
            "iframes": [],
            "videos": [],
            "audios": [],
            "svg_elements": [],
            "canvas_elements": [],
            "inputs": [],
            "textareas": [],
            "selects": [],
            "checkboxes": [],
            "radios": [],
            "divs": [],
            "spans": [],
            "paragraphs": [],
            "quotes": [],
            "codes": [],
            "pre_elements": [],
            "time_elements": [],
            "data_attributes": {},
            "custom_elements": [],
            "comments": [],
            "doctypes": "",
            "html_attributes": {},
            "body_attributes": {},
            "css_classes": {},
            "element_counts": {},
            "all_text_content": "",
            "hidden_elements": [],
            "dynamic_content": {},
            "social_media_links": [],
            "external_resources": [],
            "internal_resources": [],
            "cdn_resources": [],
            "analytics_scripts": [],
            "tracking_codes": [],
            "advertisement_elements": [],
            "cookie_consent": {},
            "error_elements": [],
            "loading_elements": [],
            "modal_elements": [],
            "tooltip_elements": [],
            "breadcrumb_elements": [],
            "pagination_elements": [],
            "search_elements": [],
            "filter_elements": [],
            "sorting_elements": [],
            "tab_elements": [],
            "accordion_elements": [],
            "carousel_elements": [],
            "gallery_elements": [],
            "map_elements": [],
            "chart_elements": [],
            "form_validations": [],
            "ajax_requests": {},
            "websockets": [],
            "service_workers": [],
            "manifest_files": [],
            "favicon": [],
            "touch_icons": [],
            "theme_colors": {},
            "responsive_breakpoints": {},
            "accessibility_attributes": {},
            "microformats": [],
            "open_graph": {},
            "twitter_cards": {},
            "json_ld_structured": [],
            "rdf_structured": [],
            "microdata_structured": [],
            "seo_elements": {},
            "performance_indicators": {},
            "security_headers": {},
            "csp_policies": {},
            "cross_origin_resources": [],
            "preload_resources": [],
            "prefetch_resources": [],
            "dns_prefetch": [],
            "preconnect_domains": []
        }
        
        # Extract structured data (JSON-LD, microdata)
        content["structured_data"] = self.extract_structured_data(soup)
        
        # Extract main content using semantic selectors
        content["main_content"] = self.extract_section_content(soup, 'main_content', base_url)
        
        # Extract navigation
        content["navigation"] = self.extract_section_content(soup, 'navigation', base_url)
        
        # Extract sidebar
        content["sidebar"] = self.extract_section_content(soup, 'sidebar', base_url)
        
        # Extract footer
        content["footer"] = self.extract_section_content(soup, 'footer', base_url)
        
        # Enhanced link extraction
        content["links"] = self.extract_links_enhanced(soup, base_url)
        
        # Enhanced image extraction
        content["images"] = self.extract_images_enhanced(soup, base_url)
        
        # Enhanced headings extraction
        content["headings"] = self.extract_headings_enhanced(soup)
        
        # Enhanced buttons extraction
        content["buttons"] = self.extract_buttons_enhanced(soup)
        
        # Enhanced forms extraction
        content["forms"] = self.extract_forms_enhanced(soup)
        
        # Enhanced lists extraction
        content["lists"] = self.extract_lists_enhanced(soup)
        
        # Enhanced tables extraction
        content["tables"] = self.extract_tables_enhanced(soup)
        
        # NEW: Extract absolutely everything
        content.update(self.extract_everything(soup, base_url))
        
        # Extract main text with better cleaning
        content["text"] = self.extract_main_text(soup)
        
        # Extract enhanced metadata
        content["metadata"] = self.extract_metadata_enhanced(soup)
        
        # Calculate performance metrics
        content["performance_metrics"] = self.calculate_performance_metrics(content)
        
        return content
    
    def extract_everything(self, soup, base_url):
        """Extract absolutely everything from the page"""
        everything = {}
        
        # Extract all scripts
        scripts = []
        for script in soup.find_all('script'):
            script_data = {
                "src": script.get('src', ''),
                "type": script.get('type', ''),
                "async": script.get('async') is not None,
                "defer": script.get('defer') is not None,
                "integrity": script.get('integrity', ''),
                "crossorigin": script.get('crossorigin', ''),
                "content": script.string or '',
                "class": script.get('class', []),
                "id": script.get('id', '')
            }
            scripts.append(script_data)
        everything["scripts"] = scripts
        
        # Extract all styles
        styles = []
        for style in soup.find_all('style'):
            style_data = {
                "type": style.get('type', ''),
                "media": style.get('media', ''),
                "content": style.string or '',
                "class": style.get('class', []),
                "id": style.get('id', '')
            }
            styles.append(style_data)
        everything["styles"] = styles
        
        # Extract all meta tags with full details
        meta_tags = []
        for meta in soup.find_all('meta'):
            meta_data = {
                "name": meta.get('name', ''),
                "property": meta.get('property', ''),
                "content": meta.get('content', ''),
                "charset": meta.get('charset', ''),
                "http_equiv": meta.get('http-equiv', ''),
                "scheme": meta.get('scheme', ''),
                "itemprop": meta.get('itemprop', '')
            }
            meta_tags.append(meta_data)
        everything["meta_tags"] = meta_tags
        
        # Extract all iframes
        iframes = []
        for iframe in soup.find_all('iframe'):
            iframe_data = {
                "src": iframe.get('src', ''),
                "srcdoc": iframe.get('srcdoc', ''),
                "width": iframe.get('width', ''),
                "height": iframe.get('height', ''),
                "name": iframe.get('name', ''),
                "allow": iframe.get('allow', ''),
                "allowfullscreen": iframe.get('allowfullscreen') is not None,
                "loading": iframe.get('loading', ''),
                "referrerpolicy": iframe.get('referrerpolicy', ''),
                "sandbox": iframe.get('sandbox', ''),
                "class": iframe.get('class', []),
                "id": iframe.get('id', '')
            }
            iframes.append(iframe_data)
        everything["iframes"] = iframes
        
        # Extract all videos
        videos = []
        for video in soup.find_all('video'):
            video_data = {
                "src": video.get('src', ''),
                "poster": video.get('poster', ''),
                "width": video.get('width', ''),
                "height": video.get('height', ''),
                "controls": video.get('controls') is not None,
                "autoplay": video.get('autoplay') is not None,
                "loop": video.get('loop') is not None,
                "muted": video.get('muted') is not None,
                "preload": video.get('preload', ''),
                "class": video.get('class', []),
                "id": video.get('id', ''),
                "sources": []
            }
            for source in video.find_all('source'):
                video_data["sources"].append({
                    "src": source.get('src', ''),
                    "type": source.get('type', '')
                })
            videos.append(video_data)
        everything["videos"] = videos
        
        # Extract all audios
        audios = []
        for audio in soup.find_all('audio'):
            audio_data = {
                "src": audio.get('src', ''),
                "controls": audio.get('controls') is not None,
                "autoplay": audio.get('autoplay') is not None,
                "loop": audio.get('loop') is not None,
                "muted": audio.get('muted') is not None,
                "preload": audio.get('preload', ''),
                "class": audio.get('class', []),
                "id": audio.get('id', ''),
                "sources": []
            }
            for source in audio.find_all('source'):
                audio_data["sources"].append({
                    "src": source.get('src', ''),
                    "type": source.get('type', '')
                })
            audios.append(audio_data)
        everything["audios"] = audios
        
        # Extract all SVG elements
        svg_elements = []
        for svg in soup.find_all('svg'):
            svg_data = {
                "width": svg.get('width', ''),
                "height": svg.get('height', ''),
                "viewbox": svg.get('viewbox', ''),
                "xmlns": svg.get('xmlns', ''),
                "content": str(svg),
                "class": svg.get('class', []),
                "id": svg.get('id', '')
            }
            svg_elements.append(svg_data)
        everything["svg_elements"] = svg_elements
        
        # Extract all canvas elements
        canvas_elements = []
        for canvas in soup.find_all('canvas'):
            canvas_data = {
                "width": canvas.get('width', ''),
                "height": canvas.get('height', ''),
                "content": canvas.string or '',
                "class": canvas.get('class', []),
                "id": canvas.get('id', '')
            }
            canvas_elements.append(canvas_data)
        everything["canvas_elements"] = canvas_elements
        
        # Extract all input elements
        inputs = []
        for input_elem in soup.find_all('input'):
            input_data = {
                "type": input_elem.get('type', ''),
                "name": input_elem.get('name', ''),
                "value": input_elem.get('value', ''),
                "placeholder": input_elem.get('placeholder', ''),
                "required": input_elem.get('required') is not None,
                "disabled": input_elem.get('disabled') is not None,
                "readonly": input_elem.get('readonly') is not None,
                "maxlength": input_elem.get('maxlength', ''),
                "min": input_elem.get('min', ''),
                "max": input_elem.get('max', ''),
                "step": input_elem.get('step', ''),
                "pattern": input_elem.get('pattern', ''),
                "class": input_elem.get('class', []),
                "id": input_elem.get('id', '')
            }
            inputs.append(input_data)
        everything["inputs"] = inputs
        
        # Extract all textarea elements
        textareas = []
        for textarea in soup.find_all('textarea'):
            textarea_data = {
                "name": textarea.get('name', ''),
                "content": textarea.string or '',
                "placeholder": textarea.get('placeholder', ''),
                "required": textarea.get('required') is not None,
                "disabled": textarea.get('disabled') is not None,
                "readonly": textarea.get('readonly') is not None,
                "maxlength": textarea.get('maxlength', ''),
                "rows": textarea.get('rows', ''),
                "cols": textarea.get('cols', ''),
                "class": textarea.get('class', []),
                "id": textarea.get('id', '')
            }
            textareas.append(textarea_data)
        everything["textareas"] = textareas
        
        # Extract all select elements
        selects = []
        for select in soup.find_all('select'):
            select_data = {
                "name": select.get('name', ''),
                "required": select.get('required') is not None,
                "disabled": select.get('disabled') is not None,
                "multiple": select.get('multiple') is not None,
                "size": select.get('size', ''),
                "options": []
            }
            for option in select.find_all('option'):
                select_data["options"].append({
                    "value": option.get('value', ''),
                    "text": option.get_text().strip(),
                    "selected": option.get('selected') is not None,
                    "disabled": option.get('disabled') is not None
                })
            selects.append(select_data)
        everything["selects"] = selects
        
        # Extract all div elements (sample for analysis)
        divs = []
        for div in soup.find_all('div')[:100]:  # Limit to first 100 for performance
            div_data = {
                "class": div.get('class', []),
                "id": div.get('id', ''),
                "text": div.get_text().strip()[:200],  # Limit text
                "child_count": len(div.find_all(recursive=False)),
                "has_events": bool(div.get('onclick') or div.get('onmouseover') or div.get('onmouseout'))
            }
            divs.append(div_data)
        everything["divs"] = divs
        
        # Extract all span elements
        spans = []
        for span in soup.find_all('span')[:50]:  # Limit to first 50
            span_data = {
                "class": span.get('class', []),
                "id": span.get('id', ''),
                "text": span.get_text().strip()[:100],
                "has_events": bool(span.get('onclick') or span.get('onmouseover'))
            }
            spans.append(span_data)
        everything["spans"] = spans
        
        # Extract all paragraphs
        paragraphs = []
        for p in soup.find_all('p'):
            p_data = {
                "class": p.get('class', []),
                "id": p.get('id', ''),
                "text": p.get_text().strip()[:300],
                "word_count": len(p.get_text().split())
            }
            paragraphs.append(p_data)
        everything["paragraphs"] = paragraphs
        
        # Extract all blockquotes
        quotes = []
        for quote in soup.find_all('blockquote'):
            quote_data = {
                "cite": quote.get('cite', ''),
                "class": quote.get('class', []),
                "id": quote.get('id', ''),
                "text": quote.get_text().strip()[:300]
            }
            quotes.append(quote_data)
        everything["quotes"] = quotes
        
        # Extract all code elements
        codes = []
        for code in soup.find_all('code'):
            code_data = {
                "class": code.get('class', []),
                "text": code.get_text().strip()[:200]
            }
            codes.append(code_data)
        everything["codes"] = codes
        
        # Extract all pre elements
        pre_elements = []
        for pre in soup.find_all('pre'):
            pre_data = {
                "class": pre.get('class', []),
                "text": pre.get_text().strip()[:500]
            }
            pre_elements.append(pre_data)
        everything["pre_elements"] = pre_elements
        
        # Extract HTML comments
        comments = []
        for comment in soup.find_all(string=True):
            if isinstance(comment, str) and comment.strip().startswith('<!--'):
                comments.append(comment.strip())
        everything["comments"] = comments
        
        # Extract HTML attributes
        html_tag = soup.find('html')
        if html_tag:
            everything["html_attributes"] = dict(html_tag.attrs)
        
        # Extract body attributes
        body_tag = soup.find('body')
        if body_tag:
            everything["body_attributes"] = dict(body_tag.attrs)
        
        # Extract all CSS classes
        css_classes = {}
        for element in soup.find_all(True):
            classes = element.get('class', [])
            for css_class in classes:
                if css_class not in css_classes:
                    css_classes[css_class] = 0
                css_classes[css_class] += 1
        everything["css_classes"] = css_classes
        
        # Count all elements
        element_counts = {}
        for element in soup.find_all(True):
            tag_name = element.name
            if tag_name not in element_counts:
                element_counts[tag_name] = 0
            element_counts[tag_name] += 1
        everything["element_counts"] = element_counts
        
        # Extract all text content
        everything["all_text_content"] = soup.get_text()
        
        # Extract hidden elements
        hidden_elements = []
        for element in soup.find_all(True):
            style = element.get('style', '')
            if style and ('display:none' in style or 'visibility:hidden' in style):
                hidden_elements.append({
                    "tag": element.name,
                    "class": element.get('class', []),
                    "id": element.get('id', ''),
                    "text": element.get_text().strip()[:100]
                })
        everything["hidden_elements"] = hidden_elements
        
        # Extract social media links
        social_domains = ['facebook.com', 'twitter.com', 'instagram.com', 'linkedin.com', 'youtube.com', 'tiktok.com']
        social_links = []
        for link in soup.find_all('a', href=True):
            href = link['href']
            if any(domain in href for domain in social_domains):
                social_links.append({
                    "url": href,
                    "text": link.get_text().strip(),
                    "platform": next(domain for domain in social_domains if domain in href)
                })
        everything["social_media_links"] = social_links
        
        # Extract external resources
        external_resources = []
        for resource in soup.find_all(['link', 'script', 'img']):
            src = resource.get('src') or resource.get('href')
            if src and (src.startswith('http://') or src.startswith('https://')):
                parsed = urlparse(src)
                if parsed.netloc != urlparse(base_url).netloc:
                    external_resources.append({
                        "url": src,
                        "type": resource.name,
                        "domain": parsed.netloc
                    })
        everything["external_resources"] = external_resources
        
        return everything
    
    def extract_structured_data(self, soup):
        """Extract structured data (JSON-LD, microdata)"""
        structured_data = {
            "json_ld": [],
            "microdata": [],
            "rdfa": []
        }
        
        # Extract JSON-LD
        for script in soup.find_all('script', type='application/ld+json'):
            try:
                data = json.loads(script.string)
                structured_data["json_ld"].append(data)
            except:
                pass
        
        # Extract microdata
        for item in soup.find_all(attrs={'itemscope': True}):
            microdata_item = self.extract_microdata_item(item)
            if microdata_item:
                structured_data["microdata"].append(microdata_item)
        
        return structured_data
    
    def extract_microdata_item(self, element):
        """Extract microdata from an element"""
        try:
            item = {}
            item_type = element.get('itemtype', '')
            if item_type:
                item['@type'] = item_type
            
            for prop in element.find_all(attrs={'itemprop': True}):
                prop_name = prop.get('itemprop')
                prop_value = prop.get('content') or prop.get_text().strip()
                if prop_name and prop_value:
                    item[prop_name] = prop_value
            
            return item if item else None
        except:
            return None
    
    def extract_section_content(self, soup, section_type, base_url):
        """Extract content from specific page sections"""
        section_content = {
            "text": "",
            "links": [],
            "images": [],
            "headings": []
        }
        
        selectors = self.content_selectors.get(section_type, [])
        
        for selector in selectors:
            elements = soup.select(selector)
            if elements:
                for element in elements:
                    # Extract text
                    text = element.get_text().strip()
                    if text:
                        section_content["text"] += text + "\n"
                    
                    # Extract links
                    for link in element.find_all('a', href=True):
                        href = link['href']
                        text = link.get_text().strip()
                        if text and not href.startswith(('javascript:', 'mailto:', 'tel:', '#')):
                            absolute_url = urljoin(base_url, href)
                            section_content["links"].append({
                                "url": absolute_url,
                                "text": text[:100]
                            })
                    
                    # Extract images
                    for img in element.find_all('img', src=True):
                        src = img['src']
                        absolute_url = urljoin(base_url, src)
                        section_content["images"].append({
                            "url": absolute_url,
                            "alt": img.get('alt', '')[:100]
                        })
                    
                    # Extract headings
                    for level in range(1, 7):
                        for heading in element.find_all(f'h{level}'):
                            text = heading.get_text().strip()
                            if text:
                                section_content["headings"].append({
                                    "level": level,
                                    "text": text[:200]
                                })
                
                break  # Use first matching selector
        
        return section_content
    
    def extract_links_enhanced(self, soup, base_url):
        """Extract links with enhanced metadata"""
        links = []
        
        for link in soup.find_all('a', href=True):
            href = link['href']
            text = link.get_text().strip()
            
            # Skip empty, javascript, and anchor links
            if (text and len(text) > 1 and 
                not href.startswith(('javascript:', 'mailto:', 'tel:', '#')) and
                href.startswith(('http://', 'https://'))):
                
                absolute_url = urljoin(base_url, href)
                
                link_data = {
                    "url": absolute_url,
                    "text": text[:100],
                    "title": link.get('title', ''),
                    "target": link.get('target', ''),
                    "rel": link.get('rel', []),
                    "class": link.get('class', []),
                    "id": link.get('id', ''),
                    "domain": urlparse(absolute_url).netloc,
                    "is_external": urlparse(base_url).netloc != urlparse(absolute_url).netloc,
                    "text_length": len(text)
                }
                
                # Categorize link
                if any(keyword in text.lower() for keyword in ['download', 'pdf', 'file']):
                    link_data["category"] = "download"
                elif any(keyword in text.lower() for keyword in ['contact', 'email', 'phone']):
                    link_data["category"] = "contact"
                elif any(keyword in text.lower() for keyword in ['buy', 'purchase', 'cart', 'shop']):
                    link_data["category"] = "ecommerce"
                elif any(keyword in text.lower() for keyword in ['login', 'signin', 'register', 'signup']):
                    link_data["category"] = "auth"
                else:
                    link_data["category"] = "general"
                
                links.append(link_data)
        
        return links
    
    def extract_images_enhanced(self, soup, base_url):
        """Extract images with enhanced metadata"""
        images = []
        
        for img in soup.find_all('img', src=True):
            src = img['src']
            absolute_url = urljoin(base_url, src)
            
            image_data = {
                "url": absolute_url,
                "alt": img.get('alt', '')[:100],
                "title": img.get('title', '')[:100],
                "width": img.get('width', ''),
                "height": img.get('height', ''),
                "srcset": img.get('srcset', ''),
                "sizes": img.get('sizes', ''),
                "loading": img.get('loading', ''),
                "class": img.get('class', []),
                "id": img.get('id', ''),
                "file_extension": src.split('.')[-1].lower() if '.' in src else 'unknown',
                "file_size": len(src)  # Rough estimate
            }
            
            # Determine image purpose
            alt_lower = image_data["alt"].lower()
            if any(keyword in alt_lower for keyword in ['logo', 'brand', 'icon']):
                image_data["purpose"] = "branding"
            elif any(keyword in alt_lower for keyword in ['banner', 'hero', 'header']):
                image_data["purpose"] = "hero"
            elif any(keyword in alt_lower for keyword in ['product', 'item', 'merchandise']):
                image_data["purpose"] = "product"
            elif any(keyword in alt_lower for keyword in ['avatar', 'profile', 'user']):
                image_data["purpose"] = "profile"
            else:
                image_data["purpose"] = "content"
            
            images.append(image_data)
        
        return images
    
    def extract_headings_enhanced(self, soup):
        """Extract headings with enhanced metadata"""
        headings = []
        
        for level in range(1, 7):
            for heading in soup.find_all(f'h{level}'):
                text = heading.get_text().strip()
                if text and len(text) > 1:
                    heading_data = {
                        "level": level,
                        "text": text[:200],
                        "id": heading.get('id', ''),
                        "class": heading.get('class', []),
                        "word_count": len(text.split()),
                        "char_count": len(text),
                        "has_anchor": bool(heading.get('id')),
                        "position": len(headings)  # Position in document
                    }
                    headings.append(heading_data)
        
        return headings
    
    def extract_buttons_enhanced(self, soup):
        """Extract buttons with enhanced metadata"""
        buttons = []
        
        for button in soup.find_all(['button', 'input'], type='submit'):
            text = button.get_text().strip() or button.get('value', '') or button.get('title', '')
            if text and len(text) > 1:
                button_data = {
                    "text": text[:100],
                    "type": button.name,
                    "class": button.get('class', []),
                    "id": button.get('id', ''),
                    "disabled": button.get('disabled') is not None,
                    "form_action": button.get('formaction', ''),
                    "word_count": len(text.split()),
                    "urgency": "high" if any(word in text.lower() for word in ['buy', 'now', 'urgent', 'limited']) else "normal"
                }
                buttons.append(button_data)
        
        return buttons
    
    def extract_forms_enhanced(self, soup):
        """Extract forms with enhanced metadata"""
        forms = []
        
        for form in soup.find_all('form'):
            action = form.get('action', '')
            method = form.get('method', 'GET').upper()
            
            form_data = {
                "action": action,
                "method": method,
                "id": form.get('id', ''),
                "class": form.get('class', []),
                "input_count": len(form.find_all(['input', 'textarea', 'select'])),
                "has_password": bool(form.find('input', type='password')),
                "has_email": bool(form.find('input', type='email')),
                "has_file_upload": bool(form.find('input', type='file')),
                "submit_buttons": len(form.find_all(['button', 'input'], type='submit')),
                "purpose": self.classify_form_purpose(form)
            }
            forms.append(form_data)
        
        return forms
    
    def classify_form_purpose(self, form):
        """Classify the purpose of a form"""
        form_text = form.get_text().lower()
        
        if any(keyword in form_text for keyword in ['login', 'signin', 'sign in', 'log in']):
            return "login"
        elif any(keyword in form_text for keyword in ['register', 'signup', 'sign up', 'create account']):
            return "registration"
        elif any(keyword in form_text for keyword in ['search', 'find', 'lookup']):
            return "search"
        elif any(keyword in form_text for keyword in ['contact', 'message', 'email']):
            return "contact"
        elif any(keyword in form_text for keyword in ['newsletter', 'subscribe']):
            return "newsletter"
        elif any(keyword in form_text for keyword in ['buy', 'purchase', 'order', 'checkout']):
            return "ecommerce"
        else:
            return "general"
    
    def extract_lists_enhanced(self, soup):
        """Extract lists with enhanced metadata"""
        lists = []
        
        for list_elem in soup.find_all(['ul', 'ol', 'dl']):
            list_type = list_elem.name
            items = []
            
            if list_type in ['ul', 'ol']:
                for li in list_elem.find_all('li', recursive=False):
                    text = li.get_text().strip()
                    if text:
                        items.append({
                            "text": text[:200],
                            "word_count": len(text.split()),
                            "has_link": bool(li.find('a')),
                            "has_emphasis": bool(li.find(['strong', 'em', 'b', 'i']))
                        })
            elif list_type == 'dl':
                pairs = []
                dt_elements = list_elem.find_all('dt')
                dd_elements = list_elem.find_all('dd')
                
                for i, (dt, dd) in enumerate(zip(dt_elements, dd_elements)):
                    dt_text = dt.get_text().strip()
                    dd_text = dd.get_text().strip()
                    if dt_text or dd_text:
                        pairs.append({
                            "term": dt_text[:100],
                            "description": dd_text[:200]
                        })
                items = pairs
            
            if items:
                lists.append({
                    "type": list_type,
                    "items": items,
                    "item_count": len(items),
                    "class": list_elem.get('class', []),
                    "id": list_elem.get('id', '')
                })
        
        return lists
    
    def extract_tables_enhanced(self, soup):
        """Extract tables with enhanced metadata"""
        tables = []
        
        for table in soup.find_all('table'):
            rows = []
            headers = []
            
            # Extract headers
            for th in table.find_all('th'):
                headers.append(th.get_text().strip())
            
            # Extract rows
            for tr in table.find_all('tr'):
                cells = []
                for td in tr.find_all('td'):
                    cells.append(td.get_text().strip())
                if cells:
                    rows.append(cells)
            
            if rows or headers:
                tables.append({
                    "headers": headers,
                    "rows": rows,
                    "row_count": len(rows),
                    "column_count": len(headers) if headers else len(rows[0]) if rows else 0,
                    "has_headers": bool(headers),
                    "class": table.get('class', []),
                    "id": table.get('id', ''),
                    "purpose": self.classify_table_purpose(headers, rows)
                })
        
        return tables
    
    def classify_table_purpose(self, headers, rows):
        """Classify the purpose of a table"""
        header_text = ' '.join(headers).lower()
        
        if any(keyword in header_text for keyword in ['price', 'cost', 'amount', '$']):
            return "pricing"
        elif any(keyword in header_text for keyword in ['name', 'product', 'item']):
            return "product_list"
        elif any(keyword in header_text for keyword in ['date', 'time', 'schedule']):
            return "schedule"
        elif any(keyword in header_text for keyword in ['contact', 'email', 'phone']):
            return "contact_list"
        elif any(keyword in header_text for keyword in ['spec', 'feature', 'detail']):
            return "specifications"
        else:
            return "data"
    
    def extract_main_text(self, soup):
        """Extract main text with intelligent content selection"""
        # Remove unwanted elements
        for element in soup(['script', 'style', 'noscript', 'meta', 'link', 'iframe', 'svg']):
            element.decompose()
        
        # Try to find main content area
        main_content = ""
        
        # Priority order for content selection
        content_selectors = [
            'main', 'article', '[role="main"]', '.content', '#content',
            '.main-content', '.page-content', '.post-content', '.entry-content'
        ]
        
        for selector in content_selectors:
            main_elem = soup.select_one(selector)
            if main_elem:
                main_content = main_elem.get_text()
                break
        
        # Fallback to body if no main content found
        if not main_content:
            main_content = soup.get_text()
        
        # Clean and format text
        lines = []
        for line in main_content.split('\n'):
            line = line.strip()
            if line and len(line) > 3:  # Skip very short lines
                lines.append(line)
        
        # Join and limit length
        text = ' '.join(lines[:100])  # First 100 meaningful lines
        
        return text[:5000] if len(text) > 5000 else text  # Limit to 5000 chars
    
    def extract_metadata_enhanced(self, soup):
        """Extract enhanced metadata"""
        metadata = {}
        
        # Standard meta tags
        for meta in soup.find_all('meta'):
            name = meta.get('name') or meta.get('property') or meta.get('http-equiv')
            content = meta.get('content', '')
            if name and content:
                metadata[name] = content[:200]
        
        # Page structure info
        metadata['page_structure'] = {
            "headings_count": len(soup.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6'])),
            "links_count": len(soup.find_all('a')),
            "images_count": len(soup.find_all('img')),
            "forms_count": len(soup.find_all('form')),
            "tables_count": len(soup.find_all('table')),
            "lists_count": len(soup.find_all(['ul', 'ol', 'dl'])),
            "has_navigation": bool(soup.find('nav')),
            "has_header": bool(soup.find('header')),
            "has_footer": bool(soup.find('footer')),
            "has_aside": bool(soup.find('aside'))
        }
        
        # Language info
        html_tag = soup.find('html')
        if html_tag:
            metadata['language'] = html_tag.get('lang', 'unknown')
        
        # Character encoding
        charset_meta = soup.find('meta', charset=True)
        if charset_meta:
            metadata['charset'] = charset_meta.get('charset', 'unknown')
        
        # Viewport
        viewport_meta = soup.find('meta', attrs={'name': 'viewport'})
        if viewport_meta:
            metadata['viewport'] = viewport_meta.get('content', 'unknown')
        
        return metadata
    
    def calculate_performance_metrics(self, content):
        """Calculate performance and quality metrics"""
        metrics = {
            "content_density": 0,
            "link_density": 0,
            "image_density": 0,
            "heading_structure": 0,
            "semantic_markup": 0,
            "accessibility_score": 0,
            "readability_score": 0
        }
        
        # Calculate densities
        total_text_length = len(content.get('text', ''))
        if total_text_length > 0:
            metrics["content_density"] = total_text_length
            metrics["link_density"] = len(content.get('links', [])) / (total_text_length / 1000)
            metrics["image_density"] = len(content.get('images', [])) / (total_text_length / 1000)
        
        # Heading structure score
        headings = content.get('headings', [])
        if headings:
            h1_count = sum(1 for h in headings if h['level'] == 1)
            has_h1 = h1_count > 0
            proper_hierarchy = all(
                any(h['level'] == i for h in headings) 
                for i in range(1, max(h['level'] for h in headings) + 1)
            )
            metrics["heading_structure"] = 100 if (has_h1 and proper_hierarchy) else 50
        
        # Semantic markup score
        semantic_elements = ['main', 'article', 'section', 'nav', 'aside', 'header', 'footer']
        # This would require access to the original soup, simplified for now
        metrics["semantic_markup"] = 75  # Placeholder
        
        # Accessibility score (basic checks)
        alt_texts = [img for img in content.get('images', []) if img.get('alt')]
        total_images = len(content.get('images', []))
        if total_images > 0:
            metrics["accessibility_score"] = (len(alt_texts) / total_images) * 100
        else:
            metrics["accessibility_score"] = 100
        
        # Readability score (simplified)
        text = content.get('text', '')
        if text:
            sentences = text.split('.')
            words = text.split()
            avg_sentence_length = len(words) / len(sentences) if sentences else 0
            
            # Simple readability calculation
            if avg_sentence_length < 15:
                metrics["readability_score"] = 90
            elif avg_sentence_length < 20:
                metrics["readability_score"] = 75
            elif avg_sentence_length < 25:
                metrics["readability_score"] = 60
            else:
                metrics["readability_score"] = 40
        
        return metrics
    
    def extract_content(self, soup, base_url):
        """Extract various content types with enhanced quality"""
        content = {
            "links": [],
            "images": [],
            "text": "",
            "metadata": {},
            "headings": [],
            "buttons": [],
            "forms": []
        }
        
        # Extract links with better filtering
        for link in soup.find_all('a', href=True):
            href = link['href']
            text = link.get_text().strip()
            
            # Skip empty, javascript, and anchor links
            if (text and len(text) > 1 and 
                not href.startswith(('javascript:', 'mailto:', 'tel:', '#')) and
                href.startswith(('http://', 'https://'))):
                
                absolute_url = urljoin(base_url, href)
                content["links"].append({
                    "url": absolute_url,
                    "text": text[:100],  # Limit text length
                    "title": link.get('title', ''),
                    "target": link.get('target', '')
                })
        
        # Extract images with better metadata
        for img in soup.find_all('img', src=True):
            src = img['src']
            absolute_url = urljoin(base_url, src)
            alt = img.get('alt', '')
            title = img.get('title', '')
            
            if absolute_url.startswith(('http://', 'https://')):
                content["images"].append({
                    "url": absolute_url,
                    "alt": alt[:100],
                    "title": title[:100],
                    "width": img.get('width', ''),
                    "height": img.get('height', '')
                })
        
        # Extract headings for structure
        for level in range(1, 7):
            for heading in soup.find_all(f'h{level}'):
                text = heading.get_text().strip()
                if text and len(text) > 1:
                    content["headings"].append({
                        "level": level,
                        "text": text[:200],
                        "id": heading.get('id', '')
                    })
        
        # Extract buttons/CTAs
        for button in soup.find_all(['button', 'input'], type='submit'):
            text = button.get_text().strip() or button.get('value', '') or button.get('title', '')
            if text and len(text) > 1:
                content["buttons"].append({
                    "text": text[:100],
                    "type": button.name,
                    "class": button.get('class', [])
                })
        
        # Extract main text with better cleaning
        for script in soup(["script", "style", "noscript", "meta", "link"]):
            script.decompose()
        
        # Get text from main content areas first
        main_content = ""
        for selector in ['main', 'article', '[role="main"]', '.content', '#content']:
            main_elem = soup.select_one(selector)
            if main_elem:
                main_content = main_elem.get_text()
                break
        
        # Fallback to body if no main content found
        if not main_content:
            main_content = soup.get_text()
        
        # Clean and format text
        lines = []
        for line in main_content.split('\n'):
            line = line.strip()
            if line and len(line) > 3:  # Skip very short lines
                lines.append(line)
        
        content["text"] = ' '.join(lines[:50])  # Limit to first 50 meaningful lines
        
        # Extract enhanced metadata
        meta_tags = {}
        for meta in soup.find_all('meta'):
            name = meta.get('name') or meta.get('property') or meta.get('http-equiv')
            content = meta.get('content', '')
            if name and content:
                meta_tags[name] = content[:200]  # Limit metadata length
        
        # Add page structure info
        meta_tags['page_structure'] = {
            "headings_count": len(content["headings"]),
            "links_count": len(content["links"]),
            "images_count": len(content["images"]),
            "buttons_count": len(content["buttons"]),
            "text_length": len(content["text"])
        }
        
        content["metadata"] = meta_tags
        
        return content
    
    async def analyze_with_ai(self, content, url):
        """Analyze content with local AI model"""
        # Check if Ollama is available first
        if not self.check_ollama_availability():
            return {"error": "AI service not available", "fallback_summary": self.generate_fallback_summary(content)}
        
        try:
            # Check if AI service is available
            try:
                async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=5)) as session:
                    async with session.get(self.ai_endpoint.replace('/api/generate', '/api/tags')) as response:
                        if response.status != 200:
                            return {"error": "AI service not available", "fallback_summary": self.generate_fallback_summary(content)}
            except:
                return {"error": "AI service not running", "fallback_summary": self.generate_fallback_summary(content)}
            
            # Extract key information for AI analysis
            page_title = content.get('title', '')
            page_text = content.get('text', '')[:2000]  # Limit text for AI
            main_headings = content.get('headings', [])[:5]  # Top 5 headings
            key_links = content.get('links', [])[:10]  # Top 10 links
            
            # Prepare enhanced prompt
            prompt = f"""Analyze this web page content from {url} and provide structured insights:

PAGE TITLE: {page_title}

MAIN HEADINGS:
{chr(10).join([f"- {h['level']}. {h['text']}" for h in main_headings])}

KEY LINKS:
{chr(10).join([f"- {link['text']} -> {link['url']}" for link in key_links[:5]])}

CONTENT SAMPLE:
{page_text}

Please analyze and provide a JSON response with these exact fields:
{{
  "main_topic": "Brief description of what this page is about",
  "page_type": "website type (homepage, blog, documentation, ecommerce, etc.)",
  "key_entities": [
    {{"name": "Entity Name", "type": "person/organization/product/other", "confidence": 0.8}}
  ],
  "important_links": [
    {{"text": "Link text", "url": "URL", "purpose": "what this link does"}}
  ],
  "sentiment": "positive/neutral/negative",
  "content_quality": "high/medium/low",
  "notable_features": ["feature1", "feature2"],
  "target_audience": "developers/general/business/etc",
  "call_to_actions": ["action1", "action2"]
}}

Respond ONLY with valid JSON, no explanations:"""
            
            # Call local AI
            payload = {
                "model": self.ai_model,
                "prompt": prompt,
                "stream": False,
                "options": {
                    "temperature": 0.1,  # Lower temperature for more consistent output
                    "max_tokens": 1000
                }
            }
            
            async with aiohttp.ClientSession() as session:
                async with session.post(self.ai_endpoint, json=payload, timeout=aiohttp.ClientTimeout(total=90)) as response:
                    if response.status == 200:
                        result = await response.json()
                        ai_response = result.get('response', '')
                        
                        # Try to parse JSON from AI response
                        try:
                            # Find JSON in response
                            json_match = re.search(r'\{.*\}', ai_response, re.DOTALL)
                            if json_match:
                                parsed_json = json.loads(json_match.group())
                                
                                # Validate and clean the response
                                return self.validate_ai_response(parsed_json)
                            
                        except json.JSONDecodeError as e:
                            # If JSON parsing fails, try to extract from the raw text
                            return self.extract_ai_from_text(ai_response, content)
                        
                        # Fallback to structured summary
                        return self.create_structured_fallback(ai_response, content)
                    else:
                        return {"error": f"AI API returned status {response.status}", "fallback_summary": self.generate_fallback_summary(content)}
                        
        except Exception as e:
            return {"error": f"AI analysis failed: {str(e)}", "fallback_summary": self.generate_fallback_summary(content)}
    
    def validate_ai_response(self, ai_response):
        """Validate and clean AI response"""
        required_fields = ["main_topic", "page_type", "key_entities", "important_links", "sentiment"]
        
        # Ensure required fields exist
        for field in required_fields:
            if field not in ai_response:
                ai_response[field] = [] if field.endswith('s') else "Unknown"
        
        # Clean entities
        if isinstance(ai_response.get("key_entities"), list):
            ai_response["key_entities"] = [
                entity if isinstance(entity, dict) else {"name": str(entity), "type": "unknown", "confidence": 0.5}
                for entity in ai_response["key_entities"][:10]
            ]
        
        # Clean links
        if isinstance(ai_response.get("important_links"), list):
            ai_response["important_links"] = [
                link if isinstance(link, dict) else {"text": str(link), "url": "", "purpose": "unknown"}
                for link in ai_response["important_links"][:10]
            ]
        
        # Clean arrays
        for array_field in ["notable_features", "call_to_actions"]:
            if array_field in ai_response and not isinstance(ai_response[array_field], list):
                ai_response[array_field] = [str(ai_response[array_field])]
            elif array_field not in ai_response:
                ai_response[array_field] = []
        
        return ai_response
    
    def extract_ai_from_text(self, ai_response, content):
        """Extract structured AI information from text response"""
        try:
            # Extract main topic from first line or sentence
            lines = ai_response.split('\n')
            main_topic = ""
            for line in lines:
                line = line.strip()
                if line and len(line) > 10:
                    main_topic = line[:200]
                    break
            
            # Extract entities from content
            text_content = content.get('text', '')
            title = content.get('title', '')
            
            # Simple entity extraction
            entities = []
            words = text_content.split()
            for word in words:
                if word.istitle() and len(word) > 3 and word not in ['The', 'This', 'That', 'With', 'From', 'For', 'And', 'But', 'GitHub', 'AI', 'Platform']:
                    entities.append({"name": word, "type": "unknown", "confidence": 0.6})
            
            # Extract important links from content
            important_links = []
            for link in content.get('links', [])[:10]:
                important_links.append({
                    "text": link.get('text', ''),
                    "url": link.get('url', ''),
                    "purpose": "navigation"
                })
            
            # Extract page type from title and content
            page_type = "website"
            title_lower = title.lower()
            if any(keyword in title_lower for keyword in ['documentation', 'docs', 'guide', 'tutorial']):
                page_type = "documentation"
            elif any(keyword in title_lower for keyword in ['blog', 'news', 'article']):
                page_type = "blog"
            elif any(keyword in title_lower for keyword in ['shop', 'store', 'marketplace']):
                page_type = "ecommerce"
            elif any(keyword in title_lower for keyword in ['github', 'gitlab', 'bitbucket']):
                page_type = "development"
            
            # Determine content quality based on content length and structure
            content_length = len(text_content)
            if content_length > 10000:
                content_quality = "high"
            elif content_length > 1000:
                content_quality = "medium"
            else:
                content_quality = "low"
            
            # Determine target audience
            target_audience = "general"
            if any(keyword in text_content.lower() for keyword in ['developer', 'code', 'programming', 'software']):
                target_audience = "developers"
            elif any(keyword in text_content.lower() for keyword in ['business', 'enterprise', 'company']):
                target_audience = "business"
            elif any(keyword in text_content.lower() for keyword in ['student', 'learn', 'education']):
                target_audience = "students"
            
            # Determine sentiment
            positive_words = ['great', 'excellent', 'amazing', 'fantastic', 'love', 'perfect', 'best', 'awesome']
            negative_words = ['bad', 'terrible', 'awful', 'worst', 'hate', 'poor', 'disappointing']
            
            text_lower = text_content.lower()
            positive_count = sum(1 for word in positive_words if word in text_lower)
            negative_count = sum(1 for word in negative_words if word in text_lower)
            
            if positive_count > negative_count:
                sentiment = "positive"
            elif negative_count > positive_count:
                sentiment = "negative"
            else:
                sentiment = "neutral"
            
            # Extract notable features
            notable_features = []
            if len(content.get('links', [])) > 50:
                notable_features.append("Extensive link network")
            if len(content.get('images', [])) > 10:
                notable_features.append("Rich media content")
            if len(content.get('headings', [])) > 10:
                notable_features.append("Well-structured content")
            if content.get('forms', []):
                notable_features.append("Interactive forms")
            
            return {
                "main_topic": main_topic,
                "page_type": page_type,
                "key_entities": entities[:10],
                "important_links": important_links,
                "sentiment": sentiment,
                "content_quality": content_quality,
                "notable_features": notable_features,
                "target_audience": target_audience,
                "call_to_actions": []
            }
            
        except Exception as e:
            return self.create_structured_fallback(ai_response, content)
    
    def create_structured_fallback(self, ai_response, content):
        """Create structured fallback from AI text response"""
        # Extract entities from content
        text_content = content.get('text', '')
        title = content.get('title', '')
        
        # Simple entity extraction
        entities = []
        words = text_content.split()
        for word in words:
            if word.istitle() and len(word) > 3 and word not in ['The', 'This', 'That', 'With', 'From', 'For', 'And', 'But']:
                entities.append({"name": word, "type": "unknown", "confidence": 0.6})
        
        # Extract important links
        important_links = []
        for link in content.get('links', [])[:5]:
            important_links.append({
                "text": link.get('text', ''),
                "url": link.get('url', ''),
                "purpose": "navigation"
            })
        
        return {
            "main_topic": ai_response[:200] if ai_response else title[:200],
            "page_type": "website",
            "key_entities": entities[:5],
            "important_links": important_links,
            "sentiment": "neutral",
            "content_quality": "medium",
            "notable_features": ["Basic content extraction"],
            "target_audience": "general",
            "call_to_actions": []
        }
    
    def generate_fallback_summary(self, content):
        """Generate basic summary without AI"""
        # Extract basic information
        if isinstance(content, dict):
            # If content is already a dict, extract text from it
            text_content = content.get('text', str(content))
        else:
            # If content is a string
            text_content = content
        
        lines = str(text_content).split('\n')[:20]  # First 20 lines
        text_sample = ' '.join(line.strip() for line in lines if line.strip())
        
        # Extract potential entities (simple pattern matching)
        words = text_sample.split()
        potential_entities = []
        
        # Look for capitalized words that might be entities
        for word in words:
            if word.istitle() and len(word) > 3 and word not in ['The', 'This', 'That', 'With', 'From', 'For']:
                potential_entities.append(word)
        
        return {
            "main_topic": text_sample[:100] + "..." if len(text_sample) > 100 else text_sample,
            "entities": potential_entities[:5],
            "important_links": [],
            "sentiment": "neutral",
            "notable_features": ["Basic content extraction"]
        }
    
    async def scrape_multiple(self, urls, verbose=False):
        """Scrape multiple URLs"""
        results = {}
        
        for url in urls:
            if verbose:
                print(f"Processing: {url}")
            
            result = await self.scrape_url(url, verbose)
            results[url] = result
            
            # Small delay to be respectful
            await asyncio.sleep(1)
        
        return results
