"""
Web Scraper Tool

This tool provides advanced web scraping capabilities for extracting
structured data from websites.
"""

import requests
from bs4 import BeautifulSoup
import json
import time
from typing import Dict, Any, List, Optional
from urllib.parse import urljoin, urlparse
from tools.tool_registry import BaseTool


class WebScraperTool(BaseTool):
    """Tool for advanced web scraping and data extraction"""
    
    def __init__(self):
        super().__init__(
            name="web_scraper_tool",
            description="Extract structured data from websites. Supports table extraction, link harvesting, content mining, and custom data extraction."
        )
        self.session = requests.Session()
        self.session.headers.update({
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
        })
    
    def execute(self, query: str, context: Dict[str, Any]) -> Any:
        """
        Execute web scraping based on query
        
        Args:
            query: User query containing scraping instructions
            context: Task analysis and other context
            
        Returns:
            Scraping result with extracted data
        """
        # Extract URL from query
        url = self._extract_url(query)
        
        if not url:
            return {
                "success": False,
                "error": "No URL found in query",
                "message": "Please provide a URL to scrape"
            }
        
        # Determine scraping type based on query
        if 'table' in query.lower():
            return self._scrape_tables(url)
        elif 'links' in query.lower():
            return self._scrape_links(url)
        elif 'images' in query.lower():
            return self._scrape_images(url)
        elif 'contact' in query.lower() or 'email' in query.lower():
            return self._scrape_contact_info(url)
        elif 'product' in query.lower() or 'price' in query.lower():
            return self._scrape_product_info(url)
        else:
            return self._scrape_general_content(url, query)
    
    def _extract_url(self, query: str) -> Optional[str]:
        """Extract URL from query text"""
        import re
        
        # Look for URLs in the query
        url_pattern = r'https?://[^\s<>"{}|\\^`\[\]]+'
        urls = re.findall(url_pattern, query)
        
        if urls:
            return urls[0]
        
        # Look for domain names
        domain_pattern = r'\b(?:[a-zA-Z0-9-]+\.)+[a-zA-Z]{2,}\b'
        domains = re.findall(domain_pattern, query)
        
        if domains:
            domain = domains[0]
            if not domain.startswith(('http://', 'https://')):
                return f"https://{domain}"
            return domain
        
        return None
    
    def _get_page_content(self, url: str) -> Optional[BeautifulSoup]:
        """Get and parse page content"""
        try:
            response = self.session.get(url, timeout=10)
            response.raise_for_status()
            return BeautifulSoup(response.text, 'html.parser')
        except Exception as e:
            print(f"Error fetching {url}: {e}")
            return None
    
    def _scrape_tables(self, url: str) -> Dict[str, Any]:
        """Scrape all tables from a webpage"""
        soup = self._get_page_content(url)
        if not soup:
            return {
                "success": False,
                "error": "Failed to fetch page content",
                "url": url
            }
        
        tables = soup.find_all('table')
        extracted_tables = []
        
        for i, table in enumerate(tables):
            table_data = {
                "table_index": i,
                "headers": [],
                "rows": []
            }
            
            # Extract headers
            header_row = table.find('tr')
            if header_row:
                headers = header_row.find_all(['th', 'td'])
                table_data["headers"] = [header.get_text().strip() for header in headers]
            
            # Extract rows
            rows = table.find_all('tr')[1:]  # Skip header row
            for row in rows:
                cells = row.find_all(['td', 'th'])
                row_data = [cell.get_text().strip() for cell in cells]
                if row_data:  # Only add non-empty rows
                    table_data["rows"].append(row_data)
            
            if table_data["headers"] or table_data["rows"]:
                extracted_tables.append(table_data)
        
        return {
            "success": True,
            "url": url,
            "tables": extracted_tables,
            "table_count": len(extracted_tables),
            "message": f"Extracted {len(extracted_tables)} tables from {url}"
        }
    
    def _scrape_links(self, url: str) -> Dict[str, Any]:
        """Scrape all links from a webpage"""
        soup = self._get_page_content(url)
        if not soup:
            return {
                "success": False,
                "error": "Failed to fetch page content",
                "url": url
            }
        
        links = []
        for link in soup.find_all('a', href=True):
            href = link['href']
            text = link.get_text().strip()
            full_url = urljoin(url, href)
            
            # Categorize link type
            link_type = "external"
            if urlparse(full_url).netloc == urlparse(url).netloc:
                link_type = "internal"
            elif href.startswith('#'):
                link_type = "anchor"
            elif href.startswith('mailto:'):
                link_type = "email"
            elif href.startswith('tel:'):
                link_type = "phone"
            
            links.append({
                "text": text,
                "href": href,
                "full_url": full_url,
                "type": link_type
            })
        
        # Group links by type
        link_groups = {}
        for link in links:
            link_type = link["type"]
            if link_type not in link_groups:
                link_groups[link_type] = []
            link_groups[link_type].append(link)
        
        return {
            "success": True,
            "url": url,
            "links": links,
            "link_groups": link_groups,
            "total_links": len(links),
            "message": f"Extracted {len(links)} links from {url}"
        }
    
    def _scrape_images(self, url: str) -> Dict[str, Any]:
        """Scrape all images from a webpage"""
        soup = self._get_page_content(url)
        if not soup:
            return {
                "success": False,
                "error": "Failed to fetch page content",
                "url": url
            }
        
        images = []
        for img in soup.find_all('img'):
            src = img.get('src', '')
            alt = img.get('alt', '')
            title = img.get('title', '')
            
            if src:
                full_url = urljoin(url, src)
                images.append({
                    "src": src,
                    "full_url": full_url,
                    "alt": alt,
                    "title": title,
                    "width": img.get('width'),
                    "height": img.get('height')
                })
        
        return {
            "success": True,
            "url": url,
            "images": images,
            "image_count": len(images),
            "message": f"Extracted {len(images)} images from {url}"
        }
    
    def _scrape_contact_info(self, url: str) -> Dict[str, Any]:
        """Scrape contact information from a webpage"""
        soup = self._get_page_content(url)
        if not soup:
            return {
                "success": False,
                "error": "Failed to fetch page content",
                "url": url
            }
        
        import re
        
        # Get all text content
        text_content = soup.get_text()
        
        # Extract emails
        email_pattern = r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b'
        emails = list(set(re.findall(email_pattern, text_content)))
        
        # Extract phone numbers (basic patterns)
        phone_patterns = [
            r'\b\d{3}[-.]?\d{3}[-.]?\d{4}\b',  # US format
            r'\b\(\d{3}\)\s?\d{3}[-.]?\d{4}\b',  # (123) 456-7890
            r'\b\+\d{1,3}[-.\s]?\d{1,4}[-.\s]?\d{1,4}[-.\s]?\d{1,9}\b'  # International
        ]
        
        phones = []
        for pattern in phone_patterns:
            phones.extend(re.findall(pattern, text_content))
        phones = list(set(phones))
        
        # Look for address patterns (basic)
        address_keywords = ['address', 'location', 'street', 'avenue', 'road', 'suite']
        potential_addresses = []
        
        lines = text_content.split('\n')
        for line in lines:
            line = line.strip()
            if any(keyword in line.lower() for keyword in address_keywords):
                if len(line) > 10 and len(line) < 200:  # Reasonable address length
                    potential_addresses.append(line)
        
        return {
            "success": True,
            "url": url,
            "contact_info": {
                "emails": emails,
                "phones": phones,
                "potential_addresses": potential_addresses[:5]  # Limit to top 5
            },
            "message": f"Extracted contact information from {url}"
        }
    
    def _scrape_product_info(self, url: str) -> Dict[str, Any]:
        """Scrape product information from a webpage"""
        soup = self._get_page_content(url)
        if not soup:
            return {
                "success": False,
                "error": "Failed to fetch page content",
                "url": url
            }
        
        import re
        
        # Look for price patterns
        price_patterns = [
            r'\$\d+(?:\.\d{2})?',  # $19.99
            r'\d+(?:\.\d{2})?\s*USD',  # 19.99 USD
            r'Price:\s*\$?\d+(?:\.\d{2})?',  # Price: $19.99
        ]
        
        text_content = soup.get_text()
        prices = []
        for pattern in price_patterns:
            prices.extend(re.findall(pattern, text_content, re.IGNORECASE))
        
        # Look for product titles (h1, h2 tags)
        titles = []
        for heading in soup.find_all(['h1', 'h2', 'h3']):
            title_text = heading.get_text().strip()
            if title_text and len(title_text) < 200:
                titles.append(title_text)
        
        # Look for descriptions (p tags, divs with certain classes)
        descriptions = []
        for desc in soup.find_all(['p', 'div'], class_=re.compile(r'description|summary|detail', re.I)):
            desc_text = desc.get_text().strip()
            if desc_text and len(desc_text) > 50:
                descriptions.append(desc_text[:500])  # Limit length
        
        return {
            "success": True,
            "url": url,
            "product_info": {
                "titles": titles[:5],
                "prices": list(set(prices)),
                "descriptions": descriptions[:3]
            },
            "message": f"Extracted product information from {url}"
        }
    
    def _scrape_general_content(self, url: str, query: str) -> Dict[str, Any]:
        """Scrape general content based on query"""
        soup = self._get_page_content(url)
        if not soup:
            return {
                "success": False,
                "error": "Failed to fetch page content",
                "url": url
            }
        
        # Extract various content types
        content = {
            "title": soup.find('title').get_text().strip() if soup.find('title') else None,
            "headings": [],
            "paragraphs": [],
            "lists": [],
            "meta_description": None
        }
        
        # Meta description
        meta_desc = soup.find('meta', attrs={'name': 'description'})
        if meta_desc:
            content["meta_description"] = meta_desc.get('content', '')
        
        # Headings
        for i in range(1, 7):
            headings = soup.find_all(f'h{i}')
            for heading in headings:
                content["headings"].append({
                    "level": i,
                    "text": heading.get_text().strip()
                })
        
        # Paragraphs
        paragraphs = soup.find_all('p')
        for p in paragraphs[:10]:  # Limit to first 10 paragraphs
            text = p.get_text().strip()
            if text and len(text) > 20:
                content["paragraphs"].append(text)
        
        # Lists
        lists = soup.find_all(['ul', 'ol'])
        for lst in lists[:5]:  # Limit to first 5 lists
            list_items = [li.get_text().strip() for li in lst.find_all('li')]
            if list_items:
                content["lists"].append({
                    "type": lst.name,
                    "items": list_items
                })
        
        return {
            "success": True,
            "url": url,
            "content": content,
            "message": f"Extracted general content from {url}"
        }
    
    def get_capabilities(self) -> List[str]:
        """Return capabilities of this tool"""
        return [
            "table_extraction",
            "link_harvesting",
            "image_extraction",
            "contact_extraction",
            "product_scraping",
            "content_mining",
            "structured_data_extraction"
        ]
    
    def validate_input(self, query: str, context: Dict[str, Any]) -> bool:
        """Validate input for web scraping"""
        # Check if query contains a URL
        url = self._extract_url(query)
        if not url:
            return False
        
        # Check for scraping-related keywords
        scraping_keywords = [
            'scrape', 'extract', 'crawl', 'harvest', 'mine',
            'table', 'links', 'images', 'contact', 'product'
        ]
        
        query_lower = query.lower()
        return any(keyword in query_lower for keyword in scraping_keywords)

