#!/usr/bin/env python3
"""
zee-clone - A professional website cloning tool
Works like wget -r -np -k with automatic organization
Version: 1.0.7 - Wget-Style Recursive Edition
"""

import os
import sys
import argparse
import requests
from urllib.parse import urlparse, urljoin, urldefrag
from bs4 import BeautifulSoup
from pathlib import Path
from typing import Set, Tuple, Dict, List, Optional
import logging
from datetime import datetime
import time
import re
import hashlib
from collections import deque, defaultdict
import mimetypes

# Configure logging with colors
class ColoredFormatter(logging.Formatter):
    """Custom formatter with colors"""
    grey = "\x1b[38;20m"
    blue = "\x1b[34;20m"
    yellow = "\x1b[33;20m"
    red = "\x1b[31;20m"
    green = "\x1b[32;20m"
    cyan = "\x1b[36;20m"
    magenta = "\x1b[35;20m"
    bold = "\x1b[1m"
    reset = "\x1b[0m"
    
    format_str = "%(message)s"
    
    FORMATS = {
        logging.DEBUG: grey + format_str + reset,
        logging.INFO: blue + format_str + reset,
        logging.WARNING: yellow + format_str + reset,
        logging.ERROR: red + format_str + reset,
        'SUCCESS': green + format_str + reset,
        'HEADER': cyan + bold + format_str + reset,
        'PROGRESS': magenta + format_str + reset
    }

    def format(self, record):
        if hasattr(record, 'custom_level'):
            log_fmt = self.FORMATS.get(record.custom_level, self.FORMATS[logging.INFO])
        else:
            log_fmt = self.FORMATS.get(record.levelno, self.FORMATS[logging.INFO])
        
        formatter = logging.Formatter(log_fmt)
        return formatter.format(record)

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

# Add console handler with colored formatter
console_handler = logging.StreamHandler()
console_handler.setFormatter(ColoredFormatter())
logger.addHandler(console_handler)

# Custom log levels
SUCCESS = 25
PROGRESS = 15
HEADER = 35
logging.addLevelName(SUCCESS, "SUCCESS")
logging.addLevelName(PROGRESS, "PROGRESS")
logging.addLevelName(HEADER, "HEADER")


class ResourceOrganizer:
    """Handles organization of downloaded resources into proper folders"""
    
    # Folder structure
    FOLDERS = {
        'css': 'css',
        'js': 'js',
        'images': 'images',
        'fonts': 'fonts',
        'videos': 'videos',
        'audio': 'audio',
        'documents': 'documents',
        'pages': 'pages',  # For HTML pages
        'other': 'other'
    }
    
    # File type mappings
    FILE_TYPES = {
        # HTML files
        '.html': 'pages',
        '.htm': 'pages',
        '.php': 'pages',
        '.asp': 'pages',
        '.aspx': 'pages',
        '.jsp': 'pages',
        
        # CSS files
        '.css': 'css',
        
        # JavaScript files
        '.js': 'js',
        '.mjs': 'js',
        '.cjs': 'js',
        
        # Images
        '.jpg': 'images',
        '.jpeg': 'images',
        '.png': 'images',
        '.gif': 'images',
        '.webp': 'images',
        '.svg': 'images',
        '.ico': 'images',
        '.bmp': 'images',
        '.tiff': 'images',
        
        # Fonts
        '.woff': 'fonts',
        '.woff2': 'fonts',
        '.ttf': 'fonts',
        '.otf': 'fonts',
        '.eot': 'fonts',
        
        # Videos
        '.mp4': 'videos',
        '.webm': 'videos',
        '.ogg': 'videos',
        '.mov': 'videos',
        '.avi': 'videos',
        '.mkv': 'videos',
        
        # Audio
        '.mp3': 'audio',
        '.wav': 'audio',
        '.aac': 'audio',
        '.flac': 'audio',
        '.m4a': 'audio',
        
        # Documents
        '.pdf': 'documents',
        '.doc': 'documents',
        '.docx': 'documents',
        '.xls': 'documents',
        '.xlsx': 'documents',
        '.ppt': 'documents',
        '.pptx': 'documents',
        '.txt': 'documents',
        '.json': 'documents',
        '.xml': 'documents',
    }
    
    def __init__(self, base_dir: Path, base_url: str):
        """
        Initialize the resource organizer.
        
        Args:
            base_dir: Base directory for the cloned website
            base_url: Base URL of the website
        """
        self.base_dir = Path(base_dir)
        self.base_url = base_url.rstrip('/')
        self.base_domain = urlparse(base_url).netloc
        self.file_mapping = {}  # Maps original URLs to relative paths
        self.url_to_path = {}   # Maps URLs to local file paths
        self.folder_paths = self._create_folders()
        self.resource_counter = defaultdict(int)  # Counter for duplicate filenames
        self.downloaded_urls = set()  # Track downloaded URLs to avoid duplicates
        
    def _create_folders(self) -> Dict[str, Path]:
        """Create all necessary folders"""
        folder_paths = {}
        for folder_name, folder_path in self.FOLDERS.items():
            path = self.base_dir / folder_path
            path.mkdir(parents=True, exist_ok=True)
            folder_paths[folder_name] = path
        return folder_paths
    
    def get_file_category(self, url: str, content_type: Optional[str] = None) -> str:
        """
        Determine the category of a file based on URL or content type.
        
        Args:
            url: File URL
            content_type: HTTP content-type header
        
        Returns:
            Category name (css, js, images, etc.)
        """
        # Remove query parameters and fragments
        url = url.split('?')[0].split('#')[0]
        
        # Try to get extension from URL
        parsed = urlparse(url)
        path = unquote(parsed.path)
        ext = os.path.splitext(path)[1].lower()
        
        if ext in self.FILE_TYPES:
            return self.FILE_TYPES[ext]
        
        # Try content-type mapping
        if content_type:
            content_type = content_type.lower()
            if 'text/html' in content_type:
                return 'pages'
            elif 'text/css' in content_type:
                return 'css'
            elif 'javascript' in content_type:
                return 'js'
            elif 'image/' in content_type:
                return 'images'
            elif 'font/' in content_type:
                return 'fonts'
            elif 'video/' in content_type:
                return 'videos'
            elif 'audio/' in content_type:
                return 'audio'
            elif 'application/pdf' in content_type:
                return 'documents'
        
        return 'other'
    
    def url_to_filename(self, url: str) -> str:
        """
        Convert a URL to a safe filename.
        
        Args:
            url: Original URL
        
        Returns:
            Safe filename
        """
        parsed = urlparse(url)
        path = parsed.path
        
        # If it's a directory-like URL (ends with /), save as index.html
        if path.endswith('/') or not os.path.basename(path):
            # Create a directory structure
            if path:
                # Remove leading slash
                rel_path = path.lstrip('/')
                if rel_path:
                    return os.path.join(rel_path, 'index.html')
            return 'index.html'
        
        # Get the filename from the path
        filename = os.path.basename(path)
        
        # Remove query parameters
        filename = filename.split('?')[0].split('#')[0]
        
        # If no extension, add .html for pages
        if not os.path.splitext(filename)[1]:
            filename += '.html'
        
        return filename
    
    def get_output_path(self, url: str, content_type: Optional[str] = None, is_page: bool = False) -> Tuple[Path, str]:
        """
        Get the output path for a resource.
        
        Args:
            url: Resource URL
            content_type: HTTP content-type header
            is_page: Whether this is an HTML page
        
        Returns:
            Tuple of (output_path, relative_path)
        """
        parsed = urlparse(url)
        
        # For pages, preserve directory structure
        if is_page or 'text/html' in (content_type or ''):
            # Get the path relative to base URL
            base_path = urlparse(self.base_url).path.rstrip('/')
            current_path = parsed.path
            
            # Remove base path if it's a subdirectory
            if base_path and current_path.startswith(base_path):
                rel_path = current_path[len(base_path):].lstrip('/')
            else:
                rel_path = current_path.lstrip('/')
            
            # Generate filename
            if not rel_path or rel_path.endswith('/'):
                filename = 'index.html'
                subdir = rel_path.rstrip('/')
            else:
                filename = os.path.basename(rel_path)
                subdir = os.path.dirname(rel_path)
            
            # Create full path
            if subdir:
                full_path = self.folder_paths['pages'] / subdir / filename
                relative_path = f"pages/{subdir}/{filename}"
            else:
                full_path = self.folder_paths['pages'] / filename
                relative_path = f"pages/{filename}"
            
            # Create parent directories
            full_path.parent.mkdir(parents=True, exist_ok=True)
            
            return full_path, relative_path
        
        # For other resources, use category-based organization
        category = self.get_file_category(url, content_type)
        filename = self.generate_filename(url, category)
        
        # Get the appropriate folder
        if category == 'css':
            output_path = self.folder_paths['css'] / filename
            relative_path = f"css/{filename}"
        elif category == 'js':
            output_path = self.folder_paths['js'] / filename
            relative_path = f"js/{filename}"
        elif category == 'images':
            output_path = self.folder_paths['images'] / filename
            relative_path = f"images/{filename}"
        elif category == 'fonts':
            output_path = self.folder_paths['fonts'] / filename
            relative_path = f"fonts/{filename}"
        elif category == 'videos':
            output_path = self.folder_paths['videos'] / filename
            relative_path = f"videos/{filename}"
        elif category == 'audio':
            output_path = self.folder_paths['audio'] / filename
            relative_path = f"audio/{filename}"
        elif category == 'documents':
            output_path = self.folder_paths['documents'] / filename
            relative_path = f"documents/{filename}"
        elif category == 'pages':
            output_path = self.folder_paths['pages'] / filename
            relative_path = f"pages/{filename}"
        else:
            output_path = self.folder_paths['other'] / filename
            relative_path = f"other/{filename}"
        
        return output_path, relative_path
    
    def generate_filename(self, url: str, category: str) -> str:
        """
        Generate a clean, unique filename for a resource.
        
        Args:
            url: Original URL
            category: Resource category
        
        Returns:
            Clean filename
        """
        parsed = urlparse(url)
        path = unquote(parsed.path)
        
        # Get the original filename
        original_name = os.path.basename(path) or f"resource_{hashlib.md5(url.encode()).hexdigest()[:8]}"
        
        # Remove query parameters
        original_name = original_name.split('?')[0].split('#')[0]
        
        # Ensure it has an extension
        if '.' not in original_name:
            # Try to guess extension from URL or use default
            if category == 'css':
                original_name += '.css'
            elif category == 'js':
                original_name += '.js'
            elif category == 'images':
                original_name += '.jpg'
            elif category == 'fonts':
                original_name += '.woff2'
            elif category == 'videos':
                original_name += '.mp4'
            elif category == 'audio':
                original_name += '.mp3'
            elif category == 'documents':
                original_name += '.bin'
            else:
                original_name += '.bin'
        
        # Handle duplicate filenames
        counter_key = f"{category}/{original_name}"
        self.resource_counter[counter_key] += 1
        
        if self.resource_counter[counter_key] > 1:
            name, ext = os.path.splitext(original_name)
            original_name = f"{name}_{self.resource_counter[counter_key]}{ext}"
        
        return original_name
    
    def is_same_domain(self, url: str) -> bool:
        """Check if URL belongs to the same domain"""
        try:
            parsed = urlparse(url)
            return parsed.netloc == self.base_domain or not parsed.netloc
        except:
            return False
    
    def normalize_url(self, url: str) -> str:
        """Normalize URL by removing fragments and standardizing"""
        url, _ = urldefrag(url)
        return url.rstrip('/')


class WebsiteCloner:
    """Main cloner class that handles recursive downloading"""
    
    def __init__(self, base_url: str, output_dir: Path, recursive: bool = True, no_parent: bool = True, 
                 convert_links: bool = True, level: int = 5, aggressive: bool = False):
        """
        Initialize the website cloner.
        
        Args:
            base_url: Base URL to clone
            output_dir: Output directory
            recursive: Enable recursive downloading
            no_parent: Don't ascend to parent directory
            convert_links: Convert links for local viewing
            level: Recursion depth limit
            aggressive: Download all resources aggressively
        """
        self.base_url = base_url.rstrip('/')
        self.output_dir = Path(output_dir)
        self.recursive = recursive
        self.no_parent = no_parent
        self.convert_links = convert_links
        self.max_depth = level
        self.aggressive = aggressive
        
        # Initialize organizer
        self.organizer = ResourceOrganizer(output_dir, base_url)
        
        # Queue for recursive downloading
        self.url_queue = deque()
        self.depth_map = {}  # Track depth of each URL
        
        # Statistics
        self.stats = {
            'url': base_url,
            'date': datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
            'files_downloaded': 0,
            'pages_downloaded': 0,
            'css_count': 0,
            'js_count': 0,
            'image_count': 0,
            'font_count': 0,
            'video_count': 0,
            'audio_count': 0,
            'document_count': 0,
            'other_count': 0,
            'total_size': 0,
            'time': 0
        }
        
        # Session for connection reuse
        self.session = requests.Session()
        self.session.headers.update({
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
            'Accept-Language': 'en-US,en;q=0.5',
            'Accept-Encoding': 'gzip, deflate',
            'DNT': '1',
            'Connection': 'keep-alive',
            'Upgrade-Insecure-Requests': '1'
        })
    
    def is_valid_url(self, url: str) -> bool:
        """Check if URL is valid for downloading"""
        if not url or url.startswith(('#', 'mailto:', 'tel:', 'javascript:', 'data:')):
            return False
        
        # Check if it's the same domain (for -np behavior)
        if self.no_parent:
            parsed_url = urlparse(url)
            parsed_base = urlparse(self.base_url)
            
            # Must be same domain
            if parsed_url.netloc and parsed_url.netloc != parsed_base.netloc:
                return False
            
            # Must not go to parent directory
            base_path = parsed_base.path.rstrip('/').split('/')
            url_path = parsed_url.path.rstrip('/').split('/')
            
            # If url path is shorter, it might be going to parent
            if len(url_path) < len(base_path):
                # Check if it's actually a parent
                for i in range(len(url_path)):
                    if i < len(base_path) and url_path[i] != base_path[i]:
                        break
                else:
                    # All matched, this is a parent directory
                    return False
        
        return True
    
    def extract_links(self, soup: BeautifulSoup, base_url: str) -> Set[str]:
        """Extract all links from HTML for recursive downloading"""
        links = set()
        
        # Find all anchor tags
        for a in soup.find_all('a', href=True):
            href = a['href']
            if href:
                full_url = urljoin(base_url, href)
                if self.is_valid_url(full_url):
                    links.add(self.organizer.normalize_url(full_url))
        
        # Also check for other link types
        for link in soup.find_all('link', href=True):
            href = link['href']
            if href:
                full_url = urljoin(base_url, href)
                if self.is_valid_url(full_url):
                    links.add(self.organizer.normalize_url(full_url))
        
        return links
    
    def extract_resources(self, soup: BeautifulSoup, base_url: str) -> Dict[str, Set[str]]:
        """Extract all resources from HTML"""
        resources = {
            'images': set(),
            'css': set(),
            'js': set(),
            'fonts': set(),
            'video': set(),
            'audio': set(),
            'documents': set(),
            'other': set()
        }
        
        # Find all image tags
        for img in soup.find_all('img'):
            if img.get('src'):
                src = img['src']
                if src and not src.startswith('data:'):
                    full_url = urljoin(base_url, src)
                    category = self.organizer.get_file_category(full_url)
                    resources[category].add(full_url)
            
            if img.get('srcset'):
                srcset = img['srcset']
                for part in srcset.split(','):
                    src = part.strip().split()[0]
                    if src and not src.startswith('data:'):
                        full_url = urljoin(base_url, src)
                        category = self.organizer.get_file_category(full_url)
                        resources[category].add(full_url)
        
        # Find all CSS links
        for link in soup.find_all('link'):
            if link.get('rel') and 'stylesheet' in link.get('rel', []):
                if link.get('href'):
                    href = link['href']
                    full_url = urljoin(base_url, href)
                    resources['css'].add(full_url)
            
            if link.get('as') == 'style' and link.get('href'):
                full_url = urljoin(base_url, link['href'])
                resources['css'].add(full_url)
            
            if link.get('rel') and any(r in ['icon', 'apple-touch-icon'] for r in link.get('rel', [])):
                if link.get('href'):
                    full_url = urljoin(base_url, link['href'])
                    resources['images'].add(full_url)
        
        # Find all JavaScript files
        for script in soup.find_all('script'):
            if script.get('src'):
                src = script['src']
                if src and not src.startswith('data:'):
                    full_url = urljoin(base_url, src)
                    resources['js'].add(full_url)
        
        # Find video sources
        for video in soup.find_all('video'):
            if video.get('src'):
                full_url = urljoin(base_url, video['src'])
                resources['video'].add(full_url)
            
            for source in video.find_all('source'):
                if source.get('src'):
                    full_url = urljoin(base_url, source['src'])
                    resources['video'].add(full_url)
        
        # Find audio sources
        for audio in soup.find_all('audio'):
            if audio.get('src'):
                full_url = urljoin(base_url, audio['src'])
                resources['audio'].add(full_url)
            
            for source in audio.find_all('source'):
                if source.get('src'):
                    full_url = urljoin(base_url, source['src'])
                    resources['audio'].add(full_url)
        
        # Find iframe sources
        for iframe in soup.find_all('iframe'):
            if iframe.get('src'):
                src = iframe['src']
                if src and not src.startswith('data:'):
                    full_url = urljoin(base_url, src)
                    resources['other'].add(full_url)
        
        # Find document links
        for a in soup.find_all('a', href=True):
            href = a['href']
            if href and not href.startswith(('#', 'mailto:', 'tel:', 'javascript:')):
                full_url = urljoin(base_url, href)
                ext = os.path.splitext(urlparse(full_url).path)[1].lower()
                if ext in ['.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx', '.txt']:
                    resources['documents'].add(full_url)
        
        # Find inline style attributes
        for tag in soup.find_all(style=True):
            style = tag['style']
            style_urls = re.findall(r'url\([\'"]?([^\'")]+)[\'"]?\)', style)
            for url in style_urls:
                if url and not url.startswith('data:'):
                    full_url = urljoin(base_url, url)
                    category = self.organizer.get_file_category(full_url)
                    resources[category].add(full_url)
        
        # Find meta tags
        for meta in soup.find_all('meta'):
            if meta.get('property', '').startswith('og:') or meta.get('name', '').startswith('twitter:'):
                if meta.get('content') and ('image' in meta.get('property', '') or 'image' in meta.get('name', '')):
                    content = meta['content']
                    if content.startswith(('http://', 'https://')):
                        resources['images'].add(content)
        
        return resources
    
    def extract_css_resources(self, css_content: str, base_url: str) -> Set[str]:
        """Extract URLs from CSS content"""
        resources = set()
        
        # Find url() patterns
        url_pattern = r'url\([\'"]?([^\'")]+)[\'"]?\)'
        urls = re.findall(url_pattern, css_content)
        
        for url in urls:
            if url and not url.startswith('data:'):
                full_url = urljoin(base_url, url.strip())
                resources.add(full_url)
        
        # Find @import statements
        import_pattern = r'@import\s+[\'"]?([^\'";]+)[\'"]?'
        imports = re.findall(import_pattern, css_content)
        
        for imp in imports:
            if imp and not imp.startswith('data:'):
                full_url = urljoin(base_url, imp.strip())
                resources.add(full_url)
        
        return resources
    
    def download_file(self, url: str, is_page: bool = False) -> Tuple[bool, int, Optional[str]]:
        """
        Download a file and save it.
        
        Args:
            url: URL to download
            is_page: Whether this is an HTML page
        
        Returns:
            Tuple of (success, file_size, content_type)
        """
        if url in self.organizer.downloaded_urls:
            return True, 0, None
        
        try:
            # Add referer for same-domain requests
            headers = {}
            if self.organizer.is_same_domain(url):
                headers['Referer'] = self.base_url
            
            response = self.session.get(url, timeout=10, stream=True, headers=headers)
            response.raise_for_status()
            
            content_type = response.headers.get('content-type', '').split(';')[0].lower()
            
            # Get output path
            output_path, relative_path = self.organizer.get_output_path(url, content_type, is_page)
            
            # Save the file
            downloaded = 0
            with open(output_path, 'wb') as f:
                for chunk in response.iter_content(chunk_size=8192):
                    if chunk:
                        f.write(chunk)
                        downloaded += len(chunk)
            
            # Store in mappings
            self.organizer.downloaded_urls.add(url)
            self.organizer.file_mapping[url] = relative_path
            self.organizer.url_to_path[url] = str(output_path)
            
            # Update statistics
            file_size_kb = downloaded / 1024
            category = self.organizer.get_file_category(url, content_type)
            
            if category == 'pages':
                self.stats['pages_downloaded'] += 1
            elif category == 'css':
                self.stats['css_count'] += 1
            elif category == 'js':
                self.stats['js_count'] += 1
            elif category == 'images':
                self.stats['image_count'] += 1
            elif category == 'fonts':
                self.stats['font_count'] += 1
            elif category == 'videos':
                self.stats['video_count'] += 1
            elif category == 'audio':
                self.stats['audio_count'] += 1
            elif category == 'documents':
                self.stats['document_count'] += 1
            else:
                self.stats['other_count'] += 1
            
            self.stats['files_downloaded'] += 1
            self.stats['total_size'] += downloaded / (1024 * 1024)
            
            # Log success
            file_size_str = f"{file_size_kb:.1f} KB" if file_size_kb < 1024 else f"{file_size_kb/1024:.1f} MB"
            logger.log(SUCCESS, f"  ✓ Downloaded: {relative_path} ({file_size_str})")
            
            return True, downloaded, content_type
            
        except requests.exceptions.RequestException as e:
            logger.error(f"  ✗ Failed to download {url}: {e}")
            return False, 0, None
        except Exception as e:
            logger.error(f"  ✗ Error: {e}")
            return False, 0, None
    
    def process_css_file(self, css_url: str, depth: int = 0) -> Set[str]:
        """Process CSS file and extract resources"""
        if depth > 3:  # Limit recursion depth
            return set()
        
        try:
            success, _, content_type = self.download_file(css_url)
            if not success:
                return set()
            
            # Get the local file path
            if css_url in self.organizer.url_to_path:
                css_path = self.organizer.url_to_path[css_url]
                
                # Read and process CSS
                with open(css_path, 'r', encoding='utf-8', errors='ignore') as f:
                    css_content = f.read()
                
                # Extract resources
                resources = self.extract_css_resources(css_content, css_url)
                
                # Process imported CSS files
                import_pattern = r'@import\s+[\'"]?([^\'";]+)[\'"]?'
                imports = re.findall(import_pattern, css_content)
                
                for imp in imports:
                    if imp and not imp.startswith('data:'):
                        imp_url = urljoin(css_url, imp.strip())
                        if imp_url not in self.organizer.downloaded_urls:
                            sub_resources = self.process_css_file(imp_url, depth + 1)
                            resources.update(sub_resources)
                
                return resources
            
        except Exception as e:
            logger.error(f"  ✗ Failed to process CSS {css_url}: {e}")
        
        return set()
    
    def process_page(self, url: str, depth: int = 0) -> Set[str]:
        """
        Process a single page: download, extract resources, and return links.
        
        Args:
            url: Page URL
            depth: Current recursion depth
        
        Returns:
            Set of links found on the page
        """
        if depth > self.max_depth:
            return set()
        
        logger.log(PROGRESS, f"\n{'  ' * depth}📄 Processing [{depth}/{self.max_depth}]: {url}")
        
        # Download the page
        success, _, content_type = self.download_file(url, is_page=True)
        if not success or 'text/html' not in (content_type or ''):
            return set()
        
        # Get the local file path
        if url not in self.organizer.url_to_path:
            return set()
        
        html_path = self.organizer.url_to_path[url]
        
        # Parse HTML
        with open(html_path, 'r', encoding='utf-8', errors='ignore') as f:
            soup = BeautifulSoup(f.read(), 'lxml')
        
        # Extract resources
        resources = self.extract_resources(soup, url)
        
        # Download CSS files first (they may contain more resources)
        for css_url in resources['css']:
            if css_url not in self.organizer.downloaded_urls:
                css_resources = self.process_css_file(css_url)
                # Add CSS resources to appropriate categories
                for res in css_resources:
                    category = self.organizer.get_file_category(res)
                    if category in resources:
                        resources[category].add(res)
        
        # Download JavaScript files
        for js_url in resources['js']:
            if js_url not in self.organizer.downloaded_urls:
                self.download_file(js_url)
        
        # Download images
        for img_url in resources['images']:
            if img_url not in self.organizer.downloaded_urls:
                self.download_file(img_url)
        
        # Download fonts
        for font_url in resources['fonts']:
            if font_url not in self.organizer.downloaded_urls:
                self.download_file(font_url)
        
        # Download videos
        for video_url in resources['video']:
            if video_url not in self.organizer.downloaded_urls:
                self.download_file(video_url)
        
        # Download audio
        for audio_url in resources['audio']:
            if audio_url not in self.organizer.downloaded_urls:
                self.download_file(audio_url)
        
        # Download documents if aggressive
        if self.aggressive:
            for doc_url in resources['documents']:
                if doc_url not in self.organizer.downloaded_urls:
                    self.download_file(doc_url)
        
        # Download other resources if aggressive
        if self.aggressive:
            for other_url in resources['other']:
                if other_url not in self.organizer.downloaded_urls:
                    self.download_file(other_url)
        
        # Extract links for recursive crawling
        if self.recursive and depth < self.max_depth:
            links = self.extract_links(soup, url)
            
            # Filter and queue links
            for link in links:
                if link not in self.organizer.downloaded_urls and link not in [item[0] for item in self.url_queue]:
                    self.url_queue.append((link, depth + 1))
        
        # Convert links to local paths if requested
        if self.convert_links:
            self.convert_html_links(soup, url)
            
            # Save the updated HTML
            with open(html_path, 'w', encoding='utf-8') as f:
                f.write(str(soup))
        
        return resources
    
    def convert_html_links(self, soup: BeautifulSoup, base_url: str):
        """Convert HTML links to point to local files"""
        
        # Update all links
        for tag in soup.find_all(['a', 'link'], href=True):
            href = tag['href']
            if href and not href.startswith(('#', 'mailto:', 'tel:', 'javascript:')):
                full_url = urljoin(base_url, href)
                if full_url in self.organizer.file_mapping:
                    tag['href'] = self.organizer.file_mapping[full_url]
        
        # Update all scripts
        for script in soup.find_all('script', src=True):
            src = script['src']
            if src:
                full_url = urljoin(base_url, src)
                if full_url in self.organizer.file_mapping:
                    script['src'] = self.organizer.file_mapping[full_url]
        
        # Update all images
        for img in soup.find_all('img', src=True):
            src = img['src']
            if src and not src.startswith('data:'):
                full_url = urljoin(base_url, src)
                if full_url in self.organizer.file_mapping:
                    img['src'] = self.organizer.file_mapping[full_url]
        
        # Update video sources
        for video in soup.find_all('video', src=True):
            src = video['src']
            if src:
                full_url = urljoin(base_url, src)
                if full_url in self.organizer.file_mapping:
                    video['src'] = self.organizer.file_mapping[full_url]
        
        for source in soup.find_all('source', src=True):
            src = source['src']
            if src:
                full_url = urljoin(base_url, src)
                if full_url in self.organizer.file_mapping:
                    source['src'] = self.organizer.file_mapping[full_url]
        
        # Update audio sources
        for audio in soup.find_all('audio', src=True):
            src = audio['src']
            if src:
                full_url = urljoin(base_url, src)
                if full_url in self.organizer.file_mapping:
                    audio['src'] = self.organizer.file_mapping[full_url]
    
    def run(self) -> bool:
        """Run the cloning process"""
        start_time = time.time()
        
        try:
            logger.log(HEADER, f"\n{ColoredFormatter.bold}🌐 TARGET: {self.base_url}{ColoredFormatter.reset}")
            logger.log(HEADER, f"{ColoredFormatter.bold}📁 OUTPUT: {self.output_dir}{ColoredFormatter.reset}")
            logger.log(HEADER, f"{ColoredFormatter.bold}⚙️  MODE: {'Recursive' if self.recursive else 'Single page'} (depth: {self.max_depth}){ColoredFormatter.reset}\n")
            
            # Start with the base URL
            self.url_queue.append((self.base_url, 0))
            
            # Process queue
            while self.url_queue:
                url, depth = self.url_queue.popleft()
                self.process_page(url, depth)
            
            # Calculate total time
            self.stats['time'] = time.time() - start_time
            
            # Create reports
            self.create_reports()
            
            # Show summary
            self.show_summary()
            
            return True
            
        except KeyboardInterrupt:
            logger.error(f"\n{ColoredFormatter.red}⚠️  Interrupted by user{ColoredFormatter.reset}")
            return False
        except Exception as e:
            logger.error(f"\n{ColoredFormatter.red}❌ Error: {e}{ColoredFormatter.reset}")
            import traceback
            traceback.print_exc()
            return False
    
    def create_reports(self):
        """Create statistics and HTML reports"""
        
        # Text statistics
        stats_path = self.output_dir / 'clone_stats.txt'
        with open(stats_path, 'w') as f:
            f.write("=" * 60 + "\n")
            f.write("ZEE-CLONE STATISTICS (v1.0.7 Wget-Style Edition)\n")
            f.write("=" * 60 + "\n\n")
            f.write(f"URL: {self.stats['url']}\n")
            f.write(f"Date: {self.stats['date']}\n")
            f.write(f"Time taken: {self.stats['time']:.2f} seconds\n")
            f.write(f"Total files downloaded: {self.stats['files_downloaded']}\n")
            f.write(f"Total size: {self.stats['total_size']:.2f} MB\n\n")
            
            f.write("BREAKDOWN BY TYPE:\n")
            f.write("-" * 40 + "\n")
            f.write(f"HTML Pages: {self.stats.get('pages_downloaded', 0)}\n")
            f.write(f"CSS files: {self.stats.get('css_count', 0)}\n")
            f.write(f"JavaScript files: {self.stats.get('js_count', 0)}\n")
            f.write(f"Images: {self.stats.get('image_count', 0)}\n")
            f.write(f"Fonts: {self.stats.get('font_count', 0)}\n")
            f.write(f"Video files: {self.stats.get('video_count', 0)}\n")
            f.write(f"Audio files: {self.stats.get('audio_count', 0)}\n")
            f.write(f"Documents: {self.stats.get('document_count', 0)}\n")
            f.write(f"Other resources: {self.stats.get('other_count', 0)}\n")
            f.write("=" * 60 + "\n\n")
            
            f.write("FOLDER STRUCTURE:\n")
            f.write("-" * 40 + "\n")
            f.write(f"📁 {self.output_dir.name}/\n")
            f.write(f"├── pages/        ({self.stats.get('pages_downloaded', 0)} files)\n")
            f.write(f"├── css/          ({self.stats.get('css_count', 0)} files)\n")
            f.write(f"├── js/           ({self.stats.get('js_count', 0)} files)\n")
            f.write(f"├── images/       ({self.stats.get('image_count', 0)} files)\n")
            f.write(f"├── fonts/        ({self.stats.get('font_count', 0)} files)\n")
            f.write(f"├── videos/       ({self.stats.get('video_count', 0)} files)\n")
            f.write(f"├── audio/        ({self.stats.get('audio_count', 0)} files)\n")
            f.write(f"├── documents/    ({self.stats.get('document_count', 0)} files)\n")
            f.write(f"└── other/        ({self.stats.get('other_count', 0)} files)\n")
        
        # HTML report
        html_report = f"""<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>zee-clone - Website Clone Report</title>
    <style>
        body {{
            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, sans-serif;
            line-height: 1.6;
            color: #333;
            max-width: 1200px;
            margin: 0 auto;
            padding: 20px;
            background: #f5f5f5;
        }}
        .header {{
            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
            color: white;
            padding: 30px;
            border-radius: 10px;
            margin-bottom: 30px;
            box-shadow: 0 4px 6px rgba(0,0,0,0.1);
        }}
        .header h1 {{
            margin: 0;
            font-size: 2.5em;
        }}
        .header p {{
            margin: 10px 0 0;
            opacity: 0.9;
        }}
        .stats-grid {{
            display: grid;
            grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
            gap: 20px;
            margin-bottom: 30px;
        }}
        .stat-card {{
            background: white;
            padding: 20px;
            border-radius: 10px;
            box-shadow: 0 2px 4px rgba(0,0,0,0.1);
            text-align: center;
        }}
        .stat-card h3 {{
            margin: 0;
            color: #764ba2;
            font-size: 2em;
        }}
        .stat-card p {{
            margin: 5px 0 0;
            color: #666;
        }}
        .section {{
            background: white;
            padding: 20px;
            border-radius: 10px;
            margin-bottom: 20px;
            box-shadow: 0 2px 4px rgba(0,0,0,0.1);
        }}
        .section h2 {{
            margin-top: 0;
            color: #444;
            border-bottom: 2px solid #f0f0f0;
            padding-bottom: 10px;
        }}
        .folder-structure {{
            background: #f8f8f8;
            padding: 15px;
            border-radius: 5px;
            font-family: monospace;
            line-height: 1.8;
        }}
        .footer {{
            text-align: center;
            color: #666;
            margin-top: 40px;
            padding: 20px;
            border-top: 1px solid #ddd;
        }}
        .badge {{
            display: inline-block;
            padding: 3px 8px;
            border-radius: 3px;
            font-size: 0.8em;
            font-weight: bold;
            margin-left: 10px;
        }}
        .badge.primary {{
            background: #667eea;
            color: white;
        }}
    </style>
</head>
<body>
    <div class="header">
        <h1>📁 zee-clone Wget-Style Report</h1>
        <p>URL: {self.stats['url']}</p>
        <p>Cloned on: {self.stats['date']}</p>
        <p><span class="badge primary">Recursive Mode</span> <span class="badge primary">Link Conversion</span></p>
    </div>

    <div class="stats-grid">
        <div class="stat-card">
            <h3>{self.stats['files_downloaded']}</h3>
            <p>Total Files</p>
        </div>
        <div class="stat-card">
            <h3>{self.stats['total_size']:.2f} MB</h3>
            <p>Total Size</p>
        </div>
        <div class="stat-card">
            <h3>{self.stats['time']:.2f}s</h3>
            <p>Time Taken</p>
        </div>
        <div class="stat-card">
            <h3>{self.stats['pages_downloaded']}</h3>
            <p>HTML Pages</p>
        </div>
    </div>

    <div class="section">
        <h2>📊 Download Statistics</h2>
        <div style="display: grid; grid-template-columns: repeat(2, 1fr); gap: 10px;">
            <p><strong>HTML Pages:</strong> {self.stats.get('pages_downloaded', 0)}</p>
            <p><strong>CSS:</strong> {self.stats.get('css_count', 0)}</p>
            <p><strong>JavaScript:</strong> {self.stats.get('js_count', 0)}</p>
            <p><strong>Images:</strong> {self.stats.get('image_count', 0)}</p>
            <p><strong>Fonts:</strong> {self.stats.get('font_count', 0)}</p>
            <p><strong>Videos:</strong> {self.stats.get('video_count', 0)}</p>
            <p><strong>Audio:</strong> {self.stats.get('audio_count', 0)}</p>
            <p><strong>Documents:</strong> {self.stats.get('document_count', 0)}</p>
            <p><strong>Other:</strong> {self.stats.get('other_count', 0)}</p>
        </div>
    </div>

    <div class="section">
        <h2>📂 Folder Structure</h2>
        <div class="folder-structure">
📁 {self.output_dir.name}/<br>
├── pages/        ({self.stats.get('pages_downloaded', 0)} files)<br>
├── css/          ({self.stats.get('css_count', 0)} files)<br>
├── js/           ({self.stats.get('js_count', 0)} files)<br>
├── images/       ({self.stats.get('image_count', 0)} files)<br>
├── fonts/        ({self.stats.get('font_count', 0)} files)<br>
├── videos/       ({self.stats.get('video_count', 0)} files)<br>
├── audio/        ({self.stats.get('audio_count', 0)} files)<br>
├── documents/    ({self.stats.get('document_count', 0)} files)<br>
└── other/        ({self.stats.get('other_count', 0)} files)
        </div>
    </div>

    <div class="footer">
        <p>Generated by zee-clone v1.0.7 | MIT License</p>
        <p><small>To view the cloned website, open pages/index.html in your browser</small></p>
    </div>
</body>
</html>"""
        
        report_path = self.output_dir / 'clone_report.html'
        with open(report_path, 'w', encoding='utf-8') as f:
            f.write(html_report)
        
        logger.log(SUCCESS, f"  ✓ Created report: clone_report.html")
    
    def show_summary(self):
        """Show summary of the cloning operation"""
        logger.log(HEADER, f"\n{ColoredFormatter.bold}{ColoredFormatter.green}✨ CLONE COMPLETE! ✨{ColoredFormatter.reset}\n")
        logger.log(SUCCESS, f"📁 Location: {self.output_dir}")
        logger.log(SUCCESS, f"⏱️  Time: {self.stats['time']:.2f} seconds")
        logger.log(SUCCESS, f"📊 Files: {self.stats['files_downloaded']} total")
        
        logger.log(HEADER, f"\n{ColoredFormatter.bold}📁 Folder Structure:{ColoredFormatter.reset}")
        logger.log(SUCCESS, f"   📁 {self.output_dir.name}/")
        logger.log(SUCCESS, f"   ├── pages/        ({self.stats.get('pages_downloaded', 0)} files)")
        logger.log(SUCCESS, f"   ├── css/          ({self.stats.get('css_count', 0)} files)")
        logger.log(SUCCESS, f"   ├── js/           ({self.stats.get('js_count', 0)} files)")
        logger.log(SUCCESS, f"   ├── images/       ({self.stats.get('image_count', 0)} files)")
        logger.log(SUCCESS, f"   ├── fonts/        ({self.stats.get('font_count', 0)} files)")
        logger.log(SUCCESS, f"   ├── videos/       ({self.stats.get('video_count', 0)} files)")
        logger.log(SUCCESS, f"   ├── audio/        ({self.stats.get('audio_count', 0)} files)")
        if self.stats.get('document_count', 0) > 0:
            logger.log(SUCCESS, f"   ├── documents/    ({self.stats.get('document_count', 0)} files)")
        if self.stats.get('other_count', 0) > 0:
            logger.log(SUCCESS, f"   └── other/        ({self.stats.get('other_count', 0)} files)")
        
        logger.log(SUCCESS, f"\n💾 Total Size: {self.stats['total_size']:.2f} MB")
        logger.log(HEADER, f"\n{ColoredFormatter.bold}📝 Reports: clone_report.html | clone_stats.txt{ColoredFormatter.reset}")
        logger.log(HEADER, f"{ColoredFormatter.bold}🌐 Open: pages/index.html in your browser{ColoredFormatter.reset}\n")


def show_banner():
    """Display a beautiful ASCII art banner"""
    banner = f"""
{ColoredFormatter.bold}{ColoredFormatter.cyan}╔══════════════════════════════════════════════════════════╗
║                                                          ║
║   ███████╗███████╗███████╗      ██████╗██╗      ██████╗ ███╗   ██╗███████╗
║   ╚══███╔╝██╔════╝██╔════╝     ██╔════╝██║     ██╔═══██╗████╗  ██║██╔════╝
║     ███╔╝ █████╗  █████╗       ██║     ██║     ██║   ██║██╔██╗ ██║█████╗  
║    ███╔╝  ██╔══╝  ██╔══╝       ██║     ██║     ██║   ██║██║╚██╗██║██╔══╝  
║   ███████╗███████╗███████╗     ╚██████╗███████╗╚██████╔╝██║ ╚████║███████╗
║   ╚══════╝╚══════╝╚══════╝      ╚═════╝╚══════╝ ╚═════╝ ╚═╝  ╚═══╝╚══════╝
║                                                          ║
║              Website Cloning Tool - Wget Style           ║
║              Version 1.0.7 | Recursive Edition           ║
║         Features: -r (recursive) -np (no-parent)         ║
║                  -k (convert links)                      ║
╚══════════════════════════════════════════════════════════╝{ColoredFormatter.reset}
    """
    print(banner)


def show_usage():
    """Display usage instructions"""
    usage = f"""
{ColoredFormatter.bold}{ColoredFormatter.yellow}📋 USAGE (Wget Style):{ColoredFormatter.reset}
    {ColoredFormatter.green}zee-clone <URL>{ColoredFormatter.reset}                          - Clone website (recursive, depth 5)
    {ColoredFormatter.green}zee-clone <URL> -r -l 3{ColoredFormatter.reset}                  - Recursive with depth 3
    {ColoredFormatter.green}zee-clone <URL> --no-parent{ColoredFormatter.reset}              - Don't go to parent directories
    {ColoredFormatter.green}zee-clone <URL> -k{ColoredFormatter.reset}                       - Convert links for local viewing
    {ColoredFormatter.green}zee-clone <URL> --all{ColoredFormatter.reset}                    - Aggressive mode (download everything)

{ColoredFormatter.bold}{ColoredFormatter.yellow}🎯 EXAMPLES (like wget):{ColoredFormatter.reset}
    {ColoredFormatter.cyan}$ zee-clone https://example.com{ColoredFormatter.reset}                    # wget -r -np -k https://example.com
    {ColoredFormatter.cyan}$ zee-clone https://example.com -l 2{ColoredFormatter.reset}               # wget -r -l 2 https://example.com
    {ColoredFormatter.cyan}$ zee-clone https://example.com --no-parent{ColoredFormatter.reset}        # wget -np https://example.com
    {ColoredFormatter.cyan}$ zee-clone https://example.com -o ./my_site{ColoredFormatter.reset}       # Custom output directory

{ColoredFormatter.bold}{ColoredFormatter.yellow}⚙️  OPTIONS:{ColoredFormatter.reset}
    {ColoredFormatter.green}-r, --recursive{ColoredFormatter.reset}          Enable recursive downloading (default: enabled)
    {ColoredFormatter.green}-l, --level <N>{ColoredFormatter.reset}          Maximum recursion depth (default: 5)
    {ColoredFormatter.green}--no-parent{ColoredFormatter.reset}              Don't ascend to parent directory (default: enabled)
    {ColoredFormatter.green}-k, --convert-links{ColoredFormatter.reset}      Convert links for local viewing (default: enabled)
    {ColoredFormatter.green}--all{ColoredFormatter.reset}                    Aggressive mode - download all resources
    {ColoredFormatter.green}-o, --output <dir>{ColoredFormatter.reset}       Output directory
    {ColoredFormatter.green}-v, --verbose{ColoredFormatter.reset}            Enable verbose output

{ColoredFormatter.bold}{ColoredFormatter.yellow}📁 ORGANIZED STRUCTURE:{ColoredFormatter.reset}
    {ColoredFormatter.cyan}cloned_site/{ColoredFormatter.reset}
    {ColoredFormatter.cyan}├── pages/        (all HTML pages, preserving structure){ColoredFormatter.reset}
    {ColoredFormatter.cyan}├── css/          (all stylesheets){ColoredFormatter.reset}
    {ColoredFormatter.cyan}├── js/           (all JavaScript files){ColoredFormatter.reset}
    {ColoredFormatter.cyan}├── images/       (all images){ColoredFormatter.reset}
    {ColoredFormatter.cyan}├── fonts/        (all font files){ColoredFormatter.reset}
    {ColoredFormatter.cyan}├── videos/       (all video files){ColoredFormatter.reset}
    {ColoredFormatter.cyan}├── audio/        (all audio files){ColoredFormatter.reset}
    {ColoredFormatter.cyan}├── documents/    (PDFs, docs, etc.){ColoredFormatter.reset}
    {ColoredFormatter.cyan}└── other/        (other resources){ColoredFormatter.reset}
    """
    print(usage)


def main():
    """Main entry point for the CLI tool."""
    # Show banner
    show_banner()
    
    # Parse arguments
    parser = argparse.ArgumentParser(
        description='Clone websites recursively like wget -r -np -k',
        formatter_class=argparse.RawDescriptionHelpFormatter,
        add_help=False
    )
    
    parser.add_argument(
        'url',
        nargs='?',
        help='URL of the website to clone'
    )
    
    parser.add_argument(
        '-r', '--recursive',
        action='store_true',
        default=True,
        help='Enable recursive downloading (default: enabled)'
    )
    
    parser.add_argument(
        '-l', '--level',
        type=int,
        default=5,
        help='Maximum recursion depth (default: 5)'
    )
    
    parser.add_argument(
        '--no-parent',
        action='store_true',
        default=True,
        help="Don't ascend to parent directory (default: enabled)"
    )
    
    parser.add_argument(
        '-k', '--convert-links',
        action='store_true',
        default=True,
        help='Convert links for local viewing (default: enabled)'
    )
    
    parser.add_argument(
        '--all',
        action='store_true',
        help='Aggressive mode - download all resources'
    )
    
    parser.add_argument(
        '-o', '--output',
        help='Output directory (default: domain name with underscores)'
    )
    
    parser.add_argument(
        '-v', '--verbose',
        action='store_true',
        help='Enable verbose output'
    )
    
    parser.add_argument(
        '-h', '--help',
        action='store_true',
        help='Show this help message'
    )
    
    args = parser.parse_args()
    
    # Show help if requested
    if args.help:
        show_usage()
        sys.exit(0)
    
    # If no URL provided, show banner and usage
    if not args.url:
        show_usage()
        sys.exit(1)
    
    # Set verbose logging if requested
    if args.verbose:
        logger.setLevel(logging.DEBUG)
    
    # Determine output directory
    if args.output:
        output_dir = Path(args.output)
    else:
        parsed = urlparse(args.url)
        folder_name = parsed.netloc.replace('.', '_')
        output_dir = Path.cwd() / folder_name
    
    # Create and run cloner
    cloner = WebsiteCloner(
        base_url=args.url,
        output_dir=output_dir,
        recursive=args.recursive,
        no_parent=args.no_parent,
        convert_links=args.convert_links,
        level=args.level,
        aggressive=args.all
    )
    
    success = cloner.run()
    
    # Exit with appropriate code
    sys.exit(0 if success else 1)


if __name__ == '__main__':
    try:
        main()
    except KeyboardInterrupt:
        logger.error(f"\n{ColoredFormatter.red}⚠️  Operation cancelled by user{ColoredFormatter.reset}")
        sys.exit(1)
    except Exception as e:
        logger.error(f"\n{ColoredFormatter.red}❌ Fatal error: {e}{ColoredFormatter.reset}")
        sys.exit(1)
