"""
PyPI Comprehensive Search and Analysis Toolkit

A high-performance, asynchronous Python library for searching and analyzing PyPI packages
with multiple search strategies, intelligent caching, and comprehensive metadata extraction.

Features:
    - Multi-strategy search with intelligent fallbacks
    - Asynchronous operations with configurable concurrency
    - Advanced caching with TTL and memory management
    - Detailed package metadata and download statistics
    - Relevance scoring with configurable algorithms
    - Rate limiting and request optimization
    - Batch operations and package comparison
    - Export capabilities for search results
    - Health monitoring and metrics collection
"""

import asyncio
import aiohttp
from typing import List, Dict, Any, Optional, Tuple, Set, Union
import json
import time
from concurrent.futures import ThreadPoolExecutor
import re
from dataclasses import dataclass, field
from urllib.parse import quote, urljoin
import logging
from pathlib import Path
import hashlib
from enum import Enum
import zlib
from datetime import datetime, timedelta
import statistics
import backoff
from aiolimiter import AsyncLimiter

# Configure structured logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)


class SearchStrategy(Enum):
    """Available search strategies in order of preference."""
    PRIMARY_JSON_API = "primary_json_api"
    SECONDARY_WEB_SCRAPING = "secondary_web_scraping"
    FALLBACK_SIMPLE_INDEX = "fallback_simple_index"
    DIRECT_PACKAGE_LOOKUP = "direct_package_lookup"


class CacheStrategy(Enum):
    """Cache storage strategies."""
    MEMORY = "memory"
    DISK = "disk"
    HYBRID = "hybrid"


@dataclass
class SearchMetrics:
    """Metrics for search performance analysis."""
    query: str
    strategy_used: SearchStrategy
    response_time: float
    results_count: int
    cache_hit: bool
    timestamp: datetime = field(default_factory=datetime.now)


@dataclass
class PackageInfo:
    """
    Comprehensive package information with enhanced metadata.
    
    Attributes:
        name (str): Canonical package name
        version (str): Latest stable version
        summary (str): Brief package description
        description (str): Detailed package description
        author (str): Primary author name
        author_email (str): Author contact email
        maintainer (str): Current maintainer
        maintainer_email (str): Maintainer contact email
        home_page (str): Project homepage URL
        project_url (str): Main project URL
        license (str): Software license
        requires_python (str): Python version requirements
        download_url (str): Direct download URL
        project_urls (Dict[str, str]): Additional project URLs
        releases (Dict[str, List[Dict]]): Version release history
        classifiers (List[str]): Trove classifiers
        downloads (Dict[str, int]): Download statistics
        score (float): Relevance score (0.0-1.0)
        last_updated (str): Last update timestamp
        health_score (float): Package health metric (0.0-1.0)
        popularity_rank (int): Popularity ranking
        dependencies (List[str]): Required dependencies
        development_status (str): Development status from classifiers
        keywords (List[str]): Package keywords
        first_release (str): Initial release date
        recent_activity (Dict[str, Any]): Recent release activity
    """
    
    name: str
    version: str
    summary: str = ""
    description: str = ""
    author: str = ""
    author_email: str = ""
    maintainer: str = ""
    maintainer_email: str = ""
    home_page: str = ""
    project_url: str = ""
    license: str = ""
    requires_python: str = ""
    download_url: str = ""
    project_urls: Dict[str, str] = field(default_factory=dict)
    releases: Dict[str, List[Dict]] = field(default_factory=dict)
    classifiers: List[str] = field(default_factory=list)
    downloads: Dict[str, int] = field(default_factory=dict)
    score: float = 0.0
    last_updated: str = ""
    health_score: float = 0.0
    popularity_rank: int = 0
    dependencies: List[str] = field(default_factory=list)
    development_status: str = ""
    keywords: List[str] = field(default_factory=list)
    first_release: str = ""
    recent_activity: Dict[str, Any] = field(default_factory=dict)

    def __post_init__(self):
        """Initialize derived attributes and validate data."""
        self._calculate_health_score()
        self._extract_development_status()
        self._extract_keywords()

    def _calculate_health_score(self) -> None:
        """Calculate comprehensive package health score."""
        scores = []
        
        # Release activity (30%)
        if self.releases:
            recent_releases = list(self.releases.keys())[-5:]  # Last 5 releases
            if len(recent_releases) >= 2:
                time_intervals = []
                for i in range(1, len(recent_releases)):
                    # Simplified time difference calculation
                    time_intervals.append(1)  # Placeholder for actual time calculation
                
                if time_intervals:
                    consistency = 1.0 - (statistics.stdev(time_intervals) / 30 if len(time_intervals) > 1 else 0)
                    scores.append(consistency * 0.3)
        
        # Download volume (25%)
        total_downloads = self.downloads.get('total_downloads', 0)
        download_score = min(total_downloads / 1000000, 1.0)  # Normalize to 1M downloads
        scores.append(download_score * 0.25)
        
        # Metadata completeness (20%)
        metadata_items = [
            self.summary, self.description, self.author, 
            self.license, self.home_page, self.requires_python
        ]
        completeness = sum(1 for item in metadata_items if item) / len(metadata_items)
        scores.append(completeness * 0.2)
        
        # Python version compatibility (15%)
        py_compat = 1.0 if self.requires_python else 0.5
        scores.append(py_compat * 0.15)
        
        # Classifiers and categorization (10%)
        classifier_score = min(len(self.classifiers) / 10, 1.0)
        scores.append(classifier_score * 0.1)
        
        self.health_score = sum(scores)

    def _extract_development_status(self) -> None:
        """Extract development status from classifiers."""
        status_map = {
            'Development Status :: 1 - Planning': 'planning',
            'Development Status :: 2 - Pre-Alpha': 'pre-alpha',
            'Development Status :: 3 - Alpha': 'alpha',
            'Development Status :: 4 - Beta': 'beta',
            'Development Status :: 5 - Production/Stable': 'stable',
            'Development Status :: 6 - Mature': 'mature',
            'Development Status :: 7 - Inactive': 'inactive'
        }
        
        for classifier in self.classifiers:
            if classifier in status_map:
                self.development_status = status_map[classifier]
                break
        else:
            self.development_status = 'unknown'

    def _extract_keywords(self) -> None:
        """Extract keywords from summary and classifiers."""
        keyword_blacklist = {'for', 'and', 'the', 'with', 'using', 'from'}
        
        # Extract from summary
        if self.summary:
            words = re.findall(r'\b[a-z]{3,}\b', self.summary.lower())
            self.keywords.extend([w for w in words if w not in keyword_blacklist][:10])
        
        # Extract from classifiers
        for classifier in self.classifiers:
            if '::' in classifier:
                parts = [p.strip().lower() for p in classifier.split('::')]
                self.keywords.extend(parts[-1].split() if parts else [])


class PyPISearch:
    """
    High-performance PyPI search utility with advanced features.
    
    This class provides comprehensive, asynchronous search capabilities for Python
    packages on PyPI with multiple search strategies, intelligent caching, and
    detailed package analysis.
    
    Key Features:
        - Multi-strategy search with automatic fallback
        - Advanced caching with multiple storage backends
        - Configurable rate limiting and concurrency
        - Comprehensive package health scoring
        - Batch operations and result comparison
        - Export capabilities for search results
        - Performance metrics and monitoring
        - Memory-efficient operations
    
    Example:
        >>> async with PyPISearch() as searcher:
        >>>     # Single search
        >>>     results = await searcher.search("requests", max_results=10)
        >>>     
        >>>     # Batch search
        >>>     batch_results = await searcher.batch_search(["numpy", "pandas"])
        >>>     
        >>>     # Package comparison
        >>>     comparison = await searcher.compare_packages(["package1", "package2"])
    """
    
    # PyPI API endpoints
    PYPI_BASE_URL = "https://pypi.org"
    PYPI_JSON_API_URL = f"{PYPI_BASE_URL}/pypi"
    PYPI_SEARCH_URL = f"{PYPI_BASE_URL}/search"
    PYPI_SIMPLE_URL = f"{PYPI_BASE_URL}/simple"
    
    def __init__(
        self,
        cache_ttl: int = 300,
        max_concurrent: int = 20,
        timeout: int = 30,
        cache_strategy: CacheStrategy = CacheStrategy.MEMORY,
        cache_dir: Optional[str] = None,
        requests_per_second: int = 10,
        enable_health_scoring: bool = True,
        enable_metrics: bool = True
    ):
        """
        Initialize the enhanced PyPI search utility.
        
        Args:
            cache_ttl: Time-to-live for cache entries in seconds
            max_concurrent: Maximum concurrent requests
            timeout: Request timeout in seconds
            cache_strategy: Cache storage strategy
            cache_dir: Directory for disk cache (required for disk/hybrid)
            requests_per_second: Rate limit for requests
            enable_health_scoring: Enable package health scoring
            enable_metrics: Enable performance metrics collection
        """
        self.cache_ttl = cache_ttl
        self.max_concurrent = max_concurrent
        self.timeout = timeout
        self.cache_strategy = cache_strategy
        self.enable_health_scoring = enable_health_scoring
        self.enable_metrics = enable_metrics
        
        # Initialize cache based on strategy
        self._cache = {}
        self._cache_dir = Path(cache_dir) if cache_dir else Path(".pypi_cache")
        if cache_strategy in [CacheStrategy.DISK, CacheStrategy.HYBRID]:
            self._cache_dir.mkdir(exist_ok=True)
        
        # Rate limiting
        self.rate_limiter = AsyncLimiter(requests_per_second, 1)
        
        # Session management
        self._session = None
        self._session_lock = asyncio.Lock()
        
        # Metrics and statistics
        self.metrics: List[SearchMetrics] = []
        self._search_stats = {
            'total_searches': 0,
            'cache_hits': 0,
            'average_response_time': 0.0,
            'successful_searches': 0
        }
        
        # Performance optimization
        self._compression_enabled = True
        
        logger.info(f"PyPISearch initialized with {max_concurrent} max concurrent requests")

    async def __aenter__(self):
        """Async context manager entry."""
        await self._ensure_session()
        return self

    async def __aexit__(self, exc_type, exc_val, exc_tb):
        """Async context manager exit."""
        await self.close()

    async def _ensure_session(self) -> None:
        """Ensure aiohttp session is available with connection pooling."""
        async with self._session_lock:
            if self._session is None:
                timeout = aiohttp.ClientTimeout(total=self.timeout)
                connector = aiohttp.TCPConnector(
                    limit=self.max_concurrent,
                    limit_per_host=self.max_concurrent // 2,
                    keepalive_timeout=30
                )
                self._session = aiohttp.ClientSession(
                    timeout=timeout,
                    connector=connector,
                    headers={
                        'User-Agent': 'PyPISearch/2.0 (Enhanced Search Toolkit)',
                        'Accept': 'application/json,text/html',
                        'Accept-Encoding': 'gzip, deflate'
                    }
                )
                logger.debug("HTTP session initialized")

    async def close(self) -> None:
        """Close the aiohttp session and cleanup resources."""
        async with self._session_lock:
            if self._session:
                await self._session.close()
                self._session = None
                logger.debug("HTTP session closed")
        
        # Save metrics if enabled
        if self.enable_metrics and self.metrics:
            await self._save_metrics()

    def _get_cache_key(self, query: str, search_type: str) -> str:
        """Generate deterministic cache key with hash for consistency."""
        key_data = f"{search_type}:{query.lower().strip()}"
        return hashlib.md5(key_data.encode()).hexdigest()

    async def _get_cached_result(self, cache_key: str) -> Optional[Any]:
        """Retrieve cached result with strategy-specific implementation."""
        if self.cache_strategy == CacheStrategy.MEMORY:
            return self._get_memory_cached(cache_key)
        elif self.cache_strategy == CacheStrategy.DISK:
            return await self._get_disk_cached(cache_key)
        elif self.cache_strategy == CacheStrategy.HYBRID:
            memory_result = self._get_memory_cached(cache_key)
            if memory_result is not None:
                return memory_result
            return await self._get_disk_cached(cache_key)
        return None

    def _get_memory_cached(self, cache_key: str) -> Optional[Any]:
        """Retrieve from memory cache."""
        if cache_key in self._cache:
            timestamp, data = self._cache[cache_key]
            if (time.time() - timestamp) < self.cache_ttl:
                self._search_stats['cache_hits'] += 1
                return data
            else:
                # Expired entry
                del self._cache[cache_key]
        return None

    async def _get_disk_cached(self, cache_key: str) -> Optional[Any]:
        """Retrieve from disk cache with compression support."""
        cache_file = self._cache_dir / f"{cache_key}.cache"
        
        if cache_file.exists():
            try:
                # Check file age
                file_age = time.time() - cache_file.stat().st_mtime
                if file_age < self.cache_ttl:
                    data = cache_file.read_bytes()
                    
                    if self._compression_enabled:
                        try:
                            data = zlib.decompress(data)
                        except zlib.error:
                            pass  # Not compressed or corrupted
                    
                    result = json.loads(data.decode())
                    self._search_stats['cache_hits'] += 1
                    
                    # Populate memory cache in hybrid mode
                    if self.cache_strategy == CacheStrategy.HYBRID:
                        self._cache[cache_key] = (time.time(), result)
                    
                    return result
                else:
                    # Expired cache file
                    cache_file.unlink()
            except (IOError, json.JSONDecodeError) as e:
                logger.warning(f"Failed to read cache file {cache_file}: {e}")
                cache_file.unlink()  # Remove corrupted cache
        
        return None

    async def _set_cached_result(self, cache_key: str, data: Any) -> None:
        """Store result in cache with strategy-specific implementation."""
        timestamp = time.time()
        
        if self.cache_strategy in [CacheStrategy.MEMORY, CacheStrategy.HYBRID]:
            self._cache[cache_key] = (timestamp, data)
        
        if self.cache_strategy in [CacheStrategy.DISK, CacheStrategy.HYBRID]:
            await self._set_disk_cached(cache_key, data, timestamp)

    async def _set_disk_cached(self, cache_key: str, data: Any, timestamp: float) -> None:
        """Store result in disk cache with compression."""
        cache_file = self._cache_dir / f"{cache_key}.cache"
        
        try:
            serialized_data = json.dumps(data).encode()
            
            if self._compression_enabled:
                serialized_data = zlib.compress(serialized_data)
            
            cache_file.write_bytes(serialized_data)
        except (IOError, TypeError) as e:
            logger.warning(f"Failed to write cache file {cache_file}: {e}")

    @backoff.on_exception(
        backoff.expo,
        (aiohttp.ClientError, asyncio.TimeoutError),
        max_tries=3,
        max_time=30
    )
    async def _make_request(self, url: str, method: str = "GET") -> Tuple[int, Any]:
        """
        Make HTTP request with retry logic and rate limiting.
        
        Args:
            url: Target URL
            method: HTTP method
            
        Returns:
            Tuple of (status_code, response_data)
        """
        async with self.rate_limiter:
            await self._ensure_session()
            
            try:
                async with self._session.request(method, url) as response:
                    if response.status == 200:
                        content_type = response.headers.get('content-type', '')
                        
                        if 'application/json' in content_type:
                            data = await response.json()
                        else:
                            data = await response.text()
                        
                        return response.status, data
                    else:
                        return response.status, None
                        
            except asyncio.TimeoutError:
                logger.warning(f"Request timeout for {url}")
                return 408, None
            except aiohttp.ClientError as e:
                logger.warning(f"Request failed for {url}: {e}")
                return 500, None

    async def search(
        self,
        query: str,
        max_results: int = 20,
        include_details: bool = True,
        strategy_priority: List[SearchStrategy] = None
    ) -> List[PackageInfo]:
        """
        Perform comprehensive PyPI package search with multiple strategies.
        
        Implements a tiered search approach:
        1. Primary JSON API (fastest, most accurate)
        2. Web scraping search (comprehensive)
        3. Simple index scanning (fallback)
        4. Direct package lookup (last resort)
        
        Args:
            query: Search query string
            max_results: Maximum number of results
            include_details: Fetch detailed package information
            strategy_priority: Custom search strategy order
            
        Returns:
            List of PackageInfo objects sorted by relevance
            
        Raises:
            ValueError: For invalid queries
            aiohttp.ClientError: For network failures
            
        Example:
            >>> results = await searcher.search("web framework", max_results=10)
            >>> for package in results:
            >>>     print(f"{package.name} ({package.health_score:.2f})")
        """
        if not query or not query.strip():
            raise ValueError("Query cannot be empty or whitespace")

        query = query.strip()
        start_time = time.time()
        
        # Update statistics
        self._search_stats['total_searches'] += 1

        # Check cache first
        cache_key = self._get_cache_key(query, "search")
        cached_result = await self._get_cached_result(cache_key)
        
        if cached_result is not None:
            logger.info(f"Cache hit for query: {query}")
            results = cached_result[:max_results]
            
            # Record metrics
            if self.enable_metrics:
                self.metrics.append(SearchMetrics(
                    query=query,
                    strategy_used=SearchStrategy.PRIMARY_JSON_API,  # Cached, so strategy doesn't matter
                    response_time=time.time() - start_time,
                    results_count=len(results),
                    cache_hit=True
                ))
            
            return results

        logger.info(f"Performing search for: {query}")
        await self._ensure_session()

        # Default strategy priority
        if strategy_priority is None:
            strategy_priority = [
                SearchStrategy.PRIMARY_JSON_API,
                SearchStrategy.SECONDARY_WEB_SCRAPING,
                SearchStrategy.FALLBACK_SIMPLE_INDEX,
                SearchStrategy.DIRECT_PACKAGE_LOOKUP
            ]

        packages = []
        used_strategy = SearchStrategy.PRIMARY_JSON_API

        # Try strategies in order until results are found
        for strategy in strategy_priority:
            try:
                if strategy == SearchStrategy.PRIMARY_JSON_API:
                    packages = await self._search_primary(query, max_results)
                elif strategy == SearchStrategy.SECONDARY_WEB_SCRAPING:
                    packages = await self._search_secondary(query, max_results)
                elif strategy == SearchStrategy.FALLBACK_SIMPLE_INDEX:
                    packages = await self._search_fallback(query, max_results)
                elif strategy == SearchStrategy.DIRECT_PACKAGE_LOOKUP:
                    packages = await self._search_direct_lookup(query, max_results)
                
                if packages:
                    used_strategy = strategy
                    logger.debug(f"Strategy {strategy} succeeded with {len(packages)} results")
                    break
                    
            except Exception as e:
                logger.warning(f"Strategy {strategy} failed: {e}")
                continue

        # Enrich with detailed information if requested
        if include_details and packages:
            packages = await self._enrich_package_details(packages)

        # Calculate relevance scores
        for package in packages:
            package.score = self._calculate_relevance_score(query, package.name, package.summary)

        # Sort by relevance and health score
        packages.sort(key=lambda x: (x.score, x.health_score), reverse=True)
        final_results = packages[:max_results]

        # Cache the results
        await self._set_cached_result(cache_key, final_results)

        # Update success statistics
        self._search_stats['successful_searches'] += 1
        
        # Calculate response time for metrics
        response_time = time.time() - start_time
        self._search_stats['average_response_time'] = (
            self._search_stats['average_response_time'] * (self._search_stats['total_searches'] - 1) + response_time
        ) / self._search_stats['total_searches']

        # Record metrics
        if self.enable_metrics:
            self.metrics.append(SearchMetrics(
                query=query,
                strategy_used=used_strategy,
                response_time=response_time,
                results_count=len(final_results),
                cache_hit=False
            ))

        logger.info(f"Search completed: {len(final_results)} results in {response_time:.2f}s")
        return final_results

    async def _search_primary(self, query: str, max_results: int) -> List[PackageInfo]:
        """
        Primary search using PyPI's JSON API with enhanced parsing.
        
        Args:
            query: Search query
            max_results: Maximum results to return
            
        Returns:
            List of PackageInfo objects
        """
        try:
            encoded_query = quote(query)
            url = f"{self.PYPI_JSON_API_URL}/?%3Aaction=search&term={encoded_query}&submit=search"
            
            status, data = await self._make_request(url)
            
            if status == 200 and data:
                return self._parse_primary_results(data, query, max_results)
                
        except Exception as e:
            logger.warning(f"Primary search failed: {e}")
            
        return []

    def _parse_primary_results(self, html: str, query: str, max_results: int) -> List[PackageInfo]:
        """
        Parse enhanced HTML response from primary search.
        
        Args:
            html: HTML content
            query: Original search query
            max_results: Maximum results to return
            
        Returns:
            List of parsed PackageInfo objects
        """
        packages = []
        
        # Enhanced regex pattern for more robust parsing
        patterns = [
            # Primary pattern for package snippets
            r'<a class="package-snippet" href="(/project/[^"]+)">\s*<h3 class="package-snippet__title">\s*<span class="package-snippet__name">([^<]+)</span>\s*<span class="package-snippet__version">([^<]+)</span>\s*</h3>\s*<p class="package-snippet__description">([^<]*)</p>',
            # Fallback pattern for alternative HTML structure
            r'<a href="(/project/[^"]+)"[^>]*>\s*([^<]+)\s*</a>\s*<span[^>]*>([^<]+)</span>\s*<p[^>]*>([^<]*)</p>'
        ]
        
        for pattern in patterns:
            matches = re.findall(pattern, html, re.DOTALL | re.IGNORECASE)
            for match in matches[:max_results]:
                if len(match) >= 4:
                    project_url, name, version, summary = match[:4]
                    
                    # Clean and validate data
                    name = self._clean_text(name)
                    version = self._clean_text(version)
                    summary = self._clean_text(summary)
                    
                    if name:  # Only add valid packages
                        score = self._calculate_relevance_score(query, name, summary)
                        
                        package = PackageInfo(
                            name=name,
                            version=version,
                            summary=summary,
                            score=score
                        )
                        packages.append(package)
            
            if packages:  # Use first pattern that yields results
                break
        
        return packages[:max_results]

    async def _search_secondary(self, query: str, max_results: int) -> List[PackageInfo]:
        """
        Secondary search using PyPI search API with web scraping.
        
        Args:
            query: Search query
            max_results: Maximum results to return
            
        Returns:
            List of PackageInfo objects
        """
        try:
            encoded_query = quote(query)
            url = f"{self.PYPI_SEARCH_URL}/?q={encoded_query}"
            
            status, data = await self._make_request(url)
            
            if status == 200 and data:
                return self._parse_secondary_results(data, query, max_results)
                
        except Exception as e:
            logger.warning(f"Secondary search failed: {e}")
            
        return []

    def _parse_secondary_results(self, html: str, query: str, max_results: int) -> List[PackageInfo]:
        """
        Parse search results from secondary search.
        
        Args:
            html: HTML content
            query: Original search query
            max_results: Maximum results to return
            
        Returns:
            List of parsed PackageInfo objects
        """
        packages = []
        
        # Pattern for search result items
        pattern = r'<a class="package-snippet"[^>]*href="[^"]*/([^"/]+)/?"[^>]*>.*?<span[^>]*>([^<]+)</span>.*?<span[^>]*>([^<]+)</span>.*?<p[^>]*>([^<]*)</p>'
        
        matches = re.findall(pattern, html, re.DOTALL)
        
        for match in matches[:max_results]:
            name, version, summary = match[0], match[2], match[3]
            
            name = self._clean_text(name)
            version = self._clean_text(version)
            summary = self._clean_text(summary)
            
            if name:
                score = self._calculate_relevance_score(query, name, summary)
                
                package = PackageInfo(
                    name=name,
                    version=version,
                    summary=summary,
                    score=score
                )
                packages.append(package)
        
        return packages

    async def _search_fallback(self, query: str, max_results: int) -> List[PackageInfo]:
        """
        Fallback search using PyPI simple index.
        
        Args:
            query: Search query
            max_results: Maximum results to return
            
        Returns:
            List of PackageInfo objects
        """
        try:
            status, data = await self._make_request(self.PYPI_SIMPLE_URL)
            
            if status == 200 and data:
                return self._parse_fallback_results(data, query, max_results)
                
        except Exception as e:
            logger.warning(f"Fallback search failed: {e}")
            
        return []

    def _parse_fallback_results(self, html: str, query: str, max_results: int) -> List[PackageInfo]:
        """
        Parse simple index for package names matching query.
        
        Args:
            html: HTML content
            query: Original search query
            max_results: Maximum results to return
            
        Returns:
            List of PackageInfo objects
        """
        packages = []
        query_lower = query.lower()
        
        # Extract package names from simple index
        pattern = r'<a[^>]*>([^<]+)</a>'
        matches = re.findall(pattern, html)
        
        for match in matches:
            name = self._clean_text(match)
            if query_lower in name.lower():
                score = self._calculate_relevance_score(query, name, "")
                
                package = PackageInfo(
                    name=name,
                    version="unknown",
                    summary="",
                    score=score
                )
                packages.append(package)
                
                if len(packages) >= max_results:
                    break
        
        return packages

    async def _search_direct_lookup(self, query: str, max_results: int) -> List[PackageInfo]:
        """
        Direct package lookup as final fallback strategy.
        
        Args:
            query: Search query (treated as package name)
            max_results: Maximum results to return
            
        Returns:
            List of PackageInfo objects
        """
        try:
            package = await self.get_package_details(query)
            if package:
                package.score = 1.0  # Exact match gets highest score
                return [package]
                
        except Exception as e:
            logger.warning(f"Direct lookup failed for {query}: {e}")
            
        return []

    async def get_package_details(self, package_name: str) -> Optional[PackageInfo]:
        """
        Get comprehensive details for a specific package.
        
        Args:
            package_name: Exact package name
            
        Returns:
            PackageInfo object or None if not found
            
        Example:
            >>> package = await searcher.get_package_details("requests")
            >>> print(f"Health score: {package.health_score:.2f}")
            >>> print(f"Dependencies: {package.dependencies}")
        """
        cache_key = self._get_cache_key(package_name, "details")
        cached_result = await self._get_cached_result(cache_key)
        
        if cached_result is not None:
            return cached_result

        try:
            url = f"{self.PYPI_JSON_API_URL}/{package_name}/json"
            status, data = await self._make_request(url)
            
            if status == 200 and data:
                package = self._parse_package_details(data)
                
                # Cache the result
                await self._set_cached_result(cache_key, package)
                
                return package
                
        except Exception as e:
            logger.error(f"Failed to get details for {package_name}: {e}")
            
        return None

    def _parse_package_details(self, data: Dict[str, Any]) -> PackageInfo:
        """
        Parse comprehensive package information from JSON API response.
        
        Args:
            data: JSON API response data
            
        Returns:
            Parsed PackageInfo object
        """
        info = data["info"]
        releases = data.get("releases", {})
        
        # Calculate enhanced download statistics
        downloads = self._calculate_download_stats(releases)
        
        # Extract dependencies
        dependencies = self._extract_dependencies(info)
        
        # Calculate recent activity
        recent_activity = self._calculate_recent_activity(releases)
        
        # Find first release
        first_release = self._find_first_release(releases)
        
        package = PackageInfo(
            name=info.get("name", ""),
            version=info.get("version", ""),
            summary=info.get("summary", ""),
            description=info.get("description", ""),
            author=info.get("author", ""),
            author_email=info.get("author_email", ""),
            maintainer=info.get("maintainer", ""),
            maintainer_email=info.get("maintainer_email", ""),
            home_page=info.get("home_page", ""),
            project_url=info.get("project_url", ""),
            license=info.get("license", ""),
            requires_python=info.get("requires_python", ""),
            download_url=info.get("download_url", ""),
            project_urls=info.get("project_urls", {}),
            releases=releases,
            classifiers=info.get("classifiers", []),
            downloads=downloads,
            dependencies=dependencies,
            first_release=first_release,
            recent_activity=recent_activity,
            last_updated=self._extract_last_updated(releases)
        )
        
        return package

    def _calculate_download_stats(self, releases: Dict[str, List[Dict]]) -> Dict[str, int]:
        """
        Calculate comprehensive download statistics.
        
        Args:
            releases: Releases information
            
        Returns:
            Dictionary with download statistics
        """
        stats = {
            "total_downloads": 0,
            "release_count": len(releases),
            "file_count": 0,
            "recent_downloads": 0,
            "average_downloads_per_release": 0
        }
        
        recent_cutoff = datetime.now() - timedelta(days=30)
        recent_downloads = 0
        release_downloads = []
        
        for version, files in releases.items():
            version_downloads = 0
            stats["file_count"] += len(files)
            
            for file_info in files:
                file_downloads = file_info.get("downloads", 0)
                stats["total_downloads"] += file_downloads
                version_downloads += file_downloads
                
                # Check if this is a recent upload
                upload_time_str = file_info.get("upload_time", "")
                if upload_time_str:
                    try:
                        upload_time = datetime.fromisoformat(upload_time_str.replace('Z', '+00:00'))
                        if upload_time > recent_cutoff:
                            recent_downloads += file_downloads
                    except (ValueError, TypeError):
                        pass
            
            release_downloads.append(version_downloads)
        
        stats["recent_downloads"] = recent_downloads
        
        if release_downloads:
            stats["average_downloads_per_release"] = sum(release_downloads) / len(release_downloads)
        
        return stats

    def _extract_dependencies(self, info: Dict[str, Any]) -> List[str]:
        """Extract package dependencies from info data."""
        dependencies = []
        
        # Check requires_dist field
        requires_dist = info.get("requires_dist", [])
        if requires_dist:
            for requirement in requires_dist:
                # Extract package name from requirement spec
                match = re.match(r'^([a-zA-Z0-9_-]+)', requirement)
                if match:
                    dependencies.append(match.group(1))
        
        return dependencies

    def _calculate_recent_activity(self, releases: Dict[str, List[Dict]]) -> Dict[str, Any]:
        """Calculate recent release activity metrics."""
        recent_cutoff = datetime.now() - timedelta(days=90)  # 3 months
        recent_releases = 0
        latest_upload = None
        
        for files in releases.values():
            for file_info in files:
                upload_time_str = file_info.get("upload_time")
                if upload_time_str:
                    try:
                        upload_time = datetime.fromisoformat(upload_time_str.replace('Z', '+00:00'))
                        if upload_time > recent_cutoff:
                            recent_releases += 1
                        
                        if latest_upload is None or upload_time > latest_upload:
                            latest_upload = upload_time
                    except (ValueError, TypeError):
                        continue
        
        return {
            "recent_releases": recent_releases,
            "latest_upload": latest_upload.isoformat() if latest_upload else None,
            "days_since_last_release": (datetime.now() - latest_upload).days if latest_upload else None
        }

    def _find_first_release(self, releases: Dict[str, List[Dict]]) -> str:
        """Find the date of the first release."""
        first_upload = None
        
        for files in releases.values():
            for file_info in files:
                upload_time_str = file_info.get("upload_time")
                if upload_time_str:
                    try:
                        upload_time = datetime.fromisoformat(upload_time_str.replace('Z', '+00:00'))
                        if first_upload is None or upload_time < first_upload:
                            first_upload = upload_time
                    except (ValueError, TypeError):
                        continue
        
        return first_upload.isoformat() if first_upload else ""

    def _extract_last_updated(self, releases: Dict[str, List[Dict]]) -> str:
        """Extract the last update timestamp from releases."""
        latest_upload = None
        
        for files in releases.values():
            for file_info in files:
                upload_time_str = file_info.get("upload_time")
                if upload_time_str:
                    try:
                        upload_time = datetime.fromisoformat(upload_time_str.replace('Z', '+00:00'))
                        if latest_upload is None or upload_time > latest_upload:
                            latest_upload = upload_time
                    except (ValueError, TypeError):
                        continue
        
        return latest_upload.isoformat() if latest_upload else ""

    async def _enrich_package_details(self, packages: List[PackageInfo]) -> List[PackageInfo]:
        """
        Enrich package list with detailed information using concurrent requests.
        
        Args:
            packages: List of basic package information
            
        Returns:
            List of enriched package information
        """
        if not packages:
            return packages

        semaphore = asyncio.Semaphore(min(self.max_concurrent, len(packages)))
        
        async def get_details(package: PackageInfo) -> PackageInfo:
            async with semaphore:
                try:
                    detailed_package = await self.get_package_details(package.name)
                    if detailed_package:
                        # Preserve original score and merge additional data
                        detailed_package.score = package.score
                        return detailed_package
                except Exception as e:
                    logger.warning(f"Failed to enrich {package.name}: {e}")
                return package

        # Process in batches for better performance
        batch_size = min(10, len(packages))
        enriched_packages = []
        
        for i in range(0, len(packages), batch_size):
            batch = packages[i:i + batch_size]
            tasks = [get_details(package) for package in batch]
            batch_results = await asyncio.gather(*tasks, return_exceptions=True)
            
            for result in batch_results:
                if isinstance(result, PackageInfo):
                    enriched_packages.append(result)
                elif isinstance(result, Exception):
                    logger.warning(f"Enrichment task failed: {result}")
        
        return enriched_packages

    def _calculate_relevance_score(self, query: str, name: str, summary: str) -> float:
        """
        Calculate advanced relevance score for search results.
        
        Uses multiple factors:
        - Exact matches
        - Partial matches
        - Word position
        - Term frequency
        - Field weighting
        
        Args:
            query: Search query
            name: Package name
            summary: Package summary
            
        Returns:
            Relevance score between 0.0 and 1.0
        """
        query_lower = query.lower() if query else None
        name_lower = name.lower() if name else None
        summary_lower = summary.lower() if summary else None
        
        score = 0.0
        max_score = 0.0
        
        # Field weights
        name_weight = 0.7
        summary_weight = 0.3
        
        # Exact matches (highest priority)
        if query_lower == name_lower:
            score += 0.8 * name_weight
            max_score += 0.8 * name_weight
        
        # Name-based scoring
        name_score = self._calculate_field_score(query_lower, name_lower)
        score += name_score * name_weight
        max_score += 1.0 * name_weight
        
        # Summary-based scoring
        if summary_lower:
            summary_score = self._calculate_field_score(query_lower, summary_lower)
            score += summary_score * summary_weight
            max_score += 1.0 * summary_weight
        
        # Normalize score
        if max_score > 0:
            final_score = score / max_score
        else:
            final_score = 0.0
        
        return min(final_score, 1.0)

    def _calculate_field_score(self, query: str, field: str) -> float:
        """Calculate relevance score for a specific field."""
        if not field:
            return 0.0
        
        score = 0.0
        
        # Exact match
        if query == field:
            return 1.0
        
        # Query contains field or field contains query
        if query in field or field in query:
            score += 0.6
        
        # Field starts with query
        if field.startswith(query):
            score += 0.4
        
        # Word-based matching
        query_words = set(query.split())
        field_words = set(field.split())
        
        if query_words:
            # Exact word matches
            exact_matches = len(query_words.intersection(field_words))
            score += (exact_matches / len(query_words)) * 0.3
            
            # Partial word matches
            partial_matches = 0
            for q_word in query_words:
                for f_word in field_words:
                    if q_word in f_word or f_word in q_word:
                        partial_matches += 1
                        break
            
            score += (partial_matches / len(query_words)) * 0.2
        
        return min(score, 1.0)

    def _clean_text(self, text: str) -> str:
        """Clean and normalize text data."""
        if not text:
            return ""
        
        # Remove extra whitespace
        text = re.sub(r'\s+', ' ', text)
        
        # Strip leading/trailing whitespace
        text = text.strip()
        
        # Remove control characters
        text = re.sub(r'[\x00-\x1f\x7f-\x9f]', '', text)
        
        return text

    async def batch_search(
        self,
        queries: List[str],
        max_results: int = 10,
        include_details: bool = True
    ) -> Dict[str, List[PackageInfo]]:
        """
        Perform multiple searches concurrently with optimized resource usage.
        
        Args:
            queries: List of search queries
            max_results: Maximum results per query
            include_details: Whether to include detailed package information
            
        Returns:
            Dictionary mapping queries to search results
            
        Example:
            >>> results = await searcher.batch_search(["requests", "numpy", "pandas"])
            >>> for query, packages in results.items():
            >>>     print(f"{query}: {len(packages)} packages")
        """
        if not queries:
            return {}
        
        # Limit concurrent searches to prevent resource exhaustion
        semaphore = asyncio.Semaphore(min(self.max_concurrent, len(queries)))
        
        async def search_with_limit(query: str) -> Tuple[str, List[PackageInfo]]:
            async with semaphore:
                try:
                    results = await self.search(query, max_results, include_details)
                    return query, results
                except Exception as e:
                    logger.error(f"Batch search failed for {query}: {e}")
                    return query, []
        
        tasks = [search_with_limit(query) for query in queries]
        results = await asyncio.gather(*tasks)
        
        return dict(results)

    async def compare_packages(self, package_names: List[str]) -> Dict[str, Any]:
        """
        Compare multiple packages across various metrics.
        
        Args:
            package_names: List of package names to compare
            
        Returns:
            Dictionary with comparison data
            
        Example:
            >>> comparison = await searcher.compare_packages(["requests", "httpx"])
            >>> print(f"Health scores: {comparison['health_scores']}")
        """
        packages = {}
        
        # Fetch package details concurrently
        tasks = [self.get_package_details(name) for name in package_names]
        results = await asyncio.gather(*tasks, return_exceptions=True)
        
        for name, result in zip(package_names, results):
            if isinstance(result, PackageInfo):
                packages[name] = result
            else:
                logger.warning(f"Failed to get details for {name}: {result}")
        
        # Generate comparison data
        comparison = {
            "health_scores": {name: pkg.health_score for name, pkg in packages.items()},
            "downloads": {name: pkg.downloads.get('total_downloads', 0) for name, pkg in packages.items()},
            "latest_versions": {name: pkg.version for name, pkg in packages.items()},
            "dependencies_count": {name: len(pkg.dependencies) for name, pkg in packages.items()},
            "development_status": {name: pkg.development_status for name, pkg in packages.items()},
            "recent_activity": {name: pkg.recent_activity.get('recent_releases', 0) for name, pkg in packages.items()}
        }
        
        return comparison

    async def get_popular_packages(self, category: str = "", limit: int = 20) -> List[PackageInfo]:
        """
        Get popular packages, optionally filtered by category.
        
        Args:
            category: Trove classifier category
            limit: Number of packages to return
            
        Returns:
            List of popular packages
        """
        # This is a simplified implementation
        # In a real scenario, you might use PyPI's popularity data or external sources
        popular_queries = ["requests", "numpy", "pandas", "matplotlib", "django", "flask"]
        
        all_packages = []
        for query in popular_queries:
            packages = await self.search(query, max_results=5, include_details=True)
            all_packages.extend(packages)
        
        # Remove duplicates and sort by downloads
        unique_packages = {}
        for package in all_packages:
            if package.name not in unique_packages:
                unique_packages[package.name] = package
        
        sorted_packages = sorted(
            unique_packages.values(),
            key=lambda x: x.downloads.get('total_downloads', 0),
            reverse=True
        )
        
        return sorted_packages[:limit]

    def get_search_statistics(self) -> Dict[str, Any]:
        """
        Get search performance and usage statistics.
        
        Returns:
            Dictionary with search statistics
        """
        return {
            'total_searches': self._search_stats['total_searches'],
            'successful_searches': self._search_stats['successful_searches'],
            'cache_hits': self._search_stats['cache_hits'],
            'cache_hit_rate': (
                self._search_stats['cache_hits'] / self._search_stats['total_searches']
                if self._search_stats['total_searches'] > 0 else 0
            ),
            'average_response_time': self._search_stats['average_response_time'],
            'current_cache_size': len(self._cache)
        }

    async def export_results(self, packages: List[PackageInfo], format: str = "json") -> str:
        """
        Export search results to various formats.
        
        Args:
            packages: List of packages to export
            format: Export format (json, csv, text)
            
        Returns:
            Exported data as string
        """
        if format == "json":
            return json.dumps([self._package_to_dict(pkg) for pkg in packages], indent=2)
        
        elif format == "csv":
            import csv
            import io
            
            output = io.StringIO()
            writer = csv.writer(output)
            
            # Write header
            writer.writerow([
                'Name', 'Version', 'Score', 'Health Score', 'Downloads',
                'Author', 'License', 'Python Version', 'Description'
            ])
            
            # Write data
            for pkg in packages:
                writer.writerow([
                    pkg.name,
                    pkg.version,
                    f"{pkg.score:.3f}",
                    f"{pkg.health_score:.3f}",
                    pkg.downloads.get('total_downloads', 0),
                    pkg.author,
                    pkg.license,
                    pkg.requires_python,
                    pkg.summary[:100]  # Truncate long descriptions
                ])
            
            return output.getvalue()
        
        elif format == "text":
            lines = []
            for pkg in packages:
                lines.extend([
                    f"Package: {pkg.name}",
                    f"Version: {pkg.version}",
                    f"Score: {pkg.score:.3f}",
                    f"Health: {pkg.health_score:.3f}",
                    f"Downloads: {pkg.downloads.get('total_downloads', 0):,}",
                    f"Author: {pkg.author}",
                    f"License: {pkg.license}",
                    f"Python: {pkg.requires_python}",
                    f"Summary: {pkg.summary}",
                    "-" * 50
                ])
            return "\n".join(lines)
        
        else:
            raise ValueError(f"Unsupported format: {format}")

    def _package_to_dict(self, package: PackageInfo) -> Dict[str, Any]:
        """Convert PackageInfo to dictionary for serialization."""
        return {
            'name': package.name,
            'version': package.version,
            'summary': package.summary,
            'description': package.description,
            'author': package.author,
            'author_email': package.author_email,
            'maintainer': package.maintainer,
            'maintainer_email': package.maintainer_email,
            'home_page': package.home_page,
            'project_url': package.project_url,
            'license': package.license,
            'requires_python': package.requires_python,
            'download_url': package.download_url,
            'project_urls': package.project_urls,
            'classifiers': package.classifiers,
            'downloads': package.downloads,
            'score': package.score,
            'health_score': package.health_score,
            'popularity_rank': package.popularity_rank,
            'dependencies': package.dependencies,
            'development_status': package.development_status,
            'keywords': package.keywords,
            'first_release': package.first_release,
            'recent_activity': package.recent_activity,
            'last_updated': package.last_updated
        }

    async def _save_metrics(self) -> None:
        """Save search metrics to disk for analysis."""
        if not self.metrics:
            return
        
        metrics_file = self._cache_dir / "search_metrics.json"
        try:
            metrics_data = [
                {
                    'query': m.query,
                    'strategy_used': m.strategy_used.value,
                    'response_time': m.response_time,
                    'results_count': m.results_count,
                    'cache_hit': m.cache_hit,
                    'timestamp': m.timestamp.isoformat()
                }
                for m in self.metrics
            ]
            
            metrics_file.write_text(json.dumps(metrics_data, indent=2))
            logger.info(f"Saved {len(self.metrics)} metrics records")
            
        except IOError as e:
            logger.warning(f"Failed to save metrics: {e}")

    def clear_cache(self, older_than: Optional[float] = None) -> None:
        """
        Clear cache with optional age-based filtering.
        
        Args:
            older_than: Clear entries older than this timestamp (seconds)
        """
        # Clear memory cache
        if older_than is None:
            self._cache.clear()
        else:
            current_time = time.time()
            expired_keys = [
                key for key, (timestamp, _) in self._cache.items()
                if current_time - timestamp > older_than
            ]
            for key in expired_keys:
                del self._cache[key]
        
        # Clear disk cache
        if self.cache_strategy in [CacheStrategy.DISK, CacheStrategy.HYBRID]:
            cache_files = list(self._cache_dir.glob("*.cache"))
            for cache_file in cache_files:
                try:
                    if older_than is None:
                        cache_file.unlink()
                    else:
                        file_age = time.time() - cache_file.stat().st_mtime
                        if file_age > older_than:
                            cache_file.unlink()
                except OSError as e:
                    logger.warning(f"Failed to delete cache file {cache_file}: {e}")
        
        logger.info("Cache cleared" + (f" (entries older than {older_than}s)" if older_than else ""))


# Convenience functions for simplified usage

async def search_package(
    query: str,
    max_results: int = 20,
    include_details: bool = True,
    timeout: int = 30,
    **kwargs
) -> List[PackageInfo]:
    """
    High-level convenience function for PyPI package search.
    
    This function provides a simple interface for the enhanced PyPI search
    capabilities with sensible defaults and automatic resource management.
    
    Args:
        query: Search query string
        max_results: Maximum number of results
        include_details: Whether to fetch detailed package information
        timeout: Request timeout in seconds
        **kwargs: Additional arguments for PyPISearch
        
    Returns:
        List of PackageInfo objects sorted by relevance
        
    Example:
        >>> import asyncio
        >>>
        >>> async def main():
        >>>     results = await search_package("web framework", max_results=5)
        >>>     for package in results:
        >>>         print(f"{package.name} v{package.version}")
        >>>         print(f"   Health: {package.health_score:.2f}")
        >>>         print(f"   Downloads: {package.downloads.get('total_downloads', 0):,}")
        >>>
        >>> asyncio.run(main())
    """
    async with PyPISearch(timeout=timeout, **kwargs) as searcher:
        return await searcher.search(query, max_results, include_details)


def search_sync(
    query: str,
    max_results: int = 20,
    include_details: bool = True,
    timeout: int = 30,
    **kwargs
) -> List[PackageInfo]:
    """
    Synchronous wrapper for PyPI package search.
    
    Note: This function creates a new event loop and should not be used
    in async contexts. For async code, use search_package instead.
    
    Args:
        query: Search query string
        max_results: Maximum number of results
        include_details: Whether to fetch detailed package information
        timeout: Request timeout in seconds
        **kwargs: Additional arguments for PyPISearch
        
    Returns:
        List of PackageInfo objects sorted by relevance
    """
    return asyncio.run(search_package(query, max_results, include_details, timeout, **kwargs))


# Enhanced demonstration and testing
async def demo():
    """Demonstrate the enhanced PyPI search capabilities."""
    print("Enhanced PyPI Search Toolkit Demo")
    print("=" * 60)
    
    async with PyPISearch(
        max_concurrent=10,
        cache_ttl=600,
        enable_health_scoring=True,
        enable_metrics=True
    ) as searcher:
        
        # Single search with details
        print("1. Single Package Search:")
        print("-" * 30)
        
        query = "http client"
        results = await searcher.search(query, max_results=3, include_details=True)
        
        for i, package in enumerate(results, 1):
            print(f"{i}. {package.name} v{package.version}")
            print(f"   Score: {package.score:.3f}, Health: {package.health_score:.3f}")
            print(f"   Downloads: {package.downloads.get('total_downloads', 0):,}")
            print(f"   Dependencies: {len(package.dependencies)}")
            print(f"   Status: {package.development_status}")
            print(f"   Summary: {package.summary[:100]}...")
            print()
        
        # Batch search
        print("\n2. Batch Search:")
        print("-" * 30)
        
        queries = ["requests", "httpx", "aiohttp"]
        batch_results = await searcher.batch_search(queries, max_results=2)
        
        for query, packages in batch_results.items():
            print(f"'{query}': {len(packages)} results")
            for pkg in packages:
                print(f"   - {pkg.name} (v{pkg.version})")
            print()
        
        # Package comparison
        print("\n3. Package Comparison:")
        print("-" * 30)
        
        comparison = await searcher.compare_packages(["requests", "httpx"])
        print("Health Scores:", comparison["health_scores"])
        print("Downloads:", comparison["downloads"])
        print("Recent Activity:", comparison["recent_activity"])
        print()
        
        # Statistics
        print("\n4. Search Statistics:")
        print("-" * 30)
        
        stats = searcher.get_search_statistics()
        for key, value in stats.items():
            print(f"{key}: {value}")
        
        print("\n5. Export Results:")
        print("-" * 30)
        
        if results:
            json_export = await searcher.export_results(results[:2], "json")
            print(f"JSON export (first 2 packages, {len(json_export)} chars)")
            
            text_export = await searcher.export_results(results[:1], "text")
            print("Text export (first package):")
            print(text_export)


if __name__ == "__main__":
    # Run the enhanced demo
    asyncio.run(demo())