"""Python Library Packager - Download and pack Python dependencies with caching support.

This module provides functionality to:
1. Read project information from projects.json or run pyprojectparse if needed
2. Download dependencies to local .cache directory
3. Pack dependencies into a distributable format
4. Support batch processing multiple projects recursively
"""

from __future__ import annotations

import argparse
import atexit
import json
import logging
import re
import shutil
import subprocess
import tarfile
import tempfile
import time
import zipfile
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import dataclass, field
from functools import cached_property
from pathlib import Path
from typing import Any, ClassVar, Final, Pattern

from sfi.pyprojectparse.models.solution import Solution
from sfi.pyprojectparse.models.project import Project
from sfi.pyprojectparse.models.dependency import Dependency

logging.basicConfig(
    level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger(__name__)

__version__ = "1.0.0"
__build__ = "20260120"

DEFAULT_CACHE_DIR = Path.home() / ".pysfi" / ".cache" / "python-libs"

MAX_DEPTH: Final[int] = 50  # Maximum recursion depth to prevent infinite loops

# Archive format constants
SUPPORTED_ARCHIVE_FORMATS: Final[tuple[str, ...]] = (
    "zip",
    "tar",
    "gztar",
    "bztar",
    "xztar",
)

# Default configuration constants
DEFAULT_MAX_WORKERS: Final[int] = 4
DEFAULT_MIRROR: Final[str] = "aliyun"
DEFAULT_OPTIMIZE: Final[bool] = True

PYPI_MIRRORS: Final[dict[str, str]] = {
    "pypi": "https://pypi.org/simple",
    "tsinghua": "https://pypi.tuna.tsinghua.edu.cn/simple",
    "aliyun": "https://mirrors.aliyun.com/pypi/simple/",
    "ustc": "https://pypi.mirrors.ustc.edu.cn/simple/",
    "douban": "https://pypi.douban.com/simple/",
    "tencent": "https://mirrors.cloud.tencent.com/pypi/simple",
}


CONFIG_FILE = Path.home() / ".pysfi" / "pylibpack.json"


@dataclass
class PyLibPackerConfig:
    """Configuration for PyLibPack with persistent settings."""

    cache_dir: Path | None = None
    mirror: str = DEFAULT_MIRROR
    optimize: bool = DEFAULT_OPTIMIZE
    max_workers: int = DEFAULT_MAX_WORKERS

    def __init__(
        self,
        cache_dir: Path | None = None,
        mirror: str = DEFAULT_MIRROR,
        optimize: bool = DEFAULT_OPTIMIZE,
        max_workers: int = DEFAULT_MAX_WORKERS,
    ):
        # Track which parameters were explicitly provided
        self._explicitly_set = {}

        if cache_dir is not None:
            self._explicitly_set["cache_dir"] = True
        if mirror != DEFAULT_MIRROR:
            self._explicitly_set["mirror"] = True
        if optimize != DEFAULT_OPTIMIZE:
            self._explicitly_set["optimize"] = True
        if max_workers != DEFAULT_MAX_WORKERS:
            self._explicitly_set["max_workers"] = True

        # Set the values
        self.cache_dir = cache_dir
        self.mirror = mirror
        self.optimize = optimize
        self.max_workers = max_workers

        # Apply defaults for unset values
        if self.cache_dir is None:
            self.cache_dir = DEFAULT_CACHE_DIR

        self.cache_dir.mkdir(parents=True, exist_ok=True)

        # Load configuration from file if it exists
        if CONFIG_FILE.exists():
            try:
                config_data = json.loads(CONFIG_FILE.read_text(encoding="utf-8"))
                # Update configuration items, but only for those not explicitly set
                for key, value in config_data.items():
                    if (
                        hasattr(self, key)
                        and isinstance(value, type(getattr(self, key)))
                        and key not in self._explicitly_set
                    ):
                        setattr(self, key, value)
            except (json.JSONDecodeError, TypeError, AttributeError) as e:
                logger.warning(f"Could not load config from {CONFIG_FILE}: {e}")

    def save(self) -> None:
        """Save current configuration to file."""
        CONFIG_FILE.parent.mkdir(parents=True, exist_ok=True)
        config_dict = {
            "cache_dir": str(self.cache_dir),
            "mirror": self.mirror,
            "optimize": self.optimize,
            "max_workers": self.max_workers,
        }
        CONFIG_FILE.write_text(json.dumps(config_dict, indent=4), encoding="utf-8")


@dataclass
class DownloadResult:
    """Result of downloading packages."""

    results: dict[str, bool] = field(default_factory=dict)
    total: int = 0
    successful: int = 0
    cached: int = 0
    downloaded: int = 0


@dataclass
class PackResult:
    """Result of packing project dependencies."""

    success: bool
    project: str
    total: int
    successful: int
    failed: int
    packages_dir: str
    extracted_packages: list[str] = field(default_factory=list)


@dataclass
class CacheMetadata:
    """Metadata for cached package."""

    name: str
    version: str | None
    path: str
    timestamp: float


DEV_TOOLS = frozenset({
    "sphinx",
    "sphinx_rtd_theme",
    "watchdog",
    "pytest",
    "coverage",
    "black",
    "mypy",
    "flake8",
    "pylint",
    "isort",
    "pre-commit",
    "tox",
    "nose",
    "unittest",
    "mock",
})
DEV_PATTERNS = frozenset({"dev", "test", "docs", "lint", "example"})
TYPING_PATTERNS = frozenset({"stubs", "typing", "types"})


@dataclass
class OptimizationRule:
    """Defines an optimization rule for a specific library.

    Attributes:
        library_name: The name of the library to apply the rule to.
        exclude_patterns: A list of patterns to exclude from the library.
        include_patterns: A list of patterns to include in the library.

    """

    library_name: str = field(default_factory=str)
    exclude_patterns: list[str] = field(default_factory=list)
    include_patterns: list[str] = field(default_factory=list)

    def __post_init__(self) -> None:
        """Compile regex patterns after initialization."""
        self.exclude_compiled: list[Pattern] = [
            re.compile(p) for p in self.exclude_patterns
        ]
        self.include_compiled: list[Pattern] = [
            re.compile(p) for p in self.include_patterns
        ]


@dataclass(frozen=False)
class SelectiveExtractionStrategy:
    """Optimization strategy that applies inclusion/exclusion rules to specific libraries.

    This strategy works as follows:
    1. First, apply universal exclusion rules (doc, test, example, demo, etc.)
    2. Then, apply library-specific exclusion rules
    3. Finally, apply inclusion rules (only files matching include patterns are kept)
    """

    # Universal exclusion patterns - applied to all libraries
    UNIVERSAL_EXCLUDE_PATTERNS: ClassVar[frozenset[str]] = frozenset({
        "doc",
        "docs",
        "test",
        "tests",
        "example",
        "examples",
        "demo",
        "demos",
        "sample",
        "samples",
        "benchmark",
        "benchmarks",
        "tutorial",
        "tutorials",
        "notebook",
        "notebooks",
        "license",
        "licenses",
    })

    def __init__(
        self,
        rules: list[OptimizationRule] | None = None,
        apply_universal_rules: bool = True,
    ):
        """Initialize the strategy with optimization rules.

        Args:
            rules: List of optimization rules to apply
            apply_universal_rules: Whether to apply universal exclusion rules (default: True)
        """
        self.rules: dict[str, OptimizationRule] = {}
        self.apply_universal_rules = apply_universal_rules

        if rules:
            for rule in rules:
                self.rules[rule.library_name.lower()] = rule

        # Default rules for common libraries
        if not rules:
            self._setup_default_rules()

        # Compile universal exclusion patterns for faster matching
        self._universal_exclude_compiled = [
            re.compile(f"(^|/)({pattern})(/|$)", re.IGNORECASE)
            for pattern in self.UNIVERSAL_EXCLUDE_PATTERNS
        ]

    def _setup_default_rules(self) -> None:
        """Setup default optimization rules for common libraries.

        This method loads JSON rule files from the rules directory and
        creates OptimizationRule objects for common libraries.
        """
        # Get the rules directory
        rules_dir = Path(__file__).parent / "rules"

        if not rules_dir.exists() or not rules_dir.is_dir():
            logger.warning(f"Rules directory not found: {rules_dir}")
            return

        # Load all JSON rule files
        for rule_file in rules_dir.glob("*.json"):
            try:
                with open(rule_file, encoding="utf-8") as f:
                    rule_data = json.load(f)

                # Convert JSON data to OptimizationRule
                rule = OptimizationRule(
                    library_name=rule_data["library_name"],
                    exclude_patterns=rule_data["exclude_patterns"],
                    include_patterns=rule_data["include_patterns"],
                )

                self.rules[rule.library_name.lower()] = rule
                logger.debug(
                    f"Loaded optimization rule for {rule.library_name} from {rule_file.name}"
                )

            except Exception as e:
                logger.warning(f"Failed to load rule from {rule_file.name}: {e}")

    def _matches_universal_exclude_pattern(self, relative_path: str) -> bool:
        """Check if file path matches any universal exclusion pattern.

        Args:
            relative_path: Relative path to the file

        Returns:
            True if path should be excluded, False otherwise
        """
        return any(
            pattern.search(relative_path)
            for pattern in self._universal_exclude_compiled
        )

    def should_extract_file(self, library_name: str, file_path: Path) -> bool:
        """Determine if a file should be extracted based on library-specific rules.

        Args:
            library_name: Name of the library
            file_path: Path to the file to check

        Returns:
            True if the file should be extracted, False otherwise
        """
        lib_name_lower = library_name.lower()
        relative_path = file_path.as_posix().lower()

        # First, apply universal exclusion rules (applied to all libraries)
        if self.apply_universal_rules and self._matches_universal_exclude_pattern(
            relative_path
        ):
            logger.debug(
                f"Excluding {file_path} from {library_name} (matches universal exclusion pattern)"
            )
            return False

        # If no specific rule exists for this library, extract everything
        if lib_name_lower not in self.rules:
            logger.debug(f"No specific rules for {library_name}, including {file_path}")
            return True

        rule = self.rules[lib_name_lower]

        logger.debug(
            f"Checking {file_path} for {library_name} with {len(rule.exclude_compiled)} exclude and {len(rule.include_compiled)} include patterns"
        )

        # Then, apply library-specific exclusion rules - if file matches any exclude pattern, skip it
        for exclude_pattern in rule.exclude_compiled:
            if exclude_pattern.search(relative_path):
                logger.debug(
                    f"Excluding {file_path} from {library_name} (matches exclude pattern: {exclude_pattern.pattern})"
                )
                return False

        # If inclusion patterns are defined, only include files that match at least one
        if rule.include_compiled:
            for include_pattern in rule.include_compiled:
                if include_pattern.search(relative_path):
                    logger.debug(
                        f"Including {file_path} from {library_name} (matches include pattern: {include_pattern.pattern})"
                    )
                    return True
            # If we have inclusion rules but the file doesn't match any, exclude it
            logger.debug(
                f"Excluding {file_path} from {library_name} (doesn't match any include patterns)"
            )
            return False

        # If no inclusion rules are defined, include the file (after exclusion check)
        logger.debug(
            f"Including {file_path} from {library_name} (passed exclusion filters)"
        )
        return True

    def get_library_names_with_rules(self) -> set[str]:
        """Get the names of libraries that have optimization rules defined.

        Returns:
            Set of library names with optimization rules
        """
        return set(self.rules.keys())


def normalize_package_name(name: str) -> str:
    """Normalize package name to lowercase with underscores.

    Args:
        name: Package name to normalize

    Returns:
        Normalized package name
    """
    return name.lower().replace("-", "_")


def should_skip_dependency(req_name: str, has_extras: bool = False) -> bool:
    """Check if a dependency should be skipped based on common patterns.

    Args:
        req_name: Package name
        has_extras: Whether the requirement has extras

    Returns:
        True if should skip, False otherwise
    """
    # Skip extras
    if has_extras:
        return True

    req_lower = req_name.lower()
    normalized_req = req_lower.replace("-", "_")

    # Skip dev/test/docs/lint/example patterns
    if any(keyword in req_lower for keyword in DEV_PATTERNS):
        return True

    # Skip typing/stubs dependencies
    if any(keyword in req_lower for keyword in TYPING_PATTERNS):
        return True

    # Skip common dev tools
    return normalized_req in DEV_TOOLS


@dataclass(frozen=False)
class LibraryCache:
    """Manage local cache for Python packages."""

    cache_dir: Path = field(default_factory=lambda: DEFAULT_CACHE_DIR)
    _dependencies_cache: dict[Path, set[str]] = field(default_factory=dict)

    def __post_init__(self) -> None:
        self.cache_dir.mkdir(parents=True, exist_ok=True)

    @cached_property
    def metadata_file(self) -> Path:
        return self.cache_dir / "metadata.json"

    @cached_property
    def wheel_files(self) -> list[Path]:
        return list(self.cache_dir.glob("*.whl"))

    @cached_property
    def sdist_files(self) -> list[Path]:
        return list(self.cache_dir.glob("*.tar.gz")) + list(
            self.cache_dir.glob("*.zip")
        )

    def collect_dependencies_from_list(self, dependency_list: list[str]) -> set[str]:
        """Recursively collect all dependencies from package files (wheel or sdist).

        Args:
            dependency_list: List of root package names to start from

        Returns:
            Set of all required package names (normalized)
        """
        all_packages: set[str] = set()
        visited: set[str] = set()
        visit_stack: dict[str, int] = {}  # Track visit depth for cycle detection

        def visit(pkg_name: str, level: int = 0) -> None:
            """Visit a package and collect its dependencies."""
            # Normalize package name for consistency
            normalized_pkg_name = pkg_name.lower().replace("-", "_")

            # Check for cycles
            if normalized_pkg_name in visit_stack:
                logger.warning(
                    f"Potential circular dependency detected: {normalized_pkg_name} (current depth: {level}, "
                    f"previous depth: {visit_stack[normalized_pkg_name]})"
                )
                return

            # Check depth limit
            if level > MAX_DEPTH:
                logger.warning(
                    f"Maximum dependency depth ({MAX_DEPTH}) reached for {normalized_pkg_name}, stopping recursion"
                )
                return

            # Skip if already visited
            if normalized_pkg_name in visited:
                return

            # Mark as visited and track depth
            visited.add(normalized_pkg_name)
            visit_stack[normalized_pkg_name] = level
            all_packages.add(normalized_pkg_name)

            # Process dependencies if package exists in map
            package_path = self.package_map.get(normalized_pkg_name)
            if package_path:
                deps = self._extract_dependencies_from_wheel(package_path)
                logger.debug(f"{'  ' * level}{normalized_pkg_name} -> {deps}")
                for dep in deps:
                    visit(dep, level + 1)

            # Remove from stack when done
            visit_stack.pop(normalized_pkg_name, None)

        for pkg_name in dependency_list:
            visit(pkg_name)

        logger.info(f"Collected {len(all_packages)} packages: {all_packages}")
        return all_packages

    @property
    def package_map(self) -> dict[str, Path]:
        """Create a mapping of package names to their file paths with improved efficiency."""
        packages: dict[str, Path] = {}

        # Process wheel files first (they take precedence)
        for wheel_file in self.wheel_files:
            pkg_name = self._extract_package_name_from_wheel(wheel_file)
            if pkg_name:
                normalized_pkg_name = normalize_package_name(pkg_name)
                packages[normalized_pkg_name] = wheel_file

        # Add sdist files only if the package isn't already in the map
        for sdist_file in self.sdist_files:
            pkg_name = self._extract_package_name_from_sdist(sdist_file)
            if pkg_name:
                normalized_pkg_name = normalize_package_name(pkg_name)
                if normalized_pkg_name not in packages:
                    packages[normalized_pkg_name] = sdist_file

        return packages

    @cached_property
    def cache_size(self) -> int:
        """Calculate total size of cache in bytes."""
        if not self.cache_dir.exists():
            return 0

        # Use generator expression for memory efficiency
        return sum(
            file_path.stat().st_size
            for file_path in self.cache_dir.rglob("*")
            if file_path.is_file()
        )

    @cached_property
    def package_count(self) -> int:
        """Get the count of packages in cache."""
        return len(self.package_map)

    @cached_property
    def cache_stats(self) -> dict[str, int]:
        """Get detailed cache statistics."""
        return {
            "total_packages": self.package_count,
            "wheel_count": len(self.wheel_files),
            "sdist_count": len(self.sdist_files),
            "cache_size_bytes": self.cache_size,
        }

    def get_package_path(
        self, package_name: str, version: str | None = None
    ) -> Path | None:
        """Get cached package path if available.

        Args:
            package_name: Name of the package
            version: Version (optional)

        Returns:
            Path to cached package or None
        """
        normalized_name = normalize_package_name(package_name)

        # First try filesystem lookup for wheel files (works even if metadata is missing)
        for whl_file in self.cache_dir.glob("*.whl"):
            parsed_name = self._extract_package_name_from_wheel(whl_file)
            if parsed_name == normalized_name:
                logger.debug(f"Cache hit (filesystem wheel): {package_name}")
                return whl_file

        # Try filesystem lookup for sdist files (.tar.gz, .zip)
        for sdist_file in self.cache_dir.glob("*.tar.gz"):
            parsed_name = self._extract_package_name_from_sdist(sdist_file)
            if parsed_name == normalized_name:
                logger.debug(f"Cache hit (filesystem sdist): {package_name}")
                return sdist_file
        for sdist_file in self.cache_dir.glob("*.zip"):
            parsed_name = self._extract_package_name_from_sdist(sdist_file)
            if parsed_name == normalized_name:
                logger.debug(f"Cache hit (filesystem sdist): {package_name}")
                return sdist_file

        # Fallback to metadata lookup
        metadata = self._load_metadata()
        for info in metadata.values():
            if info["name"] == normalized_name and (
                version is None or info.get("version") == version
            ):
                path = self.cache_dir / info["path"]
                if path.exists():
                    logger.debug(f"Cache hit (metadata): {package_name}")
                    return path

        logger.debug(f"Cache miss: {package_name}")
        return None

    @staticmethod
    def _extract_package_name_from_wheel(wheel_file: Path) -> str | None:
        """Extract package name from wheel file.

        Args:
            wheel_file: Path to wheel file

        Returns:
            Package name or None
        """
        try:
            filename = wheel_file.stem  # Remove .whl extension
            parts = filename.split("-")
            if parts:
                return normalize_package_name(parts[0])
        except Exception:
            pass
        return None

    @staticmethod
    def _extract_package_name_from_sdist(sdist_file: Path) -> str | None:
        """Extract package name from source distribution file (.tar.gz or .zip).

        Args:
            sdist_file: Path to sdist file

        Returns:
            Package name or None
        """
        try:
            # Handle .tar.gz files (e.g., package_name-1.0.0.tar.gz)
            if (
                sdist_file.suffixes
                and ".tar" in sdist_file.suffixes
                and ".gz" in sdist_file.suffixes
            ):
                # Remove both .tar.gz extensions by removing the last 7 characters (.tar.gz)
                stem_without_ext = (
                    sdist_file.stem
                )  # This removes .gz, leaving package-1.0.0.tar
                # Now remove the remaining .tar
                if stem_without_ext.endswith(".tar"):
                    stem_without_ext = stem_without_ext[:-4]  # Remove .tar
                parts = stem_without_ext.rsplit(
                    "-", 1
                )  # Split from right: ["package_name", "1.0.0"]
                if len(parts) >= 1 and parts[0]:
                    return normalize_package_name(parts[0])
            # Handle .zip files
            elif sdist_file.suffix == ".zip":
                filename = sdist_file.stem  # Remove .zip extension
                parts = filename.rsplit("-", 1)
                if len(parts) >= 1 and parts[0]:
                    return normalize_package_name(parts[0])
        except Exception as e:
            logger.debug(f"Failed to extract package name from {sdist_file}: {e}")
        return None

    def _extract_dependencies_from_wheel(self, wheel_file: Path) -> set[str]:
        """Extract dependencies from wheel METADATA file with caching.

        Args:
            wheel_file: Path to wheel or sdist file

        Returns:
            Set of package names (normalized)
        """
        # Check cache first
        if wheel_file in self._dependencies_cache:
            return self._dependencies_cache[wheel_file]

        # Check if it's an sdist file (.tar.gz or .zip)
        if wheel_file.suffix in (".gz", ".zip"):
            dependencies = self._extract_dependencies_from_sdist(wheel_file)
            self._dependencies_cache[wheel_file] = dependencies
            return dependencies

        # Early return if wheel file doesn't exist
        if not wheel_file.exists():
            logger.warning(f"Wheel file does not exist: {wheel_file}")
            self._dependencies_cache[wheel_file] = set()
            return set()

        try:
            with zipfile.ZipFile(wheel_file, "r") as zf:
                # Find metadata file
                metadata_files = [
                    name for name in zf.namelist() if name.endswith("METADATA")
                ]

                if not metadata_files:
                    dependencies = set()
                else:
                    metadata_content = zf.read(metadata_files[0]).decode(
                        "utf-8", errors="ignore"
                    )
                    dependencies = self._parse_metadata_content(metadata_content)

            # Cache the result
            self._dependencies_cache[wheel_file] = dependencies
            return dependencies
        except Exception as e:
            logger.warning(
                f"Failed to extract dependencies from {wheel_file.name}: {e}"
            )
            return set()

    def _extract_dependencies_from_sdist(self, sdist_file: Path) -> set[str]:
        """Extract dependencies from source distribution file with caching.

        Args:
            sdist_file: Path to sdist file (.tar.gz or .zip)

        Returns:
            Set of package names (normalized)
        """

        dependencies: set[str] = set()

        try:
            # Handle .tar.gz files
            if sdist_file.suffix == ".gz":
                with tarfile.open(sdist_file, "r:gz") as tf:
                    for member in tf.getmembers():
                        # Look for PKG-INFO or METADATA file in the root of the package
                        if member.name.endswith(("PKG-INFO", "METADATA")):
                            # Only use PKG-INFO/METADATA files in the root directory
                            # Count the number of slashes in the path
                            path_parts = member.name.split("/")
                            if len(path_parts) == 2 or (
                                len(path_parts) == 3
                                and path_parts[2] in ("PKG-INFO", "METADATA")
                            ):
                                content = tf.extractfile(member)
                                if content:
                                    metadata_content = content.read().decode(
                                        "utf-8", errors="ignore"
                                    )
                                    dependencies = self._parse_metadata_content(
                                        metadata_content
                                    )
                                    logger.debug(
                                        f"Extracted dependencies from {member.name} in {sdist_file.name}"
                                    )
                                    break
            # Handle .zip files
            elif sdist_file.suffix == ".zip":
                with zipfile.ZipFile(sdist_file, "r") as zf:
                    for name in zf.namelist():
                        # Look for PKG-INFO or METADATA file in the root of the package
                        if name.endswith(("PKG-INFO", "METADATA")):
                            path_parts = name.split("/")
                            if len(path_parts) == 2 or (
                                len(path_parts) == 3
                                and path_parts[2] in ("PKG-INFO", "METADATA")
                            ):
                                metadata_content = zf.read(name).decode(
                                    "utf-8", errors="ignore"
                                )
                                dependencies = self._parse_metadata_content(
                                    metadata_content
                                )
                                logger.debug(
                                    f"Extracted dependencies from {name} in {sdist_file.name}"
                                )
                                break
        except Exception as e:
            logger.warning(
                f"Failed to extract dependencies from sdist {sdist_file.name}: {e}"
            )

        return dependencies

    @staticmethod
    def _parse_metadata_content(metadata_content: str) -> set[str]:
        """Parse metadata content (PKG-INFO or METADATA) to extract dependencies.

        Args:
            metadata_content: Content of PKG-INFO or METADATA file

        Returns:
            Set of package names (normalized)
        """
        dependencies: set[str] = set()
        try:
            for line in metadata_content.splitlines():
                # Look for Requires-Dist or Requires field
                if line.startswith("Requires-Dist:") or line.startswith("Requires:"):
                    if line.startswith("Requires:"):
                        # Requires field contains comma-separated list
                        dep_str = line.split(":", 1)[1].strip()
                        for req_str in re.split(r",\s*", dep_str):
                            req_str = req_str.strip()
                            if req_str:
                                dependencies.update(
                                    LibraryCache._parse_single_requirement(req_str)
                                )
                    else:
                        # Requires-Dist field
                        dep_str = line.split(":", 1)[1].strip()
                        dependencies.update(
                            LibraryCache._parse_single_requirement(dep_str)
                        )
        except Exception as e:
            logger.debug(f"Failed to parse metadata content: {e}")

        return dependencies

    @staticmethod
    def _parse_single_requirement(req_str: str) -> set[str]:
        """Parse a single requirement string and extract package name.

        Args:
            req_str: Requirement string (e.g., "numpy>=1.20.0", "package[extra]>=1.0")

        Returns:
            Set containing the normalized package name, or empty set if should skip
        """
        try:
            # Skip extras dependencies
            if re.search(
                r'extra\s*==\s*["\']?([^"\';\s]+)["\']?', req_str, re.IGNORECASE
            ):
                logger.debug(f"Skipping extra dependency: {req_str}")
                return set()

            from packaging.requirements import Requirement

            req = Requirement(req_str)
            if not should_skip_dependency(req.name, bool(req.extras)):
                dep_name = normalize_package_name(req.name)
                logger.debug(f"Found core dependency: {dep_name}")
                return {dep_name}
        except Exception:
            pass

        return set()

    def add_package(
        self, package_name: str, package_path: Path, version: str | None = None
    ) -> None:
        """Add package to cache.

        Args:
            package_name: Name of the package
            package_path: Path to package files
            version: Package version
        """
        # Normalize package name to ensure consistency
        normalized_name = normalize_package_name(package_name)

        # Copy package files to cache (flat structure for wheels, nested for dirs)
        if package_path.is_dir():
            dest_dir = self.cache_dir / normalized_name
            if dest_dir.exists():
                shutil.rmtree(dest_dir)
            shutil.copytree(package_path, dest_dir)
            relative_path = normalized_name
        else:
            dest_file = self.cache_dir / package_path.name
            shutil.copy2(package_path, dest_file)
            relative_path = package_path.name

        # Update metadata using CacheMetadata dataclass
        metadata = self._load_metadata()
        metadata[str(package_path)] = CacheMetadata(
            name=normalized_name,
            version=version,
            path=relative_path,
            timestamp=time.time(),
        ).__dict__
        self._save_metadata(metadata)

        logger.info(f"Cached package: {normalized_name}")

    def _load_metadata(self) -> dict[str, Any]:
        """Load cache metadata.

        Returns:
            Metadata dictionary
        """
        if self.metadata_file.exists():
            try:
                with open(self.metadata_file, encoding="utf-8") as f:
                    return json.load(f)
            except Exception as e:
                logger.warning(f"Failed to load cache metadata: {e}")

        return {}

    def _save_metadata(self, metadata: dict[str, Any]) -> None:
        """Save cache metadata.

        Args:
            metadata: Metadata dictionary
        """
        with open(self.metadata_file, "w", encoding="utf-8") as f:
            json.dump(metadata, f, indent=2)

    @staticmethod
    def _should_skip_dist_info(file_path: Path) -> bool:
        """Check if the file path should be skipped because it's a dist-info directory."""
        if file_path.name.endswith(".dist-info"):
            return True
        # Check if any parent directory ends with .dist-info
        return any(part.endswith(".dist-info") for part in file_path.parts)

    def clear_cache(self) -> None:
        """Clear all cached packages."""
        if self.cache_dir.exists():
            shutil.rmtree(self.cache_dir)
            self.cache_dir.mkdir(parents=True, exist_ok=True)
        self._dependencies_cache.clear()  # Clear in-memory dependencies cache
        logger.info("Cache cleared")


@dataclass(frozen=True)
class LibraryDownloader:
    """Download Python packages from PyPI."""

    parent: PyLibPacker
    cache: LibraryCache
    _mirror: str = "pypi"

    @cached_property
    def mirror_url(self) -> str:
        return PYPI_MIRRORS.get(self._mirror, PYPI_MIRRORS["pypi"])

    @cached_property
    def pip_executable(self) -> str | None:
        return self._find_pip_executable()

    @staticmethod
    def _find_pip_executable() -> str | None:
        """Find pip executable in the system."""
        return shutil.which("pip") or shutil.which("pip3")

    def _download_package(self, dep: Dependency, dest_dir: Path) -> Path | None:
        """Download a single package without dependencies.

        Args:
            dep: Dependency to download
            dest_dir: Destination directory (typically cache_dir)

        Returns:
            Path to downloaded package file (wheel or sdist) or None
        """
        if not self.pip_executable:
            logger.error(
                "pip not found. Please install pip: python -m ensurepip --upgrade"
            )
            return None

        logger.info(f"Downloading: {dep}")

        with tempfile.TemporaryDirectory() as temp_dir:
            result = subprocess.run(
                [
                    self.pip_executable,
                    "download",
                    "--no-deps",
                    "--index-url",
                    self.mirror_url,
                    "--dest",
                    temp_dir,
                    str(dep),
                ],
                capture_output=True,
                text=True,
                check=False,
            )

            if result.returncode != 0:
                logger.warning(f"pip download failed for {dep}: {result.stderr}")
                return None

            # Prefer wheel files over sdist files
            downloaded_file = None
            for file_path in Path(temp_dir).glob("*.whl"):
                downloaded_file = file_path
                break

            # If no wheel file, look for sdist files (.tar.gz or .zip)
            if not downloaded_file:
                for file_path in Path(temp_dir).glob("*.tar.gz"):
                    downloaded_file = file_path
                    break
                for file_path in Path(temp_dir).glob("*.zip"):
                    downloaded_file = file_path
                    break

            if downloaded_file:
                self.cache.add_package(dep.name, downloaded_file, dep.version)
                shutil.copy2(downloaded_file, dest_dir / downloaded_file.name)
                logger.info(f"Downloaded: {downloaded_file.name}")
                return dest_dir / downloaded_file.name

        return None

    def download_packages(self, project: Project) -> DownloadResult:
        """Download multiple packages concurrently.

        Args:
            project: Project containing dependencies to download

        Returns:
            DownloadResult containing download statistics
        """
        results_list: list[tuple[str, bool]] = []
        cached_count = 0
        cached_packages: set[str] = set()  # Track cached package names efficiently

        dependencies = project.converted_dependencies
        logger.info(f"Total direct dependencies: {len(dependencies)}")
        logger.info(f"Using mirror: {self.mirror_url}")

        # Check cache and mark cached packages (single-threaded, safe)
        for dep in dependencies:
            if self.cache.get_package_path(dep.name, dep.version):
                results_list.append((dep.name, True))
                cached_packages.add(dep.name)
                cached_count += 1
                logger.info(f"Using cached package: {dep}")

        # Download remaining packages concurrently
        remaining_deps = [
            dep for dep in dependencies if dep.name not in cached_packages
        ]
        downloaded_count = 0

        if remaining_deps:
            with ThreadPoolExecutor(
                max_workers=self.parent.config.max_workers
            ) as executor:
                future_to_dep = {
                    executor.submit(
                        self._download_package, dep, self.cache.cache_dir
                    ): dep
                    for dep in remaining_deps
                }

                for future in as_completed(future_to_dep):
                    dep = future_to_dep[future]
                    try:
                        wheel_file = future.result()
                        results_list.append((dep.name, wheel_file is not None))
                        if wheel_file:
                            downloaded_count += 1
                    except Exception as e:
                        logger.error(f"Error processing {dep.name}: {e}")
                        results_list.append((dep.name, False))

        # Convert to dictionary for final result
        results = dict(results_list)
        successful = sum(1 for v in results.values() if v)
        logger.info(
            f"Processed {successful}/{len(dependencies)} ({cached_count} cached, {downloaded_count} downloaded)"
        )

        return DownloadResult(
            results=results,
            total=len(dependencies),
            successful=successful,
            cached=cached_count,
            downloaded=downloaded_count,
        )


@dataclass(frozen=True)
class PyLibPacker:
    """Main library packer class."""

    working_dir: Path
    config: PyLibPackerConfig

    @cached_property
    def cache(self) -> LibraryCache:
        return LibraryCache(cache_dir=self.config.cache_dir or DEFAULT_CACHE_DIR)

    @cached_property
    def downloader(self) -> LibraryDownloader:
        return LibraryDownloader(
            parent=self,
            cache=self.cache,
            _mirror=self.config.mirror,
        )

    @cached_property
    def optimization_strategy(self) -> SelectiveExtractionStrategy | None:
        return SelectiveExtractionStrategy() if self.config.optimize else None

    @cached_property
    def solution(self) -> Solution:
        return Solution.from_directory(root_dir=self.working_dir)

    @cached_property
    def projects(self) -> dict[str, Project]:
        # Return projects as a dictionary mapping project names to Project objects
        # This follows the Solution API correctly
        return {project.name: project for project in self.solution.projects.values()}

    @cached_property
    def project_count(self) -> int:
        """Get the count of projects to avoid repeated computation."""
        return len(self.projects)

    @cached_property
    def working_dir_size(self) -> int:
        """Calculate total size of the working directory in bytes."""
        if not self.working_dir.exists():
            return 0

        # Use generator expression for memory efficiency
        return sum(
            file_path.stat().st_size
            for file_path in self.working_dir.rglob("*")
            if file_path.is_file()
        )

    def pack_project(self, project: Project) -> PackResult:
        """Pack dependencies for a single project.

        Args:
            project: Project information
            output_dir: Output directory
            max_workers: Maximum concurrent downloads

        Returns:
            PackResult containing packing statistics

        Raises:
            ValueError: If project has invalid configuration
            RuntimeError: If packing fails due to system issues
        """
        logger.info(f"{120 * '='}")
        logger.info(f"Packing dependencies for project: `{project.name}`")

        download_result = self.downloader.download_packages(project)

        # Recursively collect all dependencies (pass cache instance for dependency extraction)
        all_packages = self.cache.collect_dependencies_from_list(
            list(download_result.results)
        )

        # Extract all required packages (keep order of dependency resolution)
        extracted_packages = []
        for pkg_name in all_packages:
            logger.info(f"Processing {pkg_name}")
            if pkg_name in self.cache.package_map:
                # Skip if output directory already exists
                output_pkg_dir = project.lib_dir / pkg_name
                if output_pkg_dir.exists():
                    logger.warning(f"Output directory already exists: {output_pkg_dir}")
                    continue

                package_file = self.cache.package_map[pkg_name]
                logger.info(f"Extracting {package_file.name}...")
                self._extract_package(package_file, project.lib_dir, pkg_name)
                extracted_packages.append(pkg_name)
                logger.info(f"Extracted {pkg_name}")
            else:
                logger.warning(f"Package not found in cache: {pkg_name}")
                # Attempt to download the missing package
                logger.info(f"Attempting to download missing package: {pkg_name}")

                # Create a temporary dependency object for the missing package
                missing_dep = Dependency(
                    name=pkg_name, version=None, extras=set(), requires=set()
                )

                # Try to download the missing package
                downloaded_file = self.downloader._download_package(
                    missing_dep, self.cache.cache_dir
                )
                if downloaded_file:
                    logger.info(f"Successfully downloaded missing package: {pkg_name}")
                    # Now check again if it's in the cache and extract if available
                    if pkg_name in self.cache.package_map:
                        package_file = self.cache.package_map[pkg_name]
                        logger.info(f"Extracting {package_file.name}...")
                        self._extract_package(package_file, project.lib_dir, pkg_name)
                        extracted_packages.append(pkg_name)
                        logger.info(f"Extracted {pkg_name}")
                    else:
                        logger.error(
                            f"Package {pkg_name} still not found in cache after download attempt"
                        )
                else:
                    logger.error(f"Failed to download missing package: {pkg_name}")

        # Calculate final statistics
        total_packages = len(project.dependencies)
        successful_packages = len(extracted_packages)
        failed_packages = total_packages - successful_packages

        logger.info(
            f"Pack complete for {project.name}: {successful_packages}/{total_packages}"
        )
        logger.info(f"{120 * '='}")

        return PackResult(
            success=successful_packages > 0,
            project=project.name,
            total=total_packages,
            successful=successful_packages,
            failed=failed_packages,
            packages_dir=str(project.lib_dir),
            extracted_packages=extracted_packages,
        )

    def _build_and_cache_wheel(self, sdist_file: Path, package_name: str) -> None:
        """Build wheel from sdist file and cache it for faster future access.

        Args:
            sdist_file: Path to sdist file (.tar.gz or .zip)
            package_name: Name of the package
        """
        with tempfile.TemporaryDirectory() as temp_wheel_dir:
            # Use pip wheel to build wheel from sdist
            result = subprocess.run(
                [
                    self.downloader.pip_executable or "pip",
                    "wheel",
                    "--no-deps",
                    "--wheel-dir",
                    temp_wheel_dir,
                    "--no-cache-dir",
                    str(sdist_file),
                ],
                capture_output=True,
                text=True,
                check=False,
            )

            if result.returncode != 0:
                error_msg = result.stderr.strip() if result.stderr else "Unknown error"
                logger.warning(
                    f"Failed to build wheel from sdist for {package_name}: {error_msg}"
                )
                # Log additional diagnostic information
                logger.debug(f"Build command stdout: {result.stdout}")
                logger.debug(f"Build command stderr: {result.stderr}")
                logger.debug(f"Return code: {result.returncode}")

                # Check for common compilation issues
                if "LNK1181" in error_msg or "libpq.lib" in error_msg:
                    logger.warning(
                        f"Compilation failed for {package_name} due to missing libraries. "
                        "This is common for packages requiring native compilation on Windows. "
                        "Consider using pre-compiled wheels or alternative packages."
                    )
                elif (
                    "Microsoft Visual Studio" in error_msg
                    or "compiler" in error_msg.lower()
                ):
                    logger.warning(
                        f"Compilation failed for {package_name} due to compiler issues. "
                        "Ensure you have the required build tools installed."
                    )
                return

            # Find the built wheel file
            wheel_files = list(Path(temp_wheel_dir).glob("*.whl"))
            if wheel_files:
                wheel_file = wheel_files[0]
                # Copy wheel to cache directory
                cache_wheel_path = self.cache.cache_dir / wheel_file.name
                shutil.copy2(wheel_file, cache_wheel_path)

                # Update cache metadata
                self.cache.add_package(package_name, wheel_file)

                logger.info(
                    f"Built and cached wheel: {wheel_file.name} for {package_name}"
                )
            else:
                logger.warning(f"No wheel file was built from sdist for {package_name}")

    def _extract_package(
        self, package_file: Path, dest_dir: Path, package_name: str
    ) -> None:
        """Extract package file (wheel or sdist) to destination directory with optional optimization.

        Args:
            package_file: Path to package file (wheel or sdist)
            dest_dir: Destination directory
            package_name: Name of the package being extracted
        """
        logger.info(
            f"Extracting {package_file.name} for package {package_name} to {dest_dir}"
        )

        # Handle sdist files (.tar.gz or .zip) - install using pip, and build wheel for cache
        if package_file.suffix in (".gz", ".zip"):
            self._handle_sdist_extraction(package_file, dest_dir, package_name)
            return

        # Handle wheel files with optional optimization
        self._handle_wheel_extraction(package_file, dest_dir, package_name)

    def _handle_sdist_extraction(
        self, package_file: Path, dest_dir: Path, package_name: str
    ) -> None:
        """Handle extraction of sdist files (.tar.gz or .zip)."""
        logger.info(f"Installing sdist file for {package_name} using pip...")

        # Use pip install --target to install sdist to temporary directory
        with tempfile.TemporaryDirectory() as temp_install_dir:
            result = subprocess.run(
                [
                    self.downloader.pip_executable or "pip",
                    "install",
                    "--target",
                    temp_install_dir,
                    "--no-deps",  # Don't install dependencies (we handle them separately)
                    "--no-cache-dir",
                    str(package_file),
                ],
                capture_output=True,
                text=True,
                check=False,
            )

            if result.returncode != 0:
                logger.error(
                    f"Failed to install sdist {package_file.name}: {result.stderr}"
                )
                return

            # Copy installed files to dest_dir, skipping *.dist-info directories
            temp_install_path = Path(temp_install_dir)
            # Pre-compute dist-info suffix
            dist_info_suffix = ".dist-info"

            for item in temp_install_path.iterdir():
                # Skip dist-info directories
                if item.name.endswith(dist_info_suffix):
                    logger.debug(f"Skipping dist-info directory: {item.name}")
                    continue
                dest_path = dest_dir / item.name
                if item.is_dir():
                    if dest_path.exists():
                        shutil.rmtree(dest_path)
                    shutil.copytree(item, dest_path)
                else:
                    shutil.copy2(item, dest_path)

            logger.info(
                f"Installed sdist file for {package_name} to site-packages structure"
            )

        # Build wheel from sdist and cache it for faster future access
        logger.info(f"Building wheel from sdist for {package_name}...")
        self._build_and_cache_wheel(package_file, package_name)

    def _handle_wheel_extraction(
        self, package_file: Path, dest_dir: Path, package_name: str
    ) -> None:
        """Handle extraction of wheel files with optional optimization."""
        with zipfile.ZipFile(package_file, "r") as zf:
            if self.config.optimize and self.optimization_strategy:
                self._extract_with_optimization(zf, dest_dir, package_name)
            else:
                self._extract_without_optimization(zf, dest_dir, package_name)

    def _extract_with_optimization(
        self, zf: zipfile.ZipFile, dest_dir: Path, package_name: str
    ) -> None:
        """Extract wheel with optimization strategy applied."""
        extracted_count = 0
        skipped_count = 0

        assert self.optimization_strategy is not None, "Optimization strategy is None"
        should_extract = self.optimization_strategy.should_extract_file

        for file_info in zf.filelist:
            file_path = Path(file_info.filename)

            # Skip dist-info directories
            if LibraryCache._should_skip_dist_info(
                file_path
            ):  # Use LibraryCache method
                logger.debug(f"Skipping dist-info: {file_info.filename}")
                skipped_count += 1
                continue

            if should_extract(package_name, file_path):
                zf.extract(file_info, dest_dir)
                extracted_count += 1
                logger.debug(f"Extracted {file_path} from {package_name}")
            else:
                skipped_count += 1
                logger.debug(
                    f"Skipped {file_path} from {package_name} (filtered by optimization strategy)"
                )

        logger.info(
            f"Extraction complete for {package_name}: {extracted_count} extracted, {skipped_count} skipped"
        )

    def _extract_without_optimization(
        self, zf: zipfile.ZipFile, dest_dir: Path, package_name: str
    ) -> None:
        """Extract wheel without optimization, but skip dist-info directories."""
        # Pre-compute the skip function
        should_skip = LibraryCache._should_skip_dist_info  # Use LibraryCache method

        for file_info in zf.filelist:
            file_path = Path(file_info.filename)
            # Skip dist-info directories
            if not should_skip(file_path):
                zf.extract(file_info, dest_dir)
        logger.info(
            f"All files extracted for {package_name} (no optimization applied, dist-info skipped)"
        )

    @staticmethod
    def _should_skip_dist_info(file_path: Path) -> bool:
        """Check if the file path should be skipped because it's a dist-info directory."""
        if file_path.name.endswith(".dist-info"):
            return True
        # Check if any parent directory ends with .dist-info
        return any(part.endswith(".dist-info") for part in file_path.parts)

    def run(self) -> None:
        """Pack project dependencies from base directory with concurrent processing."""
        t0 = time.perf_counter()
        project_count = self.project_count  # Use cached property

        logger.info(f"Starting to pack {project_count} projects")

        if project_count == 1:
            # Single project: process directly
            project = next(iter(self.projects.values()))
            self.pack_project(project)
        else:
            # Multiple projects: process concurrently
            logger.info(f"Packing {project_count} projects concurrently...")
            with ThreadPoolExecutor(max_workers=self.config.max_workers) as executor:
                futures = [
                    executor.submit(self.pack_project, project)
                    for project in self.projects.values()
                ]

                # Wait for all tasks to complete
                for future in as_completed(futures):
                    try:
                        future.result()
                    except Exception as e:
                        logger.error(f"Project packing failed: {e}")

        elapsed_time = time.perf_counter() - t0
        logger.info(f"Packed {project_count} projects in {elapsed_time:.2f}s")

        # Log cache statistics after packing
        logger.info(f"Cache statistics: {self.cache.cache_stats}")

    def show_stats(self):
        logger.info(f"Project count: {self.project_count}")
        logger.info(f"Total dependencies: {len(self.solution.dependencies)}")
        logger.info(f"Working directory size: {self.working_dir_size} bytes")
        logger.info(f"Cache statistics: {self.cache.cache_stats}")


def parse_args() -> argparse.Namespace:
    """Parse command-line arguments."""
    parser = argparse.ArgumentParser(
        prog="pylibpack",
        description="Python library packer with caching support",
    )

    parser.add_argument(
        "directory",
        type=str,
        nargs="?",
        default=str(Path.cwd()),
        help="Base directory containing projects",
    )
    parser.add_argument(
        "--cache-dir", type=str, default=None, help="Custom cache directory"
    )
    parser.add_argument(
        "--python-version", type=str, default=None, help="Target Python version"
    )
    parser.add_argument(
        "-j",
        "--jobs",
        type=int,
        default=DEFAULT_MAX_WORKERS,
        help="Maximum concurrent downloads",
    )
    parser.add_argument(
        "--mirror",
        type=str,
        default=DEFAULT_MIRROR,
        choices=("pypi", "tsinghua", "aliyun", "ustc", "douban", "tencent"),
        help="PyPI mirror source for faster downloads in China",
    )
    parser.add_argument("--debug", "-d", action="store_true", help="Debug mode")
    parser.add_argument(
        "--no-optimize",
        "-no",
        action="store_true",
        help="Disable package optimization (extract all files)",
    )

    # Add option to show package size
    parser.add_argument(
        "--show-size",
        action="store_true",
        help="Show packed package size",
    )
    parser.add_argument(
        "--show-stats",
        action="store_true",
        help="Show detailed project statistics",
    )
    parser.add_argument(
        "--list-optimizations",
        "-lo",
        action="store_true",
        help="List all available optimization rules",
    )
    return parser.parse_args()


def main() -> None:
    """Main entry point for pylibpack tool."""
    args = parse_args()

    # Setup logging
    if args.debug:
        logger.setLevel(logging.DEBUG)

    if args.list_optimizations:
        strategy = SelectiveExtractionStrategy()
        logging.info("Available optimization rules:")
        for lib_name in sorted(strategy.get_library_names_with_rules()):
            logging.info(f"  - {lib_name}")
        return

    # Create configuration from arguments
    config = PyLibPackerConfig(
        cache_dir=Path(args.cache_dir) if args.cache_dir else DEFAULT_CACHE_DIR,
        mirror=args.mirror,
        optimize=not args.no_optimize,
        max_workers=args.jobs,
    )

    # Register auto-save on exit
    atexit.register(config.save)

    packer = PyLibPacker(
        working_dir=Path(args.directory),
        config=config,
    )

    # Execute pack operation
    packer.run()

    if args.show_stats:
        packer.show_stats()


if __name__ == "__main__":
    main()
