"""
CVE Data Scraper for Linux Kernel Security Vulnerabilities

This script:
- Loads CVE IDs from a local JSON file (data/kernel_cves.json)
- Clones/updates Linux kernel repository and security vulnerabilities repository
- Extracts CVE details from security repo (.json/.mbox files)
- Searches for related commit URLs and affected files
- Fetches detailed patch information from the main Linux kernel repository
- Saves structured data including commit patches, metadata, and file changes
- Generates comprehensive analysis reports with success/failure statistics
- Handles missing commits and provides detailed logging throughout the process

Data Sources:
- kernel_cves.json (local CVE ID list)
- https://git.kernel.org/pub/scm/linux/security/vulns.git (security repo)
- git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git (main kernel repo)

Output: Organized directory structure with CVE data, commits, patches, and analysis reports
"""

import re
import subprocess
import time
from pathlib import Path
import logging
from typing import List, Dict, Optional
import json


# Change to 5 for processing only 5 CVEs
CVE_CNT_LIMIT = None


# Setup logging
logging.basicConfig(
    level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger(__name__)


class CVEDataScraper:
    def __init__(self, base_data_dir: str = "data"):
        self.base_data_dir = Path(base_data_dir)
        self.base_data_dir.mkdir(exist_ok=True)

        # Git repository setup
        self.linux_repo_path = self.base_data_dir / "linux_kernel_repo"
        self.vulns_repo_path = self.base_data_dir / "linux_security_vulns"

        self.cve_list = self._extract_cve_list()

    def _extract_cve_list(self) -> List[str]:
        """Load CVE IDs from the all_cves_summary.json file"""
        json_file_path = Path("data/kernel_cves.json")
        logger.info(f"Loading CVE IDs from {json_file_path}")
        with open(json_file_path, "r", encoding="utf-8") as f:
            cve_data = json.load(f)

        # Extract CVE IDs from the JSON structure
        cve_ids = []
        for entry in cve_data:
            if (
                "cve_id" in entry
                and entry["cve_id"] is not None
                and entry["cve_id"].strip()
            ):
                cve_ids.append(entry["cve_id"])

        logger.info(f"Loaded {len(cve_ids)} CVE IDs from JSON file")
        return cve_ids

    def setup_repositories(self):
        """Clone or update both Linux kernel and security vulnerabilities repositories"""
        logger.info("Setting up repositories...")

        # Setup security vulnerabilities repo
        self.setup_vulns_repo()

        # Setup Linux kernel repository
        self.setup_linux_repo()

        logger.info("All repositories setup complete")

    def setup_linux_repo(self):
        """Clone or update the Linux kernel git repository"""
        logger.info("Setting up Linux kernel git repository...")

        if not self.linux_repo_path.exists():
            logger.info("Cloning Linux kernel repository (this will take a while)...")
            cmd = [
                "git",
                "clone",
                "git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git",
                str(self.linux_repo_path),
            ]
            try:
                subprocess.run(cmd, check=True)
            except subprocess.CalledProcessError:
                logger.info("Git protocol failed, trying HTTPS...")
                cmd = [
                    "git",
                    "clone",
                    "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git",
                    str(self.linux_repo_path),
                ]
                subprocess.run(cmd, check=True)
        else:
            logger.info("Linux kernel repository exists. Updating with git pull...")
            try:
                subprocess.run(["git", "pull"], cwd=self.linux_repo_path, check=True)
                logger.info("Linux kernel repository updated successfully")
            except subprocess.CalledProcessError as e:
                logger.warning(
                    f"Git pull failed for Linux repo: {e}. Continuing with existing repo..."
                )

    def setup_vulns_repo(self):
        """Clone or update the Linux security vulnerabilities repo"""
        logger.info("Setting up Linux security vulnerabilities repository...")

        if not self.vulns_repo_path.exists():
            logger.info("Cloning Linux security vulnerabilities repo...")
            cmd = [
                "git",
                "clone",
                "https://git.kernel.org/pub/scm/linux/security/vulns.git",
                str(self.vulns_repo_path),
            ]
            subprocess.run(cmd, check=True)
        else:
            logger.info(
                "Security vulnerabilities repo exists. Updating with git pull..."
            )
            try:
                subprocess.run(["git", "pull"], cwd=self.vulns_repo_path, check=True)
                logger.info("Security vulnerabilities repository updated successfully")
            except subprocess.CalledProcessError as e:
                logger.warning(
                    f"Git pull failed for vulns repo: {e}. Continuing with existing repo..."
                )

    def process_cve_from_vulns_repo(self, cve_id: str) -> Optional[Dict]:
        """Process CVE information from the security vulnerabilities repo"""
        logger.info(f"Processing {cve_id} from security vulnerabilities repo")

        cve_dir = self.base_data_dir / cve_id
        cve_dir.mkdir(exist_ok=True)

        # Look for CVE files in the vulns repository
        cve_year = cve_id.split("-")[1]  # Extract year from CVE-YYYY-NNNNN

        # Possible file locations
        possible_paths = [
            self.vulns_repo_path / f"cve/published/{cve_year}/{cve_id}.json",
            self.vulns_repo_path / f"cve/published/{cve_year}/{cve_id}.mbox",
            self.vulns_repo_path / f"cve/{cve_year}/{cve_id}.json",
            self.vulns_repo_path / f"cve/{cve_year}/{cve_id}.mbox",
        ]

        # search recursively if not found in possible paths
        try:
            search_result = subprocess.run(
                [
                    "find",
                    str(self.vulns_repo_path),
                    "-name",
                    f"*{cve_id}*",
                    "-type",
                    "f",
                ],
                capture_output=True,
                text=True,
                check=True,
            )

            if search_result.stdout.strip():
                found_files = search_result.stdout.strip().split("\n")
                for file_path in found_files:
                    if file_path not in [str(p) for p in possible_paths]:
                        possible_paths.append(Path(file_path))

        except subprocess.CalledProcessError:
            logger.warning(f"Find command failed for {cve_id}")

        cve_data = {
            "cve_id": cve_id,
            "json_data": None,
            "mbox_data": None,
            "commit_urls": [],
            "affected_files": [],
        }

        # Process each found file
        files_found = []
        for file_path in possible_paths:
            if file_path.exists():
                files_found.append(str(file_path))
                logger.info(f"Found CVE file: {file_path}")

                try:
                    if file_path.suffix == ".json":
                        with open(file_path, "r", encoding="utf-8") as f:
                            cve_data["json_data"] = json.load(f)

                        # Copy JSON file to CVE directory
                        json_copy = cve_dir / f"{cve_id}.json"
                        with open(json_copy, "w") as f:
                            json.dump(cve_data["json_data"], f, indent=2)

                        # Extract commit info from JSON
                        self._extract_info_from_json(cve_data)

                    elif file_path.suffix == ".mbox":
                        with open(
                            file_path, "r", encoding="utf-8", errors="ignore"
                        ) as f:
                            cve_data["mbox_data"] = f.read()

                        # Copy mbox file to CVE directory
                        mbox_copy = cve_dir / f"{cve_id}.mbox"
                        with open(mbox_copy, "w", encoding="utf-8") as f:
                            f.write(cve_data["mbox_data"])

                        # Extract information from mbox
                        self._extract_info_from_mbox(cve_data)

                except Exception as e:
                    logger.error(f"Error reading {file_path}: {e}")

        if not files_found:
            logger.warning(f"No CVE files found for {cve_id} in security repo")
            return None

        # Now get commit information from Linux repo if we have commit URLs
        if cve_data["commit_urls"]:
            self.fetch_commit_info_from_linux_repo(cve_id, cve_data["commit_urls"])
        else:
            # Try to find commits by searching git log
            self.find_cve_commits_in_linux_repo(cve_id)

        # Save metadata
        metadata = {
            "cve_id": cve_id,
            "source_files": files_found,
            "json_data": cve_data["json_data"],
            "commit_urls": cve_data["commit_urls"],
            "affected_files": cve_data["affected_files"],
            "scraped_at": time.time(),
            "method": "security_vulns_repo",
        }

        metadata_path = cve_dir / "metadata.json"
        with open(metadata_path, "w") as f:
            json.dump(metadata, f, indent=2)

        logger.info(f"Successfully processed {cve_id} from security repo")
        return metadata

    def _extract_info_from_json(self, cve_data: Dict):
        """Extract commit URLs and affected files from JSON data"""
        if not cve_data["json_data"]:
            return

        json_data = cve_data["json_data"]

        # Look for commit references in various fields
        def extract_commits_from_text(text):
            if not text:
                return []

            commit_patterns = [
                r"https://git\.kernel\.org/[^/]+/[^/]+/[^/]+/[^/]+/[^/]+/commit/\?id=([0-9a-fA-F]+)",
                r"https://git\.kernel\.org/stable/c/([0-9a-fA-F]+)",
                r"https://github\.com/torvalds/linux/commit/([0-9a-fA-F]+)",
                r"commit\s+([0-9a-fA-F]{40})",
                r"([0-9a-fA-F]{40})\s+\(",
            ]

            commits = []
            for pattern in commit_patterns:
                matches = re.findall(pattern, text)
                commits.extend(matches)

            return commits

        # Search in different JSON fields
        fields_to_search = [
            "description",
            "references",
            "problemDescription",
            "mitigation",
        ]

        for field in fields_to_search:
            if field in json_data:
                field_value = json_data[field]
                if isinstance(field_value, str):
                    commits = extract_commits_from_text(field_value)
                    for commit in commits:
                        commit_url = f"https://git.kernel.org/stable/c/{commit}"
                        if commit_url not in cve_data["commit_urls"]:
                            cve_data["commit_urls"].append(commit_url)
                elif isinstance(field_value, list):
                    for item in field_value:
                        if isinstance(item, str):
                            commits = extract_commits_from_text(item)
                            for commit in commits:
                                commit_url = f"https://git.kernel.org/stable/c/{commit}"
                                if commit_url not in cve_data["commit_urls"]:
                                    cve_data["commit_urls"].append(commit_url)
                        elif isinstance(item, dict) and "url" in item:
                            commits = extract_commits_from_text(item["url"])
                            for commit in commits:
                                commit_url = f"https://git.kernel.org/stable/c/{commit}"
                                if commit_url not in cve_data["commit_urls"]:
                                    cve_data["commit_urls"].append(commit_url)

    def _extract_info_from_mbox(self, cve_data: Dict):
        """Extract commit URLs and affected files from mbox content"""
        if not cve_data["mbox_data"]:
            return

        mbox_content = cve_data["mbox_data"]

        # Extract git commit URLs
        commit_patterns = [
            r"https://git\.kernel\.org/[^/]+/[^/]+/[^/]+/[^/]+/[^/]+/commit/\?id=([0-9a-fA-F]+)",
            r"https://git\.kernel\.org/stable/c/([0-9a-fA-F]+)",
            r"https://github\.com/torvalds/linux/commit/([0-9a-fA-F]+)",
            r"commit\s+([0-9a-fA-F]{40})",
            r"([0-9a-fA-F]{40})\s+\(",
        ]

        for pattern in commit_patterns:
            matches = re.findall(pattern, mbox_content)
            for match in matches:
                commit_url = f"https://git.kernel.org/stable/c/{match}"
                if commit_url not in cve_data["commit_urls"]:
                    cve_data["commit_urls"].append(commit_url)

        # Extract affected files
        file_patterns = [
            r"The file\(s\) affected by this issue are:\s*(.*?)(?=\n\n|\nMitigation|\nThe|\Z)",
            r"Affected files:\s*(.*?)(?=\n\n|\nMitigation|\nThe|\Z)",
            r"([a-zA-Z0-9_/.-]+\.[ch])\s",
            r"diff --git a/([^\s]+)",
        ]

        for pattern in file_patterns:
            matches = re.findall(pattern, mbox_content, re.DOTALL)
            for match in matches:
                if isinstance(match, tuple):
                    match = match[0]
                # Clean up and extract file paths
                lines = match.split("\n")
                for line in lines:
                    line = line.strip()
                    if "/" in line and "." in line and len(line) > 3:
                        # file path
                        if re.match(r"^[a-zA-Z0-9_/.-]+\.[a-zA-Z0-9]+$", line):
                            if line not in cve_data["affected_files"]:
                                cve_data["affected_files"].append(line)

    def fetch_commit_info_from_linux_repo(self, cve_id: str, commit_urls: List[str]):
        """Fetch detailed commit information from Linux repository"""
        logger.info(f"Fetching commit info for {cve_id} from Linux repo")

        cve_dir = self.base_data_dir / cve_id
        commits_dir = cve_dir / "commits"
        commits_dir.mkdir(exist_ok=True)

        # Track success/failure statistics
        found_commits = []
        missing_commits = []

        for commit_url in commit_urls:
            # Extract commit hash from URL
            match = re.search(r"/c/([0-9a-fA-F]+)", commit_url)
            if not match:
                match = re.search(r"commit/([0-9a-fA-F]+)", commit_url)
            if not match:
                continue

            commit_hash = match.group(1)

            try:
                # First, try to fetch from remote if commit is not found locally
                self._ensure_commit_exists(commit_hash)

                # Get commit details
                commit_info = {"commit_hash": commit_hash, "commit_url": commit_url}

                # Get commit message and metadata
                show_cmd = ["git", "show", "--no-patch", "--format=fuller", commit_hash]
                result = subprocess.run(
                    show_cmd,
                    cwd=self.linux_repo_path,
                    capture_output=True,
                    text=True,
                    check=True,
                )
                commit_info["commit_details"] = result.stdout

                # Get the actual patch
                patch_cmd = ["git", "show", commit_hash]
                result = subprocess.run(
                    patch_cmd,
                    cwd=self.linux_repo_path,
                    capture_output=True,
                    text=True,
                    check=True,
                )
                commit_info["patch_content"] = result.stdout

                # Get list of changed files
                files_cmd = ["git", "show", "--name-only", "--format=", commit_hash]
                result = subprocess.run(
                    files_cmd,
                    cwd=self.linux_repo_path,
                    capture_output=True,
                    text=True,
                    check=True,
                )
                commit_info["changed_files"] = [
                    f.strip() for f in result.stdout.split("\n") if f.strip()
                ]

                # Get diff stats
                stats_cmd = ["git", "show", "--stat", "--format=", commit_hash]
                result = subprocess.run(
                    stats_cmd,
                    cwd=self.linux_repo_path,
                    capture_output=True,
                    text=True,
                    check=True,
                )
                commit_info["diff_stats"] = result.stdout

                # Save commit information
                commit_file = commits_dir / f"{commit_hash}.json"
                with open(commit_file, "w") as f:
                    json.dump(commit_info, f, indent=2)

                # Also save patch as separate file for easier processing
                patch_file = commits_dir / f"{commit_hash}.patch"
                with open(patch_file, "w") as f:
                    f.write(commit_info["patch_content"])

                found_commits.append(commit_hash)
                logger.info(f"Saved commit info for {commit_hash}")

            except subprocess.CalledProcessError as e:
                missing_commits.append(commit_hash)
                logger.warning(f"Commit {commit_hash} not found locally: {str(e)}")

                # Save commit as missing with URL for later reference
                missing_file = commits_dir / f"{commit_hash}_missing.json"
                missing_info = {
                    "commit_hash": commit_hash,
                    "commit_url": commit_url,
                    "status": "not_found_locally",
                    "error": str(e),
                }
                with open(missing_file, "w") as f:
                    json.dump(missing_info, f, indent=2)

            except Exception as e:
                missing_commits.append(commit_hash)
                logger.error(f"Unexpected error with commit {commit_hash}: {str(e)}")

        logger.info(
            f"Commit summary for {cve_id}: {len(found_commits)} found, {len(missing_commits)} missing"
        )

        return found_commits, missing_commits

    def _ensure_commit_exists(self, commit_hash: str):
        """Try to fetch a commit if it doesn't exist locally"""
        # First check if commit exists
        check_cmd = ["git", "cat-file", "-e", commit_hash]
        result = subprocess.run(
            check_cmd, cwd=self.linux_repo_path, capture_output=True, text=True
        )

        if result.returncode == 0:
            return  # Commit exists

        # Try to fetch the specific commit if it doesn't exist
        logger.info(f"Attempting to fetch commit {commit_hash} from remote")
        try:
            # Try to fetch the specific commit first
            fetch_cmd = ["git", "fetch", "origin", commit_hash]
            subprocess.run(
                fetch_cmd,
                cwd=self.linux_repo_path,
                capture_output=True,
                text=True,
                check=True,
            )
        except subprocess.CalledProcessError:
            # If specific commit fetch fails, try fetching all refs
            try:
                fetch_cmd = ["git", "fetch", "origin"]
                subprocess.run(
                    fetch_cmd,
                    cwd=self.linux_repo_path,
                    capture_output=True,
                    text=True,
                    check=True,
                )
            except subprocess.CalledProcessError:
                pass

    def find_cve_commits_in_linux_repo(self, cve_id: str):
        """Find commits mentioning the CVE in the Linux repository"""
        logger.info(f"Searching for {cve_id} commits in Linux repo")

        try:
            # Search for commits mentioning this CVE
            git_log_cmd = [
                "git",
                "log",
                "--all",
                "--grep",
                cve_id,
                "--pretty=format:%H",
                "--max-count=10",
            ]

            result = subprocess.run(
                git_log_cmd,
                cwd=self.linux_repo_path,
                capture_output=True,
                text=True,
                check=True,
            )

            if result.stdout.strip():
                commit_hashes = result.stdout.strip().split("\n")
                commit_urls = [
                    f"https://git.kernel.org/stable/c/{h}" for h in commit_hashes
                ]
                logger.info(f"Found {len(commit_urls)} commits for {cve_id} in git log")
                self.fetch_commit_info_from_linux_repo(cve_id, commit_urls)
            else:
                logger.info(f"No commits found for {cve_id} in git log")

        except subprocess.CalledProcessError as e:
            logger.error(f"Git search failed for {cve_id}: {e}")

    def process_single_cve(self, cve_id: str):
        """Process a single CVE using the security vulnerabilities repository"""
        logger.info(f"Processing {cve_id}")

        # Process CVE from security vulnerabilities repository
        metadata = self.process_cve_from_vulns_repo(cve_id)

        if metadata:
            logger.info(f"Successfully processed {cve_id}")
            return True
        else:
            logger.warning(f"Could not find information for {cve_id}")
            return False

    def process_all_cves(self, limit: Optional[int] = None):
        """Process all CVEs in the list"""
        cves_to_process = self.cve_list[:limit] if limit else self.cve_list

        logger.info(f"Processing {len(cves_to_process)} CVEs")

        success_count = 0
        for i, cve_id in enumerate(cves_to_process, 1):
            logger.info(f"Processing {i}/{len(cves_to_process)}: {cve_id}")

            if self.process_single_cve(cve_id):
                success_count += 1

            # Add small delay to avoid overwhelming the system
            time.sleep(0.5)

        logger.info(
            f"Completed processing. Success: {success_count}/{len(cves_to_process)}"
        )

    def get_summary_stats(self):
        """Get summary statistics of processed CVEs"""
        summary = {
            "total_cves": len(self.cve_list),
            "processed_cves": 0,
            "cves_with_commits": 0,
            "total_commits": 0,
            "found_commits": 0,
            "missing_commits": 0,
            "cves_with_files": 0,
        }

        for cve_id in self.cve_list:
            cve_dir = self.base_data_dir / cve_id
            metadata_file = cve_dir / "metadata.json"

            if metadata_file.exists():
                summary["processed_cves"] += 1

                with open(metadata_file, "r") as f:
                    metadata = json.load(f)

                if metadata.get("commit_urls"):
                    summary["cves_with_commits"] += 1
                    summary["total_commits"] += len(metadata["commit_urls"])

                if metadata.get("affected_files"):
                    summary["cves_with_files"] += 1

                # Count actual commit files found
                commits_dir = cve_dir / "commits"
                if commits_dir.exists():
                    found_files = list(commits_dir.glob("*.json"))
                    missing_files = list(commits_dir.glob("*_missing.json"))

                    # Don't count missing files in found count
                    actual_found = [
                        f for f in found_files if not f.name.endswith("_missing.json")
                    ]
                    summary["found_commits"] += len(actual_found)
                    summary["missing_commits"] += len(missing_files)

        return summary

    def analyze_missing_commits(self):
        """Analyze which commits are missing and why"""
        missing_analysis = {
            "total_missing": 0,
            "cves_with_missing": 0,
            "missing_by_cve": {},
        }

        for cve_id in self.cve_list:
            cve_dir = self.base_data_dir / cve_id
            commits_dir = cve_dir / "commits"

            if commits_dir.exists():
                missing_files = list(commits_dir.glob("*_missing.json"))
                if missing_files:
                    missing_analysis["cves_with_missing"] += 1
                    missing_analysis["total_missing"] += len(missing_files)
                    missing_analysis["missing_by_cve"][cve_id] = []

                    for missing_file in missing_files:
                        with open(missing_file, "r") as f:
                            missing_info = json.load(f)
                        missing_analysis["missing_by_cve"][cve_id].append(
                            {
                                "commit_hash": missing_info["commit_hash"],
                                "commit_url": missing_info["commit_url"],
                            }
                        )

        return missing_analysis

    def export_summary_report(self, output_file: str = "cve_analysis_report.json"):
        """Export a comprehensive summary report"""
        stats = self.get_summary_stats()
        missing_analysis = self.analyze_missing_commits()

        # Get detailed info for each processed CVE
        cve_details = {}
        for cve_id in self.cve_list:
            cve_dir = self.base_data_dir / cve_id
            metadata_file = cve_dir / "metadata.json"

            if metadata_file.exists():
                with open(metadata_file, "r") as f:
                    metadata = json.load(f)

                commits_dir = cve_dir / "commits"
                found_commits = []
                missing_commits = []

                if commits_dir.exists():
                    # Get found commits
                    found_files = [
                        f
                        for f in commits_dir.glob("*.json")
                        if not f.name.endswith("_missing.json")
                    ]
                    found_commits = [f.stem for f in found_files]

                    # Get missing commits
                    missing_files = list(commits_dir.glob("*_missing.json"))
                    for missing_file in missing_files:
                        with open(missing_file, "r") as f:
                            missing_info = json.load(f)
                        missing_commits.append(missing_info["commit_hash"])

                cve_details[cve_id] = {
                    "total_commit_urls": len(metadata.get("commit_urls", [])),
                    "found_commits": found_commits,
                    "missing_commits": missing_commits,
                    "affected_files": metadata.get("affected_files", []),
                    "source_files": metadata.get("source_files", []),
                }

        report = {
            "generation_time": time.time(),
            "summary_statistics": stats,
            "missing_commits_analysis": missing_analysis,
            "cve_details": cve_details,
        }

        output_path = self.base_data_dir / output_file
        with open(output_path, "w") as f:
            json.dump(report, f, indent=2)

        logger.info(f"Exported comprehensive report to {output_path}")
        return report


def main():
    scraper = CVEDataScraper()

    # Check if git is available
    try:
        subprocess.run(["git", "--version"], capture_output=True, check=True)
    except (subprocess.CalledProcessError, FileNotFoundError):
        logger.error("git is not installed. Please install git first.")
        return

    # Setup both repositories
    scraper.setup_repositories()

    scraper.process_all_cves(limit=CVE_CNT_LIMIT)

    # Print summary statistics
    stats = scraper.get_summary_stats()
    logger.info(f"Summary: {stats}")

    # Analyze missing commits
    missing_analysis = scraper.analyze_missing_commits()
    logger.info(
        f"Missing commits analysis: {missing_analysis['total_missing']} total missing from {missing_analysis['cves_with_missing']} CVEs"
    )

    # Export comprehensive report
    scraper.export_summary_report()

    # Print some insights
    if stats["processed_cves"] > 0:
        success_rate = (stats["found_commits"] / max(stats["total_commits"], 1)) * 100
        logger.info(
            f"Commit success rate: {success_rate:.1f}% ({stats['found_commits']}/{stats['total_commits']})"
        )

        if missing_analysis["total_missing"] > 0:
            logger.info("Sample missing commits (may be in stable/LTS branches):")
            count = 0
            for cve_id, missing_commits in missing_analysis["missing_by_cve"].items():
                if count >= 3:  # Show only first 3 CVEs with missing commits
                    break
                logger.info(f"  {cve_id}: {len(missing_commits)} missing commits")
                for commit in missing_commits[
                    :2
                ]:  # Show first 2 missing commits per CVE
                    logger.info(
                        f"    - {commit['commit_hash']} ({commit['commit_url']})"
                    )
                count += 1


if __name__ == "__main__":
    main()
