"""
Extracts features from scraped CVE data using comprehensive 48-feature logic:
https://gitlab.cee.redhat.com/prodsec/psirt/al-kernel/-/blob/main/alkernel_curver_daemon.pl
WITH DIAGNOSTICS AND TIMEOUT PROTECTION
"""

import os
import re
import json
import pandas as pd
from pathlib import Path
from typing import Dict, Optional
import logging
import time
import signal
from concurrent.futures import ProcessPoolExecutor, as_completed


# Change to 5 for processing only 5 CVEs
CVE_CNT_LIMIT = None


# Setup logging
logging.basicConfig(
    level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger(__name__)


class TimeoutException(Exception):
    pass


def timeout_handler(signum, frame):
    raise TimeoutException("CVE processing timeout")


def _process_cve_worker(
    cve_id: str, data_dir: str, timeout_seconds: int
) -> Optional[Dict]:
    """Top-level worker for processing a single CVE in a separate process.

    Created as a module-level function to be picklable by ProcessPoolExecutor.
    Each worker creates its own CVEFeatureExtractor and applies the same timeout logic.
    """
    extractor = CVEFeatureExtractor(data_dir=data_dir)
    return extractor.process_cve_with_timeout(cve_id, timeout_seconds=timeout_seconds)


class CVEFeatureExtractor:
    def __init__(self, data_dir: str = "data"):
        self.data_dir = Path(data_dir)

        # Severity mappings from Perl code
        self.severity_map = {"IMPORTANT": 0, "MODERATE": 1, "LOW": 2}

        # Feature names (48 features as in Perl code)
        self.feature_names = [
            "kasan",
            "lock",
            "write",
            "read",
            "fixprintf",
            "remote",
            "danger",
            "bpf",
            "nullptr",
            "disk",
            "init",
            "warnonly",
            "outofbounds",
            "deadlock",
            "trace",
            "usb",
            "nomemchk",
            "virt",
            "simplefix",
            "netfilter",
            "nvme",
            "race",
            "leak",
            "errorpath",
            "uaf",
            "networking",
            "notincludedcomponent",
            "skb",
            "dma",
            "logger",
            "improve",
            "kernel_panic_plus_uaf",
            "timer",
            "spectre",
            "hardware",
            "servertoclientfail",
            "tipc",
            "debugfs",
            "linus",
            "cveoutdated",
            "boot",
            "test",
            "qdisc",
            "ipv6",
            "iouring",
            "syzbot",
            "kernel_panic",
            "memory",
        ]

        # Ground truth labels:
        self.ground_truth_labels = self._load_ground_truth()

    def _load_ground_truth(self) -> Dict[str, str]:
        """Load ground truth severity labels from true labeled train and test JSON files"""
        labels = {}

        # Load from train_kernel_cves.json
        try:
            with open("data/train_kernel_cves.json", "r") as f:
                train_data = json.load(f)
            for cve in train_data:
                labels[cve["cve_id"]] = cve["severity"]
            print(f"Loaded {len(train_data)} training CVEs with labels")
        except FileNotFoundError:
            print("train_kernel_cves.json not found")

        # Load from test_kernel_cves.json
        try:
            with open("data/test_kernel_cves.json", "r") as f:
                test_data = json.load(f)
            for cve in test_data:
                labels[cve["cve_id"]] = cve["severity"]
            print(f"Loaded {len(test_data)} test CVEs with labels")
        except FileNotFoundError:
            print("test_kernel_cves.json not found")

        print(f"Total loaded: {len(labels)} CVEs with labels")
        return labels

    def clean_content(self, content: str) -> str:
        """Clean and normalize content"""
        content = re.sub(r"\s+", " ", content)
        return content.strip()

    def extract_patch_features(
        self, patch_content: str, patch_filename: str = ""
    ) -> Dict[str, bool]:
        """Extract features from patch content"""
        features = {name: False for name in self.feature_names}

        try:
            # Size check and truncation
            if len(patch_content) > 5 * 1024 * 1024:  # 5MB limit per patch
                logger.warning(
                    f"Large patch content: {len(patch_content)} bytes, truncating"
                )
                patch_content = patch_content[: 5 * 1024 * 1024]

            # Initialize counters and flags
            lines_text = 0
            lines_src = 0
            uniq_lines = set()
            uniq_lines_plus_only = set()

            # Clean patch content
            patch_content = self.clean_content(patch_content)

            # Count lines and extract unique lines
            lines = patch_content.split("\n")

            # Limiting number of lines to resolve memory issue
            if len(lines) > 50000:
                logger.warning(f"Too many lines: {len(lines)}, processing first 50000")
                lines = lines[:50000]

            for line in lines:
                line = line.strip()
                if line.startswith("+") and not line.startswith("+++"):
                    lines_src += 1
                    cleaned_line = line[1:].strip()
                    if cleaned_line and not self._is_comment_line(cleaned_line):
                        uniq_lines.add(cleaned_line)
                        uniq_lines_plus_only.add(cleaned_line)
                elif line.startswith("-") and not line.startswith("---"):
                    cleaned_line = line[1:].strip()
                    if cleaned_line and not self._is_comment_line(cleaned_line):
                        uniq_lines.add(cleaned_line)

                # Limit unique lines to prevent memory explosion
                if len(uniq_lines) > 10000:
                    logger.warning("Too many unique lines, truncating analysis")
                    break

            # Calculate text lines (approximation)
            commit_msg_match = re.search(
                r"(.*?)^diff --git", patch_content, re.MULTILINE | re.DOTALL
            )
            if commit_msg_match:
                commit_msg = commit_msg_match.group(1)
                lines_text = len(
                    [line for line in commit_msg.split("\n") if line.strip()]
                )

            # Feature extraction logic

            # 1. File path analysis
            if patch_filename:
                features.update(self._analyze_file_path(patch_filename))

            # Extract file paths from patch (limit search area)
            search_content = (
                patch_content[:100000] if len(patch_content) > 100000 else patch_content
            )
            file_paths = re.findall(r"diff --git a/([^\s]+)", search_content)

            # Limit number of file paths processed
            for file_path in file_paths[:100]:  # Process max 100 files
                features.update(self._analyze_file_path(file_path))

            # 2. Content analysis
            features.update(self._analyze_patch_content(patch_content))

            # 2.1 Critical content-based file path adjustments
            content_lower = patch_content.lower()

            # TIPC special rule: if tipc path AND specific send functions → networking + remote
            if features.get("tipc") and re.search(
                r"ip_send|udp_send|tcp_send", content_lower
            ):
                features["networking"] = True
                features["remote"] = True

            # AF_UNIX rule: if networking set but AF_UNIX in content → disable networking
            if features.get("networking") and re.search(r"af_unix", content_lower):
                features["networking"] = False

            # Content-based disk rule: ext4/xfs/PROC_FS in content → disk
            if re.search(r"ext4[^\(]|xfs|proc_fs", content_lower):
                features["disk"] = True

            # 3. Code line analysis (limit unique lines)
            if len(uniq_lines) > 5000:
                logger.warning(
                    f"Large unique line set: {len(uniq_lines)}, sampling first 5000"
                )
                uniq_lines = set(list(uniq_lines)[:5000])

            features.update(self._analyze_code_lines(uniq_lines))

            # 4. Simple fix detection
            if lines_text > 0 and lines_src > 0:
                if lines_text < 21 and lines_src < 15:
                    features["simplefix"] = True

            # 5. CVE year analysis - enhanced threshold logic
            cve_match = re.search(r"CVE-(\d{4})", patch_content)
            if cve_match:
                year = int(cve_match.group(1))
                if year <= 2023 or year > 2099:  # Enhanced threshold for outdated CVEs
                    features["cveoutdated"] = True

            features.update(self._analyze_author(patch_content))

        except TimeoutException:
            # per CVE timeout signalled -> interrupt the processing!
            raise

        except Exception as e:
            logger.error(f"Error in extract_patch_features: {str(e)}")
            # Return default features on error

        return features

    def _is_comment_line(self, line: str) -> bool:
        """Check if line is a comment"""
        line = line.strip()
        return (
            line.startswith("/*")
            or line.startswith("*")
            or line.startswith("*/")
            or line.startswith("//")
        )

    def _analyze_file_path(self, file_path: str) -> Dict[str, bool]:
        """Analyze file path for feature extraction"""
        features = {}

        # Remote/networking paths - comprehensive pattern matching
        remote_patterns = [
            r"net/(llc|ceph|tls|bluetooth|core|ethernet|ipv4|ipv6|netfilter|packet|sched|unix|wireless|mac80211|sunrpc)",
            r"include/net",
            r"drivers/block/nbd",
            r"fs/smb",
            r"fs/netfs",
            r"fs/jfs",
            r"\btcp\b",  # Standalone tcp pattern
        ]
        features["remote"] = any(
            re.search(pattern, file_path) for pattern in remote_patterns
        )

        # IPv6 specific
        features["ipv6"] = bool(re.search(r"ipv6", file_path, re.IGNORECASE))

        # Netfilter
        features["netfilter"] = bool(re.search(r"netfilter", file_path))

        # Hardware drivers
        hardware_patterns = [
            r"drivers/gpu",
            r"sound/pci",
            r"sound/soc",
            r"kernel/events",
            r"drivers/media",
        ]
        features["hardware"] = any(
            re.search(pattern, file_path) for pattern in hardware_patterns
        )

        # Server to client (SMB client)
        features["servertoclientfail"] = bool(re.search(r"fs/smb/client", file_path))

        # TIPC
        features["tipc"] = bool(re.search(r"net/tipc", file_path))

        # Networking (actual network drivers) - comprehensive pattern matching
        networking_patterns = [
            r"drivers/net",
            r"drivers/net/ipvlan",  # IPVLAN driver pattern
            r"drivers/net/ethernet",  # Ethernet driver pattern
            r"net/bluetooth",
            r"net/ipv4",
            r"net/ipv6",
            r"\btcp\b",  # Standalone tcp pattern
            r"net/ceph",  # Missing from Python
            r"gsm",
            r"net/unix",
            r"net/sched",
            r"net/sctp",
            r"drivers/infiniband",  # Missing from Python
            r"net/sunrpc",  # Missing from Python
        ]
        features["networking"] = any(
            re.search(pattern, file_path) for pattern in networking_patterns
        )

        # Trace
        features["trace"] = bool(re.search(r"trace", file_path))

        # USB/HID
        usb_patterns = [r"usb", r"i2c", r"hid", r"input/misc"]
        features["usb"] = any(re.search(pattern, file_path) for pattern in usb_patterns)

        # Virtualization
        features["virt"] = bool(re.search(r"virt|kvm", file_path))

        # BPF
        features["bpf"] = bool(re.search(r"bpf", file_path))

        # NVME
        features["nvme"] = bool(re.search(r"nvme", file_path))

        # Not included components
        excluded_patterns = [
            r"fs/afs",
            r"btrfs",
            r"lib/kunit",
            r"lib/.*kunit",
            r"arch/riscv",
            r"lib/test",
        ]
        features["notincludedcomponent"] = any(
            re.search(pattern, file_path) for pattern in excluded_patterns
        )

        # Disk/filesystem
        disk_patterns = [
            r"block/ioctl",
            r"fs/(btrfs|pstore|ext4|fat|fscache|hugetlbfs|iomap|kernfs|lockd|minix|nfs|nfs_common|nfsd|overlayfs|proc|quota|ramfs|sysfs|xfs)",
            r"drivers/md",
            r"fs/",
        ]
        features["disk"] = any(
            re.search(pattern, file_path) for pattern in disk_patterns
        )

        # TTY/serial (fixprintf equivalent)
        features["fixprintf"] = bool(re.search(r"/(tty|vt|serial)/", file_path))

        # Debugfs
        features["debugfs"] = bool(re.search(r"debugfs", file_path, re.IGNORECASE))

        # IO_URING
        features["iouring"] = bool(re.search(r"io_uring", file_path, re.IGNORECASE))

        return {k: v for k, v in features.items() if v}

    def _analyze_patch_content(self, content: str) -> Dict[str, bool]:
        """Analyze patch content for security indicators"""
        features = {}

        # Limit content size for analysis to prevent regex slowdown
        if len(content) > 1024 * 1024:  # 1MB limit for content analysis
            content = content[: 1024 * 1024]

        # Convert to lowercase for case-insensitive matching
        content_lower = content.lower()

        # KASAN/UBSAN/sanitizers
        kasan_patterns = [r"kaslr", r"kasan", r"kcsan", r"kmsan", r"ubsan", r"svace"]
        features["kasan"] = any(
            re.search(pattern, content_lower) for pattern in kasan_patterns
        )

        # Note: Linus author detection is handled comprehensively in _analyze_author method

        # Out of bounds - comprehensive pattern matching
        oob_patterns = [
            r"recursion",  # Recursion-based overflow detection
            r"address\s",
            r"buffer overflow",
            r"underflow",
            r"buffer-overflow",
            r"oob",
            r"out-of-bound",
            r"buffer boundar",
            r"out of bound",
            r"out-of-range",
            r"out of range",
            r"stack overflow",
            r"stack-overflow",
            r"bounds check",
        ]
        features["outofbounds"] = any(
            re.search(pattern, content_lower) for pattern in oob_patterns
        )

        # Printf/long line fixes
        features["fixprintf"] = bool(re.search(r"scnprintf|long line", content))

        # Race conditions
        features["race"] = bool(re.search(r"race", content_lower))

        # Error path
        features["errorpath"] = bool(
            re.search(r"error path|error handling", content_lower)
        )

        # Read operations
        read_patterns = [
            r"uninit.value in.*?\+",
            r"uninitialized value",
            r"slab.out.of.bounds read",
            r"read of size \d+",
            r"reads from",
            r"read from",
            r"mmap",
            r"read path",
            r"read-after-free",
            r"read after free",
        ]
        features["read"] = any(
            re.search(pattern, content_lower) for pattern in read_patterns
        )

        # Write operations
        write_patterns = [
            r"write to.*?of \d+ bytes",
            r"write\(\)",
            r"write_buf",
            r"free-write",
            r"free write",
            r"concurrent write",
            r"oob write",
            r"oob-write",
            r"oobw",
            r"buffer overrun",
            r"read\swrite",
            r"stack.out.of.bounds",
        ]
        features["write"] = any(
            re.search(pattern, content_lower) for pattern in write_patterns
        )

        # Adjust read/write based on priority
        if features.get("write"):
            features["read"] = False
            # Enhanced rule: write operations indicate out of bounds
            features["outofbounds"] = True
        if re.search(
            r"uninit.value in.*?\+|slab.out.of.bounds read|read of size \d+|read-after-free|read after free",
            content_lower,
        ):
            features["read"] = True
            features["write"] = False

        # KSMBD (not included component)
        if re.search(r"ksmbd", content_lower):
            features["notincludedcomponent"] = True

        # Memory leaks
        leak_patterns = [
            r"leaked",
            r"memory leak",
            r"leak",
            r"that can grow pretty large",
            r"low-memory",
            r"low memory",
        ]
        features["leak"] = any(
            re.search(pattern, content_lower) for pattern in leak_patterns
        )
        if re.search(r"commits", content_lower):  # Ignore commits-related mentions
            features["leak"] = False

        # Memory operations
        features["memory"] = bool(re.search(r"dma|page_size|huge page", content_lower))

        # SKB (socket buffers)
        features["skb"] = bool(re.search(r"skb", content_lower))

        # Initialization
        init_patterns = [
            r"setup.*init",
            r"_init",
            r"initialization",
            r"qdisc_create",
            r"_shutdown",
        ]
        features["init"] = any(
            re.search(pattern, content_lower) for pattern in init_patterns
        )

        # NULL pointer dereference
        nullptr_patterns = [
            r"null pointer deref",
            r"null-pointer-deref",
            r"nullptr",
            r"null ptr",
            r"null-ptr",
            r"null dereference",
            r"null-dereference",
        ]
        features["nullptr"] = any(
            re.search(pattern, content_lower) for pattern in nullptr_patterns
        )

        # Warning only
        warn_patterns = [
            r"warning: cpu",
            r"warning:",
            r"warn/",
            r"blocked for more than",
            r"warn\b",
            r"warn on",
            r"mismatch",
        ]
        features["warnonly"] = any(
            re.search(pattern, content_lower) for pattern in warn_patterns
        )
        if re.search(r"match warning|rip:", content_lower):
            features["warnonly"] = False

        # Spectre
        features["spectre"] = bool(re.search(r"spectre|speculation", content_lower))

        # Use-after-free
        uaf_patterns = [
            r"double-free",
            r"invalid-free",
            r"use-after-free",
            r"use after free",
            r"uaf",
            r"freed\s",
            r"divide-by-zero",
            r"divide by zero\W",
        ]
        features["uaf"] = any(
            re.search(pattern, content_lower) for pattern in uaf_patterns
        )
        if features["uaf"]:
            features["kasan"] = False

        # Internal error
        if re.search(r"internal error: oops:", content_lower) and not features.get(
            "read"
        ):
            features["kasan"] = False

        # Boot/reboot
        features["boot"] = bool(re.search(r"reboot|boot up|booting", content_lower))

        # QDISC
        features["qdisc"] = bool(re.search(r"qdisc", content_lower))

        # Test code - enhanced conditional logic
        test_pattern = re.search(r"\stest", content_lower)
        if test_pattern and not re.search(r"kernel test robot", content_lower):
            features["test"] = True
            # Exception rule: disable test if blktest or disk present
            if re.search(r"blktest", content_lower) or features.get("disk"):
                features["test"] = False

        # Locking
        features["lock"] = bool(re.search(r"locking", content_lower))

        # Kernel panic plus UAF
        if (
            re.search(r"code:", content_lower)
            and re.search(r"rip:", content_lower)
            and features.get("uaf")
        ):
            features["kernel_panic_plus_uaf"] = True
            features["warnonly"] = False

        # Syzbot
        features["syzbot"] = bool(re.search(r"syzbot", content_lower))

        # Danger indicators
        danger_patterns = [
            r"double free",
            r"to userspace",
            r"syzbot",
            r"crash",
            r"panic",
            r"kernel bug",
            r"corruption",
            r"rce",
            r"lpe",
            r"bug:",
            r"dangling pointer",
            r"_state",
        ]
        features["danger"] = any(
            re.search(pattern, content_lower) for pattern in danger_patterns
        )

        # Reduce danger if other indicators present
        if (
            features["danger"]
            and re.search(r"bug:", content_lower)
            and (
                features.get("kasan")
                or features.get("warnonly")
                or features.get("leak")
                or features.get("fixprintf")
                or features.get("bpf")
                or features.get("trace")
                or features.get("init")
            )
        ):
            features["danger"] = False

        # Kernel panic
        kernel_panic_conditions = [
            re.search(r"code:", content_lower),
            re.search(r"rip:", content_lower),
            features.get("uaf"),
            features.get("outofbounds"),
            features.get("nullptr"),
            features.get("danger"),
            features.get("memory"),
        ]

        if (
            any(kernel_panic_conditions)
            and not features.get("read")
            and not re.search(r"potential", content_lower)
            and not features.get("errorpath")
        ):
            features["kernel_panic"] = True

        # Adjust kernel panic based on other factors
        if features.get("kernel_panic"):
            if features.get("kasan") and not features.get("uaf"):
                features["kernel_panic"] = False
            if features.get("bpf"):
                features["kernel_panic"] = False
            if features.get("kernel_panic_plus_uaf"):
                features["kernel_panic"] = True
            # Comprehensive kernel panic rule: (outofbounds || remote || skb) && (ipv6 || networking || hardware || disk)
            if (
                features.get("outofbounds")
                or features.get("remote")
                or features.get("skb")
            ) and (
                features.get("ipv6")
                or features.get("networking")
                or features.get("hardware")
                or features.get("disk")
            ):
                features["kernel_panic"] = True
            if features.get("notincludedcomponent"):
                features["kernel_panic"] = False
            if features.get("leak") and not features.get("networking"):
                features["kernel_panic"] = (
                    False  # Refined leak rule with networking exception
                )
            if features.get("test") and not features.get("networking"):
                features["kernel_panic"] = (
                    False  # Refined test rule with networking exception
                )
            if re.search(r"integer overflow|arch/s390", content_lower):
                features["kernel_panic"] = False
            # Critical nullptr+test/trace interaction rule
            if (
                (features.get("nullptr") and features.get("test"))
                or (features.get("nullptr") and features.get("trace"))
            ) and not features.get("networking"):
                features["kernel_panic"] = False
            # CVE year rule - disable kernel_panic for very old CVEs
            cve_match = re.search(r"CVE-(\d{4})", content)
            if cve_match:
                year = int(cve_match.group(1))
                if year < 2022:
                    features["kernel_panic"] = False
            features["warnonly"] = False

        if features.get("memory"):
            features["kernel_panic"] = True

        # Hardware detection
        hw_patterns = [
            r"_cpu_",
            r"device",
            r"amdgpu",
            r"hardware",
            r"alsa",
            r"(\s|\-|_)arch",
            r"cpu timer/arm64",
            r"cpu",
        ]
        features["hardware"] = any(
            re.search(pattern, content_lower) for pattern in hw_patterns
        )

        # Deadlock
        features["deadlock"] = bool(
            re.search(r"deadlock|dead lock|dead-lock|lockup", content_lower)
        )

        # Improve only
        features["improve"] = bool(
            re.search(r"proper|check\s|checks\s|improv", content_lower)
        )

        # Reduce improve if networking/remote/hardware
        if (
            features.get("networking")
            or features.get("remote")
            or features.get("hardware")
        ):
            features["improve"] = False

        # Critical feature interaction rules

        # UAF disables simplefix and affects warnonly
        if features.get("uaf"):
            features["simplefix"] = False
            if not features.get("init"):
                features["warnonly"] = False

        # Hardware/UAF disables simplefix
        if features.get("hardware") or features.get("uaf"):
            features["simplefix"] = False

        # Networking disables fixprintf
        if features.get("networking"):
            features["fixprintf"] = False

        # Servertoclientfail disables remote
        if features.get("servertoclientfail"):
            features["remote"] = False

        return {k: v for k, v in features.items() if v}

    def _analyze_code_lines(self, uniq_lines: set) -> Dict[str, bool]:
        """Analyze unique code lines for specific patterns"""
        features = {}

        interrupt_count = 0
        log_count = 0

        for line in uniq_lines:
            line_lower = line.lower()

            # Locking
            lock_patterns = [
                r"mutex_lock",
                r"mutex_unlock",
                r"read_lock",
                r"read_unlock",
                r"spin_lock",
                r"spin_unlock",
                r"arch_spin_unlock",
                r"arch_spin_lock",
            ]
            if any(re.search(pattern, line_lower) for pattern in lock_patterns):
                features["lock"] = True

            # SKB
            if re.search(r"skb", line_lower):
                features["skb"] = True

            # Timer
            if re.search(r"timer", line_lower):
                features["timer"] = True

            # Spectre
            if re.search(r"spectre|speculation", line_lower):
                features["spectre"] = True

            # DMA
            if re.search(r"_dma_|dma\s", line_lower):
                features["dma"] = True

            # Interrupt
            if re.search(r"interrupt", line_lower):
                interrupt_count += 1

            # Log
            if re.search(r"log", line_lower):
                log_count += 1

            # Error path
            if re.search(r"dev_err", line_lower):
                features["errorpath"] = True

            # Init patterns
            init_patterns = [
                r"exit",
                r"init",
                r"register",
                r"flush",
                r"shutdown",
                r"disconnect",
                r"_free",
                r"_abort",
                r"_clear",
            ]
            if any(re.search(pattern, line_lower) for pattern in init_patterns):
                features["init"] = True

            # No memory check
            if re.search(r"enomem", line_lower):
                features["nomemchk"] = True

        # Set flags based on counts
        if interrupt_count > 1:
            features["dma"] = True

        if log_count > 1:
            features["logger"] = True

        return {k: v for k, v in features.items() if v}

    def _analyze_author(self, patch_content: str) -> Dict[str, bool]:
        """Analyze patch author information - comprehensive author detection"""
        features = {}

        # Prominent kernel developers (Linus = True) - comprehensive detection
        linus_patterns = [
            r"linus torvalds",
            r"eric dumazet",
            r"or cohen",
            r"sagi grimberg",
            r"keith busch",
            r"yi yang",
            r"greg kroah-hartman",
            r"qiumiao zhang",
            r"david ahern",
            r"paolo abeni",
            r"zack rusin",
            r"ilya dryomov",
            r"kuniyuki iwashima",
            r"valis",
            r"jamal hadi salim",
            r"hangyu hua",
            r"t\.feng",
            r"david s\. miller",
            r"zhiqiang liu",
            r"feilong lin",
            r"jens axboe",
            r"anthony steinhauser",
            r"thomas gleixner",
            r"luiz augusto von dentz",
            r"marcel holtmann",
            r"thadeu lima de souza cascardo",
            r"michal luczaj",
            r"sasha levin",
            r"subbaraya sundeep",
            r"tetsuo handa",
            r"florian westphal",
            r"pablo neira ayuso",
        ]

        # Less prominent developers and domains (Linus = False) - comprehensive list
        anti_linus_patterns = [
            # Individual developer names
            r"bailey forrest",
            r"willem de bruijn",
            r"catherine sullivan",
            r"christoph paasch",
            r"andy shevchenko",
            r"jarkko nikula",
            r"petr machata",
            r"jiri pirko",
            r"ido schimmel",
            r"amit cohen",
            r"xiubo li",
            r"rishabh dave",
            r"zhipeng lu",
            r"hao sun",
            r"daniel borkmann",
            r"felix fietkau",
            r"steve french",
            r"david rientjes",
            r"andy gospodarek",
            r"michael chan",
            r"hans de goede",
            r"toke\s",
            r"stanislav fomichev",
            r"hangbin liu",
            r"jason gunthorpe",
            r"akiva goldberger",
            r"moshe shemesh",
            r"tariq toukan",
            r"kees cook",
            r"johannes berg",
            r"christian k",
            r"alex deucher",
            r"jordy zomer",
            r"john stultz",
            r"sumit semwal",
            r"tom lendacky",
            r"shyam sundar",
            r"maher sanalla",
            r"saeed mahameed",
            r"paolo bonzini",
            r"yue haibing",
            r"yue sun",
            r"xingwei lee",
            r"baokun li",
            r"lu wei",
            r"li zhijian",
            r"dan carpenter",
            r"linus walleij",
            r"david sterba",
            r"filipe manana",
            r"qu wenruo",
            r"mat martineau",
            r"aleksandr mishin",
            r"alexander sergeyev",
            r"vratislav bendel",
            r"mark brown",
            # Domain patterns - these indicate less prominent/external contributors
            r"roeck-us\.net",
            r"linuxtesting\.org",
            r"intel\.com",
            r"amd\.com",
            r"huawei\.com",
            r"oppo\.com",
            r"microsoft",
            r"googlemail\.com",
            r"arm\.com",
            r"netfilter\.org",
            r"oracle\.com",
        ]

        content_lower = patch_content.lower()

        # First check for prominent developers (Linus = True)
        if any(re.search(pattern, content_lower) for pattern in linus_patterns):
            features["linus"] = True
        # Then check for less prominent developers/domains (Linus = False)
        elif any(re.search(pattern, content_lower) for pattern in anti_linus_patterns):
            features["linus"] = False

        return {k: v for k, v in features.items() if v}

    def process_cve_data_with_diagnostics(self, cve_id: str) -> Optional[Dict]:
        """Process data for a single CVE with diagnostic information - handles multiple data formats"""
        start_time = time.time()

        cve_dir = self.data_dir / cve_id

        if not cve_dir.exists():
            logger.warning(f"Directory not found for {cve_id}")
            return None

        # load metadata
        metadata_file = cve_dir / "metadata.json"

        if metadata_file.exists():
            try:
                with open(metadata_file, "r") as f:
                    _ = json.load(f)
                logger.info(f"  {cve_id}: Loaded metadata")

            except TimeoutException:
                # per CVE timeout signalled -> interrupt the processing!
                raise

            except Exception as e:
                logger.warning(f"  {cve_id}: Could not parse metadata: {e}")
        else:
            logger.info(
                f"  {cve_id}: No metadata file found, will process available data"
            )

        # Initialize features
        combined_features = {name: False for name in self.feature_names}

        # Process patches from commits directory
        commits_dir = cve_dir / "commits"
        if commits_dir.exists():
            patch_files = list(commits_dir.glob("*.patch"))

            if not patch_files:
                logger.warning(f"  {cve_id}: No patch files found in commits directory")
                return None

            logger.info(f"  {cve_id}: Found {len(patch_files)} patch files")

            total_size = 0
            for patch_file in patch_files:
                try:
                    # Check file size before processing
                    file_size = patch_file.stat().st_size
                    total_size += file_size

                    if file_size > 10 * 1024 * 1024:  # 10MB
                        logger.warning(
                            f"  {cve_id}: Large patch file {patch_file.name}: {file_size / (1024 * 1024):.1f}MB"
                        )

                    patch_start = time.time()

                    with open(patch_file, "r", encoding="utf-8", errors="ignore") as f:
                        patch_content = f.read()

                    # Extract features from this patch
                    patch_features = self.extract_patch_features(
                        patch_content, str(patch_file)
                    )

                    patch_time = time.time() - patch_start
                    if patch_time > 30:  # More than 30 seconds for one patch
                        logger.warning(
                            f"  {cve_id}: Slow patch {patch_file.name}: {patch_time:.1f}s, size: {len(patch_content)} chars"
                        )

                    # Combine features (OR operation)
                    for feature_name in self.feature_names:
                        if patch_features.get(feature_name, False):
                            combined_features[feature_name] = True

                except TimeoutException:
                    # per CVE timeout signalled -> interrupt the processing!
                    raise

                except Exception as e:
                    logger.error(f"Error processing patch {patch_file}: {str(e)}")

            if total_size > 1024 * 1024:
                logger.info(
                    f"  {cve_id}: Total data size: {total_size / (1024 * 1024):.1f}MB"
                )
            elif total_size > 1024:
                logger.info(f"  {cve_id}: Total data size: {total_size / 1024:.1f}KB")
            else:
                logger.info(f"  {cve_id}: Total data size: {total_size} bytes")
        else:
            logger.warning(f"  {cve_id}: No commits directory found")
            return None

        # Get ground truth label
        severity_label = self.ground_truth_labels.get(cve_id, "UNKNOWN")
        severity_numeric = self.severity_map.get(severity_label, -1)

        result = {
            "cve_id": cve_id,
            "severity_label": severity_label,
            "severity_numeric": severity_numeric,
            **combined_features,
        }

        total_time = time.time() - start_time
        if total_time > 60:  # More than 1 minute
            logger.warning(f"  {cve_id}: SLOW CVE - took {total_time:.1f}s")
        else:
            logger.info(f"  {cve_id}: Processed in {total_time:.1f}s")

        return result

    def process_cve_with_timeout(
        self, cve_id: str, timeout_seconds: int = 600
    ) -> Optional[Dict]:
        """Process CVE with a timeout (default 10 minutes)"""

        # Set up the timeout
        signal.signal(signal.SIGALRM, timeout_handler)
        signal.alarm(timeout_seconds)

        try:
            result = self.process_cve_data_with_diagnostics(cve_id)
            signal.alarm(0)
            return result
        except TimeoutException:
            logger.error(
                f"TIMEOUT: {cve_id} took longer than {timeout_seconds} seconds - SKIPPING"
            )
            signal.alarm(0)
            return None
        except Exception as e:
            logger.error(f"ERROR processing {cve_id}: {str(e)}")
            signal.alarm(0)
            return None

    def create_dataset(
        self,
        output_file: str = "data/cve_dataset.csv",
        *,
        parallel_workers: int | None = None,
        timeout_seconds: int = 600,
        checkpoint_every: int = 100,
    ) -> pd.DataFrame:
        """Create the complete dataset.

        Set parallel_workers > 1 to process CVEs in parallel using separate processes.
        """
        logger.info("Creating dataset...")

        all_results = []
        skipped_cves = []

        cve_ids = list(self.ground_truth_labels.keys())

        # resume from last checkpoint (if stuck)
        start_index = 0

        if CVE_CNT_LIMIT:
            cve_ids = cve_ids[:CVE_CNT_LIMIT]

        logger.info(f"Processing {len(cve_ids)} remaining CVEs")

        # Load existing results from checkpoint
        checkpoint_file = "data/cve_checkpoint_0.csv"
        if os.path.exists(checkpoint_file):
            checkpoint_df = pd.read_csv(checkpoint_file)
            all_results = checkpoint_df.to_dict("records")
            logger.info(f"LOADED {len(all_results)} CVEs from checkpoint")
        else:
            all_results = []

        # Filter out CVEs already present in checkpoint (if any)
        already_processed_ids = (
            {row["cve_id"] for row in all_results} if all_results else set()
        )
        remaining_cve_ids = [cid for cid in cve_ids if cid not in already_processed_ids]

        if parallel_workers and parallel_workers > 1:
            logger.info(
                f"Parallel mode: using {parallel_workers} workers for {len(remaining_cve_ids)} CVEs"
            )
            processed_count = len(already_processed_ids)
            with ProcessPoolExecutor(max_workers=parallel_workers) as executor:
                exec_to_cve = {
                    executor.submit(
                        _process_cve_worker, cve_id, str(self.data_dir), timeout_seconds
                    ): cve_id
                    for cve_id in remaining_cve_ids
                }
                for future in as_completed(exec_to_cve):
                    cve_id = exec_to_cve[future]
                    try:
                        result = future.result()
                        if result:
                            all_results.append(result)
                        else:
                            skipped_cves.append(cve_id)
                            logger.warning(f"SKIPPED: {cve_id}")
                    except Exception as e:
                        skipped_cves.append(cve_id)
                        logger.error(f"ERROR processing {cve_id} in worker: {e}")

                    processed_count += 1
                    logger.info(f"Processed {processed_count}/{len(cve_ids)}: {cve_id}")
                    # Save checkpoint periodically
                    if processed_count % checkpoint_every == 0:
                        checkpoint_df = pd.DataFrame(all_results)
                        checkpoint_file = f"data/cve_checkpoint_{processed_count}.csv"
                        checkpoint_df.to_csv(checkpoint_file, index=False)
                        logger.info(
                            f"CHECKPOINT: Saved {len(all_results)} CVEs to {checkpoint_file}"
                        )
        else:
            for i, cve_id in enumerate(remaining_cve_ids, start_index + 1):
                logger.info(f"Processing {i}/{len(cve_ids)}: {cve_id}")

                result = self.process_cve_with_timeout(
                    cve_id, timeout_seconds=timeout_seconds
                )
                if result:
                    all_results.append(result)
                else:
                    skipped_cves.append(cve_id)
                    logger.warning(f"SKIPPED: {cve_id}")

                # Save checkpoint periodically
                if (i - start_index) % checkpoint_every == 0:
                    checkpoint_df = pd.DataFrame(all_results)
                    checkpoint_file = f"data/cve_checkpoint_{i}.csv"
                    checkpoint_df.to_csv(checkpoint_file, index=False)
                    logger.info(
                        f"CHECKPOINT: Saved {len(all_results)} CVEs to {checkpoint_file}"
                    )

        logger.info(
            f"Completed processing. Successful: {len(all_results)}, Skipped: {len(skipped_cves)}"
        )

        if skipped_cves:
            logger.info("Skipped CVEs:")
            for cve in skipped_cves:
                logger.info(f"  - {cve}")

        df = pd.DataFrame(all_results)
        # Save to CSV
        df.to_csv(output_file, index=False)
        logger.info(f"Dataset saved to {output_file}")

        # Print summary statistics
        self._print_dataset_summary(df)

        return df

    def _print_dataset_summary(self, df: pd.DataFrame):
        """summary statistics of the dataset"""
        logger.info("\n=== Dataset Summary ===")
        logger.info(f"Total CVEs: {len(df)}")

        # Check if dataset is empty
        if len(df) == 0:
            logger.info("Dataset is empty - no features to summarize")
            return

        # Severity distribution
        logger.info("\nSeverity Distribution:")
        if "severity_label" in df.columns:
            severity_counts = df["severity_label"].value_counts()
            for severity, count in severity_counts.items():
                logger.info(f"  {severity}: {count} ({count / len(df) * 100:.1f}%)")
        else:
            logger.info("No severity_label column found")

        # Feature statistics
        logger.info("\nFeature Statistics (top 10 most common):")
        feature_counts = {}
        for feature in self.feature_names:
            if feature in df.columns:
                feature_counts[feature] = df[feature].sum()

        # Sort by frequency
        sorted_features = sorted(
            feature_counts.items(), key=lambda x: x[1], reverse=True
        )
        for feature, count in sorted_features[:10]:
            logger.info(f"  {feature}: {count} ({count / len(df) * 100:.1f}%)")


def main():
    engineer = CVEFeatureExtractor()

    # Create the dataset
    engineer.create_dataset(parallel_workers=os.cpu_count())

    logger.info("Feature extraction complete!")


if __name__ == "__main__":
    main()
