# Copyright (c) 2024 Finite State, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

"""Main report engine for orchestrating the reporting process."""

import gc
import hashlib
import logging
import platform
import threading
import time
from dataclasses import dataclass, field
from pathlib import Path

try:
    import resource  # Unix only; not available on Windows
except ImportError:
    resource = None  # type: ignore[assignment]
from collections.abc import Callable
from typing import Any

import httpx
import pandas as pd

from fs_report.api_client import APIClient
from fs_report.data_cache import DataCache
from fs_report.data_transformer import DataTransformer
from fs_report.models import Config, QueryConfig, Recipe, ReportData
from fs_report.recipe_loader import RecipeLoader
from fs_report.renderers import ReportRenderer
from fs_report.sqlite_cache import _trim_factors


@dataclass
class RecipeResult:
    """Per-recipe result data for headless JSON summary."""

    recipe: str
    output_dir: str
    files: list[str] = field(default_factory=list)
    stats: dict[str, Any] = field(default_factory=dict)


@dataclass
class RunResult:
    """Overall run result returned by ``ReportEngine.run()``."""

    success: bool
    recipes: list[RecipeResult] = field(default_factory=list)


# [REMOVED] All DuckDB-related logic and imports. Only pandas transformer is used.

# Raw API columns that _normalize_columns already extracts into flat
# columns (component_name, has_exploit, reachability_score, etc.).
# Safe to drop after per-batch scoring to reclaim ~1 KB/row.
_TRIAGE_DROP_AFTER_SCORE: frozenset[str] = frozenset(
    {
        # Nested dicts (biggest memory hogs — ~500 bytes each)
        "component",
        "project",
        "projectVersion",
        # List/dict fields consumed by _normalize_columns
        "exploitInfo",
        "reachability",
        "factors",
        # Flat API fields already normalized into snake_case columns
        "cwes",
        "attackVector",
        "epssPercentile",
        "epssScore",
        "reachabilityScore",
        "hasKnownExploit",
        "inKev",
        "inVcKev",
        "affectedFunctions",
        "findingId",
        "projectId",
        # Dot-notation columns from json_normalize (if pre-flattened)
        "component.name",
        "component.version",
        "component.vcId",
        "component.id",
        "project.name",
        "project.id",
        "projectVersion.id",
        "projectVersion.version",
    }
)


# Columns the Executive Summary actually uses.  Everything else is
# dropped per-batch during fetch to avoid accumulating ~50+ columns
# (nested dicts, exploit info, etc.) across 500K+ rows.
_EXEC_SUMMARY_KEEP: frozenset[str] = frozenset(
    {"id", "project.name", "severity", "status", "detected"}
)


def _prune_exec_summary(
    df: pd.DataFrame,
    extra_keep: frozenset[str] | None = None,
) -> pd.DataFrame:
    """Extract ``project.name`` and keep only the columns Executive Summary needs.

    This is called per-batch during fetch so large nested-dict columns
    (component, project, exploitInfo, …) are freed immediately, reducing
    peak memory from ~4-5 GB to ~100 MB on folders with 500K+ findings.
    """
    # Extract project.name from nested dict if not already flat
    if "project.name" not in df.columns and "project" in df.columns:
        df = df.copy()
        df["project.name"] = df["project"].apply(
            lambda p: p.get("name", "") if isinstance(p, dict) else ""
        )
    else:
        df = df.copy()

    keep = _EXEC_SUMMARY_KEEP | extra_keep if extra_keep else _EXEC_SUMMARY_KEEP
    drop = [c for c in df.columns if c not in keep]
    if drop:
        df = df.drop(columns=drop)
    return df


# Columns the Executive Dashboard actually uses.  Everything else is
# dropped per-batch during fetch, same pattern as Executive Summary.
_EXEC_DASHBOARD_KEEP: frozenset[str] = frozenset(
    {
        "id",
        "severity",
        "status",
        "detected",
        "category",
        "type",
        "projectId",
        "inKev",
        "in_kev",
        "hasKnownExploit",
        "has_known_exploit",
    }
)


def _prune_exec_dashboard(
    df: pd.DataFrame,
    extra_keep: frozenset[str] | None = None,
) -> pd.DataFrame:
    """Extract ``projectId`` and keep only the columns Executive Dashboard needs.

    This is called per-batch during fetch so large nested-dict columns
    (component, project, exploitInfo, …) are freed immediately.
    """
    df = df.copy()
    # Extract projectId from nested dict if not already flat
    if "projectId" not in df.columns and "project" in df.columns:
        df["projectId"] = df["project"].apply(
            lambda p: p.get("id", "") if isinstance(p, dict) else ""
        )

    keep = _EXEC_DASHBOARD_KEEP | extra_keep if extra_keep else _EXEC_DASHBOARD_KEEP
    drop = [c for c in df.columns if c not in keep]
    if drop:
        df = df.drop(columns=drop)
    return df


# Columns the Component Vulnerability Analysis transform actually uses.
# Everything else is dropped per-batch after flattening.
_CVA_KEEP: frozenset[str] = frozenset(
    {
        "id",
        "severity",
        "risk",
        "component.name",
        "component.version",
        "project.name",
        "inKev",
        "inVcKev",
        "hasKnownExploit",
        "epssPercentile",
        "reachabilityScore",
        "exploitInfo",
    }
)


def _prune_cva(
    df: pd.DataFrame,
    extra_keep: frozenset[str] | None = None,
) -> pd.DataFrame:
    """Keep only the columns Component Vulnerability Analysis needs.

    This is called per-batch (after flatten) during fetch so unused
    columns are freed immediately.
    """
    df = df.copy()
    keep = _CVA_KEEP | extra_keep if extra_keep else _CVA_KEEP
    drop = [c for c in df.columns if c not in keep]
    if drop:
        df = df.drop(columns=drop)
    return df


def _log_memory(logger: logging.Logger, label: str) -> None:
    """Log current process memory usage (RSS) for diagnostics."""
    try:
        usage = resource.getrusage(resource.RUSAGE_SELF)
        # macOS reports ru_maxrss in bytes; Linux reports in kilobytes
        if platform.system() == "Darwin":
            rss_mb = usage.ru_maxrss / (1024 * 1024)
        else:
            rss_mb = usage.ru_maxrss / 1024
        logger.info(f"[Memory] {label}: {rss_mb:.0f} MB peak RSS")
    except Exception:
        pass  # Don't let memory logging break the report


def _inject_folder_names_df(df: pd.DataFrame, pf_map: dict[str, str]) -> None:
    """Add folder_name column to DataFrame using project-to-folder mapping."""

    def _extract_pid(row: pd.Series) -> str:
        p = row.get("project") or row.get("projectId")
        if isinstance(p, dict):
            return str(p.get("id", ""))
        return str(p) if p else ""

    df["folder_name"] = df.apply(_extract_pid, axis=1).map(pf_map).fillna("")


def _inject_project_names_df(df: pd.DataFrame, project_map: dict[str, str]) -> None:
    """Add project_name column to DataFrame using project ID mapping."""

    def _extract_name(row: pd.Series) -> str:
        p = row.get("project") or row.get("projectId")
        if isinstance(p, dict):
            name = p.get("name")
            if name:
                return str(name)
            return project_map.get(str(p.get("id", "")), str(p.get("id", "")))
        return project_map.get(str(p), str(p)) if p else ""

    df["project_name"] = df.apply(_extract_name, axis=1)


# Finding type/category mapping for --finding-types flag
#
# The API has two filtering mechanisms:
#   - `type` query parameter: accepts cve, sast, thirdparty (single value only)
#   - `category` RSQL filter: accepts CVE, SAST_ANALYSIS, CREDENTIALS,
#     CONFIG_ISSUES, CRYPTO_MATERIAL (uppercase, per Swagger)
#
# For single-type requests, we use the `type` parameter (most efficient).
# For multi-type requests where all types map to a known API category, we use
# an RSQL `category=in=(...)` filter at the API level.  When some types lack a
# documented category (binary_sca, source_sca, thirdparty), we fall back to
# fetching all findings and post-filtering in Python.
CATEGORY_VALUES = {"credentials", "config_issues", "crypto_material", "sast_analysis"}
TYPE_VALUES = {"cve", "sast", "thirdparty", "binary_sca", "source_sca"}
CATEGORY_MAP = {
    "credentials": "CREDENTIALS",
    "config_issues": "CONFIG_ISSUES",
    "crypto_material": "CRYPTO_MATERIAL",
    "sast_analysis": "SAST_ANALYSIS",
    "cve": "CVE",
}
# Map CLI type names to API category values (uppercase, per Swagger).
# Only includes types that have a known API category.
# binary_sca, source_sca, thirdparty have no documented category value;
# their findings are matched by the type's own category field at runtime.
TYPE_TO_CATEGORY = {
    "cve": ["CVE"],
    "sast": ["SAST_ANALYSIS"],
    "thirdparty": [],
    "binary_sca": [],
    "source_sca": [],
}


def build_findings_type_params(finding_types: str) -> dict[str, Any]:
    """
    Build API parameters for finding type/category filtering.

    Args:
        finding_types: Comma-separated finding types from config

    Returns:
        Dict with 'type' and optional 'category_filter' keys
    """
    values = [v.strip().lower() for v in finding_types.split(",")]

    # "all" means no filtering — don't send type= param
    if "all" in values:
        return {"type": None, "category_filter": None}

    # Separate into categories (need RSQL filter) and simple types (use type param)
    categories = [
        v for v in values if v in CATEGORY_VALUES
    ]  # credentials, config_issues, etc.
    simple_types = [v for v in values if v in TYPE_VALUES]  # cve, sast, thirdparty

    # CVE-only: use type=cve directly (most efficient)
    if simple_types == ["cve"] and not categories:
        return {"type": "cve", "category_filter": None}

    # Single non-CVE type: use type param directly (API accepts sast, thirdparty)
    if (
        len(simple_types) == 1
        and simple_types[0] in ("sast", "thirdparty")
        and not categories
    ):
        return {"type": simple_types[0], "category_filter": None}

    # Multiple types, or binary_sca/source_sca (category-only), or named categories
    if (
        categories
        or len(simple_types) > 1
        or set(simple_types) & {"binary_sca", "source_sca"}
    ):
        # Build category + type sets.
        # Match on uppercase `category` field (CVE, SAST_ANALYSIS, etc.)
        # and lowercase `type` field (cve, sast, thirdparty) as fallback
        # for types without a documented category mapping.
        filter_categories: set[str] = set()
        filter_types: set[str] = set()
        for t in simple_types:
            mapped = TYPE_TO_CATEGORY.get(t, [])
            if mapped:
                filter_categories.update(mapped)
            else:
                # No category mapping — match on `type` field instead
                filter_types.add(t)
        for c in categories:
            mapped_cat = CATEGORY_MAP.get(c)
            if mapped_cat:
                filter_categories.add(mapped_cat)

        if not filter_categories and not filter_types:
            return {"type": None, "category_filter": None}

        # If ALL types map to known API categories, filter at the API level
        # using RSQL category== or category=in=() — avoids fetching extra data.
        if filter_categories and not filter_types:
            cats_sorted = sorted(filter_categories)
            if len(cats_sorted) == 1:
                cat_filter = f"category=={cats_sorted[0]}"
            else:
                cat_filter = f"category=in=({','.join(cats_sorted)})"
            return {"type": None, "category_filter": cat_filter}

        # Some types lack category mapping — must post-filter in Python
        return {
            "type": None,
            "category_filter": None,
            "post_filter_categories": filter_categories or None,
            "post_filter_types": filter_types or None,
        }

    # Default to cve only
    return {"type": "cve", "category_filter": None}


class ReportCancelled(Exception):
    """Raised when a running report is cancelled via the web UI."""


class ReportEngine:
    """Main engine for generating reports from recipes."""

    def __init__(
        self,
        config: Config,
        data_override: dict[str, Any] | None = None,
        cancel_event: threading.Event | None = None,
        on_recipe_complete: Callable[[int, int, str], None] | None = None,
        deployment_context: Any | None = None,
    ) -> None:
        """Initialize the report engine.

        Args:
            on_recipe_complete: Optional callback invoked after each recipe
                finishes.  Signature: ``(completed, total, recipe_name)``.
            deployment_context: Optional DeploymentContext for AI prompt customization.
        """
        self.config = config
        self._deployment_context = deployment_context
        self.logger = logging.getLogger(__name__)
        self._cancel_event = cancel_event
        self._on_recipe_complete = on_recipe_complete

        # Initialize cache
        self.cache = DataCache()

        # Initialize components
        self.api_client = APIClient(
            config,
            cache=self.cache,
            cache_ttl=getattr(config, "cache_ttl", 0),
        )
        # Secondary API client for cross-server version comparison
        if config.compare_domain and config.compare_auth_token:
            compare_config = Config(
                auth_token=config.compare_auth_token,
                domain=config.compare_domain,
                recipes_dir=config.recipes_dir,
                use_bundled_recipes=config.use_bundled_recipes,
                output_dir=config.output_dir,
                start_date=config.start_date,
                end_date=config.end_date,
                verbose=config.verbose,
                finding_types=config.finding_types,
                request_delay=config.request_delay,
                batch_size=config.batch_size,
            )
            self.compare_api_client: APIClient | None = APIClient(
                compare_config, cache=DataCache()
            )
        else:
            self.compare_api_client = None

        self.recipe_loader = RecipeLoader(
            config.recipes_dir,
            use_bundled=getattr(config, "use_bundled_recipes", True),
        )

        # Initialize transformer (only pandas is used)
        self.transformer = DataTransformer()
        # self.logger.info("Using Pandas transformer")

        self.renderer = ReportRenderer(
            config.output_dir, config, overwrite=getattr(config, "overwrite", False)
        )
        self.data_override = data_override

        # Cache for latest version IDs when current_version_only is enabled
        self._latest_version_ids: list[int] | None = None

        # In-memory cache: folder project IDs → latest version IDs (avoids re-resolving per report)
        self._folder_version_ids_cache: dict[str, list[int]] = {}

        # In-memory cache: findings data keyed by (endpoint, filter, finding_type, version_ids_hash)
        # Avoids redundant API fetches when multiple reports need the same data
        # Stores DataFrames (not list[dict]) to reduce peak memory.
        # Scans cache entries may still store list[dict] for internal reuse.
        self._findings_cache: dict[str, Any] = {}

        # In-memory cache: per-version findings keyed by (endpoint, version_id, finding_type)
        # Shared across Version Comparison within the same run
        self._version_findings_cache: dict[str, list[dict]] = {}

        # In-memory cache: per-project version lists from /projects/{id}/versions
        self._project_versions_cache: dict[str, list[dict]] = {}

        # Cache for _fetch_all_folders() — avoids redundant API calls per run
        self._all_folders_cache: list[dict] | None = None

        # Cache for _build_project_folder_map_from_projects() — avoids re-fetching per recipe
        self._project_map_cache: dict[str, str] | None = None

        # Files produced by the last run() call
        self.generated_files: list[str] = []

        # Resolved project name (populated by _validate_and_resolve_filters)
        self.resolved_project_name: str | None = None

        # Folder scoping state (populated by _resolve_folder_scope in run())
        self._folder_project_ids: set[str] | None = None
        self._project_folder_map: dict[str, str] = {}  # project_id -> folder_name
        self._folder_name: str | None = None
        self._folder_path: str | None = None  # e.g. "Division A / Medical Products"

    @staticmethod
    def _cache_key(*parts: str | None) -> str:
        """Build a normalized cache key from *parts*.

        ``None`` values are coerced to the empty string and parts are
        joined with ``\\x00`` so that values containing ``|`` can never
        collide with a different combination of parts.
        """
        return "\x00".join(p if p is not None else "" for p in parts)

    def _check_cancel(self) -> None:
        """Raise ``ReportCancelled`` if the cancel event has been set."""
        if self._cancel_event is not None and self._cancel_event.is_set():
            raise ReportCancelled("Report cancelled by user")

    def _validate_numeric_project_id(self, project_id: int) -> bool:
        """Check whether *project_id* is a valid project in the API.

        Returns ``True`` if the API returns project data for this ID,
        ``False`` otherwise (e.g. 404 or non-project entity).
        """
        try:
            url = f"{self.api_client.base_url}/public/v0/projects/{project_id}"
            resp = self.api_client.client.get(url)
            if resp.status_code == 404:
                return False
            resp.raise_for_status()
            data = resp.json()
            # Sanity-check: the response should contain an "id" key matching
            # the requested project ID.
            return isinstance(data, dict) and data.get("id") is not None
        except Exception:
            return False

    def _resolve_project_name(self, project_name: str) -> int | None:
        """
        Resolve a project name to its numeric API ID.

        Fetches the project list from the API and performs a case-insensitive
        name match.  Returns the numeric project ID, or None if not found.
        """
        from fs_report.models import QueryConfig, QueryParams

        projects_query = QueryConfig(
            endpoint="/public/v0/projects",
            params=QueryParams(limit=10000, archived=False, excluded=False),
        )
        projects = self.api_client.fetch_data(projects_query)

        for p in projects:
            name = p.get("name", "")
            if name.lower() == project_name.lower():
                return p.get("id")

        # Fuzzy hint: show close matches
        available = [p.get("name", "") for p in projects if p.get("name")]
        close = [n for n in available if project_name.lower() in n.lower()]
        if close:
            self.logger.info(f"Did you mean one of these? {close[:5]}")

        return None

    @staticmethod
    def _is_glob(value: str) -> bool:
        """Return True if *value* contains glob metacharacters (``*``, ``?``, ``[``)."""
        return any(c in value for c in ("*", "?", "["))

    def _resolve_project_glob(self, pattern: str) -> list[tuple[int, str]]:
        """Resolve a glob pattern against all project names.

        Returns a list of ``(id, name)`` tuples for every project whose name
        matches *pattern* (case-insensitive ``fnmatch``).  Also logs a warning
        if any project names collide when lowercased.
        """
        import fnmatch
        from collections import defaultdict

        from fs_report.models import QueryConfig, QueryParams

        projects_query = QueryConfig(
            endpoint="/public/v0/projects",
            params=QueryParams(limit=10000, archived=False, excluded=False),
        )
        projects = self.api_client.fetch_data(projects_query)

        # Case-collision check across ALL projects
        by_lower: dict[str, list[str]] = defaultdict(list)
        for p in projects:
            name = p.get("name", "")
            if name:
                by_lower[name.lower()].append(name)
        for lower, names in by_lower.items():
            distinct = sorted(set(names))
            if len(distinct) > 1:
                self.logger.warning(
                    f"Case collision: projects {distinct} map to the same "
                    f"lowercase name '{lower}'. Glob matching is case-insensitive "
                    "and will match all of them."
                )

        # Match projects against the glob pattern
        pat_lower = pattern.lower()
        matches: list[tuple[int, str]] = []
        for p in projects:
            name = p.get("name", "")
            pid = p.get("id")
            if name and pid is not None and fnmatch.fnmatch(name.lower(), pat_lower):
                matches.append((pid, name))

        return matches

    def _resolve_project_id_to_name(self, project_id: int) -> str | None:
        """Resolve a numeric project ID to its display name.

        Returns the project name, or None if the lookup fails.
        """
        try:
            url = f"{self.api_client.base_url}/public/v0/projects/{project_id}"
            resp = self.api_client.client.get(url)
            resp.raise_for_status()
            data = resp.json()
            return data.get("name") if isinstance(data, dict) else None
        except Exception:
            return None

    def _resolve_version_name(self, project_id: int, version_name: str) -> int | None:
        """Resolve a version name to its numeric API ID within a project.

        Fetches the version list for *project_id* and performs a
        case-insensitive name match.  Returns the numeric version ID,
        or None if not found.
        """
        try:
            url = (
                f"{self.api_client.base_url}"
                f"/public/v0/projects/{project_id}/versions"
            )
            resp = self.api_client.client.get(url)
            resp.raise_for_status()
            versions = resp.json()
            if not isinstance(versions, list):
                return None
        except Exception as e:
            self.logger.debug(f"Error fetching versions for project {project_id}: {e}")
            return None

        for v in versions:
            name = v.get("version") or v.get("name") or ""
            if str(name).lower() == version_name.lower():
                vid: int | None = v.get("id")
                return vid

        # Fuzzy hint: show close matches
        available = [
            str(v.get("version") or v.get("name") or "")
            for v in versions
            if v.get("version") or v.get("name")
        ]
        close = [n for n in available if version_name.lower() in n.lower()]
        if close:
            self.logger.info(f"Did you mean one of these versions? {close[:5]}")

        return None

    def _fetch_all_folders(self) -> list[dict]:
        """Fetch all folders from the API (cached per run)."""
        if self._all_folders_cache is not None:
            return self._all_folders_cache

        from fs_report.models import QueryConfig, QueryParams

        folders_query = QueryConfig(
            endpoint="/public/v0/folders",
            params=QueryParams(limit=10000),
        )
        self._all_folders_cache = self.api_client.fetch_data(folders_query)
        return self._all_folders_cache

    def _resolve_folder(self, folder_input: str) -> dict | None:
        """
        Resolve a folder name or ID to its API record.
        Returns the full folder dict, or None if not found.
        """
        folders = self._fetch_all_folders()

        # Try exact ID match first
        for f in folders:
            if str(f.get("id", "")) == folder_input:
                return f

        # Case-insensitive name match
        for f in folders:
            if f.get("name", "").lower() == folder_input.lower():
                return f

        # Fuzzy hint
        available = [f.get("name", "") for f in folders if f.get("name")]
        close = [n for n in available if folder_input.lower() in n.lower()]
        if close:
            self.logger.info(f"Did you mean one of these folders? {close[:5]}")

        return None

    def _collect_folder_tree(
        self, target_folder_id: str, all_folders: list[dict] | None = None
    ) -> tuple[set[str], dict[str, str], list[dict], dict[str, str]]:
        """
        Walk the folder tree starting from *target_folder_id* and collect:
        1. All descendant folder IDs (including the target itself).
        2. project_id -> folder_name mapping for every project in those folders.
        3. The list of subfolder dicts (for logging/display).

        Returns (folder_ids, project_folder_map, subfolder_list).
        """
        if all_folders is None:
            all_folders = self._fetch_all_folders()

        folder_by_id: dict[str, dict] = {str(f["id"]): f for f in all_folders}
        children_map: dict[str, list[str]] = {}
        for f in all_folders:
            parent = f.get("parentFolderId")
            if parent:
                children_map.setdefault(str(parent), []).append(str(f["id"]))

        # BFS to collect all descendant folder IDs
        from collections import deque

        queue: deque[str] = deque([target_folder_id])
        all_folder_ids: set[str] = set()
        subfolder_list: list[dict] = []
        while queue:
            fid = queue.popleft()
            all_folder_ids.add(fid)
            if fid != target_folder_id:
                subfolder_list.append(folder_by_id.get(fid, {}))
            for child_id in children_map.get(fid, []):
                if child_id not in all_folder_ids:
                    queue.append(child_id)

        # For each folder, fetch its projects
        from fs_report.models import QueryConfig, QueryParams

        project_folder_map: dict[str, str] = {}
        project_name_to_id: dict[str, str] = {}
        all_project_ids: set[str] = set()

        for fid in all_folder_ids:
            folder_name = folder_by_id.get(fid, {}).get("name", "Unknown")
            try:
                projects_query = QueryConfig(
                    endpoint=f"/public/v0/folders/{fid}/projects",
                    params=QueryParams(limit=10000, archived=False, excluded=False),
                )
                projects = self.api_client.fetch_data(projects_query)
                for p in projects:
                    pid = str(p.get("id", ""))
                    pname = p.get("name", "")
                    if pid:
                        all_project_ids.add(pid)
                        project_folder_map[pid] = folder_name
                        if pname:
                            project_name_to_id[pname.lower()] = pid
            except Exception as e:
                self.logger.warning(
                    f"Error fetching projects for folder '{folder_name}' ({fid}): {e}"
                )

        self.logger.info(
            f"Folder tree: {len(all_folder_ids)} folder(s), "
            f"{len(all_project_ids)} project(s)"
        )

        return all_project_ids, project_folder_map, subfolder_list, project_name_to_id

    def _build_folder_path(
        self, folder_id: str, all_folders: list[dict] | None = None
    ) -> str:
        """Build a breadcrumb-style path for a folder, e.g. 'Division A / Medical Products'."""
        if all_folders is None:
            all_folders = self._fetch_all_folders()
        folder_by_id = {str(f["id"]): f for f in all_folders}

        parts: list[str] = []
        current_id: str | None = folder_id
        seen: set[str] = set()
        while current_id and current_id not in seen:
            seen.add(current_id)
            folder = folder_by_id.get(current_id)
            if not folder:
                break
            parts.append(folder.get("name", "Unknown"))
            parent = folder.get("parentFolderId")
            current_id = str(parent) if parent else None

        parts.reverse()
        return " / ".join(parts)

    def _resolve_folder_scope(self) -> bool:
        """
        Called from run() when config.folder_filter is set.
        Resolves folder, walks tree, collects project IDs & mapping.
        Returns True on success, False on failure.
        """
        folder_input = self.config.folder_filter
        if not folder_input:
            return True

        folder = self._resolve_folder(folder_input)
        if not folder:
            self.logger.error(
                f"Could not resolve folder '{folder_input}'. "
                "Use 'fs-report list-folders' to see available folders."
            )
            return False

        folder_id = str(folder["id"])
        self._folder_name = folder.get("name", folder_input)

        # Fetch all folders once (used for tree walk and path building)
        all_folders = self._fetch_all_folders()
        self._folder_path = self._build_folder_path(folder_id, all_folders)

        project_ids, project_folder_map, subfolders, project_name_to_id = (
            self._collect_folder_tree(folder_id, all_folders)
        )

        self._folder_project_ids = project_ids
        self._project_folder_map = project_folder_map

        self.logger.info(
            f"Folder scope: '{self._folder_path}' — "
            f"{len(subfolders)} subfolder(s), {len(project_ids)} project(s)"
        )

        # If --project is also specified, validate it's within the folder
        if self.config.project_filter:
            pid = str(self.config.project_filter)
            # The filter may be a numeric ID or a project name
            if pid not in project_ids:
                # Try resolving by name (case-insensitive)
                resolved = project_name_to_id.get(pid.lower())
                if resolved:
                    pid = resolved
                else:
                    self.logger.error(
                        f"Project '{self.config.project_filter}' is not in folder "
                        f"'{self._folder_name}' or its subfolders."
                    )
                    return False

        return True

    def _build_project_folder_map_from_projects(self) -> dict[str, str]:
        """
        Build a project_id -> folder_name mapping by fetching projects.
        Used when --folder is not specified, to still populate folder_name
        from the ProjectV0.folder field.  Cached per run.
        """
        if self._project_map_cache is not None:
            return self._project_map_cache

        from fs_report.models import QueryConfig, QueryParams

        projects_query = QueryConfig(
            endpoint="/public/v0/projects",
            params=QueryParams(limit=10000, archived=False, excluded=False),
        )
        projects = self.api_client.fetch_data(projects_query)

        pf_map: dict[str, str] = {}
        for p in projects:
            pid = str(p.get("id", ""))
            folder = p.get("folder")
            if pid and folder and isinstance(folder, dict):
                pf_map[pid] = folder.get("name", "")
        self._project_map_cache = pf_map
        return pf_map

    def _get_latest_version_ids(self) -> list[int]:
        """Fetch latest version IDs for all projects (cached)."""
        if self._latest_version_ids is not None:
            return self._latest_version_ids

        self.logger.info("Fetching latest version IDs for all projects...")

        # Fetch all projects — _get_latest_version_ids_for_projects will
        # also fetch them, but the DataCache / SQLite cache makes the
        # second call essentially free.
        from fs_report.models import QueryConfig, QueryParams

        projects_query = QueryConfig(
            endpoint="/public/v0/projects",
            params=QueryParams(limit=10000, archived=False, excluded=False),
        )
        projects = self.api_client.fetch_all_with_resume(projects_query)
        self.logger.info(f"Found {len(projects)} projects")

        project_ids = [str(p["id"]) for p in projects if p.get("id")]

        # Try extracting version IDs from the data we already have.
        # This avoids a second fetch when the list response (or cache)
        # already includes defaultBranch.
        requested_set = set(project_ids)
        version_ids = self._extract_version_ids_from_projects(projects, requested_set)
        if not version_ids and project_ids:
            # Batch data lacked defaultBranch — delegate to the fallback
            # path which does per-project detail calls.
            version_ids = self._get_latest_version_ids_for_projects(project_ids)
        else:
            self.logger.info(f"Resolved {len(version_ids)} version IDs from batch data")

        self.logger.info(f"Found {len(version_ids)} latest version IDs")
        self._latest_version_ids = version_ids
        return version_ids

    def _get_latest_version_ids_for_projects(self, project_ids: list) -> list[int]:
        """Fetch the current (latest) version ID for each given project.

        Uses the project's defaultBranch.latestVersion.id which is the
        authoritative current version on the platform.

        Strategy:
        1. Try a single batch call to /public/v0/projects and extract
           defaultBranch.latestVersion.id for each requested project.
           This is fast and cache-friendly.
        2. If the batch response lacks defaultBranch data (some API versions
           omit it from list responses, or the SQLite cache may have been
           populated before this field was stored), fall back to individual
           /public/v0/projects/{id} calls with throttling.

        Results are cached in-memory so subsequent reports in the same run
        skip the API call entirely.
        """
        # Build a stable cache key from sorted project IDs
        cache_key = ",".join(str(pid) for pid in sorted(project_ids))
        if cache_key in self._folder_version_ids_cache:
            cached = self._folder_version_ids_cache[cache_key]
            self.logger.info(
                f"Using cached version IDs ({len(cached)} versions "
                f"for {len(project_ids)} projects)"
            )
            return cached

        from fs_report.models import QueryConfig, QueryParams

        # ------------------------------------------------------------------
        # Step 1: Try batch fetch
        # ------------------------------------------------------------------
        projects_query = QueryConfig(
            endpoint="/public/v0/projects",
            params=QueryParams(limit=10000, archived=False, excluded=False),
        )
        all_projects = self.api_client.fetch_all_with_resume(
            projects_query, show_progress=True
        )
        self.logger.info(
            f"Resolving latest versions for {len(project_ids)} projects "
            f"(from {len(all_projects)} total projects)"
        )

        requested_ids = {str(pid) for pid in project_ids}
        version_ids = self._extract_version_ids_from_projects(
            all_projects, requested_ids
        )

        # ------------------------------------------------------------------
        # Step 2: Fallback — per-project detail calls when batch yields nothing
        # ------------------------------------------------------------------
        if not version_ids and project_ids:
            self.logger.info(
                "Batch project list did not include defaultBranch data; "
                "falling back to per-project API calls "
                f"({len(project_ids)} projects, "
                f"delay={self.config.request_delay}s)"
            )
            version_ids = self._fetch_version_ids_per_project(project_ids)

        self._folder_version_ids_cache[cache_key] = version_ids
        return version_ids

    # ------------------------------------------------------------------
    # Helpers for _get_latest_version_ids_for_projects
    # ------------------------------------------------------------------

    @staticmethod
    def _extract_version_ids_from_projects(
        projects: list[dict], requested_ids: set[str]
    ) -> list[int]:
        """Extract defaultBranch.latestVersion.id from a list of project dicts."""
        logger = logging.getLogger(__name__)
        version_ids: list[int] = []
        for project in projects:
            pid = str(project.get("id", ""))
            if pid not in requested_ids:
                continue
            default_branch = project.get("defaultBranch") or {}
            latest_version = (
                default_branch.get("latestVersion") or {}
                if isinstance(default_branch, dict)
                else {}
            )
            version_id = (
                latest_version.get("id") if isinstance(latest_version, dict) else None
            )
            if version_id:
                version_ids.append(version_id)
            else:
                pname = project.get("name", pid)
                logger.info(
                    f"Project '{pname}' (id={pid}) has no defaultBranch.latestVersion; skipping"
                )
        return version_ids

    def _fetch_version_ids_per_project(self, project_ids: list) -> list[int]:
        """Fetch defaultBranch.latestVersion.id one project at a time.

        Used as a fallback when the batch /projects list doesn't include
        branch data. Respects --request-delay for throttling.
        """
        from tqdm import tqdm

        delay = max(0.5, self.config.request_delay)
        version_ids: list[int] = []
        for pid in tqdm(
            project_ids,
            desc="Fetching latest versions",
            unit=" projects",
            leave=True,
        ):
            try:
                url = f"{self.api_client.base_url}/public/v0/projects/{pid}"
                resp = self.api_client.client.get(url)
                resp.raise_for_status()
                data = resp.json()
                default_branch = data.get("defaultBranch") or {}
                latest_version = (
                    default_branch.get("latestVersion") or {}
                    if isinstance(default_branch, dict)
                    else {}
                )
                vid = (
                    latest_version.get("id")
                    if isinstance(latest_version, dict)
                    else None
                )
                if vid:
                    version_ids.append(vid)
                else:
                    self.logger.debug(
                        f"No defaultBranch.latestVersion for project {pid}"
                    )
            except Exception:
                self.logger.warning(
                    f"Failed to fetch version for project {pid}",
                    exc_info=True,
                )
            time.sleep(delay)
        return version_ids

    def _split_and_cache_by_version(
        self,
        records: list[dict],
        entity_type: str,
        finding_type: str = "",
        category_filter: str | None = None,
        batch_version_ids: list[int] | None = None,
    ) -> None:
        """Split batch results by projectVersion.id and cache each version individually.

        Populates both the in-memory ``_version_findings_cache`` and, when a SQLite
        cache is available, creates per-version SQLite entries using the same hash
        that a ``projectVersion=={vid}`` query would produce.  This means a later
        per-version fetch from Version Comparison can hit the
        SQLite cache entry created by a batch fetch in an earlier run.

        When ``batch_version_ids`` is provided, versions that returned zero records
        are cached as empty lists.  Without this, empty versions would be re-fetched
        on every run (the cache only stored versions that had data).

        Args:
            records: Raw API records (findings or components).
            entity_type: 'findings' or 'components'.
            finding_type: API finding type (e.g. 'cve'). Only used for findings.
            category_filter: Optional RSQL category filter. Only used for findings.
            batch_version_ids: Version IDs that were queried in this batch.
                If provided, any IDs not present in the results are cached as empty.
        """
        from collections import defaultdict

        by_version: dict[str, list[dict]] = defaultdict(list)
        for rec in records:
            pv = rec.get("projectVersion") or {}
            vid = str(pv.get("id", ""))
            if vid:
                by_version[vid].append(rec)

        # If we know which version IDs were in the batch, add empty entries
        # for versions that returned no records so they are cached as "checked".
        if batch_version_ids is not None:
            for batch_vid in batch_version_ids:
                svid = str(batch_vid)
                if svid not in by_version:
                    by_version[svid] = []

        stored_versions = 0
        for vid, version_records in by_version.items():
            if entity_type == "findings":
                cache_key = self._cache_key(
                    "findings", vid, finding_type, category_filter
                )
            else:
                cache_key = self._cache_key("components", vid)
            if cache_key not in self._version_findings_cache:
                self._version_findings_cache[cache_key] = version_records
                stored_versions += 1

                # Warm SQLite cache with a per-version entry
                if self.api_client.sqlite_cache and self.api_client.cache_ttl > 0:
                    endpoint = f"/public/v0/{entity_type}"
                    # Build params that match what a per-version query would produce
                    filter_str = f"projectVersion=={vid}"
                    if category_filter:
                        filter_str = f"{filter_str};{category_filter}"
                    params: dict[str, Any] = {"filter": filter_str, "limit": 10000}
                    if entity_type == "findings" and finding_type:
                        params["finding_type"] = finding_type
                    if entity_type == "findings":
                        params["archived"] = False
                        params["excluded"] = False

                    try:
                        if not self.api_client.sqlite_cache.is_cache_valid(
                            endpoint, params, self.api_client.cache_ttl
                        ):
                            qh = self.api_client.sqlite_cache.start_fetch(
                                endpoint, params, self.api_client.cache_ttl
                            )
                            self.api_client.sqlite_cache.store_records(
                                qh, endpoint, version_records
                            )
                            self.api_client.sqlite_cache.complete_fetch(qh)
                    except Exception:
                        self.logger.warning(
                            f"Failed to warm SQLite cache for version {vid}; continuing without cache"
                        )

        if stored_versions:
            self.logger.debug(
                f"Split batch into {stored_versions} per-version {entity_type} cache entries "
                f"(skipped {len(by_version) - stored_versions} already cached)"
            )

    def _check_version_in_cache(
        self,
        vid: str,
        entity_type: str,
        finding_type: str = "",
        category_filter: str | None = None,
    ) -> list[dict] | None:
        """Check both in-memory and SQLite caches for per-version data.

        Returns cached records or None if not found.
        """
        if entity_type == "findings":
            cache_key = self._cache_key("findings", vid, finding_type, category_filter)
        else:
            cache_key = self._cache_key("components", vid)

        # 1. In-memory cache
        if cache_key in self._version_findings_cache:
            return self._version_findings_cache[cache_key]

        # 2. SQLite cache
        if self.api_client.sqlite_cache and self.api_client.cache_ttl > 0:
            endpoint = f"/public/v0/{entity_type}"
            filter_str = f"projectVersion=={vid}"
            if category_filter:
                filter_str = f"{filter_str};{category_filter}"
            params: dict[str, Any] = {"filter": filter_str, "limit": 10000}
            if entity_type == "findings" and finding_type:
                params["finding_type"] = finding_type
            if entity_type == "findings":
                params["archived"] = False
                params["excluded"] = False

            if self.api_client.sqlite_cache.is_cache_valid(
                endpoint, params, self.api_client.cache_ttl
            ):
                cached = self.api_client.sqlite_cache.get_cached_data(
                    endpoint, params, allow_empty=True
                )
                if cached is not None:
                    # Promote to in-memory cache for fast re-reads
                    self._version_findings_cache[cache_key] = cached
                    return cached

        return None

    def _get_findings_for_versions(
        self,
        version_ids: list,
        finding_type: str,
        category_filter: str | None = None,
        entity_type: str = "findings",
        on_records: "Callable[[list[dict]], None] | None" = None,
    ) -> list[dict]:
        """Reassemble data from per-version cache, fetching any missing versions first.

        This is the main entry point for findings-based reports to get data for a
        set of versions.  It checks in-memory and SQLite caches per-version, then
        batch-fetches any missing versions from the API.

        Args:
            version_ids: Version IDs to retrieve data for.
            finding_type: API finding type (e.g. 'cve').
            category_filter: Optional RSQL category filter.
            entity_type: 'findings' or 'components'.
            on_records: Optional callback invoked with each batch of records
                as they become available (cached records first, then each
                API batch).  Used to start background work (e.g. NVD
                lookups) in parallel with the remaining fetch.

        Returns:
            Merged list of records across all requested versions.
        """
        all_records: list[dict] = []
        missing: list[int] = []

        for vid in version_ids:
            cached = self._check_version_in_cache(
                str(vid), entity_type, finding_type, category_filter
            )
            if cached is not None:
                all_records.extend(cached)
            else:
                missing.append(vid)

        # Notify callback with cached records so background work can start
        # while API batches are still being fetched.
        if on_records and all_records:
            on_records(all_records)

        cached_count = len(version_ids) - len(missing)
        if missing:
            self.logger.info(
                f"{cached_count}/{len(version_ids)} versions from cache, "
                f"fetching {len(missing)} from API ({entity_type})"
            )
            from fs_report.models import QueryConfig, QueryParams

            # Build base query WITHOUT date filters — entity cache stores all data per version
            filters: list[str] = []
            if category_filter:
                filters.append(category_filter)
            combined_filter = ";".join(filters) if filters else None

            base_query = QueryConfig(
                endpoint=f"/public/v0/{entity_type}",
                params=QueryParams(
                    limit=10000,
                    filter=combined_filter,
                    finding_type=finding_type if entity_type == "findings" else None,
                    archived=False if entity_type == "findings" else None,
                    excluded=False if entity_type == "findings" else None,
                ),
            )
            if entity_type == "components":
                # Components need type!=file filter
                comp_filters = ["type!=file"]
                if combined_filter:
                    comp_filters.append(combined_filter)
                base_query.params.filter = ";".join(comp_filters)

            new_records = self._fetch_with_version_batching(
                base_query,
                sorted(missing),
                finding_type=finding_type,
                category_filter=category_filter,
                entity_type=entity_type,
                on_records=on_records,
            )
            all_records.extend(new_records)
        else:
            self.logger.info(
                f"All {len(version_ids)} versions served from cache ({entity_type})"
            )

        return all_records

    def _volume_based_cooldown(self, records_in_batch: int) -> float:
        """Return a cooldown duration (seconds) scaled by data volume."""
        if records_in_batch > 50_000:
            return max(90.0, self.config.request_delay * 15)
        if records_in_batch > 20_000:
            return max(45.0, self.config.request_delay * 10)
        if records_in_batch > 5_000:
            return max(15.0, self.config.request_delay * 5)
        return max(5.0, self.config.request_delay * 2)

    def _fetch_with_version_batching(
        self,
        base_query: "QueryConfig",
        version_ids: list[int],
        batch_size: int | None = None,
        finding_type: str = "",
        category_filter: str | None = None,
        entity_type: str = "findings",
        on_records: "Callable[[list[dict]], None] | None" = None,
    ) -> list[dict]:
        """Fetch data in batches, filtering by version IDs.

        Before each batch, filters out versions already in per-version cache.
        After each batch, splits results and stores per-version entries.
        Uses ``skip_cache_store=True`` to avoid duplicating batch-level SQLite
        entries (per-version storage handles it).

        batch_size defaults to ``self.config.batch_size`` (CLI: ``--batch-size``,
        default 5).  Kept small to avoid overloading the server on large
        instances.  Each batch is followed by an adaptive cooldown that scales
        with the number of records fetched.
        """
        if batch_size is None:
            batch_size = self.config.batch_size
        from tqdm import tqdm

        all_results = []

        # Filter out already-cached versions first
        uncached_ids = [
            v
            for v in version_ids
            if self._check_version_in_cache(
                str(v), entity_type, finding_type, category_filter
            )
            is None
        ]

        # Collect cached results
        cached_count = len(version_ids) - len(uncached_ids)
        if cached_count > 0:
            for vid in version_ids:
                cached = self._check_version_in_cache(
                    str(vid), entity_type, finding_type, category_filter
                )
                if cached is not None:
                    all_results.extend(cached)
            self.logger.debug(
                f"Version batching: {cached_count} versions served from cache"
            )

        if not uncached_ids:
            self.logger.debug("Version batching: all versions served from cache")
            return all_results

        total_batches = (len(uncached_ids) + batch_size - 1) // batch_size

        # Log the base filter being used
        self.logger.debug(
            f"Version batching with base filter: {base_query.params.filter}"
        )

        # Split uncached version IDs into batches with progress bar
        # Track elevated cooldown state: after server errors, keep cooldowns
        # elevated for several batches so the server gets sustained recovery.
        elevated_batches_remaining = 0
        elevated_cooldown = 0.0

        with tqdm(
            range(0, len(uncached_ids), batch_size),
            desc="Fetching version batches",
            unit=" batches",
            total=total_batches,
            leave=False,
        ) as pbar:
            for i in pbar:
                batch_ids = uncached_ids[i : i + batch_size]

                # Build version filter
                version_filter = (
                    f"projectVersion=in=({','.join(str(v) for v in batch_ids)})"
                )

                # Combine with existing filter (MUST preserve base filter like type!=file)
                if base_query.params.filter:
                    combined_filter = f"{base_query.params.filter};{version_filter}"
                else:
                    combined_filter = version_filter

                self.logger.debug(
                    f"Batch {i//batch_size + 1}/{total_batches} filter: {combined_filter[:100]}..."
                )

                # Create batch query
                from fs_report.models import QueryConfig, QueryParams

                batch_query = QueryConfig(
                    endpoint=base_query.endpoint,
                    params=QueryParams(
                        limit=base_query.params.limit,
                        filter=combined_filter,
                        finding_type=base_query.params.finding_type,
                        archived=False if entity_type == "findings" else None,
                        excluded=False if entity_type == "findings" else None,
                    ),
                )

                # Use skip_cache_store to avoid duplicating batch-level SQLite entries
                batch_results = self.api_client.fetch_all_with_resume(
                    batch_query,
                    show_progress=False,
                    skip_cache_store=True,
                )

                # Split and cache per-version (in-memory + SQLite).
                # Pass batch_ids so versions with 0 results are cached as
                # empty — otherwise they'd be re-fetched on every run.
                self._split_and_cache_by_version(
                    batch_results,
                    entity_type,
                    finding_type,
                    category_filter,
                    batch_version_ids=batch_ids,
                )

                # Trim factors in-memory to avoid OOM on large instances.
                # The full factors are already persisted (trimmed) in SQLite;
                # here we apply the same trim so the in-memory accumulation
                # doesn't hold multi-MB factors arrays per finding.
                for record in batch_results:
                    raw_factors = record.get("factors")
                    if isinstance(raw_factors, list) and raw_factors:
                        record["factors"] = _trim_factors(raw_factors)

                all_results.extend(batch_results)
                if on_records and batch_results:
                    on_records(batch_results)
                pbar.set_postfix({"records": len(all_results)})

                # Adaptive cooldown between batches — scale with data volume
                if i + batch_size < len(uncached_ids):
                    records_in_batch = len(batch_results)
                    # Check if the API client hit any retries during this batch
                    retries_in_batch = getattr(self.api_client, "last_fetch_retries", 0)
                    if retries_in_batch > 0:
                        # Server was struggling — give it a long recovery and
                        # keep cooldowns elevated for the next several batches
                        # so the server gets sustained breathing room.
                        cooldown = max(120.0, self.config.request_delay * 20)
                        elevated_cooldown = max(60.0, self.config.request_delay * 10)
                        elevated_batches_remaining = 5
                        self.logger.warning(
                            f"Server had {retries_in_batch} retries in batch; "
                            f"extended cooldown {cooldown:.0f}s "
                            f"(next {elevated_batches_remaining} batches "
                            f"will use ≥{elevated_cooldown:.0f}s)"
                        )
                    elif elevated_batches_remaining > 0:
                        # Still in recovery window from a previous retry event
                        elevated_batches_remaining -= 1
                        volume_cooldown = self._volume_based_cooldown(records_in_batch)
                        cooldown = max(volume_cooldown, elevated_cooldown)
                        self.logger.info(
                            f"Recovery cooldown {cooldown:.0f}s "
                            f"({elevated_batches_remaining} elevated batches left)"
                        )
                    else:
                        cooldown = self._volume_based_cooldown(records_in_batch)
                    batches_remaining = total_batches - (i // batch_size + 1)
                    est_minutes = (batches_remaining * cooldown) / 60
                    self.logger.info(
                        f"Batch {i // batch_size + 1}/{total_batches}: "
                        f"{records_in_batch:,} records. "
                        f"Cooling down {cooldown:.0f}s. "
                        f"~{est_minutes:.0f} min remaining"
                    )
                    time.sleep(cooldown)

        return all_results

    def run(self) -> "RunResult":
        """Run the complete report generation process. Returns RunResult with success status."""
        _fail = RunResult(success=False)
        self.logger.info("Starting report generation...")

        # Load recipes
        recipes = self.recipe_loader.load_recipes()
        if not recipes:
            self.logger.warning("No recipes found in recipes directory")
            return _fail

        # Filter recipes if specific recipe is requested
        # Check both config.recipe_filter and recipe_loader.recipe_filter
        # (CLI sets recipe_loader.recipe_filter directly, not config.recipe_filter)
        explicit_recipe_requested = self.config.recipe_filter or getattr(
            self.recipe_loader, "recipe_filter", None
        )
        if self.config.recipe_filter:
            filtered_recipes = [
                r
                for r in recipes
                if r.name.lower() == self.config.recipe_filter.lower()
            ]
            if not filtered_recipes:
                self.logger.error(
                    f"Recipe '{self.config.recipe_filter}' not found. Available recipes: {[r.name for r in recipes]}"
                )
                return _fail
            recipes = filtered_recipes
            self.logger.info(f"Filtered to {len(recipes)} recipe(s)")
        elif not explicit_recipe_requested:
            # Exclude recipes with auto_run=False only when no specific recipe is requested
            auto_run_recipes = [r for r in recipes if getattr(r, "auto_run", True)]
            skipped = len(recipes) - len(auto_run_recipes)
            if skipped > 0:
                self.logger.info(
                    f"Skipping {skipped} recipe(s) with auto_run=false (use --recipe to run them)"
                )
            recipes = auto_run_recipes

        # Sort recipes by execution_order to maximize cache reuse
        # Lower order = runs first (e.g., Scan Analysis fetches scans that other reports reuse)
        recipes = sorted(recipes, key=lambda r: r.execution_order)

        self.logger.info(f"Loaded {len(recipes)} recipes")

        # Resolve folder scope first (may narrow down project set)
        if self.config.folder_filter and not self.data_override:
            if not self._resolve_folder_scope():
                return _fail

        # Resolve project name to numeric ID if needed (API filters require numeric IDs)
        if self.config.project_filter and not self.data_override:
            try:
                pid = int(self.config.project_filter)
                # Numeric ID — validate it actually refers to a project
                if not self._validate_numeric_project_id(pid):
                    self.logger.error(
                        f"No project found with ID {pid}. "
                        "This may be a project *version* ID rather than a project ID. "
                        "If so, use --version (with --project) or "
                        "--baseline-version / --current-version instead.\n"
                        "Use 'fs-report list-projects' to see available projects."
                    )
                    return _fail
                # Resolve numeric ID to project name for display
                self.resolved_project_name = self._resolve_project_id_to_name(pid)
            except ValueError:
                filter_value = self.config.project_filter
                if self._is_glob(filter_value):
                    matches = self._resolve_project_glob(filter_value)
                    if not matches:
                        self.logger.error(
                            f"No projects matched glob pattern '{filter_value}'. "
                            "Use 'fs-report list-projects' to see available projects."
                        )
                        return _fail
                    elif len(matches) == 1:
                        mid, mname = matches[0]
                        self.logger.info(
                            f"Glob '{filter_value}' matched 1 project: {mname} (ID {mid})"
                        )
                        self.resolved_project_name = mname
                        self.config.project_filter = str(mid)
                    else:
                        names = [m[1] for m in matches]
                        self.logger.info(
                            f"Glob '{filter_value}' matched {len(matches)} projects: "
                            + ", ".join(sorted(names))
                        )
                        self._folder_project_ids = {str(m[0]) for m in matches}
                        self.config.project_filter = None
                else:
                    original_name = filter_value
                    resolved_id = self._resolve_project_name(filter_value)
                    if resolved_id:
                        self.logger.info(
                            f"Resolved project '{filter_value}' to ID {resolved_id}"
                        )
                        self.resolved_project_name = original_name
                        self.config.project_filter = str(resolved_id)
                    else:
                        self.logger.error(
                            f"Could not resolve project name '{filter_value}'. "
                            "Use 'fs-report list-projects' to see available projects."
                        )
                        return _fail

        # Reject version filter with multi-project glob
        if (
            self.config.version_filter
            and not self.data_override
            and self._folder_project_ids
            and not self.config.project_filter
            and not self.config.folder_filter
        ):
            self.logger.error(
                "Version filter is not supported with project glob patterns. "
                "Use an exact project name or ID with --version."
            )
            return _fail

        # Resolve version name to numeric ID if needed (API filters require numeric IDs)
        if self.config.version_filter and not self.data_override:
            if not self.config.project_filter:
                self.logger.error(
                    "Version filter requires a project filter. "
                    "Use --project-filter to specify the project."
                )
                return _fail
            try:
                int(self.config.version_filter)
                # Already a numeric ID — no resolution needed
            except ValueError:
                try:
                    project_id = int(self.config.project_filter)
                except ValueError:
                    # project_filter should already be resolved above
                    self.logger.error(
                        f"Cannot resolve version name '{self.config.version_filter}': "
                        f"project filter '{self.config.project_filter}' is not a numeric ID."
                    )
                    return _fail
                resolved_id = self._resolve_version_name(
                    project_id, self.config.version_filter
                )
                if resolved_id:
                    self.logger.info(
                        f"Resolved version '{self.config.version_filter}' to ID {resolved_id}"
                    )
                    self.config.version_filter = str(resolved_id)
                else:
                    self.logger.error(
                        f"Could not resolve version name '{self.config.version_filter}' "
                        f"in project {self.config.project_filter}. "
                        "Use 'fs-report list-versions <project>' to see available versions."
                    )
                    return _fail

        # Process each recipe
        all_succeeded = True
        generated_files: list[str] = []
        recipe_results: list[RecipeResult] = []
        total = len(recipes)
        for idx, recipe in enumerate(recipes, 1):
            self._check_cancel()
            try:
                self.logger.info(f"[{idx}/{total}] Generating: {recipe.name} ...")

                # Require --project for recipes that declare requires_project
                if (
                    getattr(recipe, "requires_project", False) is True
                    and not self.config.project_filter
                ):
                    self.logger.error(
                        f"'{recipe.name}' requires a --project filter. "
                        "Use --project <name-or-id> to scope to a single project."
                    )
                    all_succeeded = False
                    continue

                # Require --cve for recipes that declare requires_cve
                if getattr(recipe, "requires_cve", False) is True and not getattr(
                    self.config, "cve_filter", None
                ):
                    self.logger.error(
                        f"'{recipe.name}' requires a --cve filter. "
                        "Use --cve <CVE-ID> to specify one or more CVEs."
                    )
                    all_succeeded = False
                    continue

                # Folder-scoped Remediation Package: iterate over projects
                if (
                    recipe.name == "Remediation Package"
                    and not self.config.project_filter
                    and self._folder_project_ids
                ):
                    folder_ok = self._run_remediation_folder(
                        recipe,
                        sorted(self._folder_project_ids),
                        generated_files,
                        recipe_results,
                    )
                    if not folder_ok:
                        all_succeeded = False
                    continue

                # Remediation Package without --project or --folder
                if (
                    recipe.name == "Remediation Package"
                    and not self.config.project_filter
                    and not self._folder_project_ids
                ):
                    self.logger.error(
                        "'Remediation Package' requires --project or --folder. "
                        "Use --project <name-or-id> or --folder <name-or-id>."
                    )
                    all_succeeded = False
                    continue

                # Scoped output naming: when --component or --cve is set, suffix
                # the recipe name for the output directory.
                _scoped_recipe = recipe
                scope_suffix = ""
                if getattr(self.config, "component_filter", None):
                    scope_suffix = str(self.config.component_filter)
                elif getattr(self.config, "cve_filter", None):
                    scope_suffix = str(self.config.cve_filter)
                if scope_suffix:
                    _scoped_recipe = recipe.model_copy(
                        update={"name": f"{recipe.name} - {scope_suffix}"}
                    )
                self.renderer.check_output_guard(_scoped_recipe)
                report_data = self._process_recipe(recipe)
                if report_data:
                    files = self.renderer.render(_scoped_recipe, report_data)
                    if files:
                        generated_files.extend(files)
                    # Collect extra files written by transforms (prompts, VEX JSON)
                    extra = report_data.metadata.get("additional_data", {}).get(
                        "_extra_generated_files", []
                    )
                    if extra:
                        generated_files.extend(extra)

                    # Build per-recipe result
                    recipe_output_dir = str(
                        self.renderer.output_dir
                        / self.renderer._sanitize_filename(_scoped_recipe.name)
                    )
                    row_count = 0
                    data = report_data.data
                    if hasattr(data, "__len__"):
                        row_count = len(data)
                    recipe_results.append(
                        RecipeResult(
                            recipe=_scoped_recipe.name,
                            output_dir=recipe_output_dir,
                            files=files + (extra or []),
                            stats={"finding_count": row_count},
                        )
                    )
                else:
                    self.logger.error(
                        f"No report data generated for recipe: {recipe.name}"
                    )
                    recipe_results.append(
                        RecipeResult(
                            recipe=recipe.name,
                            output_dir="",
                            files=[],
                            stats={"error": "no report data"},
                        )
                    )
                    all_succeeded = False
            except Exception as e:
                self.logger.error(f"Failed to process recipe {recipe.name}: {e}")
                recipe_results.append(
                    RecipeResult(
                        recipe=recipe.name,
                        output_dir="",
                        files=[],
                        stats={"error": str(e)},
                    )
                )
                all_succeeded = False
            finally:
                if self._on_recipe_complete:
                    try:
                        self._on_recipe_complete(idx, total, recipe.name)
                    except Exception:
                        pass
        self.generated_files = generated_files
        if generated_files:
            print("\nReports generated:")
            for f in generated_files:
                print(f"  - {f}")
        return RunResult(success=all_succeeded, recipes=recipe_results)

    # ------------------------------------------------------------------
    # Folder-scoped Remediation Package helper
    # ------------------------------------------------------------------

    def _run_remediation_folder(
        self,
        recipe: Recipe,
        project_ids: list[str],
        generated_files: list[str],
        recipe_results: list[RecipeResult],
    ) -> bool:
        """Run Remediation Package for each project in a folder.

        Creates per-project subdirectories under ``Remediation Package/``.
        Returns ``True`` if at least one project succeeded.
        """
        total_projects = len(project_ids)
        any_succeeded = False
        self.logger.info(
            f"Remediation Package: folder scope with {total_projects} project(s)"
        )

        base_output = Path(self.config.output_dir) / "Remediation Package"
        base_output.mkdir(parents=True, exist_ok=True)

        for pi, pid in enumerate(project_ids, 1):
            project_name = self._resolve_project_id_to_name(int(pid))
            if not project_name:
                project_name = pid
            self.logger.info(
                f"Remediation Package: project {pi}/{total_projects} — {project_name}"
            )

            # Create a scoped config with project_filter set to this project
            scoped_config = self.config.model_copy(update={"project_filter": pid})

            # Create a sub-engine for this project
            try:
                sub_engine = ReportEngine(
                    scoped_config,
                    deployment_context=self._deployment_context,
                )
                # Share the API client to reuse connections/cache
                sub_engine.api_client = self.api_client

                # Override the renderer to write to a per-project subdirectory
                safe_name = self.renderer._sanitize_filename(project_name)
                project_output_dir = str(base_output / safe_name)
                sub_engine.renderer = ReportRenderer(
                    project_output_dir,
                    config=scoped_config,
                    overwrite=self.renderer.overwrite,
                )

                report_data = sub_engine._process_recipe(recipe)
                if report_data:
                    # Render with the original recipe name (not scoped)
                    # since the per-project dir already provides scoping
                    files = sub_engine.renderer.render(recipe, report_data)
                    if files:
                        generated_files.extend(files)
                    extra = report_data.metadata.get("additional_data", {}).get(
                        "_extra_generated_files", []
                    )
                    if extra:
                        generated_files.extend(extra)

                    row_count = 0
                    data = report_data.data
                    if hasattr(data, "__len__"):
                        row_count = len(data)
                    recipe_results.append(
                        RecipeResult(
                            recipe=f"Remediation Package/{project_name}",
                            output_dir=project_output_dir,
                            files=files + (extra or []),
                            stats={"finding_count": row_count, "project": project_name},
                        )
                    )
                    any_succeeded = True
                else:
                    self.logger.warning(
                        f"No report data for Remediation Package — {project_name}"
                    )
            except Exception as e:
                self.logger.error(f"Remediation Package failed for {project_name}: {e}")

        return any_succeeded

    # ------------------------------------------------------------------
    # Cross-server Version Comparison helper
    # ------------------------------------------------------------------

    def _fetch_cross_server_comparison(
        self,
        _fetch_findings_primary: Any,
        _fetch_components_primary: Any,
        finding_type: str,
        category_filter: str | None,
    ) -> list[dict]:
        """Fetch findings from two different servers and build pair comparison."""
        from tqdm import tqdm as _tqdm

        from fs_report.models import QueryConfig, QueryParams

        assert self.compare_api_client is not None

        def _fetch_findings_secondary(version_id: str) -> list[dict]:
            version_filter = f"projectVersion=={version_id}"
            combined_filter = (
                f"{version_filter};{category_filter}"
                if category_filter
                else version_filter
            )
            q = QueryConfig(
                endpoint="/public/v0/findings",
                params=QueryParams(
                    limit=10000,
                    filter=combined_filter,
                    finding_type=finding_type,
                    archived=False,
                    excluded=False,
                ),
            )
            assert self.compare_api_client is not None
            return self.compare_api_client.fetch_all_with_resume(q, show_progress=False)

        def _fetch_components_secondary(version_id: str) -> list[dict]:
            q = QueryConfig(
                endpoint="/public/v0/components",
                params=QueryParams(
                    limit=10000,
                    filter=f"projectVersion=={version_id}",
                ),
            )
            assert self.compare_api_client is not None
            return self.compare_api_client.fetch_all_with_resume(q, show_progress=False)

        def _resolve_latest_version(
            api_client: APIClient, project_filter: str
        ) -> tuple[str, str, str]:
            """Resolve project → latest version. Returns (version_id, version_name, project_name)."""
            # Fetch projects
            pq = QueryConfig(
                endpoint="/public/v0/projects",
                params=QueryParams(limit=10000, archived=False, excluded=False),
            )
            projects = api_client.fetch_all_with_resume(pq, show_progress=False)
            project = None
            for p in projects:
                pid = str(p.get("id", ""))
                pname = p.get("name", "")
                if pid == project_filter or pname.lower() == project_filter.lower():
                    project = p
                    break
            if not project:
                raise ValueError(
                    f"Project '{project_filter}' not found on " f"{api_client.base_url}"
                )
            pid = str(project["id"])
            pname = project.get("name", pid)

            # Fetch versions for project
            url = f"{api_client.base_url}/public/v0/projects/{pid}/versions"
            resp = api_client.client.get(url)
            resp.raise_for_status()
            versions = resp.json()
            if not isinstance(versions, list) or not versions:
                raise ValueError(
                    f"No versions found for project '{pname}' on "
                    f"{api_client.base_url}"
                )
            # Latest = most recently created
            versions.sort(key=lambda v: v.get("created", v.get("id", "")), reverse=True)
            latest = versions[0]
            vid = str(latest.get("id", ""))
            vname = latest.get("version", latest.get("name", vid))
            return vid, vname, pname

        # ── Resolve primary side ──────────────────────────────────────
        primary_version_id = self.config.version_filter
        primary_version_name = primary_version_id or ""
        primary_project_name = ""

        if primary_version_id:
            # Explicit version ID
            self.logger.info("Cross-server: primary version ID %s", primary_version_id)
        elif self.config.project_filter:
            # Resolve project → latest version
            self.logger.info(
                "Cross-server: resolving primary project '%s'",
                self.config.project_filter,
            )
            primary_version_id, primary_version_name, primary_project_name = (
                _resolve_latest_version(self.api_client, self.config.project_filter)
            )
            self.logger.info(
                "Cross-server: primary resolved to %s (%s) in project '%s'",
                primary_version_id,
                primary_version_name,
                primary_project_name,
            )
        else:
            raise ValueError(
                "Cross-server comparison requires --project or --version "
                "to identify the primary version."
            )

        # ── Resolve secondary side ────────────────────────────────────
        secondary_version_id = self.config.compare_version
        secondary_version_name = secondary_version_id or ""
        secondary_project_name = ""

        if secondary_version_id:
            self.logger.info(
                "Cross-server: secondary version ID %s", secondary_version_id
            )
        elif self.config.compare_project:
            self.logger.info(
                "Cross-server: resolving secondary project '%s'",
                self.config.compare_project,
            )
            secondary_version_id, secondary_version_name, secondary_project_name = (
                _resolve_latest_version(
                    self.compare_api_client, self.config.compare_project
                )
            )
            self.logger.info(
                "Cross-server: secondary resolved to %s (%s) in project '%s'",
                secondary_version_id,
                secondary_version_name,
                secondary_project_name,
            )
        else:
            raise ValueError(
                "Cross-server comparison requires --compare-project or "
                "--compare-version to identify the secondary version."
            )

        # ── Fetch data from both servers ──────────────────────────────
        pair_pbar = _tqdm(
            total=4,
            desc="Fetching cross-server pair",
            unit=" requests",
            leave=False,
        )

        pair_pbar.set_postfix({"step": "Primary findings"})
        primary_findings = _fetch_findings_primary(primary_version_id)
        pair_pbar.update(1)

        pair_pbar.set_postfix({"step": "Primary components"})
        primary_components = _fetch_components_primary(primary_version_id)
        pair_pbar.update(1)

        pair_pbar.set_postfix({"step": "Secondary findings"})
        secondary_findings = _fetch_findings_secondary(secondary_version_id)
        pair_pbar.update(1)

        pair_pbar.set_postfix({"step": "Secondary components"})
        secondary_components = _fetch_components_secondary(secondary_version_id)
        pair_pbar.update(1)
        pair_pbar.close()

        # Extract version metadata from findings if not already resolved
        def _extract_meta(
            findings: list[dict], version_id: str
        ) -> tuple[str, str, str]:
            vname, created, pname = version_id, "", ""
            for f in findings:
                pv = f.get("projectVersion")
                if isinstance(pv, dict):
                    v = pv.get("version") or pv.get("name")
                    if v:
                        vname = str(v)
                    c = pv.get("created", "")
                    if c:
                        created = c
                proj = f.get("project")
                if isinstance(proj, dict):
                    n = proj.get("name")
                    if n:
                        pname = str(n)
                if vname != version_id and pname:
                    break
            return vname, created, pname

        if not primary_project_name:
            primary_version_name, primary_created, primary_project_name = _extract_meta(
                primary_findings, primary_version_id
            )
        else:
            _, primary_created, _ = _extract_meta(primary_findings, primary_version_id)

        if not secondary_project_name:
            secondary_version_name, secondary_created, secondary_project_name = (
                _extract_meta(secondary_findings, secondary_version_id)
            )
        else:
            _, secondary_created, _ = _extract_meta(
                secondary_findings, secondary_version_id
            )

        # Build labels with domain for clarity
        primary_domain = self.config.domain
        secondary_domain = self.config.compare_domain
        primary_label = f"{primary_domain} @ {primary_version_name}"
        secondary_label = f"{secondary_domain} @ {secondary_version_name}"

        # Build project name for display
        if primary_project_name and secondary_project_name:
            if primary_project_name == secondary_project_name:
                project_name = primary_project_name
            else:
                project_name = (
                    primary_project_name + " \u2194 " + secondary_project_name
                )
        else:
            project_name = (
                primary_project_name
                or secondary_project_name
                or "Cross-Server Comparison"
            )

        projects_data: list[dict] = [
            {
                "project_name": project_name,
                "is_pair_comparison": True,
                "versions": [
                    {
                        "id": primary_version_id,
                        "name": primary_label,
                        "created": primary_created,
                        "project_name": primary_project_name,
                        "findings": primary_findings,
                        "components": primary_components,
                    },
                    {
                        "id": secondary_version_id,
                        "name": secondary_label,
                        "created": secondary_created,
                        "project_name": secondary_project_name,
                        "findings": secondary_findings,
                        "components": secondary_components,
                    },
                ],
            }
        ]

        self._version_comparison_data = {"projects": projects_data}
        return [{"_vc_placeholder": True}]

    # ------------------------------------------------------------------
    # Version Comparison: specialised data fetcher
    # ------------------------------------------------------------------

    def _fetch_version_comparison_data(self, recipe: Any) -> list[dict]:
        """
        Fetch findings for every version of every in-scope project so the
        transform can show a full version-over-version progression.

        Supports:
          * ``--folder``   → projects in that folder
          * ``--project``  → single project
          * (neither)      → entire portfolio

        Stores a list of per-project dicts in
        ``self._version_comparison_data["projects"]``.
        Each dict: ``{project_name, versions: [{id, name, created, findings, components}]}``.
        """
        from tqdm import tqdm as _tqdm

        from fs_report.models import QueryConfig, QueryParams

        type_params = build_findings_type_params(self.config.finding_types)
        finding_type = type_params.get("type", "cve") or ""
        category_filter = type_params.get("category_filter")

        def _fetch_findings(version_id: str) -> list[dict]:
            # Check both in-memory and SQLite caches
            cached = self._check_version_in_cache(
                version_id, "findings", finding_type, category_filter
            )
            if cached is not None:
                return cached
            version_filter = f"projectVersion=={version_id}"
            combined_filter = (
                f"{version_filter};{category_filter}"
                if category_filter
                else version_filter
            )
            q = QueryConfig(
                endpoint="/public/v0/findings",
                params=QueryParams(
                    limit=10000,
                    filter=combined_filter,
                    finding_type=finding_type,
                    archived=False,
                    excluded=False,
                ),
            )
            result = self.api_client.fetch_all_with_resume(q, show_progress=False)
            # Store per-version in both in-memory and SQLite caches
            cache_key = self._cache_key(
                "findings", version_id, finding_type, category_filter
            )
            self._version_findings_cache[cache_key] = result
            # SQLite is already handled by fetch_all_with_resume when cache_ttl > 0
            return result

        def _fetch_components(version_id: str) -> list[dict]:
            # Check both in-memory and SQLite caches
            cached = self._check_version_in_cache(version_id, "components")
            if cached is not None:
                return cached
            q = QueryConfig(
                endpoint="/public/v0/components",
                params=QueryParams(
                    limit=10000,
                    filter=f"projectVersion=={version_id}",
                ),
            )
            result = self.api_client.fetch_all_with_resume(q, show_progress=False)
            cache_key = self._cache_key("components", version_id)
            self._version_findings_cache[cache_key] = result
            return result

        # ── Cross-server comparison ────────────────────────────────────
        if self.compare_api_client is not None:
            return self._fetch_cross_server_comparison(
                _fetch_findings,
                _fetch_components,
                finding_type,
                category_filter,
            )

        # ── Validate --baseline-version / --current-version ─────────────
        bv = self.config.baseline_version
        cv = self.config.current_version
        if bool(bv) != bool(cv):
            raise ValueError(
                "Both --baseline-version and --current-version must be provided together. "
                "Use 'fs-report list-versions <project>' to find version IDs."
            )

        # ── Short-circuit: explicit version pair ──────────────────────
        if bv and cv:
            if bv == cv:
                raise ValueError(
                    "--baseline-version and --current-version must be different version IDs."
                )

            self.logger.info(
                "Version Comparison: explicit pair (baseline=%s, current=%s)",
                bv,
                cv,
            )

            # Fetch findings + components directly (version IDs are globally unique)
            pair_pbar = _tqdm(
                total=4,
                desc="Fetching version pair",
                unit=" requests",
                leave=False,
            )
            pair_pbar.set_postfix({"step": "Baseline findings"})
            baseline_findings = _fetch_findings(bv)
            pair_pbar.update(1)

            pair_pbar.set_postfix({"step": "Baseline components"})
            baseline_components = _fetch_components(bv)
            pair_pbar.update(1)

            pair_pbar.set_postfix({"step": "Current findings"})
            current_findings = _fetch_findings(cv)
            pair_pbar.update(1)

            pair_pbar.set_postfix({"step": "Current components"})
            current_components = _fetch_components(cv)
            pair_pbar.update(1)
            pair_pbar.close()

            # Resolve version metadata from the already-fetched findings.
            # Each finding record contains projectVersion.version (name) and
            # project.name, so we can extract names without extra API calls.
            baseline_name, baseline_created = bv, ""
            current_name, current_created = cv, ""
            baseline_project_name = ""
            current_project_name = ""
            project_name = "Version Comparison"

            def _extract_version_meta(
                findings: list[dict], version_id: str
            ) -> tuple[str, str, str]:
                """Extract (version_name, created, project_name) from findings."""
                vname, created, pname = version_id, "", ""
                for f in findings:
                    pv = f.get("projectVersion")
                    if isinstance(pv, dict):
                        v = pv.get("version") or pv.get("name")
                        if v:
                            vname = str(v)
                        c = pv.get("created", "")
                        if c:
                            created = c
                    proj = f.get("project")
                    if isinstance(proj, dict):
                        n = proj.get("name")
                        if n:
                            pname = str(n)
                    if vname != version_id and pname:
                        break  # found both, stop early
                return vname, created, pname

            baseline_name, baseline_created, baseline_project_name = (
                _extract_version_meta(baseline_findings, bv)
            )
            current_name, current_created, current_project_name = _extract_version_meta(
                current_findings, cv
            )

            # Build display project name
            if baseline_project_name and current_project_name:
                if baseline_project_name == current_project_name:
                    project_name = baseline_project_name
                else:
                    project_name = (
                        baseline_project_name + " \u2194 " + current_project_name
                    )
            elif baseline_project_name:
                project_name = baseline_project_name
            elif current_project_name:
                project_name = current_project_name

            projects_data: list[dict] = [
                {
                    "project_name": project_name,
                    "is_pair_comparison": True,
                    "versions": [
                        {
                            "id": bv,
                            "name": baseline_name,
                            "created": baseline_created,
                            "project_name": baseline_project_name,
                            "findings": baseline_findings,
                            "components": baseline_components,
                        },
                        {
                            "id": cv,
                            "name": current_name,
                            "created": current_created,
                            "project_name": current_project_name,
                            "findings": current_findings,
                            "components": current_components,
                        },
                    ],
                }
            ]

            self._version_comparison_data = {"projects": projects_data}
            return [{"_vc_placeholder": True}]

        # ── Discover projects ──────────────────────────────────────────
        project_ids_and_names: list[tuple[str, str]] = []  # (pid, pname)

        projects_query = QueryConfig(
            endpoint="/public/v0/projects",
            params=QueryParams(limit=10000, archived=False, excluded=False),
        )
        all_projects = self.api_client.fetch_all_with_resume(
            projects_query,
            show_progress=False,
        )

        if self._folder_project_ids:
            self.logger.info(
                "Version Comparison: %d project(s) from folder '%s'",
                len(self._folder_project_ids),
                self._folder_name or "unknown",
            )
            for p in all_projects:
                pid = str(p.get("id", ""))
                if pid in self._folder_project_ids:
                    project_ids_and_names.append((pid, p.get("name", pid)))

        elif self.config.project_filter:
            self.logger.info(
                "Version Comparison: single project '%s'",
                self.config.project_filter,
            )
            pf = self.config.project_filter
            for p in all_projects:
                pid = str(p.get("id", ""))
                pname = p.get("name", "")
                if pid == pf or pname.lower() == pf.lower():
                    project_ids_and_names.append((pid, pname))
                    break
        else:
            self.logger.info("Version Comparison: all projects in portfolio")
            for p in all_projects:
                pid = str(p.get("id", ""))
                if pid:
                    project_ids_and_names.append((pid, p.get("name", pid)))

        if not project_ids_and_names:
            self.logger.warning("No projects found for version comparison")
            self._version_comparison_data = {}
            return []

        # ── Fetch version lists per project (with caching) ─────────────
        # Each entry: (pname, sorted_versions_list)
        project_version_lists: list[tuple[str, list[dict]]] = []

        self.logger.info(
            "Discovering versions for %d project(s)…",
            len(project_ids_and_names),
        )
        delay = getattr(self.config, "request_delay", 0.5)
        cache_hits = 0
        sqlite_cache = self.api_client.sqlite_cache
        cache_ttl = self.api_client.cache_ttl

        with _tqdm(
            project_ids_and_names,
            desc="Discovering versions",
            unit=" projects",
            leave=False,
        ) as pbar:
            for pid, pname in pbar:
                pbar.set_postfix({"project": pname[:30]})
                versions = None

                # 1. Check in-memory cache
                if pid in self._project_versions_cache:
                    versions = self._project_versions_cache[pid]
                    cache_hits += 1

                # 2. Check SQLite cache
                if versions is None and sqlite_cache and cache_ttl > 0:
                    cached = sqlite_cache.get_version_list(pid, cache_ttl)
                    if cached is not None:
                        versions = cached
                        self._project_versions_cache[pid] = versions
                        cache_hits += 1

                # 3. Fetch from API (only on cache miss)
                if versions is None:
                    try:
                        url = (
                            f"{self.api_client.base_url}"
                            f"/public/v0/projects/{pid}/versions"
                        )
                        resp = self.api_client.client.get(url)
                        resp.raise_for_status()
                        versions = resp.json()
                        if not isinstance(versions, list):
                            versions = []
                    except Exception as e:
                        self.logger.debug(
                            "Error fetching versions for %s (%s): %s",
                            pname,
                            pid,
                            e,
                        )
                        continue
                    finally:
                        # Only delay after actual API calls
                        if delay > 0:
                            time.sleep(delay)

                    # Store in both caches
                    self._project_versions_cache[pid] = versions
                    if sqlite_cache and cache_ttl > 0:
                        sqlite_cache.store_version_list(pid, versions)

                if len(versions) < 2:
                    self.logger.debug(
                        "%s has %d version(s) — skipping",
                        pname,
                        len(versions),
                    )
                    continue

                # Sort versions by configured key (default: created ascending)
                _vs = self.config.version_sort or "created"
                versions.sort(
                    key=lambda v: v.get(_vs, v.get("id", "")),
                    reverse=self.config.version_sort_desc,
                )
                project_version_lists.append((pname, versions))

        if cache_hits > 0:
            self.logger.info(
                "Version discovery: %d/%d projects served from cache",
                cache_hits,
                len(project_ids_and_names),
            )

        if not project_version_lists:
            self.logger.warning("No projects with ≥ 2 versions found")
            self._version_comparison_data = {}
            return []

        # ── Fetch findings & components for every version ──────────────
        total_versions = sum(len(vl) for _, vl in project_version_lists)
        self.logger.info(
            "Fetching findings for %d version(s) across %d project(s)…",
            total_versions,
            len(project_version_lists),
        )

        projects_data = []  # list[dict]
        with _tqdm(  # type: ignore[assignment]
            total=total_versions,
            desc="Fetching version data",
            unit=" versions",
            leave=False,
        ) as pbar:
            for pname, versions in project_version_lists:
                version_records: list[dict] = []
                for v in versions:
                    vid = str(v.get("id", ""))
                    vname = v.get("version", v.get("name", vid))
                    created = v.get("created", "")
                    pbar.set_postfix({"project": pname[:20], "version": vname[:20]})

                    findings = _fetch_findings(vid)
                    if delay > 0:
                        time.sleep(delay)
                    components = _fetch_components(vid)

                    version_records.append(
                        {
                            "id": vid,
                            "name": vname,
                            "created": created,
                            "findings": findings,
                            "components": components,
                        }
                    )
                    pbar.update(1)

                    # Throttle between versions to avoid overloading the server
                    if delay > 0:
                        time.sleep(delay)

                projects_data.append(
                    {
                        "project_name": pname,
                        "versions": version_records,
                    }
                )

        self._version_comparison_data = {"projects": projects_data}
        return [{"_vc_placeholder": True}]

    def _process_recipe(self, recipe: Recipe) -> ReportData | None:
        """Process a single recipe and return report data."""
        try:
            # Track whether entity-level caching was used (needs date post-filtering)
            needs_date_postfilter = False
            is_operational = False
            # Track whether multi-type post-filtering is needed
            post_filter_categories: set[str] | None = None
            post_filter_types: set[str] | None = None

            # --- NVD pipeline handles (initialised here so both branches see them) ---
            _nvd_on_records = None
            _nvd_collect = None
            _es_cache_key: str | None = None
            _ed_cache_key: str | None = None
            # CVA flatten/prune state (initialised here so post-accumulation code sees them)
            _is_cva = False
            _cva_pre_flattened = False
            _cva_extra_keep: frozenset[str] | None = None

            # Use override data if provided
            if self.data_override is not None:
                self.logger.info(
                    f"Using data from override file for recipe: {recipe.name}"
                )
                # For data override, we need to extract the main query data
                # The main query should match one of the keys in the override data
                endpoint = recipe.query.endpoint
                raw_data = None
                # Patch: if override is a list, use it directly
                if isinstance(self.data_override, list):  # type: ignore[unreachable]
                    raw_data = self.data_override  # type: ignore[unreachable]
                else:
                    self.logger.debug(
                        f"Override matching: endpoint={endpoint}, override keys={list(self.data_override.keys())}"
                    )
                    for key in self.data_override:
                        key_str = str(key)
                        endpoint_str = str(endpoint)
                        self.logger.debug(
                            f"Comparing key='{key_str}' to endpoint='{endpoint_str}'"
                        )
                        if key_str == endpoint_str or key_str.endswith(
                            endpoint_str.split("/")[-1]
                        ):
                            raw_data = self.data_override[key]
                            self.logger.debug(f"Override match found: key='{key_str}'")
                            break
                if raw_data is None:
                    self.logger.error(
                        f"Could not find data for endpoint {endpoint} in override data"
                    )
                    return None
            else:
                # Use robust pagination for all major findings-based reports
                _ROBUST_ENDPOINTS = {
                    "/public/v0/findings",
                    "/public/v0/scans",
                    "/public/v0/cves",
                    "/public/v0/audit",
                    "/public/v0/components",
                }
                _endpoint = recipe.query.endpoint if recipe.query else ""
                if _endpoint in _ROBUST_ENDPOINTS or recipe.name in (
                    "Version Comparison",
                    "User Activity",
                    "Scan Analysis",
                    "CVE Impact",
                    "Component List",
                ):
                    from fs_report.models import QueryConfig, QueryParams

                    if recipe.name == "Version Comparison":
                        # --- Version Comparison: two-version parallel fetch ---
                        _fetched = self._fetch_version_comparison_data(recipe)
                        raw_data = (
                            pd.DataFrame(_fetched) if _fetched else pd.DataFrame()
                        )
                        del _fetched
                    elif recipe.name == "User Activity":
                        # Build filter for audit endpoint using RSQL format
                        # The audit API supports: time=ge=START;time=le=END
                        # (The date=START,date=END format has parsing issues with commas in filter param)
                        audit_filter = f"time=ge={self.config.start_date}T00:00:00Z;time=le={self.config.end_date}T23:59:59Z"

                        unified_query = QueryConfig(
                            endpoint=recipe.query.endpoint,
                            params=QueryParams(
                                limit=recipe.query.params.limit, filter=audit_filter
                            ),
                        )
                        self.logger.info(
                            f"Fetching audit events for {recipe.name} with filter: {audit_filter}"
                        )
                        _fetched = self.api_client.fetch_all_with_resume(unified_query)
                        raw_data = (
                            pd.DataFrame(_fetched) if _fetched else pd.DataFrame()
                        )
                        del _fetched
                    elif recipe.name == "Scan Analysis":
                        # Apply project and version filtering to scans
                        # Batch folder project IDs to avoid 414 URL Too Long
                        if (
                            self._folder_project_ids
                            and not self.config.project_filter
                            and len(self._folder_project_ids) > 25
                        ):
                            folder_pids = sorted(self._folder_project_ids)
                            batch_size = 15 if len(folder_pids) > 200 else 25
                            all_scans: list[dict] = []
                            saved_pids = self._folder_project_ids
                            self.logger.info(
                                f"Batching Scan Analysis fetch: "
                                f"{len(folder_pids)} projects, batch_size={batch_size}"
                            )
                            try:
                                for i in range(0, len(folder_pids), batch_size):
                                    scan_batch_pids = set(
                                        folder_pids[i : i + batch_size]
                                    )
                                    self._folder_project_ids = scan_batch_pids
                                    scan_query = self._apply_scan_filters(recipe.query)
                                    batch_data = (
                                        self._fetch_scans_with_early_termination(
                                            scan_query
                                        )
                                    )
                                    if batch_data:
                                        all_scans.extend(batch_data)
                            finally:
                                self._folder_project_ids = saved_pids
                            _fetched = all_scans
                        else:
                            scan_query = self._apply_scan_filters(recipe.query)
                            # Use early termination to avoid fetching old scans
                            _fetched = self._fetch_scans_with_early_termination(
                                scan_query
                            )
                        raw_data = (
                            pd.DataFrame(_fetched) if _fetched else pd.DataFrame()
                        )
                        del _fetched

                        # Fetch project data for new vs existing analysis
                        # Store in a variable that will be added to additional_data later
                        self._scan_analysis_project_data = None
                        if (
                            hasattr(recipe, "project_list_query")
                            and recipe.project_list_query
                        ):
                            self.logger.info("Fetching project data for Scan Analysis")
                            project_query = QueryConfig(
                                endpoint=recipe.project_list_query.endpoint,
                                params=QueryParams(
                                    limit=recipe.project_list_query.params.limit,
                                    offset=0,
                                    archived=False,
                                    excluded=False,
                                ),
                            )
                            self._scan_analysis_project_data = (
                                self.api_client.fetch_all_with_resume(project_query)
                            )
                            self.logger.info(
                                f"Fetched {len(self._scan_analysis_project_data)} projects for new/existing analysis"
                            )
                    elif recipe.name == "CVE Impact":
                        # CVE Impact uses the /public/v0/cves endpoint which
                        # returns data pre-aggregated by CVE ID. Simple pagination,
                        # no version batching needed.

                        # Scope guard: --cve is always required.
                        # A project-scoped query (--project without --cve)
                        # can return thousands of CVEs and overwhelm the
                        # report.  For a project with N CVEs the dossier
                        # enrichment alone would issue ~3*N API calls
                        # (e.g. 4 000 CVEs → ~12 000 requests).
                        # --project is still accepted as an optional
                        # narrowing filter alongside --cve.
                        if not self.config.cve_filter:
                            self.logger.error(
                                "CVE Impact requires --cve to specify which CVE(s) to analyse. "
                                "--project alone is not supported because it can return "
                                "thousands of CVEs (e.g. 4 000 CVEs → ~12 000 API calls for "
                                "dossier enrichment).  Examples:\n"
                                "  --cve CVE-2022-37434\n"
                                "  --cve CVE-2022-37434,CVE-2023-44487\n"
                                "  --cve CVE-2022-37434 --project openwrt"
                            )
                            raw_data = pd.DataFrame()
                        else:
                            cve_filters: list[str] = []

                            # Apply --cve filter at the API level
                            if self.config.cve_filter:
                                cve_ids = [
                                    c.strip()
                                    for c in self.config.cve_filter.split(",")
                                    if c.strip()
                                ]
                                if len(cve_ids) == 1:
                                    cve_filters.append(f"cveId=={cve_ids[0]}")
                                else:
                                    cve_filters.append(
                                        f"cveId=in=({','.join(cve_ids)})"
                                    )

                            # Apply --project filter (already resolved to
                            # numeric ID by run(), so int() will not fail)
                            if self.config.project_filter:
                                cve_filters.append(
                                    f"project=={self.config.project_filter}"
                                )

                            # Apply --detected-after
                            if getattr(self.config, "detected_after", None):
                                cve_filters.append(
                                    f"detectionDate>={self.config.detected_after}T00:00:00"
                                )

                            cve_query = QueryConfig(
                                endpoint="/public/v0/cves",
                                params=QueryParams(
                                    limit=10000,
                                    filter=(
                                        ";".join(cve_filters) if cve_filters else None
                                    ),
                                ),
                            )
                            self.logger.info(
                                f"Fetching CVEs for {recipe.name}"
                                + (
                                    f" with filter: {cve_query.params.filter}"
                                    if cve_query.params.filter
                                    else ""
                                )
                            )
                            _fetched = self.api_client.fetch_all_with_resume(cve_query)
                            raw_data = (
                                pd.DataFrame(_fetched) if _fetched else pd.DataFrame()
                            )
                            del _fetched

                            # In dossier mode, enrich with per-finding reachability,
                            # CVE descriptions, and exploit details
                            if self.config.cve_filter and not raw_data.empty:
                                (
                                    self._cve_impact_reachability,
                                    self._cve_impact_descriptions,
                                    self._cve_impact_exploit_details,
                                    self._cve_impact_nvd_missing,
                                ) = self._fetch_cve_reachability(raw_data)

                    elif recipe.name in ("Component List", "License Report"):
                        # Assessment report: shows current component inventory
                        # No date filtering by default (current state, not period-bound)
                        filters = []

                        # Exclude file type components (SAST placeholders without meaningful data)
                        filters.append("type!=file")

                        # Apply --detected-after if specified (opt-in period filtering)
                        if getattr(self.config, "detected_after", None):
                            filters.append(
                                f"created>={self.config.detected_after}T00:00:00"
                            )

                        if self.config.project_filter:
                            try:
                                project_id = int(self.config.project_filter)
                                filters.append(f"project=={project_id}")
                            except ValueError:
                                filters.append(f"project=={self.config.project_filter}")
                        elif self._folder_project_ids:
                            # Folder scoping — add project=in=() filter (sorted for deterministic cache keys)
                            folder_pids = sorted(self._folder_project_ids)
                            filters.append(
                                f"project=in=({','.join(str(pid) for pid in folder_pids)})"
                            )

                        if self.config.version_filter:
                            filters.append(
                                f"projectVersion=={self.config.version_filter}"
                            )

                        combined_filter = ";".join(filters)

                        unified_query = QueryConfig(
                            endpoint=recipe.query.endpoint,
                            params=QueryParams(
                                limit=recipe.query.params.limit,
                                filter=combined_filter,
                                archived=False,
                                excluded=False,
                            ),
                        )

                        # Use batched version filtering if current_version_only is enabled
                        if (
                            self.config.current_version_only
                            and not self.config.version_filter
                        ):
                            # Scope version resolution to only the projects
                            # being queried — avoids fetching version IDs (and
                            # then components) for projects that the filter will
                            # exclude anyway.
                            if self.config.project_filter:
                                # Single project → resolve just that one version
                                version_ids = self._get_latest_version_ids_for_projects(
                                    [self.config.project_filter]
                                )
                            elif self._folder_project_ids:
                                # Folder scope → resolve only folder projects
                                version_ids = self._get_latest_version_ids_for_projects(
                                    list(self._folder_project_ids)
                                )
                            else:
                                # No project/folder filter → resolve all
                                version_ids = self._get_latest_version_ids()
                            self.logger.info(
                                f"Fetching components for {recipe.name} with --current-version-only ({len(version_ids)} versions), base filter: {combined_filter}"
                            )
                            _fetched = self._fetch_with_version_batching(
                                unified_query,
                                version_ids,
                                entity_type="components",
                            )
                            raw_data = (
                                pd.DataFrame(_fetched) if _fetched else pd.DataFrame()
                            )
                            del _fetched
                        else:
                            self.logger.info(
                                f"Fetching components for {recipe.name} with filter: {combined_filter}"
                            )
                            _fetched = self.api_client.fetch_all_with_resume(
                                unified_query
                            )
                            raw_data = (
                                pd.DataFrame(_fetched) if _fetched else pd.DataFrame()
                            )
                            del _fetched
                    elif recipe.query.endpoint == "/public/v0/findings":
                        # Report category determines period behaviour:
                        #   Operational (Executive Summary): period filters findings by detected date
                        #   Assessment  (CVA, Findings by Project, Triage): shows current state, period ignored
                        is_operational = recipe.name == "Executive Summary"

                        # Recipes whose YAML filter declares ${start}/${end} have opted
                        # into period-based date filtering regardless of category.
                        _recipe_has_date_filter = bool(
                            recipe.query
                            and recipe.query.params
                            and recipe.query.params.filter
                            and (
                                "${start}" in recipe.query.params.filter
                                or "${end}" in recipe.query.params.filter
                            )
                        )

                        # Build finding type parameters based on --finding-types flag
                        # Executive Dashboard needs all finding types regardless of CLI flag
                        _uses_gates = bool(
                            recipe.parameters and recipe.parameters.get("gates")
                        )
                        if recipe.name == "Executive Dashboard":
                            type_params = build_findings_type_params("all")
                        elif _uses_gates:
                            # Recipes with gate-based triage scoring need
                            # reachabilityScore which the API omits when the
                            # ``type=cve`` URL param is used.  Use a category
                            # RSQL filter instead so the full field set is
                            # returned.
                            type_params = {
                                "type": None,
                                "category_filter": "category==CVE",
                            }
                        else:
                            type_params = build_findings_type_params(
                                self.config.finding_types
                            )
                        finding_type = type_params.get("type", "cve") or ""
                        category_filter = type_params.get("category_filter")
                        post_filter_categories = type_params.get(
                            "post_filter_categories"
                        )
                        post_filter_types = type_params.get("post_filter_types")

                        # Whether we need to post-filter by date (set True when entity-
                        # level caching is used and date filters are NOT in the API query)
                        needs_date_postfilter = False
                        raw_data = None

                        # --- Per-batch scoring for Triage Prioritization ---
                        # Normalize + score each batch during fetch so raw
                        # API dicts are discarded immediately, reducing peak
                        # memory by an additional ~20-30% on top of chunked DF.
                        _is_triage = recipe.name == "Triage Prioritization"
                        _low_memory = getattr(self.config, "low_memory", False)
                        _triage_weights = None
                        _triage_gates = None
                        if _is_triage:
                            from fs_report.transforms.pandas.triage_prioritization import (
                                _load_gates,
                                _load_weights,
                                _normalize_columns,
                                apply_tiered_gates,
                                assign_risk_bands,
                                calculate_additive_score,
                            )

                            _triage_ad: dict[str, Any] = {"config": self.config}
                            if recipe.parameters:
                                _triage_ad["recipe_parameters"] = recipe.parameters
                            _triage_weights = _load_weights(self.config, _triage_ad)
                            _triage_gates = _load_gates(self.config, _triage_ad)

                        # --- Per-batch flattening for Findings by Project ---
                        # Pre-flatten nested dict columns and drop unused API
                        # fields per-chunk, reducing accumulated memory ~60%.
                        _is_findings_by_project = recipe.name == "Findings by Project"
                        _flatten_findings_data: Callable | None = None
                        if _is_findings_by_project:
                            from fs_report.transforms.pandas.findings_by_project import (
                                flatten_findings_data as _flatten_findings_data,
                            )

                        # Nested dict columns consumed by flatten_findings_data, plus
                        # API fields not used by the Findings by Project transform.
                        _FBP_DROP_AFTER_FLATTEN = {
                            "component",
                            "project",
                            "projectVersion",
                            "exploitInfo",
                            "cwes",
                            "attackVector",
                            "epssPercentile",
                            "epssScore",
                            "hasKnownExploit",
                            "inKev",
                            "affectedFunctions",
                            "risk",  # already extracted to cvss_score
                        }

                        # --- Per-batch column pruning for Executive Summary ---
                        # Keep only the 5 columns the transforms actually use,
                        # dropping ~50+ nested-dict/API columns per batch.
                        _is_exec_summary = recipe.name == "Executive Summary"
                        _exec_extra_keep: frozenset[str] | None = None
                        if _is_exec_summary and (
                            post_filter_categories or post_filter_types
                        ):
                            _exec_extra_keep = frozenset({"category", "type"})

                        # --- Per-batch column pruning for Executive Dashboard ---
                        _is_exec_dashboard = recipe.name == "Executive Dashboard"
                        _ed_extra_keep: frozenset[str] | None = None
                        if _is_exec_dashboard and (
                            post_filter_categories or post_filter_types
                        ):
                            _ed_extra_keep = frozenset({"category", "type"})

                        # --- Per-batch flatten + pruning for CVA ---
                        _is_cva = recipe.name == "Component Vulnerability Analysis"
                        _cva_pre_flattened = False
                        if _is_cva and (post_filter_categories or post_filter_types):
                            _cva_extra_keep = frozenset({"category", "type"})

                        # --- NVD pipeline: start background lookups during fetch ---
                        if _is_findings_by_project and not getattr(
                            self.config, "skip_nvd", False
                        ):
                            _nvd_on_records, _nvd_collect = self._start_nvd_pipeline()

                        # Build filter list for non-entity-cached paths
                        # (entity-cached paths skip this and post-filter instead)
                        filters = []
                        if category_filter:
                            filters.append(category_filter)

                        # Period-filtered reports: add detected date range filter
                        if is_operational or _recipe_has_date_filter:
                            filters.append(
                                f"detected>={self.config.start_date}T00:00:00"
                            )
                            filters.append(f"detected<={self.config.end_date}T23:59:59")

                        # Assessment reports (without date vars): apply --detected-after if specified
                        if (
                            not is_operational
                            and not _recipe_has_date_filter
                            and getattr(self.config, "detected_after", None)
                        ):
                            filters.append(
                                f"detected>={self.config.detected_after}T00:00:00"
                            )

                        if getattr(self.config, "open_only", False):
                            filters.append(
                                "status=out=(NOT_AFFECTED,FALSE_POSITIVE,RESOLVED,RESOLVED_WITH_PEDIGREE)"
                            )

                        # Scoped Remediation Package: CVE filter is applied in the
                        # transform (not at API level) so sibling CVEs on the
                        # same component are included in the action card.

                        if self.config.project_filter:
                            # Single project filter - get findings for this project
                            try:
                                project_id = int(self.config.project_filter)
                                filters.append(f"project=={project_id}")
                            except ValueError:
                                filters.append(f"project=={self.config.project_filter}")

                            if self.config.version_filter:
                                filters.append(
                                    f"projectVersion=={self.config.version_filter}"
                                )
                            elif self.config.current_version_only:
                                # Use entity-level caching (consistent with folder/scan paths)
                                try:
                                    _proj_id = int(self.config.project_filter)
                                except ValueError:
                                    _proj_id = None
                                if _proj_id is not None:
                                    latest_vids = (
                                        self._get_latest_version_ids_for_projects(
                                            [_proj_id]
                                        )
                                    )
                                    if latest_vids:
                                        self.logger.info(
                                            f"--current-version-only: scoping to latest version {latest_vids[0]}"
                                        )
                                        _vf = self._get_findings_for_versions(
                                            latest_vids,
                                            finding_type,
                                            category_filter,
                                            on_records=_nvd_on_records,
                                        )
                                        raw_data = (
                                            pd.DataFrame(_vf) if _vf else pd.DataFrame()
                                        )
                                        del _vf
                                        # Pre-flatten nested dicts for Findings by Project
                                        if (
                                            _is_findings_by_project
                                            and _flatten_findings_data is not None
                                            and not raw_data.empty
                                        ):
                                            raw_data = _flatten_findings_data(raw_data)
                                            _drop = [
                                                c
                                                for c in _FBP_DROP_AFTER_FLATTEN
                                                if c in raw_data.columns
                                            ]
                                            if _drop:
                                                raw_data = raw_data.drop(columns=_drop)
                                        # Per-batch column pruning for Executive Summary
                                        if _is_exec_summary and not raw_data.empty:
                                            raw_data = _prune_exec_summary(
                                                raw_data, _exec_extra_keep
                                            )
                                        # Per-batch column pruning for Executive Dashboard
                                        if _is_exec_dashboard and not raw_data.empty:
                                            raw_data = _prune_exec_dashboard(
                                                raw_data, _ed_extra_keep
                                            )
                                        needs_date_postfilter = True
                                    else:
                                        self.logger.warning(
                                            f"Could not resolve latest version for project {self.config.project_filter}; "
                                            "falling back to all versions"
                                        )

                            if raw_data is None:
                                combined_filter = ";".join(filters) if filters else ""
                                unified_query = QueryConfig(
                                    endpoint=recipe.query.endpoint,
                                    params=QueryParams(
                                        limit=recipe.query.params.limit,
                                        filter=combined_filter,
                                        finding_type=finding_type,
                                        archived=False,
                                        excluded=False,
                                    ),
                                )

                                self.logger.info(
                                    f"Fetching findings for {recipe.name} with type={finding_type}, filter: {combined_filter}"
                                )
                                _fetched = self.api_client.fetch_all_with_resume(
                                    unified_query
                                )
                                # Feed NVD pipeline before converting to DataFrame
                                if _nvd_on_records is not None and _fetched:
                                    _nvd_on_records(_fetched)
                                raw_data = (
                                    pd.DataFrame(_fetched)
                                    if _fetched
                                    else pd.DataFrame()
                                )
                                del _fetched
                                # Per-batch column pruning for Executive Summary
                                if _is_exec_summary and not raw_data.empty:
                                    raw_data = _prune_exec_summary(
                                        raw_data, _exec_extra_keep
                                    )
                                # Per-batch column pruning for Executive Dashboard
                                if _is_exec_dashboard and not raw_data.empty:
                                    raw_data = _prune_exec_dashboard(
                                        raw_data, _ed_extra_keep
                                    )
                        elif self._folder_project_ids:
                            # Folder scoping active — use folder's project set directly
                            # Sort for deterministic batching (ensures SQLite cache hits across runs)
                            folder_pids = sorted(self._folder_project_ids)
                            self.logger.info(
                                f"Fetching findings for {recipe.name} scoped to folder '{self._folder_name}' ({len(folder_pids)} projects)"
                            )

                            combined_filter = ";".join(filters) if filters else ""
                            unified_query = QueryConfig(
                                endpoint=recipe.query.endpoint,
                                params=QueryParams(
                                    limit=recipe.query.params.limit,
                                    filter=combined_filter,
                                    finding_type=finding_type,
                                    archived=False,
                                    excluded=False,
                                ),
                            )

                            if self.config.current_version_only:
                                # Entity-level caching: fetch per-version (shared across reports)
                                version_ids = sorted(
                                    self._get_latest_version_ids_for_projects(
                                        folder_pids
                                    )
                                )
                                self.logger.info(
                                    f"Fetching findings for {recipe.name} with --current-version-only "
                                    f"({len(version_ids)} latest versions)"
                                )
                                _vf = self._get_findings_for_versions(
                                    version_ids,
                                    finding_type,
                                    category_filter,
                                    on_records=_nvd_on_records,
                                )
                                raw_data = pd.DataFrame(_vf) if _vf else pd.DataFrame()
                                del _vf
                                # Pre-flatten nested dicts for Findings by Project
                                if (
                                    _is_findings_by_project
                                    and _flatten_findings_data is not None
                                    and not raw_data.empty
                                ):
                                    if self._project_folder_map:
                                        _inject_folder_names_df(
                                            raw_data, self._project_folder_map
                                        )
                                    raw_data = _flatten_findings_data(raw_data)
                                    _drop = [
                                        c
                                        for c in _FBP_DROP_AFTER_FLATTEN
                                        if c in raw_data.columns
                                    ]
                                    if _drop:
                                        raw_data = raw_data.drop(columns=_drop)
                                    self.logger.info(
                                        f"Pre-flattened {len(raw_data)} findings ({len(raw_data.columns)} columns after pruning)"
                                    )
                                # Per-batch column pruning for Executive Summary
                                if _is_exec_summary and not raw_data.empty:
                                    raw_data = _prune_exec_summary(
                                        raw_data, _exec_extra_keep
                                    )
                                # Per-batch column pruning for Executive Dashboard
                                if _is_exec_dashboard and not raw_data.empty:
                                    raw_data = _prune_exec_dashboard(
                                        raw_data, _ed_extra_keep
                                    )
                                needs_date_postfilter = True
                            else:
                                # Batch by project IDs — all versions
                                # Build a cache key from the query signature
                                _pid_str = ",".join(str(p) for p in sorted(folder_pids))
                                _cache_parts = f"{recipe.query.endpoint}|{combined_filter or ''}|{finding_type}|pids:{_pid_str}"
                                _cache_key = hashlib.sha256(
                                    _cache_parts.encode()
                                ).hexdigest()[:16]

                                if _cache_key in self._findings_cache:
                                    raw_data = self._findings_cache[_cache_key]
                                    self.logger.info(
                                        f"Using in-memory cached findings for {recipe.name} "
                                        f"({len(raw_data)} records, same query as a previous report)"
                                    )
                                else:
                                    # Chunked DF construction — convert each batch
                                    # to a DataFrame immediately so raw list[dict] is
                                    # freed, reducing peak memory by ~40-50%.
                                    chunks: list[pd.DataFrame] = []
                                    # Adaptive batch sizing: reduce batch size for large project counts
                                    batch_size = (
                                        15 if len(folder_pids) > 200 else 25
                                    )  # Keep URLs under server limits
                                    from tqdm import tqdm as _tqdm

                                    total_records = 0
                                    with _tqdm(
                                        range(0, len(folder_pids), batch_size),
                                        desc="Fetching folder findings",
                                        unit=" batches",
                                        leave=False,
                                    ) as pbar:
                                        for i in pbar:
                                            batch_ids = folder_pids[i : i + batch_size]
                                            project_filter_str = f"project=in=({','.join(str(pid) for pid in batch_ids)})"
                                            batch_filters = [
                                                project_filter_str
                                            ] + filters
                                            batch_combined = ";".join(batch_filters)

                                            batch_query = QueryConfig(
                                                endpoint=recipe.query.endpoint,
                                                params=QueryParams(
                                                    limit=recipe.query.params.limit,
                                                    filter=batch_combined,
                                                    finding_type=finding_type,
                                                    archived=False,
                                                    excluded=False,
                                                ),
                                            )
                                            batch_data = (
                                                self.api_client.fetch_all_with_resume(
                                                    batch_query, show_progress=False
                                                )
                                            )
                                            if batch_data:
                                                # Feed CVE IDs to NVD pipeline
                                                if _nvd_on_records is not None:
                                                    _nvd_on_records(batch_data)
                                                # Per-batch flatten for CVA
                                                if _is_cva:
                                                    from fs_report.data_transformer import (
                                                        flatten_records,
                                                    )

                                                    batch_data = flatten_records(
                                                        batch_data,
                                                        fields_to_flatten=[
                                                            "component",
                                                            "project",
                                                            "finding",
                                                        ],
                                                    )
                                                    _cva_pre_flattened = True
                                                chunk_df = pd.DataFrame(batch_data)
                                                total_records += len(batch_data)
                                                del batch_data  # Free batch memory immediately
                                                # Per-batch triage scoring
                                                if (
                                                    _is_triage
                                                    and _triage_weights is not None
                                                ):
                                                    if self._project_folder_map:
                                                        _inject_folder_names_df(
                                                            chunk_df,
                                                            self._project_folder_map,
                                                        )
                                                    chunk_df = _normalize_columns(
                                                        chunk_df
                                                    )
                                                    chunk_df = apply_tiered_gates(
                                                        chunk_df, gates=_triage_gates
                                                    )
                                                    chunk_df = calculate_additive_score(
                                                        chunk_df,
                                                        weights=_triage_weights,
                                                        gates=_triage_gates,
                                                    )
                                                    chunk_df = assign_risk_bands(
                                                        chunk_df,
                                                        weights=_triage_weights,
                                                        gates=_triage_gates,
                                                    )
                                                    _drop = [
                                                        c
                                                        for c in _TRIAGE_DROP_AFTER_SCORE
                                                        if c in chunk_df.columns
                                                    ]
                                                    if _drop:
                                                        chunk_df = chunk_df.drop(
                                                            columns=_drop
                                                        )
                                                # Per-batch flattening for Findings by Project
                                                if (
                                                    _is_findings_by_project
                                                    and _flatten_findings_data
                                                    is not None
                                                ):
                                                    if self._project_folder_map:
                                                        _inject_folder_names_df(
                                                            chunk_df,
                                                            self._project_folder_map,
                                                        )
                                                    chunk_df = _flatten_findings_data(
                                                        chunk_df
                                                    )
                                                    _drop = [
                                                        c
                                                        for c in _FBP_DROP_AFTER_FLATTEN
                                                        if c in chunk_df.columns
                                                    ]
                                                    if _drop:
                                                        chunk_df = chunk_df.drop(
                                                            columns=_drop
                                                        )
                                                # Per-batch column pruning for Executive Summary
                                                if _is_exec_summary:
                                                    chunk_df = _prune_exec_summary(
                                                        chunk_df, _exec_extra_keep
                                                    )
                                                # Per-batch column pruning for Executive Dashboard
                                                if _is_exec_dashboard:
                                                    chunk_df = _prune_exec_dashboard(
                                                        chunk_df, _ed_extra_keep
                                                    )
                                                # Per-batch column pruning for CVA
                                                if _is_cva:
                                                    chunk_df = _prune_cva(
                                                        chunk_df, _cva_extra_keep
                                                    )
                                                chunks.append(chunk_df)
                                                del chunk_df
                                            else:
                                                del batch_data
                                            pbar.set_postfix({"records": total_records})

                                            # Inter-batch delay to reduce server load
                                            # Scales with --request-delay (minimum 1s between batches)
                                            if i + batch_size < len(folder_pids):
                                                time.sleep(
                                                    max(1.0, self.config.request_delay)
                                                )
                                    raw_data = (
                                        pd.concat(chunks, ignore_index=True)
                                        if chunks
                                        else pd.DataFrame()
                                    )
                                    del chunks
                                    self._findings_cache[_cache_key] = raw_data
                                    if _is_exec_summary:
                                        _es_cache_key = _cache_key
                                    if _is_exec_dashboard:
                                        _ed_cache_key = _cache_key

                            self.logger.info(
                                f"Fetched {len(raw_data)} findings for folder scope"
                            )
                        else:
                            # No project filter - get projects scanned in the period, then their findings
                            self.logger.info(
                                f"Finding projects scanned between {self.config.start_date} and {self.config.end_date}..."
                            )

                            # Fetch scans in the period to get project IDs
                            # Use the same early termination logic as Scan Analysis
                            # Apply folder/project filters for cache reuse with Scan Analysis
                            # Note: /scans endpoint has max limit of 100
                            scan_query = QueryConfig(
                                endpoint="/public/v0/scans",
                                params=QueryParams(limit=100, sort="created:desc"),
                            )
                            scan_query = self._apply_scan_filters(scan_query)
                            scans_in_period = self._fetch_scans_with_early_termination(
                                scan_query
                            )

                            # Extract unique project IDs and track latest version per project
                            scanned_project_ids = set()
                            # Track latest version per project: {project_id: (version_id, scan_created)}
                            project_latest_version: dict[str, tuple[str, str]] = {}

                            for scan in scans_in_period:
                                # Scan structure has 'project' and 'projectVersion' at top level
                                project = scan.get("project", {})
                                project_version = scan.get("projectVersion", {})
                                scan_created = scan.get("created", "")

                                # Extract project ID
                                if isinstance(project, dict) and project.get("id"):
                                    project_id = project["id"]
                                    scanned_project_ids.add(project_id)

                                    # Extract version ID
                                    version_id = None
                                    if isinstance(project_version, dict):
                                        version_id = project_version.get("id")

                                    if version_id:
                                        # Keep track of the most recently scanned version per project
                                        if project_id not in project_latest_version:
                                            project_latest_version[project_id] = (
                                                version_id,
                                                scan_created,
                                            )
                                        else:
                                            existing_created = project_latest_version[
                                                project_id
                                            ][1]
                                            if scan_created > existing_created:
                                                project_latest_version[project_id] = (
                                                    version_id,
                                                    scan_created,
                                                )

                            self.logger.info(
                                f"Found {len(scanned_project_ids)} unique projects scanned in the period"
                            )

                            if not scanned_project_ids:
                                self.logger.warning(
                                    "No projects found with scans in the specified period"
                                )
                                raw_data = pd.DataFrame()
                            elif self.config.current_version_only:
                                # Entity-level caching: fetch per-version (shared across reports)
                                version_ids = self._get_latest_version_ids_for_projects(
                                    list(scanned_project_ids)
                                )
                                self.logger.info(
                                    f"Fetching findings for {len(version_ids)} projects (true latest version each)"
                                )
                                _vf = self._get_findings_for_versions(
                                    sorted(version_ids),
                                    finding_type,
                                    category_filter,
                                    on_records=_nvd_on_records,
                                )
                                raw_data = pd.DataFrame(_vf) if _vf else pd.DataFrame()
                                del _vf
                                # Pre-flatten nested dicts for Findings by Project
                                if (
                                    _is_findings_by_project
                                    and _flatten_findings_data is not None
                                    and not raw_data.empty
                                ):
                                    raw_data = _flatten_findings_data(raw_data)
                                    _drop = [
                                        c
                                        for c in _FBP_DROP_AFTER_FLATTEN
                                        if c in raw_data.columns
                                    ]
                                    if _drop:
                                        raw_data = raw_data.drop(columns=_drop)
                                # Per-batch column pruning for Executive Summary
                                if _is_exec_summary and not raw_data.empty:
                                    raw_data = _prune_exec_summary(
                                        raw_data, _exec_extra_keep
                                    )
                                # Per-batch column pruning for Executive Dashboard
                                if _is_exec_dashboard and not raw_data.empty:
                                    raw_data = _prune_exec_dashboard(
                                        raw_data, _ed_extra_keep
                                    )
                                needs_date_postfilter = True
                            else:
                                # Get findings for all scanned projects (all versions)
                                project_ids = list(scanned_project_ids)
                                self.logger.info(
                                    f"Fetching findings for {len(project_ids)} scanned projects (all versions)"
                                )

                                # Batch by project IDs to avoid URL length limits
                                # Chunked DF construction — convert each batch to
                                # a DataFrame immediately so raw list[dict] is freed.
                                pv_chunks: list[pd.DataFrame] = []
                                batch_size = (
                                    10 if len(project_ids) > 200 else 25
                                )  # Projects per batch (kept small — IDs are 20-digit longs)
                                from tqdm import tqdm

                                total_records = 0
                                _log_memory(
                                    self.logger,
                                    f"Before batch fetch ({len(project_ids)} projects, batch_size={batch_size})",
                                )
                                with tqdm(
                                    range(0, len(project_ids), batch_size),
                                    desc="Fetching project findings",
                                    unit=" batches",
                                    leave=False,
                                ) as pbar:
                                    for i in pbar:
                                        batch_ids = project_ids[i : i + batch_size]
                                        project_filter = f"project=in=({','.join(str(pid) for pid in batch_ids)})"

                                        batch_filters = [project_filter] + filters
                                        combined_filter = ";".join(batch_filters)

                                        batch_query = QueryConfig(
                                            endpoint=recipe.query.endpoint,
                                            params=QueryParams(
                                                limit=recipe.query.params.limit,
                                                filter=combined_filter,
                                                finding_type=finding_type,
                                                archived=False,
                                                excluded=False,
                                            ),
                                        )
                                        batch_data = (
                                            self.api_client.fetch_all_with_resume(
                                                batch_query, show_progress=False
                                            )
                                        )
                                        if batch_data:
                                            # Feed CVE IDs to NVD pipeline
                                            if _nvd_on_records is not None:
                                                _nvd_on_records(batch_data)
                                            # Per-batch flatten for CVA
                                            if _is_cva:
                                                from fs_report.data_transformer import (
                                                    flatten_records,
                                                )

                                                batch_data = flatten_records(
                                                    batch_data,
                                                    fields_to_flatten=[
                                                        "component",
                                                        "project",
                                                        "finding",
                                                    ],
                                                )
                                                _cva_pre_flattened = True
                                            chunk_df = pd.DataFrame(batch_data)
                                            total_records += len(batch_data)
                                            del batch_data  # Free batch memory immediately
                                            # Per-batch triage scoring
                                            if (
                                                _is_triage
                                                and _triage_weights is not None
                                            ):
                                                chunk_df = _normalize_columns(chunk_df)
                                                chunk_df = apply_tiered_gates(
                                                    chunk_df, gates=_triage_gates
                                                )
                                                chunk_df = calculate_additive_score(
                                                    chunk_df,
                                                    weights=_triage_weights,
                                                    gates=_triage_gates,
                                                )
                                                chunk_df = assign_risk_bands(
                                                    chunk_df,
                                                    weights=_triage_weights,
                                                    gates=_triage_gates,
                                                )
                                                _drop = [
                                                    c
                                                    for c in _TRIAGE_DROP_AFTER_SCORE
                                                    if c in chunk_df.columns
                                                ]
                                                if _drop:
                                                    chunk_df = chunk_df.drop(
                                                        columns=_drop
                                                    )
                                            # Per-batch flattening for Findings by Project
                                            if (
                                                _is_findings_by_project
                                                and _flatten_findings_data is not None
                                            ):
                                                chunk_df = _flatten_findings_data(
                                                    chunk_df
                                                )
                                                _drop = [
                                                    c
                                                    for c in _FBP_DROP_AFTER_FLATTEN
                                                    if c in chunk_df.columns
                                                ]
                                                if _drop:
                                                    chunk_df = chunk_df.drop(
                                                        columns=_drop
                                                    )
                                            # Per-batch column pruning for Executive Summary
                                            if _is_exec_summary:
                                                chunk_df = _prune_exec_summary(
                                                    chunk_df, _exec_extra_keep
                                                )
                                            # Per-batch column pruning for Executive Dashboard
                                            if _is_exec_dashboard:
                                                chunk_df = _prune_exec_dashboard(
                                                    chunk_df, _ed_extra_keep
                                                )
                                            # Per-batch column pruning for CVA
                                            if _is_cva:
                                                chunk_df = _prune_cva(
                                                    chunk_df, _cva_extra_keep
                                                )
                                            pv_chunks.append(chunk_df)
                                            del chunk_df
                                        else:
                                            del batch_data
                                        pbar.set_postfix({"records": total_records})

                                        # Inter-batch delay to reduce server load
                                        # Scales with --request-delay (minimum 1s between batches)
                                        if i + batch_size < len(project_ids):
                                            time.sleep(
                                                max(1.0, self.config.request_delay)
                                            )
                                raw_data = (
                                    pd.concat(pv_chunks, ignore_index=True)
                                    if pv_chunks
                                    else pd.DataFrame()
                                )
                                del pv_chunks

                                _log_memory(
                                    self.logger,
                                    f"After batch fetch ({total_records} findings)",
                                )
                                if _low_memory and not raw_data.empty:
                                    self.logger.info(
                                        f"[low-memory] Concat'd {total_records:,} findings "
                                        f"({len(raw_data.columns)} columns)"
                                    )
                                self.logger.info(
                                    f"Fetched {total_records} total findings for scanned projects"
                                )
                else:
                    _fetched = self.api_client.fetch_data(recipe.query)
                    raw_data = pd.DataFrame(_fetched) if _fetched else pd.DataFrame()
                    del _fetched

            # --- Date post-filtering for entity-cached paths ---
            # When _get_findings_for_versions was used, data is NOT date-filtered
            # at the API level (entity cache stores ALL findings per version).
            # Apply the date filter in-memory now.
            if (
                needs_date_postfilter
                and isinstance(raw_data, pd.DataFrame)
                and not raw_data.empty
            ):
                if is_operational or _recipe_has_date_filter:
                    start = f"{self.config.start_date}T00:00:00"
                    end = f"{self.config.end_date}T23:59:59"
                    before_count = len(raw_data)
                    detected = raw_data.get(
                        "detected", pd.Series("", index=raw_data.index)
                    ).fillna("")
                    raw_data = raw_data[(detected >= start) & (detected <= end)]
                    self.logger.debug(
                        f"Date post-filter ({self.config.start_date} to {self.config.end_date}): "
                        f"{before_count} -> {len(raw_data)} findings"
                    )
                elif (
                    not is_operational
                    and not _recipe_has_date_filter
                    and getattr(self.config, "detected_after", None)
                ):
                    before_count = len(raw_data)
                    detected = raw_data.get(
                        "detected", pd.Series("", index=raw_data.index)
                    ).fillna("")
                    raw_data = raw_data[
                        detected >= f"{self.config.detected_after}T00:00:00"
                    ]
                    self.logger.debug(
                        f"Detected-after post-filter ({self.config.detected_after}): "
                        f"{before_count} -> {len(raw_data)} findings"
                    )

            # --- Status post-filtering for entity-cached paths ---
            # Entity cache stores ALL findings per version (no status filter).
            # When --open-only is set, exclude resolved/suppressed statuses here.
            if (
                needs_date_postfilter
                and getattr(self.config, "open_only", False)
                and isinstance(raw_data, pd.DataFrame)
                and not raw_data.empty
            ):
                _RESOLVED_STATUSES = {
                    "NOT_AFFECTED",
                    "FALSE_POSITIVE",
                    "RESOLVED",
                    "RESOLVED_WITH_PEDIGREE",
                }
                before_count = len(raw_data)
                status_col = (
                    raw_data.get("status", pd.Series("", index=raw_data.index))
                    .fillna("")
                    .str.upper()
                )
                raw_data = raw_data[~status_col.isin(_RESOLVED_STATUSES)]
                self.logger.debug(
                    f"Open-only post-filter: {before_count} -> {len(raw_data)} findings"
                )

            # --- Category/type post-filtering for multi-type requests ---
            # When --finding-types has multiple types (e.g. cve,sast), we fetch
            # all findings (no type param) and post-filter here.
            # Match on `category` field (uppercase: CVE, SAST_ANALYSIS, etc.)
            # and/or `type` field (lowercase: cve, sast, thirdparty) for types
            # without a documented category mapping.
            if (
                (post_filter_categories or post_filter_types)
                and isinstance(raw_data, pd.DataFrame)
                and not raw_data.empty
            ):
                before_count = len(raw_data)
                _pf_cats = post_filter_categories or set()
                _pf_types = post_filter_types or set()
                cat_col = (
                    raw_data.get("category", pd.Series("", index=raw_data.index))
                    .fillna("")
                    .str.upper()
                )
                type_col = (
                    raw_data.get("type", pd.Series("", index=raw_data.index))
                    .fillna("")
                    .str.lower()
                )
                raw_data = raw_data[cat_col.isin(_pf_cats) | type_col.isin(_pf_types)]
                self.logger.info(
                    f"Finding-type post-filter (categories={_pf_cats}, types={_pf_types}): "
                    f"{before_count} -> {len(raw_data)} findings"
                )

            # Ensure raw_data is a DataFrame at this point
            if not isinstance(raw_data, pd.DataFrame):
                raw_data = pd.DataFrame(raw_data) if raw_data else pd.DataFrame()

            if raw_data.empty:
                self.logger.warning(f"No data returned for recipe: {recipe.name}")
                _operational_recipes = {
                    "Executive Summary",
                    "Scan Analysis",
                    "User Activity",
                }
                if self.config.project_filter and recipe.name in _operational_recipes:
                    self.logger.warning(
                        f"'{recipe.name}' is an operational report that only "
                        f"shows activity within the reporting period "
                        f"({self.config.start_date} to {self.config.end_date}). "
                        f"The project '{self.config.project_filter}' may have "
                        f"no activity in this window. Try a longer --period or "
                        f"use an assessment recipe (e.g. Triage Prioritization, "
                        f"CVE Impact) for point-in-time analysis."
                    )

            # --- Enrich with CVE details for Findings by Project ---
            # If NVD pipeline was started during fetch, collect results now.
            # Otherwise fall back to synchronous fetch for non-batched paths
            # (e.g. single project without --current-version-only).
            if recipe.name == "Findings by Project":
                if _nvd_collect is not None:
                    # Always call collect() to send termination sentinel and
                    # avoid leaking the background consumer thread.
                    self._findings_by_project_cve_details = _nvd_collect()
                elif not raw_data.empty and not getattr(self.config, "skip_nvd", False):
                    self._findings_by_project_cve_details = (
                        self._fetch_findings_cve_details(raw_data)
                    )

            # --- Apply flattening if needed ---
            if (
                recipe.name == "Component Vulnerability Analysis"
                and not _cva_pre_flattened
            ):
                # Flatten nested structures if needed (skipped when already
                # flattened per-batch during chunked fetch)
                fields_to_flatten = ["component", "project", "finding"]
                if not raw_data.empty:
                    from fs_report.data_transformer import flatten_records

                    # flatten_records expects list[dict] — convert, flatten, convert back
                    _records: list[dict[str, Any]] = raw_data.to_dict(orient="records")  # type: ignore[assignment]
                    _records = flatten_records(
                        _records, fields_to_flatten=fields_to_flatten
                    )
                    raw_data = pd.DataFrame(_records) if _records else pd.DataFrame()
                    del _records
            # Prune CVA columns for non-batch paths (after post-accumulation flatten)
            if _is_cva and not _cva_pre_flattened and not raw_data.empty:
                raw_data = _prune_cva(raw_data, _cva_extra_keep)
            # --- Inject project_name if needed ---
            # Only do this if the recipe uses project-level grouping
            # (via transform group_by or transform_function that needs it)
            uses_project = any(
                (t.group_by and "project_name" in t.group_by)
                or (t.calc and t.calc.name == "project_name")
                for t in recipe.transform or []
            ) or recipe.name in ("Executive Dashboard",)
            if uses_project:
                # Fetch all projects and build mapping
                from fs_report.models import QueryConfig, QueryParams

                project_query = QueryConfig(
                    endpoint="/public/v0/projects",
                    params=QueryParams(
                        limit=10000, offset=0, archived=False, excluded=False
                    ),
                )
                projects = self.api_client.fetch_all_with_resume(project_query)

                # Build project mapping, handling different ID formats
                project_map = {}
                for p in projects:
                    pid_val = p.get("id") or p.get("projectId")
                    project_name = p.get("name")
                    if pid_val and project_name:
                        # Convert project_id to string to ensure it's hashable
                        project_map[str(pid_val)] = project_name

                # Inject project_name using vectorized DataFrame helper
                if not raw_data.empty:
                    # Copy to avoid mutating cached DataFrame
                    raw_data = raw_data.copy()
                    _inject_project_names_df(raw_data, project_map)

            # --- Inject folder_name into raw records ---
            # Build the project-to-folder mapping (either from folder scope or from projects endpoint)
            if self._project_folder_map:
                # Folder scoping active — use the pre-built mapping
                pf_map = self._project_folder_map
            elif not raw_data.empty:
                # No folder scoping — try to extract folder from projects data
                pf_map = self._build_project_folder_map_from_projects()
            else:
                pf_map = {}

            if pf_map and not raw_data.empty and "folder_name" not in raw_data.columns:
                # Copy to avoid mutating cached DataFrame (if not already copied above)
                if "project_name" not in raw_data.columns:
                    raw_data = raw_data.copy()
                _inject_folder_names_df(raw_data, pf_map)

            # Handle additional data for multiple charts
            additional_data: dict[str, Any] = {}
            # Add config for pandas transform functions
            additional_data["config"] = self.config
            if self._deployment_context is not None:
                additional_data["deployment_context"] = self._deployment_context
            # Pass recipe parameters so transforms can access them
            if recipe.parameters:
                additional_data["recipe_parameters"] = recipe.parameters

            # Add project data for Scan Analysis (for new vs existing analysis)
            if (
                recipe.name == "Scan Analysis"
                and hasattr(self, "_scan_analysis_project_data")
                and self._scan_analysis_project_data
            ):
                additional_data["projects"] = self._scan_analysis_project_data

            # Inject CVE Impact reachability data, descriptions, and exploits
            if recipe.name == "CVE Impact":
                if (
                    hasattr(self, "_cve_impact_reachability")
                    and self._cve_impact_reachability
                ):
                    additional_data["reachability"] = self._cve_impact_reachability
                # Always inject cve_descriptions when set (even if empty dict)
                # so transforms can distinguish "not called" from "called, all failed"
                if hasattr(self, "_cve_impact_descriptions"):
                    additional_data["cve_descriptions"] = self._cve_impact_descriptions
                if (
                    hasattr(self, "_cve_impact_exploit_details")
                    and self._cve_impact_exploit_details
                ):
                    additional_data["exploit_details"] = (
                        self._cve_impact_exploit_details
                    )
                if (
                    hasattr(self, "_cve_impact_nvd_missing")
                    and self._cve_impact_nvd_missing
                ):
                    additional_data["nvd_missing_cves"] = self._cve_impact_nvd_missing

            # Inject CVE details for Findings by Project
            if recipe.name == "Findings by Project":
                if (
                    hasattr(self, "_findings_by_project_cve_details")
                    and self._findings_by_project_cve_details
                ):
                    additional_data["cve_details"] = (
                        self._findings_by_project_cve_details
                    )
                # Pass domain for FS platform link construction
                additional_data["domain"] = self.config.domain

            # Inject Version Comparison data into additional_data
            if recipe.name == "Version Comparison" and hasattr(
                self, "_version_comparison_data"
            ):
                additional_data.update(self._version_comparison_data)

            # Inject API client and domain for Remediation Package (SBOM fetching)
            if recipe.name == "Remediation Package":
                additional_data["api_client"] = self.api_client
                additional_data["domain"] = self.config.domain

                # AI master switch: --ai off → disable all AI regardless of recipe YAML
                rp = additional_data.get("recipe_parameters", {})
                if not getattr(self.config, "ai", False):
                    rp["ai_live"] = False
                    rp["ai_prompts"] = False
                    rp["ai_analysis"] = False
                else:
                    # AI enabled — respect recipe defaults, apply depth
                    if getattr(self.config, "ai_prompts", False):
                        rp["ai_prompts"] = True
                    if getattr(self.config, "ai_depth", "summary") == "full":
                        rp["ai_analysis"] = True

                # Fetch component details for enriching agent prompts
                if not raw_data.empty:
                    self._fetch_remediation_component_details(raw_data, additional_data)

            # Fetch scoped components for Executive Dashboard
            if recipe.name == "Executive Dashboard" and not raw_data.empty:
                comp_filters = ["type!=file"]
                if self.config.project_filter:
                    try:
                        _proj_id = int(self.config.project_filter)
                        comp_filters.append(f"project=={_proj_id}")
                    except ValueError:
                        comp_filters.append(f"project=={self.config.project_filter}")
                elif self._folder_project_ids:
                    folder_pids = sorted(self._folder_project_ids)
                    # Batch folder project IDs to avoid 414 URL Too Long
                    batch_size = 15 if len(folder_pids) > 200 else 25
                    all_components: list[dict] = []
                    base_filter = ";".join(comp_filters)
                    self.logger.info(
                        f"Fetching scoped components for Executive Dashboard "
                        f"in {len(range(0, len(folder_pids), batch_size))} batches "
                        f"({len(folder_pids)} projects, batch_size={batch_size})"
                    )
                    try:
                        for i in range(0, len(folder_pids), batch_size):
                            batch_ids = folder_pids[i : i + batch_size]
                            pid_filter = (
                                f"project=in=({','.join(str(p) for p in batch_ids)})"
                            )
                            batch_filter = (
                                f"{base_filter};{pid_filter}"
                                if base_filter
                                else pid_filter
                            )
                            batch_query = QueryConfig(
                                endpoint="/public/v0/components",
                                params=QueryParams(
                                    limit=10000,
                                    filter=batch_filter,
                                ),
                            )
                            batch_data = self.api_client.fetch_all_with_resume(
                                batch_query, show_progress=True
                            )
                            if batch_data:
                                all_components.extend(batch_data)
                        additional_data["components"] = all_components
                        self.logger.info(f"Fetched {len(all_components)} components")
                    except Exception as e:
                        self.logger.warning(
                            f"Could not fetch components for Executive Dashboard: {e}"
                        )
                        additional_data["components"] = []
                    # Skip the single-query path below
                    comp_filters = None  # type: ignore[assignment]

                if comp_filters is not None:
                    # For portfolio-wide (no project, no folder), extract
                    # project IDs from already-fetched findings to batch.
                    _portfolio_pids: list[int] = []
                    if (
                        not self.config.project_filter
                        and not self._folder_project_ids
                        and isinstance(raw_data, pd.DataFrame)
                        and not raw_data.empty
                    ):
                        for col in ("projectId", "project_id"):
                            if col in raw_data.columns:
                                _portfolio_pids = sorted(
                                    int(float(p))
                                    for p in raw_data[col].dropna().unique()
                                    if str(p).replace(".", "", 1).isdigit()
                                )
                                break

                    if _portfolio_pids:
                        # Batch like folder-scoped path
                        batch_size = 15 if len(_portfolio_pids) > 200 else 25
                        all_comps: list[dict] = []
                        base_filter = ";".join(comp_filters)
                        self.logger.info(
                            f"Fetching portfolio components for Executive Dashboard "
                            f"in {len(range(0, len(_portfolio_pids), batch_size))} batches "
                            f"({len(_portfolio_pids)} projects)"
                        )
                        try:
                            for i in range(0, len(_portfolio_pids), batch_size):
                                p_batch = _portfolio_pids[i : i + batch_size]
                                pid_filter = (
                                    f"project=in=({','.join(str(p) for p in p_batch)})"
                                )
                                batch_filter = (
                                    f"{base_filter};{pid_filter}"
                                    if base_filter
                                    else pid_filter
                                )
                                batch_query = QueryConfig(
                                    endpoint="/public/v0/components",
                                    params=QueryParams(
                                        limit=10000,
                                        filter=batch_filter,
                                    ),
                                )
                                batch_data = self.api_client.fetch_all_with_resume(
                                    batch_query, show_progress=True
                                )
                                if batch_data:
                                    all_comps.extend(batch_data)
                            additional_data["components"] = all_comps
                            self.logger.info(f"Fetched {len(all_comps)} components")
                        except Exception as e:
                            self.logger.warning(
                                f"Could not fetch components for Executive Dashboard: {e}"
                            )
                            additional_data["components"] = []
                    else:
                        comp_query = QueryConfig(
                            endpoint="/public/v0/components",
                            params=QueryParams(
                                limit=10000,
                                filter=";".join(comp_filters),
                            ),
                        )
                        self.logger.info(
                            f"Fetching scoped components for Executive Dashboard "
                            f"(filter: {comp_query.params.filter})"
                        )
                        try:
                            additional_data["components"] = (
                                self.api_client.fetch_all_with_resume(
                                    comp_query, show_progress=True
                                )
                            )
                            self.logger.info(
                                f"Fetched {len(additional_data['components'])} components"
                            )
                        except Exception as e:
                            self.logger.warning(
                                f"Could not fetch components for Executive Dashboard: {e}"
                            )
                            additional_data["components"] = []

            if recipe.additional_queries:
                for query_name, query_config in recipe.additional_queries.items():
                    self.logger.debug(f"Fetching additional data for {query_name}")
                    self.logger.debug(f"Query config: {query_config}")

                    additional_raw_data = self.api_client.fetch_data(query_config)

                    self.logger.debug(
                        f"Additional data for {query_name}: {len(additional_raw_data) if additional_raw_data else 0} records"
                    )

                    # Apply flattening to additional data if needed
                    if (
                        recipe.name == "Component Vulnerability Analysis"
                        and additional_raw_data
                    ):
                        from fs_report.data_transformer import flatten_records

                        self.logger.info(f"Applying flattening to {query_name} data")
                        additional_raw_data = flatten_records(
                            additional_raw_data,
                            fields_to_flatten=["component", "project", "finding"],
                        )

                    # Inject project names if needed
                    if additional_raw_data and uses_project:
                        for finding in additional_raw_data:
                            project_field = finding.get("project") or finding.get(
                                "projectId"
                            )
                            if project_field:
                                if isinstance(project_field, dict):
                                    project_name = project_field.get("name")
                                    if project_name:
                                        finding["project_name"] = project_name
                                    else:
                                        pid_str = str(
                                            project_field.get("id", project_field)
                                        )
                                        finding["project_name"] = project_map.get(
                                            pid_str, pid_str
                                        )
                                else:
                                    pid_str = str(project_field)
                                    finding["project_name"] = project_map.get(
                                        pid_str, pid_str
                                    )

                    # Apply specific transforms if available
                    if query_name == "open_issues" and recipe.open_issues_transform:
                        additional_data[query_name] = self.transformer.transform(
                            additional_raw_data,
                            recipe.open_issues_transform,
                            additional_data={"config": self.config},
                        )
                    elif (
                        query_name == "scan_frequency"
                        and recipe.scan_frequency_transform
                    ):
                        additional_data[query_name] = self.transformer.transform(
                            additional_raw_data,
                            recipe.scan_frequency_transform,
                            additional_data={"config": self.config},
                        )
                    else:
                        additional_data[query_name] = additional_raw_data

            # Add scan frequency data from main findings if transform is defined
            if recipe.scan_frequency_transform:
                self.logger.debug("Applying scan frequency transform to main data")
                additional_data["scan_frequency"] = self.transformer.transform(
                    raw_data,
                    recipe.scan_frequency_transform,
                    additional_data={"config": self.config},
                )

            # Add open issues data from main findings if transform is defined
            if recipe.open_issues_transform:
                self.logger.debug(
                    f"Applying open issues transform to {len(raw_data)} findings"
                )
                additional_data["open_issues"] = self.transformer.transform(
                    raw_data,
                    recipe.open_issues_transform,
                    additional_data={"config": self.config},
                )

            # Apply transformations (pass additional_data for join support)
            self.logger.debug(f"Applying transformations for recipe: {recipe.name}")
            self.logger.debug(f"Raw data count: {len(raw_data)}")
            if isinstance(raw_data, dict):  # type: ignore[unreachable]
                self.logger.debug(f"Raw data keys: {list(raw_data.keys())}")  # type: ignore[unreachable]
            else:
                self.logger.debug(
                    f"Raw data is a {type(raw_data).__name__}, not logging keys."
                )
            self.logger.debug(f"Additional data keys: {list(additional_data.keys())}")

            # Handle transform_function if present
            transforms_to_apply = recipe.transform
            if hasattr(recipe, "transform_function") and recipe.transform_function:
                from fs_report.models import Transform

                # Create a Transform object with the transform_function
                custom_transform = Transform(
                    transform_function=recipe.transform_function
                )
                transforms_to_apply = [custom_transform]
                self.logger.debug(
                    f"Using custom transform function: {recipe.transform_function}"
                )

            self.logger.debug(
                f"Transform count: {len(transforms_to_apply) if transforms_to_apply else 0}"
            )
            _log_memory(self.logger, f"Before transform ({recipe.name})")
            raw_data_count = len(raw_data) if hasattr(raw_data, "__len__") else 0
            transformed_data = self.transformer.transform(
                raw_data, transforms_to_apply, additional_data=additional_data
            )

            # Handle custom transform functions that return dictionaries with additional data
            if hasattr(recipe, "transform_function") and recipe.transform_function:
                # Check if transform returned a dictionary result in additional_data
                transform_result = additional_data.get("transform_result")
                if transform_result and isinstance(transform_result, dict):
                    self.logger.debug(
                        f"Processing transform result dictionary with keys: {list(transform_result.keys())}"
                    )
                    # Store all keys in additional_data
                    for key, value in transform_result.items():
                        additional_data[key] = value
                    self.logger.debug(
                        "Transform function returned dict with keys, merged into additional_data"
                    )

                    # Write VEX recommendations JSON for Triage Prioritization
                    if recipe.name == "Triage Prioritization":
                        vex_recs = transform_result.get("vex_recommendations", [])
                        if not vex_recs:
                            self.logger.info(
                                "No VEX recommendations generated (no eligible findings)"
                            )
                        if vex_recs:
                            import json

                            # Use the same sanitized directory name as the report renderer
                            sanitized_name = (
                                recipe.name.replace("/", "_")
                                .replace("\\", "_")
                                .replace(":", "_")
                                .replace("*", "_")
                                .replace("?", "_")
                                .replace('"', "_")
                                .replace("<", "_")
                                .replace(">", "_")
                                .replace("|", "_")
                                .strip(" .")
                            )
                            vex_dir = Path(self.config.output_dir) / sanitized_name
                            vex_dir.mkdir(parents=True, exist_ok=True)
                            vex_path = vex_dir / "vex_recommendations.json"
                            with open(vex_path, "w") as f:
                                json.dump(vex_recs, f, indent=2, default=str)
                            self.logger.info(
                                f"Wrote {len(vex_recs)} VEX recommendations to {vex_path}"
                            )
                            # Track for output listing
                            additional_data.setdefault(
                                "_extra_generated_files", []
                            ).append(str(vex_path))

            # When a transform returns a dict with a "main" key, extract the
            # main DataFrame as the primary report data.  The full dict is
            # already available via additional_data["transform_result"].
            if isinstance(transformed_data, dict) and "main" in transformed_data:
                self.logger.debug(
                    "Extracting 'main' DataFrame from transform result dict"
                )
                transformed_data = transformed_data["main"]

            # Promote Remediation Package transform results into additional_data
            # so the HTML template receives actions_df, remediation_summary,
            # charts, etc. as top-level template variables.
            if recipe.name == "Remediation Package":
                tr = additional_data.get("transform_result", {})
                if isinstance(tr, dict):
                    for k, v in tr.items():
                        if k != "main":
                            additional_data[k] = v

            # Apply portfolio transforms if available (for Component Vulnerability Analysis)
            portfolio_data = None
            if hasattr(recipe, "portfolio_transform") and recipe.portfolio_transform:
                self.logger.debug("Applying portfolio transforms")
                portfolio_data = self.transformer.transform(
                    raw_data,
                    recipe.portfolio_transform,
                    additional_data=additional_data,
                )
            # For CVA with transform_function, the result IS the portfolio data
            elif recipe.name == "Component Vulnerability Analysis" and hasattr(
                recipe, "transform_function"
            ):
                self.logger.debug("Setting CVA transform result as portfolio data")
                portfolio_data = transformed_data
                transformed_data = (
                    pd.DataFrame()
                )  # Empty for main data since we only need portfolio data

            # Release the Executive Summary/Dashboard _findings_cache entries
            # early — pruned DataFrames are small enough that cross-report reuse
            # is not worthwhile and we want to free memory before rendering.
            if _es_cache_key is not None:
                self._findings_cache.pop(_es_cache_key, None)
            if _ed_cache_key is not None:
                self._findings_cache.pop(_ed_cache_key, None)

            # Free raw_data now that all transforms have consumed it.
            # (For folder-scoped paths, _findings_cache may still hold a reference
            # for cross-report reuse — that's fine; del here just drops this local ref.)
            del raw_data
            gc.collect()
            _log_memory(self.logger, f"After transforms + gc ({recipe.name})")

            # Create report data
            report_data = ReportData(
                recipe_name=recipe.name,
                data=transformed_data,
                metadata={
                    "raw_count": raw_data_count,
                    "transformed_count": (
                        len(transformed_data)
                        if hasattr(transformed_data, "__len__")
                        else 1
                    ),
                    "portfolio_data": portfolio_data,
                    "recipe": recipe.model_dump(),
                    "cache_stats": self.cache.get_stats(),
                    "additional_data": additional_data,
                    "start_date": self.config.start_date,
                    "end_date": self.config.end_date,
                    "project_filter": self.config.project_filter,
                    "folder_name": self._folder_name,
                    "folder_path": self._folder_path,
                    "folder_filter": self.config.folder_filter,
                    "domain": self.config.domain,
                    "logo_path": self._resolve_logo_path(),
                },
            )

            return report_data

        except Exception as e:
            self.logger.error(f"Error processing recipe {recipe.name}: {e}")
            raise

    def get_cache_stats(self) -> dict[str, Any]:
        """Get cache statistics."""
        return self.cache.get_stats()

    def _resolve_logo_path(self) -> str | None:
        """Resolve the configured logo to a base64 data URI.

        Returns a ``data:image/...;base64,...`` string or *None* if no logo
        is configured or the file cannot be found.
        """
        import base64
        import mimetypes

        logo = getattr(self.config, "logo", None)
        if not logo:
            return None

        path = Path(logo)
        if not path.is_absolute():
            path = Path.home() / ".fs-report" / "logos" / logo

        if not path.is_file():
            self.logger.warning(f"Logo file not found: {path}")
            return None

        suffix = path.suffix.lower()
        allowed = {".png", ".jpg", ".jpeg", ".svg", ".webp"}
        if suffix not in allowed:
            self.logger.warning(
                f"Unsupported logo format '{suffix}'. Use: {', '.join(sorted(allowed))}"
            )
            return None

        size = path.stat().st_size
        if size > 500_000:
            self.logger.warning(
                f"Logo file is {size / 1024:.0f}KB (>500KB recommended maximum)"
            )

        mime = mimetypes.guess_type(str(path))[0] or "application/octet-stream"
        if suffix == ".svg":
            mime = "image/svg+xml"

        data = path.read_bytes()
        b64 = base64.b64encode(data).decode("ascii")
        return f"data:{mime};base64,{b64}"

    @staticmethod
    def _filter_findings_by_version_ids(
        findings: list[dict],
        version_ids: set[int],
    ) -> list[dict]:
        """Keep only findings whose projectVersion.id is in *version_ids*.

        Handles both nested dict keys (live API) and flat dotted keys
        (SQLite cache).  Findings with no extractable version ID are kept
        as a safety net.
        """
        result: list[dict] = []
        for f in findings:
            pv_obj = f.get("projectVersion")
            if isinstance(pv_obj, dict):
                ver_id = pv_obj.get("id")
            else:
                ver_id = f.get("projectVersion.id") or f.get("project_version_id")
            try:
                ver_int = int(ver_id) if ver_id is not None else None
            except (ValueError, TypeError):
                ver_int = None
            if ver_int is None or ver_int in version_ids:
                result.append(f)
        return result

    def _fetch_remediation_component_details(
        self,
        raw_data: "pd.DataFrame",
        additional_data: dict[str, Any],
    ) -> None:
        """Fetch component details from /public/v0/components for Remediation Package.

        Enriches agent prompts with component metadata (type, license, supplier, etc.).
        Results are stored in ``additional_data["component_details"]``.
        """
        from fs_report.models import QueryConfig, QueryParams

        # Determine project scope
        comp_filters = ["type!=file"]
        if self.config.project_filter:
            try:
                _proj_id = int(self.config.project_filter)
                comp_filters.append(f"project=={_proj_id}")
            except ValueError:
                comp_filters.append(f"project=={self.config.project_filter}")
        elif self._folder_project_ids:
            folder_pids = sorted(self._folder_project_ids)
            # Batch folder project IDs to avoid 414 URL Too Long
            batch_size = 15 if len(folder_pids) > 200 else 25
            all_components: list[dict] = []
            base_filter = ";".join(comp_filters)
            self.logger.info(
                f"Fetching component details for Remediation Package "
                f"in {len(range(0, len(folder_pids), batch_size))} batches "
                f"({len(folder_pids)} projects, batch_size={batch_size})"
            )
            try:
                for i in range(0, len(folder_pids), batch_size):
                    batch_ids = folder_pids[i : i + batch_size]
                    pid_filter = f"project=in=({','.join(str(p) for p in batch_ids)})"
                    batch_filter = (
                        f"{base_filter};{pid_filter}" if base_filter else pid_filter
                    )
                    batch_query = QueryConfig(
                        endpoint="/public/v0/components",
                        params=QueryParams(
                            limit=10000,
                            filter=batch_filter,
                        ),
                    )
                    batch_data = self.api_client.fetch_all_with_resume(
                        batch_query, show_progress=True
                    )
                    if batch_data:
                        all_components.extend(batch_data)
                additional_data["component_details"] = all_components
                self.logger.info(f"Fetched {len(all_components)} component details")
            except Exception as e:
                self.logger.warning(
                    f"Could not fetch component details for Remediation Package: {e}"
                )
                additional_data["component_details"] = []
            return

        comp_query = QueryConfig(
            endpoint="/public/v0/components",
            params=QueryParams(
                limit=10000,
                filter=";".join(comp_filters),
            ),
        )
        self.logger.info(
            f"Fetching component details for Remediation Package "
            f"(filter: {comp_query.params.filter})"
        )
        try:
            additional_data["component_details"] = (
                self.api_client.fetch_all_with_resume(comp_query, show_progress=True)
            )
            self.logger.info(
                f"Fetched {len(additional_data['component_details'])} component details"
            )
        except Exception as e:
            self.logger.warning(
                f"Could not fetch component details for Remediation Package: {e}"
            )
            additional_data["component_details"] = []

    def _fetch_cve_reachability(
        self, cve_records: "list[dict] | pd.DataFrame"
    ) -> tuple[dict[str, list[dict]], dict[str, str], dict[str, list[dict]], list[str]]:
        """Fetch per-finding reachability from /findings for dossier mode.

        For each CVE in the records, queries the findings endpoint with
        ``findingId==<cveId>`` to retrieve reachability scores per finding.

        Also fetches CVE descriptions via NVDClient batch lookup and
        exploit details from ``/findings/{findingId}/exploits`` for
        each CVE (using any finding's numeric ``id``).

        Returns:
            Tuple of:
            - reachability_map: cveId -> list of finding dicts
            - descriptions_map: cveId -> NVD description string
            - exploit_details_map: cveId -> list of exploit detail dicts
            - nvd_missing_cves: list of CVE IDs that NVD could not resolve
        """
        from fs_report.models import QueryConfig, QueryParams
        from fs_report.nvd_client import NVD_ATTRIBUTION, NVDClient

        reachability_map: dict[str, list[dict]] = {}
        descriptions_map: dict[str, str] = {}
        exploit_details_map: dict[str, list[dict]] = {}

        # Collect unique CVE IDs from the data
        cve_ids_ordered: list[str] = []
        if isinstance(cve_records, pd.DataFrame):
            # Try cveId first, then cve_id
            if "cveId" in cve_records.columns:
                cve_ids_ordered = cve_records["cveId"].dropna().unique().tolist()
            elif "cve_id" in cve_records.columns:
                cve_ids_ordered = cve_records["cve_id"].dropna().unique().tolist()
        else:
            cve_ids_seen: set[str] = set()
            for rec in cve_records:
                cve_id = rec.get("cveId") or rec.get("cve_id")
                if cve_id and cve_id not in cve_ids_seen:
                    cve_ids_seen.add(cve_id)
                    cve_ids_ordered.append(cve_id)

        if not cve_ids_ordered:
            return reachability_map, descriptions_map, exploit_details_map, []

        self.logger.info(
            f"Enriching {len(cve_ids_ordered)} CVEs with reachability data from /findings"
        )

        # Batch-fetch NVD descriptions upfront (unless --no-nvd)
        nvd_missing_cves: list[str] = []
        if not getattr(self.config, "skip_nvd", False):
            self._check_cancel()
            nvd = NVDClient(
                api_key=getattr(self.config, "nvd_api_key", None),
                cache_dir=getattr(self.config, "cache_dir", None),
                cache_ttl=max(getattr(self.config, "cache_ttl", 0) or 0, 86400),
                cancel_event=self._cancel_event,
            )
            self.logger.info(NVD_ATTRIBUTION)
            nvd_results = nvd.get_batch(cve_ids_ordered, progress=True)
            nvd_missing_cves = list(nvd.last_batch_missing)
            if nvd_missing_cves:
                self.logger.info(
                    f"NVD: {len(nvd_results)}/{len(cve_ids_ordered)} CVEs resolved"
                )
            for nvd_cve_id, nvd_rec in nvd_results.items():
                if nvd_rec.description:
                    descriptions_map[nvd_cve_id] = nvd_rec.description
        else:
            self.logger.info("Skipping NVD enrichment for CVE Impact (--no-nvd)")
            nvd_missing_cves = list(cve_ids_ordered)

        # Pre-fetch authoritative latest version IDs (from
        # defaultBranch.latestVersion.id) so we can filter per-CVE
        # findings to only the current version of each project.
        latest_version_ids: set[int] | None = None
        if self.config.current_version_only:
            try:
                auth_ids = self._get_latest_version_ids()
                # Force int conversion — the helpers don't always coerce,
                # and SQLite cache can stringify values.
                latest_version_ids = set()
                for v in auth_ids:
                    try:
                        latest_version_ids.add(int(v))
                    except (ValueError, TypeError):
                        pass
            except Exception:
                self.logger.warning(
                    "Failed to resolve authoritative version IDs",
                    exc_info=True,
                )
            if latest_version_ids:
                self.logger.info(
                    f"Version filter: {len(latest_version_ids)} authoritative "
                    f"latest version IDs"
                )
            else:
                self.logger.warning(
                    "No authoritative version IDs resolved; "
                    "version filter will be skipped"
                )
                latest_version_ids = None

        # Fetch reachability and exploit details per CVE
        for cve_id in sorted(cve_ids_ordered):
            self._check_cancel()
            finding_query = QueryConfig(
                endpoint="/public/v0/findings",
                params=QueryParams(
                    limit=10000,
                    filter=f"findingId=={cve_id}",
                ),
            )
            try:
                findings = self.api_client.fetch_all_with_resume(finding_query)
                if latest_version_ids is not None and findings:
                    before = len(findings)
                    filtered = self._filter_findings_by_version_ids(
                        findings, latest_version_ids
                    )
                    if filtered:
                        findings = filtered
                        self.logger.debug(
                            f"  Version post-filter: {before} -> "
                            f"{len(findings)} findings for {cve_id}"
                        )
                    else:
                        # Don't discard all findings — keep unfiltered so
                        # the report shows reachability data rather than
                        # UNKNOWN for every CVE.
                        self.logger.debug(
                            f"  Version filter would remove all {before} "
                            f"findings for {cve_id}; keeping unfiltered"
                        )
                reachability_map[cve_id] = findings
                self.logger.debug(
                    f"  {cve_id}: {len(findings)} findings with reachability"
                )

                # Fetch exploit details using the finding's numeric id
                if findings:
                    f0 = findings[0]
                    finding_numeric_id = f0.get("id")
                    pv_obj = f0.get("projectVersion")
                    pv_id = (
                        pv_obj.get("id")
                        if isinstance(pv_obj, dict)
                        else f0.get("project_version_id")
                    )
                    if finding_numeric_id and pv_id:
                        fid = str(finding_numeric_id)
                        pvid = str(pv_id)
                        exploits = self._fetch_cve_exploits(pvid, fid, cve_id)
                        if exploits:
                            exploit_details_map[cve_id] = exploits
            except Exception as exc:
                self.logger.warning(f"Failed to fetch reachability for {cve_id}: {exc}")
                reachability_map[cve_id] = []

        return reachability_map, descriptions_map, exploit_details_map, nvd_missing_cves

    def _start_nvd_pipeline(
        self,
    ) -> "tuple[Callable[[list[dict]], None], Callable[[], dict[str, dict[str, str]]]]":
        """Start a background NVD lookup pipeline.

        Returns ``(on_records, collect)`` where:

        * ``on_records(records)`` — callback to invoke with each batch of raw
          finding dicts as they become available.  Extracts ``findingId``
          values and queues them for the background NVD thread.
        * ``collect()`` — blocks until the pipeline is finished and returns
          the CVE details dict (same shape as ``_fetch_findings_cve_details``).

        The background thread processes CVE IDs incrementally: as soon as
        ``on_records`` is called (e.g. with cached findings), the thread
        starts NVD/OSV lookups.  Subsequent calls add new CVE IDs.  This
        overlaps NVD I/O with the remaining FS API fetch and its cooldowns.
        """
        import queue as _queue
        from concurrent.futures import ThreadPoolExecutor

        from fs_report.nvd_client import NVD_ATTRIBUTION, NVDClient

        cve_queue: _queue.Queue[set[str] | None] = _queue.Queue()

        def _consumer() -> dict[str, dict[str, str]]:
            nvd = NVDClient(
                api_key=getattr(self.config, "nvd_api_key", None),
                cache_dir=getattr(self.config, "cache_dir", None),
                cache_ttl=max(getattr(self.config, "cache_ttl", 0) or 0, 86400),
                cancel_event=self._cancel_event,
            )
            self.logger.info(NVD_ATTRIBUTION)

            seen: set[str] = set()
            results: dict[str, dict[str, str]] = {}
            batch_num = 0

            while True:
                try:
                    batch = cve_queue.get(timeout=1.0)
                except _queue.Empty:
                    continue
                if batch is None:  # sentinel
                    break
                new_ids = batch - seen
                seen.update(batch)
                if new_ids:
                    batch_num += 1
                    self.logger.info(
                        f"NVD background: looking up {len(new_ids)} new CVEs "
                        f"({len(seen)} total so far)"
                    )
                    nvd_results = nvd.get_batch(list(new_ids), progress=True)
                    results.update(
                        {
                            cve_id: {
                                "description": rec.description,
                                "cvss_v2_vector": rec.cvss_v2_vector,
                                "cvss_v3_vector": rec.cvss_v3_vector,
                            }
                            for cve_id, rec in nvd_results.items()
                        }
                    )
                    self.logger.info(
                        f"NVD background: batch {batch_num} done, "
                        f"{len(results)}/{len(seen)} CVEs resolved so far"
                    )

            self.logger.info(
                f"NVD background: completed — {len(results)}/{len(seen)} CVEs"
            )
            return results

        executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="nvd")
        future = executor.submit(_consumer)
        executor.shutdown(wait=False)
        self.logger.info("NVD/OSV pipeline started in background thread")

        def on_records(records: list[dict]) -> None:
            ids: set[str] = {r["findingId"] for r in records if r.get("findingId")}
            if ids:
                cve_queue.put(ids)

        def collect() -> dict[str, dict[str, str]]:
            cve_queue.put(None)  # sentinel
            return future.result()

        return on_records, collect

    def _fetch_findings_cve_details(
        self, raw_data: "list[dict] | pd.DataFrame"
    ) -> dict[str, dict[str, str]]:
        """Fetch CVE details (description, CVSS vectors) for Findings by Project.

        Uses NVDClient with SQLite caching, tqdm progress, and NVD API rate
        limiting instead of per-finding FS API calls.

        Returns a mapping of CVE ID -> {"description": str, "cvss_v2_vector": str, "cvss_v3_vector": str}.
        """
        from fs_report.nvd_client import NVD_ATTRIBUTION, NVDClient

        # Collect unique CVE IDs from findings
        cve_ids: list[str] = []
        if isinstance(raw_data, pd.DataFrame):
            cve_ids = (
                raw_data["findingId"].dropna().unique().tolist()
                if "findingId" in raw_data.columns
                else []
            )
        else:
            seen: set[str] = set()
            for finding in raw_data:
                cve_id = finding.get("findingId")
                if cve_id and cve_id not in seen:
                    seen.add(cve_id)
                    cve_ids.append(cve_id)

        if not cve_ids:
            return {}

        self.logger.info(
            f"Fetching CVE details for {len(cve_ids)} unique CVEs (Findings by Project)"
        )

        self._check_cancel()

        nvd = NVDClient(
            api_key=getattr(self.config, "nvd_api_key", None),
            cache_dir=getattr(self.config, "cache_dir", None),
            cache_ttl=max(getattr(self.config, "cache_ttl", 0) or 0, 86400),
            cancel_event=self._cancel_event,
        )
        self.logger.info(NVD_ATTRIBUTION)

        nvd_results = nvd.get_batch(cve_ids, progress=True)

        results: dict[str, dict[str, str]] = {
            cve_id: {
                "description": rec.description,
                "cvss_v2_vector": rec.cvss_v2_vector,
                "cvss_v3_vector": rec.cvss_v3_vector,
            }
            for cve_id, rec in nvd_results.items()
        }

        self.logger.info(f"Fetched CVE details for {len(results)}/{len(cve_ids)} CVEs")
        return results

    def _fetch_cve_exploits(
        self, project_version_id: str, finding_numeric_id: str, cve_id: str
    ) -> list[dict]:
        """Fetch exploit details from /findings/{pvId}/{findingId}/exploits.

        The response is a nested dict keyed by CVE ID::

            {
              "CVE-XXXX": {
                "request": {
                  "exploits": [
                    {
                      "url": "https://...",
                      "name": "exploit description",
                      "refsource": "github-exploits",
                      "exploit_maturity": "poc",
                      "exploit_type": "denial-of-service",
                      ...
                    }
                  ],
                  "counts": {"exploits": 5, ...},
                  "epss": {...}
                }
              }
            }

        Returns a list of exploit detail dicts, or empty list on failure.
        """
        import time

        url = (
            f"{self.api_client.base_url}/public/v0/findings"
            f"/{project_version_id}/{finding_numeric_id}/exploits"
        )
        result: list[dict] = []
        try:
            resp = self.api_client.client.get(url)
            resp.raise_for_status()
            data = resp.json()
            if isinstance(data, dict):
                # Navigate: data[cveId].request.exploits
                cve_entry = data.get(cve_id, {})
                if isinstance(cve_entry, dict):
                    request_obj = cve_entry.get("request", {})
                    if isinstance(request_obj, dict):
                        exploits_list = request_obj.get("exploits", [])
                        if isinstance(exploits_list, list):
                            result = exploits_list
            elif isinstance(data, list):
                result = data
            if result:
                self.logger.debug(f"  {cve_id}: fetched {len(result)} exploit details")
        except httpx.HTTPStatusError as exc:
            if exc.response.status_code == 404:
                self.logger.debug(f"No exploit details for {cve_id} (404)")
            else:
                self.logger.warning(
                    f"Could not fetch exploit details for {cve_id} "
                    f"(pv={project_version_id}, finding={finding_numeric_id}): {exc}"
                )
        except Exception as exc:
            self.logger.warning(
                f"Could not fetch exploit details for {cve_id} "
                f"(pv={project_version_id}, finding={finding_numeric_id}): {exc}"
            )
        # Polite delay between API calls
        time.sleep(0.3)
        return result

    def _fetch_scans_with_early_termination(
        self, query_config: "QueryConfig"
    ) -> list[dict]:
        """
        Fetch scans sorted by -created with early termination.
        Stops fetching when we've passed the start date (no more relevant scans).

        The scans endpoint does NOT support RSQL date filtering, so we must
        paginate manually and stop when scans are older than our window.

        Results are cached in-memory (same run) and SQLite (cross-run).

        Note: We extend the cutoff by 7 days to capture scans that were CREATED
        before the period but COMPLETED within it (e.g., long-running scans).
        The transform will do final filtering to include scans completed in range.
        """
        from datetime import datetime, timedelta

        from tqdm import tqdm

        # --- In-memory cache (shared across reports in the same run) ---
        _cache_parts = f"scans|{query_config.params.filter or ''}|{self.config.start_date}|{self.config.end_date}"
        _cache_key = hashlib.sha256(_cache_parts.encode()).hexdigest()[:16]
        if _cache_key in self._findings_cache:
            cached: list[dict[Any, Any]] = self._findings_cache[_cache_key]
            self.logger.info(f"Using in-memory cached scans ({len(cached)} records)")
            return cached

        # --- SQLite cache (cross-run) ---
        sqlite_params = {
            "filter": query_config.params.filter or "",
            "sort": "created:desc",
            "start_date": self.config.start_date,
            "end_date": self.config.end_date,
        }
        if (
            self.api_client.sqlite_cache
            and self.api_client.cache_ttl > 0
            and self.api_client.sqlite_cache.is_cache_valid(
                "/public/v0/scans", sqlite_params, self.api_client.cache_ttl
            )
        ):
            sqlite_cached = self.api_client.sqlite_cache.get_cached_data(
                "/public/v0/scans", sqlite_params
            )
            if sqlite_cached is not None:
                self.logger.info(
                    f"Using SQLite cached scans ({len(sqlite_cached)} records)"
                )
                self._findings_cache[_cache_key] = sqlite_cached
                return sqlite_cached

        # --- Fetch from API with early termination ---
        # Parse start date for comparison
        # Extend by 7 days to capture scans that completed in range but started earlier
        actual_start = datetime.fromisoformat(f"{self.config.start_date}T00:00:00")
        start_date = actual_start - timedelta(days=7)

        # Ensure we're sorting by created:desc (newest first)
        from fs_report.models import QueryConfig, QueryParams

        sorted_query = QueryConfig(
            endpoint=query_config.endpoint,
            params=QueryParams(
                filter=query_config.params.filter,
                sort="created:desc",  # Force sort by newest first
                limit=query_config.params.limit or 100,
                offset=0,
            ),
        )

        all_scans: list[dict[str, Any]] = []
        offset = 0
        limit = sorted_query.params.limit or 100
        done = False
        consecutive_old_pages = 0
        old_page_threshold = (
            3  # Stop after N consecutive pages where majority of scans are old
        )

        self.logger.info(
            f"Fetching scans with early termination (extended to {start_date.date()} to capture completions in {self.config.start_date} - {self.config.end_date})"
        )

        import random

        max_retries = 8
        max_pages = 500  # Hard upper bound to prevent infinite loops
        pages_fetched = 0

        with tqdm(desc="Fetching scans", unit=" records", leave=False) as pbar:
            while not done and pages_fetched < max_pages:
                # Update query with current offset
                page_query = QueryConfig(
                    endpoint=sorted_query.endpoint,
                    params=QueryParams(
                        filter=sorted_query.params.filter,
                        sort=sorted_query.params.sort,
                        limit=limit,
                        offset=offset,
                    ),
                )

                # Fetch this page with retry logic
                page_data = None
                for retry_count in range(max_retries):
                    try:
                        page_data = self.api_client.fetch_data(page_query)
                        break
                    except Exception as e:
                        wait = (2 ** min(retry_count, 6)) + random.uniform(0, 1)
                        self.logger.debug(
                            f"Transient error at offset {offset}: {e}. Retrying in {wait:.1f}s..."
                        )
                        time.sleep(wait)
                        if retry_count >= max_retries - 1:
                            if all_scans:
                                self.logger.warning(
                                    f"Max retries exceeded at offset {offset}. "
                                    f"Returning {len(all_scans)} partial results."
                                )
                                done = True
                                break
                            self.logger.error(
                                f"Max retries exceeded at offset {offset}. Aborting."
                            )
                            raise

                pages_fetched += 1
                if not page_data:
                    break

                # Check each scan and count old vs new
                old_scans_in_page = 0
                new_scans_in_page = 0

                for scan in page_data:
                    scan_created = scan.get("created")
                    scan_in_range = True  # Assume in range unless proven otherwise

                    if scan_created:
                        try:
                            # Parse the scan created timestamp
                            if isinstance(scan_created, str):
                                # Handle various timestamp formats
                                scan_dt = datetime.fromisoformat(
                                    scan_created.replace("Z", "+00:00").split("+")[0]
                                )
                            else:
                                scan_dt = scan_created

                            # Check if scan is before our start date
                            if scan_dt < start_date:
                                scan_in_range = False
                                old_scans_in_page += 1
                            else:
                                new_scans_in_page += 1
                        except (ValueError, TypeError):
                            new_scans_in_page += (
                                1  # Include scans with unparseable dates
                            )
                    else:
                        new_scans_in_page += 1  # Include scans without dates

                    # Only include scans that are in our date range
                    if scan_in_range:
                        all_scans.append(scan)

                pbar.update(len(page_data))
                pbar.set_postfix({"total": len(all_scans), "old": old_scans_in_page})

                # Check if majority of this page is old scans
                total_in_page = old_scans_in_page + new_scans_in_page
                if total_in_page > 0 and old_scans_in_page > (total_in_page / 2):
                    consecutive_old_pages += 1
                    self.logger.debug(
                        f"Page at offset {offset}: {old_scans_in_page}/{total_in_page} old, consecutive_old={consecutive_old_pages}"
                    )
                    if consecutive_old_pages >= old_page_threshold:
                        self.logger.debug(
                            f"Stopping: {old_page_threshold} consecutive majority-old pages reached"
                        )
                        done = True
                else:
                    # Reset counter if we get a page with mostly new scans
                    if consecutive_old_pages > 0:
                        self.logger.debug(
                            f"Page at offset {offset}: {new_scans_in_page}/{total_in_page} new, resetting consecutive counter"
                        )
                    consecutive_old_pages = 0

                # Only stop on truly empty pages - some APIs return partial pages mid-stream
                if len(page_data) == 0:
                    self.logger.debug(f"Stopping: Empty page at offset {offset}")
                    break
                elif len(page_data) < limit:
                    self.logger.debug(
                        f"Partial page at offset {offset}: {len(page_data)} records (continuing)"
                    )

                offset += limit

        if pages_fetched >= max_pages:
            self.logger.warning(
                f"Scan fetch hit max_pages limit ({max_pages}). "
                f"Results may be incomplete."
            )
        self.logger.info(
            f"Fetched {len(all_scans)} scans (early termination saved fetching older scans)"
        )

        # --- Store in caches ---
        self._findings_cache[_cache_key] = all_scans

        # Store in SQLite for cross-run reuse
        if self.api_client.sqlite_cache and self.api_client.cache_ttl > 0:
            qh = self.api_client.sqlite_cache.start_fetch(
                "/public/v0/scans", sqlite_params, self.api_client.cache_ttl
            )
            self.api_client.sqlite_cache.store_records(
                qh, "/public/v0/scans", all_scans
            )
            self.api_client.sqlite_cache.complete_fetch(qh)

        return all_scans

    def _apply_scan_filters(self, query_config: Any) -> Any:
        """Apply project, version, and folder filtering to scan queries."""
        from fs_report.models import QueryConfig, QueryParams

        # Start with the original filter
        original_filter = query_config.params.filter or ""

        # Build additional filters for project and version
        additional_filters = []

        if self.config.project_filter:
            try:
                project_id = int(self.config.project_filter)
                additional_filters.append(f"project=={project_id}")
                self.logger.debug(
                    f"Added project ID filter to scans: project=={project_id}"
                )
            except ValueError:
                # Not an integer, treat as project name
                additional_filters.append(f"project=={self.config.project_filter}")
                self.logger.debug(
                    f"Added project name filter to scans: project=={self.config.project_filter}"
                )
        elif self._folder_project_ids:
            # Folder scoping — add project=in=() filter for scans
            folder_pids = sorted(self._folder_project_ids)
            additional_filters.append(
                f"project=in=({','.join(str(pid) for pid in folder_pids)})"
            )
            self.logger.debug(
                f"Added folder project filter to scans: {len(folder_pids)} projects"
            )

        if self.config.version_filter:
            additional_filters.append(f"projectVersion=={self.config.version_filter}")
            self.logger.debug(
                f"Added version filter to scans: projectVersion=={self.config.version_filter}"
            )

        if getattr(self.config, "scan_types", None):
            types_list = [
                t.strip().upper() for t in str(self.config.scan_types).split(",")
            ]
            if len(types_list) == 1:
                additional_filters.append(f"type=={types_list[0]}")
            else:
                additional_filters.append(f"type=in=({','.join(types_list)})")

        if getattr(self.config, "scan_statuses", None):
            statuses_list = [
                s.strip().upper() for s in str(self.config.scan_statuses).split(",")
            ]
            if len(statuses_list) == 1:
                additional_filters.append(f"status=={statuses_list[0]}")
            else:
                additional_filters.append(f"status=in=({','.join(statuses_list)})")

        # Combine filters
        if additional_filters:
            combined_filter = ";".join(additional_filters)
            if original_filter:
                final_filter = f"{original_filter};{combined_filter}"
            else:
                final_filter = combined_filter

            # Create new query config with filters
            return QueryConfig(
                endpoint=query_config.endpoint,
                params=QueryParams(
                    filter=final_filter,
                    sort=query_config.params.sort,
                    limit=query_config.params.limit,
                    offset=query_config.params.offset,
                ),
            )

        # No additional filters, return original query
        return query_config

    def clear_cache(self) -> None:
        """Clear the data cache."""
        self.cache.clear()
        self.logger.info("Cache cleared")
