###############################################################################
# (c) Copyright 2020-2024 CERN for the benefit of the LHCb Collaboration      #
#                                                                             #
# This software is distributed under the terms of the GNU General Public      #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
#                                                                             #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization  #
# or submit itself to any jurisdiction.                                       #
###############################################################################
"""Core logic for creating minimal reproducers from failed grid jobs.

Decomposes the slicing pipeline into independently testable functions:
  1. Find the failed step in a downloaded job directory
  2. Parse Gaudi logs for event information
  3. Create modified prodConf files for reproducing failures
  4. Run the reproducer and analyze output
  5. Extract single events and package minimal reproducers
"""

import json
import re
import shutil
import subprocess
import threading
import xml.etree.ElementTree as ET
from collections.abc import Callable
from dataclasses import dataclass, field
from pathlib import Path

import yaml


@dataclass
class InputFileStatus:
    """Status of an input file from the summary XML.

    Attributes:
        lfn: The LFN of the input file (e.g. "LFN:/lhcb/...").
        status: "full" if fully processed, "part" if partially processed.
        events_read: Number of events read from this file.
    """

    lfn: str
    status: str
    events_read: int | None = None


@dataclass
class FailedStep:
    """Information about a failed step in a job."""

    step_id: str
    xml_file: Path
    log_file: Path
    prodconf_file: Path | None = None
    input_file_statuses: list[InputFileStatus] | None = None


@dataclass
class EventInfo:
    """Information about the last event processed before failure.

    Attributes:
        event_number: The overall event number (across all input files).
        slot: The processing slot number.
        can_slice_single: True if new-style HLTControlFlowMgr SUCCESS messages
            were found, allowing slicing down to a single event. False if only
            old-style RootIOAlg SUCCESS messages were found (coarser slicing).
        event_in_file: The in-file event number (only set for old-style messages).
        lfn: The LFN of the file being read (only set for old-style messages).
        suspect_lfns: LFNs most likely to contain the error.
        suspect_event_in_file: Best guess of in-file event number, or None
            if we can't narrow it down to one event in one file.
        failed_event_overall: Event number from HLTControlFlowMgr FATAL.
    """

    event_number: int
    slot: int
    can_slice_single: bool = True
    event_in_file: int | None = None
    lfn: str | None = None
    unprocessed_lfns: list[str] = field(default_factory=list)
    suspect_lfns: list[str] = field(default_factory=list)
    suspect_event_in_file: int | None = None
    failed_event_overall: int | None = None


@dataclass
class ReproducerOutput:
    """Parsed output from running a reproducer.

    Attributes:
        last_event_in_file: Last in-file event number from RootIOAlg (old style).
        last_event_lfn: LFN of the last-read file from RootIOAlg (old style).
        last_event_overall: Last overall event count from RootIOAlg (old style).
        last_processing_event: Last event from HLTControlFlowMgr SUCCESS (new style).
        failed_event_overall: Event number from HLTControlFlowMgr FATAL (bonus).
        exact_problem_event: Calculated exact in-file event that caused failure.
        can_slice_single: Whether new-style messages allow single-event precision.
        returncode: Exit code from the reproducer run.
    """

    last_event_in_file: int | None = None
    last_event_lfn: str | None = None
    last_event_overall: int | None = None
    last_processing_event: int | None = None
    failed_event_overall: int | None = None
    exact_problem_event: int | None = None
    can_slice_single: bool = False
    returncode: int = 0


CVMFS_DBASE = Path("/cvmfs/lhcb.cern.ch/lib/lhcb/DBASE")


@dataclass
class DataPackage:
    """A data package directory from CVMFS to include in the reproducer."""

    name: str
    version: str
    cvmfs_path: Path


def find_failed_step(job_dir: Path) -> FailedStep:
    """Scan summary*.xml files for a step with <success>false</success>.

    Args:
        job_dir: Path to the downloaded job directory.

    Returns:
        FailedStep with step_id, xml_file, log_file, and optionally prodconf_file.

    Raises:
        FileNotFoundError: If no XML summary files are found.
        ValueError: If no failed steps are found.
    """
    xml_files = sorted(job_dir.glob("summary*.xml"))
    if not xml_files:
        raise FileNotFoundError(f"No XML summary files found in {job_dir}")

    for xml_file in xml_files:
        try:
            tree = ET.parse(xml_file)
            root = tree.getroot()
            success_elem = root.find(".//success")

            if (
                success_elem is not None
                and success_elem.text
                and success_elem.text.strip().lower() == "false"
            ):
                step_id = xml_file.stem.replace("summary", "")
                log_file = job_dir / f"{step_id}.log"

                # Look for corresponding prodConf JSON
                prodconf_file = job_dir / f"prodConf_{step_id}.json"
                if not prodconf_file.exists():
                    # Try alternative naming patterns
                    prodconf_candidates = list(
                        job_dir.glob(f"prodConf*{step_id}*.json")
                    )
                    prodconf_file = (
                        prodconf_candidates[0] if prodconf_candidates else None
                    )

                input_file_statuses = parse_input_file_statuses(root)

                return FailedStep(
                    step_id=step_id,
                    xml_file=xml_file,
                    log_file=log_file,
                    prodconf_file=prodconf_file,
                    input_file_statuses=input_file_statuses or None,
                )
        except ET.ParseError:
            continue

    raise ValueError("No failed steps found in XML summaries")


def parse_input_file_statuses(root: ET.Element) -> list[InputFileStatus]:
    """Parse input file statuses from a summary XML root element.

    The summary XML has a structure like::

        <input>
          <file name="LFN:..." status="full">947185</file>
          <file name="LFN:..." status="part">149417</file>
        </input>

    Files with ``status="full"`` were fully processed.  Files with
    ``status="part"`` were partially processed (the error likely occurred
    here).  Files listed in the prodConf but absent from the XML were
    never opened.

    Args:
        root: The parsed XML root element.

    Returns:
        List of InputFileStatus entries.  Empty if the XML has no
        ``<input>`` section or no ``<file>`` elements with status
        attributes (e.g. after a segfault).
    """
    input_elem = root.find(".//input")
    if input_elem is None:
        return []

    statuses = []
    for file_elem in input_elem.findall("file"):
        name = file_elem.get("name", "")
        status = file_elem.get("status", "")
        if not name or not status:
            continue

        events_read = None
        if file_elem.text and file_elem.text.strip().isdigit():
            events_read = int(file_elem.text.strip())

        statuses.append(
            InputFileStatus(
                lfn=name,
                status=status,
                events_read=events_read,
            )
        )

    return statuses


def find_last_event(
    log_file: Path,
    input_file_statuses: list[InputFileStatus] | None = None,
) -> EventInfo:
    """Parse Gaudi log for event progress messages.

    Tries two patterns in order of preference:

    1. **New style** (can slice to single event)::

        HLTControlFlowMgr SUCCESS Processing event N in slot M

       For new style, also tracks LFNs that were opened but not fully processed:
       - Finds all LFNs from: RootIOAlg INFO DATAFILE='LFN:...'
       - Finds all LFNs from: RootIOHandler INFO Fully processed input file LFN:...
       - The difference is stored in EventInfo.unprocessed_lfns for testing

    2. **Old style** (can only slice to ~100-event window)::

        RootIOAlg SUCCESS Reading Event record N within LFN:..., M overall

    **Run 2 style**: If Run 2 EventSelector messages are detected, raises NotImplementedError
    as precise slicing is not yet implemented for Run 2 jobs::

        EventSelector SUCCESS Reading Event record M. Record number within stream 1: N

    Also checks for the bonus FATAL pattern::

        HLTControlFlowMgr FATAL Event N on slot M failed!

    Args:
        log_file: Path to the step log file.

    Returns:
        EventInfo with event_number, slot, and can_slice_single flag.
        For new-style logs, also includes unprocessed_lfns list.

    Raises:
        FileNotFoundError: If the log file doesn't exist.
        ValueError: If no event processing messages are found.
        NotImplementedError: If Run 2 style messages are detected.
    """
    if not log_file.exists():
        raise FileNotFoundError(f"Log file not found: {log_file}")

    new_style = re.compile(
        r"HLTControlFlowMgr\s+SUCCESS\s+Processing event\s+(\d+)\s+in slot\s+(\d+)"
    )  # DV v66r9
    old_style = re.compile(
        r"RootIOAlg\s+SUCCESS\s+Reading Event record\s+(\d+)\s+within\s+(LFN:[^,]+),\s+(\d+)\s+overall"
    )
    run2_style = re.compile(
        r"EventSelector\s+SUCCESS\s+Reading Event record\s+(\d+)\.\s+Record number within stream (\d+):\s+(\d+)"
    )
    run2_lfn = re.compile(
        r"EventSelector\s+INFO\s+Stream:EventSelector\.DataStreamTool_(\d+)\s+Def:DATAFILE='(LFN:[^']+)'"
    )
    # Track LFNs opened vs fully processed
    rootioalg_datafile = re.compile(r"RootIOAlg\s+INFO\s+DATAFILE='(LFN:[^']+)'")
    rootiohandler_processed = re.compile(
        r"RootIOHandler\s+INFO\s+Fully processed input file\s+(LFN:\S+)"
    )
    fatal_pattern = re.compile(
        r"HLTControlFlowMgr\s+FATAL\s+Event\s+(\d+)\s+on\s+slot\s+(\d+)\s+failed"
    )

    last_new = None
    last_old = None
    last_run2 = None
    failed_event: int | None = None
    # Map stream number to LFN for Run 2 style
    stream_to_lfn: dict[int, str] = {}
    # Track opened and processed LFNs
    opened_lfns: set[str] = set()
    processed_lfns: set[str] = set()

    with open(log_file) as f:
        for line in f:
            m = new_style.search(line)
            if m:
                last_new = EventInfo(
                    event_number=int(m.group(1)),
                    slot=int(m.group(2)),
                    can_slice_single=True,
                )

            m = old_style.search(line)
            if m:
                last_old = EventInfo(
                    event_number=int(m.group(3)),  # overall count
                    slot=0,
                    can_slice_single=False,
                    event_in_file=int(m.group(1)),
                    lfn=m.group(2),
                )

            m = run2_lfn.search(line)
            if m:
                stream_num = int(m.group(1))
                lfn = m.group(2)
                stream_to_lfn[stream_num] = lfn

            m = run2_style.search(line)
            if m:
                stream_num = int(m.group(2))
                last_run2 = EventInfo(
                    event_number=int(m.group(1)),  # overall event record
                    slot=0,
                    can_slice_single=True,  # Will raise NotImplementedError (Run 2 not supported)
                    event_in_file=int(m.group(3)),  # Record within stream
                    lfn=stream_to_lfn.get(stream_num),  # Get LFN from stream mapping
                )

            m = rootioalg_datafile.search(line)
            if m:
                opened_lfns.add(m.group(1))

            m = rootiohandler_processed.search(line)
            if m:
                processed_lfns.add(m.group(1))

            m = fatal_pattern.search(line)
            if m:
                failed_event = int(m.group(1))

    # Determine suspect LFNs: XML statuses are the most reliable source,
    # fall back to log-based heuristics (e.g. for segfaults where the
    # XML input section may be missing or incomplete).
    xml_suspect_lfns: list[str] = []
    if input_file_statuses:
        xml_suspect_lfns = [s.lfn for s in input_file_statuses if s.status != "full"]

    log_unprocessed_lfns = list(opened_lfns - processed_lfns)

    # Prefer new-style (allows single-event slicing)
    if last_new is not None:
        last_new.unprocessed_lfns = log_unprocessed_lfns
        last_new.suspect_lfns = xml_suspect_lfns or log_unprocessed_lfns
        last_new.failed_event_overall = failed_event
        return last_new

    # Run 2-style: precise slicing not yet implemented
    if last_run2 is not None:
        raise NotImplementedError(
            "Run 2 style event messages detected (EventSelector). "
            "Precise slicing for Run 2 jobs is not yet implemented. "
            f"Last event found: {last_run2.event_number} (in-file: {last_run2.event_in_file})"
        )

    if last_old is not None:
        last_old.suspect_lfns = xml_suspect_lfns or (
            [last_old.lfn] if last_old.lfn else []
        )
        last_old.failed_event_overall = failed_event
        # Compute suspect in-file event when we have FATAL + RootIOAlg offset
        if failed_event is not None and last_old.event_in_file is not None:
            last_old.suspect_event_in_file = last_old.event_in_file + (
                failed_event - last_old.event_number
            )
        return last_old

    raise ValueError(
        f"No event processing messages found in {log_file}. "
        "The error may have occurred before any events were processed."
    )


def get_all_input_lfns_from_prodconf(prodconf_file: Path) -> list[str]:
    """Extract all input LFNs from a prodConf JSON file.

    Args:
        prodconf_file: Path to the prodConf JSON file.

    Returns:
        List of input file LFNs.
    """
    try:
        with open(prodconf_file) as f:
            prodconf = json.load(f)
        return prodconf.get("input", {}).get("files", [])
    except (json.JSONDecodeError, OSError):
        return []


def create_reproducer_prodconf(
    prodconf_file: Path,
    output_path: Path,
    *,
    input_files: list[str] | None = None,
    first_event: int = 0,
    n_of_events: int = -1,
    print_freq: int = 1,
    application_version: str | None = None,
    binary_tag: str | None = None,
) -> Path:
    """Create a modified prodConf JSON for the reproducer.

    Args:
        prodconf_file: Path to the original prodConf JSON.
        output_path: Where to write the modified prodConf.
        input_files: Override input files list. If None, keep original.
        first_event: First event number to process.
        n_of_events: Number of events (-1 for all).
        print_freq: Event print frequency (1 for debugging).
        application_version: Override application version. If None, keep original.
        binary_tag: Override binary tag. If None, keep original.

    Returns:
        Path to the written reproducer prodConf.

    Raises:
        FileNotFoundError: If the original prodConf doesn't exist.
    """
    with open(prodconf_file) as f:
        prodconf = json.load(f)

    if input_files is not None:
        prodconf["input"]["files"] = input_files

    prodconf["input"]["first_event_number"] = first_event
    prodconf["input"]["n_of_events"] = n_of_events

    if application_version is not None:
        prodconf["application"]["version"] = application_version
    if binary_tag is not None:
        prodconf["application"]["binary_tag"] = binary_tag

    if "extra_options" in prodconf.get("options", {}):
        prodconf["options"]["extra_options"]["print_freq"] = print_freq

    with open(output_path, "w") as f:
        json.dump(prodconf, f, indent=2)

    return output_path


def run_reproducer(
    job_dir: Path,
    reproducer_prodconf: Path,
    *,
    on_line: Callable[[str], None] | None = None,
) -> tuple[subprocess.CompletedProcess, Path]:
    """Run the reproducer in a clean copy of the job directory.

    Creates a ``.slice-work`` subdirectory with a copy of the job files,
    then runs ``lb-prod-run`` with the reproducer prodConf.

    Output is streamed line-by-line so that callers can show progress.
    The optional ``on_line`` callback is invoked for each stdout line.

    Args:
        job_dir: Original job directory.
        reproducer_prodconf: Path to the reproducer prodConf JSON.
        on_line: Optional callback invoked with each stdout line (stripped).

    Returns:
        Tuple of (CompletedProcess result, path to the slice working directory).
    """
    slice_dir = job_dir / ".slice-work"

    if slice_dir.exists():
        shutil.rmtree(slice_dir)

    shutil.copytree(
        job_dir,
        slice_dir,
        ignore=lambda d, _files: (
            [".slice-work", "minimal_reproducer", "reproducer"]
            if d == str(job_dir)
            else []
        ),
    )

    # Copy the reproducer prodconf into the slice directory
    slice_prodconf = slice_dir / reproducer_prodconf.name
    shutil.copy2(reproducer_prodconf, slice_prodconf)

    proc = subprocess.Popen(
        ["lb-prod-run", slice_prodconf.name],
        cwd=slice_dir,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        text=True,
        bufsize=1,  # line-buffered
    )

    # Read stderr in a background thread to avoid deadlocks
    stderr_lines: list[str] = []

    def _drain_stderr():
        assert proc.stderr is not None
        for line in proc.stderr:
            stderr_lines.append(line)

    stderr_thread = threading.Thread(target=_drain_stderr, daemon=True)
    stderr_thread.start()

    # Read stdout line-by-line, calling the progress callback
    stdout_lines: list[str] = []
    assert proc.stdout is not None
    for line in proc.stdout:
        stdout_lines.append(line)
        if on_line is not None:
            on_line(line.rstrip("\n"))

    proc.wait()
    stderr_thread.join(timeout=5)

    result = subprocess.CompletedProcess(
        args=proc.args,
        returncode=proc.returncode,
        stdout="".join(stdout_lines),
        stderr="".join(stderr_lines),
    )

    return result, slice_dir


def run_minimal_reproducer(
    minimal_reproducer_dir: Path,
    *,
    on_line: Callable[[str], None] | None = None,
) -> tuple[subprocess.CompletedProcess, Path]:
    """Run the minimal reproducer in a clean copy of the job directory.

    Creates a ``.slice-work`` subdirectory with a copy of the job files,
    then runs ``lb-prod-run`` with the reproducer prodConf.

    Output is streamed line-by-line so that callers can show progress.
    The optional ``on_line`` callback is invoked for each stdout line.

    Args:
        minimal_reproducer_dir: Directory containing the minimal reproducer.
        on_line: Optional callback invoked with each stdout line (stripped).

    Returns:
        Tuple of (CompletedProcess result, path to the slice working directory).
    """
    slice_dir = minimal_reproducer_dir / ".test-work"

    if slice_dir.exists():
        shutil.rmtree(slice_dir)

    shutil.copytree(
        minimal_reproducer_dir,
        slice_dir,
        ignore=lambda d, _files: (
            [".test-work", "minimal_reproducer", "reproducer"]
            if d == str(minimal_reproducer_dir)
            else []
        ),
    )

    proc = subprocess.Popen(
        ["bash", "run_reproducer.sh"],
        cwd=slice_dir,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        text=True,
        bufsize=1,  # line-buffered
    )

    # Read stderr in a background thread to avoid deadlocks
    stderr_lines: list[str] = []

    def _drain_stderr():
        assert proc.stderr is not None
        for line in proc.stderr:
            stderr_lines.append(line)

    stderr_thread = threading.Thread(target=_drain_stderr, daemon=True)
    stderr_thread.start()

    # Read stdout line-by-line, calling the progress callback
    stdout_lines: list[str] = []
    assert proc.stdout is not None
    for line in proc.stdout:
        stdout_lines.append(line)
        if on_line is not None:
            on_line(line.rstrip("\n"))

    proc.wait()
    stderr_thread.join(timeout=5)

    result = subprocess.CompletedProcess(
        args=proc.args,
        returncode=proc.returncode,
        stdout="".join(stdout_lines),
        stderr="".join(stderr_lines),
    )

    return result, slice_dir


def analyze_reproducer_output(
    stdout: str,
    stderr: str = "",
    *,
    first_event_offset: int = 0,
) -> ReproducerOutput:
    """Parse reproducer output for event information and failure messages.

    When ``first_event_offset`` is non-zero the reproducer was started
    with ``first_event_number > 0``, so event numbers in the log are
    relative to that starting point.  The offset is added to
    ``exact_problem_event`` so that it reflects the true in-file
    position.

    Searches both stdout and stderr for four patterns:

    1. **New style** (sets ``can_slice_single=True``)::

        HLTControlFlowMgr SUCCESS Processing event N in slot M

    2. **Old style** (in-file ↔ overall event mapping)::

        RootIOAlg SUCCESS Reading Event record N within LFN:..., M overall

    3. **Run 2 style** (in-file ↔ overall event mapping, sets ``can_slice_single=True``)::

        EventSelector SUCCESS Reading Event record M. Record number within stream 1: N

    4. **Bonus** (explicit failure with event number)::

        HLTControlFlowMgr FATAL Event N on slot M failed!

    The FATAL message typically appears on stderr.

    Args:
        stdout: The stdout from the reproducer run.
        stderr: The stderr from the reproducer run.

    Returns:
        ReproducerOutput with parsed event information.
    """
    new_style_pattern = re.compile(
        r"HLTControlFlowMgr\s+SUCCESS\s+Processing event\s+(\d+)\s+in slot\s+(\d+)"
    )
    old_style_pattern = re.compile(
        r"RootIOAlg\s+SUCCESS\s+Reading Event record\s+(\d+)\s+within\s+(LFN:[^,]+),\s+(\d+)\s+overall"
    )
    run2_style_pattern = re.compile(
        r"EventSelector\s+SUCCESS\s+Reading Event record\s+(\d+)\.\s+Record number within stream (\d+):\s+(\d+)"
    )
    run2_lfn_pattern = re.compile(
        r"EventSelector\s+INFO\s+Stream:EventSelector\.DataStreamTool_(\d+)\s+Def:DATAFILE='(LFN:[^']+)'"
    )
    fail_pattern = re.compile(r"Event\s+(\d+)\s+on\s+slot\s+\d+\s+failed")

    output = ReproducerOutput()
    # Map stream number to LFN for Run 2 style
    stream_to_lfn: dict[int, str] = {}

    combined = stdout + "\n" + stderr
    for line in combined.split("\n"):
        m = new_style_pattern.search(line)
        if m:
            output.last_processing_event = int(m.group(1))
            output.can_slice_single = True

        m = old_style_pattern.search(line)
        if m:
            output.last_event_in_file = int(m.group(1))
            output.last_event_lfn = m.group(2)
            output.last_event_overall = int(m.group(3))

        m = run2_lfn_pattern.search(line)
        if m:
            stream_num = int(m.group(1))
            lfn = m.group(2)
            stream_to_lfn[stream_num] = lfn

        m = run2_style_pattern.search(line)
        if m:
            stream_num = int(m.group(2))
            output.last_event_in_file = int(m.group(3))  # Record within stream
            output.last_event_overall = int(m.group(1))  # Overall event record
            output.last_event_lfn = stream_to_lfn.get(
                stream_num
            )  # Get LFN from stream mapping
            output.can_slice_single = True  # Can slice to single event

        m = fail_pattern.search(line)
        if m:
            output.failed_event_overall = int(m.group(1))

    # Calculate exact problem event
    if output.last_event_in_file is not None:
        if (
            output.failed_event_overall is not None
            and output.last_event_overall is not None
        ):
            # Use FATAL event number mapped back to in-file coordinates
            offset = output.last_event_overall - output.last_event_in_file
            output.exact_problem_event = output.failed_event_overall - offset
        elif (
            output.can_slice_single
            and output.last_processing_event is not None
            and output.last_event_overall is not None
        ):
            # New-style: last processing event is the one that crashed
            offset = output.last_event_overall - output.last_event_in_file
            output.exact_problem_event = output.last_processing_event - offset
        else:
            # Old-style only, no FATAL: last read event is best guess
            output.exact_problem_event = output.last_event_in_file
    elif output.can_slice_single and output.last_processing_event is not None:
        # New-style messages only, no RootIOAlg — use overall event number directly
        output.exact_problem_event = output.last_processing_event

    # Adjust for first_event_number: only needed when we have new-style
    # messages without old-style RootIOAlg records, because old-style
    # records already provide absolute in-file event positions.
    if (
        first_event_offset
        and output.exact_problem_event is not None
        and output.last_event_in_file is None
    ):
        output.exact_problem_event += first_event_offset - 1

    return output


def create_extraction_config(
    *,
    input_lfn: str,
    first_event: int,
    n_events: int = 1,
    output_file: Path,
    pool_catalog: Path | None = None,
    prodconf_file: Path | None = None,
) -> dict:
    """Create a YAML config dict for extracting events using GaudiConf.mergeDST.

    Args:
        input_lfn: The LFN of the input file.
        first_event: The first event number to extract.
        n_events: Number of events to extract (1 for single event).
        output_file: Path for the extracted output file.
        pool_catalog: Path to pool_xml_catalog.xml if available.
        prodconf_file: Path to original prodConf to infer parameters.

    Returns:
        Dictionary suitable for writing as YAML and passing to lbexec.
    """
    config = {
        "input_files": [input_lfn],
        "first_evt": first_event,
        "evt_max": n_events,
        "print_freq": max(1, first_event // 10),
        "output_file": str(output_file),
        "input_process": "Hlt2",
        "input_type": "ROOT",
        "input_raw_format": 0.5,
        "data_type": "Upgrade",
        "simulation": False,
        "geometry_version": "run3/2024.Q1.2-v00.00",
        "conditions_version": "master",
        "xml_file_catalog": (
            str(pool_catalog.resolve())
            if pool_catalog and pool_catalog.exists()
            else ""
        ),
        "output_type": "ROOT",
        "compression": "ZSTD:6",
        "root_ioalg_name": "RootIOAlgExt",
    }

    # Infer parameters from prodConf if available
    if prodconf_file and prodconf_file.exists():
        try:
            with open(prodconf_file) as f:
                prodconf = json.load(f)

            if prodconf.get("db_tags", {}).get("dddb_tag"):
                config["simulation"] = True

            app_name = prodconf.get("application", {}).get("name", "")
            if "Gauss" in app_name or "Sim" in app_name:
                config["simulation"] = True
        except (json.JSONDecodeError, OSError):
            pass

    return config


def extract_events(
    *,
    input_lfn: str,
    first_event: int,
    n_events: int = 1,
    extract_dir: Path,
    pool_catalog: Path | None = None,
    prodconf_file: Path | None = None,
    lhcb_version: str = "LHCb/v58r2",
) -> tuple[Path | None, subprocess.CompletedProcess]:
    """Extract events from an input file using GaudiConf.mergeDST.

    Args:
        input_lfn: The LFN of the input file.
        first_event: The first event number to extract.
        n_events: Number of events to extract (1 for single event).
        extract_dir: Working directory for extraction.
        pool_catalog: Path to pool_xml_catalog.xml if available.
        prodconf_file: Path to original prodConf to infer parameters.
        lhcb_version: LHCb version string for lb-run.

    Returns:
        Tuple of (path to extracted file or None, CompletedProcess).
    """
    extract_dir.mkdir(exist_ok=True, parents=True)

    # Copy pool catalog if available
    if pool_catalog and pool_catalog.exists():
        shutil.copy2(pool_catalog, extract_dir / "pool_xml_catalog.xml")

    if n_events == 1:
        output_file = extract_dir / f"event_{first_event}.dst"
    else:
        output_file = (
            extract_dir / f"events_{first_event}_to_{first_event + n_events}.dst"
        )

    config = create_extraction_config(
        input_lfn=input_lfn,
        first_event=first_event,
        n_events=n_events,
        output_file=output_file,
        pool_catalog=extract_dir / "pool_xml_catalog.xml" if pool_catalog else None,
        prodconf_file=prodconf_file,
    )

    config_path = extract_dir / "extract_event.yaml"
    with open(config_path, "w") as f:
        yaml.dump(config, f, default_flow_style=False, sort_keys=False)

    result = subprocess.run(
        ["lb-run", lhcb_version, "lbexec", "GaudiConf.mergeDST:dst", str(config_path)],
        cwd=extract_dir,
        capture_output=True,
        text=True,
    )

    # Save log
    log_path = extract_dir / "extraction.log"
    with open(log_path, "w") as f:
        f.write(
            f"STDOUT:\n{'=' * 80}\n{result.stdout}\n\nSTDERR:\n{'=' * 80}\n{result.stderr}\n"
        )

    if output_file.exists():
        return output_file, result
    return None, result


def extract_single_event(
    *,
    input_lfn: str,
    exact_event: int,
    extract_dir: Path,
    pool_catalog: Path | None = None,
    prodconf_file: Path | None = None,
    lhcb_version: str = "LHCb/v58r2",
) -> tuple[Path | None, subprocess.CompletedProcess]:
    """Extract a single event from an input file using GaudiConf.mergeDST.

    Thin wrapper around :func:`extract_events` with ``n_events=1``.
    """
    return extract_events(
        input_lfn=input_lfn,
        first_event=exact_event,
        n_events=1,
        extract_dir=extract_dir,
        pool_catalog=pool_catalog,
        prodconf_file=prodconf_file,
        lhcb_version=lhcb_version,
    )


def download_lfn_files(
    lfns: list[str],
    output_dir: Path,
) -> list[Path]:
    """Download LFN files from the grid using dirac-dms-get-file.

    Args:
        lfns: List of LFN strings (e.g. ``"LFN:/lhcb/..."``).
        output_dir: Directory to download files into.

    Returns:
        List of paths to downloaded files.

    Raises:
        FileNotFoundError: If ``dirac-dms-get-file`` is not available.
        RuntimeError: If the download command fails.
    """
    output_dir.mkdir(exist_ok=True, parents=True)

    result = subprocess.run(
        ["dirac-dms-get-file", "-D", str(output_dir), *lfns],
        capture_output=True,
        text=True,
    )

    if result.returncode != 0:
        raise RuntimeError(
            f"dirac-dms-get-file failed (exit code {result.returncode}):\n"
            f"{result.stderr}"
        )

    # Find downloaded files by matching LFN basenames
    downloaded: list[Path] = []
    for lfn in lfns:
        # LFN format: "LFN:/lhcb/..." or just "/lhcb/..."
        lfn_path = lfn.removeprefix("LFN:")
        basename = Path(lfn_path).name
        local_path = output_dir / basename
        if local_path.exists():
            downloaded.append(local_path)

    return downloaded


def find_data_packages(prodconf_file: Path) -> list[DataPackage]:
    """Find CVMFS data packages referenced in a prodConf.

    Parses ``application.data_pkgs`` entries like ``"AnalysisProductions.v1r4315"``
    and resolves them to CVMFS directories at
    ``/cvmfs/lhcb.cern.ch/lib/lhcb/DBASE/{PackageName}/{version}``.

    Args:
        prodconf_file: Path to the prodConf JSON.

    Returns:
        List of DataPackage entries with resolved CVMFS paths.
    """
    packages = []
    try:
        with open(prodconf_file) as f:
            prodconf = json.load(f)

        data_pkgs = prodconf.get("application", {}).get("data_pkgs", [])
        for pkg_spec in data_pkgs:
            if not isinstance(pkg_spec, str) or "." not in pkg_spec:
                continue

            # Format: "PackageName.version" e.g. "AnalysisProductions.v1r4315"
            name, version = pkg_spec.rsplit(".", 1)
            cvmfs_path = CVMFS_DBASE / name / version

            if cvmfs_path.exists():
                packages.append(
                    DataPackage(
                        name=name,
                        version=version,
                        cvmfs_path=cvmfs_path,
                    )
                )

    except (json.JSONDecodeError, OSError):
        pass

    return packages


def create_reproducer_package(
    *,
    output_dir: Path,
    job_dir: Path,
    failed_step: FailedStep,
    input_lfn: str | None,
    exact_event: int | None,
    local_input_files: list[Path] | None = None,
    prodconf_file: Path | None = None,
    data_packages: list[DataPackage] | None = None,
    lhcb_version: str = "LHCb/v58r2",
    job_id: str | None = None,
) -> Path:
    """Create a minimal reproducer directory with all necessary files.

    Args:
        output_dir: Directory for the reproducer package.
        job_dir: Original job directory.
        failed_step: Information about the failed step.
        input_lfn: The problematic input file LFN.
        exact_event: The exact problematic event number (if known).
        local_input_files: Local files to include (extracted events or downloads).
        prodconf_file: Path to the original prodConf.
        data_packages: List of DataPackage entries to include from CVMFS.
        lhcb_version: LHCb version string.

    Returns:
        Path to the reproducer package directory.
    """
    output_dir.mkdir(exist_ok=True, parents=True)

    # Copy input data files and determine reproducer input references
    if local_input_files:
        input_filenames = []
        for f in local_input_files:
            if f.exists():
                shutil.copy2(f, output_dir / f.name)
                input_filenames.append(f.name)
        inputs_for_reproducer = input_filenames or [input_lfn or "UNKNOWN"]
    else:
        inputs_for_reproducer = [input_lfn or "UNKNOWN"]

    # Create reproducer prodConf
    if prodconf_file and prodconf_file.exists():
        with open(prodconf_file) as f:
            prodconf = json.load(f)

        prodconf["input"]["files"] = inputs_for_reproducer
        if local_input_files:
            prodconf["input"]["first_event_number"] = 0
        prodconf["input"]["n_of_events"] = -1

        if "extra_options" in prodconf.get("options", {}):
            prodconf["options"]["extra_options"]["print_freq"] = 1

        # Remove data_pkgs since the files are now bundled locally
        if data_packages and "data_pkgs" in prodconf.get("application", {}):
            del prodconf["application"]["data_pkgs"]

        reproducer_json = output_dir / "reproducer.json"
        with open(reproducer_json, "w") as f:
            json.dump(prodconf, f, indent=2)

    # Copy data packages from CVMFS
    if data_packages:
        for dp in data_packages:
            dest = output_dir / dp.name / dp.version
            if dest.exists():
                shutil.rmtree(dest)
            shutil.copytree(dp.cvmfs_path, dest)

    # Create run script
    run_script = output_dir / "run_reproducer.sh"
    with open(run_script, "w") as f:
        f.write("#!/bin/bash\n")
        f.write("# Minimal reproducer for problematic event\n")
        f.write("# run with --interactive to get lb-run & lbexec command\n")
        f.write(f"# Original job: {job_dir}\n")
        f.write(f"# Failed step: {failed_step.step_id}\n")
        if exact_event is not None:
            f.write(f"# Problematic event: {exact_event}\n")
        if input_lfn:
            f.write(f"# Original file: {input_lfn}\n")
        f.write("\nset -e\n\n")
        # Add data package directories to PYTHONPATH
        if data_packages:
            f.write("# Add bundled data packages to PYTHONPATH\n")
            f.write('DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"\n')
            for dp in data_packages:
                if dp.name == "AnalysisProductions":
                    f.write(
                        f'export ANALYSIS_PRODUCTIONS_BASE="$DIR/{dp.name}/{dp.version}"\n'
                    )
                f.write(
                    f'export PYTHONPATH="$DIR/{dp.name}/{dp.version}:$PYTHONPATH"\n'
                )
            f.write("\n")
        f.write("echo 'Running minimal reproducer...'\n")
        if exact_event is not None:
            f.write(f"echo 'This will process the problematic event ({exact_event})'\n")
        f.write("echo ''\n\n")
        if prodconf_file and prodconf_file.exists():
            f.write("lb-prod-run reproducer.json $@\n")
        else:
            f.write("# No prodConf available - manual setup required\n")
            f.write(f"# lb-run {lhcb_version} <application> <options>\n")
    run_script.chmod(0o755)

    # Create README
    readme = output_dir / "README.txt"
    with open(readme, "w") as f:
        f.write("  ╔═╗╔═╗  LHCb Analysis Productions\n")
        f.write("  ╠═╣╠═╝  MINIMAL REPRODUCER PACKAGE\n")
        f.write("  ╩ ╩╩    ─────────────────────────────\n")
        f.write("\n")
        f.write("=" * 80 + "\n\n")
        f.write("Created by lb-ap make-reproducer\n")
        f.write("Address problems to DPA-WP2 Mattermost!\n\n")
        f.write("""
This directory contains everything needed to reproduce the error.
If you're able to fix it yourself consider making a test case from this if appropriate.
If you need help to fix the issue, please simplify your python code in $PWD/AnalysisProductions/v1rXXXX
to make it as minimal as possible while still producing the same error ideally aim to simplify it down
to a single python file which can be ran with:

    $ gaudirun.py my_reproducer.py ProdConfXXX.py
or  $ lbexec my_reproducer:main lbexec...yaml

""")
        f.write("FILES:\n")
        if local_input_files:
            for lf in local_input_files:
                f.write(f"  - {lf.name} - Input data file\n")
        if prodconf_file and prodconf_file.exists():
            f.write("  - reproducer.json - Job configuration\n")
        f.write("  - run_reproducer.sh - Script to run the reproducer\n")
        if data_packages:
            for dp in data_packages:
                f.write(f"  - {dp.name}/{dp.version}/ - Data package from CVMFS\n")
        f.write("  - README.txt - This file\n\n")
        f.write("ORIGINAL JOB INFO:\n")
        f.write(f"  Job directory: {job_dir}\n")
        f.write(f"  Job ID: {job_id}\n")
        f.write(f"  Failed step: {failed_step.step_id}\n")
        if input_lfn:
            f.write(f"  Original file: {input_lfn}\n")
        if exact_event is not None:
            f.write(f"  Event number: {exact_event}\n")
        f.write("\nTO RUN:\n")
        f.write("  ./run_reproducer.sh\n\n")
        f.write("TO RUN INTERACTIVELY:\n")
        if prodconf_file and prodconf_file.exists():
            f.write("  ./run_reproducer.sh --interactive\n\n")
        f.write("TO RUN MANUALLY (e.g. using stack environment):\n")
        if data_packages:
            f.write("# Add bundled data packages to PYTHONPATH\n")
            f.write('DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"\n')
            for dp in data_packages:
                if dp.name == "AnalysisProductions":
                    f.write(
                        f'export ANALYSIS_PRODUCTIONS_BASE="$DIR/{dp.name}/{dp.version}"\n'
                    )
                f.write(
                    f'export PYTHONPATH="$DIR/{dp.name}/{dp.version}:$PYTHONPATH"\n'
                )
            f.write("\n")
        f.write(
            "  ... lbexec {options file} {yaml file}  # you can find the command in the interactive mode\n\n"
        )

        f.write("SHARING:\n")
        f.write("  This directory is self-contained and can be shared for debugging.\n")
        if local_input_files:
            total_kb = sum(
                lf.stat().st_size / 1024 for lf in local_input_files if lf.exists()
            )
            f.write(f"  Input data size: ~{total_kb:.1f} KB\n")

    return output_dir


def create_tarball(source_dir: Path, tarball_path: Path) -> Path:
    """Create a gzipped tarball from a directory.

    Args:
        source_dir: Directory to package.
        tarball_path: Output tarball path.

    Returns:
        Path to the created tarball.
    """
    import tarfile

    tarball_dir_name = tarball_path.stem.replace(".tar", "")

    with tarfile.open(tarball_path, "w:gz") as tar:
        for item in sorted(source_dir.rglob("*")):
            if item.is_file():
                arcname = f"{tarball_dir_name}/{item.relative_to(source_dir)}"
                tar.add(item, arcname=arcname)

    return tarball_path
