###############################################################################
# (c) Copyright 2020-2024 CERN for the benefit of the LHCb Collaboration      #
#                                                                             #
# This software is distributed under the terms of the GNU General Public      #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
#                                                                             #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization  #
# or submit itself to any jurisdiction.                                       #
###############################################################################
"""
Main CLI for LbAPLocal with enhanced UX using typer and rich.

Command-line interface for LHCb Analysis Productions,
featuring improved user experience with rich formatting, better error messages,
and integration with CWL-based workflows.
"""

import importlib.metadata
import os
import random
import shlex
import shutil
import subprocess
import time
from enum import Enum
from pathlib import Path
from typing import Annotated, Optional

import typer
import yaml
from LbAPCommon import parse_yaml, render_yaml
from rich.console import Console
from rich.live import Live
from rich.panel import Panel
from rich.progress import Progress, SpinnerColumn, TextColumn
from rich.prompt import Confirm, Prompt
from rich.table import Table
from rich.text import Text
from rich.tree import Tree

from LbAPLocal.job_slice import run_minimal_reproducer

from ..utils import (
    available_productions,
    check_production,
    inside_ap_datapkg,
    logging_subprocess_run,
    production_to_versions,
    validate_environment,
)
from .wizards import (
    checkout_wizard,
    clone_wizard,
    cwl_generate_wizard,
    test_wizard,
)

# Initialize Rich console
console = Console()
err_console = Console(stderr=True, style="bold red")

# Cooldown stamps — banner and animation have independent cooldowns
_CACHE_DIR = Path.home() / ".cache" / "lbaplocal"
_BANNER_STAMP = _CACHE_DIR / ".banner_stamp"
_ANIMATION_STAMP = _CACHE_DIR / ".animation_stamp"
_BANNER_COOLDOWN_SECS = 3600  # 1 hour
_ANIMATION_COOLDOWN_SECS = 1800  # 30 minutes

# Commands that always show the banner regardless of cooldown
_ALWAYS_BANNER_COMMANDS = {"test"}


def _stamp_expired(stamp: Path, cooldown: float) -> bool:
    """Return True if a stamp file is missing or older than cooldown seconds."""
    try:
        if stamp.exists():
            return (time.time() - stamp.stat().st_mtime) >= cooldown
    except OSError:
        pass
    return True


def _touch_stamp(stamp: Path) -> None:
    """Update a stamp file's mtime to now."""
    try:
        stamp.parent.mkdir(parents=True, exist_ok=True)
        stamp.touch()
    except OSError:
        pass


# Banner for the CLI
BANNER = """\
[dark_red]     ▄▄▄     [/dark_red] [dark_blue]▄▄▄▄▄  [/dark_blue]
[dark_red]    █   █    [/dark_red][dark_blue]█     █ [/dark_blue] [bold]LHCb Analysis Productions[/bold]
[dark_red]   █▄▄▄▄█   [/dark_red][dark_blue]█▄▄▄▄▄█ [/dark_blue]  [dim]━━━━━━━━━━━━━━━━━━━━━━━━━[/dim]
[dark_red]  █     █   [/dark_red][dark_blue]█        [/dark_blue]  [italic]Local Testing Framework[/italic]
[dark_red] █     █   [/dark_red][dark_blue]█         [/dark_blue]  [dim]v{version}[/dim]\
"""

# Plain-text target for scatter animation (no Rich markup)
_BANNER_PLAIN = [
    "     ▄▄▄      ▄▄▄▄▄  ",
    "    █   █    █     █    LHCb Analysis Productions",
    "   █▄▄▄▄█   █▄▄▄▄▄█    ━━━━━━━━━━━━━━━━━━━━━━━━━",
    "  █     █   █           Local Testing Framework",
    " █     █   █            v{version}",
]

_SCATTER_GLYPHS = "░▒▓█▀▄╱╲│─╮╯╰╭◆◇○●"

# Tips of the day — shown randomly in the banner
_TIPS = [
    "Use [bold cyan]lb-ap test --pretty[/] for a split-view of real-time logs",
    "Use [bold cyan]lb-ap list --tree[/] for a tree view of productions and jobs",
    "Use [bold cyan]lb-ap test --dry-run[/] to validate config without running",
    "Use [bold cyan]lb-ap test --pick-smallest-lfn[/] for faster testing with small files",
    "Use [bold cyan]lb-ap clone[/] with no args for an interactive wizard",
    "Use [bold cyan]lb-ap -b my-branch test ...[/] to auto-clone and test a branch",
    "Use [bold cyan]lb-ap test -i local.dst ...[/] to test with a local input file",
    "Use [bold cyan]lb-ap make-reproducer -s single-event[/] for the smallest tarball",
    "Use [bold cyan]lb-ap -C /path/to/repo list[/] to run from a different directory",
    "Use [bold cyan]lb-ap test --n-lfns 5 --pick-smallest-lfn[/] to test with multiple small files",
    "Use [bold cyan]lb-ap checkout -w[/] for an interactive checkout wizard",
    "Use [bold cyan]lb-ap versions[/] to list available production versions",
    "Clones are cached in [dim]~/.cache/lbaplocal/clones/[/] for reuse",
    "Use [bold cyan]lb-ap make-reproducer --no-run[/] to create a reproducer without running it",
]


def _get_git_branch() -> str | None:
    """Get the current git branch name, or None if not in a repo."""
    try:
        result = subprocess.run(
            ["git", "rev-parse", "--abbrev-ref", "HEAD"],
            capture_output=True,
            text=True,
            timeout=5,
        )
        if result.returncode == 0:
            return result.stdout.strip()
    except Exception:
        pass
    return None


def _check_dirac_proxy() -> bool:
    """Check if there's a valid DIRAC proxy (non-fatal)."""
    try:
        result = subprocess.run(
            ["lb-dirac", "dirac-proxy-info", "--checkvalid"],
            capture_output=True,
            timeout=10,
        )
        return result.returncode == 0
    except Exception:
        return False


def _build_banner_content(version: str) -> str:
    """Build the full banner content with logo, tip, links, and context."""
    parts = [BANNER.format(version=version)]

    # Tip of the day
    tip = random.choice(_TIPS)
    parts.append(f"\n  [yellow]Tip:[/yellow] {tip}")

    # Quick links
    parts.append(
        "\n\n  [dim]Docs:[/dim] [link=https://lhcb-ap.docs.cern.ch]lhcb-ap.docs.cern.ch[/link]"
        "  [dim]·[/dim]  "
        "[dim]Help:[/dim] DPA-WP2 Mattermost"
    )

    # Context info
    ctx_items = []
    krb = check_kerberos_ticket()
    ctx_items.append(
        "[dim]Kerberos:[/dim] [green]✓[/green]"
        if krb
        else "[dim]Kerberos:[/dim] [red]✗[/red]"
    )

    proxy = _check_dirac_proxy()
    ctx_items.append(
        "[dim]Proxy:[/dim] [green]✓[/green]"
        if proxy
        else "[dim]Proxy:[/dim] [red]✗[/red]"
    )

    cwd = Path.cwd()
    try:
        display_dir = f"~/{cwd.relative_to(Path.home())}"
    except ValueError:
        display_dir = str(cwd)
    ctx_items.append(f"[dim]Dir:[/dim] {display_dir}")

    parts.append("\n  " + "  [dim]·[/dim]  ".join(ctx_items))

    return "".join(parts)


def animate_banner(con: Console, version: str) -> None:
    """Matrix-style scatter → resolve animation for the CLI banner."""
    lines = [line.format(version=version) for line in _BANNER_PLAIN]
    max_len = max(len(line) for line in lines)
    lines = [line.ljust(max_len) for line in lines]
    rows = len(lines)

    # Build mask: True = non-space character to animate
    mask = [[c != " " for c in line] for line in lines]

    # Start fully scrambled
    current = [
        [random.choice(_SCATTER_GLYPHS) if mask[r][c] else " " for c in range(max_len)]
        for r in range(rows)
    ]

    total_steps = 18
    with Live("", refresh_per_second=30, console=con, transient=True) as live:
        for step in range(total_steps + 1):
            prob = step / total_steps
            for r in range(rows):
                for c in range(max_len):
                    if not mask[r][c]:
                        continue
                    if random.random() < prob:
                        current[r][c] = lines[r][c]
                    else:
                        current[r][c] = random.choice(_SCATTER_GLYPHS)

            # Color resolved chars: red for A, blue for P, default for text
            text = Text()
            for r, row in enumerate(current):
                for c, ch in enumerate(row):
                    if ch == lines[r][c] and mask[r][c]:
                        if c < 13:
                            text.append(ch, style="dark_red")
                        elif c < 23:
                            text.append(ch, style="dark_blue")
                        else:
                            text.append(ch, style="bold")
                    elif mask[r][c]:
                        text.append(ch, style="dim green")
                    else:
                        text.append(ch)
                text.append("\n")

            live.update(text)
            time.sleep(0.045)

        time.sleep(0.3)


# Global state for directory handling
class AppState:
    """Global application state"""

    directory: Optional[Path] = None
    branch: Optional[str] = None
    temp_clone: Optional[Path] = None


state = AppState()

# Clone cache directory
CLONE_CACHE_DIR = Path.home() / ".cache" / "lbaplocal" / "clones"

# Create the main Typer app
app = typer.Typer(
    name="lb-ap",
    help="🔬 [bold cyan]CLI for LHCb AnalysisProductions[/bold cyan]\n\n"
    "Command-line interface with rich formatting and CWL integration.",
    add_completion=True,
    rich_markup_mode="rich",
    no_args_is_help=True,
)

# Create sub-apps for organization
cwl_app = typer.Typer(
    name="cwl",
    help="🔧 [bold magenta]CWL-based workflow commands[/bold magenta]",
    rich_markup_mode="rich",
    no_args_is_help=True,
)
app.add_typer(cwl_app, name="cwl")


class CloneType(str, Enum):
    """Git clone protocol types"""

    SSH = "ssh"
    HTTPS = "https"
    KRB5 = "krb5"


class OutputFormat(str, Enum):
    """Output format options"""

    TEXT = "text"
    JSON = "json"
    YAML = "yaml"


# Helper functions for branch and Kerberos handling
def check_kerberos_ticket() -> bool:
    """Check if there's a valid Kerberos ticket for CERN.CH"""
    try:
        result = subprocess.run(
            ["klist", "-s"],
            capture_output=True,
            timeout=5,
        )
        if result.returncode != 0:
            return False

        # Check if ticket is for CERN.CH
        result = subprocess.run(
            ["klist"],
            capture_output=True,
            text=True,
            timeout=5,
        )
        return "CERN.CH" in result.stdout
    except Exception:
        return False


def get_default_clone_type() -> CloneType:
    """Determine default clone type based on environment"""
    if check_kerberos_ticket():
        console.print(
            "[green]✓ Using Kerberos authentication (ticket detected)[/green]"
        )
        return CloneType.KRB5

    # Prompt user for preferred method
    console.print("\n[yellow]No Kerberos ticket detected[/yellow]")
    console.print("Available clone methods:")
    console.print("  1. SSH (requires SSH key configured)")
    console.print("  2. HTTPS (requires username/password)")
    console.print("  3. Kerberos (requires valid ticket)")

    choice = Prompt.ask(
        "Select clone method",
        choices=["1", "2", "3"],
        default="1",
    )

    return {
        "1": CloneType.SSH,
        "2": CloneType.HTTPS,
        "3": CloneType.KRB5,
    }[choice]


def _print_git_command(cmd: list[str]) -> None:
    """Print a git command in a visible format"""
    console.print(f"[dim]$ {shlex.join(cmd)}[/dim]")


def check_git_clean() -> bool:
    """Check if git working tree is clean"""
    try:
        cmd = ["git", "status", "--porcelain"]
        _print_git_command(cmd)
        result = subprocess.run(
            cmd,
            capture_output=True,
            text=True,
            timeout=5,
        )
        return not result.stdout.strip()
    except Exception:
        return False


def version_callback(value: bool):
    """Display version information for all relevant packages"""
    if value:
        # Show animated banner (skip if output is not a terminal)
        try:
            lbaplocal_version = importlib.metadata.version("LbAPLocal")
        except importlib.metadata.PackageNotFoundError:
            lbaplocal_version = "dev"

        if console.is_terminal:
            animate_banner(console, lbaplocal_version)

        console.print(
            Panel(
                _build_banner_content(lbaplocal_version),
                border_style="dark_blue",
                padding=(1, 2),
            )
        )

        table = Table(
            title="Package Versions", show_header=True, header_style="bold magenta"
        )
        table.add_column("Package", style="cyan", no_wrap=True)
        table.add_column("Version", style="green")
        table.add_column("Status", style="yellow")

        for package in [
            "LbAPLocal",
            "LbAPCommon",
            "LbEnv",
            "LbDiracWrappers",
            "dirac-cwl-proto",
        ]:
            try:
                version = importlib.metadata.version(package)
                table.add_row(package, version, "✓ Installed")
            except importlib.metadata.PackageNotFoundError:
                table.add_row(package, "N/A", "✗ Not installed")

        console.print(table)
        raise typer.Exit()


@app.callback()
def main(
    ctx: typer.Context,
    directory: Annotated[
        Optional[Path],
        typer.Option(
            "-C",
            "--directory",
            help="Run as if started in <directory> instead of the current working directory",
            file_okay=False,
            dir_okay=True,
            resolve_path=True,
        ),
    ] = None,
    branch: Annotated[
        Optional[str],
        typer.Option(
            "--branch",
            "-b",
            help="Git branch to use (auto-clones if needed, checks out if directory exists)",
        ),
    ] = None,
    version: Annotated[
        Optional[bool],
        typer.Option(
            "--version",
            "-v",
            callback=version_callback,
            is_eager=True,
            help="Show version information and exit",
        ),
    ] = None,
):
    """
    🔬 LbAPLocal - Experimental CLI

    Enhanced command-line interface for LHCb Analysis Productions with
    rich formatting, better error messages, and CWL workflow integration.

    Use -C to run commands from a different directory:
        lb-ap-exp -C /path/to/AnalysisProductions list

    Use --branch to auto-clone or checkout a branch:
        lb-ap-exp --branch my-feature list
        lb-ap-exp -C /path/to/repo --branch my-feature test MyJob
    """
    # Determine version string
    try:
        lbaplocal_version = importlib.metadata.version("LbAPLocal")
    except importlib.metadata.PackageNotFoundError:
        lbaplocal_version = "dev"

    # Banner: always for "test", otherwise respect 1-hour cooldown
    command = ctx.invoked_subcommand
    show_banner = command in _ALWAYS_BANNER_COMMANDS or _stamp_expired(
        _BANNER_STAMP, _BANNER_COOLDOWN_SECS
    )

    if show_banner:
        # Animation: independent 30-minute cooldown
        if console.is_terminal and _stamp_expired(
            _ANIMATION_STAMP, _ANIMATION_COOLDOWN_SECS
        ):
            animate_banner(console, lbaplocal_version)
            _touch_stamp(_ANIMATION_STAMP)

        console.print(
            Panel(
                _build_banner_content(lbaplocal_version),
                border_style="dark_blue",
                padding=(1, 2),
            )
        )
        _touch_stamp(_BANNER_STAMP)

    # Store branch in global state
    if branch:
        state.branch = branch

    # Handle directory and branch logic
    if directory:
        # Directory specified - try to use it
        if not directory.exists():
            if branch:
                # Directory doesn't exist but branch specified - create and clone
                console.print(
                    f"[cyan]Creating directory and cloning with branch {branch}...[/cyan]"
                )
                directory.mkdir(parents=True, exist_ok=True)
                state.directory = directory
                os.chdir(directory)
                _auto_clone_repository(branch)
                console.print(
                    f"[bold green]Clone available at:[/bold green] {directory.resolve()}"
                )
            else:
                err_console.print(
                    f"[bold red]✗ Directory does not exist:[/bold red] {directory}"
                )
                raise typer.Exit(code=1)
        else:
            # Directory exists
            state.directory = directory
            os.chdir(directory)
            console.print(f"[dim]Working directory: {directory.resolve()}[/dim]")

            if branch:
                # Check if it's a git repo and checkout branch
                if (directory / ".git").exists():
                    _auto_checkout_branch(branch)
                    console.print(
                        f"[bold green]Repository available at:[/bold green] {directory.resolve()}"
                    )
                else:
                    # Not a git repo, but branch specified - clone here
                    _auto_clone_repository(branch)
                    console.print(
                        f"[bold green]Clone available at:[/bold green] {directory.resolve()}"
                    )
    elif branch:
        # No directory specified, but branch given - check current directory
        cwd = Path.cwd()

        # Check if we're already in AnalysisProductions
        if (cwd / ".git").exists() and cwd.name == "AnalysisProductions":
            _auto_checkout_branch(branch)
            console.print(
                f"[bold green]Repository available at:[/bold green] {cwd.resolve()}"
            )
        else:
            # Clone to temporary directory
            import tempfile

            temp_dir = Path(tempfile.mkdtemp(prefix="AnalysisProductions_"))
            state.temp_clone = temp_dir
            state.directory = temp_dir
            console.print(
                Panel(
                    f"[cyan]Branch:[/cyan] {branch}\n"
                    f"[cyan]Location:[/cyan] {temp_dir}",
                    title="🔄 Auto-Clone to Temporary Directory",
                    border_style="cyan",
                )
            )
            os.chdir(temp_dir)
            _auto_clone_repository(branch)
            console.print(
                f"[bold green]Temporary clone available at:[/bold green] {temp_dir.resolve()}"
            )
            console.print(
                "[yellow]Note:[/yellow] This directory will persist until manually deleted or system cleanup"
            )


def _get_cached_clone_path(branch: str) -> Path:
    """Get the path to a cached clone for a branch"""
    # Sanitize branch name for filesystem
    safe_branch = branch.replace("/", "_").replace("\\", "_")
    return CLONE_CACHE_DIR / safe_branch


def _is_cached_clone_usable(clone_path: Path, branch: str) -> bool:
    """Check if a cached clone exists and is usable (clean working tree, correct branch)"""
    if not clone_path.exists():
        return False

    if not (clone_path / ".git").exists():
        return False

    # Check if working tree is clean
    try:
        result = subprocess.run(
            ["git", "status", "--porcelain"],
            cwd=clone_path,
            capture_output=True,
            text=True,
            check=True,
        )
        if result.stdout.strip():
            # Working tree is dirty
            return False

        # Check if we're on the correct branch
        result = subprocess.run(
            ["git", "rev-parse", "--abbrev-ref", "HEAD"],
            cwd=clone_path,
            capture_output=True,
            text=True,
            check=True,
        )
        current_branch = result.stdout.strip()
        if current_branch != branch:
            # Wrong branch - try to checkout correct branch
            result = subprocess.run(
                ["git", "checkout", branch],
                cwd=clone_path,
                capture_output=True,
                text=True,
            )
            if result.returncode != 0:
                return False

        # Update from remote
        subprocess.run(
            ["git", "fetch", "origin"],
            cwd=clone_path,
            capture_output=True,
        )
        subprocess.run(
            ["git", "pull", "--ff-only"],
            cwd=clone_path,
            capture_output=True,
        )

        # Pull Git LFS files
        subprocess.run(
            ["git", "lfs", "install"],
            cwd=clone_path,
            capture_output=True,
            check=False,  # Don't fail if LFS is not available
        )
        subprocess.run(
            ["git", "lfs", "pull"],
            cwd=clone_path,
            capture_output=True,
            check=False,  # Don't fail if LFS is not available
        )

        return True
    except Exception:
        return False


def _is_commit_sha(ref: str) -> bool:
    """Check if a git ref is a commit SHA

    Args:
        ref: Git reference (branch, tag, or commit SHA)

    Returns:
        True if ref appears to be a commit SHA (full 40 chars or short form 7-39 chars)

    Raises:
        typer.Exit: If the SHA is too short (< 7 chars) to be safe
    """
    # Check if it looks like a hex string
    if not all(c in "0123456789abcdef" for c in ref.lower()):
        return False

    sha_len = len(ref)

    # Reject very short SHAs that could cause collisions
    if sha_len < 7:
        err_console.print(
            f"[bold red]✗ Commit SHA '{ref}' is too short ({sha_len} characters)![/bold red]\n"
            f"[yellow]Short SHAs can cause ambiguity and collisions.[/yellow]\n"
            f"[yellow]Please use:[/yellow]\n"
            f"  • Full commit SHA (40 characters)\n"
            f"  • At least 7 characters of the SHA\n"
            f"  • A branch name or tag (recommended)"
        )
        raise typer.Exit(code=1)

    # Warn about short SHAs and suggest alternatives
    if 7 <= sha_len < 40:
        console.print(
            f"[yellow]⚠ Using short commit SHA '{ref}' ({sha_len} characters)[/yellow]\n"
            f"[dim]Consider using tags or branch names for better reproducibility[/dim]"
        )

    return True


def _auto_clone_repository(branch: str, use_cache: bool = True):
    """Auto-clone AnalysisProductions repository with specified branch

    Args:
        branch: Branch name, tag, or commit SHA to clone/checkout
        use_cache: If True, try to reuse cached clone
    """
    # Check if we can reuse a cached clone
    if use_cache:
        cached_path = _get_cached_clone_path(branch)
        if _is_cached_clone_usable(cached_path, branch):
            console.print(
                Panel(
                    f"[cyan]Repository:[/cyan] AnalysisProductions\n"
                    f"[cyan]Branch:[/cyan] {branch}\n"
                    f"[cyan]Location:[/cyan] {cached_path}\n"
                    f"[cyan]Status:[/cyan] Reusing cached clone",
                    title="♻️  Cached Clone",
                    border_style="green",
                )
            )
            console.print(f"[green]✓ Reusing existing clone at {cached_path}[/green]")

            # Copy from cache to current directory
            import shutil

            for item in cached_path.iterdir():
                if item.name == ".git":
                    shutil.copytree(item, Path.cwd() / ".git", dirs_exist_ok=True)
                else:
                    if item.is_dir():
                        shutil.copytree(
                            item, Path.cwd() / item.name, dirs_exist_ok=True
                        )
                    else:
                        shutil.copy2(item, Path.cwd() / item.name)

            console.print(f"[dim]Cached clone copied to: {Path.cwd()}[/dim]")

            # Pull Git LFS files in the new location
            try:
                console.print("[dim]Pulling Git LFS files...[/dim]")
                subprocess.run(
                    ["git", "lfs", "install"],
                    capture_output=True,
                    check=False,
                )
                subprocess.run(
                    ["git", "lfs", "pull"],
                    capture_output=True,
                    check=False,
                )
            except Exception:
                pass

            return

    clone_type = get_default_clone_type()

    clone_urls = {
        CloneType.SSH: "ssh://git@gitlab.cern.ch:7999/lhcb-datapkg/AnalysisProductions.git",
        CloneType.HTTPS: "https://gitlab.cern.ch/lhcb-datapkg/AnalysisProductions.git",
        CloneType.KRB5: "https://:@gitlab.cern.ch:8443/lhcb-datapkg/AnalysisProductions.git",
    }
    clone_url = clone_urls[clone_type]

    console.print(
        Panel(
            f"[cyan]Repository:[/cyan] AnalysisProductions\n"
            f"[cyan]Branch:[/cyan] {branch}\n"
            f"[cyan]Protocol:[/cyan] {clone_type.value}\n"
            f"[cyan]Location:[/cyan] {Path.cwd()}",
            title="🔄 Auto-Clone",
            border_style="cyan",
        )
    )

    with Progress(
        SpinnerColumn(),
        TextColumn("[progress.description]{task.description}"),
        console=console,
    ) as progress:
        progress.add_task(description="Cloning repository...", total=None)

        is_sha = _is_commit_sha(branch)

        try:
            if is_sha:
                # For commit SHAs, clone without -b and then checkout the specific commit
                cmd = ["git", "clone", clone_url, "."]
                _print_git_command(cmd)
                subprocess.run(
                    cmd,
                    check=True,
                    capture_output=True,
                )
                cmd = ["git", "checkout", branch]
                _print_git_command(cmd)
                subprocess.run(
                    cmd,
                    check=True,
                    capture_output=True,
                )
                console.print(
                    f"[green]✓ Successfully cloned and checked out commit {branch[:7]}[/green]"
                )
            else:
                # For branches/tags, try clone with -b
                cmd = ["git", "clone", "-b", branch, clone_url, "."]
                _print_git_command(cmd)
                subprocess.run(
                    cmd,
                    check=True,
                    capture_output=True,
                )
                console.print(
                    f"[green]✓ Successfully cloned and checked out {branch}[/green]"
                )
        except subprocess.CalledProcessError as e:
            if is_sha:
                # SHA checkout failed
                err_console.print(
                    f"[bold red]✗ Clone or commit checkout failed:[/bold red] {e}"
                )
                raise typer.Exit(code=1)
            # If branch doesn't exist, clone master and create branch
            try:
                cmd = ["git", "clone", clone_url, "."]
                _print_git_command(cmd)
                subprocess.run(
                    cmd,
                    check=True,
                    capture_output=True,
                )
                cmd = ["git", "checkout", "-b", branch]
                _print_git_command(cmd)
                subprocess.run(
                    cmd,
                    check=True,
                    capture_output=True,
                )
                console.print(
                    f"[green]✓ Cloned repository and created branch {branch}[/green]"
                )
            except subprocess.CalledProcessError as e2:
                err_console.print(f"[bold red]✗ Clone failed:[/bold red] {e2}")
                raise typer.Exit(code=1)

        # Pull Git LFS files after clone
        try:
            console.print("[dim]Pulling Git LFS files...[/dim]")
            subprocess.run(
                ["git", "lfs", "install"],
                capture_output=True,
                check=False,
            )
            subprocess.run(
                ["git", "lfs", "pull"],
                capture_output=True,
                check=False,
            )
        except Exception:
            pass

    # Cache the clone if we're in a temp directory
    if use_cache and state.temp_clone:
        _cache_clone(branch)

    console.print(f"[dim]Clone location: {Path.cwd()}[/dim]")


def _cache_clone(branch: str):
    """Cache the current clone for future reuse"""
    cached_path = _get_cached_clone_path(branch)

    try:
        # Ensure cache directory exists
        CLONE_CACHE_DIR.mkdir(parents=True, exist_ok=True)

        # Remove old cache if it exists
        if cached_path.exists():
            shutil.rmtree(cached_path)

        shutil.copytree(Path.cwd(), cached_path)
        console.print(f"[dim]Cached clone for future use at: {cached_path}[/dim]")
    except Exception as e:
        # Don't fail if caching doesn't work
        console.print(f"[yellow]⚠ Could not cache clone: {e}[/yellow]")


def _auto_checkout_branch(branch: str):
    """Auto-checkout branch, tag, or commit SHA in existing git repository"""
    # Check if working tree is clean
    if not check_git_clean():
        err_console.print(
            "[bold red]✗ Working tree is not clean![/bold red]\n"
            "Please commit or stash your changes before switching branches."
        )
        raise typer.Exit(code=1)

    is_sha = _is_commit_sha(branch)

    if is_sha:
        console.print(f"[cyan]Checking out commit: {branch}[/cyan]")
    else:
        console.print(f"[cyan]Checking out branch: {branch}[/cyan]")

    try:
        if is_sha:
            # For commit SHAs, fetch first to ensure we have the commit
            cmd = ["git", "fetch", "origin"]
            _print_git_command(cmd)
            subprocess.run(
                cmd,
                capture_output=True,
                check=False,  # Don't fail if fetch has issues
            )

            # Checkout the commit SHA (will be in detached HEAD state)
            cmd = ["git", "checkout", branch]
            _print_git_command(cmd)
            result = subprocess.run(
                cmd,
                capture_output=True,
                text=True,
            )

            if result.returncode == 0:
                console.print(
                    f"[green]✓ Checked out commit {branch[:7]}[/green]\n"
                    f"[yellow]⚠ You are in 'detached HEAD' state[/yellow]\n"
                    f"[dim]Consider using a branch or tag name instead for better workflow[/dim]"
                )
            else:
                err_console.print(
                    f"[bold red]✗ Failed to checkout commit:[/bold red] {result.stderr}"
                )
                raise typer.Exit(code=1)
        else:
            # Try to checkout existing branch
            cmd = ["git", "checkout", branch]
            _print_git_command(cmd)
            result = subprocess.run(
                cmd,
                capture_output=True,
                text=True,
            )

            if result.returncode == 0:
                console.print(f"[green]✓ Checked out existing branch {branch}[/green]")
            else:
                # Branch doesn't exist locally, try to create from remote
                cmd = ["git", "fetch", "origin"]
                _print_git_command(cmd)
                result = subprocess.run(
                    cmd,
                    capture_output=True,
                )

                cmd = ["git", "checkout", "-b", branch, f"origin/{branch}"]
                _print_git_command(cmd)
                result = subprocess.run(
                    cmd,
                    capture_output=True,
                    text=True,
                )

                if result.returncode == 0:
                    console.print(
                        f"[green]✓ Checked out branch {branch} from remote[/green]"
                    )
                else:
                    # Create new branch
                    cmd = ["git", "checkout", "-b", branch]
                    _print_git_command(cmd)
                    result = subprocess.run(
                        cmd,
                        check=True,
                        capture_output=True,
                    )
                    console.print(f"[green]✓ Created new branch {branch}[/green]")

        # Pull Git LFS files after checkout
        try:
            console.print("[dim]Pulling Git LFS files...[/dim]")
            cmd = ["git", "lfs", "install"]
            _print_git_command(cmd)
            subprocess.run(
                cmd,
                capture_output=True,
                check=False,  # Don't fail if LFS is not available
            )
            cmd = ["git", "lfs", "pull"]
            _print_git_command(cmd)
            subprocess.run(
                cmd,
                capture_output=True,
                check=False,  # Don't fail if LFS is not available
            )
        except Exception:
            # LFS pull might fail if no LFS files or git-lfs not installed, that's okay
            pass

    except subprocess.CalledProcessError as e:
        err_console.print(f"[bold red]✗ Failed to checkout branch:[/bold red] {e}")
        raise typer.Exit(code=1)


# NOTE: login command has been removed.
# Authentication is handled automatically when needed.


@app.command()
def versions(
    working_group: Annotated[
        Optional[str], typer.Argument(help="Working group name (e.g., charm, beauty)")
    ] = None,
    production_name: Annotated[
        Optional[str], typer.Argument(help="Production name to query versions for")
    ] = None,
    output_format: Annotated[
        OutputFormat,
        typer.Option("--format", "-f", help="Output format"),
    ] = OutputFormat.TEXT,
):
    """
    📋 List available production versions

    Query available versions for Analysis Productions, optionally filtered by
    working group and production name.

    Examples:
        lb-ap-exp versions
        lb-ap-exp versions charm MyAnalysis
    """
    if not production_name and not working_group:
        err_console.print(
            "[bold yellow]⚠ Version querying currently requires both working group "
            "and production name[/bold yellow]\n"
            "Usage: [cyan]lb-ap-exp versions WORKING_GROUP PRODUCTION_NAME[/cyan]"
        )
        raise typer.Exit(code=1)

    if working_group and not production_name:
        err_console.print(
            "[bold yellow]⚠ Please specify both working group and production name[/bold yellow]\n"
            "Usage: [cyan]lb-ap-exp versions WORKING_GROUP PRODUCTION_NAME[/cyan]"
        )
        raise typer.Exit(code=1)

    try:
        versions_list = production_to_versions(working_group, production_name)
    except Exception as e:
        err_console.print(f"[bold red]✗ Failed to fetch versions:[/bold red] {e}")
        raise typer.Exit(code=1)

    if output_format == OutputFormat.TEXT:
        table = Table(
            title=f"📦 Available Versions for {production_name}",
            show_header=True,
            header_style="bold cyan",
        )
        table.add_column("Version", style="green", no_wrap=True)
        table.add_column("Status", style="yellow")

        for version in versions_list:
            table.add_row(version, "✓ Available")

        console.print(table)
    elif output_format == OutputFormat.JSON:
        import json

        console.print(
            json.dumps(
                {"production": production_name, "versions": versions_list}, indent=2
            )
        )
    elif output_format == OutputFormat.YAML:
        import yaml

        console.print(
            yaml.dump({"production": production_name, "versions": versions_list})
        )


@app.command()
def clone(
    branch_or_wg: Annotated[
        Optional[str],
        typer.Argument(help="Branch name OR working group (for production mode)"),
    ] = None,
    production_name: Annotated[
        Optional[str], typer.Argument(help="Production name (production mode only)")
    ] = None,
    version: Annotated[
        Optional[str], typer.Argument(help="Version tag (production mode only)")
    ] = None,
    new_branch: Annotated[
        Optional[str],
        typer.Argument(help="New branch name to create (production mode only)"),
    ] = None,
    clone_type: Annotated[
        Optional[CloneType],
        typer.Option(
            "--clone-type",
            "-c",
            help="Git clone protocol (auto-detected if not specified)",
        ),
    ] = None,
    force: Annotated[
        bool,
        typer.Option("--force", "-f", help="Force clone even if directory exists"),
    ] = False,
):
    """
    📥 Clone AnalysisProductions repository

    Two modes:

    **Branch Mode** (1 argument):
      Clone and checkout/create a branch for general development
      Example: lb-ap-exp clone my-feature-branch

    **Production Mode** (4 arguments):
      Clone and checkout a specific production version to a NEW branch
      Example: lb-ap-exp clone charm MyAnalysis v1.0 my-new-branch

    If no arguments: Interactive wizard

    Examples:
        lb-ap-exp clone                                    # Interactive wizard
        lb-ap-exp clone master                             # Branch mode: checkout master
        lb-ap-exp clone my-feature                         # Branch mode: create 'my-feature'
        lb-ap-exp clone charm MyAnalysis v1.0 work-branch  # Production mode
    """
    # Auto-detect clone type if not specified
    if clone_type is None:
        clone_type = get_default_clone_type()

    # Interactive mode - ask user what they want to do
    if branch_or_wg is None:
        choice = Prompt.ask(
            "\n[bold cyan]What would you like to do?[/bold cyan]\n"
            "  [green]branch[/green]     - Clone and checkout a branch (for general development)\n"
            "  [green]production[/green] - Clone and checkout a production version (creates new branch)",
            choices=["branch", "production"],
            default="branch",
        )

        if choice == "branch":
            branch_or_wg = Prompt.ask(
                "\n[bold cyan]Enter branch name to checkout/create[/bold cyan]",
                default="master",
            )
        else:  # production
            params = clone_wizard()
            if not params:
                raise typer.Exit()
            branch_or_wg = params["working_group"]
            production_name = params["production_name"]
            version = params["version"]
            new_branch = params["branch_name"]
            clone_type = CloneType(params["clone_type"])

    # Determine mode: branch clone vs production checkout
    is_production_mode = all([production_name, version, new_branch])

    # Check if directory already exists
    if Path("AnalysisProductions").exists() and not force:
        if not Confirm.ask(
            "[yellow]AnalysisProductions directory already exists. Continue?[/yellow]"
        ):
            console.print("[yellow]Operation cancelled[/yellow]")
            raise typer.Exit()

    # Determine clone URL
    clone_urls = {
        CloneType.SSH: "ssh://git@gitlab.cern.ch:7999/lhcb-datapkg/AnalysisProductions.git",
        CloneType.HTTPS: "https://gitlab.cern.ch/lhcb-datapkg/AnalysisProductions.git",
        CloneType.KRB5: "https://:@gitlab.cern.ch:8443/lhcb-datapkg/AnalysisProductions.git",
    }
    clone_url = clone_urls[clone_type]

    if is_production_mode:
        # Production checkout mode - use the old complex workflow
        working_group = branch_or_wg
        branch_name = new_branch

        # Verify version corresponds to production
        try:
            available_versions = production_to_versions(working_group, production_name)
            if version not in available_versions:
                err_console.print(
                    f"[bold red]✗ Version '{version}' not found for {production_name}[/bold red]\n"
                    f"Available versions: {', '.join(available_versions)}"
                )
                raise typer.Exit(code=1)
        except Exception as e:
            err_console.print(f"[bold red]✗ Failed to verify version:[/bold red] {e}")
            raise typer.Exit(code=1)

        console.print(
            Panel(
                f"[cyan]Repository:[/cyan] AnalysisProductions\n"
                f"[cyan]Production:[/cyan] {production_name}\n"
                f"[cyan]Version:[/cyan] {version}\n"
                f"[cyan]New Branch:[/cyan] {branch_name}\n"
                f"[cyan]Protocol:[/cyan] {clone_type.value}",
                title="🔄 Production Clone Configuration",
                border_style="cyan",
            )
        )
    else:
        # Branch mode - simple clone and checkout
        branch = branch_or_wg

        console.print(
            Panel(
                f"[cyan]Repository:[/cyan] AnalysisProductions\n"
                f"[cyan]Branch:[/cyan] {branch}\n"
                f"[cyan]Protocol:[/cyan] {clone_type.value}",
                title="🔄 Branch Clone Configuration",
                border_style="cyan",
            )
        )

    # Clone repository
    with Progress(
        SpinnerColumn(),
        TextColumn("[progress.description]{task.description}"),
        console=console,
    ) as progress:
        progress.add_task(description="Cloning repository...", total=None)

        try:
            _run_git_exp("clone", clone_url, printing=False)
        except Exception as e:
            err_console.print(
                f"[bold red]✗ Clone failed![/bold red]\n"
                f"Error: {e}\n"
                f"Check your permissions for {clone_type.value} access"
            )
            raise typer.Exit(code=1)

    if not Path("AnalysisProductions").is_dir():
        err_console.print("[bold red]✗ Clone failed - directory not created[/bold red]")
        raise typer.Exit(code=1)

    # Pull Git LFS files after clone
    os.chdir("AnalysisProductions")
    try:
        console.print("[dim]Pulling Git LFS files...[/dim]")
        _run_git_exp("lfs", "install", printing=True)
        _run_git_exp("lfs", "pull", printing=True)
    except Exception:
        # LFS pull might fail if no LFS files or git-lfs not installed, that's okay
        pass

    if is_production_mode:
        # Production mode: checkout specific production version
        try:
            _checkout_exp(
                working_group, production_name, version, branch_name, allow_deletes=True
            )
            console.print(
                Panel(
                    f"[bold green]✓ Successfully cloned and checked out {production_name}[/bold green]\n"
                    f"[cyan]Production:[/cyan] {working_group}/{production_name}\n"
                    f"[cyan]Version:[/cyan] {version}\n"
                    f"[cyan]Branch:[/cyan] {branch_name}",
                    title="Clone Complete",
                    border_style="green",
                )
            )
        except Exception as e:
            err_console.print(f"[bold red]✗ Checkout failed:[/bold red] {e}")
            raise typer.Exit(code=1)
    else:
        # Branch mode: checkout or create the specified branch
        try:
            cmd = ["git", "checkout", branch]
            _print_git_command(cmd)
            result = subprocess.run(
                cmd,
                capture_output=True,
                text=True,
            )
            if result.returncode == 0:
                console.print(
                    Panel(
                        f"[bold green]✓ Successfully cloned and checked out branch '{branch}'[/bold green]\n"
                        f"[cyan]Location:[/cyan] {Path.cwd().resolve()}",
                        title="Clone Complete",
                        border_style="green",
                    )
                )
            else:
                # Branch doesn't exist, try to create it
                cmd = ["git", "checkout", "-b", branch]
                _print_git_command(cmd)
                result = subprocess.run(
                    cmd,
                    capture_output=True,
                    text=True,
                )
                if result.returncode == 0:
                    console.print(
                        Panel(
                            f"[bold green]✓ Successfully cloned and created branch '{branch}'[/bold green]\n"
                            f"[cyan]Location:[/cyan] {Path.cwd().resolve()}",
                            title="Clone Complete",
                            border_style="green",
                        )
                    )
                else:
                    err_console.print(
                        f"[bold red]✗ Branch checkout failed:[/bold red] {result.stderr}"
                    )
                    raise typer.Exit(code=1)
        except Exception as e:
            err_console.print(f"[bold red]✗ Branch checkout failed:[/bold red] {e}")
            raise typer.Exit(code=1)


@app.command()
def checkout(
    working_group: Annotated[
        Optional[str], typer.Argument(help="Working group name")
    ] = None,
    production_name: Annotated[
        Optional[str], typer.Argument(help="Production name")
    ] = None,
    version: Annotated[
        Optional[str], typer.Argument(help="Version tag to checkout")
    ] = None,
    branch_name: Annotated[
        Optional[str], typer.Argument(help="Branch name to create")
    ] = None,
    allow_deletes: Annotated[
        bool,
        typer.Option(
            "--allow-deletes",
            help="Allow deletion of existing production directory",
        ),
    ] = False,
    interactive: Annotated[
        bool,
        typer.Option("--interactive", "-w", help="Use interactive wizard mode"),
    ] = False,
):
    """
    🔀 Checkout a specific production version

    Clean out the current copy of the specified production and restore
    it from the specified version tag.

    If arguments are omitted, an interactive wizard will guide you through the process.

    Examples:
        lb-ap-exp checkout                                 # Interactive wizard
        lb-ap-exp checkout charm MyAnalysis v1.0 my-branch
        lb-ap-exp checkout beauty Analysis v2.1 test --allow-deletes
    """
    # Use wizard if arguments missing or interactive mode requested
    if interactive or not all([working_group, production_name, version, branch_name]):
        params = checkout_wizard()
        if not params:
            raise typer.Exit()

        working_group = params["working_group"]
        production_name = params["production_name"]
        version = params["version"]
        branch_name = params["branch_name"]
        allow_deletes = params["allow_deletes"]
    try:
        _checkout_exp(
            working_group,
            production_name,
            version,
            branch_name,
            allow_deletes=allow_deletes,
        )
        console.print(
            Panel(
                f"[bold green]✓ Successfully checked out {production_name} @ {version}[/bold green]",
                title="Checkout Complete",
                border_style="green",
            )
        )
    except Exception as e:
        err_console.print(f"[bold red]✗ Checkout failed:[/bold red] {e}")
        raise typer.Exit(code=1)


@app.command(name="list")
def list_productions(
    production_name: Annotated[
        Optional[str],
        typer.Argument(help="Production name to list jobs for (optional)"),
    ] = None,
    show_tree: Annotated[
        bool,
        typer.Option("--tree", "-t", help="Display as tree structure"),
    ] = False,
):
    """
    📂 List available productions and jobs

    List all available production folders, or jobs within a specific production.

    Examples:
        lb-ap-exp list
        lb-ap-exp list MyAnalysis
        lb-ap-exp list MyAnalysis --tree
    """
    try:
        inside_ap_datapkg()
    except Exception as e:
        err_console.print(
            f"[bold red]✗ Not in AnalysisProductions directory:[/bold red] {e}"
        )
        raise typer.Exit(code=1)

    if production_name:
        # List jobs for specific production
        try:
            check_production(production_name)
        except Exception as e:
            err_console.print(f"[bold red]✗ Production not found:[/bold red] {e}")
            raise typer.Exit(code=1)

        # Parse info.yaml
        info_yaml_path = Path(production_name) / "info.yaml"
        with open(info_yaml_path, "rt") as fp:
            raw_yaml = fp.read()
        job_data = parse_yaml(render_yaml(raw_yaml))

        if show_tree:
            tree = Tree(f"[bold cyan]{production_name}[/bold cyan]")
            for job_name in job_data:
                tree.add(f"[green]{job_name}[/green]")
            console.print(tree)
        else:
            table = Table(
                title=f"📋 Jobs in {production_name}",
                show_header=True,
                header_style="bold cyan",
            )
            table.add_column("Job Name", style="green", no_wrap=True)
            table.add_column("Status", style="yellow")

            for job_name in job_data:
                table.add_row(job_name, "✓ Available")

            console.print(table)
    else:
        # List all productions
        productions = list(available_productions())

        if show_tree:
            tree = Tree("[bold cyan]📂 AnalysisProductions[/bold cyan]")
            for folder in productions:
                # Count jobs for tree view
                try:
                    info_yaml_path = Path(folder) / "info.yaml"
                    with open(info_yaml_path, "rt") as fp:
                        raw_yaml = fp.read()
                    job_data = parse_yaml(render_yaml(raw_yaml))
                    job_count = len(job_data)
                    tree.add(f"[green]{folder}[/green] [dim]({job_count} jobs)[/dim]")
                except Exception:
                    tree.add(f"[green]{folder}[/green] [dim](? jobs)[/dim]")
            console.print(tree)
        else:
            table = Table(
                title="📂 Available Productions",
                show_header=True,
                header_style="bold cyan",
            )
            table.add_column("Production", style="green", no_wrap=True)
            table.add_column("Jobs", style="cyan", justify="right")
            table.add_column("Status", style="yellow")

            for folder in productions:
                # Count jobs in each production
                try:
                    info_yaml_path = Path(folder) / "info.yaml"
                    with open(info_yaml_path, "rt") as fp:
                        raw_yaml = fp.read()
                    job_data = parse_yaml(render_yaml(raw_yaml))
                    job_count = len(job_data)
                    table.add_row(folder, str(job_count), "✓ Available")
                except Exception:
                    table.add_row(folder, "?", "⚠ Error reading info.yaml")

            console.print(table)


# NOTE: render and validate commands have been removed.
# Use `lb-ap test <production> <job> --dry-run` instead.


@app.command()
def test(
    production_name: Annotated[
        Optional[str], typer.Argument(help="Production name")
    ] = None,
    job_name: Annotated[Optional[str], typer.Argument(help="Job name to test")] = None,
    dependent_input: Annotated[
        Optional[str],
        typer.Option(
            "-i",
            "--input",
            help="Specific input file (LFN or local path)",
        ),
    ] = None,
    interactive: Annotated[
        bool,
        typer.Option("--interactive", "-w", help="Use interactive wizard mode"),
    ] = False,
    output_dir: Annotated[
        Optional[Path],
        typer.Option(
            "-o",
            "--output",
            help="Output directory for CWL files and execution",
        ),
    ] = None,
    n_lfns: Annotated[
        int,
        typer.Option(
            "--n-lfns",
            help="Number of LFNs to retrieve from Bookkeeping for testing",
        ),
    ] = 1,
    pick_smallest: Annotated[
        bool,
        typer.Option(
            "--pick-smallest-lfn",
            help="Pick the smallest file(s) for faster testing (requires --n-lfns)",
        ),
    ] = False,
    pretty: Annotated[
        bool,
        typer.Option(
            "--pretty",
            help="Show split-view real-time logs (left: stdout, right: current step log)",
        ),
    ] = False,
    download: Annotated[
        bool,
        typer.Option(
            "--download",
            help="Download input files locally instead of using remote protocol where available",
        ),
    ] = False,
    dry_run: Annotated[
        bool,
        typer.Option(
            "--dry-run",
            help="Validate and render configuration without executing the test",
        ),
    ] = False,
):
    """
    🧪 Test a job using CWL workflow

    Generate CWL workflow from production job, then execute it using a CWL runner.
    This combines YAML rendering, CWL generation, and execution in one command.

    If arguments are omitted, an interactive wizard will guide you through the process.

    Examples:
        lb-ap test                                     # Interactive wizard
        lb-ap test MyAnalysis MyJob                    # Uses 1 LFN by default
        lb-ap test MyAnalysis MyJob --download         # Download input files locally
        lb-ap test MyAnalysis MyJob --n-lfns 5         # Test with 5 LFNs
        lb-ap test MyAnalysis MyJob --pick-smallest-lfn  # Use smallest LFN
        lb-ap test MyAnalysis MyJob --pretty           # Split-view real-time logs
        lb-ap test MyAnalysis MyJob -o ./output
        lb-ap test MyAnalysis MyJob --dry-run          # Validate without executing
    """
    # Handle dry-run mode first with custom logic
    if dry_run:
        # Get available productions for wizard
        productions_list = None
        try:
            inside_ap_datapkg()
            productions_list = list(available_productions())
        except Exception:
            pass

        # Use wizard if arguments missing or interactive mode requested
        if interactive or not all([production_name, job_name]):
            params = test_wizard(productions_list)
            if not params:
                raise typer.Exit()

            production_name = params["production_name"]
            job_name = params["job_name"]

        try:
            inside_ap_datapkg()
            check_production(production_name)
        except Exception as e:
            err_console.print(f"[bold red]✗ Error:[/bold red] {e}")
            raise typer.Exit(code=1)

        # Environment validation is always performed
        console.print("[cyan]Validating environment...[/cyan]")
        try:
            validate_environment()
        except Exception as e:
            err_console.print(
                f"[bold red]✗ Environment validation failed:[/bold red] {e}"
            )
            raise typer.Exit(code=1)

        # Read and render the info.yaml
        info_yaml_path = Path(production_name) / "info.yaml"
        if not info_yaml_path.exists():
            err_console.print(
                f"[bold red]✗ info.yaml not found:[/bold red] {info_yaml_path}"
            )
            raise typer.Exit(code=1)

        from LbAPCommon import parse_yaml, render_yaml, validate_yaml

        raw_yaml = info_yaml_path.read_text()
        rendered = render_yaml(raw_yaml)

        # Show rendered YAML
        console.print("\n[bold cyan]📄 Rendered YAML:[/bold cyan]")
        from rich.syntax import Syntax

        syntax = Syntax(rendered, "yaml", theme="monokai", line_numbers=True)
        console.print(syntax)

        jobs_data = parse_yaml(rendered, production_name, None)

        if job_name not in jobs_data:
            available_jobs = list(jobs_data.keys())
            err_console.print(
                f"[bold red]✗ Job '{job_name}' not found in production '{production_name}'[/bold red]\n"
                f"Available jobs: {', '.join(available_jobs)}"
            )
            raise typer.Exit(code=1)

        # Validate the YAML
        warnings = validate_yaml(jobs_data, production_name)
        if warnings:
            console.print("\n[yellow]⚠ Validation warnings:[/yellow]")
            for warning in warnings:
                console.print(f"  [yellow]•[/yellow] {warning}")
        else:
            console.print("\n[bold green]✓ Validation passed[/bold green]")

        console.print(
            "\n[bold green]✓ Dry-run complete - configuration is valid[/bold green]"
        )
        console.print(f"[cyan]Production:[/cyan] {production_name}")
        console.print(f"[cyan]Job:[/cyan] {job_name}")
        console.print(f"[cyan]Available jobs:[/cyan] {', '.join(jobs_data.keys())}")
        return

    # For non-dry-run mode, delegate to cwl_test with skip_validation=False (always validate)
    cwl_test(
        production_name=production_name,
        job_name=job_name,
        dependent_input=dependent_input,
        n_events=None,  # Not supported in new API
        skip_validation=False,  # Always validate
        interactive=interactive,
        output_dir=output_dir,
        n_lfns=n_lfns,
        pick_smallest=pick_smallest,
        pretty=pretty,
        download=download,
    )


# NOTE: debug command has been removed.
# Use the debug bash scripts generated per CWL step instead.


# ============================================================================
# Reproducer Commands
# ============================================================================


class ReproducerStrategy(str, Enum):
    """Strategy for creating a minimal reproducer."""

    SINGLE_EVENT = "single-event"
    EVENT_WINDOW = "event-window"
    WHOLE_FILE = "whole-file"
    ALL_FILES = "all-files"


@app.command(name="make-reproducer")
def make_reproducer(
    job_id: Annotated[
        Optional[str],
        typer.Argument(help="DIRAC job ID to download and create reproducer from"),
    ] = None,
    job_dir: Annotated[
        Optional[Path],
        typer.Option(
            "--job-dir",
            "-d",
            help="Use existing downloaded job directory instead of downloading",
        ),
    ] = None,
    output_dir: Annotated[
        Optional[Path],
        typer.Option(
            "--output-dir",
            "-o",
            help="Directory for reproducer output",
        ),
    ] = None,
    strategy: Annotated[
        Optional[ReproducerStrategy],
        typer.Option(
            "--strategy",
            "-s",
            help="Reproducer strategy (prompted interactively if not given)",
        ),
    ] = None,
    no_run: Annotated[
        bool,
        typer.Option(
            "--no-run",
            help="Don't run the reproducer, just create the specification",
        ),
    ] = False,
    no_tarball: Annotated[
        bool,
        typer.Option(
            "--no-tarball",
            help="Don't create a tarball, just the reproducer directory",
        ),
    ] = False,
):
    """
    Create a minimal reproducer from a failed grid job.

    Downloads a failed job, identifies the problematic event, and creates
    a self-contained package that can be shared for debugging.

    The wizard guides you through choosing a strategy:

    - **single-event**: Extract only the problematic event (smallest tarball)
    - **whole-file**: Use the entire problematic input file
    - **all-files**: Keep all input files (warning: large tarballs)

    Examples:
        lb-ap make-reproducer 1234567
        lb-ap make-reproducer --job-dir /tmp/downloaded_job
        lb-ap make-reproducer 1234567 --strategy single-event
        lb-ap make-reproducer 1234567 --no-run
    """
    import tempfile

    from ..job_slice import (
        analyze_reproducer_output,
        create_reproducer_package,
        create_reproducer_prodconf,
        create_tarball,
        download_lfn_files,
        extract_events,
        extract_single_event,
        find_data_packages,
        find_failed_step,
        find_last_event,
        get_all_input_lfns_from_prodconf,
        run_reproducer,
    )

    # Validate arguments
    if not job_id and not job_dir:
        err_console.print(
            "[bold red]Please provide either a DIRAC job ID or --job-dir[/bold red]"
        )
        raise typer.Exit(code=1)

    # ── Step 1: Get the job directory ──────────────────────────────────────
    console.print()
    console.rule("[bold]Step 1: Obtaining job files[/bold]")
    console.print()

    if job_dir:
        job_dir = Path(job_dir).resolve()
        if not job_dir.exists():
            err_console.print(
                f"[bold red]Job directory does not exist:[/bold red] {job_dir}"
            )
            raise typer.Exit(code=1)
        console.print(f"[green]Using existing job directory:[/green] {job_dir}")
    else:
        # Download via dirac-wms-job-reproduce
        console.print(f"[cyan]Downloading job {job_id}...[/cyan]")
        download_dir = Path(tempfile.mkdtemp(prefix="lb-ap-reproducer-"))

        try:
            result = subprocess.run(
                [
                    "dirac-wms-job-reproduce",
                    "reproduce",
                    str(job_id),
                    "-o",
                    str(download_dir),
                    "--dry-run",
                ],
                capture_output=True,
                text=True,
            )
            if result.returncode != 0:
                err_console.print(
                    f"[bold red]Download failed:[/bold red]\n{result.stderr + result.stdout}"
                )
                raise typer.Exit(code=1)
        except FileNotFoundError:
            err_console.print(
                "[bold red]dirac-wms-job-reproduce not found.[/bold red]\n"
                "Make sure LHCbDIRAC is installed and the DIRAC environment is set up."
            )
            raise typer.Exit(code=1)

        # Find the actual job directory (should be a single subdirectory)
        subdirs = [d for d in download_dir.iterdir() if d.is_dir()]
        if len(subdirs) == 1:
            job_dir = subdirs[0]
        elif subdirs:
            # Multiple subdirs - look for the one matching job_id
            matching = [d for d in subdirs if job_id in d.name]
            job_dir = matching[0] if matching else subdirs[0]
        else:
            job_dir = download_dir

        console.print(f"[green]Downloaded to:[/green] {job_dir}")

    console.print()

    # ── Step 2: Find the failed step ──────────────────────────────────────
    console.rule("[bold]Step 2: Finding failed step[/bold]")
    console.print()

    try:
        failed_step = find_failed_step(job_dir)
    except (FileNotFoundError, ValueError) as e:
        err_console.print(f"[bold red]{e}[/bold red]")
        raise typer.Exit(code=1)

    console.print(f"[green]Failed step:[/green] {failed_step.step_id}")
    console.print(f"  [dim]XML:[/dim] {failed_step.xml_file.name}")
    console.print(f"  [dim]Log:[/dim] {failed_step.log_file.name}")
    if failed_step.prodconf_file:
        console.print(f"  [dim]ProdConf:[/dim] {failed_step.prodconf_file.name}")
    if failed_step.input_file_statuses:
        n_full = sum(1 for s in failed_step.input_file_statuses if s.status == "full")
        n_part = sum(1 for s in failed_step.input_file_statuses if s.status == "part")
        console.print(
            f"  [dim]Input files:[/dim] {n_full} fully processed, {n_part} partial"
        )
        for s in failed_step.input_file_statuses:
            if s.status == "part":
                console.print(
                    f"    [yellow]Partial:[/yellow] {s.lfn} ({s.events_read} events read)"
                )
    console.print()

    # ── Step 3: Parse log for event info ──────────────────────────────────
    console.rule("[bold]Step 3: Finding last processed event[/bold]")
    console.print()

    try:
        last_event = find_last_event(
            failed_step.log_file, failed_step.input_file_statuses
        )
    except (FileNotFoundError, ValueError) as e:
        err_console.print(f"[bold red]{e}[/bold red]")
        raise typer.Exit(code=1)
    except NotImplementedError as e:
        err_console.print(f"[bold red]{e}[/bold red]")
        err_console.print("[yellow]Run 2 jobs are not supported yet.[/yellow]")
        raise typer.Exit(code=1)

    console.print(f"[green]Last event indicator:[/green] {last_event.event_number}")
    console.print(f"  [dim]Slot:[/dim] {last_event.slot}")

    if last_event.can_slice_single:
        console.print(
            "[green]New-style event messages found[/green] - can slice to single event"
        )
    else:
        console.print(
            "[yellow]Only old-style event messages found[/yellow] - can slice to ~100-event window"
        )
        if last_event.lfn:
            console.print(f"  [dim]Last read file:[/dim] {last_event.lfn}")

    if last_event.failed_event_overall is not None:
        console.print(
            f"  [dim]FATAL event (overall):[/dim] {last_event.failed_event_overall}"
        )

    if last_event.suspect_event_in_file is not None and last_event.suspect_lfns:
        console.print(
            f"[yellow]The problematic event is likely around event "
            f"{last_event.suspect_event_in_file} in {last_event.suspect_lfns[0]}[/yellow]"
        )
    elif last_event.suspect_lfns:
        console.print(
            f"[yellow]The problematic file is likely:[/yellow] {last_event.suspect_lfns[0]}"
        )
        if len(last_event.suspect_lfns) > 1:
            console.print(
                f"  [dim](and {len(last_event.suspect_lfns) - 1} other file(s))[/dim]"
            )
    elif last_event.lfn:
        console.print(f"[yellow]Last file being read:[/yellow] {last_event.lfn}")

    # Determine input LFNs for reproducer: suspect_lfns (from XML/log analysis)
    # are the best source, falling back to log heuristics, then prodConf.
    if last_event.suspect_lfns:
        all_input_lfns = last_event.suspect_lfns
        input_lfn = all_input_lfns[0]
    elif last_event.unprocessed_lfns:
        all_input_lfns = last_event.unprocessed_lfns
        input_lfn = all_input_lfns[0]
    elif last_event.lfn:
        all_input_lfns = [last_event.lfn]
        input_lfn = last_event.lfn
    elif failed_step.prodconf_file:
        all_input_lfns = get_all_input_lfns_from_prodconf(failed_step.prodconf_file)
        input_lfn = all_input_lfns[0] if all_input_lfns else None
    else:
        all_input_lfns = []
        input_lfn = None

    if input_lfn:
        console.print(f"[green]Input file for reproducer:[/green] {input_lfn}")
        if len(all_input_lfns) > 1:
            console.print(f"  [dim]({len(all_input_lfns)} candidate files total)[/dim]")
    else:
        console.print("[yellow]Could not determine input file LFN[/yellow]")

    console.print()

    # ── Step 4: Ask for strategy ──────────────────────────────────────────
    console.rule("[bold]Step 4: Choose reproducer strategy[/bold]")
    console.print()

    # Read prodConf to detect if DaVinci version upgrade is applicable
    import json as _json

    version_override = None
    binary_tag_override = None
    can_upgrade_davinci = False
    if failed_step.prodconf_file and failed_step.prodconf_file.exists():
        with open(failed_step.prodconf_file) as f:
            original_prodconf = _json.load(f)
        app_name = original_prodconf.get("application", {}).get("name", "")
        app_version = original_prodconf.get("application", {}).get("version", "")
        can_upgrade_davinci = (
            app_name == "DaVinci"
            and app_version.startswith("v66r")
            and app_version != "v66r9"
        )

    if strategy is None:
        console.print(
            "[bold cyan]How would you like to create the reproducer?[/bold cyan]\n"
        )

        if can_upgrade_davinci:
            # DaVinci v66 (not v66r9): offer upgrade for precise slicing
            console.print(
                "  [green]1.[/green] [bold]Slice single event using DaVinci/v66r9[/bold] (recommended)\n"
                "     [dim]Upgrade to v66r9 for per-event logging, enabling precise\n"
                "     single-event extraction. Smallest reproducer.[/dim]\n"
            )
            console.print(
                "  [green]2.[/green] [bold]Slice to few hundred events[/bold] "
                f"(using original {app_version})\n"
                "     [dim]Keep original version. Only old-style log messages available,\n"
                "     so we can narrow to 100s of events around the failure.[/dim]\n"
            )
            console.print(
                "  [green]3.[/green] [bold]Use problematic file only[/bold] - Keep the whole input file\n"
                "     [dim]Moderate size. Uses only the file containing the problem event.[/dim]\n"
            )
            console.print(
                "  [green]4.[/green] [bold]Keep all files[/bold] - Include all input files (not recommended)\n"
                "     [dim][yellow]Warning: can produce very large tarballs![/yellow][/dim]\n"
            )

            from rich.prompt import IntPrompt

            choice = IntPrompt.ask(
                "[cyan]Select strategy (1-4)[/cyan]",
                default=1,
                choices=["1", "2", "3", "4"],
            )
            if choice == 1:
                strategy = ReproducerStrategy.SINGLE_EVENT
                version_override = "v66r9"
                binary_tag_override = "x86_64_v3-el9-gcc15-opt+g"
                console.print("[green]Using DaVinci/v66r9[/green] for precise slicing.")
            else:
                strategy = {
                    2: ReproducerStrategy.EVENT_WINDOW,
                    3: ReproducerStrategy.WHOLE_FILE,
                    4: ReproducerStrategy.ALL_FILES,
                }[choice]
        else:
            # Standard menu (already v66r9, non-DaVinci, or non-v66)
            if last_event.can_slice_single:
                console.print(
                    "  [green]1.[/green] [bold]Slice single event[/bold] - Extract only the problematic event\n"
                    "     [dim]Smallest reproducer, best for sharing. Runs the job with print_freq=1\n"
                    "     to identify the exact event, then extracts it.[/dim]\n"
                )
            else:
                console.print(
                    "  [green]1.[/green] [bold]Slice to few hundred events[/bold] - Narrow to a small event window\n"
                    "     [dim]Only old-style log messages found, so we can narrow to 100s of events\n"
                    "     around the failure but not easily to a single event.[/dim]\n"
                )
            console.print(
                "  [green]2.[/green] [bold]Use problematic file only[/bold] - Keep the whole input file\n"
                "     [dim]Moderate size. Uses only the file containing the problem event.[/dim]\n"
            )
            console.print(
                "  [green]3.[/green] [bold]Keep all files[/bold] - Include all input files (not recommended)\n"
                "     [dim][yellow]Warning: can produce very large tarballs![/yellow][/dim]\n"
            )

            from rich.prompt import IntPrompt

            choice = IntPrompt.ask(
                "[cyan]Select strategy (1-3)[/cyan]",
                default=1,
                choices=["1", "2", "3"],
            )
            if last_event.can_slice_single:
                strategy = {
                    1: ReproducerStrategy.SINGLE_EVENT,
                    2: ReproducerStrategy.WHOLE_FILE,
                    3: ReproducerStrategy.ALL_FILES,
                }[choice]
            else:
                strategy = {
                    1: ReproducerStrategy.EVENT_WINDOW,
                    2: ReproducerStrategy.WHOLE_FILE,
                    3: ReproducerStrategy.ALL_FILES,
                }[choice]

    console.print(f"[green]Strategy:[/green] {strategy.value}")
    console.print()

    # ── Step 5: Create reproducer prodConf ────────────────────────────────
    if not failed_step.prodconf_file or not failed_step.prodconf_file.exists():
        err_console.print(
            "[bold red]No prodConf JSON found for the failed step.[/bold red]"
        )
        err_console.print("[dim]Cannot create reproducer without prodConf.[/dim]")
        raise typer.Exit(code=1)

    console.rule("[bold]Step 5: Creating reproducer configuration[/bold]")
    console.print()

    # Determine input files based on strategy
    if strategy == ReproducerStrategy.ALL_FILES:
        reproducer_input_files = None  # Keep original
    elif strategy == ReproducerStrategy.WHOLE_FILE:
        reproducer_input_files = [input_lfn] if input_lfn else None
    elif input_lfn:
        # SINGLE_EVENT and EVENT_WINDOW: use all suspect LFNs
        reproducer_input_files = all_input_lfns
    else:
        reproducer_input_files = None

    reproducer_prodconf = job_dir / f"prodConf_{failed_step.step_id}_reproducer.json"

    # Estimate the suspect event number for skip calculation.
    # Use the best estimate we have: suspect_event_in_file (old-style + FATAL),
    # event_in_file (old-style approximate), or event_number (new-style overall).
    estimated_suspect_event = (
        last_event.suspect_event_in_file
        or last_event.event_in_file
        or last_event.event_number
    )

    first_event = 0
    skipped_events = False
    skip_threshold = 1000  # Only offer to skip if the suspect event is far enough in
    if estimated_suspect_event and estimated_suspect_event > skip_threshold:
        skip_to = max(0, estimated_suspect_event - 50)
        console.print(
            f"[cyan]The suspect event is around event {estimated_suspect_event}.[/cyan]\n"
            f"[cyan]Processing from event 0 with print_freq=1 may be slow.[/cyan]"
        )
        do_skip = Confirm.ask(
            f"[cyan]Skip to event {skip_to} to speed things up?[/cyan]",
            default=True,
        )
        if do_skip:
            first_event = skip_to
            skipped_events = True
        else:
            # User declined the default skip - offer more options
            console.print()
            console.print("[bold cyan]Choose starting event:[/bold cyan]\n")
            console.print("  [green]1.[/green] Start from the beginning (event 0)")
            console.print(
                f"  [green]2.[/green] Start 10 events before suspect (event {max(0, estimated_suspect_event - 10)})"
            )
            console.print("  [green]3.[/green] Custom start point")
            console.print()

            choice = IntPrompt.ask(
                "[cyan]Select option (1-3)[/cyan]",
                default=1,
                choices=["1", "2", "3"],
            )

            if choice == 1:
                first_event = 0
            elif choice == 2:
                first_event = max(0, estimated_suspect_event - 10)
                skipped_events = True
            else:  # choice == 3
                while True:
                    custom_event = IntPrompt.ask(
                        f"[cyan]Enter starting event number (0 to {estimated_suspect_event})[/cyan]",
                        default=0,
                    )
                    if custom_event < 0:
                        console.print(
                            "[yellow]Event number cannot be negative. Please try again.[/yellow]"
                        )
                    else:
                        first_event = custom_event
                        if custom_event > 0:
                            skipped_events = True
                        break
            console.print()

    if last_event.can_slice_single or version_override == "v66r9":
        console.print(
            f"[dim]Starting from event {first_event} (precise per-event slicing)[/dim]"
        )
    else:
        console.print(
            f"[dim]Starting from event {first_event} "
            f"(find rough ~100-event window around failing event)[/dim]"
        )

    if reproducer_input_files:
        console.print(
            f"[cyan]Testing {len(reproducer_input_files)} candidate file(s)[/cyan]"
        )

    create_reproducer_prodconf(
        failed_step.prodconf_file,
        reproducer_prodconf,
        input_files=reproducer_input_files,
        first_event=first_event,
        n_of_events=-1,
        print_freq=1,
        application_version=version_override,
        binary_tag=binary_tag_override,
    )
    console.print(
        f"[green]Created reproducer prodConf:[/green] {reproducer_prodconf.name}"
    )

    # If --no-run, show instructions and exit
    if no_run:
        console.print()
        console.print(
            Panel(
                f"[green]Reproducer configuration created.[/green]\n\n"
                f"[cyan]To run manually:[/cyan]\n"
                f"  cd {job_dir}\n"
                f"  lb-prod-run {reproducer_prodconf.name}\n",
                title="No-Run Mode",
                border_style="yellow",
            )
        )
        return

    # ── Step 6: Run the reproducer ────────────────────────────────────────
    console.print()
    console.rule("[bold]Step 6: Running reproducer[/bold]")
    console.print()
    console.print("[cyan]Executing reproducer with print_freq=1...[/cyan]")
    console.print(
        "[dim]This identifies the exact problematic event by logging every event.[/dim]"
    )
    console.print()

    # Setup progress display showing last 10 lines
    recent_lines: list[str] = []
    max_lines = 10

    def update_progress(line: str):
        recent_lines.append(line)
        if len(recent_lines) > max_lines:
            recent_lines.pop(0)

        # Create display text
        display_text = Text()
        for idx, log_line in enumerate(recent_lines):
            # Add a subtle separator between lines
            if idx > 0:
                display_text.append("\n")
            # Truncate very long lines to avoid wrapping issues
            truncated = log_line[:200] + "..." if len(log_line) > 200 else log_line
            display_text.append(truncated, style="dim")

        return Panel(
            display_text,
            title="[cyan]Reproducer Output (last 10 lines)[/cyan]",
            border_style="blue",
        )

    try:
        with Live(
            Panel(
                Text("Starting reproducer...", style="dim"),
                title="[cyan]Reproducer Output (last 10 lines)[/cyan]",
                border_style="blue",
            ),
            console=console,
            refresh_per_second=4,
        ) as live:

            def on_line(line: str):
                live.update(update_progress(line))

            result, slice_dir = run_reproducer(
                job_dir, reproducer_prodconf, on_line=on_line
            )
    except FileNotFoundError:
        err_console.print("[bold red]lb-prod-run not found.[/bold red]")
        err_console.print("[dim]Make sure the LHCb environment is set up.[/dim]")
        raise typer.Exit(code=1)

    # Save reproducer log
    if output_dir is None:
        output_dir = job_dir / "reproducer"
    output_dir = Path(output_dir).resolve()
    output_dir.mkdir(exist_ok=True, parents=True)

    reproducer_log = output_dir / "reproducer_run.log"
    with open(reproducer_log, "w") as f:
        f.write(
            f"STDOUT:\n{'=' * 80}\n{result.stdout}\n\nSTDERR:\n{'=' * 80}\n{result.stderr}\n"
        )

    console.print(
        f"[green]Reproducer finished[/green] (exit code: {result.returncode})"
    )
    console.print(f"[dim]Log saved to: {reproducer_log}[/dim]")
    console.print()

    # ── Step 7: Analyze output ────────────────────────────────────────────
    console.rule("[bold]Step 7: Analyzing reproducer output[/bold]")
    console.print()

    analysis = analyze_reproducer_output(
        result.stdout,
        result.stderr,
        first_event_offset=first_event,
    )
    analysis.returncode = result.returncode

    # Use the LFN from the reproducer output if available, otherwise from prodConf
    lfn_for_extraction = analysis.last_event_lfn or input_lfn

    if analysis.exact_problem_event is not None:
        if analysis.failed_event_overall is not None:
            console.print(
                f"[green]Exact problematic event (in-file):[/green] {analysis.exact_problem_event}"
            )
            console.print(
                f"  [dim]Failed event (overall):[/dim] {analysis.failed_event_overall}"
            )
            if analysis.last_event_in_file is not None:
                console.print(
                    f"  [dim]Last successful read:[/dim] {analysis.last_event_in_file} "
                    f"(overall: {analysis.last_event_overall})"
                )
        elif analysis.can_slice_single:
            console.print(
                f"[green]Probable problematic event:[/green] {analysis.exact_problem_event}"
            )
            if analysis.last_processing_event is not None:
                console.print(
                    f"  [dim]Last HLTControlFlowMgr event:[/dim] {analysis.last_processing_event}"
                )
            if analysis.last_event_in_file is not None:
                console.print(
                    f"  [dim]Last successful read:[/dim] {analysis.last_event_in_file} "
                    f"(overall: {analysis.last_event_overall})"
                )
            console.print(
                f"[yellow]No explicit FATAL message - likely segfault/crash during "
                f"event {analysis.exact_problem_event}[/yellow]"
            )
        else:
            # Old-style only, no FATAL
            console.print(
                f"[yellow]Best-guess problematic event (in-file):[/yellow] {analysis.exact_problem_event}"
            )
            if analysis.last_event_in_file is not None:
                console.print(
                    f"  [dim]Last successful read:[/dim] {analysis.last_event_in_file} "
                    f"(overall: {analysis.last_event_overall})"
                )
            console.print(
                "[yellow]Only old-style log messages found and no FATAL message.[/yellow]\n"
                "[yellow]The problematic event is approximately but not precisely identified.[/yellow]"
            )
    else:
        console.print("[yellow]Could not identify event reads in output.[/yellow]")
        console.print("[dim]Job may have failed before processing events.[/dim]")

    # Check if the error was reproduced
    if result.returncode != 0:
        console.print(
            f"\n[green]Error reproduced successfully (exit code {result.returncode})[/green]"
        )
    else:
        # If we skipped events, retry from event 0 before trying wider scope
        if skipped_events:
            console.print()
            console.print(
                "[yellow]Error not reproduced after skipping events.[/yellow]\n"
                "[cyan]Retrying from event 0 (no skip)...[/cyan]"
            )
            create_reproducer_prodconf(
                failed_step.prodconf_file,
                reproducer_prodconf,
                input_files=reproducer_input_files,
                first_event=0,
                n_of_events=-1,
                print_freq=1,
                application_version=version_override,
                binary_tag=binary_tag_override,
            )

            with Live(
                Panel(
                    Text("Retrying from event 0...", style="dim"),
                    title="[cyan]Reproducer Output (last 10 lines)[/cyan]",
                    border_style="blue",
                ),
                console=console,
                refresh_per_second=4,
            ) as live:

                def on_retry_line(line: str):
                    recent_lines.append(line)
                    if len(recent_lines) > max_lines:
                        recent_lines.pop(0)
                    display_text = Text()
                    for idx, log_line in enumerate(recent_lines):
                        if idx > 0:
                            display_text.append("\n")
                        truncated = (
                            log_line[:200] + "..." if len(log_line) > 200 else log_line
                        )
                        display_text.append(truncated, style="dim")
                    live.update(
                        Panel(
                            display_text,
                            title="[cyan]Reproducer Output (last 10 lines)[/cyan]",
                            border_style="blue",
                        )
                    )

                result, slice_dir = run_reproducer(
                    job_dir, reproducer_prodconf, on_line=on_retry_line
                )

            with open(reproducer_log, "a") as f:
                f.write(f"\n\nNO-SKIP RETRY STDOUT:\n{'=' * 80}\n{result.stdout}")
                f.write(f"\n\nNO-SKIP RETRY STDERR:\n{'=' * 80}\n{result.stderr}")

            analysis = analyze_reproducer_output(result.stdout, result.stderr)
            analysis.returncode = result.returncode
            lfn_for_extraction = analysis.last_event_lfn or input_lfn
            skipped_events = False  # Don't retry the skip path again

            if result.returncode != 0:
                console.print(
                    f"[green]Error reproduced from event 0 (exit code {result.returncode})[/green]"
                )

        if result.returncode == 0:
            console.print()
            console.print(
                Panel(
                    "[yellow]The reproducer completed successfully - the error was NOT reproduced.[/yellow]\n\n"
                    "This could mean:\n"
                    "  - The error is intermittent / non-deterministic\n"
                    "  - The error depends on specific environment conditions\n"
                    "  - The problematic event was not in the tested range\n",
                    title="Error Not Reproduced",
                    border_style="yellow",
                )
            )
            # Offer to try with wider scope (more files)
            if strategy != ReproducerStrategy.ALL_FILES:
                try_wider = Confirm.ask(
                    "[cyan]Would you like to try with a wider scope (more files/events)?[/cyan]",
                    default=True,
                )
                if try_wider:
                    if strategy in (
                        ReproducerStrategy.SINGLE_EVENT,
                        ReproducerStrategy.EVENT_WINDOW,
                    ):
                        console.print("[cyan]Retrying with whole file...[/cyan]")
                        create_reproducer_prodconf(
                            failed_step.prodconf_file,
                            reproducer_prodconf,
                            input_files=[input_lfn] if input_lfn else None,
                            first_event=0,
                            n_of_events=-1,
                            print_freq=1,
                            application_version=version_override,
                            binary_tag=binary_tag_override,
                        )
                    else:
                        console.print("[cyan]Retrying with all input files...[/cyan]")
                        create_reproducer_prodconf(
                            failed_step.prodconf_file,
                            reproducer_prodconf,
                            input_files=None,  # Keep all original files
                            first_event=0,
                            n_of_events=-1,
                            print_freq=1,
                            application_version=version_override,
                            binary_tag=binary_tag_override,
                        )

                    result, slice_dir = run_reproducer(job_dir, reproducer_prodconf)
                    with open(reproducer_log, "a") as f:
                        f.write(f"\n\nRETRY STDOUT:\n{'=' * 80}\n{result.stdout}")
                        f.write(f"\n\nRETRY STDERR:\n{'=' * 80}\n{result.stderr}")

                    analysis = analyze_reproducer_output(result.stdout, result.stderr)
                    analysis.returncode = result.returncode
                    lfn_for_extraction = analysis.last_event_lfn or input_lfn

                    if result.returncode != 0:
                        console.print(
                            f"[green]Error reproduced on retry (exit code {result.returncode})[/green]"
                        )
                    else:
                        console.print("[yellow]Error still not reproduced.[/yellow]")

    console.print()

    # ── Step 7b: Confirm the error reproduces with narrowed scope ─────────
    exact_event = analysis.exact_problem_event

    # Only attempt single-event extraction if we can precisely identify it
    can_extract_single = (
        strategy == ReproducerStrategy.SINGLE_EVENT
        and exact_event is not None
        and lfn_for_extraction
        and result.returncode != 0
        and (analysis.can_slice_single or analysis.failed_event_overall is not None)
    )

    if can_extract_single and len(all_input_lfns) > 1:
        # We narrowed from multiple files — confirm the error with just the suspect file
        console.rule("[bold]Step 7b: Confirming error with narrowed scope[/bold]")
        console.print()
        console.print(
            f"[cyan]Verifying error reproduces with just {lfn_for_extraction}[/cyan]"
        )

        confirm_prodconf = job_dir / f"prodConf_{failed_step.step_id}_confirm.json"
        create_reproducer_prodconf(
            failed_step.prodconf_file,
            confirm_prodconf,
            input_files=[lfn_for_extraction],
            first_event=0,
            n_of_events=-1,
            print_freq=1,
            application_version=version_override,
            binary_tag=binary_tag_override,
        )

        try:
            with Live(
                Panel(
                    Text("Running confirmation...", style="dim"),
                    title="[cyan]Confirmation Run[/cyan]",
                    border_style="blue",
                ),
                console=console,
                refresh_per_second=4,
            ) as live:
                confirm_lines: list[str] = []

                def on_confirm_line(line: str):
                    confirm_lines.append(line)
                    if len(confirm_lines) > max_lines:
                        confirm_lines.pop(0)
                    display_text = Text()
                    for idx, log_line in enumerate(confirm_lines):
                        if idx > 0:
                            display_text.append("\n")
                        truncated = (
                            log_line[:200] + "..." if len(log_line) > 200 else log_line
                        )
                        display_text.append(truncated, style="dim")
                    live.update(
                        Panel(
                            display_text,
                            title="[cyan]Confirmation Run[/cyan]",
                            border_style="blue",
                        )
                    )

                confirm_result, _ = run_reproducer(
                    job_dir, confirm_prodconf, on_line=on_confirm_line
                )
        except FileNotFoundError:
            confirm_result = None

        if confirm_result and confirm_result.returncode != 0:
            console.print(
                f"[green]Confirmed:[/green] error reproduces with just {lfn_for_extraction} "
                f"(exit code {confirm_result.returncode})"
            )
            # Re-analyze the confirmation output for the most accurate event info
            analysis = analyze_reproducer_output(
                confirm_result.stdout, confirm_result.stderr
            )
            analysis.returncode = confirm_result.returncode
            exact_event = analysis.exact_problem_event
            lfn_for_extraction = analysis.last_event_lfn or lfn_for_extraction

            with open(reproducer_log, "a") as f:
                f.write(
                    f"\n\nCONFIRMATION STDOUT:\n{'=' * 80}\n{confirm_result.stdout}"
                )
                f.write(
                    f"\n\nCONFIRMATION STDERR:\n{'=' * 80}\n{confirm_result.stderr}"
                )
        else:
            console.print(
                "[yellow]Error did NOT reproduce with just the narrowed file.[/yellow]\n"
                "[dim]The reproducer package will include all candidate files instead.[/dim]"
            )
            # Fall back to using all candidate files
            can_extract_single = False
            lfn_for_extraction = analysis.last_event_lfn or input_lfn

        console.print()

    if (
        not can_extract_single
        and strategy == ReproducerStrategy.SINGLE_EVENT
        and result.returncode != 0
    ):
        if not analysis.can_slice_single and analysis.failed_event_overall is None:
            console.print()
            console.print(
                "[yellow]Cannot extract a single event: only old-style log messages found "
                "and no FATAL message to pinpoint the exact event.[/yellow]"
            )
            console.print(
                "[dim]The reproducer package will include the full input file instead.[/dim]"
            )

    # ── Step 8: Prepare input files for package ───────────────────────────
    console.print()
    console.rule("[bold]Step 8: Preparing input files[/bold]")
    console.print()

    local_input_files: list[Path] | None = None
    pool_catalog = job_dir / "pool_xml_catalog.xml"

    if can_extract_single:
        # SINGLE_EVENT: extract the exact problematic event
        assert lfn_for_extraction is not None  # guaranteed by can_extract_single
        assert exact_event is not None  # guaranteed by can_extract_single
        console.print(
            f"[cyan]Extracting event {exact_event} from {lfn_for_extraction}[/cyan]"
        )
        extract_dir = slice_dir / "single_event_extraction"

        try:
            extracted_file, _extraction_result = extract_single_event(
                input_lfn=lfn_for_extraction,
                exact_event=exact_event,
                extract_dir=extract_dir,
                pool_catalog=pool_catalog if pool_catalog.exists() else None,
                prodconf_file=failed_step.prodconf_file,
            )

            if extracted_file:
                console.print(
                    f"[green]Single event extracted:[/green] {extracted_file.name}"
                )
                local_input_files = [extracted_file]
            else:
                console.print(
                    "[yellow]Event extraction did not produce output file.[/yellow]"
                )
                console.print("[dim]Check extraction log for details.[/dim]")
        except FileNotFoundError:
            console.print(
                "[yellow]lb-run not found - skipping event extraction.[/yellow]"
            )

    elif strategy == ReproducerStrategy.EVENT_WINDOW:
        # EVENT_WINDOW: extract ~200 events around the problematic event
        if exact_event is not None and lfn_for_extraction and result.returncode != 0:
            window_half = 100
            first_evt = max(0, exact_event - window_half)
            n_events = 2 * window_half
            console.print(
                f"[cyan]Extracting ~{n_events} events around event {exact_event} "
                f"from {lfn_for_extraction}[/cyan]"
            )
            console.print(
                f"[dim]Event range: {first_evt} to {first_evt + n_events}[/dim]"
            )
            extract_dir = slice_dir / "event_window_extraction"

            try:
                extracted_file, _extraction_result = extract_events(
                    input_lfn=lfn_for_extraction,
                    first_event=first_evt,
                    n_events=n_events,
                    extract_dir=extract_dir,
                    pool_catalog=pool_catalog if pool_catalog.exists() else None,
                    prodconf_file=failed_step.prodconf_file,
                )

                if extracted_file:
                    console.print(
                        f"[green]Event window extracted:[/green] {extracted_file.name}"
                    )
                    local_input_files = [extracted_file]
                else:
                    console.print(
                        "[yellow]Event window extraction did not produce output file.[/yellow]"
                    )
                    console.print("[dim]Check extraction log for details.[/dim]")
            except FileNotFoundError:
                console.print(
                    "[yellow]lb-run not found - skipping event extraction.[/yellow]"
                )
        else:
            console.print(
                "[yellow]Could not determine exact event - "
                "cannot extract event window.[/yellow]"
            )

    elif strategy == ReproducerStrategy.WHOLE_FILE:
        # WHOLE_FILE: download the problematic file
        if lfn_for_extraction:
            console.print(
                f"[cyan]Downloading {lfn_for_extraction}, this may take a while...[/cyan]"
            )
            download_dir = output_dir / "downloads"
            try:
                downloaded = download_lfn_files([lfn_for_extraction], download_dir)
                if downloaded:
                    console.print(
                        f"[green]Downloaded:[/green] {downloaded[0].name} "
                        f"({downloaded[0].stat().st_size / (1024*1024):.1f} MB)"
                    )
                    local_input_files = downloaded
                else:
                    console.print(
                        "[yellow]Download completed but file not found locally.[/yellow]"
                    )
            except FileNotFoundError:
                console.print(
                    "[yellow]dirac-dms-get-file not found - "
                    "file will be referenced by LFN.[/yellow]"
                )
            except RuntimeError as e:
                console.print(f"[yellow]Download failed: {e}[/yellow]")
        else:
            console.print("[yellow]No input LFN to download.[/yellow]")

    elif strategy == ReproducerStrategy.ALL_FILES:
        # ALL_FILES: download all original input files
        all_lfns = get_all_input_lfns_from_prodconf(failed_step.prodconf_file)
        if all_lfns:
            console.print(f"[cyan]Downloading {len(all_lfns)} input file(s)...[/cyan]")
            download_dir = output_dir / "downloads"
            try:
                downloaded = download_lfn_files(all_lfns, download_dir)
                if downloaded:
                    total_size = sum(f.stat().st_size for f in downloaded)
                    console.print(
                        f"[green]Downloaded {len(downloaded)} file(s)[/green] "
                        f"({total_size / (1024*1024):.1f} MB total)"
                    )
                    local_input_files = downloaded
                else:
                    console.print(
                        "[yellow]Download completed but no files found locally.[/yellow]"
                    )
            except FileNotFoundError:
                console.print(
                    "[yellow]dirac-dms-get-file not found - "
                    "files will be referenced by LFN.[/yellow]"
                )
            except RuntimeError as e:
                console.print(f"[yellow]Download failed: {e}[/yellow]")
        else:
            console.print("[yellow]No input LFNs found in prodConf.[/yellow]")

    # ── Step 9: Collect data package files ────────────────────────────────
    console.print()
    console.rule("[bold]Step 9: Collecting data package files[/bold]")
    console.print()

    data_packages = []
    if failed_step.prodconf_file:
        data_packages = find_data_packages(failed_step.prodconf_file)
        for dp in data_packages:
            console.print(f"  [green]Found:[/green] {dp.name}.{dp.version}")
            console.print(f"    [dim]{dp.cvmfs_path}[/dim]")

    if not data_packages:
        console.print("[dim]No data packages found in prodConf.[/dim]")

    # ── Step 10: Create reproducer package ────────────────────────────────
    console.print()
    console.rule("[bold]Step 10: Creating reproducer package[/bold]")
    console.print()

    minimal_dir = output_dir / "minimal_reproducer"
    create_reproducer_package(
        output_dir=minimal_dir,
        job_dir=job_dir,
        failed_step=failed_step,
        input_lfn=lfn_for_extraction,
        exact_event=exact_event,
        local_input_files=local_input_files,
        prodconf_file=failed_step.prodconf_file,
        data_packages=data_packages if data_packages else None,
        job_id=job_id,
    )

    console.print(f"[green]Reproducer package created:[/green] {minimal_dir}")
    console.print()

    # Show directory contents
    console.print("[blue]Package contents:[/blue]")
    for item in sorted(minimal_dir.rglob("*")):
        if item.is_file():
            size = item.stat().st_size
            size_str = (
                f"{size / 1024:.1f} KB"
                if size < 1024 * 1024
                else f"{size / (1024 * 1024):.1f} MB"
            )
            rel = item.relative_to(minimal_dir)
            console.print(f"  {rel} ({size_str})")

    # ── Step 10b: Test the reproducer package ────────────────────────────
    reproducer_json = minimal_dir / "reproducer.json"
    if reproducer_json.exists() and result.returncode != 0:
        console.print()
        console.rule("[bold]Step 10b: Testing reproducer package[/bold]")
        console.print()
        console.print(
            "[cyan]Running the reproducer package to verify it works...[/cyan]"
        )

        try:
            with Live(
                Panel(
                    Text("Running reproducer package test...", style="dim"),
                    title="[cyan]Package Test[/cyan]",
                    border_style="blue",
                ),
                console=console,
                refresh_per_second=4,
            ) as live:
                test_lines: list[str] = []

                def on_test_line(line: str):
                    test_lines.append(line)
                    if len(test_lines) > max_lines:
                        test_lines.pop(0)
                    display_text = Text()
                    for idx, log_line in enumerate(test_lines):
                        if idx > 0:
                            display_text.append("\n")
                        truncated = (
                            log_line[:200] + "..." if len(log_line) > 200 else log_line
                        )
                        display_text.append(truncated, style="dim")
                    live.update(
                        Panel(
                            display_text,
                            title="[cyan]Package Test[/cyan]",
                            border_style="blue",
                        )
                    )

                test_result, to_remove = run_minimal_reproducer(
                    minimal_dir, on_line=on_test_line
                )

            if test_result.returncode != 0:
                # Show evidence of the error from the last few lines of output
                all_output = test_result.stdout + "\n" + test_result.stderr
                evidence_lines = []
                for line in all_output.splitlines():
                    if (
                        "FATAL" in line
                        or "ERROR" in line
                        or "Segmentation" in line
                        or "Exception" in line
                    ):
                        evidence_lines.append(line.strip())
                if evidence_lines:
                    # Show the most relevant error lines (last 10)
                    shown = evidence_lines[-10:]
                    error_text = "\n".join(shown)
                    console.print(
                        Panel(
                            f"[green]Error reproduced[/green] (exit code {test_result.returncode})\n\n"
                            f"[dim]{error_text}[/dim]",
                            title="Package Test Passed",
                            border_style="green",
                        )
                    )
                    # Remove the test run directory to avoid putting
                    # it in the tarball.
                    shutil.rmtree(to_remove, ignore_errors=True)
                else:
                    # No FATAL/Exception found, show last few lines of output
                    tail = test_lines[-5:] if test_lines else []
                    tail_text = "\n".join(line.strip() for line in tail)
                    console.print(
                        Panel(
                            f"[green]Error reproduced[/green] (exit code {test_result.returncode})\n\n"
                            f"[dim]{tail_text}[/dim]",
                            title="Package Test Passed",
                            border_style="green",
                        )
                    )
            else:
                console.print(
                    Panel(
                        "[yellow]The reproducer package ran successfully - the error was "
                        "NOT reproduced.[/yellow]\n\n"
                        "The package may need manual adjustments before sharing.\n"
                        "Check that input files and event numbers are correct.",
                        title="Package Test Failed",
                        border_style="yellow",
                    )
                )

            with open(reproducer_log, "a") as f:
                f.write(f"\n\nPACKAGE TEST STDOUT:\n{'=' * 80}\n{test_result.stdout}")
                f.write(f"\n\nPACKAGE TEST STDERR:\n{'=' * 80}\n{test_result.stderr}")

        except FileNotFoundError:
            console.print(
                "[yellow]lb-prod-run not found - skipping package test.[/yellow]"
            )

    # ── Step 11: Tarball ──────────────────────────────────────────────────
    if not no_tarball:
        console.print()

        do_tarball = Confirm.ask(
            "[cyan]The reproducer package is ready. You can inspect and edit the files\n"
            f"in [bold]{minimal_dir}[/bold] before packaging.\n\n"
            "Create tarball now?[/cyan]",
            default=True,
        )
        step_id = failed_step.step_id
        event_suffix = f"_event_{exact_event}" if exact_event else ""
        tarball_name = f"minimal_reproducer_{step_id}{event_suffix}.tar.gz"

        if do_tarball:
            tarball_path = output_dir / tarball_name
            try:
                create_tarball(minimal_dir, tarball_path)
                tarball_size = tarball_path.stat().st_size
                size_str = (
                    f"{tarball_size / 1024:.1f} KB"
                    if tarball_size < 1024 * 1024
                    else f"{tarball_size / (1024 * 1024):.1f} MB"
                )
                console.print(f"[green]Tarball created:[/green] {tarball_path}")
                console.print(f"  [dim]Size: {size_str}[/dim]")
            except Exception as e:
                err_console.print(f"[bold red]Failed to create tarball:[/bold red] {e}")
        else:
            console.print(
                f"[dim]Skipped tarball. To create it later:[/dim]\n"
                f"  tar -czf {tarball_name} -C {minimal_dir.parent} {minimal_dir.name}"
            )

    # ── Final Summary ─────────────────────────────────────────────────────
    console.print()

    summary_parts = [
        "[green]Reproducer created successfully![/green]\n",
        f"[blue]Package:[/blue] {minimal_dir}",
        f"[blue]Log:[/blue] {reproducer_log}",
    ]

    if exact_event is not None:
        summary_parts.append(f"[blue]Problematic event:[/blue] {exact_event}")
    if lfn_for_extraction:
        summary_parts.append(f"[blue]Input file:[/blue] {lfn_for_extraction}")
    summary_parts.append(f"[blue]Exit code:[/blue] {result.returncode}")
    summary_parts.append("")
    summary_parts.append(
        f"[yellow]To run:[/yellow] cd {minimal_dir} && ./run_reproducer.sh"
    )

    console.print(
        Panel(
            "\n".join(summary_parts),
            title="Make-Reproducer Complete",
            border_style="green" if result.returncode != 0 else "yellow",
        )
    )


# ============================================================================
# CWL Experimental Commands
# ============================================================================


@cwl_app.command(name="generate", hidden=True)
def cwl_generate(
    production_name: Annotated[
        Optional[str], typer.Argument(help="Production name")
    ] = None,
    job_name: Annotated[Optional[str], typer.Argument(help="Job name")] = None,
    output_dir: Annotated[
        Optional[Path],
        typer.Option(
            "-o",
            "--output",
            help="Output directory for CWL files",
        ),
    ] = None,
    interactive: Annotated[
        bool,
        typer.Option("--interactive", "-w", help="Use interactive wizard mode"),
    ] = False,
):
    """
    🔧 Generate CWL workflow from production

    Convert an AnalysisProductions job into a CWL workflow definition.

    If arguments are omitted, an interactive wizard will guide you through the process.

    Examples:
        lb-ap cwl generate                             # Interactive wizard
        lb-ap cwl generate MyAnalysis MyJob
        lb-ap cwl generate MyAnalysis MyJob -o ./cwl_output
    """
    # Get available productions for wizard
    productions_list = None
    try:
        inside_ap_datapkg()
        productions_list = list(available_productions())
    except Exception:
        pass

    # Use wizard if arguments missing or interactive mode requested
    if interactive or not all([production_name, job_name]):
        params = cwl_generate_wizard(productions_list)
        if not params:
            raise typer.Exit()

        production_name = params["production_name"]
        job_name = params["job_name"]
        output_dir = params.get("output_dir") or output_dir
    console.print(
        Panel(
            "[bold yellow]⚠ CWL generation is experimental[/bold yellow]\n"
            "This feature is under active development.",
            title="Experimental Feature",
            border_style="yellow",
        )
    )

    output_dir = output_dir or Path("./cwl_generated")
    output_dir.mkdir(parents=True, exist_ok=True)

    console.print(f"[cyan]Generating CWL for {production_name}/{job_name}...[/cyan]")

    # TODO: Implement actual CWL generation logic
    console.print(
        f"[yellow]CWL generation not yet implemented[/yellow]\n"
        f"Output directory: {output_dir}"
    )


@cwl_app.command(name="run", hidden=True)
def cwl_run(
    cwl_file: Annotated[Path, typer.Argument(help="CWL workflow file")],
    inputs_file: Annotated[
        Optional[Path],
        typer.Option(
            "-i",
            "--inputs",
            help="CWL inputs file (YAML/JSON)",
        ),
    ] = None,
    executor: Annotated[
        str,
        typer.Option(
            "--executor",
            help="CWL executor to use",
        ),
    ] = "cwltool",
):
    """
    ▶️  Run a CWL workflow

    Execute a CWL workflow file using the specified executor.

    Examples:
        lb-ap cwl run workflow.cwl -i inputs.yml
        lb-ap cwl run workflow.cwl --executor cwltool
    """
    console.print(
        Panel(
            "[bold yellow]⚠ CWL execution is experimental[/bold yellow]\n"
            "This feature is under active development.",
            title="Experimental Feature",
            border_style="yellow",
        )
    )

    if not cwl_file.exists():
        err_console.print(f"[bold red]✗ CWL file not found:[/bold red] {cwl_file}")
        raise typer.Exit(code=1)

    console.print(f"[cyan]Running CWL workflow:[/cyan] {cwl_file}")

    # TODO: Implement CWL execution
    console.print("[yellow]CWL execution not yet implemented[/yellow]")


@cwl_app.command(name="validate", hidden=True)
def cwl_validate(
    cwl_file: Annotated[Path, typer.Argument(help="CWL workflow file to validate")],
):
    """
    ✅ Validate a CWL workflow

    Check if a CWL workflow file is valid according to the CWL specification.

    Examples:
        lb-ap cwl validate workflow.cwl
    """
    if not cwl_file.exists():
        err_console.print(f"[bold red]✗ CWL file not found:[/bold red] {cwl_file}")
        raise typer.Exit(code=1)

    console.print(f"[cyan]Validating CWL workflow:[/cyan] {cwl_file}")

    # TODO: Implement CWL validation using cwltool
    console.print("[yellow]CWL validation not yet implemented[/yellow]")


@cwl_app.command(name="test", deprecated=True)
def cwl_test(
    production_name: Annotated[
        Optional[str], typer.Argument(help="Production name")
    ] = None,
    job_name: Annotated[Optional[str], typer.Argument(help="Job name to test")] = None,
    dependent_input: Annotated[
        Optional[str],
        typer.Option(
            "-i",
            "--input",
            help="Specific input file (LFN or local path)",
        ),
    ] = None,
    n_events: Annotated[
        Optional[int],
        typer.Option(
            "-n",
            "--n-events",
            help="Number of events to process",
        ),
    ] = None,
    skip_validation: Annotated[
        bool,
        typer.Option(
            "--skip-validation",
            help="Skip environment validation",
        ),
    ] = False,
    interactive: Annotated[
        bool,
        typer.Option("--interactive", "-w", help="Use interactive wizard mode"),
    ] = False,
    output_dir: Annotated[
        Optional[Path],
        typer.Option(
            "-o",
            "--output",
            help="Output directory for CWL files and execution",
        ),
    ] = None,
    n_lfns: Annotated[
        int,
        typer.Option(
            "--n-lfns",
            help="Number of LFNs to retrieve from Bookkeeping for testing",
        ),
    ] = 1,
    pick_smallest: Annotated[
        bool,
        typer.Option(
            "--pick-smallest-lfn",
            help="Pick the smallest file(s) for faster testing (requires --n-lfns)",
        ),
    ] = False,
    pretty: Annotated[
        bool,
        typer.Option(
            "--pretty",
            help="Show split-view real-time logs (left: stdout, right: current step log)",
        ),
    ] = False,
    download: Annotated[
        bool,
        typer.Option(
            "--download",
            help="Download input files locally instead of using remote protocol where available",
        ),
    ] = False,
):
    """
    🧪 Test a job using CWL workflow

    Generate CWL workflow from production job, then execute it using a CWL runner.
    This combines YAML rendering, CWL generation, and execution in one command.

    If arguments are omitted, an interactive wizard will guide you through the process.

    Examples:
        lb-ap cwl test                                     # Interactive wizard
        lb-ap cwl test MyAnalysis MyJob                    # Uses 1 LFN by default
        lb-ap cwl test MyAnalysis MyJob --n-events 100
        lb-ap cwl test MyAnalysis MyJob --download         # Download input files locally
        lb-ap cwl test MyAnalysis MyJob --n-lfns 5         # Test with 5 LFNs
        lb-ap cwl test MyAnalysis MyJob --pick-smallest-lfn  # Use smallest LFN
        lb-ap cwl test MyAnalysis MyJob --pretty           # Split-view real-time logs
        lb-ap cwl test MyAnalysis MyJob -o ./output
        lb-ap cwl test MyAnalysis MyJob --executor toil-cwl-runner
    """
    executor = "dirac-cwl-run"  # Default executor
    # Get available productions for wizard
    productions_list = None
    try:
        inside_ap_datapkg()
        productions_list = list(available_productions())
    except Exception:
        pass

    # Use wizard if arguments missing or interactive mode requested
    if interactive or not all([production_name, job_name]):
        params = test_wizard(productions_list)
        if not params:
            raise typer.Exit()

        production_name = params["production_name"]
        job_name = params["job_name"]
        dependent_input = params.get("dependent_input") or dependent_input
        n_events = params.get("n_events") or n_events
        skip_validation = params.get("skip_validation", skip_validation)

    try:
        inside_ap_datapkg()
        check_production(production_name)
    except Exception as e:
        err_console.print(f"[bold red]✗ Error:[/bold red] {e}")
        raise typer.Exit(code=1)

    if not skip_validation:
        console.print("[cyan]Validating environment...[/cyan]")
        try:
            validate_environment()
        except Exception as e:
            err_console.print(
                f"[bold red]✗ Environment validation failed:[/bold red] {e}"
            )
            raise typer.Exit(code=1)

    # Set up output directory with timestamp for easy identification
    import datetime

    if not output_dir:
        timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
        output_dir = Path(f"cwl_test_{production_name}_{job_name}_{timestamp}")

    output_dir = output_dir.resolve()  # Convert to absolute path
    output_dir.mkdir(parents=True, exist_ok=True)

    console.print(
        Panel(
            f"[cyan]Production:[/cyan] {production_name}\n"
            f"[cyan]Job:[/cyan] {job_name}\n"
            f"[cyan]Events:[/cyan] {n_events or 'default'}\n"
            f"[cyan]Input:[/cyan] {dependent_input or 'default'}\n"
            f"[cyan]Executor:[/cyan] {executor}\n"
            f"[cyan]Output:[/cyan] {output_dir}",
            title="🧪 CWL Test Configuration",
            border_style="cyan",
        )
    )

    # Step 1: Generate DIRAC production request YAML using LbAPCommon
    console.print(
        "\n[bold cyan]Step 1/3: Generating DIRAC production request YAML[/bold cyan]"
    )

    # Read and render the info.yaml to check job exists
    info_yaml_path = Path(production_name) / "info.yaml"
    if not info_yaml_path.exists():
        err_console.print(
            f"[bold red]✗ info.yaml not found:[/bold red] {info_yaml_path}"
        )
        raise typer.Exit(code=1)

    # Quick check that job exists
    from LbAPCommon import parse_yaml, render_yaml

    raw_yaml = info_yaml_path.read_text()
    rendered = render_yaml(raw_yaml)
    jobs_data = parse_yaml(rendered, production_name, None)

    if job_name not in jobs_data:
        available_jobs = list(jobs_data.keys())
        err_console.print(
            f"[bold red]✗ Job '{job_name}' not found in production '{production_name}'[/bold red]\n"
            f"Available jobs: {', '.join(available_jobs)}"
        )
        raise typer.Exit(code=1)

    # Use LbAPCommon's main function to generate proper DIRAC production request
    # This queries DIRAC for input metadata and creates the full production structure
    import asyncio
    import json

    from LbAPCommon.__main__ import main as lbapcommon_main

    console.print("[cyan]Querying DIRAC for job metadata...[/cyan]")

    # Store metadata in output directory for inspection
    metadata_path = output_dir / "dirac_metadata.json"

    try:
        # Run LbAPCommon main to generate production request
        # Use --only-include to generate just the job we want
        asyncio.run(
            lbapcommon_main(
                production_name=production_name,
                ap_pkg_version="v999999999999",  # Test version
                input_file=info_yaml_path,
                output_file=metadata_path,
                server_credentials=None,
                only_include=job_name,
                dump_requests=True,  # This creates the YAML file we need
            )
        )

        # Read the generated metadata
        with open(metadata_path, "r") as f:
            metadata = json.load(f)

        # Find the production request YAML that was dumped
        # dump_requests creates files named after the production
        productions = metadata.get("productions", {})
        if not productions:
            err_console.print("[bold red]✗ No productions generated[/bold red]")
            raise typer.Exit(code=1)

        # Get the first (and should be only) production
        prod_key = list(productions.keys())[0]
        production_data = productions[prod_key]

        console.print(f"[green]✓ Generated production:[/green] {prod_key}")
        console.print(
            f"[dim]Request chain: {' → '.join([s['name'].split('#')[-1] for s in production_data['request']])}[/dim]"
        )

        # The dump_requests option creates a YAML file with the request
        # Look for it in the current directory (where metadata was written)
        request_yaml_path = metadata_path.with_name(prod_key).with_suffix(".yaml")

        if not request_yaml_path.exists():
            # If dump wasn't created, save it ourselves
            request_yaml_path = output_dir / f"{prod_key}.yaml"
            with open(request_yaml_path, "w") as f:
                yaml.dump(
                    production_data["request"],
                    f,
                    default_flow_style=False,
                    sort_keys=False,
                )
        else:
            # Move the dumped file to output directory if needed
            if request_yaml_path.parent != output_dir:
                final_yaml_path = output_dir / f"{prod_key}.yaml"
                shutil.move(str(request_yaml_path), str(final_yaml_path))
                request_yaml_path = final_yaml_path

        prod_yaml_path = request_yaml_path
        console.print(
            f"[green]✓ Saved DIRAC production request:[/green] {prod_yaml_path.name}"
        )
        console.print(f"[green]✓ Saved metadata:[/green] {metadata_path.name}")

    except Exception as e:
        err_console.print(
            f"[bold red]✗ DIRAC production request generation failed:[/bold red] {e}"
        )
        import traceback

        err_console.print(traceback.format_exc())
        raise typer.Exit(code=1)

    # Step 2: Generate CWL workflow
    console.print("\n[bold cyan]Step 2/3: Generating CWL workflow[/bold cyan]")

    from LbAPCommon.prod_request_to_cwl import (
        fromProductionRequestYAMLToCWL,
    )

    try:
        cwl_workflow, cwl_inputs, _ = fromProductionRequestYAMLToCWL(
            prod_yaml_path, production_name=None
        )
    except Exception as e:
        err_console.print(f"[bold red]✗ CWL generation failed:[/bold red] {e}")
        raise typer.Exit(code=1)

    # Save CWL workflow and inputs
    cwl_workflow_path = output_dir / f"{production_name}_{job_name}.cwl"
    cwl_inputs_path = output_dir / f"{production_name}_{job_name}-inputs.yml"

    # Convert CWL workflow object to dictionary using cwl_utils.parser.save
    from cwl_utils.parser import save

    workflow_dict = save(cwl_workflow)

    # Write workflow using ruamel.yaml for better formatting
    from ruamel.yaml import YAML

    yaml_dumper = YAML()
    yaml_dumper.default_flow_style = False
    yaml_dumper.width = 120

    with open(cwl_workflow_path, "w") as f:
        yaml_dumper.dump(workflow_dict, f)

    # Inputs is already a dict, so we can use regular yaml.dump
    # with open(cwl_inputs_path, "w") as f:
    #     yaml.dump(cwl_inputs, f, default_flow_style=False, sort_keys=False)

    console.print(f"[green]✓ Generated CWL workflow:[/green] {cwl_workflow_path}")
    # console.print(f"[green]✓ Generated CWL inputs:[/green] {cwl_inputs_path}")

    # Step 2b: Rewrite CWL for custom input (if provided)
    cwl_inputs_path = None
    if dependent_input:
        from LbAPLocal.cwl.rewrite import rewrite_cwl_for_custom_input

        console.print(
            f"\n[bold cyan]Rewriting CWL for custom input:[/bold cyan] {dependent_input}"
        )
        try:
            cwl_workflow_path, cwl_inputs_path = rewrite_cwl_for_custom_input(
                cwl_workflow_path, [dependent_input], job_name
            )
            console.print(
                f"[green]✓ Modified CWL workflow:[/green] {cwl_workflow_path}"
            )
            console.print(f"[green]✓ Custom inputs file:[/green] {cwl_inputs_path}")
        except Exception as e:
            err_console.print(f"[bold red]✗ CWL rewriting failed:[/bold red] {e}")
            import traceback

            err_console.print(traceback.format_exc())
            raise typer.Exit(code=1)

    # Step 3: Execute CWL workflow
    console.print("\n[bold cyan]Step 3/3: Executing CWL workflow[/bold cyan]")

    # Check if executor is available
    executor_check = shutil.which(executor)
    if not executor_check:
        err_console.print(
            f"[bold red]✗ CWL executor '{executor}' not found![/bold red]\n"
            f"Install with: pip install cwltool (or your preferred CWL runner)"
        )
        raise typer.Exit(code=1)

    # Set up lb-run environment for local AnalysisProductions testing
    # This creates a fake siteroot so lb-run can find AnalysisProductions.v999999999999 locally
    import os

    from LbAPCommon.hacks import project_uses_cmt, setup_lbrun_environment

    # Parse application info to check if CMT is needed
    application_name, application_version = jobs_data[job_name]["application"].rsplit(
        "/", 1
    )
    if "@" in application_version:
        application_version = application_version.split("@", 1)[0]
    require_cmt = project_uses_cmt(application_name, application_version)

    # Set up fake siteroot pointing to the output directory.
    fake_siteroot = output_dir
    fake_siteroot.mkdir(parents=True, exist_ok=True)

    # Get the absolute path to the AnalysisProductions repository
    # (the directory containing the production we're testing)
    ap_repo_path = Path.cwd().resolve()

    console.print(f"[cyan]Setting up lb-run environment in {fake_siteroot}[/cyan]")
    console.print(f"[dim]Linking AnalysisProductions repository: {ap_repo_path}[/dim]")

    # Set up the environment (modifies os.environ)
    # Args: (fake_siteroot_path, repository_path, setup_cmt)
    # This creates: fake_siteroot/DBASE/AnalysisProductions/v999999999999 -> ap_repo_path
    setup_lbrun_environment(str(fake_siteroot), str(ap_repo_path), require_cmt)

    # Create dynamic directory for auto-generated configuration files
    # This is referenced as $ANALYSIS_PRODUCTIONS_DYNAMIC in the production configs
    dynamic_dir = output_dir / "dynamic"
    dynamic_dir.mkdir(parents=True, exist_ok=True)

    # Symlink output/dynamic into ap_repo_path/dynamic
    ap_dynamic_dir = ap_repo_path / "dynamic"
    if ap_dynamic_dir.exists() or ap_dynamic_dir.is_symlink():
        if ap_dynamic_dir.is_symlink():
            ap_dynamic_dir.unlink()
        elif ap_dynamic_dir.is_dir():
            shutil.rmtree(ap_dynamic_dir)
        else:
            ap_dynamic_dir.unlink()
    ap_dynamic_dir.symlink_to(dynamic_dir.resolve())

    # Create files in dynamic
    # Note: prod_key may be a grouped name like "job1,job2" while job_name is just "job1"
    for filename, content in (
        metadata["productions"][prod_key].get("dynamic_files", {}).items()
    ):
        file_path = dynamic_dir / filename
        file_path.parent.mkdir(parents=True, exist_ok=True)
        file_path.write_text(content)
        console.print(f"[green]✓ Created dynamic file:[/green] {file_path}")

    # Create a custom environment dict with the lb-run setup to pass to subprocess
    # This ensures CMAKE_PREFIX_PATH and other variables are set for the CWL executor
    custom_env = os.environ.copy()
    custom_env["ANALYSIS_PRODUCTIONS_DYNAMIC"] = str(ap_dynamic_dir.resolve())

    # Run CWL workflow
    cwl_output_dir = output_dir / "cwl_output"
    cwl_output_dir.mkdir(parents=True, exist_ok=True)

    # Use absolute paths to avoid path resolution issues
    cmd = [
        executor,
        "--outdir",
        str(cwl_output_dir.resolve()),
    ]

    # Add dirac-cwl-run specific options for input generation
    if executor == "dirac-cwl-run":
        if not cwl_inputs_path:
            # Only add BK query options when not using custom input
            cmd.extend(["--n-lfns", str(n_lfns)])
            if pick_smallest:
                cmd.append("--pick-smallest-lfn")
            if download:
                cmd.append("--download")
            # Force regeneration to avoid interactive prompts
            cmd.append("--force-regenerate")
        # Keep temp directories for debugging
        cmd.append("--leave-tmpdir")
        # set a specific prefix for temp directories to easily identify them
        cmd.extend(["--tmpdir-prefix", f"{output_dir}/tmp_step_"])
        # Enable verbose logging to see what's happening
        cmd.append("--verbose")
        cmd.extend(["--preserve-environment", "CMAKE_PREFIX_PATH"])
        cmd.extend(["--preserve-environment", "ANALYSIS_PRODUCTIONS_DYNAMIC"])

    # Add workflow file
    cmd.append(str(cwl_workflow_path.resolve()))

    # Add inputs file if custom input was provided
    if cwl_inputs_path:
        cmd.append(str(cwl_inputs_path.resolve()))

    console.print(f"[cyan]Running:[/cyan] {shlex.join(cmd)}")
    console.print("[dim]Environment variables:[/dim]")
    console.print(
        f"[dim]  CMAKE_PREFIX_PATH={custom_env.get('CMAKE_PREFIX_PATH', 'NOT SET')}[/dim]"
    )
    console.print(
        f"[dim]  ANALYSIS_PRODUCTIONS_DYNAMIC={custom_env.get('ANALYSIS_PRODUCTIONS_DYNAMIC', 'NOT SET')}[/dim]"
    )

    # Verify the fake siteroot structure was created
    ap_link = fake_siteroot / "DBASE" / "AnalysisProductions" / "v999999999999"
    if ap_link.exists():
        console.print(
            f"[dim]  AnalysisProductions symlink: {ap_link} -> "
            f"{ap_link.resolve() if ap_link.is_symlink() else 'NOT A SYMLINK'}[/dim]\n"
        )
    else:
        console.print(
            f"[yellow]⚠ Warning: AnalysisProductions symlink not found at {ap_link}[/yellow]\n"
        )

    current_run_tmpdirs = []  # Track temp directories used in this run

    def find_log_files(include_temp_dirs=False):
        """Find and return log files, optionally including temp directories"""
        log_files = list(cwl_output_dir.glob("**/*.log"))

        if include_temp_dirs and current_run_tmpdirs:
            # Only search in temp directories from this specific run
            for tmp_dir in current_run_tmpdirs:
                if tmp_dir.exists() and tmp_dir.is_dir():
                    log_files.extend(tmp_dir.glob("*.log"))

        return sorted(set(log_files), key=lambda x: x.name)

    execution_failed = False
    max_mem_kb = 0  # Track max memory usage (only available in pretty mode)
    try:
        if pretty:
            console.print(
                Panel(
                    "[bold yellow]⚠ Pretty mode is experimental[/bold yellow]\n"
                    "This feature is under active development.",
                    title="Experimental Feature",
                    border_style="yellow",
                )
            )
            # Use Textual TUI for real-time monitoring
            from .textual_ui import run_cwl_with_textual_ui

            returncode, tmpdirs, max_mem_kb = asyncio.run(
                run_cwl_with_textual_ui(
                    cmd, custom_env, cwl_output_dir, cwl_workflow_path
                )
            )
            current_run_tmpdirs.extend(tmpdirs)
            if returncode != 0:
                execution_failed = True
                err_console.print(
                    f"[bold red]✗ CWL execution failed with exit code {returncode}[/bold red]"
                )
        else:
            # Standard output mode - stream output in real-time while parsing for temp directories
            import re
            import sys
            import threading

            debug_script_pattern = re.compile(
                r"Debug script written to:\s+(.+?/debug_interactive_.+?\.sh)"
            )

            process = subprocess.Popen(
                cmd,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                text=True,
                env=custom_env,
                bufsize=1,  # Line buffered
            )

            def _drain_stream(stream, output_stream):
                """Read lines from stream, echo them, and parse for debug scripts."""
                for line in stream:
                    output_stream.write(line)
                    output_stream.flush()
                    match = debug_script_pattern.search(line)
                    if match:
                        script_path = Path(match.group(1))
                        tmpdir = script_path.parent
                        if tmpdir.exists() and tmpdir not in current_run_tmpdirs:
                            current_run_tmpdirs.append(tmpdir)

            # Read stdout and stderr concurrently to avoid pipe buffer deadlock
            stderr_thread = threading.Thread(
                target=_drain_stream, args=(process.stderr, sys.stderr)
            )
            stderr_thread.start()
            _drain_stream(process.stdout, sys.stdout)
            stderr_thread.join()

            returncode = process.wait()
            if returncode != 0:
                execution_failed = True
                err_console.print(
                    f"[bold red]✗ CWL execution failed with exit code {returncode}[/bold red]"
                )
    except subprocess.CalledProcessError as e:
        execution_failed = True
        err_console.print(
            f"[bold red]✗ CWL execution failed with exit code {e.returncode}[/bold red]"
        )

        # Parse failed execution output for debug script paths
        if hasattr(e, "stdout") and hasattr(e, "stderr"):
            import re

            debug_script_pattern = re.compile(
                r"Debug script written to:\s+(.+?/debug_interactive_.+?\.sh)"
            )
            output = (e.stdout or "") + (e.stderr or "")
            for line in output.splitlines():
                match = debug_script_pattern.search(line)
                if match:
                    script_path = Path(match.group(1))
                    tmpdir = script_path.parent
                    if tmpdir.exists() and tmpdir not in current_run_tmpdirs:
                        current_run_tmpdirs.append(tmpdir)
    except Exception as e:
        execution_failed = True
        err_console.print(f"[bold red]✗ CWL execution failed:[/bold red] {e}")

    # If execution failed, show log files and debug scripts including temp directories
    if execution_failed:
        log_files = find_log_files(include_temp_dirs=True)
        if log_files:
            err_console.print("\n[bold red]Log files for debugging:[/bold red]")
            for log_file in log_files:
                err_console.print(f"  • {log_file.absolute()}")

        # Find and display debug scripts
        if current_run_tmpdirs:
            debug_scripts = []
            for tmp_dir in current_run_tmpdirs:
                if tmp_dir.exists() and tmp_dir.is_dir():
                    debug_scripts.extend(tmp_dir.glob("debug_interactive_*.sh"))

            if debug_scripts:
                err_console.print(
                    "\n[bold yellow]🐛 Interactive debug scripts:[/bold yellow]"
                )
                for script in sorted(debug_scripts, key=lambda x: x.name):
                    err_console.print(f"  • bash {script}")

        raise typer.Exit(code=1)

    # Success! Find and list log files in cwl_output directory only
    log_files = list(cwl_output_dir.glob("**/*.log"))

    log_files_str = ""
    if log_files:
        log_files_str = "\n[cyan]Log Files:[/cyan]\n"
        for log_file in sorted(log_files):
            # Show full absolute path
            log_files_str += f"  • {log_file.absolute()}\n"
        log_files_str += "\n"

    # Build memory usage string with warnings if applicable
    memory_str = ""
    border_style = "green"
    if max_mem_kb > 0:
        max_mem_gb = max_mem_kb / (1024 * 1024)
        if max_mem_gb >= 5.0:
            memory_str = (
                f"\n[bold red]⚠️  Peak Memory Usage: {max_mem_gb:.1f} GB "
                "(CRITICAL - Exceeded 5 GB!)[/bold red]\n"
            )
            border_style = "bold red"
        elif max_mem_gb >= 3.5:
            memory_str = f"\n[yellow]⚠  Peak Memory Usage: {max_mem_gb:.1f} GB (Warning - Exceeded 3.5 GB)[/yellow]\n"
            border_style = "yellow"
        else:
            memory_str = f"\n[blue]Peak Memory Usage: {max_mem_gb:.1f} GB[/blue]\n"

    # Find debug scripts in temp directories
    debug_scripts_str = ""
    if current_run_tmpdirs:
        debug_scripts = []
        for tmp_dir in current_run_tmpdirs:
            if tmp_dir.exists() and tmp_dir.is_dir():
                debug_scripts.extend(tmp_dir.glob("debug_interactive_*.sh"))

        if debug_scripts:
            debug_scripts_str = (
                "\n[bold yellow]🐛 Interactive Debug Scripts:[/bold yellow]\n"
            )
            for script in sorted(debug_scripts, key=lambda x: x.name)[:5]:
                debug_scripts_str += f"  • bash {script}\n"
            if len(debug_scripts) > 5:
                debug_scripts_str += (
                    f"  [dim]... and {len(debug_scripts) - 5} more[/dim]\n"
                )

    console.print(
        Panel(
            f"[bold green]✓ CWL test completed successfully![/bold green]\n\n"
            f"[bold cyan]Output Directory:[/bold cyan] {output_dir}\n\n"
            f"[cyan]Generated Files:[/cyan]\n"
            f"  • DIRAC metadata: {metadata_path.name}\n"
            f"  • Production request: {prod_yaml_path.name}\n"
            f"  • CWL workflow: {cwl_workflow_path.name}\n"
            f"  • CWL inputs: {cwl_inputs_path.name if cwl_inputs_path else 'Not Provided'}\n"
            f"  • Job outputs: {cwl_output_dir.name}/\n"
            f"{log_files_str}"
            f"{memory_str}"
            f"{debug_scripts_str}"
            f"[dim]All files are in: {output_dir}[/dim]",
            title="✨ Test Complete",
            border_style=border_style,
        )
    )


# ============================================================================
# Helper Functions
# ============================================================================


async def _run_cwl_with_pretty_logs(cmd, env, output_dir, cwl_file):
    """
    Run CWL workflow with split-view real-time log display.

    Top: Status header
    Middle row split:
      - Left: CWL executor stdout
      - Right: Workflow graph visualization
    Bottom: Current step's log file

    Args:
        cmd: Command list to execute
        env: Environment dictionary to pass to subprocess
        output_dir: CWL output directory where workflow output files are written
        cwl_file: Path to the CWL workflow file for graph visualization

    Returns:
        tuple: (exit_code, list of temp directories used, max_memory_kb)
    """
    import asyncio
    import re
    from collections import deque

    import yaml
    from rich.layout import Layout
    from rich.live import Live
    from rich.panel import Panel
    from rich.text import Text

    # Parse CWL workflow to extract steps for graph visualization (including nested workflows)
    workflow_steps = []

    def parse_workflow_steps(workflow_data, parent_id=""):
        """Recursively parse workflow steps, including nested workflows"""
        steps = []
        if "steps" not in workflow_data:
            return steps

        for step in workflow_data["steps"]:
            step_id = step.get("id", "unknown")
            full_id = f"{parent_id}/{step_id}" if parent_id else step_id

            # Get the run section to check if it's a nested workflow
            run_section = step.get("run", {})

            if isinstance(run_section, dict):
                step_label = run_section.get("label", step_id)
                step_class = run_section.get("class", "unknown")

                # Add the current step
                steps.append(
                    {
                        "id": step_id,
                        "full_id": full_id,
                        "label": step_label,
                        "class": step_class,
                        "level": parent_id.count("/"),
                        "substeps": [],
                    }
                )

                # If it's a nested Workflow, recursively parse its steps
                if step_class == "Workflow":
                    substeps = parse_workflow_steps(run_section, full_id)
                    steps[-1]["substeps"] = substeps
            else:
                # External workflow reference
                steps.append(
                    {
                        "id": step_id,
                        "full_id": full_id,
                        "label": step_id,
                        "class": "external",
                        "level": parent_id.count("/"),
                        "substeps": [],
                    }
                )

        return steps

    try:
        with open(cwl_file, "r") as f:
            cwl_data = yaml.safe_load(f)
            workflow_steps = parse_workflow_steps(cwl_data)
    except Exception:
        # If we can't parse the workflow, just show a simple message
        workflow_steps = [
            {
                "id": "workflow",
                "full_id": "workflow",
                "label": "Workflow parsing failed",
                "class": "unknown",
                "level": 0,
                "substeps": [],
            }
        ]

    # Create layout with 4 sections: header, middle (stdout + graph), logfile
    layout = Layout()
    layout.split_column(
        Layout(name="header", size=3), Layout(name="middle"), Layout(name="logfile")
    )
    # Split middle section vertically (left/right)
    layout["middle"].split_row(Layout(name="stdout"), Layout(name="graph"))

    # Storage for output lines (using deque for efficient FIFO)
    stdout_lines = deque(maxlen=50)  # Keep last 50 lines
    logfile_lines = deque(maxlen=50)

    current_step_name = "Waiting for workflow to start..."
    current_log_file = None
    current_tmpdir = None
    current_output_prefix = None
    expecting_output_prefix = False  # Track if next line should be output-prefix value
    process_running = True

    # Spinner animation and output file tracking
    spinner_frames = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]
    spinner_index = 0
    output_files = {}  # Track output files: {path: size}
    step_outputs = (
        {}
    )  # Track which files belong to which step: {step_name: [(file_path, size), ...]}
    start_time = None
    temp_directories = []  # Track temp directories used in this run
    last_completed_step = None  # Track the last step that completed
    max_memory_usage = 0  # Track max RSS+Swap in KB
    memory_warning_shown = False  # Track if we've shown the warning
    memory_critical_shown = False  # Track if we've shown the critical warning

    # Caching for expensive operations to reduce flicker
    last_file_scan_time = 0
    file_scan_interval = 0.5  # Only scan for files every 0.5 seconds
    last_prmon_check_time = 0
    prmon_check_interval = 1.0  # Only check prmon every 1 second
    cached_prmon_data = (None, None)  # (rss, swap)

    def update_display():
        """Update the display with current stdout and log file content"""
        nonlocal spinner_index, output_files, start_time
        nonlocal step_outputs, last_completed_step, max_memory_usage
        nonlocal memory_warning_shown, memory_critical_shown
        nonlocal last_file_scan_time, file_scan_interval
        nonlocal last_prmon_check_time, prmon_check_interval, cached_prmon_data

        # Update spinner
        spinner = spinner_frames[spinner_index % len(spinner_frames)]
        spinner_index += 1

        # Calculate elapsed time
        if start_time:
            elapsed = asyncio.get_event_loop().time() - start_time
            elapsed_str = f"{int(elapsed // 60)}m {int(elapsed % 60)}s"
        else:
            elapsed_str = "0s"

        # Scan for output files - but only do this expensive operation periodically
        current_time = asyncio.get_event_loop().time()
        if current_time - last_file_scan_time >= file_scan_interval:
            last_file_scan_time = current_time

            try:
                new_output_files = {}
                current_step_files = []

                # Scan final output directory
                for ext in ["*.root", "*.dst", "*.xdigi", "*.xml", "*.json", "*.txt"]:
                    for f in output_dir.glob(f"**/{ext}"):
                        if f.is_file():
                            size = f.stat().st_size
                            rel_path = str(f.relative_to(output_dir))
                            new_output_files[rel_path] = size

                # ALSO scan the current step's tmpdir for output files
                if current_tmpdir and current_tmpdir.exists():
                    for ext in [
                        "*.root",
                        "*.dst",
                        "*.xdigi",
                        "*.xml",
                        "*.json",
                        "*.txt",
                    ]:
                        for f in current_tmpdir.glob(ext):  # No ** since tmpdir is flat
                            if f.is_file():
                                size = f.stat().st_size
                                file_key = f.name  # Use just the filename as key
                                new_output_files[file_key] = size

                                # Track new files for the current step
                                if (
                                    file_key not in output_files
                                    and current_step_name
                                    != "Waiting for workflow to start..."
                                ):
                                    current_step_files.append((file_key, size))

                # Add new files to the current step's output list
                if current_step_files:
                    clean_step_name = (
                        current_step_name.replace("[job ", "")
                        .replace("[step ", "")
                        .replace("[workflow ", "")
                        .replace("]", "")
                        .strip()
                    )
                    if clean_step_name not in step_outputs:
                        step_outputs[clean_step_name] = []
                    step_outputs[clean_step_name].extend(current_step_files)

                # Update sizes of existing files in step_outputs
                for _, files_list in step_outputs.items():
                    for i, (file_path, _) in enumerate(files_list):
                        if file_path in new_output_files:
                            # Update the size tuple in place
                            files_list[i] = (file_path, new_output_files[file_path])

                output_files = new_output_files
            except Exception:
                pass

        # Parse prmon file for RSS/swap stats - but only do this expensive operation periodically
        prmon_rss, prmon_swap = cached_prmon_data  # Start with cached values

        if current_time - last_prmon_check_time >= prmon_check_interval:
            last_prmon_check_time = current_time

            # Try current tmpdir first, then fall back to searching all temp directories
            tmpdirs_to_check = []
            if current_tmpdir and current_tmpdir.exists():
                tmpdirs_to_check.append(current_tmpdir)
            # Also check other temp directories
            for tmpdir in temp_directories:
                if tmpdir.exists() and tmpdir not in tmpdirs_to_check:
                    tmpdirs_to_check.append(tmpdir)

            for tmpdir in tmpdirs_to_check:
                prmon_files = list(tmpdir.glob("prmon*.txt"))
                if prmon_files:
                    try:
                        # Use the most recently modified prmon file
                        prmon_file = max(prmon_files, key=lambda p: p.stat().st_mtime)
                        with open(prmon_file, "r") as f:
                            lines = f.readlines()
                            if len(lines) >= 2:  # Need header + at least one data line
                                # Parse header (first line)
                                header = [
                                    h.lower() for h in lines[0].strip().split("\t")
                                ]

                                # Get the last complete data line (might be incomplete if still writing)
                                # Try the last line first, fall back to second-to-last if parsing fails
                                for line_idx in [-1, -2]:
                                    if (
                                        len(lines) >= abs(line_idx) + 1
                                    ):  # Make sure we have enough lines
                                        last_line = lines[line_idx].strip()
                                        if last_line:
                                            fields = last_line.split("\t")
                                            # Make sure we have enough fields
                                            if len(fields) >= len(header):
                                                try:
                                                    # prmon format: Time\tRSS\tVMEM\tPSS\tSwap\t...
                                                    if "rss" in header:
                                                        rss_idx = header.index("rss")
                                                        prmon_rss = int(
                                                            float(fields[rss_idx])
                                                        )  # In KB
                                                    if "swap" in header:
                                                        swap_idx = header.index("swap")
                                                        prmon_swap = int(
                                                            float(fields[swap_idx])
                                                        )  # In KB
                                                    # Successfully parsed, stop searching
                                                    break
                                                except (ValueError, IndexError):
                                                    # Try previous line
                                                    continue
                                # Found valid data in this tmpdir, stop searching other tmpdirs
                                if prmon_rss is not None:
                                    break
                    except Exception:
                        # Try next tmpdir
                        continue

            # Update cache
            cached_prmon_data = (prmon_rss, prmon_swap)

        # Track max memory usage and check thresholds
        if prmon_rss is not None:
            current_memory = prmon_rss + (prmon_swap if prmon_swap else 0)  # In KB
            if current_memory > max_memory_usage:
                max_memory_usage = current_memory

        # Build header with status
        status_text = Text()
        status_text.append(f"{spinner} ", style="bold green")
        status_text.append("Running CWL Workflow ", style="bold")
        status_text.append(f"[{elapsed_str}]", style="dim")

        if output_files:
            status_text.append(" • ", style="dim")
            status_text.append(f"{len(output_files)} output file(s)", style="cyan")
            total_size = sum(output_files.values())
            if total_size > 1024 * 1024:
                size_str = f"{total_size / (1024 * 1024):.1f} MB"
            elif total_size > 1024:
                size_str = f"{total_size / 1024:.1f} KB"
            else:
                size_str = f"{total_size} bytes"
            status_text.append(f" ({size_str})", style="dim cyan")

        # Add RSS/swap stats if available
        if prmon_rss is not None or prmon_swap is not None:
            status_text.append(" • ", style="dim")
            if prmon_rss is not None:
                if prmon_rss > 1024 * 1024:
                    rss_str = f"{prmon_rss / (1024 * 1024):.1f} GB"
                elif prmon_rss > 1024:
                    rss_str = f"{prmon_rss / 1024:.1f} MB"
                else:
                    rss_str = f"{prmon_rss} KB"
                status_text.append(f"RSS: {rss_str}", style="magenta")
            if prmon_swap is not None and prmon_swap > 0:
                if prmon_rss is not None:
                    status_text.append(" ", style="dim")
                if prmon_swap > 1024 * 1024:
                    swap_str = f"{prmon_swap / (1024 * 1024):.1f} GB"
                elif prmon_swap > 1024:
                    swap_str = f"{prmon_swap / 1024:.1f} MB"
                else:
                    swap_str = f"{prmon_swap} KB"
                status_text.append(f"Swap: {swap_str}", style="yellow")

        # Display max memory usage with warnings
        if max_memory_usage > 0:
            status_text.append(" • ", style="dim")
            max_memory_gb = max_memory_usage / (1024 * 1024)

            # Determine style and warning based on thresholds
            if max_memory_gb >= 5.0:
                mem_style = "bold red"
                warning_icon = " ⚠️ "
                if not memory_critical_shown:
                    memory_critical_shown = True
                    # Add to stdout so user sees it in logs too
                    stdout_lines.append(
                        f"⚠️  CRITICAL: Memory usage exceeded 5 GB! Max: {max_memory_gb:.1f} GB"
                    )
            elif max_memory_gb >= 3.5:
                mem_style = "yellow"
                warning_icon = " ⚠ "
                if not memory_warning_shown:
                    memory_warning_shown = True
                    stdout_lines.append(
                        f"⚠  WARNING: Memory usage exceeded 3.5 GB! Current: {max_memory_gb:.1f} GB"
                    )
            else:
                mem_style = "blue"
                warning_icon = ""

            status_text.append(
                f"Max: {max_memory_gb:.1f} GB{warning_icon}", style=mem_style
            )

        # Choose header border color based on memory warnings
        header_border_style = "green"
        if max_memory_usage > 0:
            max_memory_gb = max_memory_usage / (1024 * 1024)
            if max_memory_gb >= 5.0:
                header_border_style = "bold red"
            elif max_memory_gb >= 3.5:
                header_border_style = "yellow"

        header_panel = Panel(status_text, border_style=header_border_style)

        # Get terminal height and calculate available space for each panel
        # Each panel gets roughly half the terminal height minus borders/titles
        try:
            terminal_height = console.height
            # Reserve space for header (3 lines) + borders and titles (approximately 4 lines per panel)
            panel_height = max(10, (terminal_height - 11) // 2)
        except Exception:
            panel_height = 20  # Fallback if terminal height can't be determined

        # Build stdout panel - show last N lines that fit in the panel
        if stdout_lines:
            # Get terminal width for intelligent line wrapping
            try:
                terminal_width = console.width
                # Account for panel borders and padding (roughly 6 characters)
                max_line_width = max(40, (terminal_width // 2) - 6)
            except Exception:
                max_line_width = 80

            # Get the most recent lines that will fit in the panel
            lines_to_show = list(stdout_lines)[-panel_height:]

            # Truncate very long lines intelligently
            wrapped_lines = []
            for line in lines_to_show:
                if len(line) > max_line_width:
                    # Show beginning and end of long lines
                    truncated = line[: max_line_width - 3] + "..."
                    wrapped_lines.append(truncated)
                else:
                    wrapped_lines.append(line)

            stdout_text = Text.from_ansi("\n".join(wrapped_lines))
        else:
            stdout_text = Text("No output yet...", style="dim")

        stdout_panel = Panel(
            stdout_text,
            title="[bold cyan]CWL Executor Output[/bold cyan]",
            border_style="cyan",
        )

        # Build workflow graph panel using Rich Tree
        from rich.tree import Tree

        tree = Tree("📊 [bold yellow]Workflow Steps[/bold yellow]", guide_style="dim")

        # Use spinner for running indicator (cycles through different symbols)
        running_indicators = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]
        running_symbol = running_indicators[spinner_index % len(running_indicators)]

        def add_steps_to_tree(parent_tree, steps, current_name):
            """Recursively add steps to the tree, including substeps"""
            for step in steps:
                step_id = step["id"]
                step_label = step["label"]
                full_id = step["full_id"]

                # Check if this is the currently running step (check both id and full_id)
                # Clean up the current_name to remove any bracketed prefixes
                clean_name = (
                    current_name.replace("[job ", "")
                    .replace("[step ", "")
                    .replace("[workflow ", "")
                    .replace("]", "")
                    .strip()
                )

                is_current = (
                    step_id == clean_name
                    or step_id in clean_name
                    or clean_name in step_id
                    or clean_name.startswith(step_id)
                    or any(
                        substep["id"] == clean_name
                        or substep["id"] in clean_name
                        or clean_name in substep["id"]
                        for substep in step.get("substeps", [])
                    )
                )

                if is_current:
                    # Highlight current step with animated spinner
                    step_text = f"{running_symbol} [bold green]{step_label}[/bold green] [dim cyan](running)[/dim cyan]"
                else:
                    # Not running
                    step_text = f"  [dim]{step_label}[/dim]"

                # Add the step to the tree
                step_node = parent_tree.add(step_text)

                # Only add output files to actual CommandLineTool steps, not Workflow wrappers
                # Transformation nodes (class='Workflow') should not show files directly
                if step.get("class") != "Workflow":
                    # Add output files for this step if any
                    # Try multiple keys to find matching outputs
                    step_files = []

                    # Try direct step_id match
                    if step_id in step_outputs:
                        step_files.extend(step_outputs[step_id])

                    # Try full_id match
                    if full_id in step_outputs:
                        step_files.extend(step_outputs[full_id])

                    # Try matching any key that contains or is contained in step_id
                    for key in step_outputs.keys():
                        if (step_id in key or key in step_id) and key not in [
                            step_id,
                            full_id,
                        ]:
                            step_files.extend(step_outputs[key])

                    # Add unique files
                    seen_files = set()
                    for file_path, file_size in step_files:
                        if file_path not in seen_files:
                            seen_files.add(file_path)
                            if file_size > 1024 * 1024:
                                size_str = f"{file_size / (1024 * 1024):.1f} MB"
                            elif file_size > 1024:
                                size_str = f"{file_size / 1024:.1f} KB"
                            else:
                                size_str = f"{file_size} B"

                            file_name = Path(file_path).name
                            step_node.add(
                                f"[cyan]📄 {file_name}[/cyan] [dim]({size_str})[/dim]"
                            )

                # If this step has substeps (nested workflow), add them recursively
                if step.get("substeps"):
                    add_steps_to_tree(step_node, step["substeps"], current_name)

        add_steps_to_tree(tree, workflow_steps, current_step_name)

        # Add debug info at the bottom showing current tracking state
        clean_current = (
            current_step_name.replace("[job ", "")
            .replace("[step ", "")
            .replace("[workflow ", "")
            .replace("]", "")
            .strip()
        )
        total_files = sum(len(files) for files in step_outputs.values())
        debug_info = (
            f"[dim]Tracking: {clean_current[:40]} • {len(step_outputs)} steps • {total_files} files[/dim]"
            if clean_current
            else "[dim]No step active[/dim]"
        )

        graph_panel = Panel(
            tree,
            title="[bold yellow]Workflow Progress[/bold yellow]",
            subtitle=debug_info,
            border_style="yellow",
        )

        # Build log file panel
        if current_log_file and current_log_file.exists():
            if logfile_lines:
                # Get terminal width for the bottom panel (full width)
                try:
                    terminal_width = console.width
                    # Bottom panel spans full width, account for borders
                    max_log_line_width = max(60, terminal_width - 6)
                except Exception:
                    max_log_line_width = 120

                # Get the most recent lines that will fit in the panel
                lines_to_show = list(logfile_lines)[-panel_height:]

                # Truncate long lines in log file - keep the END of lines (more important for logs)
                truncated_log_lines = []
                for line in lines_to_show:
                    if len(line) > max_log_line_width:
                        # For log lines, show the end which usually has the important info
                        truncated_log_lines.append(
                            "..." + line[-(max_log_line_width - 3) :]
                        )
                    else:
                        truncated_log_lines.append(line)

                logfile_text = Text.from_ansi("\n".join(truncated_log_lines))
            else:
                # Log file exists but no content yet - show waiting message with file info
                logfile_text = Text(
                    f"Log file found, waiting for content...\n{current_log_file}",
                    style="dim",
                )
        else:
            # Show debug info about what we're waiting for
            debug_parts = ["Waiting for step log file...\n"]
            debug_parts.append(f"Step: {current_step_name}\n")
            if current_tmpdir:
                debug_parts.append(f"TmpDir: {current_tmpdir}\n")
            if current_output_prefix:
                debug_parts.append(f"Prefix: {current_output_prefix}\n")
                if current_tmpdir:
                    debug_parts.append(
                        f"Looking for: {current_tmpdir}/*_{current_output_prefix}.log\n"
                    )
            logfile_text = Text("".join(debug_parts), style="dim")

        logfile_panel = Panel(
            logfile_text,
            title=f"[bold magenta]Step Log: {current_step_name}[/bold magenta]",
            border_style="magenta",
        )

        layout["header"].update(header_panel)
        layout["stdout"].update(stdout_panel)
        layout["graph"].update(graph_panel)
        layout["logfile"].update(logfile_panel)

        return layout

    def find_current_log_file(line):
        """Extract tmpdir and output_prefix from CWL executor output to locate log file"""
        nonlocal current_log_file, current_step_name, current_tmpdir
        nonlocal current_output_prefix, expecting_output_prefix, temp_directories

        # Pattern 1a: Look for workflow/step starting messages
        # Format: "[workflow StepName] starting step" or "[step StepName] start"
        workflow_step_match = re.search(
            r"\[(workflow|step)\s+([^\]]+)\]\s+(starting step|start)", line
        )
        if workflow_step_match:
            step = workflow_step_match.group(2)
            current_step_name = step

        # Pattern 1b: Look for cwltool job execution messages
        # Format: "[job StepName] /tmp/user/xyz$ command"
        job_match = re.search(r"\[job\s+([^\]]+)\]\s+(/tmp/[^\s$]+)\$", line)
        if job_match:
            step = job_match.group(1)
            tmpdir = Path(job_match.group(2))

            # Update tracking variables
            current_step_name = step
            current_tmpdir = tmpdir
            # Track this temp directory for later log file discovery
            if tmpdir not in temp_directories:
                temp_directories.append(tmpdir)

        # Pattern 2: Look for --output-prefix argument
        # The command output spans multiple lines with backslash continuation
        if "--output-prefix" in line:
            # Next non-empty, non-backslash line will have the prefix value
            expecting_output_prefix = True
        elif expecting_output_prefix:
            # This line should contain the actual output prefix value
            stripped = line.strip()
            # Skip lines that are just backslashes or empty
            if stripped and stripped != "\\":
                current_output_prefix = stripped
                expecting_output_prefix = False

        # Pattern 3: If we have both tmpdir and output_prefix, try to find the log file
        if current_tmpdir and current_output_prefix and current_tmpdir.exists():
            # The log file pattern is: {tmpdir}/*_{output_prefix}.log
            # Example: /tmp/roneil/4qrpajhu/DaVinci_00012345_00006789_1.log
            log_pattern = f"*_{current_output_prefix}.log"
            log_files = list(current_tmpdir.glob(log_pattern))

            if log_files:
                # Use the first (should be only one) matching log file
                if log_files[0] != current_log_file:
                    current_log_file = log_files[0]
                    # Don't overwrite current_step_name with the log file name!
                    logfile_lines.clear()
                    return True

        return False

    async def tail_log_file():
        """Continuously tail the current log file"""
        nonlocal current_log_file, current_step_name, current_tmpdir, current_output_prefix
        last_position = 0
        last_check_time = 0

        while process_running:
            current_time = asyncio.get_event_loop().time()

            # Periodically check if log file appeared (every 0.5 seconds)
            if current_time - last_check_time > 0.5:
                last_check_time = current_time

                # If we have tmpdir and output_prefix but no log file yet, try to find it
                if current_tmpdir and current_output_prefix and not current_log_file:
                    if current_tmpdir.exists():
                        log_pattern = f"*_{current_output_prefix}.log"
                        log_files = list(current_tmpdir.glob(log_pattern))
                        if log_files:
                            current_log_file = log_files[0]
                            # Don't overwrite current_step_name here either!
                            logfile_lines.clear()
                            last_position = 0

            # Tail the current log file
            if current_log_file and current_log_file.exists():
                try:
                    with open(
                        current_log_file, "r", encoding="utf-8", errors="replace"
                    ) as f:
                        # Seek to last read position
                        f.seek(last_position)
                        new_lines = f.readlines()

                        if new_lines:
                            for line in new_lines:
                                logfile_lines.append(line.rstrip())
                            last_position = f.tell()
                except Exception:
                    # File might be locked or rotated, skip this iteration
                    pass

            await asyncio.sleep(
                0.25
            )  # Poll every 0.25 seconds for responsive log updates

    async def read_stdout(stream):
        """Continuously read stdout from the subprocess"""
        nonlocal process_running
        try:
            while True:
                line = await stream.readline()
                if not line:
                    break
                line = line.decode("utf-8", errors="replace").rstrip()
                stdout_lines.append(line)
                # Try to find log file references in the output
                find_current_log_file(line)
        except Exception:
            pass
        finally:
            process_running = False

    async def update_display_loop(live):
        """Continuously update the display by calling live.update()"""
        while process_running:
            live.update(update_display())
            await asyncio.sleep(0.1)  # Update 10 times per second

    # Start the subprocess in its own process group for proper cleanup
    import os
    import signal

    # Use start_new_session instead of preexec_fn for better asyncio compatibility
    process = await asyncio.create_subprocess_exec(
        *cmd,
        stdout=asyncio.subprocess.PIPE,
        stderr=asyncio.subprocess.STDOUT,  # Merge stderr into stdout
        env=env,
        start_new_session=True,  # Create new process session/group
    )

    # Set start time
    start_time = asyncio.get_event_loop().time()

    async def cleanup_process():
        """Gracefully terminate the process and its children"""
        nonlocal process_running
        process_running = False

        if process.returncode is None:  # Process still running
            try:
                # Send SIGTERM to entire process group
                # Since we used start_new_session=True, the pid is also the pgid
                try:
                    os.killpg(process.pid, signal.SIGTERM)
                except ProcessLookupError:
                    # Process already terminated
                    pass
                except PermissionError:
                    # Fallback to killing just the main process
                    process.terminate()

                # Give processes 3 seconds to terminate gracefully
                try:
                    await asyncio.wait_for(process.wait(), timeout=3.0)
                except asyncio.TimeoutError:
                    # Force kill the entire process group
                    try:
                        os.killpg(process.pid, signal.SIGKILL)
                    except (ProcessLookupError, PermissionError):
                        pass
                    # Also try killing the main process
                    try:
                        process.kill()
                    except ProcessLookupError:
                        pass
            except Exception:
                # Fallback to simple terminate
                try:
                    process.terminate()
                    await asyncio.wait_for(process.wait(), timeout=1.0)
                except asyncio.TimeoutError:
                    try:
                        process.kill()
                    except ProcessLookupError:
                        pass

    # Display with Live - auto-refresh at 10 Hz for very smooth updates
    # Expensive operations (file scanning, prmon parsing) are cached to prevent flicker
    with Live(
        update_display(),
        refresh_per_second=10,  # Fast refresh rate - expensive operations are cached
        console=console,
        screen=False,  # Don't use alternate screen
        vertical_overflow="visible",  # Allow content to push down
    ) as live:
        try:
            # Run all tasks concurrently
            await asyncio.gather(
                tail_log_file(),
                read_stdout(process.stdout),
                update_display_loop(live),
            )

            # Wait for process to complete
            returncode = await process.wait()

            # Final update to show completion
            live.update(update_display())

        except KeyboardInterrupt:
            console.print(
                "\n[yellow]⚠️  Received interrupt signal, cleaning up...[/yellow]"
            )
            await cleanup_process()
            console.print("[yellow]✓ Cleanup complete[/yellow]")
            raise
        except Exception:
            process_running = False
            await cleanup_process()
            raise

    return returncode, temp_directories, max_memory_usage


def _run_git_exp(*args: str, printing: bool = True) -> subprocess.CompletedProcess:
    """Run a git command with rich output"""
    cmd = ("git",) + args
    if printing:
        console.print(f"[dim]$ {shlex.join(cmd)}[/dim]")

    proc = logging_subprocess_run(cmd, printing=printing)
    if proc.returncode != 0:
        raise typer.Exit(code=1)
    return proc


def _checkout_exp(
    working_group: str,
    production_name: str,
    version: str,
    branch_name: str,
    *,
    allow_deletes: bool,
):
    """Enhanced checkout with rich progress indicators"""
    inside_ap_datapkg()
    repo_dir = Path.cwd()

    # Verify version
    available_versions = production_to_versions(working_group, production_name)
    if version not in available_versions:
        raise typer.Exit(
            f"Version {version} does not correspond to {working_group}/{production_name}"
        )

    # Check git status
    proc = _run_git_exp("status", "--porcelain", printing=False)
    if proc.stdout.strip():
        err_console.print(
            "[bold red]✗ Git repository is not clean![/bold red]\n"
            "Please commit or stash your changes first."
        )
        raise typer.Exit(code=1)

    with Progress(
        SpinnerColumn(),
        TextColumn("[progress.description]{task.description}"),
        console=console,
    ) as progress:
        task = progress.add_task(description="Fetching tags...", total=None)
        _run_git_exp("fetch", "origin", "--tags", printing=False)

        progress.update(task, description="Creating branch...")
        _run_git_exp("checkout", "-b", branch_name, "origin/master", printing=False)

        # Handle existing production directory
        for child in repo_dir.iterdir():
            if child.name.lower() != production_name.lower():
                continue

            if allow_deletes:
                progress.update(task, description=f"Removing existing {child.name}...")
                shutil.rmtree(child)
            else:
                err_console.print(
                    f"[bold red]✗ {child.name} already exists![/bold red]\n"
                    f"Use --allow-deletes to remove it"
                )
                raise typer.Exit(code=1)

        # Find production in version tag
        progress.update(task, description="Finding production in version tag...")
        proc = _run_git_exp("ls-tree", "-d", version, printing=False)

        git_prod_name = None
        for line in proc.stdout.strip().splitlines():
            _, name = line.decode().split("\t")
            if name.lower() == production_name.lower():
                git_prod_name = name
                break

        if not git_prod_name:
            err_console.print(
                f"[bold red]✗ Failed to find {production_name} in {version}[/bold red]"
            )
            raise typer.Exit(code=1)

        console.print(
            f"[green]✓ Found match:[/green] {production_name} → {git_prod_name} in {version}"
        )

        # Checkout production
        progress.update(task, description=f"Checking out {version}...")
        _run_git_exp("checkout", version, "--", git_prod_name, printing=False)
        _run_git_exp(
            "commit", "-m", f"Restore {git_prod_name} from {version}", printing=False
        )


if __name__ == "__main__":
    app()
