"""
iq worker commands - Manage workers (tool runtime).

A "worker" is the runtime process that executes tools. Each tool runs as a
child process with configurable threads. The worker connects to sites and
listens on tool queues for incoming tasks.
"""

from __future__ import annotations

import asyncio
import sys
from pathlib import Path

import click
from rich.table import Table

from iq_sdk.cli import console
from iq_sdk.config import load_settings


@click.group(context_settings={"help_option_names": ["-h", "--help"]})
def worker() -> None:
    """Manage workers (tool runtime)."""
    pass


@worker.command("ls")
@click.option("--site", "-s", "site_id", help="Filter by site ID")
@click.option("--config", "-c", "config_file", help="Config file path")
@click.option("--remote", "-r", is_flag=True, help="Check remote tool availability")
@click.pass_context
def worker_list(
    ctx: click.Context,
    site_id: str | None,
    config_file: str | None,
    remote: bool,
) -> None:
    """
    List configured workers/tools and running worker instances.

    Shows all tools configured across sites and their running status.
    Use --remote to verify tool availability on remote hosts.
    """
    from iq_sdk.cli import require_project
    from iq_sdk.worker_registry import WorkerRegistry

    project_dir = require_project(ctx)

    if config_file is None:
        config_file = ctx.obj.get("config_file")

    settings = load_settings(config_file)

    # Filter sites
    if site_id:
        sites = settings.get_sites([site_id])
        if not sites:
            console.print(f"[red]Site '{site_id}' not found[/red]")
            sys.exit(1)
    else:
        sites = settings.get_sites()

    # Load worker registry
    registry_dir = project_dir / ".iq-workers"
    registry = WorkerRegistry(registry_dir)

    # Check if monitors are active - if so, don't cleanup stale entries
    monitors_active = registry.has_active_monitors()
    cleanup_stale = not monitors_active

    # Get running workers (cleanup only if no monitors)
    running_entries = registry.get_all_entries(cleanup_stale=cleanup_stale)
    running_by_tool: dict[str, list] = {}
    for entry in running_entries:
        key = f"{entry.site_id}:{entry.tool_name}"
        if key not in running_by_tool:
            running_by_tool[key] = []
        running_by_tool[key].append(entry)

    # Gather check results if needed
    check_results = {}
    if remote:
        check_results = _check_tools_on_sites(sites, settings.project_name)

    worker_id = settings.get_worker_id()

    console.print(f"[bold]Project: {settings.project_name}[/bold]")
    console.print(f"Worker ID: {worker_id}")
    if monitors_active:
        console.print("[green]● Monitor active[/green]")
    console.print()

    for site in sites:
        is_default = " (default)" if site.id == settings.site.id else ""
        console.print(f"[cyan]Site: {site.id}{is_default}[/cyan]")
        console.print(f"Host: {site.get_host()}\n")

        # Build table
        table = Table()
        table.add_column("Tool", style="white")
        table.add_column("Status")
        table.add_column("Threads")
        table.add_column("PID")

        if remote:
            table.add_column("On Host")

        # Discover tools for this site (use absolute path)
        tools_path = project_dir / settings.tools_dir
        discovered = site.discover_tools(tools_path)

        if not discovered:
            console.print("[dim]  No tools available[/dim]\n")
            continue

        # Track which tools have running instances
        shown_running = set()

        for tool_cfg, local_info in discovered:
            full_name = local_info.get_full_name(settings.project_name)
            key = f"{site.id}:{full_name}"
            running_instances = running_by_tool.get(key, [])

            # Filter to only alive instances
            alive_instances = [e for e in running_instances if e.is_alive()]

            if alive_instances:
                # Show each running instance
                for i, entry in enumerate(alive_instances):
                    tool_label = full_name if i == 0 else ""
                    row = [
                        tool_label,
                        "[green]● running[/green]",
                        str(entry.threads),
                        str(entry.pid),
                    ]
                    if remote:
                        result = check_results.get((site.id, tool_cfg.name), {})
                        on_host = "[green]✓[/green]" if result.get("exists") else "[red]✗[/red]"
                        row.append(on_host)
                    table.add_row(*row)
                shown_running.add(key)
            else:
                # Show as not running
                threads = tool_cfg.threads or site.threads or settings.worker.threads
                row = [
                    full_name,
                    "[dim]-[/dim]",
                    str(threads),
                    "[dim]-[/dim]",
                ]
                if remote:
                    result = check_results.get((site.id, tool_cfg.name), {})
                    on_host = "[green]✓[/green]" if result.get("exists") else "[red]✗[/red]"
                    row.append(on_host)
                table.add_row(*row)

        console.print(table)
        console.print()


def _check_tools_on_sites(sites: list, project_name: str) -> dict:
    """Check tool availability on sites."""
    from iq_sdk.client import IQApiClient, RemoteTool

    results = {}

    async def _check():
        for site in sites:
            try:
                async with IQApiClient.from_site(site) as client:
                    remote_tools = await RemoteTool.find_all_by_project(client, project_name)
                    remote_by_name = {t.tool_name: t for t in remote_tools}

                    for tool in site.get_tools():
                        remote = remote_by_name.get(tool.name)
                        results[(site.id, tool.name)] = {
                            "exists": remote is not None,
                            "digest_match": remote.digest if remote else None,
                        }
            except Exception as e:
                for tool in site.get_tools():
                    results[(site.id, tool.name)] = {
                        "exists": False,
                        "digest_match": None,
                        "error": str(e)[:30],
                    }

    try:
        asyncio.run(_check())
    except Exception:
        pass

    return results


@worker.command("test")
@click.option("--site", "-s", "site_id", help="Site to test (default: first configured)")
@click.option("--tool", "-t", "tool_name", help="Tool to test with (default: first available)")
@click.option("--config", "-c", "config_file", help="Config file path")
@click.pass_context
def worker_test(
    ctx: click.Context,
    site_id: str | None,
    tool_name: str | None,
    config_file: str | None,
) -> None:
    """
    Test worker connection to a site.

    Connects via Socket.IO and verifies authentication.
    Useful for debugging connection issues.
    """
    from iq_sdk.cli.tool import get_local_tools
    from iq_sdk.client.connection import test_worker_connection

    if config_file is None:
        config_file = ctx.obj.get("config_file") if ctx.obj else None

    settings = load_settings(config_file)
    project_name = settings.project_name
    worker_id = settings.get_worker_id()

    # Get target site
    if site_id:
        site = settings.get_site(site_id)
        if not site:
            console.print(f"[red]Site '{site_id}' not found[/red]")
            sys.exit(1)
    else:
        site = settings.site

    console.print("\n[bold cyan]Testing worker connection[/bold cyan]")
    console.print(f"Project: {project_name}")
    console.print(f"Site: {site.id}")
    console.print(f"Host: {site.get_host()}")
    console.print(f"Worker ID: {worker_id}\n")

    # Get tool to test with
    if tool_name:
        full_tool_name = tool_name
        # If not fully qualified, add project prefix
        if "." not in tool_name:
            full_tool_name = f"{project_name}.{tool_name}"
    else:
        # Get first local tool
        tools_path = Path("tools")
        local_tools = get_local_tools(tools_path) if tools_path.exists() else {}

        if not local_tools:
            console.print("[yellow]No local tools found. Use --tool to specify one.[/yellow]")
            sys.exit(1)

        # Get first tool
        first_base_name = next(iter(local_tools))
        first_info = local_tools[first_base_name][0]
        full_tool_name = f"{project_name}.{first_base_name}.{first_info.version}"

    console.print(f"Test tool: {full_tool_name}\n")

    # Run the test
    success = asyncio.run(
        test_worker_connection(
            site=site,
            project_name=project_name,
            tool_name=full_tool_name,
            worker_id=worker_id,
            debug=True,
        )
    )

    if success:
        console.print("\n[bold green]✓ Worker connection test passed[/bold green]")
    else:
        console.print("\n[bold red]✗ Worker connection test failed[/bold red]")
        sys.exit(1)


@worker.command("run")
@click.option(
    "--site", "-s", "site_ids", multiple=True, help="Sites to connect to (can be repeated)"
)
@click.option(
    "--tool",
    "-t",
    "tool_names",
    multiple=True,
    help="Specific tools to run (can be repeated, substring match)",
)
@click.option("--threads", type=int, help="Override threads per tool")
@click.option("--max-threads", type=int, help="Maximum threads for autoscaling")
@click.option("--autoscale", is_flag=True, help="Enable autoscaling")
@click.option("--detach", "-d", is_flag=True, help="Run in background")
@click.option(
    "--verbose", "-v", is_flag=True, help="Verbose output (show API calls, WebSocket events)"
)
@click.option("--config", "-c", "config_file", help="Config file path")
@click.option("--worker-id", "worker_id_override", help="Override worker ID")
@click.pass_context
def worker_run(
    ctx: click.Context,
    site_ids: tuple,
    tool_names: tuple,
    threads: int | None,
    max_threads: int | None,
    autoscale: bool,
    detach: bool,
    verbose: bool,
    config_file: str | None,
    worker_id_override: str | None,
) -> None:
    """
    Run worker process.

    Connects to configured sites and starts tool processes.
    Each tool runs as a child process listening on its queue.

    Use --detach to run in background (useful for production).
    """
    import os
    import shutil
    import subprocess

    from iq_sdk.cli import require_project

    project_dir = require_project(ctx)

    if config_file is None:
        config_file = ctx.obj.get("config_file")

    # Handle detached mode
    if detach:
        iq_cmd = shutil.which("iq")
        if not iq_cmd:
            python_dir = os.path.dirname(sys.executable)
            iq_cmd = os.path.join(python_dir, "iq")
            if not os.path.exists(iq_cmd):
                console.print("[red]Could not find 'iq' command[/red]")
                sys.exit(1)

        cmd = [iq_cmd, "worker", "run"]
        for sid in site_ids:
            cmd.extend(["--site", sid])
        for tn in tool_names:
            cmd.extend(["--tool", tn])
        if threads:
            cmd.extend(["--threads", str(threads)])
        if max_threads:
            cmd.extend(["--max-threads", str(max_threads)])
        if autoscale:
            cmd.append("--autoscale")
        if verbose:
            cmd.append("--verbose")
        if config_file:
            cmd.extend(["--config", config_file])
        if worker_id_override:
            cmd.extend(["--worker-id", worker_id_override])

        p = subprocess.Popen(
            cmd,
            stdout=subprocess.DEVNULL,
            stderr=subprocess.DEVNULL,
            stdin=subprocess.DEVNULL,
            start_new_session=True,
            cwd=str(project_dir),
        )
        console.print(f"[green]Worker started in background (PID: {p.pid})[/green]")
        return

    # Build overrides from CLI options
    overrides: dict = {}

    if threads:
        overrides.setdefault("worker", {})["threads"] = threads
    if max_threads:
        overrides.setdefault("worker", {})["max_threads"] = max_threads
    if autoscale:
        overrides.setdefault("worker", {})["autoscale"] = True
    if worker_id_override:
        overrides["worker_id"] = worker_id_override

    # Load settings
    settings = load_settings(config_file, overrides)

    # Get target sites
    if site_ids:
        target_sites = settings.get_sites(list(site_ids))
    else:
        target_sites = settings.get_sites()

    if not target_sites:
        console.print("[red]No sites configured[/red]")
        sys.exit(1)

    worker_id = settings.get_worker_id()

    if verbose:
        console.print(f"[bold blue]Starting worker '{worker_id}'[/bold blue]")

    # Discover tools for each site
    # Use absolute path relative to project root
    tools_path = project_dir / settings.tools_dir
    tools_to_run: list[tuple] = []

    for site in target_sites:
        discovered = site.discover_tools(tools_path)

        # Apply CLI tool filter if provided (substring matching)
        if tool_names:
            discovered = [
                (cfg, info)
                for cfg, info in discovered
                if any(tn in cfg.name or tn in info.file_name for tn in tool_names)
            ]

        for tool_config, local_info in discovered:
            tools_to_run.append((site, tool_config, local_info))

    if not tools_to_run:
        console.print("[yellow]No tools to run[/yellow]")
        if tool_names:
            console.print(f"[dim]Requested tools: {', '.join(tool_names)}[/dim]")
        console.print("[dim]Check tools/ directory and site config[/dim]")
        sys.exit(1)

    if verbose:
        console.print(f"  Worker ID: {worker_id}")
        console.print(f"  Sites: {len(target_sites)}")
        console.print(f"  Tools: {len(tools_to_run)}")

        for site, tool_settings, _local_info in tools_to_run:
            # Priority: CLI --threads > tool config > site config > global worker config
            tool_threads = (
                threads or tool_settings.threads or site.threads or settings.worker.threads
            )
            console.print(f"    → {site.id}/{tool_settings.name} (threads:{tool_threads})")

    if settings.worker.autoscale:
        console.print(f"  Autoscale: {settings.worker.min_threads}-{settings.worker.max_threads}")

    # Load tool classes via registry
    from iq_sdk.cli.tool import load_tool_class

    loaded_tools = []

    for site, tool_settings, local_info in tools_to_run:
        tool_class = load_tool_class(local_info)
        if tool_class:
            loaded_tools.append((site, tool_settings, local_info, tool_class))
        else:
            console.print(
                f"[yellow]Warning: Failed to load tool '{tool_settings.name}' "
                f"from {local_info.path}[/yellow]"
            )

    if not loaded_tools:
        console.print("[red]No tools could be loaded[/red]")
        sys.exit(1)

    if verbose:
        console.print(f"\n[dim]Loaded {len(loaded_tools)} tools[/dim]")

    # Run worker
    try:
        asyncio.run(_run_worker(settings, loaded_tools, cli_threads=threads, verbose=verbose))
    except KeyboardInterrupt:
        console.print("\n[yellow]Stopped[/yellow]")


async def _run_worker(
    settings,
    loaded_tools: list,
    cli_threads: int | None = None,
    verbose: bool = False,
) -> None:
    """Run the worker process using SiteConnection architecture."""
    from iq_sdk.client.site_connection import SiteConnection
    from iq_sdk.client.tool_worker import ToolConfig
    from iq_sdk.worker_registry import WorkerRegistry

    worker_id = settings.get_worker_id()
    project_name = settings.project_name
    project_path = str(settings.get_project_root())
    retry_config = settings.worker.retry

    # Initialize registry
    registry = WorkerRegistry()

    # Group tools by site and register them
    tools_by_site: dict = {}
    registered_entries = []

    for site, tool_settings, local_info, tool_class in loaded_tools:
        if site.id not in tools_by_site:
            tools_by_site[site.id] = {
                "site": site,
                "tools": [],
            }

        # Create ToolConfig with threads from settings cascade
        # Priority: CLI --threads > tool config > site config > global worker config
        threads = cli_threads or tool_settings.threads or site.threads or settings.worker.threads
        max_runtime = getattr(tool_settings, "max_runtime_seconds", 600)

        # Build full tool name: project.tool_name.version (e.g. "demo.my_tool.1.0")
        full_tool_name = local_info.get_full_name(project_name)

        if verbose:
            console.print(
                f"[dim]  Tool: {full_tool_name} (from {local_info.name}.{local_info.version})[/dim]"
            )

        # Register worker in registry
        entry = registry.register(
            worker_id=worker_id,
            site_id=site.id,
            tool_name=full_tool_name,
            threads=threads,
            host=site.get_host(),
        )
        registered_entries.append((site.id, full_tool_name, entry.pid))

        tools_by_site[site.id]["tools"].append(
            ToolConfig(
                name=full_tool_name,
                tool_class=tool_class,
                thread_count=threads,
                max_runtime_seconds=max_runtime,
            )
        )

    # Create and start SiteConnections
    site_connections: list[SiteConnection] = []

    for _site_id, site_data in tools_by_site.items():
        site = site_data["site"]
        tools = site_data["tools"]

        if verbose:
            console.print(f"[dim]Connecting to {site.id}...[/dim]")

        site_conn = SiteConnection(
            site=site,
            project_name=project_name,
            worker_id=worker_id,
            tools=tools,
            registry=registry,
            project_path=project_path,
            retry_config=retry_config,
            debug=verbose,
            verbose=verbose,
        )

        success = await site_conn.start()
        if not success:
            console.print(f"[red]✗ {site.id} connection failed[/red]")
            # Stop all already started connections
            for conn in site_connections:
                await conn.stop()
            return

        tool_names = [t.name for t in tools]
        console.print(f"[green]✓ Connected to {site.get_host()}[/green]")
        console.print(f"  Tools: {', '.join(tool_names)}")
        site_connections.append(site_conn)

        # Initial poll after connection status is displayed
        await site_conn.initial_poll()

    if not site_connections:
        console.print("[red]No sites available[/red]")
        return

    # All sites connected - run the main loops
    console.print("\n[bold]Running... (Ctrl+C to stop)[/bold]")

    try:
        await asyncio.gather(*[conn.run() for conn in site_connections])
    except asyncio.CancelledError:
        pass  # Graceful shutdown on Ctrl+C
    finally:
        console.print("\n[dim]Stopping all connections...[/dim]")
        for conn in site_connections:
            await conn.stop()

        # Unregister from worker registry
        for site_id, tool_name, pid in registered_entries:
            registry.unregister(site_id, tool_name, pid)

        console.print("[yellow]Worker stopped[/yellow]")


@worker.command("monitor")
@click.option("--detach", "-d", is_flag=True, help="Run in background (dual-monitor pattern)")
@click.option("--interval", default=3, help="Check interval in seconds")
@click.option("--config", "-c", "config_file", help="Config file path")
@click.option("--_monitor-index", "_monitor_index", type=int, hidden=True)
@click.option("--_partner-pid", "_partner_pid", type=int, default=0, hidden=True)
@click.pass_context
def worker_monitor(
    ctx: click.Context,
    detach: bool,
    interval: int,
    config_file: str | None,
    _monitor_index: int | None,
    _partner_pid: int,
) -> None:
    """
    Monitor and auto-restart workers.

    Watches .iq-workers/ registry files and restarts dead workers.
    The monitored "desired state" is defined by the registry files.

    Use --detach for production: starts two monitors that watch each other
    (dual-monitor pattern for self-healing).
    """
    import os
    import shutil
    import signal
    import subprocess
    import time

    from iq_sdk.cli import require_project
    from iq_sdk.worker_registry import MonitorRegistry, WorkerRegistry

    # Validate project context
    project_dir = require_project(ctx)

    if config_file is None:
        config_file = ctx.obj.get("config_file") if ctx.obj else None

    registry_dir = project_dir / ".iq-workers"
    registry_dir.mkdir(parents=True, exist_ok=True)

    mon_reg = MonitorRegistry(registry_dir)

    # Check if monitors are already running
    alive_monitors = mon_reg.get_alive_monitors()
    if alive_monitors and _monitor_index is None:
        console.print("[yellow]Monitor processes already running![/yellow]")
        for m in alive_monitors:
            console.print(f"  PID: {m.pid}")
        console.print("[dim]Use 'iq worker kill --monitors' to stop them first[/dim]")
        sys.exit(1)

    # Find iq executable
    iq_cmd = shutil.which("iq")
    if not iq_cmd:
        python_dir = os.path.dirname(sys.executable)
        iq_cmd = os.path.join(python_dir, "iq")
        if not os.path.exists(iq_cmd):
            console.print("[red]Could not find 'iq' command[/red]")
            sys.exit(1)

    cwd = str(project_dir)

    if detach:
        # Start dual monitors in background
        cmd_base = [iq_cmd, "worker", "monitor"]
        if config_file:
            cmd_base.extend(["--config", config_file])
        if interval != 3:
            cmd_base.extend(["--interval", str(interval)])

        # Start first monitor
        p1 = subprocess.Popen(
            cmd_base + ["--_monitor-index", "0"],
            stdout=subprocess.DEVNULL,
            stderr=subprocess.DEVNULL,
            stdin=subprocess.DEVNULL,
            start_new_session=True,
            cwd=cwd,
        )

        # Start second monitor
        p2 = subprocess.Popen(
            cmd_base + ["--_monitor-index", "1", "--_partner-pid", str(p1.pid)],
            stdout=subprocess.DEVNULL,
            stderr=subprocess.DEVNULL,
            stdin=subprocess.DEVNULL,
            start_new_session=True,
            cwd=cwd,
        )

        # Update first monitor with second's PID
        time.sleep(0.5)  # Give monitors time to register
        mon_reg.update_partner(0, p2.pid)

        console.print(f"[green]Monitors started (PIDs: {p1.pid}, {p2.pid})[/green]")
        console.print("[dim]Use 'iq worker kill --monitors' to stop them[/dim]")
        return

    # Check for internal monitor mode (used by dual-monitor pattern)
    if _monitor_index is not None:
        _run_monitor_loop(
            registry_dir,
            _monitor_index,
            _partner_pid,
            interval,
            config_file,
        )
        return

    # Foreground monitor mode - register as monitor 0
    console.print("[bold]Starting worker monitor...[/bold]")
    console.print(f"Project: {project_dir.name}")
    console.print(f"Watching: {registry_dir}")
    console.print(f"Interval: {interval}s")
    console.print("[dim]Press Ctrl+C to stop[/dim]\n")

    # Register this foreground monitor
    mon_reg.register(0, 0)

    def cleanup_foreground(signum, frame):
        mon_reg.unregister(0)
        console.print("\n[yellow]Monitor stopped[/yellow]")
        sys.exit(0)

    signal.signal(signal.SIGTERM, cleanup_foreground)
    signal.signal(signal.SIGINT, cleanup_foreground)

    worker_reg = WorkerRegistry(registry_dir)

    try:
        while True:
            dead_entries = worker_reg.get_dead_entries()

            if dead_entries:
                console.print(f"[yellow]Found {len(dead_entries)} dead worker(s)[/yellow]")

            for path, entry in dead_entries:
                console.print(f"  Restarting {entry.tool_name}...")

                # Use atomic rename as lock
                lock_path = path.with_suffix(".restarting")
                try:
                    path.rename(lock_path)
                    lock_path.unlink()
                except (OSError, FileNotFoundError):
                    console.print("[dim]  (already being restarted)[/dim]")
                    continue

                # Restart worker
                cmd = [
                    iq_cmd,
                    "worker",
                    "run",
                    "--site",
                    entry.site_id,
                    "--tool",
                    entry.get_filter_name(),
                    "--threads",
                    str(entry.threads),
                    "--detach",
                ]
                if config_file:
                    cmd.extend(["--config", config_file])

                try:
                    subprocess.Popen(
                        cmd,
                        stdout=subprocess.DEVNULL,
                        stderr=subprocess.DEVNULL,
                        stdin=subprocess.DEVNULL,
                        start_new_session=True,
                        cwd=cwd,
                    )
                    console.print(f"[green]✓ Restarted {entry.tool_name}[/green]")
                except Exception as e:
                    console.print(f"[red]Failed to restart {entry.tool_name}: {e}[/red]")

            time.sleep(interval)

    except KeyboardInterrupt:
        mon_reg.unregister(0)
        console.print("\n[yellow]Monitor stopped[/yellow]")


def _run_monitor_loop(
    registry_dir,
    index: int,
    partner_pid: int,
    interval: int,
    config_file: str | None,
) -> None:
    """
    Internal monitor loop for dual-monitor pattern.

    Each monitor watches the workers AND the other monitor.
    If partner dies, it restarts the partner.
    """
    import os as os_module
    import shutil
    import signal
    import subprocess
    import time

    from iq_sdk.worker_registry import MonitorRegistry, WorkerRegistry

    mon_reg = MonitorRegistry(registry_dir)
    worker_reg = WorkerRegistry(registry_dir)

    # Get project root from registry dir (.iq-workers is in project root)
    cwd = str(registry_dir.parent)

    # Find iq executable
    iq_cmd = shutil.which("iq")
    if not iq_cmd:
        python_dir = os_module.path.dirname(sys.executable)
        iq_cmd = os_module.path.join(python_dir, "iq")
        if not os_module.path.exists(iq_cmd):
            return  # Can't restart anything without iq command

    # Register self
    mon_reg.register(index, partner_pid)

    def cleanup(signum, frame):
        mon_reg.unregister(index)
        sys.exit(0)

    signal.signal(signal.SIGTERM, cleanup)
    signal.signal(signal.SIGINT, cleanup)

    # Wait for partner to start before checking (avoid race condition)
    time.sleep(interval + 1)

    try:
        while True:
            # 1. Check and restart dead workers
            dead_entries = worker_reg.get_dead_entries()

            for path, entry in dead_entries:
                # Use atomic rename as lock to prevent dual restart
                lock_path = path.with_suffix(".restarting")
                try:
                    path.rename(lock_path)
                except (OSError, FileNotFoundError):
                    # Other monitor got it first
                    continue

                try:
                    lock_path.unlink()
                except OSError:
                    pass

                cmd = [
                    iq_cmd,
                    "worker",
                    "run",
                    "--site",
                    entry.site_id,
                    "--tool",
                    entry.get_filter_name(),
                    "--threads",
                    str(entry.threads),
                    "--detach",
                ]
                if config_file:
                    cmd.extend(["--config", config_file])

                try:
                    subprocess.Popen(
                        cmd,
                        stdout=subprocess.DEVNULL,
                        stderr=subprocess.DEVNULL,
                        stdin=subprocess.DEVNULL,
                        start_new_session=True,
                        cwd=cwd,
                    )
                except Exception:
                    pass

            # 2. Check partner monitor
            partner_index = 1 - index
            partner_path = mon_reg._get_monitor_path(partner_index)
            partner_alive = False

            if partner_path.exists():
                try:
                    with open(partner_path) as f:
                        import json

                        data = json.load(f)
                    partner_entry_pid = data.get("pid", 0)

                    # Check if partner is alive
                    try:
                        os_module.kill(partner_entry_pid, 0)
                        partner_alive = True
                    except (OSError, ProcessLookupError):
                        # Partner file exists but process dead - clean up
                        try:
                            partner_path.unlink()
                        except OSError:
                            pass
                except (json.JSONDecodeError, OSError):
                    pass

            # Restart partner if not alive
            if not partner_alive:
                cmd = [
                    iq_cmd,
                    "worker",
                    "monitor",
                    "--_monitor-index",
                    str(partner_index),
                    "--_partner-pid",
                    str(os_module.getpid()),
                ]
                if config_file:
                    cmd.extend(["--config", config_file])
                if interval != 3:
                    cmd.extend(["--interval", str(interval)])

                try:
                    subprocess.Popen(
                        cmd,
                        stdout=subprocess.DEVNULL,
                        stderr=subprocess.DEVNULL,
                        stdin=subprocess.DEVNULL,
                        start_new_session=True,
                        cwd=cwd,
                    )
                except Exception:
                    pass

            time.sleep(interval)

    except Exception:
        mon_reg.unregister(index)


@worker.command("kill")
@click.argument("pattern", required=False)
@click.option("--all", "-a", "kill_all", is_flag=True, help="Kill all workers")
@click.option("--monitors", "-m", is_flag=True, help="Kill monitor processes")
@click.option("-9", "force", is_flag=True, help="Use SIGKILL instead of SIGTERM")
@click.option("-y", "skip_confirm", is_flag=True, help="Skip confirmation prompt")
@click.option("--config", "-c", "config_file", help="Config file path")
@click.pass_context
def worker_kill(
    ctx: click.Context,
    pattern: str | None,
    kill_all: bool,
    monitors: bool,
    force: bool,
    skip_confirm: bool,
    config_file: str | None,
) -> None:
    """
    Stop workers and/or monitors.

    PATTERN: Substring match for tool names to kill.

    Examples:
        iq worker kill my_tool      # Kill workers matching "my_tool"
        iq worker kill --all        # Kill all workers
        iq worker kill --monitors   # Kill monitor processes
        iq worker kill -a -m        # Kill everything
    """
    import signal as sig_module

    from iq_sdk.cli import require_project
    from iq_sdk.worker_registry import MonitorRegistry, WorkerRegistry

    project_dir = require_project(ctx)
    registry_dir = project_dir / ".iq-workers"

    if not registry_dir.exists():
        console.print("[yellow]No workers registered[/yellow]")
        return

    worker_reg = WorkerRegistry(registry_dir)
    mon_reg = MonitorRegistry(registry_dir)

    # Collect targets
    workers_to_kill = []
    monitors_to_kill = []

    if monitors or kill_all:
        monitors_to_kill = mon_reg.get_alive_monitors()

    if pattern or kill_all:
        all_entries = worker_reg.get_alive_entries()
        if kill_all:
            workers_to_kill = all_entries
        elif pattern:
            workers_to_kill = [e for e in all_entries if pattern in e.tool_name]

    if not workers_to_kill and not monitors_to_kill:
        if pattern:
            console.print(f"[yellow]No workers matching '{pattern}'[/yellow]")
        else:
            console.print("[yellow]No workers or monitors running[/yellow]")
        return

    # Show what will be killed
    if workers_to_kill:
        console.print(f"[bold]Workers to kill ({len(workers_to_kill)}):[/bold]")
        for w in workers_to_kill:
            console.print(f"  {w.tool_name} (PID: {w.pid})")

    if monitors_to_kill:
        console.print(f"[bold]Monitors to kill ({len(monitors_to_kill)}):[/bold]")
        for m in monitors_to_kill:
            console.print(f"  Monitor (PID: {m.pid})")

    signal_name = "SIGKILL" if force else "SIGTERM"
    console.print(f"\n[dim]Signal: {signal_name}[/dim]")

    # Confirm
    if not skip_confirm:
        if not click.confirm("\nProceed?"):
            console.print("[yellow]Cancelled[/yellow]")
            return

    # Kill monitors first (so they don't restart workers)
    for m in monitors_to_kill:
        try:
            sig = sig_module.SIGKILL if force else sig_module.SIGTERM
            import os

            os.kill(m.pid, sig)
            console.print(f"[green]✓ Killed monitor (PID: {m.pid})[/green]")
        except (OSError, ProcessLookupError):
            console.print(f"[dim]Monitor {m.pid} already dead[/dim]")

    # Clean up monitor files
    if monitors_to_kill:
        for i in range(2):
            path = mon_reg._get_monitor_path(i)
            if path.exists():
                try:
                    path.unlink()
                except OSError:
                    pass

    # Kill workers
    for w in workers_to_kill:
        try:
            sig = sig_module.SIGKILL if force else sig_module.SIGTERM
            import os

            os.kill(w.pid, sig)
            console.print(f"[green]✓ Killed {w.tool_name} (PID: {w.pid})[/green]")
        except (OSError, ProcessLookupError):
            console.print(f"[dim]{w.tool_name} ({w.pid}) already dead[/dim]")

        # Remove registry file
        worker_reg.remove_entry_file(w)

    total = len(workers_to_kill) + len(monitors_to_kill)
    console.print(f"\n[bold green]Killed {total} process(es)[/bold green]")


# ============================================================================
# QUEUE COMMANDS
# ============================================================================


@worker.command("queue")
@click.option("--tool", "-t", "tool_filter", help="Filter by tool name (substring match)")
@click.option("--task-id", "-id", "task_id_filter", help="Filter by task ID")
@click.option("--site", "-s", "site_id", help="Filter by site ID")
@click.option("--failed", is_flag=True, help="Show failed queue (max retries exceeded)")
@click.option("--all", "-a", "show_all", is_flag=True, help="Show all entries (no pagination)")
@click.option("--retry", "-r", is_flag=True, help="Retry matching entries")
@click.option("--yes", "-y", is_flag=True, help="Skip confirmation for retry")
@click.option("--config", "-c", "config_file", help="Config file path")
def worker_queue(
    tool_filter: str | None,
    task_id_filter: str | None,
    site_id: str | None,
    failed: bool,
    show_all: bool,
    retry: bool,
    yes: bool,
    config_file: str | None,
) -> None:
    """
    View and manage the output retry queue.

    Shows pending outputs that failed to send and are waiting for retry.

    Examples:

        # List pending outputs
        iq worker queue

        # List failed outputs (max retries exceeded)
        iq worker queue --failed

        # Filter by tool
        iq worker queue -t my_tool

        # Manually retry all pending
        iq worker queue --retry --yes
    """
    from rich.prompt import Confirm

    from iq_sdk.client.output_queue import OutputQueueManager

    settings = load_settings(config_file)
    project_path = str(settings.get_project_root())

    # Get all configured sites
    if site_id:
        sites = settings.get_sites([site_id])
    else:
        sites = settings.get_sites()

    if not sites:
        console.print("[yellow]No sites configured[/yellow]")
        return

    all_outputs = []

    for site in sites:
        queue_manager = OutputQueueManager(
            base_path=project_path,
            site_id=site.id,
            retry_config=settings.worker.retry,
        )

        if failed:
            outputs = queue_manager.get_failed_outputs(
                tool_filter=tool_filter,
                task_id_filter=task_id_filter,
            )
        else:
            outputs = queue_manager.get_pending_outputs(
                tool_filter=tool_filter,
                task_id_filter=task_id_filter,
            )

        all_outputs.extend(outputs)

    if not all_outputs:
        queue_type = "failed" if failed else "pending"
        console.print(f"[dim]No {queue_type} outputs in queue[/dim]")
        return

    # Handle retry mode
    if retry:
        if not yes:
            if not Confirm.ask(f"Retry {len(all_outputs)} output(s)?"):
                return

        asyncio.run(_retry_outputs(all_outputs, settings))
        return

    # Display mode with pagination
    _display_queue_outputs(all_outputs, show_all, failed)


def _display_queue_outputs(outputs: list, show_all: bool, failed: bool) -> None:
    """Display queue outputs with optional pagination."""
    from datetime import datetime

    from rich.prompt import Prompt

    page_size = 20
    total = len(outputs)
    shown = 0

    queue_type = "Failed" if failed else "Pending"

    while shown < total:
        # Create table for this page
        table = Table(title=f"{queue_type} Queue ({total} total)")
        table.add_column("Task ID", style="cyan")
        table.add_column("Tool", style="green")
        table.add_column("Site", style="blue")
        table.add_column("Retries", justify="right")
        table.add_column("Status")
        table.add_column("Retry At" if not failed else "Created")

        end_idx = min(shown + page_size, total) if not show_all else total

        for output in outputs[shown:end_idx]:
            now = datetime.now()
            if failed:
                time_str = output.created_at.strftime("%Y-%m-%d %H:%M")
            else:
                if output.retry_at <= now:
                    time_str = "[red]DUE[/red]"
                else:
                    delta = output.retry_at - now
                    time_str = f"in {int(delta.total_seconds())}s"

            table.add_row(
                output.task_id[:12] + "..." if len(output.task_id) > 15 else output.task_id,
                output.tool_name,
                output.site_id,
                str(output.retry_count),
                output.status,
                time_str,
            )

        console.print(table)
        shown = end_idx

        if shown < total and not show_all:
            remaining = total - shown
            choice = Prompt.ask(
                f"[dim]{remaining} more. [m]ore / [q]uit[/dim]",
                choices=["m", "q", "more", "quit"],
                default="m",
            )
            if choice in ("q", "quit"):
                break
        else:
            break


async def _retry_outputs(outputs: list, settings) -> None:
    """Retry sending queued outputs."""
    from iq_sdk.client.output_queue import OutputQueueManager
    from iq_sdk.client.remote import IQApiClient

    project_path = str(settings.get_project_root())

    from iq_sdk.client.output_queue import send_processed

    # Group by site
    by_site: dict = {}
    for output in outputs:
        if output.site_id not in by_site:
            by_site[output.site_id] = []
        by_site[output.site_id].append(output)

    success_count = 0
    fail_count = 0

    for site_id, site_outputs in by_site.items():
        # Get site config
        sites = settings.get_sites([site_id])
        if not sites:
            console.print(f"[red]Site {site_id} not found[/red]")
            continue

        site = sites[0]
        queue_manager = OutputQueueManager(
            base_path=project_path,
            site_id=site_id,
            retry_config=settings.worker.retry,
        )

        async with IQApiClient.from_site(site) as api_client:
            for output in site_outputs:
                console.print(f"  → Retrying {output.task_id}...", end=" ")

                success, error = await send_processed(
                    api_client, output.task_id, output.output, output.status
                )

                if success:
                    queue_manager.remove_output(output)
                    console.print("[green]✓[/green]")
                    success_count += 1
                else:
                    queue_manager.increment_retry(output)
                    console.print(f"[red]✗ {error}[/red]")
                    fail_count += 1

    console.print(f"\n[bold]Retried: {success_count} success, {fail_count} failed[/bold]")
