"""
Command runner with support for concurrent local and remote execution.
Uses Paramiko for SSH and subprocess for local execution.
"""

import threading
import signal
import sys
import time
import concurrent.futures
from functools import partial
from collections.abc import Callable
from concurrent.futures import ThreadPoolExecutor, Future
from datetime import datetime
from typing import Any
import traceback
import os
import paramiko

from hil_testbench.data_structs.config_data import CleanupMode
from hil_testbench.config.task_config import TaskConfig
from hil_testbench.run.execution_context import ExecutionContext
from hil_testbench.run.output_streamer import OutputStreamer
from hil_testbench.run.command_result import CommandResult
from hil_testbench.run.task_logger import TaskLogger, LogLevel
from hil_testbench.formatting import format_duration
from hil_testbench.health_monitor import HealthMonitor


def _get_command_name(command_func: Callable[..., None]) -> str:
    """Extract command name, handling functools.partial objects."""
    if isinstance(command_func, partial):
        return getattr(command_func.func, "__name__", str(command_func))
    return getattr(command_func, "__name__", str(command_func))


class CommandRunner:
    """
    Executes commands with support for:
    - Concurrent or sequential execution of multiple commands
    - Mixed local and remote (SSH) command execution
    - Live output streaming with callbacks
    - Command cancellation
    - Graceful shutdown

    Usage:
        runner = CommandRunner(max_workers=5)

        # Define commands
        def build(c):
            c.run("npm install")
            c.run("npm run build")

        def deploy(c):
            c.run("rsync -av dist/ server:/var/www/")

        # Run tasks (single or multiple)
        results = runner.run([
            (build, None),
            (deploy, "user@server.com"),
        ])

        # Run with output callback
        def log_output(line: str, is_error: bool):
            # Example: route to logger instead of direct print
            pass

        results = runner.run([
            (deploy, "user@server.com", log_output),
        ])
    """

    def __init__(
        self,
        max_workers: int = 10,
        verbose: bool = True,
        log_dir: str = "logs",
        max_bytes_main: int = 10 * 1024 * 1024,
        max_bytes_task: int = 5 * 1024 * 1024,
        max_log_file_count_main: int = 10,
        max_log_file_count_task: int = 10,
        enable_health_logging: bool = True,
        health_interval: int = 600,
        config: "TaskConfig | None" = None,  # type: ignore
    ):
        self.max_workers = max_workers
        self.verbose = verbose
        self.log_dir = log_dir
        self.max_bytes_main = max_bytes_main
        self.max_bytes_task = max_bytes_task
        self.max_log_file_count_main = max_log_file_count_main
        self.max_log_file_count_task = max_log_file_count_task
        self._executor = None
        self._running_tasks: dict[str, Future] = {}
        self._ssh_clients: dict[str, paramiko.SSHClient] = {}
        self._streamers: dict[str, OutputStreamer] = {}
        self._running_contexts: dict[str, ExecutionContext] = {}
        self._cleanup_callbacks: dict[
            str, tuple[Callable, str | None, int, str | None]
        ] = {}
        self._cancel_event = threading.Event()
        self._lock = threading.Lock()
        self._config = config
        log_level = config.run_config.log_level if config else "INFO"
        self._task_logger = TaskLogger(
            log_dir,
            max_bytes_main,
            max_bytes_task,
            max_log_file_count_main,
            max_log_file_count_task,
            log_level=log_level,
        )
        self._health_monitor: HealthMonitor | None = None
        # Track number of tasks that have actually started execution (for accurate shutdown metrics)
        self._tasks_started = 0
        if enable_health_logging:
            self._health_monitor = HealthMonitor(
                logger=self._task_logger,
                interval=health_interval,
                running_tasks_supplier=lambda: len(self._running_tasks),
                stop_event=self._cancel_event,
                log_directory=self._task_logger.get_execution_dir(),
                cpu_threshold=80.0,
                memory_threshold=85.0,
                disk_threshold_gb=10.0,
            )
            self._health_monitor.start()

            # Set up callback to push health metrics to display backend
            self._setup_health_display_callback()

        # Set up signal handlers for graceful shutdown
        signal.signal(signal.SIGINT, self._signal_handler)
        signal.signal(signal.SIGTERM, self._signal_handler)

        # Install global threading exception hook to capture uncaught thread errors
        def _threading_excepthook(args):  # type: ignore[no-redef]
            try:
                exc_type = getattr(args, "exc_type", None)
                exc_value = getattr(args, "exc_value", None)
                exc_tb = getattr(args, "exc_traceback", None)
                thr = getattr(args, "thread", None)
                tb_str = "".join(
                    traceback.format_exception(exc_type, exc_value, exc_tb)
                )
                self._task_logger.log_framework(
                    "thread_exception",
                    LogLevel.ERROR,
                    thread_name=getattr(thr, "name", "unknown"),
                    error_type=getattr(exc_type, "__name__", str(exc_type)),
                    error=str(exc_value),
                    traceback=tb_str,
                )
            except Exception:  # pylint: disable=broad-except
                pass

        try:
            threading.excepthook = _threading_excepthook  # type: ignore[attr-defined]
        except AttributeError:  # pragma: no cover
            pass

    # Public accessors for correlation id and execution directory
    def get_correlation_id(self) -> str:
        """Return the correlation ID for the current execution."""
        return self._task_logger.get_correlation_id()

    def get_execution_dir(self) -> str:
        """Return the per-execution log directory path."""
        return self._task_logger.get_execution_dir()

    def _setup_health_display_callback(self):
        """Start periodic system health sampling for the display backend (best-effort)."""
        backend = getattr(self._task_logger, "_display_backend", None)
        if not backend:
            return
        try:
            import psutil  # hil: allow-lazy - optional dependency  # type: ignore
        except ImportError:  # pragma: no cover
            psutil = None  # type: ignore

        proc = None
        if psutil:
            try:
                proc = psutil.Process(os.getpid())
            except Exception:  # pylint: disable=broad-except
                proc = None

        def update_health_display():
            while not self._cancel_event.is_set():
                try:
                    timestamp = time.time()
                    if psutil:
                        # System-wide metrics
                        cpu_sys = psutil.cpu_percent(interval=0.2)
                        mem = psutil.virtual_memory()
                        disk = psutil.disk_usage(self._task_logger.get_execution_dir())
                        backend.update_parameter(
                            "System Health",
                            "CPU System%",
                            {"value": cpu_sys, "unit": "%"},
                            timestamp,
                        )
                        backend.update_parameter(
                            "System Health",
                            "Memory%",
                            {"value": mem.percent, "unit": "%"},
                            timestamp,
                        )
                        backend.update_parameter(
                            "System Health",
                            "Memory Used GB",
                            {"value": round(mem.used / (1024**3), 2), "unit": "GB"},
                            timestamp,
                        )
                        backend.update_parameter(
                            "System Health",
                            "Disk Free GB",
                            {"value": round(disk.free / (1024**3), 2), "unit": "GB"},
                            timestamp,
                        )
                        if proc:
                            cpu_proc = proc.cpu_percent(interval=None)
                            backend.update_parameter(
                                "System Health",
                                "CPU Proc%",
                                {"value": cpu_proc, "unit": "%"},
                                timestamp,
                            )
                    else:
                        # Minimal fallback: thread count only
                        backend.update_parameter(
                            "System Health",
                            "Thread Count",
                            {"value": threading.active_count(), "unit": "threads"},
                            timestamp,
                        )
                except Exception as e:  # pylint: disable=broad-except
                    self._task_logger.log_debug(
                        "health_display_update_error", error=str(e)
                    )
                time.sleep(10)

        try:
            threading.Thread(
                target=update_health_display,
                name="HealthDisplayUpdater",
                daemon=True,
            ).start()
        except Exception:  # pylint: disable=broad-except
            pass

    def _signal_handler(self, _signum, _frame):
        """Handle interrupt signals."""
        self._task_logger.log_framework(
            "interrupt_received",
            LogLevel.WARNING,
            message="Interrupt received - cancelling tasks and shutting down...",
        )
        self._cancel_event.set()
        self.shutdown()
        self._task_logger.log_framework("runner_exiting", LogLevel.INFO)
        sys.exit(130)

    def shutdown(self):
        """Gracefully shut down all running tasks, resources, and logging."""
        # Prevent duplicate shutdown work
        if getattr(self, "_shutdown_started", False):
            return
        self._shutdown_started = True

        # Stop health monitor first (captures final snapshot internally)
        if self._health_monitor:
            try:
                self._health_monitor.stop()
            except Exception:  # pylint: disable=broad-except
                pass

        # Kill any active processes
        with self._lock:
            for command_name, ctx in list(self._running_contexts.items()):
                if ctx.has_active_process:
                    try:
                        if self.verbose:
                            self._task_logger.log_framework(
                                "shutdown_task_kill",
                                LogLevel.INFO,
                                command_name=command_name,
                            )
                        ctx.kill()
                    except Exception as e:  # pylint: disable=broad-except
                        self._task_logger.log_framework(
                            "task_kill_error",
                            LogLevel.ERROR,
                            command_name=command_name,
                            error=str(e),
                        )
                else:
                    if self.verbose:
                        self._task_logger.log_framework(
                            "task_already_completed",
                            LogLevel.INFO,
                            command_name=command_name,
                            reason="shutdown_skip",
                        )

        # Shutdown executor
        if self._executor:
            try:
                self._executor.shutdown(wait=True, cancel_futures=True)
            except Exception as e:  # pylint: disable=broad-except
                self._task_logger.log_framework(
                    "executor_shutdown_error",
                    LogLevel.WARNING,
                    error=str(e),
                )

        # Close SSH connections
        self._close_ssh_connections()

        # Clear references
        with self._lock:
            self._running_tasks = {
                k: v for k, v in self._running_tasks.items() if not v.done()
            }
            self._running_contexts.clear()
            self._streamers.clear()

        final_threads = [
            t.name for t in threading.enumerate() if t is not threading.current_thread()
        ]
        remaining_tasks = len(
            [t for t, f in self._running_tasks.items() if not f.done()]
        )
        remaining_streamers = len(self._streamers)
        remaining_ssh = len(self._ssh_clients)
        issues = remaining_tasks > 0 or remaining_streamers > 0 or remaining_ssh > 0
        level = LogLevel.WARNING if issues else LogLevel.INFO
        self._task_logger.log_framework(
            "shutdown_final",
            level,
            remaining_tasks=remaining_tasks,
            remaining_streamers=remaining_streamers,
            remaining_ssh_connections=remaining_ssh,
            remaining_threads=final_threads,
        )

        # Diagnostics (best effort)
        try:
            try:
                import psutil  # hil: allow-lazy - optional dependency  # type: ignore
            except ImportError:  # pragma: no cover
                psutil = None  # type: ignore
            pid = os.getpid()
            thread_details = [
                {
                    "name": t.name,
                    "ident": t.ident,
                    "daemon": t.daemon,
                    "alive": t.is_alive(),
                }
                for t in threading.enumerate()
            ]
            children = []
            if psutil:
                try:
                    proc = psutil.Process(pid)
                    children = [c.pid for c in proc.children(recursive=True)]
                except Exception:  # pylint: disable=broad-except
                    children = []
            stack_dump = []
            if hasattr(sys, "_current_frames"):
                frames = getattr(sys, "_current_frames")()
                for thread_id, frame in frames.items():  # type: ignore[attr-defined]
                    if thread_id != threading.current_thread().ident:
                        stack_dump.append(
                            {
                                "thread_id": thread_id,
                                "stack": traceback.format_stack(frame)[-10:],
                            }
                        )
            self._task_logger.log_framework(
                "process_exit_diagnostics",
                LogLevel.INFO,
                pid=pid,
                thread_count=len(thread_details),
                threads=repr(thread_details[:15]),
                child_pids=repr(children),
                sampled_stack_threads=len(stack_dump),
            )
        except Exception:  # pylint: disable=broad-except
            pass

        self._task_logger.log_framework(
            "runner_shutdown_completed",
            LogLevel.INFO,
            total_tasks_executed=self._tasks_started,
            had_issues=issues,
        )
        self._task_logger.close_all()

    def _close_ssh_connections(self):
        """Close all cached SSH client connections."""
        for key, client in list(self._ssh_clients.items()):
            try:
                client.close()
                if self.verbose:
                    self._task_logger.log_framework(
                        "ssh_connection_closed", LogLevel.DEBUG, key=key
                    )
            except Exception as e:  # pylint: disable=broad-except
                self._task_logger.log_framework(
                    "ssh_connection_close_error",
                    LogLevel.WARNING,
                    key=key,
                    error=str(e),
                )
        self._ssh_clients.clear()

    def _get_or_create_ssh_client(
        self, host: str, port: int = 22, password: str | None = None
    ) -> paramiko.SSHClient:
        """Create or return a cached SSH client for host.

        Supports host formats:
        - "user@hostname"
        - "hostname"
        - "hostname:2222"
        - "user@hostname:2222"
        """
        username: str | None = None
        hostname = host
        # Extract username if provided
        if "@" in hostname:
            username, hostname = hostname.split("@", 1)
        # Extract custom port if provided
        actual_port = port
        if ":" in hostname:
            host_only, port_str = hostname.split(":", 1)
            hostname = host_only
            try:
                actual_port = int(port_str)
            except ValueError:
                actual_port = port
        key = f"{(username + '@') if username else ''}{hostname}:{actual_port}"
        if key in self._ssh_clients:
            return self._ssh_clients[key]
        client = paramiko.SSHClient()
        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        try:
            client.connect(
                hostname=hostname,
                port=actual_port,
                username=username,
                password=password,
                allow_agent=True,
                look_for_keys=True,
                timeout=10,
            )
            if self.verbose:
                self._task_logger.log_framework(
                    "ssh_connection_opened", LogLevel.INFO, host=key
                )
        except Exception as e:  # pylint: disable=broad-except
            self._task_logger.log_framework(
                "ssh_connection_error", LogLevel.ERROR, host=key, error=str(e)
            )
            raise
        self._ssh_clients[key] = client
        return client

    def _execute_task(
        self,
        command_func: Callable,
        command_name: str,
        host: str | None = None,
        port: int = 22,
        password: str | None = None,
        output_callback: Callable[..., Any] | None = None,
        error_callback: Callable[[Exception], None] | None = None,
        completion_callback: Callable[[int], None] | None = None,
        immediate: bool = False,
        log_output: bool | None = None,
        sample_lines: int = 0,
        use_pty: bool = False,
        **kwargs,
    ) -> CommandResult:
        """Execute a single task function."""
        start_time = datetime.now()

        # These parameters are reserved for future behavior tuning but not
        # directly used here; keep references to avoid unused-argument warnings
        _ = (password, immediate)

        # Determine execution context
        is_remote = host is not None
        execution_context = (
            {"type": "remote", "host": host} if is_remote else {"type": "local"}
        )

        # Log task start to main log
        self._task_logger.log_command(
            "command_started",
            command_name,
            LogLevel.INFO,
            command=command_name,
            execution_type=execution_context["type"],
            host=execution_context.get("host") if is_remote else None,
        )
        # Increment started task counter for accurate shutdown reporting
        self._tasks_started += 1

        # If output_callback is a callable that returns another callable,
        # treat it as a factory and call it with execution_dir
        actual_callback = output_callback
        if output_callback and callable(output_callback):
            try:
                # Try calling it with execution_dir - if it works, it's a factory
                result = output_callback(self._task_logger.get_execution_dir())
                if callable(result):
                    self._task_logger.log_command(
                        "callback_factory_detected",
                        command_name,
                        LogLevel.DEBUG,
                        factory_type=type(output_callback).__name__,
                        result_type=type(result).__name__,
                    )
                    actual_callback = result
                else:
                    # It didn't return a callable, so it's not a factory
                    self._task_logger.log_command(
                        "callback_factory_returned_non_callable",
                        command_name,
                        LogLevel.WARNING,
                        factory_type=type(output_callback).__name__,
                        result_type=type(result).__name__,
                    )
                    actual_callback = output_callback
            except TypeError as e:
                # It doesn't accept execution_dir, so it's a direct callback
                self._task_logger.log_command(
                    "callback_direct_not_factory",
                    command_name,
                    LogLevel.DEBUG,
                    callback_type=type(output_callback).__name__,
                    error=str(e),
                )
                actual_callback = output_callback
            except Exception as e:  # pylint: disable=broad-except
                # Unexpected error during factory detection
                self._task_logger.log_command(
                    "callback_factory_error",
                    command_name,
                    LogLevel.ERROR,
                    callback_type=type(output_callback).__name__,
                    error=str(e),
                    error_type=type(e).__name__,
                )
                actual_callback = output_callback

        # Create output streamer
        streamer = OutputStreamer(
            command_name, actual_callback, self._task_logger, log_output, sample_lines
        )
        with self._lock:
            self._streamers[command_name] = streamer

        ssh_client = None
        try:
            # Create SSH client if remote
            if is_remote and host:
                ssh_client = self._get_or_create_ssh_client(host, port)

            # Create execution context
            context = ExecutionContext(
                ssh_client=ssh_client,
                streamer=streamer,
                cancel_event=self._cancel_event,
                execution_dir=self._task_logger.get_execution_dir(),
                task_name=command_name,
                use_pty=use_pty,
                logger=self._task_logger,
                config=self._config or TaskConfig(),
            )

            with self._lock:
                self._running_contexts[command_name] = context

            # Execute the task function, passing through extra kwargs
            command_func(context, **kwargs)

            end_time = datetime.now()
            duration = (end_time - start_time).total_seconds()

            # Log task completion
            duration_str = format_duration(duration)
            self._task_logger.log_command(
                "command_succeeded",
                command_name,
                LogLevel.INFO,
                command=command_name,
                duration=duration_str,
                duration_seconds=f"{duration:.2f}",
            )

            result = CommandResult(
                command_name=command_name,
                success=True,
                return_code=0,
                stdout=streamer.get_stdout() if streamer else "",
                stderr=streamer.get_stderr() if streamer else "",
                start_time=start_time,
                end_time=end_time,
                duration=duration,
            )

            if completion_callback:
                try:
                    completion_callback(0)
                except Exception as e:  # pylint: disable=broad-except
                    self._task_logger.log_command(
                        "callback_error",
                        command_name,
                        LogLevel.ERROR,
                        callback_type="completion",
                        error=str(e),
                    )

            return result

        except KeyboardInterrupt:
            end_time = datetime.now()
            duration = (end_time - start_time).total_seconds()

            self._task_logger.log_command(
                "execution_cancelled",
                command_name,
                LogLevel.WARNING,
                duration=f"{duration:.2f}",
            )

            raise

        except Exception as e:  # pylint: disable=broad-except
            end_time = datetime.now()
            duration = (end_time - start_time).total_seconds()

            self._task_logger.log_command(
                "command_failed",
                command_name,
                LogLevel.ERROR,
                command=command_name,
                duration=f"{duration:.2f}",
                error=str(e),
                error_type=type(e).__name__,
            )

            if error_callback:
                try:
                    error_callback(e)
                except Exception as callback_error:  # pylint: disable=broad-except
                    self._task_logger.log_command(
                        "callback_error",
                        command_name,
                        LogLevel.ERROR,
                        callback_type="error",
                        error=str(callback_error),
                    )

            return CommandResult(
                command_name=command_name,
                success=False,
                return_code=-1,
                stdout=streamer.get_stdout() if streamer else "",
                stderr=streamer.get_stderr() if streamer else "",
                error=e,
                start_time=start_time,
                end_time=end_time,
                duration=duration,
            )
        finally:
            with self._lock:
                if streamer:
                    self._streamers.pop(command_name, None)
                self._running_contexts.pop(command_name, None)
            self._task_logger.close_task_logger(command_name)

    def run(
        self,
        tasks: list[tuple],
        parallel: bool = True,
        password: str | None = None,
        output_callback: Callable[[str, bool], None] | None = None,
        error_callback: Callable[[Exception], None] | None = None,
        completion_callback: Callable[[int], None] | None = None,
        immediate: bool = False,
        log_output: bool | None = None,
        sample_lines: int = 0,
        # force_cleanup argument removed (was unused)
        use_pty: bool = False,
        **kwargs,
    ) -> list:
        """
        Run one or more tasks (concurrently if parallel=True).

        Args:
            tasks: List of tuples. Each tuple can be:
                (command_func, host)
                (command_func, host, per_task_output_callback)
                (command_func, host, per_task_output_callback, cleanup_callback)
            parallel: If True, run tasks concurrently; if False, run sequentially
            password: SSH password for remote tasks
            output_callback: Called for each line of output from any task
            error_callback: Called on task errors
            completion_callback: Called when any task completes
            immediate: If True, run all tasks in immediate mode
            **kwargs: Additional arguments passed to all task functions

        Returns:
            List of CommandResult objects

        Example:
            # Run tasks with specific hosts and password
            results = runner.run([
                (build, None),  # local
                (deploy, "user@server.com"),  # remote
            ], password="pass")  # pragma: allowlist secret

            # Run all tasks locally in immediate mode
            results = runner.run([
                (build, None),
                (test, None),
                (lint, None),
            ], immediate=True)
        """
        # Handle backwards compatibility
        if not tasks:
            return []

        if not parallel:
            # Sequential execution
            results = []
            for entry in tasks:
                command_name_override = None
                if len(entry) == 2:
                    if isinstance(
                        entry[1], str
                    ):  # (func, command_name) or (func, host)
                        # Try to determine if it's a host or command_name
                        if "@" in entry[1] or ":" in entry[1]:
                            command_func, host = entry
                        else:
                            command_func = entry[0]
                            host = None
                            command_name_override = entry[1]
                    else:
                        command_func, host = entry
                    per_cb = output_callback
                    cleanup_cb = None
                elif len(entry) == 3:
                    command_func, host, per_cb = entry
                    cleanup_cb = None
                elif len(entry) == 4:
                    command_func, host, per_cb, cleanup_or_name = entry
                    # Determine if 4th element is cleanup callback or command_name
                    if isinstance(cleanup_or_name, str):
                        cleanup_cb = None
                        command_name_override = cleanup_or_name
                    else:
                        cleanup_cb = cleanup_or_name
                elif len(entry) == 5:
                    command_func, host, per_cb, cleanup_cb, command_name_override = (
                        entry
                    )
                else:
                    raise ValueError(
                        "Task entry must be (func, host), (func, host, callback), "
                        "(func, host, callback, cleanup), or (func, host, callback, cleanup, command_name)"
                    )
                result = self._execute_task(
                    command_func,
                    command_name_override or _get_command_name(command_func),
                    host,
                    22,
                    password,
                    per_cb,
                    error_callback,
                    completion_callback,
                    immediate,
                    log_output,
                    sample_lines,
                    use_pty,
                    **kwargs,
                )
                results.append(result)
                if not result.success:
                    if self.verbose:
                        self._task_logger.log_framework(
                            "sequential_execution_stopped",
                            LogLevel.WARNING,
                            command_name=command_name_override
                            or _get_command_name(command_func),
                        )
                    break
            return results

        # Parallel execution
        self._executor = ThreadPoolExecutor(max_workers=self.max_workers)
        results = []

        try:
            # Submit all tasks
            future_to_task: dict[Future, dict[str, Any]] = {}
            for entry in tasks:
                command_name_override = None
                if len(entry) == 2:
                    if isinstance(
                        entry[1], str
                    ):  # (func, command_name) or (func, host)
                        # Try to determine if it's a host or command_name
                        if "@" in entry[1] or ":" in entry[1]:
                            command_func, host = entry
                        else:
                            command_func = entry[0]
                            host = None
                            command_name_override = entry[1]
                    else:
                        command_func, host = entry
                    per_cb = output_callback
                    cleanup_cb = None
                elif len(entry) == 3:
                    command_func, host, per_cb = entry
                    cleanup_cb = None
                elif len(entry) == 4:
                    command_func, host, per_cb, cleanup_or_name = entry
                    # Determine if 4th element is cleanup callback or command_name
                    if isinstance(cleanup_or_name, str):
                        cleanup_cb = None
                        command_name_override = cleanup_or_name
                    else:
                        cleanup_cb = cleanup_or_name
                elif len(entry) == 5:
                    command_func, host, per_cb, cleanup_cb, command_name_override = (
                        entry
                    )
                else:
                    raise ValueError(
                        "Task entry must be (func, host), (func, host, callback), "
                        "(func, host, callback, cleanup), or (func, host, callback, cleanup, command_name)"
                    )
                command_name = command_name_override or _get_command_name(command_func)

                # Execute cleanup on startup if requested
                cleanup_mode = (
                    self._config.run_config.cleanup_mode
                    if self._config
                    else CleanupMode.STARTUP
                )
                # Only log cleanup check if cleanup is configured
                if self.verbose and self._config and cleanup_mode != CleanupMode.NONE:
                    self._task_logger.log_framework(
                        "cleanup_mode_check",
                        LogLevel.DEBUG,
                        cleanup_mode=cleanup_mode.value,
                        command_name=command_name,
                    )
                should_run_startup_cleanup = (
                    cleanup_mode in (CleanupMode.STARTUP, CleanupMode.BOTH)
                    and cleanup_cb is not None
                )
                if should_run_startup_cleanup:
                    self._execute_cleanup(cleanup_cb, host, 22, password, command_name)  # type: ignore

                # Store cleanup callback for shutdown if needed
                should_store_for_shutdown = (
                    cleanup_mode in (CleanupMode.SHUTDOWN, CleanupMode.BOTH)
                    and cleanup_cb is not None
                )
                if should_store_for_shutdown:
                    if self.verbose:
                        self._task_logger.log_framework(
                            "storing_cleanup_callback",
                            LogLevel.DEBUG,
                            command_name=command_name,
                            has_cleanup=cleanup_cb is not None,
                        )
                    with self._lock:
                        assert cleanup_cb is not None
                        typed_cleanup_cb: Callable = cleanup_cb
                        self._cleanup_callbacks[command_name] = (
                            typed_cleanup_cb,
                            host,
                            22,
                            password,
                        )

                future = self._executor.submit(
                    self._execute_task,
                    command_func,
                    command_name,
                    host,
                    22,
                    password,
                    per_cb,
                    error_callback,
                    completion_callback,
                    immediate,
                    log_output,
                    sample_lines,
                    use_pty,
                    **kwargs,
                )
                future_to_task[future] = {
                    "command_func": command_func,
                    "host": host,
                    "command_name": command_name,
                }
                with self._lock:
                    self._running_tasks[command_name] = future

            # Collect results as they complete
            pending_futures = set(future_to_task.keys())
            while pending_futures and not self._cancel_event.is_set():
                try:
                    done, pending_futures = concurrent.futures.wait(
                        pending_futures,
                        timeout=0.5,
                        return_when=concurrent.futures.FIRST_COMPLETED,
                    )

                    for future in done:
                        meta = future_to_task[future]
                        command_func = meta["command_func"]
                        host = meta["host"]
                        command_name = meta.get("command_name") or _get_command_name(
                            command_func
                        )
                        try:
                            result = future.result()
                            results.append(result)
                        except Exception as e:  # pylint: disable=broad-except
                            self._task_logger.log_command(
                                "task_execution_error",
                                command_name,
                                LogLevel.ERROR,
                                error=str(e),
                                error_type=type(e).__name__,
                            )
                            # Synthetic crash event with traceback for visibility
                            try:
                                tb_str = "".join(
                                    traceback.format_exception(
                                        type(e), e, e.__traceback__
                                    )
                                )
                                self._task_logger.log_command(
                                    "task_crashed",
                                    command_name,
                                    LogLevel.ERROR,
                                    host=host,
                                    error=str(e),
                                    error_type=type(e).__name__,
                                    traceback=tb_str,
                                )
                            except Exception:  # pylint: disable=broad-except
                                pass
                            results.append(
                                CommandResult(
                                    command_name=command_name,
                                    success=False,
                                    return_code=-1,
                                    error=e,
                                )
                            )
                        finally:
                            with self._lock:
                                self._running_tasks.pop(command_name, None)
                except Exception as e:  # pylint: disable=broad-except
                    self._task_logger.log_framework(
                        "wait_error",
                        LogLevel.ERROR,
                        error=str(e),
                    )
                    break

            if self._cancel_event.is_set():
                if self.verbose:
                    self._task_logger.log_framework(
                        "execution_cancelled",
                        LogLevel.WARNING,
                    )

            return results
        finally:
            self._executor.shutdown(wait=False)
            self._executor = None
            self._close_ssh_connections()

    def cancel_task(self, command_name: str) -> bool:
        """
        Cancel a specific running task.

        Returns:
            True if task was found and cancelled, False otherwise
        """
        with self._lock:
            # Stop output streaming
            if command_name in self._streamers:
                self._streamers[command_name].stop()

            # Cancel the future
            if command_name in self._running_tasks:
                future = self._running_tasks[command_name]
                cancelled = future.cancel()
                if self.verbose:
                    self._task_logger.log_command(
                        "task_cancel_attempt",
                        command_name,
                        LogLevel.INFO,
                        cancelled=cancelled,
                    )
                return True

        return False

    def cancel_all(self):
        """Cancel all running tasks."""
        self._cancel_event.set()

        with self._lock:
            # Stop all streamers
            for streamer in self._streamers.values():
                streamer.stop()

            # Cancel all futures
            for _, future in list(self._running_tasks.items()):
                future.cancel()

            if self.verbose and self._running_tasks:
                self._task_logger.log_framework(
                    "all_tasks_cancelled",
                    LogLevel.WARNING,
                    count=len(self._running_tasks),
                )

        # Shutdown executor if running
        if self._executor:
            self._executor.shutdown(wait=False, cancel_futures=True)

        # Close connections
        self._close_ssh_connections()

    def get_running_tasks(self) -> list[str]:
        """Get list of currently running task names."""
        with self._lock:
            return list(self._running_tasks.keys())

    def get_task_log_file(self, command_name: str) -> str | None:
        """
        Get the log file path for a specific task.

        Args:
            command_name: Name of the task

        Returns:
            Path to the log file, or None if task hasn't been run
        """
        return self._task_logger.get_log_file(command_name)

    def get_all_log_files(self) -> dict[str, str]:
        """
        Get all log file paths for completed tasks.

        Returns:
            Dictionary mapping task names to log file paths
        """
        return self._task_logger.get_all_task_log_files()

    def get_main_log_file(self) -> str | None:
        """
        Get the main program log file path.

        Returns:
            Path to the main log file
        """
        return self._task_logger.get_main_log_file()

    def get_task_logger(self) -> "TaskLogger":  # type: ignore[name-defined]
        """Return the TaskLogger instance for structured logging factories."""
        return self._task_logger

    def _print_health_summary(self):
        """Print health monitoring summary at shutdown (suppressed for user output)."""
        # Suppressed: No console output for health summary
        return

    def _execute_shutdown_cleanups(self):
        """Execute stored cleanup callbacks during shutdown.

        Emits a summary event with counts:
          - cleanup_count: total callbacks scheduled at shutdown start
          - completed_count: successfully executed callbacks
          - failed_count: callbacks that raised exceptions
        Severity: INFO if failed_count == 0 else WARNING.
        """
        with self._lock:
            cleanup_items = list(self._cleanup_callbacks.items())
            self._cleanup_callbacks.clear()
        total = len(cleanup_items)
        completed = 0
        failed = 0

        for command_name, (cleanup_func, host, port, password) in cleanup_items:
            try:
                if self.verbose:
                    self._task_logger.log_command(
                        "shutdown_cleanup_start", command_name, LogLevel.INFO
                    )
                self._execute_cleanup(cleanup_func, host, port, password, command_name)
                if self.verbose:
                    self._task_logger.log_command(
                        "shutdown_cleanup_completed", command_name, LogLevel.INFO
                    )
                completed += 1
            except Exception as e:  # pylint: disable=broad-except
                self._task_logger.log_command(
                    "shutdown_cleanup_failed",
                    command_name,
                    LogLevel.WARNING,
                    error=str(e),
                )
                failed += 1

        if self.verbose and total > 0:
            level = LogLevel.INFO if failed == 0 else LogLevel.WARNING
            self._task_logger.log_framework(
                "shutdown_cleanup_summary",
                level,
                cleanup_count=total,
                completed_count=completed,
                failed_count=failed,
            )

    def _execute_cleanup(
        self,
        cleanup_func: Callable,
        host: str | None,
        port: int,
        password: str | None,
        command_name: str,
    ):
        """Execute cleanup callback before task execution."""
        try:
            self._task_logger.log_command("cleanup_start", command_name, LogLevel.INFO)

            # Create SSH client if remote
            ssh_client = None
            if host:
                ssh_client = self._get_or_create_ssh_client(
                    host, port, password=password
                )

            # Create minimal execution context for cleanup
            # Don't pass cancel_event for shutdown cleanup to avoid cancellation
            context = ExecutionContext(
                ssh_client=ssh_client,
                streamer=None,
                cancel_event=None,  # Don't cancel cleanup during shutdown
                execution_dir=self._task_logger.get_execution_dir(),
                task_name=command_name,
                use_pty=False,
                logger=self._task_logger,
                config=self._config or TaskConfig(),
            )

            # Execute cleanup
            cleanup_func(context)

            if self.verbose:
                self._task_logger.log_command(
                    "cleanup_completed", command_name, LogLevel.INFO
                )

        except Exception as e:  # pylint: disable=broad-except
            self._task_logger.log_command(
                "cleanup_failed", command_name, LogLevel.WARNING, error=str(e)
            )
