import json
import logging
import sys
import traceback
from pathlib import Path
import time
import os
import threading
import subprocess
from collections import defaultdict
from datetime import datetime, timezone, timedelta
from typing import Optional, Dict, Any, List, Tuple, Union, Annotated
from enum import Enum
import sqlite3
import typer
import tempfile
import hashlib
import re
from dataclasses import dataclass

class LogLineWithTimestamp:
    """Represents a log line with parsed timestamp for chronological sorting."""
    
    def __init__(self, original_line: str, file_source: str = ""):
        self.original_line = original_line.rstrip('\n\r')
        self.file_source = file_source
        self.timestamp = self._parse_timestamp()
        self.display_line = self._format_for_display()
        
    def _parse_timestamp(self) -> Optional[datetime]:
        """Parse timestamp from log line using various common formats."""
        line = self.original_line
        
        # Skip lines that already have timestamp placeholders or source indicators
        if '[--:--:--' in line or '[STDOUT]' in line or '[STDERR]' in line:
            return None
        
        # Common timestamp patterns in order of preference
        patterns = [
            # ISO format with microseconds: 2024-07-11 13:56:37.123456
            (r'(\d{4}-\d{2}-\d{2}[\s|T]\d{2}:\d{2}:\d{2}\.\d{6})', '%Y-%m-%d %H:%M:%S.%f'),
            # ISO format with milliseconds: 2024-07-11 13:56:37.123
            (r'(\d{4}-\d{2}-\d{2}[\s|T]\d{2}:\d{2}:\d{2}\.\d{3})', '%Y-%m-%d %H:%M:%S.%f'),
            # ISO format seconds: 2024-07-11 13:56:37
            (r'(\d{4}-\d{2}-\d{2}[\s|T]\d{2}:\d{2}:\d{2})', '%Y-%m-%d %H:%M:%S'),
            # Standard log format: 2024-07-11T13:56:37Z
            (r'(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z?)', '%Y-%m-%dT%H:%M:%S'),
            # Python logging format: 2024-07-11 13:56:37,123
            (r'(\d{4}-\d{2}-\d{2}[\s]\d{2}:\d{2}:\d{2},\d{3})', '%Y-%m-%d %H:%M:%S'),
            # Job ID timestamp format: 20250711-135637 (YYYYMMDD-HHMMSS)
            (r'(\d{8}-\d{6})', '%Y%m%d-%H%M%S'),
            # Syslog format: Jul 11 13:56:37
            (r'([A-Za-z]{3}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2})', '%b %d %H:%M:%S'),
            # Time only with milliseconds: 13:56:37.123456
            (r'(\d{2}:\d{2}:\d{2}\.\d{6})', '%H:%M:%S.%f'),
            # Time only with milliseconds: 13:56:37.123
            (r'(\d{2}:\d{2}:\d{2}\.\d{3})', '%H:%M:%S.%f'),
            # Time only: 13:56:37
            (r'(\d{2}:\d{2}:\d{2})', '%H:%M:%S'),
        ]
        
        for pattern, fmt in patterns:
            match = re.search(pattern, line)
            if match:
                timestamp_str = match.group(1)
                try:
                    # Handle special cases
                    if 'T' in timestamp_str and timestamp_str.endswith('Z'):
                        timestamp_str = timestamp_str[:-1]  # Remove Z
                        fmt = fmt.replace('Z?', '')
                    
                    # Parse timestamp
                    if fmt == '%b %d %H:%M:%S':
                        # For syslog format, add current year
                        current_year = datetime.now().year
                        timestamp_str = f"{current_year} {timestamp_str}"
                        fmt = '%Y %b %d %H:%M:%S'
                    elif fmt in ['%H:%M:%S.%f', '%H:%M:%S']:
                        # For time-only, add current date
                        current_date = datetime.now().strftime('%Y-%m-%d')
                        timestamp_str = f"{current_date} {timestamp_str}"
                        fmt = f'%Y-%m-%d {fmt}'
                    
                    # Handle comma decimal separator (Python logging)
                    if ',' in timestamp_str and fmt == '%Y-%m-%d %H:%M:%S':
                        timestamp_str = timestamp_str.replace(',', '.')
                        fmt = '%Y-%m-%d %H:%M:%S.%f'
                    
                    parsed_time = datetime.strptime(timestamp_str, fmt)
                    
                    # If no timezone info, assume local timezone
                    if parsed_time.tzinfo is None:
                        parsed_time = parsed_time.replace(tzinfo=timezone.utc)
                    
                    return parsed_time
                    
                except ValueError:
                    continue
        
        return None
    
    def _format_for_display(self) -> str:
        """Format the line for display with highlighted timestamp."""
        line = self.original_line
        
        # If line already has source indicators or timestamp placeholders, return as-is
        if '[STDOUT]' in line or '[STDERR]' in line or '[--:--:--' in line:
            return line
            
        if self.timestamp:
            # Extract the original timestamp from the line
            timestamp_str = self.timestamp.strftime('%H:%M:%S.%f')[:-3]  # Show milliseconds
            
            # Find the first timestamp-like pattern and replace it
            patterns = [
                r'\d{4}-\d{2}-\d{2}[\s|T]\d{2}:\d{2}:\d{2}\.?\d*',
                r'\d{8}-\d{6}',  # Job ID format: 20250711-135637
                r'\d{2}:\d{2}:\d{2}\.?\d*',
                r'[A-Za-z]{3}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2}'
            ]
            
            for pattern in patterns:
                if re.search(pattern, line):
                    # Replace with our formatted timestamp
                    line = re.sub(pattern, f"[{timestamp_str}]", line, count=1)
                    break
            else:
                # If no timestamp found in line, prepend it
                line = f"[{timestamp_str}] {line}"
            
            return line
        else:
            # No timestamp found, return original line without placeholder
            return line
    
    def __lt__(self, other):
        """Enable sorting by timestamp."""
        if self.timestamp and other.timestamp:
            return self.timestamp < other.timestamp
        elif self.timestamp:
            return False  # Lines with timestamps come after those without
        elif other.timestamp:
            return True   # Lines without timestamps come before those with
        else:
            return False  # Keep original order for lines without timestamps


def parse_and_sort_log_lines(lines: List[str], file_source: str = "") -> List[LogLineWithTimestamp]:
    """Parse log lines, extract timestamps, and sort chronologically."""
    log_lines = []
    
    for line in lines:
        if line.strip():  # Skip empty lines
            log_lines.append(LogLineWithTimestamp(line, file_source))
    
    # Sort by timestamp (lines without timestamps will be grouped together)
    log_lines.sort()
    
    return log_lines


def read_log_efficiently(file_path: str, max_lines: int = 1000, max_bytes: int = 10 * 1024 * 1024) -> List[str]:
    """
    Efficiently read log files by reading from the end for large files.
    
    Args:
        file_path: Path to the log file
        max_lines: Maximum number of lines to return
        max_bytes: Maximum bytes to read from large files (default: 10MB)
    
    Returns:
        List of log lines (most recent first for large files)
    """
    try:
        file_size = os.path.getsize(file_path)
        
        # For small files (< 1MB), read normally
        if file_size < 1024 * 1024:
            with open(file_path, 'r', encoding='utf-8', errors='replace') as f:
                lines = f.readlines()
                return [line.rstrip('\n\r') for line in lines]
        
        # For large files, read from the end
        with open(file_path, 'rb') as f:
            # Read the last max_bytes
            read_size = min(file_size, max_bytes)
            f.seek(file_size - read_size)
            
            # Read and decode
            content = f.read().decode('utf-8', errors='replace')
            lines = content.split('\n')
            
            # Remove potential incomplete first line from seeking into middle
            if len(lines) > 1 and file_size > read_size:
                lines = lines[1:]
            
            # Clean up lines and limit count
            cleaned_lines = [line.rstrip('\n\r') for line in lines if line.strip()]
            
            # Return most recent lines
            return cleaned_lines[-max_lines:] if len(cleaned_lines) > max_lines else cleaned_lines
            
    except Exception as e:
        return [f"Error reading file: {str(e)}"]


def combine_and_sort_logs_efficiently(log_files: List[Tuple[str, str]], max_lines: int = 1000) -> List[str]:
    """
    Efficiently combine multiple log files without loading everything into memory.
    
    Args:
        log_files: List of (file_path, display_name) tuples
        max_lines: Maximum lines to return from each file
    
    Returns:
        List of combined log lines (not chronologically sorted for performance)
    """
    all_log_lines = []
    
    for file_path, display_name in log_files:
        if os.path.exists(file_path):
            try:
                # Use efficient reading
                lines = read_log_efficiently(file_path, max_lines)
                
                # Add source identification to each line
                for line in lines:
                    if line.strip():  # Skip empty lines
                        # Only add source indicator if line doesn't already have one
                        if not (line.startswith('[') and ']' in line[:20]):
                            tagged_line = f"[{display_name}] {line}"
                        else:
                            tagged_line = line
                        all_log_lines.append(tagged_line)
                        
            except Exception as e:
                # Add error line if file can't be read
                all_log_lines.append(f"[{display_name}] ERROR: Could not read file: {str(e)}")
    
    return all_log_lines


# Auto-detect and setup environment
def setup_environment():
    """Auto-detect environment and set up paths correctly."""
    current_dir = Path(__file__).parent.absolute()
    
    # Try to find the sphere root directory
    sphere_root = None
    
    # Check if we're in /sphere/app (installed layout - no src subdirectory)
    if current_dir.name == "app" and current_dir.parent.name == "sphere":
        sphere_root = current_dir
    
    # Check if we're in /sphere/src (development layout)
    if not sphere_root and current_dir.name == "src":
        potential_root = current_dir.parent
        if potential_root.name == "sphere":
            sphere_root = potential_root
    
    # If not found, try common paths
    if not sphere_root:
        # Try installed layout first: /sphere/app
        if (Path("/sphere/app") / "cli.py").exists():
            sphere_root = Path("/sphere/app")
        # Try development layout: /sphere/src
        elif (Path("/sphere/src") / "cli.py").exists():
            sphere_root = Path("/sphere")
    
    # Fallback to current directory's parent
    if not sphere_root:
        sphere_root = current_dir.parent
    
    # Add to Python path (installed layout has files directly in /sphere/app, not /sphere/app/src)
    if sphere_root.name == "app":
        # Installed layout: files are in /sphere/app directly
        if str(sphere_root) not in sys.path:
            sys.path.insert(0, str(sphere_root))
    else:
        # Development layout: files are in /sphere/src
        src_path = sphere_root / "src"
        if str(src_path) not in sys.path:
            sys.path.insert(0, str(src_path))
    
    # Change working directory to sphere root
    os.chdir(sphere_root)
    
    return sphere_root

# Setup environment before importing anything else
SPHERE_ROOT = setup_environment()

import typer
from typing_extensions import Annotated
from rich.console import Console
from rich.table import Table
from rich import box

# Now import our modules (they should work with the path setup)
from config import config
from featrix_queue import (
    JobStatus,
    add_notification_email_to_session,
    clear_directory,
    get_ready_jobs,
    initialize_standard_queues,
    list_jobs_in_queue,
    print_job_dir,
    run_job,
    serialize_job,
    serialize_session,
    iterate_over_jobs_in_queue,
)
from featrix_queue import add_job_to_queue as _add_job_to_queue
from featrix_queue import create_queue as _create_queue
from featrix_queue import create_session as _create_session
from featrix_queue import get_job_position_in_queue as _get_job_position_in_queue
from featrix_queue import get_session_info as _get_session_info
from featrix_queue import list_sessions as _list_sessions
from featrix_queue import requeue_job as _requeue_job
from featrix_queue import step_session as _step_session
from featrix_queue import update_job_status as _update_job_status
from featrix_queue import watch_queue as _watch_queue
from send_email import validate_and_normalize_email

# Set up proper logging for CLI
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s [%(levelname)-8s] %(name)-45s: %(message)s',
    datefmt='%Y-%m-%d %H:%M:%S'
)
logger = logging.getLogger(__name__)

app = typer.Typer()
console = Console()


def get_short_job_id(job_id: str) -> str:
    """Extract short job ID from full job ID.
    
    Examples:
        train_es_20250710-151917_2f3286 -> 2f3286
        create_structured_data_20250710-123456_abc123 -> abc123
    """
    if '_' in job_id:
        return job_id.split('_')[-1]
    return job_id[:8]  # Fallback to first 8 chars if no underscore

def get_job_output_directory(job_id):
    """Get the output directory path for a job."""
    from featrix_queue import get_job_output_path
    # Use helper function that handles new structure
    return str(get_job_output_path(job_id))


@app.callback(invoke_without_command=True)
def callback(ctx: typer.Context):
    """
    Featrix sphere CLI
    """
    # Removed annoying startup message
    if ctx.invoked_subcommand is None:
        # If no command provided, show dashboard
        ctx.invoke(dashboard)


@app.command()
def init_app(test: bool = False):
    """Initialize the app with all necessary directories and queues."""
    logger.info("Initializing app directories and queues")
    
    if test: 
        # For testing, create minimal directories and test queues
        config.queue_dir.mkdir(exist_ok=True, parents=True)
        config.output_dir.mkdir(exist_ok=True, parents=True)
        config.data_dir.mkdir(exist_ok=True, parents=True)
        config.session_dir.mkdir(exist_ok=True, parents=True)
        config.session_private_dir.mkdir(exist_ok=True, parents=True)
        
        _create_queue("echo_task_1")
        _create_queue("echo_task_2")

    else:
        # For production, use the comprehensive initialization
        initialize_standard_queues()
        logger.info("✅ All standard directories and queues initialized")


@app.command()
def remove_app_and_all_its_data():
    """
    Remove all directories and files created by the app, except data files.
    This is a destructive operation. 
    
    After running this function, you will need 
    to run `init-app` again before performing any other operations.
    
    If `remove-app` is run on a system where the app has never been initialized,
    no error will be raised. The function will attempt to clear the directories,
    and if they do not exist, it will proceed without issue.
    
    The following directories will be cleared:
    - config.output_dir
    - config.session_dir
    - config.queue_dir
    Note: The data directory will NOT be cleared.
    """

    # Basic directories
    clear_directory(config.output_dir, absent_ok=True, remove_if_present=True)
    clear_directory(config.session_dir, absent_ok=True, remove_if_present=True)
    clear_directory(config.session_private_dir, absent_ok=True, remove_if_present=True)
    # This removes all queues
    clear_directory(config.queue_dir, absent_ok=True, remove_if_present=True)
    
    # Do NOT clear data directory!!!

    logger.info("All app data removed")

# 
# 
# 
# QUEUES
# 

@app.command()
def create_queue(queue_name: str):
    _create_queue(queue_name)


@app.command()
def clear_queue(queue_name: str):
    """Clear all jobs from a queue."""
    logger.info(f"Clearing queue: {queue_name}")
    
    queue_path = config.queue_dir / queue_name
    if queue_path.exists():
        clear_directory(queue_path)
        logger.info(f"Queue {queue_name} cleared")
    else:
        logger.warning(f"Queue {queue_name} does not exist")

# 
# 
# 
# JOBS
# 

@app.command()
def list_jobs(
    queue_name: Annotated[str, typer.Option(..., help="Queue name")],
):
    logger.info(f"Listing jobs in queue {queue_name}")

    list_jobs_in_queue(queue_name)


@app.command()
def get_job_position_in_queue(
    queue_name: Annotated[str, typer.Option(..., help="Queue name")], 
    job_id: Annotated[str, typer.Option(..., help="Job ID")]
) -> int:
    position = _get_job_position_in_queue(queue_name=queue_name, job_id=job_id)

    logger.info(f"Job {job_id} is at position {position} in queue {queue_name}")


@app.command()
def add_job(
    # This is a way to specify that the argument is required AND 
    # specified as a named argument, not a positional argument.
    queue_name: Annotated[str, typer.Option(..., help="Queue name")], 
    session_id: Annotated[str, typer.Option(..., help="Session ID")],
    job_type: Annotated[str, typer.Option(..., help="Job type")], 
    job_spec: Path = None,
    job_id: str = None
):
    
    if job_spec is None:
        job_spec = dict()
    else:
        try:
            job_spec = json.loads(job_spec.read_text())
        except json.JSONDecodeError:
            raise ValueError("Job content must be a valid JSON string")
    
    _add_job_to_queue(
        queue_name=queue_name,
        session_id=session_id,
        job_type=job_type,
        job_spec=job_spec,
        job_id=job_id,
    )


@app.command()
def update_job_status(
    queue_name: Annotated[str, typer.Option(..., help="Queue name")], 
    job_id: Annotated[str, typer.Option(..., help="Job ID")], 
    status: Annotated[JobStatus, typer.Option(..., help="New job status")]
):
    _update_job_status(queue_name=queue_name, job_id=job_id, status=status)


@app.command()
def requeue_job(queue_name: str, job_id: str):
    _requeue_job(queue_name=queue_name, job_id=job_id)


@app.command(name="unfail")
def unfail(
    job_id: str,
    queue_name: Annotated[Optional[str], typer.Option(help="Queue name. If not provided, searches all queues.")] = None
):
    """
    Reset a failed job back to READY status so it can be retried.
    
    If queue_name is not provided, searches all known queues for the job.
    
    Usage:
        ffsh unfail <job_id>
        ffsh unfail <job_id> --queue-name train_es
    """
    from featrix_queue import load_job, save_job, JobStatus, iterate_over_jobs_in_queue
    from zoneinfo import ZoneInfo
    
    # Known queues to search if queue_name not provided
    known_queues = [
        "create_structured_data",
        "train_es", 
        "train_knn",
        "build_es_projections",
        "train_single_predictor",
        "cpu_data_tasks"
    ]
    
    job = None
    found_queue = None
    
    if queue_name:
        # Try the specified queue
        try:
            job = load_job(queue_name, job_id)
            found_queue = queue_name
        except FileNotFoundError:
            logger.error(f"❌ Job {job_id} not found in queue {queue_name}")
            raise typer.Exit(1)
    else:
        # Search all queues
        for q_name in known_queues:
            try:
                job = load_job(q_name, job_id)
                found_queue = q_name
                break
            except FileNotFoundError:
                continue
        
        if job is None:
            logger.error(f"❌ Job {job_id} not found in any queue")
            logger.info(f"   Searched queues: {', '.join(known_queues)}")
            raise typer.Exit(1)
    
    # Check current status
    current_status = job.get("status")
    
    if current_status == JobStatus.READY:
        logger.info(f"✅ Job {job_id} is already READY - no action needed")
        return
    
    if current_status != JobStatus.FAILED:
        logger.warning(f"⚠️  Job {job_id} is not in FAILED status (current: {current_status})")
        logger.warning(f"   Use 'update_job_status' if you want to change it anyway")
        raise typer.Exit(1)
    
    # Reset to READY
    job["status"] = JobStatus.READY.value
    job["started_at"] = None
    job["finished_at"] = None
    # Clear error message but keep it in history if it exists
    if "error_message" in job:
        if "error_history" not in job:
            job["error_history"] = []
        job["error_history"].append({
            "error_message": job["error_message"],
            "failed_at": job.get("finished_at"),
            "cleared_at": datetime.now(tz=ZoneInfo("America/New_York")).isoformat()
        })
        job["error_message"] = None
    
    # Save the updated job
    save_job(found_queue, job_id, job, exist_ok=True)
    
    logger.info(f"✅ Job {job_id} reset from FAILED to READY")
    logger.info(f"   Queue: {found_queue}")
    logger.info(f"   Job will be picked up by queue watcher for retry")


@app.command(name="mark-done")
def mark_done(
    job_id: str,
    queue_name: Annotated[Optional[str], typer.Option(help="Queue name. If not provided, searches all queues.")] = None
):
    """
    Manually mark a job as DONE (completed successfully).
    
    Useful when a job completed successfully but wasn't marked as DONE automatically,
    or when you want to mark a job as complete to allow dependent jobs to proceed.
    
    If queue_name is not provided, searches all known queues for the job.
    
    Usage:
        ffsh mark-done <job_id>
        ffsh mark-done <job_id> --queue-name train_es
    """
    from featrix_queue import load_job, save_job, JobStatus
    from zoneinfo import ZoneInfo
    
    # Known queues to search if queue_name not provided
    known_queues = [
        "create_structured_data",
        "train_es", 
        "train_knn",
        "build_es_projections",
        "train_single_predictor",
        "cpu_data_tasks"
    ]
    
    job = None
    found_queue = None
    
    if queue_name:
        # Try the specified queue
        try:
            job = load_job(queue_name, job_id)
            found_queue = queue_name
        except FileNotFoundError:
            logger.error(f"❌ Job {job_id} not found in queue {queue_name}")
            raise typer.Exit(1)
    else:
        # Search all queues
        for q_name in known_queues:
            try:
                job = load_job(q_name, job_id)
                found_queue = q_name
                break
            except FileNotFoundError:
                continue
        
        if job is None:
            logger.error(f"❌ Job {job_id} not found in any queue")
            logger.info(f"   Searched queues: {', '.join(known_queues)}")
            raise typer.Exit(1)
    
    # Check current status
    current_status = job.get("status")
    
    if current_status == JobStatus.DONE:
        logger.info(f"✅ Job {job_id} is already DONE - no action needed")
        return
    
    # Mark as DONE
    job["status"] = JobStatus.DONE.value
    if not job.get("finished_at"):
        job["finished_at"] = datetime.now(tz=ZoneInfo("America/New_York")).isoformat()
    
    # Save the updated job
    save_job(found_queue, job_id, job, exist_ok=True)
    
    logger.info(f"✅ Job {job_id} marked as DONE")
    logger.info(f"   Queue: {found_queue}")
    logger.info(f"   Status changed: {current_status} → DONE")


@app.command()
def recover_interrupted_jobs():
    """
    Manually recover jobs that were interrupted by server restart (ALL QUEUES).
    
    This command scans all queues for jobs with RUNNING status and recovers them:
    - train_es jobs: Resume from latest checkpoint if available
    - Other jobs: Reset to READY status for retry
    - Respects retry limits to prevent infinite loops
    
    Note: Individual workers normally handle this automatically on startup.
    """
    from featrix_queue import recover_interrupted_jobs as do_recovery
    
    logger.info("🔄 Starting manual job recovery...")
    recovery_summary = do_recovery()
    
    # Print detailed summary
    print("\n" + "="*60)
    print("🎯 JOB RECOVERY SUMMARY")
    print("="*60)
    print(f"Restart reason: {recovery_summary['restart_reason']}")
    print(f"Total jobs processed: {recovery_summary['total_recovered']}")
    
    if recovery_summary['train_es_resumed_from_checkpoint']:
        print(f"\n📁 train_es jobs resumed from checkpoints:")
        for job in recovery_summary['train_es_resumed_from_checkpoint']:
            print(f"  • {job['job_id']} → epoch {job['resume_epoch']}")
    
    if recovery_summary['jobs_restarted']:
        print(f"\n🔄 Jobs restarted from beginning:")
        for job in recovery_summary['jobs_restarted']:
            print(f"  • {job['queue']}: {job['job_id']}")
    
    if recovery_summary['jobs_blocked_by_retry_limit']:
        print(f"\n🚫 Jobs blocked by retry limits:")
        for job in recovery_summary['jobs_blocked_by_retry_limit']:
            print(f"  • {job['queue']}: {job['job_id']} ({job['reason']})")
    
    print("="*60)
    
    if recovery_summary['total_recovered'] == 0:
        print("✅ No interrupted jobs found")
    else:
        resumed = len(recovery_summary['train_es_resumed_from_checkpoint'])
        restarted = len(recovery_summary['jobs_restarted'])
        blocked = len(recovery_summary['jobs_blocked_by_retry_limit'])
        print(f"✅ Recovery complete: {resumed} resumed, {restarted} restarted, {blocked} blocked")


@app.command()
def recover_queue(queue_name: str):
    """
    Manually recover interrupted jobs for a specific queue.
    
    This is useful for debugging or recovering a specific queue without
    restarting the entire worker.
    """
    from featrix_queue import _recover_queue_interrupted_jobs
    
    logger.info(f"🔄 Starting manual recovery for queue: {queue_name}")
    recovered_jobs = _recover_queue_interrupted_jobs(queue_name)
    
    if recovered_jobs:
        print(f"✅ Recovered {len(recovered_jobs)} jobs from {queue_name}:")
        for job_id in recovered_jobs:
            print(f"  • {job_id}")
    else:
        print(f"✅ No interrupted jobs found in {queue_name}")
    
    return recovered_jobs


@app.command()
def execute_job(
    queue_name: Annotated[str, typer.Option(..., help="Queue name")], 
    job_idx: int = 0,
    auto_step: bool = False,
):
    """
    queue_name: str - The name of the queue to execute the job from.
    job_idx: int - The of the job to execute. Defaults to 0, which mean
                   the oldest job in the queue. job_idx=4 would mean the 5th oldest job in the queue.
    """
    
    # Get the oldest incomplete job in the queue/
    # Contains pending jobs from oldest to most recent.
    pending_jobs = get_ready_jobs(queue_name)

    if len(pending_jobs) == 0:
        logger.info("No pending jobs found")
        return
    
    if job_idx >= len(pending_jobs):
        raise ValueError(f"Job index {job_idx} is out of bounds")

    oldest_pending_job = pending_jobs[job_idx]
    job_id = oldest_pending_job["job_id"]

    try:
        logger.info(f"Executing job {job_id} from queue {queue_name}")
        err = run_job(queue_name=queue_name, job_id=job_id, auto_step=auto_step)
        if err is not None:
            logger.error("Error executing job:")
            traceback.print_exception(type(err), err, err.__traceback__)
        else:
            logger.info(f"Job {job_id} completed successfully")
    except Exception as e:
        logger.error(f"Fatal error executing job: {e}")
        traceback.print_exc()
        


@app.command()
def show_job_dir(
    queue_name: Annotated[str, typer.Option(..., help="Queue name")], 
    job_id: Annotated[str, typer.Option(..., help="Job ID")]
):
    print_job_dir(queue_name, job_id)

# 
# 
# 
# SESSIONS
# 

@app.command()
def create_session(
    session_type: Annotated[str, typer.Option(..., help="Session type")],
    session_id: str = None,
    start: bool = False,
    input_filename: str = None
):
    _create_session(
        session_type=session_type,
        session_id=session_id,
        start=start,
        input_filename=input_filename,
    )


@app.command()
def create_predictor_session(
    target_column: Annotated[str, typer.Option(..., help="Target column")],
    target_column_type: Annotated[str, typer.Option(..., help="Target column type")],
    session_id: str = None,
    start: bool = False,
    input_filename: str = None,
    n_epochs: int = 0,
    batch_size: int = 0,
    fine_tune: bool = False,
    learning_rate: float = 0.0001,
):
    """Create a session that trains an embedding space and then a single predictor."""
    
    logger.info(f"Creating predictor session for {input_filename}")
    logger.info(f"Target: {target_column} ({target_column_type})")
    
    if target_column_type not in ["set", "scalar"]:
        logger.error("target_column_type must be 'set' or 'scalar'")
        raise typer.Exit(1)

    if not Path(input_filename).exists():
        logger.error(f"Input file does not exist: {input_filename}")
        raise typer.Exit(1)

    target_spec = {
        "target_column": target_column,
        "target_column_type": target_column_type,
        "n_epochs": n_epochs,
        "batch_size": batch_size,
        "fine_tune": fine_tune,
        "learning_rate": learning_rate,
    }
    
    session = _create_session(
        session_type="predictor",
        session_id=session_id,
        start=start,
        input_filename=input_filename,
        target_spec=target_spec,
    )
    
    logger.info(f"Created predictor session: {session['session_id']}")
    logger.info(f"Target column: {target_column} ({target_column_type})")
    logger.info(f"Training parameters: epochs={n_epochs}, batch_size={batch_size}, lr={learning_rate}")


@app.command()
def list_sessions(show_jobs: bool = False):
    logger.info("Listing sessions")
    _list_sessions(show_jobs=show_jobs)


@app.command()
def clear_sessions():
    logger.warning("Clearing all sessions")
    clear_directory(config.session_dir)
    clear_directory(config.session_private_dir)
    logger.info("Sessions cleared")


@app.command()
def step_session(session_id: Annotated[str, typer.Argument(..., help="Session ID to step")]):
    logger.info(f"Stepping session {session_id}")
    _step_session(session_id)


@app.command()
def get_session_info(session_id: Annotated[str, typer.Argument(..., help="Session ID to get info for")]):
    logger.info(f"Getting info for session {session_id}")

    session_info = _get_session_info(session_id=session_id)

    serialized_session = serialize_session(session_info['session'])
    serialized_jobs = {job_id: serialize_job(job) for job_id, job in session_info['jobs'].items()}

    logger.info(json.dumps({
        "session": serialized_session,
        "jobs": serialized_jobs,
        "job_queue_positions": session_info['job_queue_positions'],
    }, indent=4))


# TODO: add cancel-job


@app.command()
def watch_queue(
    queue_name: Annotated[str, typer.Option(..., help="Queue name")],
    skip_existing: bool = False
):
    _watch_queue(queue_name=queue_name, skip_existing=skip_existing)


#
#
#
# MESSAGING
#

@app.command()
def send_slack_msg(msg_body: str):
    from slack import send_slack_message
    send_slack_message(msg_body)


@app.command()
def send_email():
    from send_email import send_email as _send_email
    _send_email()


@app.command()
def add_notification_email(
    session_id: Annotated[str, typer.Option(..., help="Session ID")], 
    to_address: Annotated[str, typer.Option(..., help="Email address")],
):
    
    logger.info(f"Adding notification email to session {session_id}")
    
    validated_address = validate_and_normalize_email(to_address)
    if validated_address is None:
        logger.error(f"Address {to_address} is invalid.")
        raise ValueError(f"Address {to_address} is invalid.")
    
    add_notification_email_to_session(session_id, validated_address)
    logger.info(f"Notification email {validated_address} added to session {session_id}")



#
#
#
# SIMILARITY SEARCH
#

@app.command()
def find_closest_points(
    session_id: Annotated[str, typer.Option(..., help="Session ID")], 
    query_point_path: Annotated[Path, typer.Option(..., help="Path to query point file")], 
    k: Annotated[int, typer.Option(..., help="Number of closest points")] = 5,
):
    from featrix_queue import find_closest_points as _find_closest_points
    from rich.table import Table
    from rich.console import Console

    if not query_point_path.is_file():
        raise FileNotFoundError(f"Query point file {query_point_path} not found")
    
    query_point = json.loads(query_point_path.read_text())

    search_result = _find_closest_points(session_id, query_point, k)
    query_results = search_result.get("results", [])
    stats = search_result.get("stats", {})


    # 
    # Display the query and results
    # 
    console = Console()

    # Display the query record in a formatted way
    console.print("[bold underline]Query Record:[/bold underline]", style="blue")
    query_table = Table(show_header=False, box=None)
    for key, value in query_point.items():
        query_table.add_row(f"[bold]{key}[/bold]", json.dumps(value, indent=2) if isinstance(value, dict) else str(value))
    console.print(query_table)
    
    # Display query statistics
    console.print("\n[bold underline]Query Statistics:[/bold underline]", style="cyan")
    stats_table = Table(show_header=False, box=None)
    stats_table.add_row("[bold]Query Time[/bold]", f"{stats.get('query_time_seconds', 0):.6f} seconds")
    stats_table.add_row("[bold]Records Searched[/bold]", f"{stats.get('records_searched', 0):,}")
    stats_table.add_row("[bold]Vector Dimensions[/bold]", f"{stats.get('vector_dimensions', 0)}")
    stats_table.add_row("[bold]Brute Force Comparisons[/bold]", f"{stats.get('brute_force_comparisons', 0):,}")
    stats_table.add_row("[bold]Results Returned[/bold]", f"{stats.get('results_returned', 0)}")
    stats_table.add_row("[bold]Avg Distance[/bold]", f"{stats.get('avg_distance', 0):.6f}")
    stats_table.add_row("[bold]Std Distance[/bold]", f"{stats.get('std_distance', 0):.6f}")
    stats_table.add_row("[bold]Min Distance[/bold]", f"{stats.get('min_distance', 0):.6f}")
    stats_table.add_row("[bold]Max Distance[/bold]", f"{stats.get('max_distance', 0):.6f}")
    console.print(stats_table)

    # Create a rich table to display the results
    table = Table(title="Closest Records")
    # table.add_column("ID", justify="center", style="cyan", no_wrap=True)
    table.add_column("row_id", justify="center", style="red")
    table.add_column("Coordinates", justify="center", style="magenta")
    table.add_column("Distance", justify="center", style="green")
    table.add_column("Original Data", justify="left", style="yellow")

    for result in query_results:
        # print(result)
        table.add_row(
            str(result["__featrix_row_id"]),        # We had __featrix_row_offset but it's not in result..???
            str(result["coords"]),
            f"{result['distance']:.4f}",
            json.dumps(result["original_data"], indent=2)
        )

    console.print(table)


@app.command()
def kill_job(queue_name: str, job_id: str, reason: str = "manual_cancellation"):
    """
    Kill a running job and mark it as CANCELLED to prevent restart.
    
    Args:
        queue_name: Name of the queue (e.g., 'train_es', 'create_structured_data')
        job_id: Job ID to kill
        reason: Reason for cancellation (default: manual_cancellation)
    
    This command:
    1. Updates job status to CANCELLED
    2. Adds cancellation metadata to prevent recovery
    3. Sets manual_kill flag to block restart attempts
    
    The job will not be recovered by the automatic recovery system.
    """
    from featrix_queue import load_job, save_job, JobStatus
    from datetime import datetime
    from zoneinfo import ZoneInfo
    from utils import convert_to_iso
    
    try:
        # Load the job
        job = load_job(queue_name, job_id)
        current_status = job.get("status")
        
        logger.info(f"🔴 Killing job {job_id} in queue {queue_name}")
        logger.info(f"   Current status: {current_status}")
        logger.info(f"   Reason: {reason}")
        
        # Update job to CANCELLED status
        job_content = job.copy()
        job_content["status"] = JobStatus.CANCELLED
        job_content["finished_at"] = datetime.now(tz=ZoneInfo("America/New_York"))
        
        # Add cancellation metadata to prevent recovery
        job_spec = job_content.get("spec", {}).copy()
        job_spec["manual_kill"] = True
        job_spec["kill_reason"] = reason
        job_spec["killed_at"] = convert_to_iso(job_content["finished_at"])
        job_content["spec"] = job_spec
        
        # Add to recovery info to block future recovery attempts
        if "recovery_info" not in job_content:
            job_content["recovery_info"] = []
        
        # Get version info for recovery tracking
        from featrix_queue import get_version_info
        version_info = get_version_info()
        
        job_content["recovery_info"].append({
            "recovered_at": convert_to_iso(job_content["finished_at"]),
            "reason": "manual_kill",
            "previous_status": current_status,
            "kill_reason": reason,
            "manual_kill": True,
            "version": version_info,
        })
        
        # Save the updated job
        save_job(queue_name, job_id, job_content, exist_ok=True)
        
        logger.info(f"✅ Job {job_id} killed successfully")
        logger.info(f"   Status changed: {current_status} → CANCELLED")
        logger.info(f"   Recovery blocked: manual_kill flag set")
        
        # Check if this affects the session
        session_id = job.get("session_id")
        if session_id:
            logger.info(f"📋 Job belongs to session {session_id}")
            logger.info(f"   Session may be affected by this cancellation")
        
        print(f"🔴 Job {job_id} has been killed and marked CANCELLED")
        print(f"   This job will NOT restart automatically")
        print(f"   Reason: {reason}")
        
    except FileNotFoundError:
        logger.error(f"❌ Job {job_id} not found in queue {queue_name}")
        print(f"❌ Job {job_id} not found in queue {queue_name}")
        raise typer.Exit(1)
    except Exception as e:
        logger.error(f"❌ Error killing job {job_id}: {e}")
        print(f"❌ Error killing job {job_id}: {e}")
        raise typer.Exit(1)


def abort_job(queue_name: str, job_id: str):
    """
    Create an ABORT flag for a job to stop it and prevent restart.
    
    Args:
        queue_name: Name of the queue (e.g., 'train_es', 'create_structured_data')
        job_id: Job ID to abort
    
    This command creates an ABORT file in the job's output directory.
    When the job detects this flag:
    1. The job will exit immediately
    2. The job will be marked as FAILED
    3. The job will NOT be recovered on restart
    
    Use this when you want to stop a running job permanently.
    """
    from pathlib import Path
    from config import config
    
    try:
        # Check if job exists
        from featrix_queue import load_job
        job = load_job(queue_name, job_id)
        current_status = job.get("status")
        
        logger.info(f"🚫 Creating ABORT flag for job {job_id} in queue {queue_name}")
        logger.info(f"   Current status: {current_status}")
        
        # Create ABORT file in job's output directory
        job_output_dir = config.output_dir / job_id
        job_output_dir.mkdir(parents=True, exist_ok=True)
        
        abort_file = job_output_dir / "ABORT"
        with open(abort_file, 'w') as f:
            f.write(f"ABORT flag created at {datetime.now().isoformat()}\n")
            f.write(f"Queue: {queue_name}\n")
            f.write(f"Job ID: {job_id}\n")
            f.write(f"Previous status: {current_status}\n")
        
        logger.info(f"✅ ABORT flag created: {abort_file}")
        
        print(f"🚫 ABORT flag created for job {job_id}")
        print(f"   Location: {abort_file}")
        if current_status == "running":
            print(f"   The running job will detect this flag and exit")
            print(f"   The job will be marked as FAILED")
        else:
            print(f"   Current status: {current_status}")
        print(f"   The job will NOT restart automatically")
        
    except FileNotFoundError:
        logger.error(f"❌ Job {job_id} not found in queue {queue_name}")
        print(f"❌ Job {job_id} not found in queue {queue_name}")
        raise typer.Exit(1)
    except Exception as e:
        logger.error(f"❌ Error creating ABORT flag for job {job_id}: {e}")
        print(f"❌ Error creating ABORT flag for job {job_id}: {e}")
        raise typer.Exit(1)


def finish_job(queue_name: str, job_id: str):
    """
    Create a FINISH flag for a job to complete it gracefully.
    
    Args:
        queue_name: Name of the queue (e.g., 'train_es', 'create_structured_data')
        job_id: Job ID to finish
    
    This command creates a FINISH file in the job's output directory.
    When the job detects this flag:
    1. The job will break out of the training loop
    2. The model will be saved (pickle dumped)
    3. The job will be marked as COMPLETED
    4. All normal completion steps will run
    
    Use this when you want to stop training early but still save the model
    in its current state.
    """
    from pathlib import Path
    from config import config
    
    try:
        # Check if job exists
        from featrix_queue import load_job
        job = load_job(queue_name, job_id)
        current_status = job.get("status")
        
        logger.info(f"🏁 Creating FINISH flag for job {job_id} in queue {queue_name}")
        logger.info(f"   Current status: {current_status}")
        
        # Create FINISH file in job's output directory
        job_output_dir = config.output_dir / job_id
        job_output_dir.mkdir(parents=True, exist_ok=True)
        
        finish_file = job_output_dir / "FINISH"
        with open(finish_file, 'w') as f:
            f.write(f"FINISH flag created at {datetime.now().isoformat()}\n")
            f.write(f"Queue: {queue_name}\n")
            f.write(f"Job ID: {job_id}\n")
            f.write(f"Previous status: {current_status}\n")
        
        logger.info(f"✅ FINISH flag created: {finish_file}")
        
        print(f"🏁 FINISH flag created for job {job_id}")
        print(f"   Location: {finish_file}")
        if current_status == "running":
            print(f"   The running job will detect this flag and complete gracefully")
            print(f"   The model will be saved in its current state")
            print(f"   The job will be marked as COMPLETED")
        else:
            print(f"   Current status: {current_status}")
        
    except FileNotFoundError:
        logger.error(f"❌ Job {job_id} not found in queue {queue_name}")
        print(f"❌ Job {job_id} not found in queue {queue_name}")
        raise typer.Exit(1)
    except Exception as e:
        logger.error(f"❌ Error creating FINISH flag for job {job_id}: {e}")
        print(f"❌ Error creating FINISH flag for job {job_id}: {e}")
        raise typer.Exit(1)


def restart_job(queue_name: str, job_id: str):
    """
    Create a RESTART flag for a job to restart it from scratch on next system startup.
    
    Args:
        queue_name: Name of the queue (e.g., 'train_es', 'create_structured_data')
        job_id: Job ID to restart
    
    This command creates a RESTART file in the job's output directory.
    When the worker starts up:
    1. It scans job directories for RESTART flags
    2. Jobs with RESTART flags are sorted by newest first (by file modification time)
    3. Jobs are reset to READY status
    4. All progress and recovery info is cleared
    5. The RESTART flag is renamed to started.RESTART.<date>
    6. The job will run again from the beginning
    
    Use this when you want a job to restart fresh on next worker startup.
    """
    from pathlib import Path
    from config import config
    
    try:
        # Check if job exists
        from featrix_queue import load_job
        job = load_job(queue_name, job_id)
        current_status = job.get("status")
        started_at = job.get("started_at")
        
        logger.info(f"🔄 Creating RESTART flag for job {job_id} in queue {queue_name}")
        logger.info(f"   Current status: {current_status}")
        
        # Verify job is from the last 2 days
        from datetime import datetime, timedelta
        from zoneinfo import ZoneInfo
        from utils import convert_from_iso
        
        if started_at:
            if isinstance(started_at, str):
                started_at = convert_from_iso(started_at)
            
            now = datetime.now(tz=ZoneInfo("America/New_York"))
            two_days_ago = now - timedelta(days=2)
            
            if started_at < two_days_ago:
                logger.warning(f"⚠️  Job {job_id} is older than 2 days - RESTART flag may not be processed")
                print(f"⚠️  WARNING: Job started more than 2 days ago")
                print(f"   The RESTART flag may not be detected on system startup")
        
        # Create RESTART file in job's output directory
        job_output_dir = config.output_dir / job_id
        job_output_dir.mkdir(parents=True, exist_ok=True)
        
        restart_file = job_output_dir / "RESTART"
        with open(restart_file, 'w') as f:
            f.write(f"RESTART flag created at {datetime.now().isoformat()}\n")
            f.write(f"Queue: {queue_name}\n")
            f.write(f"Job ID: {job_id}\n")
            f.write(f"Previous status: {current_status}\n")
        
        logger.info(f"✅ RESTART flag created: {restart_file}")
        
        print(f"🔄 RESTART flag created for job {job_id}")
        print(f"   Location: {restart_file}")
        print(f"   Current status: {current_status}")
        print(f"   On next worker startup:")
        print(f"     - Job will be reset to READY status")
        print(f"     - All progress will be cleared")
        print(f"     - Job will run from the beginning")
        print(f"     - RESTART flag will be renamed to started.RESTART.<date>")
        
    except FileNotFoundError:
        logger.error(f"❌ Job {job_id} not found in queue {queue_name}")
        print(f"❌ Job {job_id} not found in queue {queue_name}")
        raise typer.Exit(1)
    except Exception as e:
        logger.error(f"❌ Error creating RESTART flag for job {job_id}: {e}")
        print(f"❌ Error creating RESTART flag for job {job_id}: {e}")
        raise typer.Exit(1)


@app.command()
def list_running_jobs():
    """
    List all currently running jobs across all queues.
    
    This helps you find job IDs to kill if needed.
    """
    from featrix_queue import iterate_over_jobs_in_queue, JobStatus
    from rich.console import Console
    from rich.table import Table
    
    console = Console()
    
    # Check all known queue types
    known_queues = [
        "create_structured_data",
        "train_es", 
        "train_knn",
        "build_es_projections",
        "train_single_predictor"
    ]
    
    running_jobs = []
    
    for queue_name in known_queues:
        try:
            for job in iterate_over_jobs_in_queue(queue_name):
                if job.get("status") == JobStatus.RUNNING:
                    running_jobs.append({
                        "queue": queue_name,
                        "job_id": job.get("job_id"),
                        "job_type": job.get("type"),
                        "session_id": job.get("session_id"),
                        "started_at": job.get("started_at"),
                        "progress": job.get("progress")
                    })
        except Exception as e:
            # Queue might not exist, skip it
            logger.debug(f"Could not check queue {queue_name}: {e}")
            continue
    
    if not running_jobs:
        console.print("✅ No running jobs found")
        return
    
    # Create table
    table = Table(title="🔴 Currently Running Jobs")
    table.add_column("Queue", style="cyan")
    table.add_column("Short ID", style="yellow")
    table.add_column("Type", style="green")
    table.add_column("Session", style="blue")
    table.add_column("Started", style="magenta")
    table.add_column("Progress", style="white")
    table.add_column("Kill Command", style="red")
    
    for job in running_jobs:
        # Format progress
        progress = job.get("progress")
        if progress is not None:
            if isinstance(progress, float):
                progress_str = f"{progress:.1%}"
            else:
                progress_str = str(progress)
        else:
            progress_str = "Unknown"
        
        # Format started time
        started = job.get("started_at")
        if started:
            if hasattr(started, 'strftime'):
                started_str = started.strftime("%m-%d %H:%M")
            else:
                started_str = str(started)
        else:
            started_str = "-"
        
        # Create kill command
        kill_cmd = f"python3 src/cli.py kill-job {job['queue']} {job['job_id']}"
        
        table.add_row(
            job["queue"],
            get_short_job_id(job["job_id"]),
            job["job_type"],
            job["session_id"][:10] + "..." if len(job["session_id"]) > 13 else job["session_id"],
            started_str,
            progress_str,
            kill_cmd
        )
    
    console.print(table)
    console.print(f"\n💡 Found {len(running_jobs)} running jobs")
    console.print("💡 Use the kill command shown in the table to stop a job")


@app.command()
def kill_worker(queue_name: Annotated[Optional[str], typer.Option(help="Queue name (optional)")] = None):
    """
    Kill background queue worker processes.
    
    This kills the watch_queue processes that run in the background.
    """
    import subprocess
    import signal
    import os
    
    try:
        # Find Python processes running watch_queue
        cmd = ["ps", "aux"]
        result = subprocess.run(cmd, capture_output=True, text=True)
        
        if result.returncode != 0:
            print("❌ Failed to list processes")
            return
        
        # Look for watch_queue processes
        watch_queue_processes = []
        lines = result.stdout.split('\n')
        
        for line in lines:
            if 'watch_queue' in line and 'python' in line:
                parts = line.split()
                if len(parts) >= 2:
                    pid = parts[1]
                    # Extract queue name from command line
                    if queue_name:
                        if f"watch_queue('{queue_name}')" in line or f'watch_queue("{queue_name}")' in line:
                            watch_queue_processes.append((pid, queue_name, line))
                    else:
                        # Find queue name in the command line
                        for known_queue in ["cpu_data_tasks", "create_structured_data", "train_es", "train_knn", "build_es_projections", "train_single_predictor"]:
                            if known_queue in line:
                                watch_queue_processes.append((pid, known_queue, line))
                                break
                        else:
                            # Generic watch_queue process
                            watch_queue_processes.append((pid, "unknown", line))
        
        if not watch_queue_processes:
            if queue_name:
                print(f"✅ No {queue_name} worker processes found")
            else:
                print("✅ No queue worker processes found")
            return
        
        # Kill the processes
        killed_count = 0
        for pid, found_queue, cmd_line in watch_queue_processes:
            try:
                pid_int = int(pid)
                print(f"🔴 Killing {found_queue} worker process PID {pid}")
                print(f"   Command: {cmd_line[:100]}...")
                
                # Send SIGTERM first (graceful)
                os.kill(pid_int, signal.SIGTERM)
                killed_count += 1
                
                # Wait a moment then check if it's still running
                import time
                time.sleep(1)
                
                try:
                    os.kill(pid_int, 0)  # Check if process still exists
                    # If we get here, process is still running, force kill
                    print(f"   Process still running, sending SIGKILL...")
                    os.kill(pid_int, signal.SIGKILL)
                except ProcessLookupError:
                    # Process is gone, good
                    pass
                    
            except (ValueError, ProcessLookupError) as e:
                print(f"   ⚠️  Could not kill PID {pid}: {e}")
            except PermissionError:
                print(f"   ❌ Permission denied to kill PID {pid}")
            except Exception as e:
                print(f"   ❌ Error killing PID {pid}: {e}")
        
        if killed_count > 0:
            print(f"\n✅ Killed {killed_count} worker processes")
        else:
            print("\n❌ No processes were killed")
            
    except Exception as e:
        print(f"❌ Error killing worker processes: {e}")
        raise typer.Exit(1)


@app.command()
def tail(
    job_id: str, 
    follow: bool = typer.Option(False, "--follow", "-f", help="Follow log output in real-time"),
    no_pager: bool = typer.Option(False, "--no-pager", help="Disable pager, output directly to terminal"),
    no_color: bool = typer.Option(False, "--no-color", help="Disable colored output")
):
    """
    View job logs with syntax highlighting and smart job detection.
    
    Args:
        job_id: Job ID to view logs for (can be partial match or short ID)
        follow: Follow log output in real-time (like tail -f)
        no_pager: Disable pager, output directly to terminal
        no_color: Disable colored output
    """
    from pathlib import Path
    import subprocess
    import re
    
    # Determine base directory
    base_dir = _get_base_output_dir()
    
    if not base_dir.exists():
        console.print("❌ No featrix_output directory found")
        return
    
    # Smart job name resolution
    log_path = None
    if '/' not in job_id and not job_id.endswith('.log'):
        # Find matching job directories
        matching_dirs = []
        for job_dir in base_dir.glob(f"*{job_id}*"):
            if job_dir.is_dir():
                stdout_log = job_dir / "logs" / "stdout.log"
                if stdout_log.exists():
                    matching_dirs.append((job_dir, stdout_log))
        
        if len(matching_dirs) == 0:
            console.print(f"❌ No job directories found matching '{job_id}'")
            console.print(f"🔍 Available jobs:")
            for job_dir in sorted(base_dir.glob("*")):
                if job_dir.is_dir() and (job_dir / "logs" / "stdout.log").exists():
                                         console.print(f"   {get_short_job_id(job_dir.name)}: {job_dir.name}")
            return
        elif len(matching_dirs) == 1:
            job_dir, log_path = matching_dirs[0]
        else:
            console.print(f"🤔 Multiple jobs found matching '{job_id}':")
            for i, (job_dir, stdout_log) in enumerate(matching_dirs):
                file_size = stdout_log.stat().st_size / (1024 * 1024)
                mod_time = stdout_log.stat().st_mtime
                mod_time_str = datetime.fromtimestamp(mod_time).strftime('%Y-%m-%d %H:%M:%S')
                console.print(f"   {i+1}. {job_dir.name} ({file_size:.2f}MB, {mod_time_str})")
            
            try:
                choice = input(f"Select job (1-{len(matching_dirs)}): ").strip()
                idx = int(choice) - 1
                if 0 <= idx < len(matching_dirs):
                    job_dir, log_path = matching_dirs[idx]
                else:
                    console.print("❌ Invalid selection")
                    return
            except (ValueError, KeyboardInterrupt):
                console.print("❌ Please enter a number")
                return
    else:
        # Handle as file path
        log_path = Path(job_id)
        if not log_path.is_absolute():
            if not log_path.exists():
                potential_path = base_dir / job_id
                if potential_path.exists():
                    log_path = potential_path
    
    if not log_path or not log_path.exists():
        console.print(f"❌ Log file not found: {job_id}")
        return
    
    # Check if job is running and suggest following
    if not follow:
        job_dir = log_path.parent.parent
        is_running = _is_job_running_check(job_dir)
        
        if is_running:
            console.print(f"🔥 Job is currently running")
            if not no_pager and sys.stdout.isatty():
                response = input("📺 Watch in real-time with auto-refresh? [Y/n]: ").strip().lower()
                follow = response in ['', 'y', 'yes']
            else:
                follow = True
    
    try:
        if follow:
            _watch_log_file(log_path, no_color)
        else:
            _show_log_file(log_path, no_color, not no_pager)
    except (BrokenPipeError, KeyboardInterrupt):
        console.print("\n👋 Stopped watching logs")


@app.command()
def dashboard(
    watch: bool = typer.Option(False, "--watch", "-w", help="Auto-refresh dashboard every few seconds"),
    interactive: bool = typer.Option(False, "--interactive", "-i", help="Interactive TUI mode with arrow keys"),
    no_color: bool = typer.Option(False, "--no-color", help="Disable colored output")
):
    """
    Show comprehensive job dashboard with detailed status and runtime information.
    Static by default - use --watch for auto-refresh, --interactive for TUI mode.
    """
    
    if interactive:
        # Check if redis is available for interactive dashboard
        try:
            import redis
            redis.Redis(host='localhost', port=6379, db=1, decode_responses=True).ping()
        except Exception:
            console.print("[red]❌ Interactive dashboard requires Redis to be installed and running[/red]")
            console.print("[dim]Install with: pip install redis[/dim]")
            console.print("[dim]Or use non-interactive dashboard: ffsh dashboard[/dim]")
            raise typer.Exit(1)
        _run_interactive_dashboard()
        return
    
    def show_dashboard_once():
        console.print("\n[bold blue]🚀 FEATRIX SPHERE DASHBOARD[/bold blue]")
        console.print("[dim]Real-time status from queue system[/dim]")
        console.print("=" * 80)
        
        # Get all jobs from all queues
        all_jobs = _get_all_jobs_with_details()
        
        if not all_jobs:
            console.print("[yellow]📭 No jobs found in any queue[/yellow]")
            console.print("\n💡 [dim]To create a test session:[/dim]")
            console.print("[dim]   ffsh create-session test.csv[/dim]")
            return
        
        # Show summary
        summary = _calculate_job_summary(all_jobs)
        _show_job_summary(summary)
        
        # Show recent jobs table
        _show_recent_jobs_table(all_jobs, limit=15)
        
        # Show running jobs detail
        running_jobs = [job for job in all_jobs if job["status"] == JobStatus.RUNNING]
        if running_jobs:
            _show_running_jobs_detail(running_jobs)
        
        # Show commands
        _show_available_commands()
    
    if watch:
        try:
            while True:
                console.clear()
                show_dashboard_once()
                console.print(f"\n[dim]🔄 Auto-refreshing... (Ctrl+C to stop)[/dim]")
                time.sleep(5)
        except KeyboardInterrupt:
            console.print("\n👋 Stopped watching dashboard")
    else:
        show_dashboard_once()


# Log analyzer class and global instance
class LogAnalyzer:
    """Intelligent log analysis with SQLite caching for errors and tracebacks."""
    
    def __init__(self, cache_dir=None):
        import sqlite3
        import hashlib
        from pathlib import Path
        
        self.cache_dir = cache_dir or Path.home() / ".featrix_cache"
        self.cache_dir.mkdir(exist_ok=True)
        self.db_path = self.cache_dir / "log_analysis.db"
        self.init_cache()
        
    def init_cache(self):
        """Initialize SQLite cache for traceback analysis."""
        import sqlite3
        
        self.conn = sqlite3.connect(str(self.db_path))
        self.conn.execute("""
            CREATE TABLE IF NOT EXISTS log_analysis (
                file_path TEXT,
                file_size INTEGER,
                file_mtime REAL,
                last_scanned_offset INTEGER,
                errors TEXT,
                tracebacks TEXT,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                PRIMARY KEY (file_path)
            )
        """)
        self.conn.commit()
        
    def get_cached_analysis(self, file_path):
        """Get cached analysis for a log file if still valid."""
        import sqlite3
        import os
        
        try:
            stat = os.stat(file_path)
            cursor = self.conn.execute(
                "SELECT file_size, file_mtime, last_scanned_offset, errors, tracebacks FROM log_analysis WHERE file_path = ?",
                (str(file_path),)
            )
            row = cursor.fetchone()
            
            if row:
                cached_size, cached_mtime, last_offset, errors_json, tracebacks_json = row
                
                # Check if file hasn't changed significantly
                if (abs(stat.st_size - cached_size) < 10000 and  # Allow small growth
                    abs(stat.st_mtime - cached_mtime) < 60):      # Allow 1 minute mtime diff
                    
                    import json
                    return {
                        'last_offset': last_offset,
                        'errors': json.loads(errors_json) if errors_json else [],
                        'tracebacks': json.loads(tracebacks_json) if tracebacks_json else [],
                        'from_cache': True
                    }
            return None
        except:
            return None
            
    def update_cache(self, file_path, analysis):
        """Update cache with new analysis."""
        import sqlite3
        import json
        import os
        
        try:
            stat = os.stat(file_path)
            self.conn.execute("""
                INSERT OR REPLACE INTO log_analysis 
                (file_path, file_size, file_mtime, last_scanned_offset, errors, tracebacks)
                VALUES (?, ?, ?, ?, ?, ?)
            """, (
                str(file_path),
                stat.st_size,
                stat.st_mtime,
                analysis.get('last_offset', 0),
                json.dumps(analysis.get('errors', [])),
                json.dumps(analysis.get('tracebacks', []))
            ))
            self.conn.commit()
        except:
            pass
    
    def analyze_log_file(self, file_path, max_errors=10):
        """Analyze log file for errors and tracebacks."""
        import re
        import hashlib
        import os
        from pathlib import Path
        
        file_path = Path(file_path)
        if not file_path.exists():
            return {'errors': [], 'tracebacks': [], 'from_cache': False}
        
        # Check cache first
        cached = self.get_cached_analysis(file_path)
        if cached:
            return cached
        
        errors = []
        tracebacks = []
        current_traceback = []
        in_traceback = False
        
        # Error patterns
        error_patterns = [
            re.compile(r'.*(ERROR|FATAL|FAIL|Exception|Error).*', re.IGNORECASE),
            re.compile(r'.*Traceback \(most recent call last\).*', re.IGNORECASE),
        ]
        
        try:
            file_size = file_path.stat().st_size
            
            # For large files, read from end
            if file_size > 10 * 1024 * 1024:  # 10MB
                scan_size = 2 * 1024 * 1024  # Scan last 2MB
                with open(file_path, 'rb') as f:
                    f.seek(max(0, file_size - scan_size))
                    content = f.read().decode('utf-8', errors='replace')
                lines = content.split('\n')
                start_offset = max(0, file_size - scan_size)
            else:
                # Small files, read entirely
                with open(file_path, 'r', encoding='utf-8', errors='replace') as f:
                    lines = f.readlines()
                start_offset = 0
            
            for line_num, line in enumerate(lines):
                line = line.strip()
                
                # Check for traceback start
                if 'Traceback (most recent call last)' in line:
                    if current_traceback:
                        # Finish previous traceback
                        tb_text = '\n'.join(current_traceback)
                        tb_hash = hashlib.md5(tb_text.encode()).hexdigest()[:8]
                        tracebacks.append({
                            'hash': tb_hash,
                            'text': tb_text,
                            'line_num': line_num
                        })
                    current_traceback = [line]
                    in_traceback = True
                    continue
                    
                # Continue collecting traceback
                if in_traceback:
                    current_traceback.append(line)
                    # End traceback on empty line or new log entry
                    if not line or (line and not line.startswith(' ') and ':' in line and len(line.split()) > 3):
                        if len(current_traceback) > 1:
                            tb_text = '\n'.join(current_traceback)
                            tb_hash = hashlib.md5(tb_text.encode()).hexdigest()[:8]
                            tracebacks.append({
                                'hash': tb_hash,
                                'text': tb_text,
                                'line_num': line_num
                            })
                        current_traceback = []
                        in_traceback = False
                
                # Check for error patterns
                for pattern in error_patterns:
                    if pattern.match(line):
                        errors.append({
                            'line': line,
                            'line_num': line_num,
                            'type': self._classify_error(line)
                        })
                        break
            
            # Handle final traceback
            if current_traceback and len(current_traceback) > 1:
                tb_text = '\n'.join(current_traceback)
                tb_hash = hashlib.md5(tb_text.encode()).hexdigest()[:8]
                tracebacks.append({
                    'hash': tb_hash,
                    'text': tb_text,
                    'line_num': len(lines)
                })
            
            # Deduplicate tracebacks by hash
            unique_tracebacks = {}
            for tb in tracebacks:
                unique_tracebacks[tb['hash']] = tb
            
            # Keep only recent errors
            errors = errors[-max_errors:]
            
            analysis = {
                'errors': errors,
                'tracebacks': list(unique_tracebacks.values()),
                'last_offset': start_offset + len('\n'.join(lines).encode()),
                'from_cache': False
            }
            
            # Cache the results
            self.update_cache(file_path, analysis)
            return analysis
            
        except Exception as e:
            return {'errors': [], 'tracebacks': [], 'error': str(e), 'from_cache': False}
            
    def _classify_error(self, line):
        """Classify error type from line content."""
        line_lower = line.lower()
        if 'fatal' in line_lower:
            return 'FATAL'
        elif 'error' in line_lower:
            return 'ERROR'
        elif 'fail' in line_lower:
            return 'FAIL'
        elif 'exception' in line_lower:
            return 'EXCEPTION'
        else:
            return 'OTHER'
            
    def close(self):
        """Close database connection."""
        if hasattr(self, 'conn'):
            self.conn.close()

# Global log analyzer instance
_log_analyzer = None

def get_log_analyzer():
    """Get or create global log analyzer instance."""
    global _log_analyzer
    if _log_analyzer is None:
        _log_analyzer = LogAnalyzer()
    return _log_analyzer

def count_log_errors(job_id):
    """Count errors in job log files efficiently with caching."""
    try:
        output_dir = get_job_output_directory(job_id)
        log_file = os.path.join(output_dir, "logs", "stdout.log")
        
        if not os.path.exists(log_file):
            return 0
        
        # Use the existing log analyzer for efficiency 
        analyzer = get_log_analyzer()
        analysis = analyzer.analyze_log_file(log_file, max_errors=1000)  # Count up to 1000 errors
        
        # Count different types of errors
        errors = analysis.get('errors', [])
        return len(errors)
        
    except Exception:
        return 0  # Return 0 on any error to avoid breaking the UI


class TracebackMonitor:
    """Background daemon for monitoring and tracking tracebacks across all jobs."""
    
    def __init__(self, cache_dir=None):
        # Files are installed to /sphere/app (not /sphere/app/src)
        # So traceback cache should be at /sphere/app/.traceback_cache
        if cache_dir is None:
            # If SPHERE_ROOT is /sphere/app, use .traceback_cache directly
            # If SPHERE_ROOT is /sphere, use app/.traceback_cache
            if SPHERE_ROOT.name == "app":
                self.cache_dir = SPHERE_ROOT / ".traceback_cache"
            else:
                # SPHERE_ROOT is /sphere, so use /sphere/app/.traceback_cache
                self.cache_dir = SPHERE_ROOT / "app" / ".traceback_cache"
        else:
            self.cache_dir = Path(cache_dir)
        self.cache_dir.mkdir(parents=True, exist_ok=True)
        self.traceback_cache_file = self.cache_dir / "traceback_counts.json"
        self.last_scan_file = self.cache_dir / "last_scan.txt"
        
        # Load existing traceback data
        self.traceback_counts = self.load_traceback_counts()
        self.last_scan_time = self.load_last_scan_time()
        
    def load_traceback_counts(self):
        """Load cached traceback counts."""
        try:
            if self.traceback_cache_file.exists():
                import json
                with open(self.traceback_cache_file, 'r') as f:
                    return json.load(f)
        except Exception:
            pass
        return {}
        
    def save_traceback_counts(self):
        """Save traceback counts to cache."""
        try:
            import json
            with open(self.traceback_cache_file, 'w') as f:
                json.dump(self.traceback_counts, f, indent=2)
        except Exception:
            pass
            
    def load_last_scan_time(self):
        """Load last scan time."""
        try:
            if self.last_scan_file.exists():
                with open(self.last_scan_file, 'r') as f:
                    timestamp = float(f.read().strip())
                    return datetime.fromtimestamp(timestamp, tz=timezone.utc)
        except Exception:
            pass
        return datetime.min.replace(tzinfo=timezone.utc)
        
    def save_last_scan_time(self):
        """Save current time as last scan time."""
        try:
            import time
            with open(self.last_scan_file, 'w') as f:
                f.write(str(time.time()))
        except Exception:
            pass
            
    def normalize_traceback_signature(self, traceback_lines):
        """Create normalized signature for grouping similar tracebacks."""
        if not traceback_lines:
            return "unknown_error"
        
        # Extract key components for normalization
        error_type = "UnknownError"
        call_stack = []
        
        for line in traceback_lines:
            line = line.strip()
            
            # Skip header lines
            if 'Traceback (most recent call last):' in line:
                continue
                
            # Extract function calls from stack
            if line.startswith('File "'):
                import re
                match = re.search(r'File "([^"]+)", line \d+, in (.+)', line)
                if match:
                    filepath, func_name = match.groups()
                    filename = filepath.split('/')[-1]  # Just filename
                    call_stack.append(f"{filename}::{func_name}")
            
            # Extract error type
            elif ':' in line and not line.startswith('  '):
                error_type = line.split(':')[0].strip()
        
        # Create normalized signature
        if call_stack:
            signature = f"{error_type} -> {' -> '.join(call_stack[-3:])}"  # Last 3 calls
        else:
            signature = error_type
            
        return signature
        
    def get_traceback_summary(self, limit=10):
        """Get summary of most common tracebacks."""
        sorted_tracebacks = sorted(
            self.traceback_counts.items(),
            key=lambda x: x[1]['count'],
            reverse=True
        )
        
        return sorted_tracebacks[:limit]


# Global traceback monitor instance
_traceback_monitor = None

def get_traceback_monitor():
    """Get the global traceback monitor instance."""
    global _traceback_monitor
    if _traceback_monitor is None:
        _traceback_monitor = TracebackMonitor()
    return _traceback_monitor

def _run_interactive_dashboard():
    """Run interactive TUI dashboard using Textual."""
    try:
        from textual.app import App, ComposeResult
        from textual.containers import Horizontal, Vertical, ScrollableContainer, VerticalScroll
        from textual.widgets import Static, Footer, Header
        from textual.binding import Binding
        from rich.text import Text
        from rich.panel import Panel
        from rich import box
        import os
        from watchdog.observers import Observer
        from watchdog.events import FileSystemEventHandler
        import threading
        import time
        import psutil
        import platform
        from pathlib import Path
        
        # Module-level cache for system stats
        _system_stats_cache = {}
        
        def get_system_stats():
            """Get current system resource usage statistics."""
            try:
                # Cache system stats to avoid expensive calls on every refresh
                now = time.time()
                if '_last_system_check' not in _system_stats_cache or now - _system_stats_cache['_last_system_check'] > 10:
                    # Update system stats only every 10 seconds instead of every refresh
                    
                    # CPU usage (remove the 0.1 second delay - use instant reading)
                    cpu_percent = psutil.cpu_percent(interval=0)  # Changed from 0.1 to 0 for instant reading
                    
                    # Memory usage
                    memory = psutil.virtual_memory()
                    memory_percent = memory.percent
                    
                    # Memory pressure (based on available memory and swap usage)
                    memory_pressure = "LOW"
                    if memory_percent > 90:
                        memory_pressure = "CRITICAL"
                    elif memory_percent > 80:
                        memory_pressure = "HIGH"
                    elif memory_percent > 70:
                        memory_pressure = "MEDIUM"
                    
                    # Disk usage for the current directory
                    disk = psutil.disk_usage('/')
                    disk_percent = (disk.used / disk.total) * 100
                    
                    # GPU usage (try multiple methods) - cache this expensive operation
                    gpu_percent = None
                    gpu_memory_percent = None
                    
                    try:
                        # Try nvidia-smi first (but only every 10 seconds)
                        import subprocess
                        result = subprocess.run(['nvidia-smi', '--query-gpu=utilization.gpu,memory.used,memory.total', 
                                               '--format=csv,noheader,nounits'], 
                                              capture_output=True, text=True, timeout=1)  # Reduced timeout from 2 to 1
                        if result.returncode == 0:
                            lines = result.stdout.strip().split('\n')
                            if lines and lines[0]:
                                gpu_util, mem_used, mem_total = lines[0].split(', ')
                                gpu_percent = int(gpu_util)
                                gpu_memory_percent = int(float(mem_used) / float(mem_total) * 100)
                    except:
                        pass
                    
                    # Cache the results
                    _system_stats_cache['_cached_cpu_percent'] = cpu_percent
                    _system_stats_cache['_cached_memory_percent'] = memory_percent  
                    _system_stats_cache['_cached_memory_pressure'] = memory_pressure
                    _system_stats_cache['_cached_disk_percent'] = disk_percent
                    _system_stats_cache['_cached_gpu_percent'] = gpu_percent
                    _system_stats_cache['_cached_gpu_memory_percent'] = gpu_memory_percent
                    _system_stats_cache['_last_system_check'] = now
                else:
                    # Use cached values for faster refresh
                    cpu_percent = _system_stats_cache.get('_cached_cpu_percent', 0)
                    memory_percent = _system_stats_cache.get('_cached_memory_percent', 0)
                    memory_pressure = _system_stats_cache.get('_cached_memory_pressure', 'LOW')
                    disk_percent = _system_stats_cache.get('_cached_disk_percent', 0)
                    gpu_percent = _system_stats_cache.get('_cached_gpu_percent', None)
                    gpu_memory_percent = _system_stats_cache.get('_cached_gpu_memory_percent', None)
                
                if gpu_percent is None:
                    try:
                        # Try alternative GPU monitoring
                        import GPUtil
                        gpus = GPUtil.getGPUs()
                        if gpus:
                            gpu = gpus[0]
                            gpu_percent = int(gpu.load * 100)
                            gpu_memory_percent = int(gpu.memoryUtil * 100)
                    except:
                        pass
                
                return {
                    'cpu_percent': cpu_percent,
                    'memory_percent': memory_percent,
                    'memory_pressure': memory_pressure,
                    'disk_percent': disk_percent,
                    'gpu_percent': gpu_percent,
                    'gpu_memory_percent': gpu_memory_percent
                }
                
            except Exception as e:
                return {
                    'cpu_percent': 0,
                    'memory_percent': 0,
                    'memory_pressure': "UNKNOWN",
                    'disk_percent': 0,
                    'gpu_percent': None,
                    'gpu_memory_percent': None,
                    'error': str(e)
                }
        
        # LogAnalyzer class is now defined at module level
        
        class SystemStatusBar(Static):
            """Custom status bar showing system resource usage."""
            
            def __init__(self):
                super().__init__()
                self.stats = get_system_stats()
                
            def render(self):
                self.stats = get_system_stats()  # Update stats on each render
                
                content = Text()
                
                # CPU usage
                cpu = self.stats['cpu_percent']
                if cpu > 90:
                    cpu_style = "bold red"
                elif cpu > 70:
                    cpu_style = "bold yellow"
                else:
                    cpu_style = "green"
                content.append(f"CPU: {cpu:.1f}%", style=cpu_style)
                
                content.append(" • ", style="dim white")
                
                # Memory usage
                mem = self.stats['memory_percent']
                if mem > 90:
                    mem_style = "bold red"
                elif mem > 80:
                    mem_style = "bold yellow"
                else:
                    mem_style = "green"
                content.append(f"MEM: {mem:.1f}%", style=mem_style)
                
                content.append(" • ", style="dim white")
                
                # Memory pressure
                pressure = self.stats['memory_pressure']
                if pressure == "CRITICAL":
                    pressure_style = "bold red"
                elif pressure == "HIGH":
                    pressure_style = "bold yellow"
                elif pressure == "MEDIUM":
                    pressure_style = "yellow"
                else:
                    pressure_style = "green"
                content.append(f"PRESSURE: {pressure}", style=pressure_style)
                
                content.append(" • ", style="dim white")
                
                # Disk usage
                disk = self.stats['disk_percent']
                if disk > 95:
                    disk_style = "bold red"
                elif disk > 85:
                    disk_style = "bold yellow"
                else:
                    disk_style = "green"
                content.append(f"DISK: {disk:.1f}%", style=disk_style)
                
                # GPU usage (if available)
                gpu = self.stats['gpu_percent']
                gpu_mem = self.stats['gpu_memory_percent']
                if gpu is not None:
                    content.append(" • ", style="dim white")
                    if gpu > 90:
                        gpu_style = "bold red"
                    elif gpu > 70:
                        gpu_style = "bold yellow"
                    else:
                        gpu_style = "green"
                    content.append(f"GPU: {gpu}%", style=gpu_style)
                    
                    if gpu_mem is not None:
                        content.append(" • ", style="dim white")
                        if gpu_mem > 90:
                            gpu_mem_style = "bold red"
                        elif gpu_mem > 80:
                            gpu_mem_style = "bold yellow"
                        else:
                            gpu_mem_style = "green"
                        content.append(f"VRAM: {gpu_mem}%", style=gpu_mem_style)
                
                # System info
                content.append(" • ", style="dim white")
                hostname = platform.node()
                content.append(f"{hostname}", style="dim cyan")
                
                return content
        
        class ServerStatusWidget(Static):
            """Widget for displaying server status and API logs with continuous scrolling."""
            
            def __init__(self, dashboard):
                super().__init__()
                self.dashboard = dashboard
                self.border_title = "Server Status"
                self.log_lines = []
                self.max_lines = 200
                self.api_log_buffer = []  # Rolling buffer for combined Featrix log lines
                
            def on_click(self, event):
                """Handle mouse click to select this panel."""
                self.dashboard.current_panel = "server"
                self.dashboard.update_panel_styles()
                self.dashboard.refresh_widgets()
                event.stop()  # Prevent event bubbling
                
            def find_all_featrix_log_files(self):
                """Find all log files in /var/log/featrix/ directory."""
                featrix_log_dir = "/var/log/featrix"
                
                if not os.path.exists(featrix_log_dir) or not os.path.isdir(featrix_log_dir):
                    # Fallback to single API server log if featrix log directory doesn't exist
                    possible_paths = [
                        "/var/log/api_server.log", 
                        "/sphere/logs/api_server.log",
                        "/sphere/api_server.log",
                        "logs/api_server.log",
                        "api_server.log"
                    ]
                    
                    for log_path in possible_paths:
                        if os.path.exists(log_path):
                            return [log_path]
                    return []
                
                # Get all .log files from featrix directory
                log_files = []
                try:
                    for filename in os.listdir(featrix_log_dir):
                        if filename.endswith('.log'):
                            full_path = os.path.join(featrix_log_dir, filename)
                            if os.path.isfile(full_path):
                                log_files.append(full_path)
                    
                    # Sort by modification time (most recent first)
                    log_files.sort(key=lambda f: os.path.getmtime(f), reverse=True)
                    return log_files
                    
                except (OSError, IOError):
                    return []
                
            def get_all_featrix_logs(self):
                """Get recent logs from all files in /var/log/featrix/ with continuous tailing."""
                try:
                    log_files = self.find_all_featrix_log_files()
                    
                    if not log_files:
                        return ["📍 No Featrix log files found in /var/log/featrix or fallback paths"]
                    
                    # Efficiently combine log files without loading everything into memory
                    log_file_tuples = [(log_path, os.path.basename(log_path)) for log_path in log_files]
                    all_log_lines = combine_and_sort_logs_efficiently(log_file_tuples, max_lines=self.max_lines)
                    
                    if not all_log_lines:
                        return ["📍 All Featrix log files are empty or unreadable"]
                    
                    # Store for HTTP error analysis
                    self.api_log_buffer = all_log_lines
                    
                    return all_log_lines
                    
                except Exception as e:
                    return [f"❌ Error reading Featrix logs: {str(e)}"]

            def extract_http_errors(self, log_lines):
                """Extract HTTP 4xx and 5xx errors from log lines."""
                import re
                
                # Patterns to match HTTP status codes in various log formats
                http_patterns = [
                    # Common patterns for HTTP status codes in logs
                    r'(\b[45]\d{2}\b)',  # Basic 4xx/5xx pattern
                    r'status(?:_code)?[:\s=]+([45]\d{2})',  # status: 404, status_code=500
                    r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})[^\d]*([45]\d{2})',  # IP followed by status
                    r'HTTP/\d\.\d"\s+([45]\d{2})',  # HTTP/1.1" 404
                    r'"(?:GET|POST|PUT|DELETE|PATCH)[^"]*"\s+([45]\d{2})',  # "GET /path" 404
                    r'response_status[:\s=]+([45]\d{2})',  # response_status: 500
                    r'error[:\s]+([45]\d{2})',  # error: 404
                    r'status[:\s]+([45]\d{2})',  # status: 500
                ]
                
                errors = []
                now = datetime.now()
                
                for line in log_lines:
                    line_stripped = line.strip()
                    if not line_stripped:
                        continue
                    
                    # Check each pattern
                    for pattern in http_patterns:
                        matches = re.findall(pattern, line_stripped, re.IGNORECASE)
                        for match in matches:
                            # Handle tuple results (from capture groups)
                            if isinstance(match, tuple):
                                status_code = match[-1]  # Take the last capture group (status code)
                            else:
                                status_code = match
                            
                            # Validate it's actually a 4xx or 5xx code
                            try:
                                code = int(status_code)
                                if 400 <= code <= 599:
                                    # Extract timestamp if possible
                                    timestamp_match = re.search(r'(\d{4}-\d{2}-\d{2}[T\s]\d{2}:\d{2}:\d{2})', line_stripped)
                                    timestamp_str = timestamp_match.group(1) if timestamp_match else "unknown time"
                                    
                                    # Classify error type
                                    if 400 <= code <= 499:
                                        error_type = "Client Error"
                                        if code == 404:
                                            error_type = "Not Found"
                                        elif code == 401:
                                            error_type = "Unauthorized"
                                        elif code == 403:
                                            error_type = "Forbidden"
                                        elif code == 400:
                                            error_type = "Bad Request"
                                    else:  # 5xx
                                        error_type = "Server Error"
                                        if code == 500:
                                            error_type = "Internal Error"
                                        elif code == 502:
                                            error_type = "Bad Gateway"
                                        elif code == 503:
                                            error_type = "Service Unavailable"
                                    
                                    errors.append({
                                        'status_code': code,
                                        'error_type': error_type,
                                        'timestamp': timestamp_str,
                                        'line': line_stripped[:100] + "..." if len(line_stripped) > 100 else line_stripped,
                                        'raw_line': line_stripped
                                    })
                                    break  # Found a match for this line, move to next line
                            except ValueError:
                                continue
                
                # Return last 10 unique errors, most recent first
                unique_errors = []
                seen_combinations = set()
                
                for error in reversed(errors):  # Start from most recent
                    # Create a signature to avoid duplicates
                    signature = f"{error['status_code']}_{error['error_type']}"
                    if signature not in seen_combinations:
                        unique_errors.append(error)
                        seen_combinations.add(signature)
                        if len(unique_errors) >= 10:
                            break
                
                return unique_errors
                    
            def get_deployment_status(self):
                """Get deployment monitoring status."""
                try:
                    # Check if deployment monitor is running
                    import subprocess
                    result = subprocess.run(['pgrep', '-f', 'git.*pull'], 
                                          capture_output=True, text=True)
                    
                    if result.returncode == 0:
                        pids = result.stdout.strip().split('\n')
                        return f"🔄 Deployment monitor active (PID: {', '.join(pids)})"
                    else:
                        return "💤 No deployment monitor detected"
                        
                except Exception as e:
                    return f"❓ Deployment status unknown: {str(e)[:40]}"
                    
            def get_git_status(self):
                """Get current git status."""
                try:
                    import subprocess
                    result = subprocess.run(['git', 'rev-parse', '--short', 'HEAD'], 
                                          capture_output=True, text=True, cwd='/sphere')
                    if result.returncode == 0:
                        commit = result.stdout.strip()
                        return f"📋 Current commit: {commit}"
                    else:
                        return "❓ Git status unavailable"
                except Exception:
                    return "❓ Git not available"
                    
            def render(self):
                from rich.console import Console
                from rich.text import Text
                
                console = Console(width=60)
                content = Text()
                
                # Server status header
                content.append("🖥️  SERVER STATUS\n", style="bold cyan")
                content.append("─" * 50 + "\n", style="dim white")
                
                # Deployment status
                deployment_status = self.get_deployment_status()
                content.append(f"{deployment_status}\n", style="green" if "active" in deployment_status else "yellow")
                
                # Git status
                git_status = self.get_git_status()
                content.append(f"{git_status}\n", style="blue")
                
                # Traceback monitoring status
                traceback_monitor = get_traceback_monitor()
                traceback_summary = traceback_monitor.get_traceback_summary(3)
                if traceback_summary:
                    content.append("\n🚨 RECENT TRACEBACKS:\n", style="bold red")
                    for signature, data in traceback_summary:
                        count = data['count']
                        error_type = data['error_type']
                        content.append(f"  🔴 {error_type} ({count}x): {signature[:40]}...\n", style="red")
                else:
                    content.append("\n✅ No tracebacks detected\n", style="green")
                
                # Get and analyze all Featrix logs
                log_lines = self.get_all_featrix_logs()
                http_errors = self.extract_http_errors(log_lines)
                
                # HTTP Errors section (if any found)
                if http_errors:
                    content.append("\n🚨 RECENT HTTP ERRORS\n", style="bold red")
                    content.append("─" * 50 + "\n", style="dim white")
                    
                    for error in http_errors[:5]:  # Show up to 5 recent errors
                        status_code = error['status_code']
                        error_type = error['error_type']
                        timestamp = error['timestamp']
                        line_preview = error['line']
                        
                        # Color code by error type
                        if 400 <= status_code <= 499:
                            error_style = "yellow"  # Client errors
                        else:
                            error_style = "red"     # Server errors
                        
                        content.append(f"  🔴 {status_code} {error_type}", style=error_style)
                        if timestamp != "unknown time":
                            content.append(f" @ {timestamp}", style="dim white")
                        content.append("\n")
                        
                        # Show a truncated preview of the log line
                        preview = line_preview.replace("\n", " ").strip()
                        if len(preview) > 60:
                            preview = preview[:57] + "..."
                        content.append(f"      {preview}\n", style="dim white")
                    
                    if len(http_errors) > 5:
                        remaining = len(http_errors) - 5
                        content.append(f"    ... and {remaining} more errors\n", style="dim cyan")
                    
                    content.append("\n", style="white")
                
                # Featrix logs header
                content.append("📜 RECENT FEATRIX LOGS\n", style="bold yellow")
                content.append("─" * 50 + "\n", style="dim white")
                
                # Show recent logs with syntax highlighting (no chronological sorting for performance)
                display_lines = log_lines[-15:] if len(log_lines) > 15 else log_lines
                for line in display_lines:
                    line = line.strip()
                    if not line:
                        continue
                        
                    # Highlight HTTP error lines differently
                    is_http_error = any(str(error['status_code']) in line for error in http_errors)
                    
                    # Apply basic log level coloring
                    if is_http_error:
                        content.append(f"{line}\n", style="bold red")
                    elif " ERROR " in line or " CRITICAL " in line:
                        content.append(f"{line}\n", style="red")
                    elif " WARNING " in line or " WARN " in line:
                        content.append(f"{line}\n", style="yellow")
                    elif " INFO " in line:
                        content.append(f"{line}\n", style="green")
                    elif " DEBUG " in line:
                        content.append(f"{line}\n", style="dim blue")
                    else:
                        content.append(f"{line}\n", style="dim white")
                
                # Add scroll indicator and tailing status
                if len(log_lines) > 15:
                    content.append(f"\n... ({len(log_lines) - 15} more lines) ...\n", style="dim cyan")
                
                # Show real-time streaming status
                featrix_log_count = len(self.find_all_featrix_log_files())
                if featrix_log_count > 0:
                    content.append(f"\n🔄 Streaming from {featrix_log_count} Featrix log files", style="dim green")
                else:
                    content.append("\n📍 No Featrix log files found", style="dim yellow")
                
                # Controls help
                content.append("\n🔧 Controls: ←/→ or click to navigate panels • R refresh\n", style="dim cyan")
                
                return content

        class SessionsWidget(Static):
            """Widget for displaying sessions."""
            
            def __init__(self, dashboard):
                super().__init__()
                self.dashboard = dashboard
                self.border_title = "Sessions"
                
            def on_click(self, event):
                """Handle mouse click to select this panel."""
                self.dashboard.current_panel = "sessions"
                self.dashboard.update_panel_styles()
                self.dashboard.refresh_widgets()
                event.stop()  # Prevent event bubbling
                
            def render(self):
                sessions = self.dashboard.get_all_sessions()
                if not sessions:
                    self.border_title = "Sessions"
                    content = Text()
                    content.append("\n", style="white")
                    content.append("   📭 No active sessions\n", style="dim yellow")
                    content.append("\n", style="white")
                    content.append("   💡 Use 'ffsh create-session' to start\n", style="dim cyan")
                    return content
                
                # Filter sessions by recent activity
                now = datetime.now(timezone.utc)
                filtered_sessions = []
                
                for session in sessions:
                    latest_activity = session.get("latest_activity")
                    if not latest_activity:
                        continue
                        
                    # Convert to UTC if needed
                    if latest_activity.tzinfo is None:
                        latest_activity = latest_activity.replace(tzinfo=timezone.utc)
                    elif latest_activity.tzinfo != timezone.utc:
                        latest_activity = latest_activity.astimezone(timezone.utc)
                    
                    time_diff = now - latest_activity
                    hours_ago = time_diff.total_seconds() / 3600
                    
                    # Only show sessions active within 24 hours
                    if hours_ago <= 24:
                        session["hours_ago"] = hours_ago
                        filtered_sessions.append(session)
                
                if not filtered_sessions:
                    self.border_title = "Sessions"
                    content = Text()
                    content.append("\n", style="white")
                    content.append("   📭 No recent activity (24h)\n", style="dim yellow")
                    content.append("\n", style="white")
                    content.append("   💡 Use 'ffsh recent --hours 48' for older\n", style="dim cyan")
                    return content
                
                # Sort by activity time (most recent first)
                filtered_sessions.sort(key=lambda s: s["hours_ago"])
                
                # Get terminal size for proper sizing
                try:
                    console_size = self.app.console.size
                    available_height = console_size.height - 10  # Account for headers/footers
                except:
                    available_height = 15  # Fallback
                
                # Show what fits, prioritizing recent activity
                max_sessions = max(3, available_height // 4)  # Each session takes ~4 lines
                display_sessions = filtered_sessions[:max_sessions]
                
                content = Text()
                content.append("\n", style="white")
                
                for i, session in enumerate(display_sessions):
                    # Selection indicator
                    is_selected = (i == self.dashboard.selected_session_index and 
                                 self.dashboard.current_panel == "sessions")
                    
                    # Session info
                    session_id = session["session_id"]
                    short_id = session_id[:20] if len(session_id) > 20 else session_id
                    
                    # Activity indicators
                    running_jobs = session["running_jobs"]
                    failed_jobs = session["failed_jobs"] 
                    total_jobs = session["total_jobs"]
                    
                    # Status icon and style
                    if running_jobs > 0:
                        status_icon = "🔥"
                        status_style = "bold green"
                    elif failed_jobs > 0:
                        status_icon = "❌"
                        status_style = "bold red"
                    else:
                        status_icon = "✅"
                        status_style = "bold blue"
                    
                    # Time display
                    hours_ago = session["hours_ago"]
                    if hours_ago < 1:
                        time_str = f"{int(hours_ago * 60)}m ago"
                    elif hours_ago < 12:
                        time_str = f"{hours_ago:.1f}h ago"
                    else:
                        time_str = f"{hours_ago:.0f}h ago"
                    
                    # Main session line - clean and simple
                    selector = "▶ " if is_selected else "  "
                    main_line = f"{selector}{status_icon} {short_id}\n"
                    
                    if is_selected:
                        content.append(main_line, style="reverse white")
                    else:
                        content.append(main_line, style=status_style)
                    
                    # Simple summary line - only show problems or key info
                    summary_parts = []
                    if failed_jobs > 0:
                        summary_parts.append(f"[{failed_jobs} failure{'s' if failed_jobs != 1 else ''}]")
                    elif running_jobs > 0:
                        summary_parts.append(f"[{running_jobs} running]")
                    else:
                        summary_parts.append(f"[{total_jobs} job{'s' if total_jobs != 1 else ''}]")
                    
                    summary_parts.append(time_str)
                    summary_line = f"    {' • '.join(summary_parts)}\n"
                    content.append(summary_line, style="dim white")
                    
                    # Add spacing between sessions
                    content.append("\n", style="white")
                
                # Update stored sessions for dashboard
                self.dashboard.filtered_sessions = display_sessions
                
                # Clean summary line
                total_filtered = len(filtered_sessions)
                content.append("\n", style="white")
                if total_filtered > len(display_sessions):
                    remaining = total_filtered - len(display_sessions)
                    content.append(f"   ... and {remaining} more sessions\n", style="dim cyan")
                else:
                    content.append(f"   {len(display_sessions)} active sessions\n", style="dim cyan")
                
                self.border_title = "Sessions"
                if self.dashboard.current_panel == "sessions":
                    self.border_title = "► Sessions [ACTIVE]"
                
                return content
        
        class JobsWidget(Static):
            """Widget for displaying jobs in selected session."""
            
            def __init__(self, dashboard):
                super().__init__()
                self.dashboard = dashboard
                self.border_title = "Jobs"
                
                # Initialize Redis client for live progress updates
                try:
                    import redis
                    self.redis_client = redis.Redis(host='localhost', port=6379, db=1, decode_responses=True)
                    self.redis_available = True
                except Exception:
                    self.redis_client = None
                    self.redis_available = False
                
            def on_click(self, event):
                """Handle mouse click to select this panel."""
                self.dashboard.current_panel = "jobs"
                self.dashboard.update_panel_styles()
                self.dashboard.refresh_widgets()
                event.stop()  # Prevent event bubbling
                    
            def get_live_progress(self, job_id):
                """Get live progress data from Redis for running jobs."""
                if not self.redis_available:
                    return None
                    
                try:
                    import json
                    # Check both job progress and prediction progress keys
                    keys_to_check = [
                        f"job_progress:{job_id}",
                        f"prediction_progress:{job_id}",
                        f"training_progress:{job_id}"
                    ]
                    
                    for redis_key in keys_to_check:
                        progress_data = self.redis_client.get(redis_key)
                        if progress_data:
                            return json.loads(progress_data)
                    
                    return None
                except Exception:
                    return None
        
            def render(self):
                try:
                    filtered_sessions = getattr(self.dashboard, 'filtered_sessions', [])
                    if not filtered_sessions or self.dashboard.selected_session_index >= len(filtered_sessions):
                        self.border_title = "Jobs"
                        content = Text()
                        content.append("\n", style="white")
                        content.append("   Select a session to view jobs\n", style="dim cyan")
                        return content
                    
                    selected_session = filtered_sessions[self.dashboard.selected_session_index]
                    jobs = selected_session["jobs"]
                    session_id = selected_session["session_id"]
                    short_session = session_id[:12] + "..." if len(session_id) > 15 else session_id
                    
                    if not jobs:
                        self.border_title = f"Jobs"
                        content = Text()
                        content.append("\n", style="white")
                        content.append("   No jobs in this session\n", style="dim yellow")
                        return content
                    
                    # Sort jobs by creation time (most recent first)
                    jobs.sort(key=lambda j: j.get("created_at") or datetime.min.replace(tzinfo=timezone.utc), reverse=True)
                    
                    # Get terminal height
                    try:
                        console_height = self.app.size.height - 10
                        max_items = max(3, console_height // 3)  # Each job takes ~3 lines
                    except:
                        max_items = 8
                    
                    displayed_jobs = jobs[:max_items]
                    
                    content = Text()
                    content.append("\n", style="white")
                    
                    for i, job in enumerate(displayed_jobs):
                        status = job["status"]
                        is_selected = (i == self.dashboard.selected_job_index and 
                                     self.dashboard.current_panel == "jobs")
                        
                        # Status display
                        if status == JobStatus.RUNNING:
                            status_icon = "🔥"
                            status_style = "bold green"
                        elif status == JobStatus.READY:
                            status_icon = "⏳"
                            status_style = "yellow"
                        elif status == JobStatus.DONE:
                            status_icon = "✅"
                            status_style = "bold blue"
                        elif status == JobStatus.FAILED:
                            status_icon = "❌"
                            status_style = "bold red"
                        else:
                            status_icon = "❓"
                            status_style = "dim white"
                        
                        # Job info
                        job_type = job.get("job_type", "unknown")
                        short_id = get_short_job_id(job["job_id"])
                        
                        # Get live progress from Redis for running jobs
                        live_progress = None
                        if status == JobStatus.RUNNING:
                            live_progress = self.get_live_progress(job["job_id"])
                        
                        # Progress - prefer live Redis data over job file data
                        progress = job.get("progress")
                        if live_progress and 'percentage' in live_progress:
                            progress_str = f"{live_progress['percentage']:.0f}%"
                        elif live_progress and 'current' in live_progress and 'total' in live_progress:
                            current = live_progress['current']
                            total = live_progress['total']
                            if total > 0:
                                pct = (current / total) * 100
                                progress_str = f"{pct:.0f}%"
                            else:
                                progress_str = f"{progress:.0%}" if progress else ""
                        else:
                            progress_str = f"{progress:.0%}" if progress else ""
                        
                        # Get error count for this job
                        error_count = count_log_errors(job["job_id"])
                        
                        # Main job line - clean and simple
                        selector = "▶ " if is_selected else "  "
                        job_line = f"{selector}{status_icon} {job_type}_{short_id}"
                        if progress_str:
                            job_line += f" ({progress_str})"
                        job_line += "\n"
                        
                        if is_selected:
                            content.append(job_line, style="reverse white")
                        else:
                            content.append(job_line, style=status_style)
                        
                        # Time info line with error count and live metrics - much cleaner
                        started_at = job.get("started_at")
                        error_str = ""
                        if error_count > 0:
                            error_str = f" • {error_count} error{'s' if error_count != 1 else ''}"
                        
                        # Add live training metrics if available
                        live_metrics_str = ""
                        if live_progress and status == JobStatus.RUNNING:
                            metrics_parts = []
                            if 'epoch' in live_progress:
                                metrics_parts.append(f"Epoch {live_progress['epoch']}")
                            elif 'current_epoch' in live_progress:
                                metrics_parts.append(f"Epoch {live_progress['current_epoch']}")
                            
                            if 'current_loss' in live_progress:
                                loss = live_progress['current_loss']
                                if isinstance(loss, (int, float)) and loss > 0:
                                    metrics_parts.append(f"Loss: {loss:.4f}")
                            
                            if 'validation_loss' in live_progress:
                                val_loss = live_progress['validation_loss']
                                if isinstance(val_loss, (int, float)) and val_loss > 0:
                                    metrics_parts.append(f"Val: {val_loss:.4f}")
                            
                            if metrics_parts:
                                live_metrics_str = f" • {' | '.join(metrics_parts)}"
                            
                        if started_at and status == JobStatus.RUNNING:
                            now = datetime.now(timezone.utc)
                            if started_at.tzinfo is None:
                                started_at = started_at.replace(tzinfo=timezone.utc)
                            runtime_seconds = (now - started_at).total_seconds()
                            if runtime_seconds < 3600:
                                runtime_str = f"{int(runtime_seconds/60)}m runtime"
                            elif runtime_seconds < 86400:
                                runtime_str = f"{runtime_seconds/3600:.1f}h runtime"
                            else:
                                runtime_str = f"{runtime_seconds/86400:.1f}d runtime"
                            
                            time_line = f"    Running • {runtime_str}{live_metrics_str}{error_str}\n"
                        elif started_at:
                            time_str = started_at.strftime("%m/%d %H:%M")
                            time_line = f"    Started {time_str}{error_str}\n"
                        else:
                            time_line = f"    Queued{error_str}\n"
                        
                        # Apply error styling if there are errors
                        if error_count > 0:
                            content.append(time_line, style="yellow")
                        else:
                            content.append(time_line, style="dim white")
                        
                        # Add spacing between jobs
                        if i < len(displayed_jobs) - 1:
                            content.append("\n", style="white")
                    
                    # Clean summary
                    content.append("\n", style="white")
                    if len(jobs) > len(displayed_jobs):
                        remaining = len(jobs) - len(displayed_jobs)
                        content.append(f"   ... and {remaining} more jobs\n", style="dim cyan")
                    else:
                        content.append(f"   {len(displayed_jobs)} jobs total\n", style="dim cyan")
                    
                    if self.dashboard.current_panel == "jobs":
                        content.append("\n", style="white")
                        content.append("   💡 Press Enter to browse files\n", style="dim yellow")
                    
                    self.border_title = f"Jobs"
                    if self.dashboard.current_panel == "jobs":
                        self.border_title = f"► Jobs [ACTIVE]"
                    
                    return content
                    
                except Exception as e:
                    return f"Error loading jobs: {e}"
        
        class FilesWidget(Static):
            """Widget for displaying files in selected job."""
            
            def __init__(self, dashboard):
                super().__init__()
                self.dashboard = dashboard
                self.border_title = "Files"
                
            def on_click(self, event):
                """Handle mouse click to select this panel."""
                self.dashboard.current_panel = "files"
                self.dashboard.update_panel_styles()
                self.dashboard.refresh_widgets()
                event.stop()  # Prevent event bubbling
                
            def render(self):
                filtered_sessions = getattr(self.dashboard, 'filtered_sessions', [])
                if not filtered_sessions or self.dashboard.selected_session_index >= len(filtered_sessions):
                    self.border_title = "Files"
                    content = Text()
                    content.append("\n", style="white")
                    content.append("   Select a job to view files\n", style="dim cyan")
                    return content
                
                selected_session = filtered_sessions[self.dashboard.selected_session_index]
                jobs = selected_session["jobs"]
                
                if not jobs or self.dashboard.selected_job_index >= len(jobs):
                    self.border_title = "Files"
                    content = Text()
                    content.append("\n", style="white")
                    content.append("   Select a job to view files\n", style="dim cyan")
                    return content
                
                jobs.sort(key=lambda j: j.get("created_at") or datetime.min.replace(tzinfo=timezone.utc), reverse=True)
                selected_job = jobs[self.dashboard.selected_job_index]
                
                job_id = selected_job["job_id"]
                short_id = get_short_job_id(job_id)
                output_dir = get_job_output_directory(job_id)
                
                content = Text()
                
                if not os.path.exists(output_dir):
                    content = Text()
                    content.append("\n", style="white")
                    content.append(f"   📭 No output directory for job\n", style="dim yellow")
                    content.append(f"   {short_id}\n", style="yellow")
                    self.border_title = f"Files"
                    return content
                
                try:
                    # Get all files in the job directory recursively
                    files = []
                    for root, dirs, filenames in os.walk(output_dir):
                        for filename in filenames:
                            full_path = os.path.join(root, filename)
                            rel_path = os.path.relpath(full_path, output_dir)
                            file_size = os.path.getsize(full_path)
                            
                            # Skip binary files and very large files
                            if file_size > 100 * 1024 * 1024:  # 100MB limit
                                continue
                                
                            # Check if file is likely text
                            try:
                                with open(full_path, 'rb') as f:
                                    sample = f.read(1024)
                                    # Simple binary file detection
                                    if b'\x00' in sample:
                                        continue
                            except:
                                continue
                                
                            files.append({
                                'name': rel_path,
                                'full_path': full_path,
                                'size': file_size,
                                'is_log': 'log' in rel_path.lower()
                            })
                    
                    # Sort files: logs first, then by name
                    files.sort(key=lambda f: (not f['is_log'], f['name']))
                    
                    # Ensure stdout.log is first if it exists
                    stdout_log = None
                    other_files = []
                    for f in files:
                        if f['name'] == 'logs/stdout.log':
                            stdout_log = f
                        else:
                            other_files.append(f)
                    
                    sorted_files = []
                    if stdout_log:
                        sorted_files.append(stdout_log)
                    sorted_files.extend(other_files)
                    
                    if not sorted_files:
                        content = Text()
                        content.append("\n", style="white")
                        content.append(f"   📭 No readable files found\n", style="dim yellow")
                        self.border_title = f"Files"
                        return content
                    
                    content = Text()
                    content.append("\n", style="white")
                    
                    for i, file_info in enumerate(sorted_files):
                        is_selected = (i == self.dashboard.selected_file_index and 
                                     self.dashboard.current_panel == "files")
                        
                        # File info
                        name = file_info['name']
                        display_name = name if len(name) <= 30 else f"...{name[-27:]}"
                        
                        # File size
                        size = file_info['size']
                        if size < 1024:
                            size_str = f"{size}B"
                        elif size < 1024*1024:
                            size_str = f"{size//1024}KB"
                        else:
                            size_str = f"{size//(1024*1024)}MB"
                        
                        # File icon and style
                        if file_info['is_log']:
                            file_icon = "📝"
                            file_style = "cyan"
                        elif name.endswith('.json'):
                            file_icon = "📄"
                            file_style = "yellow"
                        elif name.endswith('.csv'):
                            file_icon = "📊"
                            file_style = "green"
                        elif name.endswith('.txt'):
                            file_icon = "📄"
                            file_style = "white"
                        else:
                            file_icon = "📄"
                            file_style = "dim white"
                        
                        # Clean file line
                        selector = "▶ " if is_selected else "  "
                        file_line = f"{selector}{file_icon} {display_name}\n"
                        
                        if is_selected:
                            content.append(file_line, style="reverse white")
                        else:
                            content.append(file_line, style=file_style)
                        
                        # Size info line
                        size_line = f"    {size_str}\n"
                        content.append(size_line, style="dim white")
                        
                        # Add spacing between files
                        if i < len(sorted_files) - 1:
                            content.append("\n", style="white")
                    
                    # Store files for access by dashboard
                    self.dashboard.current_files = sorted_files
                    
                    # Clean summary
                    content.append("\n", style="white")
                    content.append(f"   {len(sorted_files)} files total\n", style="dim cyan")
                    
                except Exception as e:
                    content = Text()
                    content.append("\n", style="white")
                    content.append(f"   ❌ Error reading directory\n", style="red")
                    self.dashboard.current_files = []
                
                self.border_title = "Files"
                if self.dashboard.current_panel == "files":
                    self.border_title = "► Files [ACTIVE]"
                
                return content
        
        class LogWidget(ScrollableContainer):
            """Widget for displaying log content with level histogram and filtering."""
            
            def __init__(self, dashboard):
                super().__init__()
                self.dashboard = dashboard
                
                # Initialize attributes that are needed before clicking
                self.border_title = "Content"
                self.can_focus = True
                
                # Log level filtering - by default show everything except DEBUG and TRACE
                self.filtered_levels = {
                    'TRACE': False,    # Usually way too verbose
                    'DEBUG': False,    # Usually too verbose
                    'INFO': True,      # Important operational info
                    'NOTICE': True,    # Similar to INFO
                    'WARN': True,      # Warnings
                    'WARNING': True,   # Alternative warning format
                    'ERROR': True,     # Errors
                    'FATAL': True,     # Fatal errors
                    'CRITICAL': True,  # Critical errors
                    'FAIL': True,      # Failures
                    'EXCEPTION': True, # Exceptions
                }
                
                # Line truncation and expansion state
                self.expanded_lines = set()  # Track which line numbers are expanded
                self.max_line_length = 200   # Truncate lines longer than this
                self.traceback_patterns = [
                    'Traceback (most recent call last):',
                    'Exception in thread',
                    'Caused by:',
                    'During handling of the above exception',
                    'The above exception was the direct cause'
                ]
                
            def on_click(self, event):
                """Handle mouse click to select this panel."""
                self.dashboard.current_panel = "logs"
                self.dashboard.update_panel_styles()
                self.dashboard.refresh_widgets()
                event.stop()  # Prevent event bubbling
                
            def detect_log_level(self, line):
                """Detect log level from a line."""
                line_upper = line.upper()
                
                # Common log level patterns
                patterns = [
                    ('TRACE', ['TRACE']),
                    ('DEBUG', ['DEBUG', 'DBG']),
                    ('INFO', ['INFO', 'INFORMATION']),
                    ('NOTICE', ['NOTICE']),
                    ('WARN', ['WARN', 'WARNING']),
                    ('ERROR', ['ERROR', 'ERR']),
                    ('FATAL', ['FATAL', 'CRITICAL']),
                    ('FAIL', ['FAIL', 'FAILED']),
                    ('EXCEPTION', ['EXCEPTION', 'TRACEBACK']),
                ]
                
                # Check for explicit level markers
                for level, keywords in patterns:
                    for keyword in keywords:
                        # Look for level at word boundaries
                        if (keyword + ':' in line_upper or 
                            keyword + ']' in line_upper or 
                            keyword + ')' in line_upper or
                            '[' + keyword + ']' in line_upper or
                            '(' + keyword + ')' in line_upper or
                            line_upper.startswith(keyword + ' ') or
                            ' ' + keyword + ' ' in line_upper):
                            return level
                
                # Special patterns for common formats
                if any(pattern in line_upper for pattern in ['TRACEBACK', 'MOST RECENT CALL']):
                    return 'EXCEPTION'
                    
                return 'OTHER'  # Unknown/unclassified
                
            def detect_traceback_start(self, line):
                """Detect if a line starts a traceback."""
                return any(pattern in line for pattern in self.traceback_patterns)
                
            def is_traceback_line(self, line):
                """Check if a line is part of a traceback."""
                traceback_indicators = [
                    'File "', 'line ', 'in ', '    at ', '  File',
                    'Error:', 'Exception:', 'Warning:', 'ValueError:',
                    'TypeError:', 'KeyError:', 'AttributeError:',
                    'RuntimeError:', 'ImportError:', 'ModuleNotFoundError:',
                    '^', '~~~', '  -->', 'raise ', 'assert '
                ]
                return any(indicator in line for indicator in traceback_indicators)
                
            def truncate_line(self, line, line_num=None):
                """Truncate long lines with expand functionality. Returns (display_text, is_truncated)."""
                line_clean = line.rstrip('\n\r')
                
                # If line is short enough, return as-is
                if len(line_clean) <= self.max_line_length:
                    return line_clean, False
                
                # Check if this line is expanded
                is_expanded = line_num in self.expanded_lines if line_num is not None else False
                
                if is_expanded:
                    # Show full line with collapse indicator
                    return f"{line_clean} <<< (press space to collapse)", False
                else:
                    # Show truncated line with expand indicator
                    truncated = line_clean[:self.max_line_length]
                    remaining = len(line_clean) - self.max_line_length
                    return f"{truncated}... (+{remaining} chars, press space to expand)", True
                    
            def extract_tracebacks(self, lines):
                """Extract and group tracebacks from log lines."""
                tracebacks = []
                current_traceback = []
                in_traceback = False
                
                for i, line in enumerate(lines):
                    line_str = line.strip() if hasattr(line, 'strip') else str(line).strip()
                    
                    # Check if this line starts a new traceback
                    if self.detect_traceback_start(line_str):
                        # Save previous traceback if it exists
                        if current_traceback:
                            tracebacks.append({
                                'start_line': len(tracebacks),
                                'lines': current_traceback.copy(),
                                'error_type': self.extract_error_type(current_traceback)
                            })
                        
                        # Start new traceback
                        current_traceback = [line_str]
                        in_traceback = True
                    
                    elif in_traceback:
                        # Check if we're still in traceback
                        if self.is_traceback_line(line_str) or not line_str.strip():
                            current_traceback.append(line_str)
                        else:
                            # End of traceback
                            if current_traceback:
                                tracebacks.append({
                                    'start_line': len(tracebacks),
                                    'lines': current_traceback.copy(),
                                    'error_type': self.extract_error_type(current_traceback)
                                })
                            current_traceback = []
                            in_traceback = False
                
                # Handle final traceback
                if current_traceback:
                    tracebacks.append({
                        'start_line': len(tracebacks),
                        'lines': current_traceback.copy(),
                        'error_type': self.extract_error_type(current_traceback)
                    })
                
                return tracebacks
                
            def extract_error_type(self, traceback_lines):
                """Extract the error type from traceback lines."""
                if not traceback_lines:
                    return "UnknownError"
                
                # Look for the last line which usually contains the error
                for line in reversed(traceback_lines):
                    line = line.strip()
                    if ':' in line and not line.startswith('File '):
                        # Extract error type before the colon
                        error_part = line.split(':')[0].strip()
                        if error_part and not error_part.startswith(' '):
                            return error_part
                
                return "UnknownError"
                
            def normalize_traceback(self, traceback):
                """Normalize traceback for grouping similar errors."""
                lines = traceback.get('lines', [])
                if not lines:
                    return ""
                
                # Extract just the call stack and error type, ignore specific values
                normalized_parts = []
                
                for line in lines:
                    line = line.strip()
                    
                    # Skip the "Traceback" header line
                    if 'Traceback (most recent call last):' in line:
                        continue
                    
                    # Normalize file paths and line numbers
                    if line.startswith('File "'):
                        # Extract just the function name and file basename
                        import re
                        match = re.search(r'File "([^"]+)", line \d+, in (.+)', line)
                        if match:
                            filepath, func_name = match.groups()
                            filename = filepath.split('/')[-1]  # Just the filename
                            normalized_parts.append(f"File {filename} in {func_name}")
                    
                    # Normalize error messages by removing specific values
                    elif ':' in line and not line.startswith('  '):
                        # This is likely the error message
                        error_type = line.split(':')[0].strip()
                        normalized_parts.append(error_type)
                        break  # Only keep the error type, not the specific message
                
                return ' -> '.join(normalized_parts)
                
            def analyze_log_levels(self, lines):
                """Analyze log lines and create histogram of levels."""
                level_counts = {}
                level_lines = {}  # Store sample lines for each level
                
                for line in lines:
                    level = self.detect_log_level(line)
                    level_counts[level] = level_counts.get(level, 0) + 1
                    
                    # Store a sample line for each level (first occurrence)
                    if level not in level_lines:
                        level_lines[level] = line.strip()[:60]  # First 60 chars
                
                return level_counts, level_lines
                
            def should_show_line(self, line):
                """Check if line should be shown based on current filters."""
                level = self.detect_log_level(line)
                
                # If it's a known level, check if it's filtered
                if level in self.filtered_levels:
                    return self.filtered_levels[level]
                
                # Handle OTHER level lines based on show_other flag
                if level == 'OTHER':
                    return getattr(self, 'show_other', True)  # Default to showing OTHER
                
                return True
                
            def get_level_style(self, level):
                """Get Rich style for different log levels."""
                styles = {
                    'TRACE': 'dim white',
                    'DEBUG': 'dim cyan',
                    'INFO': 'green',
                    'NOTICE': 'blue',
                    'WARN': 'yellow',
                    'WARNING': 'yellow',
                    'ERROR': 'bold red',
                    'FATAL': 'bold red on white',
                    'CRITICAL': 'bold red on white',
                    'FAIL': 'red',
                    'EXCEPTION': 'bold red',
                    'OTHER': 'white'
                }
                return styles.get(level, 'white')
                
            def parse_validation_loss(self, lines):
                """Parse validation loss values from log lines."""
                import re
                
                validation_losses = []
                training_metrics = []
                
                # Comprehensive patterns for validation loss and training metrics
                patterns = [
                    # Standard validation loss patterns
                    re.compile(r'.*validation.*loss.*?([0-9]+\.?[0-9]*(?:[eE][+-]?[0-9]+)?)', re.IGNORECASE),
                    re.compile(r'.*val.*loss.*?([0-9]+\.?[0-9]*(?:[eE][+-]?[0-9]+)?)', re.IGNORECASE),
                    re.compile(r'.*epoch.*?([0-9]+).*validation.*?([0-9]+\.?[0-9]*(?:[eE][+-]?[0-9]+)?)', re.IGNORECASE),
                    re.compile(r'.*epoch.*?([0-9]+).*val_loss.*?([0-9]+\.?[0-9]*(?:[eE][+-]?[0-9]+)?)', re.IGNORECASE),
                    
                    # Featrix-specific patterns
                    re.compile(r'.*epoch.*?([0-9]+).*val.*?([0-9]+\.?[0-9]*(?:[eE][+-]?[0-9]+)?)', re.IGNORECASE),
                    re.compile(r'.*training.*epoch.*?([0-9]+).*loss.*?([0-9]+\.?[0-9]*(?:[eE][+-]?[0-9]+)?)', re.IGNORECASE),
                    
                    # Progress-style patterns (e.g., "Epoch 5/50 - loss: 0.1234")
                    re.compile(r'epoch\s+([0-9]+).*?loss.*?([0-9]+\.?[0-9]*(?:[eE][+-]?[0-9]+)?)', re.IGNORECASE),
                    
                    # JSON-style logs
                    re.compile(r'.*"epoch":\s*([0-9]+).*"val_loss":\s*([0-9]+\.?[0-9]*(?:[eE][+-]?[0-9]+)?)', re.IGNORECASE),
                    re.compile(r'.*"validation_loss":\s*([0-9]+\.?[0-9]*(?:[eE][+-]?[0-9]+)?)', re.IGNORECASE),
                ]
                
                for line in lines:
                    line_str = line.strip() if hasattr(line, 'strip') else str(line).strip()
                    
                    # Skip debug/trace level lines for performance
                    if any(debug_word in line_str.lower() for debug_word in ['debug:', 'trace:']):
                        continue
                    
                    for pattern in patterns:
                        match = pattern.search(line_str)
                        if match:
                            try:
                                if len(match.groups()) == 1:
                                    # Just validation loss
                                    loss = float(match.group(1))
                                    validation_losses.append(loss)
                                elif len(match.groups()) == 2:
                                    # Epoch and validation loss
                                    epoch = int(match.group(1))
                                    loss = float(match.group(2))
                                    validation_losses.append({'epoch': epoch, 'loss': loss})
                                break
                            except (ValueError, IndexError):
                                continue
                
                return validation_losses
                
            def render_validation_loss_chart(self, losses):
                """Render a simple text-based validation loss chart."""
                content = Text()
                
                if not losses:
                    return content
                
                # Extract loss values and epochs for trend analysis
                loss_values = []
                epochs = []
                for loss in losses:
                    if isinstance(loss, dict):
                        loss_values.append(loss['loss'])
                        epochs.append(loss.get('epoch', len(loss_values)))
                    else:
                        loss_values.append(loss)
                        epochs.append(len(loss_values))
                
                if len(loss_values) < 2:
                    content.append(f"📈 Val Loss: {loss_values[0]:.4f}\n", style="cyan")
                    return content
                
                # Show recent trend (last 10 values)
                recent_losses = loss_values[-10:]
                recent_epochs = epochs[-10:] if epochs else list(range(len(recent_losses)))
                
                # Calculate trend and improvement
                if len(recent_losses) >= 2:
                    trend = "📉" if recent_losses[-1] < recent_losses[0] else "📈"
                    trend_color = "green" if recent_losses[-1] < recent_losses[0] else "red"
                    
                    # Calculate improvement percentage
                    if recent_losses[0] > 0:
                        improvement = ((recent_losses[0] - recent_losses[-1]) / recent_losses[0]) * 100
                        if improvement > 0:
                            trend_text = f"↓ {improvement:.1f}%"
                        else:
                            trend_text = f"↑ {abs(improvement):.1f}%"
                    else:
                        trend_text = "changes"
                else:
                    trend = "→"
                    trend_color = "yellow"
                    trend_text = "stable"
                
                # Current and best loss
                current_loss = recent_losses[-1]
                best_loss = min(loss_values)
                current_epoch = recent_epochs[-1] if recent_epochs else len(loss_values)
                
                # Training progress info
                content.append(f"{trend} Epoch {current_epoch}: {current_loss:.4f} (best: {best_loss:.4f}) {trend_text}\n", style=trend_color)
                
                # Simple text-based chart using Unicode characters
                if len(recent_losses) >= 3:
                    # Normalize to chart width
                    chart_width = min(20, len(recent_losses))
                    min_loss = min(recent_losses)
                    max_loss = max(recent_losses)
                    
                    if max_loss > min_loss:
                        # Create simple sparkline
                        chart_chars = "▁▂▃▄▅▆▇█"
                        chart_line = "📊 "
                        
                        for loss in recent_losses[-chart_width:]:
                            normalized = (loss - min_loss) / (max_loss - min_loss) if max_loss > min_loss else 0.5
                            char_idx = min(len(chart_chars) - 1, int(normalized * len(chart_chars)))
                            chart_line += chart_chars[char_idx]
                        
                        # Add scale info
                        chart_line += f" ({min_loss:.3f}-{max_loss:.3f})"
                        content.append(chart_line + "\n", style="dim cyan")
                
                return content
                
            def render_histogram(self, level_counts):
                """Render a compact histogram of log levels."""
                content = Text()
                
                # Sort levels by severity (most important first)
                severity_order = ['FATAL', 'CRITICAL', 'ERROR', 'EXCEPTION', 'FAIL', 'WARN', 'WARNING', 'NOTICE', 'INFO', 'DEBUG', 'TRACE', 'OTHER']
                sorted_levels = [level for level in severity_order if level in level_counts and level_counts[level] > 0]
                
                if not sorted_levels:
                    content.append("📊 No log levels detected", style="dim white")
                    return content
                
                # Build histogram line
                histogram_parts = []
                for level in sorted_levels:
                    count = level_counts[level]
                    
                    # Determine if level is shown based on filters
                    if level == 'OTHER':
                        is_filtered = getattr(self, 'show_other', True)
                    else:
                        is_filtered = self.filtered_levels.get(level, True)
                    
                    # Visual indicator
                    if not is_filtered:
                        indicator = "❌"  # Filtered out
                        style = "dim strike"
                    else:
                        indicator = "✓"   # Shown
                        style = self.get_level_style(level)
                    
                    histogram_parts.append((f"{indicator}{level}({count})", style))
                
                # Add histogram line
                content.append("📊 ", style="dim white")
                for i, (part, style) in enumerate(histogram_parts):
                    content.append(part, style=style)
                    if i < len(histogram_parts) - 1:
                        content.append(" ", style="white")
                
                return content
                
            def compose(self) -> ComposeResult:
                # Create a static widget to hold the log content
                self.log_content_widget = Static("Select a job to view content")
                yield self.log_content_widget
                
            def update_content(self):
                """Update the log content based on current selection."""
                filtered_sessions = getattr(self.dashboard, 'filtered_sessions', [])
                if not filtered_sessions or self.dashboard.selected_session_index >= len(filtered_sessions):
                    self.border_title = "Content"
                    self.log_content_widget.update("Select a job to view content")
                    return
                
                selected_session = filtered_sessions[self.dashboard.selected_session_index]
                jobs = selected_session["jobs"]
                
                if not jobs or self.dashboard.selected_job_index >= len(jobs):
                    self.border_title = "Content"
                    self.log_content_widget.update("Select a job to view content")
                    return
                
                jobs.sort(key=lambda j: j.get("created_at") or datetime.min.replace(tzinfo=timezone.utc), reverse=True)
                selected_job = jobs[self.dashboard.selected_job_index]
                
                job_id = selected_job["job_id"]
                short_id = get_short_job_id(job_id)
                
                content = Text()
                
                # Determine which file(s) to show
                output_dir = get_job_output_directory(job_id)
                
                if self.dashboard.current_panel == "files" and hasattr(self.dashboard, 'current_files') and self.dashboard.current_files:
                    # Show selected file from files panel
                    if self.dashboard.selected_file_index < len(self.dashboard.current_files):
                        selected_file = self.dashboard.current_files[self.dashboard.selected_file_index]
                        file_path = selected_file['full_path']
                        file_name = selected_file['name']
                        combined_logs = False  # No longer combining logs
                    else:
                        file_path = None
                        file_name = "invalid selection"
                        combined_logs = False
                else:
                    # Default to stdout only (stderr is now redirected to stdout)
                    stdout_path = os.path.join(output_dir, "logs", "stdout.log")
                    
                    if os.path.exists(stdout_path):
                        file_path = stdout_path
                        file_name = "logs/stdout.log (all logs)"
                        combined_logs = False
                    else:
                        file_path = None
                        file_name = "no log files found"
                        combined_logs = False
                
                if not file_path or not os.path.exists(file_path):
                    content.append(f"📭 File not found: {file_name}\n", style="yellow")
                    content.append(f"📁 Expected: {file_path}\n", style="dim white")
                    self.border_title = f"Content ({short_id})"
                    self.log_content_widget.update(content)
                    return
                
                try:
                    # Get file info
                    stat_info = os.stat(file_path)
                    file_size = stat_info.st_size
                    if file_size < 1024:
                        size_str = f"{file_size}B"
                    elif file_size < 1024*1024:
                        size_str = f"{file_size//1024}KB"
                    else:
                        size_str = f"{file_size//(1024*1024)}MB"
                    
                    last_modified = datetime.fromtimestamp(stat_info.st_mtime).strftime("%H:%M:%S")
                    
                    # Read file lines efficiently (especially for large log files)
                    if file_name.endswith('.log') or 'log' in file_name.lower():
                        # Use efficient reading for log files
                        all_lines = read_log_efficiently(file_path, max_lines=5000)
                    else:
                        # For small non-log files, read normally  
                        with open(file_path, 'r', encoding='utf-8', errors='replace') as f:
                            raw_lines = f.readlines()
                        all_lines = [line.rstrip('\n\r') for line in raw_lines]
                    
                    # For log files, do enhanced analysis with histogram (optimized)
                    if file_name.endswith('.log') or 'log' in file_name.lower():
                        # Analyze log levels only on sample for large files for performance
                        sample_lines = all_lines[-2000:] if len(all_lines) > 2000 else all_lines
                        level_counts, level_lines = self.analyze_log_levels(sample_lines)
                        
                        # Header with file info
                        content.append(f"📊 {size_str} | {len(all_lines)} lines | Modified: {last_modified}\n", style="dim white")
                        content.append(f"📁 {file_path}\n", style="dim cyan")
                        content.append("🔄 All logs (stdout + stderr) sorted chronologically by timestamp\n", style="dim green")
                        
                        # Histogram display
                        histogram_content = self.render_histogram(level_counts)
                        content.append(histogram_content)
                        content.append("\n", style="white")
                        
                        # Check for validation loss data (especially for training jobs)
                        validation_losses = self.parse_validation_loss(all_lines)
                        if validation_losses:
                            val_chart = self.render_validation_loss_chart(validation_losses)
                            content.append(val_chart)
                        
                        # Filter help
                        content.append("🔧 Filters: T=trace D=debug I=info W=warn U=error F=fatal X=fail Z=exception O=other • A=all S=errors only\n", style="dim cyan")
                        content.append("📋 Export: E = export to file • C = copy to clipboard (works over SSH)\n", style="dim cyan")
                        content.append("🔧 Clipboard: Shift+C = test clipboard methods\n", style="dim cyan")
                        content.append("─" * 70 + "\n", style="dim white")
                        
                        # Filter ALL lines and display them (ScrollView handles the actual viewport)
                        filtered_lines = []
                        for line in all_lines:
                            if self.should_show_line(line):
                                filtered_lines.append(line)
                        
                        if filtered_lines:
                            # Extract and display tracebacks summary
                            tracebacks = self.extract_tracebacks(filtered_lines)
                            if tracebacks:
                                content.append(f"🚨 TRACEBACKS FOUND ({len(tracebacks)}):\n", style="bold red")
                                
                                # Group tracebacks by normalized signature
                                traceback_groups = {}
                                for tb in tracebacks:
                                    normalized = self.normalize_traceback(tb)
                                    if normalized not in traceback_groups:
                                        traceback_groups[normalized] = {'count': 0, 'example': tb}
                                    traceback_groups[normalized]['count'] += 1
                                
                                # Show grouped tracebacks
                                for normalized, group in sorted(traceback_groups.items(), key=lambda x: x[1]['count'], reverse=True):
                                    count = group['count']
                                    error_type = group['example']['error_type']
                                    count_str = f"({count}x)" if count > 1 else ""
                                    content.append(f"  🔴 {error_type} {count_str}: {normalized}\n", style="red")
                                
                                content.append("\n", style="white")
                            
                            content.append("📜 FULL LOG (Use ↑↓ to scroll, PgUp/PgDn, Home/End, Space to expand long lines):\n", style="bold cyan")
                            for line_num, line in enumerate(filtered_lines):
                                # Apply line truncation
                                display_line, is_truncated = self.truncate_line(line, line_num)
                                level = self.detect_log_level(line)
                                style = self.get_level_style(level)
                                
                                # Highlight traceback lines
                                if self.detect_traceback_start(line) or self.is_traceback_line(line):
                                    style = "bold red"
                                
                                # Apply dimmer style to truncation indicators
                                if is_truncated:
                                    # Split the line to style the truncation part differently
                                    if "... (+" in display_line:
                                        main_part, truncation_part = display_line.split("... (+", 1)
                                        content.append(main_part, style=style)
                                        content.append(f"... (+{truncation_part}\n", style="dim yellow")
                                    else:
                                        content.append(display_line + "\n", style=style)
                                else:
                                    content.append(display_line + "\n", style=style)
                            
                            # Show filtering stats
                            filtered_out = len(all_lines) - len(filtered_lines)
                            content.append(f"\n{len(filtered_lines)} lines shown • {filtered_out} filtered out • Total: {len(all_lines)} lines", style="dim white")
                        else:
                            content.append("📜 No lines match current filters\n", style="yellow")
                            content.append("Try adjusting filters: T=trace D=debug I=info W=warn U=error A=all S=errors\n", style="dim cyan")
                        
                        # Add traditional error analysis for completeness
                        analyzer = get_log_analyzer()
                        analysis = analyzer.analyze_log_file(file_path, max_errors=3)
                        
                        errors = analysis.get('errors', [])
                        if errors and self.filtered_levels.get('ERROR', True):
                            content.append("\n🚨 RECENT ERRORS:\n", style="bold red")
                            for error in errors[-3:]:  # Last 3 errors
                                error_type = error.get('type', 'ERROR')
                                error_line = error.get('line', '').strip()
                                content.append(f"  • {error_type}: {error_line}\n", style="red")
                    
                    else:
                        # Non-log files: use original display logic with level detection
                        try:
                            console_size = self.app.console.size
                            available_height = console_size.height - 15
                            max_display_lines = max(50, available_height)
                        except:
                            max_display_lines = 100
                        
                        display_lines = all_lines[-max_display_lines:] if len(all_lines) > max_display_lines else all_lines
                        total_lines = len(all_lines)
                        
                        # Header
                        content.append(f"📊 {size_str} | {total_lines} lines | Modified: {last_modified}\n", style="dim white")
                        content.append("─" * 70 + "\n", style="dim white")
                        
                        # Content with syntax highlighting based on file type
                        if file_name.endswith('.json'):
                            # JSON syntax highlighting
                            for line in display_lines:
                                line = line.rstrip('\n\r')
                                if '{' in line or '}' in line or '[' in line or ']' in line:
                                    content.append(line + "\n", style="cyan")
                                elif '"' in line and ':' in line:
                                    content.append(line + "\n", style="yellow")
                                else:
                                    content.append(line + "\n", style="white")
                        elif file_name.endswith('.csv'):
                            # CSV highlighting
                            for i, line in enumerate(display_lines):
                                line = line.rstrip('\n\r')
                                if i == 0:  # Header
                                    content.append(line + "\n", style="bold cyan")
                                else:
                                    content.append(line + "\n", style="white")
                        else:
                            # Plain text with potential log level detection
                            for line in display_lines:
                                line = line.rstrip('\n\r')
                                level = self.detect_log_level(line)
                                style = self.get_level_style(level) if level != 'OTHER' else 'white'
                                content.append(line + "\n", style=style)
                        
                        if len(all_lines) > max_display_lines:
                            skipped_lines = len(all_lines) - max_display_lines
                            content.append(f"\n... showing last {max_display_lines} of {total_lines} lines ({skipped_lines} earlier lines hidden) ...", style="dim white")
                        
                except Exception as e:
                    content.append(f"❌ Error reading file: {str(e)}\n", style="red")
                
                display_name = file_name if len(file_name) <= 20 else f"...{file_name[-17:]}"
                self.border_title = f"{display_name} ({short_id}) - {size_str}"
                self.log_content_widget.update(content)
        
        class FeatrixOutputHandler(FileSystemEventHandler):
            """File system event handler for featrix_output directory changes."""
            
            def __init__(self, app):
                super().__init__()
                self.app = app
                self.last_refresh = time.time()
                self.refresh_throttle = 1.0  # Increased back to 1.0 seconds for better performance
                
            def should_trigger_refresh(self, event):
                """Determine if this file change should trigger a refresh."""
                if event.is_directory:
                    return False
                    
                # Only care about certain file types
                relevant_extensions = ['.log', '.json', '.txt', '.progress', '.status']
                relevant_names = ['stdout.log', 'progress.json', 'api_server.log']
                
                file_path = event.src_path
                filename = os.path.basename(file_path)
                
                # Check if it's a relevant file
                is_relevant = (
                    any(file_path.endswith(ext) for ext in relevant_extensions) or
                    filename in relevant_names or
                    'log' in filename.lower() or
                    'progress' in filename.lower() or
                    'status' in filename.lower() or
                    'api_server' in filename.lower()  # Watch API server logs specifically
                )
                
                return is_relevant
                
            def trigger_refresh(self):
                """Trigger a refresh if enough time has passed."""
                now = time.time()
                # Dynamic throttle: optimized for performance
                current_throttle = 0.8 if getattr(self.app, 'has_running_jobs', False) else self.refresh_throttle
                
                if now - self.last_refresh >= current_throttle:
                    self.last_refresh = now
                    try:
                        # Update subtitle with last update time
                        time_str = time.strftime("%H:%M:%S", time.localtime(now))
                        output_dir_name = _get_base_output_dir().name
                        mode_str = "🔥 Live mode" if getattr(self.app, 'has_running_jobs', False) else "💤 Idle mode"
                        new_subtitle = f"Norton Commander Interface • 👁️ Monitoring {output_dir_name}/ • {mode_str} • ⚡ Updated {time_str}"
                        
                        # Use call_from_thread to safely update from background thread
                        self.app.call_from_thread(self.app.set_subtitle_and_refresh, new_subtitle)
                    except Exception:
                        # App might be shutting down
                        pass
                
            def on_modified(self, event):
                """Handle file modification events."""
                if self.should_trigger_refresh(event):
                    self.trigger_refresh()
                    
            def on_created(self, event):
                """Handle file creation events."""
                if self.should_trigger_refresh(event):
                    self.trigger_refresh()
        
        class NortonCommanderApp(App):
            """Norton Commander-style dashboard app with live file monitoring."""
            
            TITLE = "Featrix Sphere Cluster Monitor"
            SUB_TITLE = "Norton Commander Interface"
            
            BINDINGS = [
                Binding("j,down", "nav_down", "Down", show=False),
                Binding("k,up", "nav_up", "Up", show=False),
                Binding("h,left", "nav_left", "Left Panel", show=False),
                Binding("l,right", "nav_right", "Right Panel", show=False),
                Binding("tab", "nav_right", "Switch/Click Panel", show=True),
                Binding("enter", "select", "Enter/Select", show=True),
                Binding("escape", "back", "Back/Reset", show=True),
                Binding("1,2,3,4,5,6,7,8,9", "quick_select", "Quick Select", show=False),
                Binding("q", "quit", "Quit", show=True),
                Binding("r", "refresh", "Refresh", show=True),
                Binding("shift+w", "toggle_watch", "Toggle Watch", show=True),
                Binding("e", "export_log", "Export Log", show=True),
                Binding("c", "copy_log", "Copy Log", show=True),
                Binding("shift+c", "test_clipboard", "Test Clipboard", show=False),
                # Log navigation bindings
                Binding("pageup,ctrl+u", "log_page_up", "Page Up", show=False),
                Binding("pagedown,ctrl+d", "log_page_down", "Page Down", show=False),
                Binding("home,g", "log_home", "Start of File", show=False),
                Binding("end,shift+g", "log_end", "End of File", show=False),
                # Log filter bindings (mnemonic letters, avoiding conflicts)
                Binding("t", "filter_trace", "T:TRACE", show=False),
                Binding("d", "filter_debug", "D:DEBUG", show=False),
                Binding("i", "filter_info", "I:INFO", show=False),
                Binding("w", "filter_warn", "W:WARN", show=False),
                Binding("u", "filter_error", "U:ERROR", show=False),
                Binding("f", "filter_fatal", "F:FATAL", show=False),
                Binding("x", "filter_fail", "X:FAIL", show=False),
                Binding("z", "filter_exception", "Z:EXCEPTION", show=False),
                Binding("o", "filter_other", "O:OTHER", show=False),
                Binding("a", "filter_show_all", "A:Show All", show=False),
                Binding("s", "filter_errors_only", "S:Errors Only", show=False),
                Binding("space", "toggle_line_expansion", "Toggle Line", show=False),
            ]
            
            CSS = """
            ServerStatusWidget {
                border: solid $primary;
                width: 1fr;
                min-width: 55;
                max-width: 65;
                height: 100%;
                margin: 1;
            }
            
            SessionsWidget {
                border: solid $primary;
                width: 1fr;
                min-width: 30;
                max-width: 40;
                height: 100%;
                margin: 1;
            }
            
            JobsWidget {
                border: solid $primary;
                width: 1fr; 
                min-width: 35;
                max-width: 45;
                height: 100%;
                margin: 1;
            }
            
            FilesWidget {
                border: solid $primary;
                width: 1fr;
                min-width: 30;
                max-width: 40;
                height: 100%;
                margin: 1;
                display: none;
            }
            
            FilesWidget.show {
                display: block;
            }
            
            JobsWidget.hide {
                display: none;
            }
            
            LogWidget {
                border: solid $primary;
                width: 3fr;
                height: 100%;
                margin: 1;
            }
            
            .selected-panel {
                border: solid $accent;
                border-title-color: $accent;
                border-title-style: bold;
            }
            
            .header-text {
                color: $text;
                text-style: bold;
            }
            
            SystemStatusBar {
                background: $primary-background;
                color: $text;
                dock: bottom;
                height: 1;
                text-align: center;
            }
            """
            
            def __init__(self):
                super().__init__()
                self.selected_session_index = 0
                self.selected_job_index = 0
                self.selected_file_index = 0
                self.current_panel = "server"  # "server", "sessions", "jobs", or "files"
                self.current_files = []
                
                # File system monitoring
                self.observer = None
                self.file_handler = None
                
                # Live refresh timer for running jobs
                self.auto_refresh_timer = None
                self.has_running_jobs = False
                
                # Create widgets
                self.server_status_widget = ServerStatusWidget(self)
                self.sessions_widget = SessionsWidget(self)
                self.jobs_widget = JobsWidget(self)
                self.files_widget = FilesWidget(self)
                self.log_widget = LogWidget(self)
                self.status_bar = SystemStatusBar()
                
            def check_for_running_jobs(self):
                """Check if there are any running jobs and update refresh frequency."""
                try:
                    all_jobs = _get_all_jobs_with_details()
                    running_jobs = [job for job in all_jobs if job.get("status") == JobStatus.RUNNING]
                    
                    new_has_running_jobs = len(running_jobs) > 0
                    
                    # If running job status changed, update refresh timer
                    if new_has_running_jobs != self.has_running_jobs:
                        self.has_running_jobs = new_has_running_jobs
                        self.setup_auto_refresh()
                        
                    return len(running_jobs)
                except Exception:
                    return 0
                    
            def setup_auto_refresh(self):
                """Setup automatic refresh timer based on running job status and API log activity."""
                # Stop existing timer
                if self.auto_refresh_timer:
                    self.auto_refresh_timer.stop()
                    self.auto_refresh_timer = None
                
                # Check if we're actively tailing API server logs
                api_log_active = (hasattr(self.server_status_widget, 'current_log_path') and 
                                self.server_status_widget.current_log_path is not None)
                
                # Set refresh interval based on activity (optimized for better performance)
                if self.has_running_jobs:
                    refresh_interval = 5.0  # 5 seconds for running jobs (was 3s)
                    status_msg = "🔥 Live mode (5s refresh)"
                elif api_log_active:
                    refresh_interval = 4.0  # 4 seconds for active log tailing (was 2s)
                    status_msg = "📜 Log tailing (4s refresh)"
                else:
                    refresh_interval = 15.0  # 15 seconds when idle (was 10s)
                    status_msg = "💤 Idle mode (15s refresh)"
                
                # Schedule next refresh
                self.auto_refresh_timer = self.set_timer(refresh_interval, self.auto_refresh_callback)
                
                # Update subtitle to show refresh mode
                output_dir_name = _get_base_output_dir().name if _get_base_output_dir().exists() else "output"
                self.sub_title = f"Norton Commander Interface • 👁️ Monitoring {output_dir_name}/ • {status_msg}"
                
            def auto_refresh_callback(self):
                """Called by timer to refresh the dashboard automatically."""
                try:
                    # Check for running jobs and refresh widgets
                    running_count = self.check_for_running_jobs()
                    
                    # Scan for tracebacks periodically (every 4th refresh cycle)
                    if not hasattr(self, '_refresh_count'):
                        self._refresh_count = 0
                    self._refresh_count += 1
                    
                    if self._refresh_count % 8 == 0:  # Every 8th refresh (40s for running jobs, 120s for idle) - less frequent for better performance
                        try:
                            traceback_monitor = get_traceback_monitor()
                            # Lightweight - just check if traceback counts have changed
                            current_summary = traceback_monitor.get_traceback_summary(1)
                            if current_summary:
                                total_tracebacks = sum(data['count'] for _, data in traceback_monitor.traceback_counts.items())
                                if total_tracebacks > 0:
                                    self.sub_title = f"Norton Commander Interface • 🚨 {total_tracebacks} total tracebacks tracked"
                        except:
                            pass  # Don't let traceback scanning break the refresh
                    
                    self.refresh_widgets()
                    
                    # Schedule next refresh
                    self.setup_auto_refresh()
                    
                except Exception as e:
                    # If refresh fails, just schedule next one
                    self.setup_auto_refresh()
            
            def compose(self) -> ComposeResult:
                yield Header()
                with Horizontal():
                    yield self.server_status_widget
                    yield self.sessions_widget
                    yield self.jobs_widget  
                    yield self.files_widget
                    yield self.log_widget
                yield self.status_bar
            
            def on_mount(self):
                """Called when app starts."""
                # Start file system monitoring
                self.setup_file_watcher()
                
                # Force initial refresh to populate filtered_sessions
                self.refresh_widgets()
                
                # Ensure we start with a valid session selected
                if hasattr(self, 'filtered_sessions') and self.filtered_sessions:
                    self.selected_session_index = 0
                    self.selected_job_index = 0
                    self.current_panel = "server"
                self.update_panel_styles()
                
                # Start auto-refresh timer
                self.check_for_running_jobs()  # Initial check
                self.setup_auto_refresh()
            
            def setup_file_watcher(self):
                """Set up file system watcher for featrix_output directory and API server logs."""
                try:
                    # Get the output directory to monitor
                    output_dir = _get_base_output_dir()
                    
                    self.file_handler = FeatrixOutputHandler(self)
                    self.observer = Observer()
                    
                    watch_paths = []
                    
                    # Monitor featrix_output directory
                    if output_dir.exists():
                        self.observer.schedule(self.file_handler, str(output_dir), recursive=True)
                        watch_paths.append(output_dir.name)
                    
                    # Monitor API server log directories
                    api_log_dirs = [
                        "/var/log/featrix/",
                        "/var/log/",
                        "/sphere/logs/",
                        "/sphere/",
                        "logs/"
                    ]
                    
                    for log_dir in api_log_dirs:
                        if os.path.exists(log_dir) and os.path.isdir(log_dir):
                            self.observer.schedule(self.file_handler, log_dir, recursive=False)
                            watch_paths.append(os.path.basename(log_dir.rstrip('/')))
                    
                    # Start monitoring
                    if watch_paths:
                        self.observer.start()
                        
                        # Add status indicator that monitoring is active
                        watch_summary = ", ".join(watch_paths)
                        self.sub_title = f"Norton Commander Interface • 👁️ Monitoring {watch_summary}"
                    else:
                        self.sub_title = f"Norton Commander Interface • ⚠️ No directories to monitor"
                        
                except Exception as e:
                    self.sub_title = f"Norton Commander Interface • ❌ Monitor failed: {str(e)[:30]}"
                    
            def cleanup_file_watcher(self):
                """Clean up file system watcher."""
                if self.observer:
                    try:
                        self.observer.stop()
                        self.observer.join(timeout=1.0)
                    except Exception:
                        pass
                    finally:
                        self.observer = None
                        self.file_handler = None
            
            def update_panel_styles(self):
                """Update panel borders to show which is selected."""
                # Reset all borders
                self.server_status_widget.remove_class("selected-panel")
                self.sessions_widget.remove_class("selected-panel")
                self.jobs_widget.remove_class("selected-panel")
                self.files_widget.remove_class("selected-panel")
                self.log_widget.remove_class("selected-panel")
                
                # Control widget visibility
                if self.current_panel == "files":
                    self.jobs_widget.add_class("hide")
                    self.files_widget.add_class("show")
                else:
                    self.jobs_widget.remove_class("hide")
                    self.files_widget.remove_class("show")
                
                # Highlight current panel
                if self.current_panel == "server":
                    self.server_status_widget.add_class("selected-panel")
                elif self.current_panel == "sessions":
                    self.sessions_widget.add_class("selected-panel")
                elif self.current_panel == "jobs":
                    self.jobs_widget.add_class("selected-panel")
                elif self.current_panel == "files":
                    self.files_widget.add_class("selected-panel")
                elif self.current_panel == "logs":
                    self.log_widget.add_class("selected-panel")
            
            def get_all_sessions(self):
                """Get all sessions for the dashboard."""
                try:
                    all_jobs = _get_all_jobs_with_details()
                    sessions = {}
                    
                    for job in all_jobs:
                        session_id = job.get("session_id")
                        if session_id:
                            if session_id not in sessions:
                                sessions[session_id] = {
                                    "session_id": session_id,
                                    "jobs": [],
                                    "total_jobs": 0,
                                    "running_jobs": 0,
                                    "failed_jobs": 0,
                                    "completed_jobs": 0,
                                    "latest_activity": None
                                }
                            
                            sessions[session_id]["jobs"].append(job)
                            sessions[session_id]["total_jobs"] += 1
                            
                            # Count by status
                            status = job.get("status")
                            if status == JobStatus.RUNNING:
                                sessions[session_id]["running_jobs"] += 1
                            elif status == JobStatus.FAILED:
                                sessions[session_id]["failed_jobs"] += 1  
                            elif status == JobStatus.DONE:
                                sessions[session_id]["completed_jobs"] += 1
                            
                            # Track latest activity
                            job_time = job.get("started_at") or job.get("created_at")
                            if job_time:
                                if sessions[session_id]["latest_activity"] is None or job_time > sessions[session_id]["latest_activity"]:
                                    sessions[session_id]["latest_activity"] = job_time
                    
                    # Sort sessions by latest activity
                    session_list = list(sessions.values())
                    session_list.sort(key=lambda s: s["latest_activity"] or datetime.min.replace(tzinfo=timezone.utc), reverse=True)
                    
                    return session_list
                    
                except Exception as e:
                    return []
            
            def refresh_widgets(self):
                """Refresh all widgets."""
                self.server_status_widget.refresh()
                self.sessions_widget.refresh()
                self.jobs_widget.refresh()
                self.files_widget.refresh()
                self.log_widget.update_content()
                self.status_bar.refresh()
                self.update_panel_styles()
                
            def set_subtitle_and_refresh(self, new_subtitle):
                """Set subtitle and refresh widgets (safe for threading)."""
                self.sub_title = new_subtitle
                self.refresh_widgets()
            
            def action_nav_up(self):
                """Navigate up in current panel."""
                if self.current_panel == "server":
                    # Server panel doesn't have navigable items, just refresh
                    self.refresh_widgets()
                elif self.current_panel == "sessions":
                    if hasattr(self, 'filtered_sessions') and self.filtered_sessions:
                        if self.selected_session_index > 0:
                            self.selected_session_index -= 1
                            self.selected_job_index = 0
                elif self.current_panel == "jobs":
                    if self.selected_job_index > 0:
                        self.selected_job_index -= 1
                elif self.current_panel == "files":
                    if self.selected_file_index > 0:
                        self.selected_file_index -= 1
                elif self.current_panel == "logs":
                    # Scroll up in log content
                    self.log_widget.scroll_up()
                    return  # Don't refresh widgets for scrolling
                self.refresh_widgets()
            
            def action_nav_down(self):
                """Navigate down in current panel."""
                if self.current_panel == "server":
                    # Server panel doesn't have navigable items, just refresh
                    self.refresh_widgets()
                elif self.current_panel == "sessions":
                    if hasattr(self, 'filtered_sessions') and self.filtered_sessions:
                        if self.selected_session_index < len(self.filtered_sessions) - 1:
                            self.selected_session_index += 1
                            self.selected_job_index = 0
                elif self.current_panel == "jobs":
                    if hasattr(self, 'filtered_sessions') and self.filtered_sessions and self.selected_session_index < len(self.filtered_sessions):
                        jobs = self.filtered_sessions[self.selected_session_index]["jobs"]
                        if jobs and self.selected_job_index < len(jobs) - 1:
                            self.selected_job_index += 1
                elif self.current_panel == "files":
                    if hasattr(self, 'current_files') and self.current_files and self.selected_file_index < len(self.current_files) - 1:
                        self.selected_file_index += 1
                elif self.current_panel == "logs":
                    # Scroll down in log content
                    self.log_widget.scroll_down()
                    return  # Don't refresh widgets for scrolling
                self.refresh_widgets()
            
            def action_nav_left(self):
                """Switch to left panel."""
                if self.current_panel == "logs":
                    self.current_panel = "files"
                    self.refresh_widgets()
                elif self.current_panel == "files":
                    self.current_panel = "jobs"
                    self.refresh_widgets()
                elif self.current_panel == "jobs":
                    self.current_panel = "sessions"
                    self.refresh_widgets()
                elif self.current_panel == "sessions":
                    self.current_panel = "server"
                    self.refresh_widgets()
            
            def action_nav_right(self):
                """Switch to right panel."""
                if self.current_panel == "server":
                    self.current_panel = "sessions"
                    self.refresh_widgets()
                elif self.current_panel == "sessions":
                    self.current_panel = "jobs"
                    self.refresh_widgets()
                elif self.current_panel == "jobs":
                    self.current_panel = "files"
                    self.refresh_widgets()
                elif self.current_panel == "files":
                    self.current_panel = "logs"
                    self.refresh_widgets()
            
            def action_select(self):
                """Enter/Select action - go into selected item."""
                if self.current_panel == "server":
                    # Switch to sessions panel
                    self.current_panel = "sessions"
                    self.refresh_widgets()
                elif self.current_panel == "sessions":
                    # Switch to jobs panel to show jobs in selected session
                    self.current_panel = "jobs"
                    self.refresh_widgets()
                elif self.current_panel == "jobs":
                    # Switch to files panel to show files in selected job
                    self.current_panel = "files"
                    self.selected_file_index = 0  # Reset file selection
                    self.refresh_widgets()
            
            def action_back(self):
                """Back/Reset action - go back in hierarchy."""
                if self.current_panel == "files":
                    # Go back to jobs panel
                    self.current_panel = "jobs"
                    self.refresh_widgets()
                elif self.current_panel == "jobs":
                    # Go back to sessions panel
                    self.current_panel = "sessions"
                    self.refresh_widgets()
                elif self.current_panel == "sessions":
                    # Go back to server panel
                    self.current_panel = "server"
                    self.refresh_widgets()
                else:
                    # Already at server, reset to first session
                    if hasattr(self, 'filtered_sessions') and self.filtered_sessions:
                        self.selected_session_index = 0
                        self.selected_job_index = 0
                        self.selected_file_index = 0
                    self.refresh_widgets()
            
            def action_refresh(self):
                """Refresh the dashboard."""
                # Force refresh of sessions first
                self.refresh_widgets()
                
                # Defensive logic - ensure valid selections after refresh
                if hasattr(self, 'filtered_sessions') and self.filtered_sessions:
                    # Ensure session index is valid
                    if self.selected_session_index >= len(self.filtered_sessions):
                        self.selected_session_index = 0
                        self.selected_job_index = 0
                        self.selected_file_index = 0
                        self.current_panel = "sessions"
                    
                    # Ensure job index is valid
                    if self.selected_session_index < len(self.filtered_sessions):
                        jobs = self.filtered_sessions[self.selected_session_index]["jobs"]
                        if self.selected_job_index >= len(jobs):
                            self.selected_job_index = 0
                            self.selected_file_index = 0
                            if self.current_panel == "files":
                                self.current_panel = "jobs"
                    
                    # Ensure file index is valid
                    if hasattr(self, 'current_files') and self.current_files:
                        if self.selected_file_index >= len(self.current_files):
                            self.selected_file_index = 0
                else:
                    # No sessions available
                    self.selected_session_index = 0
                    self.selected_job_index = 0
                    self.selected_file_index = 0
                    self.current_panel = "server"
                
                self.refresh_widgets()
            
            def key_1(self): self.quick_select(0)
            def key_2(self): self.quick_select(1)
            def key_3(self): self.quick_select(2)
            def key_4(self): self.quick_select(3)
            def key_5(self): self.quick_select(4)
            def key_6(self): self.quick_select(5)
            def key_7(self): self.quick_select(6)
            def key_8(self): self.quick_select(7)
            def key_9(self): self.quick_select(8)
            
            def quick_select(self, index):
                """Quick select by number."""
                if self.current_panel == "sessions":
                    if hasattr(self, 'filtered_sessions') and self.filtered_sessions and index < len(self.filtered_sessions):
                        self.selected_session_index = index
                        self.selected_job_index = 0
                elif self.current_panel == "jobs":
                    if hasattr(self, 'filtered_sessions') and self.filtered_sessions and self.selected_session_index < len(self.filtered_sessions):
                        jobs = self.filtered_sessions[self.selected_session_index]["jobs"]
                        if index < len(jobs):
                            self.selected_job_index = index
                elif self.current_panel == "files":
                    if hasattr(self, 'current_files') and self.current_files and index < len(self.current_files):
                        self.selected_file_index = index
                self.refresh_widgets()
                
            def action_toggle_watch(self):
                """Toggle file system monitoring on/off."""
                if self.observer and self.observer.is_alive():
                    # Turn off monitoring
                    self.cleanup_file_watcher()
                    self.sub_title = "Norton Commander Interface • 👁️ Monitoring OFF (Press 'w' to enable)"
                else:
                    # Turn on monitoring
                    self.setup_file_watcher()
                    
            def action_export_log(self):
                """Export current log content to a file."""
                if not hasattr(self, 'filtered_sessions') or not self.filtered_sessions:
                    return
                
                try:
                    # Get current log content
                    log_content = self.get_current_log_content()
                    if not log_content:
                        return
                    
                    # Generate export filename
                    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
                    filename = f"exported_log_{timestamp}.txt"
                    
                    # Write to file
                    with open(filename, 'w', encoding='utf-8') as f:
                        f.write(log_content)
                    
                    # Update subtitle to show export success
                    self.sub_title = f"Norton Commander Interface • ✅ Log exported to {filename}"
                    
                except Exception as e:
                    self.sub_title = f"Norton Commander Interface • ❌ Export failed: {str(e)[:40]}"
            
            def action_copy_log(self):
                """Copy current log content to clipboard."""
                if not hasattr(self, 'filtered_sessions') or not self.filtered_sessions:
                    return
                
                try:
                    # Get current log content
                    log_content = self.get_current_log_content()
                    if not log_content:
                        return
                    
                    # Try multiple clipboard methods
                    import subprocess
                    import base64
                    import os
                    
                    clipboard_success = False
                    
                    # Method 1: OSC 52 escape sequence (works over SSH)
                    try:
                        # OSC 52 allows sending clipboard data through terminal escape sequences
                        # This works over SSH connections with most modern terminal emulators
                        encoded_content = base64.b64encode(log_content.encode('utf-8')).decode('ascii')
                        
                        # OSC 52 sequence: \033]52;c;base64_data\007
                        # c = clipboard selection, base64_data = base64 encoded content
                        osc52_sequence = f"\033]52;c;{encoded_content}\007"
                        
                        # Write directly to terminal
                        import sys
                        sys.stdout.write(osc52_sequence)
                        sys.stdout.flush()
                        
                        clipboard_success = True
                        self.sub_title = f"Norton Commander Interface • ✅ Log copied to clipboard (OSC 52)"
                        
                    except Exception as e:
                        # Fall back to traditional methods
                        pass
                    
                    # Method 2: Traditional clipboard tools (local only)
                    if not clipboard_success:
                        try:
                            # macOS
                            subprocess.run(['pbcopy'], input=log_content, text=True, check=True)
                            self.sub_title = f"Norton Commander Interface • ✅ Log copied to clipboard (pbcopy)"
                            clipboard_success = True
                        except (subprocess.CalledProcessError, FileNotFoundError):
                            try:
                                # Linux
                                subprocess.run(['xclip', '-selection', 'clipboard'], input=log_content, text=True, check=True)
                                self.sub_title = f"Norton Commander Interface • ✅ Log copied to clipboard (xclip)"
                                clipboard_success = True
                            except (subprocess.CalledProcessError, FileNotFoundError):
                                pass
                    
                    # Method 3: SSH clipboard forwarding (if available)
                    if not clipboard_success and os.environ.get('SSH_TTY'):
                        try:
                            # Check if we're in an SSH session and try ssh-copy
                            if os.path.exists('/usr/bin/ssh-copy') or os.path.exists('/usr/local/bin/ssh-copy'):
                                subprocess.run(['ssh-copy'], input=log_content, text=True, check=True)
                                self.sub_title = f"Norton Commander Interface • ✅ Log copied to clipboard (ssh-copy)"
                                clipboard_success = True
                        except:
                            pass
                    
                    # Final fallback: save to file
                    if not clipboard_success:
                        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
                        filename = f"clipboard_log_{timestamp}.txt"
                        with open(filename, 'w', encoding='utf-8') as f:
                            f.write(log_content)
                        self.sub_title = f"Norton Commander Interface • ⚠️ Clipboard failed, saved to {filename}"
                    
                except Exception as e:
                    self.sub_title = f"Norton Commander Interface • ❌ Copy failed: {str(e)[:40]}"
            
            def action_test_clipboard(self):
                """Test clipboard functionality and show available methods."""
                import subprocess
                import os
                
                methods = []
                
                # Test OSC 52 capability
                methods.append("✅ OSC 52 (SSH-compatible)")
                
                # Test traditional clipboard tools
                try:
                    subprocess.run(['pbcopy'], input="test", text=True, check=True, capture_output=True)
                    methods.append("✅ pbcopy (macOS)")
                except (subprocess.CalledProcessError, FileNotFoundError):
                    methods.append("❌ pbcopy (not available)")
                
                try:
                    subprocess.run(['xclip', '-selection', 'clipboard'], input="test", text=True, check=True, capture_output=True)
                    methods.append("✅ xclip (Linux)")
                except (subprocess.CalledProcessError, FileNotFoundError):
                    methods.append("❌ xclip (not available)")
                
                # Check SSH environment
                if os.environ.get('SSH_TTY'):
                    methods.append("ℹ️  SSH session detected")
                    if os.path.exists('/usr/bin/ssh-copy') or os.path.exists('/usr/local/bin/ssh-copy'):
                        methods.append("✅ ssh-copy available")
                    else:
                        methods.append("❌ ssh-copy not found")
                else:
                    methods.append("ℹ️  Local session")
                
                # Show results
                method_str = " • ".join(methods[:3])  # Show first 3 methods in subtitle
                self.sub_title = f"Norton Commander Interface • Clipboard: {method_str}"
            
            def get_current_log_content(self):
                """Get the current log content as plain text."""
                if not hasattr(self, 'filtered_sessions') or not self.filtered_sessions:
                    return ""
                
                if self.selected_session_index >= len(self.filtered_sessions):
                    return ""
                
                selected_session = self.filtered_sessions[self.selected_session_index]
                jobs = selected_session["jobs"]
                
                if not jobs or self.selected_job_index >= len(jobs):
                    return ""
                
                jobs.sort(key=lambda j: j.get("created_at") or datetime.min.replace(tzinfo=timezone.utc), reverse=True)
                selected_job = jobs[self.selected_job_index]
                job_id = selected_job["job_id"]
                
                # Determine which file to export
                if self.current_panel == "files" and hasattr(self, 'current_files') and self.current_files:
                    # Export selected file from files panel
                    if self.selected_file_index < len(self.current_files):
                        selected_file = self.current_files[self.selected_file_index]
                        file_path = selected_file['full_path']
                        file_name = selected_file['name']
                    else:
                        return ""
                else:
                    # Default to stdout.log
                    output_dir = get_job_output_directory(job_id)
                    file_path = os.path.join(output_dir, "logs", "stdout.log")
                    file_name = "logs/stdout.log"
                
                if not os.path.exists(file_path):
                    return f"File not found: {file_name}\nExpected: {file_path}"
                
                try:
                    # Read the file
                    with open(file_path, 'r', encoding='utf-8', errors='replace') as f:
                        all_lines = f.readlines()
                    
                    # Apply the same filtering as the display
                    filtered_lines = []
                    for line in all_lines:
                        if self.log_widget.should_show_line(line):
                            filtered_lines.append(line.rstrip('\n\r'))
                    
                    # Create export content with metadata
                    content_lines = []
                    content_lines.append(f"# Exported from Featrix Sphere Dashboard")
                    content_lines.append(f"# Job ID: {job_id}")
                    content_lines.append(f"# File: {file_name}")
                    content_lines.append(f"# Export time: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
                    content_lines.append(f"# Total lines: {len(all_lines)}")
                    content_lines.append(f"# Filtered lines: {len(filtered_lines)}")
                    content_lines.append("")
                    content_lines.extend(filtered_lines)
                    
                    return "\n".join(content_lines)
                    
                except Exception as e:
                    return f"Error reading file: {str(e)}"
            
            # Log navigation actions
            def action_log_page_up(self):
                """Page up in log content."""
                if self.current_panel == "logs":
                    self.log_widget.scroll_page_up()
                    
            def action_log_page_down(self):
                """Page down in log content."""
                if self.current_panel == "logs":
                    self.log_widget.scroll_page_down()
                    
            def action_log_home(self):
                """Jump to start of log file."""
                if self.current_panel == "logs":
                    self.log_widget.scroll_home()
                    
            def action_log_end(self):
                """Jump to end of log file."""
                if self.current_panel == "logs":
                    self.log_widget.scroll_end()
            
            # Log filter actions
            def action_filter_trace(self):
                """Toggle TRACE log level visibility."""
                self.log_widget.filtered_levels['TRACE'] = not self.log_widget.filtered_levels['TRACE']
                self.log_widget.update_content()
                
            def action_filter_debug(self):
                """Toggle DEBUG log level visibility."""
                self.log_widget.filtered_levels['DEBUG'] = not self.log_widget.filtered_levels['DEBUG']
                self.log_widget.update_content()
                
            def action_filter_info(self):
                """Toggle INFO log level visibility."""
                self.log_widget.filtered_levels['INFO'] = not self.log_widget.filtered_levels['INFO']
                self.log_widget.update_content()
                
            def action_filter_warn(self):
                """Toggle WARN/WARNING log level visibility."""
                current_state = self.log_widget.filtered_levels['WARN']
                self.log_widget.filtered_levels['WARN'] = not current_state
                self.log_widget.filtered_levels['WARNING'] = not current_state
                self.log_widget.update_content()
                
            def action_filter_error(self):
                """Toggle ERROR log level visibility."""
                self.log_widget.filtered_levels['ERROR'] = not self.log_widget.filtered_levels['ERROR']
                self.log_widget.update_content()
                
            def action_filter_fatal(self):
                """Toggle FATAL/CRITICAL log level visibility."""
                current_state = self.log_widget.filtered_levels['FATAL']
                self.log_widget.filtered_levels['FATAL'] = not current_state
                self.log_widget.filtered_levels['CRITICAL'] = not current_state
                self.log_widget.update_content()
                
            def action_filter_fail(self):
                """Toggle FAIL log level visibility."""
                self.log_widget.filtered_levels['FAIL'] = not self.log_widget.filtered_levels['FAIL']
                self.log_widget.update_content()
                
            def action_filter_exception(self):
                """Toggle EXCEPTION log level visibility."""
                self.log_widget.filtered_levels['EXCEPTION'] = not self.log_widget.filtered_levels['EXCEPTION']
                self.log_widget.update_content()
                
            def action_filter_other(self):
                """Toggle OTHER log level visibility."""
                # Toggle the display of lines that don't match known log levels
                # This requires special handling since OTHER isn't in filtered_levels
                # We'll track this with a special flag
                if not hasattr(self.log_widget, 'show_other'):
                    self.log_widget.show_other = True
                self.log_widget.show_other = not self.log_widget.show_other
                self.log_widget.update_content()
                
            def action_filter_show_all(self):
                """Show all log levels."""
                for level in self.log_widget.filtered_levels:
                    self.log_widget.filtered_levels[level] = True
                if hasattr(self.log_widget, 'show_other'):
                    self.log_widget.show_other = True
                self.log_widget.update_content()
                
            def action_filter_errors_only(self):
                """Show only error-level logs (ERROR, FATAL, CRITICAL, FAIL, EXCEPTION)."""
                # Hide all levels first
                for level in self.log_widget.filtered_levels:
                    self.log_widget.filtered_levels[level] = False
                
                # Show only error levels
                error_levels = ['ERROR', 'FATAL', 'CRITICAL', 'FAIL', 'EXCEPTION']
                for level in error_levels:
                    if level in self.log_widget.filtered_levels:
                        self.log_widget.filtered_levels[level] = True
                
                # Hide OTHER level lines
                if hasattr(self.log_widget, 'show_other'):
                    self.log_widget.show_other = False
                self.log_widget.update_content()
                
            def action_toggle_line_expansion(self):
                """Toggle expansion of long lines in logs."""
                if self.current_panel != "logs":
                    return
                
                # For now, toggle all expanded lines (in future could be cursor-based)
                if self.log_widget.expanded_lines:
                    # Collapse all
                    self.log_widget.expanded_lines.clear()
                    self.sub_title = "Norton Commander Interface • 📝 Lines collapsed"
                else:
                    # Expand first 10 long lines as demo
                    filtered_sessions = getattr(self, 'filtered_sessions', [])
                    if filtered_sessions and self.selected_session_index < len(filtered_sessions):
                        selected_session = filtered_sessions[self.selected_session_index]
                        jobs = selected_session["jobs"]
                        if jobs and self.selected_job_index < len(jobs):
                            jobs.sort(key=lambda j: j.get("created_at") or datetime.min.replace(tzinfo=timezone.utc), reverse=True)
                            selected_job = jobs[self.selected_job_index]
                            job_id = selected_job["job_id"]
                            output_dir = get_job_output_directory(job_id)
                            file_path = os.path.join(output_dir, "logs", "stdout.log")
                            
                            if os.path.exists(file_path):
                                try:
                                    with open(file_path, 'r', encoding='utf-8', errors='replace') as f:
                                        lines = f.readlines()
                                    
                                    # Find first 10 long lines and expand them
                                    long_lines_found = 0
                                    for i, line in enumerate(lines):
                                        if len(line.rstrip()) > self.log_widget.max_line_length:
                                            self.log_widget.expanded_lines.add(i)
                                            long_lines_found += 1
                                            if long_lines_found >= 10:
                                                break
                                    
                                    self.sub_title = f"Norton Commander Interface • 📝 {long_lines_found} long lines expanded"
                                except:
                                    pass
                
                # Refresh to show changes
                self.log_widget.update_content()
                
            def on_unmount(self):
                """Called when app is shutting down."""
                # Clean up auto-refresh timer
                if self.auto_refresh_timer:
                    self.auto_refresh_timer.stop()
                    self.auto_refresh_timer = None
                    
                self.cleanup_file_watcher()
                # Cleanup log analyzer
                global _log_analyzer
                if _log_analyzer:
                    _log_analyzer.close()
        
        # Run the Textual app
        app = NortonCommanderApp()
        try:
            app.run()
        finally:
            # Ensure cleanup happens even if app crashes
            app.cleanup_file_watcher()
        
    except Exception as e:
        console.print(f"❌ Error in Textual dashboard: {e}")
        import traceback
        console.print(traceback.format_exc())
    
    console.print("👋 Exited Norton Commander dashboard")


@app.command()
def monitor(
    no_color: bool = typer.Option(False, "--no-color", help="Disable colored output")
):
    """
    Monitor for new log activity in real-time across all jobs.
    """
    console.print("👁️  Monitoring for new log activity...")
    console.print("💡 Use Ctrl+C to stop")
    console.print("=" * 60)
    
    base_dir = _get_base_output_dir()
    
    if not base_dir.exists():
        console.print("❌ No output directory found to monitor")
        return
    
    try:
        # For now, show recent activity and refresh
        while True:
            _show_recent_activity()
            console.print(f"\n[dim]🔄 Refreshing in 10 seconds... (Ctrl+C to stop)[/dim]")
            time.sleep(10)
            console.clear()
    except KeyboardInterrupt:
        console.print("\n👋 Stopped monitoring")


@app.command()
def running(no_color: bool = typer.Option(False, "--no-color", help="Disable colored output")):
    """
    Show only currently running jobs.
    """
    all_jobs = _get_all_jobs_with_details()
    running_jobs = [job for job in all_jobs if job["status"] == JobStatus.RUNNING]
    
    if not running_jobs:
        console.print("🔍 No currently running jobs found")
        console.print("💡 Use 'ffsh recent' to see recently completed jobs")
        return
    
    console.print(f"🚀 Currently Running Jobs ({len(running_jobs)}):")
    console.print()
    
    for job in running_jobs:
        size_mb = job.get('log_size', 0) / (1024 * 1024)
        progress = job.get('progress')
        progress_str = f" ({progress:.1%})" if progress else ""
        short_id = get_short_job_id(job['job_id'])
        
        console.print(f"⚡ [yellow]{job['job_type']}_{short_id}[/yellow]")
        console.print(f"   📁 {job['job_id']}")
        console.print(f"   📊 {size_mb:.1f}MB{progress_str}")
        console.print(f"   🔗 ffsh tail {short_id}")
        console.print()


@app.command()
def recent(
    hours: int = typer.Option(24, "--hours", "-h", help="Show jobs from last N hours"),
    no_color: bool = typer.Option(False, "--no-color", help="Disable colored output")
):
    """
    Show recent job activity and exit.
    """
    console.print(f"📊 Recent Job Activity (Last {hours} hours)")
    console.print("=" * 50)
    
    all_jobs = _get_all_jobs_with_details()
    
    # Filter by time
    now = datetime.now(timezone.utc)
    cutoff = now - timedelta(hours=hours)
    
    recent_jobs = []
    for job in all_jobs:
        job_time = job.get("finished_at") or job.get("started_at") or job.get("created_at")
        if job_time and job_time.replace(tzinfo=timezone.utc) > cutoff:
            recent_jobs.append(job)
    
    if not recent_jobs:
        console.print(f"📭 No jobs found in the last {hours} hours")
        return
    
    # Sort by most recent
    recent_jobs.sort(key=lambda j: j.get("finished_at") or j.get("started_at") or j.get("created_at") or datetime.min.replace(tzinfo=timezone.utc), reverse=True)
    
    _show_recent_jobs_table(recent_jobs, limit=20)
    console.print(f"\n📊 Found {len(recent_jobs)} jobs in the last {hours} hours")


# Helper functions for the new commands

def _get_base_output_dir():
    """Get the base output directory, trying different common paths."""
    # Try production path first
    production_path = Path("/sphere/app/featrix_output")
    if production_path.exists():
        return production_path
    
    # Try config
    if hasattr(config, 'output_dir') and config.output_dir.exists():
        return config.output_dir
    
    # Try relative to sphere root
    sphere_output = SPHERE_ROOT / "featrix_output"
    if sphere_output.exists():
        return sphere_output
    
    # Try app/featrix_output (installed layout)
    app_output = SPHERE_ROOT / "featrix_output" if SPHERE_ROOT.name == "app" else SPHERE_ROOT / "app" / "featrix_output"  
    if app_output.exists():
        return app_output
    
    # Default fallback
    return Path("./featrix_output")


def _get_all_jobs_with_details():
    """Get all jobs from all queues with detailed information."""
    known_queues = [
        "create_structured_data",
        "train_es", 
        "train_knn",
        "build_es_projections",
        "train_single_predictor"
    ]
    
    all_jobs = []
    
    for queue_name in known_queues:
        try:
            for job in iterate_over_jobs_in_queue(queue_name):
                job_data = {
                    "queue": queue_name,
                    "job_id": job.get("job_id"),
                    "job_type": job.get("type"),
                    "session_id": job.get("session_id"),
                    "status": job.get("status"),
                    "created_at": job.get("created_at"),
                    "started_at": job.get("started_at"),
                    "finished_at": job.get("finished_at"),
                    "progress": job.get("progress"),
                }
                
                # Add log file size if available
                try:
                    base_dir = _get_base_output_dir()
                    job_dir = base_dir / job_data["job_id"]
                    log_file = job_dir / "logs" / "stdout.log"
                    if log_file.exists():
                        job_data["log_size"] = log_file.stat().st_size
                except:
                    job_data["log_size"] = 0
                
                all_jobs.append(job_data)
        except Exception as e:
            logger.debug(f"Could not check queue {queue_name}: {e}")
            continue
    
    return all_jobs


def _calculate_job_summary(all_jobs):
    """Calculate summary statistics for jobs."""
    summary = {
        'total': len(all_jobs),
        'running': 0,
        'ready': 0,
        'done': 0,
        'failed': 0,
        'cancelled': 0
    }
    
    for job in all_jobs:
        status = job.get("status")
        if status == JobStatus.RUNNING:
            summary['running'] += 1
        elif status == JobStatus.READY:
            summary['ready'] += 1
        elif status == JobStatus.DONE:
            summary['done'] += 1
        elif status == JobStatus.FAILED:
            summary['failed'] += 1
        elif status == JobStatus.CANCELLED:
            summary['cancelled'] += 1
    
    return summary


def _show_job_summary(summary):
    """Show job summary statistics."""
    console.print(f"\n📊 [bold]Job Summary:[/bold]")
    console.print(f"   🔥 Running: [green]{summary['running']}[/green]")
    console.print(f"   ⏳ Ready: [yellow]{summary['ready']}[/yellow]") 
    console.print(f"   ✅ Done: [green]{summary['done']}[/green]")
    console.print(f"   ❌ Failed: [bright_red]{summary['failed']}[/bright_red]")
    if summary['cancelled'] > 0:
        console.print(f"   🚫 Cancelled: [red]{summary['cancelled']}[/red]")
    console.print(f"   📊 Total: [blue]{summary['total']}[/blue]")


def _show_recent_jobs_table(all_jobs, limit=15):
    """Show table of recent jobs."""
    # Sort by most recent activity
    now = datetime.now(timezone.utc)
    
    def get_sort_time(job):
        job_time = job.get("finished_at") or job.get("started_at") or job.get("created_at")
        if job_time:
            if job_time.tzinfo is None:
                job_time = job_time.replace(tzinfo=timezone.utc)
            elif job_time.tzinfo != timezone.utc:
                job_time = job_time.astimezone(timezone.utc)
            return job_time
        return datetime.min.replace(tzinfo=timezone.utc)
    
    all_jobs.sort(key=get_sort_time, reverse=True)
    
    console.print(f"\n📋 [bold]Recent Jobs (Top {min(limit, len(all_jobs))} of {len(all_jobs)}):[/bold]")
    
    table = Table(box=box.ROUNDED)
    table.add_column("Status", style="bold", width=12)
    table.add_column("Queue", style="cyan", width=20)
    table.add_column("Full Job ID", style="yellow", width=45)
    table.add_column("Session ID", style="blue", no_wrap=True)  # No width limit - show full session ID
    table.add_column("Progress", style="green", width=10)
    table.add_column("Started", style="magenta", width=16)
    table.add_column("Runtime", style="white", width=12)
    table.add_column("Log Size", style="dim", width=10)
    table.add_column("Quick Commands", style="dim", width=60)
    
    for job in all_jobs[:limit]:
        status = job["status"]
        
        # Format status with emoji
        if status == JobStatus.RUNNING:
            status_str = "🔥 RUNNING"
            status_style = "bold green"
        elif status == JobStatus.READY:
            status_str = "⏳ READY"
            status_style = "bold yellow"
        elif status == JobStatus.DONE:
            status_str = "✅ DONE"
            status_style = "bold green"
        elif status == JobStatus.FAILED:
            status_str = "❌ FAILED" 
            status_style = "bold bright_red"
        elif status == JobStatus.CANCELLED:
            status_str = "🚫 CANCELLED"
            status_style = "bold red"
        else:
            status_str = f"❓ {status}"
            status_style = "dim"
        
        # Format progress
        progress = job.get("progress")
        progress_str = f"{progress:.1%}" if progress is not None else "-"
        
        # Format times
        job_time = get_sort_time(job)
        if job_time and job_time != datetime.min.replace(tzinfo=timezone.utc):
            time_diff = now - job_time
            if time_diff.total_seconds() < 60:
                time_str = f"{int(time_diff.total_seconds())}s ago"
            elif time_diff.total_seconds() < 3600:
                time_str = f"{int(time_diff.total_seconds() / 60)}m ago"
            elif time_diff.total_seconds() < 86400:
                time_str = f"{int(time_diff.total_seconds() / 3600)}h ago"
            else:
                time_str = f"{int(time_diff.days)}d ago"
            
            # Format start time
            started_str = job_time.strftime("%m/%d %H:%M:%S")
        else:
            time_str = "-"
            started_str = "-"
        
        # Calculate runtime for running jobs
        started_at = job.get("started_at")
        if status == JobStatus.RUNNING and started_at:
            if started_at.tzinfo is None:
                started_at = started_at.replace(tzinfo=timezone.utc)
            runtime_seconds = (now - started_at).total_seconds()
            if runtime_seconds < 3600:
                runtime_str = f"{int(runtime_seconds/60)}m"
            elif runtime_seconds < 86400:
                runtime_str = f"{runtime_seconds/3600:.1f}h"
            else:
                runtime_str = f"{runtime_seconds/86400:.1f}d"
        else:
            runtime_str = "-"
        
        # Format log size
        log_size = job.get("log_size", 0)
        if log_size > 1024 * 1024:
            size_str = f"{log_size / (1024*1024):.1f}MB"
        elif log_size > 1024:
            size_str = f"{log_size / 1024:.0f}KB"
        else:
            size_str = f"{log_size}B" if log_size > 0 else "-"
        
        # Format display values
        job_id_full = job["job_id"]
        short_id = get_short_job_id(job_id_full)
        session_id = job.get("session_id", "")
        
        # Quick commands
        if status == JobStatus.RUNNING:
            commands = f"ffsh tail {short_id} --follow | ffsh kill-job {job['queue']} {job_id_full}"
        else:
            commands = f"ffsh tail {short_id}"
        
        table.add_row(
            f"[{status_style}]{status_str}[/{status_style}]",
            job["queue"],
            job_id_full,
            session_id,
            progress_str,
            started_str,
            runtime_str,
            size_str,
            commands
        )
    
    console.print(table)


def _show_running_jobs_detail(running_jobs):
    """Show detailed information about running jobs."""
    console.print(f"\n🔥 [bold green]Currently Running Jobs ({len(running_jobs)}):[/bold green]")
    for job in running_jobs:
        progress = job.get("progress")
        progress_str = f" ({progress:.1%})" if progress else ""
        short_id = get_short_job_id(job["job_id"])
        
        console.print(f"   • [yellow]{job['job_id']}[/yellow] in [cyan]{job['queue']}[/cyan]{progress_str}")
        console.print(f"     [dim]Kill:[/dim] ffsh kill-job {job['queue']} {job['job_id']}")
        console.print(f"     [dim]Logs:[/dim] ffsh tail {short_id}")


def _show_available_commands():
    """Show available dashboard commands."""
    console.print(f"\n🔧 [bold]Available Commands:[/bold]")
    console.print(f"   [dim]Refresh dashboard:[/dim] ffsh dashboard")
    console.print(f"   [dim]Watch dashboard:[/dim] ffsh dashboard --watch")
    console.print(f"   [dim]List running jobs:[/dim] ffsh running")
    console.print(f"   [dim]Show recent activity:[/dim] ffsh recent")
    console.print(f"   [dim]Monitor in real-time:[/dim] ffsh monitor")
    console.print(f"   [dim]View job logs:[/dim] ffsh tail <job_id>")
    console.print(f"   [dim]Follow job logs:[/dim] ffsh tail <job_id> --follow")
    console.print()


def _show_recent_activity():
    """Show recent activity summary."""
    all_jobs = _get_all_jobs_with_details()
    
    # Filter to last hour
    now = datetime.now(timezone.utc)
    cutoff = now - timedelta(hours=1)
    
    recent_jobs = []
    for job in all_jobs:
        job_time = job.get("finished_at") or job.get("started_at") or job.get("created_at")
        if job_time and job_time.replace(tzinfo=timezone.utc) > cutoff:
            recent_jobs.append(job)
    
    console.print(f"📊 Recent Activity (Last Hour): {len(recent_jobs)} jobs")
    
    if recent_jobs:
        for job in recent_jobs[-5:]:  # Show last 5
            status = job.get("status")
            short_id = get_short_job_id(job["job_id"])
            console.print(f"   • {status} {job['job_type']}_{short_id}")


def _is_job_running_check(job_dir):
    """Check if a job is currently running."""
    try:
        for queue_name in ["create_structured_data", "train_es", "train_knn", "build_es_projections", "train_single_predictor"]:
            for job in iterate_over_jobs_in_queue(queue_name):
                if job.get("job_id") in str(job_dir):
                    return job.get("status") == JobStatus.RUNNING
    except:
        pass
    return False


def _watch_log_file(log_file_path, no_color=False, tail_lines=1000):
    """Watch log file in real-time with syntax highlighting."""
    import signal
    
    # Colors for syntax highlighting
    colors = {
        'error': '\033[91m',     # Red
        'warning': '\033[93m',   # Yellow  
        'info': '\033[92m',      # Green
        'debug': '\033[96m',     # Cyan
        'header': '\033[95m',    # Magenta
        'reset': '\033[0m'
    }
    
    if no_color:
        colors = {k: '' for k in colors}
    
    log_path = Path(log_file_path)
    
    # Signal handler for clean exit
    def signal_handler(sig, frame):
        console.print("\n👋 Stopped watching log")
        sys.exit(0)
    
    signal.signal(signal.SIGINT, signal_handler)
    
    console.print(f"👁️  Watching: {log_path}")
    console.print(f"📊 File size: {log_path.stat().st_size / (1024*1024):.2f}MB")
    console.print("=" * 60)
    
    # Read initial content
    try:
        with open(log_path, 'r', encoding='utf-8', errors='replace') as f:
            # Read last N lines
            raw_lines = f.readlines()
            if len(raw_lines) > tail_lines:
                raw_lines = raw_lines[-tail_lines:]
        
        # Show recent lines efficiently (no chronological sorting for performance)
        display_lines = [line.rstrip('\n\r') for line in raw_lines]
        console.print(f"📜 Showing last {len(display_lines)} lines")
        console.print("─" * 60)
        
        for line in display_lines:
            _print_highlighted_line(line, colors)
        
        # Follow new content
        last_size = log_path.stat().st_size
        
        while True:
            try:
                current_size = log_path.stat().st_size
                if current_size > last_size:
                    with open(log_path, 'r', encoding='utf-8', errors='replace') as f:
                        f.seek(last_size)
                        new_content = f.read()
                        new_lines = new_content.splitlines()
                        
                        # Show new lines efficiently
                        if new_lines:
                            for line in new_lines:
                                line = line.rstrip('\n\r')
                                _print_highlighted_line(line, colors)
                    last_size = current_size
                
                time.sleep(0.5)
            except FileNotFoundError:
                console.print("\n📭 Log file disappeared")
                break
    except Exception as e:
        console.print(f"❌ Error watching file: {e}")


def _show_log_file(log_file_path, no_color=False, use_pager=False):
    """Show contents of a log file with syntax highlighting."""
    import subprocess
    
    log_path = Path(log_file_path)
    
    if not log_path.exists():
        console.print(f"❌ Log file {log_path} does not exist")
        return
    
    # Colors for syntax highlighting
    colors = {
        'error': '\033[91m',     # Red
        'warning': '\033[93m',   # Yellow  
        'info': '\033[92m',      # Green
        'debug': '\033[96m',     # Cyan
        'header': '\033[95m',    # Magenta
        'reset': '\033[0m'
    }
    
    if no_color:
        colors = {k: '' for k in colors}
    
    # Get file info
    file_size = log_path.stat().st_size
    size_mb = file_size / (1024 * 1024)
    
    # Show header
    console.print(f"✅ Job: {log_path.parent.parent.name}")
    console.print(f"📄 Log: {log_path}")
    console.print(f"📏 Size: {size_mb:.2f} MB")
    console.print("=" * 80)
    
    output_lines = []
    
    def output(text):
        if use_pager:
            output_lines.append(text)
        else:
            print(text)
    
    try:
        # Use efficient reading for better performance  
        display_lines = read_log_efficiently(log_path, max_lines=1000)
        
        if size_mb < 1.0:
            # Show entire file if small
            output(f"📜 Showing all {len(display_lines)} lines\n")
        else:
            # Show last 1000 lines if large
            output(f"📜 Showing last {len(display_lines)} lines (from {size_mb:.1f}MB file)\n")
        
        for i, line in enumerate(display_lines):
            colored_line = _apply_syntax_highlighting(line, colors)
            line_num = i + 1
            output(f"{colors['header']}{line_num:5d}:{colors['reset']} {colored_line}")
    
    except Exception as e:
        output(f"❌ Error reading file: {e}")
    
    # Use pager if requested
    if use_pager and output_lines:
        try:
            full_output = '\n'.join(output_lines)
            process = subprocess.Popen(
                ['less', '-R', '-S', '-M', '-+F', '-+X'],
                stdin=subprocess.PIPE,
                text=True
            )
            process.stdin.write(full_output)
            process.stdin.close()
            process.wait()
        except (BrokenPipeError, KeyboardInterrupt):
            pass
        except FileNotFoundError:
            console.print("❌ 'less' command not found")


def _print_highlighted_line(line, colors):
    """Print a line with syntax highlighting."""
    colored_line = _apply_syntax_highlighting(line, colors)
    print(colored_line)


def _apply_syntax_highlighting(line, colors):
    """Apply syntax highlighting to a line."""
    lower_line = line.lower()
    
    if any(word in lower_line for word in ['error', 'exception', 'traceback', 'failed', 'critical']):
        return f"{colors['error']}{line}{colors['reset']}"
    elif any(word in lower_line for word in ['warning', 'warn']):
        return f"{colors['warning']}{line}{colors['reset']}"
    elif any(word in lower_line for word in ['info', 'starting', 'completed', 'success']):
        return f"{colors['info']}{line}{colors['reset']}"
    elif any(word in lower_line for word in ['debug']):
        return f"{colors['debug']}{line}{colors['reset']}"
    else:
        return line


def _get_api_client():
    """Get FeatrixSphereClient instance, trying to detect server URL."""
    try:
        import os
        from featrixsphere import FeatrixSphereClient
        
        # Try to detect server URL from environment or config
        base_url = os.getenv("FEATRIX_SPHERE_URL", "https://sphere-api.featrix.com")
        
        # If we're on a compute node, try local first
        if os.path.exists("/sphere/app"):
            try:
                # Try local API first
                local_client = FeatrixSphereClient("http://localhost:8000")
                local_client.get_stats()  # Test connection
                return local_client
            except Exception:
                pass
        
        return FeatrixSphereClient(base_url)
    except ImportError:
        console.print("❌ featrixsphere package not installed. Install with: pip install featrixsphere")
        raise typer.Exit(1)
    except Exception as e:
        console.print(f"❌ Failed to initialize API client: {e}")
        raise typer.Exit(1)


@app.command("ls")
def list_models_and_es():
    """List all models and embedding spaces."""
    try:
        client = _get_api_client()
        
        # Get all sessions from local filesystem (since API doesn't have list endpoint)
        from featrix_queue import iterate_over_sessions
        sessions = list(iterate_over_sessions())
        
        if not sessions:
            console.print("📭 No sessions found")
            return
        
        # Create table
        table = Table(title="Models and Embedding Spaces", box=box.ROUNDED)
        table.add_column("Session ID", style="cyan", no_wrap=True)
        table.add_column("Type", style="magenta")
        table.add_column("Status", style="green")
        table.add_column("ES", style="yellow", justify="center")
        table.add_column("Predictors", style="yellow", justify="center")
        table.add_column("Created", style="dim")
        
        for session in sessions[:50]:  # Limit to 50 most recent
            session_id = session.get("session_id", "unknown")
            session_type = session.get("session_type", "unknown")
            status = session.get("status", "unknown")
            created_at = session.get("created_at", "")
            
            # Check for embedding space
            has_es = "✅" if session.get("embedding_space") else "❌"
            
            # Check for predictors
            single_predictors = session.get("single_predictors", [])
            if isinstance(single_predictors, list):
                predictor_count = len([p for p in single_predictors if p])
            else:
                predictor_count = 1 if single_predictors else 0
            
            predictors_str = f"{predictor_count}" if predictor_count > 0 else "0"
            
            # Format created_at
            if created_at:
                try:
                    if isinstance(created_at, str):
                        # Try parsing ISO format or common formats
                        try:
                            dt = datetime.fromisoformat(created_at.replace('Z', '+00:00'))
                        except (ValueError, AttributeError):
                            # Fallback to string slicing
                            dt = None
                        if dt:
                            created_str = dt.strftime("%Y-%m-%d %H:%M")
                        else:
                            created_str = str(created_at)[:16]
                    else:
                        created_str = str(created_at)[:16]
                except Exception:
                    created_str = str(created_at)[:16] if created_at else "unknown"
            else:
                created_str = "unknown"
            
            table.add_row(session_id, session_type, status, has_es, predictors_str, created_str)
        
        console.print(table)
        if len(sessions) > 50:
            console.print(f"\n[dim]Showing 50 of {len(sessions)} sessions[/dim]")
            
    except Exception as e:
        console.print(f"❌ Error listing sessions: {e}")
        logger.exception("Failed to list sessions")


@app.command("top")
def show_active_training():
    """Show active training jobs and sessions."""
    try:
        client = _get_api_client()
        
        # Get running jobs
        all_jobs = _get_all_jobs_with_details()
        running_jobs = [job for job in all_jobs if job["status"] == JobStatus.RUNNING]
        
        if not running_jobs:
            console.print("📭 No active training jobs")
            return
        
        console.print(f"\n[bold blue]🚀 Active Training Jobs ({len(running_jobs)})[/bold blue]")
        console.print("=" * 80)
        
        # Group by session
        jobs_by_session = defaultdict(list)
        for job in running_jobs:
            session_id = job.get("session_id", "unknown")
            jobs_by_session[session_id].append(job)
        
        for session_id, jobs in jobs_by_session.items():
            console.print(f"\n[bold cyan]Session: {session_id}[/bold cyan]")
            
            for job in jobs:
                job_type = job.get("job_type", "unknown")
                job_id = job.get("job_id", "unknown")
                short_id = get_short_job_id(job_id)
                progress = job.get("progress")
                progress_str = f" {progress:.1%}" if progress else ""
                
                # Try to get more details from API
                try:
                    session_info = client.get_session_status(session_id)
                    status_emoji = "🟢" if session_info.status == "running" else "🟡"
                    console.print(f"  {status_emoji} [yellow]{job_type}[/yellow] ({short_id}){progress_str}")
                except Exception:
                    console.print(f"  ⚡ [yellow]{job_type}[/yellow] ({short_id}){progress_str}")
                
                # Show log size
                size_mb = job.get('log_size', 0) / (1024 * 1024)
                if size_mb > 0:
                    console.print(f"     📊 Log: {size_mb:.1f}MB | [dim]ffsh tail {short_id}[/dim]")
        
        console.print("\n[dim]💡 Use 'ffsh tail <job_id>' to view logs[/dim]")
        
    except Exception as e:
        console.print(f"❌ Error showing active training: {e}")
        logger.exception("Failed to show active training")


@app.command("cp")
def copy_session(
    session_id: str = typer.Argument(..., help="Session ID to copy"),
    target_node: str = typer.Argument(..., help="Target node hostname or IP")
):
    """Copy session to another node via rsync over SSH."""
    try:
        client = _get_api_client()
        
        # Get session info
        console.print(f"📋 Getting session info for {session_id}...")
        session = None
        try:
            session_info = client.get_session_status(session_id)
            # SessionInfo object has session_id, status, etc. but not a 'session' attribute
            # We'll load from filesystem for the actual session data
            from featrix_queue import load_session
            session = load_session(session_id)
        except Exception as e:
            console.print(f"⚠️  Could not get session from API: {e}")
            console.print("   Trying local filesystem...")
            from featrix_queue import load_session
            try:
                session = load_session(session_id)
            except FileNotFoundError:
                console.print(f"❌ Session not found: {session_id}")
                raise typer.Exit(1)
        
        # Determine source paths
        if os.path.exists("/sphere/app"):
            # On compute node
            base_dir = Path("/sphere/app")
            session_file = config.session_dir / f"{session_id}.session"
            output_dir = base_dir / "featrix_output"
        else:
            # Local development
            session_file = Path(config.session_dir) / f"{session_id}.session"
            output_dir = Path(config.output_dir) if hasattr(config, 'output_dir') else Path(config.session_dir).parent / "featrix_output"
        
        if not session_file.exists():
            console.print(f"❌ Session file not found: {session_file}")
            raise typer.Exit(1)
        
        console.print(f"📦 Copying session {session_id} to {target_node}...")
        
        # Find all related output directories
        session_timestamp = session_id.split('_')[-1] if '_' in session_id else session_id
        job_dirs = list(output_dir.glob(f"*{session_timestamp}*")) if output_dir.exists() else []
        
        # Copy session file
        console.print(f"  📄 Copying session file...")
        # Use absolute path for target
        target_session_path = f"{target_node}:/sphere/app/featrix_sessions/{session_id}.session"
        rsync_args = [
            "rsync",
            "-avz",
            "--progress",
            str(session_file),
            target_session_path
        ]
        result = subprocess.run(rsync_args, capture_output=True, text=True)
        if result.returncode != 0:
            console.print(f"❌ Failed to copy session file: {result.stderr}")
            raise typer.Exit(1)
        
        # Copy job output directories
        for job_dir in job_dirs:
            if job_dir.is_dir():
                console.print(f"  📁 Copying {job_dir.name}...")
                job_rsync = [
                    "rsync",
                    "-avz",
                    "--progress",
                    str(job_dir) + "/",
                    f"{target_node}:/sphere/app/featrix_output/{job_dir.name}/"
                ]
                result = subprocess.run(job_rsync, capture_output=True, text=True)
                if result.returncode != 0:
                    console.print(f"⚠️  Warning: Failed to copy {job_dir.name}: {result.stderr}")
        
        # Create new session on target node with metadata about origin
        console.print(f"  🔄 Creating session on target node...")
        
        # Get origin node info
        import socket
        origin_node = socket.gethostname()
        origin_metadata = {
            "copied_from": origin_node,
            "copied_at": datetime.now(timezone.utc).isoformat(),
            "original_session_id": session_id
        }
        
        # SSH to target and create session
        ssh_cmd = f"""ssh {target_node} 'cd /sphere/app && python3 -c "
from featrix_queue import load_session, save_session
import json
from datetime import datetime
from zoneinfo import ZoneInfo

# Load the copied session
session = load_session('{session_id}')

# Update metadata
if 'metadata' not in session:
    session['metadata'] = {{}}
session['metadata'].update({json.dumps(origin_metadata)})

# Save updated session
save_session('{session_id}', session, exist_ok=True)
print('Session created on target node')
"'"""
        
        result = subprocess.run(ssh_cmd, shell=True, capture_output=True, text=True)
        if result.returncode != 0:
            console.print(f"⚠️  Warning: Could not update session metadata on target: {result.stderr}")
        else:
            console.print(f"✅ Session copied successfully!")
            console.print(f"   Origin: {origin_node}")
            console.print(f"   Target: {target_node}")
            console.print(f"   Session ID: {session_id}")
        
    except Exception as e:
        console.print(f"❌ Error copying session: {e}")
        logger.exception("Failed to copy session")
        raise typer.Exit(1)


@app.command("train-more")
def train_more(
    session_id: str = typer.Argument(..., help="Session ID to continue training")
):
    """Continue training for a session (add more epochs)."""
    try:
        client = _get_api_client()
        
        console.print(f"🔄 Continuing training for session {session_id}...")
        
        # Get session status
        session_info = client.get_session_status(session_id)
        
        # Check if there's an embedding space training job
        jobs = session_info.jobs if hasattr(session_info, 'jobs') else {}
        es_jobs = [j for j_id, j in jobs.items() if j.get('type') == 'train_es']
        
        if es_jobs:
            console.print("📊 Found embedding space training - continuing ES training...")
            # For ES, we'd need to add a new train_es job with more epochs
            # This would require API support or direct queue manipulation
            console.print("⚠️  ES training continuation requires API support")
            console.print("💡 Use the API to create a new train_es job with additional epochs")
        else:
            # Check for single predictor training
            console.print("🎯 Checking for predictor training to continue...")
            models = client.get_session_models(session_id)
            
            if models.get('summary', {}).get('prediction_ready'):
                console.print("✅ Session has trained predictors")
                console.print("💡 To continue training, use:")
                console.print(f"   client.train_single_predictor('{session_id}', 'target_column', 'set', epochs=50)")
            else:
                console.print("⚠️  No completed training found to continue")
                console.print("💡 Session may still be training or needs initial training")
        
    except Exception as e:
        console.print(f"❌ Error continuing training: {e}")
        logger.exception("Failed to continue training")
        raise typer.Exit(1)




@app.command("upgrade")
def upgrade(
    branch: str = typer.Option(None, "--set", help="Git branch/tag to follow for upgrades")
):
    """Upgrade sphere installation."""
    try:
        console.print("🔄 Checking for upgrades...")
        
        # Check if we're on a compute node
        if os.path.exists("/sphere/app"):
            console.print("📦 Running on compute node - checking for updates...")
            
            # Check if auto-upgrade-monitor is running
            result = subprocess.run(
                ["supervisorctl", "status", "auto-upgrade-monitor"],
                capture_output=True,
                text=True
            )
            
            if "RUNNING" in result.stdout:
                console.print("✅ Auto-upgrade monitor is running")
                console.print("💡 Upgrades happen automatically when git changes are detected")
            else:
                console.print("⚠️  Auto-upgrade monitor is not running")
                console.print("💡 To enable: supervisorctl start auto-upgrade-monitor")
            
            # Manual upgrade option
            if branch:
                console.print(f"🔄 Setting upgrade branch to: {branch}")
                # This would require API or config file modification
                console.print("⚠️  Branch setting requires manual configuration")
            else:
                console.print("\n💡 To manually trigger upgrade:")
                console.print("   cd /home/mitch/sphere")
                console.print("   git pull")
                console.print("   sudo /home/mitch/sphere/src/node-install.sh")
        else:
            console.print("💻 Running locally - no upgrade needed")
            console.print("💡 Compute nodes upgrade automatically via auto-upgrade-monitor")
        
    except Exception as e:
        console.print(f"❌ Error checking upgrades: {e}")
        logger.exception("Failed to check upgrades")


@app.command("recreate-model-card")
def recreate_model_card(
    session_id: str = typer.Argument(..., help="Session ID to recreate model card for"),
    output_path: Optional[str] = typer.Option(None, "--output", "-o", help="Optional output path to save model card JSON (default: don't save, just return from API)")
):
    """Recreate a missing model card for a session using the API. Uses featrixsphere client to call the API endpoint."""
    import json
    
    try:
        console.print(f"📋 Recreating model card for session: {session_id}")
        
        # Get API client
        console.print("   Connecting to API...")
        client = _get_api_client()
        
        # Call the GET endpoint to recreate the model card (GET handles on-demand generation)
        console.print("   Calling API to recreate model card...")
        model_card = client._get_json(f"/compute/session/{session_id}/model_card")
        
        console.print("   ✅ Model card recreated successfully!")
        
        # Display summary
        model_info = model_card.get("model_info", {})
        data_info = model_card.get("data_info", {})
        
        console.print(f"   Best epoch: {model_info.get('best_epoch', 'N/A')}")
        console.print(f"   Total epochs: {model_info.get('total_epochs', 'N/A')}")
        console.print(f"   Columns: {data_info.get('num_columns', 'N/A')}")
        console.print(f"   Training rows: {data_info.get('train_rows', 'N/A')}")
        console.print(f"   Validation rows: {data_info.get('val_rows', 'N/A')}")
        
        # Optionally save to file
        if output_path:
            model_card_path = Path(output_path)
            model_card_path.parent.mkdir(parents=True, exist_ok=True)
            console.print(f"   Saving model card to: {model_card_path}")
            with open(model_card_path, 'w') as f:
                json.dump(model_card, f, indent=2, default=str)
            console.print(f"   ✅ Saved to: {model_card_path}")
        else:
            console.print("   💡 Tip: Use --output to save the model card to a file")
        
    except Exception as e:
        error_msg = str(e)
        if "404" in error_msg or "not found" in error_msg.lower():
            console.print(f"❌ Session not found or model card could not be created: {session_id}")
        elif "400" in error_msg or "bad request" in error_msg.lower():
            console.print(f"❌ Invalid request: {error_msg}")
        else:
            console.print(f"❌ Error recreating model card: {error_msg}")
        logger.exception("Failed to recreate model card")
        raise typer.Exit(1)


@app.command("mt")
def multitail():
    """Show all /var/log/featrix logs in a multitail session."""
    try:
        log_dir = Path("/var/log/featrix")
        
        if not log_dir.exists():
            console.print(f"❌ Log directory not found: {log_dir}")
            console.print("💡 This command is for compute nodes only")
            raise typer.Exit(1)
        
        # Find all log files
        log_files = list(log_dir.glob("*.log"))
        
        if not log_files:
            console.print("📭 No log files found in /var/log/featrix")
            return
        
        console.print(f"📜 Found {len(log_files)} log files")
        console.print("🚀 Starting multitail... (Ctrl+C to stop)\n")
        
        # Check if multitail is installed
        if subprocess.run(["which", "multitail"], capture_output=True).returncode == 0:
            # Use multitail if available
            cmd = ["multitail"] + [str(f) for f in log_files]
            subprocess.run(cmd)
        else:
            # Fallback: use tail -f on all files with labels
            console.print("⚠️  multitail not installed, using tail -f instead\n")
            
            # Open all log files
            files = {}
            for log_file in log_files:
                try:
                    f = open(log_file, 'r')
                    # Seek to end
                    f.seek(0, 2)
                    files[log_file.name] = f
                except Exception as e:
                    console.print(f"⚠️  Could not open {log_file.name}: {e}")
            
            if not files:
                console.print("❌ No log files could be opened")
                return
            
            try:
                while True:
                    # Check for new lines (simple polling)
                    for name, f in files.items():
                        line = f.readline()
                        if line:
                            console.print(f"[{name}] {line.rstrip()}")
                    time.sleep(0.1)
            except KeyboardInterrupt:
                console.print("\n👋 Stopped multitail")
            finally:
                for f in files.values():
                    f.close()
        
    except Exception as e:
        console.print(f"❌ Error starting multitail: {e}")
        logger.exception("Failed to start multitail")
        raise typer.Exit(1)


if __name__ == "__main__":
    app()
