#!/bin/bash
# Emergency Recovery Script for mcp-vector-search 2.5.18 Hang
# Usage: ./emergency_recovery.sh [instance-id]

set -e

INSTANCE_ID="${1:-i-02d8caca52b2e209b}"
REGION="${2:-us-east-1}"
USER="${3:-ubuntu}"
PID="${4:-20032}"

echo "=========================================="
echo "Emergency Recovery for mcp-vector-search"
echo "=========================================="
echo "Instance: $INSTANCE_ID"
echo "Region: $REGION"
echo "User: $USER"
echo "PID: $PID"
echo ""

# Function to run remote commands
run_remote() {
    local cmd="$1"
    echo "Running: $cmd"
    aws ssm send-command \
        --instance-ids "$INSTANCE_ID" \
        --document-name "AWS-RunShellScript" \
        --parameters "commands=[\"$cmd\"]" \
        --region "$REGION" \
        --output text \
        --query "Command.CommandId" | \
    xargs -I {} aws ssm get-command-invocation \
        --command-id {} \
        --instance-id "$INSTANCE_ID" \
        --region "$REGION" \
        --query "StandardOutputContent" \
        --output text
}

echo "Step 1: Checking if process $PID is still running..."
echo "----------------------------------------"
run_remote "ps aux | grep $PID | grep -v grep || echo 'Process not found'"
echo ""

echo "Step 2: Checking process state (if running)..."
echo "----------------------------------------"
run_remote "if ps -p $PID > /dev/null; then ps -o pid,ppid,stat,pcpu,pmem,etime,cmd -p $PID; fi"
echo ""

echo "Step 3: Checking memory usage..."
echo "----------------------------------------"
run_remote "free -h"
echo ""

echo "Step 4: Checking log file tail..."
echo "----------------------------------------"
run_remote "tail -50 /tmp/reindex-20260218-234540-gpu-v2.5.18.log 2>/dev/null || echo 'Log file not found'"
echo ""

echo "Step 5: Checking for strace (syscall analysis)..."
echo "----------------------------------------"
echo "Note: This will run for 5 seconds to capture syscalls"
run_remote "if ps -p $PID > /dev/null; then timeout 5 sudo strace -p $PID -s 200 -f 2>&1 | head -100 || echo 'Process finished or strace failed'; fi"
echo ""

echo "Step 6: Checking thread state..."
echo "----------------------------------------"
run_remote "if ps -p $PID > /dev/null; then ps -T -p $PID | head -20; fi"
echo ""

echo "=========================================="
echo "RECOVERY OPTIONS"
echo "=========================================="
echo ""
echo "Option 1: Kill the hung process (IMMEDIATE)"
echo "-------------------------------------------"
echo "Run: aws ssm send-command \\"
echo "       --instance-ids $INSTANCE_ID \\"
echo "       --document-name 'AWS-RunShellScript' \\"
echo "       --parameters 'commands=[\"sudo kill -9 $PID\"]' \\"
echo "       --region $REGION"
echo ""

echo "Option 2: Downgrade to 2.2.40 (SAFE)"
echo "-------------------------------------"
echo "Run on instance:"
echo "  pip uninstall mcp-vector-search -y"
echo "  pip install mcp-vector-search==2.2.40"
echo "  python -c 'import mcp_vector_search; print(mcp_vector_search.__version__)'"
echo ""

echo "Option 3: Disable memory monitoring (WORKAROUND)"
echo "-----------------------------------------------"
echo "Run on instance:"
echo "  export MCP_VECTOR_SEARCH_MAX_MEMORY_GB=999999"
echo "  mcp-vector-search index /path/to/project"
echo ""

echo "Option 4: Apply hotfix patch (PERMANENT FIX)"
echo "-------------------------------------------"
echo "1. Download HOTFIX_2.5.19.patch from repository"
echo "2. Apply patch: git apply HOTFIX_2.5.19.patch"
echo "3. Build and install: pip install -e ."
echo "4. Verify version: python -c 'import mcp_vector_search; print(mcp_vector_search.__version__)'"
echo ""

echo "=========================================="
echo "RECOMMENDED ACTION"
echo "=========================================="
echo "1. Kill process $PID (if still running)"
echo "2. Downgrade to version 2.2.40"
echo "3. Wait for 2.5.19 hotfix release (ETA: 3 hours)"
echo ""
echo "For immediate fix, see: INCIDENT_REPORT_2.5.18_HANG.md"
echo "=========================================="
