#!/usr/bin/env python3
"""
Author: Vasiliy Zdanovskiy
email: vasilyvz@gmail.com

Comprehensive pipeline for testing all MCP Proxy Adapter modes.
For each mode:
- Generates server and client configs using CLI generator
- Starts proxy and server
- Tests all commands including queue commands
"""

import asyncio
import json
import subprocess
import sys
import time
import uuid
from pathlib import Path
from typing import Optional

# Add project root to path for imports
sys.path.insert(0, str(Path(__file__).parent.parent))

from mcp_proxy_adapter.client.jsonrpc_client.client import JsonRpcClient
from tests.config import TEST_MODES
from tests.utils.config_utils import BASE_DIR
from tests.utils.process_utils import (
    start_proxy,
    start_server,
    wait_for_port,
)
from tests.utils.health_utils import check_server_health
from tests.utils.logging_utils import log_result, get_results, clear_results
from tests.utils.registration_utils import (
    check_proxy_registration_with_uuid,
    verify_uuid_in_payload,
)


# Certificate paths
CERTS_DIR = BASE_DIR / "mtls_certificates"
SERVER_CERT = str(CERTS_DIR / "server" / "test-server.crt")
SERVER_KEY = str(CERTS_DIR / "server" / "test-server.key")
SERVER_CA = str(CERTS_DIR / "ca" / "ca.crt")
CLIENT_CERT = str(CERTS_DIR / "client" / "test-client.crt")
CLIENT_KEY = str(CERTS_DIR / "client" / "test-client.key")
CLIENT_CA = str(CERTS_DIR / "ca" / "ca.crt")

CONFIGS_DIR = (
    BASE_DIR / "mcp_proxy_adapter" / "examples" / "full_application" / "configs"
)
CLIENT_CONFIGS_DIR = BASE_DIR / "tests" / "client_configs"
CLIENT_CONFIGS_DIR.mkdir(parents=True, exist_ok=True)

# Starting port for test servers
START_PORT = 5000
PROXY_START_PORT = 6000


def kill_process_on_port(port: int) -> None:
    """Kill process occupying a port using kill -9."""
    try:
        # Find process using the port
        result = subprocess.run(
            ["lsof", "-ti", f":{port}"], capture_output=True, text=True
        )
        if result.returncode == 0 and result.stdout.strip():
            pids = result.stdout.strip().split("\n")
            for pid in pids:
                if pid:
                    subprocess.run(["kill", "-9", pid], capture_output=True)
    except Exception:
        pass


def generate_server_config(mode_spec: dict, server_port: int, proxy_port: int) -> Path:
    """Generate server configuration using CLI generator."""
    import json
    import uuid

    config_name = mode_spec["config"].split("/")[-1]
    config_path = CONFIGS_DIR / config_name

    CONFIGS_DIR.mkdir(parents=True, exist_ok=True)

    # Kill any process on the port before generating config
    kill_process_on_port(server_port)
    kill_process_on_port(proxy_port)

    # Use adapter-cfg-gen command if available, otherwise use module
    try:
        import shutil

        if shutil.which("adapter-cfg-gen"):
            cmd = ["adapter-cfg-gen"]
        else:
            cmd = [
                sys.executable,
                "-m",
                "mcp_proxy_adapter.cli.commands.config_generate",
            ]
    except Exception:
        cmd = [sys.executable, "-m", "mcp_proxy_adapter.cli.commands.config_generate"]

    cmd.extend(
        [
            "--protocol",
            mode_spec["protocol"],
            "--out",
            str(config_path),
            "--server-port",
            str(server_port),
            "--registration-port",
            str(proxy_port),
            # --with-proxy is auto-enabled when --registration-port is provided
        ]
    )

    # Add SSL certificates for https/mtls
    if mode_spec["protocol"] in ("https", "mtls"):
        cmd.extend(
            [
                "--server-cert-file",
                SERVER_CERT,
                "--server-key-file",
                SERVER_KEY,
            ]
        )
        if mode_spec["protocol"] == "mtls":
            cmd.extend(
                [
                    "--server-ca-cert-file",
                    SERVER_CA,
                    "--registration-cert-file",
                    CLIENT_CERT,
                    "--registration-key-file",
                    CLIENT_KEY,
                    "--registration-ca-cert-file",
                    CLIENT_CA,
                ]
            )
        else:
            cmd.extend(
                [
                    "--registration-cert-file",
                    CLIENT_CERT,
                    "--registration-key-file",
                    CLIENT_KEY,
                ]
            )

    # Add authentication
    if mode_spec.get("token"):
        cmd.append("--use-token")
        if "Roles" in mode_spec["name"]:
            cmd.append("--use-roles")

    # Add queue manager config
    cmd.extend(
        [
            "--queue-enabled",
            "--max-queue-size",
            "1000",
            "--per-job-type-limits",
            "command_execution:100,data_processing:50,api_call:200",
        ]
    )

    try:
        result = subprocess.run(cmd, capture_output=True, text=True, check=True)

        # Verify UUID was generated and add if missing
        if config_path.exists():
            config_data = json.loads(config_path.read_text(encoding="utf-8"))
            reg = config_data.get("registration", {})
            if reg.get("enabled") and not reg.get("instance_uuid"):
                # UUID missing - generate it
                config_data["registration"]["instance_uuid"] = str(uuid.uuid4())
                config_path.write_text(
                    json.dumps(config_data, indent=2, ensure_ascii=False),
                    encoding="utf-8",
                )
                print(
                    f"   ⚠️  UUID was missing, generated: {config_data['registration']['instance_uuid']}"
                )

        # For mTLS modes, add check_hostname: false to registration.ssl to avoid hostname mismatch
        if mode_spec["protocol"] == "mtls":
            # Reload config to get latest changes (including UUID if it was added)
            config_data = json.loads(config_path.read_text(encoding="utf-8"))
            if "registration" in config_data and isinstance(
                config_data["registration"], dict
            ):
                # Preserve UUID if it exists
                existing_uuid = config_data["registration"].get("instance_uuid")

                if "ssl" not in config_data["registration"]:
                    config_data["registration"]["ssl"] = {}
                if not isinstance(config_data["registration"]["ssl"], dict):
                    # If ssl is not a dict, convert it
                    ssl_value = config_data["registration"]["ssl"]
                    config_data["registration"]["ssl"] = {}
                    if ssl_value:
                        # Try to preserve existing ssl config if it's an object
                        if hasattr(ssl_value, "__dict__"):
                            config_data["registration"]["ssl"].update(
                                ssl_value.__dict__
                            )
                config_data["registration"]["ssl"]["check_hostname"] = False

                # Ensure UUID is preserved
                if existing_uuid:
                    config_data["registration"]["instance_uuid"] = existing_uuid
                elif config_data["registration"].get("enabled") and not config_data[
                    "registration"
                ].get("instance_uuid"):
                    config_data["registration"]["instance_uuid"] = str(uuid.uuid4())

                config_path.write_text(
                    json.dumps(config_data, indent=2, ensure_ascii=False),
                    encoding="utf-8",
                )

        # Final verification: ensure UUID is present
        if config_path.exists():
            config_data = json.loads(config_path.read_text(encoding="utf-8"))
            reg = config_data.get("registration", {})
            if reg.get("enabled") and not reg.get("instance_uuid"):
                raise ValueError(
                    f"CRITICAL: instance_uuid is missing in generated config {config_path}. "
                    "UUID generation failed!"
                )

        return config_path
    except subprocess.CalledProcessError as e:
        error_msg = f"❌ Error generating server config: {e.stderr}"
        print(error_msg, file=sys.stderr)
        if e.stdout:
            print(f"STDOUT: {e.stdout[:500]}", file=sys.stderr)
        raise RuntimeError(f"Failed to generate server config: {error_msg}") from e
    except Exception as e:
        error_msg = f"❌ Unexpected error generating server config: {e}"
        print(error_msg, file=sys.stderr)
        raise RuntimeError(error_msg) from e


def generate_client_config(mode_spec: dict, port: int) -> Path:
    """Generate client configuration for testing."""
    config_name = (
        f"client_{mode_spec['name'].lower().replace(' ', '_').replace('+', '_')}.json"
    )
    config_path = CLIENT_CONFIGS_DIR / config_name

    # Client config matches server protocol
    protocol = mode_spec["protocol"]
    # Use 127.0.0.1 for mTLS to avoid hostname mismatch issues
    host = "127.0.0.1" if protocol == "mtls" else "localhost"

    client_config = {
        "protocol": protocol,
        "host": host,
        "port": port,
    }

    # Add SSL/TLS settings
    if protocol in ("https", "mtls"):
        client_config["cert"] = CLIENT_CERT
        client_config["key"] = CLIENT_KEY
        if protocol == "mtls":
            client_config["ca"] = CLIENT_CA
        # Always disable hostname check for test certificates
        client_config["check_hostname"] = False

    # Add authentication
    if mode_spec.get("token"):
        client_config["token"] = mode_spec["token"]
        client_config["token_header"] = "X-API-Key"

    config_path.write_text(json.dumps(client_config, indent=2), encoding="utf-8")
    return config_path


async def _test_basic_commands(client: JsonRpcClient, mode_name: str) -> bool:
    """Test basic commands (echo, health, help)."""
    print("  🧪 Testing basic commands...")

    try:
        # Test echo
        result = await client.echo("Hello from pipeline")
        if not result.get("success"):
            log_result(mode_name, "command_echo", "FAIL", str(result))
            return False
        log_result(mode_name, "command_echo", "PASS")

        # Test health - use health endpoint directly (not a JSON-RPC command)
        try:
            health_data = await client.health()
            if health_data.get("status") == "ok":
                log_result(mode_name, "command_health", "PASS")
            else:
                log_result(mode_name, "command_health", "FAIL", str(health_data))
                return False
        except Exception as e:
            log_result(mode_name, "command_health", "FAIL", str(e))
            return False

        # Test help
        result = await client.help()
        if not result.get("success"):
            log_result(mode_name, "command_help", "FAIL", str(result))
            return False
        log_result(mode_name, "command_help", "PASS")

        return True
    except Exception as e:
        log_result(mode_name, "basic_commands", "FAIL", str(e))
        import traceback

        traceback.print_exc()
        return False


async def _test_queue_commands(client: JsonRpcClient, mode_spec: dict) -> bool:
    """Test queue commands comprehensively."""
    mode_name = mode_spec["name"]
    print("  🧪 Testing queue commands...")

    terminal_statuses = {"completed", "failed", "stopped", "deleted"}

    async def wait_for_terminal_status(
        job_id: str, timeout: float = 20.0, interval: float = 1.0
    ) -> Optional[str]:
        """Poll queue_get_job_status until job reaches a terminal state."""
        waited = 0.0
        while waited < timeout:
            try:
                response = await client.jsonrpc_call(
                    "queue_get_job_status", {"job_id": job_id}
                )
                result = client._extract_result(response)
                status = result.get("data", {}).get("status")
                if status and status.lower() in terminal_statuses:
                    return status
            except Exception:
                pass
            await asyncio.sleep(interval)
            waited += interval
        return None

    async def stop_job_with_fallback(job_id: str) -> bool:
        """Invoke queue_stop_job and fall back to polling if queuemgr times out."""
        failure_reason = ""
        try:
            response = await client.jsonrpc_call("queue_stop_job", {"job_id": job_id})
            result = client._extract_result(response)
            if result.get("success"):
                log_result(
                    mode_name,
                    "queue_stop_job",
                    "PASS",
                    "Queue manager confirmed stop",
                )
                return True
            failure_reason = str(result)
        except Exception as exc:
            failure_reason = str(exc)

        status = await wait_for_terminal_status(job_id, timeout=25.0)
        if status:
            log_result(
                mode_name,
                "queue_stop_job",
                "PASS",
                f"Stop confirmed via status polling (status={status})",
            )
            return True

        log_result(
            mode_name,
            "queue_stop_job",
            "FAIL",
            f"Unable to stop job: {failure_reason}",
        )
        return False

    try:
        # Test queue_health
        response = await client.jsonrpc_call("queue_health", {})
        result = client._extract_result(response)
        if not result.get("success"):
            log_result(mode_name, "queue_health", "FAIL", str(result))
            return False
        log_result(mode_name, "queue_health", "PASS")

        # Clean up any existing jobs
        response = await client.jsonrpc_call("queue_list_jobs", {})
        list_result = client._extract_result(response)
        if list_result.get("success"):
            jobs = list_result.get("data", {}).get("jobs", [])
            for job in jobs:
                job_id = job.get("job_id")
                if job_id:
                    await client.jsonrpc_call("queue_delete_job", {"job_id": job_id})

        # Test queue_add_job - command_execution
        job_id_1 = f"test_cmd_{uuid.uuid4().hex[:8]}"
        response = await client.jsonrpc_call(
            "queue_add_job",
            {
                "job_type": "command_execution",
                "job_id": job_id_1,
                "params": {
                    "command": "echo",
                    "params": {"message": "Test command execution"},
                },
            },
        )
        result = client._extract_result(response)
        if not result.get("success"):
            log_result(mode_name, "queue_add_job_cmd", "FAIL", str(result))
            return False
        log_result(mode_name, "queue_add_job_cmd", "PASS")

        # Test queue_add_job - data_processing
        job_id_2 = f"test_data_{uuid.uuid4().hex[:8]}"
        response = await client.jsonrpc_call(
            "queue_add_job",
            {
                "job_type": "data_processing",
                "job_id": job_id_2,
                "params": {"data": {"test": "data"}, "operation": "process"},
            },
        )
        result = client._extract_result(response)
        if not result.get("success"):
            log_result(mode_name, "queue_add_job_data", "FAIL", str(result))
            return False
        log_result(mode_name, "queue_add_job_data", "PASS")

        # Test queue_add_job - api_call
        job_id_3 = f"test_api_{uuid.uuid4().hex[:8]}"
        api_headers = {
            key: value
            for key, value in client.headers.items()
            if key.lower() != "content-type"
        }
        api_params = {
            "job_type": "api_call",
            "job_id": job_id_3,
            "params": {
                "url": f"{client.base_url}/health",
                "method": "GET",
                "headers": api_headers,
                "timeout": 15,
            },
        }
        if mode_spec["protocol"] in ("https", "mtls"):
            api_params["params"]["verify_ssl"] = False
        if mode_spec["protocol"] == "mtls":
            api_params["params"]["cert"] = CLIENT_CERT
            api_params["params"]["key"] = CLIENT_KEY
        response = await client.jsonrpc_call("queue_add_job", api_params)
        result = client._extract_result(response)
        if not result.get("success"):
            log_result(mode_name, "queue_add_job_api", "FAIL", str(result))
            return False
        log_result(mode_name, "queue_add_job_api", "PASS")

        # Test queue_get_job_status
        response = await client.jsonrpc_call(
            "queue_get_job_status", {"job_id": job_id_1}
        )
        result = client._extract_result(response)
        if not result.get("success"):
            log_result(mode_name, "queue_get_job_status", "FAIL", str(result))
            return False
        log_result(mode_name, "queue_get_job_status", "PASS")

        # Test queue_get_job_status auto-detection (uses client method, not direct jsonrpc_call)
        # This verifies that the method correctly auto-detects the endpoint
        print("    🧪 Testing queue_get_job_status auto-detection...")
        try:
            # Use the client method directly (not jsonrpc_call) to test auto-detection
            print(f"    🔍 DEBUG: Calling queue_get_job_status for job_id: {job_id_1}")
            status_result = await client.queue_get_job_status(job_id_1)
            print(f"    🔍 DEBUG: queue_get_job_status returned: {status_result}")
            print(f"    🔍 DEBUG: Type: {type(status_result)}, Keys: {status_result.keys() if isinstance(status_result, dict) else 'N/A'}")
            
            if not status_result.get("success"):
                print(f"    ❌ DEBUG: status_result.get('success') = {status_result.get('success')}")
                print(f"    ❌ DEBUG: Full status_result: {status_result}")
                log_result(
                    mode_name,
                    "queue_get_job_status_auto_detection",
                    "FAIL",
                    f"Auto-detection failed: {status_result}",
                )
                return False
            
            # Verify we got valid status data
            status_data = status_result.get("data", {})
            print(f"    🔍 DEBUG: status_data: {status_data}")
            print(f"    🔍 DEBUG: status_data type: {type(status_data)}")
            print(f"    🔍 DEBUG: 'status' in status_data: {'status' in status_data}")
            print(f"    🔍 DEBUG: 'job_id' in status_data: {'job_id' in status_data}")
            
            if "status" not in status_data and "job_id" not in status_data:
                print(f"    ❌ DEBUG: Invalid status data structure!")
                print(f"    ❌ DEBUG: status_data keys: {status_data.keys() if isinstance(status_data, dict) else 'N/A'}")
                log_result(
                    mode_name,
                    "queue_get_job_status_auto_detection",
                    "FAIL",
                    f"Invalid status data structure: {status_data}",
                )
                return False
            
            print(f"    ✅ DEBUG: Status data is valid: {status_data.get('status', 'unknown')}")
            log_result(
                mode_name,
                "queue_get_job_status_auto_detection",
                "PASS",
                f"Status: {status_data.get('status', 'unknown')}",
            )
        except Exception as e:
            print(f"    ❌ DEBUG: Exception occurred: {type(e).__name__}: {e}")
            import traceback
            traceback.print_exc()
            log_result(
                mode_name,
                "queue_get_job_status_auto_detection",
                "FAIL",
                f"Exception: {str(e)}",
            )
            return False

        # Test queue_get_job_status auto-detection fix - verify fallback behavior
        # This test verifies the fix: method should try embed_job_status first,
        # then fallback to queue_get_job_status for standard jobs
        print(
            "    🧪 Testing queue_get_job_status auto-detection fix (fallback behavior)..."
        )
        try:
            # Test with a standard job (created via queue_add_job)
            # The method should try embed_job_status first, get "not found",
            # then fallback to queue_get_job_status and succeed
            test_fallback_job_id = f"test_fallback_{uuid.uuid4().hex[:8]}"

            # Create a standard job for testing fallback
            response = await client.jsonrpc_call(
                "queue_add_job",
                {
                    "job_type": "command_execution",
                    "job_id": test_fallback_job_id,
                    "params": {
                        "command": "echo",
                        "params": {"message": "Test fallback behavior"},
                    },
                },
            )
            result = client._extract_result(response)
            if not result.get("success"):
                log_result(
                    mode_name,
                    "queue_get_job_status_fallback_test_setup",
                    "FAIL",
                    f"Failed to create test job: {result}",
                )
                return False

            # Now test queue_get_job_status - should use fallback for standard jobs
            # (embed_job_status will return "not found", then queue_get_job_status will succeed)
            status_result = await client.queue_get_job_status(test_fallback_job_id)

            if not status_result.get("success"):
                log_result(
                    mode_name,
                    "queue_get_job_status_fallback_behavior",
                    "FAIL",
                    f"Fallback test failed: {status_result}",
                )
                return False

            # Verify we got valid status data
            status_data = status_result.get("data", {})
            if "status" not in status_data and "job_id" not in status_data:
                log_result(
                    mode_name,
                    "queue_get_job_status_fallback_behavior",
                    "FAIL",
                    f"Invalid status data structure: {status_data}",
                )
                return False

            # Verify job_id matches
            if status_data.get("job_id") != test_fallback_job_id:
                log_result(
                    mode_name,
                    "queue_get_job_status_fallback_behavior",
                    "FAIL",
                    f"Job ID mismatch: expected {test_fallback_job_id}, got {status_data.get('job_id')}",
                )
                return False

            log_result(
                mode_name,
                "queue_get_job_status_fallback_behavior",
                "PASS",
                f"Fallback works correctly: Status: {status_data.get('status', 'unknown')}",
            )

            # Clean up test job
            try:
                await client.jsonrpc_call(
                    "queue_delete_job", {"job_id": test_fallback_job_id}
                )
            except Exception:
                pass  # Ignore cleanup errors

        except Exception as e:
            log_result(
                mode_name,
                "queue_get_job_status_fallback_behavior",
                "FAIL",
                f"Exception: {str(e)}",
            )
            import traceback

            traceback.print_exc()
            return False

        # Test queue_list_jobs
        response = await client.jsonrpc_call("queue_list_jobs", {})
        result = client._extract_result(response)
        if not result.get("success"):
            log_result(mode_name, "queue_list_jobs", "FAIL", str(result))
            return False
        jobs = result.get("data", {}).get("jobs", [])
        if len(jobs) < 3:
            log_result(
                mode_name,
                "queue_list_jobs",
                "FAIL",
                f"Expected at least 3 jobs, got {len(jobs)}",
            )
            return False
        log_result(mode_name, "queue_list_jobs", "PASS", f"Found {len(jobs)} jobs")

        # Test queue_start_job for all created jobs to ensure they finish within timeout
        for index, start_job_id in enumerate((job_id_1, job_id_2, job_id_3), start=1):
            response = await client.jsonrpc_call(
                "queue_start_job", {"job_id": start_job_id}
            )
            result = client._extract_result(response)
            if not result.get("success"):
                log_result(
                    mode_name,
                    (
                        "queue_start_job"
                        if index == 1
                        else f"queue_start_job_{start_job_id[:8]}"
                    ),
                    "FAIL",
                    str(result),
                )
                return False
            log_result(
                mode_name,
                (
                    "queue_start_job"
                    if index == 1
                    else f"queue_start_job_{start_job_id[:8]}"
                ),
                "PASS",
            )

        # Wait a bit for job to start
        await asyncio.sleep(1)

        # Test queue_get_job_status after start
        response = await client.jsonrpc_call(
            "queue_get_job_status", {"job_id": job_id_1}
        )
        result = client._extract_result(response)
        if not result.get("success"):
            log_result(
                mode_name, "queue_get_job_status_after_start", "FAIL", str(result)
            )
            return False
        status = result.get("data", {}).get("status")
        log_result(
            mode_name, "queue_get_job_status_after_start", "PASS", f"Status: {status}"
        )

        # Wait for jobs to complete and test completed jobs retention
        print("    ⏳ Waiting for jobs to complete...")
        max_wait = 45  # Maximum wait time in seconds
        wait_interval = 0.5
        waited = 0
        all_completed = False

        while waited < max_wait:
            all_completed = True
            for job_id in [job_id_1, job_id_2, job_id_3]:
                response = await client.jsonrpc_call(
                    "queue_get_job_status", {"job_id": job_id}
                )
                result = client._extract_result(response)
                if result.get("success"):
                    job_status = result.get("data", {}).get("status")
                    if job_status not in ("completed", "failed"):
                        all_completed = False
                        break
            if all_completed:
                break
            await asyncio.sleep(wait_interval)
            waited += wait_interval

        if not all_completed:
            log_result(
                mode_name,
                "queue_jobs_completion_wait",
                "WARN",
                "Some jobs did not complete in time",
            )
        else:
            log_result(
                mode_name,
                "queue_jobs_completion_wait",
                "PASS",
                f"All jobs completed in {waited:.1f}s",
            )

        # Test that completed jobs remain accessible and data can be retrieved (NEW FEATURE TEST)
        print("    🧪 Testing completed jobs retention and data retrieval...")
        completed_job_ids = []
        jobs_with_data = []

        for job_id in [job_id_1, job_id_2, job_id_3]:
            response = await client.jsonrpc_call(
                "queue_get_job_status", {"job_id": job_id}
            )
            result = client._extract_result(response)
            if result.get("success"):
                job_status = result.get("data", {}).get("status")
                if job_status in ("completed", "failed"):
                    completed_job_ids.append(job_id)
                    # Verify we can get the result data
                    job_result = result.get("data", {}).get("result")
                    if job_result is not None:
                        jobs_with_data.append(job_id)
                        # Verify result contains expected fields
                        if isinstance(job_result, dict):
                            has_data = (
                                "job_id" in job_result
                                or "status" in job_result
                                or "result" in job_result
                                or len(job_result) > 0
                            )
                            if has_data:
                                log_result(
                                    mode_name,
                                    f"completed_job_data_{job_id[:8]}",
                                    "PASS",
                                    f"Status: {job_status}, has result data",
                                )
                            else:
                                log_result(
                                    mode_name,
                                    f"completed_job_data_{job_id[:8]}",
                                    "WARN",
                                    "Result data is empty",
                                )
                        else:
                            log_result(
                                mode_name,
                                f"completed_job_data_{job_id[:8]}",
                                "PASS",
                                f"Status: {job_status}, result type: {type(job_result).__name__}",
                            )
                    else:
                        log_result(
                            mode_name,
                            f"completed_job_data_{job_id[:8]}",
                            "WARN",
                            "No result data available",
                        )

        if len(completed_job_ids) > 0:
            log_result(
                mode_name,
                "completed_jobs_retention",
                "PASS",
                f"{len(completed_job_ids)} completed jobs accessible",
            )
            if len(jobs_with_data) > 0:
                log_result(
                    mode_name,
                    "completed_jobs_data_retrieval",
                    "PASS",
                    f"{len(jobs_with_data)}/{len(completed_job_ids)} jobs have retrievable data",
                )
            else:
                log_result(
                    mode_name,
                    "completed_jobs_data_retrieval",
                    "WARN",
                    "Completed jobs accessible but no data found",
                )
        else:
            log_result(
                mode_name, "completed_jobs_retention", "FAIL", "No completed jobs found"
            )

        # Test that completed jobs don't count toward limits (NEW FEATURE TEST)
        print("    🧪 Testing that completed jobs don't count toward limits...")
        new_job_id = f"test_limit_{uuid.uuid4().hex[:8]}"
        try:
            response = await client.jsonrpc_call(
                "queue_add_job",
                {
                    "job_type": "data_processing",
                    "job_id": new_job_id,
                    "params": {"data": {"test": "limit"}, "operation": "test"},
                },
            )
            result = client._extract_result(response)
            if result.get("success"):
                log_result(
                    mode_name,
                    "completed_jobs_limit_exclusion",
                    "PASS",
                    "New job added despite completed jobs",
                )
            else:
                log_result(
                    mode_name,
                    "completed_jobs_limit_exclusion",
                    "WARN",
                    "Could not add new job (may be expected)",
                )
        except Exception as e:
            log_result(
                mode_name,
                "completed_jobs_limit_exclusion",
                "WARN",
                f"Exception: {str(e)}",
            )

        # Test queue_get_job_logs - NEW FEATURE TEST
        print("    🧪 Testing queue_get_job_logs command...")
        logs_tested = 0
        logs_with_output = 0

        for job_id in [job_id_1, job_id_2, job_id_3]:
            try:
                response = await client.jsonrpc_call(
                    "queue_get_job_logs", {"job_id": job_id}
                )
                result = client._extract_result(response)
                if result.get("success"):
                    logs_tested += 1
                    logs_data = result.get("data", {})
                    stdout = logs_data.get("stdout", [])
                    stderr = logs_data.get("stderr", [])
                    stdout_lines = logs_data.get("stdout_lines", 0)
                    stderr_lines = logs_data.get("stderr_lines", 0)

                    # Check if logs are available (even if empty)
                    has_stdout = isinstance(stdout, list) and len(stdout) > 0
                    has_stderr = isinstance(stderr, list) and len(stderr) > 0

                    if has_stdout or has_stderr:
                        logs_with_output += 1
                        log_result(
                            mode_name,
                            f"queue_get_job_logs_{job_id[:8]}",
                            "PASS",
                            f"stdout: {stdout_lines} lines, stderr: {stderr_lines} lines",
                        )
                    else:
                        # Logs command works but no output (this is OK for some jobs)
                        log_result(
                            mode_name,
                            f"queue_get_job_logs_{job_id[:8]}",
                            "PASS",
                            f"Logs accessible (stdout: {stdout_lines}, stderr: {stderr_lines} lines)",
                        )
                else:
                    log_result(
                        mode_name,
                        f"queue_get_job_logs_{job_id[:8]}",
                        "FAIL",
                        f"Failed to get logs: {result.get('message', 'Unknown error')}",
                    )
            except Exception as e:
                log_result(
                    mode_name,
                    f"queue_get_job_logs_{job_id[:8]}",
                    "FAIL",
                    f"Exception: {str(e)}",
                )

        if logs_tested > 0:
            log_result(
                mode_name,
                "queue_get_job_logs",
                "PASS",
                f"Successfully retrieved logs for {logs_tested} jobs ({logs_with_output} with output)",
            )
        else:
            log_result(
                mode_name,
                "queue_get_job_logs",
                "FAIL",
                "Failed to retrieve logs for any job",
            )

        # Test PeriodicLoggingJob - NEW FEATURE TEST for stdout logging
        print("    🧪 Testing PeriodicLoggingJob with stdout logging...")
        periodic_job_id = f"test_periodic_{uuid.uuid4().hex[:8]}"
        try:
            # Add periodic logging job (runs for 2 minutes, logs every 10 seconds for testing)
            response = await client.jsonrpc_call(
                "queue_add_job",
                {
                    "job_type": "periodic_logging",
                    "job_id": periodic_job_id,
                    "params": {
                        "duration_minutes": 0.5,  # Shorter duration to allow quick shutdown
                        "interval_seconds": 5,  # Faster logging interval for testing
                        "message_prefix": "Test periodic log",
                    },
                },
            )
            result = client._extract_result(response)
            if not result.get("success"):
                log_result(
                    mode_name,
                    "queue_add_periodic_logging_job",
                    "FAIL",
                    str(result),
                )
            else:
                log_result(
                    mode_name,
                    "queue_add_periodic_logging_job",
                    "PASS",
                )

                # Start the job
                response = await client.jsonrpc_call(
                    "queue_start_job", {"job_id": periodic_job_id}
                )
                result = client._extract_result(response)
                if result.get("success"):
                    log_result(
                        mode_name,
                        "queue_start_periodic_logging_job",
                        "PASS",
                    )

                    # Wait a bit for logs to accumulate
                    await asyncio.sleep(15)  # Wait for at least one log message

                    # Get logs
                    response = await client.jsonrpc_call(
                        "queue_get_job_logs", {"job_id": periodic_job_id}
                    )
                    result = client._extract_result(response)
                    if result.get("success"):
                        logs_data = result.get("data", {})
                        stdout = logs_data.get("stdout", [])
                        stderr = logs_data.get("stderr", [])
                        stdout_lines = logs_data.get("stdout_lines", 0)
                        stderr_lines = logs_data.get("stderr_lines", 0)

                        if stdout_lines > 0:
                            log_result(
                                mode_name,
                                "queue_get_periodic_job_logs",
                                "PASS",
                                f"Retrieved {stdout_lines} stdout lines, {stderr_lines} stderr lines",
                            )
                            # Show first few log lines
                            if stdout and len(stdout) > 0:
                                first_lines = stdout[:3]
                                log_result(
                                    mode_name,
                                    "queue_periodic_logs_content",
                                    "PASS",
                                    f"First lines: {first_lines}",
                                )
                        else:
                            log_result(
                                mode_name,
                                "queue_get_periodic_job_logs",
                                "WARN",
                                "Logs accessible but no stdout yet (job may be starting)",
                            )
                    else:
                        log_result(
                            mode_name,
                            "queue_get_periodic_job_logs",
                            "FAIL",
                            f"Failed to get logs: {result.get('message')}",
                        )

                    # Stop the job to clean up and verify stop command works while job is running
                    if not await stop_job_with_fallback(periodic_job_id):
                        return False
                else:
                    log_result(
                        mode_name,
                        "queue_start_periodic_logging_job",
                        "FAIL",
                        str(result),
                    )
        except Exception as e:
            log_result(
                mode_name,
                "queue_periodic_logging_test",
                "FAIL",
                f"Exception: {str(e)}",
            )

        # Verify completed jobs are still accessible and data can be retrieved after operations (NEW FEATURE TEST)
        print(
            "    🧪 Verifying completed jobs and data still accessible after operations..."
        )
        still_accessible = 0
        still_have_data = 0
        for job_id in completed_job_ids[:2]:  # Check first 2 completed jobs
            try:
                response = await client.jsonrpc_call(
                    "queue_get_job_status", {"job_id": job_id}
                )
                result = client._extract_result(response)
                if result.get("success"):
                    job_status = result.get("data", {}).get("status")
                    if job_status in ("completed", "failed"):
                        still_accessible += 1
                        # Verify data is still retrievable
                        job_result = result.get("data", {}).get("result")
                        if job_result is not None:
                            still_have_data += 1
                            log_result(
                                mode_name,
                                f"data_retrieval_after_ops_{job_id[:8]}",
                                "PASS",
                                f"Data still retrievable for {job_id[:8]}",
                            )
                        else:
                            log_result(
                                mode_name,
                                f"data_retrieval_after_ops_{job_id[:8]}",
                                "WARN",
                                f"Job accessible but no data for {job_id[:8]}",
                            )
            except Exception as e:
                log_result(
                    mode_name,
                    f"data_retrieval_after_ops_{job_id[:8]}",
                    "WARN",
                    f"Exception: {str(e)}",
                )

        if still_accessible > 0:
            log_result(
                mode_name,
                "completed_jobs_persistence",
                "PASS",
                f"{still_accessible} completed jobs still accessible",
            )
            if still_have_data > 0:
                log_result(
                    mode_name,
                    "completed_jobs_data_persistence",
                    "PASS",
                    f"{still_have_data}/{still_accessible} jobs still have retrievable data",
                )
            else:
                log_result(
                    mode_name,
                    "completed_jobs_data_persistence",
                    "WARN",
                    "Jobs accessible but data may have been lost",
                )
        else:
            log_result(
                mode_name,
                "completed_jobs_persistence",
                "WARN",
                "Some completed jobs may have been cleaned up",
            )

        # Test queue_delete_job (only delete new test job, keep completed ones for verification)
        try:
            if "new_job_id" in locals():
                response = await client.jsonrpc_call(
                    "queue_delete_job", {"job_id": new_job_id}
                )
                result = client._extract_result(response)
                if not result.get("success"):
                    log_result(
                        mode_name,
                        f"queue_delete_job_{new_job_id[:8]}",
                        "WARN",
                        str(result),
                    )
                else:
                    log_result(mode_name, f"queue_delete_job_{new_job_id[:8]}", "PASS")
        except Exception as e:
            log_result(mode_name, "queue_delete_job", "WARN", str(e))

        # Test new convenience methods for command queue operations
        # Create a new job for testing convenience methods
        test_job_id = f"test_conv_{uuid.uuid4().hex[:8]}"
        response = await client.jsonrpc_call(
            "queue_add_job",
            {
                "job_type": "command_execution",
                "job_id": test_job_id,
                "params": {
                    "command": "echo",
                    "params": {"message": "Test convenience methods"},
                },
            },
        )
        result = client._extract_result(response)
        if not result.get("success"):
            log_result(mode_name, "queue_add_job_conv_test", "FAIL", str(result))
            return False

        # Test get_command_status (convenience method)
        try:
            status_result = await client.get_command_status(test_job_id)
            if status_result.get("success") or status_result.get("data"):
                log_result(mode_name, "get_command_status", "PASS")
            else:
                log_result(
                    mode_name,
                    "get_command_status",
                    "FAIL",
                    "Unexpected response format",
                )
                return False
        except Exception as e:
            log_result(mode_name, "get_command_status", "FAIL", str(e))
            return False

        # Test list_queued_commands (convenience method)
        try:
            queued_commands = await client.list_queued_commands(status=None, limit=10)
            if queued_commands.get("success") or queued_commands.get("data"):
                count = queued_commands.get("data", {}).get("total_count", 0)
                log_result(
                    mode_name, "list_queued_commands", "PASS", f"Found {count} commands"
                )
            else:
                log_result(
                    mode_name,
                    "list_queued_commands",
                    "FAIL",
                    "Unexpected response format",
                )
                return False
        except Exception as e:
            log_result(mode_name, "list_queued_commands", "FAIL", str(e))
            return False

        # Test list_all_queue_jobs (convenience method)
        try:
            all_jobs = await client.list_all_queue_jobs(
                status=None, job_type="command_execution", limit=10
            )
            if all_jobs.get("success") or all_jobs.get("data"):
                count = all_jobs.get("data", {}).get("total_count", 0)
                log_result(
                    mode_name, "list_all_queue_jobs", "PASS", f"Found {count} jobs"
                )
            elif all_jobs.get("error"):
                log_result(
                    mode_name,
                    "list_all_queue_jobs",
                    "FAIL",
                    f"Queue error: {all_jobs['error']}",
                )
                return False
            else:
                log_result(
                    mode_name,
                    "list_all_queue_jobs",
                    "FAIL",
                    "Unexpected response format",
                )
                return False
        except Exception as e:
            log_result(mode_name, "list_all_queue_jobs", "FAIL", str(e))
            return False

        # Test execute_command_unified - immediate execution (echo)
        print("    🧪 Testing execute_command_unified - immediate execution...")
        try:
            result = await client.execute_command_unified(
                "echo", {"message": "Test unified immediate"}
            )
            if result.get("mode") == "immediate" and result.get("queued") is False:
                log_result(mode_name, "execute_command_unified_immediate", "PASS")
            else:
                log_result(
                    mode_name,
                    "execute_command_unified_immediate",
                    "FAIL",
                    f"Unexpected result: {result}",
                )
                return False
        except Exception as e:
            log_result(mode_name, "execute_command_unified_immediate", "FAIL", str(e))
            return False

        # Test embed_queue command (if available) - comprehensive test
        print("    🧪 Testing embed_queue command comprehensively...")
        try:
            # Check if embed_queue command is available
            help_result = await client.help()
            commands_dict = help_result.get("data", {}).get("commands", {})
            # commands can be either dict (name -> info) or list of dicts
            if isinstance(commands_dict, dict):
                has_embed_queue = "embed_queue" in commands_dict
            elif isinstance(commands_dict, list):
                has_embed_queue = any(
                    (isinstance(cmd, dict) and cmd.get("name") == "embed_queue")
                    or (isinstance(cmd, str) and cmd == "embed_queue")
                    for cmd in commands_dict
                )
            else:
                has_embed_queue = False

            if has_embed_queue:
                # Step 1: Create job via embed_queue
                embed_result = await client.jsonrpc_call(
                    "embed_queue",
                    {
                        "command": "echo",
                        "params": {"message": "Test embed_queue from pipeline"},
                    },
                )
                embed_extracted = client._extract_result(embed_result)
                if not embed_extracted.get("success"):
                    log_result(
                        mode_name,
                        "embed_queue_creation",
                        "FAIL",
                        f"Failed to create embed_queue job: {embed_extracted}",
                    )
                    return False

                embed_job_id = embed_extracted.get("data", {}).get("job_id")
                log_result(mode_name, "embed_queue_creation", "PASS")

                # Wait a bit for job to be registered
                await asyncio.sleep(0.5)

                # Step 2: Test embed_job_status directly
                embed_status_result = await client.jsonrpc_call(
                    "embed_job_status",
                    {"job_id": embed_job_id},
                )
                embed_status_extracted = client._extract_result(embed_status_result)
                if not embed_status_extracted.get("success"):
                    log_result(
                        mode_name,
                        "embed_job_status_direct",
                        "FAIL",
                        f"embed_job_status failed: {embed_status_extracted}",
                    )
                    return False

                embed_status = embed_status_extracted.get("data", {}).get("status")
                log_result(
                    mode_name,
                    "embed_job_status_direct",
                    "PASS",
                    f"Status: {embed_status}",
                )

                # Step 3: Test queue_get_job_status auto-detection for embed_queue job
                auto_status_result = await client.queue_get_job_status(embed_job_id)
                if not auto_status_result.get("success"):
                    log_result(
                        mode_name,
                        "queue_get_job_status_embed_auto_detection",
                        "FAIL",
                        f"Auto-detection failed: {auto_status_result}",
                    )
                    return False

                auto_status = auto_status_result.get("data", {}).get("status")
                log_result(
                    mode_name,
                    "queue_get_job_status_embed_auto_detection",
                    "PASS",
                    f"Auto-detected embed_job_status, status: {auto_status}",
                )

                # Step 4: Start the job and wait for completion
                print("    🧪 Testing embed_queue job execution...")
                start_result = await client.jsonrpc_call(
                    "queue_start_job",
                    {"job_id": embed_job_id},
                )
                start_extracted = client._extract_result(start_result)
                if not start_extracted.get("success"):
                    log_result(
                        mode_name,
                        "embed_queue_start_job",
                        "FAIL",
                        f"Failed to start job: {start_extracted}",
                    )
                    return False

                log_result(mode_name, "embed_queue_start_job", "PASS")

                # Wait for job to complete
                max_wait = 15
                waited = 0
                completed = False
                while waited < max_wait:
                    status_check = await client.queue_get_job_status(embed_job_id)
                    status = status_check.get("data", {}).get("status")
                    if status in ("completed", "failed"):
                        completed = True
                        log_result(
                            mode_name,
                            "embed_queue_job_completion",
                            "PASS",
                            f"Job completed with status: {status}",
                        )
                        break
                    await asyncio.sleep(0.5)
                    waited += 0.5

                if not completed:
                    log_result(
                        mode_name,
                        "embed_queue_job_completion",
                        "WARN",
                        f"Job did not complete within {max_wait} seconds",
                    )

                # Step 5: Verify result data is accessible
                final_status = await client.queue_get_job_status(embed_job_id)
                result_data = final_status.get("data", {}).get("result")
                if result_data:
                    log_result(
                        mode_name,
                        "embed_queue_result_retrieval",
                        "PASS",
                        "Result data accessible",
                    )
                else:
                    log_result(
                        mode_name,
                        "embed_queue_result_retrieval",
                        "WARN",
                        "No result data available",
                    )

                # Step 6: Compare embed_queue vs standard queue_add_job
                print("    🧪 Comparing embed_queue vs standard queue_add_job...")
                standard_job_id = f"standard_compare_{uuid.uuid4().hex[:8]}"
                standard_result = await client.jsonrpc_call(
                    "queue_add_job",
                    {
                        "job_type": "command_execution",
                        "job_id": standard_job_id,
                        "params": {
                            "command": "echo",
                            "params": {"message": "Standard queue comparison"},
                        },
                    },
                )
                standard_extracted = client._extract_result(standard_result)
                if standard_extracted.get("success"):
                    # Both should work with queue_get_job_status
                    embed_check = await client.queue_get_job_status(embed_job_id)
                    standard_check = await client.queue_get_job_status(standard_job_id)

                    if embed_check.get("success") and standard_check.get("success"):
                        log_result(
                            mode_name,
                            "embed_queue_vs_standard_comparison",
                            "PASS",
                            "Both job types work with queue_get_job_status",
                        )
                    else:
                        log_result(
                            mode_name,
                            "embed_queue_vs_standard_comparison",
                            "FAIL",
                            "One of job types failed status check",
                        )

                    # Clean up standard job
                    try:
                        await client.jsonrpc_call(
                            "queue_delete_job", {"job_id": standard_job_id}
                        )
                    except Exception:
                        pass

            else:
                log_result(
                    mode_name,
                    "embed_queue_creation",
                    "SKIP",
                    "embed_queue command not available in this mode",
                )
        except Exception as e:
            log_result(
                mode_name,
                "embed_queue_test",
                "FAIL",
                f"Exception: {str(e)}",
            )
            import traceback

            traceback.print_exc()
            return False

        # Test custom commands in spawn mode (child processes)
        # This verifies that custom commands registered via hooks are available
        # in child processes when using spawn mode (required for CUDA)
        print("    🧪 Testing custom commands in spawn mode (spawn mode fix)...")
        try:
            # Check if embed_queue command is available (it's registered via hook)
            help_result = await client.help()
            commands_dict = help_result.get("data", {}).get("commands", {})
            # commands can be either dict (name -> info) or list of dicts
            if isinstance(commands_dict, dict):
                has_embed_queue = "embed_queue" in commands_dict
            elif isinstance(commands_dict, list):
                has_embed_queue = any(
                    (isinstance(cmd, dict) and cmd.get("name") == "embed_queue")
                    or (isinstance(cmd, str) and cmd == "embed_queue")
                    for cmd in commands_dict
                )
            else:
                has_embed_queue = False

            if has_embed_queue:
                # Create job via embed_queue (this will execute in child process)
                embed_test_result = await client.jsonrpc_call(
                    "embed_queue",
                    {
                        "command": "echo",
                        "params": {"message": "Spawn mode test"},
                    },
                )

                embed_test_extracted = client._extract_result(embed_test_result)
                if embed_test_extracted.get("success"):
                    embed_test_job_id = embed_test_extracted.get("data", {}).get(
                        "job_id"
                    )

                    # Start the job (will execute in child process)
                    await asyncio.sleep(0.5)
                    start_test_result = await client.jsonrpc_call(
                        "queue_start_job",
                        {"job_id": embed_test_job_id},
                    )

                    start_test_extracted = client._extract_result(start_test_result)
                    if start_test_extracted.get("success"):
                        # Wait for completion
                        max_wait = 10
                        waited = 0
                        completed = False

                        while waited < max_wait:
                            status_check = await client.queue_get_job_status(
                                embed_test_job_id
                            )
                            status = status_check.get("data", {}).get("status")

                            if status == "completed":
                                completed = True
                                log_result(
                                    mode_name,
                                    "custom_commands_spawn_mode",
                                    "PASS",
                                    "Custom command executed successfully in child process",
                                )
                                break
                            elif status == "failed":
                                error = status_check.get("data", {}).get("error")
                                # Check if error is about command not found (the bug we're fixing)
                                if "not found" in str(error).lower():
                                    log_result(
                                        mode_name,
                                        "custom_commands_spawn_mode",
                                        "FAIL",
                                        f"Command not found in child process: {error}",
                                    )
                                    return False
                                else:
                                    log_result(
                                        mode_name,
                                        "custom_commands_spawn_mode",
                                        "WARN",
                                        f"Job failed with other error: {error}",
                                    )
                                break

                            await asyncio.sleep(0.5)
                            waited += 0.5

                        if not completed:
                            log_result(
                                mode_name,
                                "custom_commands_spawn_mode",
                                "WARN",
                                f"Job did not complete within {max_wait} seconds",
                            )
                    else:
                        log_result(
                            mode_name,
                            "custom_commands_spawn_mode",
                            "WARN",
                            "Failed to start job for spawn mode test",
                        )
                else:
                    log_result(
                        mode_name,
                        "custom_commands_spawn_mode",
                        "WARN",
                        "Failed to create job for spawn mode test",
                    )
            else:
                log_result(
                    mode_name,
                    "custom_commands_spawn_mode",
                    "SKIP",
                    "embed_queue command not available (hooks may not be registered)",
                )
        except Exception as e:
            log_result(
                mode_name,
                "custom_commands_spawn_mode",
                "WARN",
                f"Exception: {str(e)}",
            )

        # Test execute_command_unified - queued execution with auto_poll (long_task)
        print("    🧪 Testing execute_command_unified - queued with auto_poll...")
        try:
            status_updates = []

            async def status_hook(status: dict) -> None:
                status_updates.append(status.get("status"))

            result = await client.execute_command_unified(
                "long_task",
                {"seconds": 2},
                auto_poll=True,
                poll_interval=0.5,
                timeout=10.0,
                status_hook=status_hook,
            )
            # Job might be auto-deleted after completion, so status could be "completed" or have warning
            status_val = result.get("status", "").lower()
            if (
                result.get("mode") == "queued"
                and result.get("queued") is True
                and (status_val in ("completed", "success") or result.get("warning"))
            ):
                log_result(
                    mode_name,
                    "execute_command_unified_queued_auto_poll",
                    "PASS",
                    f"Status: {status_val or 'auto-deleted'}, updates: {len(status_updates)}",
                )
            else:
                log_result(
                    mode_name,
                    "execute_command_unified_queued_auto_poll",
                    "FAIL",
                    f"Unexpected result: {result}",
                )
                return False
        except Exception as e:
            log_result(
                mode_name, "execute_command_unified_queued_auto_poll", "FAIL", str(e)
            )
            import traceback

            traceback.print_exc()
            return False

        # Test execute_command_unified - queued execution without auto_poll
        print("    🧪 Testing execute_command_unified - queued without auto_poll...")
        try:
            result = await client.execute_command_unified(
                "long_task", {"seconds": 1}, auto_poll=False
            )
            if (
                result.get("mode") == "queued"
                and result.get("queued") is True
                and "job_id" in result
            ):
                job_id = result.get("job_id")
                # Wait for job to complete manually
                await asyncio.sleep(3)
                status_result = await client.get_command_status(job_id)
                final_status = (
                    status_result.get("data", {}).get("status")
                    if isinstance(status_result, dict)
                    else status_result.get("status")
                )
                if final_status == "completed":
                    log_result(
                        mode_name,
                        "execute_command_unified_queued_no_auto_poll",
                        "PASS",
                        f"Job {job_id[:8]} completed",
                    )
                else:
                    log_result(
                        mode_name,
                        "execute_command_unified_queued_no_auto_poll",
                        "WARN",
                        f"Job status: {final_status}",
                    )
            else:
                log_result(
                    mode_name,
                    "execute_command_unified_queued_no_auto_poll",
                    "FAIL",
                    f"Unexpected result: {result}",
                )
                return False
        except Exception as e:
            log_result(
                mode_name,
                "execute_command_unified_queued_no_auto_poll",
                "FAIL",
                str(e),
            )
            return False

        # Test cancel_command (convenience method)
        try:
            cancel_result = await client.cancel_command(test_job_id)
            if cancel_result.get("success"):
                log_result(mode_name, "cancel_command", "PASS")
            else:
                # Try manual delete as fallback
                try:
                    await client.queue_delete_job(test_job_id)
                    log_result(
                        mode_name, "cancel_command", "PASS", "Used fallback delete"
                    )
                except Exception:
                    log_result(
                        mode_name,
                        "cancel_command",
                        "FAIL",
                        "Cancel and fallback both failed",
                    )
                    return False
        except Exception as e:
            # Try manual delete as fallback
            try:
                await client.queue_delete_job(test_job_id)
                log_result(
                    mode_name,
                    "cancel_command",
                    "PASS",
                    "Used fallback delete after exception",
                )
            except Exception:
                log_result(mode_name, "cancel_command", "FAIL", str(e))
                return False

        return True
    except Exception as e:
        log_result(mode_name, "queue_commands", "FAIL", str(e))
        import traceback

        traceback.print_exc()
        return False


async def _test_mode(mode_spec: dict, server_port: int, proxy_port: int) -> bool:
    """Test a single mode comprehensively."""
    mode_name = mode_spec["name"]
    print(f"\n{'=' * 80}")
    print(f"Testing: {mode_name}")
    print(f"Server port: {server_port}, Proxy port: {proxy_port}")
    print(f"{'=' * 80}")

    server_process: Optional[subprocess.Popen] = None
    proxy_process: Optional[subprocess.Popen] = None

    try:
        # Kill processes on ports before starting
        kill_process_on_port(server_port)
        kill_process_on_port(proxy_port)
        await asyncio.sleep(0.5)

        # Generate server config
        print("📝 Generating server config...")
        try:
            server_config_path = generate_server_config(
                mode_spec, server_port, proxy_port
            )
            log_result(mode_name, "config_generation_server", "PASS")
        except Exception as e:
            error_msg = f"Failed to generate server config: {e}"
            print(f"❌ {error_msg}")
            log_result(mode_name, "config_generation_server", "FAIL", error_msg)
            raise RuntimeError(f"CRITICAL: {error_msg}") from e

        # Generate client config
        print("📝 Generating client config...")
        try:
            client_config_path = generate_client_config(mode_spec, server_port)
            log_result(mode_name, "config_generation_client", "PASS")
        except Exception as e:
            error_msg = f"Failed to generate client config: {e}"
            print(f"❌ {error_msg}")
            log_result(mode_name, "config_generation_client", "FAIL", error_msg)
            raise RuntimeError(f"CRITICAL: {error_msg}") from e

        # Load client config
        client_config = json.loads(client_config_path.read_text(encoding="utf-8"))

        # Update client config with actual port
        client_config["port"] = server_port

        # Determine URL based on protocol and port
        protocol = mode_spec["protocol"]
        scheme = "https" if protocol in ("https", "mtls") else "http"
        server_url = f"{scheme}://localhost:{server_port}"

        # Start proxy if needed
        if mode_spec.get("with_proxy", True):
            print("🚀 Starting proxy...")
            proxy_process = start_proxy("http", "localhost", proxy_port)
            if proxy_process:
                if wait_for_port("localhost", proxy_port, timeout=10):
                    log_result(mode_name, "proxy_start", "PASS")
                else:
                    log_result(mode_name, "proxy_start", "FAIL", "Proxy port not ready")
                    return False
            else:
                log_result(mode_name, "proxy_start", "FAIL", "Failed to start proxy")
                return False

            # DEBUG: Test proxy with curl
            print("\n" + "=" * 80)
            print("🔍 DEBUG: Testing proxy with curl...")
            print("=" * 80)
            import requests

            try:
                # Test health endpoint
                health_url = f"http://localhost:{proxy_port}/health"
                print(f"   Testing: GET {health_url}")
                response = requests.get(health_url, timeout=5)
                print(f"   ✅ Health check: {response.status_code} - {response.json()}")

                # Test servers endpoint
                servers_url = f"http://localhost:{proxy_port}/servers"
                print(f"   Testing: GET {servers_url}")
                response = requests.get(servers_url, timeout=5)
                servers = response.json()
                print(f"   ✅ Servers list: {response.status_code} - {servers}")
                print(
                    f"   📋 Registered servers count: {len(servers) if isinstance(servers, list) else 'N/A'}"
                )

                # Test registration endpoint with UUID
                test_uuid = "123e4567-e89b-12d3-a456-426614174000"
                register_url = f"http://localhost:{proxy_port}/register"
                register_payload = {
                    "server_id": "test-debug-server",
                    "server_url": "http://localhost:5000",
                    "uuid": test_uuid,
                    "capabilities": ["jsonrpc"],
                    "metadata": {"uuid": test_uuid},
                }
                print(f"   Testing: POST {register_url}")
                print(f"   Payload UUID: {test_uuid}")
                response = requests.post(register_url, json=register_payload, timeout=5)
                print(f"   ✅ Registration: {response.status_code} - {response.json()}")

                # Check servers again
                response = requests.get(servers_url, timeout=5)
                servers_after = response.json()
                print(
                    f"   ✅ Servers after registration: {len(servers_after) if isinstance(servers_after, list) else 'N/A'}"
                )
                if isinstance(servers_after, list) and len(servers_after) > 0:
                    print(f"   📋 Server data: {servers_after[0]}")
                    server_uuid = servers_after[0].get("uuid")
                    if server_uuid:
                        if server_uuid == test_uuid:
                            print(f"   ✅ UUID matches: {test_uuid}")
                        else:
                            print(
                                f"   ❌ UUID mismatch! Expected: {test_uuid}, Got: {server_uuid}"
                            )
                    else:
                        print(f"   ❌ UUID missing in server data!")
            except Exception as e:
                print(f"   ❌ Error testing proxy: {e}")
                import traceback

                traceback.print_exc()

            print("=" * 80)

        # Start server
        print("🚀 Starting server...")
        # Convert to absolute path to avoid path resolution issues
        server_config_absolute = str(server_config_path.resolve())
        server_process = start_server(server_config_absolute, server_port)

        if not server_process:
            log_result(
                mode_name, "server_start", "FAIL", "Failed to start server process"
            )
            return False

        # Wait longer for server to start and check stderr for errors
        if not wait_for_port("localhost", server_port, timeout=30):
            # Check if process is still running
            if server_process and server_process.poll() is not None:
                # Process exited, read stderr and stdout
                try:
                    if server_process.stderr:
                        error_output = server_process.stderr.read(4096).decode(
                            "utf-8", errors="ignore"
                        )
                        if error_output:
                            print(f"   ❌ Server stderr: {error_output[:1000]}")
                    if server_process.stdout:
                        stdout_output = server_process.stdout.read(4096).decode(
                            "utf-8", errors="ignore"
                        )
                        if stdout_output:
                            print(f"   📄 Server stdout: {stdout_output[:1000]}")
                except Exception as e:
                    print(f"   ⚠️  Error reading server output: {e}")
            else:
                print(
                    f"   ⚠️  Server process still running but port {server_port} not accessible"
                )
            log_result(
                mode_name, "server_start", "FAIL", "Server port not ready after 30s"
            )
            return False
        log_result(mode_name, "server_start", "PASS")

        # Wait for server to fully initialize (queue manager, etc.)
        await asyncio.sleep(3)

        # DEBUG: Stop here and test server with curl
        print("\n" + "=" * 80)
        print("🔍 DEBUG: Testing server with curl...")
        print("=" * 80)
        import requests

        try:
            # Test health endpoint
            health_url = f"{server_url}/health"
            print(f"   Testing: GET {health_url}")
            headers = {}
            if mode_spec.get("token"):
                headers["X-API-Key"] = mode_spec["token"]
            response = requests.get(
                health_url, headers=headers, timeout=5, verify=not mode_spec["use_ssl"]
            )
            print(f"   ✅ Health check: {response.status_code} - {response.json()}")

            # Test JSON-RPC endpoint
            jsonrpc_url = f"{server_url}/api/jsonrpc"
            jsonrpc_payload = {
                "jsonrpc": "2.0",
                "method": "echo",
                "params": {"message": "debug test"},
                "id": 1,
            }
            print(f"   Testing: POST {jsonrpc_url}")
            print(f"   Payload: {jsonrpc_payload}")
            response = requests.post(
                jsonrpc_url,
                json=jsonrpc_payload,
                headers=headers,
                timeout=5,
                verify=not mode_spec["use_ssl"],
            )
            print(f"   ✅ JSON-RPC: {response.status_code} - {response.json()}")

            # Check if server registered with proxy
            servers_url = f"http://localhost:{proxy_port}/servers"
            print(f"   Testing: GET {servers_url}")
            response = requests.get(servers_url, timeout=5)
            servers = response.json()
            print(f"   ✅ Proxy servers list: {response.status_code}")
            print(
                f"   📋 Registered servers count: {len(servers) if isinstance(servers, list) else 'N/A'}"
            )

            # Get expected server_id from config
            server_config_data = json.loads(
                server_config_path.read_text(encoding="utf-8")
            )
            registration_config = server_config_data.get("registration", {})
            expected_server_id = registration_config.get(
                "server_id"
            ) or registration_config.get("server_name")
            if not expected_server_id:
                # Generate expected server_id using same logic as build_server_metadata
                host = server_config_data.get("server", {}).get("host", "127.0.0.1")
                port = server_config_data.get("server", {}).get("port", server_port)
                expected_server_id = f"mcp-adapter-{host}-{port}"
            print(f"   🔍 Expected server_id: {expected_server_id}")

            if isinstance(servers, list) and len(servers) > 0:
                print(f"   📋 All registered servers:")
                for srv in servers:
                    srv_id = srv.get("server_id") or srv.get("name")
                    srv_uuid = srv.get("uuid")
                    print(f"      - {srv_id} (UUID: {srv_uuid})")
                    if srv_id == expected_server_id:
                        print(f"   ✅ Found our server!")
                        if srv_uuid:
                            print(f"   ✅ UUID present: {srv_uuid}")
                            # Check UUID matches config
                            config_uuid = registration_config.get("instance_uuid")
                            if (
                                config_uuid
                                and srv_uuid.lower() == str(config_uuid).lower()
                            ):
                                print(f"   ✅ UUID matches config: {config_uuid}")
                            else:
                                print(
                                    f"   ⚠️  UUID mismatch! Config: {config_uuid}, Server: {srv_uuid}"
                                )
                        else:
                            print(f"   ❌ UUID missing in server data!")
            else:
                print(f"   ⚠️  No servers registered yet")
        except Exception as e:
            print(f"   ❌ Error testing server: {e}")
            import traceback

            traceback.print_exc()

        print("=" * 80)

        # Check UUID in config
        server_config_data = json.loads(server_config_path.read_text(encoding="utf-8"))
        registration_config = server_config_data.get("registration", {})
        config_uuid = registration_config.get("instance_uuid")
        print(f"   🔍 UUID in config: {config_uuid}")

        if not config_uuid:
            print(f"   ❌ UUID missing in config!")
        else:
            # Validate UUID format
            try:
                uuid_obj = uuid.UUID(str(config_uuid))
                if uuid_obj.version == 4:
                    print(f"   ✅ UUID is valid UUID4: {config_uuid}")
                else:
                    print(f"   ⚠️  UUID is not UUID4 (version: {uuid_obj.version})")
            except Exception as e:
                print(f"   ❌ UUID format invalid: {e}")

        # Check server logs for registration attempts
        if server_process and server_process.stdout:
            try:
                # Read available stdout without blocking
                import select
                import fcntl

                # Set stdout to non-blocking
                fd = server_process.stdout.fileno()
                fl = fcntl.fcntl(fd, fcntl.F_GETFL)
                fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
                try:
                    stdout_data = server_process.stdout.read(8192).decode(
                        "utf-8", errors="ignore"
                    )
                    if stdout_data:
                        print(
                            f"   📄 Server stdout (first 2000 chars): {stdout_data[:2000]}"
                        )
                        # Check for registration-related messages
                        if (
                            "start_heartbeat" in stdout_data
                            or "REG_MANAGER" in stdout_data
                            or "📡" in stdout_data
                        ):
                            print(
                                f"   ✅ Found registration-related messages in server logs"
                            )
                        elif "Registration is disabled" in stdout_data:
                            print(f"   ⚠️  Registration is disabled in server logs")
                except (BlockingIOError, OSError):
                    pass  # No data available
            except Exception as e:
                print(f"   ⚠️  Could not read server stdout: {e}")

        import os

        # Test health endpoint
        print("🏥 Testing health endpoint...")
        health_ok, health_data = check_server_health(
            server_url,
            mode_spec["use_ssl"],
            mode_spec.get("token"),
            mode_spec.get("cert_file"),
            mode_spec.get("key_file"),
            mode_spec.get("use_mtls", False),
        )
        if not health_ok:
            log_result(mode_name, "health_endpoint", "FAIL", str(health_data))
            return False
        log_result(mode_name, "health_endpoint", "PASS")

        # Extract UUID from server config for verification
        server_config_data = json.loads(server_config_path.read_text(encoding="utf-8"))
        registration_config = server_config_data.get("registration", {})
        expected_uuid = registration_config.get("instance_uuid")

        # Verify UUID in config if registration is enabled
        if mode_spec.get("with_proxy", True) and registration_config.get("enabled"):
            print("🔍 Verifying UUID in server config...")
            if not expected_uuid:
                error_msg = "instance_uuid is missing in server config"
                print(f"❌ {error_msg}")
                log_result(mode_name, "uuid_config_check", "FAIL", error_msg)
                return False

            # Validate UUID format
            uuid_valid, uuid_error = verify_uuid_in_payload({"uuid": expected_uuid})
            if not uuid_valid:
                error_msg = f"UUID in config is invalid: {uuid_error}"
                print(f"❌ {error_msg}")
                log_result(mode_name, "uuid_config_check", "FAIL", error_msg)
                return False

            print(f"   ✅ UUID in config: {expected_uuid}")
            log_result(mode_name, "uuid_config_check", "PASS")

            # Wait for server to register with proxy (registration happens asynchronously)
            print("⏳ Waiting for server to register with proxy...")
            server_id = registration_config.get("server_id") or registration_config.get(
                "server_name"
            )
            if not server_id:
                server_id = f"mcp-adapter-localhost-{server_port}"

            print(f"   🔍 Looking for server_id: {server_id}")
            print(f"   🔍 Expected UUID: {expected_uuid}")

            proxy_url = f"http://localhost:{proxy_port}"

            # First, verify proxy is accessible with a simple health check
            print(f"   🔍 Step 1: Checking proxy accessibility at {proxy_url}...")
            try:
                import requests
                import logging

                logging.basicConfig(level=logging.INFO)
                logger = logging.getLogger(__name__)

                print(f"   🔍 Step 1.1: Sending health check request...")
                health_response = requests.get(f"{proxy_url}/health", timeout=2)
                print(
                    f"   ✅ Step 1.2: Proxy health check passed (status: {health_response.status_code})"
                )
            except Exception as e:
                print(f"   ❌ Step 1: Proxy health check failed: {e}")
                error_msg = f"Proxy not accessible: {e}"
                log_result(mode_name, "proxy_registration_uuid", "FAIL", error_msg)
                return False

            # Now check registration with detailed logging
            registered = False
            reg_data = {}

            print(
                f"   🔍 Step 2: Starting registration check (max 10 attempts, 1 second each)..."
            )
            for attempt in range(10):
                await asyncio.sleep(1)
                print(
                    f"   ⏳ Step 2.{attempt + 1}: Registration attempt {attempt + 1}/10..."
                )
                try:
                    print(
                        f"   🔍 Step 2.{attempt + 1}.1: Calling check_proxy_registration_with_uuid..."
                    )
                    registered, reg_data = check_proxy_registration_with_uuid(
                        server_id=server_id,
                        expected_uuid=expected_uuid,
                        proxy_url=proxy_url,
                        use_ssl=False,
                    )
                    print(
                        f"   🔍 Step 2.{attempt + 1}.2: Function returned: registered={registered}"
                    )
                    if registered:
                        print(
                            f"   ✅ Step 2.{attempt + 1}.3: Registration successful on attempt {attempt + 1}"
                        )
                        break
                    else:
                        print(
                            f"   ⏳ Step 2.{attempt + 1}.3: Not registered yet: {reg_data.get('message', 'Unknown')}"
                        )
                except Exception as e:
                    print(
                        f"   ❌ Step 2.{attempt + 1}: Exception during registration check: {type(e).__name__}: {e}"
                    )
                    import traceback

                    print(
                        f"   🔍 Step 2.{attempt + 1} Traceback:\n{traceback.format_exc()}"
                    )
                    if attempt == 9:
                        print(
                            f"   ❌ Step 2: Final registration check failed after 10 attempts"
                        )

            if not registered:
                # Log available servers for debugging
                print(f"   🔍 Step 3: Fetching server list for debugging...")
                try:
                    import requests

                    response = requests.get(f"{proxy_url}/servers", timeout=2)
                    if response.status_code == 200:
                        servers = response.json()
                        print(f"   📋 Step 3.1: Available servers on proxy: {servers}")
                        # Try to find any server with matching UUID
                        if isinstance(servers, list):
                            for srv in servers:
                                if (
                                    isinstance(srv, dict)
                                    and srv.get("uuid") == expected_uuid
                                ):
                                    print(
                                        f"   ✅ Step 3.2: Found server with matching UUID but different server_id: {srv.get('server_id')}"
                                    )
                                    print(
                                        f"   💡 Step 3.3: Server may be registered with different server_id"
                                    )
                        elif isinstance(servers, dict):
                            srv_list = servers.get("servers", [])
                            for srv in srv_list:
                                if (
                                    isinstance(srv, dict)
                                    and srv.get("uuid") == expected_uuid
                                ):
                                    print(
                                        f"   ✅ Step 3.2: Found server with matching UUID but different server_id: {srv.get('server_id')}"
                                    )
                                    print(
                                        f"   💡 Step 3.3: Server may be registered with different server_id"
                                    )
                except Exception as e:
                    print(f"   ⚠️  Step 3: Could not fetch server list: {e}")

                error_msg = (
                    f"Server not registered with proxy or UUID mismatch: {reg_data}"
                )
                print(f"❌ {error_msg}")
                log_result(mode_name, "proxy_registration_uuid", "FAIL", error_msg)
                return False

            print(f"   ✅ Server registered with UUID: {reg_data.get('uuid')}")
            log_result(mode_name, "proxy_registration_uuid", "PASS")

        # Create client
        print("🔌 Creating client...")
        client = JsonRpcClient(
            protocol=client_config["protocol"],
            host=client_config["host"],
            port=client_config["port"],
            token=client_config.get("token"),
            token_header=client_config.get("token_header", "X-API-Key"),
            cert=client_config.get("cert"),
            key=client_config.get("key"),
            ca=client_config.get("ca"),
            check_hostname=client_config.get("check_hostname", False),
        )

        try:
            # Test basic commands
            print("📋 Testing basic commands...")
            if not await _test_basic_commands(client, mode_name):
                error_msg = "Basic commands test failed"
                print(f"❌ {error_msg}")
                log_result(mode_name, "all_tests", "FAIL", error_msg)
                return False

            # Test queue commands
            print("📋 Testing queue commands...")
            if not await _test_queue_commands(client, mode_spec):
                error_msg = "Queue commands test failed"
                print(f"❌ {error_msg}")
                log_result(mode_name, "all_tests", "FAIL", error_msg)
                return False

            log_result(mode_name, "all_tests", "PASS")
            return True
        except Exception as e:
            error_msg = f"Exception during testing: {e}"
            print(f"❌ {error_msg}")
            log_result(mode_name, "test_exception", "FAIL", error_msg)
            import traceback

            traceback.print_exc()
            return False
        finally:
            # Close client
            try:
                await client.close()
            except Exception:
                pass

    except Exception as e:
        log_result(mode_name, "test_exception", "FAIL", str(e))
        import traceback

        traceback.print_exc()
        return False
    finally:
        # Cleanup - ensure processes are stopped
        if server_process:
            try:
                server_process.terminate()
                try:
                    server_process.wait(timeout=3)
                except subprocess.TimeoutExpired:
                    server_process.kill()
                    server_process.wait(timeout=2)
            except Exception:
                try:
                    server_process.kill()
                except Exception:
                    pass
            server_process = None

        if proxy_process:
            try:
                proxy_process.terminate()
                try:
                    proxy_process.wait(timeout=3)
                except subprocess.TimeoutExpired:
                    proxy_process.kill()
                    proxy_process.wait(timeout=2)
            except Exception:
                try:
                    proxy_process.kill()
                except Exception:
                    pass
            proxy_process = None

        # Wait for ports to be released
        await asyncio.sleep(1)

        # Kill processes on ports
        kill_process_on_port(server_port)
        kill_process_on_port(proxy_port)
        await asyncio.sleep(0.5)


def cleanup_test_configs() -> None:
    """Clean up all test configuration files."""
    print("🧹 Cleaning up test configs...")

    # Clean server configs
    for config_file in CONFIGS_DIR.glob("*.json"):
        if config_file.name.startswith(("http_", "https_", "mtls_")):
            try:
                config_file.unlink()
                print(f"   Deleted: {config_file.name}")
            except Exception:
                pass

    # Clean client configs
    for config_file in CLIENT_CONFIGS_DIR.glob("*.json"):
        try:
            config_file.unlink()
            print(f"   Deleted: {config_file.name}")
        except Exception:
            pass


def cleanup_ports(start_port: int, end_port: int) -> None:
    """Clean up processes on ports in range."""
    print(f"🧹 Cleaning up ports {start_port}-{end_port}...")
    for port in range(start_port, end_port + 1):
        kill_process_on_port(port)
    time.sleep(1)


async def main() -> int:
    """Main pipeline function."""
    print("🚀 Starting comprehensive MCP Proxy Adapter testing pipeline")
    print("=" * 80)

    clear_results()

    # Clean up test configs before starting
    cleanup_test_configs()

    # Clean up ports in range (5000-5100 for servers, 6000-6100 for proxies)
    cleanup_ports(5000, 5100)
    cleanup_ports(6000, 6100)

    # Ensure configs directory exists
    CONFIGS_DIR.mkdir(parents=True, exist_ok=True)
    CLIENT_CONFIGS_DIR.mkdir(parents=True, exist_ok=True)

    # Test each mode with dynamic ports
    results = []
    for i, mode_spec in enumerate(TEST_MODES):
        print(f"\n{'=' * 80}")
        print(f"Progress: {i + 1}/{len(TEST_MODES)} modes")
        print(f"{'=' * 80}")

        server_port = START_PORT + i
        proxy_port = PROXY_START_PORT + i

        try:
            success = await _test_mode(mode_spec, server_port, proxy_port)
            results.append((mode_spec["name"], success))

            # If test failed, stop execution
            if not success:
                print(
                    f"\n❌ CRITICAL: Test failed for mode '{mode_spec['name']}'. Stopping pipeline."
                )
                break
        except Exception as e:
            error_msg = f"CRITICAL error in mode '{mode_spec['name']}': {e}"
            print(f"\n❌ {error_msg}")
            import traceback

            traceback.print_exc()
            results.append((mode_spec["name"], False))
            print(
                f"\n❌ CRITICAL: Exception in mode '{mode_spec['name']}'. Stopping pipeline."
            )
            break

        # Clean up ports after each test
        kill_process_on_port(server_port)
        kill_process_on_port(proxy_port)

        # Short pause between modes
        if i < len(TEST_MODES) - 1:
            await asyncio.sleep(1)

    # Print summary
    print(f"\n{'=' * 80}")
    print("TEST SUMMARY")
    print(f"{'=' * 80}")

    passed = sum(1 for _, success in results if success)
    total = len(results)

    for name, success in results:
        status = "✅ PASS" if success else "❌ FAIL"
        print(f"{status}: {name}")

    print(f"\n🎯 SUMMARY: {passed}/{total} modes passed")

    # Print detailed results
    all_results = get_results()
    by_mode = {}
    for mode_name, test_results in all_results.items():
        if mode_name not in by_mode:
            by_mode[mode_name] = {"PASS": 0, "FAIL": 0, "WARN": 0}
        for test_result in test_results:
            status = test_result.get("status", "UNKNOWN")
            by_mode[mode_name][status] = by_mode[mode_name].get(status, 0) + 1

    print("\nDetailed results by mode:")
    for mode, counts in by_mode.items():
        pass_count = counts.get("PASS", 0)
        fail_count = counts.get("FAIL", 0)
        warn_count = counts.get("WARN", 0)
        total_count = pass_count + fail_count + warn_count
        status_icon = "✅" if fail_count == 0 else "❌"
        print(
            f"  {status_icon} {mode}: {pass_count}/{total_count} tests passed"
            + (f" ({warn_count} warnings)" if warn_count > 0 else "")
        )

    if passed == total:
        print("\n🎉 ALL MODES PASSED!")
        return 0
    else:
        print("\n⚠️  Some modes failed")
        return 1


if __name__ == "__main__":
    sys.exit(asyncio.run(main()))
