#!/usr/bin/env python3
"""
Author: Vasiliy Zdanovskiy
email: vasilyvz@gmail.com

Comprehensive pipeline for testing all MCP Proxy Adapter modes.
For each mode:
- Generates server and client configs using CLI generator
- Starts proxy and server
- Tests all commands including queue commands
"""

import asyncio
import json
import subprocess
import sys
import time
import uuid
from pathlib import Path
from typing import Optional

from mcp_proxy_adapter.client.jsonrpc_client.client import JsonRpcClient
from tests.config import TEST_MODES
from tests.utils.config_utils import BASE_DIR
from tests.utils.process_utils import (
    start_proxy,
    start_server,
    wait_for_port,
)
from tests.utils.health_utils import check_server_health
from tests.utils.logging_utils import log_result, get_results, clear_results


# Certificate paths
CERTS_DIR = BASE_DIR / "mtls_certificates"
SERVER_CERT = str(CERTS_DIR / "server" / "test-server.crt")
SERVER_KEY = str(CERTS_DIR / "server" / "test-server.key")
SERVER_CA = str(CERTS_DIR / "ca" / "ca.crt")
CLIENT_CERT = str(CERTS_DIR / "client" / "test-client.crt")
CLIENT_KEY = str(CERTS_DIR / "client" / "test-client.key")
CLIENT_CA = str(CERTS_DIR / "ca" / "ca.crt")

CONFIGS_DIR = (
    BASE_DIR / "mcp_proxy_adapter" / "examples" / "full_application" / "configs"
)
CLIENT_CONFIGS_DIR = BASE_DIR / "tests" / "client_configs"
CLIENT_CONFIGS_DIR.mkdir(parents=True, exist_ok=True)

# Starting port for test servers
START_PORT = 5000
PROXY_START_PORT = 6000


def kill_process_on_port(port: int) -> None:
    """Kill process occupying a port using kill -9."""
    try:
        # Find process using the port
        result = subprocess.run(
            ["lsof", "-ti", f":{port}"], capture_output=True, text=True
        )
        if result.returncode == 0 and result.stdout.strip():
            pids = result.stdout.strip().split("\n")
            for pid in pids:
                if pid:
                    subprocess.run(["kill", "-9", pid], capture_output=True)
    except Exception:
        pass


def generate_server_config(mode_spec: dict, server_port: int, proxy_port: int) -> Path:
    """Generate server configuration using CLI generator."""
    config_name = mode_spec["config"].split("/")[-1]
    config_path = CONFIGS_DIR / config_name
    
    CONFIGS_DIR.mkdir(parents=True, exist_ok=True)
    
    # Kill any process on the port before generating config
    kill_process_on_port(server_port)
    kill_process_on_port(proxy_port)
    
    # Use adapter-cfg-gen command if available, otherwise use module
    try:
        import shutil

        if shutil.which("adapter-cfg-gen"):
            cmd = ["adapter-cfg-gen"]
        else:
            cmd = [
                sys.executable,
                "-m",
                "mcp_proxy_adapter.cli.commands.config_generate",
            ]
    except Exception:
        cmd = [sys.executable, "-m", "mcp_proxy_adapter.cli.commands.config_generate"]
    
    cmd.extend(
        [
            "--protocol",
            mode_spec["protocol"],
            "--out",
            str(config_path),
            "--server-port",
            str(server_port),
            "--registration-port",
            str(proxy_port),
        ]
    )
    
    # Add SSL certificates for https/mtls
    if mode_spec["protocol"] in ("https", "mtls"):
        cmd.extend(
            [
                "--server-cert-file",
                SERVER_CERT,
                "--server-key-file",
                SERVER_KEY,
            ]
        )
        if mode_spec["protocol"] == "mtls":
            cmd.extend(
                [
                    "--server-ca-cert-file",
                    SERVER_CA,
                    "--registration-cert-file",
                    CLIENT_CERT,
                    "--registration-key-file",
                    CLIENT_KEY,
                    "--registration-ca-cert-file",
                    CLIENT_CA,
                ]
            )
        else:
            cmd.extend(
                [
                    "--registration-cert-file",
                    CLIENT_CERT,
                    "--registration-key-file",
                    CLIENT_KEY,
                ]
            )
    
    # Add authentication
    if mode_spec.get("token"):
        cmd.append("--use-token")
        if "Roles" in mode_spec["name"]:
            cmd.append("--use-roles")
    
    # Add queue manager config
    cmd.extend(
        [
        "--queue-enabled",
            "--max-queue-size",
            "1000",
            "--per-job-type-limits",
            "command_execution:100,data_processing:50,api_call:200",
        ]
    )

    try:
        subprocess.run(cmd, capture_output=True, text=True, check=True)
        
        # For mTLS modes, add check_hostname: false to registration.ssl to avoid hostname mismatch
        if mode_spec["protocol"] == "mtls":
            import json

            config_data = json.loads(config_path.read_text(encoding="utf-8"))
            if "registration" in config_data and isinstance(
                config_data["registration"], dict
            ):
                if "ssl" not in config_data["registration"]:
                    config_data["registration"]["ssl"] = {}
                if not isinstance(config_data["registration"]["ssl"], dict):
                    # If ssl is not a dict, convert it
                    ssl_value = config_data["registration"]["ssl"]
                    config_data["registration"]["ssl"] = {}
                    if ssl_value:
                        # Try to preserve existing ssl config if it's an object
                        if hasattr(ssl_value, "__dict__"):
                            config_data["registration"]["ssl"].update(
                                ssl_value.__dict__
                            )
                config_data["registration"]["ssl"]["check_hostname"] = False
                config_path.write_text(
                    json.dumps(config_data, indent=2), encoding="utf-8"
                )
        
        return config_path
    except subprocess.CalledProcessError as e:
        print(f"❌ Error generating server config: {e.stderr}", file=sys.stderr)
        raise


def generate_client_config(mode_spec: dict, port: int) -> Path:
    """Generate client configuration for testing."""
    config_name = (
        f"client_{mode_spec['name'].lower().replace(' ', '_').replace('+', '_')}.json"
    )
    config_path = CLIENT_CONFIGS_DIR / config_name
    
    # Client config matches server protocol
    protocol = mode_spec["protocol"]
    # Use 127.0.0.1 for mTLS to avoid hostname mismatch issues
    host = "127.0.0.1" if protocol == "mtls" else "localhost"
    
    client_config = {
        "protocol": protocol,
        "host": host,
        "port": port,
    }
    
    # Add SSL/TLS settings
    if protocol in ("https", "mtls"):
        client_config["cert"] = CLIENT_CERT
        client_config["key"] = CLIENT_KEY
        if protocol == "mtls":
            client_config["ca"] = CLIENT_CA
        # Always disable hostname check for test certificates
        client_config["check_hostname"] = False
    
    # Add authentication
    if mode_spec.get("token"):
        client_config["token"] = mode_spec["token"]
        client_config["token_header"] = "X-API-Key"
    
    config_path.write_text(json.dumps(client_config, indent=2), encoding="utf-8")
    return config_path


async def test_basic_commands(client: JsonRpcClient, mode_name: str) -> bool:
    """Test basic commands (echo, health, help)."""
    print(f"  🧪 Testing basic commands...")
    
    try:
        # Test echo
        result = await client.echo("Hello from pipeline")
        if not result.get("success"):
            log_result(mode_name, "command_echo", "FAIL", str(result))
            return False
        log_result(mode_name, "command_echo", "PASS")
        
        # Test health - use health endpoint directly (not a JSON-RPC command)
        try:
            health_data = await client.health()
            if health_data.get("status") == "ok":
                log_result(mode_name, "command_health", "PASS")
            else:
                log_result(mode_name, "command_health", "FAIL", str(health_data))
                return False
        except Exception as e:
            log_result(mode_name, "command_health", "FAIL", str(e))
            return False
        
        # Test help
        result = await client.help()
        if not result.get("success"):
            log_result(mode_name, "command_help", "FAIL", str(result))
            return False
        log_result(mode_name, "command_help", "PASS")
        
        return True
    except Exception as e:
        log_result(mode_name, "basic_commands", "FAIL", str(e))
        import traceback

        traceback.print_exc()
        return False


async def test_queue_commands(client: JsonRpcClient, mode_name: str) -> bool:
    """Test queue commands comprehensively."""
    print(f"  🧪 Testing queue commands...")
    
    try:
        # Test queue_health
        response = await client.jsonrpc_call("queue_health", {})
        result = client._extract_result(response)
        if not result.get("success"):
            log_result(mode_name, "queue_health", "FAIL", str(result))
            return False
        log_result(mode_name, "queue_health", "PASS")
        
        # Clean up any existing jobs
        response = await client.jsonrpc_call("queue_list_jobs", {})
        list_result = client._extract_result(response)
        if list_result.get("success"):
            jobs = list_result.get("data", {}).get("jobs", [])
            for job in jobs:
                job_id = job.get("job_id")
                if job_id:
                    await client.jsonrpc_call("queue_delete_job", {"job_id": job_id})
        
        # Test queue_add_job - command_execution
        job_id_1 = f"test_cmd_{uuid.uuid4().hex[:8]}"
        response = await client.jsonrpc_call(
            "queue_add_job",
            {
                "job_type": "command_execution",
                "job_id": job_id_1,
                "params": {
                    "command": "echo",
                    "params": {"message": "Test command execution"},
                },
            },
        )
        result = client._extract_result(response)
        if not result.get("success"):
            log_result(mode_name, "queue_add_job_cmd", "FAIL", str(result))
            return False
        log_result(mode_name, "queue_add_job_cmd", "PASS")
        
        # Test queue_add_job - data_processing
        job_id_2 = f"test_data_{uuid.uuid4().hex[:8]}"
        response = await client.jsonrpc_call(
            "queue_add_job",
            {
                "job_type": "data_processing",
                "job_id": job_id_2,
                "params": {"data": {"test": "data"}, "operation": "process"},
            },
        )
        result = client._extract_result(response)
        if not result.get("success"):
            log_result(mode_name, "queue_add_job_data", "FAIL", str(result))
            return False
        log_result(mode_name, "queue_add_job_data", "PASS")
        
        # Test queue_add_job - api_call
        job_id_3 = f"test_api_{uuid.uuid4().hex[:8]}"
        response = await client.jsonrpc_call(
            "queue_add_job",
            {
                "job_type": "api_call",
                "job_id": job_id_3,
                "params": {"url": "http://localhost:8080/health", "method": "GET"},
            },
        )
        result = client._extract_result(response)
        if not result.get("success"):
            log_result(mode_name, "queue_add_job_api", "FAIL", str(result))
            return False
        log_result(mode_name, "queue_add_job_api", "PASS")
        
        # Test queue_get_job_status
        response = await client.jsonrpc_call(
            "queue_get_job_status", {"job_id": job_id_1}
        )
        result = client._extract_result(response)
        if not result.get("success"):
            log_result(mode_name, "queue_get_job_status", "FAIL", str(result))
            return False
        log_result(mode_name, "queue_get_job_status", "PASS")
        
        # Test queue_list_jobs
        response = await client.jsonrpc_call("queue_list_jobs", {})
        result = client._extract_result(response)
        if not result.get("success"):
            log_result(mode_name, "queue_list_jobs", "FAIL", str(result))
            return False
        jobs = result.get("data", {}).get("jobs", [])
        if len(jobs) < 3:
            log_result(
                mode_name,
                "queue_list_jobs",
                "FAIL",
                f"Expected at least 3 jobs, got {len(jobs)}",
            )
            return False
        log_result(mode_name, "queue_list_jobs", "PASS", f"Found {len(jobs)} jobs")
        
        # Test queue_start_job
        response = await client.jsonrpc_call("queue_start_job", {"job_id": job_id_1})
        result = client._extract_result(response)
        if not result.get("success"):
            log_result(mode_name, "queue_start_job", "FAIL", str(result))
            return False
        log_result(mode_name, "queue_start_job", "PASS")
        
        # Wait a bit for job to start
        await asyncio.sleep(1)
        
        # Test queue_get_job_status after start
        response = await client.jsonrpc_call(
            "queue_get_job_status", {"job_id": job_id_1}
        )
        result = client._extract_result(response)
        if not result.get("success"):
            log_result(
                mode_name, "queue_get_job_status_after_start", "FAIL", str(result)
            )
            return False
        status = result.get("data", {}).get("status")
        log_result(
            mode_name, "queue_get_job_status_after_start", "PASS", f"Status: {status}"
        )

        # Wait for jobs to complete and test completed jobs retention
        print(f"    ⏳ Waiting for jobs to complete...")
        max_wait = 30  # Maximum wait time in seconds
        wait_interval = 0.5
        waited = 0
        all_completed = False

        while waited < max_wait:
            all_completed = True
            for job_id in [job_id_1, job_id_2, job_id_3]:
                response = await client.jsonrpc_call(
                    "queue_get_job_status", {"job_id": job_id}
                )
                result = client._extract_result(response)
                if result.get("success"):
                    job_status = result.get("data", {}).get("status")
                    if job_status not in ("completed", "failed"):
                        all_completed = False
                        break
            if all_completed:
                break
            await asyncio.sleep(wait_interval)
            waited += wait_interval

        if not all_completed:
            log_result(
                mode_name,
                "queue_jobs_completion_wait",
                "WARN",
                "Some jobs did not complete in time",
            )
        else:
            log_result(
                mode_name,
                "queue_jobs_completion_wait",
                "PASS",
                f"All jobs completed in {waited:.1f}s",
            )

        # Test that completed jobs remain accessible and data can be retrieved (NEW FEATURE TEST)
        print(f"    🧪 Testing completed jobs retention and data retrieval...")
        completed_job_ids = []
        jobs_with_data = []

        for job_id in [job_id_1, job_id_2, job_id_3]:
            response = await client.jsonrpc_call(
                "queue_get_job_status", {"job_id": job_id}
            )
            result = client._extract_result(response)
            if result.get("success"):
                job_status = result.get("data", {}).get("status")
                if job_status in ("completed", "failed"):
                    completed_job_ids.append(job_id)
                    # Verify we can get the result data
                    job_result = result.get("data", {}).get("result")
                    if job_result is not None:
                        jobs_with_data.append(job_id)
                        # Verify result contains expected fields
                        if isinstance(job_result, dict):
                            has_data = (
                                "job_id" in job_result
                                or "status" in job_result
                                or "result" in job_result
                                or len(job_result) > 0
                            )
                            if has_data:
                                log_result(
                                    mode_name,
                                    f"completed_job_data_{job_id[:8]}",
                                    "PASS",
                                    f"Status: {job_status}, has result data",
                                )
                            else:
                                log_result(
                                    mode_name,
                                    f"completed_job_data_{job_id[:8]}",
                                    "WARN",
                                    "Result data is empty",
                                )
                        else:
                            log_result(
                                mode_name,
                                f"completed_job_data_{job_id[:8]}",
                                "PASS",
                                f"Status: {job_status}, result type: {type(job_result).__name__}",
                            )
                    else:
                        log_result(
                            mode_name,
                            f"completed_job_data_{job_id[:8]}",
                            "WARN",
                            "No result data available",
                        )

        if len(completed_job_ids) > 0:
            log_result(
                mode_name,
                "completed_jobs_retention",
                "PASS",
                f"{len(completed_job_ids)} completed jobs accessible",
            )
            if len(jobs_with_data) > 0:
                log_result(
                    mode_name,
                    "completed_jobs_data_retrieval",
                    "PASS",
                    f"{len(jobs_with_data)}/{len(completed_job_ids)} jobs have retrievable data",
                )
            else:
                log_result(
                    mode_name,
                    "completed_jobs_data_retrieval",
                    "WARN",
                    "Completed jobs accessible but no data found",
                )
        else:
            log_result(
                mode_name, "completed_jobs_retention", "FAIL", "No completed jobs found"
            )

        # Test that completed jobs don't count toward limits (NEW FEATURE TEST)
        print(f"    🧪 Testing that completed jobs don't count toward limits...")
        new_job_id = f"test_limit_{uuid.uuid4().hex[:8]}"
        try:
            response = await client.jsonrpc_call(
                "queue_add_job",
                {
                    "job_type": "data_processing",
                    "job_id": new_job_id,
                    "params": {"data": {"test": "limit"}, "operation": "test"},
                },
            )
            result = client._extract_result(response)
            if result.get("success"):
                log_result(
                    mode_name,
                    "completed_jobs_limit_exclusion",
                    "PASS",
                    "New job added despite completed jobs",
                )
            else:
                log_result(
                    mode_name,
                    "completed_jobs_limit_exclusion",
                    "WARN",
                    "Could not add new job (may be expected)",
                )
        except Exception as e:
            log_result(
                mode_name,
                "completed_jobs_limit_exclusion",
                "WARN",
                f"Exception: {str(e)}",
            )

        # Test queue_get_job_logs - NEW FEATURE TEST
        print(f"    🧪 Testing queue_get_job_logs command...")
        logs_tested = 0
        logs_with_output = 0

        for job_id in [job_id_1, job_id_2, job_id_3]:
            try:
                response = await client.jsonrpc_call(
                    "queue_get_job_logs", {"job_id": job_id}
                )
                result = client._extract_result(response)
                if result.get("success"):
                    logs_tested += 1
                    logs_data = result.get("data", {})
                    stdout = logs_data.get("stdout", [])
                    stderr = logs_data.get("stderr", [])
                    stdout_lines = logs_data.get("stdout_lines", 0)
                    stderr_lines = logs_data.get("stderr_lines", 0)

                    # Check if logs are available (even if empty)
                    has_stdout = isinstance(stdout, list) and len(stdout) > 0
                    has_stderr = isinstance(stderr, list) and len(stderr) > 0

                    if has_stdout or has_stderr:
                        logs_with_output += 1
                        log_result(
                            mode_name,
                            f"queue_get_job_logs_{job_id[:8]}",
                            "PASS",
                            f"stdout: {stdout_lines} lines, stderr: {stderr_lines} lines",
                        )
                    else:
                        # Logs command works but no output (this is OK for some jobs)
                        log_result(
                            mode_name,
                            f"queue_get_job_logs_{job_id[:8]}",
                            "PASS",
                            f"Logs accessible (stdout: {stdout_lines}, stderr: {stderr_lines} lines)",
                        )
                else:
                    log_result(
                        mode_name,
                        f"queue_get_job_logs_{job_id[:8]}",
                        "FAIL",
                        f"Failed to get logs: {result.get('message', 'Unknown error')}",
                    )
            except Exception as e:
                log_result(
                    mode_name,
                    f"queue_get_job_logs_{job_id[:8]}",
                    "FAIL",
                    f"Exception: {str(e)}",
                )

        if logs_tested > 0:
            log_result(
                mode_name,
                "queue_get_job_logs",
                "PASS",
                f"Successfully retrieved logs for {logs_tested} jobs ({logs_with_output} with output)",
            )
        else:
            log_result(
                mode_name,
                "queue_get_job_logs",
                "FAIL",
                "Failed to retrieve logs for any job",
            )

        # Test PeriodicLoggingJob - NEW FEATURE TEST for stdout logging
        print(f"    🧪 Testing PeriodicLoggingJob with stdout logging...")
        periodic_job_id = f"test_periodic_{uuid.uuid4().hex[:8]}"
        try:
            # Add periodic logging job (runs for 2 minutes, logs every 10 seconds for testing)
            response = await client.jsonrpc_call(
                "queue_add_job",
                {
                    "job_type": "periodic_logging",
                    "job_id": periodic_job_id,
                    "params": {
                        "duration_minutes": 2,
                        "interval_seconds": 10,  # Every 10 seconds for testing (not 60)
                        "message_prefix": "Test periodic log",
                    },
                },
            )
            result = client._extract_result(response)
            if not result.get("success"):
                log_result(
                    mode_name,
                    "queue_add_periodic_logging_job",
                    "FAIL",
                    str(result),
                )
            else:
                log_result(
                    mode_name,
                    "queue_add_periodic_logging_job",
                    "PASS",
                )

                # Start the job
                response = await client.jsonrpc_call(
                    "queue_start_job", {"job_id": periodic_job_id}
                )
                result = client._extract_result(response)
                if result.get("success"):
                    log_result(
                        mode_name,
                        "queue_start_periodic_logging_job",
                        "PASS",
                    )

                    # Wait a bit for logs to accumulate
                    await asyncio.sleep(15)  # Wait for at least one log message

                    # Get logs
                    response = await client.jsonrpc_call(
                        "queue_get_job_logs", {"job_id": periodic_job_id}
                    )
                    result = client._extract_result(response)
                    if result.get("success"):
                        logs_data = result.get("data", {})
                        stdout = logs_data.get("stdout", [])
                        stderr = logs_data.get("stderr", [])
                        stdout_lines = logs_data.get("stdout_lines", 0)
                        stderr_lines = logs_data.get("stderr_lines", 0)

                        if stdout_lines > 0:
                            log_result(
                                mode_name,
                                "queue_get_periodic_job_logs",
                                "PASS",
                                f"Retrieved {stdout_lines} stdout lines, {stderr_lines} stderr lines",
                            )
                            # Show first few log lines
                            if stdout and len(stdout) > 0:
                                first_lines = stdout[:3]
                                log_result(
                                    mode_name,
                                    "queue_periodic_logs_content",
                                    "PASS",
                                    f"First lines: {first_lines}",
                                )
                        else:
                            log_result(
                                mode_name,
                                "queue_get_periodic_job_logs",
                                "WARN",
                                "Logs accessible but no stdout yet (job may be starting)",
                            )
                    else:
                        log_result(
                            mode_name,
                            "queue_get_periodic_job_logs",
                            "FAIL",
                            f"Failed to get logs: {result.get('message')}",
                        )

                    # Stop the job to clean up
                    await client.jsonrpc_call(
                        "queue_stop_job", {"job_id": periodic_job_id}
                    )
                else:
                    log_result(
                        mode_name,
                        "queue_start_periodic_logging_job",
                        "FAIL",
                        str(result),
                    )
        except Exception as e:
            log_result(
                mode_name,
                "queue_periodic_logging_test",
                "FAIL",
                f"Exception: {str(e)}",
            )

        # Test queue_stop_job (on a running job if any)
        response = await client.jsonrpc_call("queue_stop_job", {"job_id": job_id_1})
        result = client._extract_result(response)
        if not result.get("success"):
            # Job might already be completed, which is fine
            log_result(
                mode_name, "queue_stop_job", "WARN", "Job may already be completed"
            )
        else:
            log_result(mode_name, "queue_stop_job", "PASS")

        # Verify completed jobs are still accessible and data can be retrieved after operations (NEW FEATURE TEST)
        print(
            f"    🧪 Verifying completed jobs and data still accessible after operations..."
        )
        still_accessible = 0
        still_have_data = 0
        for job_id in completed_job_ids[:2]:  # Check first 2 completed jobs
            try:
                response = await client.jsonrpc_call(
                    "queue_get_job_status", {"job_id": job_id}
                )
                result = client._extract_result(response)
                if result.get("success"):
                    job_status = result.get("data", {}).get("status")
                    if job_status in ("completed", "failed"):
                        still_accessible += 1
                        # Verify data is still retrievable
                        job_result = result.get("data", {}).get("result")
                        if job_result is not None:
                            still_have_data += 1
                            log_result(
                                mode_name,
                                f"data_retrieval_after_ops_{job_id[:8]}",
                                "PASS",
                                f"Data still retrievable for {job_id[:8]}",
                            )
                        else:
                            log_result(
                                mode_name,
                                f"data_retrieval_after_ops_{job_id[:8]}",
                                "WARN",
                                f"Job accessible but no data for {job_id[:8]}",
                            )
            except Exception as e:
                log_result(
                    mode_name,
                    f"data_retrieval_after_ops_{job_id[:8]}",
                    "WARN",
                    f"Exception: {str(e)}",
                )

        if still_accessible > 0:
            log_result(
                mode_name,
                "completed_jobs_persistence",
                "PASS",
                f"{still_accessible} completed jobs still accessible",
            )
            if still_have_data > 0:
                log_result(
                    mode_name,
                    "completed_jobs_data_persistence",
                    "PASS",
                    f"{still_have_data}/{still_accessible} jobs still have retrievable data",
                )
            else:
                log_result(
                    mode_name,
                    "completed_jobs_data_persistence",
                    "WARN",
                    "Jobs accessible but data may have been lost",
                )
        else:
            log_result(
                mode_name,
                "completed_jobs_persistence",
                "WARN",
                "Some completed jobs may have been cleaned up",
            )

        # Test queue_delete_job (only delete new test job, keep completed ones for verification)
        try:
            if "new_job_id" in locals():
                response = await client.jsonrpc_call(
                    "queue_delete_job", {"job_id": new_job_id}
                )
                result = client._extract_result(response)
                if not result.get("success"):
                    log_result(
                        mode_name,
                        f"queue_delete_job_{new_job_id[:8]}",
                        "WARN",
                        str(result),
                    )
                else:
                    log_result(mode_name, f"queue_delete_job_{new_job_id[:8]}", "PASS")
        except Exception as e:
            log_result(mode_name, "queue_delete_job", "WARN", str(e))
        
        # Test new convenience methods for command queue operations
        # Create a new job for testing convenience methods
        test_job_id = f"test_conv_{uuid.uuid4().hex[:8]}"
        response = await client.jsonrpc_call(
            "queue_add_job",
            {
                "job_type": "command_execution",
                "job_id": test_job_id,
                "params": {
                    "command": "echo",
                    "params": {"message": "Test convenience methods"},
                },
            },
        )
        result = client._extract_result(response)
        if not result.get("success"):
            log_result(mode_name, "queue_add_job_conv_test", "FAIL", str(result))
            return False
        
        # Test get_command_status (convenience method)
        try:
            status_result = await client.get_command_status(test_job_id)
            if status_result.get("success") or status_result.get("data"):
                log_result(mode_name, "get_command_status", "PASS")
            else:
                log_result(
                    mode_name,
                    "get_command_status",
                    "FAIL",
                    "Unexpected response format",
                )
                return False
        except Exception as e:
            log_result(mode_name, "get_command_status", "FAIL", str(e))
            return False
        
        # Test list_queued_commands (convenience method)
        try:
            queued_commands = await client.list_queued_commands(status=None, limit=10)
            if queued_commands.get("success") or queued_commands.get("data"):
                count = queued_commands.get("data", {}).get("total_count", 0)
                log_result(
                    mode_name, "list_queued_commands", "PASS", f"Found {count} commands"
                )
            else:
                log_result(
                    mode_name,
                    "list_queued_commands",
                    "FAIL",
                    "Unexpected response format",
                )
                return False
        except Exception as e:
            log_result(mode_name, "list_queued_commands", "FAIL", str(e))
            return False
        
        # Test list_all_queue_jobs (convenience method)
        try:
            all_jobs = await client.list_all_queue_jobs(
                status=None, job_type="command_execution", limit=10
            )
            if all_jobs.get("success") or all_jobs.get("data"):
                count = all_jobs.get("data", {}).get("total_count", 0)
                log_result(
                    mode_name, "list_all_queue_jobs", "PASS", f"Found {count} jobs"
                )
            else:
                log_result(
                    mode_name,
                    "list_all_queue_jobs",
                    "FAIL",
                    "Unexpected response format",
                )
                return False
        except Exception as e:
            log_result(mode_name, "list_all_queue_jobs", "FAIL", str(e))
            return False
        
        # Test cancel_command (convenience method)
        try:
            cancel_result = await client.cancel_command(test_job_id)
            if cancel_result.get("success"):
                log_result(mode_name, "cancel_command", "PASS")
            else:
                # Try manual delete as fallback
                try:
                    await client.queue_delete_job(test_job_id)
                    log_result(
                        mode_name, "cancel_command", "PASS", "Used fallback delete"
                    )
                except Exception:
                    log_result(
                        mode_name,
                        "cancel_command",
                        "FAIL",
                        "Cancel and fallback both failed",
                    )
                    return False
        except Exception as e:
            # Try manual delete as fallback
            try:
                await client.queue_delete_job(test_job_id)
                log_result(
                    mode_name,
                    "cancel_command",
                    "PASS",
                    "Used fallback delete after exception",
                )
            except Exception:
                log_result(mode_name, "cancel_command", "FAIL", str(e))
                return False
        
        return True
    except Exception as e:
        log_result(mode_name, "queue_commands", "FAIL", str(e))
        import traceback

        traceback.print_exc()
        return False


async def test_mode(mode_spec: dict, server_port: int, proxy_port: int) -> bool:
    """Test a single mode comprehensively."""
    mode_name = mode_spec["name"]
    print(f"\n{'=' * 80}")
    print(f"Testing: {mode_name}")
    print(f"Server port: {server_port}, Proxy port: {proxy_port}")
    print(f"{'=' * 80}")
    
    server_process: Optional[subprocess.Popen] = None
    proxy_process: Optional[subprocess.Popen] = None
    
    try:
        # Kill processes on ports before starting
        kill_process_on_port(server_port)
        kill_process_on_port(proxy_port)
        await asyncio.sleep(0.5)
        
        # Generate server config
        print("📝 Generating server config...")
        server_config_path = generate_server_config(mode_spec, server_port, proxy_port)
        log_result(mode_name, "config_generation_server", "PASS")
        
        # Generate client config
        print("📝 Generating client config...")
        client_config_path = generate_client_config(mode_spec, server_port)
        log_result(mode_name, "config_generation_client", "PASS")
        
        # Load client config
        client_config = json.loads(client_config_path.read_text(encoding="utf-8"))
        
        # Update client config with actual port
        client_config["port"] = server_port
        
        # Determine URL based on protocol and port
        protocol = mode_spec["protocol"]
        scheme = "https" if protocol in ("https", "mtls") else "http"
        server_url = f"{scheme}://localhost:{server_port}"
        
        # Start proxy if needed
        if mode_spec.get("with_proxy", True):
            print("🚀 Starting proxy...")
            proxy_process = start_proxy("http", "localhost", proxy_port)
            if proxy_process:
                if wait_for_port("localhost", proxy_port, timeout=10):
                    log_result(mode_name, "proxy_start", "PASS")
                else:
                    log_result(mode_name, "proxy_start", "FAIL", "Proxy port not ready")
                    return False
            else:
                log_result(mode_name, "proxy_start", "FAIL", "Failed to start proxy")
                return False
        
        # Start server
        print("🚀 Starting server...")
        server_process = start_server(str(server_config_path), server_port)
        
        # Wait longer for server to start and check stderr for errors
        if not wait_for_port("localhost", server_port, timeout=20):
            # Check if process is still running
            if server_process and server_process.poll() is not None:
                # Process exited, read stderr
                try:
                    if server_process.stderr:
                        error_output = server_process.stderr.read(2048).decode(
                            "utf-8", errors="ignore"
                        )
                        if error_output:
                            print(f"   Server error output: {error_output[:500]}")
                except Exception:
                    pass
            log_result(mode_name, "server_start", "FAIL", "Server port not ready")
            return False
        log_result(mode_name, "server_start", "PASS")
        
        # Wait for server to fully initialize (queue manager, etc.)
        await asyncio.sleep(3)
        
        # Test health endpoint
        print("🏥 Testing health endpoint...")
        health_ok, health_data = check_server_health(
            server_url,
            mode_spec["use_ssl"],
            mode_spec.get("token"),
            mode_spec.get("cert_file"),
            mode_spec.get("key_file"),
            mode_spec.get("use_mtls", False),
        )
        if not health_ok:
            log_result(mode_name, "health_endpoint", "FAIL", str(health_data))
            return False
        log_result(mode_name, "health_endpoint", "PASS")
        
        # Create client
        print("🔌 Creating client...")
        client = JsonRpcClient(
            protocol=client_config["protocol"],
            host=client_config["host"],
            port=client_config["port"],
            token=client_config.get("token"),
            token_header=client_config.get("token_header", "X-API-Key"),
            cert=client_config.get("cert"),
            key=client_config.get("key"),
            ca=client_config.get("ca"),
            check_hostname=client_config.get("check_hostname", False),
        )
        
        try:
            # Test basic commands
            print("📋 Testing basic commands...")
            if not await test_basic_commands(client, mode_name):
                return False
            
            # Test queue commands
            print("📋 Testing queue commands...")
            if not await test_queue_commands(client, mode_name):
                return False
            
            log_result(mode_name, "all_tests", "PASS")
            return True
        finally:
            # Close client
            await client.close()
        
    except Exception as e:
        log_result(mode_name, "test_exception", "FAIL", str(e))
        import traceback

        traceback.print_exc()
        return False
    finally:
        # Cleanup - ensure processes are stopped
        if server_process:
            try:
                server_process.terminate()
                try:
                    server_process.wait(timeout=3)
                except subprocess.TimeoutExpired:
                    server_process.kill()
                    server_process.wait(timeout=2)
            except Exception:
                try:
                    server_process.kill()
                except Exception:
                    pass
            server_process = None
        
        if proxy_process:
            try:
                proxy_process.terminate()
                try:
                    proxy_process.wait(timeout=3)
                except subprocess.TimeoutExpired:
                    proxy_process.kill()
                    proxy_process.wait(timeout=2)
            except Exception:
                try:
                    proxy_process.kill()
                except Exception:
                    pass
            proxy_process = None
        
        # Wait for ports to be released
        await asyncio.sleep(1)
        
        # Kill processes on ports
        kill_process_on_port(server_port)
        kill_process_on_port(proxy_port)
        await asyncio.sleep(0.5)


def cleanup_test_configs() -> None:
    """Clean up all test configuration files."""
    print("🧹 Cleaning up test configs...")
    
    # Clean server configs
    for config_file in CONFIGS_DIR.glob("*.json"):
        if config_file.name.startswith(("http_", "https_", "mtls_")):
            try:
                config_file.unlink()
                print(f"   Deleted: {config_file.name}")
            except Exception:
                pass
    
    # Clean client configs
    for config_file in CLIENT_CONFIGS_DIR.glob("*.json"):
        try:
            config_file.unlink()
            print(f"   Deleted: {config_file.name}")
        except Exception:
            pass


def cleanup_ports(start_port: int, end_port: int) -> None:
    """Clean up processes on ports in range."""
    print(f"🧹 Cleaning up ports {start_port}-{end_port}...")
    for port in range(start_port, end_port + 1):
        kill_process_on_port(port)
    time.sleep(1)


async def main() -> int:
    """Main pipeline function."""
    print("🚀 Starting comprehensive MCP Proxy Adapter testing pipeline")
    print("=" * 80)
    
    clear_results()
    
    # Clean up test configs before starting
    cleanup_test_configs()
    
    # Clean up ports in range (5000-5100 for servers, 6000-6100 for proxies)
    cleanup_ports(5000, 5100)
    cleanup_ports(6000, 6100)
    
    # Ensure configs directory exists
    CONFIGS_DIR.mkdir(parents=True, exist_ok=True)
    CLIENT_CONFIGS_DIR.mkdir(parents=True, exist_ok=True)
    
    # Test each mode with dynamic ports
    results = []
    for i, mode_spec in enumerate(TEST_MODES):
        print(f"\n{'=' * 80}")
        print(f"Progress: {i + 1}/{len(TEST_MODES)} modes")
        print(f"{'=' * 80}")
        
        server_port = START_PORT + i
        proxy_port = PROXY_START_PORT + i
        
        success = await test_mode(mode_spec, server_port, proxy_port)
        results.append((mode_spec["name"], success))
        
        # Clean up ports after each test
        kill_process_on_port(server_port)
        kill_process_on_port(proxy_port)
        
        # Short pause between modes
        if i < len(TEST_MODES) - 1:
            await asyncio.sleep(1)
    
    # Print summary
    print(f"\n{'=' * 80}")
    print("TEST SUMMARY")
    print(f"{'=' * 80}")
    
    passed = sum(1 for _, success in results if success)
    total = len(results)
    
    for name, success in results:
        status = "✅ PASS" if success else "❌ FAIL"
        print(f"{status}: {name}")
    
    print(f"\n🎯 SUMMARY: {passed}/{total} modes passed")
    
    # Print detailed results
    all_results = get_results()
    by_mode = {}
    for r in all_results:
        mode = r["mode"]
        if mode not in by_mode:
            by_mode[mode] = {"PASS": 0, "FAIL": 0}
        status = r["status"]
        by_mode[mode][status] = by_mode[mode].get(status, 0) + 1
    
    print("\nDetailed results by mode:")
    for mode, counts in by_mode.items():
        pass_count = counts.get("PASS", 0)
        fail_count = counts.get("FAIL", 0)
        total_count = pass_count + fail_count
        status_icon = "✅" if fail_count == 0 else "❌"
        print(f"  {status_icon} {mode}: {pass_count}/{total_count} tests passed")
    
    if passed == total:
        print("\n🎉 ALL MODES PASSED!")
        return 0
    else:
        print("\n⚠️  Some modes failed")
        return 1


if __name__ == "__main__":
    sys.exit(asyncio.run(main()))
