"""
Integration tests for High Availability features (Story 4-10).

Tests verify:
- HPA scaling behavior (scale-up and scale-down)
- Persistent volume data persistence across pod restarts
- Dead Letter Queue (DLQ) functionality
- Encryption at rest for persistent volumes

Prerequisites:
- Kubernetes cluster with metrics-server for HPA
- kubectl configured with access to test cluster
- Namespace 'monitoring' exists
- OTel Collector deployed with all HA features
- GCP Pub/Sub topic 'otel-collector-dlq' exists (for DLQ tests)
"""

import json
import logging
import subprocess
import time
from typing import Dict, List, Optional

import pytest

logger = logging.getLogger(__name__)


class KubernetesClient:
    """Helper class for kubectl operations."""

    def __init__(self, namespace: str = "monitoring"):
        self.namespace = namespace

    def run_kubectl(self, *args: str) -> str:
        """Execute kubectl command and return output."""
        cmd = ["kubectl", "-n", self.namespace] + list(args)
        result = subprocess.run(cmd, capture_output=True, text=True, check=True, timeout=30)
        return result.stdout.strip()

    def get_hpa(self, hpa_name: str) -> Dict:
        """Get HPA details."""
        output = self.run_kubectl("get", "hpa", hpa_name, "-o", "json")
        return json.loads(output)

    def get_statefulset(self, sts_name: str) -> Dict:
        """Get StatefulSet details."""
        output = self.run_kubectl("get", "statefulset", sts_name, "-o", "json")
        return json.loads(output)

    def get_pods(self, label_selector: str) -> List[Dict]:
        """Get pods matching label selector."""
        output = self.run_kubectl("get", "pods", "-l", label_selector, "-o", "json")
        return json.loads(output).get("items", [])

    def get_pvc(self, pvc_name: str) -> Dict:
        """Get PVC details."""
        output = self.run_kubectl("get", "pvc", pvc_name, "-o", "json")
        return json.loads(output)

    def delete_pod(self, pod_name: str) -> None:
        """Delete a pod."""
        self.run_kubectl("delete", "pod", pod_name, "--wait=false")

    def exec_in_pod(self, pod_name: str, command: List[str]) -> str:
        """Execute command in pod."""
        cmd = ["kubectl", "-n", self.namespace, "exec", pod_name, "--"] + command
        result = subprocess.run(cmd, capture_output=True, text=True, check=True, timeout=30)
        return result.stdout.strip()

    def wait_for_pod_ready(self, pod_name: str, timeout: int = 120) -> bool:
        """Wait for pod to be ready."""
        start = time.time()
        while time.time() - start < timeout:
            try:
                output = self.run_kubectl("get", "pod", pod_name, "-o", "json")
                pod = json.loads(output)
                conditions = pod["status"].get("conditions", [])
                for condition in conditions:
                    if condition["type"] == "Ready" and condition["status"] == "True":
                        return True
            except subprocess.CalledProcessError:
                pass
            time.sleep(2)
        return False


@pytest.fixture(scope="module")
def k8s_client():
    """Kubernetes client fixture."""
    return KubernetesClient()


# ============================================================================
# HPA Scaling Tests (AC #1, #2)
# ============================================================================


@pytest.mark.integration
@pytest.mark.k8s
class TestHPAScaling:
    """Test HorizontalPodAutoscaler scaling behavior."""

    def test_hpa_configuration(self, k8s_client):
        """Verify HPA is configured correctly."""
        hpa = k8s_client.get_hpa("otel-collector-hpa")

        # Verify min/max replicas (AC #1)
        spec = hpa["spec"]
        assert spec["minReplicas"] == 3, "HPA minReplicas should be 3"
        assert spec["maxReplicas"] == 10, "HPA maxReplicas should be 10"

        # Verify CPU target (AC #1)
        metrics = spec["metrics"]
        cpu_metric = next(
            (m for m in metrics if m["type"] == "Resource" and m["resource"]["name"] == "cpu"),
            None,
        )
        assert cpu_metric is not None, "HPA should have CPU metric"
        assert (
            cpu_metric["resource"]["target"]["averageUtilization"] == 70
        ), "HPA CPU target should be 70%"

        # Verify scale-down stabilization (AC #2)
        behavior = spec.get("behavior", {})
        scale_down = behavior.get("scaleDown", {})
        assert (
            scale_down.get("stabilizationWindowSeconds") == 300
        ), "HPA scale-down stabilization should be 5 minutes (300s)"

    @pytest.mark.slow
    def test_hpa_scale_up(self, k8s_client):
        """Test HPA scales up when CPU exceeds threshold."""
        # Get initial replica count
        initial_pods = k8s_client.get_pods("app=otel-collector")
        initial_count = len(initial_pods)
        assert initial_count >= 3, "Should start with at least 3 replicas"

        # Note: This test requires load generation to trigger scale-up
        # In a real environment, use tools like 'hey' or 'k6' to generate load
        print(f"Initial replica count: {initial_count}")
        print("⚠️  Manual verification required: Apply load to trigger CPU > 70%")
        print(
            "    Example: kubectl run load-generator --image=busybox -- /bin/sh -c 'while true; do wget -q -O- http://otel-collector:4318/v1/metrics; done'"
        )

        # Wait for scale-up (manual trigger in real environment)
        # In CI/CD, this would be automated with load generation

    @pytest.mark.slow
    def test_hpa_scale_down(self, k8s_client):
        """Test HPA scales down after load decreases."""
        # Note: This test requires previous scale-up and load removal
        print("⚠️  Manual verification required: Remove load and wait 5+ minutes")
        print("    Verify HPA scales down to 3 replicas after stabilization window")

        # In real environment, monitor HPA status:
        # kubectl get hpa otel-collector-hpa --watch


# ============================================================================
# Persistent Volume Tests (AC #3, #4)
# ============================================================================


@pytest.mark.integration
@pytest.mark.k8s
class TestPersistentVolumes:
    """Test persistent volume data persistence."""

    def test_pvc_exists_for_each_pod(self, k8s_client):
        """Verify each StatefulSet pod has a PVC."""
        pods = k8s_client.get_pods("app=otel-collector")
        assert len(pods) >= 3, "Should have at least 3 collector pods"

        for pod in pods:
            pod_name = pod["metadata"]["name"]
            pvc_name = f"queue-storage-{pod_name}"

            # Get PVC
            pvc = k8s_client.get_pvc(pvc_name)
            assert pvc["status"]["phase"] == "Bound", f"PVC {pvc_name} should be Bound"

            # Verify storage class (AC #8 - encryption)
            storage_class = pvc["spec"].get("storageClassName")
            assert (
                storage_class == "encrypted-ssd"
            ), f"PVC should use encrypted-ssd class, got {storage_class}"

            # Verify storage size
            requested_storage = pvc["spec"]["resources"]["requests"]["storage"]
            assert requested_storage == "10Gi", f"PVC should request 10Gi, got {requested_storage}"

    def test_persistent_queue_recovery(self, k8s_client):
        """Test queue persists across pod restart (AC #3, #4)."""
        pods = k8s_client.get_pods("app=otel-collector")
        assert len(pods) > 0, "Need at least one pod to test"

        # Pick first pod
        pod = pods[0]
        pod_name = pod["metadata"]["name"]

        # Write test data to queue directory
        test_file = "/var/otel/queue/test_persistence.txt"
        test_data = f"test-{int(time.time())}"

        cleanup_needed = False
        try:
            # Write test file
            k8s_client.exec_in_pod(pod_name, ["sh", "-c", f"echo '{test_data}' > {test_file}"])
            cleanup_needed = True

            # Verify file exists
            output = k8s_client.exec_in_pod(pod_name, ["cat", test_file])
            assert output == test_data, "Test data should be written"

            # Delete pod
            print(f"Deleting pod {pod_name}...")
            k8s_client.delete_pod(pod_name)

            # Wait for pod to be recreated
            time.sleep(5)
            assert k8s_client.wait_for_pod_ready(
                pod_name, timeout=120
            ), f"Pod {pod_name} should be ready after restart"

            # Verify data persisted
            output = k8s_client.exec_in_pod(pod_name, ["cat", test_file])
            assert output == test_data, "Data should persist across pod restart (AC #3)"

        finally:
            # Cleanup test file (guaranteed to run)
            if cleanup_needed:
                try:
                    k8s_client.exec_in_pod(pod_name, ["rm", "-f", test_file])
                except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
                    # Log but don't fail on cleanup errors
                    logger.warning(f"Failed to cleanup test file {test_file} in pod {pod_name}")


# ============================================================================
# Dead Letter Queue Tests (AC #5, #6, #7)
# ============================================================================


@pytest.mark.integration
@pytest.mark.k8s
@pytest.mark.gcp
class TestDeadLetterQueue:
    """Test Dead Letter Queue functionality."""

    def test_dlq_topic_exists(self):
        """Verify DLQ Pub/Sub topic exists."""
        try:
            result = subprocess.run(
                [
                    "gcloud",
                    "pubsub",
                    "topics",
                    "describe",
                    "otel-collector-dlq",
                    "--project=bhsf-model-monitoring-prod",
                    "--format=json",
                ],
                capture_output=True,
                text=True,
                check=True,
                timeout=10,
            )
            topic = json.loads(result.stdout)
            assert "otel-collector-dlq" in topic["name"], "DLQ topic should exist (AC #5)"

        except (subprocess.CalledProcessError, FileNotFoundError):
            pytest.skip("gcloud not available or DLQ topic not created")

    def test_dlq_subscription_exists(self):
        """Verify DLQ subscription exists."""
        try:
            result = subprocess.run(
                [
                    "gcloud",
                    "pubsub",
                    "subscriptions",
                    "describe",
                    "otel-collector-dlq-sub",
                    "--project=bhsf-model-monitoring-prod",
                    "--format=json",
                ],
                capture_output=True,
                text=True,
                check=True,
                timeout=10,
            )
            subscription = json.loads(result.stdout)
            assert (
                "otel-collector-dlq-sub" in subscription["name"]
            ), "DLQ subscription should exist (AC #5)"

        except (subprocess.CalledProcessError, FileNotFoundError):
            pytest.skip("gcloud not available or DLQ subscription not created")

    def test_dlq_replay_script_exists(self):
        """Verify DLQ replay processor exists and is executable."""
        import os

        script_path = "mca-prototype/k8s/pubsub/dlq-replay-processor.py"
        assert os.path.exists(
            script_path
        ), f"DLQ replay script should exist at {script_path} (AC #7)"
        assert os.access(script_path, os.X_OK), "DLQ replay script should be executable"

    @pytest.mark.manual
    def test_dlq_message_delivery(self):
        """Test failed telemetry is sent to DLQ (AC #5).

        This test requires manual setup:
        1. Simulate exporter failure (e.g., invalid GCP credentials)
        2. Send telemetry to Collector
        3. Wait for max retries (5 attempts)
        4. Verify message appears in DLQ topic
        """
        pytest.skip("Manual test - requires exporter failure simulation")

    @pytest.mark.manual
    def test_dlq_replay_mechanism(self):
        """Test DLQ messages can be replayed (AC #7).

        This test requires manual setup:
        1. Ensure messages exist in DLQ
        2. Run dlq-replay-processor.py
        3. Verify messages are reprocessed by Collector
        4. Verify messages are acknowledged in DLQ
        """
        pytest.skip("Manual test - requires DLQ messages")


# ============================================================================
# Encryption at Rest Tests (AC #8)
# ============================================================================


@pytest.mark.integration
@pytest.mark.k8s
class TestEncryptionAtRest:
    """Test persistent volume encryption."""

    def test_storage_class_encryption(self, k8s_client):
        """Verify StorageClass has encryption configured."""
        output = k8s_client.run_kubectl("get", "storageclass", "encrypted-ssd", "-o", "json")
        storage_class = json.loads(output)

        # Verify provisioner
        assert (
            storage_class["provisioner"] == "kubernetes.io/gce-pd"
        ), "StorageClass should use GCE persistent disk provisioner"

        # Verify SSD type
        parameters = storage_class.get("parameters", {})
        assert parameters.get("type") == "pd-ssd", "StorageClass should use pd-ssd type"

        # Note: disk-encryption-kms-key verification requires GCP-specific checks
        print(
            "⚠️  Manual verification required: Ensure disk-encryption-kms-key is configured in overlay"
        )

    def test_pvc_uses_encrypted_storage(self, k8s_client):
        """Verify PVCs use encrypted storage class (AC #8)."""
        pods = k8s_client.get_pods("app=otel-collector")
        assert len(pods) > 0, "Need at least one pod to test"

        for pod in pods[:1]:  # Test first pod only
            pod_name = pod["metadata"]["name"]
            pvc_name = f"queue-storage-{pod_name}"

            # Get PVC
            pvc = k8s_client.get_pvc(pvc_name)

            # Verify storage class
            storage_class = pvc["spec"].get("storageClassName")
            assert storage_class == "encrypted-ssd", "PVC should use encrypted-ssd class (AC #8)"

            # Note: Actual encryption verification requires GCP console or API
            print(f"✓ PVC {pvc_name} uses encrypted storage class: {storage_class}")


# ============================================================================
# Integration Tests Summary
# ============================================================================


@pytest.mark.integration
def test_ha_features_summary(k8s_client):
    """Summary test - verify all HA features are configured."""
    checks = []

    # Check 1: HPA exists
    try:
        hpa = k8s_client.get_hpa("otel-collector-hpa")
        checks.append(("HPA configured", hpa["spec"]["minReplicas"] == 3))
    except Exception as e:
        checks.append(("HPA configured", False))

    # Check 2: StatefulSet has 3+ replicas
    try:
        sts = k8s_client.get_statefulset("otel-collector")
        checks.append(("StatefulSet replicas", sts["spec"]["replicas"] >= 3))
    except Exception as e:
        checks.append(("StatefulSet replicas", False))

    # Check 3: PVCs exist
    try:
        pods = k8s_client.get_pods("app=otel-collector")
        pvc_count = 0
        for pod in pods:
            pvc_name = f"queue-storage-{pod['metadata']['name']}"
            try:
                k8s_client.get_pvc(pvc_name)
                pvc_count += 1
            except:
                pass
        checks.append(("PVCs provisioned", pvc_count >= 3))
    except Exception as e:
        checks.append(("PVCs provisioned", False))

    # Check 4: Encrypted storage class exists
    try:
        output = k8s_client.run_kubectl("get", "storageclass", "encrypted-ssd")
        checks.append(("Encrypted StorageClass", "encrypted-ssd" in output))
    except Exception as e:
        checks.append(("Encrypted StorageClass", False))

    # Print summary
    print("\n" + "=" * 80)
    print("High Availability Features Summary (Story 4-10)")
    print("=" * 80)
    for check_name, passed in checks:
        status = "✓" if passed else "✗"
        print(f"{status} {check_name}")
    print("=" * 80)

    # Assert all checks passed
    failed = [name for name, passed in checks if not passed]
    assert not failed, f"HA feature checks failed: {failed}"
