"""Comprehensive E2E addon + networking lifecycle test.

Full lifecycle with TCP connectivity verification at every stage:
internal-only, exposed (LoadBalancer/NodePort), unexposed (ClusterIP),
cross-app isolation, resource changes, credential rotation, backup, destroy.

Marked @slow — skipped by default, run with ``--runslow``.

Requires:
- K8s cluster with postgres:16, redis:7, nginx:1.27-alpine images pullable
- NetworkPolicy enforcement (k3s built-in, Calico, etc.)
- LoadBalancer/ServiceLB support (k3s built-in, MetalLB, cloud LB)

Run with: pytest tests/e2e/test_addon_lifecycle.py -v -s
"""

from __future__ import annotations

import socket
import subprocess
from typing import Any

import pytest

from kuberoku.addons.postgres import POSTGRES
from kuberoku.factory import KuberokuFactory
from kuberoku.k8s.resources import addon_pvc_name, safe_name
from tests.e2e.helpers import (
    cleanup_namespace_fixture,
    make_real_factory,
    poll_until,
    skip_if_no_cluster,
    wait_for_pods_ready,
    wait_for_service_deleted,
    wait_for_sts_ready,
)

pytestmark = pytest.mark.slow


def _get_external_address(
    factory: KuberokuFactory,
    svc_name: str,
    timeout: int = 120,
) -> tuple[str, int]:
    """Wait for a Service to get an external address.

    Returns (host, port) — either a LoadBalancer IP/hostname or a NodePort.
    """

    def _try_get() -> tuple[str, int] | None:
        svc = factory.k8s.get_service(factory.namespace, svc_name)
        svc_type = svc.get("spec", {}).get("type", "ClusterIP")
        svc_ports = svc.get("spec", {}).get("ports", [])

        if svc_type == "LoadBalancer":
            status = svc.get("status", {})
            lb = status.get("loadBalancer", {})
            for ingress in lb.get("ingress", []):
                ip = ingress.get("ip")
                hostname = ingress.get("hostname")
                host = ip or hostname
                if host:
                    port = svc_ports[0].get("port", 0) if svc_ports else 0
                    return (host, port)

        if svc_type in ("LoadBalancer", "NodePort"):
            for port_spec in svc_ports:
                node_port = port_spec.get("nodePort")
                if node_port:
                    node_ip = _get_any_node_ip()
                    if node_ip:
                        return (node_ip, node_port)

        return None

    result = poll_until(
        fn=_try_get,
        predicate=lambda addr: addr is not None,
        timeout=timeout,
        interval=1,
        message=f"Timed out waiting for external address on service {svc_name}",
    )
    assert result is not None  # for type narrowing
    return result


def _get_any_node_ip() -> str | None:
    """Get the IP of any cluster node using kubectl."""
    try:
        result = subprocess.run(
            [
                "kubectl",
                "get",
                "nodes",
                "-o",
                "jsonpath={.items[0].status.addresses[?(@.type=='InternalIP')].address}",
            ],
            capture_output=True,
            text=True,
            timeout=10,
        )
        ip = result.stdout.strip()
        if ip:
            return ip
    except Exception:
        pass
    return None


def _tcp_connect(host: str, port: int, timeout: int = 5) -> bool:
    """Attempt a raw TCP connection from the test runner host.

    Returns True if connection succeeds, False if it fails/times out.
    """
    try:
        with socket.create_connection((host, port), timeout=timeout):
            return True
    except (OSError, TimeoutError):
        return False


def _exec_tcp_probe(
    factory: KuberokuFactory,
    pod_name: str,
    target_host: str,
    target_port: int,
    timeout: int = 5,
) -> bool:
    """Probe TCP connectivity from inside a pod using nc (netcat).

    Returns True if TCP connected, False if blocked/refused/timeout.
    Uses ``sh -c "nc -w TIMEOUT HOST PORT < /dev/null"`` which works
    with busybox nc (alpine) — no ``-z`` flag needed.
    """
    try:
        factory.k8s.exec_in_pod(
            factory.namespace,
            pod_name,
            ["sh", "-c", f"nc -w {timeout} {target_host} {target_port} < /dev/null"],
        )
        return True  # nc exit 0 = TCP connected
    except Exception:
        return False  # non-zero exit = blocked/refused/timeout


# Module-level state shared between tests
_exposed_postgres: dict[str, Any] = {}


# ── Fixtures ─────────────────────────────────────────────────────


@pytest.fixture(scope="module")
def real_factory() -> KuberokuFactory:
    return make_real_factory("e2e-lifecycle")


@pytest.fixture(scope="module", autouse=True)
def cleanup_namespace(real_factory: KuberokuFactory) -> Any:
    yield
    cleanup_namespace_fixture(real_factory)


# ── Tests ────────────────────────────────────────────────────────


def test_00_skip_if_no_cluster(real_factory: KuberokuFactory) -> None:
    """Skip the entire module if no cluster is available."""
    skip_if_no_cluster()


def test_01_create_app(real_factory: KuberokuFactory) -> None:
    """Create the main app for lifecycle testing."""
    app = real_factory.apps.create("lifecycle")
    assert app.name == "lifecycle"
    print(f"\n  App created: {app.name}")
    print(f"  Namespace: {real_factory.namespace}")


def test_02_deploy_app(real_factory: KuberokuFactory) -> None:
    """Deploy nginx with port 80."""
    release = real_factory.deploy.deploy(
        "lifecycle",
        image="nginx:1.27-alpine",
        ports=[{"containerPort": 80, "protocol": "TCP"}],
        wait=False,
    )
    assert release.version >= 1
    print(f"\n  Deployed v{release.version}")

    # Wait for web pod to be ready
    pods = wait_for_pods_ready(real_factory, "lifecycle", timeout=120)
    print(f"  Web pod ready: {pods[0]['metadata']['name']}")


def test_03_create_postgres(real_factory: KuberokuFactory) -> None:
    """Create a postgres addon and wait for readiness + verify pg_isready."""
    addon = real_factory.addons.create("lifecycle", "postgres")
    assert addon.addon_type == "postgres"
    assert addon.status == "running"
    print(f"\n  Postgres created: {addon.instance_name}")

    sts_name = safe_name(real_factory.prefix, "addon", "lifecycle-postgres")
    pod = wait_for_sts_ready(real_factory, sts_name)
    print(f"  Pod ready: {pod['metadata']['name']}")

    # Verify pg_isready (Postgres may still be initializing after pod Ready)
    pod_name = f"{sts_name}-0"
    output = poll_until(
        fn=lambda: real_factory.k8s.exec_in_pod(
            real_factory.namespace,
            pod_name,
            ["pg_isready", "-U", "kuberoku"],
        ),
        predicate=lambda o: "accepting connections" in o,
        timeout=20,
        interval=2,
        message=f"pg_isready never returned 'accepting connections'",
    )
    print(f"  pg_isready: {output.strip()}")


def test_04_create_redis(real_factory: KuberokuFactory) -> None:
    """Create a redis addon and wait for readiness + verify PONG."""
    addon = real_factory.addons.create("lifecycle", "redis")
    assert addon.addon_type == "redis"
    assert addon.status == "running"
    print(f"\n  Redis created: {addon.instance_name}")

    sts_name = safe_name(real_factory.prefix, "addon", "lifecycle-redis")
    pod = wait_for_sts_ready(real_factory, sts_name)
    print(f"  Pod ready: {pod['metadata']['name']}")

    # Verify PING (redis may still be loading data after pod Ready)
    creds = real_factory.addons.credentials("lifecycle", "redis")
    pod_name = f"{sts_name}-0"
    output = poll_until(
        fn=lambda: real_factory.k8s.exec_in_pod(
            real_factory.namespace,
            pod_name,
            ["redis-cli", "-a", creds["password"], "PING"],
        ),
        predicate=lambda o: "PONG" in o,
        timeout=20,
        interval=2,
        message="redis-cli PING never returned PONG",
    )
    print(f"  redis-cli PING: {output.strip()}")


def test_05_addons_list(real_factory: KuberokuFactory) -> None:
    """Both addons visible in list."""
    addons = real_factory.addons.list("lifecycle")
    names = {a.instance_name for a in addons}
    assert names == {"postgres", "redis"}
    print(f"\n  Addons: {names}")


def test_05a_addon_urls_in_config(real_factory: KuberokuFactory) -> None:
    """Addon env vars (DATABASE_URL, REDIS_URL) present in app config."""
    config = real_factory.config.list("lifecycle")
    assert "DATABASE_URL" in config, f"DATABASE_URL missing from config: {list(config.keys())}"
    assert "REDIS_URL" in config, f"REDIS_URL missing from config: {list(config.keys())}"
    assert "postgres://" in config["DATABASE_URL"]
    assert "redis://" in config["REDIS_URL"]
    print(f"\n  DATABASE_URL: {config['DATABASE_URL'][:40]}...")
    print(f"  REDIS_URL: {config['REDIS_URL'][:40]}...")


def test_06_addon_network_policies(real_factory: KuberokuFactory) -> None:
    """Addon-allow policies exist for both addons."""
    for instance in ("postgres", "redis"):
        np_name = safe_name(
            real_factory.prefix,
            "netpol-allow",
            f"lifecycle-{instance}",
        )
        np = real_factory.k8s.get_network_policy(real_factory.namespace, np_name)
        assert np is not None
        ingress = np.get("spec", {}).get("ingress", [])
        assert len(ingress) > 0
        print(f"\n  {instance} addon-allow policy: {np_name}")


def test_07_internal_tcp_postgres(real_factory: KuberokuFactory) -> None:
    """App pod CAN reach postgres:5432 (same-app, addon-allow permits)."""
    pods = wait_for_pods_ready(real_factory, "lifecycle", timeout=120)
    pod_name = pods[0]["metadata"]["name"]
    pg_svc = safe_name(real_factory.prefix, "addon", "lifecycle-postgres")

    # NetworkPolicy rules may take 1-5s to propagate through the CNI
    poll_until(
        fn=lambda: _exec_tcp_probe(real_factory, pod_name, pg_svc, 5432),
        predicate=lambda ok: ok,
        timeout=30,
        interval=1,
        message="App pod cannot reach postgres — NetworkPolicy blocking same-app traffic",
    )
    print(f"\n  App->{pg_svc}:5432 TCP connected")


def test_08_internal_tcp_redis(real_factory: KuberokuFactory) -> None:
    """App pod CAN reach redis:6379 (same-app, addon-allow permits)."""
    pods = wait_for_pods_ready(real_factory, "lifecycle", timeout=120)
    pod_name = pods[0]["metadata"]["name"]
    rd_svc = safe_name(real_factory.prefix, "addon", "lifecycle-redis")

    poll_until(
        fn=lambda: _exec_tcp_probe(real_factory, pod_name, rd_svc, 6379),
        predicate=lambda ok: ok,
        timeout=30,
        interval=1,
        message="App pod cannot reach redis — NetworkPolicy blocking same-app traffic",
    )
    print(f"\n  App->{rd_svc}:6379 TCP connected")


def test_09_cross_app_isolation(real_factory: KuberokuFactory) -> None:
    """A second app's pod CANNOT reach the first app's postgres."""
    # Create second app
    real_factory.apps.create("isolated")
    real_factory.deploy.deploy(
        "isolated",
        image="nginx:1.27-alpine",
        ports=[{"containerPort": 80, "protocol": "TCP"}],
        wait=False,
    )

    pods = wait_for_pods_ready(real_factory, "isolated", timeout=120)
    pod_name = pods[0]["metadata"]["name"]
    pg_svc = safe_name(real_factory.prefix, "addon", "lifecycle-postgres")

    connected = _exec_tcp_probe(real_factory, pod_name, pg_svc, 5432, timeout=3)
    assert not connected, (
        "Cross-app request to postgres succeeded! "
        "NetworkPolicy not enforcing isolation. "
        "Ensure your CNI supports NetworkPolicy (Calico, Cilium, or k3s with kube-router)."
    )
    print(f"\n  isolated->{pg_svc}:5432 correctly BLOCKED")


def test_10_expose_web(real_factory: KuberokuFactory) -> None:
    """Expose web — Service becomes LoadBalancer, external TCP best-effort."""
    ep = real_factory.services.expose_on("lifecycle", "web")
    assert ep.kind == "process"
    assert ep.service_type == "LoadBalancer"
    print(f"\n  Exposed web: type={ep.service_type}")

    # Primary assertion: verify K8s state
    svc_name = safe_name(real_factory.prefix, "lifecycle", "web")
    svc = real_factory.k8s.get_service(real_factory.namespace, svc_name)
    assert svc["spec"]["type"] == "LoadBalancer"
    print(f"  Service type verified: {svc['spec']['type']}")

    # Best-effort TCP: on some environments (Colima) host can't reach VM NodePort
    try:
        host, port = _get_external_address(real_factory, svc_name, timeout=60)
        connected = _tcp_connect(host, port, timeout=10)
        status = "OK" if connected else "unreachable (VM network)"
        print(f"  External TCP to {host}:{port}: {status}")
    except TimeoutError:
        print("  External address not assigned (expected on some setups)")


def test_11_external_policy_web(real_factory: KuberokuFactory) -> None:
    """External-allow policy exists for web with exposed-by annotation."""
    np_name = safe_name(
        real_factory.prefix,
        "netpol-external",
        "lifecycle-web",
    )
    np = real_factory.k8s.get_network_policy(real_factory.namespace, np_name)
    assert np is not None
    # External policy: no `from` = allow all
    ingress = np.get("spec", {}).get("ingress", [])
    assert len(ingress) > 0
    assert "from" not in ingress[0]
    annotations = np.get("metadata", {}).get("annotations", {})
    assert f"{real_factory.domain}/exposed-by" in annotations
    assert annotations[f"{real_factory.domain}/exposed-by"] == "expose"
    print(f"\n  Web external policy: {np_name}")


def test_12_expose_postgres(real_factory: KuberokuFactory) -> None:
    """Expose postgres via gateway — addon Service stays ClusterIP, gateway LB created."""
    ep = real_factory.addons.expose_on("lifecycle", "postgres")
    assert ep.kind == "addon"
    assert ep.method == "gateway"
    assert ep.service_type == "LoadBalancer"
    assert ep.gateway_port is not None
    assert 10000 <= ep.gateway_port <= 10999
    print(f"\n  Exposed postgres via gateway: port={ep.gateway_port}")

    # Store endpoint info for test_17
    _exposed_postgres["gateway_port"] = ep.gateway_port
    _exposed_postgres["external_ip"] = ep.external_ip

    # Addon Service stays ClusterIP (traffic goes through gateway LB)
    svc_name = safe_name(real_factory.prefix, "addon", "lifecycle-postgres")
    svc = real_factory.k8s.get_service(real_factory.namespace, svc_name)
    assert svc["spec"]["type"] == "ClusterIP"
    print(f"  Addon service type: {svc['spec']['type']} (correct — gateway handles LB)")

    # Verify gateway LB Service was created
    gw_svc_name = f"{real_factory.prefix}-gw-{ep.gateway_port}"
    gw_svc = real_factory.k8s.get_service(real_factory.namespace, gw_svc_name)
    assert gw_svc["spec"]["type"] == "LoadBalancer"
    print(f"  Gateway LB service: {gw_svc_name} type={gw_svc['spec']['type']}")

    # Best-effort TCP to gateway
    try:
        host, port = _get_external_address(real_factory, gw_svc_name, timeout=60)
        connected = _tcp_connect(host, port, timeout=10)
        status = "OK" if connected else "unreachable (VM network)"
        print(f"  External TCP to {host}:{port}: {status}")
    except TimeoutError:
        print("  Gateway address not assigned yet (expected on some setups)")


def test_13_external_policy_postgres(real_factory: KuberokuFactory) -> None:
    """External-allow policy exists for postgres with addon labels."""
    np_name = safe_name(
        real_factory.prefix,
        "netpol-external",
        "lifecycle-postgres",
    )
    np = real_factory.k8s.get_network_policy(real_factory.namespace, np_name)
    assert np is not None
    ingress = np.get("spec", {}).get("ingress", [])
    assert len(ingress) > 0
    assert "from" not in ingress[0]
    np_labels = np.get("metadata", {}).get("labels", {})
    assert f"{real_factory.domain}/addon-instance" in np_labels
    print(f"\n  Postgres external policy: {np_name}")


def test_14_unexpose_web(real_factory: KuberokuFactory) -> None:
    """Unexpose web — reverts to ClusterIP, external TCP fails."""
    # Capture the external address before unexposing
    svc_name = safe_name(real_factory.prefix, "lifecycle", "web")
    host, port = _get_external_address(real_factory, svc_name, timeout=10)

    ep = real_factory.services.expose_off("lifecycle", "web")
    assert ep.service_type == "ClusterIP"
    print(f"\n  Unexposed web: type={ep.service_type}")

    # Verify Service reverted
    svc = real_factory.k8s.get_service(real_factory.namespace, svc_name)
    assert svc["spec"]["type"] == "ClusterIP"

    # Poll until external TCP no longer works (LB IP reclamation varies by env)
    poll_until(
        fn=lambda: _tcp_connect(host, port, timeout=3),
        predicate=lambda connected: not connected,
        timeout=15,
        interval=1,
        message=f"Still able to TCP connect to {host}:{port} after unexpose",
    )
    print(f"  TCP connect to {host}:{port} correctly REFUSED after unexpose")


def test_15_external_policy_deleted_web(real_factory: KuberokuFactory) -> None:
    """Web external-allow policy is gone after unexpose."""
    np_name = safe_name(
        real_factory.prefix,
        "netpol-external",
        "lifecycle-web",
    )
    try:
        real_factory.k8s.get_network_policy(real_factory.namespace, np_name)
        pytest.fail(f"Policy {np_name} should have been deleted")
    except Exception:
        print(f"\n  Web external policy deleted: {np_name}")


def test_16_web_still_reachable_internally(real_factory: KuberokuFactory) -> None:
    """After unexpose, internal same-app traffic still works."""
    pods = wait_for_pods_ready(real_factory, "lifecycle", timeout=120)
    pod_name = pods[0]["metadata"]["name"]
    web_svc = safe_name(real_factory.prefix, "lifecycle", "web")

    # wget from the web pod to its own service — should still work
    try:
        output = real_factory.k8s.exec_in_pod(
            real_factory.namespace,
            pod_name,
            ["wget", "-qO-", "-T", "5", f"http://{web_svc}:80/"],
        )
        assert "nginx" in output.lower() or "html" in output.lower()
        print(f"\n  Internal web->web:80 still works after unexpose")
    except Exception as e:
        pytest.fail(f"Internal web access broken after unexpose: {e}")


def test_17_unexpose_postgres(real_factory: KuberokuFactory) -> None:
    """Unexpose postgres — gateway deallocated, gateway LB Service removed."""
    gw_port = _exposed_postgres.get("gateway_port")
    assert gw_port is not None, "test_12 must run first to store gateway_port"

    # Capture external address from gateway LB before unexpose (best-effort)
    gw_svc_name = f"{real_factory.prefix}-gw-{gw_port}"
    gw_host: str | None = None
    gw_ext_port: int | None = None
    try:
        gw_host, gw_ext_port = _get_external_address(
            real_factory,
            gw_svc_name,
            timeout=10,
        )
    except TimeoutError:
        pass  # Gateway may not have external IP on some setups

    ep = real_factory.addons.expose_off("lifecycle", "postgres")
    assert ep.service_type == "ClusterIP"
    print(f"\n  Unexposed postgres: type={ep.service_type}")

    # Verify gateway allocation is gone
    addon_svc = safe_name(real_factory.prefix, "addon", "lifecycle-postgres")
    alloc = real_factory.gateway.get_allocation("lifecycle", addon_svc)
    assert alloc is None, f"Gateway allocation should be removed, got: {alloc}"
    print(f"  Gateway allocation removed (port {gw_port})")

    # Verify gateway LB Service was deleted (K8s deletion is async — poll)
    try:
        wait_for_service_deleted(real_factory, gw_svc_name)
        print(f"  Gateway service {gw_svc_name} deleted")
    except TimeoutError:
        pytest.fail(f"Gateway service {gw_svc_name} should have been deleted")

    # Best-effort: verify external TCP no longer works
    if gw_host and gw_ext_port:
        try:
            poll_until(
                fn=lambda: _tcp_connect(gw_host, gw_ext_port, timeout=3),
                predicate=lambda c: not c,
                timeout=10,
                interval=1,
                message="LB teardown",
            )
            print(f"  TCP to {gw_host}:{gw_ext_port} correctly REFUSED")
        except TimeoutError:
            print(f"  TCP to {gw_host}:{gw_ext_port} still reachable (LB teardown lag)")


def test_18_external_policy_deleted_postgres(real_factory: KuberokuFactory) -> None:
    """Postgres external-allow policy is gone after unexpose."""
    np_name = safe_name(
        real_factory.prefix,
        "netpol-external",
        "lifecycle-postgres",
    )
    try:
        real_factory.k8s.get_network_policy(real_factory.namespace, np_name)
        pytest.fail(f"Policy {np_name} should have been deleted")
    except Exception:
        print(f"\n  Postgres external policy deleted: {np_name}")


def test_19_postgres_still_reachable_internally(
    real_factory: KuberokuFactory,
) -> None:
    """After unexpose, same-app pod can still reach postgres internally."""
    pods = wait_for_pods_ready(real_factory, "lifecycle", timeout=120)
    pod_name = pods[0]["metadata"]["name"]
    pg_svc = safe_name(real_factory.prefix, "addon", "lifecycle-postgres")

    connected = _exec_tcp_probe(real_factory, pod_name, pg_svc, 5432)
    assert connected, "Internal postgres access broken after unexpose!"
    print(f"\n  Internal App->{pg_svc}:5432 still works after unexpose")


def test_20_cross_app_still_blocked(real_factory: KuberokuFactory) -> None:
    """After expose/unexpose cycle, cross-app isolation still holds."""
    pods = wait_for_pods_ready(real_factory, "isolated", timeout=120)
    pod_name = pods[0]["metadata"]["name"]
    pg_svc = safe_name(real_factory.prefix, "addon", "lifecycle-postgres")

    connected = _exec_tcp_probe(real_factory, pod_name, pg_svc, 5432, timeout=3)
    assert not connected, (
        "Cross-app still reaches postgres after expose/unexpose cycle! "
        "Ensure your CNI supports NetworkPolicy (Calico, Cilium, or k3s with kube-router)."
    )
    print(f"\n  isolated->{pg_svc}:5432 still BLOCKED after expose/unexpose cycle")


def test_21_postgres_credentials(real_factory: KuberokuFactory) -> None:
    """Verify credential fields are present."""
    creds = real_factory.addons.credentials("lifecycle", "postgres")
    assert creds["username"] == "kuberoku"
    assert creds["database"] == "lifecycle"
    assert "password" in creds
    assert "postgres://" in creds["url"]
    print(f"\n  Creds: user={creds['username']}, db={creds['database']}")


def test_22_credentials_rotate(real_factory: KuberokuFactory) -> None:
    """Rotate credentials — new password, config updated, still works."""
    old_creds = real_factory.addons.credentials("lifecycle", "postgres")
    new_creds = real_factory.addons.credentials_rotate("lifecycle", "postgres")
    assert new_creds["password"] != old_creds["password"]

    config = real_factory.config.list("lifecycle")
    assert new_creds["password"] in config["DATABASE_URL"]
    print(f"\n  Password rotated")

    # Verify postgres still responds with new credentials
    sts_name = safe_name(real_factory.prefix, "addon", "lifecycle-postgres")
    pod_name = f"{sts_name}-0"
    output = real_factory.k8s.exec_in_pod(
        real_factory.namespace,
        pod_name,
        ["pg_isready", "-U", "kuberoku"],
    )
    assert "accepting connections" in output
    print(f"  pg_isready after rotate: OK")


def test_23_scale_postgres(real_factory: KuberokuFactory) -> None:
    """Scale postgres memory to 512Mi."""
    addon = real_factory.addons.scale(
        "lifecycle",
        "postgres",
        memory="512Mi",
    )
    assert addon.memory == "512Mi"
    print(f"\n  Scaled postgres memory: {addon.memory}")

    # Wait for rollout after scale (triggers STS restart)
    sts_name = safe_name(real_factory.prefix, "addon", "lifecycle-postgres")
    wait_for_sts_ready(real_factory, sts_name)

    # Verify StatefulSet has new resources
    sts = real_factory.k8s.get_statefulset(real_factory.namespace, sts_name)
    container = sts["spec"]["template"]["spec"]["containers"][0]
    assert container["resources"]["requests"]["memory"] == "512Mi"


def test_24_postgres_backup(real_factory: KuberokuFactory) -> None:
    """pg_dump produces non-empty output."""
    # Wait for pod to be ready after test_23 upgrade (which triggers restart)
    sts_name = safe_name(real_factory.prefix, "addon", "lifecycle-postgres")
    wait_for_sts_ready(real_factory, sts_name)

    # Also verify pg_isready — container may be Running but postgres still starting
    # Use generous timeout: after STS scale, postgres re-initializes from scratch
    pod_name = f"{sts_name}-0"
    poll_until(
        fn=lambda: real_factory.k8s.exec_in_pod(
            real_factory.namespace,
            pod_name,
            ["pg_isready", "-U", "kuberoku"],
            container="postgres",
        ),
        predicate=lambda o: "accepting connections" in o,
        timeout=120,
        interval=2,
        message="pg_isready never returned 'accepting connections'",
    )

    output = real_factory.addons.backup("lifecycle", "postgres")
    assert len(output) > 0
    print(f"\n  Backup length: {len(output)}")
    print(f"  Preview: {output[:100]}")


def test_25_addons_exec_postgres(real_factory: KuberokuFactory) -> None:
    """addons:exec for postgres — should return pod name and psql command."""
    pod_name, cmd = real_factory.addons.exec_command("lifecycle", "postgres")
    assert "psql" in cmd[0], f"Expected psql in command, got {cmd}"
    assert "-U" in cmd, f"Expected -U flag in command: {cmd}"
    print(f"\n  exec_command: pod={pod_name}, cmd={cmd}")

    # Actually run the command to verify it works
    output = real_factory.k8s.exec_in_pod(
        real_factory.namespace,
        pod_name,
        cmd + ["-c", "SELECT 1 AS test_value;"],
    )
    assert "test_value" in output or "1" in output, f"psql query failed: {output}"
    print(f"  psql SELECT 1: {output.strip()[:100]}")


def test_26_addons_exec_redis(real_factory: KuberokuFactory) -> None:
    """addons:exec for redis — should return pod name and redis-cli command."""
    pod_name, cmd = real_factory.addons.exec_command("lifecycle", "redis")
    assert "redis-cli" in cmd[0], f"Expected redis-cli in command, got {cmd}"
    print(f"\n  exec_command: pod={pod_name}, cmd={cmd}")

    # Actually run the command to verify it connects
    output = real_factory.k8s.exec_in_pod(
        real_factory.namespace,
        pod_name,
        cmd + ["PING"],
    )
    assert "PONG" in output, f"redis-cli PING failed: {output}"
    print(f"  redis-cli PING: {output.strip()}")


def test_27_addons_exec_write_read_postgres(real_factory: KuberokuFactory) -> None:
    """Write data via psql exec, read it back — verify real DB operations."""
    pod_name, cmd = real_factory.addons.exec_command("lifecycle", "postgres")

    # Create a table and insert data
    output_create = real_factory.k8s.exec_in_pod(
        real_factory.namespace,
        pod_name,
        cmd + ["-c", "CREATE TABLE IF NOT EXISTS e2e_test (id int, val text);"],
    )
    print(f"\n  CREATE TABLE: {output_create.strip()[:80]}")

    output_insert = real_factory.k8s.exec_in_pod(
        real_factory.namespace,
        pod_name,
        cmd + ["-c", "INSERT INTO e2e_test VALUES (1, 'hello-e2e');"],
    )
    print(f"  INSERT: {output_insert.strip()[:80]}")

    # Read it back
    output_select = real_factory.k8s.exec_in_pod(
        real_factory.namespace,
        pod_name,
        cmd + ["-c", "SELECT val FROM e2e_test WHERE id = 1;"],
    )
    assert "hello-e2e" in output_select, f"Data not found: {output_select}"
    print(f"  SELECT: {output_select.strip()[:80]}")


def test_28_destroy_redis_retain_data(real_factory: KuberokuFactory) -> None:
    """Destroy redis without --delete-data — PVC retained, REDIS_URL removed."""
    real_factory.addons.destroy("lifecycle", "redis")
    addons = real_factory.addons.list("lifecycle")
    names = {a.instance_name for a in addons}
    assert "redis" not in names

    config = real_factory.config.list("lifecycle")
    assert "REDIS_URL" not in config
    print(f"\n  Redis destroyed, REDIS_URL removed")


def test_29_destroy_postgres_delete_data(real_factory: KuberokuFactory) -> None:
    """Destroy postgres with --delete-data — PVC deleted."""
    pvc_name = addon_pvc_name(
        real_factory.prefix,
        "lifecycle",
        "postgres",
        POSTGRES.volume_name,
    )

    real_factory.addons.destroy("lifecycle", "postgres", delete_data=True)

    addons = real_factory.addons.list("lifecycle")
    assert len(addons) == 0

    try:
        real_factory.k8s.get_pvc(real_factory.namespace, pvc_name)
        pytest.fail(f"PVC {pvc_name} should have been deleted")
    except Exception:
        print(f"\n  PVC deleted: {pvc_name}")

    config = real_factory.config.list("lifecycle")
    assert "DATABASE_URL" not in config
    print(f"  DATABASE_URL removed")


def test_30_no_addons_remain(real_factory: KuberokuFactory) -> None:
    """No addons left on the app."""
    addons = real_factory.addons.list("lifecycle")
    assert len(addons) == 0
    print("\n  No addons remain")


def test_31_cleanup_second_app(real_factory: KuberokuFactory) -> None:
    """Destroy the isolated app from cross-app test."""
    real_factory.apps.destroy("isolated")
    app_list = real_factory.apps.list()
    assert not any(a.name == "isolated" for a in app_list)
    print("\n  Isolated app destroyed")


def test_32_destroy_app(real_factory: KuberokuFactory) -> None:
    """Destroy the lifecycle app."""
    real_factory.apps.destroy("lifecycle")
    app_list = real_factory.apps.list()
    assert not any(a.name == "lifecycle" for a in app_list)
    print("\n  Lifecycle app destroyed")
