"""Tests for operation tracking."""

from __future__ import annotations

from datetime import UTC, datetime, timedelta
from unittest.mock import MagicMock

from fastapi.testclient import TestClient

from ilum.api.models import OperationResponse
from ilum.api.operations import OperationStore
from ilum.core.kubernetes import PodStatus
from ilum.errors import HelmError


class TestOperationStore:
    def test_create_returns_id(self) -> None:
        store = OperationStore()
        op_id = store.create(operation="enable", modules=["jupyter"])
        assert len(op_id) == 12

    def test_get_returns_created_op(self) -> None:
        store = OperationStore()
        op_id = store.create(operation="enable", modules=["jupyter"])
        op = store.get(op_id)
        assert op is not None
        assert op.status == "pending"
        assert op.operation == "enable"
        assert op.modules == ["jupyter"]

    def test_get_unknown_returns_none(self) -> None:
        store = OperationStore()
        assert store.get("nonexistent") is None

    def test_list_all_newest_first(self) -> None:
        store = OperationStore()
        id1 = store.create(operation="enable", modules=["a"])
        id2 = store.create(operation="disable", modules=["b"])
        ops = store.list_all()
        assert ops[0].id == id2
        assert ops[1].id == id1

    def test_fifo_eviction(self) -> None:
        store = OperationStore(max_size=3)
        ids = [store.create(operation="enable", modules=[f"m{i}"]) for i in range(5)]
        # First 2 should be evicted
        assert store.get(ids[0]) is None
        assert store.get(ids[1]) is None
        assert store.get(ids[2]) is not None
        assert store.get(ids[3]) is not None
        assert store.get(ids[4]) is not None

    def test_run_in_thread_success(self) -> None:
        store = OperationStore()
        op_id = store.create(operation="enable", modules=["jupyter"])

        def _ok(op_id: str, store: OperationStore) -> None:
            store.update_progress(op_id, 50, "Halfway there")

        t = store.run_in_thread(op_id, _ok)
        t.join(timeout=5)
        op = store.get(op_id)
        assert op is not None
        assert op.status == "completed"
        assert op.completed_at != ""
        assert op.progress == 100  # run_in_thread sets 100 on completion

    def test_run_in_thread_ilum_error(self) -> None:
        store = OperationStore()
        op_id = store.create(operation="enable", modules=["jupyter"])

        def _fail(op_id: str, store: OperationStore) -> None:
            raise HelmError("helm failed", error_code="ILUM-020")

        t = store.run_in_thread(op_id, _fail)
        t.join(timeout=5)
        op = store.get(op_id)
        assert op is not None
        assert op.status == "failed"
        assert "helm failed" in op.error
        assert op.error_code == "ILUM-020"

    def test_run_in_thread_generic_error(self) -> None:
        store = OperationStore()
        op_id = store.create(operation="enable", modules=["jupyter"])

        def _fail(op_id: str, store: OperationStore) -> None:
            raise RuntimeError("unexpected")

        t = store.run_in_thread(op_id, _fail)
        t.join(timeout=5)
        op = store.get(op_id)
        assert op is not None
        assert op.status == "failed"
        assert "unexpected" in op.error

    def test_completed_op_has_success_log(self) -> None:
        store = OperationStore()
        op_id = store.create(operation="enable", modules=["jupyter"])

        def _ok(op_id: str, store: OperationStore) -> None:
            pass

        t = store.run_in_thread(op_id, _ok)
        t.join(timeout=5)
        op = store.get(op_id)
        assert op is not None
        assert any(log.level == "success" for log in op.logs)
        assert any("completed successfully" in log.message for log in op.logs)

    def test_failed_op_has_error_log(self) -> None:
        store = OperationStore()
        op_id = store.create(operation="enable", modules=["jupyter"])

        def _fail(op_id: str, store: OperationStore) -> None:
            raise RuntimeError("something broke")

        t = store.run_in_thread(op_id, _fail)
        t.join(timeout=5)
        op = store.get(op_id)
        assert op is not None
        assert any(log.level == "error" for log in op.logs)
        assert any("something broke" in log.message for log in op.logs)

    def test_import_operation(self) -> None:
        store = OperationStore()
        op = OperationResponse(
            id="test123",
            status="completed",
            operation="enable",
            modules=["jupyter"],
            job_name="ilum-helm-test123",
        )
        store.import_operation(op)
        result = store.get("test123")
        assert result is not None
        assert result.status == "completed"
        assert result.job_name == "ilum-helm-test123"

    def test_import_operation_dedup(self) -> None:
        store = OperationStore()
        op = OperationResponse(
            id="test123",
            status="completed",
            operation="enable",
            modules=["jupyter"],
        )
        store.import_operation(op)
        # Import again with different status — should overwrite
        op2 = OperationResponse(
            id="test123",
            status="failed",
            operation="enable",
            modules=["jupyter"],
        )
        store.import_operation(op2)
        result = store.get("test123")
        assert result is not None
        assert result.status == "failed"

    def test_has_active_helm_operation_false_when_empty(self) -> None:
        store = OperationStore()
        assert store.has_active_helm_operation() is False

    def test_has_active_helm_operation_true_when_running(self) -> None:
        store = OperationStore()
        op_id = store.create(operation="enable", modules=["jupyter"])
        op = store.get(op_id)
        assert op is not None
        op.status = "running"
        assert store.has_active_helm_operation() is True

    def test_has_active_helm_operation_false_for_restart(self) -> None:
        store = OperationStore()
        op_id = store.create(operation="restart", modules=["jupyter"])
        op = store.get(op_id)
        assert op is not None
        op.status = "running"
        assert store.has_active_helm_operation() is False

    def test_has_active_helm_operation_false_when_completed(self) -> None:
        store = OperationStore()
        op_id = store.create(operation="enable", modules=["jupyter"])
        op = store.get(op_id)
        assert op is not None
        op.status = "completed"
        assert store.has_active_helm_operation() is False


class TestOperationProgress:
    def test_update_progress(self) -> None:
        store = OperationStore()
        op_id = store.create(operation="enable", modules=["jupyter"])
        store.update_progress(op_id, 50, "Halfway there")
        op = store.get(op_id)
        assert op is not None
        assert op.progress == 50
        assert len(op.logs) == 1
        assert op.logs[0].message == "Halfway there"
        assert op.logs[0].level == "info"

    def test_update_progress_without_message(self) -> None:
        store = OperationStore()
        op_id = store.create(operation="enable", modules=["jupyter"])
        store.update_progress(op_id, 30)
        op = store.get(op_id)
        assert op is not None
        assert op.progress == 30
        assert len(op.logs) == 0

    def test_add_log(self) -> None:
        store = OperationStore()
        op_id = store.create(operation="enable", modules=["jupyter"])
        store.add_log(op_id, "Something happened", level="warn")
        op = store.get(op_id)
        assert op is not None
        assert len(op.logs) == 1
        assert op.logs[0].message == "Something happened"
        assert op.logs[0].level == "warn"
        assert op.logs[0].timestamp != ""

    def test_multiple_progress_updates(self) -> None:
        store = OperationStore()
        op_id = store.create(operation="enable", modules=["jupyter"])
        store.update_progress(op_id, 10, "Planning")
        store.update_progress(op_id, 50, "Running Helm upgrade")
        store.update_progress(op_id, 90, "Verifying")
        op = store.get(op_id)
        assert op is not None
        assert op.progress == 90
        assert len(op.logs) == 3
        assert op.logs[0].message == "Planning"
        assert op.logs[2].message == "Verifying"

    def test_progress_in_thread_run(self) -> None:
        store = OperationStore()
        op_id = store.create(operation="enable", modules=["jupyter"])

        def _run(op_id: str, store: OperationStore) -> None:
            store.update_progress(op_id, 10, "Step 1")
            store.update_progress(op_id, 50, "Step 2")
            store.update_progress(op_id, 90, "Step 3")

        t = store.run_in_thread(op_id, _run)
        t.join(timeout=5)
        op = store.get(op_id)
        assert op is not None
        assert op.status == "completed"
        assert op.progress == 100
        assert len(op.logs) == 4  # 3 progress + 1 success
        assert op.logs[-1].level == "success"


class TestOperationCancel:
    def test_mark_cancelling(self) -> None:
        store = OperationStore()
        op_id = store.create(operation="enable", modules=["jupyter"])
        store.mark_cancelling(op_id)
        op = store.get(op_id)
        assert op is not None
        assert op.status == "cancelling"

    def test_mark_cancelled(self) -> None:
        store = OperationStore()
        op_id = store.create(operation="enable", modules=["jupyter"])
        store.mark_cancelled(op_id)
        op = store.get(op_id)
        assert op is not None
        assert op.status == "cancelled"
        assert op.completed_at != ""


class TestOperationEndpoints:
    def test_list_operations_empty(self, api_client: TestClient) -> None:
        resp = api_client.get("/api/v1/operations")
        assert resp.status_code == 200
        assert resp.json() == []

    def test_get_operation_not_found(self, api_client: TestClient) -> None:
        resp = api_client.get("/api/v1/operations/nonexistent")
        assert resp.status_code == 404

    def test_enable_creates_operation(
        self, api_client: TestClient, mock_api_manager: MagicMock
    ) -> None:
        resp = api_client.post("/api/v1/modules/jupyter/enable")
        assert resp.status_code == 202
        op_id = resp.json()["id"]

        # Should appear in operations list
        resp = api_client.get("/api/v1/operations")
        assert resp.status_code == 200
        ids = [op["id"] for op in resp.json()]
        assert op_id in ids

        # Should be retrievable by ID
        resp = api_client.get(f"/api/v1/operations/{op_id}")
        assert resp.status_code == 200
        assert resp.json()["operation"] == "enable"

    def test_operation_response_includes_job_name(
        self, api_client: TestClient, mock_api_manager: MagicMock
    ) -> None:
        resp = api_client.post("/api/v1/modules/jupyter/enable")
        assert resp.status_code == 202
        op_id = resp.json()["id"]

        resp = api_client.get(f"/api/v1/operations/{op_id}")
        assert resp.status_code == 200
        data = resp.json()
        assert "job_name" in data
        assert data["job_name"] == f"ilum-helm-{op_id}"

    def test_filter_operations_by_module(
        self, api_client: TestClient, mock_api_manager: MagicMock
    ) -> None:
        # Create an enable operation
        resp1 = api_client.post("/api/v1/modules/jupyter/enable")
        assert resp1.status_code == 202

        # Filter by module
        resp = api_client.get("/api/v1/operations?module=jupyter")
        assert resp.status_code == 200
        data = resp.json()
        assert len(data) >= 1
        assert all("jupyter" in op["modules"] for op in data)

    def test_cancel_operation_not_found(self, api_client: TestClient) -> None:
        resp = api_client.post("/api/v1/operations/nonexistent/cancel")
        assert resp.status_code == 404

    def test_cancel_operation(self, api_client: TestClient, mock_api_manager: MagicMock) -> None:
        resp = api_client.post("/api/v1/modules/jupyter/enable")
        assert resp.status_code == 202
        op_id = resp.json()["id"]

        resp = api_client.post(f"/api/v1/operations/{op_id}/cancel")
        assert resp.status_code == 200
        data = resp.json()
        assert data["id"] == op_id
        assert data["status"] == "cancelling"
        assert "Rolling back" in data["message"]

    def test_cancel_deletes_job(self, api_client: TestClient, mock_api_manager: MagicMock) -> None:
        """Cancel should delete the backing K8s Job."""
        resp = api_client.post("/api/v1/modules/jupyter/enable")
        assert resp.status_code == 202
        op_id = resp.json()["id"]

        resp = api_client.post(f"/api/v1/operations/{op_id}/cancel")
        assert resp.status_code == 200

        # Give the rollback thread a moment
        import time

        time.sleep(0.2)
        mock_api_manager.k8s.delete_job.assert_called_once()

    def test_cancel_already_cancelling(
        self, api_client: TestClient, mock_api_manager: MagicMock
    ) -> None:
        resp = api_client.post("/api/v1/modules/jupyter/enable")
        assert resp.status_code == 202
        op_id = resp.json()["id"]

        # Cancel twice
        resp = api_client.post(f"/api/v1/operations/{op_id}/cancel")
        assert resp.status_code == 200

        resp = api_client.post(f"/api/v1/operations/{op_id}/cancel")
        assert resp.status_code == 200
        data = resp.json()
        assert data["status"] in ("cancelling", "cancelled")
        assert "Already cancelling" in data["message"]

    def test_cancel_completed_operation_rejected(
        self, api_client: TestClient, mock_api_manager: MagicMock
    ) -> None:
        """B2: Cancel on a completed operation should return 'already finished'."""
        resp = api_client.post("/api/v1/modules/jupyter/enable")
        assert resp.status_code == 202
        op_id = resp.json()["id"]

        # Manually mark as completed
        import ilum.api.operations as ops_mod

        store = ops_mod._store
        assert store is not None
        op = store.get(op_id)
        assert op is not None
        op.status = "completed"

        resp = api_client.post(f"/api/v1/operations/{op_id}/cancel")
        assert resp.status_code == 200
        data = resp.json()
        assert data["status"] == "completed"
        assert "already finished" in data["message"]

    def test_cancel_failed_operation_rejected(
        self, api_client: TestClient, mock_api_manager: MagicMock
    ) -> None:
        """B2: Cancel on a failed operation should return 'already finished'."""
        resp = api_client.post("/api/v1/modules/jupyter/enable")
        assert resp.status_code == 202
        op_id = resp.json()["id"]

        # Manually mark as failed
        import ilum.api.operations as ops_mod

        store = ops_mod._store
        assert store is not None
        op = store.get(op_id)
        assert op is not None
        op.status = "failed"

        resp = api_client.post(f"/api/v1/operations/{op_id}/cancel")
        assert resp.status_code == 200
        data = resp.json()
        assert data["status"] == "failed"
        assert "already finished" in data["message"]


class TestListOperationsRefresh:
    def test_list_refreshes_job_statuses(
        self, api_client: TestClient, mock_api_manager: MagicMock
    ) -> None:
        """GET /operations should refresh job statuses from K8s.

        With readiness checking, a completed Job transitions to
        ``awaiting_readiness`` first.  When pods are ready on the same poll,
        the operation completes.
        """
        # Create an enable operation (creates a Job)
        resp = api_client.post("/api/v1/modules/jupyter/enable")
        assert resp.status_code == 202
        op_id = resp.json()["id"]

        # Simulate the backing Job having completed
        completed_job = MagicMock()
        cond = MagicMock()
        cond.type = "Complete"
        cond.status = "True"
        cond.reason = ""
        cond.message = ""
        completed_job.status.conditions = [cond]
        completed_job.status.active = None
        completed_job.status.succeeded = 1
        completed_job.status.failed = None
        mock_api_manager.k8s.get_job.return_value = completed_job

        # Pods are ready — readiness check passes on first poll
        mock_api_manager.k8s.list_pods_by_label.return_value = [
            PodStatus(
                name="jupyter-0",
                namespace="default",
                phase="Running",
                ready=True,
                restart_count=0,
            ),
        ]

        # List operations — should trigger refresh
        resp = api_client.get("/api/v1/operations")
        assert resp.status_code == 200
        data = resp.json()
        op = next(o for o in data if o["id"] == op_id)
        assert op["status"] == "completed"
        assert op["progress"] == 100

    def test_list_refreshes_failed_job(
        self, api_client: TestClient, mock_api_manager: MagicMock
    ) -> None:
        """GET /operations should detect failed Jobs."""
        resp = api_client.post("/api/v1/modules/jupyter/enable")
        assert resp.status_code == 202
        op_id = resp.json()["id"]

        # Simulate the backing Job having failed
        failed_job = MagicMock()
        cond = MagicMock()
        cond.type = "Failed"
        cond.status = "True"
        cond.reason = "BackoffLimitExceeded"
        cond.message = "limit reached"
        failed_job.status.conditions = [cond]
        failed_job.status.active = None
        failed_job.status.succeeded = None
        failed_job.status.failed = 1
        mock_api_manager.k8s.get_job.return_value = failed_job

        resp = api_client.get("/api/v1/operations")
        assert resp.status_code == 200
        data = resp.json()
        op = next(o for o in data if o["id"] == op_id)
        assert op["status"] == "failed"

    def test_refresh_marks_failed_when_job_not_found(
        self, api_client: TestClient, mock_api_manager: MagicMock
    ) -> None:
        """If the backing Job was deleted, the operation should be marked failed."""
        resp = api_client.post("/api/v1/modules/jupyter/enable")
        assert resp.status_code == 202
        op_id = resp.json()["id"]

        # Simulate the Job no longer existing
        mock_api_manager.k8s.get_job.side_effect = Exception("Not Found")

        resp = api_client.get(f"/api/v1/operations/{op_id}")
        assert resp.status_code == 200
        data = resp.json()
        assert data["status"] == "failed"
        assert "no longer exists" in data["error"]


class TestPodReadiness:
    """Tests for the pod readiness check after helm Job completion."""

    @staticmethod
    def _make_completed_job() -> MagicMock:
        """Return a MagicMock mimicking a completed K8s Job."""
        job = MagicMock()
        cond = MagicMock()
        cond.type = "Complete"
        cond.status = "True"
        cond.reason = ""
        cond.message = ""
        job.status.conditions = [cond]
        job.status.active = None
        job.status.succeeded = 1
        job.status.failed = None
        return job

    def test_job_complete_pods_ready_same_poll(
        self, api_client: TestClient, mock_api_manager: MagicMock
    ) -> None:
        """Job completes + pods ready → operation 'completed' on same poll."""
        resp = api_client.post("/api/v1/modules/jupyter/enable")
        assert resp.status_code == 202
        op_id = resp.json()["id"]

        mock_api_manager.k8s.get_job.return_value = self._make_completed_job()
        mock_api_manager.k8s.list_pods_by_label.return_value = [
            PodStatus(
                name="jupyter-0",
                namespace="default",
                phase="Running",
                ready=True,
                restart_count=0,
            ),
        ]

        resp = api_client.get(f"/api/v1/operations/{op_id}")
        assert resp.status_code == 200
        data = resp.json()
        assert data["status"] == "completed"
        assert data["progress"] == 100
        assert data["completed_at"] != ""
        assert data["helm_completed_at"] != ""

    def test_job_complete_pods_not_ready(
        self, api_client: TestClient, mock_api_manager: MagicMock
    ) -> None:
        """Job completes + pods not ready → operation stays 'awaiting_readiness'."""
        resp = api_client.post("/api/v1/modules/jupyter/enable")
        assert resp.status_code == 202
        op_id = resp.json()["id"]

        mock_api_manager.k8s.get_job.return_value = self._make_completed_job()
        mock_api_manager.k8s.list_pods_by_label.return_value = [
            PodStatus(
                name="jupyter-0",
                namespace="default",
                phase="Pending",
                ready=False,
                restart_count=0,
            ),
        ]

        resp = api_client.get(f"/api/v1/operations/{op_id}")
        assert resp.status_code == 200
        data = resp.json()
        assert data["status"] == "awaiting_readiness"
        assert data["completed_at"] == ""
        assert 80 <= data["progress"] < 100

    def test_readiness_timeout_marks_failed(
        self, api_client: TestClient, mock_api_manager: MagicMock
    ) -> None:
        """Readiness timeout → operation 'failed'."""
        resp = api_client.post("/api/v1/modules/jupyter/enable")
        assert resp.status_code == 202
        op_id = resp.json()["id"]

        mock_api_manager.k8s.get_job.return_value = self._make_completed_job()
        mock_api_manager.k8s.list_pods_by_label.return_value = [
            PodStatus(
                name="jupyter-0",
                namespace="default",
                phase="Pending",
                ready=False,
                restart_count=0,
            ),
        ]

        # First poll: Job completes → awaiting_readiness
        resp = api_client.get(f"/api/v1/operations/{op_id}")
        assert resp.json()["status"] == "awaiting_readiness"

        # Manually set helm_completed_at to the past to trigger timeout
        import ilum.api.operations as ops_mod

        store = ops_mod._store
        assert store is not None
        op = store.get(op_id)
        assert op is not None
        op.helm_completed_at = (datetime.now(UTC) - timedelta(seconds=700)).isoformat()

        # Second poll: timeout exceeded → failed
        resp = api_client.get(f"/api/v1/operations/{op_id}")
        assert resp.status_code == 200
        data = resp.json()
        assert data["status"] == "failed"
        assert "ready within" in data["error"]

    def test_disable_operation_skips_readiness(
        self, api_client: TestClient, mock_api_manager: MagicMock
    ) -> None:
        """Disable operation → immediate 'completed' (no readiness check)."""
        resp = api_client.post("/api/v1/modules/jupyter/disable")
        assert resp.status_code == 202
        op_id = resp.json()["id"]

        mock_api_manager.k8s.get_job.return_value = self._make_completed_job()
        # Don't set up list_pods_by_label — should not be called

        resp = api_client.get(f"/api/v1/operations/{op_id}")
        assert resp.status_code == 200
        data = resp.json()
        assert data["status"] == "completed"
        assert data["progress"] == 100

    def test_no_modules_skips_readiness(
        self, api_client: TestClient, mock_api_manager: MagicMock
    ) -> None:
        """Operation with no modules → immediate 'completed'."""
        # Use values update which has no modules
        resp = api_client.put(
            "/api/v1/values",
            json={"set_flags": ["ilum-core.replicaCount=2"]},
        )
        assert resp.status_code == 202
        op_id = resp.json()["id"]

        mock_api_manager.k8s.get_job.return_value = self._make_completed_job()

        resp = api_client.get(f"/api/v1/operations/{op_id}")
        assert resp.status_code == 200
        data = resp.json()
        assert data["status"] == "completed"

    def test_has_active_helm_operation_false_for_awaiting_readiness(self) -> None:
        """'awaiting_readiness' should not block new helm operations."""
        store = OperationStore()
        op_id = store.create(operation="enable", modules=["jupyter"])
        op = store.get(op_id)
        assert op is not None
        op.status = "awaiting_readiness"
        assert store.has_active_helm_operation() is False

    def test_recovery_completed_job_imports_as_awaiting_readiness(
        self, mock_api_manager: MagicMock
    ) -> None:
        """Recovery of a completed Job should import as 'awaiting_readiness'."""
        from ilum.api.job_runner import (
            ANNOTATION_MODULES,
            LABEL_MANAGED_BY,
            LABEL_OPERATION,
            LABEL_OPERATION_ID,
            MANAGED_BY_LABEL,
            recover_operations,
        )

        store = OperationStore()

        job = MagicMock()
        job.metadata.labels = {
            LABEL_MANAGED_BY: MANAGED_BY_LABEL,
            LABEL_OPERATION: "enable",
            LABEL_OPERATION_ID: "abc123",
        }
        job.metadata.annotations = {ANNOTATION_MODULES: "jupyter"}
        job.metadata.name = "ilum-helm-abc123"
        job.metadata.creation_timestamp = datetime.now(UTC)
        job.status.conditions = []
        job.status.succeeded = 1
        job.status.failed = None
        job.status.active = None
        job.status.completion_time = None

        mock_api_manager.k8s.list_jobs_by_label.return_value = [job]
        mock_api_manager.k8s.get_job_pod_logs.return_value = ""

        count = recover_operations(mock_api_manager.k8s, store, "default")
        assert count == 1

        op = store.get("abc123")
        assert op is not None
        assert op.status == "awaiting_readiness"
        assert op.progress == 80
        assert op.helm_completed_at != ""
        assert op.completed_at == ""

    def test_readiness_progress_updates(
        self, api_client: TestClient, mock_api_manager: MagicMock
    ) -> None:
        """Progress should reflect pod readiness ratio (80-99 range)."""
        resp = api_client.post("/api/v1/modules/jupyter/enable")
        assert resp.status_code == 202
        op_id = resp.json()["id"]

        mock_api_manager.k8s.get_job.return_value = self._make_completed_job()
        # 1 of 2 pods ready
        mock_api_manager.k8s.list_pods_by_label.return_value = [
            PodStatus(
                name="jupyter-0",
                namespace="default",
                phase="Running",
                ready=True,
                restart_count=0,
            ),
            PodStatus(
                name="jupyter-1",
                namespace="default",
                phase="Pending",
                ready=False,
                restart_count=0,
            ),
        ]

        resp = api_client.get(f"/api/v1/operations/{op_id}")
        assert resp.status_code == 200
        data = resp.json()
        assert data["status"] == "awaiting_readiness"
        # 80 + int((1/2) * 19) = 80 + 9 = 89
        assert data["progress"] == 89

    def test_awaiting_readiness_job_deleted_marks_failed(
        self, api_client: TestClient, mock_api_manager: MagicMock
    ) -> None:
        """Job deleted while in awaiting_readiness → operation fails."""
        resp = api_client.post("/api/v1/modules/jupyter/enable")
        assert resp.status_code == 202
        op_id = resp.json()["id"]

        # First poll: Job completes → awaiting_readiness
        mock_api_manager.k8s.get_job.return_value = self._make_completed_job()
        mock_api_manager.k8s.list_pods_by_label.return_value = [
            PodStatus(
                name="jupyter-0",
                namespace="default",
                phase="Pending",
                ready=False,
                restart_count=0,
            ),
        ]
        resp = api_client.get(f"/api/v1/operations/{op_id}")
        assert resp.json()["status"] == "awaiting_readiness"

        # Second poll: Job has been deleted (cancel/cleanup)
        mock_api_manager.k8s.get_job.side_effect = Exception("Not Found")

        resp = api_client.get(f"/api/v1/operations/{op_id}")
        assert resp.status_code == 200
        data = resp.json()
        assert data["status"] == "failed"
        assert "cancelled or superseded" in data["error"]

    def test_revision_changed_marks_failed(
        self, api_client: TestClient, mock_api_manager: MagicMock
    ) -> None:
        """Release revision changed (rollback) → operation fails with descriptive error."""
        from ilum.core.release import ReleaseInfo

        resp = api_client.post("/api/v1/modules/jupyter/enable")
        assert resp.status_code == 202
        op_id = resp.json()["id"]

        # First poll: Job completes → awaiting_readiness, captures revision=3
        mock_api_manager.k8s.get_job.return_value = self._make_completed_job()
        mock_api_manager.k8s.list_pods_by_label.return_value = [
            PodStatus(
                name="jupyter-0",
                namespace="default",
                phase="Pending",
                ready=False,
                restart_count=0,
            ),
        ]
        resp = api_client.get(f"/api/v1/operations/{op_id}")
        assert resp.json()["status"] == "awaiting_readiness"

        # Simulate rollback: revision changed from 3 to 4
        mock_api_manager.get_release_info.return_value = ReleaseInfo(
            name="ilum",
            namespace="default",
            status="deployed",
            chart="ilum",
            chart_version="6.7.0",
            revision=4,
            last_deployed="2024-01-01T00:00:00Z",
        )

        resp = api_client.get(f"/api/v1/operations/{op_id}")
        assert resp.status_code == 200
        data = resp.json()
        assert data["status"] == "failed"
        assert "rolled back or superseded" in data["error"]
        assert data["completed_at"] != ""

    def test_revision_unchanged_continues_readiness_check(
        self, api_client: TestClient, mock_api_manager: MagicMock
    ) -> None:
        """Revision matches → normal pod readiness check continues."""
        resp = api_client.post("/api/v1/modules/jupyter/enable")
        assert resp.status_code == 202
        op_id = resp.json()["id"]

        # First poll: Job completes → awaiting_readiness
        mock_api_manager.k8s.get_job.return_value = self._make_completed_job()
        mock_api_manager.k8s.list_pods_by_label.return_value = [
            PodStatus(
                name="jupyter-0",
                namespace="default",
                phase="Pending",
                ready=False,
                restart_count=0,
            ),
        ]
        resp = api_client.get(f"/api/v1/operations/{op_id}")
        assert resp.json()["status"] == "awaiting_readiness"

        # Revision unchanged (still 3) — pods still not ready
        resp = api_client.get(f"/api/v1/operations/{op_id}")
        assert resp.status_code == 200
        data = resp.json()
        assert data["status"] == "awaiting_readiness"

    def test_revision_from_logs_detects_superseded(
        self, api_client: TestClient, mock_api_manager: MagicMock
    ) -> None:
        """Recovered operation with log revision differs from current → fails."""
        import ilum.api.operations as ops_mod
        from ilum.api.models import OperationLog
        from ilum.core.release import ReleaseInfo

        store = ops_mod._store
        assert store is not None
        op = OperationResponse(
            id="recovered-op",
            status="awaiting_readiness",
            operation="enable",
            modules=["airflow"],
            progress=80,
            job_name="ilum-helm-recovered-op",
            helm_completed_at=datetime.now(UTC).isoformat(),
            logs=[
                OperationLog(
                    timestamp=datetime.now(UTC).isoformat(),
                    message=(
                        'Release "ilum" has been upgraded.\n'
                        "NAME: ilum\n"
                        "REVISION: 319\n"
                        "STATUS: deployed"
                    ),
                    level="info",
                ),
            ],
        )
        store.import_operation(op)

        # Job still exists and is completed
        mock_api_manager.k8s.get_job.return_value = self._make_completed_job()

        # Current release is at revision 324 (moved on)
        mock_api_manager.get_release_info.return_value = ReleaseInfo(
            name="ilum",
            namespace="default",
            status="deployed",
            chart="ilum",
            chart_version="6.7.0",
            revision=324,
            last_deployed="2024-01-01T00:00:00Z",
        )

        resp = api_client.get("/api/v1/operations/recovered-op")
        assert resp.status_code == 200
        data = resp.json()
        # Should detect revision mismatch (319 from logs != 324 current)
        assert data["status"] == "failed"
        assert "rolled back or superseded" in data["error"]
        assert "319" in data["error"]
        assert "324" in data["error"]
        assert data["expected_revision"] == 319

    def test_cancel_restart_skips_rollback(
        self, api_client: TestClient, mock_api_manager: MagicMock
    ) -> None:
        """E-B4: Cancel on a restart operation should NOT trigger helm rollback."""
        resp = api_client.post("/api/v1/modules/jupyter/restart")
        assert resp.status_code == 202
        op_id = resp.json()["id"]

        # Manually set the operation to running (thread may have already completed)
        import ilum.api.operations as ops_mod

        store = ops_mod._store
        assert store is not None
        op = store.get(op_id)
        assert op is not None
        op.status = "running"

        resp = api_client.post(f"/api/v1/operations/{op_id}/cancel")
        assert resp.status_code == 200
        data = resp.json()
        assert data["status"] == "cancelling"

        import time

        time.sleep(0.2)
        # Rollback should NOT have been called for restart operations
        mock_api_manager.helm.rollback.assert_not_called()

    def test_concurrency_guard_reconciles_awaiting_readiness(
        self, mock_api_manager: MagicMock
    ) -> None:
        """Stale awaiting_readiness op with deleted Job doesn't block new operations."""
        from ilum.api.job_runner import check_no_active_helm_operation

        store = OperationStore()
        op_id = store.create(operation="enable", modules=["jupyter"])
        op = store.get(op_id)
        assert op is not None
        op.status = "awaiting_readiness"
        op.job_name = f"ilum-helm-{op_id}"

        # Job has been deleted
        k8s = MagicMock()
        k8s.get_job.side_effect = Exception("Not Found")
        k8s.list_jobs_by_label.return_value = []

        # Should NOT raise — stale op should be reconciled to failed
        check_no_active_helm_operation(store, k8s, "default")

        assert op.status == "failed"
        assert "cancelled or superseded" in op.error

    def test_succeeded_job_pods_skipped_in_readiness(
        self, api_client: TestClient, mock_api_manager: MagicMock
    ) -> None:
        """Completed Job pods (migrations, init) should not block readiness.

        Airflow creates init Job pods (migrations, create-user) that finish
        with phase='Succeeded'.  These must be excluded from the readiness
        check so that the operation transitions to 'completed' when only the
        long-running service pods are Running+Ready.
        """
        resp = api_client.post("/api/v1/modules/airflow/enable")
        assert resp.status_code == 202
        op_id = resp.json()["id"]

        mock_api_manager.k8s.get_job.return_value = self._make_completed_job()
        # Mix of long-running pods (Running+Ready) and init Job pods (Succeeded)
        mock_api_manager.k8s.list_pods_by_label.return_value = [
            PodStatus(
                name="airflow-api-server-0",
                namespace="default",
                phase="Running",
                ready=True,
                restart_count=0,
            ),
            PodStatus(
                name="airflow-scheduler-0",
                namespace="default",
                phase="Running",
                ready=True,
                restart_count=0,
            ),
            PodStatus(
                name="airflow-run-migrations-abc",
                namespace="default",
                phase="Succeeded",
                ready=False,
                restart_count=0,
            ),
            PodStatus(
                name="airflow-create-user-xyz",
                namespace="default",
                phase="Succeeded",
                ready=False,
                restart_count=0,
            ),
        ]

        resp = api_client.get(f"/api/v1/operations/{op_id}")
        assert resp.status_code == 200
        data = resp.json()
        assert data["status"] == "completed"
        assert data["progress"] == 100

    def test_failed_job_pods_skipped_in_readiness(
        self, api_client: TestClient, mock_api_manager: MagicMock
    ) -> None:
        """Failed Job pods should also be skipped in readiness check."""
        resp = api_client.post("/api/v1/modules/airflow/enable")
        assert resp.status_code == 202
        op_id = resp.json()["id"]

        mock_api_manager.k8s.get_job.return_value = self._make_completed_job()
        mock_api_manager.k8s.list_pods_by_label.return_value = [
            PodStatus(
                name="airflow-api-server-0",
                namespace="default",
                phase="Running",
                ready=True,
                restart_count=0,
            ),
            PodStatus(
                name="airflow-init-failed",
                namespace="default",
                phase="Failed",
                ready=False,
                restart_count=0,
            ),
        ]

        resp = api_client.get(f"/api/v1/operations/{op_id}")
        assert resp.status_code == 200
        data = resp.json()
        assert data["status"] == "completed"
        assert data["progress"] == 100
