"""Tests for the job_runner module."""

from __future__ import annotations

from unittest.mock import MagicMock, patch

import pytest
from fastapi import HTTPException

from ilum.api.job_runner import (
    _strip_helm_notes,
    build_helm_job,
    check_no_active_helm_operation,
    find_active_helm_job,
    get_job_status,
    has_any_helm_job,
    recover_operations,
    refresh_operation_from_job,
)
from ilum.api.models import OperationResponse
from ilum.api.operations import OperationStore


class TestBuildHelmJob:
    def test_basic_job_structure(self) -> None:
        job = build_helm_job(
            op_id="abc123",
            operation="enable",
            release="ilum",
            namespace="default",
            chart_ref="ilum/ilum",
            set_flags=["ilum-jupyter.enabled=true"],
            modules=["jupyter"],
        )
        assert job.metadata.name == "ilum-helm-abc123"
        assert job.metadata.labels["app.kubernetes.io/managed-by"] == "ilum-api"
        assert job.metadata.labels["ilum.cloud/operation"] == "enable"
        assert job.metadata.labels["ilum.cloud/operation-id"] == "abc123"
        assert job.metadata.labels["ilum.cloud/release"] == "ilum"
        assert job.metadata.annotations["ilum.cloud/modules"] == "jupyter"
        assert job.metadata.annotations["ilum.cloud/chart-ref"] == "ilum/ilum"

    def test_job_spec_fields(self) -> None:
        job = build_helm_job(
            op_id="abc123",
            operation="enable",
            release="ilum",
            namespace="default",
            chart_ref="ilum/ilum",
            set_flags=[],
        )
        assert job.spec.backoff_limit == 0
        assert job.spec.active_deadline_seconds == 900
        assert job.spec.ttl_seconds_after_finished == 86400

    def test_container_restart_policy(self) -> None:
        job = build_helm_job(
            op_id="abc123",
            operation="enable",
            release="ilum",
            namespace="default",
            chart_ref="ilum/ilum",
            set_flags=[],
        )
        assert job.spec.template.spec.restart_policy == "Never"

    @patch.dict("os.environ", {"ILUM_API_IMAGE": "ilum/api:v1.0", "ILUM_SERVICE_ACCOUNT": "sa"})
    def test_container_image_from_env(self) -> None:
        job = build_helm_job(
            op_id="abc123",
            operation="enable",
            release="ilum",
            namespace="default",
            chart_ref="ilum/ilum",
            set_flags=[],
        )
        container = job.spec.template.spec.containers[0]
        assert container.image == "ilum/api:v1.0"
        assert job.spec.template.spec.service_account_name == "sa"

    def test_remote_chart_includes_repo_setup(self) -> None:
        job = build_helm_job(
            op_id="abc123",
            operation="enable",
            release="ilum",
            namespace="default",
            chart_ref="ilum/ilum",
            set_flags=[],
        )
        container = job.spec.template.spec.containers[0]
        script = container.command[2]  # /bin/sh -c <script>
        assert "helm repo add" in script
        assert "helm repo update" in script

    def test_local_chart_skips_repo_setup(self) -> None:
        job = build_helm_job(
            op_id="abc123",
            operation="enable",
            release="ilum",
            namespace="default",
            chart_ref="/opt/ilum-chart",
            set_flags=[],
        )
        container = job.spec.template.spec.containers[0]
        script = container.command[2]
        assert "helm repo add" not in script
        assert "helm upgrade" in script

    def test_set_flags_in_command(self) -> None:
        job = build_helm_job(
            op_id="abc123",
            operation="enable",
            release="ilum",
            namespace="default",
            chart_ref="ilum/ilum",
            set_flags=["ilum-jupyter.enabled=true", "ilum-jupyter.replicaCount=2"],
        )
        container = job.spec.template.spec.containers[0]
        script = container.command[2]
        assert "--set" in script
        assert "ilum-jupyter.enabled=true" in script
        assert "ilum-jupyter.replicaCount=2" in script

    def test_no_atomic_by_default(self) -> None:
        job = build_helm_job(
            op_id="abc123",
            operation="enable",
            release="ilum",
            namespace="default",
            chart_ref="ilum/ilum",
            set_flags=[],
        )
        container = job.spec.template.spec.containers[0]
        script = container.command[2]
        assert "--atomic" not in script

    def test_atomic_flag_when_requested(self) -> None:
        job = build_helm_job(
            op_id="abc123",
            operation="enable",
            release="ilum",
            namespace="default",
            chart_ref="ilum/ilum",
            set_flags=[],
            atomic=True,
        )
        container = job.spec.template.spec.containers[0]
        script = container.command[2]
        assert "--atomic" in script

    def test_reuse_values_default(self) -> None:
        job = build_helm_job(
            op_id="abc123",
            operation="enable",
            release="ilum",
            namespace="default",
            chart_ref="ilum/ilum",
            set_flags=[],
        )
        container = job.spec.template.spec.containers[0]
        script = container.command[2]
        assert "--reuse-values" in script
        assert "--reset-then-reuse-values" not in script

    def test_reset_defaults(self) -> None:
        job = build_helm_job(
            op_id="abc123",
            operation="enable",
            release="ilum",
            namespace="default",
            chart_ref="ilum/ilum",
            set_flags=[],
            reset_defaults=True,
        )
        container = job.spec.template.spec.containers[0]
        script = container.command[2]
        assert "--reset-then-reuse-values" in script

    def test_version_flag(self) -> None:
        job = build_helm_job(
            op_id="abc123",
            operation="enable",
            release="ilum",
            namespace="default",
            chart_ref="ilum/ilum",
            set_flags=[],
            version="6.8.0",
        )
        container = job.spec.template.spec.containers[0]
        script = container.command[2]
        assert "--version" in script
        assert "6.8.0" in script

    def test_no_modules_annotation_when_empty(self) -> None:
        job = build_helm_job(
            op_id="abc123",
            operation="upgrade",
            release="ilum",
            namespace="default",
            chart_ref="ilum/ilum",
            set_flags=[],
        )
        assert "ilum.cloud/modules" not in (job.metadata.annotations or {})

    def test_multiple_modules_annotation(self) -> None:
        job = build_helm_job(
            op_id="abc123",
            operation="enable",
            release="ilum",
            namespace="default",
            chart_ref="ilum/ilum",
            set_flags=[],
            modules=["jupyter", "airflow"],
        )
        assert job.metadata.annotations["ilum.cloud/modules"] == "jupyter,airflow"

    @patch.dict("os.environ", {"ILUM_API_IMAGE": "ilum/ilum-api:api-1.0.0-RC13"})
    def test_pins_api_image_in_set_flags(self) -> None:
        """Helm upgrade should pin the current API image to prevent revert."""
        job = build_helm_job(
            op_id="abc123",
            operation="enable",
            release="ilum",
            namespace="default",
            chart_ref="ilum/ilum",
            set_flags=["ilum-jupyter.enabled=true"],
        )
        script = job.spec.template.spec.containers[0].command[2]
        assert "ilum-api.image.repository=ilum/ilum-api" in script
        assert "ilum-api.image.tag=api-1.0.0-RC13" in script
        # User set_flags should come after (and thus override if needed)
        assert "ilum-jupyter.enabled=true" in script

    @patch.dict("os.environ", {"ILUM_API_IMAGE": "ilum/ilum-api:latest"}, clear=False)
    def test_skips_api_image_pin_for_default(self) -> None:
        """Don't pin the default fallback image — no useful tag to preserve."""
        job = build_helm_job(
            op_id="abc123",
            operation="enable",
            release="ilum",
            namespace="default",
            chart_ref="ilum/ilum",
            set_flags=[],
        )
        script = job.spec.template.spec.containers[0].command[2]
        assert "ilum-api.image.repository" not in script
        assert "ilum-api.image.tag" not in script


class TestStripHelmNotes:
    def test_strips_notes_section(self) -> None:
        text = (
            'Release "ilum" has been upgraded. Happy Helming!\n'
            "NOTES:\n"
            "1. Get the application URL by running:\n"
            "  kubectl port-forward svc/ilum 8080:80\n"
        )
        result = _strip_helm_notes(text)
        assert "NOTES:" not in result
        assert "Happy Helming!" in result

    def test_preserves_text_without_notes(self) -> None:
        text = "helm repo add ilum https://charts.ilum.cloud\nhelm upgrade ilum ilum/ilum"
        assert _strip_helm_notes(text) == text

    def test_strips_notes_after_upgrade_output(self) -> None:
        text = (
            "set -euo pipefail\n"
            'Release "ilum" has been upgraded.\n'
            "STATUS: deployed\n"
            "REVISION: 5\n"
            "NOTES:\n"
            "Welcome to Ilum!\n"
        )
        result = _strip_helm_notes(text)
        assert result.endswith("REVISION: 5")
        assert "Welcome to Ilum" not in result

    def test_empty_string(self) -> None:
        assert _strip_helm_notes("") == ""

    def test_notes_only(self) -> None:
        result = _strip_helm_notes("NOTES:\nSome notes here\n")
        assert result == ""


class TestGetJobStatus:
    def _make_job(
        self,
        conditions: list[tuple[str, str, str, str]] | None = None,
        active: int | None = None,
        succeeded: int | None = None,
        failed: int | None = None,
    ) -> MagicMock:
        job = MagicMock()
        if conditions:
            conds = []
            for cond_type, status, reason, message in conditions:
                c = MagicMock()
                c.type = cond_type
                c.status = status
                c.reason = reason
                c.message = message
                conds.append(c)
            job.status.conditions = conds
        else:
            job.status.conditions = []
        job.status.active = active
        job.status.succeeded = succeeded
        job.status.failed = failed
        return job

    def test_completed(self) -> None:
        job = self._make_job(conditions=[("Complete", "True", "", "")])
        status, error = get_job_status(job)
        assert status == "completed"
        assert error == ""

    def test_failed(self) -> None:
        job = self._make_job(
            conditions=[("Failed", "True", "DeadlineExceeded", "Job exceeded deadline")]
        )
        status, error = get_job_status(job)
        assert status == "failed"
        assert "DeadlineExceeded" in error
        assert "Job exceeded deadline" in error

    def test_failed_no_reason(self) -> None:
        job = self._make_job(conditions=[("Failed", "True", "", "")])
        status, error = get_job_status(job)
        assert status == "failed"

    def test_running(self) -> None:
        job = self._make_job(active=1)
        status, error = get_job_status(job)
        assert status == "running"
        assert error == ""

    def test_pending(self) -> None:
        job = self._make_job(active=None)
        status, error = get_job_status(job)
        assert status == "pending"
        assert error == ""

    def test_active_zero(self) -> None:
        job = self._make_job(active=0)
        status, error = get_job_status(job)
        assert status == "pending"

    def test_complete_condition_false_still_pending(self) -> None:
        job = self._make_job(conditions=[("Complete", "False", "", "")])
        status, error = get_job_status(job)
        assert status == "pending"

    def test_succeeded_counter_fallback(self) -> None:
        """When conditions are empty but succeeded counter > 0, report completed."""
        job = self._make_job(succeeded=1)
        status, error = get_job_status(job)
        assert status == "completed"
        assert error == ""

    def test_failed_counter_fallback(self) -> None:
        """When conditions are empty but failed counter > 0, report failed."""
        job = self._make_job(failed=1)
        status, error = get_job_status(job)
        assert status == "failed"
        assert error == "Job failed"

    def test_conditions_take_precedence_over_counters(self) -> None:
        """Conditions are checked before counters."""
        job = self._make_job(
            conditions=[("Complete", "True", "", "")],
            failed=1,
        )
        status, error = get_job_status(job)
        assert status == "completed"


class TestRefreshOperationFromJob:
    def test_refresh_completed_job(self) -> None:
        """Operation status updated when backing Job has completed."""
        k8s = MagicMock()
        job = MagicMock()
        cond = MagicMock()
        cond.type = "Complete"
        cond.status = "True"
        cond.reason = ""
        cond.message = ""
        job.status.conditions = [cond]
        job.status.active = None
        job.status.succeeded = 1
        job.status.failed = None
        k8s.get_job.return_value = job
        k8s.get_job_pod_logs.return_value = "helm upgrade completed"

        op = OperationResponse(
            id="abc123",
            status="running",
            operation="enable",
            modules=["jupyter"],
            job_name="ilum-helm-abc123",
        )
        refresh_operation_from_job(op, k8s, "default")
        assert op.status == "awaiting_readiness"
        assert op.progress == 80
        assert op.helm_completed_at != ""

    def test_refresh_completed_captures_logs(self) -> None:
        """Logs are captured at the moment of completion."""
        k8s = MagicMock()
        job = MagicMock()
        cond = MagicMock()
        cond.type = "Complete"
        cond.status = "True"
        cond.reason = ""
        cond.message = ""
        job.status.conditions = [cond]
        job.status.active = None
        job.status.succeeded = 1
        job.status.failed = None
        k8s.get_job.return_value = job
        k8s.get_job_pod_logs.return_value = "Release upgraded successfully"

        op = OperationResponse(
            id="abc123",
            status="running",
            operation="enable",
            modules=["jupyter"],
            job_name="ilum-helm-abc123",
        )
        refresh_operation_from_job(op, k8s, "default")
        assert len(op.logs) == 1
        assert "Release upgraded successfully" in op.logs[0].message
        assert op.logs[0].level == "info"

    def test_refresh_failed_captures_logs_with_error_level(self) -> None:
        """Logs captured at failure have level='error'."""
        k8s = MagicMock()
        job = MagicMock()
        cond = MagicMock()
        cond.type = "Failed"
        cond.status = "True"
        cond.reason = "BackoffLimitExceeded"
        cond.message = "Job has reached the specified backoff limit"
        job.status.conditions = [cond]
        job.status.active = None
        job.status.succeeded = None
        job.status.failed = 1
        k8s.get_job.return_value = job
        k8s.get_job_pod_logs.return_value = "Error: UPGRADE FAILED: services not found"

        op = OperationResponse(
            id="abc123",
            status="running",
            operation="enable",
            modules=["superset"],
            job_name="ilum-helm-abc123",
        )
        refresh_operation_from_job(op, k8s, "default")
        assert op.status == "failed"
        assert len(op.logs) == 1
        assert "UPGRADE FAILED" in op.logs[0].message
        assert op.logs[0].level == "error"

    def test_refresh_failed_job(self) -> None:
        """Operation status updated when backing Job has failed."""
        k8s = MagicMock()
        job = MagicMock()
        cond = MagicMock()
        cond.type = "Failed"
        cond.status = "True"
        cond.reason = "BackoffLimitExceeded"
        cond.message = "Job has reached the specified backoff limit"
        job.status.conditions = [cond]
        job.status.active = None
        job.status.succeeded = None
        job.status.failed = 1
        k8s.get_job.return_value = job
        k8s.get_job_pod_logs.return_value = ""

        op = OperationResponse(
            id="abc123",
            status="running",
            operation="enable",
            modules=["jupyter"],
            job_name="ilum-helm-abc123",
        )
        refresh_operation_from_job(op, k8s, "default")
        assert op.status == "failed"
        assert "BackoffLimitExceeded" in op.error
        assert op.completed_at != ""

    def test_refresh_preserves_existing_logs_on_pod_gone(self) -> None:
        """If pod logs can't be fetched at transition, existing logs are kept."""
        k8s = MagicMock()
        job = MagicMock()
        cond = MagicMock()
        cond.type = "Failed"
        cond.status = "True"
        cond.reason = "DeadlineExceeded"
        cond.message = "exceeded deadline"
        job.status.conditions = [cond]
        job.status.active = None
        job.status.succeeded = None
        job.status.failed = 1
        k8s.get_job.return_value = job
        k8s.get_job_pod_logs.side_effect = Exception("pod not found")

        from ilum.api.models import OperationLog

        existing_log = OperationLog(
            timestamp="2024-01-01T00:00:00", message="helm repo update", level="info"
        )
        op = OperationResponse(
            id="abc123",
            status="running",
            operation="enable",
            modules=["superset"],
            job_name="ilum-helm-abc123",
            logs=[existing_log],
        )
        refresh_operation_from_job(op, k8s, "default")
        assert op.status == "failed"
        # Existing logs should NOT be wiped
        assert len(op.logs) == 1
        assert op.logs[0].message == "helm repo update"

    def test_refresh_job_not_found_marks_failed(self) -> None:
        """If the backing Job no longer exists, mark operation as failed."""
        k8s = MagicMock()
        k8s.get_job.side_effect = Exception("Job not found")

        op = OperationResponse(
            id="abc123",
            status="running",
            operation="enable",
            modules=["jupyter"],
            job_name="ilum-helm-abc123",
        )
        refresh_operation_from_job(op, k8s, "default")
        assert op.status == "failed"
        assert "no longer exists" in op.error
        assert op.completed_at != ""

    def test_refresh_awaiting_readiness_job_deleted(self) -> None:
        """awaiting_readiness + Job deleted → operation fails as cancelled/superseded."""
        k8s = MagicMock()
        k8s.get_job.side_effect = Exception("Not Found")

        op = OperationResponse(
            id="abc123",
            status="awaiting_readiness",
            operation="enable",
            modules=["jupyter"],
            job_name="ilum-helm-abc123",
        )
        refresh_operation_from_job(op, k8s, "default")
        assert op.status == "failed"
        assert "cancelled or superseded" in op.error
        assert op.completed_at != ""

    def test_refresh_awaiting_readiness_job_still_completed(self) -> None:
        """awaiting_readiness + Job still completed → no status change."""
        k8s = MagicMock()
        job = MagicMock()
        cond = MagicMock()
        cond.type = "Complete"
        cond.status = "True"
        cond.reason = ""
        cond.message = ""
        job.status.conditions = [cond]
        job.status.active = None
        job.status.succeeded = 1
        job.status.failed = None
        k8s.get_job.return_value = job

        op = OperationResponse(
            id="abc123",
            status="awaiting_readiness",
            operation="enable",
            modules=["jupyter"],
            job_name="ilum-helm-abc123",
            helm_completed_at="2024-01-01T00:00:00",
        )
        refresh_operation_from_job(op, k8s, "default")
        assert op.status == "awaiting_readiness"
        assert op.helm_completed_at == "2024-01-01T00:00:00"

    def test_refresh_awaiting_readiness_job_failed(self) -> None:
        """awaiting_readiness + Job somehow failed → operation fails."""
        k8s = MagicMock()
        job = MagicMock()
        cond = MagicMock()
        cond.type = "Failed"
        cond.status = "True"
        cond.reason = "DeadlineExceeded"
        cond.message = "exceeded deadline"
        job.status.conditions = [cond]
        job.status.active = None
        job.status.succeeded = None
        job.status.failed = 1
        k8s.get_job.return_value = job
        k8s.get_job_pod_logs.return_value = "timeout error"

        op = OperationResponse(
            id="abc123",
            status="awaiting_readiness",
            operation="enable",
            modules=["jupyter"],
            job_name="ilum-helm-abc123",
        )
        refresh_operation_from_job(op, k8s, "default")
        assert op.status == "failed"
        assert "DeadlineExceeded" in op.error
        assert op.completed_at != ""

    def test_refresh_skips_terminal_operations(self) -> None:
        """Already completed/failed operations are not refreshed."""
        k8s = MagicMock()
        op = OperationResponse(
            id="abc123",
            status="completed",
            operation="enable",
            modules=["jupyter"],
            job_name="ilum-helm-abc123",
        )
        refresh_operation_from_job(op, k8s, "default")
        k8s.get_job.assert_not_called()

    def test_refresh_skips_ops_without_job_name(self) -> None:
        """Operations without a job_name (thread-based) are not refreshed."""
        k8s = MagicMock()
        op = OperationResponse(
            id="abc123",
            status="running",
            operation="restart",
            modules=["jupyter"],
            job_name="",
        )
        refresh_operation_from_job(op, k8s, "default")
        k8s.get_job.assert_not_called()

    def test_refresh_still_running_no_logs_captured(self) -> None:
        """Running Job keeps status as running and does NOT capture logs."""
        k8s = MagicMock()
        job = MagicMock()
        job.status.conditions = []
        job.status.active = 1
        job.status.succeeded = None
        job.status.failed = None
        k8s.get_job.return_value = job

        op = OperationResponse(
            id="abc123",
            status="running",
            operation="enable",
            modules=["jupyter"],
            job_name="ilum-helm-abc123",
        )
        refresh_operation_from_job(op, k8s, "default")
        assert op.status == "running"
        assert op.completed_at == ""
        # Log capture is NOT done by refresh_operation_from_job for running ops
        # (the router handles live log snapshots for running ops)
        k8s.get_job_pod_logs.assert_not_called()


class TestRecoverOperations:
    def test_empty_job_list(self) -> None:
        k8s = MagicMock()
        k8s.list_jobs_by_label.return_value = []
        store = OperationStore()
        count = recover_operations(k8s, store, "default")
        assert count == 0

    def test_recover_completed_job(self) -> None:
        k8s = MagicMock()
        job = MagicMock()
        job.metadata.name = "ilum-helm-abc123"
        job.metadata.labels = {
            "app.kubernetes.io/managed-by": "ilum-api",
            "ilum.cloud/operation": "enable",
            "ilum.cloud/operation-id": "abc123",
            "ilum.cloud/release": "ilum",
        }
        job.metadata.annotations = {"ilum.cloud/modules": "jupyter"}
        job.metadata.creation_timestamp = None
        cond = MagicMock()
        cond.type = "Complete"
        cond.status = "True"
        cond.reason = ""
        cond.message = ""
        job.status.conditions = [cond]
        job.status.active = None
        job.status.succeeded = 1
        job.status.failed = None
        job.status.completion_time = None
        k8s.list_jobs_by_label.return_value = [job]
        k8s.get_job_pod_logs.return_value = "done"

        store = OperationStore()
        count = recover_operations(k8s, store, "default")
        assert count == 1

        op = store.get("abc123")
        assert op is not None
        assert op.status == "awaiting_readiness"
        assert op.operation == "enable"
        assert op.modules == ["jupyter"]
        assert op.job_name == "ilum-helm-abc123"
        assert op.progress == 80
        assert op.helm_completed_at != ""

    def test_recover_uses_job_completion_time(self) -> None:
        """Recovery should use the Job's actual completion_time, not now()."""
        from datetime import UTC, datetime

        k8s = MagicMock()
        job = MagicMock()
        job.metadata.name = "ilum-helm-ts123"
        job.metadata.labels = {
            "app.kubernetes.io/managed-by": "ilum-api",
            "ilum.cloud/operation": "enable",
            "ilum.cloud/operation-id": "ts123",
        }
        job.metadata.annotations = {}
        job.metadata.creation_timestamp = None
        cond = MagicMock()
        cond.type = "Complete"
        cond.status = "True"
        cond.reason = ""
        cond.message = ""
        job.status.conditions = [cond]
        job.status.active = None
        job.status.succeeded = 1
        job.status.failed = None

        # Set a specific completion_time in the past
        past_time = datetime(2024, 6, 15, 12, 0, 0, tzinfo=UTC)
        job.status.completion_time = past_time

        k8s.list_jobs_by_label.return_value = [job]
        k8s.get_job_pod_logs.return_value = ""

        store = OperationStore()
        recover_operations(k8s, store, "default")

        op = store.get("ts123")
        assert op is not None
        assert op.helm_completed_at == past_time.isoformat()

    def test_recover_falls_back_to_now_without_completion_time(self) -> None:
        """Recovery falls back to now() when Job has no completion_time."""
        k8s = MagicMock()
        job = MagicMock()
        job.metadata.name = "ilum-helm-notime"
        job.metadata.labels = {
            "app.kubernetes.io/managed-by": "ilum-api",
            "ilum.cloud/operation": "enable",
            "ilum.cloud/operation-id": "notime",
        }
        job.metadata.annotations = {}
        job.metadata.creation_timestamp = None
        cond = MagicMock()
        cond.type = "Complete"
        cond.status = "True"
        cond.reason = ""
        cond.message = ""
        job.status.conditions = [cond]
        job.status.active = None
        job.status.succeeded = 1
        job.status.failed = None
        job.status.completion_time = None

        k8s.list_jobs_by_label.return_value = [job]
        k8s.get_job_pod_logs.return_value = ""

        store = OperationStore()
        recover_operations(k8s, store, "default")

        op = store.get("notime")
        assert op is not None
        assert op.helm_completed_at != ""  # Falls back to now()

    def test_recover_failed_job(self) -> None:
        k8s = MagicMock()
        job = MagicMock()
        job.metadata.name = "ilum-helm-def456"
        job.metadata.labels = {
            "app.kubernetes.io/managed-by": "ilum-api",
            "ilum.cloud/operation": "disable",
            "ilum.cloud/operation-id": "def456",
        }
        job.metadata.annotations = {}
        job.metadata.creation_timestamp = None
        cond = MagicMock()
        cond.type = "Failed"
        cond.status = "True"
        cond.reason = "BackoffLimitExceeded"
        cond.message = "Job has reached the specified backoff limit"
        job.status.conditions = [cond]
        job.status.active = None
        job.status.succeeded = None
        job.status.failed = 1
        k8s.list_jobs_by_label.return_value = [job]
        k8s.get_job_pod_logs.return_value = ""

        store = OperationStore()
        count = recover_operations(k8s, store, "default")
        assert count == 1

        op = store.get("def456")
        assert op is not None
        assert op.status == "failed"
        assert "BackoffLimitExceeded" in op.error

    def test_recover_running_job(self) -> None:
        k8s = MagicMock()
        job = MagicMock()
        job.metadata.name = "ilum-helm-ghi789"
        job.metadata.labels = {
            "app.kubernetes.io/managed-by": "ilum-api",
            "ilum.cloud/operation": "upgrade",
            "ilum.cloud/operation-id": "ghi789",
        }
        job.metadata.annotations = {}
        job.metadata.creation_timestamp = None
        job.status.conditions = []
        job.status.active = 1
        job.status.succeeded = None
        job.status.failed = None
        k8s.list_jobs_by_label.return_value = [job]
        k8s.get_job_pod_logs.return_value = "running..."

        store = OperationStore()
        count = recover_operations(k8s, store, "default")
        assert count == 1

        op = store.get("ghi789")
        assert op is not None
        assert op.status == "running"

    def test_dedup_existing_operations(self) -> None:
        k8s = MagicMock()
        job = MagicMock()
        job.metadata.name = "ilum-helm-abc123"
        job.metadata.labels = {
            "app.kubernetes.io/managed-by": "ilum-api",
            "ilum.cloud/operation": "enable",
            "ilum.cloud/operation-id": "abc123",
        }
        job.metadata.annotations = {}
        job.metadata.creation_timestamp = None
        cond = MagicMock()
        cond.type = "Complete"
        cond.status = "True"
        cond.reason = ""
        cond.message = ""
        job.status.conditions = [cond]
        job.status.active = None
        job.status.succeeded = 1
        job.status.failed = None
        k8s.list_jobs_by_label.return_value = [job]
        k8s.get_job_pod_logs.return_value = ""

        store = OperationStore()
        # Pre-populate the store with this operation
        store.import_operation(
            OperationResponse(
                id="abc123",
                status="running",
                operation="enable",
            )
        )

        count = recover_operations(k8s, store, "default")
        assert count == 0  # Already existed, not re-imported

    def test_skip_job_without_op_id(self) -> None:
        k8s = MagicMock()
        job = MagicMock()
        job.metadata.name = "ilum-helm-noid"
        job.metadata.labels = {"app.kubernetes.io/managed-by": "ilum-api"}
        job.metadata.annotations = {}
        job.metadata.creation_timestamp = None
        job.status.conditions = []
        job.status.active = None
        job.status.succeeded = None
        job.status.failed = None
        k8s.list_jobs_by_label.return_value = [job]

        store = OperationStore()
        count = recover_operations(k8s, store, "default")
        assert count == 0

    def test_list_jobs_failure_returns_zero(self) -> None:
        k8s = MagicMock()
        k8s.list_jobs_by_label.side_effect = Exception("k8s error")
        store = OperationStore()
        count = recover_operations(k8s, store, "default")
        assert count == 0


class TestFindActiveHelmJob:
    def test_no_jobs(self) -> None:
        k8s = MagicMock()
        k8s.list_jobs_by_label.return_value = []
        assert find_active_helm_job(k8s, "default") is None

    def test_all_terminal(self) -> None:
        k8s = MagicMock()
        job = MagicMock()
        job.metadata.name = "ilum-helm-abc123"
        cond = MagicMock()
        cond.type = "Complete"
        cond.status = "True"
        cond.reason = ""
        cond.message = ""
        job.status.conditions = [cond]
        job.status.active = None
        job.status.succeeded = 1
        job.status.failed = None
        k8s.list_jobs_by_label.return_value = [job]
        assert find_active_helm_job(k8s, "default") is None

    def test_one_active(self) -> None:
        k8s = MagicMock()
        job = MagicMock()
        job.metadata.name = "ilum-helm-active"
        job.status.conditions = []
        job.status.active = 1
        job.status.succeeded = None
        job.status.failed = None
        k8s.list_jobs_by_label.return_value = [job]
        result = find_active_helm_job(k8s, "default")
        assert result == "ilum-helm-active"

    def test_k8s_error_returns_none(self) -> None:
        k8s = MagicMock()
        k8s.list_jobs_by_label.side_effect = Exception("fail")
        assert find_active_helm_job(k8s, "default") is None


class TestHasAnyHelmJob:
    def test_no_jobs(self) -> None:
        k8s = MagicMock()
        k8s.list_jobs_by_label.return_value = []
        assert has_any_helm_job(k8s, "default") is False

    def test_completed_job_returns_true(self) -> None:
        k8s = MagicMock()
        job = MagicMock()
        cond = MagicMock()
        cond.type = "Complete"
        cond.status = "True"
        cond.reason = ""
        cond.message = ""
        job.status.conditions = [cond]
        job.status.active = None
        job.status.succeeded = 1
        job.status.failed = None
        k8s.list_jobs_by_label.return_value = [job]
        assert has_any_helm_job(k8s, "default") is True

    def test_active_job_returns_true(self) -> None:
        k8s = MagicMock()
        job = MagicMock()
        job.status.conditions = []
        job.status.active = 1
        job.status.succeeded = None
        job.status.failed = None
        k8s.list_jobs_by_label.return_value = [job]
        assert has_any_helm_job(k8s, "default") is True

    def test_k8s_error_raises(self) -> None:
        """Unlike find_active_helm_job, has_any_helm_job does NOT swallow exceptions."""
        k8s = MagicMock()
        k8s.list_jobs_by_label.side_effect = Exception("k8s API error")
        with pytest.raises(Exception, match="k8s API error"):
            has_any_helm_job(k8s, "default")


class TestConcurrencyGuard:
    def test_pass_when_clear(self) -> None:
        store = OperationStore()
        k8s = MagicMock()
        k8s.list_jobs_by_label.return_value = []
        # Should not raise
        check_no_active_helm_operation(store, k8s, "default")

    def test_409_when_in_memory_active(self) -> None:
        store = OperationStore()
        op_id = store.create(operation="enable", modules=["jupyter"])
        op = store.get(op_id)
        assert op is not None
        op.status = "running"

        k8s = MagicMock()
        k8s.list_jobs_by_label.return_value = []

        with pytest.raises(HTTPException) as exc_info:
            check_no_active_helm_operation(store, k8s, "default")
        assert exc_info.value.status_code == 409

    def test_409_when_k8s_job_active(self) -> None:
        store = OperationStore()
        k8s = MagicMock()
        job = MagicMock()
        job.metadata.name = "ilum-helm-active"
        job.status.conditions = []
        job.status.active = 1
        job.status.succeeded = None
        job.status.failed = None
        k8s.list_jobs_by_label.return_value = [job]

        with pytest.raises(HTTPException) as exc_info:
            check_no_active_helm_operation(store, k8s, "default")
        assert exc_info.value.status_code == 409

    def test_pass_when_only_restart_active(self) -> None:
        """Restart operations don't count as helm operations."""
        store = OperationStore()
        op_id = store.create(operation="restart", modules=["jupyter"])
        op = store.get(op_id)
        assert op is not None
        op.status = "running"

        k8s = MagicMock()
        k8s.list_jobs_by_label.return_value = []

        # Should not raise — restart is not a helm operation
        check_no_active_helm_operation(store, k8s, "default")

    def test_pass_when_only_completed(self) -> None:
        store = OperationStore()
        op_id = store.create(operation="enable", modules=["jupyter"])
        op = store.get(op_id)
        assert op is not None
        op.status = "completed"

        k8s = MagicMock()
        k8s.list_jobs_by_label.return_value = []

        # Should not raise
        check_no_active_helm_operation(store, k8s, "default")

    def test_reconciles_stale_running_op_from_failed_job(self) -> None:
        """In-memory 'running' op whose Job has failed should not block."""
        store = OperationStore()
        op_id = store.create(operation="enable", modules=["superset"])
        op = store.get(op_id)
        assert op is not None
        op.status = "running"
        op.job_name = f"ilum-helm-{op_id}"

        # K8s Job has actually failed
        k8s = MagicMock()
        failed_job = MagicMock()
        cond = MagicMock()
        cond.type = "Failed"
        cond.status = "True"
        cond.reason = "BackoffLimitExceeded"
        cond.message = "Job has reached the specified backoff limit"
        failed_job.status.conditions = [cond]
        failed_job.status.active = None
        failed_job.status.succeeded = None
        failed_job.status.failed = 1
        k8s.get_job.return_value = failed_job
        k8s.list_jobs_by_label.return_value = []  # no other active Jobs

        # Should NOT raise — the stale op should be reconciled to failed
        check_no_active_helm_operation(store, k8s, "default")

        # Verify the operation was updated
        assert op.status == "failed"
        assert "BackoffLimitExceeded" in op.error

    def test_reconciles_stale_running_op_from_completed_job(self) -> None:
        """In-memory 'running' op whose Job has completed should not block."""
        store = OperationStore()
        op_id = store.create(operation="enable", modules=["jupyter"])
        op = store.get(op_id)
        assert op is not None
        op.status = "running"
        op.job_name = f"ilum-helm-{op_id}"

        # K8s Job has completed
        k8s = MagicMock()
        completed_job = MagicMock()
        cond = MagicMock()
        cond.type = "Complete"
        cond.status = "True"
        cond.reason = ""
        cond.message = ""
        completed_job.status.conditions = [cond]
        completed_job.status.active = None
        completed_job.status.succeeded = 1
        completed_job.status.failed = None
        k8s.get_job.return_value = completed_job
        k8s.list_jobs_by_label.return_value = []

        # Should NOT raise
        check_no_active_helm_operation(store, k8s, "default")

        # Verify the operation was updated (awaiting_readiness, not completed,
        # because pod readiness checking happens in the router layer)
        assert op.status == "awaiting_readiness"
        assert op.progress == 80

    def test_reconciles_stale_running_op_when_job_deleted(self) -> None:
        """In-memory 'running' op whose Job was deleted should not block."""
        store = OperationStore()
        op_id = store.create(operation="enable", modules=["jupyter"])
        op = store.get(op_id)
        assert op is not None
        op.status = "running"
        op.job_name = f"ilum-helm-{op_id}"

        # K8s Job no longer exists
        k8s = MagicMock()
        k8s.get_job.side_effect = Exception("Job not found")
        k8s.list_jobs_by_label.return_value = []

        # Should NOT raise
        check_no_active_helm_operation(store, k8s, "default")

        # Verify the operation was updated
        assert op.status == "failed"
        assert "no longer exists" in op.error
