# SPDX-FileCopyrightText: 2024 UL Research Institutes
# SPDX-License-Identifier: Apache-2.0
"""Integration tests for global quota enforcement.

Tests verify that the resource-based quota system correctly:
- Enforces GPU quota limits (by hardware type: T4, L4)
- Prevents zero-cost loophole (CPU-only workflows consume admission slots)
- Counts replications/replicas correctly
- Deallocates resources on workflow termination

NOTE: These tests are marked with @pytest.mark.serial to prevent parallel execution.
They exhaust quota resources and will interfere with each other under parallel execution.
Use pytest-xdist with: pytest --dist=no tests/test_global_quota.py
Or with pytest-ordering: pytest -p no:randomly tests/test_global_quota.py
"""

from __future__ import annotations

import os
import time
from datetime import datetime, timedelta, timezone
from typing import Callable, TypeVar

import pytest
from conftest import DATA_DIR, wait_for_status

from dyff.audit.local.platform import DyffLocalPlatform
from dyff.client import Client
from dyff.schema.platform import *
from dyff.schema.requests import *

pytest.skip(allow_module_level=True)

# Mark all tests in this module as serial to prevent parallel execution
# These tests exhaust quota and will interfere with each other
pytestmark = pytest.mark.serial


T = TypeVar("T")
MOCK_INFERENCESERVICE_IMAGE = ContainerImageSource(
    host="registry.gitlab.com",
    name="dyff/workflows/inferenceservice-mock",
    digest="sha256:f7becd37affa559a60e7ef2d3a0b1e4766e6cd6438cef701d58d2653767e7e54",
    tag="0.2.1",
)


def wait_for_condition(
    get_entity_fn: Callable[[], T],
    condition: Callable[[T], bool],
    description: str,
    *,
    timeout: timedelta,
    poll_interval: float = 5.0,
) -> T:
    """Poll until condition is met or timeout."""
    start = datetime.now(timezone.utc)
    last_entity = None
    while (datetime.now(timezone.utc) - start) < timeout:
        entity = get_entity_fn()
        last_entity = entity
        if condition(entity):
            return entity
        time.sleep(poll_interval)
    raise AssertionError(
        f"Timeout waiting for {description}. "
        f"Last state: status={getattr(last_entity, 'status', 'N/A')}, "
        f"reason={getattr(last_entity, 'reason', 'N/A')}"
    )


def is_admitted(entity) -> bool:
    """Check if entity reached a non-quota terminal/active state."""
    return entity.status in ["Admitted", "Running", "Complete", "Ready"]


def is_processed(entity) -> bool:
    """Check if entity has been processed by orchestrator."""
    if is_admitted(entity):
        return True
    return entity.status == "Created" and entity.reason == "QuotaLimit"


def is_quota_limited(entity) -> bool:
    """Check if entity is quota-limited."""
    return entity.status == "Created" and entity.reason == "QuotaLimit"


def check_quotas_enabled(pytestconfig) -> bool:
    """Check if quotas are enabled via env hints."""
    if not pytestconfig.getoption("test_remote"):
        return False

    disable_quotas = os.getenv("DYFF_DISABLE_QUOTAS")
    if disable_quotas is not None and disable_quotas.lower() in {"1", "true", "yes"}:
        return False

    enable_quotas = os.getenv("DYFF_QUOTAS_ENABLED")
    if enable_quotas is not None and enable_quotas.lower() in {"0", "false", "no"}:
        return False

    return True


def probe_quota_enforcement(
    pytestconfig,
    dyffapi,
    *,
    account: str,
    dataset_id: str,
    inferenceservice_id: str,
) -> bool:
    """Deterministically verify quota enforcement via an over-quota evaluation."""
    cache_key = "dyff/quota_enforced_probe"
    cached = pytestconfig.cache.get(cache_key, None)
    if cached is not None:
        return cached

    evaluation = None
    try:
        evaluation_request = EvaluationCreateRequest(
            account=account,
            dataset=dataset_id,
            inferenceSession=EvaluationInferenceSessionRequest(
                inferenceService=inferenceservice_id,
                expires=datetime.now(timezone.utc) + timedelta(hours=1),
                replicas=1,
                useSpotPods=False,
                accelerator=Accelerator(
                    kind="GPU",
                    gpu=AcceleratorGPU(
                        hardwareTypes=["nvidia.com/gpu-t4"],
                        count=100,  # Intentionally over quota
                    ),
                ),
            ),
            replications=1,
            workersPerReplica=1,
        )
        evaluation = dyffapi.evaluations.create(evaluation_request)

        status = wait_for_condition(
            lambda: dyffapi.evaluations.get(evaluation.id),
            is_quota_limited,
            "over-quota evaluation to be quota-limited",
            timeout=timedelta(seconds=60),
        )
        enforced = status.reason == "QuotaLimit"
    except AssertionError:
        enforced = False
    finally:
        if evaluation is not None:
            try:
                dyffapi.evaluations.delete(evaluation.id)
            except Exception:
                pass

    pytestconfig.cache.set(cache_key, enforced)
    return enforced


def require_quota_enforcement_or_skip(
    pytestconfig,
    dyffapi,
    *,
    account: str,
    dataset_id: str,
    inferenceservice_id: str,
) -> None:
    """Skip tests if quotas are disabled or not enforced."""
    if not check_quotas_enabled(pytestconfig):
        pytest.skip("Quotas appear to be disabled (e.g., --disable_quotas in skaffold)")

    if not probe_quota_enforcement(
        pytestconfig,
        dyffapi,
        account=account,
        dataset_id=dataset_id,
        inferenceservice_id=inferenceservice_id,
    ):
        pytest.skip("Quotas not enforced (probe did not hit QuotaLimit)")


def wait_for_all_processed(
    get_entity_fn: Callable[[str], T],
    ids: list[str],
    description: str,
    *,
    timeout: timedelta,
    poll_interval: float = 5.0,
) -> None:
    """Wait until all entities are processed or raise."""
    start = datetime.now(timezone.utc)
    last_states: dict[str, tuple[str, str | None]] = {}
    while (datetime.now(timezone.utc) - start) < timeout:
        all_processed = True
        for entity_id in ids:
            entity = get_entity_fn(entity_id)
            last_states[entity_id] = (entity.status, entity.reason)
            if not is_processed(entity):
                all_processed = False
        if all_processed:
            return
        time.sleep(poll_interval)

    raise AssertionError(
        f"Timeout waiting for {description}. Last states: {last_states}"
    )


def create_ready_model(dyffapi, *, account: str, name: str):
    model_request = ModelCreateRequest(
        name=name,
        account=account,
        artifact=ModelArtifact(kind=ModelArtifactKind.Mock),
        storage=ModelStorage(medium=ModelStorageMedium.Mock),
        source=ModelSource(kind=ModelSourceKinds.Mock),
        resources=ModelResources(storage="0"),
    )
    model = dyffapi.models.create(model_request)
    wait_for_status(
        lambda: dyffapi.models.get(model.id),
        ["Ready", "Error"],
        timeout=timedelta(minutes=2),
    )
    model_status = dyffapi.models.get(model.id)
    assert model_status.status == "Ready", f"Model not ready: {model_status.status}"
    return model


def create_ready_inferenceservice(
    dyffapi,
    *,
    account: str,
    name: str,
    model_id: str,
):
    inferenceservice_request = InferenceServiceCreateRequest(
        account=account,
        name=name,
        model=model_id,
        runner=InferenceServiceRunner(
            kind=InferenceServiceRunnerKind.MOCK,
            resources=ModelResources(storage="1Gi", memory="2Gi"),
            image=MOCK_INFERENCESERVICE_IMAGE,
        ),
        interface=InferenceInterface(
            endpoint="openai/v1/completions",
            outputSchema=DataSchema.make_output_schema(
                DyffDataSchema(components=["text.Text"]),
            ),
            inputPipeline=[
                SchemaAdapter(
                    kind="TransformJSON",
                    configuration={"prompt": "$.text", "model": model_id},
                ),
            ],
            outputPipeline=[
                SchemaAdapter(
                    kind="ExplodeCollections",
                    configuration={"collections": ["choices"]},
                ),
                SchemaAdapter(
                    kind="TransformJSON", configuration={"text": "$.choices.text"}
                ),
            ],
        ),
    )
    inferenceservice = dyffapi.inferenceservices.create(inferenceservice_request)
    wait_for_status(
        lambda: dyffapi.inferenceservices.get(inferenceservice.id),
        ["Ready", "Error"],
        timeout=timedelta(minutes=2),
    )
    is_status = dyffapi.inferenceservices.get(inferenceservice.id)
    assert (
        is_status.status == "Ready"
    ), f"InferenceService not ready: {is_status.status}"
    return inferenceservice


def create_evaluation_request(
    *,
    account: str,
    dataset_id: str,
    inferenceservice_id: str,
    hardware_type: str,
    gpu_count: int,
    replicas: int,
    replications: int,
):
    return EvaluationCreateRequest(
        account=account,
        dataset=dataset_id,
        inferenceSession=EvaluationInferenceSessionRequest(
            inferenceService=inferenceservice_id,
            expires=datetime.now(timezone.utc) + timedelta(hours=1),
            replicas=replicas,
            useSpotPods=False,
            accelerator=Accelerator(
                kind="GPU",
                gpu=AcceleratorGPU(
                    hardwareTypes=[hardware_type],
                    count=gpu_count,
                ),
            ),
        ),
        replications=replications,
        workersPerReplica=1,
    )


@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.depends(on=["tests/test_datasets.py::test_datasets_create"])
@pytest.mark.depends(on=["tests/test_models.py::test_models_create_mock"])
def test_global_gpu_quota_enforcement(
    pytestconfig, dyffapi: Client | DyffLocalPlatform, ctx, datafiles
):
    """Test that global T4 GPU quota limits are enforced.

    Default quotas:
    - Global: 32 T4 GPUs
    - Account: 16 T4 GPUs per hardware type

    This test creates evaluations that would exceed the account quota.
    """
    if pytestconfig.getoption("skip_quotas"):
        pytest.skip()

    if pytestconfig.getoption("skip_evaluations"):
        pytest.skip()

    if not pytestconfig.getoption("test_remote"):
        pytest.skip("Quota tests require remote API with orchestrator")

    if not check_quotas_enabled(pytestconfig):
        pytest.skip("Quotas appear to be disabled (e.g., --disable_quotas in skaffold)")

    account = ctx["account"]
    dataset = ctx["dataset"]

    model = ctx["model_mock"]
    inferenceservice = None
    evaluations = []

    try:
        inferenceservice = create_ready_inferenceservice(
            dyffapi,
            account=account,
            name="quota-test-t4-inference",
            model_id=model.id,
        )

        # Ensure quotas are enforced before running the test
        require_quota_enforcement_or_skip(
            pytestconfig,
            dyffapi,
            account=account,
            dataset_id=dataset.id,
            inferenceservice_id=inferenceservice.id,
        )

        # Create 9 evaluations: 9 × 2 replicas × 2 GPUs = 36 T4 GPUs
        # This exceeds the 16 T4 GPU account quota
        for _ in range(9):
            evaluation_request = create_evaluation_request(
                account=account,
                dataset_id=dataset.id,
                inferenceservice_id=inferenceservice.id,
                hardware_type="nvidia.com/gpu-t4",
                gpu_count=2,
                replicas=2,
                replications=2,
            )
            evaluation = dyffapi.evaluations.create(evaluation_request)
            evaluations.append(evaluation)

        # Wait up to 60 seconds for all to be processed
        wait_for_all_processed(
            dyffapi.evaluations.get,
            [e.id for e in evaluations],
            "all evaluations to be processed",
            timeout=timedelta(seconds=60),
        )

        # Check results
        admitted_count = 0
        quota_limit_count = 0

        for evaluation in evaluations:
            entity = dyffapi.evaluations.get(evaluation.id)
            if entity.status in ["Admitted", "Complete", "Running"]:
                admitted_count += 1
            elif entity.status == "Created" and entity.reason == "QuotaLimit":
                quota_limit_count += 1
                # Check that message is informative
                if hasattr(entity, "message") and entity.message:
                    assert (
                        "T4 GPUs" in entity.message or "workflows" in entity.message
                    ), f"Expected informative quota message, got: {entity.message}"

        # Should hit quota limit (account quota is 16 T4 GPUs, we need 36)
        # Expect at most 4 evaluations admitted (4 × 4 GPUs = 16 GPUs)
        assert admitted_count <= 4, (
            f"Too many evaluations admitted: {admitted_count} (expected ≤ 4). "
            f"Quotas may be disabled (--disable_quotas)."
        )
        assert quota_limit_count >= 5, (
            f"Too few evaluations quota-limited: {quota_limit_count} (expected ≥ 5). "
            f"Quotas may be disabled (--disable_quotas)."
        )

    finally:
        for evaluation in evaluations:
            try:
                dyffapi.evaluations.delete(evaluation.id)
            except Exception as e:
                print(f"Cleanup failed for {evaluation.id}: {e}")

        if inferenceservice is not None:
            try:
                dyffapi.inferenceservices.delete(inferenceservice.id)
            except Exception as e:
                print(f"Cleanup failed for inference service: {e}")

        if model is not None:
            try:
                dyffapi.models.delete(model.id)
            except Exception as e:
                print(f"Cleanup failed for model: {e}")


@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.depends(
    on=["tests/test_analysis.py::test_measurements_create_python_function"]
)
def test_zero_cost_loophole_prevented(
    pytestconfig, dyffapi: Client | DyffLocalPlatform, ctx, datafiles
):
    """Test that CPU-only workflows consume quota (not free).

    Default quotas:
    - Account admission slots: 50

    This test creates many Analysis workflows to verify they consume admission slots.
    """
    if pytestconfig.getoption("skip_quotas"):
        pytest.skip()

    if pytestconfig.getoption("skip_analyses"):
        pytest.skip()

    if not pytestconfig.getoption("test_remote"):
        pytest.skip("Quota tests require remote API with orchestrator")

    if not check_quotas_enabled(pytestconfig):
        pytest.skip("Quotas appear to be disabled (e.g., --disable_quotas in skaffold)")

    account = ctx["account"]

    # Get required ctx values from test_measurements_create_python_function
    if "method_python_function" not in ctx:
        pytest.skip("No method available for Analysis creation")

    method = ctx["method_python_function"]
    evaluation = ctx["evaluation"]
    dataset = ctx["dataset"]
    inferenceservice = ctx["inferenceservice_mock"]
    model = ctx.get("model_mock")

    analyses = []

    try:
        # Create 60 Analysis workflows to exceed 50 admission slot limit
        for i in range(60):
            analysis_request = AnalysisCreateRequest(
                account=account,
                method=method.id,
                scope=AnalysisScope(
                    evaluation=evaluation.id,
                    dataset=dataset.id,
                    inferenceService=inferenceservice.id,
                    model=(model and model.id),
                ),
                arguments=[
                    AnalysisArgument(keyword="embiggen", value="smallest"),
                ],
                inputs=[
                    AnalysisInput(keyword="dataset", entity=dataset.id),
                    AnalysisInput(keyword="outputs", entity=evaluation.id),
                ],
            )
            analysis = dyffapi.measurements.create(analysis_request)
            analyses.append(analysis)

        # Wait up to 60 seconds for all to be processed
        wait_for_all_processed(
            dyffapi.measurements.get,
            [a.id for a in analyses],
            "all analyses to be processed",
            timeout=timedelta(seconds=60),
        )

        # Count how many hit quota limit
        admitted_count = 0
        quota_limit_count = 0

        for i, analysis in enumerate(analyses):
            entity = dyffapi.measurements.get(analysis.id)

            if entity.status in ["Admitted", "Complete", "Running"]:
                admitted_count += 1
            elif entity.status == "Created" and entity.reason == "QuotaLimit":
                quota_limit_count += 1

        # Should hit workflows.dyff.io/admitted quota at 50
        # NOTE: Due to eventual consistency, we may see over-admission
        assert quota_limit_count >= 8, (
            f"CPU-only workflows should hit admission quota: only {quota_limit_count} denied. "
            f"Quotas may be disabled (--disable_quotas)."
        )

    finally:
        for analysis in analyses:
            try:
                dyffapi.measurements.delete(analysis.id)
            except Exception as e:
                print(f"Cleanup failed for {analysis.id}: {e}")


@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.depends(on=["tests/test_datasets.py::test_datasets_create"])
@pytest.mark.depends(on=["tests/test_models.py::test_models_create_mock"])
def test_replicas_fill_quota(
    pytestconfig, dyffapi: Client | DyffLocalPlatform, ctx, datafiles
):
    """Test that replicas fill the GPU quota.

    Default account quota: 16 T4 GPUs

    Creates an evaluation with 8 replicas * 2 GPUs = 16 T4 GPUs to consume
    entire account quota, then verifies a second evaluation is denied.
    """
    if pytestconfig.getoption("skip_quotas"):
        pytest.skip()

    if pytestconfig.getoption("skip_evaluations"):
        pytest.skip()

    if not pytestconfig.getoption("test_remote"):
        pytest.skip("Quota tests require remote API with orchestrator")

    if not check_quotas_enabled(pytestconfig):
        pytest.skip("Quotas appear to be disabled (e.g., --disable_quotas in skaffold)")

    account = ctx["account"]
    dataset = ctx["dataset"]

    model = ctx["model_mock"]
    inferenceservice = None
    eval_1 = None
    eval_2 = None

    try:
        inferenceservice = create_ready_inferenceservice(
            dyffapi,
            account=account,
            name="quota-test-replications",
            model_id=model.id,
        )

        # Ensure quotas are enforced before running the test
        require_quota_enforcement_or_skip(
            pytestconfig,
            dyffapi,
            account=account,
            dataset_id=dataset.id,
            inferenceservice_id=inferenceservice.id,
        )

        # Create evaluation with 8 replicas: 8 × 2 = 16 T4 GPUs (entire account quota)
        eval_1_request = create_evaluation_request(
            account=account,
            dataset_id=dataset.id,
            inferenceservice_id=inferenceservice.id,
            hardware_type="nvidia.com/gpu-t4",
            gpu_count=2,
            replicas=8,
            replications=1,
        )
        eval_1 = dyffapi.evaluations.create(eval_1_request)
        # Wait for first evaluation to be admitted (not quota-limited)
        status_1 = wait_for_condition(
            lambda: dyffapi.evaluations.get(eval_1.id),
            is_processed,
            "evaluation 1 to be processed",
            timeout=timedelta(seconds=60),
        )

        # First evaluation should NOT be quota-limited (8 replicas × 2 GPUs = 16)
        assert (
            status_1.status != "Created" or status_1.reason != "QuotaLimit"
        ), f"First evaluation was quota-limited: {status_1.status}/{status_1.reason}"

        # Create second evaluation - should hit quota
        eval_2_request = create_evaluation_request(
            account=account,
            dataset_id=dataset.id,
            inferenceservice_id=inferenceservice.id,
            hardware_type="nvidia.com/gpu-t4",
            gpu_count=2,
            replicas=1,
            replications=8,
        )
        eval_2 = dyffapi.evaluations.create(eval_2_request)

        status_2 = wait_for_condition(
            lambda: dyffapi.evaluations.get(eval_2.id),
            is_processed,
            "evaluation 2 to be processed",
            timeout=timedelta(seconds=60),
        )

        # Should hit quota limit since first evaluation consumed entire account quota
        assert (
            status_2.reason == "QuotaLimit"
        ), f"Second evaluation should be quota-limited: {status_2.status}/{status_2.reason}"

        if hasattr(status_2, "message") and status_2.message:
            assert (
                "T4 GPUs" in status_2.message
            ), f"Expected T4 GPU quota message, got: {status_2.message}"

    finally:
        for evaluation in [eval_1, eval_2]:
            if evaluation is not None:
                try:
                    dyffapi.evaluations.delete(evaluation.id)
                except Exception as e:
                    print(f"Cleanup failed for {evaluation.id}: {e}")

        if inferenceservice is not None:
            try:
                dyffapi.inferenceservices.delete(inferenceservice.id)
            except Exception as e:
                print(f"Cleanup failed for inference service: {e}")

        if model is not None:
            try:
                dyffapi.models.delete(model.id)
            except Exception as e:
                print(f"Cleanup failed for model: {e}")


@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.depends(on=["tests/test_datasets.py::test_datasets_create"])
@pytest.mark.depends(on=["tests/test_models.py::test_models_create_mock"])
def test_quota_deallocation_on_termination(
    pytestconfig, dyffapi: Client | DyffLocalPlatform, ctx, datafiles
):
    """Test that quota is freed when workflows complete."""
    if pytestconfig.getoption("skip_quotas"):
        pytest.skip()

    if pytestconfig.getoption("skip_workflows"):
        pytest.skip()

    if not pytestconfig.getoption("test_remote"):
        pytest.skip("Quota tests require remote API with orchestrator")

    if not check_quotas_enabled(pytestconfig):
        pytest.skip("Quotas appear to be disabled (e.g., --disable_quotas in skaffold)")

    account = ctx["account"]
    dataset = ctx["dataset"]

    model_1 = None
    model_2 = None
    probe_inferenceservice = None

    try:
        model_1 = create_ready_model(
            dyffapi, account=account, name="quota-dealloc-test-1"
        )
        status_1 = dyffapi.models.get(model_1.id)

        assert status_1.status == "Ready", f"First model failed: {status_1.status}"

        # Ensure quotas are enforced before testing deallocation
        probe_inferenceservice = create_ready_inferenceservice(
            dyffapi,
            account=account,
            name="quota-dealloc-probe",
            model_id=model_1.id,
        )

        require_quota_enforcement_or_skip(
            pytestconfig,
            dyffapi,
            account=account,
            dataset_id=dataset.id,
            inferenceservice_id=probe_inferenceservice.id,
        )

        # Cleanup probe inferenceservice before continuing
        try:
            dyffapi.inferenceservices.delete(probe_inferenceservice.id)
        except Exception as e:
            print(f"Cleanup failed for probe inference service: {e}")
        probe_inferenceservice = None

        # Create second model - should be admitted since first completed
        model_2_request = ModelCreateRequest(
            name="quota-dealloc-test-2",
            account=account,
            artifact=ModelArtifact(kind=ModelArtifactKind.Mock),
            storage=ModelStorage(medium=ModelStorageMedium.Mock),
            source=ModelSource(kind=ModelSourceKinds.Mock),
            resources=ModelResources(storage="0"),
        )
        model_2 = dyffapi.models.create(model_2_request)

        # Wait for second model to be admitted and complete using polling
        status_2 = wait_for_condition(
            lambda: dyffapi.models.get(model_2.id),
            lambda e: is_processed(e) or e.status == "Error",
            "model 2 to be processed",
            timeout=timedelta(seconds=60),
        )
        # Second model should NOT be quota-limited (first completed and freed quota)
        assert (
            status_2.reason != "QuotaLimit"
        ), f"Second model was quota-limited: {status_2.status}/{status_2.reason}"
        assert status_2.status == "Ready", f"Second model failed: {status_2.status}"

    finally:
        if probe_inferenceservice is not None:
            try:
                dyffapi.inferenceservices.delete(probe_inferenceservice.id)
            except Exception as e:
                print(f"Cleanup failed for probe inference service: {e}")

        for model in [model_1, model_2]:
            if model is not None:
                try:
                    dyffapi.models.delete(model.id)
                except Exception as e:
                    print(f"Cleanup failed for {model.id}: {e}")


@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.depends(on=["tests/test_datasets.py::test_datasets_create"])
def test_quota_message_persistence(
    pytestconfig, dyffapi: Client | DyffLocalPlatform, ctx, datafiles
):
    """Test that quota messages persist across status updates."""
    if pytestconfig.getoption("skip_quotas"):
        pytest.skip()

    if pytestconfig.getoption("skip_evaluations"):
        pytest.skip()

    if not pytestconfig.getoption("test_remote"):
        pytest.skip("Quota tests require remote API with orchestrator")

    if not check_quotas_enabled(pytestconfig):
        pytest.skip("Quotas appear to be disabled (e.g., --disable_quotas in skaffold)")

    account = ctx["account"]
    dataset = ctx["dataset"]

    model = None
    inferenceservice = None
    evaluation = None

    try:
        model = create_ready_model(
            dyffapi, account=account, name="quota-message-test-model"
        )
        inferenceservice = create_ready_inferenceservice(
            dyffapi,
            account=account,
            name="quota-message-test",
            model_id=model.id,
        )

        # Create evaluation that will exceed quota (use high GPU count)
        evaluation_request = create_evaluation_request(
            account=account,
            dataset_id=dataset.id,
            inferenceservice_id=inferenceservice.id,
            hardware_type="nvidia.com/gpu-t4",
            gpu_count=100,
            replicas=1,
            replications=1,
        )
        evaluation = dyffapi.evaluations.create(evaluation_request)

        # Wait for orchestrator to process and set QuotaLimit using polling
        status_1 = wait_for_condition(
            lambda: dyffapi.evaluations.get(evaluation.id),
            is_quota_limited,
            "evaluation to be quota-limited",
            timeout=timedelta(seconds=60),
        )

        # Verify it hit quota limit with a message
        assert (
            status_1.reason == "QuotaLimit"
        ), f"Expected QuotaLimit, got: {status_1.reason}"

        if hasattr(status_1, "message") and status_1.message:
            original_message = status_1.message
            assert (
                "T4 GPUs" in original_message or "workflows" in original_message
            ), f"Expected informative quota message, got: {original_message}"
        else:
            pytest.fail("No message set initially - cannot validate persistence")

        # Poll to verify message persists across orchestrator cycles
        for i in range(6):
            time.sleep(5)
            status_check = dyffapi.evaluations.get(evaluation.id)
            if hasattr(status_check, "message") and status_check.message:
                if status_check.message != original_message:
                    pytest.fail(
                        f"Message changed unexpectedly! "
                        f"Original: {original_message}, Now: {status_check.message}"
                    )
            else:
                pytest.fail(f"Message was cleared after {(i + 1) * 5} seconds!")

        status_2 = dyffapi.evaluations.get(evaluation.id)

        if hasattr(status_2, "message") and status_2.message:
            # Message should persist (same as original)
            assert (
                status_2.message == original_message
            ), f"Message was cleared! Original: {original_message}, Now: {status_2.message}"
        else:
            pytest.fail("Message was cleared after status update!")

    finally:
        if evaluation is not None:
            try:
                dyffapi.evaluations.delete(evaluation.id)
            except Exception as e:
                print(f"Cleanup failed for evaluation: {e}")

        if inferenceservice is not None:
            try:
                dyffapi.inferenceservices.delete(inferenceservice.id)
            except Exception as e:
                print(f"Cleanup failed for inference service: {e}")

        if model is not None:
            try:
                dyffapi.models.delete(model.id)
            except Exception as e:
                print(f"Cleanup failed for model: {e}")


def test_quota_message_explicit_clearing(
    pytestconfig, dyffapi: Client | DyffLocalPlatform, ctx
):
    """Test that success states have no residual reason/message fields."""
    if pytestconfig.getoption("skip_quotas"):
        pytest.skip()

    if pytestconfig.getoption("skip_workflows"):
        pytest.skip()

    if not pytestconfig.getoption("test_remote"):
        pytest.skip("Quota tests require remote API with orchestrator")

    account = ctx["account"]

    model = None

    try:
        model = create_ready_model(
            dyffapi, account=account, name="quota-clear-test-model"
        )

        final_status = dyffapi.models.get(model.id)

        assert final_status.status == "Ready", f"Expected Ready: {final_status.status}"
        assert not final_status.reason, f"Unexpected reason: {final_status.reason}"
        if hasattr(final_status, "message"):
            assert (
                not final_status.message
            ), f"Unexpected message: {final_status.message}"

    finally:
        if model is not None:
            try:
                dyffapi.models.delete(model.id)
            except Exception as e:
                print(f"Cleanup failed for model: {e}")


@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.depends(on=["tests/test_datasets.py::test_datasets_create"])
def test_l4_gpu_quota_enforcement(
    pytestconfig, dyffapi: Client | DyffLocalPlatform, ctx, datafiles
):
    """Test that L4 GPU quota limits are enforced independently from T4/A10G.

    Default quotas:
    - Global: 16 L4 GPUs
    - Account: 16 L4 GPUs per hardware type

    This test verifies that L4 quotas are tracked separately.
    """
    if pytestconfig.getoption("skip_quotas"):
        pytest.skip()

    if pytestconfig.getoption("skip_evaluations"):
        pytest.skip()

    if not pytestconfig.getoption("test_remote"):
        pytest.skip("Quota tests require remote API with orchestrator")

    if not check_quotas_enabled(pytestconfig):
        pytest.skip("Quotas appear to be disabled (e.g., --disable_quotas in skaffold)")

    account = ctx["account"]
    dataset = ctx["dataset"]

    model = None
    inferenceservice = None
    eval_1 = None
    eval_2 = None

    try:
        model = create_ready_model(dyffapi, account=account, name="quota-test-l4-model")
        inferenceservice = create_ready_inferenceservice(
            dyffapi,
            account=account,
            name="quota-test-l4-inference",
            model_id=model.id,
        )

        # Ensure quotas are enforced before running the test
        require_quota_enforcement_or_skip(
            pytestconfig,
            dyffapi,
            account=account,
            dataset_id=dataset.id,
            inferenceservice_id=inferenceservice.id,
        )

        # Create evaluation with 8 replicas: 8 × 2 = 16 L4 GPUs (entire account quota)
        eval_1_request = create_evaluation_request(
            account=account,
            dataset_id=dataset.id,
            inferenceservice_id=inferenceservice.id,
            hardware_type="nvidia.com/gpu-l4",
            gpu_count=2,
            replicas=8,
            replications=1,
        )
        eval_1 = dyffapi.evaluations.create(eval_1_request)

        # Wait for first evaluation to be admitted (not quota-limited)
        status_1 = wait_for_condition(
            lambda: dyffapi.evaluations.get(eval_1.id),
            is_processed,
            "L4 evaluation 1 to be processed",
            timeout=timedelta(seconds=60),
        )

        # First evaluation should NOT be quota-limited
        assert (
            status_1.status != "Created" or status_1.reason != "QuotaLimit"
        ), f"First L4 evaluation was quota-limited: {status_1.status}/{status_1.reason}"

        # Create second evaluation - should hit quota
        eval_2_request = create_evaluation_request(
            account=account,
            dataset_id=dataset.id,
            inferenceservice_id=inferenceservice.id,
            hardware_type="nvidia.com/gpu-l4",
            gpu_count=2,
            replicas=1,
            replications=1,
        )
        eval_2 = dyffapi.evaluations.create(eval_2_request)

        # Wait for second evaluation to be processed
        status_2 = wait_for_condition(
            lambda: dyffapi.evaluations.get(eval_2.id),
            is_processed,
            "L4 evaluation 2 to be processed",
            timeout=timedelta(seconds=60),
        )

        # Should hit quota limit since first evaluation consumed entire account quota
        assert (
            status_2.reason == "QuotaLimit"
        ), f"Second L4 evaluation should be quota-limited: {status_2.status}/{status_2.reason}"

        if hasattr(status_2, "message") and status_2.message:
            assert (
                "L4 GPUs" in status_2.message
            ), f"Expected L4 GPU quota message, got: {status_2.message}"

    finally:
        for evaluation in [eval_1, eval_2]:
            if evaluation is not None:
                try:
                    dyffapi.evaluations.delete(evaluation.id)
                except Exception as e:
                    print(f"Cleanup failed for {evaluation.id}: {e}")

        if inferenceservice is not None:
            try:
                dyffapi.inferenceservices.delete(inferenceservice.id)
            except Exception as e:
                print(f"Cleanup failed for inference service: {e}")

        if model is not None:
            try:
                dyffapi.models.delete(model.id)
            except Exception as e:
                print(f"Cleanup failed for model: {e}")


@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.depends(on=["tests/test_datasets.py::test_datasets_create"])
def test_replicas_counted_correctly(
    pytestconfig, dyffapi: Client | DyffLocalPlatform, ctx, datafiles
):
    """Test that replicas multiplier is applied correctly.

    Default account quota: 16 T4 GPUs
    """
    if pytestconfig.getoption("skip_quotas"):
        pytest.skip()

    if pytestconfig.getoption("skip_evaluations"):
        pytest.skip()

    if not pytestconfig.getoption("test_remote"):
        pytest.skip("Quota tests require remote API with orchestrator")

    if not check_quotas_enabled(pytestconfig):
        pytest.skip("Quotas appear to be disabled (e.g., --disable_quotas in skaffold)")

    account = ctx["account"]
    dataset = ctx["dataset"]

    model = None
    inferenceservice = None
    eval_1 = None
    eval_2 = None
    eval_3 = None

    try:
        model = create_ready_model(
            dyffapi, account=account, name="quota-test-replicas-model"
        )
        inferenceservice = create_ready_inferenceservice(
            dyffapi,
            account=account,
            name="quota-test-replicas",
            model_id=model.id,
        )

        # Ensure quotas are enforced before running the test
        require_quota_enforcement_or_skip(
            pytestconfig,
            dyffapi,
            account=account,
            dataset_id=dataset.id,
            inferenceservice_id=inferenceservice.id,
        )

        # Each evaluation uses replicas=4, gpu_count=2 => 8 GPUs
        eval_1_request = create_evaluation_request(
            account=account,
            dataset_id=dataset.id,
            inferenceservice_id=inferenceservice.id,
            hardware_type="nvidia.com/gpu-t4",
            gpu_count=2,
            replicas=4,
            replications=2,
        )
        eval_1 = dyffapi.evaluations.create(eval_1_request)
        eval_2 = dyffapi.evaluations.create(eval_1_request)

        # First two should be admitted (total 16 GPUs)
        status_1 = wait_for_condition(
            lambda: dyffapi.evaluations.get(eval_1.id),
            is_processed,
            "evaluation 1 to be processed",
            timeout=timedelta(seconds=60),
        )
        status_2 = wait_for_condition(
            lambda: dyffapi.evaluations.get(eval_2.id),
            is_processed,
            "evaluation 2 to be processed",
            timeout=timedelta(seconds=60),
        )
        assert (
            status_1.reason != "QuotaLimit"
        ), f"Eval 1 quota-limited: {status_1.reason}"
        assert (
            status_2.reason != "QuotaLimit"
        ), f"Eval 2 quota-limited: {status_2.reason}"

        # Third should hit quota
        eval_3 = dyffapi.evaluations.create(eval_1_request)
        status_3 = wait_for_condition(
            lambda: dyffapi.evaluations.get(eval_3.id),
            is_processed,
            "evaluation 3 to be processed",
            timeout=timedelta(seconds=60),
        )
        assert status_3.reason == "QuotaLimit", (
            f"Evaluation 3 should be quota-limited but wasn't: "
            f"status={status_3.status}, reason={status_3.reason}"
        )

    finally:
        for evaluation in [eval_1, eval_2, eval_3]:
            if evaluation is not None:
                try:
                    dyffapi.evaluations.delete(evaluation.id)
                except Exception as e:
                    print(f"Cleanup failed for {evaluation.id}: {e}")

        if inferenceservice is not None:
            try:
                dyffapi.inferenceservices.delete(inferenceservice.id)
            except Exception as e:
                print(f"Cleanup failed for inference service: {e}")

        if model is not None:
            try:
                dyffapi.models.delete(model.id)
            except Exception as e:
                print(f"Cleanup failed for model: {e}")
