# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
import datetime as dt
import logging
import os
import shutil
from datetime import timedelta as td, timezone, tzinfo
from pathlib import Path
from typing import Any, Dict, Final, List, Mapping, Optional, Union
from zoneinfo import ZoneInfo

import pytest
from click.testing import CliRunner

from gcm.monitoring.cli.slurm_monitor import main
from gcm.monitoring.clock import AwareDatetime, PT
from gcm.monitoring.slurm.sinfo import (
    compute_allocated_resources,
    compute_avg_allocated_cpus_gpus,
    compute_down_nodes,
    compute_failed_jobs,
    compute_jobs_without_user,
    compute_mean_and_variance,
    compute_node_states,
    compute_per_account_slurm_log,
    compute_resources_pending,
    compute_running_and_pending_users,
    compute_total_allocated_cpus_gpus,
    compute_total_cpus_gpus,
    compute_wait_time_distribution,
)
from gcm.schemas.slurm.sacct import SacctMetrics
from gcm.schemas.slurm.sinfo import Sinfo
from gcm.schemas.slurm.sinfo_cpus_gpus import SinfoCpusGpus
from gcm.schemas.slurm.sinfo_node import SinfoNode
from gcm.schemas.slurm.sinfo_node_states import SinfoNodeStates
from gcm.schemas.slurm.slurm_log import SLURMLogAccountMetrics
from typeguard import typechecked

PT_TIMEZONE = PT
UTC_TIMEZONE = timezone.utc
TEST_PARTITION = "fake_partition"
try:
    tz_path = os.path.realpath("/etc/localtime", strict=True)
except FileNotFoundError:
    SYSTEM_TZ = ZoneInfo("Etc/UTC")
else:
    SYSTEM_TZ = ZoneInfo(os.path.relpath(tz_path, "/usr/share/zoneinfo"))


@pytest.mark.parametrize(
    "jobs, key, expected_mean, expected_var",
    [
        ([{"foo": 1.0}], "foo", 1.0, 0.0),
        ([{"foo": 1.0}, {"bar": 42}], "foo", 1.0, 0.0),
        ([{"foo": 1.0}, {"bar": 42}], "bar", 42.0, 0.0),
        ([{"foo": 1.0, "bar": 42}], "foo", 1.0, 0.0),
        (
            [
                {"foo": 1.0},
                {"foo": -1.0},
                {"bar": 100},
            ],
            "foo",
            0.0,
            2.0,
        ),
        (
            [
                {"foo": 2.0},
                {"foo": 10.0},
                {"bar": 100},
            ],
            "foo",
            6.0,
            32.0,
        ),
    ],
)
def test_compute_mean_and_variance(
    jobs: List[Mapping[str, Union[float, int]]],
    key: str,
    expected_mean: float,
    expected_var: float,
) -> None:
    eps = 1e-3

    mean, var = compute_mean_and_variance(jobs, key)

    assert abs(mean - expected_mean) < eps
    assert abs(var - expected_var) < eps


@pytest.mark.parametrize(
    "jobs, key",
    [([], "foo"), ([{"bar": 1.0}], "foo")],
)
def test_compute_mean_and_variance_throws_on_empty(
    jobs: List[Dict[str, Any]], key: str
) -> None:
    with pytest.raises(ValueError):
        compute_mean_and_variance(jobs, key)


_dt_utc = dt.datetime(2022, 6, 8, 12, 3, 40, 0).astimezone(UTC_TIMEZONE)
_dt_pt = dt.datetime(2022, 6, 8, 12, 3, 40, 0).astimezone(PT_TIMEZONE)
_dt_sys = dt.datetime(2022, 6, 8, 12, 3, 40, 0).astimezone(SYSTEM_TZ)


def _s(d: dt.datetime) -> str:
    return d.strftime("%Y-%m-%dT%H:%M:%S")


def _s_aware(d: dt.datetime) -> str:
    return d.isoformat()


@pytest.mark.parametrize(
    "jobs, expected_log, expected_mean, expected_var",
    [
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqNodes=2,
                    ReqTRES="8",
                    End="Unknown",
                    State="RUNINNG",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    ReqGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                )
            ],
            None,
            1.0,
            0.0,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqNodes=2,
                    ReqTRES="8",
                    End="Unknown",
                    State="RUNINNG",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    ReqGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="2",
                    Submit=_s(_dt_pt + td(seconds=10)),
                    Start=_s(_dt_pt + td(seconds=16)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqNodes=2,
                    ReqTRES="8",
                    End="Unknown",
                    State="RUNINNG",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    ReqGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            None,
            3.5,
            12.5,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqNodes=2,
                    ReqTRES="8",
                    End="Unknown",
                    State="PENDING",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    ReqGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="2",
                    Submit=_s(_dt_pt + td(seconds=10)),
                    Start=_s(_dt_pt + td(seconds=16)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqNodes=2,
                    ReqTRES="8",
                    End="Unknown",
                    State="RUNINNG",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    ReqGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="3",
                    Submit=_s(_dt_pt + td(seconds=0)),
                    Start="Unknown",
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqNodes=2,
                    ReqTRES="8",
                    End="Unknown",
                    State="RUNINNG",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    ReqGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            "Invalid Start 'job.Start='Unknown'' for job job.JobID='3'",
            3.5,
            12.5,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqNodes=2,
                    ReqTRES="8",
                    End="Unknown",
                    State="RUNINNG",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    ReqGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="2",
                    Submit=_s(_dt_pt + td(seconds=10)),
                    Start=_s(_dt_pt + td(seconds=16)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqNodes=2,
                    ReqTRES="8",
                    End="Unknown",
                    State="RUNINNG",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    ReqGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="3",
                    Submit="Unknown",
                    Start=_s(_dt_pt + td(seconds=16)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqNodes=2,
                    ReqTRES="8",
                    End="Unknown",
                    State="RUNINNG",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    ReqGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            "Invalid Submit 'job.Submit='Unknown'' for job job.JobID='3'",
            3.5,
            12.5,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqNodes=2,
                    ReqTRES="8",
                    End="Unknown",
                    State="RUNINNG",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    ReqGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="2",
                    Submit=_s(_dt_pt + td(seconds=10)),
                    Start=_s(_dt_pt + td(seconds=16)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqNodes=2,
                    ReqTRES="8",
                    End="Unknown",
                    State="RUNINNG",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    ReqGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="3",
                    Submit="Unknown",
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqNodes=2,
                    ReqTRES="8",
                    End="Unknown",
                    State="RUNINNG",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    ReqGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            "Invalid Submit 'job.Submit='Unknown'' for job job.JobID='3'",
            3.5,
            12.5,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqNodes=2,
                    ReqTRES="8",
                    End="Unknown",
                    State="RUNINNG",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    ReqGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            None,
            1.0,
            0.0,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqNodes=2,
                    ReqTRES="8",
                    End="Unknown",
                    State="RUNINNG",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    ReqGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="2",
                    Submit=_s(_dt_utc + td(seconds=10)),
                    Start=_s(_dt_utc + td(seconds=16)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqNodes=2,
                    ReqTRES="8",
                    End="Unknown",
                    State="RUNINNG",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    ReqGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            None,
            3.5,
            12.5,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqNodes=2,
                    ReqTRES="8",
                    End="Unknown",
                    State="RUNINNG",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    ReqGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="2",
                    Submit=_s(_dt_utc + td(seconds=10)),
                    Start=_s(_dt_utc + td(seconds=16)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqNodes=2,
                    ReqTRES="8",
                    End="Unknown",
                    State="RUNINNG",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    ReqGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="3",
                    Submit=_s(_dt_utc + td(seconds=10)),
                    Start="Unknown",
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqNodes=2,
                    ReqTRES="8",
                    End="Unknown",
                    State="PENDING",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    ReqGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            "Invalid Start 'job.Start='Unknown'' for job job.JobID='3'",
            3.5,
            12.5,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqNodes=2,
                    ReqTRES="8",
                    End="Unknown",
                    State="PENDING",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    ReqGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="2",
                    Submit=_s(_dt_utc + td(seconds=10)),
                    Start=_s(_dt_utc + td(seconds=16)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqNodes=2,
                    ReqTRES="8",
                    End="Unknown",
                    State="PENDING",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    ReqGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="3",
                    Submit="Unknown",
                    Start=_s(_dt_utc + td(seconds=16)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqNodes=2,
                    ReqTRES="8",
                    End="Unknown",
                    State="PENDING",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    ReqGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            "Invalid Submit 'job.Submit='Unknown'' for job job.JobID='3'",
            3.5,
            12.5,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqNodes=2,
                    ReqTRES="8",
                    End="Unknown",
                    State="RUNINNG",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    ReqGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="2",
                    Submit=_s(_dt_utc + td(seconds=10)),
                    Start=_s(_dt_utc + td(seconds=16)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqNodes=2,
                    ReqTRES="8",
                    End="Unknown",
                    State="RUNINNG",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    ReqGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="3",
                    Submit="Unknown",
                    Start="Unknown",
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqNodes=2,
                    ReqTRES="8",
                    End="Unknown",
                    State="PENDING",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    ReqGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            "Invalid Submit 'job.Submit='Unknown'' for job job.JobID='3'",
            3.5,
            12.5,
        ),
    ],
)
def test_compute_wait_time_distribution(
    jobs: list[SacctMetrics],
    expected_log: Optional[str],
    expected_mean: float,
    expected_var: float,
    caplog: pytest.LogCaptureFixture,
) -> None:
    eps = 1e-3

    mean, var = compute_wait_time_distribution(jobs)

    assert abs(mean - expected_mean) < eps
    assert abs(var - expected_var) < eps
    if expected_log is not None:
        assert any(
            expected_log in r.message
            for r in caplog.records
            if r.levelno == logging.ERROR
        )


@pytest.mark.parametrize(
    "jobs, expected_jobs_pending, expected_gpus_pending, expected_nodes_pending",
    [
        ([], 0, 0, 0),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="PENDING",
                    ReqGPUS=1,
                    ReqNodes=1,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                )
            ],
            1,
            1,
            1,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="SUSPENDED",
                    ReqGPUS=1,
                    ReqNodes=1,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                )
            ],
            1,
            1,
            1,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="RUNNING",
                    ReqGPUS=1,
                    ReqNodes=1,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                )
            ],
            0,
            0,
            0,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="PENDING",
                    ReqGPUS=1,
                    ReqNodes=1,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="RUNNING",
                    ReqGPUS=1,
                    ReqNodes=1,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            1,
            1,
            1,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="PENDING",
                    ReqGPUS=2,
                    ReqNodes=2,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="SUSPENDED",
                    ReqGPUS=3,
                    ReqNodes=3,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            2,
            5,
            5,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="PENDING",
                    ReqGPUS=0,
                    ReqNodes=0,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            1,
            0,
            0,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="PENDING",
                    ReqGPUS=10,
                    ReqNodes=10,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="SUSPENDED",
                    ReqGPUS=20,
                    ReqNodes=20,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="PENDING",
                    ReqGPUS=30,
                    ReqNodes=30,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            3,
            60,
            60,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="RUNNING",
                    ReqGPUS=10,
                    ReqNodes=10,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="RUNNING",
                    ReqGPUS=20,
                    ReqNodes=20,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="RUNNING",
                    ReqGPUS=30,
                    ReqNodes=30,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            0,
            0,
            0,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="PENDING",
                    ReqGPUS=10,
                    ReqNodes=10,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="RUNNING",
                    ReqGPUS=20,
                    ReqNodes=20,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="SUSPENDED",
                    ReqGPUS=30,
                    ReqNodes=30,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            2,
            40,
            40,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="CONFIGURING",
                    ReqGPUS=10,
                    ReqNodes=10,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="REQUEUE_FED",
                    ReqGPUS=20,
                    ReqNodes=20,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="REQUEUED_HOLD",
                    ReqGPUS=30,
                    ReqNodes=30,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            0,
            0,
            0,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="CONFIGURING",
                    ReqGPUS=10,
                    ReqNodes=10,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="REQUEUE_FED",
                    ReqGPUS=20,
                    ReqNodes=20,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="REQUEUED_HOLD",
                    ReqGPUS=30,
                    ReqNodes=30,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="RUNNING",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            0,
            0,
            0,
        ),
    ],
)
def test_compute_resources_pending(
    jobs: list[SacctMetrics],
    expected_jobs_pending: int,
    expected_gpus_pending: int,
    expected_nodes_pending: int,
) -> None:
    jobs_pending, gpus_pending, nodes_pending = compute_resources_pending(jobs)
    assert jobs_pending == expected_jobs_pending
    assert gpus_pending == expected_gpus_pending
    assert nodes_pending == expected_nodes_pending


@pytest.mark.parametrize(
    "jobs, expected_failed",
    [
        ([], 0),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="FAILED",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                )
            ],
            1,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="CANCELLED",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                )
            ],
            1,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="BOOT_FAIL",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                )
            ],
            1,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="DEADLINE",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                )
            ],
            1,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="NODE_FAIL",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                )
            ],
            1,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="OUT_OF_MEMORY",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                )
            ],
            1,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="PREEMPTED",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            1,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="TIMEOUT",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            1,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="FAILED",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="CANCELLED",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            2,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="FAILED",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="CANCELLED",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="BOOT_FAIL",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            3,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="FAILED",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="CANCELLED",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="PENDING",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            2,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="FAILED",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="CANCELLED",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="RUNNING",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            2,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="FAILED",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="CANCELLED",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="NODE_FAIL",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="OUT_OF_MEMORY",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="PREEMPTED",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="TIMEOUT",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    User="unixname",
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            6,
        ),
    ],
)
def test_compute_failed_jobs(jobs: list[SacctMetrics], expected_failed: int) -> None:
    assert compute_failed_jobs(jobs) == expected_failed


@pytest.mark.parametrize(
    "jobs, expected_unique_users",
    [
        ([], 0),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="PENDING",
                    User="user1",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                )
            ],
            1,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="RUNNING",
                    User="user1",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                )
            ],
            1,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="PENDING",
                    User="user1",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="RUNNING",
                    User="user2",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            2,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="PENDING",
                    User="user1",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="SUSPENDED",
                    User="user1",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            1,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="CONFIGURING",
                    User="user1",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="REQUEUE_FED",
                    User="user2",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="REQUEUED_HOLD",
                    User="user3",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            0,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="PENDING",
                    User="user1",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="RUNNING",
                    User="user1",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="SUSPENDED",
                    User="user1",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            1,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="PENDING",
                    User="user1",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="RUNNING",
                    User="user2",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="SUSPENDED",
                    User="user3",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            3,
        ),
    ],
)
def test_compute_running_and_pending_users(
    jobs: list[SacctMetrics],
    expected_unique_users: int,
) -> None:
    unique_users = compute_running_and_pending_users(jobs)
    assert unique_users == expected_unique_users


@pytest.mark.parametrize(
    "jobs, expected_jobs_without_user",
    [
        ([], 0),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="PENDING",
                    User="user1",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="RUNNING",
                    User="user2",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            0,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="RUNNING",
                    User="1234",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="SUSPENDED",
                    User="5678",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            2,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="PENDING",
                    User="user1",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="RUNNING",
                    User="1234",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="SUSPENDED",
                    User="user3",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="CONFIGURING",
                    User="9012",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            2,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="REQUEUE_FED",
                    User="3456",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="REQUEUED_HOLD",
                    User="7890",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="PENDING",
                    User="user1",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="RUNNING",
                    User="user2",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            2,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="PENDING",
                    User="user1",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="RUNNING",
                    User="1234",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="SUSPENDED",
                    User="5678",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="CONFIGURING",
                    User="9012",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="REQUEUE_FED",
                    User="3456",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            4,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="PENDING",
                    User="user1",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="RUNNING",
                    User="user2",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="SUSPENDED",
                    User="user3",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="CONFIGURING",
                    User="user4",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    JobID="1",
                    Submit=_s(_dt_pt),
                    Start=_s(_dt_pt + td(seconds=1)),
                    Account="accountname",
                    AllocCPUS=10,
                    AllocTRES="8",
                    ReqTRES="8",
                    End="Unknown",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    AllocGPUS=8,
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            0,
        ),
    ],
)
def test_compute_jobs_without_user(
    jobs: list[SacctMetrics],
    expected_jobs_without_user: int,
) -> None:
    jobs_without_user = compute_jobs_without_user(jobs)
    assert jobs_without_user == expected_jobs_without_user


@pytest.mark.parametrize(
    "start_time, end_time, jobs, expected_cpu_usage, expected_gpu_usage, expected_log, tz",
    [
        (
            _dt_sys,
            _dt_sys + td(seconds=10),
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Start=_s(_dt_sys + td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=5)),
                    AllocCPUS=5,
                    AllocGPUS=2,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            5.0,
            2.0,
            None,
            PT_TIMEZONE,
        ),
        (
            _dt_sys,
            _dt_sys + td(seconds=10),
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Start=_s(_dt_sys + td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=15)),
                    AllocCPUS=5,
                    AllocGPUS=2,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            5.0,
            2.0,
            None,
            PT_TIMEZONE,
        ),
        (
            _dt_sys,
            _dt_sys + td(seconds=10),
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Start=_s(_dt_sys + td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=5)),
                    AllocCPUS=5,
                    AllocGPUS=2,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="2",
                    Start=_s(_dt_sys + td(seconds=2)),
                    End=_s(_dt_sys + td(seconds=6)),
                    AllocCPUS=10,
                    AllocGPUS=4,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            7.5,
            3.0,
            None,
            PT_TIMEZONE,
        ),
        (
            _dt_sys,
            _dt_sys + td(seconds=10),
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Start=_s(_dt_sys + td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=5)),
                    AllocCPUS=5,
                    AllocGPUS=2,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="2",
                    Start=_s(_dt_sys - td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=15)),
                    AllocCPUS=10,
                    AllocGPUS=4,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            ((4 * 5) + (10 * 10)) / (4 + 10),
            ((4 * 2) + (10 * 4)) / (4 + 10),
            None,
            PT_TIMEZONE,
        ),
        (
            _dt_sys,
            _dt_sys + td(seconds=10),
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Start=_s(_dt_sys + td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=5)),
                    AllocCPUS=5,
                    AllocGPUS=2,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_sys),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="2",
                    Start=_s(_dt_sys - td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=15)),
                    AllocCPUS=10,
                    AllocGPUS=4,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_sys),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="3",
                    Start="Unknown",
                    End=_s(_dt_sys + td(seconds=15)),
                    AllocCPUS=100,
                    AllocGPUS=40,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_sys),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            ((4 * 5) + (10 * 10)) / (4 + 10),
            ((4 * 2) + (10 * 4)) / (4 + 10),
            "Invalid Start 'job.Start='Unknown'' for job job.JobID='3'",
            PT_TIMEZONE,
        ),
        (
            _dt_sys,
            _dt_sys + td(seconds=10),
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Start=_s(_dt_sys + td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=5)),
                    AllocCPUS=5,
                    AllocGPUS=2,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="2",
                    Start=_s(_dt_sys - td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=15)),
                    AllocCPUS=10,
                    AllocGPUS=4,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="3",
                    Start=_s(_dt_sys + td(seconds=1)),
                    End="Unknown",
                    AllocCPUS=100,
                    AllocGPUS=40,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            ((4 * 5) + (10 * 10)) / (4 + 10),
            ((4 * 2) + (10 * 4)) / (4 + 10),
            "Invalid End 'job.End='Unknown'' for job job.JobID='3'",
            PT_TIMEZONE,
        ),
        (
            _dt_sys,
            _dt_sys + td(seconds=10),
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Start=_s(_dt_sys + td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=5)),
                    AllocCPUS=5,
                    AllocGPUS=2,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="2",
                    Start=_s(_dt_sys - td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=15)),
                    AllocCPUS=10,
                    AllocGPUS=4,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="3",
                    Start="Unknown",
                    End="Unknown",
                    AllocCPUS=100,
                    AllocGPUS=40,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            ((4 * 5) + (10 * 10)) / (4 + 10),
            ((4 * 2) + (10 * 4)) / (4 + 10),
            "Invalid Start 'job.Start='Unknown'' for job job.JobID='3'",
            SYSTEM_TZ,
        ),
        (
            _dt_pt,
            _dt_pt + td(seconds=10),
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Start=_s(_dt_sys + td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=5)),
                    AllocCPUS=5,
                    AllocGPUS=2,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            5.0,
            2.0,
            None,
            PT_TIMEZONE,
        ),
        (
            _dt_pt,
            _dt_pt + td(seconds=10),
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Start=_s(_dt_sys - td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=15)),
                    AllocCPUS=5,
                    AllocGPUS=2,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            5.0,
            2.0,
            None,
            PT_TIMEZONE,
        ),
        (
            _dt_pt,
            _dt_pt + td(seconds=10),
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Start=_s(_dt_sys + td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=5)),
                    AllocCPUS=5,
                    AllocGPUS=2,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="2",
                    Start=_s(_dt_sys + td(seconds=2)),
                    End=_s(_dt_sys + td(seconds=6)),
                    AllocCPUS=10,
                    AllocGPUS=4,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            7.5,
            3.0,
            None,
            PT_TIMEZONE,
        ),
        (
            _dt_pt,
            _dt_pt + td(seconds=10),
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Start=_s(_dt_sys + td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=5)),
                    AllocCPUS=5,
                    AllocGPUS=2,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="2",
                    Start=_s(_dt_sys - td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=15)),
                    AllocCPUS=10,
                    AllocGPUS=4,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            ((4 * 5) + (10 * 10)) / (4 + 10),
            ((4 * 2) + (10 * 4)) / (4 + 10),
            None,
            PT_TIMEZONE,
        ),
        (
            _dt_pt,
            _dt_pt + td(seconds=10),
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Start=_s(_dt_sys + td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=5)),
                    AllocCPUS=5,
                    AllocGPUS=2,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="2",
                    Start=_s(_dt_sys - td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=15)),
                    AllocCPUS=10,
                    AllocGPUS=4,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="3",
                    Start="Unknown",
                    End=_s(_dt_sys + td(seconds=15)),
                    AllocCPUS=100,
                    AllocGPUS=40,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            ((4 * 5) + (10 * 10)) / (4 + 10),
            ((4 * 2) + (10 * 4)) / (4 + 10),
            "Invalid Start 'job.Start='Unknown'' for job job.JobID='3'",
            PT_TIMEZONE,
        ),
        (
            _dt_pt,
            _dt_pt + td(seconds=10),
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Start=_s(_dt_sys + td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=5)),
                    AllocCPUS=5,
                    AllocGPUS=2,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="2",
                    Start=_s(_dt_sys - td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=15)),
                    AllocCPUS=10,
                    AllocGPUS=4,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="3",
                    Start=_s_aware(_dt_pt - td(seconds=1)),
                    End="Unknown",
                    AllocCPUS=100,
                    AllocGPUS=40,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            ((4 * 5) + (10 * 10)) / (4 + 10),
            ((4 * 2) + (10 * 4)) / (4 + 10),
            "Invalid End 'job.End='Unknown'' for job job.JobID='3'",
            PT_TIMEZONE,
        ),
        (
            _dt_pt,
            _dt_pt + td(seconds=10),
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Start=_s(_dt_sys + td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=5)),
                    AllocCPUS=5,
                    AllocGPUS=2,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="2",
                    Start=_s(_dt_sys - td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=15)),
                    AllocCPUS=10,
                    AllocGPUS=4,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="3",
                    Start="Unknown",
                    End="Unknown",
                    AllocCPUS=100,
                    AllocGPUS=40,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            ((4 * 5) + (10 * 10)) / (4 + 10),
            ((4 * 2) + (10 * 4)) / (4 + 10),
            "Invalid Start 'job.Start='Unknown'' for job job.JobID='3'",
            PT_TIMEZONE,
        ),
        (
            _dt_utc,
            _dt_utc + td(seconds=10),
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Start=_s(_dt_sys + td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=5)),
                    AllocCPUS=5,
                    AllocGPUS=2,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            5.0,
            2.0,
            None,
            UTC_TIMEZONE,
        ),
        (
            _dt_utc,
            _dt_utc + td(seconds=10),
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Start=_s(_dt_sys + td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=15)),
                    AllocCPUS=5,
                    AllocGPUS=2,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            5.0,
            2.0,
            None,
            UTC_TIMEZONE,
        ),
        (
            _dt_utc,
            _dt_utc + td(seconds=10),
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Start=_s(_dt_sys + td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=5)),
                    AllocCPUS=5,
                    AllocGPUS=2,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="2",
                    Start=_s(_dt_sys + td(seconds=2)),
                    End=_s(_dt_sys + td(seconds=6)),
                    AllocCPUS=10,
                    AllocGPUS=4,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            7.5,
            3.0,
            None,
            UTC_TIMEZONE,
        ),
        (
            _dt_utc,
            _dt_utc + td(seconds=10),
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Start=_s(_dt_sys + td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=5)),
                    AllocCPUS=5,
                    AllocGPUS=2,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="2",
                    Start=_s(_dt_sys - td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=15)),
                    AllocCPUS=10,
                    AllocGPUS=4,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            ((4 * 5) + (10 * 10)) / (4 + 10),
            ((4 * 2) + (10 * 4)) / (4 + 10),
            None,
            UTC_TIMEZONE,
        ),
        (
            _dt_utc,
            _dt_utc + td(seconds=10),
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Start=_s(_dt_sys + td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=5)),
                    AllocCPUS=5,
                    AllocGPUS=2,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="2",
                    Start=_s(_dt_sys - td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=15)),
                    AllocCPUS=10,
                    AllocGPUS=4,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="3",
                    Start="Unknown",
                    End=_s_aware(_dt_utc + td(seconds=15)),
                    AllocCPUS=100,
                    AllocGPUS=40,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            ((4 * 5) + (10 * 10)) / (4 + 10),
            ((4 * 2) + (10 * 4)) / (4 + 10),
            "Invalid Start 'job.Start='Unknown'' for job job.JobID='3'",
            UTC_TIMEZONE,
        ),
        (
            _dt_utc,
            _dt_utc + td(seconds=10),
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Start=_s(_dt_sys + td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=5)),
                    AllocCPUS=5,
                    AllocGPUS=2,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="2",
                    Start=_s(_dt_sys - td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=15)),
                    AllocCPUS=10,
                    AllocGPUS=4,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="3",
                    Start=_s_aware(_dt_utc - td(seconds=1)),
                    End="Unknown",
                    AllocCPUS=100,
                    AllocGPUS=40,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            ((4 * 5) + (10 * 10)) / (4 + 10),
            ((4 * 2) + (10 * 4)) / (4 + 10),
            "Invalid End 'job.End='Unknown'' for job job.JobID='3'",
            UTC_TIMEZONE,
        ),
        (
            _dt_utc,
            _dt_utc + td(seconds=10),
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Start=_s(_dt_sys + td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=5)),
                    AllocCPUS=5,
                    AllocGPUS=2,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="2",
                    Start=_s(_dt_sys - td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=15)),
                    AllocCPUS=10,
                    AllocGPUS=4,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="3",
                    Start="Unknown",
                    End="Unknown",
                    AllocCPUS=100,
                    AllocGPUS=40,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            ((4 * 5) + (10 * 10)) / (4 + 10),
            ((4 * 2) + (10 * 4)) / (4 + 10),
            "Invalid Start 'job.Start='Unknown'' for job job.JobID='3'",
            UTC_TIMEZONE,
        ),
    ],
)
def test_compute_avg_allocated_cpus_gpus(
    caplog: pytest.LogCaptureFixture,
    start_time: AwareDatetime,
    end_time: AwareDatetime,
    jobs: list[SacctMetrics],
    expected_cpu_usage: float,
    expected_gpu_usage: float,
    expected_log: Optional[str],
    tz: tzinfo,
) -> None:
    eps = 1e-3

    cpu_usage, gpu_usage = compute_avg_allocated_cpus_gpus(
        start_time, end_time, jobs, tz
    )

    assert abs(cpu_usage - expected_cpu_usage) < eps
    assert abs(gpu_usage - expected_gpu_usage) < eps
    if expected_log is not None:
        assert any(
            expected_log in r.message
            for r in caplog.records
            if r.levelno == logging.ERROR
        )


@pytest.mark.parametrize(
    "jobs, expected_cpu_alloc, expected_gpu_alloc, expected_node_alloc",
    [
        (
            [],
            0,
            0,
            0,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Start=_s(_dt_sys + td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=5)),
                    AllocCPUS=5,
                    AllocGPUS=1,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            5,
            1,
            2,
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Start=_s(_dt_sys + td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=5)),
                    AllocCPUS=5,
                    AllocGPUS=2,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="2",
                    Start=_s(_dt_sys - td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=15)),
                    AllocCPUS=10,
                    AllocGPUS=4,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="3",
                    Start="Unknown",
                    End="Unknown",
                    AllocCPUS=100,
                    AllocGPUS=40,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            115,
            46,
            6,
        ),
    ],
)
@typechecked
def test_compute_allocated_resources(
    jobs: list[SacctMetrics],
    expected_cpu_alloc: int,
    expected_gpu_alloc: int,
    expected_node_alloc: int,
) -> None:
    actual_cpu_alloc, actual_gpu_alloc, actual_node_alloc = compute_allocated_resources(
        jobs
    )

    assert actual_cpu_alloc == expected_cpu_alloc
    assert actual_gpu_alloc == expected_gpu_alloc
    assert actual_node_alloc == expected_node_alloc


@pytest.mark.parametrize(
    "jobs, expected_logs",
    [
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Start=_s(_dt_sys + td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=5)),
                    AllocCPUS=5,
                    AllocGPUS=1,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            [
                SLURMLogAccountMetrics(
                    prefix="gcm.accountname.",
                    account="accountname",
                    derived_cluster="test_cluster",
                    total_gpus_alloc=1,
                    total_cpus_alloc=10,
                    total_nodes_alloc=1,
                    job_runtime_mean=0,
                    job_runtime_variance=0,
                    active_users=1,
                    jobs_dist_training_percent=1,
                ),
            ],
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Start=_s(_dt_sys + td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=5)),
                    AllocCPUS=5,
                    AllocGPUS=1,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Start=_s(_dt_sys + td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=5)),
                    AllocCPUS=5,
                    AllocGPUS=1,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            [
                SLURMLogAccountMetrics(
                    prefix="gcm.accountname.",
                    account="accountname",
                    derived_cluster="test_cluster",
                    total_gpus_alloc=1,
                    total_cpus_alloc=10,
                    total_nodes_alloc=1,
                    job_runtime_mean=0,
                    job_runtime_variance=0,
                    active_users=1,
                    jobs_dist_training_percent=1,
                ),
            ],
        ),
        (
            [
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="1",
                    Start=_s(_dt_sys + td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=5)),
                    AllocCPUS=5,
                    AllocGPUS=2,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
                SacctMetrics(
                    derived_cluster="test_cluster",
                    WaitingTime=None,
                    JobID="2",
                    Start=_s(_dt_sys - td(seconds=1)),
                    End=_s(_dt_sys + td(seconds=15)),
                    AllocCPUS=10,
                    AllocGPUS=4,
                    State="REQUEUE_FED",
                    User="user5",
                    ReqGPUS=40,
                    ReqNodes=40,
                    Submit=_s(_dt_pt),
                    Account="accountname2",
                    AllocTRES="8",
                    ReqTRES="8",
                    AllocNodes=2,
                    Elapsed="Unknown",
                    Suspended="Unknown",
                    RunTimeSeconds=8,
                    SuspendedSeconds=8,
                ),
            ],
            [
                SLURMLogAccountMetrics(
                    prefix="gcm.accountname.",
                    account="accountname",
                    derived_cluster="test_cluster",
                    total_gpus_alloc=1,
                    total_cpus_alloc=10,
                    total_nodes_alloc=1,
                    job_runtime_mean=0,
                    job_runtime_variance=0,
                    active_users=1,
                    jobs_dist_training_percent=1,
                ),
                SLURMLogAccountMetrics(
                    prefix="gcm.accountname2.",
                    account="accountname2",
                    derived_cluster="test_cluster",
                    total_gpus_alloc=1,
                    total_cpus_alloc=10,
                    total_nodes_alloc=1,
                    job_runtime_mean=0,
                    job_runtime_variance=0,
                    active_users=1,
                    jobs_dist_training_percent=1,
                ),
            ],
        ),
    ],
)
@typechecked
def test_compute_per_account_slurm_log(
    jobs: list[SacctMetrics],
    expected_logs: list[SLURMLogAccountMetrics],
) -> None:
    actual_logs = compute_per_account_slurm_log(jobs, "test_cluster")

    for actual, expected in zip(actual_logs, expected_logs):
        assert actual.account == expected.account


@pytest.mark.parametrize(
    "sinfo, expected_cpus, expected_gpus, expected_nodes",
    [
        (Sinfo(nodes={}), 0, 0, 0),
        (
            Sinfo(
                nodes=[
                    SinfoNode(
                        name="node1",
                        state="allocated",
                        alloc_cpus=10,
                        gres_used="gpu:volta:1",
                        gres="gpu:volta:8(S:0-1)",
                        total_cpus=80,
                        partition=TEST_PARTITION,
                    )
                ]
            ),
            10,
            1,
            1,
        ),
        (
            Sinfo(
                nodes=[
                    SinfoNode(
                        name="node1",
                        state="allocated",
                        alloc_cpus=10,
                        gres_used="gpu:volta:1",
                        gres="gpu:volta:8(S:0-1)",
                        total_cpus=80,
                        partition=TEST_PARTITION,
                    ),
                    SinfoNode(
                        name="node2",
                        state="idle",
                        alloc_cpus=0,
                        gres_used="gpu:volta:1",
                        gres="gpu:volta:8(S:0-1)",
                        total_cpus=80,
                        partition=TEST_PARTITION,
                    ),
                ]
            ),
            10,
            1,
            1,
        ),
        (
            Sinfo(
                nodes=[
                    SinfoNode(
                        name="node1",
                        state="allocated",
                        alloc_cpus=10,
                        gres_used="gpu:volta:1",
                        gres="gpu:volta:8(S:0-1)",
                        total_cpus=80,
                        partition=TEST_PARTITION,
                    ),
                    SinfoNode(
                        name="node2",
                        state="mixed",
                        alloc_cpus=5,
                        gres_used="gpu:volta:2",
                        gres="gpu:volta:8(S:0-1)",
                        total_cpus=80,
                        partition=TEST_PARTITION,
                    ),
                ]
            ),
            15,
            3,
            2,
        ),
        (
            Sinfo(
                nodes=[
                    SinfoNode(
                        name="node1",
                        state="draining",
                        alloc_cpus=10,
                        gres_used="gpu:volta:1",
                        gres="gpu:volta:8(S:0-1)",
                        total_cpus=80,
                        partition=TEST_PARTITION,
                    ),
                    SinfoNode(
                        name="node2",
                        state="mixed",
                        alloc_cpus=5,
                        gres_used="gpu:volta:2",
                        gres="gpu:volta:8(S:0-1)",
                        total_cpus=80,
                        partition=TEST_PARTITION,
                    ),
                ]
            ),
            15,
            3,
            2,
        ),
        (
            Sinfo(
                nodes=[
                    SinfoNode(
                        name="node1",
                        state="allocated",
                        alloc_cpus=10,
                        gres_used="gpu:volta:1",
                        gres="gpu:volta:8(S:0-1)",
                        total_cpus=80,
                        partition=TEST_PARTITION,
                    ),
                    SinfoNode(
                        name="node2",
                        state="mixed",
                        alloc_cpus=15,
                        gres_used="gpu:volta:0",
                        gres="gpu:volta:8(S:0-1)",
                        total_cpus=80,
                        partition=TEST_PARTITION,
                    ),
                ]
            ),
            25,
            1,
            2,
        ),
        (
            Sinfo(
                nodes=[
                    SinfoNode(
                        name="p4de-st-p4de24xlarge-745",
                        alloc_cpus=64,
                        gres_used="gpu:a100:8(IDX:0-7)",
                        state="mixed",
                        gres="gpu:volta:8(S:0-1)",
                        total_cpus=80,
                        partition=TEST_PARTITION,
                    ),
                    SinfoNode(
                        name="p4de-st-p4de24xlarge-755",
                        alloc_cpus=96,
                        gres_used="gpu:a100:8(IDX:0-7)",
                        state="allocated",
                        gres="gpu:volta:8(S:0-1)",
                        total_cpus=80,
                        partition=TEST_PARTITION,
                    ),
                ]
            ),
            160,
            16,
            2,
        ),
    ],
)
@typechecked
def test_compute_total_allocated_cpus_gpus(
    sinfo: Sinfo,
    expected_cpus: int,
    expected_gpus: int,
    expected_nodes: int,
) -> None:
    cpus, gpus, nodes = compute_total_allocated_cpus_gpus(sinfo)

    assert cpus == expected_cpus
    assert gpus == expected_gpus
    assert nodes == expected_nodes


@pytest.mark.parametrize(
    "sinfo, expected_total_cpus_gpus",
    [
        (
            Sinfo(nodes={}),
            SinfoCpusGpus(
                total_cpus_down=0,
                total_gpus_down=0,
                total_cpus_up=0,
                total_gpus_up=0,
                total_cpus_avail=0,
                total_gpus_avail=0,
            ),
        ),
        (
            Sinfo(
                nodes=[
                    SinfoNode(
                        name="node1",
                        state="allocated",
                        alloc_cpus=10,
                        gres_used="gpu:volta:1",
                        gres="gpu:volta:8(S:0-1)",
                        total_cpus=80,
                        partition=TEST_PARTITION,
                    )
                ]
            ),
            SinfoCpusGpus(
                total_cpus_down=0,
                total_gpus_down=0,
                total_cpus_up=80,
                total_gpus_up=8,
                total_cpus_avail=80,
                total_gpus_avail=8,
            ),
        ),
        (
            Sinfo(
                nodes=[
                    SinfoNode(
                        name="node1",
                        state="allocated",
                        alloc_cpus=10,
                        gres_used="gpu:volta:1",
                        gres="gpu:volta:8(S:0-1)",
                        total_cpus=80,
                        partition=TEST_PARTITION,
                    ),
                    SinfoNode(
                        name="node2",
                        state="idle",
                        alloc_cpus=0,
                        gres_used="gpu:volta:1",
                        gres="gpu:volta:8(S:0-1)",
                        total_cpus=80,
                        partition=TEST_PARTITION,
                    ),
                ]
            ),
            SinfoCpusGpus(
                total_cpus_down=0,
                total_gpus_down=0,
                total_cpus_up=160,
                total_gpus_up=16,
                total_cpus_avail=160,
                total_gpus_avail=16,
            ),
        ),
        (
            Sinfo(
                nodes=[
                    SinfoNode(
                        name="node1",
                        state="allocated",
                        alloc_cpus=10,
                        gres_used="gpu:volta:1",
                        gres="gpu:volta:8(S:0-1)",
                        total_cpus=80,
                        partition=TEST_PARTITION,
                    ),
                    SinfoNode(
                        name="node2",
                        state="mixed",
                        alloc_cpus=5,
                        gres_used="gpu:volta:2",
                        gres="gpu:volta:8(S:0-1)",
                        total_cpus=80,
                        partition=TEST_PARTITION,
                    ),
                ]
            ),
            SinfoCpusGpus(
                total_cpus_down=0,
                total_gpus_down=0,
                total_cpus_up=160,
                total_gpus_up=16,
                total_cpus_avail=160,
                total_gpus_avail=16,
            ),
        ),
        (
            Sinfo(
                nodes=[
                    SinfoNode(
                        name="node1",
                        state="draining",
                        alloc_cpus=10,
                        gres_used="gpu:volta:1",
                        gres="gpu:volta:8(S:0-1)",
                        total_cpus=80,
                        partition=TEST_PARTITION,
                    ),
                    SinfoNode(
                        name="node2",
                        state="mixed",
                        alloc_cpus=5,
                        gres_used="gpu:volta:2",
                        gres="gpu:volta:8(S:0-1)",
                        total_cpus=80,
                        partition=TEST_PARTITION,
                    ),
                ]
            ),
            SinfoCpusGpus(
                total_cpus_down=0,
                total_gpus_down=0,
                total_cpus_up=160,
                total_gpus_up=16,
                total_cpus_avail=160,
                total_gpus_avail=16,
            ),
        ),
        (
            Sinfo(
                nodes=[
                    SinfoNode(
                        name="node1",
                        state="drained",
                        alloc_cpus=10,
                        gres_used="gpu:volta:1",
                        gres="gpu:volta:8(S:0-1)",
                        total_cpus=80,
                        partition=TEST_PARTITION,
                    ),
                    SinfoNode(
                        name="node2",
                        state="drained",
                        alloc_cpus=5,
                        gres_used="gpu:volta:2",
                        gres="gpu:volta:8(S:0-1)",
                        total_cpus=80,
                        partition=TEST_PARTITION,
                    ),
                ]
            ),
            SinfoCpusGpus(
                total_cpus_down=160,
                total_gpus_down=16,
                total_cpus_up=0,
                total_gpus_up=0,
                total_cpus_avail=160,
                total_gpus_avail=16,
            ),
        ),
    ],
)
@typechecked
def test_compute_total_cpus_gpus_all_avail(
    sinfo: Sinfo,
    expected_total_cpus_gpus: SinfoCpusGpus,
) -> None:
    total_cpus_gpus = compute_total_cpus_gpus(sinfo)

    assert total_cpus_gpus == expected_total_cpus_gpus


@pytest.mark.parametrize(
    "sinfo, expected",
    [
        (
            Sinfo(nodes=[]),
            SinfoNodeStates(
                nodes_allocated=None,
                nodes_completing=None,
                nodes_down=None,
                nodes_drained=None,
                nodes_draining=None,
                nodes_fail=None,
                nodes_failing=None,
                nodes_future=None,
                nodes_idle=None,
                nodes_inval=None,
                nodes_maint=None,
                nodes_reboot_issued=None,
                nodes_reboot_requested=None,
                nodes_mixed=None,
                nodes_perfctrs=None,
                nodes_planned=None,
                nodes_power_down=None,
                nodes_powered_down=None,
                nodes_powering_down=None,
                nodes_powering_up=None,
                nodes_reserved=None,
                nodes_unknown=None,
                nodes_not_responding=None,
                nodes_unknown_state=None,
                nodes_total=None,
            ),
        ),
        (
            Sinfo(
                nodes=[
                    SinfoNode(
                        name="node1",
                        state="allocated",
                        alloc_cpus=10,
                        gres_used="gpu:volta:1",
                        gres="gpu:volta:8(S:0-1)",
                        total_cpus=80,
                        partition=TEST_PARTITION,
                    )
                ]
            ),
            SinfoNodeStates(
                nodes_allocated=1,
                nodes_completing=0,
                nodes_down=0,
                nodes_drained=0,
                nodes_draining=0,
                nodes_fail=0,
                nodes_failing=0,
                nodes_future=0,
                nodes_idle=0,
                nodes_inval=0,
                nodes_maint=0,
                nodes_reboot_issued=0,
                nodes_reboot_requested=0,
                nodes_mixed=0,
                nodes_perfctrs=0,
                nodes_planned=0,
                nodes_power_down=0,
                nodes_powered_down=0,
                nodes_powering_down=0,
                nodes_powering_up=0,
                nodes_reserved=0,
                nodes_unknown=0,
                nodes_not_responding=0,
                nodes_unknown_state=0,
                nodes_total=1,
            ),
        ),
        (
            Sinfo(
                nodes=[
                    SinfoNode(
                        name="node1",
                        state="allocated",
                        alloc_cpus=10,
                        gres_used="gpu:volta:1",
                        gres="gpu:volta:8(S:0-1)",
                        total_cpus=80,
                        partition=TEST_PARTITION,
                    ),
                    SinfoNode(
                        name="node2",
                        state="allocated",
                        alloc_cpus=10,
                        gres_used="gpu:volta:1",
                        gres="gpu:volta:8(S:0-1)",
                        total_cpus=80,
                        partition=TEST_PARTITION,
                    ),
                ]
            ),
            SinfoNodeStates(
                nodes_allocated=2,
                nodes_completing=0,
                nodes_down=0,
                nodes_drained=0,
                nodes_draining=0,
                nodes_fail=0,
                nodes_failing=0,
                nodes_future=0,
                nodes_idle=0,
                nodes_inval=0,
                nodes_maint=0,
                nodes_reboot_issued=0,
                nodes_reboot_requested=0,
                nodes_mixed=0,
                nodes_perfctrs=0,
                nodes_planned=0,
                nodes_power_down=0,
                nodes_powered_down=0,
                nodes_powering_down=0,
                nodes_powering_up=0,
                nodes_reserved=0,
                nodes_unknown=0,
                nodes_not_responding=0,
                nodes_unknown_state=0,
                nodes_total=2,
            ),
        ),
        (
            Sinfo(
                nodes=[
                    SinfoNode(
                        name="node1",
                        state="allocated*",
                        alloc_cpus=10,
                        gres_used="gpu:volta:1",
                        gres="gpu:volta:8(S:0-1)",
                        total_cpus=80,
                        partition=TEST_PARTITION,
                    ),
                    SinfoNode(
                        name="node2",
                        state="non_existing",
                        alloc_cpus=10,
                        gres_used="gpu:volta:1",
                        gres="gpu:volta:8(S:0-1)",
                        total_cpus=80,
                        partition=TEST_PARTITION,
                    ),
                ]
            ),
            SinfoNodeStates(
                nodes_allocated=1,
                nodes_completing=0,
                nodes_down=0,
                nodes_drained=0,
                nodes_draining=0,
                nodes_fail=0,
                nodes_failing=0,
                nodes_future=0,
                nodes_idle=0,
                nodes_inval=0,
                nodes_maint=0,
                nodes_reboot_issued=0,
                nodes_reboot_requested=0,
                nodes_mixed=0,
                nodes_perfctrs=0,
                nodes_planned=0,
                nodes_power_down=0,
                nodes_powered_down=0,
                nodes_powering_down=0,
                nodes_powering_up=0,
                nodes_reserved=0,
                nodes_unknown=0,
                nodes_not_responding=1,
                nodes_unknown_state=1,
                nodes_total=2,
            ),
        ),
    ],
)
@typechecked
def test_compute_node_states(
    sinfo: Sinfo,
    expected: SinfoNodeStates,
) -> None:
    node_states = compute_node_states(sinfo)

    assert node_states == expected


@pytest.mark.skipif(os.getenv("CI") is not None, reason="CI machines do not have SLURM")
@pytest.mark.skipif(shutil.which("sacct") is None, reason="Machine does not have sacct")
def test_cli(tmp_path: Path) -> None:
    fake_cluster: Final = "cluster_name"
    runner = CliRunner(mix_stderr=False)

    result = runner.invoke(
        main,
        [
            "--sink",
            "stdout",
            f"--log-folder={tmp_path}",
            "--once",
            "--cluster",
            fake_cluster,
            "--stdout",
        ],
    )
    assert result.exit_code == 0


@pytest.mark.parametrize(
    "sinfo, expected_cpus, expected_gpus, expected_nodes",
    [
        (
            Sinfo(
                nodes=[
                    SinfoNode(
                        name="shared_node",
                        state="allocated",
                        alloc_cpus=64,
                        gres_used="gpu:a100:8",
                        gres="gpu:a100:8",
                        total_cpus=128,
                        partition="partition1",
                    ),
                    SinfoNode(
                        name="shared_node",
                        state="allocated",
                        alloc_cpus=64,
                        gres_used="gpu:a100:8",
                        gres="gpu:a100:8",
                        total_cpus=128,
                        partition="partition2",
                    ),
                    SinfoNode(
                        name="unique_node",
                        state="allocated",
                        alloc_cpus=32,
                        gres_used="gpu:a100:4",
                        gres="gpu:a100:8",
                        total_cpus=64,
                        partition="partition1",
                    ),
                ]
            ),
            96,
            12,
            2,
        ),
        # Test with same node in 3 partitions
        (
            Sinfo(
                nodes=[
                    SinfoNode(
                        name="triple_shared_node",
                        state="mixed",
                        alloc_cpus=32,
                        gres_used="gpu:a100:2",
                        gres="gpu:a100:8",
                        total_cpus=128,
                        partition="part_a",
                    ),
                    SinfoNode(
                        name="triple_shared_node",
                        state="mixed",
                        alloc_cpus=32,
                        gres_used="gpu:a100:2",
                        gres="gpu:a100:8",
                        total_cpus=128,
                        partition="part_b",
                    ),
                    SinfoNode(
                        name="triple_shared_node",
                        state="mixed",
                        alloc_cpus=32,
                        gres_used="gpu:a100:2",
                        gres="gpu:a100:8",
                        total_cpus=128,
                        partition="part_c",
                    ),
                ]
            ),
            32,
            2,
            1,
        ),
    ],
)
@typechecked
def test_compute_total_allocated_cpus_gpus_deduplication(
    sinfo: Sinfo,
    expected_cpus: int,
    expected_gpus: int,
    expected_nodes: int,
) -> None:
    """Test that compute_total_allocated_cpus_gpus deduplicates nodes by name."""
    cpus, gpus, nodes = compute_total_allocated_cpus_gpus(sinfo)

    assert cpus == expected_cpus
    assert gpus == expected_gpus
    assert nodes == expected_nodes


@pytest.mark.parametrize(
    "sinfo, expected",
    [
        (
            Sinfo(
                nodes=[
                    SinfoNode(
                        name="shared_node",
                        state="idle",
                        alloc_cpus=0,
                        gres_used="gpu:a100:0",
                        gres="gpu:a100:8",
                        total_cpus=128,
                        partition="partition1",
                    ),
                    SinfoNode(
                        name="shared_node",
                        state="idle",
                        alloc_cpus=0,
                        gres_used="gpu:a100:0",
                        gres="gpu:a100:8",
                        total_cpus=128,
                        partition="partition2",
                    ),
                    SinfoNode(
                        name="allocated_node",
                        state="allocated",
                        alloc_cpus=64,
                        gres_used="gpu:a100:4",
                        gres="gpu:a100:8",
                        total_cpus=128,
                        partition="partition1",
                    ),
                ]
            ),
            SinfoNodeStates(
                nodes_allocated=1,
                nodes_completing=0,
                nodes_down=0,
                nodes_drained=0,
                nodes_draining=0,
                nodes_fail=0,
                nodes_failing=0,
                nodes_future=0,
                nodes_idle=1,
                nodes_inval=0,
                nodes_maint=0,
                nodes_reboot_issued=0,
                nodes_reboot_requested=0,
                nodes_mixed=0,
                nodes_perfctrs=0,
                nodes_planned=0,
                nodes_power_down=0,
                nodes_powered_down=0,
                nodes_powering_down=0,
                nodes_powering_up=0,
                nodes_reserved=0,
                nodes_unknown=0,
                nodes_not_responding=0,
                nodes_unknown_state=0,
                nodes_total=2,
            ),
        ),
        (
            Sinfo(
                nodes=[
                    SinfoNode(
                        name="down_node",
                        state="down",
                        alloc_cpus=0,
                        gres_used="gpu:a100:0",
                        gres="gpu:a100:8",
                        total_cpus=128,
                        partition="partition1",
                    ),
                    SinfoNode(
                        name="down_node",
                        state="down",
                        alloc_cpus=0,
                        gres_used="gpu:a100:0",
                        gres="gpu:a100:8",
                        total_cpus=128,
                        partition="partition2",
                    ),
                ]
            ),
            SinfoNodeStates(
                nodes_allocated=0,
                nodes_completing=0,
                nodes_down=1,
                nodes_drained=0,
                nodes_draining=0,
                nodes_fail=0,
                nodes_failing=0,
                nodes_future=0,
                nodes_idle=0,
                nodes_inval=0,
                nodes_maint=0,
                nodes_reboot_issued=0,
                nodes_reboot_requested=0,
                nodes_mixed=0,
                nodes_perfctrs=0,
                nodes_planned=0,
                nodes_power_down=0,
                nodes_powered_down=0,
                nodes_powering_down=0,
                nodes_powering_up=0,
                nodes_reserved=0,
                nodes_unknown=0,
                nodes_not_responding=0,
                nodes_unknown_state=0,
                nodes_total=1,
            ),
        ),
    ],
)
@typechecked
def test_compute_node_states_deduplication(
    sinfo: Sinfo,
    expected: SinfoNodeStates,
) -> None:
    """Test that compute_node_states deduplicates nodes by name."""
    node_states = compute_node_states(sinfo)

    assert node_states == expected


@pytest.mark.parametrize(
    "sinfo, expected_total_cpus_gpus",
    [
        (
            Sinfo(
                nodes=[
                    SinfoNode(
                        name="shared_node",
                        state="idle",
                        alloc_cpus=0,
                        gres_used="gpu:a100:0",
                        gres="gpu:a100:8",
                        total_cpus=128,
                        partition="partition1",
                    ),
                    SinfoNode(
                        name="shared_node",
                        state="idle",
                        alloc_cpus=0,
                        gres_used="gpu:a100:0",
                        gres="gpu:a100:8",
                        total_cpus=128,
                        partition="partition2",
                    ),
                    SinfoNode(
                        name="down_node",
                        state="down",
                        alloc_cpus=0,
                        gres_used="gpu:a100:0",
                        gres="gpu:a100:4",
                        total_cpus=64,
                        partition="partition1",
                    ),
                ]
            ),
            SinfoCpusGpus(
                total_cpus_down=64,
                total_gpus_down=4,
                total_cpus_up=128,
                total_gpus_up=8,
                total_cpus_avail=192,
                total_gpus_avail=12,
            ),
        ),
        (
            Sinfo(
                nodes=[
                    SinfoNode(
                        name="triple_down_node",
                        state="down",
                        alloc_cpus=0,
                        gres_used="gpu:a100:0",
                        gres="gpu:a100:8",
                        total_cpus=128,
                        partition="part_a",
                    ),
                    SinfoNode(
                        name="triple_down_node",
                        state="down",
                        alloc_cpus=0,
                        gres_used="gpu:a100:0",
                        gres="gpu:a100:8",
                        total_cpus=128,
                        partition="part_b",
                    ),
                    SinfoNode(
                        name="triple_down_node",
                        state="down",
                        alloc_cpus=0,
                        gres_used="gpu:a100:0",
                        gres="gpu:a100:8",
                        total_cpus=128,
                        partition="part_c",
                    ),
                ]
            ),
            SinfoCpusGpus(
                total_cpus_down=128,
                total_gpus_down=8,
                total_cpus_up=0,
                total_gpus_up=0,
                total_cpus_avail=128,
                total_gpus_avail=8,
            ),
        ),
    ],
)
@typechecked
def test_compute_total_cpus_gpus_deduplication(
    sinfo: Sinfo,
    expected_total_cpus_gpus: SinfoCpusGpus,
) -> None:
    """Test that compute_total_cpus_gpus deduplicates nodes by name."""
    result = compute_total_cpus_gpus(sinfo)

    assert result == expected_total_cpus_gpus


@pytest.mark.parametrize(
    "sinfo, expected_down_nodes",
    [
        (
            Sinfo(
                nodes=[
                    SinfoNode(
                        name="down_node",
                        state="down",
                        alloc_cpus=0,
                        gres_used="gpu:a100:0",
                        gres="gpu:a100:8",
                        total_cpus=128,
                        partition="partition1",
                    ),
                    SinfoNode(
                        name="down_node",
                        state="down",
                        alloc_cpus=0,
                        gres_used="gpu:a100:0",
                        gres="gpu:a100:8",
                        total_cpus=128,
                        partition="partition2",
                    ),
                    SinfoNode(
                        name="idle_node",
                        state="idle",
                        alloc_cpus=0,
                        gres_used="gpu:a100:0",
                        gres="gpu:a100:8",
                        total_cpus=128,
                        partition="partition1",
                    ),
                ]
            ),
            1,
        ),
        (
            Sinfo(
                nodes=[
                    SinfoNode(
                        name="triple_down_node",
                        state="down",
                        alloc_cpus=0,
                        gres_used="gpu:a100:0",
                        gres="gpu:a100:8",
                        total_cpus=128,
                        partition="part_a",
                    ),
                    SinfoNode(
                        name="triple_down_node",
                        state="down",
                        alloc_cpus=0,
                        gres_used="gpu:a100:0",
                        gres="gpu:a100:8",
                        total_cpus=128,
                        partition="part_b",
                    ),
                    SinfoNode(
                        name="triple_down_node",
                        state="down",
                        alloc_cpus=0,
                        gres_used="gpu:a100:0",
                        gres="gpu:a100:8",
                        total_cpus=128,
                        partition="part_c",
                    ),
                ]
            ),
            1,
        ),
        (
            Sinfo(
                nodes=[
                    SinfoNode(
                        name="down_node1",
                        state="down",
                        alloc_cpus=0,
                        gres_used="gpu:a100:0",
                        gres="gpu:a100:8",
                        total_cpus=128,
                        partition="partition1",
                    ),
                    SinfoNode(
                        name="down_node2",
                        state="down",
                        alloc_cpus=0,
                        gres_used="gpu:a100:0",
                        gres="gpu:a100:8",
                        total_cpus=128,
                        partition="partition2",
                    ),
                ]
            ),
            2,
        ),
        (
            Sinfo(
                nodes=[
                    SinfoNode(
                        name="idle_node",
                        state="idle",
                        alloc_cpus=0,
                        gres_used="gpu:a100:0",
                        gres="gpu:a100:8",
                        total_cpus=128,
                        partition="partition1",
                    ),
                ]
            ),
            0,
        ),
    ],
)
@typechecked
def test_compute_down_nodes_deduplication(
    sinfo: Sinfo,
    expected_down_nodes: int,
) -> None:
    """Test that compute_down_nodes deduplicates nodes by name."""
    down_nodes = compute_down_nodes(sinfo)

    assert down_nodes == expected_down_nodes
