{
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": {
          "type": "prometheus",
          "uid": "prometheus"
        },
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      }
    ]
  },
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 0,
  "id": 4,
  "links": [],
  "panels": [
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "cellOptions": {
              "mode": "gradient",
              "type": "color-background"
            },
            "filterable": false,
            "footer": {
              "reducers": []
            },
            "inspect": false,
            "minWidth": 60
          },
          "decimals": 0,
          "mappings": [
            {
              "options": {
                "from": 0,
                "result": {
                  "color": "blue",
                  "index": 0
                },
                "to": 50
              },
              "type": "range"
            },
            {
              "options": {
                "from": 50,
                "result": {
                  "color": "green",
                  "index": 1
                },
                "to": 70
              },
              "type": "range"
            },
            {
              "options": {
                "from": 70,
                "result": {
                  "color": "yellow",
                  "index": 2
                },
                "to": 85
              },
              "type": "range"
            },
            {
              "options": {
                "from": 85,
                "result": {
                  "color": "orange",
                  "index": 3
                },
                "to": 95
              },
              "type": "range"
            },
            {
              "options": {
                "from": 95,
                "result": {
                  "color": "red",
                  "index": 4
                },
                "to": 200
              },
              "type": "range"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "transparent",
                "value": 0
              }
            ]
          },
          "unit": "celsius"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Machine"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 96
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 9,
        "w": 8,
        "x": 0,
        "y": 0
      },
      "id": 8,
      "options": {
        "cellHeight": "sm",
        "frameIndex": 0,
        "showHeader": true,
        "sortBy": []
      },
      "pluginVersion": "12.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_HOT_SPOT_TEMP{gpu=\"0\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_HOT_SPOT_TEMP{gpu=\"1\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_HOT_SPOT_TEMP{gpu=\"2\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_HOT_SPOT_TEMP{gpu=\"3\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "D"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_HOT_SPOT_TEMP{gpu=\"4\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "E"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_HOT_SPOT_TEMP{gpu=\"5\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "F"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_HOT_SPOT_TEMP{gpu=\"6\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "G"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_HOT_SPOT_TEMP{gpu=\"7\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "H"
        }
      ],
      "title": "Machine GPU Core Hot Spot",
      "transformations": [
        {
          "id": "joinByField",
          "options": {
            "byField": "Hostname",
            "mode": "outer"
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "DCGM_FI_DRIVER_VERSION": true,
              "DCGM_FI_DRIVER_VERSION 1": true,
              "DCGM_FI_DRIVER_VERSION 2": true,
              "DCGM_FI_DRIVER_VERSION 3": true,
              "DCGM_FI_DRIVER_VERSION 4": true,
              "DCGM_FI_DRIVER_VERSION 5": true,
              "DCGM_FI_DRIVER_VERSION 6": true,
              "DCGM_FI_DRIVER_VERSION 7": true,
              "DCGM_FI_DRIVER_VERSION 8": true,
              "DCXP_FI_DRIVER_VERSION": true,
              "DCXP_FI_DRIVER_VERSION 1": true,
              "DCXP_FI_DRIVER_VERSION 2": true,
              "DCXP_FI_DRIVER_VERSION 3": true,
              "DCXP_FI_DRIVER_VERSION 4": true,
              "DCXP_FI_DRIVER_VERSION 5": true,
              "DCXP_FI_DRIVER_VERSION 6": true,
              "DCXP_FI_DRIVER_VERSION 7": true,
              "DCXP_FI_DRIVER_VERSION 8": true,
              "Hostname 1": true,
              "Hostname 1 1": true,
              "Hostname 1 2": true,
              "Hostname 1 3": true,
              "Hostname 1 4": true,
              "Hostname 1 5": true,
              "Hostname 1 6": true,
              "Hostname 1 7": true,
              "Hostname 1 8": true,
              "Hostname 2": true,
              "Hostname 2 1": true,
              "Hostname 2 2": true,
              "Hostname 2 3": true,
              "Hostname 2 4": true,
              "Hostname 2 5": true,
              "Hostname 2 6": true,
              "Hostname 2 7": true,
              "Hostname 2 8": true,
              "Hostname 3": true,
              "Hostname 3 1": true,
              "Hostname 3 2": true,
              "Hostname 3 3": true,
              "Hostname 3 4": true,
              "Hostname 3 5": true,
              "Hostname 3 6": true,
              "Hostname 3 7": true,
              "Hostname 3 8": true,
              "Hostname 4": true,
              "Hostname 4 1": true,
              "Hostname 4 2": true,
              "Hostname 4 3": true,
              "Hostname 4 4": true,
              "Hostname 4 5": true,
              "Hostname 4 6": true,
              "Hostname 4 7": true,
              "Hostname 4 8": true,
              "Hostname 5": true,
              "Hostname 5 1": true,
              "Hostname 5 2": true,
              "Hostname 5 3": true,
              "Hostname 5 4": true,
              "Hostname 5 5": true,
              "Hostname 5 6": true,
              "Hostname 5 7": true,
              "Hostname 5 8": true,
              "Hostname 6": true,
              "Hostname 6 1": true,
              "Hostname 6 2": true,
              "Hostname 6 3": true,
              "Hostname 6 4": true,
              "Hostname 6 5": true,
              "Hostname 6 6": true,
              "Hostname 6 7": true,
              "Hostname 6 8": true,
              "Hostname 7": true,
              "Hostname 7 1": true,
              "Hostname 7 2": true,
              "Hostname 7 3": true,
              "Hostname 7 4": true,
              "Hostname 7 5": true,
              "Hostname 7 6": true,
              "Hostname 7 7": true,
              "Hostname 7 8": true,
              "Hostname 8": true,
              "Hostname 8 1": true,
              "Hostname 8 2": true,
              "Hostname 8 3": true,
              "Hostname 8 4": true,
              "Hostname 8 5": true,
              "Hostname 8 6": true,
              "Hostname 8 7": true,
              "Hostname 8 8": true,
              "Time": true,
              "Time 1": true,
              "Time 2": true,
              "Time 3": true,
              "Time 4": true,
              "Time 5": true,
              "Time 6": true,
              "Time 7": true,
              "Time 8": true,
              "UUID": true,
              "UUID 1": true,
              "UUID 2": true,
              "UUID 3": true,
              "UUID 4": true,
              "UUID 5": true,
              "UUID 6": true,
              "UUID 7": true,
              "UUID 8": true,
              "__name__": true,
              "__name__ 1": true,
              "__name__ 2": true,
              "__name__ 3": true,
              "__name__ 4": true,
              "__name__ 5": true,
              "__name__ 6": true,
              "__name__ 7": true,
              "__name__ 8": true,
              "device": true,
              "device 1": true,
              "device 2": true,
              "device 3": true,
              "device 4": true,
              "device 5": true,
              "device 6": true,
              "device 7": true,
              "device 8": true,
              "gpu": true,
              "gpu 1": true,
              "gpu 2": true,
              "gpu 3": true,
              "gpu 4": true,
              "gpu 5": true,
              "gpu 6": true,
              "gpu 7": true,
              "gpu 8": true,
              "instance": true,
              "instance 1": true,
              "instance 2": true,
              "instance 3": true,
              "instance 4": true,
              "instance 5": true,
              "instance 6": true,
              "instance 7": true,
              "instance 8": true,
              "job": true,
              "job 1": true,
              "job 2": true,
              "job 3": true,
              "job 4": true,
              "job 5": true,
              "job 6": true,
              "job 7": true,
              "job 8": true,
              "modelName": true,
              "modelName 1": true,
              "modelName 2": true,
              "modelName 3": true,
              "modelName 4": true,
              "modelName 5": true,
              "modelName 6": true,
              "modelName 7": true,
              "modelName 8": true,
              "pci_bus_id": true,
              "pci_bus_id 1": true,
              "pci_bus_id 2": true,
              "pci_bus_id 3": true,
              "pci_bus_id 4": true,
              "pci_bus_id 5": true,
              "pci_bus_id 6": true,
              "pci_bus_id 7": true,
              "pci_bus_id 8": true,
              "reason": true,
              "reason 1": true,
              "reason 2": true,
              "reason 3": true,
              "reason 4": true,
              "reason 5": true,
              "reason 6": true,
              "reason 7": true,
              "reason 8": true,
              "source": true,
              "source 1": true,
              "source 2": true,
              "source 3": true,
              "source 4": true,
              "source 5": true,
              "source 6": true,
              "source 7": true,
              "source 8": true,
              "vm_name": true
            },
            "includeByName": {},
            "indexByName": {
              "GPU0": 1,
              "GPU1": 2,
              "GPU2": 3,
              "GPU3": 4,
              "GPU4": 5,
              "GPU5": 6,
              "GPU6": 7,
              "GPU7": 8,
              "Hostname": 0,
              "Machine": 0,
              "Value #A": 1,
              "Value #B": 2,
              "Value #C": 3,
              "Value #D": 4,
              "Value #E": 5,
              "Value #F": 6,
              "Value #G": 7,
              "Value #H": 8
            },
            "renameByName": {
              "Hostname": "Machine",
              "Value #A": "GPU0",
              "Value #B": "GPU1",
              "Value #C": "GPU2",
              "Value #D": "GPU3",
              "Value #E": "GPU4",
              "Value #F": "GPU5",
              "Value #G": "GPU6",
              "Value #H": "GPU7"
            }
          }
        },
        {
          "id": "sortBy",
          "options": {
            "fields": {},
            "sort": [
              {
                "field": "Machine"
              }
            ]
          }
        }
      ],
      "type": "table"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "cellOptions": {
              "mode": "gradient",
              "type": "color-background"
            },
            "filterable": false,
            "footer": {
              "reducers": []
            },
            "inspect": false,
            "minWidth": 60
          },
          "decimals": 0,
          "mappings": [
            {
              "options": {
                "from": 0,
                "result": {
                  "color": "blue",
                  "index": 0
                },
                "to": 50
              },
              "type": "range"
            },
            {
              "options": {
                "from": 50,
                "result": {
                  "color": "green",
                  "index": 1
                },
                "to": 70
              },
              "type": "range"
            },
            {
              "options": {
                "from": 70,
                "result": {
                  "color": "yellow",
                  "index": 2
                },
                "to": 90
              },
              "type": "range"
            },
            {
              "options": {
                "from": 90,
                "result": {
                  "color": "red",
                  "index": 3
                },
                "to": 100
              },
              "type": "range"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "transparent",
                "value": 0
              }
            ]
          },
          "unit": "%"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Machine"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 95
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 9,
        "w": 8,
        "x": 8,
        "y": 0
      },
      "id": 7,
      "options": {
        "cellHeight": "sm",
        "frameIndex": 0,
        "showHeader": true,
        "sortBy": []
      },
      "pluginVersion": "12.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_FAN_SPEED{gpu=\"0\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_FAN_SPEED{gpu=\"1\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_FAN_SPEED{gpu=\"2\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_FAN_SPEED{gpu=\"3\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "D"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_FAN_SPEED{gpu=\"4\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "E"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_FAN_SPEED{gpu=\"5\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "F"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_FAN_SPEED{gpu=\"6\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "G"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_FAN_SPEED{gpu=\"7\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "H"
        }
      ],
      "title": "Machine GPU FAN Speeds",
      "transformations": [
        {
          "id": "joinByField",
          "options": {
            "byField": "Hostname",
            "mode": "outer"
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "DCGM_FI_DRIVER_VERSION": true,
              "DCGM_FI_DRIVER_VERSION 1": true,
              "DCGM_FI_DRIVER_VERSION 2": true,
              "DCGM_FI_DRIVER_VERSION 3": true,
              "DCGM_FI_DRIVER_VERSION 4": true,
              "DCGM_FI_DRIVER_VERSION 5": true,
              "DCGM_FI_DRIVER_VERSION 6": true,
              "DCGM_FI_DRIVER_VERSION 7": true,
              "DCGM_FI_DRIVER_VERSION 8": true,
              "DCXP_FI_DRIVER_VERSION": true,
              "DCXP_FI_DRIVER_VERSION 1": true,
              "DCXP_FI_DRIVER_VERSION 2": true,
              "DCXP_FI_DRIVER_VERSION 3": true,
              "DCXP_FI_DRIVER_VERSION 4": true,
              "DCXP_FI_DRIVER_VERSION 5": true,
              "DCXP_FI_DRIVER_VERSION 6": true,
              "DCXP_FI_DRIVER_VERSION 7": true,
              "DCXP_FI_DRIVER_VERSION 8": true,
              "Hostname 1": true,
              "Hostname 1 1": true,
              "Hostname 1 2": true,
              "Hostname 1 3": true,
              "Hostname 1 4": true,
              "Hostname 1 5": true,
              "Hostname 1 6": true,
              "Hostname 1 7": true,
              "Hostname 1 8": true,
              "Hostname 2": true,
              "Hostname 2 1": true,
              "Hostname 2 2": true,
              "Hostname 2 3": true,
              "Hostname 2 4": true,
              "Hostname 2 5": true,
              "Hostname 2 6": true,
              "Hostname 2 7": true,
              "Hostname 2 8": true,
              "Hostname 3": true,
              "Hostname 3 1": true,
              "Hostname 3 2": true,
              "Hostname 3 3": true,
              "Hostname 3 4": true,
              "Hostname 3 5": true,
              "Hostname 3 6": true,
              "Hostname 3 7": true,
              "Hostname 3 8": true,
              "Hostname 4": true,
              "Hostname 4 1": true,
              "Hostname 4 2": true,
              "Hostname 4 3": true,
              "Hostname 4 4": true,
              "Hostname 4 5": true,
              "Hostname 4 6": true,
              "Hostname 4 7": true,
              "Hostname 4 8": true,
              "Hostname 5": true,
              "Hostname 5 1": true,
              "Hostname 5 2": true,
              "Hostname 5 3": true,
              "Hostname 5 4": true,
              "Hostname 5 5": true,
              "Hostname 5 6": true,
              "Hostname 5 7": true,
              "Hostname 5 8": true,
              "Hostname 6": true,
              "Hostname 6 1": true,
              "Hostname 6 2": true,
              "Hostname 6 3": true,
              "Hostname 6 4": true,
              "Hostname 6 5": true,
              "Hostname 6 6": true,
              "Hostname 6 7": true,
              "Hostname 6 8": true,
              "Hostname 7": true,
              "Hostname 7 1": true,
              "Hostname 7 2": true,
              "Hostname 7 3": true,
              "Hostname 7 4": true,
              "Hostname 7 5": true,
              "Hostname 7 6": true,
              "Hostname 7 7": true,
              "Hostname 7 8": true,
              "Hostname 8": true,
              "Hostname 8 1": true,
              "Hostname 8 2": true,
              "Hostname 8 3": true,
              "Hostname 8 4": true,
              "Hostname 8 5": true,
              "Hostname 8 6": true,
              "Hostname 8 7": true,
              "Hostname 8 8": true,
              "Time": true,
              "Time 1": true,
              "Time 2": true,
              "Time 3": true,
              "Time 4": true,
              "Time 5": true,
              "Time 6": true,
              "Time 7": true,
              "Time 8": true,
              "UUID": true,
              "UUID 1": true,
              "UUID 2": true,
              "UUID 3": true,
              "UUID 4": true,
              "UUID 5": true,
              "UUID 6": true,
              "UUID 7": true,
              "UUID 8": true,
              "__name__": true,
              "__name__ 1": true,
              "__name__ 2": true,
              "__name__ 3": true,
              "__name__ 4": true,
              "__name__ 5": true,
              "__name__ 6": true,
              "__name__ 7": true,
              "__name__ 8": true,
              "device": true,
              "device 1": true,
              "device 2": true,
              "device 3": true,
              "device 4": true,
              "device 5": true,
              "device 6": true,
              "device 7": true,
              "device 8": true,
              "gpu": true,
              "gpu 1": true,
              "gpu 2": true,
              "gpu 3": true,
              "gpu 4": true,
              "gpu 5": true,
              "gpu 6": true,
              "gpu 7": true,
              "gpu 8": true,
              "instance": true,
              "instance 1": true,
              "instance 2": true,
              "instance 3": true,
              "instance 4": true,
              "instance 5": true,
              "instance 6": true,
              "instance 7": true,
              "instance 8": true,
              "job": true,
              "job 1": true,
              "job 2": true,
              "job 3": true,
              "job 4": true,
              "job 5": true,
              "job 6": true,
              "job 7": true,
              "job 8": true,
              "modelName": true,
              "modelName 1": true,
              "modelName 2": true,
              "modelName 3": true,
              "modelName 4": true,
              "modelName 5": true,
              "modelName 6": true,
              "modelName 7": true,
              "modelName 8": true,
              "pci_bus_id": true,
              "pci_bus_id 1": true,
              "pci_bus_id 2": true,
              "pci_bus_id 3": true,
              "pci_bus_id 4": true,
              "pci_bus_id 5": true,
              "pci_bus_id 6": true,
              "pci_bus_id 7": true,
              "pci_bus_id 8": true,
              "reason": true,
              "reason 1": true,
              "reason 2": true,
              "reason 3": true,
              "reason 4": true,
              "reason 5": true,
              "reason 6": true,
              "reason 7": true,
              "reason 8": true,
              "source": true,
              "source 1": true,
              "source 2": true,
              "source 3": true,
              "source 4": true,
              "source 5": true,
              "source 6": true,
              "source 7": true,
              "source 8": true,
              "vm_name": true
            },
            "includeByName": {},
            "indexByName": {
              "GPU0": 1,
              "GPU1": 2,
              "GPU2": 3,
              "GPU3": 4,
              "GPU4": 5,
              "GPU5": 6,
              "GPU6": 7,
              "GPU7": 8,
              "Hostname": 0,
              "Machine": 0,
              "Value #A": 1,
              "Value #B": 2,
              "Value #C": 3,
              "Value #D": 4,
              "Value #E": 5,
              "Value #F": 6,
              "Value #G": 7,
              "Value #H": 8
            },
            "renameByName": {
              "Hostname": "Machine",
              "Value #A": "GPU0",
              "Value #B": "GPU1",
              "Value #C": "GPU2",
              "Value #D": "GPU3",
              "Value #E": "GPU4",
              "Value #F": "GPU5",
              "Value #G": "GPU6",
              "Value #H": "GPU7"
            }
          }
        },
        {
          "id": "sortBy",
          "options": {
            "fields": {},
            "sort": [
              {
                "field": "Machine"
              }
            ]
          }
        }
      ],
      "type": "table"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "cellOptions": {
              "mode": "gradient",
              "type": "color-background"
            },
            "filterable": false,
            "footer": {
              "reducers": []
            },
            "inspect": false,
            "minWidth": 60
          },
          "decimals": 0,
          "mappings": [
            {
              "options": {
                "from": 0,
                "result": {
                  "color": "blue",
                  "index": 0
                },
                "to": 50
              },
              "type": "range"
            },
            {
              "options": {
                "from": 50,
                "result": {
                  "color": "green",
                  "index": 1
                },
                "to": 200
              },
              "type": "range"
            },
            {
              "options": {
                "from": 200,
                "result": {
                  "color": "yellow",
                  "index": 2
                },
                "to": 300
              },
              "type": "range"
            },
            {
              "options": {
                "from": 300,
                "result": {
                  "color": "red",
                  "index": 3
                },
                "to": 500
              },
              "type": "range"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "transparent",
                "value": 0
              }
            ]
          },
          "unit": "watt"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Machine"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 98
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 9,
        "w": 8,
        "x": 16,
        "y": 0
      },
      "id": 6,
      "options": {
        "cellHeight": "sm",
        "frameIndex": 0,
        "showHeader": true,
        "sortBy": []
      },
      "pluginVersion": "12.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_POWER_USAGE{gpu=\"0\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_POWER_USAGE{gpu=\"1\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_POWER_USAGE{gpu=\"2\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_POWER_USAGE{gpu=\"3\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "D"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_POWER_USAGE{gpu=\"4\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "E"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_POWER_USAGE{gpu=\"5\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "F"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_POWER_USAGE{gpu=\"6\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "G"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_POWER_USAGE{gpu=\"7\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "H"
        }
      ],
      "title": "Machine GPU Power",
      "transformations": [
        {
          "id": "joinByField",
          "options": {
            "byField": "Hostname",
            "mode": "outer"
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "DCGM_FI_DRIVER_VERSION": true,
              "DCGM_FI_DRIVER_VERSION 1": true,
              "DCGM_FI_DRIVER_VERSION 2": true,
              "DCGM_FI_DRIVER_VERSION 3": true,
              "DCGM_FI_DRIVER_VERSION 4": true,
              "DCGM_FI_DRIVER_VERSION 5": true,
              "DCGM_FI_DRIVER_VERSION 6": true,
              "DCGM_FI_DRIVER_VERSION 7": true,
              "DCGM_FI_DRIVER_VERSION 8": true,
              "DCXP_FI_DRIVER_VERSION": true,
              "DCXP_FI_DRIVER_VERSION 1": true,
              "DCXP_FI_DRIVER_VERSION 2": true,
              "DCXP_FI_DRIVER_VERSION 3": true,
              "DCXP_FI_DRIVER_VERSION 4": true,
              "DCXP_FI_DRIVER_VERSION 5": true,
              "DCXP_FI_DRIVER_VERSION 6": true,
              "DCXP_FI_DRIVER_VERSION 7": true,
              "DCXP_FI_DRIVER_VERSION 8": true,
              "Hostname 1": true,
              "Hostname 1 1": true,
              "Hostname 1 2": true,
              "Hostname 1 3": true,
              "Hostname 1 4": true,
              "Hostname 1 5": true,
              "Hostname 1 6": true,
              "Hostname 1 7": true,
              "Hostname 1 8": true,
              "Hostname 2": true,
              "Hostname 2 1": true,
              "Hostname 2 2": true,
              "Hostname 2 3": true,
              "Hostname 2 4": true,
              "Hostname 2 5": true,
              "Hostname 2 6": true,
              "Hostname 2 7": true,
              "Hostname 2 8": true,
              "Hostname 3": true,
              "Hostname 3 1": true,
              "Hostname 3 2": true,
              "Hostname 3 3": true,
              "Hostname 3 4": true,
              "Hostname 3 5": true,
              "Hostname 3 6": true,
              "Hostname 3 7": true,
              "Hostname 3 8": true,
              "Hostname 4": true,
              "Hostname 4 1": true,
              "Hostname 4 2": true,
              "Hostname 4 3": true,
              "Hostname 4 4": true,
              "Hostname 4 5": true,
              "Hostname 4 6": true,
              "Hostname 4 7": true,
              "Hostname 4 8": true,
              "Hostname 5": true,
              "Hostname 5 1": true,
              "Hostname 5 2": true,
              "Hostname 5 3": true,
              "Hostname 5 4": true,
              "Hostname 5 5": true,
              "Hostname 5 6": true,
              "Hostname 5 7": true,
              "Hostname 5 8": true,
              "Hostname 6": true,
              "Hostname 6 1": true,
              "Hostname 6 2": true,
              "Hostname 6 3": true,
              "Hostname 6 4": true,
              "Hostname 6 5": true,
              "Hostname 6 6": true,
              "Hostname 6 7": true,
              "Hostname 6 8": true,
              "Hostname 7": true,
              "Hostname 7 1": true,
              "Hostname 7 2": true,
              "Hostname 7 3": true,
              "Hostname 7 4": true,
              "Hostname 7 5": true,
              "Hostname 7 6": true,
              "Hostname 7 7": true,
              "Hostname 7 8": true,
              "Hostname 8": true,
              "Hostname 8 1": true,
              "Hostname 8 2": true,
              "Hostname 8 3": true,
              "Hostname 8 4": true,
              "Hostname 8 5": true,
              "Hostname 8 6": true,
              "Hostname 8 7": true,
              "Hostname 8 8": true,
              "Time": true,
              "Time 1": true,
              "Time 2": true,
              "Time 3": true,
              "Time 4": true,
              "Time 5": true,
              "Time 6": true,
              "Time 7": true,
              "Time 8": true,
              "UUID": true,
              "UUID 1": true,
              "UUID 2": true,
              "UUID 3": true,
              "UUID 4": true,
              "UUID 5": true,
              "UUID 6": true,
              "UUID 7": true,
              "UUID 8": true,
              "__name__": true,
              "__name__ 1": true,
              "__name__ 2": true,
              "__name__ 3": true,
              "__name__ 4": true,
              "__name__ 5": true,
              "__name__ 6": true,
              "__name__ 7": true,
              "__name__ 8": true,
              "device": true,
              "device 1": true,
              "device 2": true,
              "device 3": true,
              "device 4": true,
              "device 5": true,
              "device 6": true,
              "device 7": true,
              "device 8": true,
              "gpu": true,
              "gpu 1": true,
              "gpu 2": true,
              "gpu 3": true,
              "gpu 4": true,
              "gpu 5": true,
              "gpu 6": true,
              "gpu 7": true,
              "gpu 8": true,
              "instance": true,
              "instance 1": true,
              "instance 2": true,
              "instance 3": true,
              "instance 4": true,
              "instance 5": true,
              "instance 6": true,
              "instance 7": true,
              "instance 8": true,
              "job": true,
              "job 1": true,
              "job 2": true,
              "job 3": true,
              "job 4": true,
              "job 5": true,
              "job 6": true,
              "job 7": true,
              "job 8": true,
              "modelName": true,
              "modelName 1": true,
              "modelName 2": true,
              "modelName 3": true,
              "modelName 4": true,
              "modelName 5": true,
              "modelName 6": true,
              "modelName 7": true,
              "modelName 8": true,
              "pci_bus_id": true,
              "pci_bus_id 1": true,
              "pci_bus_id 2": true,
              "pci_bus_id 3": true,
              "pci_bus_id 4": true,
              "pci_bus_id 5": true,
              "pci_bus_id 6": true,
              "pci_bus_id 7": true,
              "pci_bus_id 8": true,
              "reason": true,
              "reason 1": true,
              "reason 2": true,
              "reason 3": true,
              "reason 4": true,
              "reason 5": true,
              "reason 6": true,
              "reason 7": true,
              "reason 8": true,
              "source": true,
              "source 1": true,
              "source 2": true,
              "source 3": true,
              "source 4": true,
              "source 5": true,
              "source 6": true,
              "source 7": true,
              "source 8": true,
              "vm_name": true
            },
            "includeByName": {},
            "indexByName": {
              "GPU0": 1,
              "GPU1": 2,
              "GPU2": 3,
              "GPU3": 4,
              "GPU4": 5,
              "GPU5": 6,
              "GPU6": 7,
              "GPU7": 8,
              "Hostname": 0,
              "Machine": 0,
              "Value #A": 1,
              "Value #B": 2,
              "Value #C": 3,
              "Value #D": 4,
              "Value #E": 5,
              "Value #F": 6,
              "Value #G": 7,
              "Value #H": 8
            },
            "renameByName": {
              "Hostname": "Machine",
              "Value #A": "GPU0",
              "Value #B": "GPU1",
              "Value #C": "GPU2",
              "Value #D": "GPU3",
              "Value #E": "GPU4",
              "Value #F": "GPU5",
              "Value #G": "GPU6",
              "Value #H": "GPU7"
            }
          }
        },
        {
          "id": "sortBy",
          "options": {
            "fields": {},
            "sort": [
              {
                "field": "Machine"
              }
            ]
          }
        }
      ],
      "type": "table"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "cellOptions": {
              "mode": "gradient",
              "type": "color-background"
            },
            "filterable": false,
            "footer": {
              "reducers": []
            },
            "inspect": false,
            "minWidth": 60
          },
          "decimals": 0,
          "mappings": [
            {
              "options": {
                "from": 0,
                "result": {
                  "color": "blue",
                  "index": 0
                },
                "to": 50
              },
              "type": "range"
            },
            {
              "options": {
                "from": 50,
                "result": {
                  "color": "green",
                  "index": 1
                },
                "to": 70
              },
              "type": "range"
            },
            {
              "options": {
                "from": 70,
                "result": {
                  "color": "yellow",
                  "index": 2
                },
                "to": 80
              },
              "type": "range"
            },
            {
              "options": {
                "from": 80,
                "result": {
                  "color": "red",
                  "index": 3
                },
                "to": 100
              },
              "type": "range"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "transparent",
                "value": 0
              }
            ]
          },
          "unit": "celsius"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Machine"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 100
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 9,
        "w": 8,
        "x": 0,
        "y": 9
      },
      "id": 2,
      "options": {
        "cellHeight": "sm",
        "frameIndex": 0,
        "showHeader": true,
        "sortBy": []
      },
      "pluginVersion": "12.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_GPU_TEMP{gpu=\"0\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_GPU_TEMP{gpu=\"1\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_GPU_TEMP{gpu=\"2\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_GPU_TEMP{gpu=\"3\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "D"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_GPU_TEMP{gpu=\"4\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "E"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_GPU_TEMP{gpu=\"5\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "F"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_GPU_TEMP{gpu=\"6\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "G"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_GPU_TEMP{gpu=\"7\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "H"
        }
      ],
      "title": "Machine GPU Temps",
      "transformations": [
        {
          "id": "joinByField",
          "options": {
            "byField": "Hostname",
            "mode": "outer"
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "DCGM_FI_DRIVER_VERSION": true,
              "DCGM_FI_DRIVER_VERSION 1": true,
              "DCGM_FI_DRIVER_VERSION 2": true,
              "DCGM_FI_DRIVER_VERSION 3": true,
              "DCGM_FI_DRIVER_VERSION 4": true,
              "DCGM_FI_DRIVER_VERSION 5": true,
              "DCGM_FI_DRIVER_VERSION 6": true,
              "DCGM_FI_DRIVER_VERSION 7": true,
              "DCGM_FI_DRIVER_VERSION 8": true,
              "DCXP_FI_DRIVER_VERSION": true,
              "DCXP_FI_DRIVER_VERSION 1": true,
              "DCXP_FI_DRIVER_VERSION 2": true,
              "DCXP_FI_DRIVER_VERSION 3": true,
              "DCXP_FI_DRIVER_VERSION 4": true,
              "DCXP_FI_DRIVER_VERSION 5": true,
              "DCXP_FI_DRIVER_VERSION 6": true,
              "DCXP_FI_DRIVER_VERSION 7": true,
              "DCXP_FI_DRIVER_VERSION 8": true,
              "Hostname 1": true,
              "Hostname 1 1": true,
              "Hostname 1 2": true,
              "Hostname 1 3": true,
              "Hostname 1 4": true,
              "Hostname 1 5": true,
              "Hostname 1 6": true,
              "Hostname 1 7": true,
              "Hostname 1 8": true,
              "Hostname 2": true,
              "Hostname 2 1": true,
              "Hostname 2 2": true,
              "Hostname 2 3": true,
              "Hostname 2 4": true,
              "Hostname 2 5": true,
              "Hostname 2 6": true,
              "Hostname 2 7": true,
              "Hostname 2 8": true,
              "Hostname 3": true,
              "Hostname 3 1": true,
              "Hostname 3 2": true,
              "Hostname 3 3": true,
              "Hostname 3 4": true,
              "Hostname 3 5": true,
              "Hostname 3 6": true,
              "Hostname 3 7": true,
              "Hostname 3 8": true,
              "Hostname 4": true,
              "Hostname 4 1": true,
              "Hostname 4 2": true,
              "Hostname 4 3": true,
              "Hostname 4 4": true,
              "Hostname 4 5": true,
              "Hostname 4 6": true,
              "Hostname 4 7": true,
              "Hostname 4 8": true,
              "Hostname 5": true,
              "Hostname 5 1": true,
              "Hostname 5 2": true,
              "Hostname 5 3": true,
              "Hostname 5 4": true,
              "Hostname 5 5": true,
              "Hostname 5 6": true,
              "Hostname 5 7": true,
              "Hostname 5 8": true,
              "Hostname 6": true,
              "Hostname 6 1": true,
              "Hostname 6 2": true,
              "Hostname 6 3": true,
              "Hostname 6 4": true,
              "Hostname 6 5": true,
              "Hostname 6 6": true,
              "Hostname 6 7": true,
              "Hostname 6 8": true,
              "Hostname 7": true,
              "Hostname 7 1": true,
              "Hostname 7 2": true,
              "Hostname 7 3": true,
              "Hostname 7 4": true,
              "Hostname 7 5": true,
              "Hostname 7 6": true,
              "Hostname 7 7": true,
              "Hostname 7 8": true,
              "Hostname 8": true,
              "Hostname 8 1": true,
              "Hostname 8 2": true,
              "Hostname 8 3": true,
              "Hostname 8 4": true,
              "Hostname 8 5": true,
              "Hostname 8 6": true,
              "Hostname 8 7": true,
              "Hostname 8 8": true,
              "Time": true,
              "Time 1": true,
              "Time 2": true,
              "Time 3": true,
              "Time 4": true,
              "Time 5": true,
              "Time 6": true,
              "Time 7": true,
              "Time 8": true,
              "UUID": true,
              "UUID 1": true,
              "UUID 2": true,
              "UUID 3": true,
              "UUID 4": true,
              "UUID 5": true,
              "UUID 6": true,
              "UUID 7": true,
              "UUID 8": true,
              "__name__": true,
              "__name__ 1": true,
              "__name__ 2": true,
              "__name__ 3": true,
              "__name__ 4": true,
              "__name__ 5": true,
              "__name__ 6": true,
              "__name__ 7": true,
              "__name__ 8": true,
              "device": true,
              "device 1": true,
              "device 2": true,
              "device 3": true,
              "device 4": true,
              "device 5": true,
              "device 6": true,
              "device 7": true,
              "device 8": true,
              "gpu": true,
              "gpu 1": true,
              "gpu 2": true,
              "gpu 3": true,
              "gpu 4": true,
              "gpu 5": true,
              "gpu 6": true,
              "gpu 7": true,
              "gpu 8": true,
              "instance": true,
              "instance 1": true,
              "instance 2": true,
              "instance 3": true,
              "instance 4": true,
              "instance 5": true,
              "instance 6": true,
              "instance 7": true,
              "instance 8": true,
              "job": true,
              "job 1": true,
              "job 2": true,
              "job 3": true,
              "job 4": true,
              "job 5": true,
              "job 6": true,
              "job 7": true,
              "job 8": true,
              "modelName": true,
              "modelName 1": true,
              "modelName 2": true,
              "modelName 3": true,
              "modelName 4": true,
              "modelName 5": true,
              "modelName 6": true,
              "modelName 7": true,
              "modelName 8": true,
              "pci_bus_id": true,
              "pci_bus_id 1": true,
              "pci_bus_id 2": true,
              "pci_bus_id 3": true,
              "pci_bus_id 4": true,
              "pci_bus_id 5": true,
              "pci_bus_id 6": true,
              "pci_bus_id 7": true,
              "pci_bus_id 8": true,
              "reason": true,
              "reason 1": true,
              "reason 2": true,
              "reason 3": true,
              "reason 4": true,
              "reason 5": true,
              "reason 6": true,
              "reason 7": true,
              "reason 8": true,
              "source": true,
              "source 1": true,
              "source 2": true,
              "source 3": true,
              "source 4": true,
              "source 5": true,
              "source 6": true,
              "source 7": true,
              "source 8": true,
              "vm_name": true
            },
            "includeByName": {},
            "indexByName": {
              "GPU0": 1,
              "GPU1": 2,
              "GPU2": 3,
              "GPU3": 4,
              "GPU4": 5,
              "GPU5": 6,
              "GPU6": 7,
              "GPU7": 8,
              "Hostname": 0,
              "Machine": 0,
              "Value #A": 1,
              "Value #B": 2,
              "Value #C": 3,
              "Value #D": 4,
              "Value #E": 5,
              "Value #F": 6,
              "Value #G": 7,
              "Value #H": 8
            },
            "renameByName": {
              "Hostname": "Machine",
              "Value #A": "GPU0",
              "Value #B": "GPU1",
              "Value #C": "GPU2",
              "Value #D": "GPU3",
              "Value #E": "GPU4",
              "Value #F": "GPU5",
              "Value #G": "GPU6",
              "Value #H": "GPU7"
            }
          }
        },
        {
          "id": "sortBy",
          "options": {
            "fields": {},
            "sort": [
              {
                "field": "Machine"
              }
            ]
          }
        }
      ],
      "type": "table"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "cellOptions": {
              "mode": "gradient",
              "type": "color-background"
            },
            "filterable": false,
            "footer": {
              "reducers": []
            },
            "inspect": false,
            "minWidth": 60
          },
          "decimals": 0,
          "mappings": [
            {
              "options": {
                "from": 0,
                "result": {
                  "color": "blue",
                  "index": 0
                },
                "to": 50
              },
              "type": "range"
            },
            {
              "options": {
                "from": 50,
                "result": {
                  "color": "green",
                  "index": 1
                },
                "to": 70
              },
              "type": "range"
            },
            {
              "options": {
                "from": 70,
                "result": {
                  "color": "yellow",
                  "index": 2
                },
                "to": 90
              },
              "type": "range"
            },
            {
              "options": {
                "from": 90,
                "result": {
                  "color": "red",
                  "index": 3
                },
                "to": 150
              },
              "type": "range"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "transparent",
                "value": 0
              }
            ]
          },
          "unit": "celsius"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Machine"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 104
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 9,
        "w": 8,
        "x": 8,
        "y": 9
      },
      "id": 5,
      "options": {
        "cellHeight": "sm",
        "frameIndex": 0,
        "showHeader": true,
        "sortBy": []
      },
      "pluginVersion": "12.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_VRAM_TEMP{gpu=\"0\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_VRAM_TEMP{gpu=\"1\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_VRAM_TEMP{gpu=\"2\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_VRAM_TEMP{gpu=\"3\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "D"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_VRAM_TEMP{gpu=\"4\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "E"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_VRAM_TEMP{gpu=\"5\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "F"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_VRAM_TEMP{gpu=\"6\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "G"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_VRAM_TEMP{gpu=\"7\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "H"
        }
      ],
      "title": "Machine GPU VRAM Temps",
      "transformations": [
        {
          "id": "joinByField",
          "options": {
            "byField": "Hostname",
            "mode": "outer"
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "DCGM_FI_DRIVER_VERSION": true,
              "DCGM_FI_DRIVER_VERSION 1": true,
              "DCGM_FI_DRIVER_VERSION 2": true,
              "DCGM_FI_DRIVER_VERSION 3": true,
              "DCGM_FI_DRIVER_VERSION 4": true,
              "DCGM_FI_DRIVER_VERSION 5": true,
              "DCGM_FI_DRIVER_VERSION 6": true,
              "DCGM_FI_DRIVER_VERSION 7": true,
              "DCGM_FI_DRIVER_VERSION 8": true,
              "DCXP_FI_DRIVER_VERSION": true,
              "DCXP_FI_DRIVER_VERSION 1": true,
              "DCXP_FI_DRIVER_VERSION 2": true,
              "DCXP_FI_DRIVER_VERSION 3": true,
              "DCXP_FI_DRIVER_VERSION 4": true,
              "DCXP_FI_DRIVER_VERSION 5": true,
              "DCXP_FI_DRIVER_VERSION 6": true,
              "DCXP_FI_DRIVER_VERSION 7": true,
              "DCXP_FI_DRIVER_VERSION 8": true,
              "Hostname 1": true,
              "Hostname 1 1": true,
              "Hostname 1 2": true,
              "Hostname 1 3": true,
              "Hostname 1 4": true,
              "Hostname 1 5": true,
              "Hostname 1 6": true,
              "Hostname 1 7": true,
              "Hostname 1 8": true,
              "Hostname 2": true,
              "Hostname 2 1": true,
              "Hostname 2 2": true,
              "Hostname 2 3": true,
              "Hostname 2 4": true,
              "Hostname 2 5": true,
              "Hostname 2 6": true,
              "Hostname 2 7": true,
              "Hostname 2 8": true,
              "Hostname 3": true,
              "Hostname 3 1": true,
              "Hostname 3 2": true,
              "Hostname 3 3": true,
              "Hostname 3 4": true,
              "Hostname 3 5": true,
              "Hostname 3 6": true,
              "Hostname 3 7": true,
              "Hostname 3 8": true,
              "Hostname 4": true,
              "Hostname 4 1": true,
              "Hostname 4 2": true,
              "Hostname 4 3": true,
              "Hostname 4 4": true,
              "Hostname 4 5": true,
              "Hostname 4 6": true,
              "Hostname 4 7": true,
              "Hostname 4 8": true,
              "Hostname 5": true,
              "Hostname 5 1": true,
              "Hostname 5 2": true,
              "Hostname 5 3": true,
              "Hostname 5 4": true,
              "Hostname 5 5": true,
              "Hostname 5 6": true,
              "Hostname 5 7": true,
              "Hostname 5 8": true,
              "Hostname 6": true,
              "Hostname 6 1": true,
              "Hostname 6 2": true,
              "Hostname 6 3": true,
              "Hostname 6 4": true,
              "Hostname 6 5": true,
              "Hostname 6 6": true,
              "Hostname 6 7": true,
              "Hostname 6 8": true,
              "Hostname 7": true,
              "Hostname 7 1": true,
              "Hostname 7 2": true,
              "Hostname 7 3": true,
              "Hostname 7 4": true,
              "Hostname 7 5": true,
              "Hostname 7 6": true,
              "Hostname 7 7": true,
              "Hostname 7 8": true,
              "Hostname 8": true,
              "Hostname 8 1": true,
              "Hostname 8 2": true,
              "Hostname 8 3": true,
              "Hostname 8 4": true,
              "Hostname 8 5": true,
              "Hostname 8 6": true,
              "Hostname 8 7": true,
              "Hostname 8 8": true,
              "Time": true,
              "Time 1": true,
              "Time 2": true,
              "Time 3": true,
              "Time 4": true,
              "Time 5": true,
              "Time 6": true,
              "Time 7": true,
              "Time 8": true,
              "UUID": true,
              "UUID 1": true,
              "UUID 2": true,
              "UUID 3": true,
              "UUID 4": true,
              "UUID 5": true,
              "UUID 6": true,
              "UUID 7": true,
              "UUID 8": true,
              "__name__": true,
              "__name__ 1": true,
              "__name__ 2": true,
              "__name__ 3": true,
              "__name__ 4": true,
              "__name__ 5": true,
              "__name__ 6": true,
              "__name__ 7": true,
              "__name__ 8": true,
              "device": true,
              "device 1": true,
              "device 2": true,
              "device 3": true,
              "device 4": true,
              "device 5": true,
              "device 6": true,
              "device 7": true,
              "device 8": true,
              "gpu": true,
              "gpu 1": true,
              "gpu 2": true,
              "gpu 3": true,
              "gpu 4": true,
              "gpu 5": true,
              "gpu 6": true,
              "gpu 7": true,
              "gpu 8": true,
              "instance": true,
              "instance 1": true,
              "instance 2": true,
              "instance 3": true,
              "instance 4": true,
              "instance 5": true,
              "instance 6": true,
              "instance 7": true,
              "instance 8": true,
              "job": true,
              "job 1": true,
              "job 2": true,
              "job 3": true,
              "job 4": true,
              "job 5": true,
              "job 6": true,
              "job 7": true,
              "job 8": true,
              "modelName": true,
              "modelName 1": true,
              "modelName 2": true,
              "modelName 3": true,
              "modelName 4": true,
              "modelName 5": true,
              "modelName 6": true,
              "modelName 7": true,
              "modelName 8": true,
              "pci_bus_id": true,
              "pci_bus_id 1": true,
              "pci_bus_id 2": true,
              "pci_bus_id 3": true,
              "pci_bus_id 4": true,
              "pci_bus_id 5": true,
              "pci_bus_id 6": true,
              "pci_bus_id 7": true,
              "pci_bus_id 8": true,
              "reason": true,
              "reason 1": true,
              "reason 2": true,
              "reason 3": true,
              "reason 4": true,
              "reason 5": true,
              "reason 6": true,
              "reason 7": true,
              "reason 8": true,
              "source": true,
              "source 1": true,
              "source 2": true,
              "source 3": true,
              "source 4": true,
              "source 5": true,
              "source 6": true,
              "source 7": true,
              "source 8": true,
              "vm_name": true
            },
            "includeByName": {},
            "indexByName": {
              "GPU0": 1,
              "GPU1": 2,
              "GPU2": 3,
              "GPU3": 4,
              "GPU4": 5,
              "GPU5": 6,
              "GPU6": 7,
              "GPU7": 8,
              "Hostname": 0,
              "Machine": 0,
              "Value #A": 1,
              "Value #B": 2,
              "Value #C": 3,
              "Value #D": 4,
              "Value #E": 5,
              "Value #F": 6,
              "Value #G": 7,
              "Value #H": 8
            },
            "renameByName": {
              "Hostname": "Machine",
              "Value #A": "GPU0",
              "Value #B": "GPU1",
              "Value #C": "GPU2",
              "Value #D": "GPU3",
              "Value #E": "GPU4",
              "Value #F": "GPU5",
              "Value #G": "GPU6",
              "Value #H": "GPU7"
            }
          }
        },
        {
          "id": "sortBy",
          "options": {
            "fields": {},
            "sort": [
              {
                "field": "Machine"
              }
            ]
          }
        }
      ],
      "type": "table"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "cellOptions": {
              "mode": "gradient",
              "type": "color-background"
            },
            "filterable": false,
            "footer": {
              "reducers": []
            },
            "inspect": false,
            "minWidth": 60
          },
          "decimals": 0,
          "mappings": [
            {
              "options": {
                "from": 0,
                "result": {
                  "color": "blue",
                  "index": 0
                },
                "to": 0
              },
              "type": "range"
            },
            {
              "options": {
                "from": 1,
                "result": {
                  "color": "red",
                  "index": 1
                },
                "to": 100
              },
              "type": "range"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "transparent",
                "value": 0
              }
            ]
          },
          "unit": "bool"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Machine"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 99
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 9,
        "w": 8,
        "x": 16,
        "y": 9
      },
      "id": 9,
      "options": {
        "cellHeight": "sm",
        "frameIndex": 0,
        "showHeader": true,
        "sortBy": []
      },
      "pluginVersion": "12.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_CLOCKS_THROTTLE_REASON{reason=\"SwThermalSlowdown\", gpu=\"0\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_CLOCKS_THROTTLE_REASON{reason=\"SwThermalSlowdown\", gpu=\"1\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_CLOCKS_THROTTLE_REASON{reason=\"SwThermalSlowdown\", gpu=\"2\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_CLOCKS_THROTTLE_REASON{reason=\"SwThermalSlowdown\", gpu=\"3\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "D"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_CLOCKS_THROTTLE_REASON{reason=\"SwThermalSlowdown\", gpu=\"4\"} ",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "E"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_CLOCKS_THROTTLE_REASON{reason=\"SwThermalSlowdown\", gpu=\"5\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "F"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_CLOCKS_THROTTLE_REASON{reason=\"SwThermalSlowdown\", gpu=\"6\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "G"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_CLOCKS_THROTTLE_REASON{reason=\"SwThermalSlowdown\", gpu=\"7\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "H"
        }
      ],
      "title": "Machine GPU Thermal Throttle",
      "transformations": [
        {
          "id": "joinByField",
          "options": {
            "byField": "Hostname",
            "mode": "outer"
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "DCGM_FI_DRIVER_VERSION": true,
              "DCGM_FI_DRIVER_VERSION 1": true,
              "DCGM_FI_DRIVER_VERSION 2": true,
              "DCGM_FI_DRIVER_VERSION 3": true,
              "DCGM_FI_DRIVER_VERSION 4": true,
              "DCGM_FI_DRIVER_VERSION 5": true,
              "DCGM_FI_DRIVER_VERSION 6": true,
              "DCGM_FI_DRIVER_VERSION 7": true,
              "DCGM_FI_DRIVER_VERSION 8": true,
              "DCXP_FI_DRIVER_VERSION": true,
              "DCXP_FI_DRIVER_VERSION 1": true,
              "DCXP_FI_DRIVER_VERSION 2": true,
              "DCXP_FI_DRIVER_VERSION 3": true,
              "DCXP_FI_DRIVER_VERSION 4": true,
              "DCXP_FI_DRIVER_VERSION 5": true,
              "DCXP_FI_DRIVER_VERSION 6": true,
              "DCXP_FI_DRIVER_VERSION 7": true,
              "DCXP_FI_DRIVER_VERSION 8": true,
              "Hostname 1": true,
              "Hostname 1 1": true,
              "Hostname 1 2": true,
              "Hostname 1 3": true,
              "Hostname 1 4": true,
              "Hostname 1 5": true,
              "Hostname 1 6": true,
              "Hostname 1 7": true,
              "Hostname 1 8": true,
              "Hostname 2": true,
              "Hostname 2 1": true,
              "Hostname 2 2": true,
              "Hostname 2 3": true,
              "Hostname 2 4": true,
              "Hostname 2 5": true,
              "Hostname 2 6": true,
              "Hostname 2 7": true,
              "Hostname 2 8": true,
              "Hostname 3": true,
              "Hostname 3 1": true,
              "Hostname 3 2": true,
              "Hostname 3 3": true,
              "Hostname 3 4": true,
              "Hostname 3 5": true,
              "Hostname 3 6": true,
              "Hostname 3 7": true,
              "Hostname 3 8": true,
              "Hostname 4": true,
              "Hostname 4 1": true,
              "Hostname 4 2": true,
              "Hostname 4 3": true,
              "Hostname 4 4": true,
              "Hostname 4 5": true,
              "Hostname 4 6": true,
              "Hostname 4 7": true,
              "Hostname 4 8": true,
              "Hostname 5": true,
              "Hostname 5 1": true,
              "Hostname 5 2": true,
              "Hostname 5 3": true,
              "Hostname 5 4": true,
              "Hostname 5 5": true,
              "Hostname 5 6": true,
              "Hostname 5 7": true,
              "Hostname 5 8": true,
              "Hostname 6": true,
              "Hostname 6 1": true,
              "Hostname 6 2": true,
              "Hostname 6 3": true,
              "Hostname 6 4": true,
              "Hostname 6 5": true,
              "Hostname 6 6": true,
              "Hostname 6 7": true,
              "Hostname 6 8": true,
              "Hostname 7": true,
              "Hostname 7 1": true,
              "Hostname 7 2": true,
              "Hostname 7 3": true,
              "Hostname 7 4": true,
              "Hostname 7 5": true,
              "Hostname 7 6": true,
              "Hostname 7 7": true,
              "Hostname 7 8": true,
              "Hostname 8": true,
              "Hostname 8 1": true,
              "Hostname 8 2": true,
              "Hostname 8 3": true,
              "Hostname 8 4": true,
              "Hostname 8 5": true,
              "Hostname 8 6": true,
              "Hostname 8 7": true,
              "Hostname 8 8": true,
              "Time": true,
              "Time 1": true,
              "Time 2": true,
              "Time 3": true,
              "Time 4": true,
              "Time 5": true,
              "Time 6": true,
              "Time 7": true,
              "Time 8": true,
              "UUID": true,
              "UUID 1": true,
              "UUID 2": true,
              "UUID 3": true,
              "UUID 4": true,
              "UUID 5": true,
              "UUID 6": true,
              "UUID 7": true,
              "UUID 8": true,
              "__name__": true,
              "__name__ 1": true,
              "__name__ 2": true,
              "__name__ 3": true,
              "__name__ 4": true,
              "__name__ 5": true,
              "__name__ 6": true,
              "__name__ 7": true,
              "__name__ 8": true,
              "device": true,
              "device 1": true,
              "device 2": true,
              "device 3": true,
              "device 4": true,
              "device 5": true,
              "device 6": true,
              "device 7": true,
              "device 8": true,
              "gpu": true,
              "gpu 1": true,
              "gpu 2": true,
              "gpu 3": true,
              "gpu 4": true,
              "gpu 5": true,
              "gpu 6": true,
              "gpu 7": true,
              "gpu 8": true,
              "instance": true,
              "instance 1": true,
              "instance 2": true,
              "instance 3": true,
              "instance 4": true,
              "instance 5": true,
              "instance 6": true,
              "instance 7": true,
              "instance 8": true,
              "job": true,
              "job 1": true,
              "job 2": true,
              "job 3": true,
              "job 4": true,
              "job 5": true,
              "job 6": true,
              "job 7": true,
              "job 8": true,
              "modelName": true,
              "modelName 1": true,
              "modelName 2": true,
              "modelName 3": true,
              "modelName 4": true,
              "modelName 5": true,
              "modelName 6": true,
              "modelName 7": true,
              "modelName 8": true,
              "pci_bus_id": true,
              "pci_bus_id 1": true,
              "pci_bus_id 2": true,
              "pci_bus_id 3": true,
              "pci_bus_id 4": true,
              "pci_bus_id 5": true,
              "pci_bus_id 6": true,
              "pci_bus_id 7": true,
              "pci_bus_id 8": true,
              "reason": true,
              "reason 1": true,
              "reason 2": true,
              "reason 3": true,
              "reason 4": true,
              "reason 5": true,
              "reason 6": true,
              "reason 7": true,
              "reason 8": true,
              "source": true,
              "source 1": true,
              "source 2": true,
              "source 3": true,
              "source 4": true,
              "source 5": true,
              "source 6": true,
              "source 7": true,
              "source 8": true,
              "vm_name": true
            },
            "includeByName": {},
            "indexByName": {
              "GPU0": 1,
              "GPU1": 2,
              "GPU2": 3,
              "GPU3": 4,
              "GPU4": 5,
              "GPU5": 6,
              "GPU6": 7,
              "GPU7": 8,
              "Hostname": 0,
              "Machine": 0,
              "Value #A": 1,
              "Value #B": 2,
              "Value #C": 3,
              "Value #D": 4,
              "Value #E": 5,
              "Value #F": 6,
              "Value #G": 7,
              "Value #H": 8
            },
            "renameByName": {
              "Hostname": "Machine",
              "Value #A": "GPU0",
              "Value #B": "GPU1",
              "Value #C": "GPU2",
              "Value #D": "GPU3",
              "Value #E": "GPU4",
              "Value #F": "GPU5",
              "Value #G": "GPU6",
              "Value #H": "GPU7"
            }
          }
        },
        {
          "id": "sortBy",
          "options": {
            "fields": {},
            "sort": [
              {
                "field": "Machine"
              }
            ]
          }
        }
      ],
      "type": "table"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "description": "Cont of AER in system logs per GPU:  Advanced Error Reporting by PCI Express devices",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "cellOptions": {
              "mode": "gradient",
              "type": "color-background"
            },
            "filterable": false,
            "footer": {
              "reducers": []
            },
            "inspect": false,
            "minWidth": 60
          },
          "decimals": 0,
          "mappings": [
            {
              "options": {
                "from": 0,
                "result": {
                  "color": "blue",
                  "index": 0
                },
                "to": 50
              },
              "type": "range"
            },
            {
              "options": {
                "from": 51,
                "result": {
                  "color": "green",
                  "index": 1
                },
                "to": 70
              },
              "type": "range"
            },
            {
              "options": {
                "from": 71,
                "result": {
                  "color": "yellow",
                  "index": 2
                },
                "to": 89
              },
              "type": "range"
            },
            {
              "options": {
                "from": 90,
                "result": {
                  "color": "red",
                  "index": 3
                },
                "to": 150
              },
              "type": "range"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "transparent",
                "value": 0
              }
            ]
          },
          "unit": "none"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Machine"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 104
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 0,
        "y": 18
      },
      "id": 10,
      "options": {
        "cellHeight": "sm",
        "frameIndex": 0,
        "showHeader": true,
        "sortBy": []
      },
      "pluginVersion": "12.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "GPU_AER_TOTAL_ERRORS{gpu=\"0\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "GPU_AER_TOTAL_ERRORS{gpu=\"1\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "GPU_AER_TOTAL_ERRORS{gpu=\"2\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "GPU_AER_TOTAL_ERRORS{gpu=\"3\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "D"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "GPU_AER_TOTAL_ERRORS{gpu=\"4\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "E"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "GPU_AER_TOTAL_ERRORS{gpu=\"5\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "F"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "GPU_AER_TOTAL_ERRORS{gpu=\"6\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "G"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "GPU_AER_TOTAL_ERRORS{gpu=\"7\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "H"
        }
      ],
      "title": "Machine GPU AER_ ERRORS",
      "transformations": [
        {
          "id": "joinByField",
          "options": {
            "byField": "Hostname",
            "mode": "outer"
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "DCGM_FI_DRIVER_VERSION": true,
              "DCGM_FI_DRIVER_VERSION 1": true,
              "DCGM_FI_DRIVER_VERSION 2": true,
              "DCGM_FI_DRIVER_VERSION 3": true,
              "DCGM_FI_DRIVER_VERSION 4": true,
              "DCGM_FI_DRIVER_VERSION 5": true,
              "DCGM_FI_DRIVER_VERSION 6": true,
              "DCGM_FI_DRIVER_VERSION 7": true,
              "DCGM_FI_DRIVER_VERSION 8": true,
              "DCXP_FI_DRIVER_VERSION": true,
              "DCXP_FI_DRIVER_VERSION 1": true,
              "DCXP_FI_DRIVER_VERSION 2": true,
              "DCXP_FI_DRIVER_VERSION 3": true,
              "DCXP_FI_DRIVER_VERSION 4": true,
              "DCXP_FI_DRIVER_VERSION 5": true,
              "DCXP_FI_DRIVER_VERSION 6": true,
              "DCXP_FI_DRIVER_VERSION 7": true,
              "DCXP_FI_DRIVER_VERSION 8": true,
              "Hostname 1": true,
              "Hostname 1 1": true,
              "Hostname 1 2": true,
              "Hostname 1 3": true,
              "Hostname 1 4": true,
              "Hostname 1 5": true,
              "Hostname 1 6": true,
              "Hostname 1 7": true,
              "Hostname 1 8": true,
              "Hostname 2": true,
              "Hostname 2 1": true,
              "Hostname 2 2": true,
              "Hostname 2 3": true,
              "Hostname 2 4": true,
              "Hostname 2 5": true,
              "Hostname 2 6": true,
              "Hostname 2 7": true,
              "Hostname 2 8": true,
              "Hostname 3": true,
              "Hostname 3 1": true,
              "Hostname 3 2": true,
              "Hostname 3 3": true,
              "Hostname 3 4": true,
              "Hostname 3 5": true,
              "Hostname 3 6": true,
              "Hostname 3 7": true,
              "Hostname 3 8": true,
              "Hostname 4": true,
              "Hostname 4 1": true,
              "Hostname 4 2": true,
              "Hostname 4 3": true,
              "Hostname 4 4": true,
              "Hostname 4 5": true,
              "Hostname 4 6": true,
              "Hostname 4 7": true,
              "Hostname 4 8": true,
              "Hostname 5": true,
              "Hostname 5 1": true,
              "Hostname 5 2": true,
              "Hostname 5 3": true,
              "Hostname 5 4": true,
              "Hostname 5 5": true,
              "Hostname 5 6": true,
              "Hostname 5 7": true,
              "Hostname 5 8": true,
              "Hostname 6": true,
              "Hostname 6 1": true,
              "Hostname 6 2": true,
              "Hostname 6 3": true,
              "Hostname 6 4": true,
              "Hostname 6 5": true,
              "Hostname 6 6": true,
              "Hostname 6 7": true,
              "Hostname 6 8": true,
              "Hostname 7": true,
              "Hostname 7 1": true,
              "Hostname 7 2": true,
              "Hostname 7 3": true,
              "Hostname 7 4": true,
              "Hostname 7 5": true,
              "Hostname 7 6": true,
              "Hostname 7 7": true,
              "Hostname 7 8": true,
              "Hostname 8": true,
              "Hostname 8 1": true,
              "Hostname 8 2": true,
              "Hostname 8 3": true,
              "Hostname 8 4": true,
              "Hostname 8 5": true,
              "Hostname 8 6": true,
              "Hostname 8 7": true,
              "Hostname 8 8": true,
              "Time": true,
              "Time 1": true,
              "Time 2": true,
              "Time 3": true,
              "Time 4": true,
              "Time 5": true,
              "Time 6": true,
              "Time 7": true,
              "Time 8": true,
              "UUID": true,
              "UUID 1": true,
              "UUID 2": true,
              "UUID 3": true,
              "UUID 4": true,
              "UUID 5": true,
              "UUID 6": true,
              "UUID 7": true,
              "UUID 8": true,
              "__name__": true,
              "__name__ 1": true,
              "__name__ 2": true,
              "__name__ 3": true,
              "__name__ 4": true,
              "__name__ 5": true,
              "__name__ 6": true,
              "__name__ 7": true,
              "__name__ 8": true,
              "device": true,
              "device 1": true,
              "device 2": true,
              "device 3": true,
              "device 4": true,
              "device 5": true,
              "device 6": true,
              "device 7": true,
              "device 8": true,
              "gpu": true,
              "gpu 1": true,
              "gpu 2": true,
              "gpu 3": true,
              "gpu 4": true,
              "gpu 5": true,
              "gpu 6": true,
              "gpu 7": true,
              "gpu 8": true,
              "instance": true,
              "instance 1": true,
              "instance 2": true,
              "instance 3": true,
              "instance 4": true,
              "instance 5": true,
              "instance 6": true,
              "instance 7": true,
              "instance 8": true,
              "job": true,
              "job 1": true,
              "job 2": true,
              "job 3": true,
              "job 4": true,
              "job 5": true,
              "job 6": true,
              "job 7": true,
              "job 8": true,
              "modelName": true,
              "modelName 1": true,
              "modelName 2": true,
              "modelName 3": true,
              "modelName 4": true,
              "modelName 5": true,
              "modelName 6": true,
              "modelName 7": true,
              "modelName 8": true,
              "pci_bus_id": true,
              "pci_bus_id 1": true,
              "pci_bus_id 2": true,
              "pci_bus_id 3": true,
              "pci_bus_id 4": true,
              "pci_bus_id 5": true,
              "pci_bus_id 6": true,
              "pci_bus_id 7": true,
              "pci_bus_id 8": true,
              "reason": true,
              "reason 1": true,
              "reason 2": true,
              "reason 3": true,
              "reason 4": true,
              "reason 5": true,
              "reason 6": true,
              "reason 7": true,
              "reason 8": true,
              "source": true,
              "source 1": true,
              "source 2": true,
              "source 3": true,
              "source 4": true,
              "source 5": true,
              "source 6": true,
              "source 7": true,
              "source 8": true
            },
            "indexByName": {
              "GPU0": 1,
              "GPU1": 2,
              "GPU2": 3,
              "GPU3": 4,
              "GPU4": 5,
              "GPU5": 6,
              "GPU6": 7,
              "GPU7": 8,
              "Hostname": 0,
              "Machine": 0,
              "Value #A": 1,
              "Value #B": 2,
              "Value #C": 3,
              "Value #D": 4,
              "Value #E": 5,
              "Value #F": 6,
              "Value #G": 7,
              "Value #H": 8
            },
            "renameByName": {
              "Hostname": "Machine",
              "Value #A": "GPU0",
              "Value #B": "GPU1",
              "Value #C": "GPU2",
              "Value #D": "GPU3",
              "Value #E": "GPU4",
              "Value #F": "GPU5",
              "Value #G": "GPU6",
              "Value #H": "GPU7"
            }
          }
        },
        {
          "id": "sortBy",
          "options": {
            "fields": {},
            "sort": [
              {
                "field": "Machine"
              }
            ]
          }
        }
      ],
      "type": "table"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "cellOptions": {
              "mode": "gradient",
              "type": "color-background"
            },
            "filterable": false,
            "footer": {
              "reducers": []
            },
            "inspect": false,
            "minWidth": 60
          },
          "decimals": 0,
          "mappings": [
            {
              "options": {
                "from": 0,
                "result": {
                  "color": "blue",
                  "index": 0
                },
                "to": 50
              },
              "type": "range"
            },
            {
              "options": {
                "from": 50,
                "result": {
                  "color": "green",
                  "index": 1
                },
                "to": 70
              },
              "type": "range"
            },
            {
              "options": {
                "from": 70,
                "result": {
                  "color": "yellow",
                  "index": 2
                },
                "to": 90
              },
              "type": "range"
            },
            {
              "options": {
                "from": 90,
                "result": {
                  "color": "red",
                  "index": 3
                },
                "to": 100
              },
              "type": "range"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "transparent",
                "value": 0
              }
            ]
          },
          "unit": "%"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Machine"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 95
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 8,
        "y": 18
      },
      "id": 14,
      "options": {
        "cellHeight": "sm",
        "frameIndex": 0,
        "showHeader": true,
        "sortBy": []
      },
      "pluginVersion": "12.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_GPU_UTIL{gpu=\"0\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_GPU_UTIL{gpu=\"1\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_GPU_UTIL{gpu=\"2\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_GPU_UTIL{gpu=\"3\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "D"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_GPU_UTIL{gpu=\"4\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "E"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_GPU_UTIL{gpu=\"5\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "F"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_GPU_UTIL{gpu=\"6\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "G"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_GPU_UTIL{gpu=\"7\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "H"
        }
      ],
      "title": "Machine GPU Utilization",
      "transformations": [
        {
          "id": "joinByField",
          "options": {
            "byField": "Hostname",
            "mode": "outer"
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "DCGM_FI_DRIVER_VERSION": true,
              "DCGM_FI_DRIVER_VERSION 1": true,
              "DCGM_FI_DRIVER_VERSION 2": true,
              "DCGM_FI_DRIVER_VERSION 3": true,
              "DCGM_FI_DRIVER_VERSION 4": true,
              "DCGM_FI_DRIVER_VERSION 5": true,
              "DCGM_FI_DRIVER_VERSION 6": true,
              "DCGM_FI_DRIVER_VERSION 7": true,
              "DCGM_FI_DRIVER_VERSION 8": true,
              "DCXP_FI_DRIVER_VERSION": true,
              "DCXP_FI_DRIVER_VERSION 1": true,
              "DCXP_FI_DRIVER_VERSION 2": true,
              "DCXP_FI_DRIVER_VERSION 3": true,
              "DCXP_FI_DRIVER_VERSION 4": true,
              "DCXP_FI_DRIVER_VERSION 5": true,
              "DCXP_FI_DRIVER_VERSION 6": true,
              "DCXP_FI_DRIVER_VERSION 7": true,
              "DCXP_FI_DRIVER_VERSION 8": true,
              "Hostname 1": true,
              "Hostname 1 1": true,
              "Hostname 1 2": true,
              "Hostname 1 3": true,
              "Hostname 1 4": true,
              "Hostname 1 5": true,
              "Hostname 1 6": true,
              "Hostname 1 7": true,
              "Hostname 1 8": true,
              "Hostname 2": true,
              "Hostname 2 1": true,
              "Hostname 2 2": true,
              "Hostname 2 3": true,
              "Hostname 2 4": true,
              "Hostname 2 5": true,
              "Hostname 2 6": true,
              "Hostname 2 7": true,
              "Hostname 2 8": true,
              "Hostname 3": true,
              "Hostname 3 1": true,
              "Hostname 3 2": true,
              "Hostname 3 3": true,
              "Hostname 3 4": true,
              "Hostname 3 5": true,
              "Hostname 3 6": true,
              "Hostname 3 7": true,
              "Hostname 3 8": true,
              "Hostname 4": true,
              "Hostname 4 1": true,
              "Hostname 4 2": true,
              "Hostname 4 3": true,
              "Hostname 4 4": true,
              "Hostname 4 5": true,
              "Hostname 4 6": true,
              "Hostname 4 7": true,
              "Hostname 4 8": true,
              "Hostname 5": true,
              "Hostname 5 1": true,
              "Hostname 5 2": true,
              "Hostname 5 3": true,
              "Hostname 5 4": true,
              "Hostname 5 5": true,
              "Hostname 5 6": true,
              "Hostname 5 7": true,
              "Hostname 5 8": true,
              "Hostname 6": true,
              "Hostname 6 1": true,
              "Hostname 6 2": true,
              "Hostname 6 3": true,
              "Hostname 6 4": true,
              "Hostname 6 5": true,
              "Hostname 6 6": true,
              "Hostname 6 7": true,
              "Hostname 6 8": true,
              "Hostname 7": true,
              "Hostname 7 1": true,
              "Hostname 7 2": true,
              "Hostname 7 3": true,
              "Hostname 7 4": true,
              "Hostname 7 5": true,
              "Hostname 7 6": true,
              "Hostname 7 7": true,
              "Hostname 7 8": true,
              "Hostname 8": true,
              "Hostname 8 1": true,
              "Hostname 8 2": true,
              "Hostname 8 3": true,
              "Hostname 8 4": true,
              "Hostname 8 5": true,
              "Hostname 8 6": true,
              "Hostname 8 7": true,
              "Hostname 8 8": true,
              "Time": true,
              "Time 1": true,
              "Time 2": true,
              "Time 3": true,
              "Time 4": true,
              "Time 5": true,
              "Time 6": true,
              "Time 7": true,
              "Time 8": true,
              "UUID": true,
              "UUID 1": true,
              "UUID 2": true,
              "UUID 3": true,
              "UUID 4": true,
              "UUID 5": true,
              "UUID 6": true,
              "UUID 7": true,
              "UUID 8": true,
              "__name__": true,
              "__name__ 1": true,
              "__name__ 2": true,
              "__name__ 3": true,
              "__name__ 4": true,
              "__name__ 5": true,
              "__name__ 6": true,
              "__name__ 7": true,
              "__name__ 8": true,
              "device": true,
              "device 1": true,
              "device 2": true,
              "device 3": true,
              "device 4": true,
              "device 5": true,
              "device 6": true,
              "device 7": true,
              "device 8": true,
              "gpu": true,
              "gpu 1": true,
              "gpu 2": true,
              "gpu 3": true,
              "gpu 4": true,
              "gpu 5": true,
              "gpu 6": true,
              "gpu 7": true,
              "gpu 8": true,
              "instance": true,
              "instance 1": true,
              "instance 2": true,
              "instance 3": true,
              "instance 4": true,
              "instance 5": true,
              "instance 6": true,
              "instance 7": true,
              "instance 8": true,
              "job": true,
              "job 1": true,
              "job 2": true,
              "job 3": true,
              "job 4": true,
              "job 5": true,
              "job 6": true,
              "job 7": true,
              "job 8": true,
              "modelName": true,
              "modelName 1": true,
              "modelName 2": true,
              "modelName 3": true,
              "modelName 4": true,
              "modelName 5": true,
              "modelName 6": true,
              "modelName 7": true,
              "modelName 8": true,
              "pci_bus_id": true,
              "pci_bus_id 1": true,
              "pci_bus_id 2": true,
              "pci_bus_id 3": true,
              "pci_bus_id 4": true,
              "pci_bus_id 5": true,
              "pci_bus_id 6": true,
              "pci_bus_id 7": true,
              "pci_bus_id 8": true,
              "reason": true,
              "reason 1": true,
              "reason 2": true,
              "reason 3": true,
              "reason 4": true,
              "reason 5": true,
              "reason 6": true,
              "reason 7": true,
              "reason 8": true,
              "source": true,
              "source 1": true,
              "source 2": true,
              "source 3": true,
              "source 4": true,
              "source 5": true,
              "source 6": true,
              "source 7": true,
              "source 8": true,
              "vm_name": true
            },
            "includeByName": {},
            "indexByName": {
              "GPU0": 1,
              "GPU1": 2,
              "GPU2": 3,
              "GPU3": 4,
              "GPU4": 5,
              "GPU5": 6,
              "GPU6": 7,
              "GPU7": 8,
              "Hostname": 0,
              "Machine": 0,
              "Value #A": 1,
              "Value #B": 2,
              "Value #C": 3,
              "Value #D": 4,
              "Value #E": 5,
              "Value #F": 6,
              "Value #G": 7,
              "Value #H": 8
            },
            "renameByName": {
              "Hostname": "Machine",
              "Value #A": "GPU0",
              "Value #B": "GPU1",
              "Value #C": "GPU2",
              "Value #D": "GPU3",
              "Value #E": "GPU4",
              "Value #F": "GPU5",
              "Value #G": "GPU6",
              "Value #H": "GPU7"
            }
          }
        },
        {
          "id": "sortBy",
          "options": {
            "fields": {},
            "sort": [
              {
                "field": "Machine"
              }
            ]
          }
        }
      ],
      "type": "table"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "description": "Current error state for GPU (0=Unknown, 1=No Error, 2=Error, 3=VM Passthrough)",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "cellOptions": {
              "mode": "gradient",
              "type": "color-background"
            },
            "filterable": false,
            "footer": {
              "reducers": []
            },
            "inspect": false,
            "minWidth": 60
          },
          "decimals": 0,
          "mappings": [
            {
              "options": {
                "from": 0,
                "result": {
                  "color": "blue",
                  "index": 0,
                  "text": "Unknown"
                },
                "to": 0
              },
              "type": "range"
            },
            {
              "options": {
                "from": 1,
                "result": {
                  "color": "green",
                  "index": 1,
                  "text": "No Error"
                },
                "to": 1
              },
              "type": "range"
            },
            {
              "options": {
                "from": 2,
                "result": {
                  "color": "red",
                  "index": 2,
                  "text": "Error"
                },
                "to": 2
              },
              "type": "range"
            },
            {
              "options": {
                "from": 3,
                "result": {
                  "color": "purple",
                  "index": 3,
                  "text": "VM Passthrough"
                },
                "to": 3
              },
              "type": "range"
            },
            {
              "options": {
                "from": 4,
                "result": {
                  "color": "gray",
                  "index": 4,
                  "text": "N/A"
                },
                "to": 4
              },
              "type": "range"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "transparent",
                "value": 0
              }
            ]
          },
          "unit": "none"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Machine"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 121
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 16,
        "y": 18
      },
      "id": 11,
      "options": {
        "cellHeight": "sm",
        "frameIndex": 0,
        "showHeader": true,
        "sortBy": []
      },
      "pluginVersion": "12.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "GPU_ERROR_STATE{gpu=\"0\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "GPU_ERROR_STATE{gpu=\"1\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "GPU_ERROR_STATE{gpu=\"2\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "GPU_ERROR_STATE{gpu=\"3\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "D"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "GPU_ERROR_STATE{gpu=\"4\"} ",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "E"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "GPU_ERROR_STATE{gpu=\"5\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "F"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "GPU_ERROR_STATE{gpu=\"6\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "G"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "GPU_ERROR_STATE{gpu=\"7\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "H"
        }
      ],
      "title": "Machine GPU Error State",
      "transformations": [
        {
          "id": "joinByField",
          "options": {
            "byField": "Hostname",
            "mode": "outer"
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "DCGM_FI_DRIVER_VERSION": true,
              "DCGM_FI_DRIVER_VERSION 1": true,
              "DCGM_FI_DRIVER_VERSION 2": true,
              "DCGM_FI_DRIVER_VERSION 3": true,
              "DCGM_FI_DRIVER_VERSION 4": true,
              "DCGM_FI_DRIVER_VERSION 5": true,
              "DCGM_FI_DRIVER_VERSION 6": true,
              "DCGM_FI_DRIVER_VERSION 7": true,
              "DCGM_FI_DRIVER_VERSION 8": true,
              "DCXP_FI_DRIVER_VERSION": true,
              "DCXP_FI_DRIVER_VERSION 1": true,
              "DCXP_FI_DRIVER_VERSION 2": true,
              "DCXP_FI_DRIVER_VERSION 3": true,
              "DCXP_FI_DRIVER_VERSION 4": true,
              "DCXP_FI_DRIVER_VERSION 5": true,
              "DCXP_FI_DRIVER_VERSION 6": true,
              "DCXP_FI_DRIVER_VERSION 7": true,
              "DCXP_FI_DRIVER_VERSION 8": true,
              "Hostname 1": true,
              "Hostname 1 1": true,
              "Hostname 1 2": true,
              "Hostname 1 3": true,
              "Hostname 1 4": true,
              "Hostname 1 5": true,
              "Hostname 1 6": true,
              "Hostname 1 7": true,
              "Hostname 1 8": true,
              "Hostname 2": true,
              "Hostname 2 1": true,
              "Hostname 2 2": true,
              "Hostname 2 3": true,
              "Hostname 2 4": true,
              "Hostname 2 5": true,
              "Hostname 2 6": true,
              "Hostname 2 7": true,
              "Hostname 2 8": true,
              "Hostname 3": true,
              "Hostname 3 1": true,
              "Hostname 3 2": true,
              "Hostname 3 3": true,
              "Hostname 3 4": true,
              "Hostname 3 5": true,
              "Hostname 3 6": true,
              "Hostname 3 7": true,
              "Hostname 3 8": true,
              "Hostname 4": true,
              "Hostname 4 1": true,
              "Hostname 4 2": true,
              "Hostname 4 3": true,
              "Hostname 4 4": true,
              "Hostname 4 5": true,
              "Hostname 4 6": true,
              "Hostname 4 7": true,
              "Hostname 4 8": true,
              "Hostname 5": true,
              "Hostname 5 1": true,
              "Hostname 5 2": true,
              "Hostname 5 3": true,
              "Hostname 5 4": true,
              "Hostname 5 5": true,
              "Hostname 5 6": true,
              "Hostname 5 7": true,
              "Hostname 5 8": true,
              "Hostname 6": true,
              "Hostname 6 1": true,
              "Hostname 6 2": true,
              "Hostname 6 3": true,
              "Hostname 6 4": true,
              "Hostname 6 5": true,
              "Hostname 6 6": true,
              "Hostname 6 7": true,
              "Hostname 6 8": true,
              "Hostname 7": true,
              "Hostname 7 1": true,
              "Hostname 7 2": true,
              "Hostname 7 3": true,
              "Hostname 7 4": true,
              "Hostname 7 5": true,
              "Hostname 7 6": true,
              "Hostname 7 7": true,
              "Hostname 7 8": true,
              "Hostname 8": true,
              "Hostname 8 1": true,
              "Hostname 8 2": true,
              "Hostname 8 3": true,
              "Hostname 8 4": true,
              "Hostname 8 5": true,
              "Hostname 8 6": true,
              "Hostname 8 7": true,
              "Hostname 8 8": true,
              "Time": true,
              "Time 1": true,
              "Time 2": true,
              "Time 3": true,
              "Time 4": true,
              "Time 5": true,
              "Time 6": true,
              "Time 7": true,
              "Time 8": true,
              "UUID": true,
              "UUID 1": true,
              "UUID 2": true,
              "UUID 3": true,
              "UUID 4": true,
              "UUID 5": true,
              "UUID 6": true,
              "UUID 7": true,
              "UUID 8": true,
              "__name__": true,
              "__name__ 1": true,
              "__name__ 2": true,
              "__name__ 3": true,
              "__name__ 4": true,
              "__name__ 5": true,
              "__name__ 6": true,
              "__name__ 7": true,
              "__name__ 8": true,
              "device": true,
              "device 1": true,
              "device 2": true,
              "device 3": true,
              "device 4": true,
              "device 5": true,
              "device 6": true,
              "device 7": true,
              "device 8": true,
              "gpu": true,
              "gpu 1": true,
              "gpu 2": true,
              "gpu 3": true,
              "gpu 4": true,
              "gpu 5": true,
              "gpu 6": true,
              "gpu 7": true,
              "gpu 8": true,
              "instance": true,
              "instance 1": true,
              "instance 2": true,
              "instance 3": true,
              "instance 4": true,
              "instance 5": true,
              "instance 6": true,
              "instance 7": true,
              "instance 8": true,
              "job": true,
              "job 1": true,
              "job 2": true,
              "job 3": true,
              "job 4": true,
              "job 5": true,
              "job 6": true,
              "job 7": true,
              "job 8": true,
              "modelName": true,
              "modelName 1": true,
              "modelName 2": true,
              "modelName 3": true,
              "modelName 4": true,
              "modelName 5": true,
              "modelName 6": true,
              "modelName 7": true,
              "modelName 8": true,
              "pci_bus_id": true,
              "pci_bus_id 1": true,
              "pci_bus_id 2": true,
              "pci_bus_id 3": true,
              "pci_bus_id 4": true,
              "pci_bus_id 5": true,
              "pci_bus_id 6": true,
              "pci_bus_id 7": true,
              "pci_bus_id 8": true,
              "reason": true,
              "reason 1": true,
              "reason 2": true,
              "reason 3": true,
              "reason 4": true,
              "reason 5": true,
              "reason 6": true,
              "reason 7": true,
              "reason 8": true,
              "source": true,
              "source 1": true,
              "source 2": true,
              "source 3": true,
              "source 4": true,
              "source 5": true,
              "source 6": true,
              "source 7": true,
              "source 8": true
            },
            "includeByName": {},
            "indexByName": {
              "GPU0": 1,
              "GPU1": 2,
              "GPU2": 3,
              "GPU3": 4,
              "GPU4": 5,
              "GPU5": 6,
              "GPU6": 7,
              "GPU7": 8,
              "Hostname": 0,
              "Machine": 0,
              "Value #A": 1,
              "Value #B": 2,
              "Value #C": 3,
              "Value #D": 4,
              "Value #E": 5,
              "Value #F": 6,
              "Value #G": 7,
              "Value #H": 8
            },
            "renameByName": {
              "Hostname": "Machine",
              "Value #A": "GPU0",
              "Value #B": "GPU1",
              "Value #C": "GPU2",
              "Value #D": "GPU3",
              "Value #E": "GPU4",
              "Value #F": "GPU5",
              "Value #G": "GPU6",
              "Value #H": "GPU7"
            }
          }
        },
        {
          "id": "sortBy",
          "options": {
            "fields": {},
            "sort": [
              {
                "field": "Machine"
              }
            ]
          }
        }
      ],
      "type": "table"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "description": "APT_UPGRADABLE_PACKAGES Updated once every 1h\n",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "cellOptions": {
              "type": "auto"
            },
            "footer": {
              "reducers": []
            },
            "inspect": false,
            "minWidth": 50
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": 0
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "job"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 149
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 0,
        "y": 26
      },
      "id": 12,
      "options": {
        "cellHeight": "sm",
        "showHeader": true,
        "sortBy": [
          {
            "desc": false,
            "displayName": "job"
          }
        ]
      },
      "pluginVersion": "12.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "APT_UPGRADABLE_PACKAGES",
          "format": "table",
          "instant": false,
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Pending Updates",
      "transformations": [
        {
          "id": "groupBy",
          "options": {
            "fields": {
              "Value": {
                "aggregations": [
                  "last"
                ],
                "operation": "aggregate"
              },
              "job": {
                "aggregations": [],
                "operation": "groupby"
              }
            }
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {},
            "indexByName": {},
            "renameByName": {
              "Value (last)": "Updateds"
            }
          }
        }
      ],
      "type": "table"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "description": "GPU drivers\n",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "cellOptions": {
              "type": "auto"
            },
            "footer": {
              "reducers": []
            },
            "inspect": false,
            "minWidth": 50
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": 0
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "GPU Model Name"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 196
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "GPU Count"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 86
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Hostname"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 119
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 8,
        "w": 16,
        "x": 8,
        "y": 26
      },
      "id": 13,
      "options": {
        "cellHeight": "sm",
        "showHeader": true,
        "sortBy": [
          {
            "desc": false,
            "displayName": "Hostname"
          }
        ]
      },
      "pluginVersion": "12.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "count(DCGM_FI_DEV_FB_USED / 1024 / 1024) by (Hostname)",
          "format": "table",
          "instant": false,
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_SM_CLOCK{Hostname=~\".+\"}",
          "format": "table",
          "hide": false,
          "instant": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "B"
        }
      ],
      "title": "GPU Driver and count",
      "transformations": [
        {
          "id": "joinByField",
          "options": {
            "byField": "Hostname",
            "mode": "outer"
          }
        },
        {
          "id": "groupBy",
          "options": {
            "fields": {
              "DCGM_FI_DRIVER_VERSION": {
                "aggregations": [
                  "lastNotNull"
                ],
                "operation": "aggregate"
              },
              "Hostname": {
                "aggregations": [],
                "operation": "groupby"
              },
              "Value #A": {
                "aggregations": [
                  "lastNotNull"
                ],
                "operation": "aggregate"
              },
              "Value #B": {
                "aggregations": [
                  "lastNotNull"
                ]
              },
              "modelName": {
                "aggregations": [
                  "lastNotNull"
                ],
                "operation": "aggregate"
              }
            }
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {},
            "includeByName": {},
            "indexByName": {
              "DCGM_FI_DRIVER_VERSION (lastNotNull)": 3,
              "Hostname": 0,
              "Value #A (lastNotNull)": 2,
              "modelName (lastNotNull)": 1
            },
            "renameByName": {
              "DCGM_FI_DRIVER_VERSION (lastNotNull)": "DRIVER_VERSION",
              "Value #A (lastNotNull)": "GPU Count",
              "modelName (lastNotNull)": "GPU Model Name"
            }
          }
        }
      ],
      "type": "table"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "cellOptions": {
              "type": "auto"
            },
            "footer": {
              "reducers": []
            },
            "inspect": false,
            "minWidth": 50
          },
          "decimals": 1,
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": 0
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 33,
        "w": 24,
        "x": 0,
        "y": 34
      },
      "id": 1,
      "options": {
        "cellHeight": "sm",
        "frameIndex": 4,
        "showHeader": true,
        "sortBy": [
          {
            "desc": false,
            "displayName": "Uptime(h)"
          }
        ]
      },
      "pluginVersion": "12.3.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "(sum by(instance) (irate(node_cpu_seconds_total{mode!=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance) (irate(node_cpu_seconds_total[$__rate_interval]))) * 100\r\n",
          "format": "table",
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "node_uname_info",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "(sum by(instance) (node_memory_MemTotal_bytes{} - node_memory_MemFree_bytes{} - node_memory_Buffers_bytes{} - node_memory_Cached_bytes{})) / sum by(instance) (node_memory_MemTotal_bytes{}) * 100\r\n",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "(node_time_seconds - on(instance) node_boot_time_seconds ) / 60 /60",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "D"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "round(max by(instance)(rate(node_network_transmit_bytes_total[1m])*8/1000000), 0.1)\r\n\r\n\r\n\r\n",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "E"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "round(max by(instance)(rate(node_network_receive_bytes_total[1m])*8/1000000), 0.1)\r\n",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "F"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "(1 - avg by(instance)(node_filesystem_avail_bytes{mountpoint=\"/\"}/node_filesystem_size_bytes{mountpoint=\"/\"})) * 100\r\n",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "G"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "(1 - avg by(instance)(node_filesystem_avail_bytes{mountpoint=\"/var/lib/docker\"}/node_filesystem_size_bytes{mountpoint=\"/var/lib/docker\"})) * 100 \r\n",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "H"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "avg by(instance)(node_cpu_temperature{})\r\n",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "I"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "(1 - avg by(instance)(node_filesystem_avail_bytes{mountpoint=\"/var/lib/mfs\"}/node_filesystem_size_bytes{mountpoint=\"/var/lib/mfs\"})) * 100",
          "format": "table",
          "hide": false,
          "instant": false,
          "range": true,
          "refId": "J"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "(1 - avg by(instance)(node_filesystem_avail_bytes{mountpoint=~\"/mnt/mfschunks1|/mnt/chunks4\"}/node_filesystem_size_bytes{mountpoint=~\"/mnt/mfschunks1|/mnt/chunks4\"})) * 100",
          "format": "table",
          "hide": false,
          "instant": false,
          "range": true,
          "refId": "K"
        }
      ],
      "title": "Machine Overview",
      "transformations": [
        {
          "id": "joinByField",
          "options": {
            "byField": "instance",
            "mode": "outer"
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "Time 1": true,
              "Time 10": true,
              "Time 11": true,
              "Time 2": true,
              "Time 3": true,
              "Time 4": true,
              "Time 5": true,
              "Time 6": true,
              "Time 7": true,
              "Time 8": true,
              "Time 9": true,
              "Value #B": true,
              "__name__": true,
              "domainname": true,
              "instance": true,
              "job": true,
              "machine": true,
              "sysname": true,
              "version": true
            },
            "indexByName": {
              "Time 1": 4,
              "Time 2": 7,
              "Time 3": 14,
              "Time 4": 16,
              "Time 5": 18,
              "Time 6": 20,
              "Time 7": 22,
              "Time 8": 24,
              "Time 9": 26,
              "Value #A": 5,
              "Value #B": 13,
              "Value #C": 15,
              "Value #D": 17,
              "Value #E": 19,
              "Value #F": 21,
              "Value #G": 23,
              "Value #H": 25,
              "Value #I": 6,
              "__name__": 8,
              "domainname": 9,
              "instance": 3,
              "job": 10,
              "machine": 2,
              "nodename": 0,
              "release": 1,
              "sysname": 11,
              "version": 12
            },
            "renameByName": {
              "Time 2": "",
              "Time 5": "",
              "Value #A": "CPU %",
              "Value #C": "Memory Used %",
              "Value #D": "Uptime(h)",
              "Value #E": "Network_transmit Mbps",
              "Value #F": "Network_receive Mbps ",
              "Value #G": "Root FS Disk Usage %",
              "Value #H": "Docker Disk Usage %",
              "Value #I": "CPU Temp C",
              "Value #J": "mfs Disk Usage %",
              "Value #K": "mfschunks Usage %",
              "nodename": "Machine",
              "release": "Kernel"
            }
          }
        },
        {
          "id": "sortBy",
          "options": {
            "fields": {},
            "sort": [
              {
                "field": "Machine"
              }
            ]
          }
        }
      ],
      "type": "table"
    }
  ],
  "preload": false,
  "refresh": "10s",
  "schemaVersion": 42,
  "tags": [
    "Prometheuse",
    "Nvidia",
    "Linux"
  ],
  "templating": {
    "list": [
      {
        "baseFilters": [],
        "datasource": {
          "type": "prometheus",
          "uid": "prometheus"
        },
        "filters": [
          {
            "key": "Hostname",
            "operator": "!=",
            "value": ""
          }
        ],
        "name": "Filters",
        "type": "adhoc"
      }
    ]
  },
  "time": {
    "from": "now-10s",
    "to": "now"
  },
  "timepicker": {},
  "timezone": "",
  "title": "DC OverView",
  "uid": "dc-overview-main",
  "version": 2
}