{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "description": "GPU observability for HAMi v2.10.0 and later: scheduler allocation, host and container vGPU usage, plus optional NVIDIA DCGM telemetry.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": null, "links": [ { "asDropdown": false, "icon": "external link", "includeVars": false, "keepTime": false, "tags": [], "targetBlank": true, "title": "HAMi project", "tooltip": "", "type": "link", "url": "https://github.com/Project-HAMi/HAMi" } ], "liveNow": false, "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 100, "panels": [], "title": "Cluster overview", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Number of physical GPUs known to the scheduler for the selected node(s).", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, "id": 1, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "10.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "count(hami_gpu_memory_limit_bytes{node=~\"$node\"})", "instant": true, "legendFormat": "GPUs", "refId": "A" } ], "title": "Physical GPUs", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Total schedulable GPU memory across the selected node(s).", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 4, "w": 5, "x": 4, "y": 1 }, "id": 2, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "10.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(hami_gpu_memory_limit_bytes{node=~\"$node\"})", "instant": true, "legendFormat": "Total", "refId": "A" } ], "title": "Total GPU memory", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "GPU memory the scheduler has allocated to pods across the selected node(s).", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 4, "w": 5, "x": 9, "y": 1 }, "id": 3, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "10.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(hami_gpu_memory_allocated_bytes{node=~\"$node\"})", "instant": true, "legendFormat": "Allocated", "refId": "A" } ], "title": "Allocated GPU memory", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Allocated GPU memory as a share of total schedulable GPU memory. Green below 75%, amber 75-90%, red above 90%.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 100, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 75 }, { "color": "red", "value": 90 } ] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 4, "w": 5, "x": 14, "y": 1 }, "id": 4, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "10.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "100 * sum(hami_gpu_memory_allocated_bytes{node=~\"$node\"}) / clamp_min(sum(hami_gpu_memory_limit_bytes{node=~\"$node\"}), 1)", "instant": true, "legendFormat": "Allocated %", "refId": "A" } ], "title": "GPU memory allocated %", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Number of containers sharing GPUs across the selected node(s).", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 4, "w": 5, "x": 19, "y": 1 }, "id": 5, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "10.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(hami_gpu_shared_count{node=~\"$node\"})", "instant": true, "legendFormat": "Shared", "refId": "A" } ], "title": "Shared containers", "type": "stat" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, "id": 101, "panels": [], "title": "Physical GPUs (host)", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Real GPU memory in use on each physical device, as read by the vGPU monitor.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "mappings": [], "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, "id": 10, "options": { "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "hami_host_gpu_memory_used_bytes{node=~\"$node\"}", "legendFormat": "{{device_type}} idx{{device_index}} ({{device_uuid}})", "refId": "A" } ], "title": "Host GPU memory used", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Physical GPU utilization per device (0-100%).", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "mappings": [], "max": 100, "min": 0, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, "id": 11, "options": { "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "hami_host_gpu_utilization_ratio{node=~\"$node\"}", "legendFormat": "{{device_type}} idx{{device_index}} ({{device_uuid}})", "refId": "A" } ], "title": "Host GPU utilization", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Scheduler-allocated GPU memory and physical host memory used, grouped by physical GPU.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "mappings": [], "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "/host used/" }, "properties": [ { "id": "custom.lineStyle", "value": { "dash": [10, 10], "fill": "dash" } }, { "id": "custom.fillOpacity", "value": 0 } ] } ] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 14 }, "id": 12, "options": { "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "hami_gpu_memory_allocated_bytes{node=~\"$node\"}", "legendFormat": "allocated {{node}} idx{{device_index}} ({{device_uuid}})", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "hami_host_gpu_memory_used_bytes{node=~\"$node\"}", "legendFormat": "host used {{node}} idx{{device_index}} ({{device_uuid}})", "refId": "B" } ], "title": "GPU memory: allocated vs host used", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 }, "id": 102, "panels": [], "title": "Scheduler / allocation", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Scheduler view of GPU memory: allocated versus the configured limit, summed per node.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "mappings": [], "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "/limit/" }, "properties": [ { "id": "custom.lineStyle", "value": { "dash": [10, 10], "fill": "dash" } }, { "id": "custom.fillOpacity", "value": 0 } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 23 }, "id": 20, "options": { "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum by (node) (hami_gpu_memory_allocated_bytes{node=~\"$node\"})", "legendFormat": "allocated {{node}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum by (node) (hami_gpu_memory_limit_bytes{node=~\"$node\"})", "legendFormat": "limit {{node}}", "refId": "B" } ], "title": "GPU memory: allocated vs limit", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Allocated memory on each GPU as a share of its capacity. Green below 75%, amber 75-90%, red above 90%.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.75 }, { "color": "red", "value": 0.9 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 23 }, "id": 21, "options": { "displayMode": "gradient", "minVizHeight": 10, "minVizWidth": 0, "orientation": "horizontal", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showUnfilled": true, "valueMode": "color" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "hami_node_gpu_memory_allocated_ratio{node=~\"$node\"}", "instant": true, "legendFormat": "{{node}} idx{{device_index}}", "range": false, "refId": "A" } ], "title": "Node GPU memory allocated ratio", "type": "bargauge" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Percentage of GPU compute cores the scheduler has allocated per device.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 100, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 75 }, { "color": "red", "value": 90 } ] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 31 }, "id": 22, "options": { "displayMode": "gradient", "minVizHeight": 10, "minVizWidth": 0, "orientation": "horizontal", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showUnfilled": true, "valueMode": "color" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "hami_gpu_core_allocated_ratio{node=~\"$node\"}", "instant": true, "legendFormat": "{{node}} idx{{device_index}} ({{device_uuid}})", "range": false, "refId": "A" } ], "title": "GPU core allocated ratio", "type": "bargauge" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Number of containers sharing each GPU device.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "min": 0, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 31 }, "id": 23, "options": { "displayMode": "basic", "minVizHeight": 10, "minVizWidth": 0, "orientation": "horizontal", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showUnfilled": false, "valueMode": "color" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "hami_gpu_shared_count{node=~\"$node\"}", "instant": true, "legendFormat": "{{node}} idx{{device_index}} ({{device_uuid}})", "range": false, "refId": "A" } ], "title": "GPU shared count per device", "type": "bargauge" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Current GPU inventory reported by the HAMi scheduler. The sample value is allocated memory in bytes; total memory is exposed by HAMi as a label in MiB.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byName", "options": "Value" }, "properties": [ { "id": "unit", "value": "bytes" } ] } ] }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 39 }, "id": 24, "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": ["sum"], "show": false }, "showHeader": true, "sortBy": [ { "displayName": "Value", "desc": true } ] }, "pluginVersion": "10.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "hami_node_gpu_overview{node=~\"$node\"}", "format": "table", "instant": true, "legendFormat": "__auto", "refId": "A" } ], "title": "GPU inventory and allocation", "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true, "zone": true }, "indexByName": {}, "renameByName": { "node": "Node", "device_uuid": "GPU UUID", "device_index": "Index", "device_type": "Device type", "device_cores": "Total cores", "device_memory_limit": "Total memory (MiB)", "Value": "Allocated memory" } } } ], "type": "table" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 46 }, "id": 103, "panels": [], "title": "vGPU / container workloads", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Per-container vGPU memory used versus its limit, for pods in the selected namespace(s).", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "mappings": [], "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "/limit/" }, "properties": [ { "id": "custom.lineStyle", "value": { "dash": [10, 10], "fill": "dash" } }, { "id": "custom.fillOpacity", "value": 0 } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 47 }, "id": 30, "options": { "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "hami_vgpu_memory_used_bytes{namespace=~\"$namespace\"}", "legendFormat": "used {{namespace}}/{{pod}}/{{container}} v{{vdevice_index}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "hami_vgpu_memory_limit_bytes{namespace=~\"$namespace\"}", "legendFormat": "limit {{namespace}}/{{pod}}/{{container}} v{{vdevice_index}}", "refId": "B" } ], "title": "vGPU memory: used vs limit", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Per-container GPU compute utilization inside the vGPU (0-100%).", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "mappings": [], "max": 100, "min": 0, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 47 }, "id": 31, "options": { "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "hami_container_device_utilization_ratio{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}}/{{pod}}/{{container}} v{{vdevice_index}}", "refId": "A" } ], "title": "Container device utilization", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "vGPU memory used as a share of each container's limit. Green below 75%, amber 75-90%, red above 90%.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 100, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 75 }, { "color": "red", "value": 90 } ] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 55 }, "id": 32, "options": { "displayMode": "gradient", "minVizHeight": 10, "minVizWidth": 0, "orientation": "horizontal", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showUnfilled": true, "valueMode": "color" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "100 * hami_vgpu_memory_used_bytes{namespace=~\"$namespace\"} / clamp_min(hami_vgpu_memory_limit_bytes{namespace=~\"$namespace\"}, 1)", "instant": true, "legendFormat": "{{namespace}}/{{pod}}/{{container}} v{{vdevice_index}}", "range": false, "refId": "A" } ], "title": "vGPU memory used % of limit", "type": "bargauge" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Containers using the most vGPU memory right now, in the selected namespace(s).", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byName", "options": "Value" }, "properties": [ { "id": "custom.cellOptions", "value": { "mode": "gradient", "type": "gauge" } }, { "id": "custom.width", "value": 160 }, { "id": "unit", "value": "bytes" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 55 }, "id": 33, "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": ["sum"], "show": false }, "showHeader": true, "sortBy": [ { "displayName": "Value", "desc": true } ] }, "pluginVersion": "10.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "topk(10, hami_vgpu_memory_used_bytes{namespace=~\"$namespace\"})", "format": "table", "instant": true, "legendFormat": "__auto", "refId": "A" } ], "title": "Top 10 containers by vGPU memory", "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true, "device_uuid": false, "zone": true }, "indexByName": {}, "renameByName": { "container": "Container", "device_uuid": "GPU UUID", "namespace": "Namespace", "pod": "Pod", "vdevice_index": "vGPU", "Value": "Used" } } } ], "type": "table" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 63 }, "id": 104, "panels": [], "title": "NVIDIA hardware telemetry (optional DCGM Exporter)", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Number of selected NVIDIA GPUs whose latest DCGM XID error value is non-zero.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 5, "w": 8, "x": 0, "y": 64 }, "id": 40, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "10.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum((DCGM_FI_DEV_XID_ERRORS > bool 0) and on (UUID) label_replace(hami_gpu_memory_limit_bytes{node=~\"$node\"}, \"UUID\", \"$1\", \"device_uuid\", \"(.*)\"))", "instant": true, "legendFormat": "GPUs with XID errors", "refId": "A" } ], "title": "GPUs reporting XID errors", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Average temperature across the selected NVIDIA GPUs, reported by DCGM Exporter.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 100, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 83 }, { "color": "red", "value": 87 } ] }, "unit": "celsius" }, "overrides": [] }, "gridPos": { "h": 5, "w": 8, "x": 8, "y": 64 }, "id": 44, "options": { "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "pluginVersion": "10.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "avg(DCGM_FI_DEV_GPU_TEMP and on (UUID) label_replace(hami_gpu_memory_limit_bytes{node=~\"$node\"}, \"UUID\", \"$1\", \"device_uuid\", \"(.*)\"))", "instant": true, "legendFormat": "Average temperature", "range": false, "refId": "A" } ], "title": "Average GPU temperature", "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Combined power draw of the selected NVIDIA GPUs, reported by DCGM Exporter.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "min": 0, "unit": "watt" }, "overrides": [] }, "gridPos": { "h": 5, "w": 8, "x": 16, "y": 64 }, "id": 45, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "10.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(DCGM_FI_DEV_POWER_USAGE and on (UUID) label_replace(hami_gpu_memory_limit_bytes{node=~\"$node\"}, \"UUID\", \"$1\", \"device_uuid\", \"(.*)\"))", "legendFormat": "Total power", "range": true, "refId": "A" } ], "title": "Total GPU power usage", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Physical NVIDIA GPU temperature reported by DCGM Exporter for GPUs on the selected HAMi node(s).", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "mappings": [], "unit": "celsius", "min": 0, "max": 120 }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 69 }, "id": 41, "options": { "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "DCGM_FI_DEV_GPU_TEMP and on (UUID) label_replace(hami_gpu_memory_limit_bytes{node=~\"$node\"}, \"UUID\", \"$1\", \"device_uuid\", \"(.*)\")", "legendFormat": "{{modelName}} gpu{{gpu}} ({{UUID}})", "refId": "A" } ], "title": "GPU temperature", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Physical NVIDIA GPU power draw reported by DCGM Exporter for GPUs on the selected HAMi node(s).", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "mappings": [], "unit": "watt", "min": 0 }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 69 }, "id": 42, "options": { "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "DCGM_FI_DEV_POWER_USAGE and on (UUID) label_replace(hami_gpu_memory_limit_bytes{node=~\"$node\"}, \"UUID\", \"$1\", \"device_uuid\", \"(.*)\")", "legendFormat": "{{modelName}} gpu{{gpu}} ({{UUID}})", "refId": "A" } ], "title": "GPU power usage", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Physical NVIDIA GPU SM clock reported by DCGM Exporter. DCGM reports MHz; the query converts it to Hz for Grafana.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "mappings": [], "unit": "hertz", "min": 0 }, "overrides": [] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 77 }, "id": 43, "options": { "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "(DCGM_FI_DEV_SM_CLOCK and on (UUID) label_replace(hami_gpu_memory_limit_bytes{node=~\"$node\"}, \"UUID\", \"$1\", \"device_uuid\", \"(.*)\")) * 1000000", "legendFormat": "{{modelName}} gpu{{gpu}} ({{UUID}})", "refId": "A" } ], "title": "GPU SM clock (DCGM)", "type": "timeseries" } ], "refresh": "30s", "schemaVersion": 39, "tags": ["hami", "gpu", "vgpu"], "templating": { "list": [ { "current": {}, "hide": 0, "includeAll": false, "label": "Data source", "multi": false, "name": "datasource", "options": [], "query": "prometheus", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "current": {}, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "definition": "label_values(hami_gpu_memory_limit_bytes, node)", "hide": 0, "includeAll": true, "label": "Node", "multi": true, "name": "node", "options": [], "query": { "qryType": 1, "query": "label_values(hami_gpu_memory_limit_bytes, node)", "refId": "StandardVariableQuery" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1, "type": "query" }, { "current": {}, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "definition": "label_values(hami_vgpu_memory_used_bytes, namespace)", "hide": 0, "includeAll": true, "label": "Namespace", "multi": true, "name": "namespace", "options": [], "query": { "qryType": 1, "query": "label_values(hami_vgpu_memory_used_bytes, namespace)", "refId": "StandardVariableQuery" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1, "type": "query" } ] }, "time": { "from": "now-6h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "HAMi vGPU metrics", "uid": "hami-vgpu-metrics", "version": 1, "weekStart": "" }