--- apiVersion: v1 kind: ConfigMap metadata: name: dashboard-vllm namespace: monitoring labels: grafana_dashboard: "1" annotations: grafana_folder: AI/ML data: vllm.json: |- { "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "description": "vLLM inference server monitoring dashboard for OpenClaw Brain on GKE", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": null, "links": [], "liveNow": false, "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 100, "panels": [], "title": "Request Status", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Total number of requests currently being processed", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 100 }, { "color": "red", "value": 500 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 5, "w": 4, "x": 0, "y": 1 }, "id": 1, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", "expr": "sum(vllm:num_requests_running)", "legendFormat": "Running", "range": true, "refId": "A" } ], "title": "Running Requests", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Total number of requests waiting in the queue", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 50 }, { "color": "red", "value": 200 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 5, "w": 4, "x": 4, "y": 1 }, "id": 2, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", "expr": "sum(vllm:num_requests_waiting)", "legendFormat": "Waiting", "range": true, "refId": "A" } ], "title": "Queue Length", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Rate of successful requests", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 5, "w": 4, "x": 8, "y": 1 }, "id": 3, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", "expr": "sum(rate(vllm:request_success_total[5m]))", "legendFormat": "Success rate", "range": true, "refId": "A" } ], "title": "Request Success Rate", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Number of distinct models currently loaded", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 5, "w": 4, "x": 12, "y": 1 }, "id": 20, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", "expr": "count(vllm:num_requests_running by (model))", "legendFormat": "Models", "range": true, "refId": "A" } ], "title": "Loaded Models", "type": "stat" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 }, "id": 101, "panels": [], "title": "Token Throughput", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Input token throughput rate per second (prompt tokens)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Tokens/sec", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "lines", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 7 }, "id": 4, "options": { "legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", "expr": "sum(rate(vllm:prompt_tokens_total[5m]))", "legendFormat": "Input tokens/sec", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", "expr": "sum(rate(vllm:prompt_tokens_total[5m])) by (model)", "hide": false, "legendFormat": "{{model}}", "range": true, "refId": "B" } ], "title": "Input Token Throughput", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Output token throughput rate per second (generation tokens)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Tokens/sec", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "lines", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 7 }, "id": 5, "options": { "legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", "expr": "sum(rate(vllm:generation_tokens_total[5m]))", "legendFormat": "Output tokens/sec", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", "expr": "sum(rate(vllm:generation_tokens_total[5m])) by (model)", "hide": false, "legendFormat": "{{model}}", "range": true, "refId": "B" } ], "title": "Output Token Throughput", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 15 }, "id": 102, "panels": [], "title": "Latency", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Time per output token (TPOT) - measures generation speed. Lower is better.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "ms", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "lines", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 50 }, { "color": "red", "value": 200 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 16 }, "id": 6, "options": { "legend": { "calcs": ["mean", "max", "p95"], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", "expr": "histogram_quantile(0.50, sum(rate(vllm:time_per_output_token_seconds_bucket[5m])) by (le, model)) * 1000", "legendFormat": "p50 {{model}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(vllm:time_per_output_token_seconds_bucket[5m])) by (le, model)) * 1000", "hide": false, "legendFormat": "p95 {{model}}", "range": true, "refId": "B" } ], "title": "TPOT (Time Per Output Token)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Time to first token (TTFT) - measures initial latency. Lower is better.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "ms", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "lines", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 500 }, { "color": "red", "value": 2000 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 16 }, "id": 7, "options": { "legend": { "calcs": ["mean", "max", "p95"], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", "expr": "histogram_quantile(0.50, sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le, model)) * 1000", "legendFormat": "p50 {{model}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le, model)) * 1000", "hide": false, "legendFormat": "p95 {{model}}", "range": true, "refId": "B" } ], "title": "TTFT (Time to First Token)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Request success and error rates over time", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "req/s", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "lines", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "reqps" }, "overrides": [ { "matcher": { "id": "byName", "options": "Errors" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 16 }, "id": 8, "options": { "legend": { "calcs": ["mean", "max", "sum"], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", "expr": "sum(rate(vllm:request_success_total[5m]))", "legendFormat": "Successful", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", "expr": "sum(rate(vllm:request_success_total{status=\"failure\"}[5m]))", "hide": false, "legendFormat": "Errors", "range": true, "refId": "B" } ], "title": "Request Success vs Error Rate", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 }, "id": 103, "panels": [], "title": "GPU & Cache", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "GPU cache utilization percentage. High usage indicates memory pressure.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "%", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "lines", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line+area" } }, "mappings": [], "max": 1.0, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.75 }, { "color": "red", "value": 0.90 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 25 }, "id": 9, "options": { "legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", "expr": "sum(vllm:gpu_cache_usage_perc) by (instance)", "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "GPU Cache Usage", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "KV cache prefix hit rate. Higher values indicate better cache reuse efficiency.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "%", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "lines", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 25 }, "id": 11, "options": { "legend": { "calcs": ["mean", "max", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", "expr": "sum(vllm:gpu_prefix_cache_hit_rate) by (instance)", "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "KV Cache Hit Rate", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 33 }, "id": 104, "panels": [], "title": "Per-Model Metrics", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Token throughput breakdown by model (input + output)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Tokens/sec", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "lines", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 34 }, "id": 12, "options": { "legend": { "calcs": ["mean", "max", "sum"], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", "expr": "sum(rate(vllm:prompt_tokens_total[5m])) by (model)", "legendFormat": "in:{{model}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", "expr": "sum(rate(vllm:generation_tokens_total[5m])) by (model)", "hide": false, "legendFormat": "out:{{model}}", "range": true, "refId": "B" } ], "title": "Per-Model Token Throughput", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Per-model TPOT breakdown for latency comparison", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "ms", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "lines", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 50 }, { "color": "red", "value": 200 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 34 }, "id": 13, "options": { "legend": { "calcs": ["mean", "max", "p95"], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", "expr": "histogram_quantile(0.50, sum(rate(vllm:time_per_output_token_seconds_bucket[5m])) by (le, model)) * 1000", "legendFormat": "p50-{{model}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(vllm:time_per_output_token_seconds_bucket[5m])) by (le, model)) * 1000", "hide": false, "legendFormat": "p95-{{model}}", "range": true, "refId": "B" } ], "title": "Per-Model TPOT", "type": "timeseries" } ], "refresh": "30s", "schemaVersion": 38, "style": "dark", "tags": [ "vllm", "llm", "inference", "openclaw" ], "templating": { "list": [ { "current": { "selected": false, "text": "Prometheus", "value": "Prometheus" }, "hide": 0, "includeAll": false, "label": "Datasource", "multi": false, "name": "datasource", "options": [], "query": "prometheus", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" } ] }, "time": { "from": "now-3h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "vLLM Inference Server", "uid": "vllm-inference-server", "version": 1, "weekStart": "" }