6183159974
- Install Prometheus + Grafana via kube-prometheus-stack (ClusterIP only, no public ingress) - Deploy Tailscale Operator for secure VPN access to internal services - Add CNPG/PostgreSQL monitoring dashboards - Add vLLM inference monitoring dashboards (tokens, latency, GPU) - Add Cilium networking dashboards (policy, traffic, drops) - Update infra-controllers staging kustomization to include all controllers - Add monitoring-configs Flux sync for dashboard deployment - Update README with monitoring architecture and access instructions - Remove broken stale monitoring files (Azure Key Vault refs, wrong domains) Access: kubectl port-forward or Tailscale VPN (replace auth key before deploy)
630 lines
21 KiB
YAML
630 lines
21 KiB
YAML
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: dashboard-cilium
|
|
labels:
|
|
grafana_dashboard: "1"
|
|
annotations:
|
|
grafana_folder: Networking
|
|
data:
|
|
cilium-networking.json: |-
|
|
{
|
|
"annotations": {
|
|
"list": [
|
|
{
|
|
"builtIn": 1,
|
|
"datasource": {
|
|
"type": "grafana",
|
|
"uid": "-- Grafana --"
|
|
},
|
|
"enable": true,
|
|
"hide": true,
|
|
"iconColor": "rgba(0, 211, 255, 1)",
|
|
"name": "Annotations & Alerts",
|
|
"type": "dashboard"
|
|
}
|
|
]
|
|
},
|
|
"description": "Cilium networking dashboard for monitoring network flows, policy denials, drops, and connectivity",
|
|
"editable": true,
|
|
"fiscalYearStartMonth": 0,
|
|
"graphTooltip": 1,
|
|
"id": null,
|
|
"links": [],
|
|
"liveNow": false,
|
|
"panels": [
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
|
"id": 100,
|
|
"panels": [],
|
|
"title": "Network Traffic Overview",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "$datasource" },
|
|
"description": "Total bytes received across the cluster",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": { "mode": "palette-classic" },
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 20,
|
|
"lineWidth": 2,
|
|
"showPoints": "never"
|
|
},
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
|
"unit": "bytes"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 },
|
|
"id": 1,
|
|
"options": {
|
|
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
|
|
"tooltip": { "mode": "multi", "sort": "desc" }
|
|
},
|
|
"title": "Network Received Bytes Total",
|
|
"type": "timeseries",
|
|
"targets": [
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "$datasource" },
|
|
"expr": "rate(cilium_network_received_bytes_total[5m])",
|
|
"legendFormat": "{{namespace}} - {{pod}}",
|
|
"refId": "A"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "$datasource" },
|
|
"description": "Total bytes sent across the cluster",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": { "mode": "palette-classic" },
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 20,
|
|
"lineWidth": 2,
|
|
"showPoints": "never"
|
|
},
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
|
"unit": "bytes"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 },
|
|
"id": 2,
|
|
"options": {
|
|
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
|
|
"tooltip": { "mode": "multi", "sort": "desc" }
|
|
},
|
|
"title": "Network Sent Bytes Total",
|
|
"type": "timeseries",
|
|
"targets": [
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "$datasource" },
|
|
"expr": "rate(cilium_network_sent_bytes_total[5m])",
|
|
"legendFormat": "{{namespace}} - {{pod}}",
|
|
"refId": "A"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "$datasource" },
|
|
"description": "Bandwidth received per namespace",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": { "mode": "palette-classic" },
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 20,
|
|
"lineWidth": 2,
|
|
"showPoints": "never"
|
|
},
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
|
"unit": "Bps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
|
|
"id": 3,
|
|
"options": {
|
|
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
|
|
"tooltip": { "mode": "multi", "sort": "desc" }
|
|
},
|
|
"title": "Bandwidth Received Per Namespace",
|
|
"type": "timeseries",
|
|
"targets": [
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "$datasource" },
|
|
"expr": "sum by (namespace) (rate(cilium_network_received_bytes_total[5m]))",
|
|
"legendFormat": "{{namespace}}",
|
|
"refId": "A"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "$datasource" },
|
|
"description": "Bandwidth sent per namespace",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": { "mode": "palette-classic" },
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 20,
|
|
"lineWidth": 2,
|
|
"showPoints": "never"
|
|
},
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
|
"unit": "Bps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 },
|
|
"id": 4,
|
|
"options": {
|
|
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
|
|
"tooltip": { "mode": "multi", "sort": "desc" }
|
|
},
|
|
"title": "Bandwidth Sent Per Namespace",
|
|
"type": "timeseries",
|
|
"targets": [
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "$datasource" },
|
|
"expr": "sum by (namespace) (rate(cilium_network_sent_bytes_total[5m]))",
|
|
"legendFormat": "{{namespace}}",
|
|
"refId": "A"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 17 },
|
|
"id": 200,
|
|
"panels": [],
|
|
"title": "Network Policy Enforcement",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "$datasource" },
|
|
"description": "Policy L3/L4 denied connections",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": { "mode": "palette-classic" },
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"lineWidth": 2,
|
|
"showPoints": "never"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{ "color": "green", "value": null },
|
|
{ "color": "red", "value": 1 }
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 18 },
|
|
"id": 5,
|
|
"options": {
|
|
"legend": { "calcs": ["sum", "max"], "displayMode": "table", "placement": "bottom" },
|
|
"tooltip": { "mode": "multi", "sort": "desc" }
|
|
},
|
|
"title": "Policy L3/L4 Denied Total",
|
|
"type": "timeseries",
|
|
"targets": [
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "$datasource" },
|
|
"expr": "increase(cilium_policy_l3_l4_denied_total[5m])",
|
|
"legendFormat": "{{namespace}} - {{pod}}",
|
|
"refId": "A"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "$datasource" },
|
|
"description": "Policy L3/L4 allowed connections",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": { "mode": "palette-classic" },
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 20,
|
|
"lineWidth": 2,
|
|
"showPoints": "never"
|
|
},
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 18 },
|
|
"id": 6,
|
|
"options": {
|
|
"legend": { "calcs": ["sum", "max"], "displayMode": "table", "placement": "bottom" },
|
|
"tooltip": { "mode": "multi", "sort": "desc" }
|
|
},
|
|
"title": "Policy L3/L4 Allowed Total",
|
|
"type": "timeseries",
|
|
"targets": [
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "$datasource" },
|
|
"expr": "increase(cilium_policy_l3_l4_allowed_total[5m])",
|
|
"legendFormat": "{{namespace}} - {{pod}}",
|
|
"refId": "A"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "$datasource" },
|
|
"description": "Dropped packets by reason",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": { "mode": "palette-classic" },
|
|
"custom": {
|
|
"drawStyle": "bars",
|
|
"fillOpacity": 80,
|
|
"lineWidth": 1,
|
|
"showPoints": "never"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{ "color": "green", "value": null },
|
|
{ "color": "yellow", "value": 10 },
|
|
{ "color": "red", "value": 100 }
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 26 },
|
|
"id": 7,
|
|
"options": {
|
|
"legend": { "calcs": ["sum", "max"], "displayMode": "table", "placement": "bottom" },
|
|
"tooltip": { "mode": "multi", "sort": "desc" }
|
|
},
|
|
"title": "Drop Total By Reason",
|
|
"type": "timeseries",
|
|
"targets": [
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "$datasource" },
|
|
"expr": "increase(cilium_drop_total{reason!=\"\"}[5m])",
|
|
"legendFormat": "{{reason}}",
|
|
"refId": "A"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "$datasource" },
|
|
"description": "Total drops across all reasons",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": { "mode": "thresholds" },
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{ "color": "green", "value": null },
|
|
{ "color": "yellow", "value": 50 },
|
|
{ "color": "red", "value": 200 }
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 26 },
|
|
"id": 8,
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "auto",
|
|
"orientation": "auto",
|
|
"reduceOptions": { "calcs": ["sum"], "fields": "", "values": false },
|
|
"textMode": "auto"
|
|
},
|
|
"title": "Total Drops",
|
|
"type": "stat",
|
|
"targets": [
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "$datasource" },
|
|
"expr": "sum(increase(cilium_drop_total[5m]))",
|
|
"legendFormat": "Total",
|
|
"refId": "A"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 34 },
|
|
"id": 300,
|
|
"panels": [],
|
|
"title": "Service Connectivity & TCP",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "$datasource" },
|
|
"description": "TCP connection count and duration",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": { "mode": "palette-classic" },
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 20,
|
|
"lineWidth": 2,
|
|
"showPoints": "never"
|
|
},
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 35 },
|
|
"id": 9,
|
|
"options": {
|
|
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
|
|
"tooltip": { "mode": "multi", "sort": "desc" }
|
|
},
|
|
"title": "TCP Connection Count",
|
|
"type": "timeseries",
|
|
"targets": [
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "$datasource" },
|
|
"expr": "rate(cilium_network_tcp_duration_seconds_count[5m])",
|
|
"legendFormat": "{{namespace}} - {{pod}}",
|
|
"refId": "A"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "$datasource" },
|
|
"description": "TCP connection duration (latency)",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": { "mode": "palette-classic" },
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 20,
|
|
"lineWidth": 2,
|
|
"showPoints": "never"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{ "color": "green", "value": null },
|
|
{ "color": "yellow", "value": 0.1 },
|
|
{ "color": "red", "value": 0.5 }
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 35 },
|
|
"id": 10,
|
|
"options": {
|
|
"legend": { "calcs": ["mean", "max", "p99"], "displayMode": "table", "placement": "bottom" },
|
|
"tooltip": { "mode": "multi", "sort": "desc" }
|
|
},
|
|
"title": "TCP Connection Duration (Latency)",
|
|
"type": "timeseries",
|
|
"targets": [
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "$datasource" },
|
|
"expr": "rate(cilium_network_tcp_duration_seconds_sum[5m]) / rate(cilium_network_tcp_duration_seconds_count[5m])",
|
|
"legendFormat": "{{namespace}} - {{pod}}",
|
|
"refId": "A"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 43 },
|
|
"id": 400,
|
|
"panels": [],
|
|
"title": "DNS Query Monitoring",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "$datasource" },
|
|
"description": "DNS query rate from Cilium",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": { "mode": "palette-classic" },
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 20,
|
|
"lineWidth": 2,
|
|
"showPoints": "never"
|
|
},
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 44 },
|
|
"id": 11,
|
|
"options": {
|
|
"legend": { "calcs": ["sum", "max"], "displayMode": "table", "placement": "bottom" },
|
|
"tooltip": { "mode": "multi", "sort": "desc" }
|
|
},
|
|
"title": "DNS Query Rate",
|
|
"type": "timeseries",
|
|
"targets": [
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "$datasource" },
|
|
"expr": "rate(cilium_dns_queries_total[5m])",
|
|
"legendFormat": "{{namespace}} - {{pod}}",
|
|
"refId": "A"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "$datasource" },
|
|
"description": "DNS query errors from Cilium",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": { "mode": "palette-classic" },
|
|
"custom": {
|
|
"drawStyle": "bars",
|
|
"fillOpacity": 80,
|
|
"lineWidth": 1,
|
|
"showPoints": "never"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{ "color": "green", "value": null },
|
|
{ "color": "red", "value": 1 }
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 44 },
|
|
"id": 12,
|
|
"options": {
|
|
"legend": { "calcs": ["sum", "max"], "displayMode": "table", "placement": "bottom" },
|
|
"tooltip": { "mode": "multi", "sort": "desc" }
|
|
},
|
|
"title": "DNS Query Errors",
|
|
"type": "timeseries",
|
|
"targets": [
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "$datasource" },
|
|
"expr": "rate(cilium_dns_queries_total{status=\"error\"}[5m])",
|
|
"legendFormat": "{{namespace}} - {{pod}}",
|
|
"refId": "A"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 52 },
|
|
"id": 500,
|
|
"panels": [],
|
|
"title": "Load Balancer Metrics",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "$datasource" },
|
|
"description": "Load balancer backend selection count",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": { "mode": "palette-classic" },
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 20,
|
|
"lineWidth": 2,
|
|
"showPoints": "never"
|
|
},
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 53 },
|
|
"id": 13,
|
|
"options": {
|
|
"legend": { "calcs": ["sum"], "displayMode": "table", "placement": "bottom" },
|
|
"tooltip": { "mode": "multi", "sort": "desc" }
|
|
},
|
|
"title": "Load Balancer Backend Selections",
|
|
"type": "timeseries",
|
|
"targets": [
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "$datasource" },
|
|
"expr": "rate(cilium_lb_backends_total[5m])",
|
|
"legendFormat": "{{type}} - {{address}}:{{port}}",
|
|
"refId": "A"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "$datasource" },
|
|
"description": "Service proxy TCP connections tracked by Cilium load balancer",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": { "mode": "palette-classic" },
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 20,
|
|
"lineWidth": 2,
|
|
"showPoints": "never"
|
|
},
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 53 },
|
|
"id": 14,
|
|
"options": {
|
|
"legend": { "calcs": ["sum", "max"], "displayMode": "table", "placement": "bottom" },
|
|
"tooltip": { "mode": "multi", "sort": "desc" }
|
|
},
|
|
"title": "Service Proxy TCP Connections",
|
|
"type": "timeseries",
|
|
"targets": [
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "$datasource" },
|
|
"expr": "rate(cilium_service_proxy_tcp_connections_total[5m])",
|
|
"legendFormat": "{{namespace}} - {{pod}}",
|
|
"refId": "A"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"refresh": "30s",
|
|
"schemaVersion": 39,
|
|
"tags": ["cilium", "networking", "kubernetes"],
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"current": {
|
|
"selected": false,
|
|
"text": "prometheus",
|
|
"value": "prometheus"
|
|
},
|
|
"hide": 2,
|
|
"includeAll": false,
|
|
"label": "Datasource",
|
|
"multi": false,
|
|
"name": "datasource",
|
|
"options": [],
|
|
"query": "prometheus",
|
|
"refresh": 1,
|
|
"regex": "",
|
|
"skipUrlSync": false,
|
|
"type": "datasource"
|
|
}
|
|
]
|
|
},
|
|
"time": {
|
|
"from": "now-3h",
|
|
"to": "now"
|
|
},
|
|
"timepicker": {
|
|
"refresh_intervals": [
|
|
"10s",
|
|
"30s",
|
|
"1m",
|
|
"5m",
|
|
"15m",
|
|
"30m",
|
|
"1h",
|
|
"2h",
|
|
"1d"
|
|
]
|
|
},
|
|
"timezone": "",
|
|
"title": "Cilium Networking Dashboard",
|
|
"uid": "cilium-networking-dashboard"
|
|
}
|