sirius/gcloud-lab
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
gcloud-lab/apps/base/monitoring/dashboards/dashboards-cnpg.yaml

1102 lines
30 KiB
YAML
Raw Permalink Normal View History

feat: add Prometheus, Grafana, and Tailscale monitoring stack - Install Prometheus + Grafana via kube-prometheus-stack (ClusterIP only, no public ingress) - Deploy Tailscale Operator for secure VPN access to internal services - Add CNPG/PostgreSQL monitoring dashboards - Add vLLM inference monitoring dashboards (tokens, latency, GPU) - Add Cilium networking dashboards (policy, traffic, drops) - Update infra-controllers staging kustomization to include all controllers - Add monitoring-configs Flux sync for dashboard deployment - Update README with monitoring architecture and access instructions - Remove broken stale monitoring files (Azure Key Vault refs, wrong domains) Access: kubectl port-forward or Tailscale VPN (replace auth key before deploy)
2026-04-26 02:31:53 +00:00
apiVersion: v1
kind: ConfigMap
metadata:
name: cnpg-postgres-dashboard
add dashboards to namespace
2026-05-04 20:38:22 +00:00
namespace: monitoring
feat: add Prometheus, Grafana, and Tailscale monitoring stack - Install Prometheus + Grafana via kube-prometheus-stack (ClusterIP only, no public ingress) - Deploy Tailscale Operator for secure VPN access to internal services - Add CNPG/PostgreSQL monitoring dashboards - Add vLLM inference monitoring dashboards (tokens, latency, GPU) - Add Cilium networking dashboards (policy, traffic, drops) - Update infra-controllers staging kustomization to include all controllers - Add monitoring-configs Flux sync for dashboard deployment - Update README with monitoring architecture and access instructions - Remove broken stale monitoring files (Azure Key Vault refs, wrong domains) Access: kubectl port-forward or Tailscale VPN (replace auth key before deploy)
2026-04-26 02:31:53 +00:00
labels:
grafana_dashboard: "1"
annotations:
grafana_folder: Database
data:
cnpg-postgres-dashboard.json: >
{
"__inputs": [],
"__requires": [],
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"description": "CloudNativePG (CNPG) PostgreSQL monitoring dashboard. Queries use $cluster variable for multi-cluster support.",
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"id": null,
"links": [
{
"title": "CNPG Grafana Dashboards (official)",
"url": "https://github.com/cloudnative-pg/cloudnative-pg/tree/main/docs/src/cookbooks/grafana-dashboards"
}
],
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 100,
"title": "Cluster Overview",
"type": "row"
},
{
"datasource": {
"uid": "${datasource}"
},
"description": "CNPG cluster instance roles. healthy = primary + replicas.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "yellow",
"value": 0
},
{
"color": "green",
"value": 1
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 4,
"x": 0,
"y": 1
},
"id": 1,
"options": {
"colorMode": "background",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"datasource": {
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "sum by (role) (pg_replication_in_state{cluster=\"$cluster\", state=\"streaming\"})",
"legendFormat": "{{role}}",
"range": true,
"refId": "A"
}
],
"title": "Replica Count",
"type": "stat"
},
{
"datasource": {
"uid": "${datasource}"
},
"description": "Number of ready CNPG instances in the cluster.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "yellow",
"value": 0
},
{
"color": "green",
"value": 1
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 4,
"x": 4,
"y": 1
},
"id": 2,
"options": {
"colorMode": "background",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"datasource": {
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "sum(pg_up{cluster=\"$cluster\"})",
"legendFormat": "instances",
"range": true,
"refId": "A"
}
],
"title": "Instances Up",
"type": "stat"
},
{
"datasource": {
"uid": "${datasource}"
},
"description": "WAL sender count — indicates active replication connections.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 4,
"x": 8,
"y": 1
},
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"datasource": {
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "pg_stat_activity_count{cluster=\"$cluster\", datname=~\".+\", state=\"active\"}",
"legendFormat": "WAL senders",
"range": true,
"refId": "A"
}
],
"title": "Active Connections",
"type": "stat"
},
{
"datasource": {
"uid": "${datasource}"
},
"description": "Total size of the PostgreSQL database cluster.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 50000000000
},
{
"color": "red",
"value": 100000000000
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 4,
"x": 12,
"y": 1
},
"id": 4,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"datasource": {
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "pg_database_size_bytes{cluster=\"$cluster\", datname=\"postgres\"}",
"legendFormat": "DB size",
"range": true,
"refId": "A"
}
],
"title": "Database Size",
"type": "stat"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 6
},
"id": 200,
"title": "Replication & WAL",
"type": "row"
},
{
"datasource": {
"uid": "${datasource}"
},
"description": "Replication lag in bytes between primary and replicas.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "lag (bytes)",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "lines",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 100000000
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 7
},
"id": 10,
"options": {
"legend": {
"calcs": ["max", "mean"],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "pg_replication_lag_bytes{cluster=\"$cluster\"}",
"legendFormat": "{{pod}}",
"range": true,
"refId": "A"
}
],
"title": "Replication Lag (bytes)",
"type": "timeseries"
},
{
"datasource": {
"uid": "${datasource}"
},
"description": "WAL generation rate — volume of WAL being produced by the primary.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "W/sec",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "lines",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 7
},
"id": 11,
"options": {
"legend": {
"calcs": ["max", "mean"],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "rate(pg_stat_database_xact_commit{cluster=\"$cluster\"}[5m])",
"legendFormat": "commits/sec",
"range": true,
"refId": "A"
}
],
"title": "Transaction Commit Rate",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 16
},
"id": 300,
"title": "Performance & Queries",
"type": "row"
},
{
"datasource": {
"uid": "${datasource}"
},
"description": "Database transaction rollback rate.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "rollbacks/sec",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "lines",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 17
},
"id": 20,
"options": {
"legend": {
"calcs": ["max", "mean"],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "rate(pg_stat_database_xact_rollback{cluster=\"$cluster\"}[5m])",
"legendFormat": "{{datname}}",
"range": true,
"refId": "A"
}
],
"title": "Transaction Rollback Rate",
"type": "timeseries"
},
{
"datasource": {
"uid": "${datasource}"
},
"description": "Deadlocks detected — should be zero or near-zero.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "deadlocks/sec",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "bars",
"fillOpacity": 80,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 0
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 17
},
"id": 21,
"options": {
"legend": {
"calcs": ["max", "sum"],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "rate(pg_stat_database_deadlocks{cluster=\"$cluster\"}[5m])",
"legendFormat": "{{datname}}",
"range": true,
"refId": "A"
}
],
"title": "Deadlock Rate",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 26
},
"id": 400,
"title": "Storage & Disk I/O",
"type": "row"
},
{
"datasource": {
"uid": "${datasource}"
},
"description": "Temporary file creation rate — high values may indicate queries exceeding work_mem.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "bytes/sec",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "lines",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 27
},
"id": 30,
"options": {
"legend": {
"calcs": ["max", "mean"],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "rate(pg_stat_database_temp_bytes{cluster=\"$cluster\"}[5m])",
"legendFormat": "{{datname}}",
"range": true,
"refId": "A"
}
],
"title": "Temp File Creation Rate",
"type": "timeseries"
},
{
"datasource": {
"uid": "${datasource}"
},
"description": "Checkpointer stats — sync rate. High values indicate heavy write activity.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "syncs/sec",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "lines",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 27
},
"id": 31,
"options": {
"legend": {
"calcs": ["max", "mean"],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "rate(pg_stat_bgwriter_buffers_alloc{cluster=\"$cluster\"}[5m])",
"legendFormat": "buffer allocs/sec",
"range": true,
"refId": "A"
}
],
"title": "Buffer Allocation Rate",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 36
},
"id": 500,
"title": "Connection Pooling",
"type": "row"
},
{
"datasource": {
"uid": "${datasource}"
},
"description": "Active and idle connections per database. Approaching max_connections is a problem.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "connections",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "lines",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "normal"
},
"thresholdsStyle": {
"mode": "off"
}
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 37
},
"id": 40,
"options": {
"legend": {
"calcs": ["max", "mean"],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "sum by (datname, state) (pg_stat_activity_count{cluster=\"$cluster\"})",
"legendFormat": "{{datname}} - {{state}}",
"range": true,
"refId": "A"
}
],
"title": "Connections by State",
"type": "timeseries"
},
{
"datasource": {
"uid": "${datasource}"
},
"description": "Percentage of max_connections in use. Alert if approaching 80%.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "%",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "lines",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line+area"
}
},
"max": 100,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 50
},
{
"color": "red",
"value": 80
}
]
},
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 37
},
"id": 41,
"options": {
"legend": {
"calcs": ["max", "mean"],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "(\n sum(pg_stat_activity_count{cluster=\"$cluster\"})\n /\n pg_settings_max_connections{cluster=\"$cluster\"}\n) * 100",
"legendFormat": "connection utilization %",
"range": true,
"refId": "A"
}
],
"title": "Connection Utilization %",
"type": "timeseries"
}
],
"refresh": "30s",
"schemaVersion": 39,
"style": "dark",
"tags": [
"cnpg",
"cloudnativepg",
"postgresql",
"database"
],
"templating": {
"list": [
{
"current": {
"selected": false,
"text": "prometheus-community-kube-prometheus-stack-prometheus",
"value": "prometheus-community-kube-prometheus-stack-prometheus"
},
"hide": 0,
"includeAll": false,
"label": "Datasource",
"multi": false,
"name": "datasource",
"options": [],
"query": "prometheus-community-kube-prometheus-stack-prometheus",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
},
{
"current": {
"selected": false,
"text": "All",
"value": "$__all"
},
"datasource": {
"uid": "${datasource}"
},
"definition": "label_values(pg_up, cluster)",
"hide": 0,
"includeAll": true,
"label": "Cluster",
"multi": false,
"name": "cluster",
"options": [],
"query": {
"query": "label_values(pg_up, cluster)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"type": "query"
}
]
},
"time": {
"from": "now-3h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "CNPG / PostgreSQL — Cluster Monitoring",
"uid": "cnpg-postgres-cluster",
"version": 1
add dashboards to namespace
2026-05-04 20:38:22 +00:00
}
Reference in a new issue Copy permalink