6183159974
- Install Prometheus + Grafana via kube-prometheus-stack (ClusterIP only, no public ingress) - Deploy Tailscale Operator for secure VPN access to internal services - Add CNPG/PostgreSQL monitoring dashboards - Add vLLM inference monitoring dashboards (tokens, latency, GPU) - Add Cilium networking dashboards (policy, traffic, drops) - Update infra-controllers staging kustomization to include all controllers - Add monitoring-configs Flux sync for dashboard deployment - Update README with monitoring architecture and access instructions - Remove broken stale monitoring files (Azure Key Vault refs, wrong domains) Access: kubectl port-forward or Tailscale VPN (replace auth key before deploy)
86 lines
2.3 KiB
YAML
86 lines
2.3 KiB
YAML
apiVersion: helm.toolkit.fluxcd.io/v2
|
|
kind: HelmRelease
|
|
metadata:
|
|
name: kube-prometheus-stack
|
|
namespace: monitoring
|
|
spec:
|
|
interval: 30m
|
|
chart:
|
|
spec:
|
|
chart: kube-prometheus-stack
|
|
version: "63.x"
|
|
sourceRef:
|
|
kind: HelmRepository
|
|
name: prometheus-community
|
|
namespace: monitoring
|
|
interval: 12h
|
|
targetNamespace: monitoring
|
|
install:
|
|
createNamespace: true
|
|
crds: Create
|
|
upgrade:
|
|
crds: CreateReplace
|
|
driftDetection:
|
|
mode: enabled
|
|
ignore:
|
|
- paths: ["/metadata/annotations/prometheus-operator-validated"]
|
|
target:
|
|
kind: PrometheusRule
|
|
values:
|
|
# Alertmanager disabled - using Grafana-native alerting
|
|
alertmanager:
|
|
enabled: false
|
|
|
|
# Prometheus configuration
|
|
prometheus:
|
|
prometheusSpec:
|
|
retention: 7d
|
|
# Monitor pods from all namespaces (needed for CNPG, vLLM, etc.)
|
|
podMonitorNamespaceSelector: {}
|
|
podMonitorSelectorNilUsesHelmValues: false
|
|
probeSelectorNilUsesHelmValues: false
|
|
ruleSelectorNilUsesHelmValues: false
|
|
serviceMonitorSelectorNilUsesHelmValues: false
|
|
storageSpec:
|
|
volumeClaimTemplate:
|
|
spec:
|
|
accessModes: ["ReadWriteOnce"]
|
|
resources:
|
|
requests:
|
|
storage: 10Gi
|
|
# ServiceMonitor for Prometheus itself
|
|
serviceMonitor:
|
|
selfMonitor: true
|
|
|
|
# Grafana configuration - NO PUBLIC INGRESS, internal only
|
|
grafana:
|
|
# Admin credentials
|
|
adminUser: admin
|
|
# Password injected from monitoring-grafana-admin secret
|
|
adminPassword: "change-me-after-deploy"
|
|
|
|
# No ingress - accessed via Tailscale port-forward or kubectl port-forward
|
|
ingress:
|
|
enabled: false
|
|
|
|
# Service - internal only (ClusterIP)
|
|
service:
|
|
type: ClusterIP
|
|
port: 3000
|
|
|
|
# Sidecar to auto-load dashboards from ConfigMaps
|
|
sidecar:
|
|
dashboards:
|
|
enabled: true
|
|
label: grafana_dashboard
|
|
labelValue: "1"
|
|
searchNamespace: all
|
|
folderAnnotation: grafana_folder
|
|
provider:
|
|
allowUiUpdates: true
|
|
folders:
|
|
- folder: Kubernetes
|
|
orgId: 1
|
|
type: file
|
|
datasources:
|
|
enabled: true
|