diff --git a/.github/workflows/osint-dashboard-infra.yml b/.github/workflows/osint-dashboard-infra.yml new file mode 100644 index 0000000..3e60261 --- /dev/null +++ b/.github/workflows/osint-dashboard-infra.yml @@ -0,0 +1,130 @@ +name: OSINT Dashboard Infrastructure + +on: + push: + branches: [master] + paths: + - 'apps/base/osint-dashboard/**' + - 'apps/staging/osint-dashboard/**' + - 'clusters/devops-lab/**' + pull_request: + paths: + - 'apps/base/osint-dashboard/**' + workflow_dispatch: + +env: + REGISTRY: gcr.io/devops-lab-cluster + CHART_PATH: apps/base/osint-dashboard + +permissions: + contents: read + security-events: write + pull-requests: write + +jobs: + lint: + name: Lint Helm Chart + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Helm + uses: azure/setup-helm@v3 + with: + version: v3.14.0 + + - name: Set up chart-testing + uses: helm/chart-testing-action@v2 + + - name: Run helm lint + run: | + helm lint ${{ env.CHART_PATH }} + helm lint ${{ env.CHART_PATH }} -f ${{ env.CHART_PATH }}/values.yaml + + - name: Run chart-testing lint + run: | + ct lint --target-branch ${{ github.event.pull_request.base.ref || github.ref_name }} --chart-dirs apps/base --validate-maintainers=false + + template: + name: Render Templates + runs-on: ubuntu-latest + needs: lint + if: github.event_name == 'pull_request' + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Helm + uses: azure/setup-helm@v3 + with: + version: v3.14.0 + + - name: Render templates (dev) + run: | + helm template osint-dashboard ${{ env.CHART_PATH }} --namespace customer1 --values ${{ env.CHART_PATH }}/values.yaml --output-template-files > /dev/null + + - name: Render templates (prod override) + run: | + helm template osint-dashboard ${{ env.CHART_PATH }} --namespace customer1 --values ${{ env.CHART_PATH }}/values.yaml --set postgresql.instances=3 --set nats.replicaCount=3 --set redis.replica.replicaCount=2 --set minio.replicaCount=4 --output-template-files > /dev/null + + validate-yaml: + name: Validate YAML + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install yamllint + run: pip install yamllint + + - name: Lint YAML files + run: | + yamllint -d relaxed --ignore '*/gotk-components.yaml' ${{ env.CHART_PATH }}/Chart.yaml ${{ env.CHART_PATH }}/values.yaml ${{ env.CHART_PATH }}/templates/ + + security-scan: + name: Security Scan + runs-on: ubuntu-latest + needs: lint + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Run Trivy Helm chart scanner + uses: aquasecurity/trivy-action@master + with: + scan-type: 'config' + scan-ref: ${{ env.CHART_PATH }}/templates/ + format: 'sarif' + output: 'trivy-results.sarif' + severity: 'CRITICAL,HIGH' + + - name: Upload Trivy results + uses: github/codeql-action/upload-sarif@v3 + if: always() + with: + sarif_file: 'trivy-results.sarif' + + deploy-staging: + name: Deploy to Staging + runs-on: ubuntu-latest + needs: [lint, validate-yaml] + if: github.ref == 'refs/heads/master' && github.event_name == 'push' + environment: + name: staging + url: https://dashboard.siriusdevops.com + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Flux Kustomization already applied + run: | + echo "Flux CD will automatically pick up changes from master branch." + echo "Kustomization: customer1 -> apps/staging/customer1" + echo "No manual deploy step needed — GitOps loop handles it." + + - name: Notify deployment + if: always() + run: | + echo "Deployment triggered via Flux CD GitOps loop" + echo "Check Flux status: flux get kustomizations -n flux-system" diff --git a/apps/base/customer1/hermes-db/kafka-broker.yaml b/apps/base/customer1/hermes-db/kafka-broker.yaml new file mode 100644 index 0000000..04f86b1 --- /dev/null +++ b/apps/base/customer1/hermes-db/kafka-broker.yaml @@ -0,0 +1,128 @@ +# Kafka broker (KRaft mode — no ZooKeeper required) +# Single-broker for dev/staging; scale replicas for production +apiVersion: v1 +kind: Service +metadata: + name: trading-kafka + namespace: customer1 + labels: + app: trading-kafka +spec: + clusterIP: None + selector: + app: trading-kafka + ports: + - name: internal + port: 9092 + targetPort: 9092 + - name: controller + port: 9093 + targetPort: 9093 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: trading-kafka-config + namespace: customer1 +data: + server.properties: | + process.roles=broker,controller + node.id=1 + controller.quorum.voters=1@trading-kafka-0.trading-kafka.customer1.svc.cluster.local:9093 + listeners=PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093 + advertised.listeners=PLAINTEXT://trading-kafka-0.trading-kafka.customer1.svc.cluster.local:9092 + listener.security.protocol.map=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT + controller.listener.names=CONTROLLER + inter.broker.listener.name=PLAINTEXT + log.dirs=/var/lib/kafka/data + num.partitions=3 + default.replication.factor=1 + offsets.topic.replication.factor=1 + transaction.state.log.replication.factor=1 + transaction.state.log.min.isr=1 + auto.create.topics.enable=true +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: trading-kafka + namespace: customer1 + labels: + app: trading-kafka +spec: + serviceName: trading-kafka + replicas: 1 + selector: + matchLabels: + app: trading-kafka + template: + metadata: + labels: + app: trading-kafka + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + containers: + - name: kafka + image: apache/kafka:3.9.0 + ports: + - containerPort: 9092 + name: internal + - containerPort: 9093 + name: controller + env: + - name: KAFKA_HEAP_OPTS + value: "-Xmx512M -Xms256M" + - name: CLUSTER_ID + value: "trading-kafka-cluster-01" + command: + - /bin/bash + - -c + - | + export KAFKA_CLUSTER_ID="$(/opt/kafka/bin/kafka-storage.sh random-uuid)" + /opt/kafka/bin/kafka-storage.sh format -t $KAFKA_CLUSTER_ID -c /etc/kafka/server.properties --ignore-formatted + exec /opt/kafka/bin/kafka-server-start.sh /etc/kafka/server.properties + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 1000m + memory: 1Gi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + capabilities: + drop: + - ALL + volumeMounts: + - name: config + mountPath: /etc/kafka + - name: data + mountPath: /var/lib/kafka/data + readinessProbe: + tcpSocket: + port: 9092 + initialDelaySeconds: 30 + periodSeconds: 10 + failureThreshold: 5 + livenessProbe: + tcpSocket: + port: 9092 + initialDelaySeconds: 60 + periodSeconds: 30 + volumes: + - name: config + configMap: + name: trading-kafka-config + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/apps/base/customer1/hermes-db/kustomization.yaml b/apps/base/customer1/hermes-db/kustomization.yaml index 9f9677f..7047ec6 100644 --- a/apps/base/customer1/hermes-db/kustomization.yaml +++ b/apps/base/customer1/hermes-db/kustomization.yaml @@ -9,3 +9,5 @@ resources: - trading-data-db.yaml - agent-memory-db.yaml - hermes-scheduled-backup.yaml + - kafka-broker.yaml + - redis-cluster.yaml diff --git a/apps/base/customer1/hermes-db/redis-cluster.yaml b/apps/base/customer1/hermes-db/redis-cluster.yaml new file mode 100644 index 0000000..fadd4d4 --- /dev/null +++ b/apps/base/customer1/hermes-db/redis-cluster.yaml @@ -0,0 +1,103 @@ +# Redis single-instance for trading platform caching +apiVersion: v1 +kind: Service +metadata: + name: trading-redis + namespace: customer1 + labels: + app: trading-redis +spec: + selector: + app: trading-redis + ports: + - port: 6379 + targetPort: 6379 + name: redis + type: ClusterIP +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: trading-redis-config + namespace: customer1 +data: + redis.conf: | + maxmemory 256mb + maxmemory-policy allkeys-lru + save 900 1 + save 300 10 + save 60 10000 + appendonly yes + appendfsync everysec + dir /data +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: trading-redis + namespace: customer1 + labels: + app: trading-redis +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: trading-redis + template: + metadata: + labels: + app: trading-redis + spec: + securityContext: + runAsNonRoot: true + runAsUser: 999 + fsGroup: 999 + containers: + - name: redis + image: redis:7.4-alpine + ports: + - containerPort: 6379 + name: redis + args: + - redis-server + - /etc/redis/redis.conf + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: config + mountPath: /etc/redis + - name: data + mountPath: /data + readinessProbe: + exec: + command: + - redis-cli + - ping + initialDelaySeconds: 5 + periodSeconds: 5 + livenessProbe: + exec: + command: + - redis-cli + - ping + initialDelaySeconds: 10 + periodSeconds: 10 + volumes: + - name: config + configMap: + name: trading-redis-config + - name: data + emptyDir: {} diff --git a/apps/base/customer1/kustomization.yaml b/apps/base/customer1/kustomization.yaml index 51573fe..1ac48ec 100644 --- a/apps/base/customer1/kustomization.yaml +++ b/apps/base/customer1/kustomization.yaml @@ -6,4 +6,5 @@ resources: - http-route.yaml - service.yaml - siriusdevops-site - - waitlist-api \ No newline at end of file + - waitlist-api + - trading-platform diff --git a/apps/base/customer1/trading-platform/configmaps/dashboard-config.yaml b/apps/base/customer1/trading-platform/configmaps/dashboard-config.yaml new file mode 100644 index 0000000..4fcdaee --- /dev/null +++ b/apps/base/customer1/trading-platform/configmaps/dashboard-config.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: trading-dashboard-config + namespace: customer1 +data: + DB_HOST: "hermes-pgdb-rw.customer1.svc.cluster.local" + DB_PORT: "5432" + DB_NAME: "trading_data" + DATA_SERVICE_URL: "http://trading-data-service.customer1.svc.cluster.local" + EXECUTE_SERVICE_URL: "http://trading-execute-service.customer1.svc.cluster.local" + NEWS_SERVICE_URL: "http://trading-news-service.customer1.svc.cluster.local" diff --git a/apps/base/customer1/trading-platform/configmaps/data-service-config.yaml b/apps/base/customer1/trading-platform/configmaps/data-service-config.yaml new file mode 100644 index 0000000..c9dc9e2 --- /dev/null +++ b/apps/base/customer1/trading-platform/configmaps/data-service-config.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: trading-data-service-config + namespace: customer1 +data: + DB_HOST: "hermes-pgdb-rw.customer1.svc.cluster.local" + DB_PORT: "5432" + DB_NAME: "trading_data" + REDIS_HOST: "trading-redis.customer1.svc.cluster.local" + REDIS_PORT: "6379" + KAFKA_BROKER: "trading-kafka.customer1.svc.cluster.local:9092" + LOG_LEVEL: "info" diff --git a/apps/base/customer1/trading-platform/configmaps/execute-service-config.yaml b/apps/base/customer1/trading-platform/configmaps/execute-service-config.yaml new file mode 100644 index 0000000..090157f --- /dev/null +++ b/apps/base/customer1/trading-platform/configmaps/execute-service-config.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: trading-execute-service-config + namespace: customer1 +data: + DB_HOST: "hermes-pgdb-rw.customer1.svc.cluster.local" + DB_PORT: "5432" + DB_NAME: "trading_data" + REDIS_HOST: "trading-redis.customer1.svc.cluster.local" + REDIS_PORT: "6379" + KAFKA_BROKER: "trading-kafka.customer1.svc.cluster.local:9092" + LOG_LEVEL: "info" diff --git a/apps/base/customer1/trading-platform/configmaps/kustomization.yaml b/apps/base/customer1/trading-platform/configmaps/kustomization.yaml new file mode 100644 index 0000000..e34cabf --- /dev/null +++ b/apps/base/customer1/trading-platform/configmaps/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - data-service-config.yaml + - execute-service-config.yaml + - news-service-config.yaml + - dashboard-config.yaml diff --git a/apps/base/customer1/trading-platform/configmaps/news-service-config.yaml b/apps/base/customer1/trading-platform/configmaps/news-service-config.yaml new file mode 100644 index 0000000..ec4e416 --- /dev/null +++ b/apps/base/customer1/trading-platform/configmaps/news-service-config.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: trading-news-service-config + namespace: customer1 +data: + DB_HOST: "hermes-pgdb-rw.customer1.svc.cluster.local" + DB_PORT: "5432" + DB_NAME: "trading_data" + REDIS_HOST: "trading-redis.customer1.svc.cluster.local" + REDIS_PORT: "6379" + NEWS_FETCH_INTERVAL: "300" + LOG_LEVEL: "info" diff --git a/apps/base/customer1/trading-platform/dashboard/deployment.yaml b/apps/base/customer1/trading-platform/dashboard/deployment.yaml new file mode 100644 index 0000000..c2f3c1e --- /dev/null +++ b/apps/base/customer1/trading-platform/dashboard/deployment.yaml @@ -0,0 +1,79 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: trading-dashboard + namespace: customer1 + labels: + app: trading-dashboard +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: trading-dashboard + template: + metadata: + labels: + app: trading-dashboard + annotations: + checksum/config: trading-dashboard-config + spec: + terminationGracePeriodSeconds: 30 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + containers: + - name: dashboard + image: ghcr.io/sirius0xdev/trading-dashboard:latest + imagePullPolicy: Always + ports: + - containerPort: 8000 + name: http + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + envFrom: + - configMapRef: + name: trading-dashboard-config + env: + - name: DB_USER + valueFrom: + secretKeyRef: + name: trading-db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: trading-db-credentials + key: password + startupProbe: + httpGet: + path: /api/health + port: 8000 + initialDelaySeconds: 15 + periodSeconds: 5 + failureThreshold: 6 + livenessProbe: + httpGet: + path: /api/health + port: 8000 + initialDelaySeconds: 30 + periodSeconds: 30 + readinessProbe: + httpGet: + path: /api/health + port: 8000 + initialDelaySeconds: 5 + periodSeconds: 10 diff --git a/apps/base/customer1/trading-platform/dashboard/kustomization.yaml b/apps/base/customer1/trading-platform/dashboard/kustomization.yaml new file mode 100644 index 0000000..5b98e94 --- /dev/null +++ b/apps/base/customer1/trading-platform/dashboard/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - deployment.yaml + - service.yaml diff --git a/apps/base/customer1/trading-platform/dashboard/service.yaml b/apps/base/customer1/trading-platform/dashboard/service.yaml new file mode 100644 index 0000000..73bca2f --- /dev/null +++ b/apps/base/customer1/trading-platform/dashboard/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: trading-dashboard-svc + namespace: customer1 + annotations: + tailscale.com/expose: "true" + tailscale.com/hostname: "trading-dashboard" + tailscale.com/tags: "tag:k8s-operator" + tailscale.com/ports: "http:80" +spec: + selector: + app: trading-dashboard + ports: + - port: 80 + targetPort: 8000 + name: http + type: ClusterIP diff --git a/apps/base/customer1/trading-platform/data-service/deployment.yaml b/apps/base/customer1/trading-platform/data-service/deployment.yaml new file mode 100644 index 0000000..f1f88a7 --- /dev/null +++ b/apps/base/customer1/trading-platform/data-service/deployment.yaml @@ -0,0 +1,79 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: trading-data-service + namespace: customer1 + labels: + app: trading-data-service +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: trading-data-service + template: + metadata: + labels: + app: trading-data-service + annotations: + checksum/config: trading-data-service-config + spec: + terminationGracePeriodSeconds: 30 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + containers: + - name: data-service + image: ghcr.io/sirius0xdev/trading-data-service:latest + imagePullPolicy: Always + ports: + - containerPort: 8001 + name: http + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + envFrom: + - configMapRef: + name: trading-data-service-config + env: + - name: DB_USER + valueFrom: + secretKeyRef: + name: trading-db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: trading-db-credentials + key: password + startupProbe: + httpGet: + path: /health + port: 8001 + initialDelaySeconds: 15 + periodSeconds: 5 + failureThreshold: 6 + livenessProbe: + httpGet: + path: /health + port: 8001 + initialDelaySeconds: 30 + periodSeconds: 30 + readinessProbe: + httpGet: + path: /health + port: 8001 + initialDelaySeconds: 5 + periodSeconds: 10 diff --git a/apps/base/customer1/trading-platform/data-service/kustomization.yaml b/apps/base/customer1/trading-platform/data-service/kustomization.yaml new file mode 100644 index 0000000..5b98e94 --- /dev/null +++ b/apps/base/customer1/trading-platform/data-service/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - deployment.yaml + - service.yaml diff --git a/apps/base/customer1/trading-platform/data-service/service.yaml b/apps/base/customer1/trading-platform/data-service/service.yaml new file mode 100644 index 0000000..562d721 --- /dev/null +++ b/apps/base/customer1/trading-platform/data-service/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: trading-data-service + namespace: customer1 + annotations: + tailscale.com/expose: "true" + tailscale.com/hostname: "trading-data-service" + tailscale.com/tags: "tag:k8s-operator" + tailscale.com/ports: "http:80" +spec: + selector: + app: trading-data-service + ports: + - port: 80 + targetPort: 8001 + name: http + type: ClusterIP diff --git a/apps/base/customer1/trading-platform/execute-service/deployment.yaml b/apps/base/customer1/trading-platform/execute-service/deployment.yaml new file mode 100644 index 0000000..2881071 --- /dev/null +++ b/apps/base/customer1/trading-platform/execute-service/deployment.yaml @@ -0,0 +1,79 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: trading-execute-service + namespace: customer1 + labels: + app: trading-execute-service +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: trading-execute-service + template: + metadata: + labels: + app: trading-execute-service + annotations: + checksum/config: trading-execute-service-config + spec: + terminationGracePeriodSeconds: 30 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + containers: + - name: execute-service + image: ghcr.io/sirius0xdev/trading-execute-service:latest + imagePullPolicy: Always + ports: + - containerPort: 8002 + name: http + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + envFrom: + - configMapRef: + name: trading-execute-service-config + env: + - name: DB_USER + valueFrom: + secretKeyRef: + name: trading-db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: trading-db-credentials + key: password + startupProbe: + httpGet: + path: /health + port: 8002 + initialDelaySeconds: 15 + periodSeconds: 5 + failureThreshold: 6 + livenessProbe: + httpGet: + path: /health + port: 8002 + initialDelaySeconds: 30 + periodSeconds: 30 + readinessProbe: + httpGet: + path: /health + port: 8002 + initialDelaySeconds: 5 + periodSeconds: 10 diff --git a/apps/base/customer1/trading-platform/execute-service/kustomization.yaml b/apps/base/customer1/trading-platform/execute-service/kustomization.yaml new file mode 100644 index 0000000..5b98e94 --- /dev/null +++ b/apps/base/customer1/trading-platform/execute-service/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - deployment.yaml + - service.yaml diff --git a/apps/base/customer1/trading-platform/execute-service/service.yaml b/apps/base/customer1/trading-platform/execute-service/service.yaml new file mode 100644 index 0000000..49a0b42 --- /dev/null +++ b/apps/base/customer1/trading-platform/execute-service/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: trading-execute-service + namespace: customer1 + annotations: + tailscale.com/expose: "true" + tailscale.com/hostname: "trading-execute-service" + tailscale.com/tags: "tag:k8s-operator" + tailscale.com/ports: "http:80" +spec: + selector: + app: trading-execute-service + ports: + - port: 80 + targetPort: 8002 + name: http + type: ClusterIP diff --git a/apps/base/customer1/trading-platform/kustomization.yaml b/apps/base/customer1/trading-platform/kustomization.yaml new file mode 100644 index 0000000..f7775fe --- /dev/null +++ b/apps/base/customer1/trading-platform/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: customer1 + +resources: + - data-service + - execute-service + - news-service + - dashboard + - configmaps + - secrets + - network-policies + - routes diff --git a/apps/base/customer1/trading-platform/network-policies/kustomization.yaml b/apps/base/customer1/trading-platform/network-policies/kustomization.yaml new file mode 100644 index 0000000..0e9a94a --- /dev/null +++ b/apps/base/customer1/trading-platform/network-policies/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - trading-network-policies.yaml diff --git a/apps/base/customer1/trading-platform/network-policies/trading-network-policies.yaml b/apps/base/customer1/trading-platform/network-policies/trading-network-policies.yaml new file mode 100644 index 0000000..a553f07 --- /dev/null +++ b/apps/base/customer1/trading-platform/network-policies/trading-network-policies.yaml @@ -0,0 +1,109 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: trading-platform-netpol + namespace: customer1 +spec: + podSelector: + matchExpressions: + - key: app + operator: In + values: + - trading-data-service + - trading-execute-service + - trading-news-service + - trading-dashboard + policyTypes: + - Ingress + - Egress + ingress: + # Allow from Gateway / ingress controller + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: customer1 + ports: + - port: 8000 + protocol: TCP + - port: 8001 + protocol: TCP + - port: 8002 + protocol: TCP + - port: 8003 + protocol: TCP + # Allow inter-service communication + - from: + - podSelector: + matchExpressions: + - key: app + operator: In + values: + - trading-data-service + - trading-execute-service + - trading-news-service + - trading-dashboard + ports: + - port: 8000 + protocol: TCP + - port: 8001 + protocol: TCP + - port: 8002 + protocol: TCP + - port: 8003 + protocol: TCP + egress: + # Allow DNS + - to: + - namespaceSelector: {} + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + # Allow DB access + - to: + - podSelector: + matchLabels: + cnpg.io/cluster: hermes-pgdb + ports: + - port: 5432 + protocol: TCP + # Allow Redis access + - to: + - podSelector: + matchLabels: + app: trading-redis + ports: + - port: 6379 + protocol: TCP + # Allow Kafka access + - to: + - podSelector: + matchLabels: + app: trading-kafka + ports: + - port: 9092 + protocol: TCP + # Allow inter-service egress + - to: + - podSelector: + matchExpressions: + - key: app + operator: In + values: + - trading-data-service + - trading-execute-service + - trading-news-service + - trading-dashboard + ports: + - port: 8000 + protocol: TCP + - port: 8001 + protocol: TCP + - port: 8002 + protocol: TCP + - port: 8003 + protocol: TCP diff --git a/apps/base/customer1/trading-platform/news-service/deployment.yaml b/apps/base/customer1/trading-platform/news-service/deployment.yaml new file mode 100644 index 0000000..dda5c98 --- /dev/null +++ b/apps/base/customer1/trading-platform/news-service/deployment.yaml @@ -0,0 +1,79 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: trading-news-service + namespace: customer1 + labels: + app: trading-news-service +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: trading-news-service + template: + metadata: + labels: + app: trading-news-service + annotations: + checksum/config: trading-news-service-config + spec: + terminationGracePeriodSeconds: 30 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + containers: + - name: news-service + image: ghcr.io/sirius0xdev/trading-news-service:latest + imagePullPolicy: Always + ports: + - containerPort: 8003 + name: http + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + envFrom: + - configMapRef: + name: trading-news-service-config + env: + - name: DB_USER + valueFrom: + secretKeyRef: + name: trading-db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: trading-db-credentials + key: password + startupProbe: + httpGet: + path: /health + port: 8003 + initialDelaySeconds: 15 + periodSeconds: 5 + failureThreshold: 6 + livenessProbe: + httpGet: + path: /health + port: 8003 + initialDelaySeconds: 30 + periodSeconds: 30 + readinessProbe: + httpGet: + path: /health + port: 8003 + initialDelaySeconds: 5 + periodSeconds: 10 diff --git a/apps/base/customer1/trading-platform/news-service/kustomization.yaml b/apps/base/customer1/trading-platform/news-service/kustomization.yaml new file mode 100644 index 0000000..5b98e94 --- /dev/null +++ b/apps/base/customer1/trading-platform/news-service/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - deployment.yaml + - service.yaml diff --git a/apps/base/customer1/trading-platform/news-service/service.yaml b/apps/base/customer1/trading-platform/news-service/service.yaml new file mode 100644 index 0000000..0556352 --- /dev/null +++ b/apps/base/customer1/trading-platform/news-service/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: trading-news-service + namespace: customer1 + annotations: + tailscale.com/expose: "true" + tailscale.com/hostname: "trading-news-service" + tailscale.com/tags: "tag:k8s-operator" + tailscale.com/ports: "http:80" +spec: + selector: + app: trading-news-service + ports: + - port: 80 + targetPort: 8003 + name: http + type: ClusterIP diff --git a/apps/base/customer1/trading-platform/routes/http-routes.yaml b/apps/base/customer1/trading-platform/routes/http-routes.yaml new file mode 100644 index 0000000..794af95 --- /dev/null +++ b/apps/base/customer1/trading-platform/routes/http-routes.yaml @@ -0,0 +1,53 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: trading-dashboard-route + namespace: customer1 +spec: + parentRefs: + - name: external-http-gateway + hostnames: + - "sirius-sec.com" + - "www.sirius-sec.com" + rules: + - matches: + - path: + type: PathPrefix + value: /trade + backendRefs: + - name: trading-dashboard-svc + port: 80 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: trading-api-routes + namespace: customer1 +spec: + parentRefs: + - name: external-http-gateway + hostnames: + - "sirius-sec.com" + - "www.sirius-sec.com" + rules: + - matches: + - path: + type: PathPrefix + value: /api/data + backendRefs: + - name: trading-data-service + port: 80 + - matches: + - path: + type: PathPrefix + value: /api/execute + backendRefs: + - name: trading-execute-service + port: 80 + - matches: + - path: + type: PathPrefix + value: /api/news + backendRefs: + - name: trading-news-service + port: 80 diff --git a/apps/base/customer1/trading-platform/routes/kustomization.yaml b/apps/base/customer1/trading-platform/routes/kustomization.yaml new file mode 100644 index 0000000..e025033 --- /dev/null +++ b/apps/base/customer1/trading-platform/routes/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - http-routes.yaml diff --git a/apps/base/customer1/trading-platform/secrets/kustomization.yaml b/apps/base/customer1/trading-platform/secrets/kustomization.yaml new file mode 100644 index 0000000..c7525ed --- /dev/null +++ b/apps/base/customer1/trading-platform/secrets/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - trading-secrets.yaml diff --git a/apps/base/customer1/trading-platform/secrets/trading-secrets.yaml b/apps/base/customer1/trading-platform/secrets/trading-secrets.yaml new file mode 100644 index 0000000..d2c10e1 --- /dev/null +++ b/apps/base/customer1/trading-platform/secrets/trading-secrets.yaml @@ -0,0 +1,32 @@ +apiVersion: ENC[AES256_GCM,data:j4g=,iv:Bb3E3dbyD1MUIsthCltT2rRNNorduAL7QmPG8oRy31g=,tag:htxjh/00z6RGQNNyKCUviw==,type:str] +kind: ENC[AES256_GCM,data:k3FRKBzN,iv:Pvtu7bnGWQNl9pkjtU8GpHjpeg00oFXatf3jxxtQEpw=,tag:zj/yAy/DpCfLO7MdH3hiMA==,type:str] +metadata: + name: ENC[AES256_GCM,data:ZpbqQd2YyEfnugJ27JVAmqBf8Q7m75B0,iv:4njbS+2AYJuxzNERSOjOH8wmoI1KM0ocUT7OgeoGEgo=,tag:spwlHcpeQM0G8O3VPHoDOg==,type:str] + namespace: ENC[AES256_GCM,data:4ErEXcBaI8Yp,iv:IKHgZ6Gm5X21Atnnm2xOFU11IgSfw5X5Wdnl25EDyOI=,tag:znkdsNDkIb4mxBY4yJbX9g==,type:str] +type: ENC[AES256_GCM,data:dd7uKLw+,iv:qRkV8K+ytp55rLGNIP1lG2yZ+LENt/FkdDiWzi/1tik=,tag:aUWQ+ZOQS7/hXcnceCyrTQ==,type:str] +stringData: + #ENC[AES256_GCM,data:Xol7d8ednDll9VKfZ62jZRdARcmz8UJ/6ovDaw8OHrPb72aUdA==,iv:TvGk+LiK0+maCioF5daeWDTNKOSRA5pBFf9PgKL/9Z4=,tag:34kee3TQNsCdIpyEtE12mA==,type:comment] + news-api-key: ENC[AES256_GCM,data:eNMLhs55u3bwVqD4l2tQhgq1+wp/9rCa,iv:7BjlJqgJbg6BqdXxNphnIWKA/LYFZJc3qqJ360/EteY=,tag:hs5vfj+zP9OCq2tDucItIg==,type:str] + market-data-api-key: ENC[AES256_GCM,data:ol3aAC9ijFcFUo7jEVQv7mKjCLpUXBuaVThwRzG/fQ==,iv:YouuIdmj6aK0tuYJtDuiOt91gnUtK2xdshVZxy+gCH4=,tag:MAs8mS/+ndC8AYdI+WdFcA==,type:str] + #ENC[AES256_GCM,data:Si0AKl5/sWrYDNSYiC35iD7vtvP9CzAb,iv:cyBrgR5cFSta+bPdyyCUXrKH68Hi84UbGFqiWskQb7s=,tag:w9+sotMjBTuyfcGYIHfN+w==,type:comment] + service-auth-token: ENC[AES256_GCM,data:NCu/mBKdGIFNXTA5n7M53UXjMaZy9UdzImC90GPU,iv:JPLeNfTDWV3VLbVIUidL6w1IrJN09LTGIyPn0tJlj90=,tag:RIzC8FfGXBmE2PRZzgaAww==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age1uuxf066xuuqgvjppxfcmqkwfcufnwp3wcwnl9h20g9k4l8nkw9jsaungf7 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBFTzBrN3V5elJmNjNGZ28y + Q1JOQUt5eURqSFZNZkJUNWMrV1NTZEFiYkgwCnlDSTFYTW8zcU02M0NJdXZjYSti + Ylh0bWpJd0k2MWt6VjNNTUlNU0pTUVUKLS0tIENHUEU4VElXbC96bXBGRmo3QXpQ + ZkxxUDRubGt0dnRoQXVtS2xFSnhTRkUKs+rcKiZvgA7mffGo7GkkFL4vWnTIGAIn + RXwlbDNPEhiK+6lh/TgkV2CYXDBt1Hwfk4fzhZknYjY3Psp0ufvY2Q== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2026-05-21T04:08:04Z" + mac: ENC[AES256_GCM,data:EEuPQ1n7qAab7xkQYNt4rxzNy+u6PSYn+hFUTv3ZwmKbMHW7zQY3BgkwalDVrTC4ZNOLy3tGjVDB6v0KXmwXiXwvYL0Y17h8zRiU4id+zQl+oeZTMCFoUZ5Piz69DxO06cMaZF7+6K+9uQ0JLkZsnY3hb82xKAKbl9E/MFdvr6s=,iv:ldIAz5IKNFnbvcNpzo9qX6n0evix7tsLcTPiouB8lfk=,tag:KXDYrz4vfXkWH0cUHsOUdw==,type:str] + pgp: [] + unencrypted_suffix: _unencrypted + version: 3.9.4 diff --git a/apps/base/osint-dashboard/Chart.yaml b/apps/base/osint-dashboard/Chart.yaml new file mode 100644 index 0000000..2bff7a8 --- /dev/null +++ b/apps/base/osint-dashboard/Chart.yaml @@ -0,0 +1,15 @@ +# OSINT Dashboard — Helm Chart +apiVersion: v2 +name: osint-dashboard +description: Real-time geospatial OSINT dashboard infrastructure +type: application +version: 0.1.0 +appVersion: "1.0.0" +keywords: + - osint + - dashboard + - geospatial + - real-time +maintainers: + - name: sec-ops + email: sec-ops@osint-dashboard.local diff --git a/apps/base/osint-dashboard/templates/_helpers.tpl b/apps/base/osint-dashboard/templates/_helpers.tpl new file mode 100644 index 0000000..fb1122f --- /dev/null +++ b/apps/base/osint-dashboard/templates/_helpers.tpl @@ -0,0 +1,67 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "osint-dashboard.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "osint-dashboard.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 64 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 64 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 64 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "osint-dashboard.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "osint-dashboard.labels" -}} +helm.sh/chart: {{ include "osint-dashboard.chart" . }} +{{ include "osint-dashboard.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "osint-dashboard.selectorLabels" -}} +app.kubernetes.io/name: {{ include "osint-dashboard.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Namespace labels +*/}} +{{- define "osint-dashboard.namespaceLabels" -}} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "osint-dashboard.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "osint-dashboard.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/apps/base/osint-dashboard/templates/ingress/ingress.yaml b/apps/base/osint-dashboard/templates/ingress/ingress.yaml new file mode 100644 index 0000000..1893432 --- /dev/null +++ b/apps/base/osint-dashboard/templates/ingress/ingress.yaml @@ -0,0 +1,47 @@ +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: {{ $host.host | replace "." "-" | trunc 50 | trimSuffix "-" }} + namespace: {{ $.Values.namespace }} + labels: + {{- include "osint-dashboard.labels" $ | nindent 4 }} +spec: + parentRefs: + - name: external-http-gateway + hostnames: + - "{{ $host.host }}" + rules: + {{- range $rule := $host.paths }} + - matches: + - path: + type: PathPrefix + value: {{ $rule.path }} + backendRefs: + - name: dashboard-web + port: 3000 + weight: 100 + {{- end }} +--- +{{- end }} + +# TLS Certificate resources +{{- range $tls := .Values.ingress.tls }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ $tls.secretName }} + namespace: {{ $.Values.namespace }} + labels: + {{- include "osint-dashboard.labels" $ | nindent 4 }} +spec: + secretName: {{ $tls.secretName }} + issuerRef: + name: {{ $.Values.ingress.certManager.clusterIssuerName }} + kind: ClusterIssuer + dnsNames: + {{- toYaml $tls.hosts | nindent 4 }} +--- +{{- end }} +{{- end }} diff --git a/apps/base/osint-dashboard/templates/minio/buckets-init-job.yaml b/apps/base/osint-dashboard/templates/minio/buckets-init-job.yaml new file mode 100644 index 0000000..a069a8e --- /dev/null +++ b/apps/base/osint-dashboard/templates/minio/buckets-init-job.yaml @@ -0,0 +1,61 @@ +{{- if .Values.minio.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: minio-buckets-init + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: object-storage + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded +spec: + template: + metadata: + labels: + {{- include "osint-dashboard.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: object-storage + spec: + restartPolicy: OnFailure + securityContext: + runAsNonRoot: true + runAsUser: 1000 + containers: + - name: mc + image: "{{ .Values.minio.image.repository }}:{{ .Values.minio.image.tag }}" + envFrom: + - secretRef: + name: {{ .Values.minio.credentialsSecret }} + command: + - /bin/sh + - -c + args: + - | + # Wait for MinIO to be ready + until curl -sf http://minio:{{ .Values.minio.ports.api }}/minio/health/live; do + echo "Waiting for MinIO..." + sleep 2 + done + + # Configure mc alias + mc alias set myminio http://minio:{{ .Values.minio.ports.api }} $MINIO_ROOT_USER $MINIO_ROOT_PASSWORD + + # Create buckets + mc mb --ignore-existing myminio/osint-video-clips + mc mb --ignore-existing myminio/osint-satellite-tiles + mc mb --ignore-existing myminio/osint-data-dumps + + echo "MinIO buckets initialized successfully" + resources: + requests: + cpu: "100m" + memory: "64Mi" + limits: + cpu: "500m" + memory: "256Mi" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] +{{- end }} diff --git a/apps/base/osint-dashboard/templates/minio/credentials-secret.yaml b/apps/base/osint-dashboard/templates/minio/credentials-secret.yaml new file mode 100644 index 0000000..e66560a --- /dev/null +++ b/apps/base/osint-dashboard/templates/minio/credentials-secret.yaml @@ -0,0 +1,15 @@ +{{- if .Values.minio.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.minio.credentialsSecret }} + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + annotations: + # SOPS will encrypt this +type: Opaque +stringData: + MINIO_ROOT_USER: CHANGE_ME_USE_SOPS + MINIO_ROOT_PASSWORD: CHANGE_ME_USE_SOPS +{{- end }} diff --git a/apps/base/osint-dashboard/templates/minio/service.yaml b/apps/base/osint-dashboard/templates/minio/service.yaml new file mode 100644 index 0000000..be37a1d --- /dev/null +++ b/apps/base/osint-dashboard/templates/minio/service.yaml @@ -0,0 +1,49 @@ +{{- if .Values.minio.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: minio + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: object-storage +spec: + type: ClusterIP + ports: + - port: {{ .Values.minio.ports.api }} + targetPort: {{ .Values.minio.ports.api }} + protocol: TCP + name: api + - port: {{ .Values.minio.ports.console }} + targetPort: {{ .Values.minio.ports.console }} + protocol: TCP + name: console + selector: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: object-storage +--- +apiVersion: v1 +kind: Service +metadata: + name: minio-headless + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: object-storage + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" +spec: + clusterIP: None + ports: + - port: {{ .Values.minio.ports.api }} + targetPort: {{ .Values.minio.ports.api }} + protocol: TCP + name: api + - port: {{ .Values.minio.ports.console }} + targetPort: {{ .Values.minio.ports.console }} + protocol: TCP + name: console + selector: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: object-storage +{{- end }} diff --git a/apps/base/osint-dashboard/templates/minio/statefulset.yaml b/apps/base/osint-dashboard/templates/minio/statefulset.yaml new file mode 100644 index 0000000..17b7bed --- /dev/null +++ b/apps/base/osint-dashboard/templates/minio/statefulset.yaml @@ -0,0 +1,84 @@ +{{- if .Values.minio.enabled }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: minio + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: object-storage +spec: + serviceName: minio-headless + replicas: {{ .Values.minio.replicaCount }} + selector: + matchLabels: + {{- include "osint-dashboard.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: object-storage + template: + metadata: + labels: + {{- include "osint-dashboard.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: object-storage + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "{{ .Values.minio.ports.api }}" + prometheus.io/path: "/minio/v2/metrics/cluster" + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + containers: + - name: minio + image: "{{ .Values.minio.image.repository }}:{{ .Values.minio.image.tag }}" + ports: + - name: api + containerPort: {{ .Values.minio.ports.api }} + - name: console + containerPort: {{ .Values.minio.ports.console }} + resources: + {{- toYaml .Values.minio.resources | nindent 12 }} + envFrom: + - secretRef: + name: {{ .Values.minio.credentialsSecret }} + env: + - name: MINIO_SERVER_URL + value: "http://minio.{{ .Values.namespace }}.svc:{{ .Values.minio.ports.api }}" + args: + - server + - "--console-address" + - ":{{ .Values.minio.ports.console }}" + - "--address" + - ":{{ .Values.minio.ports.api }}" + # Distributed mode: all 4 pods + - "http://minio-{0...3}.minio-headless.{{ .Values.namespace }}.svc/data" + volumeMounts: + - name: data + mountPath: /data + readinessProbe: + httpGet: + path: /minio/health/live + port: api + initialDelaySeconds: 5 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /minio/health/live + port: api + initialDelaySeconds: 10 + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.minio.storage.size }} + storageClassName: {{ .Values.minio.storage.storageClass }} +{{- end }} diff --git a/apps/base/osint-dashboard/templates/monitoring/alertmanager.yaml b/apps/base/osint-dashboard/templates/monitoring/alertmanager.yaml new file mode 100644 index 0000000..e745a12 --- /dev/null +++ b/apps/base/osint-dashboard/templates/monitoring/alertmanager.yaml @@ -0,0 +1,134 @@ +{{- if .Values.monitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: alertmanager-config + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: monitoring +data: + alertmanager.yml: | + global: + resolve_timeout: 5m + + route: + group_by: ['alertname', 'namespace'] + group_wait: 30s + group_interval: 5m + repeat_interval: 4h + receiver: 'default' + routes: + - match: + severity: critical + receiver: 'pager' + repeat_interval: 1h + - match: + severity: warning + receiver: 'slack' + + receivers: + - name: 'default' + email_configs: + - to: CHANGE_ME_USE_SOPS + from: monitoring@{{ .Values.monitoring.defaultEmailDomain }} + smarthost: CHANGE_ME_USE_SOPS + auth_username: CHANGE_ME_USE_SOPS + auth_password: CHANGE_ME_USE_SOPS + + - name: 'pager' + webhook_configs: + - url: CHANGE_ME_USE_SOPS + send_resolved: true + + - name: 'slack' + slack_configs: + - api_url: CHANGE_ME_USE_SOPS + channel: '#osint-alerts' + send_resolved: true + title: '{{ .GroupLabels.alertname }}' + text: >- + {{ range .Alerts }} + *Alert:* {{ .Labels.alertname }} + *Severity:* {{ .Labels.severity }} + *Summary:* {{ .Annotations.summary }} + {{ end }} + + inhibit_rules: + - source_match: + severity: 'critical' + target_match: + severity: 'warning' + equal: ['alertname', 'namespace'] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: alertmanager + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: monitoring +spec: + replicas: 1 + selector: + matchLabels: + {{- include "osint-dashboard.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: monitoring + template: + metadata: + labels: + {{- include "osint-dashboard.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: monitoring + spec: + securityContext: + runAsNonRoot: true + runAsUser: 65534 + fsGroup: 65534 + containers: + - name: alertmanager + image: "{{ .Values.monitoring.alertmanager.image.repository }}:{{ .Values.monitoring.alertmanager.image.tag }}" + ports: + - containerPort: {{ .Values.monitoring.alertmanager.port }} + name: web + args: + - "--config.file=/etc/alertmanager/alertmanager.yml" + - "--storage.path=/alertmanager" + resources: + {{- toYaml .Values.monitoring.alertmanager.resources | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/alertmanager + - name: data + mountPath: /alertmanager + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumes: + - name: config + configMap: + name: alertmanager-config + strategy: + type: Recreate +--- +apiVersion: v1 +kind: Service +metadata: + name: alertmanager + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: monitoring +spec: + type: ClusterIP + ports: + - port: {{ .Values.monitoring.alertmanager.port }} + targetPort: web + protocol: TCP + name: web + selector: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: monitoring +{{- end }} diff --git a/apps/base/osint-dashboard/templates/monitoring/grafana.yaml b/apps/base/osint-dashboard/templates/monitoring/grafana.yaml new file mode 100644 index 0000000..cf3be67 --- /dev/null +++ b/apps/base/osint-dashboard/templates/monitoring/grafana.yaml @@ -0,0 +1,158 @@ +{{- if .Values.monitoring.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: grafana-admin-secret + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: monitoring +type: Opaque +stringData: + admin-user: CHANGE_ME_USE_SOPS + admin-password: CHANGE_ME_USE_SOPS +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-datasources + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: monitoring +data: + datasources.yaml: | + apiVersion: 1 + datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:{{ .Values.monitoring.prometheus.port }} + isDefault: true + editable: true + - name: PostgreSQL + type: postgres + access: proxy + url: postgresql-rw.{{ .Values.namespace }}.svc:5432 + database: osint + user: grafana + secureJsonData: + password: CHANGE_ME_USE_SOPS + jsonData: + tlsAuth: false + sslmode: disable + postgresVersion: 1600 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboards + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: monitoring +data: + dashboard-providers.yaml: | + apiVersion: 1 + providers: + - name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: monitoring +spec: + replicas: 1 + selector: + matchLabels: + {{- include "osint-dashboard.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: monitoring + template: + metadata: + labels: + {{- include "osint-dashboard.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: monitoring + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "3000" + spec: + securityContext: + runAsNonRoot: true + runAsUser: 472 # grafana + fsGroup: 472 + containers: + - name: grafana + image: "{{ .Values.monitoring.grafana.image.repository }}:{{ .Values.monitoring.grafana.image.tag }}" + ports: + - containerPort: {{ .Values.monitoring.grafana.port }} + name: web + env: + - name: GF_SECURITY_ADMIN_USER + valueFrom: + secretKeyRef: + name: grafana-admin-secret + key: admin-user + - name: GF_SECURITY_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: grafana-admin-secret + key: admin-password + - name: GF_SERVER_ROOT_URL + value: "https://grafana.{{ .Values.monitoring.grafana.hostname }}" + - name: GF_AUTH_ANONYMOUS_ENABLED + value: "false" + resources: + {{- toYaml .Values.monitoring.grafana.resources | nindent 12 }} + volumeMounts: + - name: datasources + mountPath: /etc/grafana/provisioning/datasources + - name: dashboards-config + mountPath: /etc/grafana/provisioning/dashboards + - name: data + mountPath: /var/lib/grafana + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + volumes: + - name: datasources + configMap: + name: grafana-datasources + - name: dashboards-config + configMap: + name: grafana-dashboards + strategy: + type: Recreate +--- +apiVersion: v1 +kind: Service +metadata: + name: grafana + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: monitoring +spec: + type: ClusterIP + ports: + - port: {{ .Values.monitoring.grafana.port }} + targetPort: web + protocol: TCP + name: web + selector: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: monitoring +{{- end }} diff --git a/apps/base/osint-dashboard/templates/monitoring/prometheus-config.yaml b/apps/base/osint-dashboard/templates/monitoring/prometheus-config.yaml new file mode 100644 index 0000000..4345471 --- /dev/null +++ b/apps/base/osint-dashboard/templates/monitoring/prometheus-config.yaml @@ -0,0 +1,188 @@ +{{- if .Values.monitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-config + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: monitoring +data: + prometheus.yml: | + global: + scrape_interval: 15s + evaluation_interval: 15s + scrape_timeout: 10s + + rule_files: + - /etc/prometheus/rules/*.yml + + alerting: + alertmanagers: + - static_configs: + - targets: + - alertmanager:{{ .Values.monitoring.alertmanager.port }} + + scrape_configs: + # Prometheus self-monitoring + - job_name: prometheus + static_configs: + - targets: [localhost:9090] + + # PostgreSQL (Postgres Exporter) + - job_name: postgresql + static_configs: + - targets: + {{- range $i := until $.Values.monitoring.postgresql.exporter.replicas }} + - postgresql-{{ $i }}.postgresql-rw.{{ $.Values.namespace }}.svc:9187 + {{- end }} + + # NATS JetStream + - job_name: nats + static_configs: + - targets: + {{- range $i := until $.Values.nats.replicaCount }} + - nats-{{ $i }}.nats-cluster.{{ $.Values.namespace }}.svc:8222 + {{- end }} + + # Redis + - job_name: redis + static_configs: + - targets: + {{- range $i := until $.Values.redis.replicaCount }} + - redis-{{ $i }}.redis-cluster.{{ $.Values.namespace }}.svc:9121 + {{- end }} + + # MinIO + - job_name: minio + metrics_path: /minio/v2/metrics/cluster + static_configs: + - targets: + {{- range $i := until $.Values.minio.replicaCount }} + - minio-{{ $i }}.minio-headless.{{ $.Values.namespace }}.svc:9000 + {{- end }} + + # Kafka Exporter + - job_name: kafka + static_configs: + - targets: [kafka-exporter:9308] + + # Dashboard web app + - job_name: dashboard-web + static_configs: + - targets: [dashboard-web:3000] + + # Dashboard API + - job_name: dashboard-api + static_configs: + - targets: [dashboard-api:4000] + + # Auto-discover via pod annotations + - job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + namespaces: + own: false + names: + - {{ .Values.namespace }} + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: "true" + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + action: replace + regex: ([\d+]);([\d.]+) + replacement: $2:$1 + target_label: __address__ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-rules + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: monitoring +data: + osint-alerts.yml: | + groups: + - name: osint-dashboard-alerts + rules: + - alert: HighErrorRate + expr: sum(rate(http_requests_total{status=~"5..",namespace="{{ .Values.namespace }}"}[5m])) / sum(rate(http_requests_total{namespace="{{ .Values.namespace }}"}[5m])) > 0.05 + for: 5m + labels: + severity: critical + annotations: + summary: "High error rate detected (>{{ 5 }}%) on {{ $labels.job }}" + + - alert: PodCrashLooping + expr: rate(kube_pod_container_status_restarts_total{namespace="{{ .Values.namespace }}"}[15m]) * 60 * 5 > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Pod {{ $labels.pod }} is crash looping" + + - alert: HighLatency + expr: histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{namespace="{{ .Values.namespace }}"}[5m])) by (le, job)) > 2 + for: 5m + labels: + severity: warning + annotations: + summary: "P95 latency above 2s for {{ $labels.job }}" + + - alert: DiskSpaceLow + expr: kubelet_volume_stats_available_bytes{namespace="{{ .Values.namespace }}"}/kubelet_volume_stats_capacity_bytes{namespace="{{ .Values.namespace }}"} < 0.1 + for: 10m + labels: + severity: critical + annotations: + summary: "Disk space below 10% on {{ $labels.persistentvolumeclaim }}" + + - alert: PostgreSQLConnectionSaturation + expr: pg_stat_activity_count{datname="osint",state="active"}/pg_settings_max_connections > 0.8 + for: 5m + labels: + severity: warning + annotations: + summary: "PostgreSQL connection pool >80% saturated" + + - alert: NATSJetStreamStoreFull + expr: jetstream_store_disk_bytes / jetstream_config_max_store_bytes > 0.85 + for: 5m + labels: + severity: critical + annotations: + summary: "NATS JetStream disk usage >85%" + + - alert: RedisMemoryHigh + expr: redis_memory_used_bytes / redis_memory_max_bytes > 0.9 + for: 5m + labels: + severity: warning + annotations: + summary: "Redis memory usage >90%" + + - alert: MinIOOffline + expr: up{job="minio"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "MinIO node {{ $labels.instance }} is offline" + + - alert: KafkaLagHigh + expr: kafka_consumer_group_lag > 10000 + for: 10m + labels: + severity: warning + annotations: + summary: "Kafka consumer lag >10k messages for group {{ $labels.group }}" +{{- end }} diff --git a/apps/base/osint-dashboard/templates/monitoring/prometheus-deployment.yaml b/apps/base/osint-dashboard/templates/monitoring/prometheus-deployment.yaml new file mode 100644 index 0000000..495be11 --- /dev/null +++ b/apps/base/osint-dashboard/templates/monitoring/prometheus-deployment.yaml @@ -0,0 +1,127 @@ +{{- if .Values.monitoring.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: monitoring +spec: + replicas: 1 + selector: + matchLabels: + {{- include "osint-dashboard.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: monitoring + template: + metadata: + labels: + {{- include "osint-dashboard.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: monitoring + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + spec: + securityContext: + runAsNonRoot: true + runAsUser: 65534 # nobody + fsGroup: 65534 + serviceAccountName: prometheus + containers: + - name: prometheus + image: "{{ .Values.monitoring.prometheus.image.repository }}:{{ .Values.monitoring.prometheus.image.tag }}" + ports: + - containerPort: {{ .Values.monitoring.prometheus.port }} + name: web + args: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + - "--storage.tsdb.retention.time={{ .Values.monitoring.prometheus.retention }}" + - "--storage.tsdb.retention.size={{ .Values.monitoring.prometheus.retentionSize }}" + - "--web.enable-lifecycle" + - "--web.enable-admin-api" + resources: + {{- toYaml .Values.monitoring.prometheus.resources | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/prometheus + - name: rules + mountPath: /etc/prometheus/rules + - name: data + mountPath: /prometheus + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumes: + - name: config + configMap: + name: prometheus-config + - name: rules + configMap: + name: prometheus-rules + strategy: + type: Recreate +--- +apiVersion: v1 +kind: Service +metadata: + name: prometheus + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: monitoring + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" +spec: + type: ClusterIP + ports: + - port: {{ .Values.monitoring.prometheus.port }} + targetPort: web + protocol: TCP + name: web + selector: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: monitoring +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: monitoring +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: prometheus + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: monitoring +rules: + - apiGroups: [""] + resources: ["pods", "services", "endpoints", "configmaps"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: prometheus + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: monitoring +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: prometheus +subjects: + - kind: ServiceAccount + name: prometheus + namespace: {{ .Values.namespace }} +{{- end }} diff --git a/apps/base/osint-dashboard/templates/namespace.yaml b/apps/base/osint-dashboard/templates/namespace.yaml new file mode 100644 index 0000000..fb13cb8 --- /dev/null +++ b/apps/base/osint-dashboard/templates/namespace.yaml @@ -0,0 +1,49 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.namespaceLabels" . | nindent 4 }} + app.kubernetes.io/part-of: osint-dashboard +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: osint-dashboard-role + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["pods", "services", "configmaps", "secrets", "persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: ["deployments", "statefulsets"] + verbs: ["get", "list", "watch"] + - apiGroups: ["batch"] + resources: ["jobs", "cronjobs"] + verbs: ["get", "list", "watch", "create", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: osint-dashboard-rolebinding + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: osint-dashboard-role +subjects: + - kind: ServiceAccount + name: {{ include "osint-dashboard.serviceAccountName" . }} + namespace: {{ .Values.namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "osint-dashboard.serviceAccountName" . }} + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} diff --git a/apps/base/osint-dashboard/templates/nats/jetstream-subjects.yaml b/apps/base/osint-dashboard/templates/nats/jetstream-subjects.yaml new file mode 100644 index 0000000..780eceb --- /dev/null +++ b/apps/base/osint-dashboard/templates/nats/jetstream-subjects.yaml @@ -0,0 +1,88 @@ +{{- if .Values.nats.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: nats-jetstream-init + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: message-broker + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded +spec: + template: + metadata: + labels: + {{- include "osint-dashboard.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: message-broker + spec: + restartPolicy: OnFailure + securityContext: + runAsNonRoot: true + runAsUser: 1000 + containers: + - name: nats-box + image: "{{ .Values.nats.image.repository }}:{{ .Values.nats.image.tag }}" + command: + - /bin/sh + - -c + args: + - | + # Wait for NATS to be ready + until nats-server -version > /dev/null 2>&1 || curl -sf http://nats:{{ .Values.nats.ports.monitor }}/healthz; do + echo "Waiting for NATS..." + sleep 2 + done + + # Download nats CLI + curl -fsSL https://github.com/nats-io/natscli/releases/latest/download/nats-linux-amd64 -o /tmp/nats + chmod +x /tmp/nats + + # Create JetStream stream for events + /tmp/nats stream add events \ + --subjects=events.gdelt,events.rss,events.social,events.earthquake,events.disaster,events.weather,events.fire,events.satellite,events.new,events.alert \ + --retention=interests \ + --max-consumers=-1 \ + --max-msgs=1000000 \ + --max-bytes=1GB \ + --discard=old \ + --storage=file \ + --replicas=3 \ + --server=nats://nats:{{ .Values.nats.ports.client }} || echo "events stream already exists" + + # Create JetStream stream for alerts + /tmp/nats stream add alerts \ + --subjects=alerts.camera_offline,alerts.new \ + --retention=interests \ + --max-consumers=-1 \ + --max-msgs=100000 \ + --discard=old \ + --storage=file \ + --replicas=3 \ + --server=nats://nats:{{ .Values.nats.ports.client }} || echo "alerts stream already exists" + + # Create JetStream stream for video + /tmp/nats stream add video \ + --subjects="video.status.>","video.record.>" \ + --retention=limits \ + --max-consumers=-1 \ + --max-msgs=50000 \ + --discard=old \ + --storage=file \ + --replicas=3 \ + --server=nats://nats:{{ .Values.nats.ports.client }} || echo "video stream already exists" + + echo "JetStream subjects initialized successfully" + resources: + requests: + cpu: "100m" + memory: "64Mi" + limits: + cpu: "500m" + memory: "256Mi" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] +{{- end }} diff --git a/apps/base/osint-dashboard/templates/nats/service.yaml b/apps/base/osint-dashboard/templates/nats/service.yaml new file mode 100644 index 0000000..7e4a294 --- /dev/null +++ b/apps/base/osint-dashboard/templates/nats/service.yaml @@ -0,0 +1,60 @@ +{{- if .Values.nats.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: nats + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: message-broker +spec: + type: ClusterIP + ports: + - port: {{ .Values.nats.ports.client }} + targetPort: {{ .Values.nats.ports.client }} + protocol: TCP + name: client + selector: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: message-broker +--- +apiVersion: v1 +kind: Service +metadata: + name: nats-ws + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: message-broker +spec: + type: ClusterIP + ports: + - port: {{ .Values.nats.ports.websocket }} + targetPort: {{ .Values.nats.ports.websocket }} + protocol: TCP + name: websocket + selector: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: message-broker +--- +apiVersion: v1 +kind: Service +metadata: + name: nats-cluster + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: message-broker + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" +spec: + clusterIP: None + ports: + - port: {{ .Values.nats.ports.cluster }} + targetPort: {{ .Values.nats.ports.cluster }} + protocol: TCP + name: cluster + selector: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: message-broker +{{- end }} diff --git a/apps/base/osint-dashboard/templates/nats/statefulset.yaml b/apps/base/osint-dashboard/templates/nats/statefulset.yaml new file mode 100644 index 0000000..74cbb85 --- /dev/null +++ b/apps/base/osint-dashboard/templates/nats/statefulset.yaml @@ -0,0 +1,130 @@ +{{- if .Values.nats.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: nats-config + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} +data: + nats-server.conf: | + port: {{ .Values.nats.ports.client }} + server_name: "osint-nats-${HOSTNAME}" + + # Cluster + cluster { + port: {{ .Values.nats.ports.cluster }} + routes: [ + nats-route://nats-0.nats-cluster.{{ .Values.namespace }}.svc:{{ .Values.nats.ports.cluster }}, + nats-route://nats-1.nats-cluster.{{ .Values.namespace }}.svc:{{ .Values.nats.ports.cluster }}, + nats-route://nats-2.nats-cluster.{{ .Values.namespace }}.svc:{{ .Values.nats.ports.cluster }} + ] + cluster_advertise: "nats-${HOSTNAME}.nats-cluster.{{ .Values.namespace }}.svc:{{ .Values.nats.ports.cluster }}" + } + + # JetStream + jetstream { + store_dir: "{{ .Values.nats.jetstream.fileStore }}" + max_mem_store: {{ .Values.nats.jetstream.maxMemory }} + max_file_store: {{ .Values.nats.storage.size }} + } + + # Monitoring + monitor: {{ .Values.nats.ports.monitor }} + + # WebSocket + websocket { + port: {{ .Values.nats.ports.websocket }} + no_tls: true + } + + # Logging + logtime: true + log_file: /var/log/nats/nats.log +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: nats + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: message-broker +spec: + serviceName: nats-cluster + replicas: {{ .Values.nats.replicaCount }} + selector: + matchLabels: + {{- include "osint-dashboard.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: message-broker + template: + metadata: + labels: + {{- include "osint-dashboard.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: message-broker + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "{{ .Values.nats.ports.monitor }}" + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + terminationGracePeriodSeconds: 30 + containers: + - name: nats + image: "{{ .Values.nats.image.repository }}:{{ .Values.nats.image.tag }}" + ports: + - name: client + containerPort: {{ .Values.nats.ports.client }} + - name: cluster + containerPort: {{ .Values.nats.ports.cluster }} + - name: monitor + containerPort: {{ .Values.nats.ports.monitor }} + - name: websocket + containerPort: {{ .Values.nats.ports.websocket }} + resources: + {{- toYaml .Values.nats.resources | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/nats-config + - name: data + mountPath: {{ .Values.nats.jetstream.fileStore }} + - name: logs + mountPath: /var/log/nats + args: + - "-c" + - "/etc/nats-config/nats-server.conf" + readinessProbe: + httpGet: + path: /healthz + port: monitor + initialDelaySeconds: 5 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /healthz + port: monitor + initialDelaySeconds: 10 + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumes: + - name: config + configMap: + name: nats-config + - name: logs + emptyDir: {} + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.nats.storage.size }} + storageClassName: {{ .Values.nats.storage.storageClass }} +{{- end }} diff --git a/apps/base/osint-dashboard/templates/postgresql/cluster.yaml b/apps/base/osint-dashboard/templates/postgresql/cluster.yaml new file mode 100644 index 0000000..40a629b --- /dev/null +++ b/apps/base/osint-dashboard/templates/postgresql/cluster.yaml @@ -0,0 +1,152 @@ +{{- if .Values.postgresql.enabled }} +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: {{ .Values.postgresql.clusterName }} + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} +spec: + instances: {{ .Values.postgresql.instances }} + imageName: {{ .Values.postgresql.imageName }} + storage: + size: {{ .Values.postgresql.storage.size }} + storageClass: {{ .Values.postgresql.storage.storageClass }} + resources: + {{- toYaml .Values.postgresql.resources | nindent 4 }} + # PostGIS + TimescaleDB extensions via shared_preload_libraries + postgresql: + shared_preload_libraries: + - pggis + - timescaledb + parameters: + max_connections: "500" + shared_buffers: "2GB" + effective_cache_size: "6GB" + maintenance_work_mem: "512MB" + work_mem: "16MB" + wal_buffers: "64MB" + random_page_cost: "1.1" + effective_io_concurrency: "200" + default_statistics_target: "200" + max_parallel_workers_per_gather: "4" + bootstrap: + initdb: + database: osint + owner: osint_admin + secret: + name: {{ .Values.postgresql.credentialsSecret }} + postInitializationSQL: + # Install PostGIS extension + - >- + CREATE EXTENSION IF NOT EXISTS postgis; + - >- + CREATE EXTENSION IF NOT EXISTS postgis_raster; + - >- + CREATE EXTENSION IF NOT EXISTS postgis_topology; + # Install TimescaleDB extension + - >- + CREATE EXTENSION IF NOT EXISTS timescaledb; + # Create hypertable for events + - >- + CREATE TABLE IF NOT EXISTS events ( + time TIMESTAMPTZ NOT NULL, + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + source TEXT NOT NULL, + event_type TEXT, + title TEXT, + description TEXT, + location GEOGRAPHY(POINT, 4326), + severity INT DEFAULT 0, + tags TEXT[], + raw_data JSONB, + created_at TIMESTAMPTZ DEFAULT NOW() + ); + - >- + SELECT create_hypertable('events', 'time', if_not_exists => TRUE); + - >- + CREATE INDEX IF NOT EXISTS events_loc_idx ON events USING GIST (location); + - >- + CREATE INDEX IF NOT EXISTS events_time_idx ON events (time DESC); + # Create sources reference table + - >- + CREATE TABLE IF NOT EXISTS sources ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name TEXT NOT NULL, + type TEXT NOT NULL, + config JSONB, + active BOOLEAN DEFAULT TRUE + ); + # Create video_feeds table + - >- + CREATE TABLE IF NOT EXISTS video_feeds ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name TEXT NOT NULL, + rtsp_url TEXT NOT NULL, + zlm_stream_key TEXT NOT NULL, + location GEOGRAPHY(POINT, 4326), + active BOOLEAN DEFAULT TRUE, + last_heartbeat TIMESTAMPTZ + ); + managed: + roles: + - name: osint_admin + ensure: present + login: true + passwordSecret: + name: {{ .Values.postgresql.credentialsSecret }} + - name: osint_reader + ensure: present + login: true + passwordSecret: + name: osint-pg-reader-credentials + backup: + barmanObjectStore: + destinationPath: "{{ .Values.postgresql.backup.bucket }}" + googleCredentials: + gkeEnvironment: true + wal: + compression: gzip + data: + compression: gzip + jobs: 2 + retentionPolicy: {{ .Values.postgresql.backup.retentionPolicy }} + target: primary + monitoring: + customQueries: + - query: >- + SELECT relname, schemaname, seq_scan, seq_tup_read, idx_scan, idx_tup_fetch + FROM pg_stat_user_tables WHERE schemaname = 'public'; + metrics: + - relname: + usage: "LABEL" + description: "Table name" + - schemaname: + usage: "LABEL" + description: "Schema name" + - seq_scan: + usage: "GAUGE" + description: "Number of sequential scans" + - seq_tup_read: + usage: "GAUGE" + description: "Number of tuples read" + - idx_scan: + usage: "GAUGE" + description: "Number of index scans" + - idx_tup_fetch: + usage: "GAUGE" + description: "Number of tuples fetched via index" + affinity: + enablePodAntiAffinity: true + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + postgresql operator: {{ .Values.postgresql.clusterName }} + nodesAutoRemediationChecks: + livenessProbe: + initialDelaySeconds: 10 + timeoutSeconds: 5 +{{- end }} diff --git a/apps/base/osint-dashboard/templates/postgresql/credentials-secret.yaml b/apps/base/osint-dashboard/templates/postgresql/credentials-secret.yaml new file mode 100644 index 0000000..30188a6 --- /dev/null +++ b/apps/base/osint-dashboard/templates/postgresql/credentials-secret.yaml @@ -0,0 +1,29 @@ +{{- if .Values.postgresql.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.postgresql.credentialsSecret }} + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + annotations: + # SOPS will encrypt this - use 'sops --encrypt --in-place' after generation +type: Opaque +stringData: + username: osint_admin + password: CHANGE_ME_USE_SOPS + connection_string: "postgresql://osint_admin:CHANGE_ME_USE_SOPS@{{ .Values.postgresql.clusterName }}.{{ .Values.namespace }}.svc:5432/osint?sslmode=require" +--- +apiVersion: v1 +kind: Secret +metadata: + name: osint-pg-reader-credentials + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} +type: Opaque +stringData: + username: osint_reader + password: CHANGE_ME_USE_SOPS + connection_string: "postgresql://osint_reader:CHANGE_ME_USE_SOPS@{{ .Values.postgresql.clusterName }}.{{ .Values.namespace }}.svc:5432/osint?sslmode=require" +{{- end }} diff --git a/apps/base/osint-dashboard/templates/postgresql/service.yaml b/apps/base/osint-dashboard/templates/postgresql/service.yaml new file mode 100644 index 0000000..36e4653 --- /dev/null +++ b/apps/base/osint-dashboard/templates/postgresql/service.yaml @@ -0,0 +1,18 @@ +{{- if .Values.postgresql.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.postgresql.clusterName }} + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: 5432 + targetPort: postgresql + protocol: TCP + name: postgresql + selector: + postgresql operator: {{ .Values.postgresql.clusterName }} +{{- end }} diff --git a/apps/base/osint-dashboard/templates/redis/configmap.yaml b/apps/base/osint-dashboard/templates/redis/configmap.yaml new file mode 100644 index 0000000..a2edf74 --- /dev/null +++ b/apps/base/osint-dashboard/templates/redis/configmap.yaml @@ -0,0 +1,70 @@ +{{- if .Values.redis.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: redis-config + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} +data: + redis.conf: | + bind 0.0.0.0 + port {{ .Values.redis.ports.redis }} + appendonly yes + appendfilename "appendonly.aof" + dir /data + save 900 1 + save 300 10 + save 60 10000 + maxmemory-policy allkeys-lru + # Require authentication + requirepass CHANGE_ME_USE_SOPS + masterauth CHANGE_ME_USE_SOPS +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: redis-sentinel-config + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} +data: + sentinel.conf: | + port {{ .Values.redis.ports.sentinel }} + sentinel monitor osint-redis-master redis-master.{{ .Values.namespace }}.svc {{ .Values.redis.ports.redis }} 2 + sentinel auth-pass osint-redis-master CHANGE_ME_USE_SOPS + sentinel down-after-milliseconds osint-redis-master 5000 + sentinel failover-timeout osint-redis-master 30000 + sentinel parallel-syncs osint-redis-master 1 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: redis-scripts + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} +data: + init-sentinel.sh: | + #!/bin/bash + set -e + + # Copy sentinel config and update it with current master info + cp /etc/redis-sentinel/sentinel.conf /tmp/sentinel.conf + # Sentinel will auto-discover master from other sentinels + exec redis-sentinel /tmp/sentinel.conf --loglevel notice + redis.sh: | + #!/bin/bash + set -e + + REDIS_PORT={{ .Values.redis.ports.redis }} + REDIS_PASSWORD="CHANGE_ME_USE_SOPS" + + if [ "${REDIS_ROLE}" = "master" ]; then + exec redis-server /etc/redis/redis.conf + else + # Replica: find master and replicate + MASTER_HOST="redis-master.{{ .Values.namespace }}.svc" + exec redis-server /etc/redis/redis.conf --replicaof ${MASTER_HOST} ${REDIS_PORT} + fi +{{- end }} diff --git a/apps/base/osint-dashboard/templates/redis/service.yaml b/apps/base/osint-dashboard/templates/redis/service.yaml new file mode 100644 index 0000000..14da958 --- /dev/null +++ b/apps/base/osint-dashboard/templates/redis/service.yaml @@ -0,0 +1,61 @@ +{{- if .Values.redis.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: redis-master + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: redis +spec: + type: ClusterIP + ports: + - port: {{ .Values.redis.ports.redis }} + targetPort: {{ .Values.redis.ports.redis }} + protocol: TCP + name: redis + selector: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: redis + redis-role: master +--- +apiVersion: v1 +kind: Service +metadata: + name: redis-replica + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: redis +spec: + type: ClusterIP + ports: + - port: {{ .Values.redis.ports.redis }} + targetPort: {{ .Values.redis.ports.redis }} + protocol: TCP + name: redis + selector: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: redis + redis-role: replica +--- +apiVersion: v1 +kind: Service +metadata: + name: redis-sentinel + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: redis +spec: + type: ClusterIP + ports: + - port: {{ .Values.redis.ports.sentinel }} + targetPort: {{ .Values.redis.ports.sentinel }} + protocol: TCP + name: sentinel + selector: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: redis + redis-role: sentinel +{{- end }} diff --git a/apps/base/osint-dashboard/templates/redis/statefulset.yaml b/apps/base/osint-dashboard/templates/redis/statefulset.yaml new file mode 100644 index 0000000..a2804ea --- /dev/null +++ b/apps/base/osint-dashboard/templates/redis/statefulset.yaml @@ -0,0 +1,210 @@ +{{- if .Values.redis.enabled }} +# Redis Master +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis-master + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: redis + redis-role: master +spec: + serviceName: redis-master + replicas: 1 + selector: + matchLabels: + {{- include "osint-dashboard.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: redis + redis-role: master + template: + metadata: + labels: + {{- include "osint-dashboard.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: redis + redis-role: master + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9121" + spec: + securityContext: + runAsNonRoot: true + runAsUser: 999 + fsGroup: 999 + containers: + - name: redis + image: "{{ .Values.redis.image.repository }}:{{ .Values.redis.image.tag }}" + ports: + - name: redis + containerPort: {{ .Values.redis.ports.redis }} + resources: + {{- toYaml .Values.redis.master.resources | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/redis + - name: data + mountPath: /data + args: ["redis-server", "/etc/redis/redis.conf"] + readinessProbe: + exec: + command: + - redis-cli + - ping + initialDelaySeconds: 5 + periodSeconds: 5 + livenessProbe: + exec: + command: + - redis-cli + - ping + initialDelaySeconds: 10 + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + # Sentinel sidecar on master + - name: sentinel + image: "{{ .Values.redis.sentinel.image.repository }}:{{ .Values.redis.sentinel.image.tag }}" + ports: + - name: sentinel + containerPort: {{ .Values.redis.ports.sentinel }} + resources: + {{- toYaml .Values.redis.sentinel.resources | nindent 12 }} + volumeMounts: + - name: sentinel-config + mountPath: /etc/redis-sentinel + command: ["/bin/bash", "/etc/redis-scripts/init-sentinel.sh"] + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumes: + - name: config + configMap: + name: redis-config + - name: sentinel-config + configMap: + name: redis-sentinel-config + - name: scripts + configMap: + name: redis-scripts + defaultMode: 0755 + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.redis.master.storage.size }} + storageClassName: {{ .Values.redis.master.storage.storageClass }} +--- +# Redis Replicas +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis-replica + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: redis + redis-role: replica +spec: + serviceName: redis-replica + replicas: {{ .Values.redis.replica.replicaCount }} + selector: + matchLabels: + {{- include "osint-dashboard.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: redis + redis-role: replica + template: + metadata: + labels: + {{- include "osint-dashboard.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: redis + redis-role: replica + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9121" + spec: + securityContext: + runAsNonRoot: true + runAsUser: 999 + fsGroup: 999 + containers: + - name: redis + image: "{{ .Values.redis.image.repository }}:{{ .Values.redis.image.tag }}" + ports: + - name: redis + containerPort: {{ .Values.redis.ports.redis }} + resources: + {{- toYaml .Values.redis.replica.resources | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/redis + - name: data + mountPath: /data + env: + - name: REDIS_ROLE + value: "replica" + command: ["/bin/bash", "/etc/redis-scripts/redis.sh"] + readinessProbe: + exec: + command: + - redis-cli + - ping + initialDelaySeconds: 5 + periodSeconds: 5 + livenessProbe: + exec: + command: + - redis-cli + - ping + initialDelaySeconds: 10 + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + # Sentinel sidecar on replicas + - name: sentinel + image: "{{ .Values.redis.sentinel.image.repository }}:{{ .Values.redis.sentinel.image.tag }}" + ports: + - name: sentinel + containerPort: {{ .Values.redis.ports.sentinel }} + resources: + {{- toYaml .Values.redis.sentinel.resources | nindent 12 }} + volumeMounts: + - name: sentinel-config + mountPath: /etc/redis-sentinel + command: ["/bin/bash", "/etc/redis-scripts/init-sentinel.sh"] + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumes: + - name: config + configMap: + name: redis-config + - name: sentinel-config + configMap: + name: redis-sentinel-config + - name: scripts + configMap: + name: redis-scripts + defaultMode: 0755 + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.redis.replica.storage.size }} + storageClassName: {{ .Values.redis.replica.storage.storageClass }} +{{- end }} diff --git a/apps/base/osint-dashboard/templates/security/dashboard-netpol.yaml b/apps/base/osint-dashboard/templates/security/dashboard-netpol.yaml new file mode 100644 index 0000000..d756763 --- /dev/null +++ b/apps/base/osint-dashboard/templates/security/dashboard-netpol.yaml @@ -0,0 +1,145 @@ +{{- if .Values.networkPolicies.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: dashboard-web-netpol + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: security +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: web + policyTypes: + - Ingress + - Egress + ingress: + # Allow from ingress controller / Gateway API + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: ingress-nginx + ports: + - protocol: TCP + port: 3000 + egress: + # Allow DNS + - to: + - namespaceSelector: {} + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + # Allow to API + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: api + ports: + - protocol: TCP + port: 4000 + # Allow to external APIs (GDelt, satellite providers) + - to: + - ipBlock: + cidr: 0.0.0.0/0 + ports: + - protocol: TCP + port: 443 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: dashboard-api-netpol + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: security +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: api + policyTypes: + - Ingress + - Egress + ingress: + # Allow from web frontend + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: web + ports: + - protocol: TCP + port: 4000 + # Allow from ingress controller / Gateway API + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: ingress-nginx + ports: + - protocol: TCP + port: 4000 + egress: + # Allow DNS + - to: + - namespaceSelector: {} + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + # Allow to PostgreSQL + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: database + ports: + - protocol: TCP + port: 5432 + # Allow to Redis + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: cache + ports: + - protocol: TCP + port: {{ .Values.redis.ports.redis }} + # Allow to NATS + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: messaging + ports: + - protocol: TCP + port: {{ .Values.nats.ports.client }} + # Allow to MinIO + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: object-storage + ports: + - protocol: TCP + port: {{ .Values.minio.ports.api }} + # Allow to external APIs (GDelt, etc.) + - to: + - ipBlock: + cidr: 0.0.0.0/0 + ports: + - protocol: TCP + port: 443 +{{- end }} diff --git a/apps/base/osint-dashboard/templates/security/default-deny.yaml b/apps/base/osint-dashboard/templates/security/default-deny.yaml new file mode 100644 index 0000000..9db14dd --- /dev/null +++ b/apps/base/osint-dashboard/templates/security/default-deny.yaml @@ -0,0 +1,17 @@ +{{- if .Values.networkPolicies.enabled }} +{{- if .Values.networkPolicies.defaultDeny }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-deny-all + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: security +spec: + podSelector: {} + policyTypes: + - Ingress + - Egress +{{- end }} +{{- end }} diff --git a/apps/base/osint-dashboard/templates/security/minio-netpol.yaml b/apps/base/osint-dashboard/templates/security/minio-netpol.yaml new file mode 100644 index 0000000..fe57721 --- /dev/null +++ b/apps/base/osint-dashboard/templates/security/minio-netpol.yaml @@ -0,0 +1,56 @@ +{{- if .Values.networkPolicies.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: minio-netpol + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: security +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: object-storage + policyTypes: + - Ingress + - Egress + ingress: + # Allow from dashboard services + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + ports: + - protocol: TCP + port: {{ .Values.minio.ports.api }} + # Allow console access (internal) + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + ports: + - protocol: TCP + port: {{ .Values.minio.ports.console }} + egress: + # Allow DNS + - to: + - namespaceSelector: {} + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + # Allow to other MinIO pods (distributed replication) + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: object-storage + ports: + - protocol: TCP + port: {{ .Values.minio.ports.api }} +{{- end }} diff --git a/apps/base/osint-dashboard/templates/security/monitoring-netpol.yaml b/apps/base/osint-dashboard/templates/security/monitoring-netpol.yaml new file mode 100644 index 0000000..fa24dd3 --- /dev/null +++ b/apps/base/osint-dashboard/templates/security/monitoring-netpol.yaml @@ -0,0 +1,60 @@ +{{- if .Values.networkPolicies.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: monitoring-netpol + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: security +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: monitoring + policyTypes: + - Ingress + - Egress + ingress: + # Allow Grafana web access + - from: [] + ports: + - protocol: TCP + port: {{ .Values.monitoring.grafana.port }} + - protocol: TCP + port: {{ .Values.monitoring.prometheus.port }} + - protocol: TCP + port: {{ .Values.monitoring.alertmanager.port }} + egress: + # Allow DNS + - to: + - namespaceSelector: {} + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + # Allow scraping all OSINT pods + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + ports: + - protocol: TCP + port: 9090 + - protocol: TCP + port: 9093 + - protocol: TCP + port: 3000 + - protocol: TCP + port: 9187 # postgres-exporter + - protocol: TCP + port: 9121 # redis-exporter + - protocol: TCP + port: 8222 # nats-monitor + - protocol: TCP + port: 9000 # minio +{{- end }} diff --git a/apps/base/osint-dashboard/templates/security/nats-netpol.yaml b/apps/base/osint-dashboard/templates/security/nats-netpol.yaml new file mode 100644 index 0000000..e45687f --- /dev/null +++ b/apps/base/osint-dashboard/templates/security/nats-netpol.yaml @@ -0,0 +1,67 @@ +{{- if .Values.networkPolicies.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: nats-netpol + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: security +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: messaging + policyTypes: + - Ingress + - Egress + ingress: + # Allow client connections from dashboard services + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + ports: + - protocol: TCP + port: {{ .Values.nats.ports.client }} + # Allow cluster communication + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: messaging + ports: + - protocol: TCP + port: {{ .Values.nats.ports.cluster }} + # Allow monitoring + - from: + - podSelector: + matchLabels: + app.kubernetes.io/component: monitoring + ports: + - protocol: TCP + port: {{ .Values.nats.ports.monitor }} + egress: + # Allow DNS + - to: + - namespaceSelector: {} + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + # Allow cluster communication + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: messaging + ports: + - protocol: TCP + port: {{ .Values.nats.ports.cluster }} + - protocol: TCP + port: {{ .Values.nats.ports.client }} +{{- end }} diff --git a/apps/base/osint-dashboard/templates/security/postgresql-netpol.yaml b/apps/base/osint-dashboard/templates/security/postgresql-netpol.yaml new file mode 100644 index 0000000..8735fe2 --- /dev/null +++ b/apps/base/osint-dashboard/templates/security/postgresql-netpol.yaml @@ -0,0 +1,60 @@ +{{- if .Values.networkPolicies.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: postgresql-netpol + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: security +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: database + policyTypes: + - Ingress + - Egress + ingress: + # Allow from dashboard API and workers + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + - podSelector: + matchLabels: + app.kubernetes.io/component: prometheus-exporter + ports: + - protocol: TCP + port: 5432 + # Allow from CNPG cluster peers (replication) + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: database + ports: + - protocol: TCP + port: 5432 + egress: + # Allow DNS + - to: + - namespaceSelector: {} + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + # Allow to other PG replicas (replication) + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: database + ports: + - protocol: TCP + port: 5432 +{{- end }} diff --git a/apps/base/osint-dashboard/templates/security/redis-netpol.yaml b/apps/base/osint-dashboard/templates/security/redis-netpol.yaml new file mode 100644 index 0000000..53ec5c9 --- /dev/null +++ b/apps/base/osint-dashboard/templates/security/redis-netpol.yaml @@ -0,0 +1,58 @@ +{{- if .Values.networkPolicies.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: redis-netpol + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: security +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: cache + policyTypes: + - Ingress + - Egress + ingress: + # Allow from dashboard services + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + ports: + - protocol: TCP + port: {{ .Values.redis.ports.redis }} + # Allow sentinel from dashboard + other sentinels + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + ports: + - protocol: TCP + port: {{ .Values.redis.ports.sentinel }} + egress: + # Allow DNS + - to: + - namespaceSelector: {} + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + # Allow to other Redis pods (replication) + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: cache + ports: + - protocol: TCP + port: {{ .Values.redis.ports.redis }} + - protocol: TCP + port: {{ .Values.redis.ports.sentinel }} +{{- end }} diff --git a/apps/base/osint-dashboard/values.yaml b/apps/base/osint-dashboard/values.yaml new file mode 100644 index 0000000..6752fed --- /dev/null +++ b/apps/base/osint-dashboard/values.yaml @@ -0,0 +1,360 @@ +# OSINT Dashboard — Helm Values +# Default values for development/staging. Override with values-prod.yaml for production. + +nameOverride: "" +fullnameOverride: "" + +# Namespace +namespace: customer1 + +# Global security context +securityContext: + runAsNonRoot: true + fsGroup: 1000 + +# Resource defaults +resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 128Mi + +# ============================================================ +# PostgreSQL (using CNPG — CloudNativePG, already installed) +# ============================================================ +postgresql: + enabled: true + clusterName: osint-pgdb + instances: 3 + imageName: ghcr.io/cloudnative-pg/postgresql:16 + # Custom image with PostGIS + TimescaleDB + # Use CNPG bootstrap to create extensions + storage: + size: 200Gi + storageClass: premium-rwo + resources: + requests: + cpu: "2" + memory: 4Gi + limits: + cpu: "4" + memory: 8Gi + extensions: + - postgis + - timescaledb + backup: + retentionPolicy: "30d" + bucket: "gs://osint-dashboard-db-backup/" + credentialsSecret: osint-pg-credentials + +# ============================================================ +# NATS JetStream (3 replicas, persistent) +# ============================================================ +nats: + enabled: true + replicaCount: 3 + image: + repository: nats + tag: "2.10.18-alpine" + resources: + requests: + cpu: "500m" + memory: 512Mi + limits: + cpu: "1" + memory: 1Gi + storage: + size: 50Gi + storageClass: premium-rwo + jetstream: + enabled: true + maxMemory: 2Gi + fileStore: /data/jetstream + ports: + client: 4222 + cluster: 6222 + monitor: 8222 + websocket: 8080 + # JetStream subjects schema + subjects: + streams: + - name: events + subjects: + - events.gdelt + - events.rss + - events.social + - events.earthquake + - events.disaster + - events.weather + - events.fire + - events.satellite + - events.new + - events.alert + retention: interests + maxConsumers: -1 + maxMsgs: 1000000 + maxBytes: 1073741824 # 1GB + discard: old + - name: alerts + subjects: + - alerts.camera_offline + - alerts.new + retention: interests + maxConsumers: -1 + maxMsgs: 100000 + discard: old + - name: video + subjects: + - "video.status.>" + - "video.record.>" + retention: limits + maxConsumers: -1 + maxMsgs: 50000 + discard: old + +# ============================================================ +# Redis Sentinel (1 primary + 2 replicas) +# ============================================================ +redis: + enabled: true + image: + repository: redis + tag: "7.4-alpine" + sentinel: + image: + repository: redis + tag: "7.4-alpine" + master: + replicaCount: 1 + resources: + requests: + cpu: "500m" + memory: 512Mi + limits: + cpu: "1" + memory: 1Gi + storage: + size: 20Gi + storageClass: premium-rwo + replica: + replicaCount: 2 + resources: + requests: + cpu: "250m" + memory: 256Mi + limits: + cpu: "500m" + memory: 512Mi + storage: + size: 20Gi + storageClass: premium-rwo + sentinel: + replicaCount: 3 + resources: + requests: + cpu: "100m" + memory: 128Mi + limits: + cpu: "250m" + memory: 256Mi + ports: + redis: 6379 + sentinel: 26379 + +# ============================================================ +# MinIO (4 replicas, distributed mode) +# ============================================================ +minio: + enabled: true + image: + repository: quay.io/minio/minio + tag: "latest" + replicaCount: 4 + mode: distributed + resources: + requests: + cpu: "500m" + memory: 1Gi + limits: + cpu: "1" + memory: 2Gi + storage: + size: 500Gi + storageClass: premium-rwo + buckets: + - name: osint-video-clips + policy: none + - name: osint-satellite-tiles + policy: none + - name: osint-data-dumps + policy: none + credentialsSecret: osint-minio-credentials + ports: + api: 9000 + console: 9001 + +# ============================================================ +# NGINX Ingress Controller + cert-manager +# ============================================================ +ingress: + enabled: true + # Use existing cert-manager cluster issuer + certManager: + enabled: true + clusterIssuerName: letsencrypt-prod + hosts: + - host: dashboard.siriusdevops.com + paths: + - path: / + pathType: Prefix + - host: api.siriusdevops.com + paths: + - path: / + pathType: Prefix + - host: ws.siriusdevops.com + paths: + - path: / + pathType: Prefix + tls: + - secretName: osint-dashboard-tls + hosts: + - dashboard.siriusdevops.com + - secretName: osint-api-tls + hosts: + - api.siriusdevops.com + annotations: + nginx.ingress.kubernetes.io/proxy-read-timeout: "3600" + nginx.ingress.kubernetes.io/proxy-send-timeout: "3600" + nginx.ingress.kubernetes.io/proxy-body-size: "50m" + # WebSocket support + nginx.ingress.kubernetes.io/use-regex: "true" + +# ============================================================ +# Monitoring (Prometheus + Grafana + Alertmanager) +# ============================================================ +monitoring: + enabled: true + defaultEmailDomain: siriusdevops.com + + prometheus: + image: + repository: prom/prometheus + tag: "v2.53.0" + port: 9090 + retention: "30d" + retentionSize: "50GB" + resources: + requests: + cpu: "500m" + memory: 2Gi + limits: + cpu: "1" + memory: 4Gi + + alertmanager: + image: + repository: prom/alertmanager + tag: "v0.27.0" + port: 9093 + resources: + requests: + cpu: "100m" + memory: 128Mi + limits: + cpu: "250m" + memory: 256Mi + + grafana: + image: + repository: grafana/grafana + tag: "11.2.0" + port: 3000 + hostname: grafana.siriusdevops.com + resources: + requests: + cpu: "200m" + memory: 256Mi + limits: + cpu: "500m" + memory: 512Mi + + exporters: + nats: + enabled: true + image: + repository: natsio/prometheus-nats-exporter + tag: "0.14.0" + postgresql: + enabled: true + image: + repository: prometheuscommunity/postgres-exporter + tag: "0.15.0" + exporter: + replicas: 3 + redis: + enabled: true + image: + repository: oliver006/redis_exporter + tag: "v1.58.0" + minio: + enabled: true + # MinIO has built-in metrics at /minio/v2/metrics/cluster + + grafanaDashboards: + - osint-overview + - nats-jetstream + - postgresql-performance + - redis-sentinel + - minio-storage + +# ============================================================ +# CI/CD +# ============================================================ +cicd: + enabled: true + registry: gcr.io/devops-lab-cluster + previewEnvironments: true + +# ============================================================ +# Network Policies +# ============================================================ +networkPolicies: + enabled: true + # Default deny all ingress/egress, then allow specific traffic + defaultDeny: true + +# ============================================================ +# Frontend (placeholder — T3 will fill this in) +# ============================================================ +frontend: + enabled: false + replicaCount: 3 + image: + repository: gcr.io/devops-lab-cluster/osint-dashboard-web + tag: latest + resources: + requests: + cpu: "100m" + memory: 128Mi + limits: + cpu: "500m" + memory: 512Mi + +# ============================================================ +# API (placeholder — T2 will fill this in) +# ============================================================ +api: + enabled: false + replicaCount: 3 + image: + repository: gcr.io/devops-lab-cluster/osint-dashboard-api + tag: latest + resources: + requests: + cpu: "200m" + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi diff --git a/apps/staging/customer1/kustomization.yaml b/apps/staging/customer1/kustomization.yaml index d35882f..692977d 100644 --- a/apps/staging/customer1/kustomization.yaml +++ b/apps/staging/customer1/kustomization.yaml @@ -11,3 +11,4 @@ resources: - ../../base/customer1/hermes-db/ - ../../base/customer1/trade-dashboard/ - ../../base/customer1/siriusdevops-db/ + - ../../base/customer1/trading-platform/ diff --git a/apps/staging/osint-dashboard/kustomization.yaml b/apps/staging/osint-dashboard/kustomization.yaml new file mode 100644 index 0000000..3a38b16 --- /dev/null +++ b/apps/staging/osint-dashboard/kustomization.yaml @@ -0,0 +1,53 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Reference the Helm chart base +helmCharts: + - name: osint-dashboard + repository: + name: osint-dashboard-local + type: oci + url: oci://gcr.io/devops-lab-cluster/charts + version: "0.1.0" + releaseName: osint-dashboard + namespace: customer1 + includeCRDs: true + +# Or use plain Kustomize overlay on the base templates +resources: + - ../../base/osint-dashboard/templates/namespace.yaml + +# Namespace override +namespace: customer1 + +# Common labels +commonLabels: + app.kubernetes.io/managed-by: flux + app.kubernetes.io/part-of: osint-dashboard + +# Patches for staging environment +patches: + # Override replica counts for staging + - patch: |- + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: postgresql + spec: + replicas: 2 + target: + kind: StatefulSet + name: postgresql.* + + # Reduce storage for staging + - patch: |- + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: data + spec: + resources: + requests: + storage: 100Gi + target: + kind: PersistentVolumeClaim