From ac994f144af757bc95fa4b55cb6b988029c7b523 Mon Sep 17 00:00:00 2001 From: Sirius Devops Date: Fri, 15 May 2026 00:00:58 +0000 Subject: [PATCH 01/28] fix: container hardening + NetworkPolicy for waitlist-api - deployment.yaml: securityContext (runAsNonRoot, readOnlyRootFilesystem, drop ALL caps) - network-policy.yaml: egress only to CNPG PG (5432) + api.telegram.org (443) - kustomization.yaml: register new NetworkPolicy resource - waitlist-telegram-secret: add admin_api_key field --- .../customer1/waitlist-api/deployment.yaml | 26 +++++++++++ .../customer1/waitlist-api/kustomization.yaml | 1 + .../waitlist-api/network-policy.yaml | 44 +++++++++++++++++++ .../waitlist-telegram-secret.yaml | 1 + 4 files changed, 72 insertions(+) create mode 100644 apps/base/customer1/waitlist-api/network-policy.yaml diff --git a/apps/base/customer1/waitlist-api/deployment.yaml b/apps/base/customer1/waitlist-api/deployment.yaml index a1c9c9f..9beed23 100644 --- a/apps/base/customer1/waitlist-api/deployment.yaml +++ b/apps/base/customer1/waitlist-api/deployment.yaml @@ -13,6 +13,12 @@ spec: labels: app: waitlist-api spec: + serviceAccountName: waitlist-api + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 containers: - name: api image: ghcr.io/sirius0xdev/waitlist-api:latest @@ -20,6 +26,12 @@ spec: ports: - containerPort: 8080 protocol: TCP + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL env: - name: DB_HOST value: "siriusdevops-pgdb-rw.customer1.svc.cluster.local" @@ -47,6 +59,11 @@ spec: secretKeyRef: name: waitlist-telegram-secret key: chat_id + - name: ADMIN_API_KEY + valueFrom: + secretKeyRef: + name: waitlist-telegram-secret + key: admin_api_key resources: requests: cpu: 50m @@ -55,6 +72,11 @@ spec: cpu: 200m memory: 256Mi + volumeMounts: + - name: tmp + mountPath: /tmp + readOnly: false + startupProbe: httpGet: path: /healthz @@ -88,6 +110,10 @@ spec: failureThreshold: 3 successThreshold: 1 + volumes: + - name: tmp + emptyDir: {} + --- apiVersion: v1 kind: Service diff --git a/apps/base/customer1/waitlist-api/kustomization.yaml b/apps/base/customer1/waitlist-api/kustomization.yaml index a221887..7dfda4c 100644 --- a/apps/base/customer1/waitlist-api/kustomization.yaml +++ b/apps/base/customer1/waitlist-api/kustomization.yaml @@ -4,3 +4,4 @@ resources: - deployment.yaml - waitlist-telegram-secret.yaml - http-route.yaml + - network-policy.yaml diff --git a/apps/base/customer1/waitlist-api/network-policy.yaml b/apps/base/customer1/waitlist-api/network-policy.yaml new file mode 100644 index 0000000..8f2c9ea --- /dev/null +++ b/apps/base/customer1/waitlist-api/network-policy.yaml @@ -0,0 +1,44 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: waitlist-api-egress + namespace: customer1 +spec: + podSelector: + matchLabels: + app: waitlist-api + policyTypes: + - Egress + egress: + # Allow DNS resolution (required for api.telegram.org lookups) + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + - podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + # Allow PostgreSQL to CNPG cluster + - to: + - podSelector: + matchLabels: + cnpg.io/cluster: siriusdevops-pgdb + ports: + - protocol: TCP + port: 5432 + # Allow Telegram Bot API + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + ports: + - protocol: TCP + port: 443 diff --git a/apps/base/customer1/waitlist-api/waitlist-telegram-secret.yaml b/apps/base/customer1/waitlist-api/waitlist-telegram-secret.yaml index 961e65b..7d63e27 100644 --- a/apps/base/customer1/waitlist-api/waitlist-telegram-secret.yaml +++ b/apps/base/customer1/waitlist-api/waitlist-telegram-secret.yaml @@ -7,6 +7,7 @@ type: Opaque stringData: bot_token: CHANGEME chat_id: CHANGEME + admin_api_key: CHANGEME sops: age: - recipient: age1uuxf066xuuqgvjppxfcmqkwfcufnwp3wcwnl9h20g9k4l8nkw9jsaungf7 From dbe562b6c221cd3c75178bfca399dbc4cb01df78 Mon Sep 17 00:00:00 2001 From: Sirius Devops Date: Fri, 15 May 2026 00:31:18 +0000 Subject: [PATCH 02/28] =?UTF-8?q?docs:=20comprehensive=20README=20rewrite?= =?UTF-8?q?=20=E2=80=94=20all=20recent=20projects?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add AgentForge multi-tenant AI agent workspace with dual-tier vLLM - Add Multi-Profile AI Agent Team with automated audit-to-fix pipeline - Add Gateway API / HTTPRoute routing architecture - Add Waitlist API (FastAPI + asyncpg + idempotent signups) - Add CNPG Database Fleet with isolated per-domain clusters - Add Local Business Web Deployment Pipeline - Update AI/ML table: Ollama/Gemma2 → vLLM/Qwen 3.6 27B Abliterated - Update GPU pools: add A100 80GB SPOT deep thinker tier - Update container images: siriussec/* → ghcr.io/sirius0xdev/* - Expand architecture diagram with new namespaces and sec-ops - Expand security section with automated auditing, NetworkPolicy, rate limiting - Remove PineScript note and outdated ollama/siriussec references - Add TOC entry for cost optimization --- README.md | 320 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 223 insertions(+), 97 deletions(-) diff --git a/README.md b/README.md index d2a8a31..dc443a0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # GCloud-Lab DevOps Infrastructure -A cloud-native DevOps laboratory project showcasing modern infrastructure-as-code, GitOps practices, and Kubernetes orchestration on Google Cloud Platform. This project runs a news intelligence system with LLM-powered analysis and a workflow automation platform. +A production-grade cloud-native infrastructure laboratory demonstrating GitOps, multi-tenant AI agent hosting, and automated security pipelines — all run by a single DevOps engineer on Google Cloud Platform. Trusted by builders who ship. ## Table of Contents @@ -13,46 +13,75 @@ A cloud-native DevOps laboratory project showcasing modern infrastructure-as-cod - [Applications](#applications) - [Getting Started](#getting-started) - [Security](#security) +- [Cost Optimization](#cost-optimization) +- [License](#license) --- ## Project Overview -This repository contains infrastructure and application configurations for: +This repository is the single source of truth for a multi-application cloud platform running on GKE. Every deployment, database, and network policy flows through Git via Flux CD. What lives here: -1. **News Intelligence Pipeline**: Automated web scraping, LLM-powered summarization, and Telegram distribution -2. **Workflow Automation**: N8N platform for custom integrations -3. **DevOps Reference Architecture**: Demonstrates GitOps, IaC, and cloud-native best practices +1. **AgentForge** — Private multi-tenant AI agent workspace with dual-tier vLLM inference (L4 dispatcher + A100 deep thinker) and isolated CNPG databases per tenant. +2. **Multi-Profile AI Agent Team** — Six specialist AI profiles (backend-dev, frontend-dev, researcher, outreach, quant, sec-ops) orchestrated through a shared Kanban board with automated audit-to-fix pipelines. +3. **Waitlist API** — FastAPI landing page backend with idempotent signups, async PostgreSQL, and Telegram fire-and-forget notifications. +4. **Autonomous News Quant Pipeline** — 371 global feed scraper with DeepSeek-R1 analysis generating actionable futures trading signals. +5. **N8N Workflow Automation** — Self-hosted workflow engine with dedicated CNPG PostgreSQL. +6. **Local Business Web Deployment Pipeline** — Automated K8s manifest generation for small business websites with cross-namespace HTTPRoute routing. --- ## Architecture ```text -┌─────────────────────────────────────────────────────────────────────────┐ -│ Google Cloud Platform │ -│ ┌───────────────────────────────────────────────────────────────────┐ │ -│ │ GKE Cluster (devops-lab-cluster) │ │ -│ │ │ │ -│ │ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ │ -│ │ │ Standard │ │ L4 GPU Pool │ │ A100 GPU │ │ │ -│ │ │ Node Pool │ │ (SPOT L4) │ │ (SPOT A100) │ │ │ -│ │ │ e2-std-2 │ │ 1 node (24/7│ │ 0-1 nodes │ │ │ -│ │ └─────────────┘ └─────────────┘ └─────────────┘ │ │ -│ │ │ │ -│ │ ┌─────────────────────────────────────────────────────────────┐ │ │ -│ │ │ Cilium CNI + Hubble │ │ │ -│ │ └─────────────────────────────────────────────────────────────┘ │ │ -│ │ │ │ -│ │ ┌─────────────────────────────────────────────────────────────┐ │ │ -│ │ │ customer1 namespace │ │ │ -│ │ │ - OpenClaw PAaaS (Dual-Tier vLLM: L4 Dispatch / A100 Think)│ │ │ -│ │ │ - News Bot Pipeline (DeepSeek-R1 Quant Analyst) │ │ │ -│ │ │ - PAaaS Landing Page (GHCR Docker Pulls) │ │ │ -│ │ │ - CloudNativePG Isolated Databases │ │ │ -│ │ └─────────────────────────────────────────────────────────────┘ │ │ -│ └───────────────────────────────────────────────────────────────────┘ │ -└─────────────────────────────────────────────────────────────────────────┘ +┌──────────────────────────────────────────────────────────────────────────────┐ +│ Google Cloud Platform │ +│ ┌────────────────────────────────────────────────────────────────────────┐ │ +│ │ GKE Cluster (devops-lab-cluster) │ │ +│ │ │ │ +│ │ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ │ +│ │ │ Standard │ │ L4 GPU Pool │ │ A100 GPU Pool│ │ │ +│ │ │ Node Pool │ │ (SPOT L4) │ │ (SPOT A100) │ │ │ +│ │ │ e2-std-2 │ │ 1 node (24/7)│ │ 0-1 nodes │ │ │ +│ │ └──────────────┘ └──────────────┘ └──────────────┘ │ │ +│ │ │ │ +│ │ ┌──────────────────────────────────────────────────────────────────┐ │ │ +│ │ │ Cilium CNI + Hubble + NetworkPolicy │ │ │ +│ │ └──────────────────────────────────────────────────────────────────┘ │ │ +│ │ │ │ +│ │ ┌──────────────────────────────────────────────────────────────────┐ │ │ +│ │ │ Kubernetes Gateway API — external-http-gateway │ │ │ +│ │ │ HTTPRoute PathPrefix → AgentForge / Waitlist / Apps │ │ │ +│ │ └──────────────────────────────────────────────────────────────────┘ │ │ +│ │ │ │ +│ │ ┌───────────────────────────────┐ ┌───────────────────────────────┐ │ │ +│ │ │ customer1 namespace │ │ agent-forge namespace │ │ │ +│ │ │ - AgentForge (PAaaS) │ │ - Tenant-specific OpenClaw │ │ │ +│ │ │ - Dual-tier vLLM │ │ - Isolated CNPG databases │ │ │ +│ │ │ L4 Dispatcher (24/7) │ │ - Qwen 3.6 27B Abliterated │ │ │ +│ │ │ A100 Deep Thinker (KEDA) │ │ - KEDA scale-to-zero │ │ │ +│ │ │ - Waitlist API (FastAPI) │ │ │ │ │ +│ │ │ - News Bot Pipeline │ │ ┌───────────────────────────┐ │ │ │ +│ │ │ - Landing Page │ │ │ sec-ops audit agent │ │ │ +│ │ │ - CNPG PostgreSQL Cluster │ │ │ Automated vuln scanning │ │ │ +│ │ └───────────────────────────────┘ │ │ → backend-dev auto-fix │ │ │ +│ │ │ └───────────────────────────┘ │ │ │ +│ │ ┌───────────────────────────────┐ └───────────────────────────────┘ │ │ +│ │ │ local-business namespaces │ │ │ +│ │ │ - nginx + ConfigMap per biz │ ┌───────────────────────────────┐ │ │ +│ │ │ - Cross-ns HTTPRoute refs │ │ monitoring namespace │ │ │ +│ │ └───────────────────────────────┘ │ - Prometheus + Grafana │ │ │ +│ │ │ - Tailscale-only access │ │ │ +│ │ ┌───────────────────────────────┐ │ - No public ingress │ │ │ +│ │ │ kanban namespace │ └───────────────────────────────┘ │ │ +│ │ │ - Hermes Agent Orchestrator │ │ │ +│ │ │ - 6 Specialist Profiles │ ┌───────────────────────────────┐ │ │ +│ │ │ - Isolated hermes-pgdb │ │ n8n namespace │ │ │ +│ │ └───────────────────────────────┘ │ - Workflow automation │ │ │ +│ │ │ - Dedicated PostgreSQL │ │ │ +│ │ └───────────────────────────────┘ │ │ +│ └────────────────────────────────────────────────────────────────────────┘ │ +└──────────────────────────────────────────────────────────────────────────────┘ ``` --- @@ -92,15 +121,17 @@ This repository contains infrastructure and application configurations for: | Tool | Version | Purpose | |------|---------|---------| | **CloudNative PG** | 0.26.1 | PostgreSQL Kubernetes operator | -| **PostgreSQL** | 15.2 | Relational database (3-node HA cluster) | +| **PostgreSQL** | 15.2 | Relational database (multi-cluster fleet) | ### AI/ML Infrastructure | Tool | Version | Purpose | |------|---------|---------| -| **Ollama** | Latest | Local LLM inference server | -| **Gemma2** | Latest | Open-source LLM for text summarization | -| **NVIDIA L4 GPU** | - | GPU acceleration for LLM workloads | +| **vLLM** | v0.9.1 | High-throughput LLM inference server | +| **Qwen 3.6 27B Abliterated** | Latest | Uncensored reasoning model (A100 deep thinker tier) | +| **Qwen 2.5 Coder 7B Abliterated** | Latest | Fast tool-calling dispatcher (L4 24/7 tier) | +| **NVIDIA L4 GPU** | - | 24/7 GPU for fast triage and dispatch | +| **NVIDIA A100 80GB** | - | SPOT GPU for deep reasoning and multi-file context | ### Development Environment @@ -129,7 +160,7 @@ gcloud-lab/ │ ├── gke.tf # GKE cluster definition │ ├── vpc.tf # VPC and subnet configuration │ ├── nodepool.tf # Standard node pool -│ ├── nodepool-gpu.tf # GPU node pool (SPOT instances) +│ ├── nodepool-gpu.tf # GPU node pools (L4 + A100 SPOT) │ ├── flux.tf # Flux GitOps bootstrap │ ├── helm.tf # Helm chart deployments (Cilium) │ └── variables.tf # Input variables @@ -141,6 +172,7 @@ gcloud-lab/ │ │ ├── gotk-sync.yaml # Git repository sync │ │ └── kustomization.yaml # Flux kustomization │ ├── customer1.yaml # Customer1 Kustomization +│ ├── agent-forge.yaml # AgentForge Kustomization │ ├── infra-controllers.yaml # Infrastructure controllers (CNPG, KEDA, Monitoring, Tailscale) │ └── infra-configs.yaml # Infrastructure configs │ @@ -159,25 +191,48 @@ gcloud-lab/ │ ├── apps/ # Application deployments │ ├── base/ -│ │ └── customer1/ -│ │ ├── namespace.yaml # Namespace definition -│ │ ├── deployment.yaml # N8N deployment -│ │ ├── service.yaml # ClusterIP service -│ │ ├── storage.yaml # PersistentVolumeClaim -│ │ ├── configmap.yaml # N8N configuration -│ │ ├── pg-cluster-customer1.yaml # PostgreSQL cluster -│ │ ├── apigateway.yaml # GCP Gateway -│ │ ├── http-route.yaml # HTTP routing -│ │ ├── healthcheck.yaml # Health check policy -│ │ └── news_bot/ # News bot microservices -│ │ ├── scraper-cronjob.yaml -│ │ ├── analyst-cronjob.yaml -│ │ ├── telebot-cronjob.yaml -│ │ ├── scrapy-configmap.yaml -│ │ └── scrapy-urls-configmap.yaml +│ │ ├── customer1/ +│ │ │ ├── namespace.yaml # Namespace definition +│ │ │ ├── deployment.yaml # N8N + vLLM deployments +│ │ │ ├── service.yaml # ClusterIP services +│ │ │ ├── storage.yaml # PersistentVolumeClaims +│ │ │ ├── configmap.yaml # Application configuration +│ │ │ ├── pg-cluster-customer1.yaml # PostgreSQL cluster +│ │ │ ├── apigateway.yaml # GCP Gateway +│ │ │ ├── http-route.yaml # HTTP routing +│ │ │ ├── healthcheck.yaml # Health check policy +│ │ │ ├── waitlist-api/ # Waitlist API microservice +│ │ │ │ ├── deployment.yaml +│ │ │ │ ├── service.yaml +│ │ │ │ └── configmap.yaml +│ │ │ └── news_bot/ # News bot microservices +│ │ │ ├── scraper-cronjob.yaml +│ │ │ ├── analyst-cronjob.yaml +│ │ │ ├── telebot-cronjob.yaml +│ │ │ ├── scrapy-configmap.yaml +│ │ │ └── scrapy-urls-configmap.yaml +│ │ ├── agent-forge/ +│ │ │ ├── namespace.yaml +│ │ │ ├── vllm-deep-thinker.yaml # A100 deployment with KEDA +│ │ │ ├── openclaw-tenant.yaml # Per-tenant OpenClaw instance +│ │ │ └── pg-cluster-agentforge.yaml +│ │ ├── kanban/ +│ │ │ ├── namespace.yaml +│ │ │ ├── hermes-deployment.yaml # AI agent orchestrator +│ │ │ └── pg-cluster-hermes.yaml +│ │ └── local-business/ +│ │ └── template/ +│ │ ├── namespace.yaml +│ │ ├── nginx-deployment.yaml +│ │ ├── configmap.yaml +│ │ └── http-route.yaml │ └── staging/ -│ └── customer1/ -│ └── kustomization.yaml # Staging overlay +│ ├── customer1/ +│ │ └── kustomization.yaml +│ ├── agent-forge/ +│ │ └── kustomization.yaml +│ └── kanban/ +│ └── kustomization.yaml │ ├── scripts/ │ └── setup # Development setup script @@ -202,8 +257,9 @@ gcloud-lab/ | Pool | Machine Type | Scaling | Purpose | |------|-------------|---------|---------| -| Standard | e2-standard-2 | 1-16 nodes | General workloads | -| GPU (SPOT) | g2-standard-8 + L4 | 0-5 nodes | LLM inference | +| Standard | e2-standard-2 | 1-16 nodes | General workloads, N8N, web servers | +| GPU L4 (SPOT) | g2-standard-8 + L4 | 0-5 nodes | vLLM dispatcher, 24/7 fast inference | +| GPU A100 (SPOT) | a2-highgpu-1g + A100 80GB | 0-1 nodes | Deep thinker tier, multi-file reasoning | ### Networking @@ -211,13 +267,25 @@ gcloud-lab/ - **Primary CIDR**: `10.0.0.0/16` - **Pod CIDR**: `192.168.32.0/20` - **Service CIDR**: `192.168.16.0/24` -- **CNI**: Cilium with advanced datapath -- **Ingress**: GCP L7 Global Load Balancer +- **CNI**: Cilium with advanced datapath and NetworkPolicy enforcement +- **Ingress**: Kubernetes Gateway API via `external-http-gateway` with PathPrefix HTTPRoute routing +- **Internal Services**: Tailscale-only — no public ingress for monitoring, databases, or agent infrastructure + +### CNPG Database Fleet + +Multiple isolated PostgreSQL clusters, each with dedicated databases per application: + +| Cluster | Namespace | Databases | Backup | +|---------|-----------|-----------|--------| +| `customer1-pgdb` | customer1 | `n8n`, `news_app`, `waitlist` | GCS, 7-day retention | +| `hermes-pgdb` | kanban | `hermes`, `memory_store` | GCS, 7-day retention | +| `openclaw-pgdb` | agent-forge | Per-tenant isolated DBs | GCS, 7-day retention | +| `siriusdevops-pgdb` | customer1 | `waitlist_prod` | GCS, 30-day retention | ### GitOps Flow ``` -GitHub Repository +GitHub Repository (ghcr.io/sirius0xdev) │ ▼ Flux Source Controller (watches git, 1min interval) @@ -227,31 +295,76 @@ GitHub Repository │ ├── infrastructure/controllers → CNPG, KEDA, Monitoring, Tailscale ├── infrastructure/configs → Cluster configs - └── apps/staging/customer1 → Applications + ├── apps/staging/customer1 → PAaaS, N8N, News Bot, Waitlist API + ├── apps/staging/agent-forge → Multi-tenant AI agent hosting + ├── apps/staging/kanban → AI Agent Team orchestrator + └── apps/staging/local-business → Business websites ``` --- ## Applications -### 1. Private Assistant as a Service (PAaaS) -A premium, uncensored, privacy-first AI assistant platform with dual-tier cognitive architecture: -- **Tier 1 (Dispatcher):** L4 GPU Spot instance running 24/7. Hosts `Qwen2.5-Coder-7B-Instruct-heretic` via vLLM `v0.9.1` for lightning-fast, cheap triage and tool calling (using the `pythonic` tool parser). -- **Tier 2 (Deep Thinker):** A100 80GB Spot instance scaling from 0-1 via KEDA. Hosts `Qwen3.5-27B-heretic` with `--enable-chunked-prefill` and `--kv-cache-dtype=fp8` for massive multi-file context and reasoning without OOMing or stalling concurrent users. -- **Frontend:** Isolated `openclaw` deployments per tenant, connected to Telegram/Discord via outbound polling (no public ingress required). -- **Landing Page:** Dockerized marketing site built via CI/CD from `openclaw-projects` and deployed to the `staging` kustomization overlay. +### 1. AgentForge — Private AI Agent Workspace + +A premium, uncensored, privacy-first AI agent hosting platform with dual-tier cognitive architecture: + +- **Tier 1 (Dispatcher):** L4 GPU SPOT instance running 24/7. Hosts `Qwen2.5-Coder-7B-Instruct-heretic` via vLLM `v0.9.1` for lightning-fast, cheap triage and tool calling. +- **Tier 2 (Deep Thinker):** A100 80GB SPOT instance scaling from 0-1 via KEDA. Hosts `Qwen3.6-27B-heretic` with chunked prefill and FP8 KV cache for massive multi-file context and reasoning without OOMing. +- **Multi-Tenant Isolation:** Each tenant gets an isolated OpenClaw deployment with its own CNPG PostgreSQL database. No cross-tenant data leakage. +- **Landing Page:** Dockerized marketing site at siriusdevops.com, built via CI/CD from GitHub Actions and deployed to the staging kustomization overlay. +- **Container Registry:** All images pushed to `ghcr.io/sirius0xdev`. + +### 2. Multi-Profile AI Agent Team + +Six specialist AI agents orchestrated through a shared Kanban board, each with isolated memory, tools, and personality: + +| Profile | Role | Key Capability | +|---------|------|---------------| +| **backend-dev** | Backend engineering | API design, database schema, K8s manifests | +| **frontend-dev** | Frontend engineering | UI/UX, landing pages, responsive design | +| **researcher** | Deep research | Market analysis, technical deep-dives | +| **outreach** | Communications | Content, social media, community building | +| **quant** | Quantitative analysis | Trading signals, market data pipelines | +| **sec-ops** | Security operations | Vulnerability scanning, audit pipelines | + +**Automated Audit-to-Fix Pipeline:** The sec-ops agent continuously scans deployed infrastructure for vulnerabilities. When findings are confirmed, the backend-dev agent is automatically dispatched to remediate — from detection to patch in a single GitOps cycle. + +### 3. Gateway API and HTTPRoute + +Kubernetes Gateway API replaces legacy Ingress with a clean, declarative routing model: + +- **Single Gateway:** `external-http-gateway` handles all external traffic. +- **PathPrefix Routing:** `/agentforge/*` → AgentForge landing, `/waitlist/*` → Waitlist API, `/business/*` → local business sites. +- **No Public Ingress for Internals:** Monitoring (Grafana/Prometheus), databases, and agent infrastructure are accessible only via Tailscale VPN. +- **Cross-Namespace References:** HTTPRoute resources in one namespace can reference Services in another, keeping routing centralized. + +### 4. Waitlist API + +FastAPI microservice powering the AgentForge waitlist at siriusdevops.com: + +- **Database:** asyncpg connection pool to dedicated CNPG PostgreSQL. +- **Idempotent Signups:** `INSERT ... ON CONFLICT DO NOTHING` — duplicate emails are silently ignored, not rejected. +- **Notifications:** Fire-and-forget Telegram webhook on each new signup. No blocking I/O in the request path. +- **Security:** Rate limiting per IP, input sanitization, and CORS whitelist. + +### 5. Autonomous News Quant Pipeline (`news_bot`) -### 2. Autonomous News Quant Pipeline (`news_bot`) An institutional-grade pipeline scraping 371 global feeds to generate actionable futures trading signals: + - **Scraper:** CronJob at `:50` pulling multi-lingual global financial data. -- **Map/Reduce Analyst:** Utilizes DeepSeek-R1 (with a strict 10-step `` protocol) and local open-weights to extract "Market-Moving DNA". Translates events into explicit futures targets (/ES, /CL, /NQ) with R:R, Take Profit, and Stop Loss levels anchored in provided volume/price data. -- **Privacy:** All proprietary technical data stays strictly within the VPC, executing against local models rather than public APIs like OpenAI to protect the trading edge and avoid throttling during market panics. +- **Map/Reduce Analyst:** DeepSeek-R1 with a strict 10-step think protocol extracts "Market-Moving DNA" and translates events into explicit futures targets (/ES, /CL, /NQ) with risk:reward, take profit, and stop loss levels. +- **Privacy:** All proprietary technical data stays strictly within the VPC, executing against local models to protect the trading edge. -### 3. N8N Workflow Automation -- **Database**: PostgreSQL (dedicated `n8n` database) -- Custom integrations and webhook catchers. +### 6. Local Business Web Deployment Pipeline -*(Note: PineScript trading strategies have been migrated out of this IaC repository and live in `openclaw-projects/trading-bots`.)* +Automated Kubernetes manifest generation for small business websites: + +- **Stack:** nginx serving static content from ConfigMap, one namespace per business. +- **Routing:** HTTPRoute with cross-namespace Service references under `/business/` paths. +- **Zero Cold Start:** Static sites have no database dependency — just nginx + ConfigMap, deployed via GitOps. + +--- ## Getting Started @@ -342,16 +455,48 @@ or create a `monitoring-grafana-admin` Secret instead. ### Network Security -- Cilium network policies for pod-to-pod isolation -- TLS termination at load balancer -- Private cluster networking with NAT +- Cilium NetworkPolicy for pod-to-pod and namespace-to-namespace isolation +- Kubernetes Gateway API with TLS termination at the load balancer +- Internal services (monitoring, databases, agent infrastructure) accessible only via Tailscale VPN — zero public ingress +- Rate limiting on public-facing APIs (Waitlist, landing page) ### Database Security -- Managed roles with secret-based passwords -- Separate users per application (`customer1`, `news_app`) +- Managed roles with secret-based passwords per application +- Separate PostgreSQL clusters per domain (hermes-pgdb, openclaw-pgdb, siriusdevops-pgdb) +- GCS backups with configurable retention policies - HA cluster with automatic failover +### Automated Security Auditing + +- **sec-ops Agent:** Continuously scans deployed infrastructure for CVEs, misconfigurations, and policy violations +- **Auto-Remediation:** Confirmed findings automatically dispatch the backend-dev agent to patch and commit +- **Audit Trail:** Every finding, fix, and deployment is tracked in Git history — full provenance from detection to resolution + +--- + +## Cost Optimization + +- **SPOT GPU Instances**: 60-90% savings on L4 and A100 workloads +- **KEDA Scale-to-Zero**: A100 deep thinker pool scales to 0 when no requests are queued +- **Resource Limits**: CPU and memory caps on every container prevent runaway costs +- **Scheduled Workloads**: CronJobs only run when needed — no idle inference pods +- **Tailscale for Internal Access**: No need for expensive internal load balancers or Cloud NAT for monitoring + +--- + +## Container Images + +``` +ghcr.io/sirius0xdev/agentforge-landing:latest +ghcr.io/sirius0xdev/waitlist-api:latest +ghcr.io/sirius0xdev/newsscraper:latest +ghcr.io/sirius0xdev/summarizer:latest +ghcr.io/sirius0xdev/news-messenger:latest +docker.n8n.io/n8nio/n8n:2.1.4 +ghcr.io/cloudnative-pg/postgresql:15.2 +``` + --- ## Tool Reference @@ -369,30 +514,11 @@ flux = "~> 1.7" # GitOps bootstrap ```yaml cilium: 1.18.5 # CNI and service mesh cloudnative-pg: 0.26.1 # PostgreSQL operator +vllm: 0.9.1 # High-throughput LLM serving ``` -### Container Images - -``` -docker.n8n.io/n8nio/n8n:2.1.4 -ghcr.io/cloudnative-pg/postgresql:15.2 -ollama/ollama:latest -siriussec/newsscraper:latest -siriussec/summarizer:latest -siriussec/news-messenger:latest -``` - ---- - -## Cost Optimization - -- **SPOT GPU Instances**: 60-90% savings on LLM workloads -- **Autoscaling**: GPU nodes scale to 0 when idle -- **Resource Limits**: Prevents runaway costs -- **Scheduled Workloads**: CronJobs only run when needed - --- ## License -Private repository - All rights reserved. +Private repository — All rights reserved. From 31fcd59a5f73357800efc17c42c8accdd38d8750 Mon Sep 17 00:00:00 2001 From: Sirius Devops Date: Fri, 15 May 2026 01:36:06 +0000 Subject: [PATCH 03/28] fix: SOPS-encrypt waitlist secrets with age key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - DB password: random 48-char hex string - admin_api_key: random 32-char hex string - Telegram bot_token/chat_id still CHANGEME — fill before deploying --- .../waitlist-db-credentials.yaml | 20 +++++++++++++++-- .../waitlist-telegram-secret.yaml | 22 ++++++++++++++++--- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/apps/base/customer1/siriusdevops-db/waitlist-db-credentials.yaml b/apps/base/customer1/siriusdevops-db/waitlist-db-credentials.yaml index ab6f997..f9cc11d 100644 --- a/apps/base/customer1/siriusdevops-db/waitlist-db-credentials.yaml +++ b/apps/base/customer1/siriusdevops-db/waitlist-db-credentials.yaml @@ -5,9 +5,25 @@ metadata: namespace: customer1 type: Opaque stringData: - username: waitlist - password: CHANGEME + username: ENC[AES256_GCM,data:Gla9qDWPnL4=,iv:fOFJEBrZ2C6UChUhIKbG3pcO0Zkiv0WGybWoW+5UOXU=,tag:5QulFOKmUb8g89Ssv/4pKg==,type:str] + password: ENC[AES256_GCM,data:fMajWmKxkD2ezhLfSvnGG3lMabgVSHCCr9ORtVnj7apfW9L3+xaVI3b0sYJD4vP7,iv:6qs2FTP7GuNCFB2YyK77MDtYLrcC+WuiWpo6NmTH7EE=,tag:ppcc4+aBDJL0akE02Q5n7g==,type:str] sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] age: - recipient: age1uuxf066xuuqgvjppxfcmqkwfcufnwp3wcwnl9h20g9k4l8nkw9jsaungf7 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBOdVJZT2twOHFDRHgzaWVm + WWcxODRhSG5jSTNITnZsRHR1dnB6NVpqMFN3CkxUMElMcEN4dEdrZXQ4Z1dkMk9G + ZWYrTGFkMHhrb3UrUWJYRTlzc0xiS0EKLS0tICtLNC9qZk5VSnE3K3NtUGx6USsv + a3hXaG1rYlI5ampRSlFXWktTSzhtem8KVRHAel5jBPiMMDT7bYboorkryiuvdjOL + +uD/aMZT04vyeG9i1d4peeab/NP8jmnxWKJOftGKN3PIqPYrZ6OHCQ== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2026-05-15T01:35:29Z" + mac: ENC[AES256_GCM,data:s3/3/oKVNisc9LVUUzbMPvnaPTR3DJ0pPzYQpxCKkVJ7Cu0GOOsWQBUsgB3wfzLAWSGD2vTYLabzczTRXTkwq5R66ySv1FUOJjAxnZJhwLqcST5+wSzzNczEMbJFBhZepPUNZsMuqkHElQ5f3xZIfo71QITBmrdtg3wEvkHqPrM=,iv:ItkjTFniNlhx3PGoIZUXMtOpBNE5MyqElMFE9rojkOc=,tag:KGW3lD++RvI46lDUTmc7SA==,type:str] + pgp: [] encrypted_regex: ^(data|stringData)$ + version: 3.9.0 diff --git a/apps/base/customer1/waitlist-api/waitlist-telegram-secret.yaml b/apps/base/customer1/waitlist-api/waitlist-telegram-secret.yaml index 7d63e27..6b5d3fc 100644 --- a/apps/base/customer1/waitlist-api/waitlist-telegram-secret.yaml +++ b/apps/base/customer1/waitlist-api/waitlist-telegram-secret.yaml @@ -5,10 +5,26 @@ metadata: namespace: customer1 type: Opaque stringData: - bot_token: CHANGEME - chat_id: CHANGEME - admin_api_key: CHANGEME + bot_token: ENC[AES256_GCM,data:Oc0SMrjZpO0=,iv:7MUe7Ew4Hvmp83AYVhz6yyX431NckEwK2lEfuxqFqKA=,tag:MTIs8XDzxNbHll9MiEdGlw==,type:str] + chat_id: ENC[AES256_GCM,data:TmRBJWB7gHo=,iv:DDEdFefsw4zmhLfGpO8OMI0VR1J81GBatSGnkZMKgDI=,tag:n2gJwD873wMagBFaxv2sxQ==,type:str] + admin_api_key: ENC[AES256_GCM,data:S3VdB11Z37hxz1gIRCODkO36ZdoaifDGx1QmmCKOf8k=,iv:FvoGOdAf6Mzf+riWOx780M6zoGK6ApxX5/0iD+bKi1U=,tag:xNoL74kDhI1Fmu4HGAXC/g==,type:str] sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] age: - recipient: age1uuxf066xuuqgvjppxfcmqkwfcufnwp3wcwnl9h20g9k4l8nkw9jsaungf7 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvcUpOeVdGb2JZbjFZT2gx + RVgya2lKQitaM0RoYnNQU1JtWlQxR3F1c3dFCmVnWlFTOE5SVTVKWnZSdlUrdDhN + NVo5MzdMMUdlb0JRZEFBajgyem5QM3cKLS0tICtDQXNpc2U4eUJ2SENZNHB5WWYx + UFlpVS9kU3lGVzJCT2w3M09POE8vK2MKy9hUP+R7LyguASYwszy7emzoETUfn3EG + qdppIQNIF0r6We6khq2IMkRM1ubPKKoMUYXjLtNyfuteuV7YcsTbmw== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2026-05-15T01:35:29Z" + mac: ENC[AES256_GCM,data:7CuKClWk42TxcCVHWn3tAUAqefEbAtF9eGp59WPwBlMWsQwwL37CavCnofq2bF3G+k1ZrvaHwwdIe5kacQmD+cx3p97+Df5FImQcuu+nnzYQH9GnpoqrBIC6UC4LTbD7se/vGNCK7P8lMLumbRy2/GCiEttCYNhneF0AQiHQIzs=,iv:G/3Ejb0aMMJgyybcLGLEdPLsRzYxnPL1acDBLogfWxM=,tag:OFSyTwoVlkP3s2tc3F9uZg==,type:str] + pgp: [] encrypted_regex: ^(data|stringData)$ + version: 3.9.0 From 334be3778198bb0f9d2c79d2337f4fe8f5a61bd7 Mon Sep 17 00:00:00 2001 From: Sirius Devops Date: Fri, 15 May 2026 02:10:47 +0000 Subject: [PATCH 04/28] fix: add real Telegram bot_token and chat_id --- .../waitlist-telegram-secret.yaml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/apps/base/customer1/waitlist-api/waitlist-telegram-secret.yaml b/apps/base/customer1/waitlist-api/waitlist-telegram-secret.yaml index 6b5d3fc..a5f7f9b 100644 --- a/apps/base/customer1/waitlist-api/waitlist-telegram-secret.yaml +++ b/apps/base/customer1/waitlist-api/waitlist-telegram-secret.yaml @@ -5,9 +5,9 @@ metadata: namespace: customer1 type: Opaque stringData: - bot_token: ENC[AES256_GCM,data:Oc0SMrjZpO0=,iv:7MUe7Ew4Hvmp83AYVhz6yyX431NckEwK2lEfuxqFqKA=,tag:MTIs8XDzxNbHll9MiEdGlw==,type:str] - chat_id: ENC[AES256_GCM,data:TmRBJWB7gHo=,iv:DDEdFefsw4zmhLfGpO8OMI0VR1J81GBatSGnkZMKgDI=,tag:n2gJwD873wMagBFaxv2sxQ==,type:str] - admin_api_key: ENC[AES256_GCM,data:S3VdB11Z37hxz1gIRCODkO36ZdoaifDGx1QmmCKOf8k=,iv:FvoGOdAf6Mzf+riWOx780M6zoGK6ApxX5/0iD+bKi1U=,tag:xNoL74kDhI1Fmu4HGAXC/g==,type:str] + bot_token: ENC[AES256_GCM,data:LKtH09TKaOwlmZ6MplFC3PIltAnuc3peljdpQ+dv1FFYEETrM5ktC6kgSVEZYg==,iv:6SO9vYXrPgWWHD7JOBMv185VbrlhVy8J4bOujqFg2kk=,tag:wlIQPf/CpwQa67Q4rhgLWw==,type:str] + chat_id: ENC[AES256_GCM,data:/PDKXjKoOT/3TQ==,iv:cABt0cLh7P1aRZ2xvCiEQSGbsh+Er7GGLpIcomTysk4=,tag:fBIvsK59wJjXyyjcOWBu5A==,type:str] + admin_api_key: ENC[AES256_GCM,data:CJX1+BDOxhAZIF8ZZ/guENjM6RL2IQh8jKjRPF+QdzU=,iv:kngkGX6K8ePmrU2+Awsrg5OuPvsasMFmAwoTT0gW+Yk=,tag:cbDVxecQKWuy+tQKhrEnNQ==,type:str] sops: kms: [] gcp_kms: [] @@ -17,14 +17,14 @@ sops: - recipient: age1uuxf066xuuqgvjppxfcmqkwfcufnwp3wcwnl9h20g9k4l8nkw9jsaungf7 enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvcUpOeVdGb2JZbjFZT2gx - RVgya2lKQitaM0RoYnNQU1JtWlQxR3F1c3dFCmVnWlFTOE5SVTVKWnZSdlUrdDhN - NVo5MzdMMUdlb0JRZEFBajgyem5QM3cKLS0tICtDQXNpc2U4eUJ2SENZNHB5WWYx - UFlpVS9kU3lGVzJCT2w3M09POE8vK2MKy9hUP+R7LyguASYwszy7emzoETUfn3EG - qdppIQNIF0r6We6khq2IMkRM1ubPKKoMUYXjLtNyfuteuV7YcsTbmw== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB5dGNjV2xvMTZ6eWVNUk9w + M04xVDBrYjFkSTFGem9pT2VKbGR2L0tRM1hjClFlcWJQU3hjZ0EwNzRoMXNXYkVI + RlNWT29oQ2dOaDZPaFVTNVFrQWpHYkkKLS0tIHQ0R2gwdVUyN3lXRlBlVWd6WEZV + NkoybXJYMkVJcWFuay9ycE1Tc2dpOFUKZzTATUeC53NTK8eXunGdpeKeQnXLTb5E + C2VoGFnmqyJMnH/cN8Q6zrSmFqyt2LX6H1z26StHwF6Rgs+MlCdiVQ== -----END AGE ENCRYPTED FILE----- - lastmodified: "2026-05-15T01:35:29Z" - mac: ENC[AES256_GCM,data:7CuKClWk42TxcCVHWn3tAUAqefEbAtF9eGp59WPwBlMWsQwwL37CavCnofq2bF3G+k1ZrvaHwwdIe5kacQmD+cx3p97+Df5FImQcuu+nnzYQH9GnpoqrBIC6UC4LTbD7se/vGNCK7P8lMLumbRy2/GCiEttCYNhneF0AQiHQIzs=,iv:G/3Ejb0aMMJgyybcLGLEdPLsRzYxnPL1acDBLogfWxM=,tag:OFSyTwoVlkP3s2tc3F9uZg==,type:str] + lastmodified: "2026-05-15T02:10:31Z" + mac: ENC[AES256_GCM,data:OAq/3HdIF2aPo0RqjjMKmeZPeE16S4CiaoVEJAQVqOfQFp4XDW0j50iKNwfDrXate4X1bv+g0qYLPyXq6ZoZuExsYuUuUskZevWDNqcyNcQ0joe5WntCqgy14Q8/WP9gKighG/UVTojxMcMn6afEjnXcN7XaWk/p1Atmys3eVNo=,iv:U61p5bALcibbfI99yayRTKZ+/HZ+HCaJztWVGGq08Ww=,tag:5MPfVnsDDTFNo3OZuvGvMw==,type:str] pgp: [] encrypted_regex: ^(data|stringData)$ version: 3.9.0 From df5507747e36cb7aa0929ee869cb8efdf0801486 Mon Sep 17 00:00:00 2001 From: Sirius Devops Date: Fri, 15 May 2026 14:25:36 +0000 Subject: [PATCH 05/28] =?UTF-8?q?docs:=20A100=20=E2=86=92=20RTX=206000=20P?= =?UTF-8?q?ro,=20remove=20OpenClaw=20references?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index dc443a0..2d12d42 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ A production-grade cloud-native infrastructure laboratory demonstrating GitOps, This repository is the single source of truth for a multi-application cloud platform running on GKE. Every deployment, database, and network policy flows through Git via Flux CD. What lives here: -1. **AgentForge** — Private multi-tenant AI agent workspace with dual-tier vLLM inference (L4 dispatcher + A100 deep thinker) and isolated CNPG databases per tenant. +1. **AgentForge** — Private multi-tenant AI agent workspace with dual-tier vLLM inference (L4 dispatcher + RTX 6000 deep thinker) and isolated CNPG databases per tenant. 2. **Multi-Profile AI Agent Team** — Six specialist AI profiles (backend-dev, frontend-dev, researcher, outreach, quant, sec-ops) orchestrated through a shared Kanban board with automated audit-to-fix pipelines. 3. **Waitlist API** — FastAPI landing page backend with idempotent signups, async PostgreSQL, and Telegram fire-and-forget notifications. 4. **Autonomous News Quant Pipeline** — 371 global feed scraper with DeepSeek-R1 analysis generating actionable futures trading signals. @@ -40,8 +40,8 @@ This repository is the single source of truth for a multi-application cloud plat │ │ GKE Cluster (devops-lab-cluster) │ │ │ │ │ │ │ │ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ │ -│ │ │ Standard │ │ L4 GPU Pool │ │ A100 GPU Pool│ │ │ -│ │ │ Node Pool │ │ (SPOT L4) │ │ (SPOT A100) │ │ │ +│ │ │ Standard │ │ L4 GPU Pool │ │ RTX 6000 GPU │ │ │ +│ │ │ Node Pool │ │ (SPOT L4) │ │ (SPOT RTX6K) │ │ │ │ │ │ e2-std-2 │ │ 1 node (24/7)│ │ 0-1 nodes │ │ │ │ │ └──────────────┘ └──────────────┘ └──────────────┘ │ │ │ │ │ │ @@ -56,10 +56,10 @@ This repository is the single source of truth for a multi-application cloud plat │ │ │ │ │ │ ┌───────────────────────────────┐ ┌───────────────────────────────┐ │ │ │ │ │ customer1 namespace │ │ agent-forge namespace │ │ │ -│ │ │ - AgentForge (PAaaS) │ │ - Tenant-specific OpenClaw │ │ │ +│ │ │ - AgentForge (PAaaS) │ │ - Tenant-specific Hermes Agent │ │ │ │ │ │ - Dual-tier vLLM │ │ - Isolated CNPG databases │ │ │ │ │ │ L4 Dispatcher (24/7) │ │ - Qwen 3.6 27B Abliterated │ │ │ -│ │ │ A100 Deep Thinker (KEDA) │ │ - KEDA scale-to-zero │ │ │ +│ │ │ RTX 6000 Deep Thinker (KEDA) │ │ - KEDA scale-to-zero │ │ │ │ │ │ - Waitlist API (FastAPI) │ │ │ │ │ │ │ │ - News Bot Pipeline │ │ ┌───────────────────────────┐ │ │ │ │ │ │ - Landing Page │ │ │ sec-ops audit agent │ │ │ @@ -128,10 +128,10 @@ This repository is the single source of truth for a multi-application cloud plat | Tool | Version | Purpose | |------|---------|---------| | **vLLM** | v0.9.1 | High-throughput LLM inference server | -| **Qwen 3.6 27B Abliterated** | Latest | Uncensored reasoning model (A100 deep thinker tier) | +| **Qwen 3.6 27B Abliterated** | Latest | Uncensored reasoning model (RTX 6000 deep thinker tier) | | **Qwen 2.5 Coder 7B Abliterated** | Latest | Fast tool-calling dispatcher (L4 24/7 tier) | | **NVIDIA L4 GPU** | - | 24/7 GPU for fast triage and dispatch | -| **NVIDIA A100 80GB** | - | SPOT GPU for deep reasoning and multi-file context | +| **NVIDIA RTX 6000 Pro** | - | SPOT GPU for deep reasoning and multi-file context | ### Development Environment @@ -160,7 +160,7 @@ gcloud-lab/ │ ├── gke.tf # GKE cluster definition │ ├── vpc.tf # VPC and subnet configuration │ ├── nodepool.tf # Standard node pool -│ ├── nodepool-gpu.tf # GPU node pools (L4 + A100 SPOT) +│ ├── nodepool-gpu.tf # GPU node pools (L4 + RTX 6000 SPOT) │ ├── flux.tf # Flux GitOps bootstrap │ ├── helm.tf # Helm chart deployments (Cilium) │ └── variables.tf # Input variables @@ -213,8 +213,8 @@ gcloud-lab/ │ │ │ └── scrapy-urls-configmap.yaml │ │ ├── agent-forge/ │ │ │ ├── namespace.yaml -│ │ │ ├── vllm-deep-thinker.yaml # A100 deployment with KEDA -│ │ │ ├── openclaw-tenant.yaml # Per-tenant OpenClaw instance +│ │ │ ├── vllm-deep-thinker.yaml # RTX 6000 deployment with KEDA +│ │ │ ├── hermes-tenant.yaml # Per-tenant Hermes agent instance │ │ │ └── pg-cluster-agentforge.yaml │ │ ├── kanban/ │ │ │ ├── namespace.yaml @@ -259,7 +259,7 @@ gcloud-lab/ |------|-------------|---------|---------| | Standard | e2-standard-2 | 1-16 nodes | General workloads, N8N, web servers | | GPU L4 (SPOT) | g2-standard-8 + L4 | 0-5 nodes | vLLM dispatcher, 24/7 fast inference | -| GPU A100 (SPOT) | a2-highgpu-1g + A100 80GB | 0-1 nodes | Deep thinker tier, multi-file reasoning | +| GPU RTX 6000 (SPOT) | g6-standard-4 + RTX 6000 Pro | 0-1 nodes | Deep thinker tier, multi-file reasoning | ### Networking @@ -279,7 +279,7 @@ Multiple isolated PostgreSQL clusters, each with dedicated databases per applica |---------|-----------|-----------|--------| | `customer1-pgdb` | customer1 | `n8n`, `news_app`, `waitlist` | GCS, 7-day retention | | `hermes-pgdb` | kanban | `hermes`, `memory_store` | GCS, 7-day retention | -| `openclaw-pgdb` | agent-forge | Per-tenant isolated DBs | GCS, 7-day retention | +| `hermes-tenant-pgdb` | agent-forge | Per-tenant isolated DBs | GCS, 7-day retention | | `siriusdevops-pgdb` | customer1 | `waitlist_prod` | GCS, 30-day retention | ### GitOps Flow @@ -310,9 +310,9 @@ GitHub Repository (ghcr.io/sirius0xdev) A premium, uncensored, privacy-first AI agent hosting platform with dual-tier cognitive architecture: - **Tier 1 (Dispatcher):** L4 GPU SPOT instance running 24/7. Hosts `Qwen2.5-Coder-7B-Instruct-heretic` via vLLM `v0.9.1` for lightning-fast, cheap triage and tool calling. -- **Tier 2 (Deep Thinker):** A100 80GB SPOT instance scaling from 0-1 via KEDA. Hosts `Qwen3.6-27B-heretic` with chunked prefill and FP8 KV cache for massive multi-file context and reasoning without OOMing. -- **Multi-Tenant Isolation:** Each tenant gets an isolated OpenClaw deployment with its own CNPG PostgreSQL database. No cross-tenant data leakage. -- **Landing Page:** Dockerized marketing site at siriusdevops.com, built via CI/CD from GitHub Actions and deployed to the staging kustomization overlay. +- **Tier 2 (Deep Thinker):** RTX 6000 Pro Spot instance scaling from 0-1 via KEDA. Hosts `Qwen3.5-27B-heretic` with `--enable-chunked-prefill` and `--kv-cache-dtype=fp8` for massive multi-file context and reasoning without OOMing or stalling concurrent users. +- **Frontend:** Isolated Hermes agent profiles per tenant, connected to Telegram/Discord via outbound polling (no public ingress required). +- **Landing Page:** Dockerized marketing site built via CI/CD from `hermes-projects` and deployed to the `staging` kustomization overlay. - **Container Registry:** All images pushed to `ghcr.io/sirius0xdev`. ### 2. Multi-Profile AI Agent Team @@ -463,7 +463,7 @@ or create a `monitoring-grafana-admin` Secret instead. ### Database Security - Managed roles with secret-based passwords per application -- Separate PostgreSQL clusters per domain (hermes-pgdb, openclaw-pgdb, siriusdevops-pgdb) +- Separate PostgreSQL clusters per domain (hermes-pgdb, hermes-tenant-pgdb, siriusdevops-pgdb) - GCS backups with configurable retention policies - HA cluster with automatic failover @@ -477,8 +477,8 @@ or create a `monitoring-grafana-admin` Secret instead. ## Cost Optimization -- **SPOT GPU Instances**: 60-90% savings on L4 and A100 workloads -- **KEDA Scale-to-Zero**: A100 deep thinker pool scales to 0 when no requests are queued +- **SPOT GPU Instances**: 60-90% savings on L4 and RTX 6000 workloads +- **KEDA Scale-to-Zero**: RTX 6000 deep thinker pool scales to 0 when no requests are queued - **Resource Limits**: CPU and memory caps on every container prevent runaway costs - **Scheduled Workloads**: CronJobs only run when needed — no idle inference pods - **Tailscale for Internal Access**: No need for expensive internal load balancers or Cloud NAT for monitoring From 65e174a86dbae75cbe5cf32ea6c99df62bc9540e Mon Sep 17 00:00:00 2001 From: Sirius Devops Date: Fri, 15 May 2026 14:56:10 +0000 Subject: [PATCH 06/28] feat: deploy siriusdevops.com main site to GKE Add siriusdevops-site manifests: - Deployment + ClusterIP Service (nginx:alpine, port 80) - HTTPRoute for siriusdevops.com via external-http-gateway - Kustomization overlay wiring --- apps/base/customer1/http-route.yaml | 15 ++++ apps/base/customer1/kustomization.yaml | 1 + .../siriusdevops-site/deployment.yaml | 77 +++++++++++++++++++ .../siriusdevops-site/kustomization.yaml | 4 + 4 files changed, 97 insertions(+) create mode 100644 apps/base/customer1/siriusdevops-site/deployment.yaml create mode 100644 apps/base/customer1/siriusdevops-site/kustomization.yaml diff --git a/apps/base/customer1/http-route.yaml b/apps/base/customer1/http-route.yaml index aee5e9d..e90bd5d 100644 --- a/apps/base/customer1/http-route.yaml +++ b/apps/base/customer1/http-route.yaml @@ -13,3 +13,18 @@ spec: - name: n8n-service # The name of your Kubernetes Service port: 80 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: siriusdevops-route + namespace: customer1 +spec: + parentRefs: + - name: external-http-gateway + hostnames: + - "siriusdevops.com" + rules: + - backendRefs: + - name: siriusdevops-site-svc + port: 80 diff --git a/apps/base/customer1/kustomization.yaml b/apps/base/customer1/kustomization.yaml index 61aeb0f..5a94142 100644 --- a/apps/base/customer1/kustomization.yaml +++ b/apps/base/customer1/kustomization.yaml @@ -5,3 +5,4 @@ resources: - namespace.yaml - http-route.yaml - service.yaml + - siriusdevops-site diff --git a/apps/base/customer1/siriusdevops-site/deployment.yaml b/apps/base/customer1/siriusdevops-site/deployment.yaml new file mode 100644 index 0000000..cc0d4c7 --- /dev/null +++ b/apps/base/customer1/siriusdevops-site/deployment.yaml @@ -0,0 +1,77 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: siriusdevops-site + namespace: customer1 +spec: + replicas: 1 + selector: + matchLabels: + app: siriusdevops-site + template: + metadata: + labels: + app: siriusdevops-site + spec: + containers: + - name: nginx + image: ghcr.io/sirius0xdev/siriusdevops:latest + imagePullPolicy: Always + ports: + - containerPort: 80 + protocol: TCP + resources: + requests: + cpu: 10m + memory: 64Mi + limits: + cpu: 50m + memory: 128Mi + + startupProbe: + httpGet: + path: / + port: 80 + scheme: HTTP + initialDelaySeconds: 15 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 30 + successThreshold: 1 + + readinessProbe: + httpGet: + path: / + port: 80 + scheme: HTTP + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 3 + successThreshold: 1 + + livenessProbe: + httpGet: + path: / + port: 80 + scheme: HTTP + initialDelaySeconds: 40 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + successThreshold: 1 + +--- +apiVersion: v1 +kind: Service +metadata: + name: siriusdevops-site-svc + namespace: customer1 +spec: + selector: + app: siriusdevops-site + ports: + - protocol: TCP + port: 80 + targetPort: 80 + type: ClusterIP diff --git a/apps/base/customer1/siriusdevops-site/kustomization.yaml b/apps/base/customer1/siriusdevops-site/kustomization.yaml new file mode 100644 index 0000000..9c2d28b --- /dev/null +++ b/apps/base/customer1/siriusdevops-site/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - deployment.yaml From c12b31a47a46eac3d735d0b0bec8e27195b06e7e Mon Sep 17 00:00:00 2001 From: sirius0xdev Date: Sat, 16 May 2026 13:31:58 -0400 Subject: [PATCH 07/28] Update kustomization.yaml --- infrastructure/gpus/staging/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/gpus/staging/kustomization.yaml b/infrastructure/gpus/staging/kustomization.yaml index 37d0cdd..bea67ef 100644 --- a/infrastructure/gpus/staging/kustomization.yaml +++ b/infrastructure/gpus/staging/kustomization.yaml @@ -1,5 +1,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - # - ../base/vllm-servers/ + - ../base/vllm-servers/ # - ../base/keda-gpu-scaling/ From e4ce253e804fbf88c02343a2c19a3b8dcf2e05e6 Mon Sep 17 00:00:00 2001 From: sirius0xdev Date: Sat, 16 May 2026 22:35:02 -0400 Subject: [PATCH 08/28] Update rtx6000-vllm.yaml --- infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml b/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml index c490324..dc2adb2 100644 --- a/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml +++ b/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml @@ -39,7 +39,7 @@ spec: - --tokenizer-mode=hf - --gpu-memory-utilization=0.90 #- --enforce-eager - - --max-model-len=98304 + - --max-model-len=136876 - --enable-auto-tool-choice - --kv-cache-dtype=fp8 - --max-num-batched-tokens=32768 @@ -51,6 +51,7 @@ spec: - --enable-prefix-caching - --tool-call-parser=qwen3_xml - --reasoning-parser=qwen3 + - --speculative-config '{"method": "mtp", "num_speculative_tokens": 2}' ports: - containerPort: 8000 From 728a2dfe7771ba03715bced01fba1fbabcc1c6a5 Mon Sep 17 00:00:00 2001 From: sirius0xdev Date: Sat, 16 May 2026 22:57:54 -0400 Subject: [PATCH 09/28] Update rtx6000-vllm.yaml --- infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml b/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml index dc2adb2..14bb3bb 100644 --- a/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml +++ b/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml @@ -51,7 +51,7 @@ spec: - --enable-prefix-caching - --tool-call-parser=qwen3_xml - --reasoning-parser=qwen3 - - --speculative-config '{"method": "mtp", "num_speculative_tokens": 2}' + - --speculative-config='{"method": "mtp", "num_speculative_tokens": 2}' ports: - containerPort: 8000 From 3836578bb843a4c9f2cb7b08396bf0142d55fb6d Mon Sep 17 00:00:00 2001 From: sirius0xdev Date: Sat, 16 May 2026 22:58:38 -0400 Subject: [PATCH 10/28] Update rtx6000-vllm.yaml --- infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml b/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml index 14bb3bb..b91d403 100644 --- a/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml +++ b/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml @@ -51,7 +51,7 @@ spec: - --enable-prefix-caching - --tool-call-parser=qwen3_xml - --reasoning-parser=qwen3 - - --speculative-config='{"method": "mtp", "num_speculative_tokens": 2}' + - --speculative-config {"method": "mtp", "num_speculative_tokens": 2} ports: - containerPort: 8000 From 0899aa1b97f022241cf5beec0b9d4edeed63b32b Mon Sep 17 00:00:00 2001 From: sirius0xdev Date: Sat, 16 May 2026 22:59:03 -0400 Subject: [PATCH 11/28] Update rtx6000-vllm.yaml --- infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml b/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml index b91d403..d95af39 100644 --- a/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml +++ b/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml @@ -51,7 +51,7 @@ spec: - --enable-prefix-caching - --tool-call-parser=qwen3_xml - --reasoning-parser=qwen3 - - --speculative-config {"method": "mtp", "num_speculative_tokens": 2} + - --speculative-config "{"method": "mtp", "num_speculative_tokens": 2}" ports: - containerPort: 8000 From 18aa77923240c42592dafa81b17b27f82015ea37 Mon Sep 17 00:00:00 2001 From: sirius0xdev Date: Sat, 16 May 2026 23:01:39 -0400 Subject: [PATCH 12/28] Update rtx6000-vllm.yaml --- infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml b/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml index d95af39..abaab2b 100644 --- a/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml +++ b/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml @@ -51,7 +51,8 @@ spec: - --enable-prefix-caching - --tool-call-parser=qwen3_xml - --reasoning-parser=qwen3 - - --speculative-config "{"method": "mtp", "num_speculative_tokens": 2}" + - --speculative-config + - --'{"method": "mtp", "num_speculative_tokens": 2}' ports: - containerPort: 8000 From aa1f5be3238f07246739583420727f1116cf7f7d Mon Sep 17 00:00:00 2001 From: sirius0xdev Date: Sat, 16 May 2026 23:10:18 -0400 Subject: [PATCH 13/28] Update rtx6000-vllm.yaml --- infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml b/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml index abaab2b..38b4018 100644 --- a/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml +++ b/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml @@ -52,7 +52,7 @@ spec: - --tool-call-parser=qwen3_xml - --reasoning-parser=qwen3 - --speculative-config - - --'{"method": "mtp", "num_speculative_tokens": 2}' + - '{"method": "mtp", "num_speculative_tokens": 2}' ports: - containerPort: 8000 From 1a6dd510765d96914127f2cdfb3b74ce2cf0caf4 Mon Sep 17 00:00:00 2001 From: sirius0xdev Date: Sat, 16 May 2026 23:11:34 -0400 Subject: [PATCH 14/28] Update rtx6000-vllm.yaml --- infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml b/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml index 38b4018..a6e67b3 100644 --- a/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml +++ b/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml @@ -51,7 +51,7 @@ spec: - --enable-prefix-caching - --tool-call-parser=qwen3_xml - --reasoning-parser=qwen3 - - --speculative-config + - --"speculative-config" - '{"method": "mtp", "num_speculative_tokens": 2}' ports: From c987ad613231de9ff8ce6f8a98a480d1a4928a73 Mon Sep 17 00:00:00 2001 From: sirius0xdev Date: Sun, 17 May 2026 03:30:39 -0400 Subject: [PATCH 15/28] Update kustomization.yaml --- infrastructure/gpus/staging/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/gpus/staging/kustomization.yaml b/infrastructure/gpus/staging/kustomization.yaml index bea67ef..77459c3 100644 --- a/infrastructure/gpus/staging/kustomization.yaml +++ b/infrastructure/gpus/staging/kustomization.yaml @@ -1,5 +1,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - ../base/vllm-servers/ + # - ../base/vllm-servers/ # - ../base/keda-gpu-scaling/ From 4ba4f082995a994dc05a3d1fd7c4f7c4c2da4c84 Mon Sep 17 00:00:00 2001 From: sirius0xdev Date: Sun, 17 May 2026 14:42:59 -0400 Subject: [PATCH 16/28] Update kustomization.yaml --- infrastructure/gpus/staging/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/gpus/staging/kustomization.yaml b/infrastructure/gpus/staging/kustomization.yaml index 77459c3..bea67ef 100644 --- a/infrastructure/gpus/staging/kustomization.yaml +++ b/infrastructure/gpus/staging/kustomization.yaml @@ -1,5 +1,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - # - ../base/vllm-servers/ + - ../base/vllm-servers/ # - ../base/keda-gpu-scaling/ From af4cb3d321e65d560263906bae4ef337f2bec8ff Mon Sep 17 00:00:00 2001 From: sirius0xdev Date: Sun, 17 May 2026 16:42:06 -0400 Subject: [PATCH 17/28] Update rtx6000-vllm.yaml --- infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml b/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml index a6e67b3..070b1e7 100644 --- a/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml +++ b/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml @@ -51,7 +51,7 @@ spec: - --enable-prefix-caching - --tool-call-parser=qwen3_xml - --reasoning-parser=qwen3 - - --"speculative-config" + - --speculative-config - '{"method": "mtp", "num_speculative_tokens": 2}' ports: From d079d1239f3b320df615b34284f5bc408c1ae04a Mon Sep 17 00:00:00 2001 From: Sirius Devops Date: Sun, 17 May 2026 22:15:00 +0000 Subject: [PATCH 18/28] feat: add trading platform K8s deployment infrastructure - Add Helm chart with 19 templates (Deployments, Services, Ingress, ConfigMaps, Secrets, NetworkPolicy, cert-manager) - Add Dockerfiles for 4 microservices (dashboard, data-service, execute-service, news-service) - Add CI/CD workflows (build-test, build-push, deploy) - Add raw K8s manifests, per-service Helm charts, and deploy scripts - Add SOPS-encrypted secrets template and config - Configure deployment to customer1 namespace - Include infrastructure components: PostgreSQL, Redis, Kafka --- .../.github/workflows/build-push.yml | 109 ++++++ .../.github/workflows/build-test.yml | 135 ++++++++ trading-platform/.github/workflows/deploy.yml | 132 ++++++++ trading-platform/README.md | 112 +++++++ .../deploy/ci-cd/.github/workflows/ci-cd.yml | 236 +++++++++++++ .../docker-compose/docker-compose.dev.yml | 204 ++++++++++++ .../deploy/dockerfiles/api-gateway.Dockerfile | 26 ++ .../deploy/dockerfiles/dashboard.Dockerfile | 40 +++ .../dockerfiles/data-service.Dockerfile | 32 ++ .../dockerfiles/execute-service.Dockerfile | 36 ++ .../dockerfiles/news-service.Dockerfile | 41 +++ trading-platform/deploy/helm/Chart.yaml | 34 ++ .../deploy/helm/_helpers.tpl.template | 60 ++++ .../deploy/helm/api-gateway/Chart.yaml | 6 + .../helm/api-gateway/templates/_helpers.tpl | 61 ++++ .../api-gateway/templates/deployment.yaml | 57 ++++ .../helm/api-gateway/templates/hpa.yaml | 32 ++ .../helm/api-gateway/templates/ingress.yaml | 61 ++++ .../api-gateway/templates/networkpolicy.yaml | 29 ++ .../helm/api-gateway/templates/service.yaml | 15 + .../api-gateway/templates/serviceaccount.yaml | 12 + .../deploy/helm/api-gateway/values.yaml | 80 +++++ .../deploy/helm/dashboard/Chart.yaml | 6 + .../helm/dashboard/templates/_helpers.tpl | 61 ++++ .../helm/dashboard/templates/deployment.yaml | 57 ++++ .../deploy/helm/dashboard/templates/hpa.yaml | 32 ++ .../dashboard/templates/networkpolicy.yaml | 29 ++ .../helm/dashboard/templates/service.yaml | 15 + .../dashboard/templates/serviceaccount.yaml | 12 + .../deploy/helm/dashboard/values.yaml | 80 +++++ .../deploy/helm/data-service/Chart.yaml | 6 + .../helm/data-service/templates/_helpers.tpl | 61 ++++ .../data-service/templates/deployment.yaml | 57 ++++ .../helm/data-service/templates/hpa.yaml | 32 ++ .../data-service/templates/networkpolicy.yaml | 29 ++ .../helm/data-service/templates/service.yaml | 15 + .../templates/serviceaccount.yaml | 12 + .../deploy/helm/data-service/values.yaml | 80 +++++ .../deploy/helm/execute-service/Chart.yaml | 6 + .../execute-service/templates/_helpers.tpl | 61 ++++ .../execute-service/templates/deployment.yaml | 57 ++++ .../helm/execute-service/templates/hpa.yaml | 32 ++ .../templates/networkpolicy.yaml | 29 ++ .../execute-service/templates/service.yaml | 15 + .../templates/serviceaccount.yaml | 12 + .../deploy/helm/execute-service/values.yaml | 80 +++++ .../deploy/helm/news-service/Chart.yaml | 6 + .../helm/news-service/templates/_helpers.tpl | 61 ++++ .../news-service/templates/deployment.yaml | 57 ++++ .../helm/news-service/templates/hpa.yaml | 32 ++ .../news-service/templates/networkpolicy.yaml | 29 ++ .../helm/news-service/templates/service.yaml | 15 + .../templates/serviceaccount.yaml | 12 + .../deploy/helm/news-service/values.yaml | 80 +++++ .../k8s/base/api-gateway-deployment.yaml | 71 ++++ .../deploy/k8s/base/api-gateway-hpa.yaml | 27 ++ .../deploy/k8s/base/api-gateway-service.yaml | 18 + .../cert-manager/api-gateway-mtls-cert.yaml | 24 ++ .../k8s/base/cert-manager/ca-issuer.yaml | 8 + .../base/cert-manager/cluster-issuers.yaml | 29 ++ .../cert-manager/dashboard-mtls-cert.yaml | 24 ++ .../cert-manager/data-service-mtls-cert.yaml | 24 ++ .../execute-service-mtls-cert.yaml | 24 ++ .../deploy/k8s/base/cert-manager/issuer.yaml | 24 ++ .../cert-manager/news-service-mtls-cert.yaml | 24 ++ .../deploy/k8s/base/dashboard-deployment.yaml | 71 ++++ .../deploy/k8s/base/dashboard-hpa.yaml | 27 ++ .../deploy/k8s/base/dashboard-service.yaml | 18 + .../k8s/base/data-service-deployment.yaml | 71 ++++ .../deploy/k8s/base/data-service-hpa.yaml | 27 ++ .../deploy/k8s/base/data-service-service.yaml | 18 + .../k8s/base/execute-service-deployment.yaml | 71 ++++ .../deploy/k8s/base/execute-service-hpa.yaml | 27 ++ .../k8s/base/execute-service-service.yaml | 18 + trading-platform/deploy/k8s/base/ingress.yaml | 40 +++ .../deploy/k8s/base/namespace.yaml | 18 + .../k8s/base/news-service-deployment.yaml | 71 ++++ .../deploy/k8s/base/news-service-hpa.yaml | 27 ++ .../deploy/k8s/base/news-service-service.yaml | 18 + trading-platform/deploy/mtls/README.md | 20 ++ trading-platform/deploy/scripts/deploy.sh | 141 ++++++++ .../deploy/scripts/generate-mtls-certs.sh | 66 ++++ .../dockerfiles/dashboard/.dockerignore | 10 + .../dockerfiles/dashboard/Dockerfile | 31 ++ .../dockerfiles/data-service/.dockerignore | 10 + .../dockerfiles/data-service/Dockerfile | 23 ++ .../dockerfiles/execute-service/.dockerignore | 10 + .../dockerfiles/execute-service/Dockerfile | 23 ++ .../dockerfiles/news-service/.dockerignore | 10 + .../dockerfiles/news-service/Dockerfile | 30 ++ trading-platform/helm/.sops.yaml | 19 ++ .../helm/trading-platform/.sops.yaml | 19 ++ .../helm/trading-platform/Chart.yaml | 12 + .../helm/trading-platform/templates/NOTES.txt | 29 ++ .../trading-platform/templates/_helpers.tpl | 94 ++++++ .../templates/cert-manager/certificates.yaml | 128 +++++++ .../trading-platform/templates/configmap.yaml | 21 ++ .../templates/dashboard/deployment.yaml | 64 ++++ .../templates/dashboard/service.yaml | 40 +++ .../templates/data-service/deployment.yaml | 89 +++++ .../templates/data-service/service.yaml | 48 +++ .../templates/execute-service/deployment.yaml | 112 +++++++ .../templates/execute-service/service.yaml | 50 +++ .../templates/infrastructure/kafka.yaml | 142 ++++++++ .../templates/infrastructure/postgres.yaml | 111 +++++++ .../templates/infrastructure/redis.yaml | 107 ++++++ .../trading-platform/templates/ingress.yaml | 48 +++ .../trading-platform/templates/namespace.yaml | 12 + .../network-policies/network-policies.yaml | 311 ++++++++++++++++++ .../templates/news-service/deployment.yaml | 85 +++++ .../templates/news-service/service.yaml | 48 +++ .../trading-platform/templates/secrets.yaml | 45 +++ .../trading-platform/trading-secrets.yaml | 33 ++ .../helm/trading-platform/values.yaml | 306 +++++++++++++++++ trading-platform/helm/trading-secrets.yaml | 33 ++ 115 files changed, 5917 insertions(+) create mode 100644 trading-platform/.github/workflows/build-push.yml create mode 100644 trading-platform/.github/workflows/build-test.yml create mode 100644 trading-platform/.github/workflows/deploy.yml create mode 100644 trading-platform/README.md create mode 100644 trading-platform/deploy/ci-cd/.github/workflows/ci-cd.yml create mode 100644 trading-platform/deploy/docker-compose/docker-compose.dev.yml create mode 100644 trading-platform/deploy/dockerfiles/api-gateway.Dockerfile create mode 100644 trading-platform/deploy/dockerfiles/dashboard.Dockerfile create mode 100644 trading-platform/deploy/dockerfiles/data-service.Dockerfile create mode 100644 trading-platform/deploy/dockerfiles/execute-service.Dockerfile create mode 100644 trading-platform/deploy/dockerfiles/news-service.Dockerfile create mode 100644 trading-platform/deploy/helm/Chart.yaml create mode 100644 trading-platform/deploy/helm/_helpers.tpl.template create mode 100644 trading-platform/deploy/helm/api-gateway/Chart.yaml create mode 100644 trading-platform/deploy/helm/api-gateway/templates/_helpers.tpl create mode 100644 trading-platform/deploy/helm/api-gateway/templates/deployment.yaml create mode 100644 trading-platform/deploy/helm/api-gateway/templates/hpa.yaml create mode 100644 trading-platform/deploy/helm/api-gateway/templates/ingress.yaml create mode 100644 trading-platform/deploy/helm/api-gateway/templates/networkpolicy.yaml create mode 100644 trading-platform/deploy/helm/api-gateway/templates/service.yaml create mode 100644 trading-platform/deploy/helm/api-gateway/templates/serviceaccount.yaml create mode 100644 trading-platform/deploy/helm/api-gateway/values.yaml create mode 100644 trading-platform/deploy/helm/dashboard/Chart.yaml create mode 100644 trading-platform/deploy/helm/dashboard/templates/_helpers.tpl create mode 100644 trading-platform/deploy/helm/dashboard/templates/deployment.yaml create mode 100644 trading-platform/deploy/helm/dashboard/templates/hpa.yaml create mode 100644 trading-platform/deploy/helm/dashboard/templates/networkpolicy.yaml create mode 100644 trading-platform/deploy/helm/dashboard/templates/service.yaml create mode 100644 trading-platform/deploy/helm/dashboard/templates/serviceaccount.yaml create mode 100644 trading-platform/deploy/helm/dashboard/values.yaml create mode 100644 trading-platform/deploy/helm/data-service/Chart.yaml create mode 100644 trading-platform/deploy/helm/data-service/templates/_helpers.tpl create mode 100644 trading-platform/deploy/helm/data-service/templates/deployment.yaml create mode 100644 trading-platform/deploy/helm/data-service/templates/hpa.yaml create mode 100644 trading-platform/deploy/helm/data-service/templates/networkpolicy.yaml create mode 100644 trading-platform/deploy/helm/data-service/templates/service.yaml create mode 100644 trading-platform/deploy/helm/data-service/templates/serviceaccount.yaml create mode 100644 trading-platform/deploy/helm/data-service/values.yaml create mode 100644 trading-platform/deploy/helm/execute-service/Chart.yaml create mode 100644 trading-platform/deploy/helm/execute-service/templates/_helpers.tpl create mode 100644 trading-platform/deploy/helm/execute-service/templates/deployment.yaml create mode 100644 trading-platform/deploy/helm/execute-service/templates/hpa.yaml create mode 100644 trading-platform/deploy/helm/execute-service/templates/networkpolicy.yaml create mode 100644 trading-platform/deploy/helm/execute-service/templates/service.yaml create mode 100644 trading-platform/deploy/helm/execute-service/templates/serviceaccount.yaml create mode 100644 trading-platform/deploy/helm/execute-service/values.yaml create mode 100644 trading-platform/deploy/helm/news-service/Chart.yaml create mode 100644 trading-platform/deploy/helm/news-service/templates/_helpers.tpl create mode 100644 trading-platform/deploy/helm/news-service/templates/deployment.yaml create mode 100644 trading-platform/deploy/helm/news-service/templates/hpa.yaml create mode 100644 trading-platform/deploy/helm/news-service/templates/networkpolicy.yaml create mode 100644 trading-platform/deploy/helm/news-service/templates/service.yaml create mode 100644 trading-platform/deploy/helm/news-service/templates/serviceaccount.yaml create mode 100644 trading-platform/deploy/helm/news-service/values.yaml create mode 100644 trading-platform/deploy/k8s/base/api-gateway-deployment.yaml create mode 100644 trading-platform/deploy/k8s/base/api-gateway-hpa.yaml create mode 100644 trading-platform/deploy/k8s/base/api-gateway-service.yaml create mode 100644 trading-platform/deploy/k8s/base/cert-manager/api-gateway-mtls-cert.yaml create mode 100644 trading-platform/deploy/k8s/base/cert-manager/ca-issuer.yaml create mode 100644 trading-platform/deploy/k8s/base/cert-manager/cluster-issuers.yaml create mode 100644 trading-platform/deploy/k8s/base/cert-manager/dashboard-mtls-cert.yaml create mode 100644 trading-platform/deploy/k8s/base/cert-manager/data-service-mtls-cert.yaml create mode 100644 trading-platform/deploy/k8s/base/cert-manager/execute-service-mtls-cert.yaml create mode 100644 trading-platform/deploy/k8s/base/cert-manager/issuer.yaml create mode 100644 trading-platform/deploy/k8s/base/cert-manager/news-service-mtls-cert.yaml create mode 100644 trading-platform/deploy/k8s/base/dashboard-deployment.yaml create mode 100644 trading-platform/deploy/k8s/base/dashboard-hpa.yaml create mode 100644 trading-platform/deploy/k8s/base/dashboard-service.yaml create mode 100644 trading-platform/deploy/k8s/base/data-service-deployment.yaml create mode 100644 trading-platform/deploy/k8s/base/data-service-hpa.yaml create mode 100644 trading-platform/deploy/k8s/base/data-service-service.yaml create mode 100644 trading-platform/deploy/k8s/base/execute-service-deployment.yaml create mode 100644 trading-platform/deploy/k8s/base/execute-service-hpa.yaml create mode 100644 trading-platform/deploy/k8s/base/execute-service-service.yaml create mode 100644 trading-platform/deploy/k8s/base/ingress.yaml create mode 100644 trading-platform/deploy/k8s/base/namespace.yaml create mode 100644 trading-platform/deploy/k8s/base/news-service-deployment.yaml create mode 100644 trading-platform/deploy/k8s/base/news-service-hpa.yaml create mode 100644 trading-platform/deploy/k8s/base/news-service-service.yaml create mode 100644 trading-platform/deploy/mtls/README.md create mode 100755 trading-platform/deploy/scripts/deploy.sh create mode 100755 trading-platform/deploy/scripts/generate-mtls-certs.sh create mode 100644 trading-platform/dockerfiles/dashboard/.dockerignore create mode 100644 trading-platform/dockerfiles/dashboard/Dockerfile create mode 100644 trading-platform/dockerfiles/data-service/.dockerignore create mode 100644 trading-platform/dockerfiles/data-service/Dockerfile create mode 100644 trading-platform/dockerfiles/execute-service/.dockerignore create mode 100644 trading-platform/dockerfiles/execute-service/Dockerfile create mode 100644 trading-platform/dockerfiles/news-service/.dockerignore create mode 100644 trading-platform/dockerfiles/news-service/Dockerfile create mode 100644 trading-platform/helm/.sops.yaml create mode 100644 trading-platform/helm/trading-platform/.sops.yaml create mode 100644 trading-platform/helm/trading-platform/Chart.yaml create mode 100644 trading-platform/helm/trading-platform/templates/NOTES.txt create mode 100644 trading-platform/helm/trading-platform/templates/_helpers.tpl create mode 100644 trading-platform/helm/trading-platform/templates/cert-manager/certificates.yaml create mode 100644 trading-platform/helm/trading-platform/templates/configmap.yaml create mode 100644 trading-platform/helm/trading-platform/templates/dashboard/deployment.yaml create mode 100644 trading-platform/helm/trading-platform/templates/dashboard/service.yaml create mode 100644 trading-platform/helm/trading-platform/templates/data-service/deployment.yaml create mode 100644 trading-platform/helm/trading-platform/templates/data-service/service.yaml create mode 100644 trading-platform/helm/trading-platform/templates/execute-service/deployment.yaml create mode 100644 trading-platform/helm/trading-platform/templates/execute-service/service.yaml create mode 100644 trading-platform/helm/trading-platform/templates/infrastructure/kafka.yaml create mode 100644 trading-platform/helm/trading-platform/templates/infrastructure/postgres.yaml create mode 100644 trading-platform/helm/trading-platform/templates/infrastructure/redis.yaml create mode 100644 trading-platform/helm/trading-platform/templates/ingress.yaml create mode 100644 trading-platform/helm/trading-platform/templates/namespace.yaml create mode 100644 trading-platform/helm/trading-platform/templates/network-policies/network-policies.yaml create mode 100644 trading-platform/helm/trading-platform/templates/news-service/deployment.yaml create mode 100644 trading-platform/helm/trading-platform/templates/news-service/service.yaml create mode 100644 trading-platform/helm/trading-platform/templates/secrets.yaml create mode 100644 trading-platform/helm/trading-platform/trading-secrets.yaml create mode 100644 trading-platform/helm/trading-platform/values.yaml create mode 100644 trading-platform/helm/trading-secrets.yaml diff --git a/trading-platform/.github/workflows/build-push.yml b/trading-platform/.github/workflows/build-push.yml new file mode 100644 index 0000000..c0353e6 --- /dev/null +++ b/trading-platform/.github/workflows/build-push.yml @@ -0,0 +1,109 @@ +# Build and push container images to Artifact Registry +name: Build & Push Images + +on: + push: + branches: [main, develop] + paths: + - "trading-platform/**" + - "!trading-platform/infra/**" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + GCP_PROJECT_ID: customer1-gke + GCP_REGION: us-central1 + ARTIFACT_REGISTRY: us-central1-docker.pkg.dev/${{ env.GCP_PROJECT_ID }}/trading + +jobs: + build-and-push: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@v4 + + - name: Authenticate to Google Cloud + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: projects/${{ env.GCP_PROJECT_ID }}/locations/global/workloadIdentityPools/github-pool/providers/github-provider + service_account: ci-builder@${{ env.GCP_PROJECT_ID }}.iam.gserviceaccount.com + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v2 + + - name: Configure Docker for Artifact Registry + run: gcloud auth configure-docker ${{ env.GCP_REGION }}-docker.pkg.dev --quiet + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Generate image tags + id: tags + run: | + SHORT_SHA="${GITHUB_SHA::8}" + BRANCH="${GITHUB_REF#refs/heads/}" + echo "tag_sha=${SHORT_SHA}" >> $GITHUB_OUTPUT + echo "tag_branch=${BRANCH}" >> $GITHUB_OUTPUT + echo "tag_latest=${BRANCH}" >> $GITHUB_OUTPUT + + # ---- Execute Service ---- + - name: Build and push execute-service + uses: docker/build-push-action@v5 + with: + context: trading-platform/execute-service + file: trading-platform/infra/dockerfiles/execute-service/Dockerfile + push: true + tags: | + ${{ env.ARTIFACT_REGISTRY }}/execute-service:${{ steps.tags.outputs.tag_sha }} + ${{ env.ARTIFACT_REGISTRY }}/execute-service:${{ steps.tags.outputs.tag_branch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + # ---- News Service ---- + - name: Build and push news-service + uses: docker/build-push-action@v5 + with: + context: trading-platform/news-service + file: trading-platform/infra/dockerfiles/news-service/Dockerfile + push: true + tags: | + ${{ env.ARTIFACT_REGISTRY }}/news-service:${{ steps.tags.outputs.tag_sha }} + ${{ env.ARTIFACT_REGISTRY }}/news-service:${{ steps.tags.outputs.tag_branch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + # ---- Data Service ---- + - name: Build and push data-service + uses: docker/build-push-action@v5 + with: + context: trading-platform/data-service + file: trading-platform/infra/dockerfiles/data-service/Dockerfile + push: true + tags: | + ${{ env.ARTIFACT_REGISTRY }}/data-service:${{ steps.tags.outputs.tag_sha }} + ${{ env.ARTIFACT_REGISTRY }}/data-service:${{ steps.tags.outputs.tag_branch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + # ---- Dashboard ---- + - name: Build and push dashboard + uses: docker/build-push-action@v5 + with: + context: trading-platform/dashboard + file: trading-platform/infra/dockerfiles/dashboard/Dockerfile + push: true + tags: | + ${{ env.ARTIFACT_REGISTRY }}/dashboard:${{ steps.tags.outputs.tag_sha }} + ${{ env.ARTIFACT_REGISTRY }}/dashboard:${{ steps.tags.outputs.tag_branch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Notify deployment pipeline + run: | + echo "Images pushed successfully with tag ${{ steps.tags.outputs.tag_sha }}" + # This can trigger the deploy workflow via repository dispatch + # or be used by the deploy workflow as a workflow_run trigger diff --git a/trading-platform/.github/workflows/build-test.yml b/trading-platform/.github/workflows/build-test.yml new file mode 100644 index 0000000..da4f7f2 --- /dev/null +++ b/trading-platform/.github/workflows/build-test.yml @@ -0,0 +1,135 @@ +# Build and test on pull requests +name: Build & Test + +on: + pull_request: + branches: [main, develop] + paths: + - "trading-platform/execute-service/**" + - "trading-platform/news-service/**" + - "trading-platform/data-service/**" + - "trading-platform/dashboard/**" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + # ---- Python services ---- + test-execute-service: + runs-on: ubuntu-latest + defaults: + run: + working-directory: trading-platform/execute-service + steps: + - uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install uv + uses: astral-sh/setup-uv@v3 + with: + version: "latest" + - name: Install dependencies + run: uv sync --all-extras --dev + - name: Run tests + run: uv run pytest --asyncio-mode=auto -v --tb=short + - name: Lint + run: uv run ruff check app/ tests/ + + test-news-service: + runs-on: ubuntu-latest + defaults: + run: + working-directory: trading-platform/news-service + steps: + - uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install dependencies + run: | + pip install -r requirements.txt + pip install pytest pytest-asyncio + - name: Run tests + run: python -m pytest -v --tb=short || true + + test-data-service: + runs-on: ubuntu-latest + defaults: + run: + working-directory: trading-platform/data-service + steps: + - uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install uv + uses: astral-sh/setup-uv@v3 + with: + version: "latest" + - name: Install dependencies + run: uv sync --all-extras --dev + - name: Run tests + run: uv run pytest --asyncio-mode=auto -v --tb=short + + # ---- Dashboard ---- + test-dashboard: + runs-on: ubuntu-latest + defaults: + run: + working-directory: trading-platform/dashboard + steps: + - uses: actions/checkout@v4 + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + cache: "npm" + cache-dependency-path: trading-platform/dashboard/package-lock.json + - name: Install dependencies + run: npm ci + - name: Build + run: npm run build + - name: Lint + run: npm run lint + + # ---- Docker build validation ---- + docker-build-check: + needs: [test-execute-service, test-news-service, test-data-service, test-dashboard] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build execute-service + uses: docker/build-push-action@v5 + with: + context: trading-platform/execute-service + file: trading-platform/infra/dockerfiles/execute-service/Dockerfile + push: false + load: false + - name: Build news-service + uses: docker/build-push-action@v5 + with: + context: trading-platform/news-service + file: trading-platform/infra/dockerfiles/news-service/Dockerfile + push: false + load: false + - name: Build data-service + uses: docker/build-push-action@v5 + with: + context: trading-platform/data-service + file: trading-platform/infra/dockerfiles/data-service/Dockerfile + push: false + load: false + - name: Build dashboard + uses: docker/build-push-action@v5 + with: + context: trading-platform/dashboard + file: trading-platform/infra/dockerfiles/dashboard/Dockerfile + push: false + load: false diff --git a/trading-platform/.github/workflows/deploy.yml b/trading-platform/.github/workflows/deploy.yml new file mode 100644 index 0000000..d59c553 --- /dev/null +++ b/trading-platform/.github/workflows/deploy.yml @@ -0,0 +1,132 @@ +# Deploy to staging/prod via Helm on GKE +name: Deploy + +on: + workflow_dispatch: + inputs: + environment: + description: "Target environment" + required: true + default: "staging" + type: choice + options: + - staging + - production + image_tag: + description: "Container image tag (SHA or branch name)" + required: true + type: string + workflow_run: + workflows: ["Build & Push Images"] + types: [completed] + branches: [main, develop] + +permissions: + contents: read + id-token: write + +jobs: + deploy: + runs-on: ubuntu-latest + if: >- + github.event_name == 'workflow_dispatch' || + (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') + environment: ${{ github.event.inputs.environment || (github.ref == 'refs/heads/main' && 'production' || 'staging') }} + steps: + - uses: actions/checkout@v4 + + - name: Authenticate to Google Cloud + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: projects/customer1-gke/locations/global/workloadIdentityPools/github-pool/providers/github-provider + service_account: ci-deployer@customer1-gke.iam.gserviceaccount.com + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v2 + + - name: Configure kubectl for GKE + run: | + gcloud container clusters get-credentials \ + ${{ github.event.inputs.environment || (github.ref == 'refs/heads/main' && 'prod' || 'staging') }}-cluster \ + --region us-central1 \ + --project customer1-gke + + - name: Install Helm + uses: azure/setup-helm@v3 + with: + version: v3.14.0 + + - name: Install SOPS + Age + run: | + curl -Lo /tmp/sops.zip https://github.com/getsops/sops/releases/download/v3.8.1/sops-v3.8.1_linux.amd64.zip + unzip /tmp/sops.zip -d /tmp/ + sudo mv /tmp/sops /usr/local/bin/sops + go install github.com/getsops/gopgs@latest || true + go install filippo.io/age/cmd/age@latest || true + + - name: Create namespace + run: | + kubectl create namespace trading --dry-run=client -o yaml | kubectl apply -f - + + - name: Decrypt secrets + run: | + # Copy age key for SOPS decryption + mkdir -p /etc/sops + echo "${{ secrets.SOPS_AGE_KEY }}" > /etc/sops/age.key + chmod 600 /etc/sops/age.key + export SOPS_AGE_KEY_FILE=/etc/sops/age.key + # Decrypt secrets + sops -d trading-platform/infra/helm/trading-platform/trading-secrets.yaml > trading-platform/infra/helm/trading-platform/trading-secrets-decrypted.yaml + + - name: Deploy with Helm + run: | + IMAGE_TAG="${{ github.event.inputs.image_tag }}" + ENVIRONMENT="${{ github.event.inputs.environment || (github.ref == 'refs/heads/main' && 'production' || 'staging') }}" + + helm upgrade --install trading-platform \ + trading-platform/infra/helm/trading-platform \ + --namespace trading \ + --create-namespace \ + --set global.environment=${ENVIRONMENT} \ + --set executeService.image.tag=${IMAGE_TAG} \ + --set newsService.image.tag=${IMAGE_TAG} \ + --set dataService.image.tag=${IMAGE_TAG} \ + --set dashboard.image.tag=${IMAGE_TAG} \ + --wait \ + --timeout 10m \ + --atomic + + - name: Apply decrypted secrets + run: | + export SOPS_AGE_KEY_FILE=/etc/sops/age.key + sops -d trading-platform/infra/helm/trading-platform/trading-secrets.yaml | kubectl apply -f - + + - name: Verify deployment + run: | + echo "=== Pod Status ===" + kubectl get pods -n trading + echo "" + echo "=== Service Status ===" + kubectl get svc -n trading + echo "" + echo "=== Ingress ===" + kubectl get ingress -n trading + + - name: Post-deployment smoke test + run: | + # Wait for readiness + kubectl wait --for=condition=available --timeout=5m \ + deployment/execute-service -n trading + kubectl wait --for=condition=available --timeout=5m \ + deployment/news-service -n trading + kubectl wait --for=condition=available --timeout=5m \ + deployment/data-service -n trading + kubectl wait --for=condition=available --timeout=5m \ + deployment/dashboard -n trading + echo "All services deployed and healthy" + + - name: Rollback on failure + if: failure() + run: | + helm rollback trading-platform -n trading --timeout 10m || true + echo "Rolled back to previous release" diff --git a/trading-platform/README.md b/trading-platform/README.md new file mode 100644 index 0000000..5ba817d --- /dev/null +++ b/trading-platform/README.md @@ -0,0 +1,112 @@ +# Trading Platform — Kubernetes Deployment + +Kubernetes deployment infrastructure for the trading platform microservices running on GKE (customer1 namespace). + +## Directory Structure + +``` +trading-platform/ +├── dockerfiles/ # Multi-stage Dockerfiles for each service +│ ├── dashboard/ # Next.js frontend (port 3000) +│ ├── data-service/ # Data pipeline service (port 8000) +│ ├── execute-service/ # Trading engine: Hyperliquid + Solana (port 8000) +│ └── news-service/ # CNPG connector + Kafka producer (port 8000) +├── helm/ # Helm chart for full platform deployment +│ ├── Chart.yaml # Chart metadata +│ ├── values.yaml # Default values (images, replicas, resources, infra) +│ ├── .sops.yaml # SOPS configuration for secret encryption +│ ├── trading-secrets.yaml # SOPS-encrypted secrets template +│ └── templates/ # 19 Kubernetes manifest templates +│ ├── _helpers.tpl # Template helpers +│ ├── namespace.yaml # Namespace resource +│ ├── configmap.yaml # Shared ConfigMap +│ ├── secrets.yaml # Secrets (SOPS-encrypted via trading-secrets.yaml) +│ ├── ingress.yaml # GCE Ingress for all services +│ ├── NOTES.txt # Post-install notes +│ ├── dashboard/ # Dashboard Deployment + Service +│ ├── data-service/ # Data Service Deployment + Service +│ ├── execute-service/ # Execute Service Deployment + Service +│ ├── news-service/ # News Service Deployment + Service +│ ├── infrastructure/ # PostgreSQL, Redis, Kafka +│ ├── network-policies/ # Default deny + explicit allow policies +│ └── cert-manager/ # Certificates & issuers +├── deploy/ # Additional deployment resources +│ ├── k8s/base/ # Raw K8s manifests (non-Helm fallback) +│ ├── helm/ # Individual per-service Helm charts +│ ├── dockerfiles/ # Alternative Dockerfiles (api-gateway, services) +│ ├── docker-compose/ # Local dev compose files +│ ├── scripts/ # deploy.sh, generate-mtls-certs.sh +│ └── mtls/ # mTLS documentation +└── .github/workflows/ # CI/CD pipelines + ├── build-test.yml # Build + unit tests on PR + ├── build-push.yml # Build + push to GAR on merge + └── deploy.yml # Helm deploy to GKE on push to master +``` + +## Services + +| Service | Port | Description | +|---------|------|-------------| +| Dashboard | 3000 | Next.js trading dashboard | +| Data Service | 8000 | Data pipeline, Postgres + Redis + Kafka consumers | +| Execute Service | 8000 | Trading engine with Hyperliquid + Solana integration | +| News Service | 8000 | CryptoPanic/GNews connector, Kafka producer | + +## Infrastructure Components + +- **PostgreSQL 17** — Primary database for trades, orders, user data +- **Redis 7** — Caching layer with 3-node cluster +- **Kafka 3.9** (KRaft mode) — Event streaming (trades, orders, news topics) +- **GCE Ingress** — External traffic routing with TLS termination +- **Cert-Manager** — Automatic TLS certificates (Let's Encrypt + internal CA) +- **Network Policies** — Default deny ingress/egress with explicit allow rules + +## Deploying + +### Prerequisites + +- GKE cluster: `customer1-gke` (us-central1) +- Helm 3 installed locally or in CI +- SOPS configured with Age key (`trading-secrets.yaml` must be encrypted) +- Access to `us-central1-docker.pkg.dev/customer1-gke/trading` registry + +### Quick Deploy + +```bash +# 1. Encrypt secrets (must use the SOPS Age key) +cd helm +sops -e -i trading-secrets.yaml + +# 2. Install/upgrade the Helm release +helm upgrade --install trading-platform ./helm \ + --namespace customer1 \ + --create-namespace \ + --values helm/values.yaml \ + --set global.environment=production +``` + +### CI/CD + +- **PR opened** → `build-test.yml` runs unit tests +- **Merged to master** → `build-push.yml` builds images and pushes to GAR +- **Push to master** → `deploy.yml` runs `helm upgrade` on GKE + +## Secrets + +Secrets are managed via [SOPS](https://github.com/getsops/sops) with Age encryption. +The `.sops.yaml` file configures which keys to use for each path. + +```bash +# Encrypt the secrets file +sops -e -i helm/trading-secrets.yaml + +# Decrypt (for debugging) +sops -d helm/trading-secrets.yaml +``` + +**Never commit unencrypted secrets to git.** + +## Namespace + +The platform deploys into the `customer1` namespace on the GKE cluster. +Update `global.namespace` in `helm/values.yaml` or override via `--set` during install. diff --git a/trading-platform/deploy/ci-cd/.github/workflows/ci-cd.yml b/trading-platform/deploy/ci-cd/.github/workflows/ci-cd.yml new file mode 100644 index 0000000..b6993ee --- /dev/null +++ b/trading-platform/deploy/ci-cd/.github/workflows/ci-cd.yml @@ -0,0 +1,236 @@ +name: Build, Test, and Deploy Trading Platform + +on: + push: + branches: [main] + paths: + - 'trading-platform/**' + pull_request: + branches: [main] + paths: + - 'trading-platform/**' + workflow_dispatch: + inputs: + environment: + description: 'Deploy environment' + type: choice + options: + - staging + - production + default: staging + +env: + REGISTRY: ghcr.io + IMAGE_PREFIX: ${{ github.repository_owner }}/trading-platform + +permissions: + contents: read + packages: write + +jobs: + # ── Test All Services ────────────────────────────────────────────────── + test-python-services: + name: Test Python Services + runs-on: ubuntu-latest + strategy: + matrix: + service: [execute-service, data-service, news-service] + defaults: + run: + working-directory: trading-platform/${{ matrix.service }} + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + working-directory: trading-platform/${{ matrix.service }} + + - name: Run tests with coverage + run: | + pytest tests/ --cov=app --cov-report=xml --cov-report=term-missing -v + working-directory: trading-platform/${{ matrix.service }} + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + file: trading-platform/${{ matrix.service }}/coverage.xml + flags: ${{ matrix.service }} + + test-dashboard: + name: Test Dashboard (Next.js) + runs-on: ubuntu-latest + defaults: + run: + working-directory: trading-platform/dashboard + steps: + - uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + cache-dependency-path: trading-platform/dashboard/package-lock.json + + - name: Install dependencies + run: npm ci + working-directory: trading-platform/dashboard + + - name: Run linting + run: npm run lint + working-directory: trading-platform/dashboard + + - name: Build application + run: npm run build + working-directory: trading-platform/dashboard + + test-api-gateway: + name: Lint API Gateway Configs + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Validate nginx config syntax + run: | + docker run --rm -v $(pwd)/deploy/k8s/base/gateway:/etc/nginx/conf.d:ro nginx:1.25-alpine nginx -t + + # ── Build and Push Container Images ───────────────────────────────────── + build-and-push: + needs: [test-python-services, test-dashboard, test-api-gateway] + name: Build & Push Images + runs-on: ubuntu-latest + if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' + strategy: + matrix: + service: [execute-service, data-service, news-service, api-gateway, dashboard] + steps: + - uses: actions/checkout@v4 + + - name: Log in to Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}/${{ matrix.service }} + tags: | + type=sha,prefix= + type=ref,event=branch + type=semver,pattern={{version}} + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + file: trading-platform/deploy/dockerfiles/${{ matrix.service }}.Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + # ── Deploy to Kubernetes (Helm) ──────────────────────────────────────── + deploy-staging: + needs: [build-and-push] + name: Deploy to Staging + runs-on: ubuntu-latest + if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.inputs.environment == 'staging') + environment: staging + steps: + - uses: actions/checkout@v4 + + - name: Set up kubectl + uses: azure/setup-kubectl@v3 + with: + version: 'v1.29.0' + + - name: Configure kubeconfig + run: | + echo "${{ secrets.STAGING_KUBECONFIG }}" | base64 -d > $HOME/.kube/config + env: + STAGING_KUBECONFIG: ${{ secrets.STAGING_KUBECONFIG }} + + - name: Install Helm + uses: azure/setup-helm@v3 + with: + version: 'v3.14.0' + + - name: Deploy with Helm (staging) + run: | + helm upgrade --install trading-platform-staging \\ + deploy/helm/trading-platform \\ + --namespace customer1-staging \\ + --create-namespace \\ + --set image.tag=${{ github.sha }} \\ + --wait --timeout 10m + + - name: Verify deployment + run: | + kubectl rollout status deployment/execute-service -n customer1-staging --timeout=5m + kubectl rollout status deployment/data-service -n customer1-staging --timeout=5m + kubectl rollout status deployment/news-service -n customer1-staging --timeout=5m + kubectl rollout status deployment/api-gateway -n customer1-staging --timeout=5m + kubectl rollout status deployment/dashboard -n customer1-staging --timeout=5m + + deploy-production: + needs: [deploy-staging] + name: Deploy to Production + runs-on: ubuntu-latest + if: github.event_name == 'workflow_dispatch' && github.event.inputs.environment == 'production' + environment: production + steps: + - uses: actions/checkout@v4 + + - name: Set up kubectl + uses: azure/setup-kubectl@v3 + with: + version: 'v1.29.0' + + - name: Configure kubeconfig + run: | + echo "${{ secrets.PRODUCTION_KUBECONFIG }}" | base64 -d > $HOME/.kube/config + + - name: Install Helm + uses: azure/setup-helm@v3 + with: + version: 'v3.14.0' + + - name: Deploy with Helm (production) + run: | + helm upgrade --install trading-platform-production \\ + deploy/helm/trading-platform \\ + --namespace customer1 \\ + --create-namespace \\ + --set image.tag=${{ github.sha }} \\ + --values deploy/helm/trading-platform/values-production.yaml \\ + --wait --timeout 15m + + - name: Verify deployment + run: | + kubectl rollout status deployment/execute-service -n customer1 --timeout=5m + kubectl rollout status deployment/data-service -n customer1 --timeout=5m + kubectl rollout status deployment/news-service -n customer1 --timeout=5m + kubectl rollout status deployment/api-gateway -n customer1 --timeout=5m + kubectl rollout status deployment/dashboard -n customer1 --timeout=5m + + - name: Run post-deployment health checks + run: | + # Check all services respond to health endpoints + for service in execute-service data-service news-service api-gateway dashboard; do + echo "Health check: $service" + kubectl run healthcheck-$service --rm --restart=Never --image=curlimages/curl \\ + --command -- curl -sf http://$service:$(kubectl get svc $service -o jsonpath='{.spec.ports[0].port}')/health || exit 1 + done diff --git a/trading-platform/deploy/docker-compose/docker-compose.dev.yml b/trading-platform/deploy/docker-compose/docker-compose.dev.yml new file mode 100644 index 0000000..b9080d4 --- /dev/null +++ b/trading-platform/deploy/docker-compose/docker-compose.dev.yml @@ -0,0 +1,204 @@ +# ============================================================================= +# Docker Compose — Local Development Environment +# ============================================================================= +# Brings up all microservices + infrastructure for local development +# +# Usage: +# docker compose -f trading-platform/deploy/docker-compose/docker-compose.dev.yml up -d +# docker compose -f trading-platform/deploy/docker-compose/docker-compose.dev.yml down -v +# docker compose -f trading-platform/deploy/docker-compose/docker-compose.dev.yml logs -f execute-service +# ============================================================================= + +x-common-env: &common-env + TRADING_ENV: development + LOG_LEVEL: debug + +services: + # ── Infrastructure ──────────────────────────────────────────────────── + + postgres: + image: postgres:16-alpine + container_name: trading-postgres-dev + environment: + POSTGRES_USER: trading + POSTGRES_PASSWORD: trading_dev_password + POSTGRES_DB: trading_db + ports: + - "5432:5432" + volumes: + - postgres-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U trading -d trading_db"] + interval: 5s + timeout: 3s + retries: 5 + restart: unless-stopped + networks: + - trading-network + + redis: + image: redis:7-alpine + container_name: trading-redis-dev + ports: + - "6379:6379" + volumes: + - redis-data:/data + command: redis-server --appendonly yes --maxmemory 256mb --maxmemory-policy allkeys-lru + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 5 + restart: unless-stopped + networks: + - trading-network + + kafka: + image: apache/kafka:3.7.0 + container_name: trading-kafka-dev + ports: + - "9092:9092" + - "9093:9093" + environment: + KAFKA_NODE_ID: 1 + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093 + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT + KAFKA_CLUSTER_ID: MkU3OEVBNTcwT0FBQT0= + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" + KAFKA_LOG_RETENTION_HOURS: 24 + KAFKA_LOG_SEGMENT_BYTES: 1073741824 + volumes: + - kafka-data:/var/lib/kafka/data + healthcheck: + test: ["CMD-SHELL", "kafka-topics.sh --bootstrap-server localhost:9092 --list || exit 1"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + restart: unless-stopped + networks: + - trading-network + + # ── Microservices ───────────────────────────────────────────────────── + + data-service: + build: + context: ../../.. + dockerfile: trading-platform/deploy/dockerfiles/data-service.Dockerfile + container_name: trading-data-service-dev + environment: + <<: *common-env + DATABASE_URL: postgresql+asyncpg://trading:trading_dev_password@postgres:5432/trading_db + REDIS_URL: redis://redis:6379/0 + KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + LOG_LEVEL: debug + ports: + - "8001:8001" + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + kafka: + condition: service_healthy + restart: unless-stopped + networks: + - trading-network + + execute-service: + build: + context: ../../.. + dockerfile: trading-platform/deploy/dockerfiles/execute-service.Dockerfile + container_name: trading-execute-service-dev + environment: + <<: *common-env + EXECUTE_DATABASE_URL: sqlite+aiosqlite:///./execute.db + EXECUTE_JWT_SECRET_KEY: dev-secret-change-me + EXECUTE_HYPERLIQUID_TESTNET: "true" + EXECUTE_MTLS_ENABLED: "false" + EXECUTE_MARKET_DATA_SERVICE_URL: http://data-service:8001 + LOG_LEVEL: debug + ports: + - "8000:8000" + depends_on: + data-service: + condition: service_healthy + restart: unless-stopped + networks: + - trading-network + + news-service: + build: + context: ../../.. + dockerfile: trading-platform/deploy/dockerfiles/news-service.Dockerfile + container_name: trading-news-service-dev + environment: + <<: *common-env + DATABASE_URL: postgresql+asyncpg://trading:trading_dev_password@postgres:5432/trading_db + KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + REDIS_URL: redis://redis:6379/1 + LOG_LEVEL: debug + ports: + - "8002:8002" + depends_on: + postgres: + condition: service_healthy + kafka: + condition: service_healthy + restart: unless-stopped + networks: + - trading-network + + api-gateway: + build: + context: ../../.. + dockerfile: trading-platform/deploy/dockerfiles/api-gateway.Dockerfile + container_name: trading-api-gateway-dev + environment: + <<: *common-env + ports: + - "8080:8080" + - "8443:8443" + depends_on: + execute-service: + condition: service_healthy + data-service: + condition: service_healthy + news-service: + condition: service_healthy + restart: unless-stopped + networks: + - trading-network + + dashboard: + build: + context: ../../.. + dockerfile: trading-platform/deploy/dockerfiles/dashboard.Dockerfile + container_name: trading-dashboard-dev + environment: + NEXT_PUBLIC_API_URL: http://localhost:8080 + NODE_ENV: development + ports: + - "3000:3000" + depends_on: + api-gateway: + condition: service_started + restart: unless-stopped + networks: + - trading-network + +volumes: + postgres-data: + redis-data: + kafka-data: + +networks: + trading-network: + driver: bridge diff --git a/trading-platform/deploy/dockerfiles/api-gateway.Dockerfile b/trading-platform/deploy/dockerfiles/api-gateway.Dockerfile new file mode 100644 index 0000000..2e63688 --- /dev/null +++ b/trading-platform/deploy/dockerfiles/api-gateway.Dockerfile @@ -0,0 +1,26 @@ +# ============================================================================= +# API Gateway Dockerfile — Nginx-based reverse proxy with rate limiting +# ============================================================================= +FROM nginx:1.25-alpine AS production + +# Copy custom nginx configuration +COPY deploy/k8s/base/gateway/nginx.conf /etc/nginx/nginx.conf +COPY deploy/k8s/base/gateway/conf.d/ /etc/nginx/conf.d/ + +# Create required directories +RUN mkdir -p /etc/nginx/ssl \ + /etc/nginx/conf.d \ + /var/cache/nginx \ + /var/run/nginx \ + /var/log/nginx \ + && touch /var/run/nginx/nginx.pid + +# Security: run as nginx user (already exists in alpine image) +USER nginx + +EXPOSE 8080 8443 + +HEALTHCHECK --interval=15s --timeout=3s --start-period=10s --retries=3 \ + CMD wget --no-verbose --tries=1 --spider http://localhost:8080/health || exit 1 + +CMD ["nginx", "-g", "daemon off;"] diff --git a/trading-platform/deploy/dockerfiles/dashboard.Dockerfile b/trading-platform/deploy/dockerfiles/dashboard.Dockerfile new file mode 100644 index 0000000..816d558 --- /dev/null +++ b/trading-platform/deploy/dockerfiles/dashboard.Dockerfile @@ -0,0 +1,40 @@ +# ============================================================================= +# Dashboard Dockerfile — Next.js multi-stage build with static export +# ============================================================================= +FROM node:20-alpine AS builder + +WORKDIR /app + +# Install dependencies first (better layer caching) +COPY trading-platform/dashboard/package*.json ./ +RUN npm ci + +# Copy source and build +COPY trading-platform/dashboard/ ./ +RUN npm run build + +# Production stage +FROM node:20-alpine AS production + +# Security: non-root user +RUN addgroup -g 1001 -S nodejs && adduser -S nextjs -u 1001 + +WORKDIR /app + +# Copy built output and package.json from builder +COPY --from=builder /app/package.json ./package.json +COPY --from=builder /app/.next/standalone ./ +COPY --from=builder /app/.next/static ./.next/static +COPY --from=builder /app/public ./public + +USER nextjs + +EXPOSE 3000 + +ENV NODE_ENV=production +ENV PORT=3000 + +HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 \ + CMD wget --no-verbose --tries=1 --spider http://localhost:3000/ || exit 1 + +CMD ["node", "server.js"] diff --git a/trading-platform/deploy/dockerfiles/data-service.Dockerfile b/trading-platform/deploy/dockerfiles/data-service.Dockerfile new file mode 100644 index 0000000..902d71e --- /dev/null +++ b/trading-platform/deploy/dockerfiles/data-service.Dockerfile @@ -0,0 +1,32 @@ +# ============================================================================= +# Data Service Dockerfile — Multi-stage build +# ============================================================================= +FROM python:3.12-slim AS builder + +WORKDIR /build +COPY trading-platform/data-service/pyproject.toml ./ +RUN pip install --no-cache-dir --prefix=/install . + +# Production stage +FROM python:3.12-slim AS production + +# Security: non-root user +RUN useradd -m --system appuser + +# Copy dependencies from builder +COPY --from=builder /install /usr/local + +# Copy application code +WORKDIR /app +COPY --chown=appuser:appuser trading-platform/data-service/data_service/ ./data_service/ +COPY --chown=appuser:appuser trading-platform/data-service/pyproject.toml ./ + +USER appuser + +# Health check +HEALTHCHECK --interval=15s --timeout=5s --start-period=15s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8001/health')" || exit 1 + +EXPOSE 8001 + +CMD ["uvicorn", "data_service.app.main:app", "--host", "0.0.0.0", "--port", "8001"] diff --git a/trading-platform/deploy/dockerfiles/execute-service.Dockerfile b/trading-platform/deploy/dockerfiles/execute-service.Dockerfile new file mode 100644 index 0000000..0e1c2fe --- /dev/null +++ b/trading-platform/deploy/dockerfiles/execute-service.Dockerfile @@ -0,0 +1,36 @@ +# ============================================================================= +# Execute Service Dockerfile — Multi-stage build for minimal image +# ============================================================================= +# Build stage: compile dependencies +FROM python:3.12-slim AS builder + +WORKDIR /build +COPY trading-platform/execute-service/pyproject.toml ./ +RUN pip install --no-cache-dir --prefix=/install . + +# Production stage +FROM python:3.12-slim AS production + +# Security: non-root user +RUN useradd -m --system appuser + +# Copy dependencies from builder +COPY --from=builder /install /usr/local + +# Copy application code +WORKDIR /app +COPY --chown=appuser:appuser trading-platform/execute-service/app/ ./app/ +COPY --chown=appuser:appuser trading-platform/execute-service/pyproject.toml ./ + +# Create required directories with proper permissions +RUN mkdir -p /tmp /app/data && chown -R appuser:appuser /tmp /app/data + +USER appuser + +# Health check using Python (curl not in slim) +HEALTHCHECK --interval=15s --timeout=5s --start-period=15s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8000/health')" || exit 1 + +EXPOSE 8000 + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/trading-platform/deploy/dockerfiles/news-service.Dockerfile b/trading-platform/deploy/dockerfiles/news-service.Dockerfile new file mode 100644 index 0000000..770cb5e --- /dev/null +++ b/trading-platform/deploy/dockerfiles/news-service.Dockerfile @@ -0,0 +1,41 @@ +# ============================================================================= +# News Service Dockerfile — Multi-stage build with NLTK data +# ============================================================================= +FROM python:3.12-slim AS builder + +WORKDIR /build +COPY trading-platform/news-service/requirements.txt ./ +RUN pip install --no-cache-dir --prefix=/install -r requirements.txt + +# Production stage +FROM python:3.12-slim AS production + +# Install system dependencies +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + gcc libpq-dev && \ + rm -rf /var/lib/apt/lists/* + +# Install Python dependencies from builder +COPY --from=builder /install /usr/local + +# Download NLTK data for textblob +RUN python -c "import nltk; nltk.download('punkt'); nltk.download('punkt_tab'); nltk.download('averaged_perceptron_tagger')" + +# Security: non-root user +RUN useradd -m --system appuser + +# Copy application code +WORKDIR /app +COPY --chown=appuser:appuser trading-platform/news-service/app/ ./app/ +COPY --chown=appuser:appuser trading-platform/news-service/requirements.txt ./ + +USER appuser + +# Health check +HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8002/health')" || exit 1 + +EXPOSE 8002 + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8002", "--workers", "4"] diff --git a/trading-platform/deploy/helm/Chart.yaml b/trading-platform/deploy/helm/Chart.yaml new file mode 100644 index 0000000..f7d278d --- /dev/null +++ b/trading-platform/deploy/helm/Chart.yaml @@ -0,0 +1,34 @@ +# ============================================================================= +# Trading Platform — Root Helm Chart +# ============================================================================= +apiVersion: v2 +name: trading-platform +description: Helm chart for the entire trading platform microservices +type: application +version: 0.1.0 +appVersion: "0.1.0" + +dependencies: + - name: api-gateway + version: "0.1.0" + repository: "file://../api-gateway" + - name: execute-service + version: "0.1.0" + repository: "file://../execute-service" + - name: data-service + version: "0.1.0" + repository: "file://../data-service" + - name: news-service + version: "0.1.0" + repository: "file://../news-service" + - name: dashboard + version: "0.1.0" + repository: "file://../dashboard" + - name: cert-manager + version: "1.14.0" + repository: https://charts.jetstack.io + condition: cert-manager.enabled + - name: ingress-nginx + version: "4.9.0" + repository: https://kubernetes.github.io/ingress-nginx + condition: ingress-nginx.enabled diff --git a/trading-platform/deploy/helm/_helpers.tpl.template b/trading-platform/deploy/helm/_helpers.tpl.template new file mode 100644 index 0000000..3b76fa7 --- /dev/null +++ b/trading-platform/deploy/helm/_helpers.tpl.template @@ -0,0 +1,60 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define ".name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define ".fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define ".chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define ".labels" -}} +helm.sh/chart: {{ include ".chart" . }} +{{ include ".selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define ".selectorLabels" -}} +app.kubernetes.io/name: {{ include ".name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define ".serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include ".fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/trading-platform/deploy/helm/api-gateway/Chart.yaml b/trading-platform/deploy/helm/api-gateway/Chart.yaml new file mode 100644 index 0000000..5e0f40d --- /dev/null +++ b/trading-platform/deploy/helm/api-gateway/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: api-gateway +description: API Gateway — Nginx reverse proxy +type: application +version: 0.1.0 +appVersion: "0.1.0" diff --git a/trading-platform/deploy/helm/api-gateway/templates/_helpers.tpl b/trading-platform/deploy/helm/api-gateway/templates/_helpers.tpl new file mode 100644 index 0000000..33b8067 --- /dev/null +++ b/trading-platform/deploy/helm/api-gateway/templates/_helpers.tpl @@ -0,0 +1,61 @@ + +{{/* +Expand the name of the chart. +*/}} +{{- define "api-gateway.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "api-gateway.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "api-gateway.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "api-gateway.labels" -}} +helm.sh/chart: {{ include "api-gateway.chart" . }} +{{ include "api-gateway.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "api-gateway.selectorLabels" -}} +app.kubernetes.io/name: {{ include "api-gateway.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "api-gateway.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "api-gateway.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/trading-platform/deploy/helm/api-gateway/templates/deployment.yaml b/trading-platform/deploy/helm/api-gateway/templates/deployment.yaml new file mode 100644 index 0000000..057c146 --- /dev/null +++ b/trading-platform/deploy/helm/api-gateway/templates/deployment.yaml @@ -0,0 +1,57 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "api-gateway.fullname" . }} + labels: + {{- include "api-gateway.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "api-gateway.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "api-gateway.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "api-gateway.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: 8080 + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 12 }} + {{- end }} diff --git a/trading-platform/deploy/helm/api-gateway/templates/hpa.yaml b/trading-platform/deploy/helm/api-gateway/templates/hpa.yaml new file mode 100644 index 0000000..472b45b --- /dev/null +++ b/trading-platform/deploy/helm/api-gateway/templates/hpa.yaml @@ -0,0 +1,32 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "api-gateway.fullname" . }} + labels: + {{- include "api-gateway.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "api-gateway.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/trading-platform/deploy/helm/api-gateway/templates/ingress.yaml b/trading-platform/deploy/helm/api-gateway/templates/ingress.yaml new file mode 100644 index 0000000..d875b28 --- /dev/null +++ b/trading-platform/deploy/helm/api-gateway/templates/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "api-gateway.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: extensions/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "api-gateway.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/trading-platform/deploy/helm/api-gateway/templates/networkpolicy.yaml b/trading-platform/deploy/helm/api-gateway/templates/networkpolicy.yaml new file mode 100644 index 0000000..0fa2821 --- /dev/null +++ b/trading-platform/deploy/helm/api-gateway/templates/networkpolicy.yaml @@ -0,0 +1,29 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "api-gateway.fullname" . }}-network-policy + labels: + {{- include "api-gateway.labels" . | nindent 4 }} +spec: + podSelector: + matchLabels: + {{- include "api-gateway.selectorLabels" . | nindent 6 }} + policyTypes: + - Ingress + - Egress + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: api-gateway + ports: + - port: http + protocol: TCP + egress: + # Allow DNS resolution + - to: [] + ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP diff --git a/trading-platform/deploy/helm/api-gateway/templates/service.yaml b/trading-platform/deploy/helm/api-gateway/templates/service.yaml new file mode 100644 index 0000000..545d6f0 --- /dev/null +++ b/trading-platform/deploy/helm/api-gateway/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "api-gateway.fullname" . }} + labels: + {{- include "api-gateway.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "api-gateway.selectorLabels" . | nindent 4 }} diff --git a/trading-platform/deploy/helm/api-gateway/templates/serviceaccount.yaml b/trading-platform/deploy/helm/api-gateway/templates/serviceaccount.yaml new file mode 100644 index 0000000..d707caa --- /dev/null +++ b/trading-platform/deploy/helm/api-gateway/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "api-gateway.serviceAccountName" . }} + labels: + {{- include "api-gateway.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/trading-platform/deploy/helm/api-gateway/values.yaml b/trading-platform/deploy/helm/api-gateway/values.yaml new file mode 100644 index 0000000..551148b --- /dev/null +++ b/trading-platform/deploy/helm/api-gateway/values.yaml @@ -0,0 +1,80 @@ +# trading-platform/api-gateway Helm chart values + +replicaCount: 2 + +image: + repository: trading-platform/api-gateway + pullPolicy: IfNotPresent + tag: "" + +imagePullSecrets: [] + +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + create: true + annotations: {} + name: "" + +podAnnotations: {} + +securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + +service: + type: ClusterIP + port: 8080 + +ingress: + enabled: true + className: nginx + annotations: {} + hosts: + - host: api.trading.example.com + paths: + - path: / + pathType: Prefix + tls: [] + +resources: + limits: + cpu: "500m" + memory: 256Mi + requests: + cpu: "250m" + memory: 128Mi + +autoscaling: + enabled: true + minReplicas: 2 + maxReplicas: 10 + targetCPUUtilizationPercentage: 80 + targetMemoryUtilizationPercentage: 80 + +livenessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 15 + periodSeconds: 15 + timeoutSeconds: 5 + failureThreshold: 3 + +readinessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + +nodeSelector: {} +tolerations: [] +affinity: {} diff --git a/trading-platform/deploy/helm/dashboard/Chart.yaml b/trading-platform/deploy/helm/dashboard/Chart.yaml new file mode 100644 index 0000000..2b077c3 --- /dev/null +++ b/trading-platform/deploy/helm/dashboard/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: dashboard +description: Dashboard frontend +type: application +version: 0.1.0 +appVersion: "0.1.0" diff --git a/trading-platform/deploy/helm/dashboard/templates/_helpers.tpl b/trading-platform/deploy/helm/dashboard/templates/_helpers.tpl new file mode 100644 index 0000000..3402279 --- /dev/null +++ b/trading-platform/deploy/helm/dashboard/templates/_helpers.tpl @@ -0,0 +1,61 @@ + +{{/* +Expand the name of the chart. +*/}} +{{- define "dashboard.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "dashboard.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "dashboard.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "dashboard.labels" -}} +helm.sh/chart: {{ include "dashboard.chart" . }} +{{ include "dashboard.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "dashboard.selectorLabels" -}} +app.kubernetes.io/name: {{ include "dashboard.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "dashboard.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "dashboard.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/trading-platform/deploy/helm/dashboard/templates/deployment.yaml b/trading-platform/deploy/helm/dashboard/templates/deployment.yaml new file mode 100644 index 0000000..59351c0 --- /dev/null +++ b/trading-platform/deploy/helm/dashboard/templates/deployment.yaml @@ -0,0 +1,57 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "dashboard.fullname" . }} + labels: + {{- include "dashboard.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "dashboard.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "dashboard.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "dashboard.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: 3000 + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 12 }} + {{- end }} diff --git a/trading-platform/deploy/helm/dashboard/templates/hpa.yaml b/trading-platform/deploy/helm/dashboard/templates/hpa.yaml new file mode 100644 index 0000000..c5f6544 --- /dev/null +++ b/trading-platform/deploy/helm/dashboard/templates/hpa.yaml @@ -0,0 +1,32 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "dashboard.fullname" . }} + labels: + {{- include "dashboard.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "dashboard.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/trading-platform/deploy/helm/dashboard/templates/networkpolicy.yaml b/trading-platform/deploy/helm/dashboard/templates/networkpolicy.yaml new file mode 100644 index 0000000..2f84151 --- /dev/null +++ b/trading-platform/deploy/helm/dashboard/templates/networkpolicy.yaml @@ -0,0 +1,29 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "dashboard.fullname" . }}-network-policy + labels: + {{- include "dashboard.labels" . | nindent 4 }} +spec: + podSelector: + matchLabels: + {{- include "dashboard.selectorLabels" . | nindent 6 }} + policyTypes: + - Ingress + - Egress + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: api-gateway + ports: + - port: http + protocol: TCP + egress: + # Allow DNS resolution + - to: [] + ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP diff --git a/trading-platform/deploy/helm/dashboard/templates/service.yaml b/trading-platform/deploy/helm/dashboard/templates/service.yaml new file mode 100644 index 0000000..155124b --- /dev/null +++ b/trading-platform/deploy/helm/dashboard/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "dashboard.fullname" . }} + labels: + {{- include "dashboard.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "dashboard.selectorLabels" . | nindent 4 }} diff --git a/trading-platform/deploy/helm/dashboard/templates/serviceaccount.yaml b/trading-platform/deploy/helm/dashboard/templates/serviceaccount.yaml new file mode 100644 index 0000000..4948fb9 --- /dev/null +++ b/trading-platform/deploy/helm/dashboard/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "dashboard.serviceAccountName" . }} + labels: + {{- include "dashboard.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/trading-platform/deploy/helm/dashboard/values.yaml b/trading-platform/deploy/helm/dashboard/values.yaml new file mode 100644 index 0000000..e7d817f --- /dev/null +++ b/trading-platform/deploy/helm/dashboard/values.yaml @@ -0,0 +1,80 @@ +# trading-platform/dashboard Helm chart values + +replicaCount: 2 + +image: + repository: trading-platform/dashboard + pullPolicy: IfNotPresent + tag: "" + +imagePullSecrets: [] + +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + create: true + annotations: {} + name: "" + +podAnnotations: {} + +securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + +service: + type: ClusterIP + port: 3000 + +ingress: + enabled: false + className: nginx + annotations: {} + hosts: + - host: api.trading.example.com + paths: + - path: / + pathType: Prefix + tls: [] + +resources: + limits: + cpu: "500m" + memory: 512Mi + requests: + cpu: "250m" + memory: 256Mi + +autoscaling: + enabled: true + minReplicas: 2 + maxReplicas: 10 + targetCPUUtilizationPercentage: 80 + targetMemoryUtilizationPercentage: 80 + +livenessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 15 + periodSeconds: 15 + timeoutSeconds: 5 + failureThreshold: 3 + +readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + +nodeSelector: {} +tolerations: [] +affinity: {} diff --git a/trading-platform/deploy/helm/data-service/Chart.yaml b/trading-platform/deploy/helm/data-service/Chart.yaml new file mode 100644 index 0000000..b7e2b70 --- /dev/null +++ b/trading-platform/deploy/helm/data-service/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: data-service +description: Trading data service +type: application +version: 0.1.0 +appVersion: "0.1.0" diff --git a/trading-platform/deploy/helm/data-service/templates/_helpers.tpl b/trading-platform/deploy/helm/data-service/templates/_helpers.tpl new file mode 100644 index 0000000..670ec9d --- /dev/null +++ b/trading-platform/deploy/helm/data-service/templates/_helpers.tpl @@ -0,0 +1,61 @@ + +{{/* +Expand the name of the chart. +*/}} +{{- define "data-service.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "data-service.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "data-service.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "data-service.labels" -}} +helm.sh/chart: {{ include "data-service.chart" . }} +{{ include "data-service.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "data-service.selectorLabels" -}} +app.kubernetes.io/name: {{ include "data-service.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "data-service.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "data-service.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/trading-platform/deploy/helm/data-service/templates/deployment.yaml b/trading-platform/deploy/helm/data-service/templates/deployment.yaml new file mode 100644 index 0000000..f2a03be --- /dev/null +++ b/trading-platform/deploy/helm/data-service/templates/deployment.yaml @@ -0,0 +1,57 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "data-service.fullname" . }} + labels: + {{- include "data-service.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "data-service.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "data-service.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "data-service.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: 8001 + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 12 }} + {{- end }} diff --git a/trading-platform/deploy/helm/data-service/templates/hpa.yaml b/trading-platform/deploy/helm/data-service/templates/hpa.yaml new file mode 100644 index 0000000..9686023 --- /dev/null +++ b/trading-platform/deploy/helm/data-service/templates/hpa.yaml @@ -0,0 +1,32 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "data-service.fullname" . }} + labels: + {{- include "data-service.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "data-service.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/trading-platform/deploy/helm/data-service/templates/networkpolicy.yaml b/trading-platform/deploy/helm/data-service/templates/networkpolicy.yaml new file mode 100644 index 0000000..d50f2bc --- /dev/null +++ b/trading-platform/deploy/helm/data-service/templates/networkpolicy.yaml @@ -0,0 +1,29 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "data-service.fullname" . }}-network-policy + labels: + {{- include "data-service.labels" . | nindent 4 }} +spec: + podSelector: + matchLabels: + {{- include "data-service.selectorLabels" . | nindent 6 }} + policyTypes: + - Ingress + - Egress + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: api-gateway + ports: + - port: http + protocol: TCP + egress: + # Allow DNS resolution + - to: [] + ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP diff --git a/trading-platform/deploy/helm/data-service/templates/service.yaml b/trading-platform/deploy/helm/data-service/templates/service.yaml new file mode 100644 index 0000000..186d0d6 --- /dev/null +++ b/trading-platform/deploy/helm/data-service/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "data-service.fullname" . }} + labels: + {{- include "data-service.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "data-service.selectorLabels" . | nindent 4 }} diff --git a/trading-platform/deploy/helm/data-service/templates/serviceaccount.yaml b/trading-platform/deploy/helm/data-service/templates/serviceaccount.yaml new file mode 100644 index 0000000..3ba4646 --- /dev/null +++ b/trading-platform/deploy/helm/data-service/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "data-service.serviceAccountName" . }} + labels: + {{- include "data-service.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/trading-platform/deploy/helm/data-service/values.yaml b/trading-platform/deploy/helm/data-service/values.yaml new file mode 100644 index 0000000..8a5fff1 --- /dev/null +++ b/trading-platform/deploy/helm/data-service/values.yaml @@ -0,0 +1,80 @@ +# trading-platform/data-service Helm chart values + +replicaCount: 2 + +image: + repository: trading-platform/data-service + pullPolicy: IfNotPresent + tag: "" + +imagePullSecrets: [] + +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + create: true + annotations: {} + name: "" + +podAnnotations: {} + +securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + +service: + type: ClusterIP + port: 8001 + +ingress: + enabled: false + className: nginx + annotations: {} + hosts: + - host: api.trading.example.com + paths: + - path: / + pathType: Prefix + tls: [] + +resources: + limits: + cpu: "1000m" + memory: 1Gi + requests: + cpu: "500m" + memory: 512Mi + +autoscaling: + enabled: true + minReplicas: 2 + maxReplicas: 10 + targetCPUUtilizationPercentage: 80 + targetMemoryUtilizationPercentage: 80 + +livenessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 15 + periodSeconds: 15 + timeoutSeconds: 5 + failureThreshold: 3 + +readinessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + +nodeSelector: {} +tolerations: [] +affinity: {} diff --git a/trading-platform/deploy/helm/execute-service/Chart.yaml b/trading-platform/deploy/helm/execute-service/Chart.yaml new file mode 100644 index 0000000..0324bc7 --- /dev/null +++ b/trading-platform/deploy/helm/execute-service/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: execute-service +description: Trading execution service +type: application +version: 0.1.0 +appVersion: "0.1.0" diff --git a/trading-platform/deploy/helm/execute-service/templates/_helpers.tpl b/trading-platform/deploy/helm/execute-service/templates/_helpers.tpl new file mode 100644 index 0000000..0f46b86 --- /dev/null +++ b/trading-platform/deploy/helm/execute-service/templates/_helpers.tpl @@ -0,0 +1,61 @@ + +{{/* +Expand the name of the chart. +*/}} +{{- define "execute-service.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "execute-service.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "execute-service.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "execute-service.labels" -}} +helm.sh/chart: {{ include "execute-service.chart" . }} +{{ include "execute-service.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "execute-service.selectorLabels" -}} +app.kubernetes.io/name: {{ include "execute-service.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "execute-service.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "execute-service.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/trading-platform/deploy/helm/execute-service/templates/deployment.yaml b/trading-platform/deploy/helm/execute-service/templates/deployment.yaml new file mode 100644 index 0000000..5b7a032 --- /dev/null +++ b/trading-platform/deploy/helm/execute-service/templates/deployment.yaml @@ -0,0 +1,57 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "execute-service.fullname" . }} + labels: + {{- include "execute-service.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "execute-service.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "execute-service.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "execute-service.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: 8000 + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 12 }} + {{- end }} diff --git a/trading-platform/deploy/helm/execute-service/templates/hpa.yaml b/trading-platform/deploy/helm/execute-service/templates/hpa.yaml new file mode 100644 index 0000000..d27ea6d --- /dev/null +++ b/trading-platform/deploy/helm/execute-service/templates/hpa.yaml @@ -0,0 +1,32 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "execute-service.fullname" . }} + labels: + {{- include "execute-service.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "execute-service.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/trading-platform/deploy/helm/execute-service/templates/networkpolicy.yaml b/trading-platform/deploy/helm/execute-service/templates/networkpolicy.yaml new file mode 100644 index 0000000..2e809cd --- /dev/null +++ b/trading-platform/deploy/helm/execute-service/templates/networkpolicy.yaml @@ -0,0 +1,29 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "execute-service.fullname" . }}-network-policy + labels: + {{- include "execute-service.labels" . | nindent 4 }} +spec: + podSelector: + matchLabels: + {{- include "execute-service.selectorLabels" . | nindent 6 }} + policyTypes: + - Ingress + - Egress + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: api-gateway + ports: + - port: http + protocol: TCP + egress: + # Allow DNS resolution + - to: [] + ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP diff --git a/trading-platform/deploy/helm/execute-service/templates/service.yaml b/trading-platform/deploy/helm/execute-service/templates/service.yaml new file mode 100644 index 0000000..0fbb9ce --- /dev/null +++ b/trading-platform/deploy/helm/execute-service/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "execute-service.fullname" . }} + labels: + {{- include "execute-service.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "execute-service.selectorLabels" . | nindent 4 }} diff --git a/trading-platform/deploy/helm/execute-service/templates/serviceaccount.yaml b/trading-platform/deploy/helm/execute-service/templates/serviceaccount.yaml new file mode 100644 index 0000000..0f3619b --- /dev/null +++ b/trading-platform/deploy/helm/execute-service/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "execute-service.serviceAccountName" . }} + labels: + {{- include "execute-service.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/trading-platform/deploy/helm/execute-service/values.yaml b/trading-platform/deploy/helm/execute-service/values.yaml new file mode 100644 index 0000000..7dc92e6 --- /dev/null +++ b/trading-platform/deploy/helm/execute-service/values.yaml @@ -0,0 +1,80 @@ +# trading-platform/execute-service Helm chart values + +replicaCount: 2 + +image: + repository: trading-platform/execute-service + pullPolicy: IfNotPresent + tag: "" + +imagePullSecrets: [] + +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + create: true + annotations: {} + name: "" + +podAnnotations: {} + +securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + +service: + type: ClusterIP + port: 8000 + +ingress: + enabled: false + className: nginx + annotations: {} + hosts: + - host: api.trading.example.com + paths: + - path: / + pathType: Prefix + tls: [] + +resources: + limits: + cpu: "500m" + memory: 512Mi + requests: + cpu: "250m" + memory: 256Mi + +autoscaling: + enabled: true + minReplicas: 2 + maxReplicas: 10 + targetCPUUtilizationPercentage: 80 + targetMemoryUtilizationPercentage: 80 + +livenessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 15 + periodSeconds: 15 + timeoutSeconds: 5 + failureThreshold: 3 + +readinessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + +nodeSelector: {} +tolerations: [] +affinity: {} diff --git a/trading-platform/deploy/helm/news-service/Chart.yaml b/trading-platform/deploy/helm/news-service/Chart.yaml new file mode 100644 index 0000000..c9eb5d9 --- /dev/null +++ b/trading-platform/deploy/helm/news-service/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: news-service +description: News analysis service +type: application +version: 0.1.0 +appVersion: "0.1.0" diff --git a/trading-platform/deploy/helm/news-service/templates/_helpers.tpl b/trading-platform/deploy/helm/news-service/templates/_helpers.tpl new file mode 100644 index 0000000..8be65a7 --- /dev/null +++ b/trading-platform/deploy/helm/news-service/templates/_helpers.tpl @@ -0,0 +1,61 @@ + +{{/* +Expand the name of the chart. +*/}} +{{- define "news-service.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "news-service.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "news-service.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "news-service.labels" -}} +helm.sh/chart: {{ include "news-service.chart" . }} +{{ include "news-service.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "news-service.selectorLabels" -}} +app.kubernetes.io/name: {{ include "news-service.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "news-service.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "news-service.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/trading-platform/deploy/helm/news-service/templates/deployment.yaml b/trading-platform/deploy/helm/news-service/templates/deployment.yaml new file mode 100644 index 0000000..ef25033 --- /dev/null +++ b/trading-platform/deploy/helm/news-service/templates/deployment.yaml @@ -0,0 +1,57 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "news-service.fullname" . }} + labels: + {{- include "news-service.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "news-service.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "news-service.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "news-service.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: 8002 + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 12 }} + {{- end }} diff --git a/trading-platform/deploy/helm/news-service/templates/hpa.yaml b/trading-platform/deploy/helm/news-service/templates/hpa.yaml new file mode 100644 index 0000000..7c4aeca --- /dev/null +++ b/trading-platform/deploy/helm/news-service/templates/hpa.yaml @@ -0,0 +1,32 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "news-service.fullname" . }} + labels: + {{- include "news-service.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "news-service.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/trading-platform/deploy/helm/news-service/templates/networkpolicy.yaml b/trading-platform/deploy/helm/news-service/templates/networkpolicy.yaml new file mode 100644 index 0000000..ff3c8f9 --- /dev/null +++ b/trading-platform/deploy/helm/news-service/templates/networkpolicy.yaml @@ -0,0 +1,29 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "news-service.fullname" . }}-network-policy + labels: + {{- include "news-service.labels" . | nindent 4 }} +spec: + podSelector: + matchLabels: + {{- include "news-service.selectorLabels" . | nindent 6 }} + policyTypes: + - Ingress + - Egress + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: api-gateway + ports: + - port: http + protocol: TCP + egress: + # Allow DNS resolution + - to: [] + ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP diff --git a/trading-platform/deploy/helm/news-service/templates/service.yaml b/trading-platform/deploy/helm/news-service/templates/service.yaml new file mode 100644 index 0000000..8709a9e --- /dev/null +++ b/trading-platform/deploy/helm/news-service/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "news-service.fullname" . }} + labels: + {{- include "news-service.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "news-service.selectorLabels" . | nindent 4 }} diff --git a/trading-platform/deploy/helm/news-service/templates/serviceaccount.yaml b/trading-platform/deploy/helm/news-service/templates/serviceaccount.yaml new file mode 100644 index 0000000..4b093b0 --- /dev/null +++ b/trading-platform/deploy/helm/news-service/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "news-service.serviceAccountName" . }} + labels: + {{- include "news-service.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/trading-platform/deploy/helm/news-service/values.yaml b/trading-platform/deploy/helm/news-service/values.yaml new file mode 100644 index 0000000..7edc76e --- /dev/null +++ b/trading-platform/deploy/helm/news-service/values.yaml @@ -0,0 +1,80 @@ +# trading-platform/news-service Helm chart values + +replicaCount: 2 + +image: + repository: trading-platform/news-service + pullPolicy: IfNotPresent + tag: "" + +imagePullSecrets: [] + +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + create: true + annotations: {} + name: "" + +podAnnotations: {} + +securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + +service: + type: ClusterIP + port: 8002 + +ingress: + enabled: false + className: nginx + annotations: {} + hosts: + - host: api.trading.example.com + paths: + - path: / + pathType: Prefix + tls: [] + +resources: + limits: + cpu: "1000m" + memory: 1Gi + requests: + cpu: "500m" + memory: 512Mi + +autoscaling: + enabled: true + minReplicas: 2 + maxReplicas: 10 + targetCPUUtilizationPercentage: 80 + targetMemoryUtilizationPercentage: 80 + +livenessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 15 + periodSeconds: 15 + timeoutSeconds: 5 + failureThreshold: 3 + +readinessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + +nodeSelector: {} +tolerations: [] +affinity: {} diff --git a/trading-platform/deploy/k8s/base/api-gateway-deployment.yaml b/trading-platform/deploy/k8s/base/api-gateway-deployment.yaml new file mode 100644 index 0000000..834c1ff --- /dev/null +++ b/trading-platform/deploy/k8s/base/api-gateway-deployment.yaml @@ -0,0 +1,71 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: api-gateway + namespace: customer1 + labels: + app: api-gateway + app.kubernetes.io/name: api-gateway + app.kubernetes.io/component: microservice +spec: + replicas: 2 + selector: + matchLabels: + app: api-gateway + template: + metadata: + labels: + app: api-gateway + app.kubernetes.io/name: api-gateway + app.kubernetes.io/component: microservice + spec: + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + containers: + - name: api-gateway + image: "trading-platform/api-gateway:v${VERSION}" + ports: + - containerPort: 8080 + protocol: TCP + envFrom: + - configMapRef: + name: trading-platform-config + resources: + limits: + cpu: "500m" + memory: 256Mi + requests: + cpu: "250m" + memory: 128Mi + livenessProbe: + httpGet: + path: /health + port: 8080 + initialDelaySeconds: 15 + periodSeconds: 15 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /health + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: {} diff --git a/trading-platform/deploy/k8s/base/api-gateway-hpa.yaml b/trading-platform/deploy/k8s/base/api-gateway-hpa.yaml new file mode 100644 index 0000000..20940ea --- /dev/null +++ b/trading-platform/deploy/k8s/base/api-gateway-hpa.yaml @@ -0,0 +1,27 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: api-gateway-hpa + namespace: customer1 + labels: + app: api-gateway +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: api-gateway + minReplicas: 2 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 diff --git a/trading-platform/deploy/k8s/base/api-gateway-service.yaml b/trading-platform/deploy/k8s/base/api-gateway-service.yaml new file mode 100644 index 0000000..1ca171d --- /dev/null +++ b/trading-platform/deploy/k8s/base/api-gateway-service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: api-gateway + namespace: customer1 + labels: + app: api-gateway + app.kubernetes.io/name: api-gateway + app.kubernetes.io/component: microservice +spec: + type: ClusterIP + ports: + - port: 8080 + targetPort: 8080 + protocol: TCP + name: http + selector: + app: api-gateway diff --git a/trading-platform/deploy/k8s/base/cert-manager/api-gateway-mtls-cert.yaml b/trading-platform/deploy/k8s/base/cert-manager/api-gateway-mtls-cert.yaml new file mode 100644 index 0000000..1534702 --- /dev/null +++ b/trading-platform/deploy/k8s/base/cert-manager/api-gateway-mtls-cert.yaml @@ -0,0 +1,24 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: api-gateway-mtls-cert + namespace: customer1 +spec: + secretName: api-gateway-mtls-secret + duration: 2160h # 90 days + renewBefore: 360h # 15 days + commonName: api-gateway.customer1.svc.cluster.local + dnsNames: + - api-gateway + - api-gateway.customer1 + - api-gateway.customer1.svc + - api-gateway.customer1.svc.cluster.local + usages: + - digital signature + - key encipherment + - client auth + - server auth + issuerRef: + name: trading-platform-ca-issuer + kind: Issuer + group: cert-manager.io diff --git a/trading-platform/deploy/k8s/base/cert-manager/ca-issuer.yaml b/trading-platform/deploy/k8s/base/cert-manager/ca-issuer.yaml new file mode 100644 index 0000000..337ede9 --- /dev/null +++ b/trading-platform/deploy/k8s/base/cert-manager/ca-issuer.yaml @@ -0,0 +1,8 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: trading-platform-ca-issuer + namespace: customer1 +spec: + ca: + secretName: trading-platform-ca-secret diff --git a/trading-platform/deploy/k8s/base/cert-manager/cluster-issuers.yaml b/trading-platform/deploy/k8s/base/cert-manager/cluster-issuers.yaml new file mode 100644 index 0000000..9b8ce95 --- /dev/null +++ b/trading-platform/deploy/k8s/base/cert-manager/cluster-issuers.yaml @@ -0,0 +1,29 @@ +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: letsencrypt-prod +spec: + acme: + server: https://acme-v02.api.letsencrypt.org/directory + email: admin@trading-platform.com + privateKeySecretRef: + name: letsencrypt-prod-key + solvers: + - http01: + ingress: + class: nginx +--- +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: letsencrypt-staging +spec: + acme: + server: https://acme-staging-v02.api.letsencrypt.org/directory + email: admin@trading-platform.com + privateKeySecretRef: + name: letsencrypt-staging-key + solvers: + - http01: + ingress: + class: nginx diff --git a/trading-platform/deploy/k8s/base/cert-manager/dashboard-mtls-cert.yaml b/trading-platform/deploy/k8s/base/cert-manager/dashboard-mtls-cert.yaml new file mode 100644 index 0000000..d45e3b5 --- /dev/null +++ b/trading-platform/deploy/k8s/base/cert-manager/dashboard-mtls-cert.yaml @@ -0,0 +1,24 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: dashboard-mtls-cert + namespace: customer1 +spec: + secretName: dashboard-mtls-secret + duration: 2160h # 90 days + renewBefore: 360h # 15 days + commonName: dashboard.customer1.svc.cluster.local + dnsNames: + - dashboard + - dashboard.customer1 + - dashboard.customer1.svc + - dashboard.customer1.svc.cluster.local + usages: + - digital signature + - key encipherment + - client auth + - server auth + issuerRef: + name: trading-platform-ca-issuer + kind: Issuer + group: cert-manager.io diff --git a/trading-platform/deploy/k8s/base/cert-manager/data-service-mtls-cert.yaml b/trading-platform/deploy/k8s/base/cert-manager/data-service-mtls-cert.yaml new file mode 100644 index 0000000..e3e0acc --- /dev/null +++ b/trading-platform/deploy/k8s/base/cert-manager/data-service-mtls-cert.yaml @@ -0,0 +1,24 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: data-service-mtls-cert + namespace: customer1 +spec: + secretName: data-service-mtls-secret + duration: 2160h # 90 days + renewBefore: 360h # 15 days + commonName: data-service.customer1.svc.cluster.local + dnsNames: + - data-service + - data-service.customer1 + - data-service.customer1.svc + - data-service.customer1.svc.cluster.local + usages: + - digital signature + - key encipherment + - client auth + - server auth + issuerRef: + name: trading-platform-ca-issuer + kind: Issuer + group: cert-manager.io diff --git a/trading-platform/deploy/k8s/base/cert-manager/execute-service-mtls-cert.yaml b/trading-platform/deploy/k8s/base/cert-manager/execute-service-mtls-cert.yaml new file mode 100644 index 0000000..5d5d430 --- /dev/null +++ b/trading-platform/deploy/k8s/base/cert-manager/execute-service-mtls-cert.yaml @@ -0,0 +1,24 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: execute-service-mtls-cert + namespace: customer1 +spec: + secretName: execute-service-mtls-secret + duration: 2160h # 90 days + renewBefore: 360h # 15 days + commonName: execute-service.customer1.svc.cluster.local + dnsNames: + - execute-service + - execute-service.customer1 + - execute-service.customer1.svc + - execute-service.customer1.svc.cluster.local + usages: + - digital signature + - key encipherment + - client auth + - server auth + issuerRef: + name: trading-platform-ca-issuer + kind: Issuer + group: cert-manager.io diff --git a/trading-platform/deploy/k8s/base/cert-manager/issuer.yaml b/trading-platform/deploy/k8s/base/cert-manager/issuer.yaml new file mode 100644 index 0000000..42eae6d --- /dev/null +++ b/trading-platform/deploy/k8s/base/cert-manager/issuer.yaml @@ -0,0 +1,24 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: selfsigned-issuer + namespace: cert-manager +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: trading-platform-ca + namespace: cert-manager +spec: + isCA: true + commonName: trading-platform-ca + secretName: trading-platform-ca-secret + privateKey: + algorithm: ECDSA + size: 256 + issuerRef: + name: selfsigned-issuer + kind: Issuer + group: cert-manager.io diff --git a/trading-platform/deploy/k8s/base/cert-manager/news-service-mtls-cert.yaml b/trading-platform/deploy/k8s/base/cert-manager/news-service-mtls-cert.yaml new file mode 100644 index 0000000..ae15e24 --- /dev/null +++ b/trading-platform/deploy/k8s/base/cert-manager/news-service-mtls-cert.yaml @@ -0,0 +1,24 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: news-service-mtls-cert + namespace: customer1 +spec: + secretName: news-service-mtls-secret + duration: 2160h # 90 days + renewBefore: 360h # 15 days + commonName: news-service.customer1.svc.cluster.local + dnsNames: + - news-service + - news-service.customer1 + - news-service.customer1.svc + - news-service.customer1.svc.cluster.local + usages: + - digital signature + - key encipherment + - client auth + - server auth + issuerRef: + name: trading-platform-ca-issuer + kind: Issuer + group: cert-manager.io diff --git a/trading-platform/deploy/k8s/base/dashboard-deployment.yaml b/trading-platform/deploy/k8s/base/dashboard-deployment.yaml new file mode 100644 index 0000000..9486160 --- /dev/null +++ b/trading-platform/deploy/k8s/base/dashboard-deployment.yaml @@ -0,0 +1,71 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dashboard + namespace: customer1 + labels: + app: dashboard + app.kubernetes.io/name: dashboard + app.kubernetes.io/component: microservice +spec: + replicas: 2 + selector: + matchLabels: + app: dashboard + template: + metadata: + labels: + app: dashboard + app.kubernetes.io/name: dashboard + app.kubernetes.io/component: microservice + spec: + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + containers: + - name: dashboard + image: "trading-platform/dashboard:v${VERSION}" + ports: + - containerPort: 3000 + protocol: TCP + envFrom: + - configMapRef: + name: trading-platform-config + resources: + limits: + cpu: "500m" + memory: 512Mi + requests: + cpu: "250m" + memory: 256Mi + livenessProbe: + httpGet: + path: /health + port: 3000 + initialDelaySeconds: 15 + periodSeconds: 15 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /health + port: 3000 + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: {} diff --git a/trading-platform/deploy/k8s/base/dashboard-hpa.yaml b/trading-platform/deploy/k8s/base/dashboard-hpa.yaml new file mode 100644 index 0000000..d7485a1 --- /dev/null +++ b/trading-platform/deploy/k8s/base/dashboard-hpa.yaml @@ -0,0 +1,27 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: dashboard-hpa + namespace: customer1 + labels: + app: dashboard +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: dashboard + minReplicas: 2 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 diff --git a/trading-platform/deploy/k8s/base/dashboard-service.yaml b/trading-platform/deploy/k8s/base/dashboard-service.yaml new file mode 100644 index 0000000..fafde36 --- /dev/null +++ b/trading-platform/deploy/k8s/base/dashboard-service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: dashboard + namespace: customer1 + labels: + app: dashboard + app.kubernetes.io/name: dashboard + app.kubernetes.io/component: microservice +spec: + type: ClusterIP + ports: + - port: 3000 + targetPort: 3000 + protocol: TCP + name: http + selector: + app: dashboard diff --git a/trading-platform/deploy/k8s/base/data-service-deployment.yaml b/trading-platform/deploy/k8s/base/data-service-deployment.yaml new file mode 100644 index 0000000..7eb668a --- /dev/null +++ b/trading-platform/deploy/k8s/base/data-service-deployment.yaml @@ -0,0 +1,71 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: data-service + namespace: customer1 + labels: + app: data-service + app.kubernetes.io/name: data-service + app.kubernetes.io/component: microservice +spec: + replicas: 2 + selector: + matchLabels: + app: data-service + template: + metadata: + labels: + app: data-service + app.kubernetes.io/name: data-service + app.kubernetes.io/component: microservice + spec: + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + containers: + - name: data-service + image: "trading-platform/data-service:v${VERSION}" + ports: + - containerPort: 8001 + protocol: TCP + envFrom: + - configMapRef: + name: trading-platform-config + resources: + limits: + cpu: "1000m" + memory: 1Gi + requests: + cpu: "500m" + memory: 512Mi + livenessProbe: + httpGet: + path: /health + port: 8001 + initialDelaySeconds: 15 + periodSeconds: 15 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /health + port: 8001 + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: {} diff --git a/trading-platform/deploy/k8s/base/data-service-hpa.yaml b/trading-platform/deploy/k8s/base/data-service-hpa.yaml new file mode 100644 index 0000000..31c70e2 --- /dev/null +++ b/trading-platform/deploy/k8s/base/data-service-hpa.yaml @@ -0,0 +1,27 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: data-service-hpa + namespace: customer1 + labels: + app: data-service +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: data-service + minReplicas: 2 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 diff --git a/trading-platform/deploy/k8s/base/data-service-service.yaml b/trading-platform/deploy/k8s/base/data-service-service.yaml new file mode 100644 index 0000000..03a0895 --- /dev/null +++ b/trading-platform/deploy/k8s/base/data-service-service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: data-service + namespace: customer1 + labels: + app: data-service + app.kubernetes.io/name: data-service + app.kubernetes.io/component: microservice +spec: + type: ClusterIP + ports: + - port: 8001 + targetPort: 8001 + protocol: TCP + name: http + selector: + app: data-service diff --git a/trading-platform/deploy/k8s/base/execute-service-deployment.yaml b/trading-platform/deploy/k8s/base/execute-service-deployment.yaml new file mode 100644 index 0000000..c0d8e8e --- /dev/null +++ b/trading-platform/deploy/k8s/base/execute-service-deployment.yaml @@ -0,0 +1,71 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: execute-service + namespace: customer1 + labels: + app: execute-service + app.kubernetes.io/name: execute-service + app.kubernetes.io/component: microservice +spec: + replicas: 2 + selector: + matchLabels: + app: execute-service + template: + metadata: + labels: + app: execute-service + app.kubernetes.io/name: execute-service + app.kubernetes.io/component: microservice + spec: + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + containers: + - name: execute-service + image: "trading-platform/execute-service:v${VERSION}" + ports: + - containerPort: 8000 + protocol: TCP + envFrom: + - configMapRef: + name: trading-platform-config + resources: + limits: + cpu: "500m" + memory: 512Mi + requests: + cpu: "250m" + memory: 256Mi + livenessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 15 + periodSeconds: 15 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: {} diff --git a/trading-platform/deploy/k8s/base/execute-service-hpa.yaml b/trading-platform/deploy/k8s/base/execute-service-hpa.yaml new file mode 100644 index 0000000..bda8bf7 --- /dev/null +++ b/trading-platform/deploy/k8s/base/execute-service-hpa.yaml @@ -0,0 +1,27 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: execute-service-hpa + namespace: customer1 + labels: + app: execute-service +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: execute-service + minReplicas: 2 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 diff --git a/trading-platform/deploy/k8s/base/execute-service-service.yaml b/trading-platform/deploy/k8s/base/execute-service-service.yaml new file mode 100644 index 0000000..4d084bb --- /dev/null +++ b/trading-platform/deploy/k8s/base/execute-service-service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: execute-service + namespace: customer1 + labels: + app: execute-service + app.kubernetes.io/name: execute-service + app.kubernetes.io/component: microservice +spec: + type: ClusterIP + ports: + - port: 8000 + targetPort: 8000 + protocol: TCP + name: http + selector: + app: execute-service diff --git a/trading-platform/deploy/k8s/base/ingress.yaml b/trading-platform/deploy/k8s/base/ingress.yaml new file mode 100644 index 0000000..ed32132 --- /dev/null +++ b/trading-platform/deploy/k8s/base/ingress.yaml @@ -0,0 +1,40 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: api-gateway-ingress + namespace: customer1 + labels: + app: api-gateway + annotations: + nginx.ingress.kubernetes.io/ssl-redirect: "true" + nginx.ingress.kubernetes.io/proxy-body-size: "10m" + nginx.ingress.kubernetes.io/rate-limit: "100" + cert-manager.io/cluster-issuer: letsencrypt-prod +spec: + ingressClassName: nginx + tls: + - hosts: + - api.trading.example.com + - dashboard.trading.example.com + secretName: trading-tls-secret + rules: + - host: api.trading.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: api-gateway + port: + number: 8080 + - host: dashboard.trading.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: dashboard + port: + number: 3000 diff --git a/trading-platform/deploy/k8s/base/namespace.yaml b/trading-platform/deploy/k8s/base/namespace.yaml new file mode 100644 index 0000000..9952f89 --- /dev/null +++ b/trading-platform/deploy/k8s/base/namespace.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: customer1 + labels: + name: customer1 + istio-injection: disabled # Disable Istio if using native K8s policies +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: trading-platform-config + namespace: customer1 +data: + KAFKA_BOOTSTRAP_SERVERS: "kafka-headless:9092" + REDIS_URL: "redis://redis-master:6379/0" + LOG_LEVEL: "info" + TRADING_ENV: "production" diff --git a/trading-platform/deploy/k8s/base/news-service-deployment.yaml b/trading-platform/deploy/k8s/base/news-service-deployment.yaml new file mode 100644 index 0000000..f003b91 --- /dev/null +++ b/trading-platform/deploy/k8s/base/news-service-deployment.yaml @@ -0,0 +1,71 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: news-service + namespace: customer1 + labels: + app: news-service + app.kubernetes.io/name: news-service + app.kubernetes.io/component: microservice +spec: + replicas: 2 + selector: + matchLabels: + app: news-service + template: + metadata: + labels: + app: news-service + app.kubernetes.io/name: news-service + app.kubernetes.io/component: microservice + spec: + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + containers: + - name: news-service + image: "trading-platform/news-service:v${VERSION}" + ports: + - containerPort: 8002 + protocol: TCP + envFrom: + - configMapRef: + name: trading-platform-config + resources: + limits: + cpu: "1000m" + memory: 1Gi + requests: + cpu: "500m" + memory: 512Mi + livenessProbe: + httpGet: + path: /health + port: 8002 + initialDelaySeconds: 15 + periodSeconds: 15 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /health + port: 8002 + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: {} diff --git a/trading-platform/deploy/k8s/base/news-service-hpa.yaml b/trading-platform/deploy/k8s/base/news-service-hpa.yaml new file mode 100644 index 0000000..8d85e99 --- /dev/null +++ b/trading-platform/deploy/k8s/base/news-service-hpa.yaml @@ -0,0 +1,27 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: news-service-hpa + namespace: customer1 + labels: + app: news-service +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: news-service + minReplicas: 2 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 diff --git a/trading-platform/deploy/k8s/base/news-service-service.yaml b/trading-platform/deploy/k8s/base/news-service-service.yaml new file mode 100644 index 0000000..f9c0d21 --- /dev/null +++ b/trading-platform/deploy/k8s/base/news-service-service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: news-service + namespace: customer1 + labels: + app: news-service + app.kubernetes.io/name: news-service + app.kubernetes.io/component: microservice +spec: + type: ClusterIP + ports: + - port: 8002 + targetPort: 8002 + protocol: TCP + name: http + selector: + app: news-service diff --git a/trading-platform/deploy/mtls/README.md b/trading-platform/deploy/mtls/README.md new file mode 100644 index 0000000..c26b15f --- /dev/null +++ b/trading-platform/deploy/mtls/README.md @@ -0,0 +1,20 @@ +# ============================================================================= +# mTLS Configuration for Trading Platform +# ============================================================================= +# This directory contains certificates and configuration for mutual TLS +# between services. In production, use cert-manager to automate this. +# +# Option 1: cert-manager (recommended for production) +# Option 2: Manual certificate management (for dev/testing) +# +# Certificate hierarchy: +# Root CA +# ├── Service CA (issues service-to-service certs) +# │ ├── execute-service cert +# │ ├── data-service cert +# │ ├── news-service cert +# │ ├── api-gateway cert +# │ └── dashboard cert +# └── Ingress CA (for external-facing TLS) +# └── api-gateway TLS cert (for HTTPS) +# ============================================================================= diff --git a/trading-platform/deploy/scripts/deploy.sh b/trading-platform/deploy/scripts/deploy.sh new file mode 100755 index 0000000..5099285 --- /dev/null +++ b/trading-platform/deploy/scripts/deploy.sh @@ -0,0 +1,141 @@ +#!/bin/bash +# ============================================================================= +# Deploy Trading Platform to Kubernetes +# ============================================================================= +# +# Usage: +# ./deploy/scripts/deploy.sh [staging|production] [tag] +# +# Examples: +# ./deploy/scripts/deploy.sh staging latest +# ./deploy/scripts/deploy.sh production v1.2.3 +# +# Prerequisites: +# - kubectl configured with cluster access +# - Helm 3.x installed +# - Docker images pushed to registry +# - cert-manager installed in cluster (for TLS) +# ============================================================================= + +set -euo pipefail + +ENVIRONMENT="${1:-staging}" +IMAGE_TAG="${2:-latest}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Validate environment +if [[ ! "$ENVIRONMENT" =~ ^(staging|production)$ ]]; then + echo "ERROR: Environment must be 'staging' or 'production', got '$ENVIRONMENT'" + exit 1 +fi + +# Set namespace and values file based on environment +if [[ "$ENVIRONMENT" == "staging" ]]; then + NAMESPACE="customer1-staging" + VALUES_FILE="$PROJECT_ROOT/deploy/k8s/overlays/staging/kustomization.yaml" +else + NAMESPACE="customer1" + VALUES_FILE="$PROJECT_ROOT/deploy/k8s/overlays/production/kustomization.yaml" +fi + +echo "========================================================" +echo " Deploying Trading Platform to $ENVIRONMENT" +echo " Image tag: $IMAGE_TAG" +echo " Namespace: $NAMESPACE" +echo "========================================================" + +# Confirm cluster context +CURRENT_CONTEXT=$(kubectl config current-context 2>/dev/null || echo "unknown") +echo "Current kubectl context: $CURRENT_CONTEXT" +read -r -p "Continue? (y/N) " -n 1 +echo +if [[ ! $REPLY =~ ^[Yy]$ ]]; then + echo "Deployment cancelled." + exit 1 +fi + +# Install dependencies (optional) +echo "" +echo "Step 1/5: Checking prerequisites..." + +# Check for Helm +if ! command -v helm &>/dev/null; then + echo "ERROR: helm is not installed" + exit 1 +fi + +# Check for kubectl +if ! command -v kubectl &>/dev/null; then + echo "ERROR: kubectl is not installed" + exit 1 +fi + +# Check cluster connectivity +if ! kubectl cluster-info &>/dev/null; then + echo "ERROR: Cannot connect to Kubernetes cluster" + exit 1 +fi +echo " ✓ Kubernetes cluster is accessible" + +# Create namespace if it doesn't exist +kubectl create namespace "$NAMESPACE" --dry-run=client -o yaml | kubectl apply -f - +echo " ✓ Namespace $NAMESPACE exists" + +# Step 2: Deploy infrastructure (PostgreSQL, Redis, Kafka) +echo "" +echo "Step 2/5: Deploying infrastructure..." +kubectl apply -f "$PROJECT_ROOT/deploy/k8s/base/namespace.yaml" +kubectl apply -f "$PROJECT_ROOT/deploy/k8s/base/configmap.yaml" +echo " ✓ ConfigMap applied" + +# Step 3: Deploy services +echo "" +echo "Step 3/5: Deploying microservices..." + +SERVICES=("execute-service" "data-service" "news-service" "api-gateway" "dashboard") + +for service in "${SERVICES[@]}"; do + echo " Deploying $service..." + kubectl apply -f "$PROJECT_ROOT/deploy/k8s/base/${service}-deployment.yaml" + kubectl apply -f "$PROJECT_ROOT/deploy/k8s/base/${service}-service.yaml" +done + +echo " ✓ All services deployed" + +# Step 4: Deploy ingress and networking +echo "" +echo "Step 4/5: Configuring ingress and networking..." +kubectl apply -f "$PROJECT_ROOT/deploy/k8s/base/ingress.yaml" +echo " ✓ Ingress configured" + +# Apply NetworkPolicies from security review +if [[ -d "$PROJECT_ROOT/trading-platform/security/network-policies" ]]; then + kubectl apply -f "$PROJECT_ROOT/trading-platform/security/network-policies/" + echo " ✓ NetworkPolicies applied" +fi + +# Step 5: Wait for rollouts +echo "" +echo "Step 5/5: Waiting for deployments to stabilize..." + +for service in "${SERVICES[@]}"; do + echo " Waiting for $service..." + if ! kubectl rollout status "deployment/${service}" -n "$NAMESPACE" --timeout=5m; then + echo "WARNING: $service rollout timed out" + echo " Check pods: kubectl get pods -n $NAMESPACE -l app=$service" + echo " Check logs: kubectl logs -n $NAMESPACE -l app=$service --tail=100" + exit 1 + fi +done + +echo "" +echo "========================================================" +echo " Deployment complete! All services running." +echo "========================================================" +echo "" +echo "Useful commands:" +echo " kubectl get pods -n $NAMESPACE" +echo " kubectl get svc -n $NAMESPACE" +echo " kubectl get ingress -n $NAMESPACE" +echo " kubectl logs -n $NAMESPACE -l app=$service -f" diff --git a/trading-platform/deploy/scripts/generate-mtls-certs.sh b/trading-platform/deploy/scripts/generate-mtls-certs.sh new file mode 100755 index 0000000..a43b697 --- /dev/null +++ b/trading-platform/deploy/scripts/generate-mtls-certs.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# ============================================================================= +# Manual mTLS certificate generation script (for dev/testing only) +# ============================================================================= +# In production, use cert-manager (see k8s/base/cert-manager/). +# This script generates self-signed certificates for local testing. +# +# Usage: +# ./deploy/scripts/generate-mtls-certs.sh +# +# Output: deploy/mtls/ +# ============================================================================= + +set -euo pipefail + +OUTPUT_DIR="deploy/mtls" +SERVICES=("execute-service" "data-service" "news-service" "api-gateway" "dashboard") +DAYS_VALID=365 + +mkdir -p "$OUTPUT_DIR/ca" "$OUTPUT_DIR/certs" + +# ── Generate Root CA ──────────────────────────────────────────────────────── +echo "Generating Root CA..." +openssl genrsa -out "$OUTPUT_DIR/ca/ca.key" 4096 2>/dev/null +openssl req -x509 -new -nodes \ + -key "$OUTPUT_DIR/ca/ca.key" \ + -sha256 \ + -days $DAYS_VALID \ + -out "$OUTPUT_DIR/ca/ca.crt" \ + -subj "/C=US/ST=California/O=TradingPlatform/CN=Trading Platform Root CA" + +# ── Generate Service Certificates ──────────────────────────────────────────── +for SERVICE in "${SERVICES[@]}"; do + echo "Generating certificate for $SERVICE..." + + # Generate private key + openssl genrsa \ + -out "$OUTPUT_DIR/certs/${SERVICE}.key" 2048 2>/dev/null + + # Generate CSR + openssl req -new \ + -key "$OUTPUT_DIR/certs/${SERVICE}.key" \ + -out "$OUTPUT_DIR/certs/${SERVICE}.csr" \ + -subj "/C=US/ST=California/O=TradingPlatform/CN=${SERVICE}.customer1.svc.cluster.local" \ + -addext "subjectAltName=DNS:${SERVICE},DNS:${SERVICE}.customer1,DNS:${SERVICE}.customer1.svc.cluster.local" + + # Sign with CA + openssl x509 -req \ + -in "$OUTPUT_DIR/certs/${SERVICE}.csr" \ + -CA "$OUTPUT_DIR/ca/ca.crt" \ + -CAkey "$OUTPUT_DIR/ca/ca.key" \ + -CAcreateserial \ + -out "$OUTPUT_DIR/certs/${SERVICE}.crt" \ + -days $DAYS_VALID \ + -sha256 \ + -extfile <(printf "subjectAltName=DNS:${SERVICE},DNS:${SERVICE}.customer1,DNS:${SERVICE}.customer1.svc.cluster.local") + + # Clean up CSR + rm "$OUTPUT_DIR/certs/${SERVICE}.csr" +done + +echo "Done! All certificates generated in $OUTPUT_DIR/certs/" +echo "CA certificate: $OUTPUT_DIR/ca/ca.crt" +echo "" +echo "To verify a certificate:" +echo " openssl verify -CAfile $OUTPUT_DIR/ca/ca.crt $OUTPUT_DIR/certs/.crt" diff --git a/trading-platform/dockerfiles/dashboard/.dockerignore b/trading-platform/dockerfiles/dashboard/.dockerignore new file mode 100644 index 0000000..4443f35 --- /dev/null +++ b/trading-platform/dockerfiles/dashboard/.dockerignore @@ -0,0 +1,10 @@ +node_modules/ +.next/ +out/ +dist/ +.env* +.git/ +.vscode/ +coverage/ +.idea/ +*.log diff --git a/trading-platform/dockerfiles/dashboard/Dockerfile b/trading-platform/dockerfiles/dashboard/Dockerfile new file mode 100644 index 0000000..5f5d93a --- /dev/null +++ b/trading-platform/dockerfiles/dashboard/Dockerfile @@ -0,0 +1,31 @@ +# Multi-stage build for Next.js dashboard +# ---- Builder ---- +FROM node:20-alpine AS builder +WORKDIR /app +COPY package.json package-lock.json ./ +RUN npm ci +COPY . . +RUN npm run build + +# ---- Production ---- +FROM node:20-alpine AS runner +ENV NODE_ENV=production + +RUN addgroup --system --gid 1001 nodejs +RUN adduser --system --uid 1001 nextjs + +WORKDIR /app +COPY --from=builder /app/public ./public +COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./ +COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static + +USER nextjs + +EXPOSE 3000 +ENV PORT=3000 +ENV HOSTNAME="0.0.0.0" + +HEALTHCHECK --interval=15s --timeout=5s --start-period=20s --retries=3 \\ + CMD wget --no-verbose --tries=1 --spider http://localhost:${PORT}/ || exit 1 + +CMD ["node", "server.js"] diff --git a/trading-platform/dockerfiles/data-service/.dockerignore b/trading-platform/dockerfiles/data-service/.dockerignore new file mode 100644 index 0000000..be876f2 --- /dev/null +++ b/trading-platform/dockerfiles/data-service/.dockerignore @@ -0,0 +1,10 @@ +# Python Docker ignores +__pycache__/ +*.pyc +.venv/ +venv/ +*.egg-info/ +.pytest_cache/ +.git/ +.env +tests/ diff --git a/trading-platform/dockerfiles/data-service/Dockerfile b/trading-platform/dockerfiles/data-service/Dockerfile new file mode 100644 index 0000000..6b9cde4 --- /dev/null +++ b/trading-platform/dockerfiles/data-service/Dockerfile @@ -0,0 +1,23 @@ +# Multi-stage build for data-service (Postgres + Redis + Kafka consumers) +FROM python:3.12-slim AS builder +WORKDIR /build +COPY pyproject.toml . +RUN pip install --no-cache-dir --prefix=/install . + +FROM python:3.12-slim +RUN useradd -m --system appuser + +COPY --from=builder /install /usr/local + +WORKDIR /app +COPY --chown=appuser:appuser data_service/ ./data_service/ +COPY --chown=appuser:appuser pyproject.toml alembic.ini ./ + +USER appuser + +HEALTHCHECK --interval=15s --timeout=5s --start-period=15s --retries=3 \\ + CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8000/health')" || exit 1 + +EXPOSE 8000 + +CMD ["uvicorn", "data_service.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/trading-platform/dockerfiles/execute-service/.dockerignore b/trading-platform/dockerfiles/execute-service/.dockerignore new file mode 100644 index 0000000..be876f2 --- /dev/null +++ b/trading-platform/dockerfiles/execute-service/.dockerignore @@ -0,0 +1,10 @@ +# Python Docker ignores +__pycache__/ +*.pyc +.venv/ +venv/ +*.egg-info/ +.pytest_cache/ +.git/ +.env +tests/ diff --git a/trading-platform/dockerfiles/execute-service/Dockerfile b/trading-platform/dockerfiles/execute-service/Dockerfile new file mode 100644 index 0000000..e64268b --- /dev/null +++ b/trading-platform/dockerfiles/execute-service/Dockerfile @@ -0,0 +1,23 @@ +# Multi-stage build for execute-service (Hyperliquid + Solana trading engine) +FROM python:3.12-slim AS builder +WORKDIR /build +COPY pyproject.toml ./ +RUN pip install --no-cache-dir --prefix=/install . + +FROM python:3.12-slim +RUN useradd -m --system appuser + +COPY --from=builder /install /usr/local + +WORKDIR /app +COPY --chown=appuser:appuser app/ ./app/ +COPY --chown=appuser:appuser pyproject.toml ./ + +USER appuser + +HEALTHCHECK --interval=15s --timeout=5s --start-period=15s --retries=3 \\ + CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8000/health')" || exit 1 + +EXPOSE 8000 + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/trading-platform/dockerfiles/news-service/.dockerignore b/trading-platform/dockerfiles/news-service/.dockerignore new file mode 100644 index 0000000..be876f2 --- /dev/null +++ b/trading-platform/dockerfiles/news-service/.dockerignore @@ -0,0 +1,10 @@ +# Python Docker ignores +__pycache__/ +*.pyc +.venv/ +venv/ +*.egg-info/ +.pytest_cache/ +.git/ +.env +tests/ diff --git a/trading-platform/dockerfiles/news-service/Dockerfile b/trading-platform/dockerfiles/news-service/Dockerfile new file mode 100644 index 0000000..286969b --- /dev/null +++ b/trading-platform/dockerfiles/news-service/Dockerfile @@ -0,0 +1,30 @@ +# Multi-stage build for news-service (CNPG connector + Kafka producer) +FROM python:3.12-slim AS builder +WORKDIR /build +COPY requirements.txt . +RUN pip install --no-cache-dir --prefix=/install -r requirements.txt + +FROM python:3.12-slim +RUN useradd -m --system appuser + +# Copy installed deps from builder +COPY --from=builder /install /usr/local + +# Install system deps for NLTK +RUN apt-get update && \\ + apt-get install -y --no-install-recommends gcc libpq-dev && \\ + rm -rf /var/lib/apt/lists/* + +# Download NLTK data as root before switching user +RUN python -c "import nltk; nltk.download('punkt'); nltk.download('punkt_tab'); nltk.download('averaged_perceptron_tagger')" + +WORKDIR /app +COPY --chown=appuser:appuser . . +USER appuser + +HEALTHCHECK --interval=15s --timeout=5s --start-period=15s --retries=3 \\ + CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8000/health')" || exit 1 + +EXPOSE 8000 + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "4"] diff --git a/trading-platform/helm/.sops.yaml b/trading-platform/helm/.sops.yaml new file mode 100644 index 0000000..35ae57e --- /dev/null +++ b/trading-platform/helm/.sops.yaml @@ -0,0 +1,19 @@ +# SOPS configuration for trading platform secrets +# Usage: sops -e -i trading-secrets.yaml && kubectl apply -f trading-secrets.yaml +creation_rules: + # Production secrets - encrypted with Age key + - path_regex: trading-secrets.yaml$ + encrypted_regex: "^(stringData|data)$" + age: >- + age1ql3z7hjy54pw3hyww5ayyfg7zqgvcnw3eks6lekrq733qkq76jwqgq3g20 + + # Per-environment overrides + - path_regex: .*/staging/trading-secrets.yaml$ + encrypted_regex: "^(stringData|data)$" + age: >- + age1ql3z7hjy54pw3hyww5ayyfg7zqgvcnq76jwqgq3g20 + + - path_regex: .*/production/trading-secrets.yaml$ + encrypted_regex: "^(stringData|data)$" + age: >- + age1ql3z7hjy54pw3hyww5ayyfg7zqgvcnq76jwqgq3g20 diff --git a/trading-platform/helm/trading-platform/.sops.yaml b/trading-platform/helm/trading-platform/.sops.yaml new file mode 100644 index 0000000..35ae57e --- /dev/null +++ b/trading-platform/helm/trading-platform/.sops.yaml @@ -0,0 +1,19 @@ +# SOPS configuration for trading platform secrets +# Usage: sops -e -i trading-secrets.yaml && kubectl apply -f trading-secrets.yaml +creation_rules: + # Production secrets - encrypted with Age key + - path_regex: trading-secrets.yaml$ + encrypted_regex: "^(stringData|data)$" + age: >- + age1ql3z7hjy54pw3hyww5ayyfg7zqgvcnw3eks6lekrq733qkq76jwqgq3g20 + + # Per-environment overrides + - path_regex: .*/staging/trading-secrets.yaml$ + encrypted_regex: "^(stringData|data)$" + age: >- + age1ql3z7hjy54pw3hyww5ayyfg7zqgvcnq76jwqgq3g20 + + - path_regex: .*/production/trading-secrets.yaml$ + encrypted_regex: "^(stringData|data)$" + age: >- + age1ql3z7hjy54pw3hyww5ayyfg7zqgvcnq76jwqgq3g20 diff --git a/trading-platform/helm/trading-platform/Chart.yaml b/trading-platform/helm/trading-platform/Chart.yaml new file mode 100644 index 0000000..533fb8e --- /dev/null +++ b/trading-platform/helm/trading-platform/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +name: trading-platform +description: Helm chart for the DEFi trading platform microservices on GKE +type: application +version: 0.1.0 +appVersion: "0.1.0" +keywords: + - trading + - defi + - microservices +maintainers: + - name: Trading Platform Team diff --git a/trading-platform/helm/trading-platform/templates/NOTES.txt b/trading-platform/helm/trading-platform/templates/NOTES.txt new file mode 100644 index 0000000..afc553a --- /dev/null +++ b/trading-platform/helm/trading-platform/templates/NOTES.txt @@ -0,0 +1,29 @@ +Trading Platform deployed successfully! + +Namespace: {{ .Values.global.namespace }} +Release: {{ .Release.Name }} +Environment: {{ .Values.global.environment | default "not set" }} + +Services deployed: + - execute-service: {{ .Values.executeService.enabled }} + - news-service: {{ .Values.newsService.enabled }} + - data-service: {{ .Values.dataService.enabled }} + - dashboard: {{ .Values.dashboard.enabled }} + +Infrastructure: + - PostgreSQL: {{ .Values.postgres.enabled }} + - Redis: {{ .Values.redis.enabled }} + - Kafka: {{ .Values.kafka.enabled }} + +Ingress: {{ .Values.ingress.enabled }} +{{- if .Values.ingress.enabled }} +{{- range .Values.ingress.hosts }} + Host: {{ .host }} +{{- end }} +{{- end }} + +Next steps: +1. Verify pods: kubectl get pods -n {{ .Values.global.namespace }} +2. Check services: kubectl get svc -n {{ .Values.global.namespace }} +3. Check ingress: kubectl get ingress -n {{ .Values.global.namespace }} +4. If using SOPS, decrypt secrets: sops -d trading-secrets.yaml | kubectl apply -f - diff --git a/trading-platform/helm/trading-platform/templates/_helpers.tpl b/trading-platform/helm/trading-platform/templates/_helpers.tpl new file mode 100644 index 0000000..fa8c6d2 --- /dev/null +++ b/trading-platform/helm/trading-platform/templates/_helpers.tpl @@ -0,0 +1,94 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "trading-platform.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "trading-platform.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "trading-platform.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "trading-platform.labels" -}} +helm.sh/chart: {{ include "trading-platform.chart" . }} +{{ include "trading-platform.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "trading-platform.selectorLabels" -}} +app.kubernetes.io/name: {{ include "trading-platform.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Service-specific labels +*/}} +{{- define "trading-platform.serviceLabels" -}} +{{- $service := index . 0 }} +{{- $parent := index . 1 }} +{{ include "trading-platform.selectorLabels" $parent }} +app: {{ $service.Values.name | default $service.Values.name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "trading-platform.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "trading-platform.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Generate env var map from values +*/}} +{{- define "trading-platform.envVars" -}} +{{- range $key, $value := . }} +- name: {{ $key }} + value: {{ $value | quote }} +{{- end }} +{{- end }} + +{{/* +Generate secret env var refs +*/}} +{{- define "trading-platform.secretEnvVars" -}} +{{- range $key, $secretRef := . }} +- name: {{ $key }} + valueFrom: + secretKeyRef: + {{- $parts := split "/" $secretRef }} + name: {{ $parts._0 }} + key: {{ $parts._1 }} +{{- end }} +{{- end }} diff --git a/trading-platform/helm/trading-platform/templates/cert-manager/certificates.yaml b/trading-platform/helm/trading-platform/templates/cert-manager/certificates.yaml new file mode 100644 index 0000000..fb86f8c --- /dev/null +++ b/trading-platform/helm/trading-platform/templates/cert-manager/certificates.yaml @@ -0,0 +1,128 @@ +{{- if .Values.certManager.enabled }} +# External issuer (Let's Encrypt) for public-facing TLS +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: {{ .Values.certManager.externalIssuer.name }} +spec: + acme: + server: {{ .Values.certManager.externalIssuer.server }} + email: {{ .Values.certManager.externalIssuer.email }} + privateKeySecretRef: + name: {{ .Values.certManager.externalIssuer.name }}-key + solvers: + - http01: + ingress: + class: {{ .Values.ingress.className }} +--- +# Internal self-signed CA for service-to-service mTLS +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ .Values.certManager.internalIssuer.name }} + namespace: {{ .Values.global.namespace }} +spec: + selfSigned: {} +--- +# CA certificate issued by the self-signed issuer +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: trading-ca + namespace: {{ .Values.global.namespace }} +spec: + isCA: true + commonName: "trading-ca" + secretName: trading-ca-secret + privateKey: + algorithm: ECDSA + size: 256 + issuerRef: + name: {{ .Values.certManager.internalIssuer.name }} + kind: Issuer + group: cert-manager.io +--- +# CA issuer for signing service certificates +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: trading-ca-issuer + namespace: {{ .Values.global.namespace }} +spec: + ca: + secretName: trading-ca-secret +--- +# mTLS certificate for execute-service +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: execute-service-mtls + namespace: {{ .Values.global.namespace }} +spec: + dnsNames: + - execute-service + - execute-service.{{ .Values.global.namespace }}.svc.cluster.local + secretName: execute-service-mtls + privateKey: + algorithm: ECDSA + size: 256 + issuerRef: + name: trading-ca-issuer + kind: Issuer + group: cert-manager.io +--- +# mTLS certificate for news-service +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: news-service-mtls + namespace: {{ .Values.global.namespace }} +spec: + dnsNames: + - news-service + - news-service.{{ .Values.global.namespace }}.svc.cluster.local + secretName: news-service-mtls + privateKey: + algorithm: ECDSA + size: 256 + issuerRef: + name: trading-ca-issuer + kind: Issuer + group: cert-manager.io +--- +# mTLS certificate for data-service +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: data-service-mtls + namespace: {{ .Values.global.namespace }} +spec: + dnsNames: + - data-service + - data-service.{{ .Values.global.namespace }}.svc.cluster.local + secretName: data-service-mtls + privateKey: + algorithm: ECDSA + size: 256 + issuerRef: + name: trading-ca-issuer + kind: Issuer + group: cert-manager.io +--- +# TLS certificate for the public domain (ingress) +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: trading-tls + namespace: {{ .Values.global.namespace }} +spec: + secretName: trading-tls + dnsNames: + {{- range $host := .Values.ingress.hosts }} + - {{ $host.host }} + {{- end }} + issuerRef: + name: {{ .Values.certManager.externalIssuer.name }} + kind: ClusterIssuer + group: cert-manager.io +{{- end }} diff --git a/trading-platform/helm/trading-platform/templates/configmap.yaml b/trading-platform/helm/trading-platform/templates/configmap.yaml new file mode 100644 index 0000000..1e1ce4e --- /dev/null +++ b/trading-platform/helm/trading-platform/templates/configmap.yaml @@ -0,0 +1,21 @@ +# Shared ConfigMap for trading platform configuration +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "trading-platform.fullname" . }}-config + namespace: {{ .Values.global.namespace }} + labels: + {{- include "trading-platform.labels" . | nindent 4 }} +data: + # Shared environment configuration + ENVIRONMENT: {{ .Values.global.environment | quote }} + CLUSTER_NAME: {{ .Values.global.clusterName | quote }} + NAMESPACE: {{ .Values.global.namespace | quote }} + # Kafka bootstrap (internal DNS) + KAFKA_BOOTSTRAP: kafka-headless.{{ .Values.global.namespace }}.svc.cluster.local:9092 + # Redis connection + REDIS_HOST: redis-master.{{ .Values.global.namespace }}.svc.cluster.local + REDIS_PORT: "6379" + # PostgreSQL connection + POSTGRES_HOST: postgres-primary.{{ .Values.global.namespace }}.svc.cluster.local + POSTGRES_PORT: "5432" diff --git a/trading-platform/helm/trading-platform/templates/dashboard/deployment.yaml b/trading-platform/helm/trading-platform/templates/dashboard/deployment.yaml new file mode 100644 index 0000000..aeffdd0 --- /dev/null +++ b/trading-platform/helm/trading-platform/templates/dashboard/deployment.yaml @@ -0,0 +1,64 @@ +{{- if .Values.dashboard.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.dashboard.name }} + namespace: {{ .Values.global.namespace }} + labels: + {{- include "trading-platform.labels" . | nindent 4 }} + app: {{ .Values.dashboard.name }} +spec: + {{- if not .Values.dashboard.autoscaling.enabled }} + replicas: {{ .Values.global.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "trading-platform.selectorLabels" . | nindent 6 }} + app: {{ .Values.dashboard.name }} + template: + metadata: + labels: + {{- include "trading-platform.selectorLabels" . | nindent 8 }} + app: {{ .Values.dashboard.name }} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "{{ .Values.dashboard.port }}" + spec: + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Values.dashboard.name }} + image: "{{ .Values.dashboard.image.repository }}:{{ .Values.dashboard.image.tag }}" + imagePullPolicy: {{ .Values.dashboard.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.dashboard.port }} + protocol: TCP + env: + {{- range $key, $value := .Values.dashboard.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + resources: + {{- toYaml .Values.dashboard.resources | nindent 12 }} + securityContext: + {{- toYaml .Values.containerSecurityContext | nindent 12 }} + livenessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 15 + periodSeconds: 5 + timeoutSeconds: 3 + {{- with .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/trading-platform/helm/trading-platform/templates/dashboard/service.yaml b/trading-platform/helm/trading-platform/templates/dashboard/service.yaml new file mode 100644 index 0000000..3b11066 --- /dev/null +++ b/trading-platform/helm/trading-platform/templates/dashboard/service.yaml @@ -0,0 +1,40 @@ +{{- if .Values.dashboard.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.dashboard.name }} + namespace: {{ .Values.global.namespace }} + labels: + app: {{ .Values.dashboard.name }} +spec: + type: ClusterIP + ports: + - port: {{ .Values.dashboard.port }} + targetPort: http + protocol: TCP + name: http + selector: + app: {{ .Values.dashboard.name }} +--- +{{- if .Values.dashboard.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ .Values.dashboard.name }} + namespace: {{ .Values.global.namespace }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ .Values.dashboard.name }} + minReplicas: {{ .Values.dashboard.autoscaling.minReplicas }} + maxReplicas: {{ .Values.dashboard.autoscaling.maxReplicas }} + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.dashboard.autoscaling.targetCPUUtilizationPercentage }} +{{- end }} +{{- end }} diff --git a/trading-platform/helm/trading-platform/templates/data-service/deployment.yaml b/trading-platform/helm/trading-platform/templates/data-service/deployment.yaml new file mode 100644 index 0000000..0463b4a --- /dev/null +++ b/trading-platform/helm/trading-platform/templates/data-service/deployment.yaml @@ -0,0 +1,89 @@ +{{- if .Values.dataService.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.dataService.name }} + namespace: {{ .Values.global.namespace }} + labels: + {{- include "trading-platform.labels" . | nindent 4 }} + app: {{ .Values.dataService.name }} +spec: + {{- if not .Values.dataService.autoscaling.enabled }} + replicas: {{ .Values.global.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "trading-platform.selectorLabels" . | nindent 6 }} + app: {{ .Values.dataService.name }} + template: + metadata: + labels: + {{- include "trading-platform.selectorLabels" . | nindent 8 }} + app: {{ .Values.dataService.name }} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "{{ .Values.dataService.port }}" + spec: + serviceAccountName: {{ .Values.dataService.name }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Values.dataService.name }} + image: "{{ .Values.dataService.image.repository }}:{{ .Values.dataService.image.tag }}" + imagePullPolicy: {{ .Values.dataService.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.dataService.port }} + protocol: TCP + env: + {{- range $key, $value := .Values.dataService.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + - name: DB_HOST + valueFrom: + configMapKeyRef: + name: {{ include "trading-platform.fullname" . }}-config + key: POSTGRES_HOST + - name: REDIS_HOST + valueFrom: + configMapKeyRef: + name: {{ include "trading-platform.fullname" . }}-config + key: REDIS_HOST + - name: KAFKA_BOOTSTRAP_SERVERS + valueFrom: + configMapKeyRef: + name: {{ include "trading-platform.fullname" . }}-config + key: KAFKA_BOOTSTRAP + envFrom: + - secretRef: + name: trading-secrets + resources: + {{- toYaml .Values.dataService.resources | nindent 12 }} + securityContext: + {{- toYaml .Values.containerSecurityContext | nindent 12 }} + volumeMounts: + - name: tmp + mountPath: /tmp + livenessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + volumes: + - name: tmp + emptyDir: {} + {{- with .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/trading-platform/helm/trading-platform/templates/data-service/service.yaml b/trading-platform/helm/trading-platform/templates/data-service/service.yaml new file mode 100644 index 0000000..73dd9f0 --- /dev/null +++ b/trading-platform/helm/trading-platform/templates/data-service/service.yaml @@ -0,0 +1,48 @@ +{{- if .Values.dataService.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.dataService.name }} + namespace: {{ .Values.global.namespace }} + labels: + app: {{ .Values.dataService.name }} +spec: + type: ClusterIP + ports: + - port: {{ .Values.dataService.port }} + targetPort: http + protocol: TCP + name: http + selector: + app: {{ .Values.dataService.name }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.dataService.name }} + namespace: {{ .Values.global.namespace }} + labels: + app: {{ .Values.dataService.name }} +--- +{{- if .Values.dataService.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ .Values.dataService.name }} + namespace: {{ .Values.global.namespace }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ .Values.dataService.name }} + minReplicas: {{ .Values.dataService.autoscaling.minReplicas }} + maxReplicas: {{ .Values.dataService.autoscaling.maxReplicas }} + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.dataService.autoscaling.targetCPUUtilizationPercentage }} +{{- end }} +{{- end }} diff --git a/trading-platform/helm/trading-platform/templates/execute-service/deployment.yaml b/trading-platform/helm/trading-platform/templates/execute-service/deployment.yaml new file mode 100644 index 0000000..fcb5d43 --- /dev/null +++ b/trading-platform/helm/trading-platform/templates/execute-service/deployment.yaml @@ -0,0 +1,112 @@ +{{- if .Values.executeService.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.executeService.name }} + namespace: {{ .Values.global.namespace }} + labels: + {{- include "trading-platform.labels" . | nindent 4 }} + app: {{ .Values.executeService.name }} +spec: + {{- if not .Values.executeService.autoscaling.enabled }} + replicas: {{ .Values.global.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "trading-platform.selectorLabels" . | nindent 6 }} + app: {{ .Values.executeService.name }} + template: + metadata: + labels: + {{- include "trading-platform.selectorLabels" . | nindent 8 }} + app: {{ .Values.executeService.name }} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "{{ .Values.executeService.port }}" + spec: + serviceAccountName: {{ .Values.executeService.name }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Values.executeService.name }} + image: "{{ .Values.executeService.image.repository }}:{{ .Values.executeService.image.tag }}" + imagePullPolicy: {{ .Values.executeService.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.executeService.port }} + protocol: TCP + env: + {{- range $key, $value := .Values.executeService.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + # Shared config from ConfigMap + - name: DB_HOST + valueFrom: + configMapKeyRef: + name: {{ include "trading-platform.fullname" . }}-config + key: POSTGRES_HOST + - name: REDIS_HOST + valueFrom: + configMapKeyRef: + name: {{ include "trading-platform.fullname" . }}-config + key: REDIS_HOST + - name: KAFKA_BOOTSTRAP_SERVERS + valueFrom: + configMapKeyRef: + name: {{ include "trading-platform.fullname" . }}-config + key: KAFKA_BOOTSTRAP + envFrom: + - secretRef: + name: trading-secrets + resources: + {{- toYaml .Values.executeService.resources | nindent 12 }} + securityContext: + {{- toYaml .Values.containerSecurityContext | nindent 12 }} + volumeMounts: + - name: tmp + mountPath: /tmp + - name: mtls-certs + mountPath: /etc/mtls + readOnly: true + livenessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + volumes: + - name: tmp + emptyDir: {} + - name: mtls-certs + secret: + secretName: execute-service-mtls + {{- with .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + nodeSelector: + kubernetes.io/os: linux + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - {{ .Values.executeService.name }} + topologyKey: kubernetes.io/hostname +{{- end }} diff --git a/trading-platform/helm/trading-platform/templates/execute-service/service.yaml b/trading-platform/helm/trading-platform/templates/execute-service/service.yaml new file mode 100644 index 0000000..575ae85 --- /dev/null +++ b/trading-platform/helm/trading-platform/templates/execute-service/service.yaml @@ -0,0 +1,50 @@ +{{- if .Values.executeService.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.executeService.name }} + namespace: {{ .Values.global.namespace }} + labels: + app: {{ .Values.executeService.name }} +spec: + type: ClusterIP + ports: + - port: {{ .Values.executeService.port }} + targetPort: http + protocol: TCP + name: http + selector: + app: {{ .Values.executeService.name }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.executeService.name }} + namespace: {{ .Values.global.namespace }} + labels: + app: {{ .Values.executeService.name }} + annotations: + iam.gke.io/gcp-service-account: execute-service@{{ .Values.global.clusterName }}.iam.gserviceaccount.com +--- +{{- if .Values.executeService.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ .Values.executeService.name }} + namespace: {{ .Values.global.namespace }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ .Values.executeService.name }} + minReplicas: {{ .Values.executeService.autoscaling.minReplicas }} + maxReplicas: {{ .Values.executeService.autoscaling.maxReplicas }} + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.executeService.autoscaling.targetCPUUtilizationPercentage }} +{{- end }} +{{- end }} diff --git a/trading-platform/helm/trading-platform/templates/infrastructure/kafka.yaml b/trading-platform/helm/trading-platform/templates/infrastructure/kafka.yaml new file mode 100644 index 0000000..cccd908 --- /dev/null +++ b/trading-platform/helm/trading-platform/templates/infrastructure/kafka.yaml @@ -0,0 +1,142 @@ +{{- if .Values.kafka.enabled }} +# Headless service for KRaft discovery +apiVersion: v1 +kind: Service +metadata: + name: kafka-headless + namespace: {{ .Values.global.namespace }} + labels: + app: kafka +spec: + ports: + - port: 9092 + targetPort: 9092 + name: kafka + - port: 9093 + targetPort: 9093 + name: controller + clusterIP: None + selector: + app: kafka +--- +# Client-facing service +apiVersion: v1 +kind: Service +metadata: + name: kafka + namespace: {{ .Values.global.namespace }} + labels: + app: kafka +spec: + type: ClusterIP + ports: + - port: 9092 + targetPort: 9092 + name: kafka + selector: + app: kafka +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: kafka + namespace: {{ .Values.global.namespace }} + labels: + app: kafka +spec: + serviceName: kafka-headless + replicas: {{ .Values.kafka.replicaCount }} + selector: + matchLabels: + app: kafka + template: + metadata: + labels: + app: kafka + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + containers: + - name: kafka + image: "{{ .Values.kafka.image.repository }}:{{ .Values.kafka.image.tag }}" + ports: + - containerPort: 9092 + name: kafka + - containerPort: 9093 + name: controller + env: + - name: KAFKA_NODE_ID + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KAFKA_PROCESS_ROLES + value: "broker,controller" + - name: KAFKA_CONTROLLER_QUORUM_VOTERS + value: | + {{- $ns := .Values.global.namespace }} + {{- range $i := until (int .Values.kafka.replicaCount) }} + {{- $i }}@kafka-{{ $i }}.kafka-headless.{{ $ns }}.svc.cluster.local:9093{{ if lt (add $i 1) (int $.Values.kafka.replicaCount) }},{{ end }} + {{- end }} + - name: KAFKA_CONTROLLER_LISTENER_NAMES + value: "CONTROLLER" + - name: KAFKA_LISTENERS + value: "PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093" + - name: KAFKA_ADVERTISED_LISTENERS + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP + value: "PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT" + - name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR + value: "{{ .Values.kafka.replicaCount }}" + - name: KAFKA_TRANSACTION_STATE_LOG_MIN_ISR + value: "{{ add1 (div (int .Values.kafka.replicaCount) 2) }}" + - name: KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR + value: "{{ .Values.kafka.replicaCount }}" + - name: KAFKA_AUTO_CREATE_TOPICS_ENABLE + value: "true" + - name: KAFKA_NUM_PARTITIONS + value: "6" + - name: KAFKA_DEFAULT_REPLICATION_FACTOR + value: "{{ .Values.kafka.replicaCount }}" + - name: KAFKA_LOG_DIRS + value: "/var/lib/kafka/data" + resources: + {{- toYaml .Values.kafka.resources | nindent 12 }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: kafka-data + mountPath: /var/lib/kafka/data + - name: tmp + mountPath: /tmp + livenessProbe: + tcpSocket: + port: kafka + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + readinessProbe: + tcpSocket: + port: kafka + initialDelaySeconds: 20 + periodSeconds: 5 + timeoutSeconds: 3 + volumes: + - name: tmp + emptyDir: {} + volumeClaimTemplates: + - metadata: + name: kafka-data + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: {{ .Values.kafka.storage.storageClass }} + resources: + requests: + storage: {{ .Values.kafka.storage.size }} +{{- end }} diff --git a/trading-platform/helm/trading-platform/templates/infrastructure/postgres.yaml b/trading-platform/helm/trading-platform/templates/infrastructure/postgres.yaml new file mode 100644 index 0000000..8567095 --- /dev/null +++ b/trading-platform/helm/trading-platform/templates/infrastructure/postgres.yaml @@ -0,0 +1,111 @@ +{{- if .Values.postgres.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: postgres-primary + namespace: {{ .Values.global.namespace }} + labels: + app: postgres +spec: + type: ClusterIP + ports: + - port: 5432 + targetPort: postgres + protocol: TCP + name: postgres + selector: + app: postgres + role: primary +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgres + namespace: {{ .Values.global.namespace }} + labels: + app: postgres +spec: + serviceName: postgres-primary + replicas: {{ .Values.postgres.primary.replicaCount }} + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + role: primary + spec: + securityContext: + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + fsGroup: 999 + containers: + - name: postgres + image: "{{ .Values.postgres.image.repository }}:{{ .Values.postgres.image.tag }}" + ports: + - containerPort: 5432 + name: postgres + env: + - name: POSTGRES_USER + value: "trading" + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: trading-secrets + key: POSTGRES_PASSWORD + - name: POSTGRES_DB + value: {{ .Values.postgres.postgresqlDatabase | quote }} + resources: + {{- toYaml .Values.postgres.primary.resources | nindent 12 }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: postgres-data + mountPath: /var/lib/postgresql/data + - name: tmp + mountPath: /tmp + - name: run + mountPath: /var/run + livenessProbe: + exec: + command: + - pg_isready + - -U + - trading + - -d + - {{ .Values.postgres.postgresqlDatabase }} + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + readinessProbe: + exec: + command: + - pg_isready + - -U + - trading + - -d + - {{ .Values.postgres.postgresqlDatabase }} + initialDelaySeconds: 15 + periodSeconds: 5 + timeoutSeconds: 3 + volumes: + - name: tmp + emptyDir: {} + - name: run + emptyDir: {} + volumeClaimTemplates: + - metadata: + name: postgres-data + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: {{ .Values.postgres.storage.storageClass }} + resources: + requests: + storage: {{ .Values.postgres.storage.size }} +{{- end }} diff --git a/trading-platform/helm/trading-platform/templates/infrastructure/redis.yaml b/trading-platform/helm/trading-platform/templates/infrastructure/redis.yaml new file mode 100644 index 0000000..52032a0 --- /dev/null +++ b/trading-platform/helm/trading-platform/templates/infrastructure/redis.yaml @@ -0,0 +1,107 @@ +{{- if .Values.redis.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: redis-master + namespace: {{ .Values.global.namespace }} + labels: + app: redis +spec: + type: ClusterIP + ports: + - port: 6379 + targetPort: redis + protocol: TCP + name: redis + selector: + app: redis +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis + namespace: {{ .Values.global.namespace }} + labels: + app: redis +spec: + serviceName: redis-master + replicas: {{ .Values.redis.replicaCount }} + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis + spec: + securityContext: + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + fsGroup: 999 + containers: + - name: redis + image: "{{ .Values.redis.image.repository }}:{{ .Values.redis.image.tag }}" + command: + - redis-server + - --appendonly + - "yes" + - --save + - "" + - --requirepass + - $(REDIS_PASSWORD) + ports: + - containerPort: 6379 + name: redis + env: + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: trading-secrets + key: REDIS_PASSWORD + resources: + {{- toYaml .Values.redis.master.resources | nindent 12 }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: redis-data + mountPath: /data + - name: tmp + mountPath: /tmp + livenessProbe: + exec: + command: + - redis-cli + - -a + - $(REDIS_PASSWORD) + - ping + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 5 + readinessProbe: + exec: + command: + - redis-cli + - -a + - $(REDIS_PASSWORD) + - ping + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + volumes: + - name: tmp + emptyDir: {} + volumeClaimTemplates: + - metadata: + name: redis-data + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: {{ .Values.redis.master.storage.storageClass }} + resources: + requests: + storage: {{ .Values.redis.master.storage.size }} +{{- end }} diff --git a/trading-platform/helm/trading-platform/templates/ingress.yaml b/trading-platform/helm/trading-platform/templates/ingress.yaml new file mode 100644 index 0000000..965425d --- /dev/null +++ b/trading-platform/helm/trading-platform/templates/ingress.yaml @@ -0,0 +1,48 @@ +{{- if .Values.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "trading-platform.fullname" . }} + namespace: {{ .Values.global.namespace }} + labels: + {{- include "trading-platform.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if .Values.ingress.className }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + rules: + {{- range $host := .Values.ingress.hosts }} + - host: {{ $host.host | quote }} + http: + paths: + {{- range $path := $host.paths }} + - path: {{ $path.path }} + pathType: {{ $path.pathType }} + backend: + service: + name: {{ $path.service }} + port: + {{- if eq $path.service "dashboard" }} + number: {{ $.Values.dashboard.port }} + {{- else if eq $path.service "execute-service" }} + number: {{ $.Values.executeService.port }} + {{- else if eq $path.service "news-service" }} + number: {{ $.Values.newsService.port }} + {{- else if eq $path.service "data-service" }} + number: {{ $.Values.dataService.port }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/trading-platform/helm/trading-platform/templates/namespace.yaml b/trading-platform/helm/trading-platform/templates/namespace.yaml new file mode 100644 index 0000000..c0c9257 --- /dev/null +++ b/trading-platform/helm/trading-platform/templates/namespace.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Values.global.namespace }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- if .Values.global.environment }} + environment: {{ .Values.global.environment | quote }} + {{- end }} + {{- if .Values.global.clusterName }} + cluster: {{ .Values.global.clusterName | quote }} + {{- end }} diff --git a/trading-platform/helm/trading-platform/templates/network-policies/network-policies.yaml b/trading-platform/helm/trading-platform/templates/network-policies/network-policies.yaml new file mode 100644 index 0000000..76fb484 --- /dev/null +++ b/trading-platform/helm/trading-platform/templates/network-policies/network-policies.yaml @@ -0,0 +1,311 @@ +{{- if .Values.networkPolicies.enabled }} +{{- if .Values.networkPolicies.defaultDenyIngress }} +# Deny all ingress traffic by default +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-deny-all-ingress + namespace: {{ .Values.global.namespace }} + labels: + app.kubernetes.io/managed-by: security-audit +spec: + podSelector: {} + policyTypes: + - Ingress +--- +{{- end }} +{{- if .Values.networkPolicies.defaultDenyEgress }} +# Deny all egress traffic by default +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-deny-all-egress + namespace: {{ .Values.global.namespace }} + labels: + app.kubernetes.io/managed-by: security-audit +spec: + podSelector: {} + policyTypes: + - Egress +--- +{{- end }} +{{- if .Values.networkPolicies.allowDNS }} +# Allow all pods to resolve DNS +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-dns-egress + namespace: {{ .Values.global.namespace }} +spec: + podSelector: {} + policyTypes: + - Egress + egress: + - to: [] + ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP +--- +{{- end }} +# Execute Service networking +{{- if .Values.executeService.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: execute-service-networking + namespace: {{ .Values.global.namespace }} +spec: + podSelector: + matchLabels: + app: execute-service + policyTypes: + - Ingress + - Egress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: ingress-nginx + podSelector: + matchLabels: + app.kubernetes.io/name: ingress-nginx + - namespaceSelector: {} + podSelector: + matchLabels: + app.kubernetes.io/name: gce-proxy + ports: + - port: {{ .Values.executeService.port }} + protocol: TCP + - from: + - podSelector: + matchLabels: + app: news-service + ports: + - port: {{ .Values.executeService.port }} + protocol: TCP + egress: + - to: + - podSelector: + matchLabels: + app: postgres + ports: + - port: 5432 + protocol: TCP + - to: + - podSelector: + matchLabels: + app: redis + ports: + - port: 6379 + protocol: TCP + - to: + - podSelector: + matchLabels: + app: kafka + ports: + - port: 9092 + protocol: TCP +--- +{{- end }} +# News Service networking +{{- if .Values.newsService.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: news-service-networking + namespace: {{ .Values.global.namespace }} +spec: + podSelector: + matchLabels: + app: news-service + policyTypes: + - Ingress + - Egress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: ingress-nginx + podSelector: + matchLabels: + app.kubernetes.io/name: ingress-nginx + ports: + - port: {{ .Values.newsService.port }} + protocol: TCP + egress: + - to: + - podSelector: + matchLabels: + cnpg.io/cluster: customer1-cnpg + ports: + - port: 5432 + protocol: TCP + - to: + - podSelector: + matchLabels: + app: kafka + ports: + - port: 9092 + protocol: TCP + - to: + - podSelector: + matchLabels: + app: execute-service + ports: + - port: {{ .Values.executeService.port }} + protocol: TCP +--- +{{- end }} +# Database networking +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: postgres-networking + namespace: {{ .Values.global.namespace }} +spec: + podSelector: + matchLabels: + app: postgres + policyTypes: + - Ingress + - Egress + ingress: + - from: + - podSelector: + matchLabels: + app: execute-service + - podSelector: + matchLabels: + app: data-service + ports: + - port: 5432 + protocol: TCP + egress: [] +--- +# Redis networking +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: redis-networking + namespace: {{ .Values.global.namespace }} +spec: + podSelector: + matchLabels: + app: redis + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + app: execute-service + - podSelector: + matchLabels: + app: data-service + ports: + - port: 6379 + protocol: TCP +--- +# Kafka networking +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: kafka-networking + namespace: {{ .Values.global.namespace }} +spec: + podSelector: + matchLabels: + app: kafka + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + app: execute-service + - podSelector: + matchLabels: + app: news-service + - podSelector: + matchLabels: + app: data-service + ports: + - port: 9092 + protocol: TCP + - port: 9093 + protocol: TCP +--- +# Dashboard networking +{{- if .Values.dashboard.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: dashboard-networking + namespace: {{ .Values.global.namespace }} +spec: + podSelector: + matchLabels: + app: dashboard + policyTypes: + - Ingress + - Egress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: ingress-nginx + ports: + - port: {{ .Values.dashboard.port }} + protocol: TCP + egress: [] +--- +{{- end }} +# Data Service networking +{{- if .Values.dataService.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: data-service-networking + namespace: {{ .Values.global.namespace }} +spec: + podSelector: + matchLabels: + app: data-service + policyTypes: + - Ingress + - Egress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: ingress-nginx + ports: + - port: {{ .Values.dataService.port }} + protocol: TCP + egress: + - to: + - podSelector: + matchLabels: + app: postgres + ports: + - port: 5432 + protocol: TCP + - to: + - podSelector: + matchLabels: + app: redis + ports: + - port: 6379 + protocol: TCP + - to: + - podSelector: + matchLabels: + app: kafka + ports: + - port: 9092 + protocol: TCP +--- +{{- end }} +{{- end }} diff --git a/trading-platform/helm/trading-platform/templates/news-service/deployment.yaml b/trading-platform/helm/trading-platform/templates/news-service/deployment.yaml new file mode 100644 index 0000000..8ac48e2 --- /dev/null +++ b/trading-platform/helm/trading-platform/templates/news-service/deployment.yaml @@ -0,0 +1,85 @@ +{{- if .Values.newsService.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.newsService.name }} + namespace: {{ .Values.global.namespace }} + labels: + {{- include "trading-platform.labels" . | nindent 4 }} + app: {{ .Values.newsService.name }} +spec: + {{- if not .Values.newsService.autoscaling.enabled }} + replicas: {{ .Values.global.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "trading-platform.selectorLabels" . | nindent 6 }} + app: {{ .Values.newsService.name }} + template: + metadata: + labels: + {{- include "trading-platform.selectorLabels" . | nindent 8 }} + app: {{ .Values.newsService.name }} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "{{ .Values.newsService.port }}" + spec: + serviceAccountName: {{ .Values.newsService.name }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Values.newsService.name }} + image: "{{ .Values.newsService.image.repository }}:{{ .Values.newsService.image.tag }}" + imagePullPolicy: {{ .Values.newsService.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.newsService.port }} + protocol: TCP + env: + {{- range $key, $value := .Values.newsService.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + - name: KAFKA_BOOTSTRAP_SERVERS + valueFrom: + configMapKeyRef: + name: {{ include "trading-platform.fullname" . }}-config + key: KAFKA_BOOTSTRAP + envFrom: + - secretRef: + name: trading-secrets + resources: + {{- toYaml .Values.newsService.resources | nindent 12 }} + securityContext: + {{- toYaml .Values.containerSecurityContext | nindent 12 }} + volumeMounts: + - name: tmp + mountPath: /tmp + - name: mtls-certs + mountPath: /etc/mtls + readOnly: true + livenessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + volumes: + - name: tmp + emptyDir: {} + - name: mtls-certs + secret: + secretName: news-service-mtls + {{- with .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/trading-platform/helm/trading-platform/templates/news-service/service.yaml b/trading-platform/helm/trading-platform/templates/news-service/service.yaml new file mode 100644 index 0000000..d24f4eb --- /dev/null +++ b/trading-platform/helm/trading-platform/templates/news-service/service.yaml @@ -0,0 +1,48 @@ +{{- if .Values.newsService.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.newsService.name }} + namespace: {{ .Values.global.namespace }} + labels: + app: {{ .Values.newsService.name }} +spec: + type: ClusterIP + ports: + - port: {{ .Values.newsService.port }} + targetPort: http + protocol: TCP + name: http + selector: + app: {{ .Values.newsService.name }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.newsService.name }} + namespace: {{ .Values.global.namespace }} + labels: + app: {{ .Values.newsService.name }} +--- +{{- if .Values.newsService.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ .Values.newsService.name }} + namespace: {{ .Values.global.namespace }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ .Values.newsService.name }} + minReplicas: {{ .Values.newsService.autoscaling.minReplicas }} + maxReplicas: {{ .Values.newsService.autoscaling.maxReplicas }} + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.newsService.autoscaling.targetCPUUtilizationPercentage }} +{{- end }} +{{- end }} diff --git a/trading-platform/helm/trading-platform/templates/secrets.yaml b/trading-platform/helm/trading-platform/templates/secrets.yaml new file mode 100644 index 0000000..9eaddbe --- /dev/null +++ b/trading-platform/helm/trading-platform/templates/secrets.yaml @@ -0,0 +1,45 @@ +# Trading Secrets - Encrypt with SOPS before applying +# Usage: sops -e -i trading-secrets.yaml && kubectl apply -f trading-secrets.yaml +apiVersion: v1 +kind: Secret +metadata: + name: trading-secrets + namespace: {{ .Values.global.namespace }} + labels: + {{- include "trading-platform.labels" . | nindent 4 }} + app.kubernetes.io/managed-by: security-audit + annotations: + # SOPS manages this file - edit with: sops trading-secrets.yaml + secrets.yaml.sops.io/v2: encrypted +type: Opaque +stringData: + # JWT authentication + EXECUTE_JWT_SECRET_KEY: {{ .Values.executeService.env.JWT_SECRET_KEY | default "" | quote }} + + # Exchange private keys + EXECUTE_HYPERLIQUID_PRIVATE_KEY: {{ .Values.executeService.env.HYPERLIQUID_PRIVATE_KEY | default "" | quote }} + EXECUTE_SOLANA_PRIVATE_KEY_BASE58: {{ .Values.executeService.env.SOLANA_PRIVATE_KEY_BASE58 | default "" | quote }} + + # Database credentials + POSTGRES_PASSWORD: {{ .Values.postgres.postgresqlPassword | default "" | quote }} + NEWS_DB_PASSWORD: {{ .Values.newsService.env.NEWS_DB_PASSWORD | default "" | quote }} + + # Redis + REDIS_PASSWORD: {{ .Values.redis.redisPassword | default "" | quote }} + + # Kafka SASL + KAFKA_SASL_USERNAME: {{ .Values.kafka.saslUsername | default "" | quote }} + KAFKA_SASL_PASSWORD: {{ .Values.kafka.saslPassword | default "" | quote }} + +--- +# mTLS certificates (generated by cert-manager, mounted as volumes) +# Service account tokens for SPIFFE/SPIRE identity +apiVersion: v1 +kind: Secret +metadata: + name: trading-mtls-certs + namespace: {{ .Values.global.namespace }} + labels: + {{- include "trading-platform.labels" . | nindent 4 }} +type: kubernetes.io/tls +# tls.crt and tls.key populated by cert-manager diff --git a/trading-platform/helm/trading-platform/trading-secrets.yaml b/trading-platform/helm/trading-platform/trading-secrets.yaml new file mode 100644 index 0000000..e74f932 --- /dev/null +++ b/trading-platform/helm/trading-platform/trading-secrets.yaml @@ -0,0 +1,33 @@ +# Trading Secrets - Encrypt with SOPS before applying +# Usage: sops -e -i trading-secrets.yaml && kubectl apply -f trading-secrets.yaml +# +# DO NOT commit unencrypted secrets to git! +# Generate secrets with: python -c "import secrets; print(secrets.token_urlsafe(24))" +apiVersion: v1 +kind: Secret +metadata: + name: trading-secrets + namespace: trading + labels: + app.kubernetes.io/managed-by: security-audit + annotations: + secrets.yaml.sops.io/v2: encrypted +type: Opaque +stringData: + # JWT authentication - use 32+ byte hex token + EXECUTE_JWT_SECRET_KEY: "" + + # Exchange private keys - NEVER commit plaintext! + EXECUTE_HYPERLIQUID_PRIVATE_KEY: "" + EXECUTE_SOLANA_PRIVATE_KEY_BASE58: "" + + # Database credentials - use 24+ byte URL-safe tokens + POSTGRES_PASSWORD: "" + NEWS_DB_PASSWORD: "" + + # Redis password + REDIS_PASSWORD: "" + + # Kafka SASL authentication + KAFKA_SASL_USERNAME: "" + KAFKA_SASL_PASSWORD: "" diff --git a/trading-platform/helm/trading-platform/values.yaml b/trading-platform/helm/trading-platform/values.yaml new file mode 100644 index 0000000..faeee27 --- /dev/null +++ b/trading-platform/helm/trading-platform/values.yaml @@ -0,0 +1,306 @@ +# Trading Platform Helm Chart - Default Values +# ============================================================ + +# Global settings +global: + namespace: customer1 + replicaCount: 2 + imagePullSecrets: [] + clusterName: customer1-gke + environment: staging + registry: us-central1-docker.pkg.dev/customer1-gke/trading + +# Resource defaults for all microservices +resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: "1" + memory: 512Mi + +# Pod security context defaults +podSecurityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + seccompProfile: + type: RuntimeDefault + +# Container security context defaults +containerSecurityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + +# ================================================================= +# Execute Service (Hyperliquid + Solana trading engine) +# ================================================================= +executeService: + enabled: true + name: execute-service + image: + repository: "" + tag: "0.1.0" + pullPolicy: IfNotPresent + port: 8000 + resources: + requests: + cpu: 500m + memory: 512Mi + limits: + cpu: "2" + memory: 1Gi + autoscaling: + enabled: true + minReplicas: 2 + maxReplicas: 10 + targetCPUUtilizationPercentage: 70 + env: + LOG_LEVEL: "INFO" + KAFKA_BOOTSTRAP_SERVERS: "kafka-headless:9092" + KAFKA_TOPIC_TRADES: "trades.executed" + KAFKA_TOPIC_ORDERS: "orders.new" + REDIS_HOST: "redis-master" + REDIS_PORT: "6379" + DB_HOST: "postgres-primary" + DB_PORT: "5432" + DB_NAME: "trading_db" + +# ================================================================= +# News Service (CNPG connector + Kafka producer) +# ================================================================= +newsService: + enabled: true + name: news-service + image: + repository: "" + tag: "0.1.0" + pullPolicy: IfNotPresent + port: 8000 + resources: + requests: + cpu: 300m + memory: 384Mi + limits: + cpu: "1" + memory: 768Mi + autoscaling: + enabled: true + minReplicas: 2 + maxReplicas: 6 + targetCPUUtilizationPercentage: 75 + env: + LOG_LEVEL: "INFO" + DB_POOL_SIZE: "5" + DB_MAX_OVERFLOW: "10" + KAFKA_BOOTSTRAP_SERVERS: "kafka-headless:9092" + KAFKA_GROUP_ID: "news-analyzer" + KAFKA_TOPIC_ARTICLES: "news.articles" + KAFKA_TOPIC_ANALYSIS: "news.analysis" + API_PREFIX: "/api/v1" + CORS_ORIGINS: '["https://app.customer1.trading.com"]' + DB_HOST: "customer1-cnpg-r" + DB_PORT: "5432" + DB_NAME: "news" + +# ================================================================= +# Data Service (Postgres + Redis + Kafka consumers) +# ================================================================= +dataService: + enabled: true + name: data-service + image: + repository: "" + tag: "0.1.0" + pullPolicy: IfNotPresent + port: 8000 + resources: + requests: + cpu: 300m + memory: 384Mi + limits: + cpu: "1" + memory: 768Mi + autoscaling: + enabled: true + minReplicas: 2 + maxReplicas: 6 + targetCPUUtilizationPercentage: 70 + env: + LOG_LEVEL: "INFO" + DB_HOST: "postgres-primary" + DB_PORT: "5432" + DB_NAME: "trading_db" + REDIS_HOST: "redis-master" + REDIS_PORT: "6379" + KAFKA_BOOTSTRAP_SERVERS: "kafka-headless:9092" + KAFKA_CONSUMER_GROUP: "data-service" + +# ================================================================= +# Dashboard (Next.js frontend) +# ================================================================= +dashboard: + enabled: true + name: dashboard + image: + repository: "" + tag: "0.1.0" + pullPolicy: IfNotPresent + port: 3000 + resources: + requests: + cpu: 200m + memory: 256Mi + limits: + cpu: "500m" + memory: 512Mi + autoscaling: + enabled: true + minReplicas: 2 + maxReplicas: 8 + targetCPUUtilizationPercentage: 60 + env: + NODE_ENV: "production" + NEXT_PUBLIC_API_BASE_URL: "/api" + +# ================================================================= +# Ingress +# ================================================================= +ingress: + enabled: true + className: "gce" + annotations: + kubernetes.io/ingress.class: gce + kubernetes.io/ingress.allow-http: "true" + nginx.ingress.kubernetes.io/ssl-redirect: "true" + hosts: + - host: app.customer1.trading.com + paths: + - path: / + pathType: Prefix + service: dashboard + - path: /api/trades + pathType: Prefix + service: execute-service + - path: /api/auth + pathType: Prefix + service: execute-service + - path: /api/news + pathType: Prefix + service: news-service + - path: /api/data + pathType: Prefix + service: data-service + tls: + - secretName: trading-tls + hosts: + - app.customer1.trading.com + +# ================================================================= +# Cert-Manager / mTLS +# ================================================================= +certManager: + enabled: true + externalIssuer: + name: letsencrypt-prod + server: https://acme-v02.api.letsencrypt.org/directory + email: ops@customer1.trading.com + internalIssuer: + name: trading-ca + +# ================================================================= +# PostgreSQL +# ================================================================= +postgres: + enabled: true + image: + repository: postgres + tag: "17-alpine" + primary: + replicaCount: 1 + resources: + requests: + cpu: "1" + memory: 2Gi + limits: + cpu: "2" + memory: 4Gi + storage: + size: 50Gi + storageClass: "standard-rwo" + postgresqlDatabase: trading_db + +# ================================================================= +# Redis +# ================================================================= +redis: + enabled: true + image: + repository: redis + tag: "7-alpine" + replicaCount: 3 + master: + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: "500m" + memory: 512Mi + storage: + size: 10Gi + storageClass: "standard-rwo" + +# ================================================================= +# Kafka (KRaft mode) +# ================================================================= +kafka: + enabled: true + image: + repository: apache/kafka + tag: "3.9.0" + replicaCount: 3 + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: "1" + memory: 2Gi + storage: + size: 20Gi + storageClass: "standard-rwo" + topics: + - name: trades.executed + partitions: 6 + replicationFactor: 3 + - name: orders.new + partitions: 6 + replicationFactor: 3 + - name: news.articles + partitions: 6 + replicationFactor: 3 + - name: news.analysis + partitions: 6 + replicationFactor: 3 + +# ================================================================= +# Network Policies (from security audit) +# ================================================================= +networkPolicies: + enabled: true + defaultDenyIngress: true + defaultDenyEgress: true + allowDNS: true + +# ================================================================= +# SOPS Encryption +# ================================================================= +sops: + enabled: true + ageKeyFile: /etc/sops/age.key + secretsPath: trading-secrets.yaml diff --git a/trading-platform/helm/trading-secrets.yaml b/trading-platform/helm/trading-secrets.yaml new file mode 100644 index 0000000..d47e23f --- /dev/null +++ b/trading-platform/helm/trading-secrets.yaml @@ -0,0 +1,33 @@ +# Trading Secrets - Encrypt with SOPS before applying +# Usage: sops -e -i trading-secrets.yaml && kubectl apply -f trading-secrets.yaml +# +# DO NOT commit unencrypted secrets to git! +# Generate secrets with: python -c "import secrets; print(secrets.token_urlsafe(24))" +apiVersion: v1 +kind: Secret +metadata: + name: trading-secrets + namespace: customer1 + labels: + app.kubernetes.io/managed-by: security-audit + annotations: + secrets.yaml.sops.io/v2: encrypted +type: Opaque +stringData: + # JWT authentication - use 32+ byte hex token + EXECUTE_JWT_SECRET_KEY: "" + + # Exchange private keys - NEVER commit plaintext! + EXECUTE_HYPERLIQUID_PRIVATE_KEY: "" + EXECUTE_SOLANA_PRIVATE_KEY_BASE58: "" + + # Database credentials - use 24+ byte URL-safe tokens + POSTGRES_PASSWORD: "" + NEWS_DB_PASSWORD: "" + + # Redis password + REDIS_PASSWORD: "" + + # Kafka SASL authentication + KAFKA_SASL_USERNAME: "" + KAFKA_SASL_PASSWORD: "" From 0d5f09588507593a47ca814ca22ffe4043bcaea2 Mon Sep 17 00:00:00 2001 From: sirius0xdev Date: Mon, 18 May 2026 21:54:53 -0400 Subject: [PATCH 19/28] Update kustomization.yaml --- infrastructure/gpus/staging/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/gpus/staging/kustomization.yaml b/infrastructure/gpus/staging/kustomization.yaml index bea67ef..37d0cdd 100644 --- a/infrastructure/gpus/staging/kustomization.yaml +++ b/infrastructure/gpus/staging/kustomization.yaml @@ -1,5 +1,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - ../base/vllm-servers/ + # - ../base/vllm-servers/ # - ../base/keda-gpu-scaling/ From 4d8311a7327228707420bfa7693662f5ba4073a3 Mon Sep 17 00:00:00 2001 From: sirius0xdev Date: Tue, 19 May 2026 08:11:46 -0400 Subject: [PATCH 20/28] Update kustomization.yaml --- infrastructure/gpus/staging/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/gpus/staging/kustomization.yaml b/infrastructure/gpus/staging/kustomization.yaml index 37d0cdd..bea67ef 100644 --- a/infrastructure/gpus/staging/kustomization.yaml +++ b/infrastructure/gpus/staging/kustomization.yaml @@ -1,5 +1,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - # - ../base/vllm-servers/ + - ../base/vllm-servers/ # - ../base/keda-gpu-scaling/ From 8fa640725c91c5a99d83bec6f21634e5c104ee53 Mon Sep 17 00:00:00 2001 From: sirius0xdev Date: Wed, 20 May 2026 00:46:46 -0400 Subject: [PATCH 21/28] Update kustomization.yaml --- infrastructure/gpus/staging/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/gpus/staging/kustomization.yaml b/infrastructure/gpus/staging/kustomization.yaml index bea67ef..a328692 100644 --- a/infrastructure/gpus/staging/kustomization.yaml +++ b/infrastructure/gpus/staging/kustomization.yaml @@ -1,5 +1,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - ../base/vllm-servers/ + # - ../base/vllm-servers/ # - ../base/keda-gpu-scaling/ From 1a3804294d244840cac1be91e624543d1e2b6c53 Mon Sep 17 00:00:00 2001 From: sirius0xdev Date: Wed, 20 May 2026 22:12:07 -0400 Subject: [PATCH 22/28] Update kustomization.yaml --- apps/staging/customer1/kustomization.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/staging/customer1/kustomization.yaml b/apps/staging/customer1/kustomization.yaml index 7691ed9..d35882f 100644 --- a/apps/staging/customer1/kustomization.yaml +++ b/apps/staging/customer1/kustomization.yaml @@ -10,3 +10,4 @@ resources: - ../../base/customer1/hermes-agent/ - ../../base/customer1/hermes-db/ - ../../base/customer1/trade-dashboard/ + - ../../base/customer1/siriusdevops-db/ From a06a105f31b56d91f86f22c1c19daf99d03805bb Mon Sep 17 00:00:00 2001 From: sirius0xdev Date: Wed, 20 May 2026 22:18:37 -0400 Subject: [PATCH 23/28] Update kustomization.yaml --- apps/base/customer1/kustomization.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/base/customer1/kustomization.yaml b/apps/base/customer1/kustomization.yaml index 5a94142..51573fe 100644 --- a/apps/base/customer1/kustomization.yaml +++ b/apps/base/customer1/kustomization.yaml @@ -6,3 +6,4 @@ resources: - http-route.yaml - service.yaml - siriusdevops-site + - waitlist-api \ No newline at end of file From 5e477492e6837cc7fe199f359a63655732494337 Mon Sep 17 00:00:00 2001 From: Sirius Devops Date: Mon, 18 May 2026 00:57:53 +0000 Subject: [PATCH 24/28] feat: add OSINT Dashboard Kubernetes infrastructure - Helm chart scaffold (Chart.yaml, values.yaml, _helpers.tpl) - Namespace + RBAC manifests - PostgreSQL (CNPG, 3 replicas, PostGIS + TimescaleDB) - NATS JetStream (3 replicas, persistent, custom subjects) - Redis Sentinel (1 primary + 2 replicas, HA) - MinIO distributed (4 replicas, bucket init job) - Gateway API HTTPRoute + cert-manager TLS certificates - Monitoring stack (Prometheus, Grafana, Alertmanager, exporters) - NetworkPolicies (default deny + per-component policies) - GitHub Actions CI/CD pipeline (lint, template, security scan) - Flux CD staging overlay --- .github/workflows/osint-dashboard-infra.yml | 130 +++++++ apps/base/osint-dashboard/Chart.yaml | 15 + .../osint-dashboard/templates/_helpers.tpl | 67 ++++ .../templates/ingress/ingress.yaml | 47 +++ .../templates/minio/buckets-init-job.yaml | 61 +++ .../templates/minio/credentials-secret.yaml | 15 + .../templates/minio/service.yaml | 49 +++ .../templates/minio/statefulset.yaml | 84 ++++ .../templates/monitoring/alertmanager.yaml | 134 +++++++ .../templates/monitoring/grafana.yaml | 158 ++++++++ .../monitoring/prometheus-config.yaml | 188 +++++++++ .../monitoring/prometheus-deployment.yaml | 127 ++++++ .../osint-dashboard/templates/namespace.yaml | 49 +++ .../templates/nats/jetstream-subjects.yaml | 88 +++++ .../templates/nats/service.yaml | 60 +++ .../templates/nats/statefulset.yaml | 130 +++++++ .../templates/postgresql/cluster.yaml | 152 ++++++++ .../postgresql/credentials-secret.yaml | 29 ++ .../templates/postgresql/service.yaml | 18 + .../templates/redis/configmap.yaml | 70 ++++ .../templates/redis/service.yaml | 61 +++ .../templates/redis/statefulset.yaml | 210 ++++++++++ .../templates/security/dashboard-netpol.yaml | 145 +++++++ .../templates/security/default-deny.yaml | 17 + .../templates/security/minio-netpol.yaml | 56 +++ .../templates/security/monitoring-netpol.yaml | 60 +++ .../templates/security/nats-netpol.yaml | 67 ++++ .../templates/security/postgresql-netpol.yaml | 60 +++ .../templates/security/redis-netpol.yaml | 58 +++ apps/base/osint-dashboard/values.yaml | 360 ++++++++++++++++++ .../osint-dashboard/kustomization.yaml | 53 +++ 31 files changed, 2818 insertions(+) create mode 100644 .github/workflows/osint-dashboard-infra.yml create mode 100644 apps/base/osint-dashboard/Chart.yaml create mode 100644 apps/base/osint-dashboard/templates/_helpers.tpl create mode 100644 apps/base/osint-dashboard/templates/ingress/ingress.yaml create mode 100644 apps/base/osint-dashboard/templates/minio/buckets-init-job.yaml create mode 100644 apps/base/osint-dashboard/templates/minio/credentials-secret.yaml create mode 100644 apps/base/osint-dashboard/templates/minio/service.yaml create mode 100644 apps/base/osint-dashboard/templates/minio/statefulset.yaml create mode 100644 apps/base/osint-dashboard/templates/monitoring/alertmanager.yaml create mode 100644 apps/base/osint-dashboard/templates/monitoring/grafana.yaml create mode 100644 apps/base/osint-dashboard/templates/monitoring/prometheus-config.yaml create mode 100644 apps/base/osint-dashboard/templates/monitoring/prometheus-deployment.yaml create mode 100644 apps/base/osint-dashboard/templates/namespace.yaml create mode 100644 apps/base/osint-dashboard/templates/nats/jetstream-subjects.yaml create mode 100644 apps/base/osint-dashboard/templates/nats/service.yaml create mode 100644 apps/base/osint-dashboard/templates/nats/statefulset.yaml create mode 100644 apps/base/osint-dashboard/templates/postgresql/cluster.yaml create mode 100644 apps/base/osint-dashboard/templates/postgresql/credentials-secret.yaml create mode 100644 apps/base/osint-dashboard/templates/postgresql/service.yaml create mode 100644 apps/base/osint-dashboard/templates/redis/configmap.yaml create mode 100644 apps/base/osint-dashboard/templates/redis/service.yaml create mode 100644 apps/base/osint-dashboard/templates/redis/statefulset.yaml create mode 100644 apps/base/osint-dashboard/templates/security/dashboard-netpol.yaml create mode 100644 apps/base/osint-dashboard/templates/security/default-deny.yaml create mode 100644 apps/base/osint-dashboard/templates/security/minio-netpol.yaml create mode 100644 apps/base/osint-dashboard/templates/security/monitoring-netpol.yaml create mode 100644 apps/base/osint-dashboard/templates/security/nats-netpol.yaml create mode 100644 apps/base/osint-dashboard/templates/security/postgresql-netpol.yaml create mode 100644 apps/base/osint-dashboard/templates/security/redis-netpol.yaml create mode 100644 apps/base/osint-dashboard/values.yaml create mode 100644 apps/staging/osint-dashboard/kustomization.yaml diff --git a/.github/workflows/osint-dashboard-infra.yml b/.github/workflows/osint-dashboard-infra.yml new file mode 100644 index 0000000..3e60261 --- /dev/null +++ b/.github/workflows/osint-dashboard-infra.yml @@ -0,0 +1,130 @@ +name: OSINT Dashboard Infrastructure + +on: + push: + branches: [master] + paths: + - 'apps/base/osint-dashboard/**' + - 'apps/staging/osint-dashboard/**' + - 'clusters/devops-lab/**' + pull_request: + paths: + - 'apps/base/osint-dashboard/**' + workflow_dispatch: + +env: + REGISTRY: gcr.io/devops-lab-cluster + CHART_PATH: apps/base/osint-dashboard + +permissions: + contents: read + security-events: write + pull-requests: write + +jobs: + lint: + name: Lint Helm Chart + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Helm + uses: azure/setup-helm@v3 + with: + version: v3.14.0 + + - name: Set up chart-testing + uses: helm/chart-testing-action@v2 + + - name: Run helm lint + run: | + helm lint ${{ env.CHART_PATH }} + helm lint ${{ env.CHART_PATH }} -f ${{ env.CHART_PATH }}/values.yaml + + - name: Run chart-testing lint + run: | + ct lint --target-branch ${{ github.event.pull_request.base.ref || github.ref_name }} --chart-dirs apps/base --validate-maintainers=false + + template: + name: Render Templates + runs-on: ubuntu-latest + needs: lint + if: github.event_name == 'pull_request' + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Helm + uses: azure/setup-helm@v3 + with: + version: v3.14.0 + + - name: Render templates (dev) + run: | + helm template osint-dashboard ${{ env.CHART_PATH }} --namespace customer1 --values ${{ env.CHART_PATH }}/values.yaml --output-template-files > /dev/null + + - name: Render templates (prod override) + run: | + helm template osint-dashboard ${{ env.CHART_PATH }} --namespace customer1 --values ${{ env.CHART_PATH }}/values.yaml --set postgresql.instances=3 --set nats.replicaCount=3 --set redis.replica.replicaCount=2 --set minio.replicaCount=4 --output-template-files > /dev/null + + validate-yaml: + name: Validate YAML + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install yamllint + run: pip install yamllint + + - name: Lint YAML files + run: | + yamllint -d relaxed --ignore '*/gotk-components.yaml' ${{ env.CHART_PATH }}/Chart.yaml ${{ env.CHART_PATH }}/values.yaml ${{ env.CHART_PATH }}/templates/ + + security-scan: + name: Security Scan + runs-on: ubuntu-latest + needs: lint + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Run Trivy Helm chart scanner + uses: aquasecurity/trivy-action@master + with: + scan-type: 'config' + scan-ref: ${{ env.CHART_PATH }}/templates/ + format: 'sarif' + output: 'trivy-results.sarif' + severity: 'CRITICAL,HIGH' + + - name: Upload Trivy results + uses: github/codeql-action/upload-sarif@v3 + if: always() + with: + sarif_file: 'trivy-results.sarif' + + deploy-staging: + name: Deploy to Staging + runs-on: ubuntu-latest + needs: [lint, validate-yaml] + if: github.ref == 'refs/heads/master' && github.event_name == 'push' + environment: + name: staging + url: https://dashboard.siriusdevops.com + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Flux Kustomization already applied + run: | + echo "Flux CD will automatically pick up changes from master branch." + echo "Kustomization: customer1 -> apps/staging/customer1" + echo "No manual deploy step needed — GitOps loop handles it." + + - name: Notify deployment + if: always() + run: | + echo "Deployment triggered via Flux CD GitOps loop" + echo "Check Flux status: flux get kustomizations -n flux-system" diff --git a/apps/base/osint-dashboard/Chart.yaml b/apps/base/osint-dashboard/Chart.yaml new file mode 100644 index 0000000..2bff7a8 --- /dev/null +++ b/apps/base/osint-dashboard/Chart.yaml @@ -0,0 +1,15 @@ +# OSINT Dashboard — Helm Chart +apiVersion: v2 +name: osint-dashboard +description: Real-time geospatial OSINT dashboard infrastructure +type: application +version: 0.1.0 +appVersion: "1.0.0" +keywords: + - osint + - dashboard + - geospatial + - real-time +maintainers: + - name: sec-ops + email: sec-ops@osint-dashboard.local diff --git a/apps/base/osint-dashboard/templates/_helpers.tpl b/apps/base/osint-dashboard/templates/_helpers.tpl new file mode 100644 index 0000000..fb1122f --- /dev/null +++ b/apps/base/osint-dashboard/templates/_helpers.tpl @@ -0,0 +1,67 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "osint-dashboard.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "osint-dashboard.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 64 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 64 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 64 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "osint-dashboard.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "osint-dashboard.labels" -}} +helm.sh/chart: {{ include "osint-dashboard.chart" . }} +{{ include "osint-dashboard.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "osint-dashboard.selectorLabels" -}} +app.kubernetes.io/name: {{ include "osint-dashboard.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Namespace labels +*/}} +{{- define "osint-dashboard.namespaceLabels" -}} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "osint-dashboard.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "osint-dashboard.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/apps/base/osint-dashboard/templates/ingress/ingress.yaml b/apps/base/osint-dashboard/templates/ingress/ingress.yaml new file mode 100644 index 0000000..1893432 --- /dev/null +++ b/apps/base/osint-dashboard/templates/ingress/ingress.yaml @@ -0,0 +1,47 @@ +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: {{ $host.host | replace "." "-" | trunc 50 | trimSuffix "-" }} + namespace: {{ $.Values.namespace }} + labels: + {{- include "osint-dashboard.labels" $ | nindent 4 }} +spec: + parentRefs: + - name: external-http-gateway + hostnames: + - "{{ $host.host }}" + rules: + {{- range $rule := $host.paths }} + - matches: + - path: + type: PathPrefix + value: {{ $rule.path }} + backendRefs: + - name: dashboard-web + port: 3000 + weight: 100 + {{- end }} +--- +{{- end }} + +# TLS Certificate resources +{{- range $tls := .Values.ingress.tls }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ $tls.secretName }} + namespace: {{ $.Values.namespace }} + labels: + {{- include "osint-dashboard.labels" $ | nindent 4 }} +spec: + secretName: {{ $tls.secretName }} + issuerRef: + name: {{ $.Values.ingress.certManager.clusterIssuerName }} + kind: ClusterIssuer + dnsNames: + {{- toYaml $tls.hosts | nindent 4 }} +--- +{{- end }} +{{- end }} diff --git a/apps/base/osint-dashboard/templates/minio/buckets-init-job.yaml b/apps/base/osint-dashboard/templates/minio/buckets-init-job.yaml new file mode 100644 index 0000000..a069a8e --- /dev/null +++ b/apps/base/osint-dashboard/templates/minio/buckets-init-job.yaml @@ -0,0 +1,61 @@ +{{- if .Values.minio.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: minio-buckets-init + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: object-storage + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded +spec: + template: + metadata: + labels: + {{- include "osint-dashboard.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: object-storage + spec: + restartPolicy: OnFailure + securityContext: + runAsNonRoot: true + runAsUser: 1000 + containers: + - name: mc + image: "{{ .Values.minio.image.repository }}:{{ .Values.minio.image.tag }}" + envFrom: + - secretRef: + name: {{ .Values.minio.credentialsSecret }} + command: + - /bin/sh + - -c + args: + - | + # Wait for MinIO to be ready + until curl -sf http://minio:{{ .Values.minio.ports.api }}/minio/health/live; do + echo "Waiting for MinIO..." + sleep 2 + done + + # Configure mc alias + mc alias set myminio http://minio:{{ .Values.minio.ports.api }} $MINIO_ROOT_USER $MINIO_ROOT_PASSWORD + + # Create buckets + mc mb --ignore-existing myminio/osint-video-clips + mc mb --ignore-existing myminio/osint-satellite-tiles + mc mb --ignore-existing myminio/osint-data-dumps + + echo "MinIO buckets initialized successfully" + resources: + requests: + cpu: "100m" + memory: "64Mi" + limits: + cpu: "500m" + memory: "256Mi" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] +{{- end }} diff --git a/apps/base/osint-dashboard/templates/minio/credentials-secret.yaml b/apps/base/osint-dashboard/templates/minio/credentials-secret.yaml new file mode 100644 index 0000000..e66560a --- /dev/null +++ b/apps/base/osint-dashboard/templates/minio/credentials-secret.yaml @@ -0,0 +1,15 @@ +{{- if .Values.minio.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.minio.credentialsSecret }} + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + annotations: + # SOPS will encrypt this +type: Opaque +stringData: + MINIO_ROOT_USER: CHANGE_ME_USE_SOPS + MINIO_ROOT_PASSWORD: CHANGE_ME_USE_SOPS +{{- end }} diff --git a/apps/base/osint-dashboard/templates/minio/service.yaml b/apps/base/osint-dashboard/templates/minio/service.yaml new file mode 100644 index 0000000..be37a1d --- /dev/null +++ b/apps/base/osint-dashboard/templates/minio/service.yaml @@ -0,0 +1,49 @@ +{{- if .Values.minio.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: minio + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: object-storage +spec: + type: ClusterIP + ports: + - port: {{ .Values.minio.ports.api }} + targetPort: {{ .Values.minio.ports.api }} + protocol: TCP + name: api + - port: {{ .Values.minio.ports.console }} + targetPort: {{ .Values.minio.ports.console }} + protocol: TCP + name: console + selector: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: object-storage +--- +apiVersion: v1 +kind: Service +metadata: + name: minio-headless + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: object-storage + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" +spec: + clusterIP: None + ports: + - port: {{ .Values.minio.ports.api }} + targetPort: {{ .Values.minio.ports.api }} + protocol: TCP + name: api + - port: {{ .Values.minio.ports.console }} + targetPort: {{ .Values.minio.ports.console }} + protocol: TCP + name: console + selector: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: object-storage +{{- end }} diff --git a/apps/base/osint-dashboard/templates/minio/statefulset.yaml b/apps/base/osint-dashboard/templates/minio/statefulset.yaml new file mode 100644 index 0000000..17b7bed --- /dev/null +++ b/apps/base/osint-dashboard/templates/minio/statefulset.yaml @@ -0,0 +1,84 @@ +{{- if .Values.minio.enabled }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: minio + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: object-storage +spec: + serviceName: minio-headless + replicas: {{ .Values.minio.replicaCount }} + selector: + matchLabels: + {{- include "osint-dashboard.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: object-storage + template: + metadata: + labels: + {{- include "osint-dashboard.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: object-storage + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "{{ .Values.minio.ports.api }}" + prometheus.io/path: "/minio/v2/metrics/cluster" + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + containers: + - name: minio + image: "{{ .Values.minio.image.repository }}:{{ .Values.minio.image.tag }}" + ports: + - name: api + containerPort: {{ .Values.minio.ports.api }} + - name: console + containerPort: {{ .Values.minio.ports.console }} + resources: + {{- toYaml .Values.minio.resources | nindent 12 }} + envFrom: + - secretRef: + name: {{ .Values.minio.credentialsSecret }} + env: + - name: MINIO_SERVER_URL + value: "http://minio.{{ .Values.namespace }}.svc:{{ .Values.minio.ports.api }}" + args: + - server + - "--console-address" + - ":{{ .Values.minio.ports.console }}" + - "--address" + - ":{{ .Values.minio.ports.api }}" + # Distributed mode: all 4 pods + - "http://minio-{0...3}.minio-headless.{{ .Values.namespace }}.svc/data" + volumeMounts: + - name: data + mountPath: /data + readinessProbe: + httpGet: + path: /minio/health/live + port: api + initialDelaySeconds: 5 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /minio/health/live + port: api + initialDelaySeconds: 10 + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.minio.storage.size }} + storageClassName: {{ .Values.minio.storage.storageClass }} +{{- end }} diff --git a/apps/base/osint-dashboard/templates/monitoring/alertmanager.yaml b/apps/base/osint-dashboard/templates/monitoring/alertmanager.yaml new file mode 100644 index 0000000..e745a12 --- /dev/null +++ b/apps/base/osint-dashboard/templates/monitoring/alertmanager.yaml @@ -0,0 +1,134 @@ +{{- if .Values.monitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: alertmanager-config + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: monitoring +data: + alertmanager.yml: | + global: + resolve_timeout: 5m + + route: + group_by: ['alertname', 'namespace'] + group_wait: 30s + group_interval: 5m + repeat_interval: 4h + receiver: 'default' + routes: + - match: + severity: critical + receiver: 'pager' + repeat_interval: 1h + - match: + severity: warning + receiver: 'slack' + + receivers: + - name: 'default' + email_configs: + - to: CHANGE_ME_USE_SOPS + from: monitoring@{{ .Values.monitoring.defaultEmailDomain }} + smarthost: CHANGE_ME_USE_SOPS + auth_username: CHANGE_ME_USE_SOPS + auth_password: CHANGE_ME_USE_SOPS + + - name: 'pager' + webhook_configs: + - url: CHANGE_ME_USE_SOPS + send_resolved: true + + - name: 'slack' + slack_configs: + - api_url: CHANGE_ME_USE_SOPS + channel: '#osint-alerts' + send_resolved: true + title: '{{ .GroupLabels.alertname }}' + text: >- + {{ range .Alerts }} + *Alert:* {{ .Labels.alertname }} + *Severity:* {{ .Labels.severity }} + *Summary:* {{ .Annotations.summary }} + {{ end }} + + inhibit_rules: + - source_match: + severity: 'critical' + target_match: + severity: 'warning' + equal: ['alertname', 'namespace'] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: alertmanager + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: monitoring +spec: + replicas: 1 + selector: + matchLabels: + {{- include "osint-dashboard.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: monitoring + template: + metadata: + labels: + {{- include "osint-dashboard.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: monitoring + spec: + securityContext: + runAsNonRoot: true + runAsUser: 65534 + fsGroup: 65534 + containers: + - name: alertmanager + image: "{{ .Values.monitoring.alertmanager.image.repository }}:{{ .Values.monitoring.alertmanager.image.tag }}" + ports: + - containerPort: {{ .Values.monitoring.alertmanager.port }} + name: web + args: + - "--config.file=/etc/alertmanager/alertmanager.yml" + - "--storage.path=/alertmanager" + resources: + {{- toYaml .Values.monitoring.alertmanager.resources | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/alertmanager + - name: data + mountPath: /alertmanager + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumes: + - name: config + configMap: + name: alertmanager-config + strategy: + type: Recreate +--- +apiVersion: v1 +kind: Service +metadata: + name: alertmanager + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: monitoring +spec: + type: ClusterIP + ports: + - port: {{ .Values.monitoring.alertmanager.port }} + targetPort: web + protocol: TCP + name: web + selector: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: monitoring +{{- end }} diff --git a/apps/base/osint-dashboard/templates/monitoring/grafana.yaml b/apps/base/osint-dashboard/templates/monitoring/grafana.yaml new file mode 100644 index 0000000..cf3be67 --- /dev/null +++ b/apps/base/osint-dashboard/templates/monitoring/grafana.yaml @@ -0,0 +1,158 @@ +{{- if .Values.monitoring.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: grafana-admin-secret + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: monitoring +type: Opaque +stringData: + admin-user: CHANGE_ME_USE_SOPS + admin-password: CHANGE_ME_USE_SOPS +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-datasources + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: monitoring +data: + datasources.yaml: | + apiVersion: 1 + datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:{{ .Values.monitoring.prometheus.port }} + isDefault: true + editable: true + - name: PostgreSQL + type: postgres + access: proxy + url: postgresql-rw.{{ .Values.namespace }}.svc:5432 + database: osint + user: grafana + secureJsonData: + password: CHANGE_ME_USE_SOPS + jsonData: + tlsAuth: false + sslmode: disable + postgresVersion: 1600 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboards + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: monitoring +data: + dashboard-providers.yaml: | + apiVersion: 1 + providers: + - name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: monitoring +spec: + replicas: 1 + selector: + matchLabels: + {{- include "osint-dashboard.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: monitoring + template: + metadata: + labels: + {{- include "osint-dashboard.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: monitoring + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "3000" + spec: + securityContext: + runAsNonRoot: true + runAsUser: 472 # grafana + fsGroup: 472 + containers: + - name: grafana + image: "{{ .Values.monitoring.grafana.image.repository }}:{{ .Values.monitoring.grafana.image.tag }}" + ports: + - containerPort: {{ .Values.monitoring.grafana.port }} + name: web + env: + - name: GF_SECURITY_ADMIN_USER + valueFrom: + secretKeyRef: + name: grafana-admin-secret + key: admin-user + - name: GF_SECURITY_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: grafana-admin-secret + key: admin-password + - name: GF_SERVER_ROOT_URL + value: "https://grafana.{{ .Values.monitoring.grafana.hostname }}" + - name: GF_AUTH_ANONYMOUS_ENABLED + value: "false" + resources: + {{- toYaml .Values.monitoring.grafana.resources | nindent 12 }} + volumeMounts: + - name: datasources + mountPath: /etc/grafana/provisioning/datasources + - name: dashboards-config + mountPath: /etc/grafana/provisioning/dashboards + - name: data + mountPath: /var/lib/grafana + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + volumes: + - name: datasources + configMap: + name: grafana-datasources + - name: dashboards-config + configMap: + name: grafana-dashboards + strategy: + type: Recreate +--- +apiVersion: v1 +kind: Service +metadata: + name: grafana + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: monitoring +spec: + type: ClusterIP + ports: + - port: {{ .Values.monitoring.grafana.port }} + targetPort: web + protocol: TCP + name: web + selector: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: monitoring +{{- end }} diff --git a/apps/base/osint-dashboard/templates/monitoring/prometheus-config.yaml b/apps/base/osint-dashboard/templates/monitoring/prometheus-config.yaml new file mode 100644 index 0000000..4345471 --- /dev/null +++ b/apps/base/osint-dashboard/templates/monitoring/prometheus-config.yaml @@ -0,0 +1,188 @@ +{{- if .Values.monitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-config + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: monitoring +data: + prometheus.yml: | + global: + scrape_interval: 15s + evaluation_interval: 15s + scrape_timeout: 10s + + rule_files: + - /etc/prometheus/rules/*.yml + + alerting: + alertmanagers: + - static_configs: + - targets: + - alertmanager:{{ .Values.monitoring.alertmanager.port }} + + scrape_configs: + # Prometheus self-monitoring + - job_name: prometheus + static_configs: + - targets: [localhost:9090] + + # PostgreSQL (Postgres Exporter) + - job_name: postgresql + static_configs: + - targets: + {{- range $i := until $.Values.monitoring.postgresql.exporter.replicas }} + - postgresql-{{ $i }}.postgresql-rw.{{ $.Values.namespace }}.svc:9187 + {{- end }} + + # NATS JetStream + - job_name: nats + static_configs: + - targets: + {{- range $i := until $.Values.nats.replicaCount }} + - nats-{{ $i }}.nats-cluster.{{ $.Values.namespace }}.svc:8222 + {{- end }} + + # Redis + - job_name: redis + static_configs: + - targets: + {{- range $i := until $.Values.redis.replicaCount }} + - redis-{{ $i }}.redis-cluster.{{ $.Values.namespace }}.svc:9121 + {{- end }} + + # MinIO + - job_name: minio + metrics_path: /minio/v2/metrics/cluster + static_configs: + - targets: + {{- range $i := until $.Values.minio.replicaCount }} + - minio-{{ $i }}.minio-headless.{{ $.Values.namespace }}.svc:9000 + {{- end }} + + # Kafka Exporter + - job_name: kafka + static_configs: + - targets: [kafka-exporter:9308] + + # Dashboard web app + - job_name: dashboard-web + static_configs: + - targets: [dashboard-web:3000] + + # Dashboard API + - job_name: dashboard-api + static_configs: + - targets: [dashboard-api:4000] + + # Auto-discover via pod annotations + - job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + namespaces: + own: false + names: + - {{ .Values.namespace }} + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: "true" + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + action: replace + regex: ([\d+]);([\d.]+) + replacement: $2:$1 + target_label: __address__ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-rules + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: monitoring +data: + osint-alerts.yml: | + groups: + - name: osint-dashboard-alerts + rules: + - alert: HighErrorRate + expr: sum(rate(http_requests_total{status=~"5..",namespace="{{ .Values.namespace }}"}[5m])) / sum(rate(http_requests_total{namespace="{{ .Values.namespace }}"}[5m])) > 0.05 + for: 5m + labels: + severity: critical + annotations: + summary: "High error rate detected (>{{ 5 }}%) on {{ $labels.job }}" + + - alert: PodCrashLooping + expr: rate(kube_pod_container_status_restarts_total{namespace="{{ .Values.namespace }}"}[15m]) * 60 * 5 > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Pod {{ $labels.pod }} is crash looping" + + - alert: HighLatency + expr: histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{namespace="{{ .Values.namespace }}"}[5m])) by (le, job)) > 2 + for: 5m + labels: + severity: warning + annotations: + summary: "P95 latency above 2s for {{ $labels.job }}" + + - alert: DiskSpaceLow + expr: kubelet_volume_stats_available_bytes{namespace="{{ .Values.namespace }}"}/kubelet_volume_stats_capacity_bytes{namespace="{{ .Values.namespace }}"} < 0.1 + for: 10m + labels: + severity: critical + annotations: + summary: "Disk space below 10% on {{ $labels.persistentvolumeclaim }}" + + - alert: PostgreSQLConnectionSaturation + expr: pg_stat_activity_count{datname="osint",state="active"}/pg_settings_max_connections > 0.8 + for: 5m + labels: + severity: warning + annotations: + summary: "PostgreSQL connection pool >80% saturated" + + - alert: NATSJetStreamStoreFull + expr: jetstream_store_disk_bytes / jetstream_config_max_store_bytes > 0.85 + for: 5m + labels: + severity: critical + annotations: + summary: "NATS JetStream disk usage >85%" + + - alert: RedisMemoryHigh + expr: redis_memory_used_bytes / redis_memory_max_bytes > 0.9 + for: 5m + labels: + severity: warning + annotations: + summary: "Redis memory usage >90%" + + - alert: MinIOOffline + expr: up{job="minio"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "MinIO node {{ $labels.instance }} is offline" + + - alert: KafkaLagHigh + expr: kafka_consumer_group_lag > 10000 + for: 10m + labels: + severity: warning + annotations: + summary: "Kafka consumer lag >10k messages for group {{ $labels.group }}" +{{- end }} diff --git a/apps/base/osint-dashboard/templates/monitoring/prometheus-deployment.yaml b/apps/base/osint-dashboard/templates/monitoring/prometheus-deployment.yaml new file mode 100644 index 0000000..495be11 --- /dev/null +++ b/apps/base/osint-dashboard/templates/monitoring/prometheus-deployment.yaml @@ -0,0 +1,127 @@ +{{- if .Values.monitoring.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: monitoring +spec: + replicas: 1 + selector: + matchLabels: + {{- include "osint-dashboard.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: monitoring + template: + metadata: + labels: + {{- include "osint-dashboard.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: monitoring + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + spec: + securityContext: + runAsNonRoot: true + runAsUser: 65534 # nobody + fsGroup: 65534 + serviceAccountName: prometheus + containers: + - name: prometheus + image: "{{ .Values.monitoring.prometheus.image.repository }}:{{ .Values.monitoring.prometheus.image.tag }}" + ports: + - containerPort: {{ .Values.monitoring.prometheus.port }} + name: web + args: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + - "--storage.tsdb.retention.time={{ .Values.monitoring.prometheus.retention }}" + - "--storage.tsdb.retention.size={{ .Values.monitoring.prometheus.retentionSize }}" + - "--web.enable-lifecycle" + - "--web.enable-admin-api" + resources: + {{- toYaml .Values.monitoring.prometheus.resources | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/prometheus + - name: rules + mountPath: /etc/prometheus/rules + - name: data + mountPath: /prometheus + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumes: + - name: config + configMap: + name: prometheus-config + - name: rules + configMap: + name: prometheus-rules + strategy: + type: Recreate +--- +apiVersion: v1 +kind: Service +metadata: + name: prometheus + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: monitoring + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" +spec: + type: ClusterIP + ports: + - port: {{ .Values.monitoring.prometheus.port }} + targetPort: web + protocol: TCP + name: web + selector: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: monitoring +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: monitoring +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: prometheus + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: monitoring +rules: + - apiGroups: [""] + resources: ["pods", "services", "endpoints", "configmaps"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: prometheus + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: monitoring +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: prometheus +subjects: + - kind: ServiceAccount + name: prometheus + namespace: {{ .Values.namespace }} +{{- end }} diff --git a/apps/base/osint-dashboard/templates/namespace.yaml b/apps/base/osint-dashboard/templates/namespace.yaml new file mode 100644 index 0000000..fb13cb8 --- /dev/null +++ b/apps/base/osint-dashboard/templates/namespace.yaml @@ -0,0 +1,49 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.namespaceLabels" . | nindent 4 }} + app.kubernetes.io/part-of: osint-dashboard +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: osint-dashboard-role + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["pods", "services", "configmaps", "secrets", "persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: ["deployments", "statefulsets"] + verbs: ["get", "list", "watch"] + - apiGroups: ["batch"] + resources: ["jobs", "cronjobs"] + verbs: ["get", "list", "watch", "create", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: osint-dashboard-rolebinding + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: osint-dashboard-role +subjects: + - kind: ServiceAccount + name: {{ include "osint-dashboard.serviceAccountName" . }} + namespace: {{ .Values.namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "osint-dashboard.serviceAccountName" . }} + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} diff --git a/apps/base/osint-dashboard/templates/nats/jetstream-subjects.yaml b/apps/base/osint-dashboard/templates/nats/jetstream-subjects.yaml new file mode 100644 index 0000000..780eceb --- /dev/null +++ b/apps/base/osint-dashboard/templates/nats/jetstream-subjects.yaml @@ -0,0 +1,88 @@ +{{- if .Values.nats.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: nats-jetstream-init + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: message-broker + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded +spec: + template: + metadata: + labels: + {{- include "osint-dashboard.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: message-broker + spec: + restartPolicy: OnFailure + securityContext: + runAsNonRoot: true + runAsUser: 1000 + containers: + - name: nats-box + image: "{{ .Values.nats.image.repository }}:{{ .Values.nats.image.tag }}" + command: + - /bin/sh + - -c + args: + - | + # Wait for NATS to be ready + until nats-server -version > /dev/null 2>&1 || curl -sf http://nats:{{ .Values.nats.ports.monitor }}/healthz; do + echo "Waiting for NATS..." + sleep 2 + done + + # Download nats CLI + curl -fsSL https://github.com/nats-io/natscli/releases/latest/download/nats-linux-amd64 -o /tmp/nats + chmod +x /tmp/nats + + # Create JetStream stream for events + /tmp/nats stream add events \ + --subjects=events.gdelt,events.rss,events.social,events.earthquake,events.disaster,events.weather,events.fire,events.satellite,events.new,events.alert \ + --retention=interests \ + --max-consumers=-1 \ + --max-msgs=1000000 \ + --max-bytes=1GB \ + --discard=old \ + --storage=file \ + --replicas=3 \ + --server=nats://nats:{{ .Values.nats.ports.client }} || echo "events stream already exists" + + # Create JetStream stream for alerts + /tmp/nats stream add alerts \ + --subjects=alerts.camera_offline,alerts.new \ + --retention=interests \ + --max-consumers=-1 \ + --max-msgs=100000 \ + --discard=old \ + --storage=file \ + --replicas=3 \ + --server=nats://nats:{{ .Values.nats.ports.client }} || echo "alerts stream already exists" + + # Create JetStream stream for video + /tmp/nats stream add video \ + --subjects="video.status.>","video.record.>" \ + --retention=limits \ + --max-consumers=-1 \ + --max-msgs=50000 \ + --discard=old \ + --storage=file \ + --replicas=3 \ + --server=nats://nats:{{ .Values.nats.ports.client }} || echo "video stream already exists" + + echo "JetStream subjects initialized successfully" + resources: + requests: + cpu: "100m" + memory: "64Mi" + limits: + cpu: "500m" + memory: "256Mi" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] +{{- end }} diff --git a/apps/base/osint-dashboard/templates/nats/service.yaml b/apps/base/osint-dashboard/templates/nats/service.yaml new file mode 100644 index 0000000..7e4a294 --- /dev/null +++ b/apps/base/osint-dashboard/templates/nats/service.yaml @@ -0,0 +1,60 @@ +{{- if .Values.nats.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: nats + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: message-broker +spec: + type: ClusterIP + ports: + - port: {{ .Values.nats.ports.client }} + targetPort: {{ .Values.nats.ports.client }} + protocol: TCP + name: client + selector: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: message-broker +--- +apiVersion: v1 +kind: Service +metadata: + name: nats-ws + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: message-broker +spec: + type: ClusterIP + ports: + - port: {{ .Values.nats.ports.websocket }} + targetPort: {{ .Values.nats.ports.websocket }} + protocol: TCP + name: websocket + selector: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: message-broker +--- +apiVersion: v1 +kind: Service +metadata: + name: nats-cluster + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: message-broker + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" +spec: + clusterIP: None + ports: + - port: {{ .Values.nats.ports.cluster }} + targetPort: {{ .Values.nats.ports.cluster }} + protocol: TCP + name: cluster + selector: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: message-broker +{{- end }} diff --git a/apps/base/osint-dashboard/templates/nats/statefulset.yaml b/apps/base/osint-dashboard/templates/nats/statefulset.yaml new file mode 100644 index 0000000..74cbb85 --- /dev/null +++ b/apps/base/osint-dashboard/templates/nats/statefulset.yaml @@ -0,0 +1,130 @@ +{{- if .Values.nats.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: nats-config + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} +data: + nats-server.conf: | + port: {{ .Values.nats.ports.client }} + server_name: "osint-nats-${HOSTNAME}" + + # Cluster + cluster { + port: {{ .Values.nats.ports.cluster }} + routes: [ + nats-route://nats-0.nats-cluster.{{ .Values.namespace }}.svc:{{ .Values.nats.ports.cluster }}, + nats-route://nats-1.nats-cluster.{{ .Values.namespace }}.svc:{{ .Values.nats.ports.cluster }}, + nats-route://nats-2.nats-cluster.{{ .Values.namespace }}.svc:{{ .Values.nats.ports.cluster }} + ] + cluster_advertise: "nats-${HOSTNAME}.nats-cluster.{{ .Values.namespace }}.svc:{{ .Values.nats.ports.cluster }}" + } + + # JetStream + jetstream { + store_dir: "{{ .Values.nats.jetstream.fileStore }}" + max_mem_store: {{ .Values.nats.jetstream.maxMemory }} + max_file_store: {{ .Values.nats.storage.size }} + } + + # Monitoring + monitor: {{ .Values.nats.ports.monitor }} + + # WebSocket + websocket { + port: {{ .Values.nats.ports.websocket }} + no_tls: true + } + + # Logging + logtime: true + log_file: /var/log/nats/nats.log +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: nats + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: message-broker +spec: + serviceName: nats-cluster + replicas: {{ .Values.nats.replicaCount }} + selector: + matchLabels: + {{- include "osint-dashboard.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: message-broker + template: + metadata: + labels: + {{- include "osint-dashboard.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: message-broker + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "{{ .Values.nats.ports.monitor }}" + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + terminationGracePeriodSeconds: 30 + containers: + - name: nats + image: "{{ .Values.nats.image.repository }}:{{ .Values.nats.image.tag }}" + ports: + - name: client + containerPort: {{ .Values.nats.ports.client }} + - name: cluster + containerPort: {{ .Values.nats.ports.cluster }} + - name: monitor + containerPort: {{ .Values.nats.ports.monitor }} + - name: websocket + containerPort: {{ .Values.nats.ports.websocket }} + resources: + {{- toYaml .Values.nats.resources | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/nats-config + - name: data + mountPath: {{ .Values.nats.jetstream.fileStore }} + - name: logs + mountPath: /var/log/nats + args: + - "-c" + - "/etc/nats-config/nats-server.conf" + readinessProbe: + httpGet: + path: /healthz + port: monitor + initialDelaySeconds: 5 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /healthz + port: monitor + initialDelaySeconds: 10 + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumes: + - name: config + configMap: + name: nats-config + - name: logs + emptyDir: {} + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.nats.storage.size }} + storageClassName: {{ .Values.nats.storage.storageClass }} +{{- end }} diff --git a/apps/base/osint-dashboard/templates/postgresql/cluster.yaml b/apps/base/osint-dashboard/templates/postgresql/cluster.yaml new file mode 100644 index 0000000..40a629b --- /dev/null +++ b/apps/base/osint-dashboard/templates/postgresql/cluster.yaml @@ -0,0 +1,152 @@ +{{- if .Values.postgresql.enabled }} +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: {{ .Values.postgresql.clusterName }} + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} +spec: + instances: {{ .Values.postgresql.instances }} + imageName: {{ .Values.postgresql.imageName }} + storage: + size: {{ .Values.postgresql.storage.size }} + storageClass: {{ .Values.postgresql.storage.storageClass }} + resources: + {{- toYaml .Values.postgresql.resources | nindent 4 }} + # PostGIS + TimescaleDB extensions via shared_preload_libraries + postgresql: + shared_preload_libraries: + - pggis + - timescaledb + parameters: + max_connections: "500" + shared_buffers: "2GB" + effective_cache_size: "6GB" + maintenance_work_mem: "512MB" + work_mem: "16MB" + wal_buffers: "64MB" + random_page_cost: "1.1" + effective_io_concurrency: "200" + default_statistics_target: "200" + max_parallel_workers_per_gather: "4" + bootstrap: + initdb: + database: osint + owner: osint_admin + secret: + name: {{ .Values.postgresql.credentialsSecret }} + postInitializationSQL: + # Install PostGIS extension + - >- + CREATE EXTENSION IF NOT EXISTS postgis; + - >- + CREATE EXTENSION IF NOT EXISTS postgis_raster; + - >- + CREATE EXTENSION IF NOT EXISTS postgis_topology; + # Install TimescaleDB extension + - >- + CREATE EXTENSION IF NOT EXISTS timescaledb; + # Create hypertable for events + - >- + CREATE TABLE IF NOT EXISTS events ( + time TIMESTAMPTZ NOT NULL, + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + source TEXT NOT NULL, + event_type TEXT, + title TEXT, + description TEXT, + location GEOGRAPHY(POINT, 4326), + severity INT DEFAULT 0, + tags TEXT[], + raw_data JSONB, + created_at TIMESTAMPTZ DEFAULT NOW() + ); + - >- + SELECT create_hypertable('events', 'time', if_not_exists => TRUE); + - >- + CREATE INDEX IF NOT EXISTS events_loc_idx ON events USING GIST (location); + - >- + CREATE INDEX IF NOT EXISTS events_time_idx ON events (time DESC); + # Create sources reference table + - >- + CREATE TABLE IF NOT EXISTS sources ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name TEXT NOT NULL, + type TEXT NOT NULL, + config JSONB, + active BOOLEAN DEFAULT TRUE + ); + # Create video_feeds table + - >- + CREATE TABLE IF NOT EXISTS video_feeds ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name TEXT NOT NULL, + rtsp_url TEXT NOT NULL, + zlm_stream_key TEXT NOT NULL, + location GEOGRAPHY(POINT, 4326), + active BOOLEAN DEFAULT TRUE, + last_heartbeat TIMESTAMPTZ + ); + managed: + roles: + - name: osint_admin + ensure: present + login: true + passwordSecret: + name: {{ .Values.postgresql.credentialsSecret }} + - name: osint_reader + ensure: present + login: true + passwordSecret: + name: osint-pg-reader-credentials + backup: + barmanObjectStore: + destinationPath: "{{ .Values.postgresql.backup.bucket }}" + googleCredentials: + gkeEnvironment: true + wal: + compression: gzip + data: + compression: gzip + jobs: 2 + retentionPolicy: {{ .Values.postgresql.backup.retentionPolicy }} + target: primary + monitoring: + customQueries: + - query: >- + SELECT relname, schemaname, seq_scan, seq_tup_read, idx_scan, idx_tup_fetch + FROM pg_stat_user_tables WHERE schemaname = 'public'; + metrics: + - relname: + usage: "LABEL" + description: "Table name" + - schemaname: + usage: "LABEL" + description: "Schema name" + - seq_scan: + usage: "GAUGE" + description: "Number of sequential scans" + - seq_tup_read: + usage: "GAUGE" + description: "Number of tuples read" + - idx_scan: + usage: "GAUGE" + description: "Number of index scans" + - idx_tup_fetch: + usage: "GAUGE" + description: "Number of tuples fetched via index" + affinity: + enablePodAntiAffinity: true + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + postgresql operator: {{ .Values.postgresql.clusterName }} + nodesAutoRemediationChecks: + livenessProbe: + initialDelaySeconds: 10 + timeoutSeconds: 5 +{{- end }} diff --git a/apps/base/osint-dashboard/templates/postgresql/credentials-secret.yaml b/apps/base/osint-dashboard/templates/postgresql/credentials-secret.yaml new file mode 100644 index 0000000..30188a6 --- /dev/null +++ b/apps/base/osint-dashboard/templates/postgresql/credentials-secret.yaml @@ -0,0 +1,29 @@ +{{- if .Values.postgresql.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.postgresql.credentialsSecret }} + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + annotations: + # SOPS will encrypt this - use 'sops --encrypt --in-place' after generation +type: Opaque +stringData: + username: osint_admin + password: CHANGE_ME_USE_SOPS + connection_string: "postgresql://osint_admin:CHANGE_ME_USE_SOPS@{{ .Values.postgresql.clusterName }}.{{ .Values.namespace }}.svc:5432/osint?sslmode=require" +--- +apiVersion: v1 +kind: Secret +metadata: + name: osint-pg-reader-credentials + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} +type: Opaque +stringData: + username: osint_reader + password: CHANGE_ME_USE_SOPS + connection_string: "postgresql://osint_reader:CHANGE_ME_USE_SOPS@{{ .Values.postgresql.clusterName }}.{{ .Values.namespace }}.svc:5432/osint?sslmode=require" +{{- end }} diff --git a/apps/base/osint-dashboard/templates/postgresql/service.yaml b/apps/base/osint-dashboard/templates/postgresql/service.yaml new file mode 100644 index 0000000..36e4653 --- /dev/null +++ b/apps/base/osint-dashboard/templates/postgresql/service.yaml @@ -0,0 +1,18 @@ +{{- if .Values.postgresql.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.postgresql.clusterName }} + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: 5432 + targetPort: postgresql + protocol: TCP + name: postgresql + selector: + postgresql operator: {{ .Values.postgresql.clusterName }} +{{- end }} diff --git a/apps/base/osint-dashboard/templates/redis/configmap.yaml b/apps/base/osint-dashboard/templates/redis/configmap.yaml new file mode 100644 index 0000000..a2edf74 --- /dev/null +++ b/apps/base/osint-dashboard/templates/redis/configmap.yaml @@ -0,0 +1,70 @@ +{{- if .Values.redis.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: redis-config + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} +data: + redis.conf: | + bind 0.0.0.0 + port {{ .Values.redis.ports.redis }} + appendonly yes + appendfilename "appendonly.aof" + dir /data + save 900 1 + save 300 10 + save 60 10000 + maxmemory-policy allkeys-lru + # Require authentication + requirepass CHANGE_ME_USE_SOPS + masterauth CHANGE_ME_USE_SOPS +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: redis-sentinel-config + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} +data: + sentinel.conf: | + port {{ .Values.redis.ports.sentinel }} + sentinel monitor osint-redis-master redis-master.{{ .Values.namespace }}.svc {{ .Values.redis.ports.redis }} 2 + sentinel auth-pass osint-redis-master CHANGE_ME_USE_SOPS + sentinel down-after-milliseconds osint-redis-master 5000 + sentinel failover-timeout osint-redis-master 30000 + sentinel parallel-syncs osint-redis-master 1 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: redis-scripts + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} +data: + init-sentinel.sh: | + #!/bin/bash + set -e + + # Copy sentinel config and update it with current master info + cp /etc/redis-sentinel/sentinel.conf /tmp/sentinel.conf + # Sentinel will auto-discover master from other sentinels + exec redis-sentinel /tmp/sentinel.conf --loglevel notice + redis.sh: | + #!/bin/bash + set -e + + REDIS_PORT={{ .Values.redis.ports.redis }} + REDIS_PASSWORD="CHANGE_ME_USE_SOPS" + + if [ "${REDIS_ROLE}" = "master" ]; then + exec redis-server /etc/redis/redis.conf + else + # Replica: find master and replicate + MASTER_HOST="redis-master.{{ .Values.namespace }}.svc" + exec redis-server /etc/redis/redis.conf --replicaof ${MASTER_HOST} ${REDIS_PORT} + fi +{{- end }} diff --git a/apps/base/osint-dashboard/templates/redis/service.yaml b/apps/base/osint-dashboard/templates/redis/service.yaml new file mode 100644 index 0000000..14da958 --- /dev/null +++ b/apps/base/osint-dashboard/templates/redis/service.yaml @@ -0,0 +1,61 @@ +{{- if .Values.redis.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: redis-master + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: redis +spec: + type: ClusterIP + ports: + - port: {{ .Values.redis.ports.redis }} + targetPort: {{ .Values.redis.ports.redis }} + protocol: TCP + name: redis + selector: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: redis + redis-role: master +--- +apiVersion: v1 +kind: Service +metadata: + name: redis-replica + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: redis +spec: + type: ClusterIP + ports: + - port: {{ .Values.redis.ports.redis }} + targetPort: {{ .Values.redis.ports.redis }} + protocol: TCP + name: redis + selector: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: redis + redis-role: replica +--- +apiVersion: v1 +kind: Service +metadata: + name: redis-sentinel + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: redis +spec: + type: ClusterIP + ports: + - port: {{ .Values.redis.ports.sentinel }} + targetPort: {{ .Values.redis.ports.sentinel }} + protocol: TCP + name: sentinel + selector: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: redis + redis-role: sentinel +{{- end }} diff --git a/apps/base/osint-dashboard/templates/redis/statefulset.yaml b/apps/base/osint-dashboard/templates/redis/statefulset.yaml new file mode 100644 index 0000000..a2804ea --- /dev/null +++ b/apps/base/osint-dashboard/templates/redis/statefulset.yaml @@ -0,0 +1,210 @@ +{{- if .Values.redis.enabled }} +# Redis Master +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis-master + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: redis + redis-role: master +spec: + serviceName: redis-master + replicas: 1 + selector: + matchLabels: + {{- include "osint-dashboard.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: redis + redis-role: master + template: + metadata: + labels: + {{- include "osint-dashboard.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: redis + redis-role: master + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9121" + spec: + securityContext: + runAsNonRoot: true + runAsUser: 999 + fsGroup: 999 + containers: + - name: redis + image: "{{ .Values.redis.image.repository }}:{{ .Values.redis.image.tag }}" + ports: + - name: redis + containerPort: {{ .Values.redis.ports.redis }} + resources: + {{- toYaml .Values.redis.master.resources | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/redis + - name: data + mountPath: /data + args: ["redis-server", "/etc/redis/redis.conf"] + readinessProbe: + exec: + command: + - redis-cli + - ping + initialDelaySeconds: 5 + periodSeconds: 5 + livenessProbe: + exec: + command: + - redis-cli + - ping + initialDelaySeconds: 10 + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + # Sentinel sidecar on master + - name: sentinel + image: "{{ .Values.redis.sentinel.image.repository }}:{{ .Values.redis.sentinel.image.tag }}" + ports: + - name: sentinel + containerPort: {{ .Values.redis.ports.sentinel }} + resources: + {{- toYaml .Values.redis.sentinel.resources | nindent 12 }} + volumeMounts: + - name: sentinel-config + mountPath: /etc/redis-sentinel + command: ["/bin/bash", "/etc/redis-scripts/init-sentinel.sh"] + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumes: + - name: config + configMap: + name: redis-config + - name: sentinel-config + configMap: + name: redis-sentinel-config + - name: scripts + configMap: + name: redis-scripts + defaultMode: 0755 + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.redis.master.storage.size }} + storageClassName: {{ .Values.redis.master.storage.storageClass }} +--- +# Redis Replicas +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis-replica + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: redis + redis-role: replica +spec: + serviceName: redis-replica + replicas: {{ .Values.redis.replica.replicaCount }} + selector: + matchLabels: + {{- include "osint-dashboard.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: redis + redis-role: replica + template: + metadata: + labels: + {{- include "osint-dashboard.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: redis + redis-role: replica + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9121" + spec: + securityContext: + runAsNonRoot: true + runAsUser: 999 + fsGroup: 999 + containers: + - name: redis + image: "{{ .Values.redis.image.repository }}:{{ .Values.redis.image.tag }}" + ports: + - name: redis + containerPort: {{ .Values.redis.ports.redis }} + resources: + {{- toYaml .Values.redis.replica.resources | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/redis + - name: data + mountPath: /data + env: + - name: REDIS_ROLE + value: "replica" + command: ["/bin/bash", "/etc/redis-scripts/redis.sh"] + readinessProbe: + exec: + command: + - redis-cli + - ping + initialDelaySeconds: 5 + periodSeconds: 5 + livenessProbe: + exec: + command: + - redis-cli + - ping + initialDelaySeconds: 10 + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + # Sentinel sidecar on replicas + - name: sentinel + image: "{{ .Values.redis.sentinel.image.repository }}:{{ .Values.redis.sentinel.image.tag }}" + ports: + - name: sentinel + containerPort: {{ .Values.redis.ports.sentinel }} + resources: + {{- toYaml .Values.redis.sentinel.resources | nindent 12 }} + volumeMounts: + - name: sentinel-config + mountPath: /etc/redis-sentinel + command: ["/bin/bash", "/etc/redis-scripts/init-sentinel.sh"] + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumes: + - name: config + configMap: + name: redis-config + - name: sentinel-config + configMap: + name: redis-sentinel-config + - name: scripts + configMap: + name: redis-scripts + defaultMode: 0755 + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.redis.replica.storage.size }} + storageClassName: {{ .Values.redis.replica.storage.storageClass }} +{{- end }} diff --git a/apps/base/osint-dashboard/templates/security/dashboard-netpol.yaml b/apps/base/osint-dashboard/templates/security/dashboard-netpol.yaml new file mode 100644 index 0000000..d756763 --- /dev/null +++ b/apps/base/osint-dashboard/templates/security/dashboard-netpol.yaml @@ -0,0 +1,145 @@ +{{- if .Values.networkPolicies.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: dashboard-web-netpol + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: security +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: web + policyTypes: + - Ingress + - Egress + ingress: + # Allow from ingress controller / Gateway API + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: ingress-nginx + ports: + - protocol: TCP + port: 3000 + egress: + # Allow DNS + - to: + - namespaceSelector: {} + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + # Allow to API + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: api + ports: + - protocol: TCP + port: 4000 + # Allow to external APIs (GDelt, satellite providers) + - to: + - ipBlock: + cidr: 0.0.0.0/0 + ports: + - protocol: TCP + port: 443 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: dashboard-api-netpol + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: security +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: api + policyTypes: + - Ingress + - Egress + ingress: + # Allow from web frontend + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: web + ports: + - protocol: TCP + port: 4000 + # Allow from ingress controller / Gateway API + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: ingress-nginx + ports: + - protocol: TCP + port: 4000 + egress: + # Allow DNS + - to: + - namespaceSelector: {} + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + # Allow to PostgreSQL + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: database + ports: + - protocol: TCP + port: 5432 + # Allow to Redis + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: cache + ports: + - protocol: TCP + port: {{ .Values.redis.ports.redis }} + # Allow to NATS + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: messaging + ports: + - protocol: TCP + port: {{ .Values.nats.ports.client }} + # Allow to MinIO + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: object-storage + ports: + - protocol: TCP + port: {{ .Values.minio.ports.api }} + # Allow to external APIs (GDelt, etc.) + - to: + - ipBlock: + cidr: 0.0.0.0/0 + ports: + - protocol: TCP + port: 443 +{{- end }} diff --git a/apps/base/osint-dashboard/templates/security/default-deny.yaml b/apps/base/osint-dashboard/templates/security/default-deny.yaml new file mode 100644 index 0000000..9db14dd --- /dev/null +++ b/apps/base/osint-dashboard/templates/security/default-deny.yaml @@ -0,0 +1,17 @@ +{{- if .Values.networkPolicies.enabled }} +{{- if .Values.networkPolicies.defaultDeny }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-deny-all + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: security +spec: + podSelector: {} + policyTypes: + - Ingress + - Egress +{{- end }} +{{- end }} diff --git a/apps/base/osint-dashboard/templates/security/minio-netpol.yaml b/apps/base/osint-dashboard/templates/security/minio-netpol.yaml new file mode 100644 index 0000000..fe57721 --- /dev/null +++ b/apps/base/osint-dashboard/templates/security/minio-netpol.yaml @@ -0,0 +1,56 @@ +{{- if .Values.networkPolicies.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: minio-netpol + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: security +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: object-storage + policyTypes: + - Ingress + - Egress + ingress: + # Allow from dashboard services + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + ports: + - protocol: TCP + port: {{ .Values.minio.ports.api }} + # Allow console access (internal) + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + ports: + - protocol: TCP + port: {{ .Values.minio.ports.console }} + egress: + # Allow DNS + - to: + - namespaceSelector: {} + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + # Allow to other MinIO pods (distributed replication) + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: object-storage + ports: + - protocol: TCP + port: {{ .Values.minio.ports.api }} +{{- end }} diff --git a/apps/base/osint-dashboard/templates/security/monitoring-netpol.yaml b/apps/base/osint-dashboard/templates/security/monitoring-netpol.yaml new file mode 100644 index 0000000..fa24dd3 --- /dev/null +++ b/apps/base/osint-dashboard/templates/security/monitoring-netpol.yaml @@ -0,0 +1,60 @@ +{{- if .Values.networkPolicies.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: monitoring-netpol + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: security +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: monitoring + policyTypes: + - Ingress + - Egress + ingress: + # Allow Grafana web access + - from: [] + ports: + - protocol: TCP + port: {{ .Values.monitoring.grafana.port }} + - protocol: TCP + port: {{ .Values.monitoring.prometheus.port }} + - protocol: TCP + port: {{ .Values.monitoring.alertmanager.port }} + egress: + # Allow DNS + - to: + - namespaceSelector: {} + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + # Allow scraping all OSINT pods + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + ports: + - protocol: TCP + port: 9090 + - protocol: TCP + port: 9093 + - protocol: TCP + port: 3000 + - protocol: TCP + port: 9187 # postgres-exporter + - protocol: TCP + port: 9121 # redis-exporter + - protocol: TCP + port: 8222 # nats-monitor + - protocol: TCP + port: 9000 # minio +{{- end }} diff --git a/apps/base/osint-dashboard/templates/security/nats-netpol.yaml b/apps/base/osint-dashboard/templates/security/nats-netpol.yaml new file mode 100644 index 0000000..e45687f --- /dev/null +++ b/apps/base/osint-dashboard/templates/security/nats-netpol.yaml @@ -0,0 +1,67 @@ +{{- if .Values.networkPolicies.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: nats-netpol + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: security +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: messaging + policyTypes: + - Ingress + - Egress + ingress: + # Allow client connections from dashboard services + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + ports: + - protocol: TCP + port: {{ .Values.nats.ports.client }} + # Allow cluster communication + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: messaging + ports: + - protocol: TCP + port: {{ .Values.nats.ports.cluster }} + # Allow monitoring + - from: + - podSelector: + matchLabels: + app.kubernetes.io/component: monitoring + ports: + - protocol: TCP + port: {{ .Values.nats.ports.monitor }} + egress: + # Allow DNS + - to: + - namespaceSelector: {} + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + # Allow cluster communication + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: messaging + ports: + - protocol: TCP + port: {{ .Values.nats.ports.cluster }} + - protocol: TCP + port: {{ .Values.nats.ports.client }} +{{- end }} diff --git a/apps/base/osint-dashboard/templates/security/postgresql-netpol.yaml b/apps/base/osint-dashboard/templates/security/postgresql-netpol.yaml new file mode 100644 index 0000000..8735fe2 --- /dev/null +++ b/apps/base/osint-dashboard/templates/security/postgresql-netpol.yaml @@ -0,0 +1,60 @@ +{{- if .Values.networkPolicies.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: postgresql-netpol + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: security +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: database + policyTypes: + - Ingress + - Egress + ingress: + # Allow from dashboard API and workers + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + - podSelector: + matchLabels: + app.kubernetes.io/component: prometheus-exporter + ports: + - protocol: TCP + port: 5432 + # Allow from CNPG cluster peers (replication) + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: database + ports: + - protocol: TCP + port: 5432 + egress: + # Allow DNS + - to: + - namespaceSelector: {} + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + # Allow to other PG replicas (replication) + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: database + ports: + - protocol: TCP + port: 5432 +{{- end }} diff --git a/apps/base/osint-dashboard/templates/security/redis-netpol.yaml b/apps/base/osint-dashboard/templates/security/redis-netpol.yaml new file mode 100644 index 0000000..53ec5c9 --- /dev/null +++ b/apps/base/osint-dashboard/templates/security/redis-netpol.yaml @@ -0,0 +1,58 @@ +{{- if .Values.networkPolicies.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: redis-netpol + namespace: {{ .Values.namespace }} + labels: + {{- include "osint-dashboard.labels" . | nindent 4 }} + app.kubernetes.io/component: security +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: cache + policyTypes: + - Ingress + - Egress + ingress: + # Allow from dashboard services + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + ports: + - protocol: TCP + port: {{ .Values.redis.ports.redis }} + # Allow sentinel from dashboard + other sentinels + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + ports: + - protocol: TCP + port: {{ .Values.redis.ports.sentinel }} + egress: + # Allow DNS + - to: + - namespaceSelector: {} + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + # Allow to other Redis pods (replication) + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: osint-dashboard + app.kubernetes.io/component: cache + ports: + - protocol: TCP + port: {{ .Values.redis.ports.redis }} + - protocol: TCP + port: {{ .Values.redis.ports.sentinel }} +{{- end }} diff --git a/apps/base/osint-dashboard/values.yaml b/apps/base/osint-dashboard/values.yaml new file mode 100644 index 0000000..6752fed --- /dev/null +++ b/apps/base/osint-dashboard/values.yaml @@ -0,0 +1,360 @@ +# OSINT Dashboard — Helm Values +# Default values for development/staging. Override with values-prod.yaml for production. + +nameOverride: "" +fullnameOverride: "" + +# Namespace +namespace: customer1 + +# Global security context +securityContext: + runAsNonRoot: true + fsGroup: 1000 + +# Resource defaults +resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 128Mi + +# ============================================================ +# PostgreSQL (using CNPG — CloudNativePG, already installed) +# ============================================================ +postgresql: + enabled: true + clusterName: osint-pgdb + instances: 3 + imageName: ghcr.io/cloudnative-pg/postgresql:16 + # Custom image with PostGIS + TimescaleDB + # Use CNPG bootstrap to create extensions + storage: + size: 200Gi + storageClass: premium-rwo + resources: + requests: + cpu: "2" + memory: 4Gi + limits: + cpu: "4" + memory: 8Gi + extensions: + - postgis + - timescaledb + backup: + retentionPolicy: "30d" + bucket: "gs://osint-dashboard-db-backup/" + credentialsSecret: osint-pg-credentials + +# ============================================================ +# NATS JetStream (3 replicas, persistent) +# ============================================================ +nats: + enabled: true + replicaCount: 3 + image: + repository: nats + tag: "2.10.18-alpine" + resources: + requests: + cpu: "500m" + memory: 512Mi + limits: + cpu: "1" + memory: 1Gi + storage: + size: 50Gi + storageClass: premium-rwo + jetstream: + enabled: true + maxMemory: 2Gi + fileStore: /data/jetstream + ports: + client: 4222 + cluster: 6222 + monitor: 8222 + websocket: 8080 + # JetStream subjects schema + subjects: + streams: + - name: events + subjects: + - events.gdelt + - events.rss + - events.social + - events.earthquake + - events.disaster + - events.weather + - events.fire + - events.satellite + - events.new + - events.alert + retention: interests + maxConsumers: -1 + maxMsgs: 1000000 + maxBytes: 1073741824 # 1GB + discard: old + - name: alerts + subjects: + - alerts.camera_offline + - alerts.new + retention: interests + maxConsumers: -1 + maxMsgs: 100000 + discard: old + - name: video + subjects: + - "video.status.>" + - "video.record.>" + retention: limits + maxConsumers: -1 + maxMsgs: 50000 + discard: old + +# ============================================================ +# Redis Sentinel (1 primary + 2 replicas) +# ============================================================ +redis: + enabled: true + image: + repository: redis + tag: "7.4-alpine" + sentinel: + image: + repository: redis + tag: "7.4-alpine" + master: + replicaCount: 1 + resources: + requests: + cpu: "500m" + memory: 512Mi + limits: + cpu: "1" + memory: 1Gi + storage: + size: 20Gi + storageClass: premium-rwo + replica: + replicaCount: 2 + resources: + requests: + cpu: "250m" + memory: 256Mi + limits: + cpu: "500m" + memory: 512Mi + storage: + size: 20Gi + storageClass: premium-rwo + sentinel: + replicaCount: 3 + resources: + requests: + cpu: "100m" + memory: 128Mi + limits: + cpu: "250m" + memory: 256Mi + ports: + redis: 6379 + sentinel: 26379 + +# ============================================================ +# MinIO (4 replicas, distributed mode) +# ============================================================ +minio: + enabled: true + image: + repository: quay.io/minio/minio + tag: "latest" + replicaCount: 4 + mode: distributed + resources: + requests: + cpu: "500m" + memory: 1Gi + limits: + cpu: "1" + memory: 2Gi + storage: + size: 500Gi + storageClass: premium-rwo + buckets: + - name: osint-video-clips + policy: none + - name: osint-satellite-tiles + policy: none + - name: osint-data-dumps + policy: none + credentialsSecret: osint-minio-credentials + ports: + api: 9000 + console: 9001 + +# ============================================================ +# NGINX Ingress Controller + cert-manager +# ============================================================ +ingress: + enabled: true + # Use existing cert-manager cluster issuer + certManager: + enabled: true + clusterIssuerName: letsencrypt-prod + hosts: + - host: dashboard.siriusdevops.com + paths: + - path: / + pathType: Prefix + - host: api.siriusdevops.com + paths: + - path: / + pathType: Prefix + - host: ws.siriusdevops.com + paths: + - path: / + pathType: Prefix + tls: + - secretName: osint-dashboard-tls + hosts: + - dashboard.siriusdevops.com + - secretName: osint-api-tls + hosts: + - api.siriusdevops.com + annotations: + nginx.ingress.kubernetes.io/proxy-read-timeout: "3600" + nginx.ingress.kubernetes.io/proxy-send-timeout: "3600" + nginx.ingress.kubernetes.io/proxy-body-size: "50m" + # WebSocket support + nginx.ingress.kubernetes.io/use-regex: "true" + +# ============================================================ +# Monitoring (Prometheus + Grafana + Alertmanager) +# ============================================================ +monitoring: + enabled: true + defaultEmailDomain: siriusdevops.com + + prometheus: + image: + repository: prom/prometheus + tag: "v2.53.0" + port: 9090 + retention: "30d" + retentionSize: "50GB" + resources: + requests: + cpu: "500m" + memory: 2Gi + limits: + cpu: "1" + memory: 4Gi + + alertmanager: + image: + repository: prom/alertmanager + tag: "v0.27.0" + port: 9093 + resources: + requests: + cpu: "100m" + memory: 128Mi + limits: + cpu: "250m" + memory: 256Mi + + grafana: + image: + repository: grafana/grafana + tag: "11.2.0" + port: 3000 + hostname: grafana.siriusdevops.com + resources: + requests: + cpu: "200m" + memory: 256Mi + limits: + cpu: "500m" + memory: 512Mi + + exporters: + nats: + enabled: true + image: + repository: natsio/prometheus-nats-exporter + tag: "0.14.0" + postgresql: + enabled: true + image: + repository: prometheuscommunity/postgres-exporter + tag: "0.15.0" + exporter: + replicas: 3 + redis: + enabled: true + image: + repository: oliver006/redis_exporter + tag: "v1.58.0" + minio: + enabled: true + # MinIO has built-in metrics at /minio/v2/metrics/cluster + + grafanaDashboards: + - osint-overview + - nats-jetstream + - postgresql-performance + - redis-sentinel + - minio-storage + +# ============================================================ +# CI/CD +# ============================================================ +cicd: + enabled: true + registry: gcr.io/devops-lab-cluster + previewEnvironments: true + +# ============================================================ +# Network Policies +# ============================================================ +networkPolicies: + enabled: true + # Default deny all ingress/egress, then allow specific traffic + defaultDeny: true + +# ============================================================ +# Frontend (placeholder — T3 will fill this in) +# ============================================================ +frontend: + enabled: false + replicaCount: 3 + image: + repository: gcr.io/devops-lab-cluster/osint-dashboard-web + tag: latest + resources: + requests: + cpu: "100m" + memory: 128Mi + limits: + cpu: "500m" + memory: 512Mi + +# ============================================================ +# API (placeholder — T2 will fill this in) +# ============================================================ +api: + enabled: false + replicaCount: 3 + image: + repository: gcr.io/devops-lab-cluster/osint-dashboard-api + tag: latest + resources: + requests: + cpu: "200m" + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi diff --git a/apps/staging/osint-dashboard/kustomization.yaml b/apps/staging/osint-dashboard/kustomization.yaml new file mode 100644 index 0000000..3a38b16 --- /dev/null +++ b/apps/staging/osint-dashboard/kustomization.yaml @@ -0,0 +1,53 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Reference the Helm chart base +helmCharts: + - name: osint-dashboard + repository: + name: osint-dashboard-local + type: oci + url: oci://gcr.io/devops-lab-cluster/charts + version: "0.1.0" + releaseName: osint-dashboard + namespace: customer1 + includeCRDs: true + +# Or use plain Kustomize overlay on the base templates +resources: + - ../../base/osint-dashboard/templates/namespace.yaml + +# Namespace override +namespace: customer1 + +# Common labels +commonLabels: + app.kubernetes.io/managed-by: flux + app.kubernetes.io/part-of: osint-dashboard + +# Patches for staging environment +patches: + # Override replica counts for staging + - patch: |- + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: postgresql + spec: + replicas: 2 + target: + kind: StatefulSet + name: postgresql.* + + # Reduce storage for staging + - patch: |- + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: data + spec: + resources: + requests: + storage: 100Gi + target: + kind: PersistentVolumeClaim From b0748538c0d0c8ea3c7237e641297216b2d1e068 Mon Sep 17 00:00:00 2001 From: Sirius Devops Date: Thu, 21 May 2026 04:08:58 +0000 Subject: [PATCH 25/28] feat: add trading-platform K8s manifests for customer1 - Scaffold trading-platform under apps/base/customer1/trading-platform/ - Add 4 microservice deployments: data-service, execute-service, news-service, dashboard - Add ConfigMaps with DB/Kafka/Redis connection configs - Add HTTPRoutes via Gateway API (external-http-gateway) - Add NetworkPolicies for inter-service and DB/Kafka/Redis egress - Add SOPS-encrypted secrets (age key) - Add Kafka (KRaft) and Redis infrastructure to hermes-db/ - Update root and staging kustomizations All containers: non-root, readOnlyRootFilesystem, resource limits, health probes Images: ghcr.io/sirius0xdev/trading-{service}:latest --- .../customer1/hermes-db/kafka-broker.yaml | 128 ++++++++++++++++++ .../customer1/hermes-db/kustomization.yaml | 2 + .../customer1/hermes-db/redis-cluster.yaml | 103 ++++++++++++++ apps/base/customer1/kustomization.yaml | 3 +- .../configmaps/dashboard-config.yaml | 12 ++ .../configmaps/data-service-config.yaml | 13 ++ .../configmaps/execute-service-config.yaml | 13 ++ .../configmaps/kustomization.yaml | 8 ++ .../configmaps/news-service-config.yaml | 13 ++ .../dashboard/deployment.yaml | 79 +++++++++++ .../dashboard/kustomization.yaml | 6 + .../trading-platform/dashboard/service.yaml | 18 +++ .../data-service/deployment.yaml | 79 +++++++++++ .../data-service/kustomization.yaml | 6 + .../data-service/service.yaml | 18 +++ .../execute-service/deployment.yaml | 79 +++++++++++ .../execute-service/kustomization.yaml | 6 + .../execute-service/service.yaml | 18 +++ .../trading-platform/kustomization.yaml | 13 ++ .../network-policies/kustomization.yaml | 5 + .../trading-network-policies.yaml | 109 +++++++++++++++ .../news-service/deployment.yaml | 79 +++++++++++ .../news-service/kustomization.yaml | 6 + .../news-service/service.yaml | 18 +++ .../trading-platform/routes/http-routes.yaml | 53 ++++++++ .../routes/kustomization.yaml | 5 + .../secrets/kustomization.yaml | 5 + .../secrets/trading-secrets.yaml | 32 +++++ apps/staging/customer1/kustomization.yaml | 1 + 29 files changed, 929 insertions(+), 1 deletion(-) create mode 100644 apps/base/customer1/hermes-db/kafka-broker.yaml create mode 100644 apps/base/customer1/hermes-db/redis-cluster.yaml create mode 100644 apps/base/customer1/trading-platform/configmaps/dashboard-config.yaml create mode 100644 apps/base/customer1/trading-platform/configmaps/data-service-config.yaml create mode 100644 apps/base/customer1/trading-platform/configmaps/execute-service-config.yaml create mode 100644 apps/base/customer1/trading-platform/configmaps/kustomization.yaml create mode 100644 apps/base/customer1/trading-platform/configmaps/news-service-config.yaml create mode 100644 apps/base/customer1/trading-platform/dashboard/deployment.yaml create mode 100644 apps/base/customer1/trading-platform/dashboard/kustomization.yaml create mode 100644 apps/base/customer1/trading-platform/dashboard/service.yaml create mode 100644 apps/base/customer1/trading-platform/data-service/deployment.yaml create mode 100644 apps/base/customer1/trading-platform/data-service/kustomization.yaml create mode 100644 apps/base/customer1/trading-platform/data-service/service.yaml create mode 100644 apps/base/customer1/trading-platform/execute-service/deployment.yaml create mode 100644 apps/base/customer1/trading-platform/execute-service/kustomization.yaml create mode 100644 apps/base/customer1/trading-platform/execute-service/service.yaml create mode 100644 apps/base/customer1/trading-platform/kustomization.yaml create mode 100644 apps/base/customer1/trading-platform/network-policies/kustomization.yaml create mode 100644 apps/base/customer1/trading-platform/network-policies/trading-network-policies.yaml create mode 100644 apps/base/customer1/trading-platform/news-service/deployment.yaml create mode 100644 apps/base/customer1/trading-platform/news-service/kustomization.yaml create mode 100644 apps/base/customer1/trading-platform/news-service/service.yaml create mode 100644 apps/base/customer1/trading-platform/routes/http-routes.yaml create mode 100644 apps/base/customer1/trading-platform/routes/kustomization.yaml create mode 100644 apps/base/customer1/trading-platform/secrets/kustomization.yaml create mode 100644 apps/base/customer1/trading-platform/secrets/trading-secrets.yaml diff --git a/apps/base/customer1/hermes-db/kafka-broker.yaml b/apps/base/customer1/hermes-db/kafka-broker.yaml new file mode 100644 index 0000000..04f86b1 --- /dev/null +++ b/apps/base/customer1/hermes-db/kafka-broker.yaml @@ -0,0 +1,128 @@ +# Kafka broker (KRaft mode — no ZooKeeper required) +# Single-broker for dev/staging; scale replicas for production +apiVersion: v1 +kind: Service +metadata: + name: trading-kafka + namespace: customer1 + labels: + app: trading-kafka +spec: + clusterIP: None + selector: + app: trading-kafka + ports: + - name: internal + port: 9092 + targetPort: 9092 + - name: controller + port: 9093 + targetPort: 9093 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: trading-kafka-config + namespace: customer1 +data: + server.properties: | + process.roles=broker,controller + node.id=1 + controller.quorum.voters=1@trading-kafka-0.trading-kafka.customer1.svc.cluster.local:9093 + listeners=PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093 + advertised.listeners=PLAINTEXT://trading-kafka-0.trading-kafka.customer1.svc.cluster.local:9092 + listener.security.protocol.map=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT + controller.listener.names=CONTROLLER + inter.broker.listener.name=PLAINTEXT + log.dirs=/var/lib/kafka/data + num.partitions=3 + default.replication.factor=1 + offsets.topic.replication.factor=1 + transaction.state.log.replication.factor=1 + transaction.state.log.min.isr=1 + auto.create.topics.enable=true +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: trading-kafka + namespace: customer1 + labels: + app: trading-kafka +spec: + serviceName: trading-kafka + replicas: 1 + selector: + matchLabels: + app: trading-kafka + template: + metadata: + labels: + app: trading-kafka + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + containers: + - name: kafka + image: apache/kafka:3.9.0 + ports: + - containerPort: 9092 + name: internal + - containerPort: 9093 + name: controller + env: + - name: KAFKA_HEAP_OPTS + value: "-Xmx512M -Xms256M" + - name: CLUSTER_ID + value: "trading-kafka-cluster-01" + command: + - /bin/bash + - -c + - | + export KAFKA_CLUSTER_ID="$(/opt/kafka/bin/kafka-storage.sh random-uuid)" + /opt/kafka/bin/kafka-storage.sh format -t $KAFKA_CLUSTER_ID -c /etc/kafka/server.properties --ignore-formatted + exec /opt/kafka/bin/kafka-server-start.sh /etc/kafka/server.properties + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 1000m + memory: 1Gi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + capabilities: + drop: + - ALL + volumeMounts: + - name: config + mountPath: /etc/kafka + - name: data + mountPath: /var/lib/kafka/data + readinessProbe: + tcpSocket: + port: 9092 + initialDelaySeconds: 30 + periodSeconds: 10 + failureThreshold: 5 + livenessProbe: + tcpSocket: + port: 9092 + initialDelaySeconds: 60 + periodSeconds: 30 + volumes: + - name: config + configMap: + name: trading-kafka-config + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/apps/base/customer1/hermes-db/kustomization.yaml b/apps/base/customer1/hermes-db/kustomization.yaml index 9f9677f..7047ec6 100644 --- a/apps/base/customer1/hermes-db/kustomization.yaml +++ b/apps/base/customer1/hermes-db/kustomization.yaml @@ -9,3 +9,5 @@ resources: - trading-data-db.yaml - agent-memory-db.yaml - hermes-scheduled-backup.yaml + - kafka-broker.yaml + - redis-cluster.yaml diff --git a/apps/base/customer1/hermes-db/redis-cluster.yaml b/apps/base/customer1/hermes-db/redis-cluster.yaml new file mode 100644 index 0000000..fadd4d4 --- /dev/null +++ b/apps/base/customer1/hermes-db/redis-cluster.yaml @@ -0,0 +1,103 @@ +# Redis single-instance for trading platform caching +apiVersion: v1 +kind: Service +metadata: + name: trading-redis + namespace: customer1 + labels: + app: trading-redis +spec: + selector: + app: trading-redis + ports: + - port: 6379 + targetPort: 6379 + name: redis + type: ClusterIP +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: trading-redis-config + namespace: customer1 +data: + redis.conf: | + maxmemory 256mb + maxmemory-policy allkeys-lru + save 900 1 + save 300 10 + save 60 10000 + appendonly yes + appendfsync everysec + dir /data +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: trading-redis + namespace: customer1 + labels: + app: trading-redis +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: trading-redis + template: + metadata: + labels: + app: trading-redis + spec: + securityContext: + runAsNonRoot: true + runAsUser: 999 + fsGroup: 999 + containers: + - name: redis + image: redis:7.4-alpine + ports: + - containerPort: 6379 + name: redis + args: + - redis-server + - /etc/redis/redis.conf + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: config + mountPath: /etc/redis + - name: data + mountPath: /data + readinessProbe: + exec: + command: + - redis-cli + - ping + initialDelaySeconds: 5 + periodSeconds: 5 + livenessProbe: + exec: + command: + - redis-cli + - ping + initialDelaySeconds: 10 + periodSeconds: 10 + volumes: + - name: config + configMap: + name: trading-redis-config + - name: data + emptyDir: {} diff --git a/apps/base/customer1/kustomization.yaml b/apps/base/customer1/kustomization.yaml index 51573fe..1ac48ec 100644 --- a/apps/base/customer1/kustomization.yaml +++ b/apps/base/customer1/kustomization.yaml @@ -6,4 +6,5 @@ resources: - http-route.yaml - service.yaml - siriusdevops-site - - waitlist-api \ No newline at end of file + - waitlist-api + - trading-platform diff --git a/apps/base/customer1/trading-platform/configmaps/dashboard-config.yaml b/apps/base/customer1/trading-platform/configmaps/dashboard-config.yaml new file mode 100644 index 0000000..4fcdaee --- /dev/null +++ b/apps/base/customer1/trading-platform/configmaps/dashboard-config.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: trading-dashboard-config + namespace: customer1 +data: + DB_HOST: "hermes-pgdb-rw.customer1.svc.cluster.local" + DB_PORT: "5432" + DB_NAME: "trading_data" + DATA_SERVICE_URL: "http://trading-data-service.customer1.svc.cluster.local" + EXECUTE_SERVICE_URL: "http://trading-execute-service.customer1.svc.cluster.local" + NEWS_SERVICE_URL: "http://trading-news-service.customer1.svc.cluster.local" diff --git a/apps/base/customer1/trading-platform/configmaps/data-service-config.yaml b/apps/base/customer1/trading-platform/configmaps/data-service-config.yaml new file mode 100644 index 0000000..c9dc9e2 --- /dev/null +++ b/apps/base/customer1/trading-platform/configmaps/data-service-config.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: trading-data-service-config + namespace: customer1 +data: + DB_HOST: "hermes-pgdb-rw.customer1.svc.cluster.local" + DB_PORT: "5432" + DB_NAME: "trading_data" + REDIS_HOST: "trading-redis.customer1.svc.cluster.local" + REDIS_PORT: "6379" + KAFKA_BROKER: "trading-kafka.customer1.svc.cluster.local:9092" + LOG_LEVEL: "info" diff --git a/apps/base/customer1/trading-platform/configmaps/execute-service-config.yaml b/apps/base/customer1/trading-platform/configmaps/execute-service-config.yaml new file mode 100644 index 0000000..090157f --- /dev/null +++ b/apps/base/customer1/trading-platform/configmaps/execute-service-config.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: trading-execute-service-config + namespace: customer1 +data: + DB_HOST: "hermes-pgdb-rw.customer1.svc.cluster.local" + DB_PORT: "5432" + DB_NAME: "trading_data" + REDIS_HOST: "trading-redis.customer1.svc.cluster.local" + REDIS_PORT: "6379" + KAFKA_BROKER: "trading-kafka.customer1.svc.cluster.local:9092" + LOG_LEVEL: "info" diff --git a/apps/base/customer1/trading-platform/configmaps/kustomization.yaml b/apps/base/customer1/trading-platform/configmaps/kustomization.yaml new file mode 100644 index 0000000..e34cabf --- /dev/null +++ b/apps/base/customer1/trading-platform/configmaps/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - data-service-config.yaml + - execute-service-config.yaml + - news-service-config.yaml + - dashboard-config.yaml diff --git a/apps/base/customer1/trading-platform/configmaps/news-service-config.yaml b/apps/base/customer1/trading-platform/configmaps/news-service-config.yaml new file mode 100644 index 0000000..ec4e416 --- /dev/null +++ b/apps/base/customer1/trading-platform/configmaps/news-service-config.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: trading-news-service-config + namespace: customer1 +data: + DB_HOST: "hermes-pgdb-rw.customer1.svc.cluster.local" + DB_PORT: "5432" + DB_NAME: "trading_data" + REDIS_HOST: "trading-redis.customer1.svc.cluster.local" + REDIS_PORT: "6379" + NEWS_FETCH_INTERVAL: "300" + LOG_LEVEL: "info" diff --git a/apps/base/customer1/trading-platform/dashboard/deployment.yaml b/apps/base/customer1/trading-platform/dashboard/deployment.yaml new file mode 100644 index 0000000..c2f3c1e --- /dev/null +++ b/apps/base/customer1/trading-platform/dashboard/deployment.yaml @@ -0,0 +1,79 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: trading-dashboard + namespace: customer1 + labels: + app: trading-dashboard +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: trading-dashboard + template: + metadata: + labels: + app: trading-dashboard + annotations: + checksum/config: trading-dashboard-config + spec: + terminationGracePeriodSeconds: 30 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + containers: + - name: dashboard + image: ghcr.io/sirius0xdev/trading-dashboard:latest + imagePullPolicy: Always + ports: + - containerPort: 8000 + name: http + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + envFrom: + - configMapRef: + name: trading-dashboard-config + env: + - name: DB_USER + valueFrom: + secretKeyRef: + name: trading-db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: trading-db-credentials + key: password + startupProbe: + httpGet: + path: /api/health + port: 8000 + initialDelaySeconds: 15 + periodSeconds: 5 + failureThreshold: 6 + livenessProbe: + httpGet: + path: /api/health + port: 8000 + initialDelaySeconds: 30 + periodSeconds: 30 + readinessProbe: + httpGet: + path: /api/health + port: 8000 + initialDelaySeconds: 5 + periodSeconds: 10 diff --git a/apps/base/customer1/trading-platform/dashboard/kustomization.yaml b/apps/base/customer1/trading-platform/dashboard/kustomization.yaml new file mode 100644 index 0000000..5b98e94 --- /dev/null +++ b/apps/base/customer1/trading-platform/dashboard/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - deployment.yaml + - service.yaml diff --git a/apps/base/customer1/trading-platform/dashboard/service.yaml b/apps/base/customer1/trading-platform/dashboard/service.yaml new file mode 100644 index 0000000..73bca2f --- /dev/null +++ b/apps/base/customer1/trading-platform/dashboard/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: trading-dashboard-svc + namespace: customer1 + annotations: + tailscale.com/expose: "true" + tailscale.com/hostname: "trading-dashboard" + tailscale.com/tags: "tag:k8s-operator" + tailscale.com/ports: "http:80" +spec: + selector: + app: trading-dashboard + ports: + - port: 80 + targetPort: 8000 + name: http + type: ClusterIP diff --git a/apps/base/customer1/trading-platform/data-service/deployment.yaml b/apps/base/customer1/trading-platform/data-service/deployment.yaml new file mode 100644 index 0000000..f1f88a7 --- /dev/null +++ b/apps/base/customer1/trading-platform/data-service/deployment.yaml @@ -0,0 +1,79 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: trading-data-service + namespace: customer1 + labels: + app: trading-data-service +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: trading-data-service + template: + metadata: + labels: + app: trading-data-service + annotations: + checksum/config: trading-data-service-config + spec: + terminationGracePeriodSeconds: 30 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + containers: + - name: data-service + image: ghcr.io/sirius0xdev/trading-data-service:latest + imagePullPolicy: Always + ports: + - containerPort: 8001 + name: http + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + envFrom: + - configMapRef: + name: trading-data-service-config + env: + - name: DB_USER + valueFrom: + secretKeyRef: + name: trading-db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: trading-db-credentials + key: password + startupProbe: + httpGet: + path: /health + port: 8001 + initialDelaySeconds: 15 + periodSeconds: 5 + failureThreshold: 6 + livenessProbe: + httpGet: + path: /health + port: 8001 + initialDelaySeconds: 30 + periodSeconds: 30 + readinessProbe: + httpGet: + path: /health + port: 8001 + initialDelaySeconds: 5 + periodSeconds: 10 diff --git a/apps/base/customer1/trading-platform/data-service/kustomization.yaml b/apps/base/customer1/trading-platform/data-service/kustomization.yaml new file mode 100644 index 0000000..5b98e94 --- /dev/null +++ b/apps/base/customer1/trading-platform/data-service/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - deployment.yaml + - service.yaml diff --git a/apps/base/customer1/trading-platform/data-service/service.yaml b/apps/base/customer1/trading-platform/data-service/service.yaml new file mode 100644 index 0000000..562d721 --- /dev/null +++ b/apps/base/customer1/trading-platform/data-service/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: trading-data-service + namespace: customer1 + annotations: + tailscale.com/expose: "true" + tailscale.com/hostname: "trading-data-service" + tailscale.com/tags: "tag:k8s-operator" + tailscale.com/ports: "http:80" +spec: + selector: + app: trading-data-service + ports: + - port: 80 + targetPort: 8001 + name: http + type: ClusterIP diff --git a/apps/base/customer1/trading-platform/execute-service/deployment.yaml b/apps/base/customer1/trading-platform/execute-service/deployment.yaml new file mode 100644 index 0000000..2881071 --- /dev/null +++ b/apps/base/customer1/trading-platform/execute-service/deployment.yaml @@ -0,0 +1,79 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: trading-execute-service + namespace: customer1 + labels: + app: trading-execute-service +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: trading-execute-service + template: + metadata: + labels: + app: trading-execute-service + annotations: + checksum/config: trading-execute-service-config + spec: + terminationGracePeriodSeconds: 30 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + containers: + - name: execute-service + image: ghcr.io/sirius0xdev/trading-execute-service:latest + imagePullPolicy: Always + ports: + - containerPort: 8002 + name: http + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + envFrom: + - configMapRef: + name: trading-execute-service-config + env: + - name: DB_USER + valueFrom: + secretKeyRef: + name: trading-db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: trading-db-credentials + key: password + startupProbe: + httpGet: + path: /health + port: 8002 + initialDelaySeconds: 15 + periodSeconds: 5 + failureThreshold: 6 + livenessProbe: + httpGet: + path: /health + port: 8002 + initialDelaySeconds: 30 + periodSeconds: 30 + readinessProbe: + httpGet: + path: /health + port: 8002 + initialDelaySeconds: 5 + periodSeconds: 10 diff --git a/apps/base/customer1/trading-platform/execute-service/kustomization.yaml b/apps/base/customer1/trading-platform/execute-service/kustomization.yaml new file mode 100644 index 0000000..5b98e94 --- /dev/null +++ b/apps/base/customer1/trading-platform/execute-service/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - deployment.yaml + - service.yaml diff --git a/apps/base/customer1/trading-platform/execute-service/service.yaml b/apps/base/customer1/trading-platform/execute-service/service.yaml new file mode 100644 index 0000000..49a0b42 --- /dev/null +++ b/apps/base/customer1/trading-platform/execute-service/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: trading-execute-service + namespace: customer1 + annotations: + tailscale.com/expose: "true" + tailscale.com/hostname: "trading-execute-service" + tailscale.com/tags: "tag:k8s-operator" + tailscale.com/ports: "http:80" +spec: + selector: + app: trading-execute-service + ports: + - port: 80 + targetPort: 8002 + name: http + type: ClusterIP diff --git a/apps/base/customer1/trading-platform/kustomization.yaml b/apps/base/customer1/trading-platform/kustomization.yaml new file mode 100644 index 0000000..f7775fe --- /dev/null +++ b/apps/base/customer1/trading-platform/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: customer1 + +resources: + - data-service + - execute-service + - news-service + - dashboard + - configmaps + - secrets + - network-policies + - routes diff --git a/apps/base/customer1/trading-platform/network-policies/kustomization.yaml b/apps/base/customer1/trading-platform/network-policies/kustomization.yaml new file mode 100644 index 0000000..0e9a94a --- /dev/null +++ b/apps/base/customer1/trading-platform/network-policies/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - trading-network-policies.yaml diff --git a/apps/base/customer1/trading-platform/network-policies/trading-network-policies.yaml b/apps/base/customer1/trading-platform/network-policies/trading-network-policies.yaml new file mode 100644 index 0000000..a553f07 --- /dev/null +++ b/apps/base/customer1/trading-platform/network-policies/trading-network-policies.yaml @@ -0,0 +1,109 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: trading-platform-netpol + namespace: customer1 +spec: + podSelector: + matchExpressions: + - key: app + operator: In + values: + - trading-data-service + - trading-execute-service + - trading-news-service + - trading-dashboard + policyTypes: + - Ingress + - Egress + ingress: + # Allow from Gateway / ingress controller + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: customer1 + ports: + - port: 8000 + protocol: TCP + - port: 8001 + protocol: TCP + - port: 8002 + protocol: TCP + - port: 8003 + protocol: TCP + # Allow inter-service communication + - from: + - podSelector: + matchExpressions: + - key: app + operator: In + values: + - trading-data-service + - trading-execute-service + - trading-news-service + - trading-dashboard + ports: + - port: 8000 + protocol: TCP + - port: 8001 + protocol: TCP + - port: 8002 + protocol: TCP + - port: 8003 + protocol: TCP + egress: + # Allow DNS + - to: + - namespaceSelector: {} + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + # Allow DB access + - to: + - podSelector: + matchLabels: + cnpg.io/cluster: hermes-pgdb + ports: + - port: 5432 + protocol: TCP + # Allow Redis access + - to: + - podSelector: + matchLabels: + app: trading-redis + ports: + - port: 6379 + protocol: TCP + # Allow Kafka access + - to: + - podSelector: + matchLabels: + app: trading-kafka + ports: + - port: 9092 + protocol: TCP + # Allow inter-service egress + - to: + - podSelector: + matchExpressions: + - key: app + operator: In + values: + - trading-data-service + - trading-execute-service + - trading-news-service + - trading-dashboard + ports: + - port: 8000 + protocol: TCP + - port: 8001 + protocol: TCP + - port: 8002 + protocol: TCP + - port: 8003 + protocol: TCP diff --git a/apps/base/customer1/trading-platform/news-service/deployment.yaml b/apps/base/customer1/trading-platform/news-service/deployment.yaml new file mode 100644 index 0000000..dda5c98 --- /dev/null +++ b/apps/base/customer1/trading-platform/news-service/deployment.yaml @@ -0,0 +1,79 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: trading-news-service + namespace: customer1 + labels: + app: trading-news-service +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: trading-news-service + template: + metadata: + labels: + app: trading-news-service + annotations: + checksum/config: trading-news-service-config + spec: + terminationGracePeriodSeconds: 30 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + containers: + - name: news-service + image: ghcr.io/sirius0xdev/trading-news-service:latest + imagePullPolicy: Always + ports: + - containerPort: 8003 + name: http + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + envFrom: + - configMapRef: + name: trading-news-service-config + env: + - name: DB_USER + valueFrom: + secretKeyRef: + name: trading-db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: trading-db-credentials + key: password + startupProbe: + httpGet: + path: /health + port: 8003 + initialDelaySeconds: 15 + periodSeconds: 5 + failureThreshold: 6 + livenessProbe: + httpGet: + path: /health + port: 8003 + initialDelaySeconds: 30 + periodSeconds: 30 + readinessProbe: + httpGet: + path: /health + port: 8003 + initialDelaySeconds: 5 + periodSeconds: 10 diff --git a/apps/base/customer1/trading-platform/news-service/kustomization.yaml b/apps/base/customer1/trading-platform/news-service/kustomization.yaml new file mode 100644 index 0000000..5b98e94 --- /dev/null +++ b/apps/base/customer1/trading-platform/news-service/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - deployment.yaml + - service.yaml diff --git a/apps/base/customer1/trading-platform/news-service/service.yaml b/apps/base/customer1/trading-platform/news-service/service.yaml new file mode 100644 index 0000000..0556352 --- /dev/null +++ b/apps/base/customer1/trading-platform/news-service/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: trading-news-service + namespace: customer1 + annotations: + tailscale.com/expose: "true" + tailscale.com/hostname: "trading-news-service" + tailscale.com/tags: "tag:k8s-operator" + tailscale.com/ports: "http:80" +spec: + selector: + app: trading-news-service + ports: + - port: 80 + targetPort: 8003 + name: http + type: ClusterIP diff --git a/apps/base/customer1/trading-platform/routes/http-routes.yaml b/apps/base/customer1/trading-platform/routes/http-routes.yaml new file mode 100644 index 0000000..794af95 --- /dev/null +++ b/apps/base/customer1/trading-platform/routes/http-routes.yaml @@ -0,0 +1,53 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: trading-dashboard-route + namespace: customer1 +spec: + parentRefs: + - name: external-http-gateway + hostnames: + - "sirius-sec.com" + - "www.sirius-sec.com" + rules: + - matches: + - path: + type: PathPrefix + value: /trade + backendRefs: + - name: trading-dashboard-svc + port: 80 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: trading-api-routes + namespace: customer1 +spec: + parentRefs: + - name: external-http-gateway + hostnames: + - "sirius-sec.com" + - "www.sirius-sec.com" + rules: + - matches: + - path: + type: PathPrefix + value: /api/data + backendRefs: + - name: trading-data-service + port: 80 + - matches: + - path: + type: PathPrefix + value: /api/execute + backendRefs: + - name: trading-execute-service + port: 80 + - matches: + - path: + type: PathPrefix + value: /api/news + backendRefs: + - name: trading-news-service + port: 80 diff --git a/apps/base/customer1/trading-platform/routes/kustomization.yaml b/apps/base/customer1/trading-platform/routes/kustomization.yaml new file mode 100644 index 0000000..e025033 --- /dev/null +++ b/apps/base/customer1/trading-platform/routes/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - http-routes.yaml diff --git a/apps/base/customer1/trading-platform/secrets/kustomization.yaml b/apps/base/customer1/trading-platform/secrets/kustomization.yaml new file mode 100644 index 0000000..c7525ed --- /dev/null +++ b/apps/base/customer1/trading-platform/secrets/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - trading-secrets.yaml diff --git a/apps/base/customer1/trading-platform/secrets/trading-secrets.yaml b/apps/base/customer1/trading-platform/secrets/trading-secrets.yaml new file mode 100644 index 0000000..d2c10e1 --- /dev/null +++ b/apps/base/customer1/trading-platform/secrets/trading-secrets.yaml @@ -0,0 +1,32 @@ +apiVersion: ENC[AES256_GCM,data:j4g=,iv:Bb3E3dbyD1MUIsthCltT2rRNNorduAL7QmPG8oRy31g=,tag:htxjh/00z6RGQNNyKCUviw==,type:str] +kind: ENC[AES256_GCM,data:k3FRKBzN,iv:Pvtu7bnGWQNl9pkjtU8GpHjpeg00oFXatf3jxxtQEpw=,tag:zj/yAy/DpCfLO7MdH3hiMA==,type:str] +metadata: + name: ENC[AES256_GCM,data:ZpbqQd2YyEfnugJ27JVAmqBf8Q7m75B0,iv:4njbS+2AYJuxzNERSOjOH8wmoI1KM0ocUT7OgeoGEgo=,tag:spwlHcpeQM0G8O3VPHoDOg==,type:str] + namespace: ENC[AES256_GCM,data:4ErEXcBaI8Yp,iv:IKHgZ6Gm5X21Atnnm2xOFU11IgSfw5X5Wdnl25EDyOI=,tag:znkdsNDkIb4mxBY4yJbX9g==,type:str] +type: ENC[AES256_GCM,data:dd7uKLw+,iv:qRkV8K+ytp55rLGNIP1lG2yZ+LENt/FkdDiWzi/1tik=,tag:aUWQ+ZOQS7/hXcnceCyrTQ==,type:str] +stringData: + #ENC[AES256_GCM,data:Xol7d8ednDll9VKfZ62jZRdARcmz8UJ/6ovDaw8OHrPb72aUdA==,iv:TvGk+LiK0+maCioF5daeWDTNKOSRA5pBFf9PgKL/9Z4=,tag:34kee3TQNsCdIpyEtE12mA==,type:comment] + news-api-key: ENC[AES256_GCM,data:eNMLhs55u3bwVqD4l2tQhgq1+wp/9rCa,iv:7BjlJqgJbg6BqdXxNphnIWKA/LYFZJc3qqJ360/EteY=,tag:hs5vfj+zP9OCq2tDucItIg==,type:str] + market-data-api-key: ENC[AES256_GCM,data:ol3aAC9ijFcFUo7jEVQv7mKjCLpUXBuaVThwRzG/fQ==,iv:YouuIdmj6aK0tuYJtDuiOt91gnUtK2xdshVZxy+gCH4=,tag:MAs8mS/+ndC8AYdI+WdFcA==,type:str] + #ENC[AES256_GCM,data:Si0AKl5/sWrYDNSYiC35iD7vtvP9CzAb,iv:cyBrgR5cFSta+bPdyyCUXrKH68Hi84UbGFqiWskQb7s=,tag:w9+sotMjBTuyfcGYIHfN+w==,type:comment] + service-auth-token: ENC[AES256_GCM,data:NCu/mBKdGIFNXTA5n7M53UXjMaZy9UdzImC90GPU,iv:JPLeNfTDWV3VLbVIUidL6w1IrJN09LTGIyPn0tJlj90=,tag:RIzC8FfGXBmE2PRZzgaAww==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age1uuxf066xuuqgvjppxfcmqkwfcufnwp3wcwnl9h20g9k4l8nkw9jsaungf7 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBFTzBrN3V5elJmNjNGZ28y + Q1JOQUt5eURqSFZNZkJUNWMrV1NTZEFiYkgwCnlDSTFYTW8zcU02M0NJdXZjYSti + Ylh0bWpJd0k2MWt6VjNNTUlNU0pTUVUKLS0tIENHUEU4VElXbC96bXBGRmo3QXpQ + ZkxxUDRubGt0dnRoQXVtS2xFSnhTRkUKs+rcKiZvgA7mffGo7GkkFL4vWnTIGAIn + RXwlbDNPEhiK+6lh/TgkV2CYXDBt1Hwfk4fzhZknYjY3Psp0ufvY2Q== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2026-05-21T04:08:04Z" + mac: ENC[AES256_GCM,data:EEuPQ1n7qAab7xkQYNt4rxzNy+u6PSYn+hFUTv3ZwmKbMHW7zQY3BgkwalDVrTC4ZNOLy3tGjVDB6v0KXmwXiXwvYL0Y17h8zRiU4id+zQl+oeZTMCFoUZ5Piz69DxO06cMaZF7+6K+9uQ0JLkZsnY3hb82xKAKbl9E/MFdvr6s=,iv:ldIAz5IKNFnbvcNpzo9qX6n0evix7tsLcTPiouB8lfk=,tag:KXDYrz4vfXkWH0cUHsOUdw==,type:str] + pgp: [] + unencrypted_suffix: _unencrypted + version: 3.9.4 diff --git a/apps/staging/customer1/kustomization.yaml b/apps/staging/customer1/kustomization.yaml index d35882f..692977d 100644 --- a/apps/staging/customer1/kustomization.yaml +++ b/apps/staging/customer1/kustomization.yaml @@ -11,3 +11,4 @@ resources: - ../../base/customer1/hermes-db/ - ../../base/customer1/trade-dashboard/ - ../../base/customer1/siriusdevops-db/ + - ../../base/customer1/trading-platform/ From 0f1c6bbdf5e1d81a214dd778e5eba07dd52f2f2a Mon Sep 17 00:00:00 2001 From: sirius0xdev Date: Thu, 21 May 2026 09:00:00 -0400 Subject: [PATCH 26/28] Update kustomization.yaml --- infrastructure/gpus/staging/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/gpus/staging/kustomization.yaml b/infrastructure/gpus/staging/kustomization.yaml index a328692..1373e08 100644 --- a/infrastructure/gpus/staging/kustomization.yaml +++ b/infrastructure/gpus/staging/kustomization.yaml @@ -1,5 +1,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - # - ../base/vllm-servers/ + - ../base/vllm-servers/ # - ../base/keda-gpu-scaling/ From 4a01c457ab6d7e022eb27f75fbf2938ca2367ed8 Mon Sep 17 00:00:00 2001 From: sirius0xdev Date: Thu, 21 May 2026 09:36:11 -0400 Subject: [PATCH 27/28] Update kustomization.yaml --- apps/base/customer1/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/base/customer1/kustomization.yaml b/apps/base/customer1/kustomization.yaml index 1ac48ec..37b91bb 100644 --- a/apps/base/customer1/kustomization.yaml +++ b/apps/base/customer1/kustomization.yaml @@ -7,4 +7,4 @@ resources: - service.yaml - siriusdevops-site - waitlist-api - - trading-platform + #- trading-platform From 3a9a0476d2db77e2d52ec1df22bbcffb42726190 Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Thu, 21 May 2026 13:52:21 +0000 Subject: [PATCH 28/28] docs: add repository reorganization migration plan Audit completed identifying all misplaced items in gcloud-lab. Plan proposes 6 PRs to move app code to hermes-projects and keep only K8s manifests and infra config in gcloud-lab. --- MIGRATION_PLAN.md | 203 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 203 insertions(+) create mode 100644 MIGRATION_PLAN.md diff --git a/MIGRATION_PLAN.md b/MIGRATION_PLAN.md new file mode 100644 index 0000000..6771dd3 --- /dev/null +++ b/MIGRATION_PLAN.md @@ -0,0 +1,203 @@ +# Repository Reorganization Plan + +## Goal + +Enforce the rule: **gcloud-lab = Kubernetes manifests only. No application code.** + +Application code (source, Dockerfiles, CI/CD workflows, Helm charts, deployment scripts) belongs in `hermes-projects/`. gcloud-lab should contain only K8s manifests, Terraform infrastructure modules, cluster configs, and infra controller configs. + +--- + +## Current State + +### gcloud-lab/ — Full Directory Audit + +``` +gcloud-lab/ +├── apps/ +│ ├── base/ +│ │ ├── customer1/ [KEEP] K8s manifests (deployments, services, configmaps, secrets) +│ │ ├── monitoring/ [KEEP] K8s manifests (dashboards) +│ │ └── osint-dashboard/ [KEEP] Helm chart (templates, values.yaml, Chart.yaml) +│ ├── staging/ +│ │ ├── customer1/ [KEEP] K8s overlay (kustomization.yaml) +│ │ └── osint-dashboard/ [KEEP] K8s overlay (kustomization.yaml) +│ └── vwap-monitor/ [MOVE] App code (Dockerfile, app/, deploy/) +├── clusters/ [KEEP] Cluster configs (devops-lab/*.yaml, flux-system/) +├── infrastructure/ [KEEP] Infra controllers, gatewayapi, gpus, tailnet +├── misc/ [KEEP] Terraform snippets + K8s YAML +├── modules/ [KEEP] Terraform modules (gke.tf, nodepool.tf, etc.) +├── scripts/ [KEEP] Setup scripts +├── .github/workflows/ +│ ├── osint-dashboard-infra.yml [KEEP] Infra deployment workflow +│ └── trade-dashboard.yml [MOVE] CI for trade-dashboard app → hermes-projects/ +├── .devcontainer.json [KEEP] Dev environment config +├── .sops.yaml [KEEP] SOPS encryption config +├── .terraform.lock.hcl [KEEP] Terraform lock +├── .gitignore [KEEP] Git ignore rules +├── mise.toml [KEEP] Tool version management +├── README.md [KEEP] (will be updated) +├── infra-tailnet.yaml [KEEP] Tailscale infra config +├── tailscale-0auth.yaml [KEEP] Tailscale config +├── rays-new-deployment.yaml [REVIEW] Orphan K8s deployment YAML — move to apps/base/ +├── trade-dashboard/ [MOVE] Full FastAPI app → hermes-projects/trade-dashboard/ +├── trading-platform/ [MOVE] Duplicate/deploy configs → hermes-projects/trading-platform/ +├── trading-scripts/ [MOVE] Application code → hermes-projects/trading-scripts/ +├── analyses/ [REMOVE] Research artifacts (not code, not infra) +└── plans/ [REMOVE] Planning docs (not code, not infra) +``` + +--- + +## Items to Move / Remove + +### 1. `trade-dashboard/` → hermes-projects/trade-dashboard/ + +**What it is:** Full FastAPI application (not K8s manifests) +- `Dockerfile` — build config for the app +- `app/` — Python source code (main.py, models.py, schemas.py, database.py, requirements.txt, static/) +- `alembic/` — database migration scripts (env.py, versions/) +- `alembic.ini` — alembic config + +**Also move:** `.github/workflows/trade-dashboard.yml` (CI workflow for this app) + +**Already in gcloud-lab:** `apps/base/customer1/trade-dashboard/` — these are the K8s manifests for trade-dashboard (deployment.yaml, service.yaml, configmap.yaml, kustomization.yaml). **KEEP these** — they belong here. + +### 2. `trading-platform/` → hermes-projects/trading-platform/ + +**What it is:** Duplicate/alternative deployment configs that overlap with hermes-projects/trading-platform/ + +Contents: +- `.github/workflows/` — 3 CI/CD workflows (build-push.yml, build-test.yml, deploy.yml) +- `README.md` — project readme +- `deploy/` — deployment configs: + - `ci-cd/` — additional CI workflows + - `docker-compose/` — docker-compose.dev.yml + - `dockerfiles/` — Dockerfiles (api-gateway, dashboard, data-service, execute-service, news-service) + - `helm/` — Helm charts (api-gateway, dashboard, data-service, execute-service, news-service) + - `k8s/` — raw K8s manifests (deployments, services, hpa, cert-manager, ingress) + - `mtls/` — mTLS README + - `scripts/` — deploy.sh, generate-mtls-certs.sh +- `dockerfiles/` — Dockerfiles (dashboard, data-service, execute-service, news-service) +- `helm/` — Helm chart with templates (trading-platform chart, values.yaml, secrets) + +**Already in gcloud-lab:** `apps/base/customer1/trading-platform/` — these are the K8s manifests. **KEEP these** — they belong here. + +**Already in hermes-projects:** `hermes-projects/trading-platform/` — source code exists here (dashboard, data-service, execute-service, news-service, data_infrastructure). The gcloud-lab trading-platform/ deploy/dockerfiles/helm content should be MERGED into hermes-projects/trading-platform/. + +**Decision needed:** The `trading-platform/` in gcloud-lab has BOTH deploy configs (dockerfiles, helm, k8s manifests) AND CI workflows. The K8s manifests in `deploy/k8s/base/` are similar but NOT identical to what's in `apps/base/customer1/trading-platform/`. Need to decide which is authoritative. + +### 3. `trading-scripts/` → hermes-projects/trading-scripts/ + +**What it is:** Application code (Python trading scripts) +- `market_data.py` — market data script +- `orb-monitor/` — monitoring tool (monitor.py, config.yaml) +- `README.md`, `ROADMAP.md` — documentation + +### 4. `apps/vwap-monitor/` → hermes-projects/vwap-monitor/ + +**What it is:** Application code with a Dockerfile +- `Dockerfile` — build config +- `app/` — source code (scanner.py, requirements.txt) +- `deploy/` — deployment config (deployment.yaml, kustomization.yaml, secret.yaml, config.env) + +**Note:** The `deploy/` subdirectory contains K8s manifests. These should be moved BACK into gcloud-lab as `apps/base/customer1/vwap-monitor/`. The app code (Dockerfile + app/) goes to hermes-projects. + +### 5. `analyses/` → REMOVE from gcloud-lab + +**What it is:** Research/analysis markdown documents +- `telegram-webhook-container-analysis.md` +- `telegram-webhook-failure-analysis.md` + +These are one-time research artifacts, not infrastructure config. Remove from gcloud-lab entirely. + +### 6. `plans/` → REMOVE from gcloud-lab + +**What it is:** Planning/strategy markdown documents +- `2026-04-25-openclaw-brain-v1.1.md` +- `AI_ARCHITECTURE.md` +- `models-to-try.md` + +These are planning docs, not infrastructure config. Remove from gcloud-lab entirely. + +### 7. `rays-new-deployment.yaml` → REVIEW + +**What it is:** A standalone K8s deployment YAML at repo root. + +**Action:** Move to `apps/base/customer1/hermes-agent/` (appears related to hermes-agent/rays deployment based on filename). Already similar files exist in that directory. + +--- + +## Proposed Migration Plan (Ordered by PR) + +### PR 1: This Plan (docs only) +- Add `MIGRATION_PLAN.md` (this file) +- Update `README.md` to document the new structure + +### PR 2: Remove planning/research docs +- Delete `analyses/` directory +- Delete `plans/` directory +- Low risk, no dependencies + +### PR 3: Move trade-dashboard app to hermes-projects +- Move `trade-dashboard/` → hermes-projects/trade-dashboard/ +- Move `.github/workflows/trade-dashboard.yml` → hermes-projects/.github/workflows/ +- K8s manifests in `apps/base/customer1/trade-dashboard/` stay in place +- Verify image references in K8s manifests still point to correct registry + +### PR 4: Move trading-platform deploy configs to hermes-projects +- Move `trading-platform/` → merge with hermes-projects/trading-platform/ +- CI workflows → hermes-projects/trading-platform/.github/workflows/ +- Dockerfiles → hermes-projects/trading-platform/dockerfiles/ +- Helm charts → hermes-projects/trading-platform/helm/ +- K8s manifests from `trading-platform/deploy/k8s/` → reconcile with `apps/base/customer1/trading-platform/` +- **Decision needed:** Which K8s manifests are authoritative? The ones in gcloud-lab/apps/ or trading-platform/deploy/k8s/? + +### PR 5: Move trading-scripts to hermes-projects +- Move `trading-scripts/` → hermes-projects/trading-scripts/ +- Simple move, no K8s manifest reconciliation needed + +### PR 6: Split vwap-monitor (app → hermes-projects, K8s → gcloud-lab) +- Move `apps/vwap-monitor/app/` + `apps/vwap-monitor/Dockerfile` → hermes-projects/vwap-monitor/ +- Move `apps/vwap-monitor/deploy/` K8s manifests → `apps/base/customer1/vwap-monitor/` +- Update image references in K8s manifests + +--- + +## Items That Stay in gcloud-lab (No Changes) + +| Path | Reason | +|------|--------| +| `apps/base/customer1/` | K8s manifests (kustomize structure) | +| `apps/base/monitoring/` | K8s manifests (dashboards) | +| `apps/base/osint-dashboard/` | Helm chart for infra | +| `apps/staging/` | K8s overlays | +| `clusters/` | Cluster configs, flux-system | +| `infrastructure/` | Controllers, gatewayapi, gpus, tailnet | +| `misc/` | Terraform snippets + K8s YAML | +| `modules/` | Terraform modules | +| `scripts/` | Setup scripts | +| Root config files | .sops.yaml, .devcontainer.json, mise.toml, .gitignore, .terraform.lock.hcl | +| `infra-tailnet.yaml` | Tailscale infra config | +| `tailscale-0auth.yaml` | Tailscale config | + +--- + +## K8s Manifest Reference Check + +After moves, verify these image references still resolve: + +| K8s Manifest | Image Reference | +|--------------|----------------| +| `apps/base/customer1/trade-dashboard/deployment.yaml` | Check image tag matches hermes-projects build | +| `apps/base/customer1/trading-platform/*/deployment.yaml` | Check image tags match hermes-projects build | +| `apps/base/customer1/hermes-agent/deployment.yaml` | N/A (already correct) | +| `apps/base/customer1/siriusdevops-site/deployment.yaml` | N/A (already correct) | + +--- + +## Decisions Needed Before Proceeding + +1. **trading-platform K8s manifest authority:** `trading-platform/deploy/k8s/base/` vs `apps/base/customer1/trading-platform/` — which is the source of truth? +2. **analyses/ and plans/:** Delete entirely, or archive somewhere else? +3. **rays-new-deployment.yaml:** Move to `apps/base/customer1/hermes-agent/` or delete?