From 4f80f1eede6a9bbdf1bb79ebf4e1d38f3bcc2c2b Mon Sep 17 00:00:00 2001 From: sirius0xdev Date: Fri, 16 Jan 2026 20:33:37 +0000 Subject: [PATCH] deploy news bot --- .claude/settings.local.json | 7 + README.md | 397 ++++++++++++++++++ .../customer1/news_bot/analyst-cronjob.yaml | 71 ++-- .../news_bot/analyst-gemini-cronjob.yaml | 31 ++ apps/base/customer1/news_bot/configmap.yaml | 6 +- .../customer1/news_bot/gemini-apikey.yaml | 23 + .../news_bot/news-user-db-secret.yaml | 24 ++ .../customer1/news_bot/scraper-cronjob.yaml | 35 ++ .../customer1/news_bot/scrapy-configmap.yaml | 18 + .../news_bot/scrapy-urls-configmap.yaml | 184 ++++++++ .../customer1/news_bot/telebot-cronjob.yaml | 31 ++ .../customer1/news_bot/telebot-secret.yaml | 23 + apps/base/customer1/pg-cluster-customer1.yaml | 6 + customer1-db-credentials.yamlv | 9 - mise.toml | 2 + modules/nodepool-gpu.tf | 43 ++ 16 files changed, 864 insertions(+), 46 deletions(-) create mode 100644 .claude/settings.local.json create mode 100644 README.md create mode 100644 apps/base/customer1/news_bot/analyst-gemini-cronjob.yaml create mode 100644 apps/base/customer1/news_bot/gemini-apikey.yaml create mode 100644 apps/base/customer1/news_bot/news-user-db-secret.yaml create mode 100644 apps/base/customer1/news_bot/scraper-cronjob.yaml create mode 100644 apps/base/customer1/news_bot/scrapy-configmap.yaml create mode 100644 apps/base/customer1/news_bot/scrapy-urls-configmap.yaml create mode 100644 apps/base/customer1/news_bot/telebot-cronjob.yaml create mode 100644 apps/base/customer1/news_bot/telebot-secret.yaml delete mode 100644 customer1-db-credentials.yamlv create mode 100644 modules/nodepool-gpu.tf diff --git a/.claude/settings.local.json b/.claude/settings.local.json new file mode 100644 index 0000000..d0df7d3 --- /dev/null +++ b/.claude/settings.local.json @@ -0,0 +1,7 @@ +{ + "permissions": { + "allow": [ + "Bash(tree:*)" + ] + } +} diff --git a/README.md b/README.md new file mode 100644 index 0000000..f67cd83 --- /dev/null +++ b/README.md @@ -0,0 +1,397 @@ +# GCloud-Lab DevOps Infrastructure + +A cloud-native DevOps laboratory project showcasing modern infrastructure-as-code, GitOps practices, and Kubernetes orchestration on Google Cloud Platform. This project runs a news intelligence system with LLM-powered analysis and a workflow automation platform. + +## Table of Contents + +- [Project Overview](#project-overview) +- [Architecture](#architecture) +- [DevOps Tools & Technologies](#devops-tools--technologies) +- [Project Structure](#project-structure) +- [Infrastructure Components](#infrastructure-components) +- [Applications](#applications) +- [Getting Started](#getting-started) +- [Security](#security) + +--- + +## Project Overview + +This repository contains infrastructure and application configurations for: + +1. **News Intelligence Pipeline**: Automated web scraping, LLM-powered summarization, and Telegram distribution +2. **Workflow Automation**: N8N platform for custom integrations +3. **DevOps Reference Architecture**: Demonstrates GitOps, IaC, and cloud-native best practices + +--- + +## Architecture + +``` +┌─────────────────────────────────────────────────────────────────────────┐ +│ Google Cloud Platform │ +│ ┌───────────────────────────────────────────────────────────────────┐ │ +│ │ GKE Cluster (devops-lab-cluster) │ │ +│ │ │ │ +│ │ ┌─────────────┐ ┌─────────────┐ ┌─────────────────────────┐ │ │ +│ │ │ Standard │ │ GPU Pool │ │ Flux CD (GitOps) │ │ │ +│ │ │ Node Pool │ │ (SPOT L4) │ │ - Source Controller │ │ │ +│ │ │ e2-std-2 │ │ g2-std-8 │ │ - Kustomize Controller│ │ │ +│ │ │ 1-16 nodes │ │ 0-5 nodes │ │ - Helm Controller │ │ │ +│ │ └─────────────┘ └─────────────┘ └─────────────────────────┘ │ │ +│ │ │ │ +│ │ ┌─────────────────────────────────────────────────────────────┐ │ │ +│ │ │ Cilium CNI + Hubble │ │ │ +│ │ │ (Network Policies + Observability) │ │ │ +│ │ └─────────────────────────────────────────────────────────────┘ │ │ +│ │ │ │ +│ │ ┌───────────────────────┐ ┌─────────────────────────────────┐ │ │ +│ │ │ customer1 namespace │ │ cnpg-system namespace │ │ │ +│ │ │ ┌─────────────────┐ │ │ ┌───────────────────────────┐ │ │ │ +│ │ │ │ N8N │ │ │ │ CloudNative PG Operator │ │ │ │ +│ │ │ │ (Workflows) │ │ │ └───────────────────────────┘ │ │ │ +│ │ │ └─────────────────┘ │ └─────────────────────────────────┘ │ │ +│ │ │ ┌─────────────────┐ │ │ │ +│ │ │ │ News Scraper │ │ ┌─────────────────────────────────┐ │ │ +│ │ │ │ (CronJob :00) │ │ │ PostgreSQL HA Cluster │ │ │ +│ │ │ └─────────────────┘ │ │ ┌─────┐ ┌─────┐ ┌─────┐ │ │ │ +│ │ │ ┌─────────────────┐ │ │ │ DB1 │ │ DB2 │ │ DB3 │ │ │ │ +│ │ │ │ News Analyst │◄─┼──┼──│(RW) │ │(RO) │ │(RO) │ │ │ │ +│ │ │ │ (CronJob :15) │ │ │ └─────┘ └─────┘ └─────┘ │ │ │ +│ │ │ │ + Ollama/Gemma2 │ │ └─────────────────────────────────┘ │ │ +│ │ │ └─────────────────┘ │ │ │ +│ │ │ ┌─────────────────┐ │ │ │ +│ │ │ │ Telegram Bot │ │ │ │ +│ │ │ │ (CronJob :20) │ │ │ │ +│ │ │ └─────────────────┘ │ │ │ +│ │ └───────────────────────┘ │ │ +│ └───────────────────────────────────────────────────────────────────┘ │ +│ │ +│ ┌─────────────────────────────────────────────────────────────────┐ │ +│ │ GCP L7 Global Load Balancer (HTTPS) │ │ +│ │ n8n.sirius-sec.com │ │ +│ └─────────────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## DevOps Tools & Technologies + +### Infrastructure as Code (IaC) + +| Tool | Version | Purpose | +|------|---------|---------| +| **Terraform** | 1.7+ | Infrastructure provisioning for GCP resources | +| **Google Provider** | 7.14.1 | Terraform provider for GCP | +| **Helm Provider** | Latest | Terraform provider for Helm charts | +| **Flux Provider** | 1.7.6 | Terraform provider for Flux bootstrap | + +### Container Orchestration & Networking + +| Tool | Version | Purpose | +|------|---------|---------| +| **Google Kubernetes Engine (GKE)** | Latest | Managed Kubernetes cluster | +| **Cilium** | 1.18.5 | CNI plugin with eBPF-based networking | +| **Hubble** | 1.18.5 | Network observability and monitoring | +| **Kubernetes Gateway API** | v1 | Ingress routing and traffic management | + +### GitOps & Configuration Management + +| Tool | Version | Purpose | +|------|---------|---------| +| **Flux CD** | 1.7.6 | GitOps continuous delivery | +| **Kustomize** | v1beta1 | Kubernetes manifest customization | +| **Helm** | 3+ | Kubernetes package manager | +| **SOPS** | Latest | Secrets encryption in Git | +| **Age** | Latest | Modern encryption for SOPS | + +### Database + +| Tool | Version | Purpose | +|------|---------|---------| +| **CloudNative PG** | 0.26.1 | PostgreSQL Kubernetes operator | +| **PostgreSQL** | 15.2 | Relational database (3-node HA cluster) | + +### AI/ML Infrastructure + +| Tool | Version | Purpose | +|------|---------|---------| +| **Ollama** | Latest | Local LLM inference server | +| **Gemma2** | Latest | Open-source LLM for text summarization | +| **NVIDIA L4 GPU** | - | GPU acceleration for LLM workloads | + +### Development Environment + +| Tool | Version | Purpose | +|------|---------|---------| +| **Mise** | Latest | Development tool version manager | +| **Dev Containers** | Latest | Consistent development environment | +| **k9s** | Latest | Kubernetes CLI dashboard | + +--- + +## Project Structure + +``` +gcloud-lab/ +├── modules/ # Terraform IaC modules +│ ├── providers.tf # Provider configurations +│ ├── gke.tf # GKE cluster definition +│ ├── vpc.tf # VPC and subnet configuration +│ ├── nodepool.tf # Standard node pool +│ ├── nodepool-gpu.tf # GPU node pool (SPOT instances) +│ ├── flux.tf # Flux GitOps bootstrap +│ ├── helm.tf # Helm chart deployments (Cilium) +│ └── variables.tf # Input variables +│ +├── clusters/ # Cluster configurations +│ └── devops-lab/ +│ ├── flux-system/ # Flux CD components +│ │ ├── gotk-components.yaml # Flux controllers +│ │ ├── gotk-sync.yaml # Git repository sync +│ │ └── kustomization.yaml # Flux kustomization +│ ├── customer1.yaml # Customer1 Kustomization +│ ├── infra-controllers.yaml # Infrastructure controllers +│ └── infra-configs.yaml # Infrastructure configs +│ +├── infrastructure/ # Infrastructure components +│ ├── controllers/ +│ │ ├── base/ +│ │ │ └── cnpg/ # CloudNative PG operator +│ │ │ ├── repository.yaml # Helm repository +│ │ │ └── release.yaml # Helm release +│ │ └── staging/ +│ │ └── kustomization.yaml +│ └── configs/ +│ └── staging/ +│ └── kustomization.yaml +│ +├── apps/ # Application deployments +│ ├── base/ +│ │ └── customer1/ +│ │ ├── namespace.yaml # Namespace definition +│ │ ├── deployment.yaml # N8N deployment +│ │ ├── service.yaml # ClusterIP service +│ │ ├── storage.yaml # PersistentVolumeClaim +│ │ ├── configmap.yaml # N8N configuration +│ │ ├── pg-cluster-customer1.yaml # PostgreSQL cluster +│ │ ├── apigateway.yaml # GCP Gateway +│ │ ├── http-route.yaml # HTTP routing +│ │ ├── healthcheck.yaml # Health check policy +│ │ └── news_bot/ # News bot microservices +│ │ ├── scraper-cronjob.yaml +│ │ ├── analyst-cronjob.yaml +│ │ ├── telebot-cronjob.yaml +│ │ ├── scrapy-configmap.yaml +│ │ └── scrapy-urls-configmap.yaml +│ └── staging/ +│ └── customer1/ +│ └── kustomization.yaml # Staging overlay +│ +├── scripts/ +│ └── setup # Development setup script +│ +├── .devcontainer.json # Dev container configuration +├── mise.toml # Tool version management +├── age.agekey # SOPS encryption key +└── README.md # This file +``` + +--- + +## Infrastructure Components + +### GKE Cluster + +- **Name**: `devops-lab-cluster` +- **Region**: `us-central1-a` +- **Network**: Custom VPC with dual-stack IPv4/IPv6 + +### Node Pools + +| Pool | Machine Type | Scaling | Purpose | +|------|-------------|---------|---------| +| Standard | e2-standard-2 | 1-16 nodes | General workloads | +| GPU (SPOT) | g2-standard-8 + L4 | 0-5 nodes | LLM inference | + +### Networking + +- **VPC**: `devops-lab-network` +- **Primary CIDR**: `10.0.0.0/16` +- **Pod CIDR**: `192.168.32.0/20` +- **Service CIDR**: `192.168.16.0/24` +- **CNI**: Cilium with advanced datapath +- **Ingress**: GCP L7 Global Load Balancer + +### GitOps Flow + +``` +GitHub Repository + │ + ▼ + Flux Source Controller (watches git, 1min interval) + │ + ▼ + Flux Kustomize Controller (applies manifests) + │ + ├── infrastructure/controllers → CNPG Operator + ├── infrastructure/configs → Cluster configs + └── apps/staging/customer1 → Applications +``` + +--- + +## Applications + +### N8N Workflow Automation + +- **URL**: `https://n8n.sirius-sec.com` +- **Image**: `docker.n8n.io/n8nio/n8n:2.1.4` +- **Database**: PostgreSQL (dedicated `n8n` database) +- **Storage**: 1GB persistent volume + +### News Intelligence Pipeline + +A three-stage data pipeline running as Kubernetes CronJobs: + +| Stage | Schedule | Container | Purpose | +|-------|----------|-----------|---------| +| Scraper | `:00` hourly | `siriussec/newsscraper` | Scrapes 100+ global news sources | +| Analyst | `:15` hourly | `siriussec/summarizer` + `ollama/ollama` | LLM-powered summarization | +| Telegram | `:20` hourly | `siriussec/news-messenger` | Distributes summaries to Telegram | + +**News Sources Coverage**: +- North America: NPR, AP News, CBC, etc. +- Europe: BBC, Reuters, The Guardian, etc. +- Asia: SCMP, Al Jazeera, Times of India, etc. +- Africa: BBC Africa, News24, etc. +- South America: Buenos Aires Herald, etc. + +--- + +## Getting Started + +### Prerequisites + +- Google Cloud account with billing enabled +- GitHub account with repository access +- `gcloud` CLI authenticated +- Terraform 1.7+ + +### Local Development Setup + +```bash +# Install tools via mise +./scripts/setup + +# Or manually +mise trust && mise install +``` + +### Infrastructure Deployment + +```bash +cd modules + +# Initialize Terraform +terraform init + +# Set required variables +export TF_VAR_github_token="your-token" +export TF_VAR_github_org="your-org" +export TF_VAR_github_repository="gcloud-lab" + +# Plan and apply +terraform plan +terraform apply +``` + +### Accessing the Cluster + +```bash +# Configure kubectl +gcloud container clusters get-credentials devops-lab-cluster \ + --zone us-central1-a \ + --project devops-lab-cluster + +# Verify connection +kubectl get nodes + +# Use k9s for interactive management +k9s +``` + +--- + +## Security + +### Secrets Management + +- **Encryption**: SOPS with Age encryption +- **Key Storage**: `age.agekey` (do not commit unencrypted) +- **Flux Integration**: Automatic decryption during deployment + +### Pod Security + +- Non-root containers (UID 1000) +- Filesystem group enforcement +- Privilege escalation disabled +- Resource limits enforced + +### Network Security + +- Cilium network policies for pod-to-pod isolation +- TLS termination at load balancer +- Private cluster networking with NAT + +### Database Security + +- Managed roles with secret-based passwords +- Separate users per application (`customer1`, `news_app`) +- HA cluster with automatic failover + +--- + +## Tool Reference + +### Terraform Providers + +```hcl +google = "~> 7.14" # GCP resources +helm = "~> 2.0" # Helm chart management +flux = "~> 1.7" # GitOps bootstrap +``` + +### Helm Charts + +```yaml +cilium: 1.18.5 # CNI and service mesh +cloudnative-pg: 0.26.1 # PostgreSQL operator +``` + +### Container Images + +``` +docker.n8n.io/n8nio/n8n:2.1.4 +ghcr.io/cloudnative-pg/postgresql:15.2 +ollama/ollama:latest +siriussec/newsscraper:latest +siriussec/summarizer:latest +siriussec/news-messenger:latest +``` + +--- + +## Cost Optimization + +- **SPOT GPU Instances**: 60-90% savings on LLM workloads +- **Autoscaling**: GPU nodes scale to 0 when idle +- **Resource Limits**: Prevents runaway costs +- **Scheduled Workloads**: CronJobs only run when needed + +--- + +## License + +Private repository - All rights reserved. diff --git a/apps/base/customer1/news_bot/analyst-cronjob.yaml b/apps/base/customer1/news_bot/analyst-cronjob.yaml index c004014..4d2626c 100644 --- a/apps/base/customer1/news_bot/analyst-cronjob.yaml +++ b/apps/base/customer1/news_bot/analyst-cronjob.yaml @@ -2,44 +2,47 @@ apiVersion: batch/v1 kind: CronJob metadata: name: news-analyst + namespace: customer1 spec: schedule: "15 * * * *" jobTemplate: spec: template: spec: - volumes: - - name: ollama-storage - emptyDir: {} - - initContainers: - - name: model-puller - image: ollama/ollama:latest - volumeMounts: - - name: ollama-storage - mountPath: /root/.ollama - command: ["/bin/sh", "-c"] - args: ["ollama serve & sleep 5 && ollama pull gemma2"] - - containers: - - name: analyst - image: siriussec/summarizer:latest - envFrom: - - configMapRef: - name: news-app-config - env: - valueFrom: - secretKeyRef: - name: db-secrets - key: password - - - name: ollama-sidecar - image: ollama/ollama:latest - ports: - - containerPort: 11434 - volumeMounts: + volumes: - name: ollama-storage - mountPath: /root/.ollama - resources: - limits: - nvidia.com/gpu: 1 + emptyDir: {} + + initContainers: + - name: model-puller + image: ollama/ollama:latest + volumeMounts: + - name: ollama-storage + mountPath: /root/.ollama + command: ["/bin/sh", "-c"] + args: ["ollama serve & sleep 5 && ollama pull gemma2"] + + containers: + - name: analyst + image: siriussec/summarizer:latest + + envFrom: + - configMapRef: + name: news-app-config + + + - secretRef: + name: news-user-password + + + - name: ollama-sidecar + image: ollama/ollama:latest + ports: + - containerPort: 11434 + volumeMounts: + - name: ollama-storage + mountPath: /root/.ollama + resources: + limits: + nvidia.com/gpu: 1 + restartPolicy: OnFailure diff --git a/apps/base/customer1/news_bot/analyst-gemini-cronjob.yaml b/apps/base/customer1/news_bot/analyst-gemini-cronjob.yaml new file mode 100644 index 0000000..5b57bf6 --- /dev/null +++ b/apps/base/customer1/news_bot/analyst-gemini-cronjob.yaml @@ -0,0 +1,31 @@ + +apiVersion: batch/v1 +kind: CronJob +metadata: + name: news-analyst-gemini + namespace: customer1 + +spec: + schedule: "15 * * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: analyst-gemini + image: siriussec/summarizer:1.5 + + env: + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: news-user-password + key: password + envFrom: + - configMapRef: + name: news-app-config + + + - secretRef: + name: gemini-apikey + restartPolicy: OnFailure diff --git a/apps/base/customer1/news_bot/configmap.yaml b/apps/base/customer1/news_bot/configmap.yaml index 96444c4..cb45108 100644 --- a/apps/base/customer1/news_bot/configmap.yaml +++ b/apps/base/customer1/news_bot/configmap.yaml @@ -2,14 +2,14 @@ apiVersion: v1 kind: ConfigMap metadata: name: news-app-config - namespace: default + namespace: customer1 data: DB_HOST: "customer1-pgdb-rw" DB_NAME: "news_app_db" DB_USER: "news_app" DB_PORT: "5432" - LLM_URL: "http://localhost:11434/api/generate" # Used for sidecar setup - MODEL_NAME: "gemma2" + LLM_URL: "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash:generateContent" + MODEL_NAME: "gemini-2.0-flash" SUMMARY_PROMPT: | You are a professional news editor. Combine these findings into a Telegram-friendly update with emojis and bold headers. Translate everything to english diff --git a/apps/base/customer1/news_bot/gemini-apikey.yaml b/apps/base/customer1/news_bot/gemini-apikey.yaml new file mode 100644 index 0000000..daabee1 --- /dev/null +++ b/apps/base/customer1/news_bot/gemini-apikey.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +data: + GEMINI_API_KEY: ENC[AES256_GCM,data:EcyraxRLXiWWQVhJNmmUGut6AmSMA8BvFg0X/PoP2ckclcFas8pU7GfsmMXFD9arcuCJpg==,iv:52YNqcPGWQqQY4jdfc8DtP6dROTsxJqUDahXzEN3ycI=,tag:Q/9l0mDG+5sjTLKt5Jg4sw==,type:str] +kind: Secret +metadata: + creationTimestamp: null + name: gemini-apikey + namespace: customer1 +sops: + age: + - recipient: age1uuxf066xuuqgvjppxfcmqkwfcufnwp3wcwnl9h20g9k4l8nkw9jsaungf7 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBTcGdQb0tMdWtiTkl3aWZt + dnl4ZHo3aHZ4bGgvR1Rva0E5dlloQlRGY0d3CmMwOUJWb3NlWEtRVEk3STJ5MWVk + d21ZNzkyeHBZQTF3dWhYdUlSZEtQQkEKLS0tIEUrU3M2UGh5Y1BPTUhRQi9NK2VN + Vi83QzhjcG45U1B1WTJGcklFdkRvQjQK/jAjKf3wCZCpHp5naJoitHfN1yEEqmoN + p5AMB97oTtyVhvu3wkRrxqHB8LKNWZifaJf8g13To1OgFh9azC/Kng== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2026-01-16T20:31:27Z" + mac: ENC[AES256_GCM,data:7y9LrLu+bngdTuhb1ucIDnvg3jpKgXW6rXMpS6O08nxHHBKAKA6Jl80PsfOE0aQNSsIM2ERfrdGmfLkNh470oB/erGvEHxRLKcf8IStPxoPpry99XffiVo+QJrCPIzqgKy1tw37bqBNrm5LJkhUmugzf/IVt6WKiGCZhpbXjB7E=,iv:RfK084/orIGcRwXMeAip0BKXJ3eYJoRmEgLF72ER9vA=,tag:NKqZKIrt3yVkGL4ZkqcZKA==,type:str] + encrypted_regex: ^(data|stringData)$ + version: 3.11.0 diff --git a/apps/base/customer1/news_bot/news-user-db-secret.yaml b/apps/base/customer1/news_bot/news-user-db-secret.yaml new file mode 100644 index 0000000..5afa585 --- /dev/null +++ b/apps/base/customer1/news_bot/news-user-db-secret.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +data: + password: ENC[AES256_GCM,data:j+FF3ertQpe+uty07da0EBAHC4c=,iv:lLtlRJzY2lJwsmVh73oCiyBmp+N+KYJTvuKbK9u2vqA=,tag:ZACpQOW70kqJ+FfGbSIaHA==,type:str] + username: ENC[AES256_GCM,data:u/Lrk+pDBzinjWW/,iv:F3JU0RkH54visaQMfiJokQCl/g5De4pasRCCgZWhPBo=,tag:kNtUjgKoyflrkBNBDEf49A==,type:str] +kind: Secret +metadata: + creationTimestamp: null + name: news-user-password + namespace: customer1 +sops: + age: + - recipient: age1uuxf066xuuqgvjppxfcmqkwfcufnwp3wcwnl9h20g9k4l8nkw9jsaungf7 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA5dllDdkRXRzF2UFdNTUZ6 + eWZDTkJwdlROT0JIYXdyOVRaV3pMK3JpQ3p3CndaTnY4MjZqWjYrTlVRdXlhbDdy + bis0dFIzSVNPbGVqTXNzOEdVZEo2QU0KLS0tIHhLQXlYeEUxY0Y1L2QyYVlSR3ZH + OXNNdXdMT2NjOWx4M2Vld3lrSkVVb3MK8dnZBDOdc2kWAgibRc4SZC0St2Qdv2sS + aYGdiI7mPtuk2mJa0PWFzpFOAiE31JSg7anMmAhGERX++ErgzGj6lQ== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2026-01-16T20:32:11Z" + mac: ENC[AES256_GCM,data:J38DV9WAiUE3Y/7Lgz6qbjBgIIQpbEZ6Li8NLZ1lVh3QMZGJm0/V76mbQfKJR+XuRy5bJ5uDJS9ftm6VPomH1/hYWlubrMgajnXrXRryyPbJ7uJbbwOUdI3W5GzFTxU8Ek3YKo5NKgKIEDzwfPJXvb2kXizMgA6U9L5Ru+K4jek=,iv:mvINcPrPSnDw3Pey+96WZ7JSWveh2Yaljirde+SsQuo=,tag:HBqRgMXmsnDAnKNy/Wkcig==,type:str] + encrypted_regex: ^(data|stringData)$ + version: 3.11.0 diff --git a/apps/base/customer1/news_bot/scraper-cronjob.yaml b/apps/base/customer1/news_bot/scraper-cronjob.yaml new file mode 100644 index 0000000..4247b7d --- /dev/null +++ b/apps/base/customer1/news_bot/scraper-cronjob.yaml @@ -0,0 +1,35 @@ + +apiVersion: batch/v1 +kind: CronJob +metadata: + name: news-scraper + namespace: customer1 +spec: + schedule: "0 * * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: news-scraper + image: siriussec/newsscraper:2.1 + + envFrom: + - configMapRef: + name: scrapy-config + env: + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: news-user-password + key: password + volumeMounts: + - name: url-storage + mountPath: /app/urls.txt + subPath: urls.txt + volumes: + - name: url-storage + configMap: + name: scrapy-urls + restartPolicy: OnFailure + diff --git a/apps/base/customer1/news_bot/scrapy-configmap.yaml b/apps/base/customer1/news_bot/scrapy-configmap.yaml new file mode 100644 index 0000000..b04d018 --- /dev/null +++ b/apps/base/customer1/news_bot/scrapy-configmap.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: scrapy-config + namespace: customer1 +data: + # Database Connection + DB_HOST: "customer1-pgdb-rw" + DB_NAME: "news_app_db" + DB_USER: "news_app" + DB_PORT: "5432" + + # Scrapy Settings + LOG_LEVEL: "INFO" + USER_AGENT: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) NewsScraper/1.0" + + # Proxy Endpoint (The URL part only) + PROXY_ENDPOINT: "" diff --git a/apps/base/customer1/news_bot/scrapy-urls-configmap.yaml b/apps/base/customer1/news_bot/scrapy-urls-configmap.yaml new file mode 100644 index 0000000..356ef60 --- /dev/null +++ b/apps/base/customer1/news_bot/scrapy-urls-configmap.yaml @@ -0,0 +1,184 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: scrapy-urls + namespace: customer1 +data: + urls.txt: | + https://www.npr.org + https://www.bbc.com/news + https://www.reuters.com + https://www.khmertimeskh.com + https://apnews.com + # USA + https://www.npr.org + https://www.pbs.org/newshour + https://www.usatoday.com + https://www.cbsnews.com + https://www.nbcnews.com + + # Canada + https://www.cbc.ca/news + https://www.ctvnews.ca + https://globalnews.ca + https://nationalpost.com + https://www.thestar.com + + # Mexico + https://www.eluniversal.com.mx + https://www.milenio.com + https://www.jornada.com.mx + https://www.excelsior.com.mx + https://aristeguinoticias.com + + # --- SOUTH AMERICA --- + # Brazil + https://g1.globo.com + https://www.uol.com.br + https://agenciabrasil.ebc.com.br + https://www.metropoles.com + https://www.terra.com.br/noticias + + # Argentina + https://www.infobae.com + https://www.clarin.com + https://www.lanacion.com.ar + https://www.pagina12.com.ar + https://www.cronista.com + + # Colombia + https://www.eltiempo.com + https://www.elespectador.com + https://www.semana.com + https://www.bluradio.com + https://www.rcnradio.com + + # --- EUROPE --- + # United Kingdom + https://www.bbc.com/news + https://www.theguardian.com/uk + https://news.sky.com + https://www.independent.co.uk + https://metro.co.uk + + # France + https://www.france24.com/en + https://www.lefigaro.fr + https://www.20minutes.fr + https://www.francetvinfo.fr + https://www.lemonde.fr + + # Germany + https://www.dw.com/en + https://www.tagesschau.de + https://www.spiegel.de + https://www.zeit.de + https://www.bild.de + + # Spain + https://elpais.com + https://www.elmundo.es + https://www.rtve.es/noticias + https://www.20minutos.es + https://www.elconfidencial.com + + # Italy + https://www.ansa.it + https://www.corriere.it + https://www.repubblica.it + https://www.lastampa.it + https://tg24.sky.it + + # Russia (State & Independent mix) + https://tass.com + https://www.interfax.ru + https://www.rt.com + https://www.themoscowtimes.com + https://meduza.io/en + + # --- ASIA --- + # China + https://www.xinhuanet.com/english + https://www.chinadaily.com.cn + https://www.globaltimes.cn + https://www.cgtn.com + https://www.scmp.com + + # India + https://www.ndtv.com + https://timesofindia.indiatimes.com + https://indianexpress.com + https://www.thehindu.com + https://www.hindustantimes.com + + # Japan + https://www3.nhk.or.jp/nhkworld + https://www.japantimes.co.jp + https://www.asahi.com/ajw + https://mainichi.jp/english + https://english.kyodonews.net + + # South Korea + https://en.yna.co.kr + https://www.koreaherald.com + https://koreajoongangdaily.joins.com + https://www.donga.com/en + https://english.chosun.com + + # --- AFRICA --- + # South Africa + https://www.news24.com + https://www.iol.co.za + https://www.dailymaverick.co.za + https://www.sabcnews.com + https://www.timeslive.co.za + https://www.al.com + https://www.adn.com + https://www.azcentral.com + https://www.arkansasonline.com + https://www.latimes.com + https://www.denverpost.com + https://www.courant.com + https://www.delawareonline.com + https://www.tampabay.com + https://www.ajc.com + https://www.staradvertiser.com + https://www.idahostatesman.com + https://www.chicagotribune.com + https://www.indystar.com + https://www.desmoinesregister.com + https://www.kansas.com + https://www.courier-journal.com + https://www.nola.com + https://www.pressherald.com + https://www.baltimoresun.com + https://www.bostonglobe.com + https://www.freep.com + https://www.startribune.com + https://www.clarionledger.com + https://www.stltoday.com + https://www.missoulian.com + https://www.omaha.com + https://www.lvrj.com + https://www.unionleader.com + https://www.nj.com + https://www.abqjournal.com + https://www.nytimes.com + https://www.charlotteobserver.com + https://www.inforum.com + https://www.cleveland.com + https://www.oklahoman.com + https://www.oregonlive.com + https://www.inquirer.com + https://www.providencejournal.com + https://www.thestate.com + https://www.argusleader.com + https://www.tennessean.com + https://www.dallasnews.com + https://www.sltrib.com + https://www.burlingtonfreepress.com + https://www.pilotonline.com + https://www.seattletimes.com + https://www.wvgazettemail.com + https://www.jsonline.com + https://www.trib.com diff --git a/apps/base/customer1/news_bot/telebot-cronjob.yaml b/apps/base/customer1/news_bot/telebot-cronjob.yaml new file mode 100644 index 0000000..4775120 --- /dev/null +++ b/apps/base/customer1/news_bot/telebot-cronjob.yaml @@ -0,0 +1,31 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: news-messenger + namespace: customer1 +spec: + schedule: "20 * * * *" # Runs at :20 every hour + jobTemplate: + spec: + template: + spec: + containers: + - name: messenger + image: siriussec/news-messenger:2.3 + envFrom: + - configMapRef: + name: scrapy-config + env: + - name: TELEGRAM_TOKEN + valueFrom: + secretKeyRef: + name: telegram-secrets + key: token + - name: TELEGRAM_CHAT_ID + value: "-1003545141546" # Your Channel ID + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: news-user-password + key: password + restartPolicy: OnFailure diff --git a/apps/base/customer1/news_bot/telebot-secret.yaml b/apps/base/customer1/news_bot/telebot-secret.yaml new file mode 100644 index 0000000..f9f63f3 --- /dev/null +++ b/apps/base/customer1/news_bot/telebot-secret.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +data: + token: ENC[AES256_GCM,data:quiF2P1bgdY9wNtnDMIjEw60eId5M8JFq/tpUaP9fAxfwsBfP4wHXRQSeZcv66mJtKXKhb98dP+0EF4ZRwyGRA==,iv:KzN4juy8re0FKbGeMb7NVCeAETye/xs1VQVQjEumGnU=,tag:0nvnqazH/P5NOaWpWO3Wbg==,type:str] +kind: Secret +metadata: + creationTimestamp: null + name: telegram-secrets + namespace: customer1 +sops: + age: + - recipient: age1uuxf066xuuqgvjppxfcmqkwfcufnwp3wcwnl9h20g9k4l8nkw9jsaungf7 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBhb3V6QW8wSE43UUMzY2xl + djQzTWdkbkF0OUkvMDJKRC9UTFhkZHJIQzJzCmxnU2FtNVhQNmhNbjB2dEpFU2oy + UnlYYTdJZzB4VU12Mm9FK1p5Zko3aGMKLS0tIFJNczl5Zk5ZWHZQMVNDRG1TT202 + TWl1S1lacFVBTEVGYlVSZ29DMTJkOFEKuiXvNWxMNo3DuqDOxf7W5XExY+mE++pi + jQx/KxZjplwqGmJFuBtUMPD4XA8nx4lYut/MuKlYyfsrPWXVSAbSDA== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2026-01-16T20:31:09Z" + mac: ENC[AES256_GCM,data:/PKZzN0A6BQKtJAPNNFBuDBtgGJqI6lk7EXlee10g1rRobeHrtgbV5Ihe7jmH+awu+GcEeYRiekQaN57yYcWQd9R5+S27OgOTs1gxwDmMWcQgxivkvHf+6CywdNIwwUDmhd69sLU+j2kTpM+xQnMyTHrMX+IW59FFaCnmUkgOZo=,iv:KU4i+WRfMb9OR7kEdlTFhXBRi53lihG1enkJMGCqN7o=,tag:QsSn6RG7/ud38lTmSZAw6Q==,type:str] + encrypted_regex: ^(data|stringData)$ + version: 3.11.0 diff --git a/apps/base/customer1/pg-cluster-customer1.yaml b/apps/base/customer1/pg-cluster-customer1.yaml index b384b9c..150d005 100644 --- a/apps/base/customer1/pg-cluster-customer1.yaml +++ b/apps/base/customer1/pg-cluster-customer1.yaml @@ -3,6 +3,7 @@ kind: Cluster metadata: name: customer1-pgdb namespace: customer1 + spec: managed: roles: @@ -11,6 +12,11 @@ spec: login: true passwordSecret: name: customer1-db-credentials + - name: news_app + ensure: present + login: true + passwordSecret: + name: news-user-password instances: 3 imageName: ghcr.io/cloudnative-pg/postgresql:15.2 storage: diff --git a/customer1-db-credentials.yamlv b/customer1-db-credentials.yamlv deleted file mode 100644 index 78c04bb..0000000 --- a/customer1-db-credentials.yamlv +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v1 -data: - password: cGV0dWsxazE3MTE2NjY= - username: Y3VzdG9tZXIx -kind: Secret -metadata: - creationTimestamp: null - name: customer1-db-credentials - namespace: customer1 diff --git a/mise.toml b/mise.toml index d9f9c23..12d1b42 100644 --- a/mise.toml +++ b/mise.toml @@ -1,3 +1,5 @@ [tools] gcloud = "latest" k9s = "latest" +sops = "latest" +terraform = "latest" diff --git a/modules/nodepool-gpu.tf b/modules/nodepool-gpu.tf new file mode 100644 index 0000000..f7ca663 --- /dev/null +++ b/modules/nodepool-gpu.tf @@ -0,0 +1,43 @@ +resource "google_container_node_pool" "gpu_pool" { + name = "gpu-pool-l4" + location = "us-central1-a" + cluster = google_container_cluster.primary.name + + initial_node_count = 0 + + autoscaling { + min_node_count = 0 + max_node_count = 5 + } + + node_config { + machine_type = "g2-standard-8" # Optimized for NVIDIA L4 + + guest_accelerator { + type = "nvidia-l4" + count = 1 + } + + # Use SPOT instances to save ~60-90% on GPU costs + spot = true + + oauth_scopes = [ + "https://www.googleapis.com/auth/cloud-platform" + ] + + labels = { + workload = "llm-analyst" + } + + taint { + key = "nvidia.com/gpu" + value = "present" + effect = "NO_SCHEDULE" + } + } + + management { + auto_repair = true + auto_upgrade = true + } +}