From a3ad07d1918893e48bbeec2eb40d3ef940011f4f Mon Sep 17 00:00:00 2001 From: sirius0xdev Date: Sat, 2 May 2026 21:31:41 +0000 Subject: [PATCH] feat: add dedicated CNPG cluster for Hermes agent memory and trading data - New hermes-pgdb cluster (Postgres 15.2, 1 instance, 20Gi) - 3 databases: hermes_memory, trading_data, agent_memory - 3 roles with SOPS-encrypted secrets (age encryption) - Daily backups to gs://customer1_db_backup/hermes-backups/ - Wired into Flux pipeline via staging kustomization --- .sops.yaml | 2 + .../customer1/hermes-db/agent-memory-db.yaml | 10 ++++ .../hermes-db/hermes-db-credentials.yaml | 24 ++++++++ .../hermes-db/hermes-scheduled-backup.yaml | 12 ++++ .../customer1/hermes-db/kustomization.yaml | 11 ++++ .../hermes-db/memory-db-credentials.yaml | 24 ++++++++ .../hermes-db/pg-cluster-hermes.yaml | 55 +++++++++++++++++++ .../customer1/hermes-db/trading-data-db.yaml | 10 ++++ .../hermes-db/trading-db-credentials.yaml | 24 ++++++++ apps/staging/customer1/kustomization.yaml | 1 + 10 files changed, 173 insertions(+) create mode 100644 .sops.yaml create mode 100644 apps/base/customer1/hermes-db/agent-memory-db.yaml create mode 100644 apps/base/customer1/hermes-db/hermes-db-credentials.yaml create mode 100644 apps/base/customer1/hermes-db/hermes-scheduled-backup.yaml create mode 100644 apps/base/customer1/hermes-db/kustomization.yaml create mode 100644 apps/base/customer1/hermes-db/memory-db-credentials.yaml create mode 100644 apps/base/customer1/hermes-db/pg-cluster-hermes.yaml create mode 100644 apps/base/customer1/hermes-db/trading-data-db.yaml create mode 100644 apps/base/customer1/hermes-db/trading-db-credentials.yaml diff --git a/.sops.yaml b/.sops.yaml new file mode 100644 index 0000000..d70d674 --- /dev/null +++ b/.sops.yaml @@ -0,0 +1,2 @@ +creation_rules: + - age: age1uuxf066xuuqgvjppxfcmqkwfcufnwp3wcwnl9h20g9k4l8nkw9jsaungf7 diff --git a/apps/base/customer1/hermes-db/agent-memory-db.yaml b/apps/base/customer1/hermes-db/agent-memory-db.yaml new file mode 100644 index 0000000..6077e56 --- /dev/null +++ b/apps/base/customer1/hermes-db/agent-memory-db.yaml @@ -0,0 +1,10 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Database +metadata: + name: agent-memory-db + namespace: customer1 +spec: + cluster: + name: hermes-pgdb + name: agent_memory + owner: memory diff --git a/apps/base/customer1/hermes-db/hermes-db-credentials.yaml b/apps/base/customer1/hermes-db/hermes-db-credentials.yaml new file mode 100644 index 0000000..93a207c --- /dev/null +++ b/apps/base/customer1/hermes-db/hermes-db-credentials.yaml @@ -0,0 +1,24 @@ +apiVersion: ENC[AES256_GCM,data:txA=,iv:gCFjwoYIBFCTXOTBjND+corfessRLGUh313T8aVsO0M=,tag:V/SjrvWG2vLrFTBP9CpWNA==,type:str] +kind: ENC[AES256_GCM,data:ZgEuMlxy,iv:Wp4ZPGH4ID7t8BGEsnJ/Rm7wIVm9rWPQYLfj3CJmo5s=,tag:bd1KCKO6eKgsJtJ4YcWW4A==,type:str] +metadata: + name: ENC[AES256_GCM,data:0iXVLY1JDCujP/JQWELIr5dP4uSB,iv:Das8czGXBlxz+hC8KWh9jZwXJCQ12v+P6tSL4DUL1S4=,tag:l4kKkLd4LvRxECg31ob2fg==,type:str] + namespace: ENC[AES256_GCM,data:p7hF20bNI6CI,iv:FNB+sXTcgKQdE/1MZuYlxNukMSx5mWNXqVOtnxbvOCo=,tag:lOmXwn+jF3JapOGPWzEz2w==,type:str] +type: ENC[AES256_GCM,data:NLgmDHre,iv:xMs4To5h+yPgQuYOVP2pu53cA1yqsh8d91trr/gvAWg=,tag:1FNpJ4VqHwfXP5L2T0jG1w==,type:str] +stringData: + username: ENC[AES256_GCM,data:Kyn8uqkj,iv:0BMWC+W8RiAOhKeaWwCwpoxug0t5nHBRPqRfeGZPaF0=,tag:ECYB03gU8kk5mWKu5ERbEg==,type:str] + password: ENC[AES256_GCM,data:iWw+TY9t68laPLRstPN7gybhyUP3bz6lpRTSnSdAgW0=,iv:hFOncdpn0AoQ5JywZxkwnrOnE+jb0rFIeoU2FcTnkqY=,tag:f+rjg2kfGNTfR3gb8kVRBA==,type:str] +sops: + age: + - recipient: age1uuxf066xuuqgvjppxfcmqkwfcufnwp3wcwnl9h20g9k4l8nkw9jsaungf7 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBRc2haZVlrUEJUV2F5bHFm + WG5qV2xVYzl4UE5aM3VBNE9JUnJ4YVI5V2dzCkVJM2NDZDd6MjZsdElYR09aRkgw + MXZ5anlGVW1laHl3T2k0M2ZiUHZ3cjgKLS0tIG9od1JtU1lJY0ZrSUdtWWMxV1FM + Vk92L1NWVGpkMWRUdmNpV1AxS1RSMzQKjITIDbSQp+aaKDzR48ENFmuY+dMTYPaQ + 1P8zng4fQptC9RBvps327Z5kMIAtS52zYeoYgBwgdXNtTsaoO4ddmA== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2026-05-02T21:29:40Z" + mac: ENC[AES256_GCM,data:ioHm0Go1hjO4VYKhEiu3e2Pdz+FzwDjlzQ5rWNbmuoCRW3bdqXpIvGNMyALBX7e+UCo+PwlNWiu3JrhszxdLptsySdCpzz0z04CZoQCbej66xdi4U49VvMVCJyZaRP5ZGFysXQghz0m0z4QoAcMSyZZGh8f8PnLewxORiXwrS7c=,iv:HXjO4oABWJCSUHN4dQiMXV1dFR0bxpDYSJXhBQ4G970=,tag:LIIY8pv+0/U/IfaB2D8vuQ==,type:str] + unencrypted_suffix: _unencrypted + version: 3.12.2 diff --git a/apps/base/customer1/hermes-db/hermes-scheduled-backup.yaml b/apps/base/customer1/hermes-db/hermes-scheduled-backup.yaml new file mode 100644 index 0000000..f2b52e7 --- /dev/null +++ b/apps/base/customer1/hermes-db/hermes-scheduled-backup.yaml @@ -0,0 +1,12 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: ScheduledBackup +metadata: + name: hermes-daily-backup + namespace: customer1 +spec: + schedule: "0 4 * * *" # Daily at 04:00 UTC + suspend: false + immediate: false + backupOwnerReference: self + cluster: + name: hermes-pgdb diff --git a/apps/base/customer1/hermes-db/kustomization.yaml b/apps/base/customer1/hermes-db/kustomization.yaml new file mode 100644 index 0000000..9f9677f --- /dev/null +++ b/apps/base/customer1/hermes-db/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - pg-cluster-hermes.yaml + - hermes-db-credentials.yaml + - trading-db-credentials.yaml + - memory-db-credentials.yaml + - trading-data-db.yaml + - agent-memory-db.yaml + - hermes-scheduled-backup.yaml diff --git a/apps/base/customer1/hermes-db/memory-db-credentials.yaml b/apps/base/customer1/hermes-db/memory-db-credentials.yaml new file mode 100644 index 0000000..1151b82 --- /dev/null +++ b/apps/base/customer1/hermes-db/memory-db-credentials.yaml @@ -0,0 +1,24 @@ +apiVersion: ENC[AES256_GCM,data:2mQ=,iv:LNl1ueDj/seVjsPVQ20/CQJnbEr7Fd/BqMogFKbWgCo=,tag:fVqYeJV9BqKWifP1idQKjg==,type:str] +kind: ENC[AES256_GCM,data:44E4DRzE,iv:zIn6ZIrZm44IJuLFERN+WTgctZ/04KkmxURtjsUFUXs=,tag:DbRfwktJT4jdc4TR39QYqQ==,type:str] +metadata: + name: ENC[AES256_GCM,data:rQCpZs4/HHeRnO0OgCr65HhYK84W,iv:dUiKDy3zsp7EO+wd/QI0W0M8dPHYKSDzmCP80Y+XIIM=,tag:bKWz+FI/PtCewuSaVB1cUA==,type:str] + namespace: ENC[AES256_GCM,data:cGonbW6DupwQ,iv:6EGZKJeC77OSgUikYPRRkAPraIdRLFLaQ/FD2+C4gco=,tag:TSl9wo35AMfjU62gcq0lsg==,type:str] +type: ENC[AES256_GCM,data:xXPsacph,iv:u2HydlElviV+Pvs7nsZutnQbK2TIRm88uk7u9/fJels=,tag:h8f7Dy8JpvAJzUWwzjsMUw==,type:str] +stringData: + username: ENC[AES256_GCM,data:VF38x+hU,iv:epSlTbTaVeOkpUZkLQ5QHuidsh5JEQkPz0HpqYQIvLI=,tag:HjZ/F29xirkNR3IMzyA+Dw==,type:str] + password: ENC[AES256_GCM,data:Wu015YXUwNMuIIAUZ4Vd7BDfQS137VuQynL4e7rqtw8=,iv:7xA4FdSOXPJTwkA7guYR1IhPHjlXXDjtCaRcdGJ52kQ=,tag:kw/oxGWhPeXDkBKvxg3vog==,type:str] +sops: + age: + - recipient: age1uuxf066xuuqgvjppxfcmqkwfcufnwp3wcwnl9h20g9k4l8nkw9jsaungf7 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBsMkE4N2R3bTVRUWtZdkVi + N1Q4ZEo5ZHdmZjFNdVAydHdMeXkzZURFZVFBCmJxWnVLQ0F6OU52Y3FTa0N3Y2ZU + M2trTjdZL3E3R1lXa29lOWlGTVFCenMKLS0tIHVhVllpTXNzQ0hSblhnYW56eHIv + OW9sRDJLcU54TmdXdGpsTCtlUlR5NmMK7n5nIwAzxJgvJw9VqStiCVCE9wuNzMX2 + NaREHasP+XtJpB3AGkNgwBr/LW7cZKs5uvy8vKUFqhqpk/hcDsHjhg== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2026-05-02T21:29:40Z" + mac: ENC[AES256_GCM,data:1vcokwfS2/qfWqMWpfTl1wmsgMcx9tUbMv7IP13lzBc1j7YirW4xs+a52hxGFityPJgWvp4/XwdUbt06l33IF7dykN6vT9lFK4792Kx6KzdmCqP2eawtT2GKT1ewfmO1Mjr/YJ77o1qmaek7BPwNv+8vlDg0Wlerbt0jsy2SqCI=,iv:O98sE1/kNbUYaljxDevO9Ow0CbDz7VWcTCRY3wvwF6s=,tag:Yy1kzaKc1JCW6Wkny7ITHg==,type:str] + unencrypted_suffix: _unencrypted + version: 3.12.2 diff --git a/apps/base/customer1/hermes-db/pg-cluster-hermes.yaml b/apps/base/customer1/hermes-db/pg-cluster-hermes.yaml new file mode 100644 index 0000000..158fd4e --- /dev/null +++ b/apps/base/customer1/hermes-db/pg-cluster-hermes.yaml @@ -0,0 +1,55 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: hermes-pgdb + namespace: customer1 + +spec: + instances: 1 + imageName: ghcr.io/cloudnative-pg/postgresql:15.2 + storage: + size: 20Gi + + managed: + roles: + - name: hermes + ensure: present + login: true + passwordSecret: + name: hermes-db-credentials + - name: trading + ensure: present + login: true + passwordSecret: + name: trading-db-credentials + - name: memory + ensure: present + login: true + passwordSecret: + name: memory-db-credentials + + bootstrap: + initdb: + database: hermes_memory + owner: hermes + secret: + name: hermes-db-credentials + + serviceAccountTemplate: + metadata: + name: cnpg-backup-sa + annotations: + iam.gke.io/gcp-service-account: cnpg-backup-sa@devops-lab-cluster.iam.gserviceaccount.com + + backup: + barmanObjectStore: + destinationPath: "gs://customer1_db_backup/hermes-backups/" + googleCredentials: + gkeEnvironment: true + wal: + compression: gzip + data: + compression: gzip + jobs: 2 + retentionPolicy: "30d" + target: primary diff --git a/apps/base/customer1/hermes-db/trading-data-db.yaml b/apps/base/customer1/hermes-db/trading-data-db.yaml new file mode 100644 index 0000000..88da3e0 --- /dev/null +++ b/apps/base/customer1/hermes-db/trading-data-db.yaml @@ -0,0 +1,10 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Database +metadata: + name: trading-data-db + namespace: customer1 +spec: + cluster: + name: hermes-pgdb + name: trading_data + owner: trading diff --git a/apps/base/customer1/hermes-db/trading-db-credentials.yaml b/apps/base/customer1/hermes-db/trading-db-credentials.yaml new file mode 100644 index 0000000..ad1da95 --- /dev/null +++ b/apps/base/customer1/hermes-db/trading-db-credentials.yaml @@ -0,0 +1,24 @@ +apiVersion: ENC[AES256_GCM,data:GSk=,iv:EtxocdVlzVEPk6rHozVdjwOgjWqZNdJfG2PWrVRF1Ik=,tag:pBINgiS3tVGO6iQ0seax/g==,type:str] +kind: ENC[AES256_GCM,data:T5+vJT+t,iv:GEuW09nF9g3ZFOtFx6kJfgkx+U80QYXkVMQDJoAKh7E=,tag:OTsAfg7FiOoe4C6eyZCLyQ==,type:str] +metadata: + name: ENC[AES256_GCM,data:iM8e0gyP/bheft8exNWIuLAu9AObog==,iv:73wEhFh6OVB+vZEvYY0uEQj4U7NgDoOstWmka57J72Y=,tag:vRTgvDqoX3ccZDIwybiH0w==,type:str] + namespace: ENC[AES256_GCM,data:65gK9YhIlftj,iv:uZC14McBjB1LTMhalbPH2KJinstaaZyvlVDDe2FvKCE=,tag:j2yuQ6bD5VRYhAV0JhxflQ==,type:str] +type: ENC[AES256_GCM,data:rjywm84O,iv:Yp9tttUgySGPSMRiSmuMff544kp0+k09t0Q8jGHWjiM=,tag:xgsEUcmI20kfJVsEU+zTBQ==,type:str] +stringData: + username: ENC[AES256_GCM,data:MHbgLPzGjQ==,iv:hwINFxef3rnIetuD1aICeTly2Y9oVieQoRHTfKtEAyk=,tag:hnAirU6ib5aOpRf6JyYZpA==,type:str] + password: ENC[AES256_GCM,data:jrxJhvglCEa+Kwed1mrtT+JnrpmQ4leVMOv1jVVcGw8=,iv:jHyO1uw3eo9uJfwQBZcG/UdVwnpkkr1fsrb9uFCE7B8=,tag:V4VeDoD/I0RE62x2pIFiOA==,type:str] +sops: + age: + - recipient: age1uuxf066xuuqgvjppxfcmqkwfcufnwp3wcwnl9h20g9k4l8nkw9jsaungf7 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBnM0RsNm5UTCsxdWJJRjdp + d1ZZcitVZ1p3aGgzY3YvWVk5aVZIUzZxSWpnCmk1S0NwUXpqWWw1bUoxbW5Kcm9X + cjdlb3pqbkpvQm95K2lxY1JtNlRRZm8KLS0tIERyZFl5M0hJUDdxZlBTaGxGbm1z + M0U5ajRJeTI3TmltOWFoR1R3Sk1LQlkKht8ze+iqhdwucRbqiNMAnwvs+idhs159 + JBnY8Y90O1UBAB/yUoACVjGf32d3c6iCSOi2L35FopTjXNREgcU5wQ== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2026-05-02T21:29:40Z" + mac: ENC[AES256_GCM,data:Bjl+US5pStZ+xiCJ1l1xtDl4bVldkqme5fxMQN5WNsvalo/bKXElUpxVLf52n5jWcZjkjNHWuM407db3NQmSwXqz1o6+qmwIgfv3hz3JViUh0JSiSX/tDz8+yLEQ7fPVkvehxfWaZN2wJhyHhuPvkOmz9Fm+YSxqbw9DaWr2BkI=,iv:83n0XewztuAnIdb/CLPmseQLLbo/uw+rD8t67lxD1J8=,tag:KiKtZKPCX5BRfJfZ8xdDkg==,type:str] + unencrypted_suffix: _unencrypted + version: 3.12.2 diff --git a/apps/staging/customer1/kustomization.yaml b/apps/staging/customer1/kustomization.yaml index 278d710..974b250 100644 --- a/apps/staging/customer1/kustomization.yaml +++ b/apps/staging/customer1/kustomization.yaml @@ -8,3 +8,4 @@ resources: - ../../base/customer1/uncensored-bot/ - ../../base/customer1/paaas-landing/ - ../../base/customer1/hermes-agent/ + - ../../base/customer1/hermes-db/