From 25850b5b0490a7dc77d4d3d77baa081eea82c5b5 Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Tue, 26 May 2026 00:37:36 +0000 Subject: [PATCH 1/4] replace manual Kafka StatefulSet with Strimzi operator CR Remove hand-rolled KRaft StatefulSet that kept failing (mount conflicts, advertised.listeners issues). Replace with a single Strimzi Kafka CR. Strimzi handles: - KRaft setup, storage, scaling - Broker config & advertised listeners (auto-resolved) - Topic Operator for proper topic lifecycle - Bootstrap service at trading-kafka-kafka-bootstrap.customer1.svc.cluster.local:9092 Requires Strimzi 1.0.0 operator installed in the cluster first: kubectl apply -f https://strimzi.io/install/latest?namespace=customer1 -n customer1 --- .../siriusdevops-db/kafka-broker.yaml | 129 ------------------ .../siriusdevops-db/kustomization.yaml | 4 +- .../siriusdevops-db/trading-kafka.yaml | 39 ++++++ .../configmaps/data-service-config.yaml | 2 +- .../configmaps/execute-service-config.yaml | 2 +- 5 files changed, 43 insertions(+), 133 deletions(-) delete mode 100644 apps/base/customer1/siriusdevops-db/kafka-broker.yaml create mode 100644 apps/base/customer1/siriusdevops-db/trading-kafka.yaml diff --git a/apps/base/customer1/siriusdevops-db/kafka-broker.yaml b/apps/base/customer1/siriusdevops-db/kafka-broker.yaml deleted file mode 100644 index 8dd9a2e..0000000 --- a/apps/base/customer1/siriusdevops-db/kafka-broker.yaml +++ /dev/null @@ -1,129 +0,0 @@ -# Kafka broker (KRaft mode — no ZooKeeper required) -# Single-broker for dev/staging; scale replicas for production -apiVersion: v1 -kind: Service -metadata: - name: trading-kafka - namespace: customer1 - labels: - app: trading-kafka -spec: - clusterIP: None - selector: - app: trading-kafka - ports: - - name: internal - port: 9092 - targetPort: 9092 - - name: controller - port: 9093 - targetPort: 9093 ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: trading-kafka-config - namespace: customer1 -data: - server.properties: | - process.roles=broker,controller - node.id=1 - controller.quorum.voters=1@trading-kafka-0.trading-kafka.customer1.svc.cluster.local:9093 - listeners=PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093 - advertised.listeners=PLAINTEXT://trading-kafka-0.trading-kafka.customer1.svc.cluster.local:9092 - listener.security.protocol.map=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT - controller.listener.names=CONTROLLER - inter.broker.listener.name=PLAINTEXT - log.dirs=/var/lib/kafka/data - num.partitions=3 - default.replication.factor=1 - offsets.topic.replication.factor=1 - transaction.state.log.replication.factor=1 - transaction.state.log.min.isr=1 - auto.create.topics.enable=true ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: trading-kafka - namespace: customer1 - labels: - app: trading-kafka -spec: - serviceName: trading-kafka - replicas: 1 - selector: - matchLabels: - app: trading-kafka - template: - metadata: - labels: - app: trading-kafka - spec: - securityContext: - runAsNonRoot: true - runAsUser: 1000 - fsGroup: 1000 - containers: - - name: kafka - image: apache/kafka:3.9.0 - ports: - - containerPort: 9092 - name: internal - - containerPort: 9093 - name: controller - env: - - name: KAFKA_HEAP_OPTS - value: "-Xmx512M -Xms256M" - - name: CLUSTER_ID - value: "trading-kafka-cluster-01" - command: - - /bin/bash - - -c - - | - export KAFKA_CLUSTER_ID="$(/opt/kafka/bin/kafka-storage.sh random-uuid)" - /opt/kafka/bin/kafka-storage.sh format -t $KAFKA_CLUSTER_ID -c /opt/kafka/config/server.properties --ignore-formatted - exec /opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties - resources: - requests: - cpu: 250m - memory: 512Mi - limits: - cpu: 1000m - memory: 1Gi - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: false - capabilities: - drop: - - ALL - volumeMounts: - - name: config - mountPath: /opt/kafka/config/server.properties - subPath: server.properties - - name: data - mountPath: /var/lib/kafka/data - readinessProbe: - tcpSocket: - port: 9092 - initialDelaySeconds: 30 - periodSeconds: 10 - failureThreshold: 5 - livenessProbe: - tcpSocket: - port: 9092 - initialDelaySeconds: 60 - periodSeconds: 30 - volumes: - - name: config - configMap: - name: trading-kafka-config - volumeClaimTemplates: - - metadata: - name: data - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi diff --git a/apps/base/customer1/siriusdevops-db/kustomization.yaml b/apps/base/customer1/siriusdevops-db/kustomization.yaml index 7af80be..e19a542 100644 --- a/apps/base/customer1/siriusdevops-db/kustomization.yaml +++ b/apps/base/customer1/siriusdevops-db/kustomization.yaml @@ -11,5 +11,5 @@ resources: # Trading platform DB (migrated from hermes-pgdb) - trading-db-credentials.yaml - trading-data-db.yaml - # Kafka broker (moved from hermes-db) - - kafka-broker.yaml + # Kafka via Strimzi operator (replaces manual StatefulSet) + - trading-kafka.yaml diff --git a/apps/base/customer1/siriusdevops-db/trading-kafka.yaml b/apps/base/customer1/siriusdevops-db/trading-kafka.yaml new file mode 100644 index 0000000..a241ebd --- /dev/null +++ b/apps/base/customer1/siriusdevops-db/trading-kafka.yaml @@ -0,0 +1,39 @@ +apiVersion: kafka.strimzi.io/v1beta2 +kind: Kafka +metadata: + name: trading-kafka + namespace: customer1 +spec: + kafka: + replicas: 1 + listeners: + - name: plain + port: 9092 + type: internal + tls: false + storage: + type: persistent-claim + size: 10Gi + config: + auto.create.topics.enable: true + default.replication.factor: 1 + min.insync.replicas: 1 + offsets.topic.replication.factor: 1 + transaction.state.log.replication.factor: 1 + transaction.state.log.min.isr: 1 + template: + pod: + securityContext: + runAsUser: 1000 + runAsNonRoot: true + containers: + - name: kafka + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 1000m + memory: 1Gi + entityOperator: + topicOperator: {} diff --git a/apps/base/customer1/trading-platform/configmaps/data-service-config.yaml b/apps/base/customer1/trading-platform/configmaps/data-service-config.yaml index e355247..179284c 100644 --- a/apps/base/customer1/trading-platform/configmaps/data-service-config.yaml +++ b/apps/base/customer1/trading-platform/configmaps/data-service-config.yaml @@ -9,5 +9,5 @@ data: DB_NAME: "trading_data" REDIS_HOST: "trading-redis.customer1.svc.cluster.local" REDIS_PORT: "6379" - KAFKA_BOOTSTRAP_SERVERS: "trading-kafka.customer1.svc.cluster.local:9092" + KAFKA_BOOTSTRAP_SERVERS: "trading-kafka-kafka-bootstrap.customer1.svc.cluster.local:9092" LOG_LEVEL: "info" diff --git a/apps/base/customer1/trading-platform/configmaps/execute-service-config.yaml b/apps/base/customer1/trading-platform/configmaps/execute-service-config.yaml index 55fba5f..e62bbcf 100644 --- a/apps/base/customer1/trading-platform/configmaps/execute-service-config.yaml +++ b/apps/base/customer1/trading-platform/configmaps/execute-service-config.yaml @@ -9,5 +9,5 @@ data: EXECUTE_DB_NAME: "trading_data" EXECUTE_REDIS_HOST: "trading-redis.customer1.svc.cluster.local" EXECUTE_REDIS_PORT: "6379" - KAFKA_BOOTSTRAP_SERVERS: "trading-kafka.customer1.svc.cluster.local:9092" + KAFKA_BOOTSTRAP_SERVERS: "trading-kafka-kafka-bootstrap.customer1.svc.cluster.local:9092" EXECUTE_LOG_LEVEL: "info" From 94f6b04c281273e9d7697f7c9e5b32c73b606bf7 Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Tue, 26 May 2026 00:49:08 +0000 Subject: [PATCH 2/4] add Strimzi operator as kustomize remote URL (GitOps) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reference Strimzi 1.0.0 install bundle directly from kustomization. No separate install step needed — kubectl apply -k handles CRDs + operator in one shot before the Kafka CR. Pinned to 1.0.0 for reproducible deployments. --- apps/base/customer1/siriusdevops-db/kustomization.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/base/customer1/siriusdevops-db/kustomization.yaml b/apps/base/customer1/siriusdevops-db/kustomization.yaml index e19a542..a57f838 100644 --- a/apps/base/customer1/siriusdevops-db/kustomization.yaml +++ b/apps/base/customer1/siriusdevops-db/kustomization.yaml @@ -11,5 +11,7 @@ resources: # Trading platform DB (migrated from hermes-pgdb) - trading-db-credentials.yaml - trading-data-db.yaml - # Kafka via Strimzi operator (replaces manual StatefulSet) + # Strimzi Kafka operator (CRDs + operator deployment) + - https://strimzi.io/install/1.0.0?namespace=customer1 + # Kafka cluster (applied after operator CRDs are available) - trading-kafka.yaml From 47aae85ed348d2ec004ea79fdd5d1c95d4c90b76 Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Tue, 26 May 2026 01:09:56 +0000 Subject: [PATCH 3/4] split Strimzi operator into separate kustomization for Flux dependsOn Flux dry-run fails when CRDs are in the same kustomization as resources that reference them - CRDs need to exist before validation. Two kustomizations now: strimzi/ - Strimzi 1.0.0 CRDs + operator siriusdevops-db - DBs, Kafka CR (depends on strimzi CRDs) Wire them in Flux: Kustomization 'siriusdevops-strimzi' -> path: strimzi/ Kustomization 'siriusdevops-db' -> path: siriusdevops-db/ dependsOn: [{name: siriusdevops-strimzi}] --- apps/base/customer1/siriusdevops-db/kustomization.yaml | 4 +--- .../customer1/siriusdevops-db/strimzi/kustomization.yaml | 6 ++++++ 2 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 apps/base/customer1/siriusdevops-db/strimzi/kustomization.yaml diff --git a/apps/base/customer1/siriusdevops-db/kustomization.yaml b/apps/base/customer1/siriusdevops-db/kustomization.yaml index a57f838..4f1f076 100644 --- a/apps/base/customer1/siriusdevops-db/kustomization.yaml +++ b/apps/base/customer1/siriusdevops-db/kustomization.yaml @@ -11,7 +11,5 @@ resources: # Trading platform DB (migrated from hermes-pgdb) - trading-db-credentials.yaml - trading-data-db.yaml - # Strimzi Kafka operator (CRDs + operator deployment) - - https://strimzi.io/install/1.0.0?namespace=customer1 - # Kafka cluster (applied after operator CRDs are available) + # Kafka cluster (depends on strimzi/ kustomization for CRDs) - trading-kafka.yaml diff --git a/apps/base/customer1/siriusdevops-db/strimzi/kustomization.yaml b/apps/base/customer1/siriusdevops-db/strimzi/kustomization.yaml new file mode 100644 index 0000000..353bf9b --- /dev/null +++ b/apps/base/customer1/siriusdevops-db/strimzi/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + # Strimzi Kafka operator - CRDs + cluster operator + # Pinned to 1.0.0 for reproducible deployments + - https://strimzi.io/install/1.0.0?namespace=customer1 From d920cc7c633ee8c530a065e2b79c51ca3f41808f Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Tue, 26 May 2026 01:30:28 +0000 Subject: [PATCH 4/4] add customer1-strimzi Flux Kustomization + force:true on customer1 Split Strimzi operator deployment into its own Flux Kustomization so CRDs are applied before the Kafka CR in customer1. Chain: infrastructure-controllers -> customer1-strimzi -> customer1 customer1 now has force: true to skip dry-run validation when CRDs are recently applied but not yet visible to the APIServer cache. --- .../customer1-strimzi/kustomization.yaml | 5 +++++ clusters/devops-lab/customer1-strimzi.yaml | 21 +++++++++++++++++++ clusters/devops-lab/customer1.yaml | 3 ++- 3 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 apps/staging/customer1-strimzi/kustomization.yaml create mode 100644 clusters/devops-lab/customer1-strimzi.yaml diff --git a/apps/staging/customer1-strimzi/kustomization.yaml b/apps/staging/customer1-strimzi/kustomization.yaml new file mode 100644 index 0000000..97df51c --- /dev/null +++ b/apps/staging/customer1-strimzi/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: customer1 +resources: + - ../../base/customer1/siriusdevops-db/strimzi diff --git a/clusters/devops-lab/customer1-strimzi.yaml b/clusters/devops-lab/customer1-strimzi.yaml new file mode 100644 index 0000000..e3d87fd --- /dev/null +++ b/clusters/devops-lab/customer1-strimzi.yaml @@ -0,0 +1,21 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: customer1-strimzi + namespace: flux-system +spec: + interval: 1m0s + retryInterval: 1m + timeout: 5m + dependsOn: + - name: infrastructure-controllers + sourceRef: + kind: GitRepository + name: flux-system + path: ../../apps/staging/customer1-strimzi + prune: true + force: true + decryption: + provider: sops + secretRef: + name: sops-age diff --git a/clusters/devops-lab/customer1.yaml b/clusters/devops-lab/customer1.yaml index 0b9fc11..9d05dfb 100644 --- a/clusters/devops-lab/customer1.yaml +++ b/clusters/devops-lab/customer1.yaml @@ -8,12 +8,13 @@ spec: retryInterval: 1m timeout: 5m dependsOn: - - name: infrastructure-controllers + - name: customer1-strimzi sourceRef: kind: GitRepository name: flux-system path: ../../apps/staging/customer1 prune: true + force: true decryption: provider: sops secretRef: