From d012cac7f4a62058e09cb5cacc02c92dcd3ff8ba Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Tue, 26 May 2026 00:37:36 +0000 Subject: [PATCH] replace manual Kafka StatefulSet with Strimzi operator CR Remove hand-rolled KRaft StatefulSet that kept failing (mount conflicts, advertised.listeners issues). Replace with a single Strimzi Kafka CR. Strimzi handles: - KRaft setup, storage, scaling - Broker config & advertised listeners (auto-resolved) - Topic Operator for proper topic lifecycle - Bootstrap service at trading-kafka-kafka-bootstrap.customer1.svc.cluster.local:9092 Requires Strimzi 1.0.0 operator installed in the cluster first: kubectl apply -f https://strimzi.io/install/latest?namespace=customer1 -n customer1 --- .../siriusdevops-db/kafka-broker.yaml | 129 ------------------ .../siriusdevops-db/kustomization.yaml | 4 +- .../siriusdevops-db/trading-kafka.yaml | 39 ++++++ .../configmaps/data-service-config.yaml | 2 +- .../configmaps/execute-service-config.yaml | 2 +- 5 files changed, 43 insertions(+), 133 deletions(-) delete mode 100644 apps/base/customer1/siriusdevops-db/kafka-broker.yaml create mode 100644 apps/base/customer1/siriusdevops-db/trading-kafka.yaml diff --git a/apps/base/customer1/siriusdevops-db/kafka-broker.yaml b/apps/base/customer1/siriusdevops-db/kafka-broker.yaml deleted file mode 100644 index 8dd9a2e..0000000 --- a/apps/base/customer1/siriusdevops-db/kafka-broker.yaml +++ /dev/null @@ -1,129 +0,0 @@ -# Kafka broker (KRaft mode — no ZooKeeper required) -# Single-broker for dev/staging; scale replicas for production -apiVersion: v1 -kind: Service -metadata: - name: trading-kafka - namespace: customer1 - labels: - app: trading-kafka -spec: - clusterIP: None - selector: - app: trading-kafka - ports: - - name: internal - port: 9092 - targetPort: 9092 - - name: controller - port: 9093 - targetPort: 9093 ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: trading-kafka-config - namespace: customer1 -data: - server.properties: | - process.roles=broker,controller - node.id=1 - controller.quorum.voters=1@trading-kafka-0.trading-kafka.customer1.svc.cluster.local:9093 - listeners=PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093 - advertised.listeners=PLAINTEXT://trading-kafka-0.trading-kafka.customer1.svc.cluster.local:9092 - listener.security.protocol.map=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT - controller.listener.names=CONTROLLER - inter.broker.listener.name=PLAINTEXT - log.dirs=/var/lib/kafka/data - num.partitions=3 - default.replication.factor=1 - offsets.topic.replication.factor=1 - transaction.state.log.replication.factor=1 - transaction.state.log.min.isr=1 - auto.create.topics.enable=true ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: trading-kafka - namespace: customer1 - labels: - app: trading-kafka -spec: - serviceName: trading-kafka - replicas: 1 - selector: - matchLabels: - app: trading-kafka - template: - metadata: - labels: - app: trading-kafka - spec: - securityContext: - runAsNonRoot: true - runAsUser: 1000 - fsGroup: 1000 - containers: - - name: kafka - image: apache/kafka:3.9.0 - ports: - - containerPort: 9092 - name: internal - - containerPort: 9093 - name: controller - env: - - name: KAFKA_HEAP_OPTS - value: "-Xmx512M -Xms256M" - - name: CLUSTER_ID - value: "trading-kafka-cluster-01" - command: - - /bin/bash - - -c - - | - export KAFKA_CLUSTER_ID="$(/opt/kafka/bin/kafka-storage.sh random-uuid)" - /opt/kafka/bin/kafka-storage.sh format -t $KAFKA_CLUSTER_ID -c /opt/kafka/config/server.properties --ignore-formatted - exec /opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties - resources: - requests: - cpu: 250m - memory: 512Mi - limits: - cpu: 1000m - memory: 1Gi - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: false - capabilities: - drop: - - ALL - volumeMounts: - - name: config - mountPath: /opt/kafka/config/server.properties - subPath: server.properties - - name: data - mountPath: /var/lib/kafka/data - readinessProbe: - tcpSocket: - port: 9092 - initialDelaySeconds: 30 - periodSeconds: 10 - failureThreshold: 5 - livenessProbe: - tcpSocket: - port: 9092 - initialDelaySeconds: 60 - periodSeconds: 30 - volumes: - - name: config - configMap: - name: trading-kafka-config - volumeClaimTemplates: - - metadata: - name: data - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi diff --git a/apps/base/customer1/siriusdevops-db/kustomization.yaml b/apps/base/customer1/siriusdevops-db/kustomization.yaml index 7af80be..e19a542 100644 --- a/apps/base/customer1/siriusdevops-db/kustomization.yaml +++ b/apps/base/customer1/siriusdevops-db/kustomization.yaml @@ -11,5 +11,5 @@ resources: # Trading platform DB (migrated from hermes-pgdb) - trading-db-credentials.yaml - trading-data-db.yaml - # Kafka broker (moved from hermes-db) - - kafka-broker.yaml + # Kafka via Strimzi operator (replaces manual StatefulSet) + - trading-kafka.yaml diff --git a/apps/base/customer1/siriusdevops-db/trading-kafka.yaml b/apps/base/customer1/siriusdevops-db/trading-kafka.yaml new file mode 100644 index 0000000..a241ebd --- /dev/null +++ b/apps/base/customer1/siriusdevops-db/trading-kafka.yaml @@ -0,0 +1,39 @@ +apiVersion: kafka.strimzi.io/v1beta2 +kind: Kafka +metadata: + name: trading-kafka + namespace: customer1 +spec: + kafka: + replicas: 1 + listeners: + - name: plain + port: 9092 + type: internal + tls: false + storage: + type: persistent-claim + size: 10Gi + config: + auto.create.topics.enable: true + default.replication.factor: 1 + min.insync.replicas: 1 + offsets.topic.replication.factor: 1 + transaction.state.log.replication.factor: 1 + transaction.state.log.min.isr: 1 + template: + pod: + securityContext: + runAsUser: 1000 + runAsNonRoot: true + containers: + - name: kafka + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 1000m + memory: 1Gi + entityOperator: + topicOperator: {} diff --git a/apps/base/customer1/trading-platform/configmaps/data-service-config.yaml b/apps/base/customer1/trading-platform/configmaps/data-service-config.yaml index e355247..179284c 100644 --- a/apps/base/customer1/trading-platform/configmaps/data-service-config.yaml +++ b/apps/base/customer1/trading-platform/configmaps/data-service-config.yaml @@ -9,5 +9,5 @@ data: DB_NAME: "trading_data" REDIS_HOST: "trading-redis.customer1.svc.cluster.local" REDIS_PORT: "6379" - KAFKA_BOOTSTRAP_SERVERS: "trading-kafka.customer1.svc.cluster.local:9092" + KAFKA_BOOTSTRAP_SERVERS: "trading-kafka-kafka-bootstrap.customer1.svc.cluster.local:9092" LOG_LEVEL: "info" diff --git a/apps/base/customer1/trading-platform/configmaps/execute-service-config.yaml b/apps/base/customer1/trading-platform/configmaps/execute-service-config.yaml index 55fba5f..e62bbcf 100644 --- a/apps/base/customer1/trading-platform/configmaps/execute-service-config.yaml +++ b/apps/base/customer1/trading-platform/configmaps/execute-service-config.yaml @@ -9,5 +9,5 @@ data: EXECUTE_DB_NAME: "trading_data" EXECUTE_REDIS_HOST: "trading-redis.customer1.svc.cluster.local" EXECUTE_REDIS_PORT: "6379" - KAFKA_BOOTSTRAP_SERVERS: "trading-kafka.customer1.svc.cluster.local:9092" + KAFKA_BOOTSTRAP_SERVERS: "trading-kafka-kafka-bootstrap.customer1.svc.cluster.local:9092" EXECUTE_LOG_LEVEL: "info"