replace manual Kafka StatefulSet with Strimzi operator CR

Remove hand-rolled KRaft StatefulSet that kept failing (mount conflicts,
advertised.listeners issues). Replace with a single Strimzi Kafka CR.

Strimzi handles:
- KRaft setup, storage, scaling
- Broker config & advertised listeners (auto-resolved)
- Topic Operator for proper topic lifecycle
- Bootstrap service at trading-kafka-kafka-bootstrap.customer1.svc.cluster.local:9092

Requires Strimzi 1.0.0 operator installed in the cluster first:
  kubectl apply -f https://strimzi.io/install/latest?namespace=customer1 -n customer1
This commit is contained in:
Hermes Agent 2026-05-26 00:37:36 +00:00
parent 572f9f2c0f
commit d012cac7f4
5 changed files with 43 additions and 133 deletions

View file

@ -1,129 +0,0 @@
# Kafka broker (KRaft mode — no ZooKeeper required)
# Single-broker for dev/staging; scale replicas for production
apiVersion: v1
kind: Service
metadata:
name: trading-kafka
namespace: customer1
labels:
app: trading-kafka
spec:
clusterIP: None
selector:
app: trading-kafka
ports:
- name: internal
port: 9092
targetPort: 9092
- name: controller
port: 9093
targetPort: 9093
---
apiVersion: v1
kind: ConfigMap
metadata:
name: trading-kafka-config
namespace: customer1
data:
server.properties: |
process.roles=broker,controller
node.id=1
controller.quorum.voters=1@trading-kafka-0.trading-kafka.customer1.svc.cluster.local:9093
listeners=PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093
advertised.listeners=PLAINTEXT://trading-kafka-0.trading-kafka.customer1.svc.cluster.local:9092
listener.security.protocol.map=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
controller.listener.names=CONTROLLER
inter.broker.listener.name=PLAINTEXT
log.dirs=/var/lib/kafka/data
num.partitions=3
default.replication.factor=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
auto.create.topics.enable=true
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: trading-kafka
namespace: customer1
labels:
app: trading-kafka
spec:
serviceName: trading-kafka
replicas: 1
selector:
matchLabels:
app: trading-kafka
template:
metadata:
labels:
app: trading-kafka
spec:
securityContext:
runAsNonRoot: true
runAsUser: 1000
fsGroup: 1000
containers:
- name: kafka
image: apache/kafka:3.9.0
ports:
- containerPort: 9092
name: internal
- containerPort: 9093
name: controller
env:
- name: KAFKA_HEAP_OPTS
value: "-Xmx512M -Xms256M"
- name: CLUSTER_ID
value: "trading-kafka-cluster-01"
command:
- /bin/bash
- -c
- |
export KAFKA_CLUSTER_ID="$(/opt/kafka/bin/kafka-storage.sh random-uuid)"
/opt/kafka/bin/kafka-storage.sh format -t $KAFKA_CLUSTER_ID -c /opt/kafka/config/server.properties --ignore-formatted
exec /opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties
resources:
requests:
cpu: 250m
memory: 512Mi
limits:
cpu: 1000m
memory: 1Gi
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: false
capabilities:
drop:
- ALL
volumeMounts:
- name: config
mountPath: /opt/kafka/config/server.properties
subPath: server.properties
- name: data
mountPath: /var/lib/kafka/data
readinessProbe:
tcpSocket:
port: 9092
initialDelaySeconds: 30
periodSeconds: 10
failureThreshold: 5
livenessProbe:
tcpSocket:
port: 9092
initialDelaySeconds: 60
periodSeconds: 30
volumes:
- name: config
configMap:
name: trading-kafka-config
volumeClaimTemplates:
- metadata:
name: data
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10Gi

View file

@ -11,5 +11,5 @@ resources:
# Trading platform DB (migrated from hermes-pgdb) # Trading platform DB (migrated from hermes-pgdb)
- trading-db-credentials.yaml - trading-db-credentials.yaml
- trading-data-db.yaml - trading-data-db.yaml
# Kafka broker (moved from hermes-db) # Kafka via Strimzi operator (replaces manual StatefulSet)
- kafka-broker.yaml - trading-kafka.yaml

View file

@ -0,0 +1,39 @@
apiVersion: kafka.strimzi.io/v1beta2
kind: Kafka
metadata:
name: trading-kafka
namespace: customer1
spec:
kafka:
replicas: 1
listeners:
- name: plain
port: 9092
type: internal
tls: false
storage:
type: persistent-claim
size: 10Gi
config:
auto.create.topics.enable: true
default.replication.factor: 1
min.insync.replicas: 1
offsets.topic.replication.factor: 1
transaction.state.log.replication.factor: 1
transaction.state.log.min.isr: 1
template:
pod:
securityContext:
runAsUser: 1000
runAsNonRoot: true
containers:
- name: kafka
resources:
requests:
cpu: 250m
memory: 512Mi
limits:
cpu: 1000m
memory: 1Gi
entityOperator:
topicOperator: {}

View file

@ -9,5 +9,5 @@ data:
DB_NAME: "trading_data" DB_NAME: "trading_data"
REDIS_HOST: "trading-redis.customer1.svc.cluster.local" REDIS_HOST: "trading-redis.customer1.svc.cluster.local"
REDIS_PORT: "6379" REDIS_PORT: "6379"
KAFKA_BOOTSTRAP_SERVERS: "trading-kafka.customer1.svc.cluster.local:9092" KAFKA_BOOTSTRAP_SERVERS: "trading-kafka-kafka-bootstrap.customer1.svc.cluster.local:9092"
LOG_LEVEL: "info" LOG_LEVEL: "info"

View file

@ -9,5 +9,5 @@ data:
EXECUTE_DB_NAME: "trading_data" EXECUTE_DB_NAME: "trading_data"
EXECUTE_REDIS_HOST: "trading-redis.customer1.svc.cluster.local" EXECUTE_REDIS_HOST: "trading-redis.customer1.svc.cluster.local"
EXECUTE_REDIS_PORT: "6379" EXECUTE_REDIS_PORT: "6379"
KAFKA_BOOTSTRAP_SERVERS: "trading-kafka.customer1.svc.cluster.local:9092" KAFKA_BOOTSTRAP_SERVERS: "trading-kafka-kafka-bootstrap.customer1.svc.cluster.local:9092"
EXECUTE_LOG_LEVEL: "info" EXECUTE_LOG_LEVEL: "info"