replace manual Kafka StatefulSet with Strimzi operator CR
Remove hand-rolled KRaft StatefulSet that kept failing (mount conflicts, advertised.listeners issues). Replace with a single Strimzi Kafka CR. Strimzi handles: - KRaft setup, storage, scaling - Broker config & advertised listeners (auto-resolved) - Topic Operator for proper topic lifecycle - Bootstrap service at trading-kafka-kafka-bootstrap.customer1.svc.cluster.local:9092 Requires Strimzi 1.0.0 operator installed in the cluster first: kubectl apply -f https://strimzi.io/install/latest?namespace=customer1 -n customer1
This commit is contained in:
parent
572f9f2c0f
commit
d012cac7f4
5 changed files with 43 additions and 133 deletions
|
|
@ -1,129 +0,0 @@
|
|||
# Kafka broker (KRaft mode — no ZooKeeper required)
|
||||
# Single-broker for dev/staging; scale replicas for production
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: trading-kafka
|
||||
namespace: customer1
|
||||
labels:
|
||||
app: trading-kafka
|
||||
spec:
|
||||
clusterIP: None
|
||||
selector:
|
||||
app: trading-kafka
|
||||
ports:
|
||||
- name: internal
|
||||
port: 9092
|
||||
targetPort: 9092
|
||||
- name: controller
|
||||
port: 9093
|
||||
targetPort: 9093
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: trading-kafka-config
|
||||
namespace: customer1
|
||||
data:
|
||||
server.properties: |
|
||||
process.roles=broker,controller
|
||||
node.id=1
|
||||
controller.quorum.voters=1@trading-kafka-0.trading-kafka.customer1.svc.cluster.local:9093
|
||||
listeners=PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093
|
||||
advertised.listeners=PLAINTEXT://trading-kafka-0.trading-kafka.customer1.svc.cluster.local:9092
|
||||
listener.security.protocol.map=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
|
||||
controller.listener.names=CONTROLLER
|
||||
inter.broker.listener.name=PLAINTEXT
|
||||
log.dirs=/var/lib/kafka/data
|
||||
num.partitions=3
|
||||
default.replication.factor=1
|
||||
offsets.topic.replication.factor=1
|
||||
transaction.state.log.replication.factor=1
|
||||
transaction.state.log.min.isr=1
|
||||
auto.create.topics.enable=true
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: trading-kafka
|
||||
namespace: customer1
|
||||
labels:
|
||||
app: trading-kafka
|
||||
spec:
|
||||
serviceName: trading-kafka
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: trading-kafka
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: trading-kafka
|
||||
spec:
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
fsGroup: 1000
|
||||
containers:
|
||||
- name: kafka
|
||||
image: apache/kafka:3.9.0
|
||||
ports:
|
||||
- containerPort: 9092
|
||||
name: internal
|
||||
- containerPort: 9093
|
||||
name: controller
|
||||
env:
|
||||
- name: KAFKA_HEAP_OPTS
|
||||
value: "-Xmx512M -Xms256M"
|
||||
- name: CLUSTER_ID
|
||||
value: "trading-kafka-cluster-01"
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
export KAFKA_CLUSTER_ID="$(/opt/kafka/bin/kafka-storage.sh random-uuid)"
|
||||
/opt/kafka/bin/kafka-storage.sh format -t $KAFKA_CLUSTER_ID -c /opt/kafka/config/server.properties --ignore-formatted
|
||||
exec /opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: 1000m
|
||||
memory: 1Gi
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: false
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /opt/kafka/config/server.properties
|
||||
subPath: server.properties
|
||||
- name: data
|
||||
mountPath: /var/lib/kafka/data
|
||||
readinessProbe:
|
||||
tcpSocket:
|
||||
port: 9092
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
failureThreshold: 5
|
||||
livenessProbe:
|
||||
tcpSocket:
|
||||
port: 9092
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 30
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: trading-kafka-config
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: data
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
|
|
@ -11,5 +11,5 @@ resources:
|
|||
# Trading platform DB (migrated from hermes-pgdb)
|
||||
- trading-db-credentials.yaml
|
||||
- trading-data-db.yaml
|
||||
# Kafka broker (moved from hermes-db)
|
||||
- kafka-broker.yaml
|
||||
# Kafka via Strimzi operator (replaces manual StatefulSet)
|
||||
- trading-kafka.yaml
|
||||
|
|
|
|||
39
apps/base/customer1/siriusdevops-db/trading-kafka.yaml
Normal file
39
apps/base/customer1/siriusdevops-db/trading-kafka.yaml
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
apiVersion: kafka.strimzi.io/v1beta2
|
||||
kind: Kafka
|
||||
metadata:
|
||||
name: trading-kafka
|
||||
namespace: customer1
|
||||
spec:
|
||||
kafka:
|
||||
replicas: 1
|
||||
listeners:
|
||||
- name: plain
|
||||
port: 9092
|
||||
type: internal
|
||||
tls: false
|
||||
storage:
|
||||
type: persistent-claim
|
||||
size: 10Gi
|
||||
config:
|
||||
auto.create.topics.enable: true
|
||||
default.replication.factor: 1
|
||||
min.insync.replicas: 1
|
||||
offsets.topic.replication.factor: 1
|
||||
transaction.state.log.replication.factor: 1
|
||||
transaction.state.log.min.isr: 1
|
||||
template:
|
||||
pod:
|
||||
securityContext:
|
||||
runAsUser: 1000
|
||||
runAsNonRoot: true
|
||||
containers:
|
||||
- name: kafka
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: 1000m
|
||||
memory: 1Gi
|
||||
entityOperator:
|
||||
topicOperator: {}
|
||||
|
|
@ -9,5 +9,5 @@ data:
|
|||
DB_NAME: "trading_data"
|
||||
REDIS_HOST: "trading-redis.customer1.svc.cluster.local"
|
||||
REDIS_PORT: "6379"
|
||||
KAFKA_BOOTSTRAP_SERVERS: "trading-kafka.customer1.svc.cluster.local:9092"
|
||||
KAFKA_BOOTSTRAP_SERVERS: "trading-kafka-kafka-bootstrap.customer1.svc.cluster.local:9092"
|
||||
LOG_LEVEL: "info"
|
||||
|
|
|
|||
|
|
@ -9,5 +9,5 @@ data:
|
|||
EXECUTE_DB_NAME: "trading_data"
|
||||
EXECUTE_REDIS_HOST: "trading-redis.customer1.svc.cluster.local"
|
||||
EXECUTE_REDIS_PORT: "6379"
|
||||
KAFKA_BOOTSTRAP_SERVERS: "trading-kafka.customer1.svc.cluster.local:9092"
|
||||
KAFKA_BOOTSTRAP_SERVERS: "trading-kafka-kafka-bootstrap.customer1.svc.cluster.local:9092"
|
||||
EXECUTE_LOG_LEVEL: "info"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue