gcloud-lab/apps/base/customer1/openclaw/vllm-gemma.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: openclaw-brain-vllm
  namespace: customer1
  labels:
    app: openclaw-brain
spec:
  replicas: 1
  selector:
    matchLabels:
      app: openclaw-brain
  template:
    metadata:
      labels:
        app: openclaw-brain
    spec:
      nodeSelector:
        cloud.google.com/gke-accelerator: "nvidia-a100-80gb"
      tolerations:
        - key: "nvidia.com/gpu-a100-80gb"
          operator: "Equal"
          value: "present"
          effect: "NoSchedule"
      containers:
        - name: vllm-brain
          image: vllm/vllm-openai:latest
          command: ["python3", "-m", "vllm.entrypoints.openai.api_server"]
          env:
            - name: HF_TOKEN
              value: ""
            - name: VLLM_TOKENIZER_MODE
              value: "auto"
          args:
            - --model=mlabonne/Qwen3.5-32B-Heretic
            - --host=0.0.0.0
            - --port=8000
            - --tensor-parallel-size=1
            - --gpu-memory-utilization=0.78          # ← Critical: lower than default 0.90
            - --max-model-len=16384                  # Start conservative (you can raise to 32768 later)
            - --max-num-batched-tokens=8192
            - --max-num-seqs=4                       # Lower for single-user / low-concurrency
            - --enforce-eager                        # Disables CUDA graphs → big memory saver at startup
            - --disable-custom-all-reduce
            - --trust-remote-code
            - --dtype=auto
            ##- --limit-mm-per-prompt='{"image"=1, "video"=0}'  # Optional: limit vision if not heavily using images yet 
          ports:
            - containerPort: 8000
          resources:
            limits:
              nvidia.com/gpu: 1
              memory: "80Gi"
              cpu: "16"
            requests:
              nvidia.com/gpu: 1
              memory: "60Gi"
              cpu: "8"
          volumeMounts:
            - name: model-cache
              mountPath: /root/.cache/huggingface
      volumes:
        - name: model-cache
          persistentVolumeClaim:
            claimName: vllm-model-cache-pvc
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: vllm-model-cache-pvc
  namespace: customer1
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 100Gi
---
apiVersion: v1
kind: Service
metadata:
  name: openclaw-brain-service
  namespace: customer1
spec:
  selector:
    app: openclaw-brain
  ports:
    - protocol: TCP
      port: 8000
      targetPort: 8000
  type: ClusterIP