diff --git a/infrastructure/gpus/base/vllm-servers/a100-vllm.yaml b/infrastructure/gpus/base/vllm-servers/a100-vllm.yaml index 6adfa02..71206da 100644 --- a/infrastructure/gpus/base/vllm-servers/a100-vllm.yaml +++ b/infrastructure/gpus/base/vllm-servers/a100-vllm.yaml @@ -1,19 +1,19 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: openclaw-brain-vllm + name: a100-brain-vllm namespace: customer1 labels: - app: openclaw-brain + app: a100-brain spec: replicas: 1 selector: matchLabels: - app: openclaw-brain + app: a100-brain template: metadata: labels: - app: openclaw-brain + app: a100-brain spec: nodeSelector: cloud.google.com/gke-accelerator: "nvidia-a100-80gb" @@ -32,35 +32,37 @@ spec: - name: TOKENIZER_MODE value: "hf" args: - - --model=Youssofal/Qwen3.6-27B-Abliterated-Heretic-Uncensored-BF16 + - --model=sakamakismile/Huihui-Qwen3.6-27B-abliterated-NVFP4-MTP - --host=0.0.0.0 - --port=8000 - --tensor-parallel-size=1 - --tokenizer-mode=hf - --gpu-memory-utilization=0.95 - - --kv-cache-dtype=fp8 - - --max-model-len=131072 + - --kv-cache-dtype=fp8 + - --max-model-len=262144 - --enable-auto-tool-choice - --enable-chunked-prefill - - --max-num-batched-tokens=8192 - - --max-num-seqs=4 + - --max-num-batched-tokens=16384 + - --max-num-seqs=2 - --trust-remote-code - --dtype=auto - --enable-prefix-caching - --tool-call-parser=qwen3_xml - --reasoning-parser=qwen3 - --disable-custom-all-reduce + - --attention-backend=flash_attn + - --quantization=modelopt ports: - containerPort: 8000 resources: limits: nvidia.com/gpu: 1 - memory: "100Gi" - cpu: "16" + + requests: nvidia.com/gpu: 1 - memory: "80Gi" - cpu: "8" + memory: "100Gi" + cpu: "24" startupProbe: httpGet: @@ -99,12 +101,12 @@ spec: volumes: - name: model-cache persistentVolumeClaim: - claimName: vllm-model-qwen3.6-27b-uncensored + claimName: vllm-model-qwen3.6-27b-uncensored-4bit --- apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: vllm-model-qwen3.6-27b-uncensored + name: vllm-model-qwen3.6-27b-uncensored-4bit namespace: customer1 spec: accessModes: @@ -116,11 +118,11 @@ spec: apiVersion: v1 kind: Service metadata: - name: openclaw-brain-service + name: a100-brain-service namespace: customer1 spec: selector: - app: openclaw-brain + app: a100-brain ports: - protocol: TCP port: 8000 diff --git a/infrastructure/gpus/base/vllm-servers/kustomization.yaml b/infrastructure/gpus/base/vllm-servers/kustomization.yaml index 15c8559..cc1428c 100644 --- a/infrastructure/gpus/base/vllm-servers/kustomization.yaml +++ b/infrastructure/gpus/base/vllm-servers/kustomization.yaml @@ -1,6 +1,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - #- a100-vllm.yaml + - a100-vllm.yaml - rtx6000-vllm.yaml #- vllm-l4.yaml