From 680a16e4a88ef12f5bb7c9a81be013b134e269fd Mon Sep 17 00:00:00 2001 From: sirius0xdev Date: Mon, 4 May 2026 21:53:08 +0000 Subject: [PATCH] try new vllm config --- .../gpus/base/vllm-servers/rtx6000-vllm.yaml | 12 ++++++------ infrastructure/gpus/staging/kustomization.yaml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml b/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml index bcb0cb4..92d6519 100644 --- a/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml +++ b/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml @@ -38,14 +38,14 @@ spec: - --port=8000 - --tensor-parallel-size=1 - --tokenizer-mode=hf - - --gpu-memory-utilization=0.95 + - --gpu-memory-utilization=0.98 - --max-model-len=262144 - --enable-auto-tool-choice - - --enable-chunked-prefill - - --max-num-batched-tokens=4096 + - --disable-chunked-prefill + - --max-num-batched-tokens=65536 - --max-num-seqs=8 - --trust-remote-code - - --dtype=auto + - --dtype=fp8 - --enable-prefix-caching - --tool-call-parser=qwen3_xml - --reasoning-parser=qwen3 @@ -59,8 +59,8 @@ spec: requests: nvidia.com/gpu: 1 - memory: "120Gi" - cpu: "8" + memory: "250Gi" + cpu: "40" startupProbe: httpGet: diff --git a/infrastructure/gpus/staging/kustomization.yaml b/infrastructure/gpus/staging/kustomization.yaml index 3187a27..adcabb6 100644 --- a/infrastructure/gpus/staging/kustomization.yaml +++ b/infrastructure/gpus/staging/kustomization.yaml @@ -1,5 +1,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - #- ../base/vllm-servers/ + - ../base/vllm-servers/ # - ../base/keda-gpu-scaling/