diff --git a/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml b/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml index c490324..dc2adb2 100644 --- a/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml +++ b/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml @@ -39,7 +39,7 @@ spec: - --tokenizer-mode=hf - --gpu-memory-utilization=0.90 #- --enforce-eager - - --max-model-len=98304 + - --max-model-len=136876 - --enable-auto-tool-choice - --kv-cache-dtype=fp8 - --max-num-batched-tokens=32768 @@ -51,6 +51,7 @@ spec: - --enable-prefix-caching - --tool-call-parser=qwen3_xml - --reasoning-parser=qwen3 + - --speculative-config '{"method": "mtp", "num_speculative_tokens": 2}' ports: - containerPort: 8000