diff --git a/apps/base/customer1/hermes-agent/configmap.yaml b/apps/base/customer1/hermes-agent/configmap.yaml index 4768fce..455f89f 100644 --- a/apps/base/customer1/hermes-agent/configmap.yaml +++ b/apps/base/customer1/hermes-agent/configmap.yaml @@ -14,7 +14,7 @@ data: type: openai api_key: EMPTY base_url: http://rtx6000-brain-service.customer1.svc.cluster.local:8000/v1 - context_length: 131072 + context_length: 176072 qwen-vllm: type: openai api_key: EMPTY diff --git a/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml b/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml index 1062754..dac27e4 100644 --- a/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml +++ b/infrastructure/gpus/base/vllm-servers/rtx6000-vllm.yaml @@ -38,11 +38,12 @@ spec: - --tensor-parallel-size=1 - --tokenizer-mode=hf - --gpu-memory-utilization=0.95 - - --max-model-len=237864 + - --max-model-len=98304 - --enable-auto-tool-choice - --kv-cache-dtype=fp8 - - --max-num-batched-tokens=32768 - - --max-num-seqs=16 + - --max-num-batched-tokens=65536 + - --max-num-seqs=128 + - --enable-chunked-prefill - --trust-remote-code - --dtype=auto - --enable-prefix-caching @@ -59,7 +60,7 @@ spec: requests: nvidia.com/gpu: 1 memory: "150Gi" - cpu: "24" + cpu: "32" startupProbe: httpGet: