feat: optimize cost via L4 spot instances and vLLM performance tuning
This commit is contained in:
parent
12aef25aa0
commit
43eb51d5b7
3 changed files with 5 additions and 3 deletions
|
|
@ -24,7 +24,7 @@ spec:
|
|||
effect: "NoSchedule"
|
||||
containers:
|
||||
- name: vllm-brain
|
||||
image: vllm/vllm-openai:latest
|
||||
image: vllm/vllm-openai:v0.4.2
|
||||
command: ["python3", "-m", "vllm.entrypoints.openai.api_server"]
|
||||
env:
|
||||
- name: HF_TOKEN
|
||||
|
|
@ -44,6 +44,8 @@ spec:
|
|||
- --tool-call-parser=qwen3_coder
|
||||
- --reasoning-parser=qwen3
|
||||
- --enable-prefix-caching
|
||||
- --enable-chunked-prefill
|
||||
- --kv-cache-dtype=fp8
|
||||
##- --limit-mm-per-prompt='{"image"=1, "video"=0}' # Optional: limit vision if not heavily using images yet
|
||||
ports:
|
||||
- containerPort: 8000
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ spec:
|
|||
effect: "NoSchedule"
|
||||
containers:
|
||||
- name: vllm-brain-l4
|
||||
image: vllm/vllm-openai:latest
|
||||
image: vllm/vllm-openai:v0.4.2
|
||||
command: ["python3", "-m", "vllm.entrypoints.openai.api_server"]
|
||||
env:
|
||||
- name: HF_TOKEN
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ resource "google_container_node_pool" "gpu_pool" {
|
|||
}
|
||||
|
||||
|
||||
spot = false
|
||||
spot = true
|
||||
|
||||
oauth_scopes = [
|
||||
"https://www.googleapis.com/auth/cloud-platform"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue