refactor: upgrade vLLM brain to uncensored Qwen 2.5 32B AWQ
This commit is contained in:
parent
e68774da48
commit
55a3e9eb28
1 changed files with 6 additions and 4 deletions
|
|
@ -23,16 +23,18 @@ spec:
|
|||
value: "present"
|
||||
effect: "NO_SCHEDULE"
|
||||
containers:
|
||||
- name: vllm-gemma
|
||||
- name: vllm-brain
|
||||
image: vllm/vllm-openai:latest
|
||||
command: ["python3", "-m", "vllm.entrypoints.openai.api_server"]
|
||||
args:
|
||||
- "--model"
|
||||
- "cognitivecomputations/dolphin-2.8-gemma-2b-v0" # Uncensored/Distilled Dolphin-Gemma model
|
||||
- "cognitivecomputations/dolphin-2.9.4-qwen2.5-32b-AWQ" # Uncensored 32B Reasoning Heavyweight (Quantized)
|
||||
- "--quantization"
|
||||
- "awq"
|
||||
- "--dtype"
|
||||
- "bfloat16"
|
||||
- "half"
|
||||
- "--max-model-len"
|
||||
- "8192"
|
||||
- "16384"
|
||||
ports:
|
||||
- containerPort: 8000
|
||||
resources:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue