Initial SQL-only 1C adapter baseline
This commit is contained in:
@@ -0,0 +1,20 @@
|
||||
# Copy to .env on the deployment host and adjust values there.
|
||||
# Do not commit real tokens, private paths, or credentials.
|
||||
|
||||
VLLM_CONTAINER_NAME=llm-vllm-text
|
||||
VLLM_IMAGE=vllm/vllm-openai:latest
|
||||
VLLM_MODEL_ID=Qwen/Qwen3-4B-Instruct-2507
|
||||
VLLM_SERVED_MODEL_NAME=qwen3-4b-instruct
|
||||
VLLM_LORA_MODULES=qwen3-4b-1c=/models/adapters/1c/qwen3-4b-1c-lora-v1
|
||||
VLLM_MAX_LORAS=1
|
||||
VLLM_HOST_PORT=8000
|
||||
VLLM_GPU_MEMORY_UTILIZATION=0.90
|
||||
VLLM_MAX_MODEL_LEN=32768
|
||||
VLLM_DTYPE=auto
|
||||
|
||||
# Local model/cache paths on docker-gpu.cin.su.
|
||||
HOST_MODELS_DIR=/models
|
||||
HOST_HF_CACHE_DIR=/models/cache/huggingface
|
||||
|
||||
# Set on the host only if the model source requires it.
|
||||
HF_TOKEN=
|
||||
@@ -0,0 +1,42 @@
|
||||
services:
|
||||
vllm:
|
||||
image: ${VLLM_IMAGE:-vllm/vllm-openai:latest}
|
||||
container_name: ${VLLM_CONTAINER_NAME:-llm-vllm-text}
|
||||
restart: unless-stopped
|
||||
ipc: host
|
||||
ports:
|
||||
- "${VLLM_HOST_PORT:-8000}:8000"
|
||||
environment:
|
||||
HF_HOME: /root/.cache/huggingface
|
||||
HUGGING_FACE_HUB_TOKEN: ${HF_TOKEN:-}
|
||||
volumes:
|
||||
- ${HOST_MODELS_DIR:-/models}:/models
|
||||
- ${HOST_HF_CACHE_DIR:-/models/cache/huggingface}:/root/.cache/huggingface
|
||||
command:
|
||||
- --model
|
||||
- ${VLLM_MODEL_ID:-Qwen/Qwen3-4B-Instruct-2507}
|
||||
- --served-model-name
|
||||
- ${VLLM_SERVED_MODEL_NAME:-qwen3-4b-instruct}
|
||||
- --enable-lora
|
||||
- --max-loras
|
||||
- ${VLLM_MAX_LORAS:-1}
|
||||
- --lora-modules
|
||||
- ${VLLM_LORA_MODULES:-qwen3-4b-1c=/models/adapters/1c/qwen3-4b-1c-lora-v1}
|
||||
- --host
|
||||
- 0.0.0.0
|
||||
- --port
|
||||
- "8000"
|
||||
- --gpu-memory-utilization
|
||||
- ${VLLM_GPU_MEMORY_UTILIZATION:-0.90}
|
||||
- --max-model-len
|
||||
- ${VLLM_MAX_MODEL_LEN:-32768}
|
||||
- --dtype
|
||||
- ${VLLM_DTYPE:-auto}
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities:
|
||||
- gpu
|
||||
Reference in New Issue
Block a user