43 lines
1.2 KiB
YAML
43 lines
1.2 KiB
YAML
services:
|
|
vllm:
|
|
image: ${VLLM_IMAGE:-vllm/vllm-openai:latest}
|
|
container_name: ${VLLM_CONTAINER_NAME:-llm-vllm-text}
|
|
restart: unless-stopped
|
|
ipc: host
|
|
ports:
|
|
- "${VLLM_HOST_PORT:-8000}:8000"
|
|
environment:
|
|
HF_HOME: /root/.cache/huggingface
|
|
HUGGING_FACE_HUB_TOKEN: ${HF_TOKEN:-}
|
|
volumes:
|
|
- ${HOST_MODELS_DIR:-/models}:/models
|
|
- ${HOST_HF_CACHE_DIR:-/models/cache/huggingface}:/root/.cache/huggingface
|
|
command:
|
|
- --model
|
|
- ${VLLM_MODEL_ID:-Qwen/Qwen3-4B-Instruct-2507}
|
|
- --served-model-name
|
|
- ${VLLM_SERVED_MODEL_NAME:-qwen3-4b-instruct}
|
|
- --enable-lora
|
|
- --max-loras
|
|
- ${VLLM_MAX_LORAS:-1}
|
|
- --lora-modules
|
|
- ${VLLM_LORA_MODULES:-qwen3-4b-1c=/models/adapters/1c/qwen3-4b-1c-lora-v1}
|
|
- --host
|
|
- 0.0.0.0
|
|
- --port
|
|
- "8000"
|
|
- --gpu-memory-utilization
|
|
- ${VLLM_GPU_MEMORY_UTILIZATION:-0.90}
|
|
- --max-model-len
|
|
- ${VLLM_MAX_MODEL_LEN:-32768}
|
|
- --dtype
|
|
- ${VLLM_DTYPE:-auto}
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
- driver: nvidia
|
|
count: all
|
|
capabilities:
|
|
- gpu
|