44 lines
1.1 KiB
YAML
44 lines
1.1 KiB
YAML
services:
|
|
llama-cpp:
|
|
image: ${LLAMA_IMAGE:-ghcr.io/ggml-org/llama.cpp:server-cuda}
|
|
container_name: ${LLAMA_CONTAINER_NAME:-llm-llama-devstral-1c}
|
|
restart: unless-stopped
|
|
ports:
|
|
- "${LLAMA_HOST_PORT:-8080}:8080"
|
|
volumes:
|
|
- ${HOST_MODELS_DIR:-/models}:/models
|
|
command:
|
|
- --host
|
|
- 0.0.0.0
|
|
- --port
|
|
- "8080"
|
|
- --model
|
|
- ${LLAMA_MODEL_PATH}
|
|
- --alias
|
|
- ${LLAMA_SERVED_MODEL_NAME:-devstral-1c-q4}
|
|
- --ctx-size
|
|
- ${LLAMA_CTX_SIZE:-32768}
|
|
- --n-gpu-layers
|
|
- ${LLAMA_GPU_LAYERS:-999}
|
|
- --threads
|
|
- ${LLAMA_THREADS:-8}
|
|
- --parallel
|
|
- ${LLAMA_PARALLEL:-1}
|
|
- --fit
|
|
- ${LLAMA_FIT:-off}
|
|
- --reasoning
|
|
- ${LLAMA_REASONING:-off}
|
|
- --reasoning-format
|
|
- ${LLAMA_REASONING_FORMAT:-none}
|
|
- --chat-template
|
|
- ${LLAMA_CHAT_TEMPLATE:-mistral-v7-tekken}
|
|
- --skip-chat-parsing
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
- driver: nvidia
|
|
count: all
|
|
capabilities:
|
|
- gpu
|