52 lines
1.3 KiB
YAML
52 lines
1.3 KiB
YAML
name: llm-embeddings
|
|
|
|
services:
|
|
qwen3-embedding:
|
|
image: ${EMBEDDING_IMAGE:-ghcr.io/ggml-org/llama.cpp@sha256:3e8914c1aab600a330ada97c10fb5fb02ff1c15ac39f2ece218db125cf54594e}
|
|
container_name: ${EMBEDDING_CONTAINER_NAME:-llm-qwen3-embedding}
|
|
restart: unless-stopped
|
|
ports:
|
|
- "${EMBEDDING_HOST_PORT:-8082}:8080"
|
|
volumes:
|
|
- ${HOST_LLAMA_CACHE_DIR:-Z:/LLM/models/cache/llama.cpp}:/root/.cache/llama.cpp
|
|
command:
|
|
- --host
|
|
- 0.0.0.0
|
|
- --port
|
|
- "8080"
|
|
- --hf-repo
|
|
- ${EMBEDDING_HF_REPO:-Qwen/Qwen3-Embedding-0.6B-GGUF:Q8_0}
|
|
- --alias
|
|
- ${EMBEDDING_SERVED_MODEL_NAME:-qwen3-embedding-0.6b}
|
|
- --embedding
|
|
- --pooling
|
|
- last
|
|
- --ctx-size
|
|
- ${EMBEDDING_CTX_SIZE:-8192}
|
|
- --batch-size
|
|
- ${EMBEDDING_BATCH_SIZE:-1024}
|
|
- --ubatch-size
|
|
- ${EMBEDDING_UBATCH_SIZE:-1024}
|
|
- --threads
|
|
- ${EMBEDDING_THREADS:-12}
|
|
- --parallel
|
|
- ${EMBEDDING_PARALLEL:-1}
|
|
- --n-gpu-layers
|
|
- "0"
|
|
healthcheck:
|
|
test:
|
|
- CMD
|
|
- curl
|
|
- --fail
|
|
- --silent
|
|
- http://localhost:8080/health
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 30
|
|
start_period: 10m
|
|
deploy:
|
|
resources:
|
|
limits:
|
|
cpus: ${EMBEDDING_CPU_LIMIT:-8.0}
|
|
memory: ${EMBEDDING_MEMORY_LIMIT:-4G}
|