20 lines
779 B
Bash
20 lines
779 B
Bash
# Qwen3-Coder Q6 GPU llama.cpp settings for docker-gpu.cin.su.
|
|
# This model is too large to keep online together with SDXL/vLLM on a 24 GiB RTX 4090.
|
|
|
|
LLAMA_CONTAINER_NAME=llm-llama-qwen3-coder-q6-test
|
|
LLAMA_IMAGE=ghcr.io/ggml-org/llama.cpp:server-cuda
|
|
LLAMA_HOST_PORT=8081
|
|
LLAMA_MODEL_PATH=/models/gguf/1c/qwen3-coder-30b-a3b-instruct-q6_k/Qwen3-Coder-30B-A3B-Instruct-Q6_K.gguf
|
|
LLAMA_SERVED_MODEL_NAME=qwen3-coder-1c-q6
|
|
LLAMA_CTX_SIZE=8192
|
|
LLAMA_GPU_LAYERS=auto
|
|
LLAMA_THREADS=12
|
|
LLAMA_PARALLEL=1
|
|
LLAMA_FIT=on
|
|
LLAMA_REASONING=auto
|
|
LLAMA_REASONING_FORMAT=none
|
|
LLAMA_CHAT_TEMPLATE=chatml
|
|
# Optional future LoRA path after converting the trained adapter to GGUF for llama.cpp:
|
|
# LLAMA_LORA_PATH=/models/adapters/1c/qwen3-coder-30b-a3b-1c-lora-v1.gguf
|
|
HOST_MODELS_DIR=Z:/LLM/models
|