Initial SQL-only 1C adapter baseline

2026-07-22 03:03:47 +03:00
commit e2503b77e7
545 changed files with 184711 additions and 0 deletions
@@ -0,0 +1,17 @@
+# Copy to .env on the deployment host and adjust values there.
+# Do not commit real tokens, private paths, or credentials.
+
+LLAMA_CONTAINER_NAME=llm-llama-devstral-1c
+LLAMA_IMAGE=ghcr.io/ggml-org/llama.cpp:server-cuda
+LLAMA_HOST_PORT=8080
+LLAMA_MODEL_PATH=/models/gguf/1c/devstral-small-2-24b-instruct-2512-q4_k_m/mistralai_Devstral-Small-2-24B-Instruct-2512-Q4_K_M.gguf
+LLAMA_SERVED_MODEL_NAME=devstral-1c-q4
+LLAMA_CTX_SIZE=32768
+LLAMA_GPU_LAYERS=999
+LLAMA_THREADS=8
+LLAMA_PARALLEL=1
+LLAMA_FIT=off
+LLAMA_REASONING=off
+LLAMA_REASONING_FORMAT=none
+LLAMA_CHAT_TEMPLATE=mistral-v7-tekken
+HOST_MODELS_DIR=/models
@@ -0,0 +1,43 @@
+services:
+  llama-cpp:
+    image: ${LLAMA_IMAGE:-ghcr.io/ggml-org/llama.cpp:server-cuda}
+    container_name: ${LLAMA_CONTAINER_NAME:-llm-llama-devstral-1c}
+    restart: unless-stopped
+    ports:
+      - "${LLAMA_HOST_PORT:-8080}:8080"
+    volumes:
+      - ${HOST_MODELS_DIR:-/models}:/models
+    command:
+      - --host
+      - 0.0.0.0
+      - --port
+      - "8080"
+      - --model
+      - ${LLAMA_MODEL_PATH}
+      - --alias
+      - ${LLAMA_SERVED_MODEL_NAME:-devstral-1c-q4}
+      - --ctx-size
+      - ${LLAMA_CTX_SIZE:-32768}
+      - --n-gpu-layers
+      - ${LLAMA_GPU_LAYERS:-999}
+      - --threads
+      - ${LLAMA_THREADS:-8}
+      - --parallel
+      - ${LLAMA_PARALLEL:-1}
+      - --fit
+      - ${LLAMA_FIT:-off}
+      - --reasoning
+      - ${LLAMA_REASONING:-off}
+      - --reasoning-format
+      - ${LLAMA_REASONING_FORMAT:-none}
+      - --chat-template
+      - ${LLAMA_CHAT_TEMPLATE:-mistral-v7-tekken}
+      - --skip-chat-parsing
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: all
+              capabilities:
+                - gpu
@@ -0,0 +1,19 @@
+# Qwen3-Coder Q6 GPU llama.cpp settings for docker-gpu.cin.su.
+# This model is too large to keep online together with SDXL/vLLM on a 24 GiB RTX 4090.
+
+LLAMA_CONTAINER_NAME=llm-llama-qwen3-coder-q6-test
+LLAMA_IMAGE=ghcr.io/ggml-org/llama.cpp:server-cuda
+LLAMA_HOST_PORT=8081
+LLAMA_MODEL_PATH=/models/gguf/1c/qwen3-coder-30b-a3b-instruct-q6_k/Qwen3-Coder-30B-A3B-Instruct-Q6_K.gguf
+LLAMA_SERVED_MODEL_NAME=qwen3-coder-1c-q6
+LLAMA_CTX_SIZE=8192
+LLAMA_GPU_LAYERS=auto
+LLAMA_THREADS=12
+LLAMA_PARALLEL=1
+LLAMA_FIT=on
+LLAMA_REASONING=auto
+LLAMA_REASONING_FORMAT=none
+LLAMA_CHAT_TEMPLATE=chatml
+# Optional future LoRA path after converting the trained adapter to GGUF for llama.cpp:
+# LLAMA_LORA_PATH=/models/adapters/1c/qwen3-coder-30b-a3b-1c-lora-v1.gguf
+HOST_MODELS_DIR=Z:/LLM/models