services:
  llama-cpp:
    image: ${LLAMA_IMAGE:-ghcr.io/ggml-org/llama.cpp:server-cuda}
    container_name: ${LLAMA_CONTAINER_NAME:-llm-llama-devstral-1c}
    restart: unless-stopped
    ports:
      - "${LLAMA_HOST_PORT:-8080}:8080"
    volumes:
      - ${HOST_MODELS_DIR:-/models}:/models
    command:
      - --host
      - 0.0.0.0
      - --port
      - "8080"
      - --model
      - ${LLAMA_MODEL_PATH}
      - --alias
      - ${LLAMA_SERVED_MODEL_NAME:-devstral-1c-q4}
      - --ctx-size
      - ${LLAMA_CTX_SIZE:-32768}
      - --n-gpu-layers
      - ${LLAMA_GPU_LAYERS:-999}
      - --threads
      - ${LLAMA_THREADS:-8}
      - --parallel
      - ${LLAMA_PARALLEL:-1}
      - --fit
      - ${LLAMA_FIT:-off}
      - --reasoning
      - ${LLAMA_REASONING:-off}
      - --reasoning-format
      - ${LLAMA_REASONING_FORMAT:-none}
      - --chat-template
      - ${LLAMA_CHAT_TEMPLATE:-mistral-v7-tekken}
      - --skip-chat-parsing
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities:
                - gpu