72 lines
2.1 KiB
PowerShell
72 lines
2.1 KiB
PowerShell
param(
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidateSet("start", "stop", "restart", "status", "logs")]
|
|
[string]$Action,
|
|
|
|
[string]$DockerHost = "ssh://test-docker",
|
|
[string]$ContainerName = "llm-qwen3-coder-q6-cpu-test",
|
|
[string]$HostPort = "18086",
|
|
[string]$ModelDir = "/home/test/llm/models/qwen3-coder-30b-a3b-instruct-q6_k",
|
|
[string]$ModelFile = "Qwen3-Coder-30B-A3B-Instruct-Q6_K.gguf",
|
|
[string]$ServedModelName = "qwen3-coder-1c-q6-cpu",
|
|
[int]$Threads = 64,
|
|
[int]$ContextSize = 8192,
|
|
[int]$BatchSize = 512,
|
|
[int]$Tail = 80
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
function Start-CpuModel {
|
|
$existing = docker -H $DockerHost ps -a --filter "name=^$ContainerName$" --format "{{.Names}}"
|
|
if ($existing -contains $ContainerName) {
|
|
docker -H $DockerHost start $ContainerName
|
|
return
|
|
}
|
|
|
|
$args = @(
|
|
"-H", $DockerHost, "run", "-d",
|
|
"--name", $ContainerName,
|
|
"--restart", "unless-stopped",
|
|
"-p", "${HostPort}:8080",
|
|
"-v", "${ModelDir}:/models:ro",
|
|
"ghcr.io/ggml-org/llama.cpp:server",
|
|
"--host", "0.0.0.0",
|
|
"--port", "8080",
|
|
"--model", "/models/$ModelFile",
|
|
"--alias", $ServedModelName,
|
|
"--ctx-size", "$ContextSize",
|
|
"--threads", "$Threads",
|
|
"--parallel", "1",
|
|
"--batch-size", "$BatchSize"
|
|
)
|
|
docker @args
|
|
}
|
|
|
|
switch ($Action) {
|
|
"start" {
|
|
Start-CpuModel
|
|
}
|
|
"stop" {
|
|
docker -H $DockerHost stop $ContainerName
|
|
}
|
|
"restart" {
|
|
$existing = docker -H $DockerHost ps -a --filter "name=^$ContainerName$" --format "{{.Names}}"
|
|
if ($existing -contains $ContainerName) {
|
|
docker -H $DockerHost restart $ContainerName
|
|
} else {
|
|
Start-CpuModel
|
|
}
|
|
}
|
|
"status" {
|
|
docker -H $DockerHost ps -a --filter "name=^$ContainerName$"
|
|
Write-Host ""
|
|
docker -H $DockerHost stats --no-stream --format "table {{.Name}}`t{{.CPUPerc}}`t{{.MemUsage}}" $ContainerName 2>$null
|
|
}
|
|
"logs" {
|
|
docker -H $DockerHost logs --tail $Tail $ContainerName
|
|
}
|
|
}
|
|
|
|
exit $LASTEXITCODE
|