122 lines
3.3 KiB
PowerShell
122 lines
3.3 KiB
PowerShell
param(
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$Profile,
|
|
|
|
[string]$DockerHost = "ssh://docker-gpu",
|
|
[string]$SshTarget = "docker-gpu",
|
|
[int]$WaitSeconds = 180,
|
|
[switch]$NoWait,
|
|
[switch]$PlanOnly
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
$root = Split-Path -Parent (Split-Path -Parent $MyInvocation.MyCommand.Path)
|
|
$manager = Join-Path $root "scripts\manage_model_service.ps1"
|
|
$profilesPath = Join-Path $root "config\gpu_profiles.json"
|
|
|
|
if (-not (Test-Path -LiteralPath $manager)) {
|
|
throw "Service manager not found: $manager"
|
|
}
|
|
|
|
if (-not (Test-Path -LiteralPath $profilesPath)) {
|
|
throw "GPU profiles config not found: $profilesPath"
|
|
}
|
|
|
|
$profiles = Get-Content -LiteralPath $profilesPath -Raw | ConvertFrom-Json
|
|
$profileProperty = $profiles.PSObject.Properties[$Profile]
|
|
if (-not $profileProperty) {
|
|
$available = ($profiles.PSObject.Properties.Name | Sort-Object) -join ", "
|
|
throw "Unknown GPU profile '$Profile'. Available profiles: $available"
|
|
}
|
|
|
|
function Invoke-ManagedService {
|
|
param(
|
|
[Parameter(Mandatory = $true)][string]$Service,
|
|
[Parameter(Mandatory = $true)][ValidateSet("start", "stop", "status")][string]$Action
|
|
)
|
|
|
|
$args = @(
|
|
"-ExecutionPolicy", "Bypass",
|
|
"-File", $manager,
|
|
"-Service", $Service,
|
|
"-Action", $Action,
|
|
"-DockerHost", $DockerHost,
|
|
"-SshTarget", $SshTarget
|
|
)
|
|
|
|
if ($PlanOnly) {
|
|
Write-Host "PLAN powershell $($args -join ' ')"
|
|
return
|
|
}
|
|
|
|
Write-Host "[$Action] $Service"
|
|
& powershell @args
|
|
if ($LASTEXITCODE -ne 0) {
|
|
if ($Action -eq "stop") {
|
|
Write-Warning "Stop failed for $Service; continuing because the container may already be stopped or absent."
|
|
return
|
|
}
|
|
throw "Service action failed: $Action $Service"
|
|
}
|
|
}
|
|
|
|
function Wait-HttpEndpoint {
|
|
param(
|
|
[Parameter(Mandatory = $true)][string]$Name,
|
|
[Parameter(Mandatory = $true)][string]$Url
|
|
)
|
|
|
|
if ($NoWait -or $PlanOnly) {
|
|
return
|
|
}
|
|
|
|
$deadline = (Get-Date).AddSeconds($WaitSeconds)
|
|
Write-Host "[wait] $Name $Url"
|
|
while ((Get-Date) -lt $deadline) {
|
|
try {
|
|
Invoke-WebRequest -Uri $Url -UseBasicParsing -TimeoutSec 5 | Out-Null
|
|
Write-Host "[ok] $Name"
|
|
return
|
|
} catch {
|
|
Start-Sleep -Seconds 5
|
|
}
|
|
}
|
|
Write-Warning "Endpoint did not become ready in $WaitSeconds seconds: $Name $Url"
|
|
}
|
|
|
|
function Show-GpuStatus {
|
|
if ($PlanOnly) {
|
|
return
|
|
}
|
|
Write-Host ""
|
|
Write-Host "Containers:"
|
|
docker --host $DockerHost ps --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"
|
|
Write-Host ""
|
|
Write-Host "GPU:"
|
|
ssh $SshTarget "nvidia-smi --query-gpu=index,name,memory.used,memory.total,utilization.gpu --format=csv,noheader,nounits"
|
|
}
|
|
|
|
$plan = $profileProperty.Value
|
|
$stops = @($plan.stops)
|
|
$starts = @($plan.starts)
|
|
$waitEndpoints = @($plan.wait)
|
|
|
|
Write-Host "GPU profile: $Profile"
|
|
Write-Host "Stop : $($stops -join ', ')"
|
|
Write-Host "Start: $($starts -join ', ')"
|
|
|
|
foreach ($service in $stops) {
|
|
Invoke-ManagedService -Service $service -Action stop
|
|
}
|
|
|
|
foreach ($service in $starts) {
|
|
Invoke-ManagedService -Service $service -Action start
|
|
}
|
|
|
|
foreach ($endpoint in $waitEndpoints) {
|
|
Wait-HttpEndpoint -Name $endpoint.name -Url $endpoint.url
|
|
}
|
|
|
|
Show-GpuStatus
|