param( [Parameter(Mandatory = $true)] [string]$Profile, [string]$DockerHost = "ssh://docker-gpu", [string]$SshTarget = "docker-gpu", [int]$WaitSeconds = 180, [switch]$NoWait, [switch]$PlanOnly ) $ErrorActionPreference = "Stop" $root = Split-Path -Parent (Split-Path -Parent $MyInvocation.MyCommand.Path) $manager = Join-Path $root "scripts\manage_model_service.ps1" $profilesPath = Join-Path $root "config\gpu_profiles.json" if (-not (Test-Path -LiteralPath $manager)) { throw "Service manager not found: $manager" } if (-not (Test-Path -LiteralPath $profilesPath)) { throw "GPU profiles config not found: $profilesPath" } $profiles = Get-Content -LiteralPath $profilesPath -Raw | ConvertFrom-Json $profileProperty = $profiles.PSObject.Properties[$Profile] if (-not $profileProperty) { $available = ($profiles.PSObject.Properties.Name | Sort-Object) -join ", " throw "Unknown GPU profile '$Profile'. Available profiles: $available" } function Invoke-ManagedService { param( [Parameter(Mandatory = $true)][string]$Service, [Parameter(Mandatory = $true)][ValidateSet("start", "stop", "status")][string]$Action ) $args = @( "-ExecutionPolicy", "Bypass", "-File", $manager, "-Service", $Service, "-Action", $Action, "-DockerHost", $DockerHost, "-SshTarget", $SshTarget ) if ($PlanOnly) { Write-Host "PLAN powershell $($args -join ' ')" return } Write-Host "[$Action] $Service" & powershell @args if ($LASTEXITCODE -ne 0) { if ($Action -eq "stop") { Write-Warning "Stop failed for $Service; continuing because the container may already be stopped or absent." return } throw "Service action failed: $Action $Service" } } function Wait-HttpEndpoint { param( [Parameter(Mandatory = $true)][string]$Name, [Parameter(Mandatory = $true)][string]$Url ) if ($NoWait -or $PlanOnly) { return } $deadline = (Get-Date).AddSeconds($WaitSeconds) Write-Host "[wait] $Name $Url" while ((Get-Date) -lt $deadline) { try { Invoke-WebRequest -Uri $Url -UseBasicParsing -TimeoutSec 5 | Out-Null Write-Host "[ok] $Name" return } catch { Start-Sleep -Seconds 5 } } Write-Warning "Endpoint did not become ready in $WaitSeconds seconds: $Name $Url" } function Show-GpuStatus { if ($PlanOnly) { return } Write-Host "" Write-Host "Containers:" docker --host $DockerHost ps --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" Write-Host "" Write-Host "GPU:" ssh $SshTarget "nvidia-smi --query-gpu=index,name,memory.used,memory.total,utilization.gpu --format=csv,noheader,nounits" } $plan = $profileProperty.Value $stops = @($plan.stops) $starts = @($plan.starts) $waitEndpoints = @($plan.wait) Write-Host "GPU profile: $Profile" Write-Host "Stop : $($stops -join ', ')" Write-Host "Start: $($starts -join ', ')" foreach ($service in $stops) { Invoke-ManagedService -Service $service -Action stop } foreach ($service in $starts) { Invoke-ManagedService -Service $service -Action start } foreach ($endpoint in $waitEndpoints) { Wait-HttpEndpoint -Name $endpoint.name -Url $endpoint.url } Show-GpuStatus