AUFGABE C: /free-Auto-Thrashing-Schutz, Scheduled Task Barby-VRAMGuard (90s, reboot-persistent)
This commit is contained in:
parent
d23270d2ec
commit
73ddd042d8
95
vram-guard-free-2026-07-24.ps1
Normal file
95
vram-guard-free-2026-07-24.ps1
Normal file
|
|
@ -0,0 +1,95 @@
|
||||||
|
# vram-guard-free.ps1 — #54-Folgeauftrag: /free-Auto-Thrashing-Schutz
|
||||||
|
# Additiv, reversibel. Erkennt das 44-Min-Thrashing-Muster (VRAM > 90% UND
|
||||||
|
# derselbe Queue-Job haengt ueber mehrere Checks ohne Fortschritt) und ruft
|
||||||
|
# ComfyUI POST /free {"unload_models":true,"free_memory":true} -- genau der
|
||||||
|
# von Worker A/B in vram-arbitrierung-und-routingkarte-20260724.md empfohlene
|
||||||
|
# Schritt ("main/Orchestrator-Ebene, nicht in diesem Schritt umgesetzt").
|
||||||
|
# Laeuft als Scheduled Task alle 90s, idempotent, kein Dauerprozess.
|
||||||
|
#
|
||||||
|
# Rueckweg: Scheduled Task "Barby-VRAMGuard" loeschen (Unregister-ScheduledTask),
|
||||||
|
# Skript selbst tut nichts destruktives ausser dem ComfyUI-eigenen /free-Call
|
||||||
|
# (entlaedt Modelle aus VRAM -- kein Datenverlust, ComfyUI laedt bei Bedarf neu).
|
||||||
|
|
||||||
|
$ErrorActionPreference = "Stop"
|
||||||
|
$StateFile = "M:\vram-guard-state.json"
|
||||||
|
$LogFile = "M:\vram-guard.log"
|
||||||
|
$ComfyBase = "http://127.0.0.1:8188"
|
||||||
|
$VramThresholdPct = 90
|
||||||
|
$StallChecksNeeded = 3 # 3x hintereinander (~4.5 Min bei 90s-Trigger) derselbe Job + VRAM>90% -> Thrashing-Verdacht
|
||||||
|
|
||||||
|
function Write-Log($msg) {
|
||||||
|
$line = "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] $msg"
|
||||||
|
Add-Content -Path $LogFile -Value $line -Encoding utf8
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
$nvOut = & nvidia-smi --query-gpu=memory.used,memory.total,utilization.gpu --format=csv,noheader,nounits 2>$null
|
||||||
|
if (-not $nvOut) { Write-Log "nvidia-smi kein Output -- Skip"; exit 0 }
|
||||||
|
$parts = $nvOut.Trim() -split ',\s*'
|
||||||
|
$used = [double]$parts[0]
|
||||||
|
$total = [double]$parts[1]
|
||||||
|
$util = [double]$parts[2]
|
||||||
|
$pct = [math]::Round(($used / $total) * 100, 1)
|
||||||
|
} catch {
|
||||||
|
Write-Log "nvidia-smi Fehler: $_ -- Skip"
|
||||||
|
exit 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# ComfyUI-Queue abfragen (laufender Job = Fingerprint fuer Stall-Erkennung)
|
||||||
|
$queueId = $null
|
||||||
|
try {
|
||||||
|
$q = Invoke-RestMethod -Uri "$ComfyBase/queue" -TimeoutSec 5
|
||||||
|
if ($q.queue_running -and $q.queue_running.Count -gt 0) {
|
||||||
|
$queueId = $q.queue_running[0][1] # prompt_id des laufenden Jobs
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
Write-Log "ComfyUI /queue nicht erreichbar ($_) -- Skip (kein Fehlalarm ohne Server)"
|
||||||
|
exit 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# State laden/initialisieren
|
||||||
|
$state = @{ last_queue_id = $null; stall_count = 0; last_check = "" }
|
||||||
|
if (Test-Path $StateFile) {
|
||||||
|
try { $state = Get-Content $StateFile -Raw | ConvertFrom-Json } catch {}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (-not $queueId) {
|
||||||
|
# Keine laufende Job -- kein Thrashing-Kontext, State zuruecksetzen
|
||||||
|
if ($state.stall_count -ne 0) { Write-Log "Queue leer -- Stall-Counter zurueckgesetzt (VRAM ${pct}%)" }
|
||||||
|
$state = @{ last_queue_id = $null; stall_count = 0; last_check = (Get-Date -Format 'o') }
|
||||||
|
$state | ConvertTo-Json | Set-Content $StateFile -Encoding utf8
|
||||||
|
exit 0
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($pct -lt $VramThresholdPct) {
|
||||||
|
# VRAM unauffaellig -- normaler Betrieb, State fuer diesen Job zuruecksetzen
|
||||||
|
$state = @{ last_queue_id = $queueId; stall_count = 0; last_check = (Get-Date -Format 'o') }
|
||||||
|
$state | ConvertTo-Json | Set-Content $StateFile -Encoding utf8
|
||||||
|
exit 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# VRAM > Schwelle: pruefen ob es DERSELBE Job wie beim letzten Check ist
|
||||||
|
if ($state.last_queue_id -eq $queueId) {
|
||||||
|
$state.stall_count = [int]$state.stall_count + 1
|
||||||
|
} else {
|
||||||
|
$state.stall_count = 1
|
||||||
|
}
|
||||||
|
$state.last_queue_id = $queueId
|
||||||
|
$state.last_check = (Get-Date -Format 'o')
|
||||||
|
|
||||||
|
Write-Log "VRAM ${pct}% (>=${VramThresholdPct}%), GPU-Util ${util}%, Job $queueId, Stall-Count $($state.stall_count)/$StallChecksNeeded"
|
||||||
|
|
||||||
|
if ($state.stall_count -ge $StallChecksNeeded) {
|
||||||
|
Write-Log "THRASHING-VERDACHT: VRAM>=${VramThresholdPct}% + derselbe Job seit $StallChecksNeeded Checks -> rufe POST /free auf"
|
||||||
|
try {
|
||||||
|
$body = @{ unload_models = $true; free_memory = $true } | ConvertTo-Json
|
||||||
|
$resp = Invoke-RestMethod -Uri "$ComfyBase/free" -Method Post -Body $body -ContentType "application/json" -TimeoutSec 15
|
||||||
|
Write-Log "/free aufgerufen -- Antwort: $($resp | ConvertTo-Json -Compress)"
|
||||||
|
} catch {
|
||||||
|
Write-Log "/free-Aufruf fehlgeschlagen: $_"
|
||||||
|
}
|
||||||
|
# Counter zuruecksetzen nach Eingriff, um nicht im Sekundentakt erneut zu triggern
|
||||||
|
$state.stall_count = 0
|
||||||
|
}
|
||||||
|
|
||||||
|
$state | ConvertTo-Json | Set-Content $StateFile -Encoding utf8
|
||||||
Loading…
Reference in a new issue