One owner for the card, not two neighbours (V-486)
CW2 is a ROCm process, so it registers on the KFD like any contender. Running it as its own systemd unit made mavgpud yield llama-server to it every few seconds. The gemma-4-12b arm was down for eight minutes on 2026-08-09 and routing had silently fallen back to the resident model. So mavgpud takes an `stt` block and runs the transcriber itself. `foreign` now excludes every child rather than one pid, which is the fix. Yielding is all or nothing, because a job that wants the card wants all of it. Idle unloading stays llama-server's alone: CW2 holds 1.6GB and unloading it would only send the next voice turn to the homesrv floor. Maven still talks to the transcriber directly on 8081. There is no proxy, because with no idle timer there is nothing for one to measure. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013ptwopxyo3Z2kwFckHkLvN
This commit is contained in:
+46
-1
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
@@ -8,6 +9,7 @@ import (
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fakeKFD builds the sysfs shape the workstation actually has: one directory
|
||||
@@ -47,6 +49,24 @@ func TestForeignExcludesOurChild(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The transcriber is a ROCm process on the same card, so it registers on the
|
||||
// KFD exactly like a contender does. Reading it as one is what happened on
|
||||
// 2026-08-09 while CW2 ran under its own systemd unit: mavgpud yielded, waited
|
||||
// five polls, loaded the model, yielded again, and never held it for a whole
|
||||
// minute. Excluding every child is the fix and this is the test of it.
|
||||
func TestForeignExcludesEveryChild(t *testing.T) {
|
||||
p := probe{kfdRoot: fakeKFD(t, map[int]int64{478104: 12791693312, 999: 4096, 1001: 1717986918})}
|
||||
|
||||
ours := p.foreign(999, 1001)
|
||||
if len(ours) != 1 || ours[0].PID != 478104 {
|
||||
t.Fatalf("only the CPT run is a contender, got %+v", ours)
|
||||
}
|
||||
// A child that is not running reports pid 0, which must exclude nothing.
|
||||
if got := p.foreign(999, 0); len(got) != 2 {
|
||||
t.Errorf("a stopped child excludes nobody: got %d contenders, want 2", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
// An empty KFD tree is the state that permits a start, so it must read as empty
|
||||
// rather than as an error the caller has to interpret.
|
||||
func TestForeignEmptyAndMissing(t *testing.T) {
|
||||
@@ -81,7 +101,7 @@ func TestFreeVRAM(t *testing.T) {
|
||||
// rather than hanging or proxying into a closed port. Maven reads this endpoint
|
||||
// on a timer forever, including while the workstation is busy.
|
||||
func TestHealthAndProxyRefuseWhenNotReady(t *testing.T) {
|
||||
s := &supervisor{run: newRunner("/bin/true", nil, "")}
|
||||
s := &supervisor{run: newRunner("fake", "/bin/true", nil, "")}
|
||||
h := s.handler(mustURL(t, "http://127.0.0.1:1"))
|
||||
|
||||
for _, path := range []string{"/health", "/v1/chat/completions"} {
|
||||
@@ -101,3 +121,28 @@ func mustURL(t *testing.T, s string) *url.URL {
|
||||
}
|
||||
return u
|
||||
}
|
||||
|
||||
// Yielding is all or nothing. A CPT run wants the whole card, so handing back
|
||||
// the language model while the transcriber keeps 1.6GB mapped would leave the
|
||||
// other job failing its allocation, which is the outcome yielding exists to
|
||||
// prevent.
|
||||
func TestYieldStopsEveryChild(t *testing.T) {
|
||||
idle := "while : ; do sleep 1 ; done"
|
||||
s := &supervisor{
|
||||
cfg: config{EvictAfter: 1, StopGrace: duration(2 * time.Second)},
|
||||
probe: probe{kfdRoot: fakeKFD(t, map[int]int64{478104: 12791693312})},
|
||||
run: newRunner("llama-server", fakeServer(t, idle), nil, ""),
|
||||
stt: newRunner("cw2", fakeServer(t, idle), nil, ""),
|
||||
}
|
||||
for _, r := range s.children() {
|
||||
if err := r.start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
s.tick(context.Background())
|
||||
for _, r := range s.children() {
|
||||
if r.running() {
|
||||
t.Errorf("%s outlived the yield", r.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user