Compare commits
15 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| e52c616592 | |||
| 2b97bac51e | |||
| ab42db2b87 | |||
| 94d553570d | |||
| 2e97b905b4 | |||
| fbcca449be | |||
| 2076e4a788 | |||
| 30eb6add1b | |||
| dc266056d1 | |||
| 1c786b7156 | |||
| a3af10a830 | |||
| c0de473382 | |||
| 3e534340bf | |||
| 1a704d704d | |||
| e57adcb001 |
@@ -66,3 +66,6 @@ coverage.out
|
||||
/HANDOFF.md
|
||||
/models/stt
|
||||
/models/tts
|
||||
|
||||
# root .env — MAVEN_AMBIENT_TOKEN and friends, same class as deploy/telegram.env
|
||||
.env
|
||||
|
||||
@@ -28,6 +28,14 @@ model is a one-line change to `phraser.model_path` in `deploy/mavend.json`.
|
||||
See `docs/rearchitecture.md` for the target architecture, `docs/design.md` for the folded design spec, and
|
||||
`AGENTS.md` for local-preview + model-download recipes.
|
||||
|
||||
**Model work is moving to the workstation** (owner's call, 2026-08-02). homesrv cannot grow a
|
||||
GPU and the workstation has 16GB of VRAM. So the resident model, STT and TTS become preferred
|
||||
remotes with a floor on homesrv. The workstation is never assumed up. Fall back silently when
|
||||
it would only do the job better. Name the gap when the 1.7B cannot do it at all. The embedder
|
||||
stays on homesrv permanently, because it backs that floor. Read `docs/offload.md` before
|
||||
touching a daemon seam or adding a model caller. Vikunja #483 is the umbrella, #484 to #487
|
||||
are the work.
|
||||
|
||||
## Build & test
|
||||
|
||||
CGO daemons (`mavend`, `mavsttd`, `mavttsd`, `mavenclient`) need the vendored toolchain
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The card is an AMD 7900 GRE with 16GB, driven by amdgpu and ROCm. Everything
|
||||
// here reads sysfs and forks nothing: rocm-smi is not even installed on the
|
||||
// workstation, and a poll that costs a subprocess every second is a poll that
|
||||
// gets tuned down until it is useless.
|
||||
|
||||
// gpuProc — one process holding the compute engine.
|
||||
type gpuProc struct {
|
||||
PID int
|
||||
Comm string
|
||||
VRAM int64 // bytes, as the kernel accounts them to this process
|
||||
}
|
||||
|
||||
// probe reads the two sysfs trees the supervisor decides from.
|
||||
//
|
||||
// kfdRoot is /sys/class/kfd/kfd/proc, one directory per ROCm process. The
|
||||
// directory appears when the process initialises HIP, which is well before it
|
||||
// allocates anything large. That is the whole reason this works: the job that
|
||||
// is about to want the card announces itself while it is still starting up,
|
||||
// so we see the contender rather than only the winner of an allocation race.
|
||||
//
|
||||
// drmDev is /sys/class/drm/cardN/device, which reports total and used VRAM for
|
||||
// the card as a whole.
|
||||
type probe struct {
|
||||
kfdRoot string
|
||||
drmDev string
|
||||
}
|
||||
|
||||
// foreign lists every ROCm process that is not ours. selfPID is the supervisor's
|
||||
// llama-server child, or 0 when it is not running.
|
||||
//
|
||||
// An unreadable kfd tree returns no processes and no error. That is deliberate
|
||||
// and it is the safe direction only because startVRAM also has to agree before
|
||||
// anything launches: a supervisor that cannot see the KFD never sees free VRAM
|
||||
// either, because the CPT run holding the card shows up in the drm totals.
|
||||
func (p probe) foreign(selfPID int) []gpuProc {
|
||||
entries, err := os.ReadDir(p.kfdRoot)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var out []gpuProc
|
||||
for _, e := range entries {
|
||||
pid, err := strconv.Atoi(e.Name())
|
||||
if err != nil || pid == selfPID {
|
||||
continue
|
||||
}
|
||||
out = append(out, gpuProc{
|
||||
PID: pid,
|
||||
Comm: readComm(pid),
|
||||
VRAM: p.procVRAM(e.Name()),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// procVRAM sums the per-node vram_* files under one process directory. The
|
||||
// suffix is the KFD topology node id (vram_35881 on this card), so it is
|
||||
// globbed rather than named, and a machine with two cards sums both.
|
||||
func (p probe) procVRAM(pid string) int64 {
|
||||
matches, err := filepath.Glob(filepath.Join(p.kfdRoot, pid, "vram_*"))
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
var total int64
|
||||
for _, m := range matches {
|
||||
total += readInt(m)
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// freeVRAM reports the bytes the card has left. Used only to decide whether to
|
||||
// start: a shortfall here means llama-server would refuse to load anyway. It is
|
||||
// never used to decide to stop, because by the time free VRAM has dropped the
|
||||
// other job has already failed its allocation, which is exactly the outcome
|
||||
// yielding exists to prevent.
|
||||
func (p probe) freeVRAM() int64 {
|
||||
total := readInt(filepath.Join(p.drmDev, "mem_info_vram_total"))
|
||||
used := readInt(filepath.Join(p.drmDev, "mem_info_vram_used"))
|
||||
if total <= 0 {
|
||||
return 0
|
||||
}
|
||||
if free := total - used; free > 0 {
|
||||
return free
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func readInt(path string) int64 {
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
n, err := strconv.ParseInt(strings.TrimSpace(string(b)), 10, 64)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// readComm names the contender for the log. The log is the instrument for the
|
||||
// open question in Vikunja #488: whether a process can want this card without
|
||||
// ever registering on the KFD, which a Vulkan or video-decode job would.
|
||||
func readComm(pid int) string {
|
||||
b, err := os.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "comm"))
|
||||
if err != nil {
|
||||
return "?"
|
||||
}
|
||||
return strings.TrimSpace(string(b))
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// fakeKFD builds the sysfs shape the workstation actually has: one directory
|
||||
// per ROCm process, each holding a vram_<node> file. Sampled from the live box
|
||||
// on 02-08-2026, where the CPT run appeared as proc/478104/vram_35881.
|
||||
func fakeKFD(t *testing.T, vramByPID map[int]int64) string {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
for pid, vram := range vramByPID {
|
||||
dir := filepath.Join(root, strconv.Itoa(pid))
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := filepath.Join(dir, "vram_35881")
|
||||
if err := os.WriteFile(f, []byte(strconv.FormatInt(vram, 10)+"\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
||||
func TestForeignExcludesOurChild(t *testing.T) {
|
||||
root := fakeKFD(t, map[int]int64{478104: 12791693312, 999: 4096})
|
||||
p := probe{kfdRoot: root}
|
||||
|
||||
all := p.foreign(0)
|
||||
if len(all) != 2 {
|
||||
t.Fatalf("with no child running, both processes are foreign, got %d", len(all))
|
||||
}
|
||||
|
||||
ours := p.foreign(999)
|
||||
if len(ours) != 1 || ours[0].PID != 478104 {
|
||||
t.Fatalf("our own llama-server must not count as a contender, got %+v", ours)
|
||||
}
|
||||
if ours[0].VRAM != 12791693312 {
|
||||
t.Errorf("per-process VRAM = %d, want the value from vram_35881", ours[0].VRAM)
|
||||
}
|
||||
}
|
||||
|
||||
// An empty KFD tree is the state that permits a start, so it must read as empty
|
||||
// rather than as an error the caller has to interpret.
|
||||
func TestForeignEmptyAndMissing(t *testing.T) {
|
||||
if got := (probe{kfdRoot: t.TempDir()}).foreign(0); len(got) != 0 {
|
||||
t.Errorf("empty kfd tree: got %d processes, want 0", len(got))
|
||||
}
|
||||
if got := (probe{kfdRoot: "/nonexistent"}).foreign(0); got != nil {
|
||||
t.Errorf("missing kfd tree: got %+v, want nil", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFreeVRAM(t *testing.T) {
|
||||
dev := t.TempDir()
|
||||
write := func(name, v string) {
|
||||
if err := os.WriteFile(filepath.Join(dev, name), []byte(v), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// The live numbers from the workstation while the CPT run held the card.
|
||||
write("mem_info_vram_total", "17163091968\n")
|
||||
write("mem_info_vram_used", "13396389888\n")
|
||||
p := probe{drmDev: dev}
|
||||
if got, want := p.freeVRAM(), int64(3766702080); got != want {
|
||||
t.Errorf("freeVRAM = %d, want %d", got, want)
|
||||
}
|
||||
if got := (probe{drmDev: "/nonexistent"}).freeVRAM(); got != 0 {
|
||||
t.Errorf("unreadable card reports %d free, want 0 so nothing starts", got)
|
||||
}
|
||||
}
|
||||
|
||||
// With no model loaded the supervisor must still answer, and it must answer 503
|
||||
// rather than hanging or proxying into a closed port. Maven reads this endpoint
|
||||
// on a timer forever, including while the workstation is busy.
|
||||
func TestHealthAndProxyRefuseWhenNotReady(t *testing.T) {
|
||||
s := &supervisor{run: newRunner("/bin/true", nil, "")}
|
||||
h := s.handler(mustURL(t, "http://127.0.0.1:1"))
|
||||
|
||||
for _, path := range []string{"/health", "/v1/chat/completions"} {
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil))
|
||||
if w.Code != http.StatusServiceUnavailable {
|
||||
t.Errorf("%s with no model: got %d, want 503", path, w.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func mustURL(t *testing.T, s string) *url.URL {
|
||||
t.Helper()
|
||||
u, err := url.Parse(s)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return u
|
||||
}
|
||||
@@ -0,0 +1,247 @@
|
||||
// mavgpud — the workstation's GPU supervisor.
|
||||
//
|
||||
// It runs on the workstation (an AMD 7900 GRE, 16GB), not on homesrv, and it is
|
||||
// deployed separately from the Maven daemons. Maven does not participate in any
|
||||
// of this and never asks for a start: it reads /health through internal/llm.Pair
|
||||
// and either gets the big model or falls back to the resident 1.7B.
|
||||
//
|
||||
// The rule, from Vikunja #488: keep llama-server loaded whenever the card is
|
||||
// free, unload it when it has been idle too long or when another process needs
|
||||
// the card. Not on demand, because a 7-14B takes tens of seconds to load and a
|
||||
// world question would be answered by a gap every time the card had been quiet.
|
||||
// Not always on, because that holds 16GB against the owner's own jobs.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/http/httputil"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
type config struct {
|
||||
Listen string `json:"listen"` // what Maven talks to
|
||||
LlamaAddr string `json:"llama_addr"` // where llama-server binds
|
||||
LlamaBin string `json:"llama_bin"`
|
||||
// LlamaArgs must include the flags that bind LlamaAddr. They are passed
|
||||
// through untouched so the model, context size and layer count stay the
|
||||
// owner's business and not this daemon's schema.
|
||||
LlamaArgs []string `json:"llama_args"`
|
||||
|
||||
KFDRoot string `json:"kfd_root"`
|
||||
DRMDevice string `json:"drm_device"`
|
||||
|
||||
Poll duration `json:"poll"`
|
||||
IdleTimeout duration `json:"idle_timeout"`
|
||||
StopGrace duration `json:"stop_grace"`
|
||||
MinFreeVRAM int64 `json:"min_free_vram_bytes"`
|
||||
// EvictAfter and StartAfter are counted in polls, not seconds. Both exist
|
||||
// to damp flapping: a one-tick blip from a short-lived rocm process must
|
||||
// not evict the model, and a card that has just been released must not be
|
||||
// grabbed before the previous job has finished unmapping.
|
||||
EvictAfter int `json:"evict_after_polls"`
|
||||
StartAfter int `json:"start_after_polls"`
|
||||
}
|
||||
|
||||
func defaults() config {
|
||||
return config{
|
||||
Listen: ":8080",
|
||||
LlamaAddr: "127.0.0.1:8081",
|
||||
KFDRoot: "/sys/class/kfd/kfd/proc",
|
||||
DRMDevice: "/sys/class/drm/card1/device",
|
||||
Poll: duration(time.Second),
|
||||
IdleTimeout: duration(15 * time.Minute),
|
||||
StopGrace: duration(20 * time.Second),
|
||||
MinFreeVRAM: 15 << 30,
|
||||
EvictAfter: 2,
|
||||
StartAfter: 5,
|
||||
}
|
||||
}
|
||||
|
||||
// duration lets the config file say "15m" instead of counting nanoseconds.
|
||||
type duration time.Duration
|
||||
|
||||
func (d *duration) UnmarshalJSON(b []byte) error {
|
||||
var s string
|
||||
if err := json.Unmarshal(b, &s); err != nil {
|
||||
return err
|
||||
}
|
||||
v, err := time.ParseDuration(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*d = duration(v)
|
||||
return nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
path := flag.String("config", "/etc/mavgpud.json", "config file")
|
||||
flag.Parse()
|
||||
|
||||
cfg := defaults()
|
||||
b, err := os.ReadFile(*path)
|
||||
if err != nil {
|
||||
log.Fatalf("mavgpud: read config: %v", err)
|
||||
}
|
||||
if err := json.Unmarshal(b, &cfg); err != nil {
|
||||
log.Fatalf("mavgpud: parse config: %v", err)
|
||||
}
|
||||
if cfg.LlamaBin == "" {
|
||||
log.Fatal("mavgpud: llama_bin is required")
|
||||
}
|
||||
|
||||
base := "http://" + cfg.LlamaAddr
|
||||
run := newRunner(cfg.LlamaBin, cfg.LlamaArgs, base+"/health")
|
||||
sup := &supervisor{
|
||||
cfg: cfg,
|
||||
probe: probe{kfdRoot: cfg.KFDRoot, drmDev: cfg.DRMDevice},
|
||||
run: run,
|
||||
}
|
||||
sup.touch()
|
||||
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer cancel()
|
||||
|
||||
target, err := url.Parse(base)
|
||||
if err != nil {
|
||||
log.Fatalf("mavgpud: llama_addr: %v", err)
|
||||
}
|
||||
srv := &http.Server{Addr: cfg.Listen, Handler: sup.handler(target)}
|
||||
go func() {
|
||||
log.Printf("mavgpud: listening on %s, model %s", cfg.Listen, cfg.LlamaBin)
|
||||
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
log.Fatalf("mavgpud: listen: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
sup.loop(ctx)
|
||||
|
||||
// The card must come back before we do. A supervisor that exits leaving
|
||||
// llama-server holding 14GB is worse than one that never ran.
|
||||
shut, done := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer done()
|
||||
_ = srv.Shutdown(shut)
|
||||
run.stop(time.Duration(cfg.StopGrace))
|
||||
}
|
||||
|
||||
type supervisor struct {
|
||||
cfg config
|
||||
probe probe
|
||||
run *runner
|
||||
|
||||
lastReq atomic.Int64 // unix nanos of the last request Maven sent
|
||||
|
||||
foreignStreak int
|
||||
clearStreak int
|
||||
}
|
||||
|
||||
func (s *supervisor) touch() { s.lastReq.Store(time.Now().UnixNano()) }
|
||||
|
||||
func (s *supervisor) idle() time.Duration {
|
||||
return time.Since(time.Unix(0, s.lastReq.Load()))
|
||||
}
|
||||
|
||||
// handler serves the two things the workstation exposes.
|
||||
//
|
||||
// /health is answered locally and always, with no GPU cost and no round trip,
|
||||
// because it is the only thing Maven reads and Maven reads it on a timer
|
||||
// forever. Everything else is llama-server's API, reverse-proxied. Proxying
|
||||
// rather than pointing Maven straight at llama-server is what makes the idle
|
||||
// window measurable: the supervisor cannot otherwise know when the model was
|
||||
// last used.
|
||||
func (s *supervisor) handler(target *url.URL) http.Handler {
|
||||
proxy := httputil.NewSingleHostReverseProxy(target)
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.run.isReady() {
|
||||
http.Error(w, "model not loaded", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"status":"ok"}`))
|
||||
})
|
||||
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.run.isReady() {
|
||||
http.Error(w, "model not loaded", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
s.touch()
|
||||
proxy.ServeHTTP(w, r)
|
||||
})
|
||||
return mux
|
||||
}
|
||||
|
||||
func (s *supervisor) loop(ctx context.Context) {
|
||||
t := time.NewTicker(time.Duration(s.cfg.Poll))
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
s.tick(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// tick is the whole decision. Yielding is checked before starting, and presence
|
||||
// on the KFD is what triggers it — not a VRAM threshold. A ROCm process
|
||||
// registers under /sys/class/kfd/kfd/proc when it initialises HIP, before it
|
||||
// allocates, so we see a contender during its startup rather than after it has
|
||||
// already failed to get the memory it wanted.
|
||||
func (s *supervisor) tick(ctx context.Context) {
|
||||
others := s.probe.foreign(s.run.pid())
|
||||
if len(others) > 0 {
|
||||
s.foreignStreak++
|
||||
s.clearStreak = 0
|
||||
} else {
|
||||
s.foreignStreak = 0
|
||||
s.clearStreak++
|
||||
}
|
||||
|
||||
if s.run.running() {
|
||||
s.run.refreshReady(ctx)
|
||||
switch {
|
||||
case s.foreignStreak >= s.cfg.EvictAfter:
|
||||
log.Printf("mavgpud: yielding the card to %s", describe(others))
|
||||
s.run.stop(time.Duration(s.cfg.StopGrace))
|
||||
case s.idle() > time.Duration(s.cfg.IdleTimeout):
|
||||
log.Printf("mavgpud: idle for %s, unloading", s.idle().Round(time.Second))
|
||||
s.run.stop(time.Duration(s.cfg.StopGrace))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if s.clearStreak < s.cfg.StartAfter {
|
||||
return
|
||||
}
|
||||
if free := s.probe.freeVRAM(); free < s.cfg.MinFreeVRAM {
|
||||
return
|
||||
}
|
||||
s.touch() // the idle clock starts at load, not at the last request before it
|
||||
if err := s.run.start(); err != nil {
|
||||
log.Printf("mavgpud: start llama-server: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// describe names the contenders in the log. This log is the instrument for the
|
||||
// open question in #488: whether polling the KFD misses a job that wants the
|
||||
// card without registering there.
|
||||
func describe(procs []gpuProc) string {
|
||||
out := ""
|
||||
for i, p := range procs {
|
||||
if i > 0 {
|
||||
out += ", "
|
||||
}
|
||||
out += p.Comm
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"net/http"
|
||||
"os/exec"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// runner owns one llama-server process. Owning it is the point of the daemon:
|
||||
// the workstation cannot keep a 7-14B resident, because that holds 16GB against
|
||||
// the owner's CPT runs, Correx and the manga-recap pipeline. So the thing that
|
||||
// stays up is this, which costs no VRAM, and the model comes and goes under it.
|
||||
type runner struct {
|
||||
bin string
|
||||
args []string
|
||||
// ready is llama-server's own /health, which answers "is a model loaded".
|
||||
// Loading a 7-14B takes tens of seconds, so started is not ready.
|
||||
readyURL string
|
||||
|
||||
mu sync.Mutex
|
||||
cmd *exec.Cmd
|
||||
ready bool
|
||||
http *http.Client
|
||||
}
|
||||
|
||||
func newRunner(bin string, args []string, readyURL string) *runner {
|
||||
return &runner{
|
||||
bin: bin, args: args, readyURL: readyURL,
|
||||
http: &http.Client{Timeout: 2 * time.Second},
|
||||
}
|
||||
}
|
||||
|
||||
// pid is the child's, or 0. The GPU probe needs it to tell our own model apart
|
||||
// from a contender.
|
||||
func (r *runner) pid() int {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.cmd == nil || r.cmd.Process == nil {
|
||||
return 0
|
||||
}
|
||||
return r.cmd.Process.Pid
|
||||
}
|
||||
|
||||
func (r *runner) running() bool { return r.pid() != 0 }
|
||||
|
||||
// isReady reports the cached readiness. The supervisor loop refreshes it; the
|
||||
// health handler only reads, so answering /health never costs a round trip.
|
||||
func (r *runner) isReady() bool {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return r.ready
|
||||
}
|
||||
|
||||
// start launches llama-server. It returns as soon as the process exists, not
|
||||
// when the model is loaded.
|
||||
func (r *runner) start() error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.cmd != nil {
|
||||
return nil
|
||||
}
|
||||
cmd := exec.Command(r.bin, r.args...)
|
||||
// Own process group, so stop kills anything llama-server spawned rather
|
||||
// than leaving it holding VRAM after we have declared the card yielded.
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
if err := cmd.Start(); err != nil {
|
||||
return err
|
||||
}
|
||||
r.cmd, r.ready = cmd, false
|
||||
log.Printf("mavgpud: started llama-server pid=%d", cmd.Process.Pid)
|
||||
go func() {
|
||||
err := cmd.Wait()
|
||||
r.mu.Lock()
|
||||
r.cmd, r.ready = nil, false
|
||||
r.mu.Unlock()
|
||||
log.Printf("mavgpud: llama-server exited: %v", err)
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// stop ends llama-server and waits for the VRAM to come back. SIGTERM first so
|
||||
// it unmaps cleanly, SIGKILL after the grace window. Returning before the
|
||||
// process is gone would let the supervisor report a free card while 14GB is
|
||||
// still mapped, which is the one lie that would make yielding useless.
|
||||
func (r *runner) stop(grace time.Duration) {
|
||||
r.mu.Lock()
|
||||
cmd := r.cmd
|
||||
r.ready = false
|
||||
r.mu.Unlock()
|
||||
if cmd == nil || cmd.Process == nil {
|
||||
return
|
||||
}
|
||||
pgid := -cmd.Process.Pid
|
||||
_ = syscall.Kill(pgid, syscall.SIGTERM)
|
||||
deadline := time.Now().Add(grace)
|
||||
for time.Now().Before(deadline) {
|
||||
if !r.running() {
|
||||
return
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
log.Printf("mavgpud: llama-server did not exit in %s, killing", grace)
|
||||
_ = syscall.Kill(pgid, syscall.SIGKILL)
|
||||
}
|
||||
|
||||
// refreshReady asks llama-server whether the model is loaded. Called once per
|
||||
// supervisor tick, never per request.
|
||||
func (r *runner) refreshReady(ctx context.Context) {
|
||||
if !r.running() {
|
||||
return
|
||||
}
|
||||
ok := false
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, r.readyURL, nil)
|
||||
if err == nil {
|
||||
resp, err := r.http.Do(req)
|
||||
if err == nil {
|
||||
ok = resp.StatusCode == http.StatusOK
|
||||
resp.Body.Close()
|
||||
}
|
||||
}
|
||||
r.mu.Lock()
|
||||
was := r.ready
|
||||
r.ready = ok
|
||||
r.mu.Unlock()
|
||||
if ok && !was {
|
||||
log.Printf("mavgpud: model ready")
|
||||
}
|
||||
}
|
||||
+126
@@ -0,0 +1,126 @@
|
||||
# Offloading model work to the workstation
|
||||
|
||||
*Last verified: 2026-08-02 @ 5c05163. Living doc: correct it in place, do not append.*
|
||||
|
||||
Owner's call, 2026-08-02. Vikunja #483 is the umbrella. Tasks #484 to #487 are the
|
||||
work, and this file holds the shape and the rules all four must obey.
|
||||
|
||||
## The goal
|
||||
|
||||
homesrv cannot grow a GPU. The workstation has 16GB of VRAM. Move the model work
|
||||
to the workstation and leave homesrv running the logic that must be always-on,
|
||||
deterministic and cheap.
|
||||
|
||||
## Why this is tractable
|
||||
|
||||
The split already exists structurally. `mavsttd` and `mavttsd` are separate
|
||||
daemons that core reaches over a socket, not linked libraries. Moving them off-box
|
||||
is a transport change, not a redesign.
|
||||
|
||||
The microphone is at the workstation, because that is where the owner sits and
|
||||
homesrv is headless. So speech-to-text and the wake word are already on the
|
||||
workstation side by construction. Audio never has to cross the LAN. Only the core
|
||||
turn does.
|
||||
|
||||
## The constraint that shapes everything
|
||||
|
||||
The workstation's GPU is often busy: CPT runs, experiments, Correx, the manga-recap
|
||||
pipeline. It also sleeps. homesrv does not.
|
||||
|
||||
So an offloaded model is never *the* model. It is the preferred one, with a floor
|
||||
on homesrv. That is the shape the cascade already has, where a router error falls
|
||||
through to the classifier.
|
||||
|
||||
## The degradation rule
|
||||
|
||||
Two cases, and the line between them is sharp.
|
||||
|
||||
**Fall back silently** when the workstation model would only do the job *better*:
|
||||
routing, phrasing, a nudge. Falling back costs nothing that exists today, because
|
||||
the resident Qwen3-1.7B is today's production quality. The owner should not be told
|
||||
that his reply was phrased by the smaller model.
|
||||
|
||||
**Name the gap** when the resident model cannot do the job *at all*. A world
|
||||
question that a 1.7B answers by inventing is the case. A wrong answer is worse
|
||||
than "не могу сейчас". This is the rule CLAUDE.md already states for a sibling
|
||||
service being down.
|
||||
|
||||
Nothing in between. A turn never breaks on the workstation being asleep.
|
||||
|
||||
## Admission control, not a scheduler
|
||||
|
||||
There is no GPU arbiter. That is a service with its own failure modes, and nothing
|
||||
here needs work *distributed*. It needs admission control. The workstation
|
||||
advertises free VRAM over a health endpoint, and Maven treats it as one more query
|
||||
source that claims a turn or passes. llama-server also refuses to load when VRAM is
|
||||
short, so the failure is detectable without cooperation from the owner's other
|
||||
jobs.
|
||||
|
||||
The caller must be able to ask "is this peer usable right now" without a turn
|
||||
hanging on a timeout. A dead remote is a normal state, not an error state.
|
||||
`internal/llm.Pair` is that check on the Maven side. A prober caches the answer,
|
||||
so `Available()` is an atomic read and no turn pays for a health check.
|
||||
|
||||
llama-server does not stay up on the workstation. It cannot: a resident 7-14B
|
||||
would hold 16GB against the owner's CPT runs. So a supervisor there owns its
|
||||
lifecycle, keeps it loaded while the card is free, and unloads it on idle or
|
||||
when another process needs the card (owner's call, 2026-08-02, Vikunja #488).
|
||||
|
||||
That supervisor is still not a scheduler, and the distinction is worth holding.
|
||||
It arbitrates nothing between callers. It reports whether it can take work and
|
||||
manages one process to back that answer. Maven never asks it to start anything
|
||||
and never learns that it did.
|
||||
|
||||
## What stays on homesrv, permanently
|
||||
|
||||
The **embedder** (multilingual-e5-small, ONNX, CPU). It backs the classifier, which
|
||||
must answer while the GPU is saturated. It is also cheap enough on CPU that moving
|
||||
it buys nothing. Four callers:
|
||||
|
||||
| Caller | What for |
|
||||
|---|---|
|
||||
| `internal/router/classifier.go` | the routing floor |
|
||||
| `cmd/mavend/actions_query.go` (`queryEmbed`) | memory recall |
|
||||
| `cmd/mavend/feeds.go` | ingest embedding for every RSS item |
|
||||
| `internal/crawl/watch.go` | ingest embedding for every crawled page |
|
||||
|
||||
`internal/speaker` becomes a fifth once it lands.
|
||||
|
||||
## Inventory: what runs a model on homesrv today
|
||||
|
||||
The **resident model** is one llama-server with seven callers:
|
||||
|
||||
| Caller | What for |
|
||||
|---|---|
|
||||
| `cmd/mavend/voicewire.go` | routing |
|
||||
| `cmd/mavend/replier_llm.go` | replies |
|
||||
| `cmd/mavend/tick.go` | digestion worker: `PhraseNudge`, `PhraseReminder` |
|
||||
| `cmd/mavend/capture.go` | capture summarisation (unreachable, see #480) |
|
||||
| `cmd/mavend/mail.go` | mail extraction (off, no IMAP) |
|
||||
| `cmd/mavend/kiwixwire.go` | answering from a Kiwix, search or crawl passage |
|
||||
| `memoryeval.go`, `modelswap.go` | admin and evals |
|
||||
|
||||
Then the embedder above, **whisper.cpp** in `mavsttd`, and **piper** in `mavttsd`.
|
||||
`mavwaked` uses no model at all: an energy-threshold VAD over 30ms frames.
|
||||
|
||||
## Order
|
||||
|
||||
1. **Transport** (#484). Nothing else is possible until a seam can cross a host.
|
||||
`internal/netaddr` landed in PR #92. A seam address now carries its own scheme,
|
||||
and a scheme-less one is still unix. A tcp seam requires a shared token, because
|
||||
the filesystem permission that authenticated the unix socket is gone.
|
||||
2. **The resident model** (#485). Biggest quality delta. A 16GB card runs a 7-14B,
|
||||
which fixes what the 1.7B gets wrong: world knowledge, and the persona the CPT
|
||||
targets. The degradation path is already written and measured, since the
|
||||
classifier scores 68.8% full accuracy at p50 16.6µs on its own.
|
||||
3. **Speech-to-text and text-to-speech** (#486). They gain a real margin, but on
|
||||
quality alone, and both already work.
|
||||
4. **The wake word** (#487). Independent of all of the above.
|
||||
|
||||
## Assumptions
|
||||
|
||||
- The LAN is trusted enough that wireguard is supported but not required (owner's
|
||||
call). What crosses the wire is still his utterances. That is why the tcp seam
|
||||
carries its own token instead of assuming a network boundary.
|
||||
- The workstation is not expected to be up. Every child task must still serve a
|
||||
turn while it is down.
|
||||
+24
-13
@@ -8,13 +8,15 @@ import (
|
||||
"net"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/netaddr"
|
||||
)
|
||||
|
||||
// Client — the module side of the boundary. Wraps a unix-socket connection
|
||||
// and satisfies CoreAPI, so a module imports ipc, holds a CoreAPI, and is
|
||||
// agnostic to whether it's been wired in-process (tests / daemon-embedded)
|
||||
// or over this socket (full topology). The swappability is the seam auth
|
||||
// will insert into without touching module code.
|
||||
// Client — the module side of the boundary. Wraps a connection to core and
|
||||
// satisfies CoreAPI, so a module imports ipc, holds a CoreAPI, and is
|
||||
// agnostic to whether it's been wired in-process (tests / daemon-embedded),
|
||||
// over a local unix socket, or over tcp to another host. The swappability is
|
||||
// the seam auth will insert into without touching module code.
|
||||
//
|
||||
// One Client ⇒ one conn ⇒ one concurrent request at a time. A module that
|
||||
// wants parallel requests opens one Client per goroutine; the store is the
|
||||
@@ -22,7 +24,8 @@ import (
|
||||
// per-Client lock keeps frame interleaving impossible by construction.
|
||||
type Client struct {
|
||||
conn net.Conn
|
||||
path string // kept so a dropped conn can be re-dialed (core restart)
|
||||
path string // the address as configured, kept for errors and logs
|
||||
addr netaddr.Addr // parsed, so a dropped conn can be re-dialed (core restart)
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
@@ -81,14 +84,22 @@ var readOnlyMethods = map[Method]bool{
|
||||
MethodPing: true,
|
||||
}
|
||||
|
||||
// Dial connects to a core socket at path and returns a Client. The module
|
||||
// owns its Client lifecycle; Close on shutdown.
|
||||
// Dial connects to core at path and returns a Client. The module owns its
|
||||
// Client lifecycle; Close on shutdown.
|
||||
//
|
||||
// path is a netaddr seam address: a bare path is the unix socket it has
|
||||
// always been, and "tcp://host:port?token=..." reaches a core on another
|
||||
// host. See internal/netaddr.
|
||||
func Dial(path string) (*Client, error) {
|
||||
c, err := net.Dial("unix", path)
|
||||
addr, err := netaddr.Parse(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ipc: dial %s: %w", path, err)
|
||||
return nil, err
|
||||
}
|
||||
return &Client{conn: c, path: path}, nil
|
||||
c, err := netaddr.Dial(addr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ipc: dial %s: %w", addr, err)
|
||||
}
|
||||
return &Client{conn: c, path: path, addr: addr}, nil
|
||||
}
|
||||
|
||||
func (c *Client) Close() error {
|
||||
@@ -189,9 +200,9 @@ func (c *Client) call(ctx context.Context, m Method, params, result any) error {
|
||||
// re-dials clean. Caller holds c.mu.
|
||||
func (c *Client) roundtrip(m Method, raw json.RawMessage, resp *Response) error {
|
||||
if c.conn == nil {
|
||||
conn, err := net.Dial("unix", c.path)
|
||||
conn, err := netaddr.Dial(c.addr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: dial %s: %v", errWriteLost, c.path, err)
|
||||
return fmt.Errorf("%w: dial %s: %v", errWriteLost, c.addr, err)
|
||||
}
|
||||
c.conn = conn
|
||||
}
|
||||
|
||||
+20
-40
@@ -8,11 +8,11 @@ import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"os"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/netaddr"
|
||||
"github.com/kami/maven/internal/store"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
@@ -446,6 +446,7 @@ func mapErr(err error) error {
|
||||
type Server struct {
|
||||
api atomic.Value // stores CoreAPI
|
||||
path string
|
||||
addr netaddr.Addr
|
||||
|
||||
ln net.Listener
|
||||
wg sync.WaitGroup
|
||||
@@ -610,31 +611,29 @@ type CheckFunc func(ctx context.Context, m Method, params json.RawMessage) error
|
||||
// MethodAssertStepUp dispatch calls this instead of going through CoreAPI.
|
||||
type StepUpFunc func(ctx context.Context) error
|
||||
|
||||
// Listen creates a Server bound to path. path's parent dir must exist and be
|
||||
// 0700 (we chmod it if we own it); the socket file itself is created 0600 so
|
||||
// only the same unix user can connect — the current "auth floor", same radius
|
||||
// as wg at the network boundary. Removing a stale socket at path first lets
|
||||
// the daemon restart cleanly.
|
||||
// Listen creates a Server bound to path.
|
||||
//
|
||||
// A bare path is a unix socket, unchanged: its parent dir is 0700 and the
|
||||
// socket file itself is 0600, so only the same unix user can connect — the
|
||||
// current "auth floor", same radius as wg at the network boundary. A stale
|
||||
// socket is removed first so the daemon restarts cleanly.
|
||||
//
|
||||
// A "tcp://host:port?token=..." address binds a network listener instead, for
|
||||
// a module that lives on another host. There is no filesystem there to be the
|
||||
// auth floor, so netaddr checks the shared token before this package sees the
|
||||
// connection and a token is mandatory. See internal/netaddr.
|
||||
func Listen(path string, api CoreAPI) (*Server, error) {
|
||||
_ = os.Remove(path) // stale socket from a crashed daemon; ignore missing
|
||||
if err := os.MkdirAll(parentDir(path), 0o700); err != nil {
|
||||
return nil, fmt.Errorf("ipc: mkdir socket dir: %w", err)
|
||||
}
|
||||
// umask could widen the perms on socket creation; tighten then chmod to
|
||||
// be explicit. 0600 ⇒ read+write by owner only.
|
||||
oldMask := unix.Umask(0o077)
|
||||
ln, err := net.Listen("unix", path)
|
||||
unix.Umask(oldMask)
|
||||
addr, err := netaddr.Parse(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ipc: listen %s: %w", path, err)
|
||||
return nil, err
|
||||
}
|
||||
if err := os.Chmod(path, 0o600); err != nil {
|
||||
_ = ln.Close()
|
||||
_ = os.Remove(path)
|
||||
return nil, fmt.Errorf("ipc: chmod socket: %w", err)
|
||||
ln, err := netaddr.Listen(addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s := &Server{
|
||||
path: path,
|
||||
addr: addr,
|
||||
ln: ln,
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
@@ -1268,7 +1267,7 @@ func (s *Server) Close() error {
|
||||
// missing the seal costs every write since the last clean shutdown.
|
||||
log.Printf("ipc: %d connection(s) still busy after %s, closing anyway", s.liveConns(), closeGrace)
|
||||
}
|
||||
_ = os.Remove(s.path)
|
||||
netaddr.Cleanup(s.addr)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1338,25 +1337,6 @@ func (s *Server) Path() string { return s.path }
|
||||
// while the server is serving (dispatch loads api once per request via atomic).
|
||||
func (s *Server) SetAPI(api CoreAPI) { s.api.Store(api) }
|
||||
|
||||
func parentDir(p string) string {
|
||||
if i := lastIndexByte(p, '/'); i >= 0 {
|
||||
if i == 0 {
|
||||
return "/"
|
||||
}
|
||||
return p[:i]
|
||||
}
|
||||
return "."
|
||||
}
|
||||
|
||||
func lastIndexByte(s string, b byte) int {
|
||||
for i := len(s) - 1; i >= 0; i-- {
|
||||
if s[i] == b {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// peerCaller — read SO_PEERCRED off a unix conn to identify the connecting
|
||||
// process. Returns ok=false on a non-unix conn or a platform without
|
||||
// SO_PEERCRED; the caller then proceeds without a Caller (the socket perms
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"net/http"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Pair — a preferred model on another host, with the resident one as the floor.
|
||||
//
|
||||
// homesrv cannot grow a GPU and the workstation has 16GB of VRAM, so the big
|
||||
// model runs there and the resident Qwen3-1.7B stays here. See docs/offload.md.
|
||||
// The workstation is never assumed up: its GPU is often busy with CPT runs and
|
||||
// the manga-recap pipeline, and the machine sleeps. So the remote is preferred,
|
||||
// never required, and Pair is what makes "preferred" mean something precise.
|
||||
//
|
||||
// This is admission control, not a scheduler. There is no arbiter deciding who
|
||||
// gets the card. A prober asks the remote whether it will take work, caches the
|
||||
// answer, and every request reads that cached answer in nanoseconds. Routing
|
||||
// sits on the hot path at p50 825ms and must never wait on a machine that may
|
||||
// be asleep, so no request ever pays for a health check itself.
|
||||
//
|
||||
// Pair satisfies nothing by itself. Callers pick a method by which half of the
|
||||
// degradation rule they live under:
|
||||
//
|
||||
// - Complete falls back silently. For routing, replies, and nudge phrasing,
|
||||
// where the big model is only better and the 1.7B is today's shipping
|
||||
// quality. He is not told which model phrased his reply.
|
||||
// - CompleteRemote returns ErrRemoteUnavailable instead of falling back. For
|
||||
// a world question, or a long Kiwix or search passage, where a 1.7B
|
||||
// confabulates rather than summarises. A named gap beats an invented
|
||||
// answer.
|
||||
type Pair struct {
|
||||
remote *Client
|
||||
floor *Client
|
||||
|
||||
// up — the cached admission answer, written only by the prober goroutine
|
||||
// and read by every request. Atomic so the read costs nanoseconds and no
|
||||
// request ever contends with the prober.
|
||||
up atomic.Bool
|
||||
|
||||
health string
|
||||
interval time.Duration
|
||||
http *http.Client
|
||||
stop chan struct{}
|
||||
}
|
||||
|
||||
// ErrRemoteUnavailable — the workstation model was required and is not
|
||||
// answering. Callers on the naming half of the degradation rule turn this into
|
||||
// a gap in the reply ("не могу сейчас"), never into a guess from the floor.
|
||||
var ErrRemoteUnavailable = errors.New("llm: workstation model unavailable")
|
||||
|
||||
// ErrNoFloor — a Pair was built with no resident model to fall back to. A
|
||||
// configuration mistake: the floor is the whole point.
|
||||
var ErrNoFloor = errors.New("llm: no floor client")
|
||||
|
||||
// NewPair builds the two-model arrangement. remote may be nil, which is the
|
||||
// unconfigured deploy and must behave exactly as the box behaves today: every
|
||||
// call goes to the floor and nothing probes anything.
|
||||
//
|
||||
// health is the URL the prober asks. llama-server's /health answers "is a model
|
||||
// loaded and ready", which is the useful signal here, because llama-server
|
||||
// refuses to load at all when VRAM is short. That makes a busy card detectable
|
||||
// without any cooperation from the owner's other jobs.
|
||||
func NewPair(remote, floor *Client, health string, interval time.Duration) *Pair {
|
||||
p := &Pair{
|
||||
remote: remote,
|
||||
floor: floor,
|
||||
health: health,
|
||||
interval: interval,
|
||||
http: &http.Client{Timeout: probeTimeout},
|
||||
stop: make(chan struct{}),
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// probeTimeout — a remote that cannot answer /health this fast is not going to
|
||||
// serve a turn either. Short on purpose: the prober runs on its own goroutine,
|
||||
// but a slow probe still delays the moment Maven notices the card came back.
|
||||
const probeTimeout = 2 * time.Second
|
||||
|
||||
// Start begins probing. It returns immediately, and the first probe runs before
|
||||
// the first tick so a remote that is already up is used on the first turn
|
||||
// rather than after one interval of falling back. Safe to call with a nil
|
||||
// remote; it does nothing.
|
||||
func (p *Pair) Start(ctx context.Context) {
|
||||
if p.remote == nil || p.health == "" {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
p.probe(ctx)
|
||||
t := time.NewTicker(p.interval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-p.stop:
|
||||
return
|
||||
case <-t.C:
|
||||
p.probe(ctx)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Stop ends the prober. Idempotent.
|
||||
func (p *Pair) Stop() {
|
||||
select {
|
||||
case <-p.stop:
|
||||
default:
|
||||
close(p.stop)
|
||||
}
|
||||
}
|
||||
|
||||
// Available reports whether the workstation will take work right now. It reads
|
||||
// a cached flag, so it is safe to call per turn on the hot path. A false answer
|
||||
// is never stale in the direction that matters: the worst case is that Maven
|
||||
// falls back for up to one probe interval after the card frees up.
|
||||
func (p *Pair) Available() bool {
|
||||
return p.remote != nil && p.up.Load()
|
||||
}
|
||||
|
||||
func (p *Pair) probe(ctx context.Context) {
|
||||
ctx, cancel := context.WithTimeout(ctx, probeTimeout)
|
||||
defer cancel()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.health, nil)
|
||||
if err != nil {
|
||||
p.set(false)
|
||||
return
|
||||
}
|
||||
resp, err := p.http.Do(req)
|
||||
if err != nil {
|
||||
p.set(false)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
p.set(resp.StatusCode == http.StatusOK)
|
||||
}
|
||||
|
||||
// set records the admission answer and logs only the transitions. A machine
|
||||
// that sleeps every night would otherwise write one line per interval forever.
|
||||
func (p *Pair) set(up bool) {
|
||||
if p.up.Swap(up) == up {
|
||||
return
|
||||
}
|
||||
if up {
|
||||
log.Printf("llm: workstation model available at %s", p.health)
|
||||
} else {
|
||||
log.Printf("llm: workstation model unavailable, falling back to the resident model")
|
||||
}
|
||||
}
|
||||
|
||||
// Complete runs r on the workstation when it will take work, and on the
|
||||
// resident model otherwise. A remote that fails mid-request falls back too: the
|
||||
// admission answer is a cache and can be one interval out of date, so an error
|
||||
// here is expected rather than exceptional.
|
||||
//
|
||||
// This is the silent half of the degradation rule. It must be indistinguishable
|
||||
// from today's behaviour when the workstation is down.
|
||||
func (p *Pair) Complete(ctx context.Context, r Req) (string, error) {
|
||||
if p.floor == nil {
|
||||
return "", ErrNoFloor
|
||||
}
|
||||
if p.Available() {
|
||||
out, err := p.remote.Complete(ctx, r)
|
||||
if err == nil {
|
||||
return out, nil
|
||||
}
|
||||
// The cached answer was wrong. Correct it now rather than sending the
|
||||
// next request into the same hole, then fall back.
|
||||
p.set(false)
|
||||
}
|
||||
return p.floor.Complete(ctx, r)
|
||||
}
|
||||
|
||||
// CompleteRemote runs r on the workstation or refuses. It never falls back,
|
||||
// because for a world question the resident 1.7B does not answer worse, it
|
||||
// invents. Callers turn ErrRemoteUnavailable into a named gap.
|
||||
func (p *Pair) CompleteRemote(ctx context.Context, r Req) (string, error) {
|
||||
if !p.Available() {
|
||||
return "", ErrRemoteUnavailable
|
||||
}
|
||||
out, err := p.remote.Complete(ctx, r)
|
||||
if err != nil {
|
||||
p.set(false)
|
||||
return "", errors.Join(ErrRemoteUnavailable, err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// completionServer stands in for a llama-server. It counts what reached it, so
|
||||
// a test can say which of the two models answered.
|
||||
func completionServer(t *testing.T, reply string, hits *atomic.Int64) *httptest.Server {
|
||||
t.Helper()
|
||||
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"` + reply + `"}}]}`))
|
||||
}))
|
||||
t.Cleanup(s.Close)
|
||||
return s
|
||||
}
|
||||
|
||||
func healthServer(t *testing.T, ok *atomic.Bool) *httptest.Server {
|
||||
t.Helper()
|
||||
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if !ok.Load() {
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
t.Cleanup(s.Close)
|
||||
return s
|
||||
}
|
||||
|
||||
// waitFor polls until cond holds or the deadline passes. The prober runs on its
|
||||
// own goroutine, so a test has to wait for it rather than assume it has run.
|
||||
func waitFor(t *testing.T, cond func() bool) bool {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if cond() {
|
||||
return true
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// The unconfigured deploy. No remote, no probing, every call to the floor —
|
||||
// exactly what the box does today.
|
||||
func TestNoRemoteGoesToTheFloor(t *testing.T) {
|
||||
var floorHits atomic.Int64
|
||||
floor := completionServer(t, "floor", &floorHits)
|
||||
|
||||
p := NewPair(nil, New(floor.URL, time.Second), "", time.Second)
|
||||
p.Start(context.Background())
|
||||
defer p.Stop()
|
||||
|
||||
if p.Available() {
|
||||
t.Fatal("a Pair with no remote reports available")
|
||||
}
|
||||
out, err := p.Complete(context.Background(), Req{User: "привет"})
|
||||
if err != nil {
|
||||
t.Fatalf("complete: %v", err)
|
||||
}
|
||||
if out != "floor" || floorHits.Load() != 1 {
|
||||
t.Fatalf("out = %q, floor hits = %d", out, floorHits.Load())
|
||||
}
|
||||
}
|
||||
|
||||
// The workstation is up, so it answers and the resident model is not touched.
|
||||
func TestAvailableRemoteAnswers(t *testing.T) {
|
||||
var remoteHits, floorHits atomic.Int64
|
||||
remote := completionServer(t, "remote", &remoteHits)
|
||||
floor := completionServer(t, "floor", &floorHits)
|
||||
up := &atomic.Bool{}
|
||||
up.Store(true)
|
||||
health := healthServer(t, up)
|
||||
|
||||
p := NewPair(New(remote.URL, time.Second), New(floor.URL, time.Second), health.URL, 20*time.Millisecond)
|
||||
p.Start(context.Background())
|
||||
defer p.Stop()
|
||||
if !waitFor(t, p.Available) {
|
||||
t.Fatal("prober never saw the remote come up")
|
||||
}
|
||||
|
||||
out, err := p.Complete(context.Background(), Req{User: "привет"})
|
||||
if err != nil {
|
||||
t.Fatalf("complete: %v", err)
|
||||
}
|
||||
if out != "remote" || floorHits.Load() != 0 {
|
||||
t.Fatalf("out = %q, floor hits = %d", out, floorHits.Load())
|
||||
}
|
||||
}
|
||||
|
||||
// The card is busy, so /health refuses and Complete degrades silently. This is
|
||||
// the constraint from 483: the workstation being down is indistinguishable from
|
||||
// today's behaviour.
|
||||
func TestBusyCardFallsBackSilently(t *testing.T) {
|
||||
var remoteHits, floorHits atomic.Int64
|
||||
remote := completionServer(t, "remote", &remoteHits)
|
||||
floor := completionServer(t, "floor", &floorHits)
|
||||
health := healthServer(t, &atomic.Bool{}) // never ok
|
||||
|
||||
p := NewPair(New(remote.URL, time.Second), New(floor.URL, time.Second), health.URL, 20*time.Millisecond)
|
||||
p.Start(context.Background())
|
||||
defer p.Stop()
|
||||
time.Sleep(60 * time.Millisecond)
|
||||
|
||||
out, err := p.Complete(context.Background(), Req{User: "привет"})
|
||||
if err != nil {
|
||||
t.Fatalf("complete: %v", err)
|
||||
}
|
||||
if out != "floor" || remoteHits.Load() != 0 {
|
||||
t.Fatalf("out = %q, remote hits = %d", out, remoteHits.Load())
|
||||
}
|
||||
}
|
||||
|
||||
// The cached admission answer can be one interval out of date, so a remote that
|
||||
// dies between probes must still not break the turn.
|
||||
func TestRemoteErrorMidRequestFallsBack(t *testing.T) {
|
||||
var floorHits atomic.Int64
|
||||
dead := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
defer dead.Close()
|
||||
floor := completionServer(t, "floor", &floorHits)
|
||||
up := &atomic.Bool{}
|
||||
up.Store(true)
|
||||
health := healthServer(t, up)
|
||||
|
||||
p := NewPair(New(dead.URL, time.Second), New(floor.URL, time.Second), health.URL, time.Hour)
|
||||
p.Start(context.Background())
|
||||
defer p.Stop()
|
||||
if !waitFor(t, p.Available) {
|
||||
t.Fatal("prober never saw the remote come up")
|
||||
}
|
||||
|
||||
out, err := p.Complete(context.Background(), Req{User: "привет"})
|
||||
if err != nil {
|
||||
t.Fatalf("complete: %v", err)
|
||||
}
|
||||
if out != "floor" || floorHits.Load() != 1 {
|
||||
t.Fatalf("out = %q, floor hits = %d", out, floorHits.Load())
|
||||
}
|
||||
// The failed request must have corrected the cached answer, so the next
|
||||
// one does not walk into the same hole.
|
||||
if p.Available() {
|
||||
t.Fatal("a failed remote request left the admission answer up")
|
||||
}
|
||||
}
|
||||
|
||||
// The naming half of the degradation rule. A world question must not be handed
|
||||
// to the resident model, because it answers by inventing.
|
||||
func TestCompleteRemoteNamesTheGap(t *testing.T) {
|
||||
var floorHits atomic.Int64
|
||||
floor := completionServer(t, "floor", &floorHits)
|
||||
health := healthServer(t, &atomic.Bool{}) // never ok
|
||||
|
||||
p := NewPair(New("http://127.0.0.1:1", time.Second), New(floor.URL, time.Second), health.URL, 20*time.Millisecond)
|
||||
p.Start(context.Background())
|
||||
defer p.Stop()
|
||||
time.Sleep(60 * time.Millisecond)
|
||||
|
||||
if _, err := p.CompleteRemote(context.Background(), Req{User: "почему небо голубое"}); !errors.Is(err, ErrRemoteUnavailable) {
|
||||
t.Fatalf("err = %v, want ErrRemoteUnavailable", err)
|
||||
}
|
||||
if floorHits.Load() != 0 {
|
||||
t.Fatalf("CompleteRemote fell back to the floor %d times", floorHits.Load())
|
||||
}
|
||||
}
|
||||
|
||||
// Routing sits on the hot path and must never pay for a health check. Available
|
||||
// reads a cached flag, so it costs no network at all.
|
||||
func TestAvailableDoesNotProbe(t *testing.T) {
|
||||
var probes atomic.Int64
|
||||
health := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
probes.Add(1)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer health.Close()
|
||||
|
||||
p := NewPair(New("http://127.0.0.1:1", time.Second), New("http://127.0.0.1:1", time.Second), health.URL, time.Hour)
|
||||
p.Start(context.Background())
|
||||
defer p.Stop()
|
||||
if !waitFor(t, p.Available) {
|
||||
t.Fatal("prober never ran")
|
||||
}
|
||||
|
||||
before := probes.Load()
|
||||
for range 1000 {
|
||||
p.Available()
|
||||
}
|
||||
if got := probes.Load(); got != before {
|
||||
t.Fatalf("1000 Available calls made %d probes", got-before)
|
||||
}
|
||||
}
|
||||
|
||||
// A Pair with no floor is a configuration mistake, and it must say so rather
|
||||
// than silently having nowhere to degrade to.
|
||||
func TestNoFloorIsAnError(t *testing.T) {
|
||||
p := NewPair(nil, nil, "", time.Second)
|
||||
if _, err := p.Complete(context.Background(), Req{User: "привет"}); !errors.Is(err, ErrNoFloor) {
|
||||
t.Fatalf("err = %v, want ErrNoFloor", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,277 @@
|
||||
// Package netaddr parses a daemon seam address and dials or binds it.
|
||||
//
|
||||
// Every seam between Maven's daemons used to be a unix socket with the
|
||||
// network hardcoded at the call site — two dials in internal/ipc, one listen,
|
||||
// and the same pair again in internal/worker. That is correct for co-located
|
||||
// daemons and it is the reason a module cannot live on another host. This
|
||||
// package moves the choice into the address string so a deploy picks the
|
||||
// transport, not a recompile:
|
||||
//
|
||||
// /run/maven/stt.sock unix (the default, unchanged)
|
||||
// unix:///run/maven/stt.sock unix (explicit, same thing)
|
||||
// tcp://workstation:9310?token=hunter2 tcp
|
||||
//
|
||||
// A scheme-less address is unix and behaves exactly as it did before this
|
||||
// package existed: same 0700 parent dir, same 0600 socket, same bytes on the
|
||||
// wire with no handshake in front of them.
|
||||
//
|
||||
// Over TCP the filesystem permission that authenticated the unix socket is
|
||||
// gone, and what crosses this seam is audio of the owner speaking and the
|
||||
// text of his turns. So a TCP seam carries a shared token, checked before the
|
||||
// first protocol frame is read. Wireguard is supported underneath and is not
|
||||
// required.
|
||||
package netaddr
|
||||
|
||||
import (
|
||||
"crypto/subtle"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// ErrUnauthorized — the peer presented a token the listener does not accept,
|
||||
// or presented none when one is required.
|
||||
var ErrUnauthorized = errors.New("netaddr: unauthorized")
|
||||
|
||||
// Addr is a parsed seam endpoint.
|
||||
type Addr struct {
|
||||
// Network is "unix" or "tcp".
|
||||
Network string
|
||||
// Address is the socket path (unix) or host:port (tcp).
|
||||
Address string
|
||||
// Token is the shared secret for a tcp seam. Empty for unix, where the
|
||||
// filesystem does the same job.
|
||||
Token string
|
||||
}
|
||||
|
||||
// String renders the address for logs and errors. The token is never included.
|
||||
func (a Addr) String() string {
|
||||
if a.Network == "unix" {
|
||||
return a.Address
|
||||
}
|
||||
return a.Network + "://" + a.Address
|
||||
}
|
||||
|
||||
// IsUnix reports whether this seam is a unix socket, and so is local, is
|
||||
// authenticated by file permissions, and needs no handshake.
|
||||
func (a Addr) IsUnix() bool { return a.Network == "unix" }
|
||||
|
||||
// Parse reads a seam address. Anything without a "scheme://" prefix is a unix
|
||||
// socket path, which keeps every existing config and every default working
|
||||
// untouched.
|
||||
func Parse(s string) (Addr, error) {
|
||||
if !strings.Contains(s, "://") {
|
||||
return Addr{Network: "unix", Address: s}, nil
|
||||
}
|
||||
u, err := url.Parse(s)
|
||||
if err != nil {
|
||||
return Addr{}, fmt.Errorf("netaddr: parse %q: %w", s, err)
|
||||
}
|
||||
switch u.Scheme {
|
||||
case "unix":
|
||||
return Addr{Network: "unix", Address: u.Path}, nil
|
||||
case "tcp":
|
||||
if u.Host == "" {
|
||||
return Addr{}, fmt.Errorf("netaddr: %q has no host:port", s)
|
||||
}
|
||||
return Addr{Network: "tcp", Address: u.Host, Token: u.Query().Get("token")}, nil
|
||||
default:
|
||||
return Addr{}, fmt.Errorf("netaddr: unsupported scheme %q", u.Scheme)
|
||||
}
|
||||
}
|
||||
|
||||
// MustParse is Parse for a literal known good at compile time. It panics on a
|
||||
// bad address, so use it in tests and constants, never on config input.
|
||||
func MustParse(s string) Addr {
|
||||
a, err := Parse(s)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
// handshakeTimeout bounds the token exchange. A peer that cannot write one
|
||||
// short line in this long is not going to serve a turn either.
|
||||
const handshakeTimeout = 5 * time.Second
|
||||
|
||||
// greeting prefixes the token line. Versioned so a later mTLS seam can be
|
||||
// told apart from this one on the wire.
|
||||
const greeting = "MAVEN1 "
|
||||
|
||||
// Dial connects to a. On a tcp seam it sends the token and waits for the
|
||||
// listener to accept it, so a returned conn is already authorized and the
|
||||
// caller can write its first protocol frame.
|
||||
func Dial(a Addr) (net.Conn, error) {
|
||||
return DialTimeout(a, 0)
|
||||
}
|
||||
|
||||
// DialTimeout is Dial with a bound on the connect. Zero means the operating
|
||||
// system default. The token exchange gets its own timeout either way.
|
||||
func DialTimeout(a Addr, timeout time.Duration) (net.Conn, error) {
|
||||
var c net.Conn
|
||||
var err error
|
||||
if timeout > 0 {
|
||||
c, err = net.DialTimeout(a.Network, a.Address, timeout)
|
||||
} else {
|
||||
c, err = net.Dial(a.Network, a.Address)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if a.IsUnix() {
|
||||
return c, nil
|
||||
}
|
||||
if err := clientHandshake(c, a.Token); err != nil {
|
||||
_ = c.Close()
|
||||
return nil, err
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func clientHandshake(c net.Conn, token string) error {
|
||||
_ = c.SetDeadline(time.Now().Add(handshakeTimeout))
|
||||
defer c.SetDeadline(time.Time{})
|
||||
if _, err := c.Write([]byte(greeting + token + "\n")); err != nil {
|
||||
return fmt.Errorf("netaddr: send token: %w", err)
|
||||
}
|
||||
var reply [3]byte
|
||||
if _, err := readFull(c, reply[:]); err != nil {
|
||||
return fmt.Errorf("%w: %v", ErrUnauthorized, err)
|
||||
}
|
||||
if string(reply[:]) != "ok\n" {
|
||||
return ErrUnauthorized
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Listener wraps a net.Listener so Accept performs the token check for a tcp
|
||||
// seam. A connection that fails the check is closed and never surfaces, so
|
||||
// the protocol above this layer only ever sees authorized peers.
|
||||
type Listener struct {
|
||||
net.Listener
|
||||
addr Addr
|
||||
}
|
||||
|
||||
// Accept returns the next authorized connection. Unauthorized peers are
|
||||
// dropped and Accept keeps waiting: a bad token is a rejected stranger, not a
|
||||
// reason to stop serving.
|
||||
func (l *Listener) Accept() (net.Conn, error) {
|
||||
for {
|
||||
c, err := l.Listener.Accept()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if l.addr.IsUnix() {
|
||||
return c, nil
|
||||
}
|
||||
if err := serverHandshake(c, l.addr.Token); err != nil {
|
||||
_ = c.Close()
|
||||
continue
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Addr reports the parsed seam address this listener was built from.
|
||||
func (l *Listener) SeamAddr() Addr { return l.addr }
|
||||
|
||||
func serverHandshake(c net.Conn, want string) error {
|
||||
_ = c.SetDeadline(time.Now().Add(handshakeTimeout))
|
||||
defer c.SetDeadline(time.Time{})
|
||||
// The line is bounded: greeting, token, newline. Read a byte at a time so
|
||||
// nothing of the first protocol frame is consumed when the token is short.
|
||||
line := make([]byte, 0, 128)
|
||||
var b [1]byte
|
||||
for {
|
||||
if _, err := readFull(c, b[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
if b[0] == '\n' {
|
||||
break
|
||||
}
|
||||
line = append(line, b[0])
|
||||
if len(line) > 512 {
|
||||
return ErrUnauthorized
|
||||
}
|
||||
}
|
||||
got, ok := strings.CutPrefix(string(line), greeting)
|
||||
if !ok {
|
||||
return ErrUnauthorized
|
||||
}
|
||||
if subtle.ConstantTimeCompare([]byte(got), []byte(want)) != 1 {
|
||||
return ErrUnauthorized
|
||||
}
|
||||
if _, err := c.Write([]byte("ok\n")); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func readFull(c net.Conn, p []byte) (int, error) {
|
||||
n := 0
|
||||
for n < len(p) {
|
||||
m, err := c.Read(p[n:])
|
||||
n += m
|
||||
if err != nil {
|
||||
return n, err
|
||||
}
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// Listen binds a. A unix seam gets the perms it has always had: parent dir
|
||||
// 0700, socket 0600, and any stale socket from a crashed daemon removed
|
||||
// first. A tcp seam must carry a token, because there is no filesystem to
|
||||
// stand in for one.
|
||||
func Listen(a Addr) (*Listener, error) {
|
||||
if a.IsUnix() {
|
||||
ln, err := listenUnix(a.Address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Listener{Listener: ln, addr: a}, nil
|
||||
}
|
||||
if a.Token == "" {
|
||||
return nil, fmt.Errorf("netaddr: listen %s: tcp seam requires a token", a)
|
||||
}
|
||||
ln, err := net.Listen("tcp", a.Address)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("netaddr: listen %s: %w", a, err)
|
||||
}
|
||||
return &Listener{Listener: ln, addr: a}, nil
|
||||
}
|
||||
|
||||
func listenUnix(path string) (net.Listener, error) {
|
||||
_ = os.Remove(path) // stale socket from a crashed daemon; ignore missing
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
return nil, fmt.Errorf("netaddr: mkdir socket dir: %w", err)
|
||||
}
|
||||
// umask could widen the perms on socket creation; tighten then chmod to
|
||||
// be explicit. 0600 ⇒ read+write by owner only.
|
||||
oldMask := unix.Umask(0o077)
|
||||
ln, err := net.Listen("unix", path)
|
||||
unix.Umask(oldMask)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("netaddr: listen %s: %w", path, err)
|
||||
}
|
||||
if err := os.Chmod(path, 0o600); err != nil {
|
||||
_ = ln.Close()
|
||||
_ = os.Remove(path)
|
||||
return nil, fmt.Errorf("netaddr: chmod socket: %w", err)
|
||||
}
|
||||
return ln, nil
|
||||
}
|
||||
|
||||
// Cleanup removes the socket file behind a unix seam. It is a no-op for tcp.
|
||||
func Cleanup(a Addr) {
|
||||
if a.IsUnix() && a.Address != "" {
|
||||
_ = os.Remove(a.Address)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
package netaddr
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A scheme-less address must stay unix. Every deploy in the tree writes a bare
|
||||
// path, so this is the test that says the transport change costs them nothing.
|
||||
func TestParseSchemelessIsUnix(t *testing.T) {
|
||||
a, err := Parse("/run/maven/stt.sock")
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if !a.IsUnix() {
|
||||
t.Fatalf("want unix, got %q", a.Network)
|
||||
}
|
||||
if a.Address != "/run/maven/stt.sock" {
|
||||
t.Fatalf("address = %q", a.Address)
|
||||
}
|
||||
if a.Token != "" {
|
||||
t.Fatalf("unix seam carries a token: %q", a.Token)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParse(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
net, addr, tk string
|
||||
wantErr bool
|
||||
}{
|
||||
{in: "", net: "unix", addr: ""},
|
||||
{in: "unix:///run/maven/core.sock", net: "unix", addr: "/run/maven/core.sock"},
|
||||
{in: "tcp://workstation:9310", net: "tcp", addr: "workstation:9310"},
|
||||
{in: "tcp://workstation:9310?token=hunter2", net: "tcp", addr: "workstation:9310", tk: "hunter2"},
|
||||
{in: "tcp://", wantErr: true},
|
||||
{in: "udp://workstation:9310", wantErr: true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
a, err := Parse(c.in)
|
||||
if c.wantErr {
|
||||
if err == nil {
|
||||
t.Errorf("Parse(%q) = %v, want error", c.in, a)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
t.Errorf("Parse(%q): %v", c.in, err)
|
||||
continue
|
||||
}
|
||||
if a.Network != c.net || a.Address != c.addr || a.Token != c.tk {
|
||||
t.Errorf("Parse(%q) = %+v, want %s/%s/%s", c.in, a, c.net, c.addr, c.tk)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The token must never reach a log line.
|
||||
func TestStringHidesToken(t *testing.T) {
|
||||
a := MustParse("tcp://workstation:9310?token=hunter2")
|
||||
if got := a.String(); got != "tcp://workstation:9310" {
|
||||
t.Fatalf("String() = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A unix seam must round-trip with no handshake in front of the payload: the
|
||||
// first bytes the listener sees are the caller's, exactly as before.
|
||||
func TestUnixRoundTripHasNoHandshake(t *testing.T) {
|
||||
a := MustParse(filepath.Join(t.TempDir(), "s.sock"))
|
||||
ln, err := Listen(a)
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
defer ln.Close()
|
||||
go echoOnce(ln)
|
||||
|
||||
c, err := Dial(a)
|
||||
if err != nil {
|
||||
t.Fatalf("dial: %v", err)
|
||||
}
|
||||
defer c.Close()
|
||||
if got := roundTrip(t, c, "hello"); got != "hello" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTCPRoundTripWithToken(t *testing.T) {
|
||||
ln, addr := listenLoopback(t, "s3cret")
|
||||
defer ln.Close()
|
||||
go echoOnce(ln)
|
||||
|
||||
c, err := Dial(addr)
|
||||
if err != nil {
|
||||
t.Fatalf("dial: %v", err)
|
||||
}
|
||||
defer c.Close()
|
||||
if got := roundTrip(t, c, "hello"); got != "hello" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTCPWrongTokenIsRejected(t *testing.T) {
|
||||
ln, addr := listenLoopback(t, "s3cret")
|
||||
defer ln.Close()
|
||||
// Accept keeps waiting past the bad peer, so nothing here should ever
|
||||
// reach the echo. A conn that does means the token was not checked.
|
||||
go echoOnce(ln)
|
||||
|
||||
bad := addr
|
||||
bad.Token = "wrong"
|
||||
if _, err := Dial(bad); !errors.Is(err, ErrUnauthorized) {
|
||||
t.Fatalf("dial with wrong token: err = %v, want ErrUnauthorized", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A stranger that speaks the protocol instead of the greeting is dropped, and
|
||||
// the listener stays up for the peer that follows it.
|
||||
func TestTCPUngreetedPeerDoesNotKillTheListener(t *testing.T) {
|
||||
ln, addr := listenLoopback(t, "s3cret")
|
||||
defer ln.Close()
|
||||
go echoOnce(ln)
|
||||
|
||||
raw, err := net.Dial("tcp", addr.Address)
|
||||
if err != nil {
|
||||
t.Fatalf("raw dial: %v", err)
|
||||
}
|
||||
if _, err := raw.Write([]byte("GET / HTTP/1.1\n")); err != nil {
|
||||
t.Fatalf("raw write: %v", err)
|
||||
}
|
||||
raw.Close()
|
||||
|
||||
c, err := Dial(addr)
|
||||
if err != nil {
|
||||
t.Fatalf("dial after stranger: %v", err)
|
||||
}
|
||||
defer c.Close()
|
||||
if got := roundTrip(t, c, "still here"); got != "still here" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A tcp seam with no token is a misconfiguration, and it must fail at bind
|
||||
// rather than serve the owner's turns to anyone who connects.
|
||||
func TestTCPListenRequiresToken(t *testing.T) {
|
||||
if _, err := Listen(MustParse("tcp://127.0.0.1:0")); err == nil {
|
||||
t.Fatal("listen on a tokenless tcp seam succeeded")
|
||||
}
|
||||
}
|
||||
|
||||
func listenLoopback(t *testing.T, token string) (*Listener, Addr) {
|
||||
t.Helper()
|
||||
ln, err := Listen(Addr{Network: "tcp", Address: "127.0.0.1:0", Token: token})
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
return ln, Addr{Network: "tcp", Address: ln.Addr().String(), Token: token}
|
||||
}
|
||||
|
||||
func echoOnce(ln *Listener) {
|
||||
c, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer c.Close()
|
||||
buf := make([]byte, 256)
|
||||
n, err := c.Read(buf)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_, _ = c.Write(buf[:n])
|
||||
}
|
||||
|
||||
func roundTrip(t *testing.T, c net.Conn, msg string) string {
|
||||
t.Helper()
|
||||
if _, err := c.Write([]byte(msg)); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
buf := make([]byte, 256)
|
||||
n, err := c.Read(buf)
|
||||
if err != nil {
|
||||
t.Fatalf("read: %v", err)
|
||||
}
|
||||
return string(buf[:n])
|
||||
}
|
||||
@@ -24,6 +24,8 @@ import (
|
||||
"net"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/netaddr"
|
||||
)
|
||||
|
||||
// Client — one connection to one worker module. NOT goroutine-safe for
|
||||
@@ -38,15 +40,25 @@ type Client struct {
|
||||
dial func() (net.Conn, error)
|
||||
}
|
||||
|
||||
// Dial opens a Client to the worker socket at path. The first call lazily
|
||||
// Dial opens a Client to the worker module at path. The first call lazily
|
||||
// dials; subsequent calls reuse the conn (a fresh dial happens on next call
|
||||
// after a teardown). Lazy dial keeps a worker that's restarting from
|
||||
// blocking core's startup; core attempts the dial on first use.
|
||||
//
|
||||
// path is a netaddr seam address. A bare path is the unix socket it has
|
||||
// always been; "tcp://workstation:9310?token=..." reaches a module on another
|
||||
// host, which is how stt and tts move to the machine with the GPU and the
|
||||
// microphone. A bad address surfaces on the first call, not here, because
|
||||
// Dial does not fail — see internal/netaddr.
|
||||
func Dial(path string) *Client {
|
||||
addr, err := netaddr.Parse(path)
|
||||
return &Client{
|
||||
path: path,
|
||||
dial: func() (net.Conn, error) {
|
||||
return net.Dial("unix", path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return netaddr.Dial(addr)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
+22
-34
@@ -7,10 +7,11 @@
|
||||
// which module to dial; mixing the two is a config error caught cleanly by
|
||||
// the wire, not a runtime goroutine panic). One Server per module process.
|
||||
//
|
||||
// Socket perms mirror ipc.Server: dir 0700, socket 0600 ⇒ same unix user.
|
||||
// The module has no key, so the floor is "same user"; the wg/mTLS layers
|
||||
// are out of scope here (this socket never crosses the network radius —
|
||||
// it's local-only, point-to-point between two processes on the box).
|
||||
// The seam address decides the transport. On the default unix socket the
|
||||
// perms mirror ipc.Server — dir 0700, socket 0600 ⇒ same unix user — and that
|
||||
// is the whole auth floor, because the seam never leaves the box. A tcp
|
||||
// address moves the module to another host and takes that floor away, so
|
||||
// netaddr checks a shared token before the first frame. See internal/netaddr.
|
||||
package worker
|
||||
|
||||
import (
|
||||
@@ -18,11 +19,10 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
"github.com/kami/maven/internal/netaddr"
|
||||
)
|
||||
|
||||
// Server — a worker module process's listener. Wires either a Transcriber,
|
||||
@@ -34,6 +34,7 @@ type Server struct {
|
||||
s Synthesizer
|
||||
|
||||
path string
|
||||
addr netaddr.Addr
|
||||
ln net.Listener
|
||||
|
||||
wg sync.WaitGroup
|
||||
@@ -61,25 +62,24 @@ func NewSynthesizerServer(path string, s Synthesizer) *Server {
|
||||
// two separate processes per the restart-free / fail-independent invariant).
|
||||
func (srv *Server) SetSynthesizer(s Synthesizer) { srv.s = s }
|
||||
|
||||
// Listen binds the unix socket with 0700 dir + 0600 socket perms (same floor
|
||||
// as internal/ipc). A stale socket at path is removed first so the worker
|
||||
// process restarts cleanly after a crash, no manual cleanup needed.
|
||||
// Listen binds the seam address the Server was built with.
|
||||
//
|
||||
// A bare path is a unix socket with 0700 dir + 0600 socket perms, the same
|
||||
// floor as internal/ipc, and a stale socket is removed first so the worker
|
||||
// process restarts cleanly after a crash. A "tcp://host:port?token=..."
|
||||
// address binds a network listener instead, so this module can run on the
|
||||
// workstation while core stays on homesrv; the token is mandatory there,
|
||||
// because there is no filesystem to be the auth floor. See internal/netaddr.
|
||||
func (srv *Server) Listen() error {
|
||||
_ = os.Remove(srv.path)
|
||||
if err := os.MkdirAll(parentDir(srv.path), 0o700); err != nil {
|
||||
return fmt.Errorf("worker: mkdir socket dir: %w", err)
|
||||
}
|
||||
oldMask := unix.Umask(0o077)
|
||||
ln, err := net.Listen("unix", srv.path)
|
||||
unix.Umask(oldMask)
|
||||
addr, err := netaddr.Parse(srv.path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("worker: listen %s: %w", srv.path, err)
|
||||
return err
|
||||
}
|
||||
if err := os.Chmod(srv.path, 0o600); err != nil {
|
||||
_ = ln.Close()
|
||||
_ = os.Remove(srv.path)
|
||||
return fmt.Errorf("worker: chmod socket: %w", err)
|
||||
ln, err := netaddr.Listen(addr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
srv.addr = addr
|
||||
srv.ln = ln
|
||||
return nil
|
||||
}
|
||||
@@ -193,7 +193,7 @@ func (srv *Server) Close() error {
|
||||
}
|
||||
err := srv.ln.Close()
|
||||
srv.wg.Wait()
|
||||
_ = os.Remove(srv.path)
|
||||
netaddr.Cleanup(srv.addr)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -214,15 +214,3 @@ func marshalResult(v any) json.RawMessage {
|
||||
b, _ := json.Marshal(v)
|
||||
return b
|
||||
}
|
||||
|
||||
func parentDir(p string) string {
|
||||
for i := len(p) - 1; i >= 0; i-- {
|
||||
if p[i] == '/' {
|
||||
if i == 0 {
|
||||
return "/"
|
||||
}
|
||||
return p[:i]
|
||||
}
|
||||
}
|
||||
return "."
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user