diff --git a/.gitignore b/.gitignore index b6b2424..d240800 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ /mavwaked /mavmaild /mavupdate +/mavgpud # Certs (private keys, don't commit) certs/ diff --git a/Makefile b/Makefile index 1721e39..7fbf5ff 100644 --- a/Makefile +++ b/Makefile @@ -16,11 +16,11 @@ PIPER_BIN := $(shell pwd)/deps/piper/piper PIPER_MODEL := $(shell pwd)/models/tts/ru_RU-irina-medium.onnx PIPER_ESPEAK := $(shell pwd)/deps/piper/espeak-ng-data -.PHONY: simulate stt-fixtures test-stt-golden all build build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav clean test fmt-check vet run-stt run-tts run-web download-embedder deps-go eval-router eval-recall eval-phrasing eval-models +.PHONY: simulate stt-fixtures test-stt-golden all build build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav clean test fmt-check vet run-stt run-tts run-web download-embedder deps-go eval-router eval-recall eval-phrasing eval-models build-gpud all: build -build: build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav build-mail build-update +build: build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav build-mail build-update build-gpud build-stt: CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \ @@ -59,6 +59,12 @@ build-mail: build-update: $(GO) build $(GOFLAGS) -o mavupdate ./cmd/mavupdate/ +# mavgpud runs on the workstation, not here. It is built with the rest so a +# broken supervisor is caught by `make build` on homesrv rather than by the +# workstation refusing to serve. Copy the binary over, do not `make deploy` it. +build-gpud: + $(GO) build $(GOFLAGS) -o mavgpud ./cmd/mavgpud/ + run-web: build-web ./mavweb -addr :9200 -voice 127.0.0.1:9100 diff --git a/deploy/mavgpud.json b/deploy/mavgpud.json new file mode 100644 index 0000000..41d001d --- /dev/null +++ b/deploy/mavgpud.json @@ -0,0 +1,32 @@ +{ + "listen": ":8080", + "llama_addr": "127.0.0.1:10000", + "llama_bin": "llama-server", + "llama_args": [ + "-m", "/mnt/D/AI/gemma4/gemma-4-12B-it-qat-UD-Q4_K_XL.gguf", + "-md", "/mnt/D/AI/gemma4/mtp-gemma-4-12B-it-BF16.gguf", + "-ngl", "99", + "-fa", "on", + "-np", "1", + "--host", "127.0.0.1", + "--port", "10000", + "--ctx-size", "32768", + "--threads", "6", + "--batch-size", "2048", + "--ubatch-size", "512", + "--jinja", + "--chat-template-kwargs", "{\"enable_thinking\":false}", + "--spec-type", "draft-mtp", + "--spec-draft-n-max", "2" + ], + + "kfd_root": "/sys/class/kfd/kfd/proc", + "drm_device": "/sys/class/drm/card1/device", + + "poll": "1s", + "idle_timeout": "15m", + "stop_grace": "20s", + "min_free_vram_bytes": 10737418240, + "evict_after_polls": 2, + "start_after_polls": 5 +} diff --git a/deploy/mavgpud.service b/deploy/mavgpud.service new file mode 100644 index 0000000..b200baa --- /dev/null +++ b/deploy/mavgpud.service @@ -0,0 +1,24 @@ +[Unit] +# Runs on the workstation (bugmachine), not on homesrv. Install as a systemd +# user unit and turn on lingering, so the card is supervised after a reboot +# with nobody logged in: +# +# scp mavgpud workpc:~/.local/bin/mavgpud +# scp deploy/mavgpud.json workpc:~/.config/mavgpud.json +# scp deploy/mavgpud.service workpc:~/.config/systemd/user/mavgpud.service +# ssh workpc 'systemctl --user daemon-reload && systemctl --user enable --now mavgpud' +# sudo loginctl enable-linger kami +Description=Maven GPU supervisor (holds llama-server while the card is free) +After=network.target + +[Service] +ExecStart=%h/.local/bin/mavgpud -config %h/.config/mavgpud.json +Restart=always +RestartSec=5 +# The card must come back when the supervisor goes down. mavgpud stops +# llama-server on SIGTERM, so give it longer than stop_grace to do that. +KillSignal=SIGTERM +TimeoutStopSec=60 + +[Install] +WantedBy=default.target diff --git a/docs/offload.md b/docs/offload.md index 7b2957f..4c6a6c8 100644 --- a/docs/offload.md +++ b/docs/offload.md @@ -71,6 +71,22 @@ It arbitrates nothing between callers. It reports whether it can take work and manages one process to back that answer. Maven never asks it to start anything and never learns that it did. +Contention is decided by presence under `/sys/class/kfd/kfd/proc`, not by a VRAM +threshold. A ROCm process registers there when it initialises HIP, before it +allocates anything. So the supervisor sees a contender during that job's startup, +and yields before the job loses the memory it asked for. A +threshold reads the card too late. By the time free VRAM has dropped, the other +job has already lost the allocation race. Free VRAM is still read, but only as a +precondition for loading, never as the eviction signal. One blind spot is known. +A job can take the card without registering on the KFD, as a Vulkan or a +video-decode job would. `describe()` logs every contender's comm, and that log is +how we find out whether the blind spot is real. + +`mavgpud` runs from a systemd unit on the workstation with +`deploy/mavgpud.json` as its config, and `llama_args` is passed to llama-server +untouched. The model, the context size, the layer count and the MTP flags are the +owner's business and not this daemon's schema. + ## What stays on homesrv, permanently The **embedder** (multilingual-e5-small, ONNX, CPU). It backs the classifier, which