diff --git a/.gitignore b/.gitignore index b6b2424..d240800 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ /mavwaked /mavmaild /mavupdate +/mavgpud # Certs (private keys, don't commit) certs/ diff --git a/Makefile b/Makefile index 1721e39..7fbf5ff 100644 --- a/Makefile +++ b/Makefile @@ -16,11 +16,11 @@ PIPER_BIN := $(shell pwd)/deps/piper/piper PIPER_MODEL := $(shell pwd)/models/tts/ru_RU-irina-medium.onnx PIPER_ESPEAK := $(shell pwd)/deps/piper/espeak-ng-data -.PHONY: simulate stt-fixtures test-stt-golden all build build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav clean test fmt-check vet run-stt run-tts run-web download-embedder deps-go eval-router eval-recall eval-phrasing eval-models +.PHONY: simulate stt-fixtures test-stt-golden all build build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav clean test fmt-check vet run-stt run-tts run-web download-embedder deps-go eval-router eval-recall eval-phrasing eval-models build-gpud all: build -build: build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav build-mail build-update +build: build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav build-mail build-update build-gpud build-stt: CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \ @@ -59,6 +59,12 @@ build-mail: build-update: $(GO) build $(GOFLAGS) -o mavupdate ./cmd/mavupdate/ +# mavgpud runs on the workstation, not here. It is built with the rest so a +# broken supervisor is caught by `make build` on homesrv rather than by the +# workstation refusing to serve. Copy the binary over, do not `make deploy` it. +build-gpud: + $(GO) build $(GOFLAGS) -o mavgpud ./cmd/mavgpud/ + run-web: build-web ./mavweb -addr :9200 -voice 127.0.0.1:9100 diff --git a/deploy/mavgpud.json b/deploy/mavgpud.json new file mode 100644 index 0000000..fb33c2f --- /dev/null +++ b/deploy/mavgpud.json @@ -0,0 +1,31 @@ +{ + "listen": ":8080", + "llama_addr": "127.0.0.1:10000", + "llama_bin": "llama-server", + "llama_args": [ + "-m", "/mnt/D/AI/gemma4/gemma-4-12B-it-qat-UD-Q4_K_XL.gguf", + "-md", "/mnt/D/AI/gemma4/mtp-gemma-4-12B-it-BF16.gguf", + "-ngl", "99", + "-fa", "on", + "-np", "1", + "--host", "127.0.0.1", + "--port", "10000", + "--ctx-size", "32768", + "--threads", "6", + "--batch-size", "2048", + "--ubatch-size", "512", + "--jinja", + "--spec-type", "draft-mtp", + "--spec-draft-n-max", "2" + ], + + "kfd_root": "/sys/class/kfd/kfd/proc", + "drm_device": "/sys/class/drm/card1/device", + + "poll": "1s", + "idle_timeout": "15m", + "stop_grace": "20s", + "min_free_vram_bytes": 10737418240, + "evict_after_polls": 2, + "start_after_polls": 5 +} diff --git a/deploy/mavgpud.service b/deploy/mavgpud.service new file mode 100644 index 0000000..b200baa --- /dev/null +++ b/deploy/mavgpud.service @@ -0,0 +1,24 @@ +[Unit] +# Runs on the workstation (bugmachine), not on homesrv. Install as a systemd +# user unit and turn on lingering, so the card is supervised after a reboot +# with nobody logged in: +# +# scp mavgpud workpc:~/.local/bin/mavgpud +# scp deploy/mavgpud.json workpc:~/.config/mavgpud.json +# scp deploy/mavgpud.service workpc:~/.config/systemd/user/mavgpud.service +# ssh workpc 'systemctl --user daemon-reload && systemctl --user enable --now mavgpud' +# sudo loginctl enable-linger kami +Description=Maven GPU supervisor (holds llama-server while the card is free) +After=network.target + +[Service] +ExecStart=%h/.local/bin/mavgpud -config %h/.config/mavgpud.json +Restart=always +RestartSec=5 +# The card must come back when the supervisor goes down. mavgpud stops +# llama-server on SIGTERM, so give it longer than stop_grace to do that. +KillSignal=SIGTERM +TimeoutStopSec=60 + +[Install] +WantedBy=default.target