Label LLM eval runs with the model llama-server has loaded

The bake-off in #278/#250 needs two models' scores side by side, and the
report names only carried the config, so the rows were indistinguishable.
ModelID reads /v1/models instead of taking a string that goes stale.
New target: make eval-models MAVEN_LLM_URL=...

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CGeSZxh1DCtRxmFVSYVGvJ
This commit is contained in:
kami
2026-07-31 02:17:20 +04:00
parent abe9b28719
commit 94eb92fb15
3 changed files with 83 additions and 5 deletions
+16 -1
View File
@@ -16,7 +16,7 @@ PIPER_BIN := $(shell pwd)/deps/piper/piper
PIPER_MODEL := $(shell pwd)/models/tts/ru_RU-irina-medium.onnx
PIPER_ESPEAK := $(shell pwd)/deps/piper/espeak-ng-data
.PHONY: all build build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav clean test run-stt run-tts run-web download-embedder deps-go eval-router
.PHONY: all build build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav clean test run-stt run-tts run-web download-embedder deps-go eval-router eval-models
all: build
@@ -83,6 +83,21 @@ MAVEN_ONNX_LIB ?= $(shell pwd)/deps/onnxruntime-linux-x64-1.26.0/lib/libonnxrunt
eval-router:
MAVEN_ONNX_LIB="$(MAVEN_ONNX_LIB)" $(GO) test -v -count=1 ./internal/router/eval/
# eval-models — score ONE llama-server against the same fixture, for the
# resident-model bake-off (#278, #250). Start a server with the gguf you want,
# then:
#
# make eval-models MAVEN_LLM_URL=http://127.0.0.1:18100
#
# The report names carry the model llama-server reports, so runs from two
# checkpoints stay apart. Only the LLM test runs — the classifier baselines do
# not depend on the model and take the ONNX runtime with them.
MAVEN_LLM_URL ?= http://127.0.0.1:18099
eval-models:
MAVEN_LLM_URL="$(MAVEN_LLM_URL)" $(GO) test -v -count=1 -timeout 60m \
-run TestLLMRouterBaseline ./internal/router/eval/
run-stt: build-stt
LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
./mavsttd -socket /tmp/maven/stt.sock -model $(WHISPER_MODEL)