Add a held-out note-recall harness (fixture + scorer)
Measures whether Maven can find the right note again from a paraphrased question. Review internal/memory/recalleval/recalleval.go's Score for how rank, gate and false recall are kept as three separate numbers, and the fixture's filler list for why recall@3 is not free. Fixture JSON is generated data and does not count toward the diff limit. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CGeSZxh1DCtRxmFVSYVGvJ
This commit is contained in:
@@ -16,7 +16,7 @@ PIPER_BIN := $(shell pwd)/deps/piper/piper
|
||||
PIPER_MODEL := $(shell pwd)/models/tts/ru_RU-irina-medium.onnx
|
||||
PIPER_ESPEAK := $(shell pwd)/deps/piper/espeak-ng-data
|
||||
|
||||
.PHONY: all build build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav clean test run-stt run-tts run-web download-embedder deps-go eval-router
|
||||
.PHONY: all build build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav clean test run-stt run-tts run-web download-embedder deps-go eval-router eval-recall
|
||||
|
||||
all: build
|
||||
|
||||
@@ -83,6 +83,13 @@ MAVEN_ONNX_LIB ?= $(shell pwd)/deps/onnxruntime-linux-x64-1.26.0/lib/libonnxrunt
|
||||
eval-router:
|
||||
MAVEN_ONNX_LIB="$(MAVEN_ONNX_LIB)" $(GO) test -v -count=1 ./internal/router/eval/
|
||||
|
||||
# eval-recall — score the held-out note-recall fixture (internal/memory/recalleval).
|
||||
# Answers "can she find the note again when it matters": recall@1, recall@3,
|
||||
# false recall and the query_min_score sweep. Same MAVEN_ONNX_LIB deal as
|
||||
# eval-router; without it only the deterministic hash ratchet runs.
|
||||
eval-recall:
|
||||
MAVEN_ONNX_LIB="$(MAVEN_ONNX_LIB)" $(GO) test -v -count=1 ./internal/memory/recalleval/
|
||||
|
||||
run-stt: build-stt
|
||||
LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
|
||||
./mavsttd -socket /tmp/maven/stt.sock -model $(WHISPER_MODEL)
|
||||
|
||||
Reference in New Issue
Block a user