make: add eval-reach (V-405)
Scores the ecosystem reach fixture. Same MAVEN_ONNX_LIB deal as eval-router: without it only the deterministic hash ratchet runs.
This commit is contained in:
@@ -16,7 +16,7 @@ PIPER_BIN := $(shell pwd)/deps/piper/piper
|
||||
PIPER_MODEL := $(shell pwd)/models/tts/ru_RU-irina-medium.onnx
|
||||
PIPER_ESPEAK := $(shell pwd)/deps/piper/espeak-ng-data
|
||||
|
||||
.PHONY: simulate stt-fixtures test-stt-golden all build build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav clean test fmt-check vet run-stt run-tts run-web download-embedder deps-go eval-router eval-recall eval-phrasing eval-models build-gpud
|
||||
.PHONY: simulate stt-fixtures test-stt-golden all build build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav clean test fmt-check vet run-stt run-tts run-web download-embedder deps-go eval-router eval-reach eval-recall eval-phrasing eval-models build-gpud
|
||||
|
||||
all: build
|
||||
|
||||
@@ -119,6 +119,14 @@ MAVEN_ONNX_LIB ?= $(shell pwd)/deps/onnxruntime-linux-x64-1.26.0/lib/libonnxrunt
|
||||
eval-router:
|
||||
MAVEN_ONNX_LIB="$(MAVEN_ONNX_LIB)" $(GO) test -v -count=1 ./internal/router/eval/
|
||||
|
||||
# eval-reach — score the held-out ecosystem reach fixture (internal/router/eval,
|
||||
# ru_ecosystem_v1.json). Answers "does a real Russian utterance actually arrive
|
||||
# at Praxis or Hexis", which routing accuracy alone does not say. Same
|
||||
# MAVEN_ONNX_LIB deal as eval-router; without it only the deterministic hash
|
||||
# ratchet runs. Vikunja #405.
|
||||
eval-reach:
|
||||
MAVEN_ONNX_LIB="$(MAVEN_ONNX_LIB)" $(GO) test -v -count=1 -run 'Reach|Praxis' ./internal/router/eval/
|
||||
|
||||
# eval-recall — score the held-out note-recall fixture (internal/memory/recalleval).
|
||||
# Answers "can she find the note again when it matters": recall@1, recall@3,
|
||||
# false recall and the query_min_score sweep. Same MAVEN_ONNX_LIB deal as
|
||||
|
||||
Reference in New Issue
Block a user