Score how often a real utterance reaches Hexis/Praxis (ecosystem reach fixture) #160

Merged
claude merged 21 commits from task/405-score-how-often-a-real-utterance-reaches into master 2026-08-04 18:50:49 +02:00
Showing only changes of commit a9067a5754 - Show all commits
+9 -1
View File
@@ -16,7 +16,7 @@ PIPER_BIN := $(shell pwd)/deps/piper/piper
PIPER_MODEL := $(shell pwd)/models/tts/ru_RU-irina-medium.onnx
PIPER_ESPEAK := $(shell pwd)/deps/piper/espeak-ng-data
.PHONY: simulate stt-fixtures test-stt-golden all build build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav clean test fmt-check vet run-stt run-tts run-web download-embedder deps-go eval-router eval-recall eval-phrasing eval-models build-gpud
.PHONY: simulate stt-fixtures test-stt-golden all build build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav clean test fmt-check vet run-stt run-tts run-web download-embedder deps-go eval-router eval-reach eval-recall eval-phrasing eval-models build-gpud
all: build
@@ -119,6 +119,14 @@ MAVEN_ONNX_LIB ?= $(shell pwd)/deps/onnxruntime-linux-x64-1.26.0/lib/libonnxrunt
eval-router:
MAVEN_ONNX_LIB="$(MAVEN_ONNX_LIB)" $(GO) test -v -count=1 ./internal/router/eval/
# eval-reach — score the held-out ecosystem reach fixture (internal/router/eval,
# ru_ecosystem_v1.json). Answers "does a real Russian utterance actually arrive
# at Praxis or Hexis", which routing accuracy alone does not say. Same
# MAVEN_ONNX_LIB deal as eval-router; without it only the deterministic hash
# ratchet runs. Vikunja #405.
eval-reach:
MAVEN_ONNX_LIB="$(MAVEN_ONNX_LIB)" $(GO) test -v -count=1 -run 'Reach|Praxis' ./internal/router/eval/
# eval-recall — score the held-out note-recall fixture (internal/memory/recalleval).
# Answers "can she find the note again when it matters": recall@1, recall@3,
# false recall and the query_min_score sweep. Same MAVEN_ONNX_LIB deal as