Add golden-audio STT tests against real whisper.cpp (#288)
Four committed WAV fixtures go through the real whisper.cpp binding in cmd/mavsttd, so a wrong model, a wrong language hint, a broken resample or a regressed silence gate fails `make test` instead of surfacing as Maven mishearing him. The fixtures are piper-synthesised, not recorded: scripts/gen-stt-fixtures.sh drives the vendored piper with the ru_RU-irina voice Maven already speaks with, so nothing of the owner's voice is committed and every fixture is reproducible. 360K total for three Russian clips and one English. Matching is tolerant on purpose. Golden transcripts move with the model, so each case asserts intent-carrying keywords (prefix match, so Russian inflection does not fail it) plus a word error rate ceiling, not an exact string. The matcher is unit-tested on its own and needs no model. TestGoldenAudioTranscription skips when models/stt/ggml-small.bin is absent, so `make test` still passes on a box without models. TestGoldenFixturesAreCanonical runs everywhere and checks the WAVs are 16k mono s16le and would clear mavsttd's own silence gate.
This commit is contained in:
@@ -16,7 +16,7 @@ PIPER_BIN := $(shell pwd)/deps/piper/piper
|
||||
PIPER_MODEL := $(shell pwd)/models/tts/ru_RU-irina-medium.onnx
|
||||
PIPER_ESPEAK := $(shell pwd)/deps/piper/espeak-ng-data
|
||||
|
||||
.PHONY: all build build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav clean test fmt-check vet run-stt run-tts run-web download-embedder deps-go eval-router eval-recall eval-phrasing eval-models
|
||||
.PHONY: stt-fixtures test-stt-golden all build build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav clean test fmt-check vet run-stt run-tts run-web download-embedder deps-go eval-router eval-recall eval-phrasing eval-models
|
||||
|
||||
all: build
|
||||
|
||||
@@ -139,6 +139,17 @@ eval-models:
|
||||
MAVEN_LLM_URL="$(MAVEN_LLM_URL)" $(GO) test -v -count=1 -timeout 60m \
|
||||
-run TestLLMRouterBaseline ./internal/router/eval/
|
||||
|
||||
# stt-fixtures — regenerate the golden STT audio in cmd/mavsttd/testdata from
|
||||
# the piper voices (#288). The committed WAVs are synthesised, never recorded,
|
||||
# so this is the only way they should ever change. TestGoldenAudioTranscription
|
||||
# then scores them against ggml-small; it self-skips when the model is absent.
|
||||
stt-fixtures:
|
||||
./scripts/gen-stt-fixtures.sh
|
||||
|
||||
test-stt-golden:
|
||||
CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
|
||||
$(GO) test -v -count=1 -run TestGolden ./cmd/mavsttd/
|
||||
|
||||
run-stt: build-stt
|
||||
LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
|
||||
./mavsttd -socket /tmp/maven/stt.sock -model $(WHISPER_MODEL)
|
||||
|
||||
Reference in New Issue
Block a user