kami
62cc072f8c
Four committed WAV fixtures go through the real whisper.cpp binding in cmd/mavsttd, so a wrong model, a wrong language hint, a broken resample or a regressed silence gate fails `make test` instead of surfacing as Maven mishearing him. The fixtures are piper-synthesised, not recorded: scripts/gen-stt-fixtures.sh drives the vendored piper with the ru_RU-irina voice Maven already speaks with, so nothing of the owner's voice is committed and every fixture is reproducible. 360K total for three Russian clips and one English. Matching is tolerant on purpose. Golden transcripts move with the model, so each case asserts intent-carrying keywords (prefix match, so Russian inflection does not fail it) plus a word error rate ceiling, not an exact string. The matcher is unit-tested on its own and needs no model. TestGoldenAudioTranscription skips when models/stt/ggml-small.bin is absent, so `make test` still passes on a box without models. TestGoldenFixturesAreCanonical runs everywhere and checks the WAVs are 16k mono s16le and would clear mavsttd's own silence gate.
324 lines
9.8 KiB
Go
324 lines
9.8 KiB
Go
package main
|
||
|
||
// Golden-audio STT tests (Vikunja #288).
|
||
//
|
||
// These push real audio through the real whisper.cpp binding, so a bad model
|
||
// path, a wrong language hint, a broken resample or a regressed silence gate
|
||
// is caught by `make test` rather than by the owner talking to a daemon that
|
||
// mishears him.
|
||
//
|
||
// The fixtures are piper-synthesised, not recorded — see
|
||
// scripts/gen-stt-fixtures.sh. Nothing of the owner's voice is committed, and
|
||
// any fixture can be rebuilt from the script plus a voice model.
|
||
//
|
||
// Matching is deliberately tolerant. Golden transcripts are model-dependent:
|
||
// swapping ggml-small for a different whisper build moves punctuation, casing
|
||
// and the odd word ending, and an exact-string assertion would turn every
|
||
// model swap into a fixture rewrite. Each case therefore asserts two things —
|
||
// the words that carry the intent are present, and the word error rate
|
||
// against the reference stays under a per-case ceiling.
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"testing"
|
||
"unicode"
|
||
|
||
"github.com/kami/maven/internal/audio"
|
||
"github.com/kami/maven/internal/worker"
|
||
)
|
||
|
||
// goldenModelPath — the whisper model the golden tests run against. Same file
|
||
// the Makefile's run-stt target uses. Overridable so a box that keeps its
|
||
// models elsewhere can still run these.
|
||
func goldenModelPath() string {
|
||
if p := os.Getenv("MAVEN_WHISPER_MODEL"); p != "" {
|
||
return p
|
||
}
|
||
return filepath.Join("..", "..", "models", "stt", "ggml-small.bin")
|
||
}
|
||
|
||
type goldenCase struct {
|
||
Name string `json:"name"`
|
||
WAV string `json:"wav"`
|
||
Lang string `json:"lang"`
|
||
Text string `json:"text"`
|
||
Keywords []string `json:"keywords"`
|
||
MaxWER float64 `json:"max_wer"`
|
||
}
|
||
|
||
type goldenManifest struct {
|
||
Cases []goldenCase `json:"cases"`
|
||
}
|
||
|
||
func loadGoldenManifest(t *testing.T) goldenManifest {
|
||
t.Helper()
|
||
raw, err := os.ReadFile(filepath.Join("testdata", "golden_v1.json"))
|
||
if err != nil {
|
||
t.Fatalf("read golden manifest: %v", err)
|
||
}
|
||
var m goldenManifest
|
||
if err := json.Unmarshal(raw, &m); err != nil {
|
||
t.Fatalf("parse golden manifest: %v", err)
|
||
}
|
||
if len(m.Cases) == 0 {
|
||
t.Fatal("golden manifest has no cases")
|
||
}
|
||
return m
|
||
}
|
||
|
||
// normalizeTranscript lowercases, drops punctuation, folds the Russian ё onto
|
||
// е (whisper is inconsistent about it and the router does not care), and
|
||
// collapses whitespace. Everything the comparison does happens on this form.
|
||
func normalizeTranscript(s string) []string {
|
||
var b strings.Builder
|
||
for _, r := range strings.ToLower(s) {
|
||
switch {
|
||
case r == 'ё':
|
||
b.WriteRune('е')
|
||
case unicode.IsLetter(r) || unicode.IsDigit(r):
|
||
b.WriteRune(r)
|
||
default:
|
||
b.WriteRune(' ')
|
||
}
|
||
}
|
||
return strings.Fields(b.String())
|
||
}
|
||
|
||
// wordErrorRate is the Levenshtein distance between two word sequences,
|
||
// divided by the length of the reference. 0 means identical; it can exceed 1
|
||
// when the hypothesis is much longer than the reference.
|
||
func wordErrorRate(ref, hyp []string) float64 {
|
||
if len(ref) == 0 {
|
||
if len(hyp) == 0 {
|
||
return 0
|
||
}
|
||
return 1
|
||
}
|
||
prev := make([]int, len(hyp)+1)
|
||
cur := make([]int, len(hyp)+1)
|
||
for j := range prev {
|
||
prev[j] = j
|
||
}
|
||
for i := 1; i <= len(ref); i++ {
|
||
cur[0] = i
|
||
for j := 1; j <= len(hyp); j++ {
|
||
cost := 1
|
||
if ref[i-1] == hyp[j-1] {
|
||
cost = 0
|
||
}
|
||
cur[j] = min(prev[j]+1, min(cur[j-1]+1, prev[j-1]+cost))
|
||
}
|
||
prev, cur = cur, prev
|
||
}
|
||
return float64(prev[len(hyp)]) / float64(len(ref))
|
||
}
|
||
|
||
// missingKeywords returns the keywords absent from the hypothesis. A keyword
|
||
// matches on prefix, so a different case ending ("воды" vs "воду") does not
|
||
// fail the assertion — the router's stage-0 grammar is stem-shaped too.
|
||
func missingKeywords(keywords []string, hyp []string) []string {
|
||
var missing []string
|
||
for _, kw := range keywords {
|
||
want := normalizeTranscript(kw)
|
||
if len(want) == 0 {
|
||
continue
|
||
}
|
||
if !containsSeq(hyp, want) {
|
||
missing = append(missing, kw)
|
||
}
|
||
}
|
||
return missing
|
||
}
|
||
|
||
func containsSeq(hyp, want []string) bool {
|
||
for i := 0; i+len(want) <= len(hyp); i++ {
|
||
ok := true
|
||
for j, w := range want {
|
||
// Prefix match, so inflection differences pass but
|
||
// distinct words do not.
|
||
if !looseWordMatch(hyp[i+j], w) {
|
||
ok = false
|
||
break
|
||
}
|
||
}
|
||
if ok {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
func looseWordMatch(got, want string) bool {
|
||
if got == want {
|
||
return true
|
||
}
|
||
g, w := []rune(got), []rune(want)
|
||
n := len(w) - 1
|
||
if len(w) > 6 {
|
||
n = len(w) - 2
|
||
}
|
||
// Words of three runes or fewer have no room for a safe prefix: require
|
||
// an exact match rather than letting "час" pass for "часть".
|
||
if n < 3 || len(g) < n {
|
||
return false
|
||
}
|
||
return string(g[:n]) == string(w[:n])
|
||
}
|
||
|
||
// --- the model-backed test -------------------------------------------------
|
||
|
||
func TestGoldenAudioTranscription(t *testing.T) {
|
||
m := loadGoldenManifest(t)
|
||
|
||
model := goldenModelPath()
|
||
if _, err := os.Stat(model); err != nil {
|
||
t.Skipf("whisper model %s absent (%v) — set MAVEN_WHISPER_MODEL or see AGENTS.md", model, err)
|
||
}
|
||
|
||
// Same gate thresholds as mavsttd's defaults, so a regression in the
|
||
// silence gate shows up here as an empty transcript.
|
||
h, err := newWhisperHandler(model, 300, 0.01)
|
||
if err != nil {
|
||
t.Fatalf("load whisper model %s: %v", model, err)
|
||
}
|
||
defer h.Close()
|
||
|
||
for _, c := range m.Cases {
|
||
t.Run(c.Name, func(t *testing.T) {
|
||
path := filepath.Join("testdata", c.WAV)
|
||
raw, err := os.ReadFile(path)
|
||
if err != nil {
|
||
t.Skipf("fixture %s absent (%v) — run scripts/gen-stt-fixtures.sh", path, err)
|
||
}
|
||
format, pcm, err := audio.PCMFromWAV(raw)
|
||
if err != nil {
|
||
t.Fatalf("%s is not canonical 16k mono PCM: %v", path, err)
|
||
}
|
||
|
||
resp, err := h.Transcribe(context.Background(), worker.TranscribeReq{
|
||
Audio: audio.Audio{Format: format, Bytes: pcm},
|
||
Lang: c.Lang,
|
||
})
|
||
if err != nil {
|
||
t.Fatalf("transcribe %s: %v", c.WAV, err)
|
||
}
|
||
t.Logf("%s → %q (confidence %.3f)", c.WAV, resp.Text, resp.Confidence)
|
||
|
||
if strings.TrimSpace(resp.Text) == "" {
|
||
t.Fatalf("%s transcribed to empty text — the silence gate ate real speech", c.WAV)
|
||
}
|
||
if resp.Confidence <= 0 {
|
||
t.Errorf("%s: confidence %v, want > 0", c.WAV, resp.Confidence)
|
||
}
|
||
|
||
hyp := normalizeTranscript(resp.Text)
|
||
ref := normalizeTranscript(c.Text)
|
||
|
||
if missing := missingKeywords(c.Keywords, hyp); len(missing) > 0 {
|
||
t.Errorf("%s: missing keywords %v in %q", c.WAV, missing, resp.Text)
|
||
}
|
||
if wer := wordErrorRate(ref, hyp); wer > c.MaxWER {
|
||
t.Errorf("%s: WER %.2f > %.2f\n want: %q\n got: %q", c.WAV, wer, c.MaxWER, c.Text, resp.Text)
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
// TestGoldenFixturesAreCanonical checks the committed audio without needing a
|
||
// model, so a fixture regenerated at the wrong sample rate fails on every box.
|
||
func TestGoldenFixturesAreCanonical(t *testing.T) {
|
||
m := loadGoldenManifest(t)
|
||
for _, c := range m.Cases {
|
||
path := filepath.Join("testdata", c.WAV)
|
||
raw, err := os.ReadFile(path)
|
||
if err != nil {
|
||
t.Errorf("fixture %s missing: %v", path, err)
|
||
continue
|
||
}
|
||
format, pcm, err := audio.PCMFromWAV(raw)
|
||
if err != nil {
|
||
t.Errorf("%s: %v", path, err)
|
||
continue
|
||
}
|
||
if !format.IsValid() {
|
||
t.Errorf("%s: format %+v is not canonical", path, format)
|
||
}
|
||
a := audio.Audio{Format: format, Bytes: pcm}
|
||
if d := a.Duration(); d < 0.5 || d > 10 {
|
||
t.Errorf("%s: duration %.2fs outside the sane 0.5–10s fixture range", path, d)
|
||
}
|
||
// The fixture must clear mavsttd's own silence gate, otherwise the
|
||
// model test below would be asserting on a gated empty string.
|
||
if reason := gateReason(pcmToF32(pcm), whisperSampleRate, 300, 0.01); reason != "" {
|
||
t.Errorf("%s: would be gated as %s", path, reason)
|
||
}
|
||
if len(c.Keywords) == 0 {
|
||
t.Errorf("%s: manifest case has no keywords", c.Name)
|
||
}
|
||
if c.MaxWER <= 0 || c.MaxWER > 1 {
|
||
t.Errorf("%s: max_wer %v outside (0,1]", c.Name, c.MaxWER)
|
||
}
|
||
}
|
||
}
|
||
|
||
func pcmToF32(b []byte) []float32 {
|
||
out := make([]float32, len(b)/2)
|
||
for i := range out {
|
||
s := int16(b[i*2]) | int16(b[i*2+1])<<8
|
||
out[i] = float32(s) / 32768.0
|
||
}
|
||
return out
|
||
}
|
||
|
||
// --- matcher unit tests (no model, no fixtures) ----------------------------
|
||
|
||
func TestNormalizeTranscript(t *testing.T) {
|
||
got := normalizeTranscript(" Ещё, Раз... ")
|
||
want := []string{"еще", "раз"}
|
||
if len(got) != len(want) || got[0] != want[0] || got[1] != want[1] {
|
||
t.Fatalf("normalizeTranscript = %v, want %v", got, want)
|
||
}
|
||
}
|
||
|
||
func TestWordErrorRate(t *testing.T) {
|
||
cases := []struct {
|
||
name string
|
||
ref, hyp string
|
||
want float64
|
||
}{
|
||
{"identical", "напомни мне через час", "Напомни мне через час.", 0},
|
||
{"one substitution", "напомни мне через час", "напомни мне через день", 0.25},
|
||
{"one deletion", "напомни мне через час", "напомни мне час", 0.25},
|
||
{"empty hypothesis", "напомни мне", "", 1},
|
||
{"both empty", "", "", 0},
|
||
}
|
||
for _, c := range cases {
|
||
t.Run(c.name, func(t *testing.T) {
|
||
got := wordErrorRate(normalizeTranscript(c.ref), normalizeTranscript(c.hyp))
|
||
if got != c.want {
|
||
t.Fatalf("WER = %v, want %v", got, c.want)
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
func TestMissingKeywords(t *testing.T) {
|
||
hyp := normalizeTranscript("Отметь, что я выпил воду.")
|
||
if got := missingKeywords([]string{"воды", "отметь"}, hyp); len(got) != 0 {
|
||
t.Fatalf("missingKeywords = %v, want none (inflection must not fail the match)", got)
|
||
}
|
||
if got := missingKeywords([]string{"календарю"}, hyp); len(got) != 1 {
|
||
t.Fatalf("missingKeywords = %v, want the absent keyword reported", got)
|
||
}
|
||
// A short word must match exactly — no 4-rune prefix shortcut that would
|
||
// let "час" pass for "часть".
|
||
hyp2 := normalizeTranscript("через час")
|
||
if got := missingKeywords([]string{"часть"}, hyp2); len(got) != 1 {
|
||
t.Fatalf("missingKeywords = %v, want %q reported missing", got, "часть")
|
||
}
|
||
}
|