Maven/cmd/mavsttd/golden_test.go

package main

// Golden-audio STT tests (Vikunja #288).
//
// These push real audio through the real whisper.cpp binding. What that
// covers, precisely, is two things: the model still transcribes known speech
// well enough for the router to act on it, and the silence gate still lets real
// speech through. A regression in either shows up in `make test` rather than in
// the owner talking to a daemon that mishears him.
//
// It is worth being exact about what is NOT covered, because this comment used
// to claim more. Nothing here resamples: audio.PCMFromWAV refuses anything that
// is not 16 kHz mono s16, the fixtures arrive at 16 kHz from ffmpeg, and there
// is no conversion step between the WAV and whisper_full. Nothing here
// exercises language selection either: the hint comes out of the manifest
// already correct and goes straight into the request, so how mavsttd chooses a
// language is untested. And a wrong model path is not caught when it is the
// default one, because a box without the model skips; an explicitly set
// MAVEN_WHISPER_MODEL that does not exist is a failure, since that is a
// mistake and not an absence.
//
// The fixtures are piper-synthesised, not recorded — see
// scripts/gen-stt-fixtures.sh. Nothing of the owner's voice is committed, and
// any fixture can be rebuilt from the script plus a voice model.
//
// Matching is deliberately tolerant. Golden transcripts are model-dependent:
// swapping ggml-small for a different whisper build moves punctuation, casing
// and the odd word ending, and an exact-string assertion would turn every
// model swap into a fixture rewrite. Each case therefore asserts two things —
// the words that carry the intent are present, and the word error rate
// against the reference stays under a per-case ceiling.

import (
	"context"
	"encoding/json"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"unicode"

	"github.com/kami/maven/internal/audio"
	"github.com/kami/maven/internal/worker"
)

// goldenModelPath — the whisper model the golden tests run against. Same file
// the Makefile's run-stt target uses. Overridable so a box that keeps its
// models elsewhere can still run these.
func goldenModelPath() string {
	if p := os.Getenv("MAVEN_WHISPER_MODEL"); p != "" {
		return p
	}
	return filepath.Join("..", "..", "models", "stt", "ggml-small.bin")
}

type goldenCase struct {
	Name     string   `json:"name"`
	WAV      string   `json:"wav"`
	Lang     string   `json:"lang"`
	Text     string   `json:"text"`
	Keywords []string `json:"keywords"`
	// MeasuredWER is what this case scored when the ceiling was last set, so
	// a model swap is a diff to a recorded number rather than silence.
	MeasuredWER float64 `json:"measured_wer"`
	MaxWER      float64 `json:"max_wer"`
}

type goldenManifest struct {
	Cases []goldenCase `json:"cases"`
}

func loadGoldenManifest(t *testing.T) goldenManifest {
	t.Helper()
	raw, err := os.ReadFile(filepath.Join("testdata", "golden_v1.json"))
	if err != nil {
		t.Fatalf("read golden manifest: %v", err)
	}
	var m goldenManifest
	if err := json.Unmarshal(raw, &m); err != nil {
		t.Fatalf("parse golden manifest: %v", err)
	}
	if len(m.Cases) == 0 {
		t.Fatal("golden manifest has no cases")
	}
	return m
}

// normalizeTranscript lowercases, drops punctuation, folds the Russian ё onto
// е (whisper is inconsistent about it and the router does not care), and
// collapses whitespace. Everything the comparison does happens on this form.
func normalizeTranscript(s string) []string {
	var b strings.Builder
	for _, r := range strings.ToLower(s) {
		switch {
		case r == 'ё':
			b.WriteRune('е')
		case unicode.IsLetter(r) || unicode.IsDigit(r):
			b.WriteRune(r)
		default:
			b.WriteRune(' ')
		}
	}
	return strings.Fields(b.String())
}

// wordErrorRate is the Levenshtein distance between two word sequences,
// divided by the length of the reference. 0 means identical; it can exceed 1
// when the hypothesis is much longer than the reference.
func wordErrorRate(ref, hyp []string) float64 {
	if len(ref) == 0 {
		if len(hyp) == 0 {
			return 0
		}
		return 1
	}
	prev := make([]int, len(hyp)+1)
	cur := make([]int, len(hyp)+1)
	for j := range prev {
		prev[j] = j
	}
	for i := 1; i <= len(ref); i++ {
		cur[0] = i
		for j := 1; j <= len(hyp); j++ {
			cost := 1
			if ref[i-1] == hyp[j-1] {
				cost = 0
			}
			cur[j] = min(prev[j]+1, min(cur[j-1]+1, prev[j-1]+cost))
		}
		prev, cur = cur, prev
	}
	return float64(prev[len(hyp)]) / float64(len(ref))
}

// missingKeywords returns the keywords absent from the hypothesis. A keyword
// matches on prefix, so a different case ending ("воды" vs "воду") does not
// fail the assertion — the router's stage-0 grammar is stem-shaped too.
func missingKeywords(keywords []string, hyp []string) []string {
	var missing []string
	for _, kw := range keywords {
		want := normalizeTranscript(kw)
		if len(want) == 0 {
			continue
		}
		if !containsSeq(hyp, want) {
			missing = append(missing, kw)
		}
	}
	return missing
}

func containsSeq(hyp, want []string) bool {
	for i := 0; i+len(want) <= len(hyp); i++ {
		ok := true
		for j, w := range want {
			// Prefix match, so inflection differences pass but
			// distinct words do not.
			if !looseWordMatch(hyp[i+j], w) {
				ok = false
				break
			}
		}
		if ok {
			return true
		}
	}
	return false
}

// looseWordMatch reports whether got is want, or an inflection of it.
//
// A shared prefix alone is not enough. "воды" retains three runes, so "водка"
// used to satisfy the ru_fact keyword and the test passed on whisper hearing
// "выпил водки". "disk" retains "dis", which "display", "distance" and
// "discuss" all match. So the hypothesis is also capped in length: a case
// ending adds a rune or two, it does not add a syllable. Short words get no
// slack at all, because there is nothing left of them after a prefix cut.
func looseWordMatch(got, want string) bool {
	if got == want {
		return true
	}
	g, w := []rune(got), []rune(want)
	n := len(w) - 1
	if len(w) > 6 {
		n = len(w) - 2
	}
	// Words of three runes or fewer have no room for a safe prefix: require
	// an exact match rather than letting "час" pass for "часть".
	if n < 3 || len(g) < n {
		return false
	}
	extra := 2
	if len(w) <= 4 {
		extra = 0
	}
	if len(g) > len(w)+extra {
		return false
	}
	return string(g[:n]) == string(w[:n])
}

// --- the model-backed test -------------------------------------------------

func TestGoldenAudioTranscription(t *testing.T) {
	m := loadGoldenManifest(t)

	model := goldenModelPath()
	if _, err := os.Stat(model); err != nil {
		// An explicit override that points at nothing is a mistake, not a box
		// without the model. Skipping there made a typo look like a pass.
		if os.Getenv("MAVEN_WHISPER_MODEL") != "" {
			t.Fatalf("MAVEN_WHISPER_MODEL=%s does not exist: %v", model, err)
		}
		t.Skipf("whisper model %s absent (%v) — set MAVEN_WHISPER_MODEL or see AGENTS.md", model, err)
	}

	// Same gate thresholds as mavsttd's defaults, so a regression in the
	// silence gate shows up here as an empty transcript.
	h, err := newWhisperHandler(model, 300, 0.01)
	if err != nil {
		t.Fatalf("load whisper model %s: %v", model, err)
	}
	defer h.Close()

	for _, c := range m.Cases {
		t.Run(c.Name, func(t *testing.T) {
			path := filepath.Join("testdata", c.WAV)
			raw, err := os.ReadFile(path)
			if err != nil {
				// Not a skip. A fixture the generator failed to write is a
				// broken checkout, and skipping made `make test` green on one.
				t.Fatalf("fixture %s absent (%v) — run scripts/gen-stt-fixtures.sh", path, err)
			}
			format, pcm, err := audio.PCMFromWAV(raw)
			if err != nil {
				t.Fatalf("%s is not canonical 16k mono PCM: %v", path, err)
			}

			resp, err := h.Transcribe(context.Background(), worker.TranscribeReq{
				Audio: audio.Audio{Format: format, Bytes: pcm},
				Lang:  c.Lang,
			})
			if err != nil {
				t.Fatalf("transcribe %s: %v", c.WAV, err)
			}
			t.Logf("%s → %q (confidence %.3f)", c.WAV, resp.Text, resp.Confidence)

			if strings.TrimSpace(resp.Text) == "" {
				t.Fatalf("%s transcribed to empty text — the silence gate ate real speech", c.WAV)
			}
			if resp.Confidence <= 0 {
				t.Errorf("%s: confidence %v, want > 0", c.WAV, resp.Confidence)
			}

			hyp := normalizeTranscript(resp.Text)
			ref := normalizeTranscript(c.Text)

			if missing := missingKeywords(c.Keywords, hyp); len(missing) > 0 {
				t.Errorf("%s: missing keywords %v in %q", c.WAV, missing, resp.Text)
			}
			wer := wordErrorRate(ref, hyp)
			if wer > c.MaxWER {
				t.Errorf("%s: WER %.2f > %.2f (measured %.2f when the ceiling was set)\n  want: %q\n  got:  %q",
					c.WAV, wer, c.MaxWER, c.MeasuredWER, c.Text, resp.Text)
			}
			t.Logf("%s: WER %.2f (ceiling %.2f, was %.2f)", c.WAV, wer, c.MaxWER, c.MeasuredWER)
		})
	}
}

// TestGoldenFixturesAreCanonical checks the committed audio without needing a
// model, so a fixture regenerated at the wrong sample rate fails on every box.
func TestGoldenFixturesAreCanonical(t *testing.T) {
	m := loadGoldenManifest(t)
	for _, c := range m.Cases {
		path := filepath.Join("testdata", c.WAV)
		raw, err := os.ReadFile(path)
		if err != nil {
			t.Errorf("fixture %s missing: %v", path, err)
			continue
		}
		format, pcm, err := audio.PCMFromWAV(raw)
		if err != nil {
			t.Errorf("%s: %v", path, err)
			continue
		}
		if !format.IsValid() {
			t.Errorf("%s: format %+v is not canonical", path, format)
		}
		a := audio.Audio{Format: format, Bytes: pcm}
		if d := a.Duration(); d < 0.5 || d > 10 {
			t.Errorf("%s: duration %.2fs outside the sane 0.5–10s fixture range", path, d)
		}
		// The fixture must clear mavsttd's own silence gate, otherwise the
		// model test below would be asserting on a gated empty string.
		if reason := gateReason(pcmSamples(pcm), whisperSampleRate, 300, 0.01); reason != "" {
			t.Errorf("%s: would be gated as %s", path, reason)
		}
		if len(c.Keywords) == 0 {
			t.Errorf("%s: manifest case has no keywords", c.Name)
		}
		// An empty reference makes wordErrorRate return 1 for every
		// hypothesis, so the WER assertion fires with nothing useful to say.
		if len(normalizeTranscript(c.Text)) == 0 {
			t.Errorf("%s: manifest case has no reference text", c.Name)
		}
		if c.Lang != "ru" && c.Lang != "en" {
			t.Errorf("%s: lang %q is not one of the two languages mavsttd is run with", c.Name, c.Lang)
		}
		if c.MaxWER <= 0 || c.MaxWER > 1 {
			t.Errorf("%s: max_wer %v outside (0,1]", c.Name, c.MaxWER)
		}
		if c.MeasuredWER > c.MaxWER {
			t.Errorf("%s: measured_wer %v is above max_wer %v, so the ceiling was never met", c.Name, c.MeasuredWER, c.MaxWER)
		}
	}
}

// --- matcher unit tests (no model, no fixtures) ----------------------------

func TestNormalizeTranscript(t *testing.T) {
	got := normalizeTranscript(" Ещё,  Раз... ")
	want := []string{"еще", "раз"}
	if len(got) != len(want) || got[0] != want[0] || got[1] != want[1] {
		t.Fatalf("normalizeTranscript = %v, want %v", got, want)
	}
}

func TestWordErrorRate(t *testing.T) {
	cases := []struct {
		name     string
		ref, hyp string
		want     float64
	}{
		{"identical", "напомни мне через час", "Напомни мне через час.", 0},
		{"one substitution", "напомни мне через час", "напомни мне через день", 0.25},
		{"one deletion", "напомни мне через час", "напомни мне час", 0.25},
		{"empty hypothesis", "напомни мне", "", 1},
		{"both empty", "", "", 0},
	}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			got := wordErrorRate(normalizeTranscript(c.ref), normalizeTranscript(c.hyp))
			if got != c.want {
				t.Fatalf("WER = %v, want %v", got, c.want)
			}
		})
	}
}

func TestMissingKeywords(t *testing.T) {
	hyp := normalizeTranscript("Отметь, что я выпил воду.")
	if got := missingKeywords([]string{"воды", "отметь"}, hyp); len(got) != 0 {
		t.Fatalf("missingKeywords = %v, want none (inflection must not fail the match)", got)
	}
	if got := missingKeywords([]string{"календарю"}, hyp); len(got) != 1 {
		t.Fatalf("missingKeywords = %v, want the absent keyword reported", got)
	}
	// A short word must match exactly — no 4-rune prefix shortcut that would
	// let "час" pass for "часть".
	hyp2 := normalizeTranscript("через час")
	if got := missingKeywords([]string{"часть"}, hyp2); len(got) != 1 {
		t.Fatalf("missingKeywords = %v, want %q reported missing", got, "часть")
	}
	// A prefix is not a word. These are different words that share one, and
	// each of them used to satisfy the keyword it is paired with.
	different := [][2]string{
		{"воды", "Я выпил водки."},
		{"disk", "check the display"},
		{"disk", "we should discuss it"},
		{"server", "a serverless function"},
	}
	for _, d := range different {
		if got := missingKeywords([]string{d[0]}, normalizeTranscript(d[1])); len(got) != 1 {
			t.Errorf("keyword %q was satisfied by %q", d[0], d[1])
		}
	}
	// And the inflections still pass, which is the whole point of the loose
	// match.
	same := [][2]string{
		{"воды", "выпил воду"},
		{"напомни", "напомните мне"},
		{"календарю", "по календаре"},
		{"restart", "restarted the server"},
	}
	for _, d := range same {
		if got := missingKeywords([]string{d[0]}, normalizeTranscript(d[1])); len(got) != 0 {
			t.Errorf("keyword %q was not matched by %q", d[0], d[1])
		}
	}
}