diff --git a/cmd/mavwaked/silero_test.go b/cmd/mavwaked/silero_test.go new file mode 100644 index 0000000..25fee35 --- /dev/null +++ b/cmd/mavwaked/silero_test.go @@ -0,0 +1,149 @@ +package main + +// What this measures. The energy threshold cannot tell a voice from a +// television, and every utterance it accepts becomes a turn. So the test that +// matters is not "does silero find speech" — it is "does it decline what the +// energy threshold accepts". +// +// Speech is the four piper fixtures mavsttd already scores against. They are +// synthesised, so nothing of the owner's voice is committed. Non-speech is +// white noise at the same loudness, which is the cheapest thing that fools an +// energy floor and the honest floor for this claim. +// +// Both halves skip without models/vad/silero_vad.onnx and MAVEN_ONNX_LIB, +// like the TestONNX measurements in internal/router/eval. + +import ( + "math" + "math/rand" + "os" + "path/filepath" + "testing" +) + +const wavHeader = 44 // 16kHz mono s16le, written by piper + +func loadSilero(t *testing.T) *sileroVAD { + t.Helper() + model := filepath.Join("..", "..", "models", "vad", "silero_vad.onnx") + lib := os.Getenv("MAVEN_ONNX_LIB") + if _, err := os.Stat(model); err != nil { + t.Skipf("missing %s: %v", model, err) + } + if lib == "" { + t.Skip("MAVEN_ONNX_LIB unset") + } + s, err := newSileroVAD(model, lib) + if err != nil { + t.Skipf("silero unavailable: %v", err) + } + return s +} + +// feedAll runs a whole clip through a VAD and reports how many utterances it +// produced and how many frames it called speech. +func feedAll(v *VAD, pcm []int16) (utterances, speechFrames int) { + for i := 0; i+frameSamples <= len(pcm); i += frameSamples { + frame := pcm[i : i+frameSamples] + utt, state := v.Feed(frame) + if state == StateSpeech { + speechFrames++ + } + if utt.Bytes != nil { + utterances++ + } + } + return utterances, speechFrames +} + +func readFixture(t *testing.T, name string) []int16 { + t.Helper() + raw, err := os.ReadFile(filepath.Join("..", "mavsttd", "testdata", name)) + if err != nil { + t.Skipf("missing fixture %s: %v", name, err) + } + if len(raw) <= wavHeader { + t.Fatalf("%s: %d bytes, no audio", name, len(raw)) + } + return PCMToI16(raw[wavHeader:]) +} + +// noise returns white noise scaled to the same RMS as ref. Same loudness, +// nothing said. +func noise(ref []int16, seed int64) []int16 { + target := frameRMS(ref) + r := rand.New(rand.NewSource(seed)) + out := make([]int16, len(ref)) + for i := range out { + out[i] = int16(r.NormFloat64() * target * 32768.0) + } + return out +} + +func TestSileroHearsSpeechAndDeclinesNoise(t *testing.T) { + s := loadSilero(t) + defer s.Close() + + for _, name := range []string{"ru_fact.wav", "ru_query.wav", "ru_reminder.wav", "en_act.wav"} { + pcm := readFixture(t, name) + + v := NewVAD(0, 0, 0, 0) + v.UseSilero(s, defaultSileroThreshold) + _, spoke := feedAll(v, pcm) + if spoke == 0 { + t.Errorf("%s: silero heard no speech in a spoken clip", name) + } + + s.Reset() + v2 := NewVAD(0, 0, 0, 0) + v2.UseSilero(s, defaultSileroThreshold) + _, heard := feedAll(v2, noise(pcm, 7)) + + energy := NewVAD(0, 0, 0, 0) + _, energyHeard := feedAll(energy, noise(pcm, 7)) + + t.Logf("%s: speech frames — silero on speech %d, silero on noise %d, energy on noise %d", + name, spoke, heard, energyHeard) + if heard >= energyHeard { + t.Errorf("%s: silero called %d noise frames speech, energy called %d — no improvement", + name, heard, energyHeard) + } + s.Reset() + } +} + +// BenchmarkSileroFrame answers the only performance question that matters +// here: one 30ms frame must cost far less than 30ms on one core, or the +// detector cannot run always-on beside everything else on that machine. +func BenchmarkSileroFrame(b *testing.B) { + s := loadSilero(&testing.T{}) + if s == nil { + b.Skip("silero unavailable") + } + defer s.Close() + frame := make([]int16, frameSamples) + for i := range frame { + frame[i] = int16(i%400 - 200) + } + for i := 0; i < b.N; i++ { + s.Speech(frame, defaultSileroThreshold) + } +} + +// TestSileroRechunksAcrossFrames pins the reason this file exists. The capture +// frame is 480 samples and the model window is 512, so a detector that ran one +// inference per frame would be feeding the model a shape it does not accept. +func TestSileroRechunksAcrossFrames(t *testing.T) { + s := loadSilero(t) + defer s.Close() + + silence := make([]int16, frameSamples) + for i := 0; i < 20; i++ { + if _, p := s.Speech(silence, defaultSileroThreshold); math.IsNaN(p) { + t.Fatalf("frame %d: probability is NaN", i) + } + } + if len(s.pending) >= sileroWindow { + t.Errorf("pending grew to %d samples, so windows are not being consumed", len(s.pending)) + } +}