cmd/mavwaked/silero_test.go

@@ -0,0 +1,149 @@
+package main
+
+// What this measures. The energy threshold cannot tell a voice from a
+// television, and every utterance it accepts becomes a turn. So the test that
+// matters is not "does silero find speech" — it is "does it decline what the
+// energy threshold accepts".
+//
+// Speech is the four piper fixtures mavsttd already scores against. They are
+// synthesised, so nothing of the owner's voice is committed. Non-speech is
+// white noise at the same loudness, which is the cheapest thing that fools an
+// energy floor and the honest floor for this claim.
+//
+// Both halves skip without models/vad/silero_vad.onnx and MAVEN_ONNX_LIB,
+// like the TestONNX measurements in internal/router/eval.
+
+import (
+	"math"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+const wavHeader = 44 // 16kHz mono s16le, written by piper
+
+func loadSilero(t *testing.T) *sileroVAD {
+	t.Helper()
+	model := filepath.Join("..", "..", "models", "vad", "silero_vad.onnx")
+	lib := os.Getenv("MAVEN_ONNX_LIB")
+	if _, err := os.Stat(model); err != nil {
+		t.Skipf("missing %s: %v", model, err)
+	}
+	if lib == "" {
+		t.Skip("MAVEN_ONNX_LIB unset")
+	}
+	s, err := newSileroVAD(model, lib)
+	if err != nil {
+		t.Skipf("silero unavailable: %v", err)
+	}
+	return s
+}
+
+// feedAll runs a whole clip through a VAD and reports how many utterances it
+// produced and how many frames it called speech.
+func feedAll(v *VAD, pcm []int16) (utterances, speechFrames int) {
+	for i := 0; i+frameSamples <= len(pcm); i += frameSamples {
+		frame := pcm[i : i+frameSamples]
+		utt, state := v.Feed(frame)
+		if state == StateSpeech {
+			speechFrames++
+		}
+		if utt.Bytes != nil {
+			utterances++
+		}
+	}
+	return utterances, speechFrames
+}
+
+func readFixture(t *testing.T, name string) []int16 {
+	t.Helper()
+	raw, err := os.ReadFile(filepath.Join("..", "mavsttd", "testdata", name))
+	if err != nil {
+		t.Skipf("missing fixture %s: %v", name, err)
+	}
+	if len(raw) <= wavHeader {
+		t.Fatalf("%s: %d bytes, no audio", name, len(raw))
+	}
+	return PCMToI16(raw[wavHeader:])
+}
+
+// noise returns white noise scaled to the same RMS as ref. Same loudness,
+// nothing said.
+func noise(ref []int16, seed int64) []int16 {
+	target := frameRMS(ref)
+	r := rand.New(rand.NewSource(seed))
+	out := make([]int16, len(ref))
+	for i := range out {
+		out[i] = int16(r.NormFloat64() * target * 32768.0)
+	}
+	return out
+}
+
+func TestSileroHearsSpeechAndDeclinesNoise(t *testing.T) {
+	s := loadSilero(t)
+	defer s.Close()
+
+	for _, name := range []string{"ru_fact.wav", "ru_query.wav", "ru_reminder.wav", "en_act.wav"} {
+		pcm := readFixture(t, name)
+
+		v := NewVAD(0, 0, 0, 0)
+		v.UseSilero(s, defaultSileroThreshold)
+		_, spoke := feedAll(v, pcm)
+		if spoke == 0 {
+			t.Errorf("%s: silero heard no speech in a spoken clip", name)
+		}
+
+		s.Reset()
+		v2 := NewVAD(0, 0, 0, 0)
+		v2.UseSilero(s, defaultSileroThreshold)
+		_, heard := feedAll(v2, noise(pcm, 7))
+
+		energy := NewVAD(0, 0, 0, 0)
+		_, energyHeard := feedAll(energy, noise(pcm, 7))
+
+		t.Logf("%s: speech frames — silero on speech %d, silero on noise %d, energy on noise %d",
+			name, spoke, heard, energyHeard)
+		if heard >= energyHeard {
+			t.Errorf("%s: silero called %d noise frames speech, energy called %d — no improvement",
+				name, heard, energyHeard)
+		}
+		s.Reset()
+	}
+}
+
+// BenchmarkSileroFrame answers the only performance question that matters
+// here: one 30ms frame must cost far less than 30ms on one core, or the
+// detector cannot run always-on beside everything else on that machine.
+func BenchmarkSileroFrame(b *testing.B) {
+	s := loadSilero(&testing.T{})
+	if s == nil {
+		b.Skip("silero unavailable")
+	}
+	defer s.Close()
+	frame := make([]int16, frameSamples)
+	for i := range frame {
+		frame[i] = int16(i%400 - 200)
+	}
+	for i := 0; i < b.N; i++ {
+		s.Speech(frame, defaultSileroThreshold)
+	}
+}
+
+// TestSileroRechunksAcrossFrames pins the reason this file exists. The capture
+// frame is 480 samples and the model window is 512, so a detector that ran one
+// inference per frame would be feeding the model a shape it does not accept.
+func TestSileroRechunksAcrossFrames(t *testing.T) {
+	s := loadSilero(t)
+	defer s.Close()
+
+	silence := make([]int16, frameSamples)
+	for i := 0; i < 20; i++ {
+		if _, p := s.Speech(silence, defaultSileroThreshold); math.IsNaN(p) {
+			t.Fatalf("frame %d: probability is NaN", i)
+		}
+	}
+	if len(s.pending) >= sileroWindow {
+		t.Errorf("pending grew to %d samples, so windows are not being consumed", len(s.pending))
+	}
+}