diff --git a/.gitignore b/.gitignore index d240800..30023b1 100644 --- a/.gitignore +++ b/.gitignore @@ -70,3 +70,5 @@ coverage.out # root .env — MAVEN_AMBIENT_TOKEN and friends, same class as deploy/telegram.env .env +# silero-vad, downloaded (see AGENTS.md) +/models/vad/ diff --git a/AGENTS.md b/AGENTS.md index 1901f9c..bccb2b3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -95,6 +95,22 @@ model: the code puts `query: ` in front of a question and `passage: ` in front of a stored note, which is how e5 was trained. The quantized file is the one that is downloaded, deployed and measured. +## Voice activity model for mavwaked + +`mavwaked` decides an utterance has started with silero-vad when `-vad-model` +points at it, and with an energy threshold when it does not. The model is 2.3MB +and is not committed: + +```sh +mkdir -p models/vad +curl -sL -o models/vad/silero_vad.onnx \ + https://github.com/snakers4/silero-vad/raw/master/src/silero_vad/data/silero_vad.onnx +``` + +It needs the same `libonnxruntime.so` the embedder needs, passed as `-onnx-lib` +or read from `MAVEN_ONNX_LIB`. The measurement is +`docs/evals/2026-08-09-silero-vad.md`, and the tests skip without the file. + **Also need ONNX Runtime** (`libonnxruntime.so`): ```sh diff --git a/cmd/mavwaked/main.go b/cmd/mavwaked/main.go index 65e4a55..f1bfa26 100644 --- a/cmd/mavwaked/main.go +++ b/cmd/mavwaked/main.go @@ -5,12 +5,17 @@ // is detected sends it as a PushToTalk frame to the voice server. The reply // audio is played back through aplay(1). // -// No wake-word model yet (MVP uses voice-activity-only trigger). The -// SurfaceVoice auth layer caps all commands at L0 (no destructive acts), -// making accidental triggers safe by design. A proper wake-word engine -// (openWakeWord / Silero VAD ONNX) is the planned upgrade — the VAD shape -// (30ms frames, 16kHz PCM) matches silero-vad's input interface exactly, so -// swapping energy-threshold for ONNX-inference is a local change in vad.go. +// Voice activity is silero-vad when -vad-model points at the graph, and an +// energy threshold when it does not. Silero declines noise the threshold +// accepts: 0 frames against 68 to 99 on the four fixtures, measured in +// docs/evals/2026-08-09-silero-vad.md. Note that the model window is 512 +// samples and the capture frame is 480, so silero.go re-chunks. This comment +// used to say the two matched, which was true of silero v4. +// +// There is still no wake-word model, so anything spoken near the microphone +// becomes a turn (V-487 stage two). The SurfaceVoice auth layer caps all +// commands at L0 (no destructive acts), which is what makes an accidental +// trigger safe rather than expensive. // // While a reply is playing the capture side is muted (half-duplex): without // it, Maven's own voice comes back in through the mic and she answers @@ -73,6 +78,9 @@ func run(args []string) error { bargeIn := flag.Bool("barge-in", false, "cut Maven off when he talks over her (needs a room-tuned -barge-in-rms)") bargeRMS := flag.Int("barge-in-rms", defaultBargeRMS, "RMS x10000 a frame must clear to count as barge-in") bargeFrames := flag.Int("barge-in-frames", defaultBargeFrames, "consecutive frames over -barge-in-rms before playback is cut") + vadModel := flag.String("vad-model", "", "silero-vad onnx file; empty runs the energy threshold instead") + vadThreshold := flag.Float64("vad-threshold", defaultSileroThreshold, "speech probability a frame must clear") + onnxLib := flag.String("onnx-lib", os.Getenv("MAVEN_ONNX_LIB"), "libonnxruntime.so, needed with -vad-model") flag.CommandLine.Parse(args) ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP) @@ -82,8 +90,20 @@ func run(args []string) error { vc := voice.Dial(*addr) defer vc.Close() - // VAD engine. + // VAD engine. A model that will not load is logged and not fatal: the + // energy threshold is worse, and it is a great deal better than a + // listening client that refuses to start. vad := NewVAD(*minRMS, *speechMs, *silenceMs, *maxMs) + if *vadModel != "" { + s, err := newSileroVAD(*vadModel, *onnxLib) + if err != nil { + log.Printf("mavwaked: silero unavailable, energy threshold unchanged: %v", err) + } else { + defer s.Close() + vad.UseSilero(s, *vadThreshold) + log.Printf("mavwaked: silero-vad from %s, threshold %.2f", *vadModel, *vadThreshold) + } + } // Audio source. var src io.ReadCloser diff --git a/cmd/mavwaked/silero.go b/cmd/mavwaked/silero.go new file mode 100644 index 0000000..6c015fb --- /dev/null +++ b/cmd/mavwaked/silero.go @@ -0,0 +1,171 @@ +package main + +// silero-vad, the speech detector that replaces the energy threshold (V-487). +// +// Why an energy threshold is not a voice activity detector. It answers "is +// this frame loud", and a fan, a door and a television are all loud. mavwaked +// sends every utterance it accepts to speech-to-text and then to the daemon, +// so a false trigger is a turn Maven takes on something nobody said to her. +// Silero answers "is this frame speech", which is the question. +// +// It is 2.3MB of ONNX and runs on one CPU core in real time. That is not an +// aside: this is the one model in the system that may never be offloaded or +// gated on GPU admission, because a wake path that waits on a card is not a +// wake path. +// +// Nil is a working value. Without -vad-model the daemon runs the energy VAD +// exactly as it did before this file existed. + +import ( + "fmt" + "sync" + + ort "github.com/yalue/onnxruntime_go" +) + +const ( + // sileroWindow — samples per inference at 16kHz. The model is fixed at + // 512 and does not accept another size, which is why this file + // re-chunks rather than reusing the 480-sample capture frame. main.go + // used to claim the two matched; that was true of silero v4. + sileroWindow = 512 + + // sileroContext — samples of the previous window prepended to each + // inference, as the reference implementation does. Without it the first + // milliseconds of every window are judged with no history and speech + // onsets score low. + sileroContext = 64 + + // sileroState — the LSTM state carried between windows, [2][1][128]. + sileroStateDim = 128 + + // defaultSileroThreshold — probability above which a window is speech. + // 0.5 is the reference default. Raising it costs speech onsets, which + // are the quietest part of an utterance. + defaultSileroThreshold = 0.5 +) + +// sileroVAD holds one ONNX session and the streaming state around it. It is +// fed 30ms capture frames and answers per frame, buffering across calls +// because 480 samples never line up with a 512-sample window. +type sileroVAD struct { + mu sync.Mutex + session *ort.DynamicAdvancedSession + + pending []float32 // samples not yet part of a full window + context [sileroContext]float32 // tail of the previous window + state []float32 // [2][1][128], carried between windows + last float64 // most recent probability, held between windows + sr []int64 +} + +// newSileroVAD loads the graph. The ONNX environment is initialised here when +// nothing else has done it, because mavwaked has no embedder to do it first. +func newSileroVAD(modelPath, libPath string) (*sileroVAD, error) { + if !ort.IsInitialized() { + if libPath != "" { + ort.SetSharedLibraryPath(libPath) + } + if err := ort.InitializeEnvironment(); err != nil { + return nil, fmt.Errorf("silero: onnx runtime: %w", err) + } + } + s, err := ort.NewDynamicAdvancedSession(modelPath, + []string{"input", "state", "sr"}, []string{"output", "stateN"}, nil) + if err != nil { + return nil, fmt.Errorf("silero: load %s: %w", modelPath, err) + } + return &sileroVAD{ + session: s, + state: make([]float32, 2*sileroStateDim), + sr: []int64{16000}, + }, nil +} + +// Speech reports whether the frame carries speech, and the probability behind +// that answer. A frame that completes no window inherits the previous +// probability, so the caller sees one answer per frame either way. +func (s *sileroVAD) Speech(frame []int16, threshold float64) (bool, float64) { + s.mu.Lock() + defer s.mu.Unlock() + + for _, v := range frame { + s.pending = append(s.pending, float32(v)/32768.0) + } + for len(s.pending) >= sileroWindow { + p, err := s.infer(s.pending[:sileroWindow]) + if err != nil { + // A failed inference must not silence the microphone. Hold the + // last answer and let the next window try again. + break + } + s.last = p + s.pending = s.pending[sileroWindow:] + } + return s.last >= threshold, s.last +} + +// infer runs one window and rolls the state and the context forward. +func (s *sileroVAD) infer(window []float32) (float64, error) { + in := make([]float32, sileroContext+sileroWindow) + copy(in, s.context[:]) + copy(in[sileroContext:], window) + + inT, err := ort.NewTensor(ort.NewShape(1, int64(len(in))), in) + if err != nil { + return 0, err + } + defer inT.Destroy() + stT, err := ort.NewTensor(ort.NewShape(2, 1, sileroStateDim), s.state) + if err != nil { + return 0, err + } + defer stT.Destroy() + srT, err := ort.NewTensor(ort.NewShape(1), s.sr) + if err != nil { + return 0, err + } + defer srT.Destroy() + + out, err := ort.NewEmptyTensor[float32](ort.NewShape(1, 1)) + if err != nil { + return 0, err + } + defer out.Destroy() + next, err := ort.NewEmptyTensor[float32](ort.NewShape(2, 1, sileroStateDim)) + if err != nil { + return 0, err + } + defer next.Destroy() + + if err := s.session.Run( + []ort.Value{inT, stT, srT}, + []ort.Value{out, next}, + ); err != nil { + return 0, err + } + copy(s.state, next.GetData()) + copy(s.context[:], in[len(in)-sileroContext:]) + return float64(out.GetData()[0]), nil +} + +// Reset drops the streaming state. Called at every utterance boundary and +// after barge-in, so echo-era history never scores the next sentence. +func (s *sileroVAD) Reset() { + s.mu.Lock() + defer s.mu.Unlock() + s.pending = s.pending[:0] + s.context = [sileroContext]float32{} + for i := range s.state { + s.state[i] = 0 + } + s.last = 0 +} + +// Close releases the session. +func (s *sileroVAD) Close() error { + if s == nil || s.session == nil { + return nil + } + return s.session.Destroy() +} diff --git a/cmd/mavwaked/silero_test.go b/cmd/mavwaked/silero_test.go new file mode 100644 index 0000000..25fee35 --- /dev/null +++ b/cmd/mavwaked/silero_test.go @@ -0,0 +1,149 @@ +package main + +// What this measures. The energy threshold cannot tell a voice from a +// television, and every utterance it accepts becomes a turn. So the test that +// matters is not "does silero find speech" — it is "does it decline what the +// energy threshold accepts". +// +// Speech is the four piper fixtures mavsttd already scores against. They are +// synthesised, so nothing of the owner's voice is committed. Non-speech is +// white noise at the same loudness, which is the cheapest thing that fools an +// energy floor and the honest floor for this claim. +// +// Both halves skip without models/vad/silero_vad.onnx and MAVEN_ONNX_LIB, +// like the TestONNX measurements in internal/router/eval. + +import ( + "math" + "math/rand" + "os" + "path/filepath" + "testing" +) + +const wavHeader = 44 // 16kHz mono s16le, written by piper + +func loadSilero(t *testing.T) *sileroVAD { + t.Helper() + model := filepath.Join("..", "..", "models", "vad", "silero_vad.onnx") + lib := os.Getenv("MAVEN_ONNX_LIB") + if _, err := os.Stat(model); err != nil { + t.Skipf("missing %s: %v", model, err) + } + if lib == "" { + t.Skip("MAVEN_ONNX_LIB unset") + } + s, err := newSileroVAD(model, lib) + if err != nil { + t.Skipf("silero unavailable: %v", err) + } + return s +} + +// feedAll runs a whole clip through a VAD and reports how many utterances it +// produced and how many frames it called speech. +func feedAll(v *VAD, pcm []int16) (utterances, speechFrames int) { + for i := 0; i+frameSamples <= len(pcm); i += frameSamples { + frame := pcm[i : i+frameSamples] + utt, state := v.Feed(frame) + if state == StateSpeech { + speechFrames++ + } + if utt.Bytes != nil { + utterances++ + } + } + return utterances, speechFrames +} + +func readFixture(t *testing.T, name string) []int16 { + t.Helper() + raw, err := os.ReadFile(filepath.Join("..", "mavsttd", "testdata", name)) + if err != nil { + t.Skipf("missing fixture %s: %v", name, err) + } + if len(raw) <= wavHeader { + t.Fatalf("%s: %d bytes, no audio", name, len(raw)) + } + return PCMToI16(raw[wavHeader:]) +} + +// noise returns white noise scaled to the same RMS as ref. Same loudness, +// nothing said. +func noise(ref []int16, seed int64) []int16 { + target := frameRMS(ref) + r := rand.New(rand.NewSource(seed)) + out := make([]int16, len(ref)) + for i := range out { + out[i] = int16(r.NormFloat64() * target * 32768.0) + } + return out +} + +func TestSileroHearsSpeechAndDeclinesNoise(t *testing.T) { + s := loadSilero(t) + defer s.Close() + + for _, name := range []string{"ru_fact.wav", "ru_query.wav", "ru_reminder.wav", "en_act.wav"} { + pcm := readFixture(t, name) + + v := NewVAD(0, 0, 0, 0) + v.UseSilero(s, defaultSileroThreshold) + _, spoke := feedAll(v, pcm) + if spoke == 0 { + t.Errorf("%s: silero heard no speech in a spoken clip", name) + } + + s.Reset() + v2 := NewVAD(0, 0, 0, 0) + v2.UseSilero(s, defaultSileroThreshold) + _, heard := feedAll(v2, noise(pcm, 7)) + + energy := NewVAD(0, 0, 0, 0) + _, energyHeard := feedAll(energy, noise(pcm, 7)) + + t.Logf("%s: speech frames — silero on speech %d, silero on noise %d, energy on noise %d", + name, spoke, heard, energyHeard) + if heard >= energyHeard { + t.Errorf("%s: silero called %d noise frames speech, energy called %d — no improvement", + name, heard, energyHeard) + } + s.Reset() + } +} + +// BenchmarkSileroFrame answers the only performance question that matters +// here: one 30ms frame must cost far less than 30ms on one core, or the +// detector cannot run always-on beside everything else on that machine. +func BenchmarkSileroFrame(b *testing.B) { + s := loadSilero(&testing.T{}) + if s == nil { + b.Skip("silero unavailable") + } + defer s.Close() + frame := make([]int16, frameSamples) + for i := range frame { + frame[i] = int16(i%400 - 200) + } + for i := 0; i < b.N; i++ { + s.Speech(frame, defaultSileroThreshold) + } +} + +// TestSileroRechunksAcrossFrames pins the reason this file exists. The capture +// frame is 480 samples and the model window is 512, so a detector that ran one +// inference per frame would be feeding the model a shape it does not accept. +func TestSileroRechunksAcrossFrames(t *testing.T) { + s := loadSilero(t) + defer s.Close() + + silence := make([]int16, frameSamples) + for i := 0; i < 20; i++ { + if _, p := s.Speech(silence, defaultSileroThreshold); math.IsNaN(p) { + t.Fatalf("frame %d: probability is NaN", i) + } + } + if len(s.pending) >= sileroWindow { + t.Errorf("pending grew to %d samples, so windows are not being consumed", len(s.pending)) + } +} diff --git a/cmd/mavwaked/vad.go b/cmd/mavwaked/vad.go index d6d3b10..08698f4 100644 --- a/cmd/mavwaked/vad.go +++ b/cmd/mavwaked/vad.go @@ -68,6 +68,37 @@ type VAD struct { // follows the room's ambient level. Initialised to minRMS; updated // on each silence frame. floorRMS float64 + + // speech is silero-vad, or nil. When it is set the energy floor decides + // nothing: the question becomes "is this speech" rather than "is this + // loud", and the noise floor is not even tracked. Everything after that + // answer — the speech hold, the silence hold, the length cap, the + // buffer — is the same state machine either way, which is why the + // detector goes here and not around this type. + speech *sileroVAD + speechMin float64 +} + +// UseSilero swaps the energy threshold for the model. Passing nil is a +// no-op, so a caller that could not load the graph keeps a working VAD. +func (v *VAD) UseSilero(s *sileroVAD, threshold float64) { + if s == nil { + return + } + if threshold <= 0 { + threshold = defaultSileroThreshold + } + v.speech = s + v.speechMin = threshold +} + +// isSpeech answers the one question the state machine asks of a frame. +func (v *VAD) isSpeech(frame []int16, rms float64) bool { + if v.speech != nil { + ok, _ := v.speech.Speech(frame, v.speechMin) + return ok + } + return rms >= v.floorRMS } // NewVAD creates a VAD with the given thresholds. Zero values use defaults. @@ -110,7 +141,7 @@ func (v *VAD) State() SpeechState { return v.state } // should send the audio to the voice server before feeding more frames. func (v *VAD) Feed(frame []int16) (_ audio.Audio, state SpeechState) { rms := frameRMS(frame) - isSpeech := rms >= v.floorRMS + isSpeech := v.isSpeech(frame, rms) switch v.state { case StateSilence: @@ -175,6 +206,9 @@ func (v *VAD) Feed(frame []int16) (_ audio.Audio, state SpeechState) { func (v *VAD) Reset() { v.reset() } func (v *VAD) reset() { + if v.speech != nil { + v.speech.Reset() + } v.state = StateSilence v.speechFrames = 0 v.silenceFrames = 0 diff --git a/docs/evals/2026-08-09-silero-vad.md b/docs/evals/2026-08-09-silero-vad.md new file mode 100644 index 0000000..97e3f94 --- /dev/null +++ b/docs/evals/2026-08-09-silero-vad.md @@ -0,0 +1,63 @@ +# silero-vad against the energy threshold in mavwaked + +*Measured 2026-08-09 on homesrv. V-487, stage one of two.* + +mavwaked decided an utterance had started by comparing frame energy to an +adaptive floor. That answers "is this frame loud". A fan, a door and a +television are all loud, and every utterance mavwaked accepts becomes a turn. + +silero-vad answers "is this frame speech". It is 2.3MB of ONNX and it replaces +the comparison and nothing else. The speech hold, the silence hold, the length +cap and the utterance buffer are the same state machine either way. + +## What it declines + +Speech is the four piper fixtures `mavsttd` already scores against, so nothing +of the owner's voice is committed. Non-speech is white noise at the same RMS as the clip beside it. That is the +cheapest thing that fools an energy floor. + +| clip | silero, speech frames on speech | silero on noise | energy on noise | +|---|---|---|---| +| ru_fact.wav | 59 | 0 | 68 | +| ru_query.wav | 69 | 0 | 79 | +| ru_reminder.wav | 80 | 0 | 89 | +| en_act.wav | 90 | 0 | 99 | + +The energy threshold accepts every noise clip as a complete utterance. Silero +calls not one frame of any of them speech, and still hears all four spoken +clips. `TestSileroHearsSpeechAndDeclinesNoise` is that table. + +White noise is a floor, not a proof. It says nothing about a television, which +is speech, or about a fan, which is narrowband. Those need room recordings and +this box has none. + +## What it costs + +`BenchmarkSileroFrame` on the homesrv laptop (Ryzen 5 5600U), one 30ms frame +through the model including the re-chunking: + + 509µs per frame + +That is 1.7% of one core, on the slower of the two machines. The detector runs +on the workstation beside the microphone, never on the GPU. This number is what +says it does not need one. + +## The window is 512 samples, not 480 + +`cmd/mavwaked/main.go` claimed the frame contract matched silero's input +exactly. That was true of silero v4. Version 5 takes exactly 512 samples at 16kHz, plus 64 samples of context from +the previous window. So `sileroVAD` buffers across capture frames, and a frame +completing no window inherits the previous probability. `TestSileroRechunksAcrossFrames` pins it. + +## Still an energy gate by default + +`-vad-model` is empty in the code default, so a deployment that does not pass +it runs exactly what shipped before. Barge-in is untouched and deliberately so. It reads frame energy while she is +speaking, which is a different question from whether the frame is speech. + +## Not done here + +The wake word. This is stage one of the two V-487 asks for. The second needs a +keyword model that does not exist yet. The pretrained openWakeWord keywords are +English, and a Russian one has to be trained. Until then anything spoken near +the microphone still becomes a turn. It is now merely required to be speech.