Hear speech instead of loudness in mavwaked (V-487)

silero-vad replaces the energy threshold when -vad-model points at it. Everything after the speech decision is the same state machine: the speech hold, the silence hold, the length cap and the utterance buffer. The model window is 512 samples and the capture frame is 480, so silero.go re-chunks across frames. main.go claimed the two matched, which was true of silero v4. Stage two, the wake word, is not here. It needs a Russian keyword model that does not exist yet.
2026-08-09 01:43:32 +04:00
parent 6d5801bb1f
commit a99932b427
6 changed files with 314 additions and 8 deletions
@@ -5,12 +5,17 @@
 // is detected sends it as a PushToTalk frame to the voice server. The reply
 // audio is played back through aplay(1).
 //
-// No wake-word model yet (MVP uses voice-activity-only trigger). The
-// SurfaceVoice auth layer caps all commands at L0 (no destructive acts),
-// making accidental triggers safe by design. A proper wake-word engine
-// (openWakeWord / Silero VAD ONNX) is the planned upgrade — the VAD shape
-// (30ms frames, 16kHz PCM) matches silero-vad's input interface exactly, so
-// swapping energy-threshold for ONNX-inference is a local change in vad.go.
+// Voice activity is silero-vad when -vad-model points at the graph, and an
+// energy threshold when it does not. Silero declines noise the threshold
+// accepts: 0 frames against 68 to 99 on the four fixtures, measured in
+// docs/evals/2026-08-09-silero-vad.md. Note that the model window is 512
+// samples and the capture frame is 480, so silero.go re-chunks. This comment
+// used to say the two matched, which was true of silero v4.
+//
+// There is still no wake-word model, so anything spoken near the microphone
+// becomes a turn (V-487 stage two). The SurfaceVoice auth layer caps all
+// commands at L0 (no destructive acts), which is what makes an accidental
+// trigger safe rather than expensive.
 //
 // While a reply is playing the capture side is muted (half-duplex): without
 // it, Maven's own voice comes back in through the mic and she answers
@@ -73,6 +78,9 @@ func run(args []string) error {
 	bargeIn := flag.Bool("barge-in", false, "cut Maven off when he talks over her (needs a room-tuned -barge-in-rms)")
 	bargeRMS := flag.Int("barge-in-rms", defaultBargeRMS, "RMS x10000 a frame must clear to count as barge-in")
 	bargeFrames := flag.Int("barge-in-frames", defaultBargeFrames, "consecutive frames over -barge-in-rms before playback is cut")
+	vadModel := flag.String("vad-model", "", "silero-vad onnx file; empty runs the energy threshold instead")
+	vadThreshold := flag.Float64("vad-threshold", defaultSileroThreshold, "speech probability a frame must clear")
+	onnxLib := flag.String("onnx-lib", os.Getenv("MAVEN_ONNX_LIB"), "libonnxruntime.so, needed with -vad-model")
 	flag.CommandLine.Parse(args)

 	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
@@ -82,8 +90,20 @@ func run(args []string) error {
 	vc := voice.Dial(*addr)
 	defer vc.Close()

-	// VAD engine.
+	// VAD engine. A model that will not load is logged and not fatal: the
+	// energy threshold is worse, and it is a great deal better than a
+	// listening client that refuses to start.
 	vad := NewVAD(*minRMS, *speechMs, *silenceMs, *maxMs)
+	if *vadModel != "" {
+		s, err := newSileroVAD(*vadModel, *onnxLib)
+		if err != nil {
+			log.Printf("mavwaked: silero unavailable, energy threshold unchanged: %v", err)
+		} else {
+			defer s.Close()
+			vad.UseSilero(s, *vadThreshold)
+			log.Printf("mavwaked: silero-vad from %s, threshold %.2f", *vadModel, *vadThreshold)
+		}
+	}

 	// Audio source.
 	var src io.ReadCloser
@@ -0,0 +1,171 @@
+package main
+
+// silero-vad, the speech detector that replaces the energy threshold (V-487).
+//
+// Why an energy threshold is not a voice activity detector. It answers "is
+// this frame loud", and a fan, a door and a television are all loud. mavwaked
+// sends every utterance it accepts to speech-to-text and then to the daemon,
+// so a false trigger is a turn Maven takes on something nobody said to her.
+// Silero answers "is this frame speech", which is the question.
+//
+// It is 2.3MB of ONNX and runs on one CPU core in real time. That is not an
+// aside: this is the one model in the system that may never be offloaded or
+// gated on GPU admission, because a wake path that waits on a card is not a
+// wake path.
+//
+// Nil is a working value. Without -vad-model the daemon runs the energy VAD
+// exactly as it did before this file existed.
+
+import (
+	"fmt"
+	"sync"
+
+	ort "github.com/yalue/onnxruntime_go"
+)
+
+const (
+	// sileroWindow — samples per inference at 16kHz. The model is fixed at
+	// 512 and does not accept another size, which is why this file
+	// re-chunks rather than reusing the 480-sample capture frame. main.go
+	// used to claim the two matched; that was true of silero v4.
+	sileroWindow = 512
+
+	// sileroContext — samples of the previous window prepended to each
+	// inference, as the reference implementation does. Without it the first
+	// milliseconds of every window are judged with no history and speech
+	// onsets score low.
+	sileroContext = 64
+
+	// sileroState — the LSTM state carried between windows, [2][1][128].
+	sileroStateDim = 128
+
+	// defaultSileroThreshold — probability above which a window is speech.
+	// 0.5 is the reference default. Raising it costs speech onsets, which
+	// are the quietest part of an utterance.
+	defaultSileroThreshold = 0.5
+)
+
+// sileroVAD holds one ONNX session and the streaming state around it. It is
+// fed 30ms capture frames and answers per frame, buffering across calls
+// because 480 samples never line up with a 512-sample window.
+type sileroVAD struct {
+	mu      sync.Mutex
+	session *ort.DynamicAdvancedSession
+
+	pending []float32               // samples not yet part of a full window
+	context [sileroContext]float32  // tail of the previous window
+	state   []float32               // [2][1][128], carried between windows
+	last    float64                 // most recent probability, held between windows
+	sr      []int64
+}
+
+// newSileroVAD loads the graph. The ONNX environment is initialised here when
+// nothing else has done it, because mavwaked has no embedder to do it first.
+func newSileroVAD(modelPath, libPath string) (*sileroVAD, error) {
+	if !ort.IsInitialized() {
+		if libPath != "" {
+			ort.SetSharedLibraryPath(libPath)
+		}
+		if err := ort.InitializeEnvironment(); err != nil {
+			return nil, fmt.Errorf("silero: onnx runtime: %w", err)
+		}
+	}
+	s, err := ort.NewDynamicAdvancedSession(modelPath,
+		[]string{"input", "state", "sr"}, []string{"output", "stateN"}, nil)
+	if err != nil {
+		return nil, fmt.Errorf("silero: load %s: %w", modelPath, err)
+	}
+	return &sileroVAD{
+		session: s,
+		state:   make([]float32, 2*sileroStateDim),
+		sr:      []int64{16000},
+	}, nil
+}
+
+// Speech reports whether the frame carries speech, and the probability behind
+// that answer. A frame that completes no window inherits the previous
+// probability, so the caller sees one answer per frame either way.
+func (s *sileroVAD) Speech(frame []int16, threshold float64) (bool, float64) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	for _, v := range frame {
+		s.pending = append(s.pending, float32(v)/32768.0)
+	}
+	for len(s.pending) >= sileroWindow {
+		p, err := s.infer(s.pending[:sileroWindow])
+		if err != nil {
+			// A failed inference must not silence the microphone. Hold the
+			// last answer and let the next window try again.
+			break
+		}
+		s.last = p
+		s.pending = s.pending[sileroWindow:]
+	}
+	return s.last >= threshold, s.last
+}
+
+// infer runs one window and rolls the state and the context forward.
+func (s *sileroVAD) infer(window []float32) (float64, error) {
+	in := make([]float32, sileroContext+sileroWindow)
+	copy(in, s.context[:])
+	copy(in[sileroContext:], window)
+
+	inT, err := ort.NewTensor(ort.NewShape(1, int64(len(in))), in)
+	if err != nil {
+		return 0, err
+	}
+	defer inT.Destroy()
+	stT, err := ort.NewTensor(ort.NewShape(2, 1, sileroStateDim), s.state)
+	if err != nil {
+		return 0, err
+	}
+	defer stT.Destroy()
+	srT, err := ort.NewTensor(ort.NewShape(1), s.sr)
+	if err != nil {
+		return 0, err
+	}
+	defer srT.Destroy()
+
+	out, err := ort.NewEmptyTensor[float32](ort.NewShape(1, 1))
+	if err != nil {
+		return 0, err
+	}
+	defer out.Destroy()
+	next, err := ort.NewEmptyTensor[float32](ort.NewShape(2, 1, sileroStateDim))
+	if err != nil {
+		return 0, err
+	}
+	defer next.Destroy()
+
+	if err := s.session.Run(
+		[]ort.Value{inT, stT, srT},
+		[]ort.Value{out, next},
+	); err != nil {
+		return 0, err
+	}
+	copy(s.state, next.GetData())
+	copy(s.context[:], in[len(in)-sileroContext:])
+	return float64(out.GetData()[0]), nil
+}
+
+// Reset drops the streaming state. Called at every utterance boundary and
+// after barge-in, so echo-era history never scores the next sentence.
+func (s *sileroVAD) Reset() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.pending = s.pending[:0]
+	s.context = [sileroContext]float32{}
+	for i := range s.state {
+		s.state[i] = 0
+	}
+	s.last = 0
+}
+
+// Close releases the session.
+func (s *sileroVAD) Close() error {
+	if s == nil || s.session == nil {
+		return nil
+	}
+	return s.session.Destroy()
+}
@@ -68,6 +68,37 @@ type VAD struct {
 	// follows the room's ambient level. Initialised to minRMS; updated
 	// on each silence frame.
 	floorRMS float64
+
+	// speech is silero-vad, or nil. When it is set the energy floor decides
+	// nothing: the question becomes "is this speech" rather than "is this
+	// loud", and the noise floor is not even tracked. Everything after that
+	// answer — the speech hold, the silence hold, the length cap, the
+	// buffer — is the same state machine either way, which is why the
+	// detector goes here and not around this type.
+	speech    *sileroVAD
+	speechMin float64
+}
+
+// UseSilero swaps the energy threshold for the model. Passing nil is a
+// no-op, so a caller that could not load the graph keeps a working VAD.
+func (v *VAD) UseSilero(s *sileroVAD, threshold float64) {
+	if s == nil {
+		return
+	}
+	if threshold <= 0 {
+		threshold = defaultSileroThreshold
+	}
+	v.speech = s
+	v.speechMin = threshold
+}
+
+// isSpeech answers the one question the state machine asks of a frame.
+func (v *VAD) isSpeech(frame []int16, rms float64) bool {
+	if v.speech != nil {
+		ok, _ := v.speech.Speech(frame, v.speechMin)
+		return ok
+	}
+	return rms >= v.floorRMS
 }

 // NewVAD creates a VAD with the given thresholds. Zero values use defaults.
@@ -110,7 +141,7 @@ func (v *VAD) State() SpeechState { return v.state }
 // should send the audio to the voice server before feeding more frames.
 func (v *VAD) Feed(frame []int16) (_ audio.Audio, state SpeechState) {
 	rms := frameRMS(frame)
-	isSpeech := rms >= v.floorRMS
+	isSpeech := v.isSpeech(frame, rms)

 	switch v.state {
 	case StateSilence:
@@ -175,6 +206,9 @@ func (v *VAD) Feed(frame []int16) (_ audio.Audio, state SpeechState) {
 func (v *VAD) Reset() { v.reset() }

 func (v *VAD) reset() {
+	if v.speech != nil {
+		v.speech.Reset()
+	}
 	v.state = StateSilence
 	v.speechFrames = 0
 	v.silenceFrames = 0