cmd/mavwaked/main.go

@@ -12,10 +12,12 @@
 // samples and the capture frame is 480, so silero.go re-chunks. This comment
 // used to say the two matched, which was true of silero v4.
 //
-// There is still no wake-word model, so anything spoken near the microphone
-// becomes a turn (V-487 stage two). The SurfaceVoice auth layer caps all
-// commands at L0 (no destructive acts), which is what makes an accidental
-// trigger safe rather than expensive.
+// The keyword is "Мэйвен" and it is required, when -wake-model points at the
+// head (V-487 stage two). Without it anything spoken near the microphone
+// becomes a turn, which the SurfaceVoice auth layer makes safe rather than
+// expensive: it caps all commands at L0, no destructive acts. It does not cap
+// reading, so an open gate still lets the room hear his facts read back.
+// wakeword.go holds the cadence and wakefeatures.go the three models.
 //
 // The conn carries both directions. mavwaked sends utterances and receives
 // proactive nudges on it, and it is opened at startup rather than at the first
@@ -62,6 +64,12 @@ const (
 	defaultAddr     = "127.0.0.1:9100"
 	defaultLang     = "ru"
 	defaultReadSize = 4096 // max PCM bytes per read from arecord (fits multiple frames)
+
+	// defaultWakeWindowMs — how long the keyword stays good for. He says
+	// "Мэйвен" and then a sentence, and the VAD does not close the utterance
+	// until he stops, so this has to outlive the word by the length of what
+	// follows it. It is spent on dispatch: one keyword, one turn.
+	defaultWakeWindowMs = 8000
 )
 
 func main() {
@@ -86,6 +94,11 @@ func run(args []string) error {
 	vadModel := flag.String("vad-model", "", "silero-vad onnx file; empty runs the energy threshold instead")
 	vadThreshold := flag.Float64("vad-threshold", defaultSileroThreshold, "speech probability a frame must clear")
 	onnxLib := flag.String("onnx-lib", os.Getenv("MAVEN_ONNX_LIB"), "libonnxruntime.so, needed with -vad-model")
+	wakeModel := flag.String("wake-model", "", "keyword head onnx; empty ships every utterance, as before V-487")
+	wakeMel := flag.String("wake-mel", "", "melspectrogram.onnx, required with -wake-model")
+	wakeEmbed := flag.String("wake-embed", "", "embedding_model.onnx, required with -wake-model")
+	wakeThreshold := flag.Float64("wake-threshold", defaultWakeThreshold, "score the keyword must clear")
+	wakeWindowMs := flag.Int("wake-window-ms", defaultWakeWindowMs, "ms an utterance may still start after the keyword")
 	flag.CommandLine.Parse(args)
 
 	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
@@ -171,6 +184,21 @@ func run(args []string) error {
 	}
 	sess := newSession(vad, newAplayPlayer(), &voiceSender{vc: vc}, *lang, barge)
 
+	// Keyword gate. A model that will not load is logged and not fatal, for
+	// the same reason silero's is not: an open gate is the daemon he had
+	// yesterday, and a daemon that refuses to start is not.
+	if *wakeModel != "" {
+		w, err := newWakeWord(*wakeMel, *wakeEmbed, *wakeModel, *onnxLib, *wakeThreshold)
+		if err != nil {
+			log.Printf("mavwaked: wake word unavailable, every utterance is a turn: %v", err)
+		} else {
+			defer w.Close()
+			sess.UseWakeWord(w, time.Duration(*wakeWindowMs)*time.Millisecond)
+			log.Printf("mavwaked: wake word from %s, threshold %.3f, window %dms",
+				*wakeModel, *wakeThreshold, *wakeWindowMs)
+		}
+	}
+
 	// Listen for nudges alongside capture. Connect eagerly so mavend has a
 	// voice session before he has said anything: without one, a nudge routed
 	// to voice finds nobody home and goes to the away channels instead.

cmd/mavwaked/session.go

@@ -20,6 +20,15 @@ type utteranceSender interface {
 	Send(ctx context.Context, utt audio.Audio, lang string) (audio.Audio, error)
 }
 
+// keywordGate answers whether the keyword has just been spoken. The
+// production one is wakeWord; tests substitute a recorder, because a gate that
+// can only be exercised with three ONNX files is a gate nobody tests.
+type keywordGate interface {
+	Feed(frame []int16) bool
+	Reset()
+	Score() float64
+}
+
 // bargeInConfig holds the two numbers barge-in needs. Zero Frames disables
 // barge-in entirely — the half-duplex gate still runs.
 type bargeInConfig struct {
@@ -63,6 +72,14 @@ type session struct {
 	// whenever playback ends.
 	loudFrames int
 
+	// wake is the keyword gate, or nil when no model was loaded. wakeUntil is
+	// how long a keyword stays good for: he says "Мэйвен" and then a sentence,
+	// and the VAD does not close the utterance until he stops, so the window
+	// has to outlive the word by the length of what follows it.
+	wake       keywordGate
+	wakeWindow time.Duration
+	wakeUntil  time.Time
+
 	// pending holds a nudge the push receiver handed over, waiting for the
 	// capture loop to speak it. It is the one field written from another
 	// goroutine, hence the mutex; everything else in this struct belongs to
@@ -76,6 +93,8 @@ type session struct {
 	bargeIns   int // times playback was cut because he spoke over her
 	sent       int // utterances shipped to the daemon
 	nudges     int // proactive pushes spoken through the speaker
+	wakes      int // times the keyword opened the gate
+	ignored    int // complete utterances dropped because the keyword was absent
 
 	// loudSum and loudSeen accumulate the energy of suppressed frames, so
 	// the operator can read what the room actually measures and set
@@ -88,6 +107,12 @@ func newSession(vad *VAD, p player, s utteranceSender, lang string, barge bargeI
 	return &session{vad: vad, player: p, sender: s, lang: lang, barge: barge, now: time.Now}
 }
 
+// UseWakeWord puts the keyword gate in front of dispatch. Without it every
+// utterance is shipped, which is what mavwaked did before V-487 stage two.
+func (s *session) UseWakeWord(w keywordGate, window time.Duration) {
+	s.wake, s.wakeWindow = w, window
+}
+
 // frameDuration is the wall time one captured frame represents.
 const frameDuration = defaultFrameMs * time.Millisecond
 
@@ -147,6 +172,7 @@ func (s *session) feed(ctx context.Context, frame []byte) error {
 		s.bargeIns++
 		s.loudFrames = 0
 		s.vad.Reset()
+		s.resetWake()
 		log.Printf("mavwaked: barge-in — stopped playback")
 		s.replayRecent()
 		return nil
@@ -157,13 +183,28 @@ func (s *session) feed(ctx context.Context, frame []byte) error {
 	if s.loudFrames != 0 {
 		s.loudFrames = 0
 		s.vad.Reset()
+		// The wake word saw nothing during playback, so what it holds is from
+		// before she spoke. Judging what he says next on it would score a
+		// sentence that ended a reply ago.
+		s.resetWake()
 	}
 
 	if s.startPendingNudge() {
 		return nil
 	}
 
-	utt, state := s.vad.Feed(PCMToI16(frame))
+	// The keyword is scored on the same frames the VAD sees, and only on the
+	// ones that reach here: every path above returns while she is speaking, so
+	// her own voice saying "Мэйвен" cannot wake her.
+	pcm := PCMToI16(frame)
+	if s.wake != nil && s.wake.Feed(pcm) {
+		s.wakes++
+		s.wakeUntil = s.now().Add(s.wakeWindow)
+		log.Printf("mavwaked: keyword heard (score %.3f), listening for %s",
+			s.wake.Score(), s.wakeWindow)
+	}
+
+	utt, state := s.vad.Feed(pcm)
 	if state == StateSpeech || utt.Bytes == nil {
 		return nil
 	}
@@ -218,6 +259,25 @@ func (s *session) startPendingNudge() bool {
 	return true
 }
 
+// awake reports whether an utterance ending now was addressed to her.
+//
+// With no wake word loaded every utterance is, which is exactly what mavwaked
+// did before this gate existed. An operator with no model file gets the old
+// daemon rather than a daemon that refuses to hear anything.
+func (s *session) awake() bool {
+	if s.wake == nil {
+		return true
+	}
+	return s.now().Before(s.wakeUntil)
+}
+
+// resetWake drops the gate's streaming state when there is a gate.
+func (s *session) resetWake() {
+	if s.wake != nil {
+		s.wake.Reset()
+	}
+}
+
 // keepRecent stores a copy of one barge-in trigger frame, keeping at most
 // barge.Frames of them.
 func (s *session) keepRecent(frame []byte) {
@@ -258,6 +318,19 @@ func (s *session) replayRecent() {
 // whole backlog straight into the VAD, and a Send error did the same on every
 // failed turn, so a dead socket drove a retry loop off nothing but backlog.
 func (s *session) dispatch(ctx context.Context, utt audio.Audio) error {
+	if !s.awake() {
+		s.ignored++
+		log.Printf("mavwaked: utterance ignored, keyword not heard (%.2fs, %d ignored so far)",
+			utt.Duration(), s.ignored)
+		s.vad.Reset()
+		s.resetWake()
+		return nil
+	}
+	// One keyword, one turn. A window that renewed itself on every reply would
+	// leave the microphone open for as long as he kept talking, which is the
+	// state this gate exists to end.
+	s.wakeUntil = time.Time{}
+
 	log.Printf("mavwaked: utterance complete (%.2fs, %d bytes), sending...", utt.Duration(), len(utt.Bytes))
 	start := s.now()
 	reply, err := s.sender.Send(ctx, utt, s.lang)
@@ -286,6 +359,7 @@ func (s *session) dispatch(ctx context.Context, utt audio.Audio) error {
 // recorded before she started speaking.
 func (s *session) dropBacklog(start time.Time) {
 	s.vad.Reset()
+	s.resetWake()
 	s.loudFrames = 0
 	s.recent = s.recent[:0]
 	if elapsed := s.now().Sub(start); elapsed > 0 {