diff --git a/cmd/mavwaked/main.go b/cmd/mavwaked/main.go index 4f0c5a4..f346705 100644 --- a/cmd/mavwaked/main.go +++ b/cmd/mavwaked/main.go @@ -12,10 +12,12 @@ // samples and the capture frame is 480, so silero.go re-chunks. This comment // used to say the two matched, which was true of silero v4. // -// There is still no wake-word model, so anything spoken near the microphone -// becomes a turn (V-487 stage two). The SurfaceVoice auth layer caps all -// commands at L0 (no destructive acts), which is what makes an accidental -// trigger safe rather than expensive. +// The keyword is "Мэйвен" and it is required, when -wake-model points at the +// head (V-487 stage two). Without it anything spoken near the microphone +// becomes a turn, which the SurfaceVoice auth layer makes safe rather than +// expensive: it caps all commands at L0, no destructive acts. It does not cap +// reading, so an open gate still lets the room hear his facts read back. +// wakeword.go holds the cadence and wakefeatures.go the three models. // // The conn carries both directions. mavwaked sends utterances and receives // proactive nudges on it, and it is opened at startup rather than at the first @@ -62,6 +64,12 @@ const ( defaultAddr = "127.0.0.1:9100" defaultLang = "ru" defaultReadSize = 4096 // max PCM bytes per read from arecord (fits multiple frames) + + // defaultWakeWindowMs — how long the keyword stays good for. He says + // "Мэйвен" and then a sentence, and the VAD does not close the utterance + // until he stops, so this has to outlive the word by the length of what + // follows it. It is spent on dispatch: one keyword, one turn. + defaultWakeWindowMs = 8000 ) func main() { @@ -86,6 +94,11 @@ func run(args []string) error { vadModel := flag.String("vad-model", "", "silero-vad onnx file; empty runs the energy threshold instead") vadThreshold := flag.Float64("vad-threshold", defaultSileroThreshold, "speech probability a frame must clear") onnxLib := flag.String("onnx-lib", os.Getenv("MAVEN_ONNX_LIB"), "libonnxruntime.so, needed with -vad-model") + wakeModel := flag.String("wake-model", "", "keyword head onnx; empty ships every utterance, as before V-487") + wakeMel := flag.String("wake-mel", "", "melspectrogram.onnx, required with -wake-model") + wakeEmbed := flag.String("wake-embed", "", "embedding_model.onnx, required with -wake-model") + wakeThreshold := flag.Float64("wake-threshold", defaultWakeThreshold, "score the keyword must clear") + wakeWindowMs := flag.Int("wake-window-ms", defaultWakeWindowMs, "ms an utterance may still start after the keyword") flag.CommandLine.Parse(args) ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP) @@ -171,6 +184,21 @@ func run(args []string) error { } sess := newSession(vad, newAplayPlayer(), &voiceSender{vc: vc}, *lang, barge) + // Keyword gate. A model that will not load is logged and not fatal, for + // the same reason silero's is not: an open gate is the daemon he had + // yesterday, and a daemon that refuses to start is not. + if *wakeModel != "" { + w, err := newWakeWord(*wakeMel, *wakeEmbed, *wakeModel, *onnxLib, *wakeThreshold) + if err != nil { + log.Printf("mavwaked: wake word unavailable, every utterance is a turn: %v", err) + } else { + defer w.Close() + sess.UseWakeWord(w, time.Duration(*wakeWindowMs)*time.Millisecond) + log.Printf("mavwaked: wake word from %s, threshold %.3f, window %dms", + *wakeModel, *wakeThreshold, *wakeWindowMs) + } + } + // Listen for nudges alongside capture. Connect eagerly so mavend has a // voice session before he has said anything: without one, a nudge routed // to voice finds nobody home and goes to the away channels instead. diff --git a/cmd/mavwaked/session.go b/cmd/mavwaked/session.go index 8c0d600..b99ae0e 100644 --- a/cmd/mavwaked/session.go +++ b/cmd/mavwaked/session.go @@ -20,6 +20,15 @@ type utteranceSender interface { Send(ctx context.Context, utt audio.Audio, lang string) (audio.Audio, error) } +// keywordGate answers whether the keyword has just been spoken. The +// production one is wakeWord; tests substitute a recorder, because a gate that +// can only be exercised with three ONNX files is a gate nobody tests. +type keywordGate interface { + Feed(frame []int16) bool + Reset() + Score() float64 +} + // bargeInConfig holds the two numbers barge-in needs. Zero Frames disables // barge-in entirely — the half-duplex gate still runs. type bargeInConfig struct { @@ -63,6 +72,14 @@ type session struct { // whenever playback ends. loudFrames int + // wake is the keyword gate, or nil when no model was loaded. wakeUntil is + // how long a keyword stays good for: he says "Мэйвен" and then a sentence, + // and the VAD does not close the utterance until he stops, so the window + // has to outlive the word by the length of what follows it. + wake keywordGate + wakeWindow time.Duration + wakeUntil time.Time + // pending holds a nudge the push receiver handed over, waiting for the // capture loop to speak it. It is the one field written from another // goroutine, hence the mutex; everything else in this struct belongs to @@ -76,6 +93,8 @@ type session struct { bargeIns int // times playback was cut because he spoke over her sent int // utterances shipped to the daemon nudges int // proactive pushes spoken through the speaker + wakes int // times the keyword opened the gate + ignored int // complete utterances dropped because the keyword was absent // loudSum and loudSeen accumulate the energy of suppressed frames, so // the operator can read what the room actually measures and set @@ -88,6 +107,12 @@ func newSession(vad *VAD, p player, s utteranceSender, lang string, barge bargeI return &session{vad: vad, player: p, sender: s, lang: lang, barge: barge, now: time.Now} } +// UseWakeWord puts the keyword gate in front of dispatch. Without it every +// utterance is shipped, which is what mavwaked did before V-487 stage two. +func (s *session) UseWakeWord(w keywordGate, window time.Duration) { + s.wake, s.wakeWindow = w, window +} + // frameDuration is the wall time one captured frame represents. const frameDuration = defaultFrameMs * time.Millisecond @@ -147,6 +172,7 @@ func (s *session) feed(ctx context.Context, frame []byte) error { s.bargeIns++ s.loudFrames = 0 s.vad.Reset() + s.resetWake() log.Printf("mavwaked: barge-in — stopped playback") s.replayRecent() return nil @@ -157,13 +183,28 @@ func (s *session) feed(ctx context.Context, frame []byte) error { if s.loudFrames != 0 { s.loudFrames = 0 s.vad.Reset() + // The wake word saw nothing during playback, so what it holds is from + // before she spoke. Judging what he says next on it would score a + // sentence that ended a reply ago. + s.resetWake() } if s.startPendingNudge() { return nil } - utt, state := s.vad.Feed(PCMToI16(frame)) + // The keyword is scored on the same frames the VAD sees, and only on the + // ones that reach here: every path above returns while she is speaking, so + // her own voice saying "Мэйвен" cannot wake her. + pcm := PCMToI16(frame) + if s.wake != nil && s.wake.Feed(pcm) { + s.wakes++ + s.wakeUntil = s.now().Add(s.wakeWindow) + log.Printf("mavwaked: keyword heard (score %.3f), listening for %s", + s.wake.Score(), s.wakeWindow) + } + + utt, state := s.vad.Feed(pcm) if state == StateSpeech || utt.Bytes == nil { return nil } @@ -218,6 +259,25 @@ func (s *session) startPendingNudge() bool { return true } +// awake reports whether an utterance ending now was addressed to her. +// +// With no wake word loaded every utterance is, which is exactly what mavwaked +// did before this gate existed. An operator with no model file gets the old +// daemon rather than a daemon that refuses to hear anything. +func (s *session) awake() bool { + if s.wake == nil { + return true + } + return s.now().Before(s.wakeUntil) +} + +// resetWake drops the gate's streaming state when there is a gate. +func (s *session) resetWake() { + if s.wake != nil { + s.wake.Reset() + } +} + // keepRecent stores a copy of one barge-in trigger frame, keeping at most // barge.Frames of them. func (s *session) keepRecent(frame []byte) { @@ -258,6 +318,19 @@ func (s *session) replayRecent() { // whole backlog straight into the VAD, and a Send error did the same on every // failed turn, so a dead socket drove a retry loop off nothing but backlog. func (s *session) dispatch(ctx context.Context, utt audio.Audio) error { + if !s.awake() { + s.ignored++ + log.Printf("mavwaked: utterance ignored, keyword not heard (%.2fs, %d ignored so far)", + utt.Duration(), s.ignored) + s.vad.Reset() + s.resetWake() + return nil + } + // One keyword, one turn. A window that renewed itself on every reply would + // leave the microphone open for as long as he kept talking, which is the + // state this gate exists to end. + s.wakeUntil = time.Time{} + log.Printf("mavwaked: utterance complete (%.2fs, %d bytes), sending...", utt.Duration(), len(utt.Bytes)) start := s.now() reply, err := s.sender.Send(ctx, utt, s.lang) @@ -286,6 +359,7 @@ func (s *session) dispatch(ctx context.Context, utt audio.Audio) error { // recorded before she started speaking. func (s *session) dropBacklog(start time.Time) { s.vad.Reset() + s.resetWake() s.loudFrames = 0 s.recent = s.recent[:0] if elapsed := s.now().Sub(start); elapsed > 0 {