package main // The capture session: what happens to one 30ms frame, given whether Maven is // currently speaking. Split out of main.go's processFrame so the decision is // testable without a mic, a speaker, or a daemon (Vikunja #287). import ( "context" "log" "time" "github.com/kami/maven/internal/audio" ) // utteranceSender ships one complete utterance to the voice server and // returns the reply audio to play. The real one round-trips over the voice // wire; tests substitute a recorder. type utteranceSender interface { Send(ctx context.Context, utt audio.Audio, lang string) (audio.Audio, error) } // bargeInConfig holds the two numbers barge-in needs. Zero Frames disables // barge-in entirely — the half-duplex gate still runs. type bargeInConfig struct { // RMS is the normalised energy a frame must exceed to count as him // talking over her rather than the mic hearing her. It is deliberately // far above the VAD's own floor: the speaker leaks into the mic at // roughly ambient level, a person talking at the mic does not. RMS float64 // Frames is how many consecutive frames must clear RMS before playback // is cut. One loud frame is a door closing; five in a row is a voice. Frames int } // Enabled reports whether barge-in should be attempted at all. func (c bargeInConfig) Enabled() bool { return c.Frames > 0 && c.RMS > 0 } // session is the per-client capture state machine. type session struct { vad *VAD player player sender utteranceSender lang string barge bargeInConfig // now is the clock, swapped in tests. The round-trip backlog is measured // in wall time, because that is the only thing that says how much room // went into the pipe while the daemon was thinking. now func() time.Time // discard is how many buffered frames still have to be thrown away // before capture means anything again. See dispatch. discard int // recent holds the last few frames seen during playback, so the ones // that proved he was interrupting can be replayed into the VAD after the // barge-in reset instead of being clipped off the front of his sentence. recent [][]byte // loudFrames counts consecutive over-threshold frames seen while she is // speaking. Reset whenever a frame falls back under the threshold, and // whenever playback ends. loudFrames int // counters, read by tests and logged on the way out. suppressed int // frames dropped because she was speaking dropped int // frames dropped as round-trip backlog bargeIns int // times playback was cut because he spoke over her sent int // utterances shipped to the daemon // loudSum and loudSeen accumulate the energy of suppressed frames, so // the operator can read what the room actually measures and set // -barge-in-rms from data instead of guessing. loudSum float64 loudSeen int } func newSession(vad *VAD, p player, s utteranceSender, lang string, barge bargeInConfig) *session { return &session{vad: vad, player: p, sender: s, lang: lang, barge: barge, now: time.Now} } // frameDuration is the wall time one captured frame represents. const frameDuration = defaultFrameMs * time.Millisecond // suppressLogEvery — how many suppressed frames between energy reports. 200 // frames is six seconds of her talking, so this is roughly one line per reply. const suppressLogEvery = 200 // feed processes one 30ms PCM frame. // // While the player is running the capture side is muted: the VAD is not fed // and no utterance can be produced, so Maven's own reply cannot come back in // as a new command. The one thing that gets through is barge-in — sustained // energy well above the speaker's leak level cuts playback, and capture // resumes on the very next frame with a clean VAD. func (s *session) feed(ctx context.Context, frame []byte) error { // Backlog first, before anything looks at this frame. These are frames // the microphone captured while the round-trip blocked; they arrive in a // burst at pipe speed and they are not a command, not an answer and not // an interruption. if s.discard > 0 { s.discard-- s.dropped++ return nil } if s.player.Playing() { s.suppressed++ rms := frameRMS(PCMToI16(frame)) s.loudSum += rms s.loudSeen++ if s.loudSeen >= suppressLogEvery { // The doc comment asks for energy "well above the speaker's leak // level" and never says what that is. This is what it is. log.Printf("mavwaked: suppressed %d frames while speaking, mean rms %.4f (barge-in threshold %.4f)", s.loudSeen, s.loudSum/float64(s.loudSeen), s.barge.RMS) s.loudSum, s.loudSeen = 0, 0 } if !s.barge.Enabled() { return nil } if rms < s.barge.RMS { s.loudFrames = 0 s.recent = s.recent[:0] return nil } s.loudFrames++ s.keepRecent(frame) if s.loudFrames < s.barge.Frames { return nil } // He is talking over her. Cut her off, drop the VAD state that // accumulated from the echo, and start listening for real — starting // with the frames that proved he was talking. Those used to be // thrown away, which clipped the first 150ms off his interruption, // and on a short one that is the whole first word. s.player.Stop() s.bargeIns++ s.loudFrames = 0 s.vad.Reset() log.Printf("mavwaked: barge-in — stopped playback") s.replayRecent() return nil } // Not speaking. If we just stopped, make sure no echo-era state leaks // into the next utterance. if s.loudFrames != 0 { s.loudFrames = 0 s.vad.Reset() } utt, state := s.vad.Feed(PCMToI16(frame)) if state == StateSpeech || utt.Bytes == nil { return nil } return s.dispatch(ctx, utt) } // keepRecent stores a copy of one barge-in trigger frame, keeping at most // barge.Frames of them. func (s *session) keepRecent(frame []byte) { if len(s.recent) >= s.barge.Frames { copy(s.recent, s.recent[1:]) s.recent = s.recent[:len(s.recent)-1] } s.recent = append(s.recent, append([]byte(nil), frame...)) } // replayRecent feeds the trigger frames back into the freshly reset VAD, so // his interruption starts where he started it. // // Feed cannot complete an utterance here: closing one needs silenceMs of // trailing quiet and these frames are all above the barge-in threshold, which // is far above the VAD floor. Any utterance it did return would be a fragment // of a sentence he is still speaking, so it is not dispatched. func (s *session) replayRecent() { for _, f := range s.recent { s.vad.Feed(PCMToI16(f)) } s.recent = s.recent[:0] } // dispatch ships a complete utterance and plays whatever comes back. // // Every return path here has to deal with the backlog. Nothing reads the // microphone while Send is in flight, so the audio piles up in arecord's pipe // and the kernel buffer, and it arrives in a burst the moment this returns. A // round-trip is p50 2.7s through the LLM router, which is around 90 frames of // room, of him finishing his sentence, of the television. // // This used to reset the VAD on the reply path only, and for the wrong reason: // the comment said the VAD had been accumulating during the round-trip, when // in fact its state is exactly what Feed left it as. The two paths that had no // reset are the ones that mattered, because neither of them starts playback // and so neither is covered by the half-duplex gate. A text-only turn fed the // whole backlog straight into the VAD, and a Send error did the same on every // failed turn, so a dead socket drove a retry loop off nothing but backlog. func (s *session) dispatch(ctx context.Context, utt audio.Audio) error { log.Printf("mavwaked: utterance complete (%.2fs, %d bytes), sending...", utt.Duration(), len(utt.Bytes)) start := s.now() reply, err := s.sender.Send(ctx, utt, s.lang) defer s.dropBacklog(start) if err != nil { return err } // Count what was shipped, not what was attempted. This used to run // before the error check, so failed round-trips counted as sent. s.sent++ if len(reply.Bytes) == 0 { log.Printf("mavwaked: empty reply audio (text only)") return nil } s.player.Play(reply) return nil } // dropBacklog resets the VAD and arranges for the frames captured during the // round-trip to be thrown away as they arrive. // // Discarding them is also what keeps barge-in honest. The Frames guard is // documented as "long enough that a door or a cough does not cut her off", // which assumes the frames are real time. Draining a backlog delivers five // frames in microseconds, so without this she could be cut off by audio // recorded before she started speaking. func (s *session) dropBacklog(start time.Time) { s.vad.Reset() s.loudFrames = 0 s.recent = s.recent[:0] if elapsed := s.now().Sub(start); elapsed > 0 { s.discard = int(elapsed / frameDuration) } }