Stop mavwaked from hearing itself, and add barge-in (#287)
Playback was `go playAudio(reply)` — fire and forget, nobody holding the process handle. Two audible consequences fell out of that. She answered herself. The capture loop kept feeding the VAD while the speaker was running, so her own reply came back in through the mic, tripped the VAD, and was shipped to the daemon as a fresh command. There is no acoustic echo canceller in this pipeline, so the fix is half-duplex: while she is speaking, the capture side is muted. That part is unconditional — it repairs a defect, it is not a new capability. And talking over her did nothing, because there was no handle to cancel. -barge-in now cuts playback when sustained energy clears a room-tuned threshold (-barge-in-rms, default 0.12 normalised, over -barge-in-frames consecutive frames, default 5). It is off by default: without an echo canceller the only way to tell "he is talking over her" from "the mic is hearing her" is that he is much louder, and how much louder depends on where the mic sits. The frame decision moved out of main.go into session.feed, behind a player and an utteranceSender interface, so all of it is testable with no mic, no speaker and no daemon. Nine tests cover the self-hearing case, the off-by-default case, the consecutive-frame requirement, speaker-leak-level audio not triggering, capturing the interrupting utterance after a cut, and failed round-trips not starting playback. The other seven items on #287 (partial STT, per-segment retry, mic profiles, noise-floor calibration, short-response-while-speaking) are untouched and stay on the task.
This commit is contained in:
kami
+33
-92
@@ -12,8 +12,15 @@
|
||||
// (30ms frames, 16kHz PCM) matches silero-vad's input interface exactly, so
|
||||
// swapping energy-threshold for ONNX-inference is a local change in vad.go.
|
||||
//
|
||||
// While a reply is playing the capture side is muted (half-duplex): without
|
||||
// it, Maven's own voice comes back in through the mic and she answers
|
||||
// herself. -barge-in punches one hole in that gate — sustained energy well
|
||||
// above the speaker's leak level cuts playback so he can talk over her. It is
|
||||
// off by default because the threshold is room-specific; see playback.go.
|
||||
//
|
||||
// usage:
|
||||
// mavwaked # default ALSA device, 127.0.0.1:9100
|
||||
// mavwaked -barge-in # let him interrupt her mid-reply
|
||||
// mavwaked -device hw:1,0 -addr 10.42.0.1:9100
|
||||
// mavwaked -test file.wav # read from file, no arecord
|
||||
package main
|
||||
@@ -60,6 +67,9 @@ func run(args []string) error {
|
||||
silenceMs := flag.Int("silence-ms", defaultSilenceMs, "silence ms to end utterance")
|
||||
maxMs := flag.Int("max-ms", defaultMaxMs, "max utterance ms")
|
||||
testFile := flag.String("test", "", "read PCM from file instead of arecord (testing only)")
|
||||
bargeIn := flag.Bool("barge-in", false, "cut Maven off when he talks over her (needs a room-tuned -barge-in-rms)")
|
||||
bargeRMS := flag.Int("barge-in-rms", defaultBargeRMS, "RMS x10000 a frame must clear to count as barge-in")
|
||||
bargeFrames := flag.Int("barge-in-frames", defaultBargeFrames, "consecutive frames over -barge-in-rms before playback is cut")
|
||||
flag.CommandLine.Parse(args)
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
|
||||
@@ -117,14 +127,21 @@ func run(args []string) error {
|
||||
|
||||
defer src.Close()
|
||||
|
||||
return captureLoop(ctx, src, vad, vc, *lang)
|
||||
var barge bargeInConfig
|
||||
if *bargeIn {
|
||||
barge = bargeInConfig{RMS: float64(*bargeRMS) / 10000.0, Frames: *bargeFrames}
|
||||
log.Printf("mavwaked: barge-in on (rms %.4f x %d frames)", barge.RMS, barge.Frames)
|
||||
}
|
||||
sess := newSession(vad, newAplayPlayer(), &voiceSender{vc: vc}, *lang, barge)
|
||||
|
||||
return captureLoop(ctx, src, sess)
|
||||
}
|
||||
|
||||
// captureLoop reads PCM from src, runs VAD, and sends complete utterances to
|
||||
// the voice server. Returns when ctx is done or src is exhausted.
|
||||
func captureLoop(ctx context.Context, src io.Reader, vad *VAD, vc *voice.Client, lang string) error {
|
||||
// captureLoop reads PCM from src and hands whole frames to the session.
|
||||
// Returns when ctx is done or src is exhausted.
|
||||
func captureLoop(ctx context.Context, src io.Reader, sess *session) error {
|
||||
br := bufio.NewReaderSize(src, defaultReadSize)
|
||||
frameBytes := vad.FrameSamples() * 2 // 480 samples × 2 bytes = 960 bytes per 30ms
|
||||
frameBytes := sess.vad.FrameSamples() * 2 // 480 samples × 2 bytes = 960 bytes per 30ms
|
||||
|
||||
log.Printf("mavwaked: capture loop starting (frame=%d bytes, %dms)",
|
||||
frameBytes, defaultFrameMs)
|
||||
@@ -147,7 +164,7 @@ func captureLoop(ctx context.Context, src io.Reader, vad *VAD, vc *voice.Client,
|
||||
// Flush partial frame.
|
||||
partial = append(partial, buf[:n]...)
|
||||
if len(partial) >= frameBytes {
|
||||
if err := processFrame(partial[:frameBytes], vad, vc, lang); err != nil {
|
||||
if err := sess.feed(ctx, partial[:frameBytes]); err != nil {
|
||||
log.Printf("mavwaked: process frame: %v", err)
|
||||
}
|
||||
partial = partial[frameBytes:]
|
||||
@@ -165,107 +182,31 @@ func captureLoop(ctx context.Context, src io.Reader, vad *VAD, vc *voice.Client,
|
||||
partial = nil
|
||||
}
|
||||
|
||||
if err := processFrame(full, vad, vc, lang); err != nil {
|
||||
if err := sess.feed(ctx, full); err != nil {
|
||||
log.Printf("mavwaked: process frame: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// processFrame feeds one 30ms PCM frame to the VAD and sends any completed
|
||||
// utterance to the voice server.
|
||||
func processFrame(frame []byte, vad *VAD, vc *voice.Client, lang string) error {
|
||||
samples := PCMToI16(frame)
|
||||
utt, state := vad.Feed(samples)
|
||||
// voiceSender is the production utteranceSender: one PushToTalk round-trip
|
||||
// over the voice wire. SurfaceVoice (not the default SurfacePCClient that
|
||||
// c.PushToTalk uses) caps everything at L0, which is what makes an accidental
|
||||
// VAD trigger safe.
|
||||
type voiceSender struct{ vc *voice.Client }
|
||||
|
||||
if state == StateSpeech {
|
||||
// Speech is in progress; nothing to send yet.
|
||||
return nil
|
||||
}
|
||||
|
||||
if utt.Bytes == nil {
|
||||
// Still in silence, or short speech that didn't trigger.
|
||||
return nil
|
||||
}
|
||||
|
||||
// We have a complete utterance — send it to the voice server.
|
||||
return sendUtterance(context.Background(), utt, vc, lang)
|
||||
}
|
||||
|
||||
// sendUtterance sends audio to the voice server and plays the reply.
|
||||
func sendUtterance(ctx context.Context, utt audio.Audio, vc *voice.Client, lang string) error {
|
||||
dur := utt.Duration()
|
||||
log.Printf("mavwaked: utterance complete (%.2fs, %d bytes), sending...",
|
||||
dur, len(utt.Bytes))
|
||||
|
||||
// Use SendRequest directly so we can set SurfaceVoice instead of the
|
||||
// default SurfacePCClient that c.PushToTalk uses.
|
||||
func (s *voiceSender) Send(ctx context.Context, utt audio.Audio, lang string) (audio.Audio, error) {
|
||||
var resp voice.PushToTalkResp
|
||||
err := vc.SendRequest(ctx, voice.MethodPushToTalk, voice.PushToTalkReq{
|
||||
err := s.vc.SendRequest(ctx, voice.MethodPushToTalk, voice.PushToTalkReq{
|
||||
Audio: utt,
|
||||
Lang: lang,
|
||||
Surface: voice.SurfaceVoice,
|
||||
}, &resp)
|
||||
if err != nil {
|
||||
return fmt.Errorf("push-to-talk: %w", err)
|
||||
return audio.Audio{}, fmt.Errorf("push-to-talk: %w", err)
|
||||
}
|
||||
|
||||
log.Printf("mavwaked: reply: %q (%.2fs audio)", resp.ReplyText, resp.ReplyAudio.Duration())
|
||||
|
||||
// Play the reply audio.
|
||||
if len(resp.ReplyAudio.Bytes) > 0 {
|
||||
go playAudio(resp.ReplyAudio)
|
||||
} else {
|
||||
log.Printf("mavwaked: empty reply audio (text only)")
|
||||
}
|
||||
|
||||
if len(resp.RoutedChannels) > 0 {
|
||||
log.Printf("mavwaked: also routed to: %v", resp.RoutedChannels)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// playAudio pipes PCM audio to aplay(1) for playback. Runs in a goroutine.
|
||||
func playAudio(a audio.Audio) {
|
||||
// Build WAV header for aplay (or pipe raw PCM with the right format flags).
|
||||
cmd := exec.Command("aplay",
|
||||
"-f", "S16_LE",
|
||||
"-r", fmt.Sprintf("%d", a.Format.SampleRate),
|
||||
"-c", fmt.Sprintf("%d", a.Format.Channels),
|
||||
"-t", "raw",
|
||||
)
|
||||
|
||||
stdin, err := cmd.StdinPipe()
|
||||
if err != nil {
|
||||
log.Printf("mavwaked: aplay stdin pipe: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
log.Printf("mavwaked: start aplay: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Write audio to aplay's stdin.
|
||||
if _, err := stdin.Write(a.Bytes); err != nil {
|
||||
log.Printf("mavwaked: write to aplay: %v", err)
|
||||
}
|
||||
_ = stdin.Close()
|
||||
|
||||
// Wait for playback to finish (with a timeout).
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- cmd.Wait()
|
||||
}()
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
log.Printf("mavwaked: aplay: %v", err)
|
||||
}
|
||||
case <-time.After(30 * time.Second):
|
||||
log.Printf("mavwaked: aplay timeout, killing")
|
||||
_ = cmd.Process.Kill()
|
||||
<-done
|
||||
}
|
||||
return resp.ReplyAudio, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user