mavwaked: throw away the round-trip backlog before it becomes a turn
Nothing reads the microphone while Send is in flight, so the audio piles up in arecord's pipe and arrives in a burst the moment dispatch returns. A round-trip is p50 2.7s through the LLM router, which is about 90 frames of room, of him finishing his sentence, of the television. The old code reset the VAD on the reply path only, and for a reason that was not true: the comment said the VAD had been accumulating during the round-trip, when its state is exactly what Feed left it as. The two paths with no reset are the ones that mattered, because neither starts playback and so neither is covered by the half-duplex gate. A text-only turn fed the whole backlog into the VAD, and a Send error did the same on every failed turn, so a dead socket drove a retry loop off backlog alone. The backlog was scored for barge-in too. Five frames delivered in microseconds cut her off with audio recorded before she started speaking, which is the opposite of what the five-frame guard is for. Both are fixed by the same mechanism: measure the wall time the round-trip took, convert it to frames, and discard that many before anything looks at them. Barge-in also threw away the 150ms that proved he was talking. The VAD started from the next frame, so the first word of a short interruption was clipped before whisper saw it. Those frames are kept in a small ring and replayed after the reset. A stuck aplay was worse than before this feature existed. Playing() gates all capture, so a wedged child made her deaf rather than silent, for the full 30s ceiling inherited from the fire-and-forget version. The mute window is bounded by the reply's own duration plus a margin now. Three smaller ones. "-barge-in -barge-in-rms 0" logged "barge-in on" and then did nothing. The sent counter incremented before the error check, so failed round-trips counted as shipped. And the threshold the operator has to guess is now reported: mavwaked logs the mean energy of the frames it suppressed while speaking, so he can set it from data. Found in review of #76.
This commit is contained in:
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"math"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/audio"
|
||||
)
|
||||
@@ -280,3 +281,147 @@ func TestBargeInConfigEnabled(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// slowSender models the real thing: a round-trip takes wall-clock time, and
|
||||
// the microphone keeps recording into a pipe nobody is reading.
|
||||
type slowSender struct {
|
||||
fakeSender
|
||||
clock *time.Time
|
||||
took time.Duration
|
||||
}
|
||||
|
||||
func (s *slowSender) Send(ctx context.Context, utt audio.Audio, lang string) (audio.Audio, error) {
|
||||
*s.clock = s.clock.Add(s.took)
|
||||
return s.fakeSender.Send(ctx, utt, lang)
|
||||
}
|
||||
|
||||
// newSlowSession wires a session whose round-trip takes took of wall time.
|
||||
func newSlowSession(barge bargeInConfig, reply audio.Audio, err error, took time.Duration) (*session, *fakePlayer, *slowSender) {
|
||||
now := time.Unix(0, 0)
|
||||
p := &fakePlayer{}
|
||||
snd := &slowSender{fakeSender: fakeSender{reply: reply, err: err}, clock: &now, took: took}
|
||||
sess := newSession(NewVAD(0, 0, 0, 0), p, snd, "ru", barge)
|
||||
sess.now = func() time.Time { return now }
|
||||
return sess, p, snd
|
||||
}
|
||||
|
||||
// A text-only turn starts no playback, so the half-duplex gate does not cover
|
||||
// it. The backlog captured during the round-trip has to be dropped anyway, or
|
||||
// three seconds of room arrives at pipe speed and becomes a command.
|
||||
func TestSessionDropsBacklogAfterAnEmptyReply(t *testing.T) {
|
||||
sess, p, snd := newSlowSession(bargeInConfig{}, audio.Audio{Format: audio.PCM16kMono}, nil, 3*time.Second)
|
||||
|
||||
speakThenPause(t, sess)
|
||||
if p.plays != 0 || len(snd.sent) != 1 {
|
||||
t.Fatalf("plays = %d, sent = %d; want one text-only turn", p.plays, len(snd.sent))
|
||||
}
|
||||
// The tail of speakThenPause already spent a couple of them.
|
||||
if want := int(3 * time.Second / frameDuration); sess.discard+sess.dropped != want {
|
||||
t.Fatalf("discard %d + dropped %d frames, want %d (3s of backlog)", sess.discard, sess.dropped, want)
|
||||
}
|
||||
|
||||
// The burst: the whole backlog, all of it him still talking.
|
||||
loud := frameAt(0.35)
|
||||
for i := 0; i < sess.discard; i++ {
|
||||
if err := sess.feed(context.Background(), loud); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if len(snd.sent) != 1 {
|
||||
t.Errorf("the backlog was sent as a second utterance (sent = %d)", len(snd.sent))
|
||||
}
|
||||
if sess.dropped == 0 {
|
||||
t.Error("no frames were counted as backlog")
|
||||
}
|
||||
}
|
||||
|
||||
// Same on the error path. A dead daemon used to seed the next spurious trigger
|
||||
// on every failed turn, so a dead socket drove a retry loop off backlog alone.
|
||||
func TestSessionDropsBacklogAfterASendError(t *testing.T) {
|
||||
sess, _, _ := newSlowSession(bargeInConfig{}, audio.Audio{}, errors.New("boom"), 3*time.Second)
|
||||
|
||||
// Not speakThenPause: the dispatch returns the send error, which that
|
||||
// helper treats as fatal.
|
||||
loud := frameAt(0.35)
|
||||
for i := 0; i < (defaultSpeechMs+defaultFrameMs-1)/defaultFrameMs+5; i++ {
|
||||
_ = sess.feed(context.Background(), loud)
|
||||
}
|
||||
for i := 0; i < (defaultSilenceMs+defaultFrameMs-1)/defaultFrameMs+2; i++ {
|
||||
_ = sess.feed(context.Background(), silentBytes())
|
||||
}
|
||||
if sess.discard == 0 {
|
||||
t.Fatal("a failed round-trip left the backlog to be fed into the VAD")
|
||||
}
|
||||
}
|
||||
|
||||
// Barge-in must not be triggerable by the backlog. Those frames are him
|
||||
// finishing the sentence he started before she answered, delivered in
|
||||
// microseconds, and the five-frame guard assumes real time.
|
||||
func TestSessionBacklogCannotBargeIn(t *testing.T) {
|
||||
sess, p, _ := newSlowSession(bargeInConfig{RMS: 0.12, Frames: 5}, replyAudio(), nil, 3*time.Second)
|
||||
|
||||
speakThenPause(t, sess)
|
||||
if p.plays != 1 || !p.Playing() {
|
||||
t.Fatalf("plays = %d, playing = %v; want the reply playing", p.plays, p.Playing())
|
||||
}
|
||||
|
||||
loud := frameAt(0.35)
|
||||
backlog := sess.discard
|
||||
if backlog < 5 {
|
||||
t.Fatalf("discard = %d, want a real backlog", backlog)
|
||||
}
|
||||
for i := 0; i < backlog; i++ {
|
||||
if err := sess.feed(context.Background(), loud); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if p.stops != 0 {
|
||||
t.Fatalf("she was cut off by audio recorded before she started speaking (stops = %d)", p.stops)
|
||||
}
|
||||
|
||||
// Real-time speech after the backlog still interrupts her.
|
||||
for i := 0; i < 5; i++ {
|
||||
if err := sess.feed(context.Background(), loud); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if p.stops != 1 {
|
||||
t.Fatalf("stops = %d, want 1 — barge-in must still work after the backlog", p.stops)
|
||||
}
|
||||
}
|
||||
|
||||
// The frames that proved he was interrupting are replayed into the VAD, so his
|
||||
// first word is not clipped. Five trigger frames plus five real ones reach the
|
||||
// 300ms speech threshold; without the replay the first five are lost and no
|
||||
// utterance is produced at all.
|
||||
func TestSessionReplaysTheBargeInTriggerFrames(t *testing.T) {
|
||||
sess, p, snd := newTestSession(bargeInConfig{RMS: 0.12, Frames: 5})
|
||||
speakThenPause(t, sess)
|
||||
|
||||
veryLoud := frameAt(0.35)
|
||||
for i := 0; i < 5; i++ {
|
||||
_ = sess.feed(context.Background(), veryLoud)
|
||||
}
|
||||
if p.stops != 1 {
|
||||
t.Fatalf("expected barge-in, stops = %d", p.stops)
|
||||
}
|
||||
if p.Playing() {
|
||||
t.Fatal("fake player still playing after Stop")
|
||||
}
|
||||
|
||||
speechFrames := (defaultSpeechMs + defaultFrameMs - 1) / defaultFrameMs
|
||||
for i := 0; i < speechFrames-5; i++ {
|
||||
if err := sess.feed(context.Background(), veryLoud); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
silenceFrames := (defaultSilenceMs+defaultFrameMs-1)/defaultFrameMs + 2
|
||||
for i := 0; i < silenceFrames; i++ {
|
||||
if err := sess.feed(context.Background(), silentBytes()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if len(snd.sent) != 2 {
|
||||
t.Fatalf("sent %d utterances, want 2 — the 150ms that triggered barge-in was clipped", len(snd.sent))
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user