d3fcc1dfdb
Nothing reads the microphone while Send is in flight, so the audio piles up in arecord's pipe and arrives in a burst the moment dispatch returns. A round-trip is p50 2.7s through the LLM router, which is about 90 frames of room, of him finishing his sentence, of the television. The old code reset the VAD on the reply path only, and for a reason that was not true: the comment said the VAD had been accumulating during the round-trip, when its state is exactly what Feed left it as. The two paths with no reset are the ones that mattered, because neither starts playback and so neither is covered by the half-duplex gate. A text-only turn fed the whole backlog into the VAD, and a Send error did the same on every failed turn, so a dead socket drove a retry loop off backlog alone. The backlog was scored for barge-in too. Five frames delivered in microseconds cut her off with audio recorded before she started speaking, which is the opposite of what the five-frame guard is for. Both are fixed by the same mechanism: measure the wall time the round-trip took, convert it to frames, and discard that many before anything looks at them. Barge-in also threw away the 150ms that proved he was talking. The VAD started from the next frame, so the first word of a short interruption was clipped before whisper saw it. Those frames are kept in a small ring and replayed after the reset. A stuck aplay was worse than before this feature existed. Playing() gates all capture, so a wedged child made her deaf rather than silent, for the full 30s ceiling inherited from the fire-and-forget version. The mute window is bounded by the reply's own duration plus a margin now. Three smaller ones. "-barge-in -barge-in-rms 0" logged "barge-in on" and then did nothing. The sent counter incremented before the error check, so failed round-trips counted as shipped. And the threshold the operator has to guess is now reported: mavwaked logs the mean energy of the frames it suppressed while speaking, so he can set it from data. Found in review of #76.
153 lines
4.3 KiB
Go
153 lines
4.3 KiB
Go
package main
|
|
|
|
// Reply playback, and the half-duplex gate around it (Vikunja #287).
|
|
//
|
|
// Before this, playback was `go playAudio(reply)` — fire and forget, with no
|
|
// handle on the running aplay. Two things fell out of that, and both are
|
|
// audible:
|
|
//
|
|
// 1. Self-trigger. The capture loop keeps feeding the VAD while the speaker
|
|
// is playing, so Maven's own reply comes back in through the mic, trips
|
|
// the VAD, and is sent to the daemon as a fresh utterance. She answers
|
|
// herself. There is no acoustic echo canceller in this pipeline, so the
|
|
// only correct fix is half-duplex: while she is speaking, the capture
|
|
// side is muted.
|
|
//
|
|
// 2. No barge-in. Talking over her did nothing — there was nothing to
|
|
// cancel, because nobody held the process handle.
|
|
//
|
|
// The two are the same mechanism seen from opposite sides, so they live
|
|
// together here. Echo suppression is unconditional (it fixes a bug). Barge-in
|
|
// is off unless -barge-in is passed, because it needs a room-specific energy
|
|
// threshold: with no echo canceller, the only way to tell "he is talking over
|
|
// her" from "the mic is hearing her" is that he is louder, and how much
|
|
// louder depends on where the mic sits relative to the speaker.
|
|
|
|
import (
|
|
"log"
|
|
"os/exec"
|
|
"strconv"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/kami/maven/internal/audio"
|
|
)
|
|
|
|
// playbackMargin is the slack over the reply's own duration before a stuck
|
|
// aplay is killed. Enough for ALSA to open the device and drain its buffer,
|
|
// short enough that a busy device does not cost her a turn.
|
|
const playbackMargin = 2 * time.Second
|
|
|
|
// player plays one reply at a time and can be cut off mid-utterance.
|
|
type player interface {
|
|
// Play starts playback of a, replacing anything already playing, and
|
|
// returns immediately.
|
|
Play(a audio.Audio)
|
|
// Stop ends playback now. A no-op when nothing is playing.
|
|
Stop()
|
|
// Playing reports whether audio is currently going out of the speaker.
|
|
Playing() bool
|
|
}
|
|
|
|
// aplayPlayer pipes raw PCM to aplay(1). Stop kills the child, which is what
|
|
// makes barge-in instant rather than "instant at the end of the sentence".
|
|
type aplayPlayer struct {
|
|
mu sync.Mutex
|
|
cmd *exec.Cmd
|
|
playing bool
|
|
// gen rises on every Play/Stop so a finishing playback cannot clear the
|
|
// playing flag of the one that replaced it.
|
|
gen uint64
|
|
}
|
|
|
|
func newAplayPlayer() *aplayPlayer { return &aplayPlayer{} }
|
|
|
|
func (p *aplayPlayer) Play(a audio.Audio) {
|
|
if len(a.Bytes) == 0 {
|
|
return
|
|
}
|
|
p.Stop()
|
|
|
|
cmd := exec.Command("aplay",
|
|
"-f", "S16_LE",
|
|
"-r", strconv.Itoa(a.Format.SampleRate),
|
|
"-c", strconv.Itoa(a.Format.Channels),
|
|
"-t", "raw",
|
|
)
|
|
stdin, err := cmd.StdinPipe()
|
|
if err != nil {
|
|
log.Printf("mavwaked: aplay stdin pipe: %v", err)
|
|
return
|
|
}
|
|
if err := cmd.Start(); err != nil {
|
|
log.Printf("mavwaked: start aplay: %v", err)
|
|
_ = stdin.Close()
|
|
return
|
|
}
|
|
|
|
p.mu.Lock()
|
|
p.gen++
|
|
gen := p.gen
|
|
p.cmd = cmd
|
|
p.playing = true
|
|
p.mu.Unlock()
|
|
|
|
// Bound the mute window by the reply itself. Playing() gates all capture
|
|
// now, so a wedged aplay does not merely go silent, it makes her deaf for
|
|
// as long as the flag is set. The old ceiling was a flat 30s inherited
|
|
// from the fire-and-forget version, where it only bounded a leaked
|
|
// goroutine. A reply cannot legitimately take longer than it lasts.
|
|
limit := time.Duration(a.Duration()*float64(time.Second)) + playbackMargin
|
|
|
|
go func() {
|
|
if _, err := stdin.Write(a.Bytes); err != nil {
|
|
// Broken pipe is the expected outcome of Stop().
|
|
log.Printf("mavwaked: write to aplay: %v", err)
|
|
}
|
|
_ = stdin.Close()
|
|
|
|
done := make(chan error, 1)
|
|
go func() { done <- cmd.Wait() }()
|
|
select {
|
|
case err := <-done:
|
|
if err != nil {
|
|
log.Printf("mavwaked: aplay: %v", err)
|
|
}
|
|
case <-time.After(limit):
|
|
log.Printf("mavwaked: aplay did not finish %.1fs of audio within %s, killing (capture was muted the whole time)",
|
|
a.Duration(), limit)
|
|
if pr := cmd.Process; pr != nil {
|
|
_ = pr.Kill()
|
|
}
|
|
<-done
|
|
}
|
|
|
|
p.mu.Lock()
|
|
if p.gen == gen {
|
|
p.playing = false
|
|
p.cmd = nil
|
|
}
|
|
p.mu.Unlock()
|
|
}()
|
|
}
|
|
|
|
func (p *aplayPlayer) Stop() {
|
|
p.mu.Lock()
|
|
cmd := p.cmd
|
|
if cmd != nil {
|
|
p.gen++
|
|
p.playing = false
|
|
p.cmd = nil
|
|
}
|
|
p.mu.Unlock()
|
|
if cmd != nil && cmd.Process != nil {
|
|
_ = cmd.Process.Kill()
|
|
}
|
|
}
|
|
|
|
func (p *aplayPlayer) Playing() bool {
|
|
p.mu.Lock()
|
|
defer p.mu.Unlock()
|
|
return p.playing
|
|
}
|