Maven/cmd/mavwaked/session.go

package main

// The capture session: what happens to one 30ms frame, given whether Maven is
// currently speaking. Split out of main.go's processFrame so the decision is
// testable without a mic, a speaker, or a daemon (Vikunja #287).

import (
	"context"
	"log"
	"time"

	"github.com/kami/maven/internal/audio"
)

// utteranceSender ships one complete utterance to the voice server and
// returns the reply audio to play. The real one round-trips over the voice
// wire; tests substitute a recorder.
type utteranceSender interface {
	Send(ctx context.Context, utt audio.Audio, lang string) (audio.Audio, error)
}

// bargeInConfig holds the two numbers barge-in needs. Zero Frames disables
// barge-in entirely — the half-duplex gate still runs.
type bargeInConfig struct {
	// RMS is the normalised energy a frame must exceed to count as him
	// talking over her rather than the mic hearing her. It is deliberately
	// far above the VAD's own floor: the speaker leaks into the mic at
	// roughly ambient level, a person talking at the mic does not.
	RMS float64
	// Frames is how many consecutive frames must clear RMS before playback
	// is cut. One loud frame is a door closing; five in a row is a voice.
	Frames int
}

// Enabled reports whether barge-in should be attempted at all.
func (c bargeInConfig) Enabled() bool { return c.Frames > 0 && c.RMS > 0 }

// session is the per-client capture state machine.
type session struct {
	vad    *VAD
	player player
	sender utteranceSender
	lang   string
	barge  bargeInConfig

	// now is the clock, swapped in tests. The round-trip backlog is measured
	// in wall time, because that is the only thing that says how much room
	// went into the pipe while the daemon was thinking.
	now func() time.Time

	// discard is how many buffered frames still have to be thrown away
	// before capture means anything again. See dispatch.
	discard int

	// recent holds the last few frames seen during playback, so the ones
	// that proved he was interrupting can be replayed into the VAD after the
	// barge-in reset instead of being clipped off the front of his sentence.
	recent [][]byte

	// loudFrames counts consecutive over-threshold frames seen while she is
	// speaking. Reset whenever a frame falls back under the threshold, and
	// whenever playback ends.
	loudFrames int

	// counters, read by tests and logged on the way out.
	suppressed int // frames dropped because she was speaking
	dropped    int // frames dropped as round-trip backlog
	bargeIns   int // times playback was cut because he spoke over her
	sent       int // utterances shipped to the daemon

	// loudSum and loudSeen accumulate the energy of suppressed frames, so
	// the operator can read what the room actually measures and set
	// -barge-in-rms from data instead of guessing.
	loudSum  float64
	loudSeen int
}

func newSession(vad *VAD, p player, s utteranceSender, lang string, barge bargeInConfig) *session {
	return &session{vad: vad, player: p, sender: s, lang: lang, barge: barge, now: time.Now}
}

// frameDuration is the wall time one captured frame represents.
const frameDuration = defaultFrameMs * time.Millisecond

// suppressLogEvery — how many suppressed frames between energy reports. 200
// frames is six seconds of her talking, so this is roughly one line per reply.
const suppressLogEvery = 200

// feed processes one 30ms PCM frame.
//
// While the player is running the capture side is muted: the VAD is not fed
// and no utterance can be produced, so Maven's own reply cannot come back in
// as a new command. The one thing that gets through is barge-in — sustained
// energy well above the speaker's leak level cuts playback, and capture
// resumes on the very next frame with a clean VAD.
func (s *session) feed(ctx context.Context, frame []byte) error {
	// Backlog first, before anything looks at this frame. These are frames
	// the microphone captured while the round-trip blocked; they arrive in a
	// burst at pipe speed and they are not a command, not an answer and not
	// an interruption.
	if s.discard > 0 {
		s.discard--
		s.dropped++
		return nil
	}

	if s.player.Playing() {
		s.suppressed++
		rms := frameRMS(PCMToI16(frame))
		s.loudSum += rms
		s.loudSeen++
		if s.loudSeen >= suppressLogEvery {
			// The doc comment asks for energy "well above the speaker's leak
			// level" and never says what that is. This is what it is.
			log.Printf("mavwaked: suppressed %d frames while speaking, mean rms %.4f (barge-in threshold %.4f)",
				s.loudSeen, s.loudSum/float64(s.loudSeen), s.barge.RMS)
			s.loudSum, s.loudSeen = 0, 0
		}
		if !s.barge.Enabled() {
			return nil
		}
		if rms < s.barge.RMS {
			s.loudFrames = 0
			s.recent = s.recent[:0]
			return nil
		}
		s.loudFrames++
		s.keepRecent(frame)
		if s.loudFrames < s.barge.Frames {
			return nil
		}
		// He is talking over her. Cut her off, drop the VAD state that
		// accumulated from the echo, and start listening for real — starting
		// with the frames that proved he was talking. Those used to be
		// thrown away, which clipped the first 150ms off his interruption,
		// and on a short one that is the whole first word.
		s.player.Stop()
		s.bargeIns++
		s.loudFrames = 0
		s.vad.Reset()
		log.Printf("mavwaked: barge-in — stopped playback")
		s.replayRecent()
		return nil
	}

	// Not speaking. If we just stopped, make sure no echo-era state leaks
	// into the next utterance.
	if s.loudFrames != 0 {
		s.loudFrames = 0
		s.vad.Reset()
	}

	utt, state := s.vad.Feed(PCMToI16(frame))
	if state == StateSpeech || utt.Bytes == nil {
		return nil
	}
	return s.dispatch(ctx, utt)
}

// keepRecent stores a copy of one barge-in trigger frame, keeping at most
// barge.Frames of them.
func (s *session) keepRecent(frame []byte) {
	if len(s.recent) >= s.barge.Frames {
		copy(s.recent, s.recent[1:])
		s.recent = s.recent[:len(s.recent)-1]
	}
	s.recent = append(s.recent, append([]byte(nil), frame...))
}

// replayRecent feeds the trigger frames back into the freshly reset VAD, so
// his interruption starts where he started it.
//
// Feed cannot complete an utterance here: closing one needs silenceMs of
// trailing quiet and these frames are all above the barge-in threshold, which
// is far above the VAD floor. Any utterance it did return would be a fragment
// of a sentence he is still speaking, so it is not dispatched.
func (s *session) replayRecent() {
	for _, f := range s.recent {
		s.vad.Feed(PCMToI16(f))
	}
	s.recent = s.recent[:0]
}

// dispatch ships a complete utterance and plays whatever comes back.
//
// Every return path here has to deal with the backlog. Nothing reads the
// microphone while Send is in flight, so the audio piles up in arecord's pipe
// and the kernel buffer, and it arrives in a burst the moment this returns. A
// round-trip is p50 2.7s through the LLM router, which is around 90 frames of
// room, of him finishing his sentence, of the television.
//
// This used to reset the VAD on the reply path only, and for the wrong reason:
// the comment said the VAD had been accumulating during the round-trip, when
// in fact its state is exactly what Feed left it as. The two paths that had no
// reset are the ones that mattered, because neither of them starts playback
// and so neither is covered by the half-duplex gate. A text-only turn fed the
// whole backlog straight into the VAD, and a Send error did the same on every
// failed turn, so a dead socket drove a retry loop off nothing but backlog.
func (s *session) dispatch(ctx context.Context, utt audio.Audio) error {
	log.Printf("mavwaked: utterance complete (%.2fs, %d bytes), sending...", utt.Duration(), len(utt.Bytes))
	start := s.now()
	reply, err := s.sender.Send(ctx, utt, s.lang)
	defer s.dropBacklog(start)
	if err != nil {
		return err
	}
	// Count what was shipped, not what was attempted. This used to run
	// before the error check, so failed round-trips counted as sent.
	s.sent++
	if len(reply.Bytes) == 0 {
		log.Printf("mavwaked: empty reply audio (text only)")
		return nil
	}
	s.player.Play(reply)
	return nil
}

// dropBacklog resets the VAD and arranges for the frames captured during the
// round-trip to be thrown away as they arrive.
//
// Discarding them is also what keeps barge-in honest. The Frames guard is
// documented as "long enough that a door or a cough does not cut her off",
// which assumes the frames are real time. Draining a backlog delivers five
// frames in microseconds, so without this she could be cut off by audio
// recorded before she started speaking.
func (s *session) dropBacklog(start time.Time) {
	s.vad.Reset()
	s.loudFrames = 0
	s.recent = s.recent[:0]
	if elapsed := s.now().Sub(start); elapsed > 0 {
		s.discard = int(elapsed / frameDuration)
	}
}