Maven/cmd/mavwaked/session.go

package main

// The capture session: what happens to one 30ms frame, given whether Maven is
// currently speaking. Split out of main.go's processFrame so the decision is
// testable without a mic, a speaker, or a daemon (Vikunja #287).

import (
	"context"
	"log"
	"sync"
	"time"

	"github.com/kami/maven/internal/audio"
)

// utteranceSender ships one complete utterance to the voice server and
// returns the reply audio to play. The real one round-trips over the voice
// wire; tests substitute a recorder.
type utteranceSender interface {
	Send(ctx context.Context, utt audio.Audio, lang string) (audio.Audio, error)
}

// keywordGate answers whether the keyword has just been spoken. The
// production one is wakeWord; tests substitute a recorder, because a gate that
// can only be exercised with three ONNX files is a gate nobody tests.
type keywordGate interface {
	Feed(frame []int16) bool
	Reset()
	Score() float64
}

// bargeInConfig holds the two numbers barge-in needs. Zero Frames disables
// barge-in entirely — the half-duplex gate still runs.
type bargeInConfig struct {
	// RMS is the normalised energy a frame must exceed to count as him
	// talking over her rather than the mic hearing her. It is deliberately
	// far above the VAD's own floor: the speaker leaks into the mic at
	// roughly ambient level, a person talking at the mic does not.
	RMS float64
	// Frames is how many consecutive frames must clear RMS before playback
	// is cut. One loud frame is a door closing; five in a row is a voice.
	Frames int
}

// Enabled reports whether barge-in should be attempted at all.
func (c bargeInConfig) Enabled() bool { return c.Frames > 0 && c.RMS > 0 }

// session is the per-client capture state machine.
type session struct {
	vad    *VAD
	player player
	sender utteranceSender
	lang   string
	barge  bargeInConfig

	// now is the clock, swapped in tests. The round-trip backlog is measured
	// in wall time, because that is the only thing that says how much room
	// went into the pipe while the daemon was thinking.
	now func() time.Time

	// discard is how many buffered frames still have to be thrown away
	// before capture means anything again. See dispatch.
	discard int

	// recent holds the last few frames seen during playback, so the ones
	// that proved he was interrupting can be replayed into the VAD after the
	// barge-in reset instead of being clipped off the front of his sentence.
	recent [][]byte

	// loudFrames counts consecutive over-threshold frames seen while she is
	// speaking. Reset whenever a frame falls back under the threshold, and
	// whenever playback ends.
	loudFrames int

	// wake is the keyword gate, or nil when no model was loaded. wakeUntil is
	// how long a keyword stays good for: he says "Мэйвен" and then a sentence,
	// and the VAD does not close the utterance until he stops, so the window
	// has to outlive the word by the length of what follows it.
	wake       keywordGate
	wakeWindow time.Duration
	wakeUntil  time.Time

	// pending holds a nudge the push receiver handed over, waiting for the
	// capture loop to speak it. It is the one field written from another
	// goroutine, hence the mutex; everything else in this struct belongs to
	// the capture loop alone.
	nudgeMu sync.Mutex
	pending *audio.Audio

	// counters, read by tests and logged on the way out.
	suppressed int // frames dropped because she was speaking
	dropped    int // frames dropped as round-trip backlog
	bargeIns   int // times playback was cut because he spoke over her
	sent       int // utterances shipped to the daemon
	nudges     int // proactive pushes spoken through the speaker
	wakes      int // times the keyword opened the gate
	ignored    int // complete utterances dropped because the keyword was absent

	// loudSum and loudSeen accumulate the energy of suppressed frames, so
	// the operator can read what the room actually measures and set
	// -barge-in-rms from data instead of guessing.
	loudSum  float64
	loudSeen int
}

func newSession(vad *VAD, p player, s utteranceSender, lang string, barge bargeInConfig) *session {
	return &session{vad: vad, player: p, sender: s, lang: lang, barge: barge, now: time.Now}
}

// UseWakeWord puts the keyword gate in front of dispatch. Without it every
// utterance is shipped, which is what mavwaked did before V-487 stage two.
func (s *session) UseWakeWord(w keywordGate, window time.Duration) {
	s.wake, s.wakeWindow = w, window
}

// frameDuration is the wall time one captured frame represents.
const frameDuration = defaultFrameMs * time.Millisecond

// suppressLogEvery — how many suppressed frames between energy reports. 200
// frames is six seconds of her talking, so this is roughly one line per reply.
const suppressLogEvery = 200

// feed processes one 30ms PCM frame.
//
// While the player is running the capture side is muted: the VAD is not fed
// and no utterance can be produced, so Maven's own reply cannot come back in
// as a new command. The one thing that gets through is barge-in — sustained
// energy well above the speaker's leak level cuts playback, and capture
// resumes on the very next frame with a clean VAD.
func (s *session) feed(ctx context.Context, frame []byte) error {
	// Backlog first, before anything looks at this frame. These are frames
	// the microphone captured while the round-trip blocked; they arrive in a
	// burst at pipe speed and they are not a command, not an answer and not
	// an interruption.
	if s.discard > 0 {
		s.discard--
		s.dropped++
		return nil
	}

	if s.player.Playing() {
		s.suppressed++
		rms := frameRMS(PCMToI16(frame))
		s.loudSum += rms
		s.loudSeen++
		if s.loudSeen >= suppressLogEvery {
			// The doc comment asks for energy "well above the speaker's leak
			// level" and never says what that is. This is what it is.
			log.Printf("mavwaked: suppressed %d frames while speaking, mean rms %.4f (barge-in threshold %.4f)",
				s.loudSeen, s.loudSum/float64(s.loudSeen), s.barge.RMS)
			s.loudSum, s.loudSeen = 0, 0
		}
		if !s.barge.Enabled() {
			return nil
		}
		if rms < s.barge.RMS {
			s.loudFrames = 0
			s.recent = s.recent[:0]
			return nil
		}
		s.loudFrames++
		s.keepRecent(frame)
		if s.loudFrames < s.barge.Frames {
			return nil
		}
		// He is talking over her. Cut her off, drop the VAD state that
		// accumulated from the echo, and start listening for real — starting
		// with the frames that proved he was talking. Those used to be
		// thrown away, which clipped the first 150ms off his interruption,
		// and on a short one that is the whole first word.
		s.player.Stop()
		s.bargeIns++
		s.loudFrames = 0
		s.vad.Reset()
		s.resetWake()
		log.Printf("mavwaked: barge-in — stopped playback")
		s.replayRecent()
		return nil
	}

	// Not speaking. If we just stopped, make sure no echo-era state leaks
	// into the next utterance.
	if s.loudFrames != 0 {
		s.loudFrames = 0
		s.vad.Reset()
		// The wake word saw nothing during playback, so what it holds is from
		// before she spoke. Judging what he says next on it would score a
		// sentence that ended a reply ago.
		s.resetWake()
	}

	if s.startPendingNudge() {
		return nil
	}

	// The keyword is scored on the same frames the VAD sees, and only on the
	// ones that reach here: every path above returns while she is speaking, so
	// her own voice saying "Мэйвен" cannot wake her.
	pcm := PCMToI16(frame)
	if s.wake != nil && s.wake.Feed(pcm) {
		s.wakes++
		s.wakeUntil = s.now().Add(s.wakeWindow)
		log.Printf("mavwaked: keyword heard (score %.3f), listening for %s",
			s.wake.Score(), s.wakeWindow)
	}

	utt, state := s.vad.Feed(pcm)
	if state == StateSpeech || utt.Bytes == nil {
		return nil
	}
	return s.dispatch(ctx, utt)
}

// Nudge hands proactive audio to the session, to be spoken as soon as the
// capture loop finds a quiet moment. Safe to call from the push receiver
// goroutine; nothing else here is.
//
// A nudge arriving while one is already waiting REPLACES it. That is the
// contract internal/voice states for PushHandler: the next nudge replaces the
// stale one in his attention rather than dogpiling on it.
func (s *session) Nudge(a audio.Audio) {
	if len(a.Bytes) == 0 {
		return
	}
	s.nudgeMu.Lock()
	if s.pending != nil {
		log.Printf("mavwaked: nudge replaced one still waiting to be spoken")
	}
	s.pending = &a
	s.nudgeMu.Unlock()
}

// takeNudge removes and returns the waiting nudge, or nil.
func (s *session) takeNudge() *audio.Audio {
	s.nudgeMu.Lock()
	defer s.nudgeMu.Unlock()
	a := s.pending
	s.pending = nil
	return a
}

// startPendingNudge speaks a waiting nudge and reports whether it started
// one. It runs on the capture loop, past the half-duplex gate, so a nudge
// never cuts across a reply and never plays into a backlog drain.
//
// The VAD is reset first. Playback is about to suppress every frame until it
// ends, and a half-heard sentence left in the VAD would splice onto whatever
// he says afterwards. Barge-in needs no special case: it reads the player,
// and the player does not care which audio it is playing.
func (s *session) startPendingNudge() bool {
	a := s.takeNudge()
	if a == nil {
		return false
	}
	s.vad.Reset()
	s.nudges++
	log.Printf("mavwaked: speaking nudge (%.2fs audio)", a.Duration())
	s.player.Play(*a)
	return true
}

// awake reports whether an utterance ending now was addressed to her.
//
// With no wake word loaded every utterance is, which is exactly what mavwaked
// did before this gate existed. An operator with no model file gets the old
// daemon rather than a daemon that refuses to hear anything.
func (s *session) awake() bool {
	if s.wake == nil {
		return true
	}
	return s.now().Before(s.wakeUntil)
}

// resetWake drops the gate's streaming state when there is a gate.
func (s *session) resetWake() {
	if s.wake != nil {
		s.wake.Reset()
	}
}

// keepRecent stores a copy of one barge-in trigger frame, keeping at most
// barge.Frames of them.
func (s *session) keepRecent(frame []byte) {
	if len(s.recent) >= s.barge.Frames {
		copy(s.recent, s.recent[1:])
		s.recent = s.recent[:len(s.recent)-1]
	}
	s.recent = append(s.recent, append([]byte(nil), frame...))
}

// replayRecent feeds the trigger frames back into the freshly reset VAD, so
// his interruption starts where he started it.
//
// Feed cannot complete an utterance here: closing one needs silenceMs of
// trailing quiet and these frames are all above the barge-in threshold, which
// is far above the VAD floor. Any utterance it did return would be a fragment
// of a sentence he is still speaking, so it is not dispatched.
func (s *session) replayRecent() {
	for _, f := range s.recent {
		s.vad.Feed(PCMToI16(f))
	}
	s.recent = s.recent[:0]
}

// dispatch ships a complete utterance and plays whatever comes back.
//
// Every return path here has to deal with the backlog. Nothing reads the
// microphone while Send is in flight, so the audio piles up in arecord's pipe
// and the kernel buffer, and it arrives in a burst the moment this returns. A
// round-trip is p50 2.7s through the LLM router, which is around 90 frames of
// room, of him finishing his sentence, of the television.
//
// This used to reset the VAD on the reply path only, and for the wrong reason:
// the comment said the VAD had been accumulating during the round-trip, when
// in fact its state is exactly what Feed left it as. The two paths that had no
// reset are the ones that mattered, because neither of them starts playback
// and so neither is covered by the half-duplex gate. A text-only turn fed the
// whole backlog straight into the VAD, and a Send error did the same on every
// failed turn, so a dead socket drove a retry loop off nothing but backlog.
func (s *session) dispatch(ctx context.Context, utt audio.Audio) error {
	if !s.awake() {
		s.ignored++
		log.Printf("mavwaked: utterance ignored, keyword not heard (%.2fs, %d ignored so far)",
			utt.Duration(), s.ignored)
		s.vad.Reset()
		s.resetWake()
		return nil
	}
	// One keyword, one turn. A window that renewed itself on every reply would
	// leave the microphone open for as long as he kept talking, which is the
	// state this gate exists to end.
	s.wakeUntil = time.Time{}

	log.Printf("mavwaked: utterance complete (%.2fs, %d bytes), sending...", utt.Duration(), len(utt.Bytes))
	start := s.now()
	reply, err := s.sender.Send(ctx, utt, s.lang)
	defer s.dropBacklog(start)
	if err != nil {
		return err
	}
	// Count what was shipped, not what was attempted. This used to run
	// before the error check, so failed round-trips counted as sent.
	s.sent++
	if len(reply.Bytes) == 0 {
		log.Printf("mavwaked: empty reply audio (text only)")
		return nil
	}
	s.player.Play(reply)
	return nil
}

// dropBacklog resets the VAD and arranges for the frames captured during the
// round-trip to be thrown away as they arrive.
//
// Discarding them is also what keeps barge-in honest. The Frames guard is
// documented as "long enough that a door or a cough does not cut her off",
// which assumes the frames are real time. Draining a backlog delivers five
// frames in microseconds, so without this she could be cut off by audio
// recorded before she started speaking.
func (s *session) dropBacklog(start time.Time) {
	s.vad.Reset()
	s.resetWake()
	s.loudFrames = 0
	s.recent = s.recent[:0]
	if elapsed := s.now().Sub(start); elapsed > 0 {
		s.discard = int(elapsed / frameDuration)
	}
}