Files
Maven/cmd/mavend/voice.go

543 lines
23 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Package main is mavend's voice wiring + reactive handler.
//
// Two responsibilities for the audio path:
//
// 1. CONSTRUCTION: read cfg.Voice, build the stt/tts/transcribers (Stub
// in-process by default, Remote via worker socket when configured),
// the router (stage-0 grammar + HashEmbedder classifier seeded with
// floor examples — production swaps in the ONNX multilingual model
// later), the voice TCP listener, the sessions registry, the
// voicesink, and wire the voicesink into the dispatcher's Voice slot.
//
// 2. HANDLER: a concrete voice.Handler that processes PushToTalk
// requests: stt → router → action → replier → tts → reply. The
// handler is what makes the audio round-trip "live". It wires to the
// CoreAPI in-process (the daemon already has it as ipc.NewStoreAPI(st)
// for module-IPC — the reactive path uses the same CoreAPI off the
// same store; both are the "core = the only key-holder" path through
// the daemon-embedded adapter).
//
// The "actions" handled today (per spec order; some deferred):
//
// - IntentFact: WriteFact via CoreAPI. The router's Slots.Key/Value feed
// the write; Source = "tap:voice" (the voice path is a tap, value=1.0
// confidence — the user said it out loud, maven trusts the capture).
// - IntentReminder: CreateReminder via CoreAPI. The router already
// resolved relative→absolute at capture ("in 4h" → fire_ts); the
// CoreAPI stores it as-is.
// - IntentAct: the tool executor runs the matched fn against the store's
// ENABLED allowlist (internal/tool). A verb not on it is scaffolded as a
// 'proposed' tool a human enables on the authed mavweb surface (never
// voice). Destructive tools run only after a spoken confirm turn.
// - IntentNote: chroma/vector-store deferred. The handler replies
// "saved" without persisting — a stub on the way to chroma.
// - IntentQuery: RAG-over-chroma deferred. The handler replies "I'll
// look that up later" — same shape as the other deferred slots.
// - Clarify: the router's stage-3 confidence gate fired; reply "didn't
// catch that, can you rephrase?"
//
// The Replier (voice.StubReplier today) renders the reply TEXT across all
// these branches. The TTS synthesiser (tts.Stub today) renders that text
// to audio. The PushToTalkResp carries BOTH so the client can play (audio)
// AND log (text) for tests asserting the round-trip.
package main
import (
"context"
"errors"
"fmt"
"log"
"strings"
"sync"
"time"
"unicode"
"github.com/kami/maven/internal/audio"
"github.com/kami/maven/internal/dialogue"
"github.com/kami/maven/internal/ipc"
"github.com/kami/maven/internal/memory"
"github.com/kami/maven/internal/pattern"
"github.com/kami/maven/internal/phraser"
"github.com/kami/maven/internal/router"
"github.com/kami/maven/internal/store"
"github.com/kami/maven/internal/stt"
"github.com/kami/maven/internal/tool"
"github.com/kami/maven/internal/tts"
"github.com/kami/maven/internal/ttsnorm"
"github.com/kami/maven/internal/voice"
"github.com/kami/maven/internal/weather"
)
// reactiveHandler — voice.Handler implementation. One method: turn a
// PushToTalkReq into a reply (audio + text). The handler is concurrency-
// safe (the wired stt/tts/router/api all are); called from per-conn
// goroutines on the voice.Server.
type reactiveHandler struct {
stt stt.Transcriber
tts tts.Synthesizer
router *router.Router
embedder router.Embedder // reused for note write/query (same model as the classifier)
api ipc.CoreAPI
tools *tool.Executor
matcher *tool.Matcher
phraser phraser.Phraser
replier voice.Replier
now func() time.Time
weatherProvider weather.Provider
weatherLocation string // default location for weather queries
memStore memory.Store
dataStore *store.Store // direct store access for event extraction + pattern detection
// queryMinScore — the note-recall confidence gate. Top cosine below this ⇒
// "I don't know" instead of a guess. Tuned for the ONNX embedder; a knob, not
// load-bearing math (same posture as the presence thresholds). Set by
// wireVoice from VoiceConfig; default 0.55.
queryMinScore float64
// queryMinMargin — the second half of that gate: how far the top hit must
// beat the runner-up. 0 ⇒ margin off.
queryMinMargin float64
// timeParser — used as a fallback for stage-0 reminder grammar matches
// (where the extractor didn't run). Shared with the router's extractor.
// The production dateparser will replace StubDateTimeParser here too.
timeParser router.DateTimeParser
// dialogueSessions carries slots across turns for follow-ups (single-user
// box → one session slot, keyed voiceDialogueID). nil ⇒ no carry-over.
dialogueSessions *dialogue.SessionStore
// clarifyStore parks the request behind an open question she asked (see
// clarify.go). nil ⇒ she falls back to the canned "не поняла" reply.
clarifyStore *dialogue.ClarifyStore
// clarifyMaxAttempts — questions per request before she gives up out loud.
// 0 ⇒ dialogue.DefaultMaxAttempts (3). Set from VoiceConfig.
clarifyMaxAttempts int
// extractor parses the answer to an open question, with the same parsers
// the router's own stage-2 uses.
extractor router.Extractor
// pending destructive-act confirmation. A destructive act replies with a
// "выполнить X? да/нет" prompt and parks here; the NEXT utterance is read as
// the y/n answer. ponytail: single slot, single-user box — a second act
// while one waits overwrites it (last-asked wins); expires after confirmTTL.
mu sync.Mutex
pending *pendingAct
pendingRoutine *pendingRoutineConfirm // routine proposal awaiting y/n
pendingHexis *pendingHexisExec // mutating Hexis capability awaiting y/n
ecosystem *ecosystemWiring // nexus + hexis + praxis clients
}
// HandlePushToTalk — the full reactive round-trip. Each step's failure
// surfaces as a short reply text + empty audio OR an error; the voice
// server translates an error into a wire RpcError. Today the handler
// prefers a canned error-reply over an error return (a user-facing "didn't
// catch that" is better than a wire error the client surfaces as
// "internal"); the only error returned is a synthesizer fault (no audio
// to ship back).
func (h *reactiveHandler) HandlePushToTalk(ctx context.Context, req voice.PushToTalkReq, _ uint64) (voice.PushToTalkResp, error) {
// 1. stt — transcribe the audio.
text, _, err := h.stt.Transcribe(ctx, req.Audio)
if err != nil {
log.Printf("voice: stt error: %v", err)
return h.reply(ctx, "не получилось разобрать речь — попробуй ещё раз.", nil)
}
if text == "" {
return h.reply(ctx, "ничего не услышала — попробуй ещё раз.", nil)
}
log.Printf("voice: stt → %q", text)
// 2-5. the shared turn pipeline (confirm → clarify → route → dialogue →
// action → replier), identical to the text path.
replyText := h.runTurn(ctx, text)
// 6. tts — synthesise the reply text; return to the voice server which
// ships it back on the conn.
return h.reply(ctx, replyText, nil)
}
// handleText — the core reactive path without stt/tts. Used by the IPC Chat
// endpoint (and eventually by telegram). Splits out the audio bookends from
// HandlePushToTalk so text channels share the same routing logic.
func (h *reactiveHandler) handleText(ctx context.Context, text string) string {
log.Printf("voice: handleText: %q", text)
return h.runTurn(ctx, text)
}
// runTurn — the reactive turn pipeline shared by the voice and text entry
// points: confirm answer → expired-clarify notice → clarify answer → quiet
// toggle → route → dialogue merge → clarify question → action → replier.
// Takes the already-transcribed utterance, returns the reply text; the voice
// path wraps it in stt/tts, the text path returns it as-is.
//
// The ordering is load-bearing — see the step comments.
func (h *reactiveHandler) runTurn(ctx context.Context, text string) string {
// 1. confirm turn — if a destructive act is parked, this utterance is its
// y/n answer, not a fresh command. Handled before routing so "да" doesn't
// get classified as some other intent.
if reply, handled := h.resolveConfirm(ctx, text); handled {
return reply
}
// 2. expired clarify — a question was parked but its TTL ran out, so the
// request behind it is gone. Say that out loud (see clarify.go) and carry
// on: these words are still routed as a fresh utterance below, with the
// notice glued in front of whatever the fresh routing answers. Checked
// BEFORE the answer path: reading a parked question drops an expired one.
expiredNotice := h.clarifyExpiredNotice()
// 3. clarify answer — if she asked a live question last turn, this
// utterance is its answer, not a fresh command. After the confirm check: a
// y/n gate is armed by her own prompt and is the narrower claim on the
// utterance.
if reply, handled := h.resolveClarifyAnswer(ctx, text); handled {
return reply
}
// 4. quiet-hours toggle — keyword match, not classifier-dependent.
// "тихий режим" / "quiet on" would route through the classifier
// unreliably (it's a command, not a free-form query), so we match it
// before routing. Same pattern as the confirm turn above.
if reply, handled := h.resolveQuietToggle(ctx, text); handled {
return withNotice(expiredNotice, reply)
}
// 5. router — classify the utterance.
dec, err := h.router.Route(ctx, text, h.now())
if err != nil {
// ErrNoIntents ⇒ classifier unseeded (cold boot). reply with a
// "still warming up" rather than a wire error.
if errors.Is(err, router.ErrNoIntents) {
return withNotice(expiredNotice, "я ещё не понимаю свободную речь — скоро научусь.")
}
log.Printf("voice: router error: %v", err)
return withNotice(expiredNotice, "не получилось разобрать команду.")
}
log.Printf("voice: route result: intent=%s slots=%+v", dec.Intent, dec.Slots)
// 6. dialogue — fill this turn's missing slots from a prior same-intent
// turn (follow-ups like «напомни завтра» → «…позвонить маме»), then remember
// this turn for the next follow-up. Only same-intent, non-expired, non-
// clarify turns carry (see followUpMerge). Best-effort: nil store ⇒ skipped.
if h.dialogueSessions != nil {
now := h.now()
prev := h.dialogueSessions.Get(voiceDialogueID, now)
dec = followUpMerge(prev, dec, now)
if !dec.Clarify {
h.rememberTurn(prev, dec, now)
}
}
// 7. clarify — she is not sure. If one named thing is missing, ask about it
// and park the request (clarify.go); otherwise the replier's canned reply
// stands.
if dec.Clarify {
if question, asked := h.askClarify(dec); asked {
return withNotice(expiredNotice, question)
}
}
// 8. action — execute the decision's intent. errors here surface as
// short reply text (the user wants to know the action didn't land);
// the round-trip stays alive.
replyText := h.applyAction(ctx, dec)
log.Printf("voice: applyAction returned: %q", replyText)
// 9. replier — phrase the reply across the router decision.
if replyText == "" {
replyText = h.replier.Reply(dec)
}
return withNotice(expiredNotice, replyText)
}
// applyAction — executes the router's Decision. Intent-by-intent:
//
// - IntentFact: WriteFact via CoreAPI. Source = "tap:voice" (a voice
// capture is a tap; confidence 1.0).
// - IntentReminder: CreateReminder via CoreAPI.
// - IntentAct: tool-executor deferred (no-op today; the reply says so).
// - IntentNote / IntentQuery: chroma/RAG deferred (no-op; reply says so).
// - Clarify: the router's stage-3 fired; no action.
//
// Returns "" when the Replier should phrase the reply (the default path);
// returns a non-empty string when the action path wants to OVERRIDE the
// reply text (e.g. an action error the user should hear SPECIFICALLY, not
// a generic "ok"). Errors surface as a short reply text the user hears.
func (h *reactiveHandler) applyAction(ctx context.Context, dec router.Decision) string {
if dec.Clarify {
return "" // the Replier phrases clarify
}
if handler, ok := actionHandlers[dec.Intent]; ok {
return handler(h, ctx, dec)
}
return ""
}
// detectPattern extracts an event from the written fact and runs the pattern
// detector. If a stable recurring pattern is found and no proposed routine
// exists for this action+object yet, one is created and the user is prompted
// to confirm via the park() mechanism. Returns the suggestion phrase when a
// new proposal was created and parked; "" otherwise.
func (h *reactiveHandler) detectPattern(ctx context.Context, factID int64, key, value string, ts time.Time) string {
ev := pattern.Extract(factID, key, value, ts)
if ev == nil {
return "" // not an actionable event
}
if _, err := h.dataStore.CreateEvent(ctx, factID, ev.Action, ev.Object, ts); err != nil {
log.Printf("voice: create event: %v", err)
return ""
}
// Detect+propose (Vikunja #43) is shared with the digestion tick's
// proactive scan — see patterns.go. Event *extraction* above stays here,
// tied to this fact write; detection over the accumulated history does
// not need to happen right now for the voice path to have already done
// its job — it's dedupe-safe to also let the next tick find the same
// pattern independently.
r, id, err := detectAndPropose(ctx, h.dataStore, ev.Action, ev.Object, ts)
if err != nil {
log.Printf("voice: detect pattern %s/%s: %v", ev.Action, ev.Object, err)
return ""
}
if r == nil {
return "" // not enough data, too irregular, or already proposed/decided
}
log.Printf("voice: proposed routine: %s/%s every %.1f days", r.Action, r.Object, r.IntervalDays)
// Park the proposal for voice confirmation.
phrase := pattern.PhraseRoutine(r)
h.mu.Lock()
h.pendingRoutine = &pendingRoutineConfirm{
routineID: id,
action: r.Action,
object: r.Object,
interval: r.IntervalDays,
phrase: phrase,
expiry: ts.Add(confirmTTL),
}
h.mu.Unlock()
return phrase
}
// resolveQuietToggle — pre-route keyword check. Returns (reply, true) when
// the utterance is a quiet-on/off command; ("", false) otherwise. Called from
// runTurn BEFORE the router so a classifier miscue can't drop it — which means
// both the voice path and the text path (mavweb /api/chat, telegram) reach it,
// so a false positive here is a network-reachable way to flip a daemon-wide
// setting. See classifyQuietToggle for the matching rule.
func (h *reactiveHandler) resolveQuietToggle(ctx context.Context, text string) (string, bool) {
on, off := classifyQuietToggle(text)
if !on && !off {
return "", false
}
val := "false"
reply := "тихий режим выключен."
if on {
val = "true"
reply = "тихий режим включён. буду реже напоминать."
}
if _, err := h.api.WriteFact(ctx, ipc.WriteFactReq{
Ts: h.now(),
Kind: "config",
Key: "quiet_hours",
Value: val,
Source: "tap:voice",
Confidence: 1.0,
}); err != nil {
log.Printf("voice: write quiet_hours: %v", err)
return "не получилось переключить тихий режим.", true
}
return reply, true
}
// quietInflections — the inflectional endings a stem may carry and still be
// the same word. Adjective/adverb/noun/verb endings, all ≤3 letters. This is
// what separates "тихий"/"тихом"/"тихо" (stem "тих" + a real ending) from
// "тихонько"/"потихоньку", which are different words: "онько" is not an
// ending, and "потихоньку" doesn't start with the stem at all.
var quietInflections = []string{
"", "а", "е", "и", "й", "о", "у", "ы", "ю", "я",
"ая", "ее", "ей", "ем", "ие", "ий", "им", "их", "ия", "ию", "ое", "ой", "ом", "ую", "ые", "ый", "ым", "ых", "ья",
"ами", "ого", "ому", "ыми", "ать", "ить", "ять",
}
// quietStem reports whether tok is the given stem carrying at most one
// inflectional ending. Word boundaries come from tokenisation (see
// quietTokens), not from a regexp — Go's \b is ASCII-oriented and treats every
// Cyrillic letter as a non-word character, so `\bтих\b` would happily match
// inside "тихонько". Comparing whole tokens sidesteps that entirely.
func quietStem(tok, stem string) bool {
if !strings.HasPrefix(tok, stem) {
return false
}
suffix := tok[len(stem):]
for _, e := range quietInflections {
if suffix == e {
return true
}
}
return false
}
// quietTokens splits an utterance into lowercase word tokens, dropping
// punctuation and spacing. Unicode-aware, so Cyrillic words tokenise the same
// way ASCII ones do.
func quietTokens(text string) []string {
return strings.FieldsFunc(strings.ToLower(strings.TrimSpace(text)), func(r rune) bool {
return !unicode.IsLetter(r) && !unicode.IsDigit(r)
})
}
// quietPhrase matches a pattern (a sequence of stems) against the token list.
// Multi-word patterns match any contiguous run of tokens — "включи тихий
// режим" carries "тихий режим". Single-word patterns match ONLY when they are
// the whole utterance: bare "тихо" is a command, but "в комнате тихо" is a
// remark about the room and must not flip a daemon-wide setting.
func quietPhrase(tokens, pattern []string) bool {
if len(pattern) == 0 || len(tokens) < len(pattern) {
return false
}
if len(pattern) == 1 {
return len(tokens) == 1 && quietStem(tokens[0], pattern[0])
}
for i := 0; i+len(pattern) <= len(tokens); i++ {
hit := true
for j, stem := range pattern {
if !quietStem(tokens[i+j], stem) {
hit = false
break
}
}
if hit {
return true
}
}
return false
}
// quietOffPhrases / quietOnPhrases — the toggle vocabulary, as stem sequences.
var (
quietOffPhrases = [][]string{
{"quiet", "off"}, {"quiet", "end"},
{"громк", "режим"}, {"шумн", "режим"},
{"отмен", "тих"}, {"выключ", "тих"}, {"не", "тих"},
}
quietOnPhrases = [][]string{
{"quiet", "on"}, {"quiet", "mode"},
{"тих", "режим"}, {"не", "шум"}, {"не", "беспоко"},
{"тих"},
}
)
// classifyQuietToggle reads an utterance as a quiet-mode command. OFF is
// resolved before ON for the same reason classifyConfirm checks negatives
// first: the OFF phrases are built out of the ON words ("выключи тихий"
// contains "тихий"), so scanning ON first would shadow them and "выключи
// тихий режим" would turn quiet mode on. Negation wins.
func classifyQuietToggle(text string) (on, off bool) {
tokens := quietTokens(text)
for _, p := range quietOffPhrases {
if quietPhrase(tokens, p) {
return false, true
}
}
for _, p := range quietOnPhrases {
if quietPhrase(tokens, p) {
return true, false
}
}
return false, false
}
// replySystem answers system-observable queries using the handler's clock
// and (in future) system interfaces. The decision's utterance is parsed
// for keywords to determine what the user is asking about.
func (h *reactiveHandler) replySystem(ctx context.Context, dec router.Decision) string {
u := strings.ToLower(dec.Utterance)
now := h.now()
// stage-0 grammars catch the exact time/date patterns, but duration
// queries ("сколько времени прошло") bypass the grammar's build filter
// and can reach replySystem via the classifier path. Guard against them.
if hasDurationWords(u) {
return "пока не умею отвечать на этот вопрос."
}
switch {
case strings.Contains(u, "час") || strings.Contains(u, "врем"):
// "который час в киеве" — she keeps one clock, so any named place gets
// the honest answer. Never local time dressed up as the city's.
if mentionsUnknownPlace(u) {
return onlyLocalTimeReply
}
return "сейчас " + ruClock(now)
case strings.Contains(u, "день") || strings.Contains(u, "числ"):
// "какое число завтра" — answer for the day the user asked about,
// not today. Reuses the router's calendar day-word parser.
day := now
prefix := "сегодня"
if d, ok := router.ParseCalendarDate(u, now); ok {
day = d
prefix = dayPrefix(now, d)
} else if mentionsUnknownDay(u) {
// He named a day she cannot work out ("в пятницу", "через неделю").
// Answering today's date here would be the same silent wrong answer
// this arm was fixed for, so say what she can do instead.
return onlyNearDaysReply
}
dow := ruWeekdays[day.Weekday()]
month := ruMonths[day.Month()-1]
return fmt.Sprintf("%s %s, %d %s %d года", prefix, dow, day.Day(), month, day.Year())
case strings.Contains(u, "кто дома") || strings.Contains(u, "человек дома"):
return "присутствие пока не подключено к голосовому запросу."
case strings.Contains(u, "памят") || strings.Contains(u, "процессор") || strings.Contains(u, "загрузк") || strings.Contains(u, "статус") || strings.Contains(u, "работа") || strings.Contains(u, "сервис") || strings.Contains(u, "диск") || strings.Contains(u, "ip") || strings.Contains(u, "аптайм") || strings.Contains(u, "трафик") || strings.Contains(u, "интернет"):
return "системная статистика пока не подключена."
default:
return "пока не умею отвечать на этот вопрос."
}
}
// chatHistory collects dialogue turns from the session store for the current
// conversation. Returns prior user utterances (newest last) up to a depth of
// 4 turns. Returns nil when there's no session or no history.
func (h *reactiveHandler) chatHistory() []dialogue.Turn {
if h.dialogueSessions == nil {
return nil
}
now := h.now()
prev := h.dialogueSessions.Get(voiceDialogueID, now)
if prev == nil {
return nil
}
// History already includes the immediate prior turn (set by the dialogue
// merge at lines 373-395), plus up to 3 more from deeper history.
out := make([]dialogue.Turn, 0, 1+len(prev.History))
out = append(out, dialogue.Turn{
Intent: prev.Intent,
Slots: prev.Slots,
Text: prev.Slots.Text,
})
out = append(out, prev.History...)
return out
}
// reply wraps a text reply through TTS to produce a PushToTalkResp. If TTS
// fails, the response carries an empty audio + the text — the client can
// still display text if it can't play. The routedChannels field is
// reserved for a future "the dispatcher also forwarded to ntfy/telegram"
// reply (today the reactive path doesn't dispatch nudges; that's the loop
// tick's job).
func (h *reactiveHandler) reply(ctx context.Context, text string, _ []string) (voice.PushToTalkResp, error) {
spoken := ttsnorm.Speakable(text)
log.Printf("voice: reply → %q", text)
audioOut, err := h.tts.Synthesize(ctx, spoken)
if err != nil {
log.Printf("voice: tts error: %v", err)
return voice.PushToTalkResp{ReplyText: text, ReplyAudio: audio.Audio{}}, nil
}
return voice.PushToTalkResp{ReplyText: text, ReplyAudio: audioOut}, nil
}