b2eb08bb51
runTurn computed the notice at step 2, after the confirm check had already returned. So he could be asked a question, walk off until it expired, come back and say "да" to a confirm that was still parked. The confirm answered and he never heard that the older request had been let go, even though the store had dropped it. Every other exit from runTurn carries the notice. The notice is now taken first and every early return wraps in withNotice, including the clarify answer path, where it is empty in practice because one dialogue id holds one question. Found in review of #50. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01TrVSBKe3RFDF4fGYKWYQnX
396 lines
18 KiB
Go
396 lines
18 KiB
Go
// Package main is mavend's voice wiring + reactive handler.
|
|
//
|
|
// Two responsibilities for the audio path:
|
|
//
|
|
// 1. CONSTRUCTION: read cfg.Voice, build the stt/tts/transcribers (Stub
|
|
// in-process by default, Remote via worker socket when configured),
|
|
// the router (stage-0 grammar + HashEmbedder classifier seeded with
|
|
// floor examples — production swaps in the ONNX multilingual model
|
|
// later), the voice TCP listener, the sessions registry, the
|
|
// voicesink, and wire the voicesink into the dispatcher's Voice slot.
|
|
//
|
|
// 2. HANDLER: a concrete voice.Handler that processes PushToTalk
|
|
// requests: stt → router → action → replier → tts → reply. The
|
|
// handler is what makes the audio round-trip "live". It wires to the
|
|
// CoreAPI in-process (the daemon already has it as ipc.NewStoreAPI(st)
|
|
// for module-IPC — the reactive path uses the same CoreAPI off the
|
|
// same store; both are the "core = the only key-holder" path through
|
|
// the daemon-embedded adapter).
|
|
//
|
|
// The "actions" handled today (per spec order; some deferred):
|
|
//
|
|
// - IntentFact: WriteFact via CoreAPI. The router's Slots.Key/Value feed
|
|
// the write; Source = "tap:voice" (the voice path is a tap, value=1.0
|
|
// confidence — the user said it out loud, maven trusts the capture).
|
|
// - IntentReminder: CreateReminder via CoreAPI. The router already
|
|
// resolved relative→absolute at capture ("in 4h" → fire_ts); the
|
|
// CoreAPI stores it as-is.
|
|
// - IntentAct: the tool executor runs the matched fn against the store's
|
|
// ENABLED allowlist (internal/tool). A verb not on it is scaffolded as a
|
|
// 'proposed' tool a human enables on the authed mavweb surface (never
|
|
// voice). Destructive tools run only after a spoken confirm turn.
|
|
// - IntentNote: chroma/vector-store deferred. The handler replies
|
|
// "saved" without persisting — a stub on the way to chroma.
|
|
// - IntentQuery: RAG-over-chroma deferred. The handler replies "I'll
|
|
// look that up later" — same shape as the other deferred slots.
|
|
// - Clarify: the router's stage-3 confidence gate fired; reply "didn't
|
|
// catch that, can you rephrase?"
|
|
//
|
|
// The Replier (voice.StubReplier today) renders the reply TEXT across all
|
|
// these branches. The TTS synthesiser (tts.Stub today) renders that text
|
|
// to audio. The PushToTalkResp carries BOTH so the client can play (audio)
|
|
// AND log (text) for tests asserting the round-trip.
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/kami/maven/internal/audio"
|
|
"github.com/kami/maven/internal/crawl"
|
|
"github.com/kami/maven/internal/dialogue"
|
|
"github.com/kami/maven/internal/ipc"
|
|
"github.com/kami/maven/internal/memory"
|
|
"github.com/kami/maven/internal/phraser"
|
|
"github.com/kami/maven/internal/router"
|
|
"github.com/kami/maven/internal/store"
|
|
"github.com/kami/maven/internal/stt"
|
|
"github.com/kami/maven/internal/tool"
|
|
"github.com/kami/maven/internal/tts"
|
|
"github.com/kami/maven/internal/ttsnorm"
|
|
"github.com/kami/maven/internal/voice"
|
|
"github.com/kami/maven/internal/weather"
|
|
)
|
|
|
|
// reactiveHandler — voice.Handler implementation. One method: turn a
|
|
// PushToTalkReq into a reply (audio + text). The handler is concurrency-
|
|
// safe (the wired stt/tts/router/api all are); called from per-conn
|
|
// goroutines on the voice.Server.
|
|
type reactiveHandler struct {
|
|
stt stt.Transcriber
|
|
tts tts.Synthesizer
|
|
router *router.Router
|
|
embedder router.Embedder // reused for note write/query (same model as the classifier)
|
|
api ipc.CoreAPI
|
|
tools *tool.Executor
|
|
matcher *tool.Matcher
|
|
phraser phraser.Phraser
|
|
replier voice.Replier
|
|
now func() time.Time
|
|
|
|
// crawler reads a web page he names out loud (queryWeb). nil ⇒ on-demand
|
|
// page reading is off, which is the default: no `crawl` block, no fetch.
|
|
crawler *crawl.Crawler
|
|
|
|
// feedsOn — whether any RSS feed is configured (config.Feeds). It changes
|
|
// only what she SAYS when asked and nothing is there: "ленты не настроены"
|
|
// instead of "ничего нового", which are different truths.
|
|
feedsOn bool
|
|
|
|
// home — the Home Assistant client (Vikunja #256). nil ⇒ the house is not
|
|
// configured, which is the default: no `smarthome` block, no reads, no
|
|
// switches. Control does not go through this field — it goes through the
|
|
// act allowlist and tool.Executor, like every other mutating act.
|
|
home *homeWiring
|
|
|
|
// netscan — the LAN scanner (Vikunja #257). nil ⇒ off, which is the
|
|
// default. A scan is a read, so it has no allowlist row; what keeps it
|
|
// safe is that its range comes from config and from nowhere else.
|
|
netscan *netWiring
|
|
|
|
weatherProvider weather.Provider
|
|
weatherLocation string // default location for weather queries
|
|
|
|
memStore memory.Store
|
|
dataStore *store.Store // direct store access for event extraction + pattern detection
|
|
|
|
// queryMinScore — the note-recall confidence gate. Top cosine below this ⇒
|
|
// "I don't know" instead of a guess. Tuned for the ONNX embedder; a knob, not
|
|
// load-bearing math (same posture as the presence thresholds). Set by
|
|
// wireVoice from VoiceConfig; default 0.55.
|
|
queryMinScore float64
|
|
// queryMinMargin — the second half of that gate: how far the top hit must
|
|
// beat the runner-up. 0 ⇒ margin off.
|
|
queryMinMargin float64
|
|
|
|
// timeParser — used as a fallback for stage-0 reminder grammar matches
|
|
// (where the extractor didn't run). Shared with the router's extractor.
|
|
// The production dateparser will replace StubDateTimeParser here too.
|
|
timeParser router.DateTimeParser
|
|
|
|
// dialogueSessions carries slots across turns for follow-ups (single-user
|
|
// box → one session slot, keyed voiceDialogueID). nil ⇒ no carry-over.
|
|
dialogueSessions *dialogue.SessionStore
|
|
|
|
// clarifyStore parks the request behind an open question she asked (see
|
|
// clarify.go). nil ⇒ she falls back to the canned "не поняла" reply.
|
|
clarifyStore *dialogue.ClarifyStore
|
|
|
|
// clarifyMaxAttempts — questions per request before she gives up out loud.
|
|
// 0 ⇒ dialogue.DefaultMaxAttempts (3). Set from VoiceConfig.
|
|
clarifyMaxAttempts int
|
|
|
|
// extractor parses the answer to an open question, with the same parsers
|
|
// the router's own stage-2 uses.
|
|
extractor router.Extractor
|
|
|
|
// pending destructive-act confirmation. A destructive act replies with a
|
|
// "выполнить X? да/нет" prompt and parks here; the NEXT utterance is read as
|
|
// the y/n answer. ponytail: single slot, single-user box — a second act
|
|
// while one waits overwrites it (last-asked wins); expires after confirmTTL.
|
|
mu sync.Mutex
|
|
pending *pendingAct
|
|
pendingRoutine *pendingRoutineConfirm // routine proposal awaiting y/n
|
|
pendingHexis *pendingHexisExec // mutating Hexis capability awaiting y/n
|
|
|
|
ecosystem *ecosystemWiring // nexus + hexis + praxis clients
|
|
}
|
|
|
|
// HandlePushToTalk — the full reactive round-trip. Each step's failure
|
|
// surfaces as a short reply text + empty audio OR an error; the voice
|
|
// server translates an error into a wire RpcError. Today the handler
|
|
// prefers a canned error-reply over an error return (a user-facing "didn't
|
|
// catch that" is better than a wire error the client surfaces as
|
|
// "internal"); the only error returned is a synthesizer fault (no audio
|
|
// to ship back).
|
|
func (h *reactiveHandler) HandlePushToTalk(ctx context.Context, req voice.PushToTalkReq, _ uint64) (voice.PushToTalkResp, error) {
|
|
// 1. stt — transcribe the audio.
|
|
text, _, err := h.stt.Transcribe(ctx, req.Audio)
|
|
if err != nil {
|
|
log.Printf("voice: stt error: %v", err)
|
|
return h.reply(ctx, "не получилось разобрать речь — попробуй ещё раз.", nil)
|
|
}
|
|
if text == "" {
|
|
return h.reply(ctx, "ничего не услышала — попробуй ещё раз.", nil)
|
|
}
|
|
log.Printf("voice: stt → %q", text)
|
|
|
|
// 2-5. the shared turn pipeline (confirm → clarify → route → dialogue →
|
|
// action → replier), identical to the text path.
|
|
replyText := h.runTurn(ctx, text)
|
|
|
|
// 6. tts — synthesise the reply text; return to the voice server which
|
|
// ships it back on the conn.
|
|
return h.reply(ctx, replyText, nil)
|
|
}
|
|
|
|
// handleText — the core reactive path without stt/tts. Used by the IPC Chat
|
|
// endpoint (and eventually by telegram). Splits out the audio bookends from
|
|
// HandlePushToTalk so text channels share the same routing logic.
|
|
func (h *reactiveHandler) handleText(ctx context.Context, text string) string {
|
|
log.Printf("voice: handleText: %q", text)
|
|
return h.runTurn(ctx, text)
|
|
}
|
|
|
|
// runTurn — the reactive turn pipeline shared by the voice and text entry
|
|
// points: expired-clarify notice → confirm answer → clarify answer → quiet
|
|
// toggle → route → dialogue merge → clarify question → action → replier.
|
|
// Takes the already-transcribed utterance, returns the reply text; the voice
|
|
// path wraps it in stt/tts, the text path returns it as-is.
|
|
//
|
|
// The ordering is load-bearing — see the step comments.
|
|
func (h *reactiveHandler) runTurn(ctx context.Context, text string) string {
|
|
// 1. expired clarify — a question was parked but its TTL ran out, so the
|
|
// request behind it is gone. Say that out loud (see clarify.go) and carry
|
|
// on: these words are still routed as a fresh utterance below, with the
|
|
// notice glued in front of whatever the fresh routing answers. Checked
|
|
// BEFORE the answer path: reading a parked question drops an expired one.
|
|
//
|
|
// Taken before the confirm check, not after, because a confirm turn returns
|
|
// early. He can be asked a question, walk off, come back and say "да" to a
|
|
// confirm that is still parked; computing the notice after that return meant
|
|
// he answered the confirm and never heard that the older request was let go.
|
|
expiredNotice := h.clarifyExpiredNotice()
|
|
|
|
// 2. confirm turn — if a destructive act is parked, this utterance is its
|
|
// y/n answer, not a fresh command. Handled before routing so "да" doesn't
|
|
// get classified as some other intent.
|
|
if reply, handled := h.resolveConfirm(ctx, text); handled {
|
|
return withNotice(expiredNotice, reply)
|
|
}
|
|
|
|
// 3. clarify answer — if she asked a live question last turn, this
|
|
// utterance is its answer, not a fresh command. After the confirm check: a
|
|
// y/n gate is armed by her own prompt and is the narrower claim on the
|
|
// utterance.
|
|
// A live question and an expired one cannot both exist for one dialogue id,
|
|
// so the notice is empty here in practice. withNotice anyway: every exit
|
|
// from runTurn carries it, and that is what stops the next one from
|
|
// forgetting.
|
|
if reply, handled := h.resolveClarifyAnswer(ctx, text); handled {
|
|
return withNotice(expiredNotice, reply)
|
|
}
|
|
|
|
// 4. quiet-hours toggle — keyword match, not classifier-dependent.
|
|
// "тихий режим" / "quiet on" would route through the classifier
|
|
// unreliably (it's a command, not a free-form query), so we match it
|
|
// before routing. Same pattern as the confirm turn above.
|
|
if reply, handled := h.resolveQuietToggle(ctx, text); handled {
|
|
return withNotice(expiredNotice, reply)
|
|
}
|
|
|
|
// 5. router — classify the utterance.
|
|
dec, err := h.router.Route(ctx, text, h.now())
|
|
if err != nil {
|
|
// ErrNoIntents ⇒ classifier unseeded (cold boot). reply with a
|
|
// "still warming up" rather than a wire error.
|
|
if errors.Is(err, router.ErrNoIntents) {
|
|
return withNotice(expiredNotice, "я ещё не понимаю свободную речь — скоро научусь.")
|
|
}
|
|
log.Printf("voice: router error: %v", err)
|
|
return withNotice(expiredNotice, "не получилось разобрать команду.")
|
|
}
|
|
log.Printf("voice: route result: intent=%s slots=%+v", dec.Intent, dec.Slots)
|
|
|
|
// 6. dialogue — fill this turn's missing slots from a prior same-intent
|
|
// turn (follow-ups like «напомни завтра» → «…позвонить маме»), then remember
|
|
// this turn for the next follow-up. Only same-intent, non-expired, non-
|
|
// clarify turns carry (see followUpMerge). Best-effort: nil store ⇒ skipped.
|
|
if h.dialogueSessions != nil {
|
|
now := h.now()
|
|
prev := h.dialogueSessions.Get(voiceDialogueID, now)
|
|
dec = followUpMerge(prev, dec, now)
|
|
if !dec.Clarify {
|
|
h.rememberTurn(prev, dec, now)
|
|
}
|
|
}
|
|
|
|
// 7. clarify — she is not sure. If one named thing is missing, ask about it
|
|
// and park the request (clarify.go); otherwise the replier's canned reply
|
|
// stands.
|
|
if dec.Clarify {
|
|
if question, asked := h.askClarify(dec); asked {
|
|
return withNotice(expiredNotice, question)
|
|
}
|
|
}
|
|
|
|
// 8. action — execute the decision's intent. errors here surface as
|
|
// short reply text (the user wants to know the action didn't land);
|
|
// the round-trip stays alive.
|
|
replyText := h.applyAction(ctx, dec)
|
|
log.Printf("voice: applyAction returned: %q", replyText)
|
|
|
|
// 9. replier — phrase the reply across the router decision.
|
|
if replyText == "" {
|
|
replyText = h.replier.Reply(dec)
|
|
}
|
|
return withNotice(expiredNotice, replyText)
|
|
}
|
|
|
|
// applyAction — executes the router's Decision. Intent-by-intent:
|
|
//
|
|
// - IntentFact: WriteFact via CoreAPI. Source = "tap:voice" (a voice
|
|
// capture is a tap; confidence 1.0).
|
|
// - IntentReminder: CreateReminder via CoreAPI.
|
|
// - IntentAct: tool-executor deferred (no-op today; the reply says so).
|
|
// - IntentNote / IntentQuery: chroma/RAG deferred (no-op; reply says so).
|
|
// - Clarify: the router's stage-3 fired; no action.
|
|
//
|
|
// Returns "" when the Replier should phrase the reply (the default path);
|
|
// returns a non-empty string when the action path wants to OVERRIDE the
|
|
// reply text (e.g. an action error the user should hear SPECIFICALLY, not
|
|
// a generic "ok"). Errors surface as a short reply text the user hears.
|
|
func (h *reactiveHandler) applyAction(ctx context.Context, dec router.Decision) string {
|
|
if dec.Clarify {
|
|
return "" // the Replier phrases clarify
|
|
}
|
|
if handler, ok := actionHandlers[dec.Intent]; ok {
|
|
return handler(h, ctx, dec)
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// replySystem answers system-observable queries using the handler's clock
|
|
// and (in future) system interfaces. The decision's utterance is parsed
|
|
// for keywords to determine what the user is asking about.
|
|
func (h *reactiveHandler) replySystem(ctx context.Context, dec router.Decision) string {
|
|
u := strings.ToLower(dec.Utterance)
|
|
now := h.now()
|
|
|
|
// stage-0 grammars catch the exact time/date patterns, but duration
|
|
// queries ("сколько времени прошло") bypass the grammar's build filter
|
|
// and can reach replySystem via the classifier path. Guard against them.
|
|
if hasDurationWords(u) {
|
|
return "пока не умею отвечать на этот вопрос."
|
|
}
|
|
|
|
switch {
|
|
case strings.Contains(u, "час") || strings.Contains(u, "врем"):
|
|
// "который час в киеве" — she keeps one clock, so any named place gets
|
|
// the honest answer. Never local time dressed up as the city's.
|
|
if mentionsUnknownPlace(u) {
|
|
return onlyLocalTimeReply
|
|
}
|
|
return "сейчас " + ruClock(now)
|
|
case strings.Contains(u, "день") || strings.Contains(u, "числ"):
|
|
// "какое число завтра" — answer for the day the user asked about,
|
|
// not today. Reuses the router's calendar day-word parser.
|
|
day := now
|
|
prefix := "сегодня"
|
|
if d, ok := router.ParseCalendarDate(u, now); ok {
|
|
day = d
|
|
prefix = dayPrefix(now, d)
|
|
} else if mentionsUnknownDay(u) {
|
|
// He named a day she cannot work out ("в пятницу", "через неделю").
|
|
// Answering today's date here would be the same silent wrong answer
|
|
// this arm was fixed for, so say what she can do instead.
|
|
return onlyNearDaysReply
|
|
}
|
|
dow := ruWeekdays[day.Weekday()]
|
|
month := ruMonths[day.Month()-1]
|
|
return fmt.Sprintf("%s %s, %d %s %d года", prefix, dow, day.Day(), month, day.Year())
|
|
case strings.Contains(u, "кто дома") || strings.Contains(u, "человек дома"):
|
|
return "присутствие пока не подключено к голосовому запросу."
|
|
case strings.Contains(u, "памят") || strings.Contains(u, "процессор") || strings.Contains(u, "загрузк") || strings.Contains(u, "статус") || strings.Contains(u, "работа") || strings.Contains(u, "сервис") || strings.Contains(u, "диск") || strings.Contains(u, "ip") || strings.Contains(u, "аптайм") || strings.Contains(u, "трафик") || strings.Contains(u, "интернет"):
|
|
return "системная статистика пока не подключена."
|
|
default:
|
|
return "пока не умею отвечать на этот вопрос."
|
|
}
|
|
}
|
|
|
|
// chatHistory collects dialogue turns from the session store for the current
|
|
// conversation. Returns prior user utterances (newest last) up to a depth of
|
|
// 4 turns. Returns nil when there's no session or no history.
|
|
func (h *reactiveHandler) chatHistory() []dialogue.Turn {
|
|
if h.dialogueSessions == nil {
|
|
return nil
|
|
}
|
|
now := h.now()
|
|
prev := h.dialogueSessions.Get(voiceDialogueID, now)
|
|
if prev == nil {
|
|
return nil
|
|
}
|
|
// History already includes the immediate prior turn (set by the dialogue
|
|
// merge at lines 373-395), plus up to 3 more from deeper history.
|
|
out := make([]dialogue.Turn, 0, 1+len(prev.History))
|
|
out = append(out, dialogue.Turn{
|
|
Intent: prev.Intent,
|
|
Slots: prev.Slots,
|
|
Text: prev.Slots.Text,
|
|
})
|
|
out = append(out, prev.History...)
|
|
return out
|
|
}
|
|
|
|
// reply wraps a text reply through TTS to produce a PushToTalkResp. If TTS
|
|
// fails, the response carries an empty audio + the text — the client can
|
|
// still display text if it can't play. The routedChannels field is
|
|
// reserved for a future "the dispatcher also forwarded to ntfy/telegram"
|
|
// reply (today the reactive path doesn't dispatch nudges; that's the loop
|
|
// tick's job).
|
|
func (h *reactiveHandler) reply(ctx context.Context, text string, _ []string) (voice.PushToTalkResp, error) {
|
|
spoken := ttsnorm.Speakable(text)
|
|
log.Printf("voice: reply → %q", text)
|
|
audioOut, err := h.tts.Synthesize(ctx, spoken)
|
|
if err != nil {
|
|
log.Printf("voice: tts error: %v", err)
|
|
return voice.PushToTalkResp{ReplyText: text, ReplyAudio: audio.Audio{}}, nil
|
|
}
|
|
return voice.PushToTalkResp{ReplyText: text, ReplyAudio: audioOut}, nil
|
|
}
|