// Package main is mavend's voice wiring + reactive handler. // // Two responsibilities for the audio path: // // 1. CONSTRUCTION: read cfg.Voice, build the stt/tts/transcribers (Stub // in-process by default, Remote via worker socket when configured), // the router (stage-0 grammar + HashEmbedder classifier seeded with // floor examples — production swaps in the ONNX multilingual model // later), the voice TCP listener, the sessions registry, the // voicesink, and wire the voicesink into the dispatcher's Voice slot. // // 2. HANDLER: a concrete voice.Handler that processes PushToTalk // requests: stt → router → action → replier → tts → reply. The // handler is what makes the audio round-trip "live". It wires to the // CoreAPI in-process (the daemon already has it as ipc.NewStoreAPI(st) // for module-IPC — the reactive path uses the same CoreAPI off the // same store; both are the "core = the only key-holder" path through // the daemon-embedded adapter). // // The "actions" handled today (per spec order; some deferred): // // - IntentFact: WriteFact via CoreAPI. The router's Slots.Key/Value feed // the write; Source = "tap:voice" (the voice path is a tap, value=1.0 // confidence — the user said it out loud, maven trusts the capture). // - IntentReminder: CreateReminder via CoreAPI. The router already // resolved relative→absolute at capture ("in 4h" → fire_ts); the // CoreAPI stores it as-is. // - IntentAct: the tool executor runs the matched fn against the store's // ENABLED allowlist (internal/tool). A verb not on it is scaffolded as a // 'proposed' tool a human enables on the authed mavweb surface (never // voice). Destructive tools run only after a spoken confirm turn. // - IntentNote: chroma/vector-store deferred. The handler replies // "saved" without persisting — a stub on the way to chroma. // - IntentQuery: RAG-over-chroma deferred. The handler replies "I'll // look that up later" — same shape as the other deferred slots. // - Clarify: the router's stage-3 confidence gate fired; reply "didn't // catch that, can you rephrase?" // // The Replier (voice.StubReplier today) renders the reply TEXT across all // these branches. The TTS synthesiser (tts.Stub today) renders that text // to audio. The PushToTalkResp carries BOTH so the client can play (audio) // AND log (text) for tests asserting the round-trip. package main import ( "context" "errors" "fmt" "log" "strings" "sync" "time" "github.com/kami/maven/internal/audio" "github.com/kami/maven/internal/dialogue" "github.com/kami/maven/internal/ipc" "github.com/kami/maven/internal/memory" "github.com/kami/maven/internal/pattern" "github.com/kami/maven/internal/phraser" "github.com/kami/maven/internal/router" "github.com/kami/maven/internal/store" "github.com/kami/maven/internal/stt" "github.com/kami/maven/internal/tool" "github.com/kami/maven/internal/tts" "github.com/kami/maven/internal/ttsnorm" "github.com/kami/maven/internal/voice" "github.com/kami/maven/internal/weather" ) // reactiveHandler — voice.Handler implementation. One method: turn a // PushToTalkReq into a reply (audio + text). The handler is concurrency- // safe (the wired stt/tts/router/api all are); called from per-conn // goroutines on the voice.Server. type reactiveHandler struct { stt stt.Transcriber tts tts.Synthesizer router *router.Router embedder router.Embedder // reused for note write/query (same model as the classifier) api ipc.CoreAPI tools *tool.Executor matcher *tool.Matcher phraser phraser.Phraser replier voice.Replier now func() time.Time weatherProvider weather.Provider weatherLocation string // default location for weather queries memStore memory.Store dataStore *store.Store // direct store access for event extraction + pattern detection // queryMinScore — the note-recall confidence gate. Top cosine below this ⇒ // "I don't know" instead of a guess. Tuned for the ONNX embedder; a knob, not // load-bearing math (same posture as the presence thresholds). Set by // wireVoice from VoiceConfig; default 0.55. queryMinScore float64 // queryMinMargin — the second half of that gate: how far the top hit must // beat the runner-up. 0 ⇒ margin off. queryMinMargin float64 // timeParser — used as a fallback for stage-0 reminder grammar matches // (where the extractor didn't run). Shared with the router's extractor. // The production dateparser will replace StubDateTimeParser here too. timeParser router.DateTimeParser // dialogueSessions carries slots across turns for follow-ups (single-user // box → one session slot, keyed voiceDialogueID). nil ⇒ no carry-over. dialogueSessions *dialogue.SessionStore // clarifyStore parks the request behind an open question she asked (see // clarify.go). nil ⇒ she falls back to the canned "не поняла" reply. clarifyStore *dialogue.ClarifyStore // clarifyMaxAttempts — questions per request before she gives up out loud. // 0 ⇒ dialogue.DefaultMaxAttempts (3). Set from VoiceConfig. clarifyMaxAttempts int // extractor parses the answer to an open question, with the same parsers // the router's own stage-2 uses. extractor router.Extractor // pending destructive-act confirmation. A destructive act replies with a // "выполнить X? да/нет" prompt and parks here; the NEXT utterance is read as // the y/n answer. ponytail: single slot, single-user box — a second act // while one waits overwrites it (last-asked wins); expires after confirmTTL. mu sync.Mutex pending *pendingAct pendingRoutine *pendingRoutineConfirm // routine proposal awaiting y/n pendingHexis *pendingHexisExec // mutating Hexis capability awaiting y/n ecosystem *ecosystemWiring // nexus + hexis + praxis clients } // HandlePushToTalk — the full reactive round-trip. Each step's failure // surfaces as a short reply text + empty audio OR an error; the voice // server translates an error into a wire RpcError. Today the handler // prefers a canned error-reply over an error return (a user-facing "didn't // catch that" is better than a wire error the client surfaces as // "internal"); the only error returned is a synthesizer fault (no audio // to ship back). func (h *reactiveHandler) HandlePushToTalk(ctx context.Context, req voice.PushToTalkReq, _ uint64) (voice.PushToTalkResp, error) { // 1. stt — transcribe the audio. text, _, err := h.stt.Transcribe(ctx, req.Audio) if err != nil { log.Printf("voice: stt error: %v", err) return h.reply(ctx, "не получилось разобрать речь — попробуй ещё раз.", nil) } if text == "" { return h.reply(ctx, "ничего не услышала — попробуй ещё раз.", nil) } log.Printf("voice: stt → %q", text) // 1b. confirm turn — if a destructive act is parked, this utterance is its // y/n answer, not a fresh command. Handled before routing so "да" doesn't // get classified as some other intent. if reply, handled := h.resolveConfirm(ctx, text); handled { return h.reply(ctx, reply, nil) } // 1b2. expired clarify — a question was parked but its TTL ran out, so the // request behind it is gone. Say that out loud (see clarify.go) and carry // on: these words are still routed as a fresh utterance below, with the // notice glued in front of whatever the fresh routing answers. Checked // BEFORE the answer path: reading a parked question drops an expired one. expiredNotice := h.clarifyExpiredNotice() // 1b3. clarify answer — if she asked a live question last turn, this // utterance is its answer, not a fresh command. After the confirm check: a // y/n gate is armed by her own prompt and is the narrower claim on the // utterance. if reply, handled := h.resolveClarifyAnswer(ctx, text); handled { return h.reply(ctx, reply, nil) } // 1c. quiet-hours toggle — keyword match, not classifier-dependent. // "тихий режим" / "quiet on" would route through the classifier // unreliably (it's a command, not a free-form query), so we match it // before routing. Same pattern as the confirm turn above. if reply, handled := h.resolveQuietToggle(ctx, text); handled { return h.reply(ctx, withNotice(expiredNotice, reply), nil) } // 2. router — classify the utterance. dec, err := h.router.Route(ctx, text, h.now()) if err != nil { // ErrNoIntents ⇒ classifier unseeded (cold boot). reply with a // "still warming up" rather than a wire error. if errors.Is(err, router.ErrNoIntents) { return h.reply(ctx, withNotice(expiredNotice, "я ещё не понимаю свободную речь — скоро научусь."), nil) } log.Printf("voice: router error: %v", err) return h.reply(ctx, withNotice(expiredNotice, "не получилось разобрать команду."), nil) } // 2b. dialogue — fill this turn's missing slots from a prior same-intent // turn (follow-ups like «напомни завтра» → «…позвонить маме»), then remember // this turn for the next follow-up. Only same-intent, non-expired, non- // clarify turns carry (see followUpMerge). Best-effort: nil store ⇒ skipped. if h.dialogueSessions != nil { now := h.now() prev := h.dialogueSessions.Get(voiceDialogueID, now) dec = followUpMerge(prev, dec, now) if !dec.Clarify { h.rememberTurn(prev, dec, now) } } // 2c. clarify — she is not sure. If one named thing is missing, ask about it // and park the request (clarify.go); otherwise the replier's canned reply // stands. if dec.Clarify { if question, asked := h.askClarify(dec); asked { return h.reply(ctx, withNotice(expiredNotice, question), nil) } } // 3. action — execute the decision's intent. errors here surface as // short reply text (the user wants to know the action didn't land); // the round-trip stays alive. replyText := h.applyAction(ctx, dec) // 4. replier — phrase the reply across the router decision. if replyText == "" { replyText = h.replier.Reply(dec) } // 5. tts — synthesise the reply text; return to the voice server which // ships it back on the conn. return h.reply(ctx, withNotice(expiredNotice, replyText), nil) } // handleText — the core reactive path without stt/tts: confirm check → // route → dialogue → action → replier. Used by the IPC Chat endpoint // (and eventually by telegram). Splits out the audio bookends from // HandlePushToTalk so text channels share the same routing logic. func (h *reactiveHandler) handleText(ctx context.Context, text string) string { log.Printf("voice: handleText: %q", text) // 1b. confirm turn — if a destructive act is parked, this utterance is its // y/n answer. Same check as HandlePushToTalk. if reply, handled := h.resolveConfirm(ctx, text); handled { return reply } // 1b2/1b3. expired clarify then clarify answer — same order and reasons as // HandlePushToTalk. expiredNotice := h.clarifyExpiredNotice() if reply, handled := h.resolveClarifyAnswer(ctx, text); handled { return reply } // 2. router — classify the utterance. dec, err := h.router.Route(ctx, text, h.now()) if err != nil { if errors.Is(err, router.ErrNoIntents) { return withNotice(expiredNotice, "я ещё не понимаю свободную речь — скоро научусь.") } log.Printf("voice: handleText router error: %v", err) return withNotice(expiredNotice, "не получилось разобрать команду.") } log.Printf("voice: route result: intent=%s slots=%+v", dec.Intent, dec.Slots) // 2b. dialogue — same as HandlePushToTalk. if h.dialogueSessions != nil { now := h.now() prev := h.dialogueSessions.Get(voiceDialogueID, now) dec = followUpMerge(prev, dec, now) if !dec.Clarify { h.rememberTurn(prev, dec, now) } } // 2c. clarify — same as HandlePushToTalk: ask about the one missing thing. if dec.Clarify { if question, asked := h.askClarify(dec); asked { return withNotice(expiredNotice, question) } } // 3. action — execute the decision's intent. replyText := h.applyAction(ctx, dec) log.Printf("voice: applyAction returned: %q", replyText) // 4. replier — phrase the reply when applyAction returned "". if replyText == "" { replyText = h.replier.Reply(dec) } return withNotice(expiredNotice, replyText) } // applyAction — executes the router's Decision. Intent-by-intent: // // - IntentFact: WriteFact via CoreAPI. Source = "tap:voice" (a voice // capture is a tap; confidence 1.0). // - IntentReminder: CreateReminder via CoreAPI. // - IntentAct: tool-executor deferred (no-op today; the reply says so). // - IntentNote / IntentQuery: chroma/RAG deferred (no-op; reply says so). // - Clarify: the router's stage-3 fired; no action. // // Returns "" when the Replier should phrase the reply (the default path); // returns a non-empty string when the action path wants to OVERRIDE the // reply text (e.g. an action error the user should hear SPECIFICALLY, not // a generic "ok"). Errors surface as a short reply text the user hears. func (h *reactiveHandler) applyAction(ctx context.Context, dec router.Decision) string { if dec.Clarify { return "" // the Replier phrases clarify } if handler, ok := actionHandlers[dec.Intent]; ok { return handler(h, ctx, dec) } return "" } // detectPattern extracts an event from the written fact and runs the pattern // detector. If a stable recurring pattern is found and no proposed routine // exists for this action+object yet, one is created and the user is prompted // to confirm via the park() mechanism. Returns the suggestion phrase when a // new proposal was created and parked; "" otherwise. func (h *reactiveHandler) detectPattern(ctx context.Context, factID int64, key, value string, ts time.Time) string { ev := pattern.Extract(factID, key, value, ts) if ev == nil { return "" // not an actionable event } if _, err := h.dataStore.CreateEvent(ctx, factID, ev.Action, ev.Object, ts); err != nil { log.Printf("voice: create event: %v", err) return "" } // Detect+propose (Vikunja #43) is shared with the digestion tick's // proactive scan — see patterns.go. Event *extraction* above stays here, // tied to this fact write; detection over the accumulated history does // not need to happen right now for the voice path to have already done // its job — it's dedupe-safe to also let the next tick find the same // pattern independently. r, id, err := detectAndPropose(ctx, h.dataStore, ev.Action, ev.Object, ts) if err != nil { log.Printf("voice: detect pattern %s/%s: %v", ev.Action, ev.Object, err) return "" } if r == nil { return "" // not enough data, too irregular, or already proposed/decided } log.Printf("voice: proposed routine: %s/%s every %.1f days", r.Action, r.Object, r.IntervalDays) // Park the proposal for voice confirmation. phrase := pattern.PhraseRoutine(r) h.mu.Lock() h.pendingRoutine = &pendingRoutineConfirm{ routineID: id, action: r.Action, object: r.Object, interval: r.IntervalDays, phrase: phrase, expiry: ts.Add(confirmTTL), } h.mu.Unlock() return phrase } // resolveQuietToggle — pre-route keyword check. Returns (reply, true) when // the utterance is a quiet-on/off command; ("", false) otherwise. Called from // HandlePushToTalk BEFORE the router so a classifier miscue can't drop it. func (h *reactiveHandler) resolveQuietToggle(ctx context.Context, text string) (string, bool) { u := strings.ToLower(strings.TrimSpace(text)) var on, off bool // Match as whole-token phrases so "тихий" in "тихий режим включи" still // catches, but "тихий" alone in "очень тихий сегодня день" doesn't fire. // The confirm turn is handled above, so "да"/"нет" won't reach here. for _, kw := range []string{"quiet on", "quiet mode", "тихий режим", "тихий", "не шуми", "не беспокоить", "тихо"} { if strings.Contains(u, kw) { on = true break } } if !on { for _, kw := range []string{"quiet off", "quiet end", "громкий режим", "шумный режим", "отмени тихий", "выключи тихий", "не тихо"} { if strings.Contains(u, kw) { off = true break } } } if !on && !off { return "", false } val := "false" reply := "тихий режим выключен." if on { val = "true" reply = "тихий режим включён. буду реже напоминать." } if _, err := h.api.WriteFact(ctx, ipc.WriteFactReq{ Ts: h.now(), Kind: "config", Key: "quiet_hours", Value: val, Source: "tap:voice", Confidence: 1.0, }); err != nil { log.Printf("voice: write quiet_hours: %v", err) return "не получилось переключить тихий режим.", true } return reply, true } // replySystem answers system-observable queries using the handler's clock // and (in future) system interfaces. The decision's utterance is parsed // for keywords to determine what the user is asking about. func (h *reactiveHandler) replySystem(ctx context.Context, dec router.Decision) string { u := strings.ToLower(dec.Utterance) now := h.now() // stage-0 grammars catch the exact time/date patterns, but duration // queries ("сколько времени прошло") bypass the grammar's build filter // and can reach replySystem via the classifier path. Guard against them. if hasDurationWords(u) { return "пока не умею отвечать на этот вопрос." } switch { case strings.Contains(u, "час") || strings.Contains(u, "врем"): // "который час в киеве" — she keeps one clock, so any named place gets // the honest answer. Never local time dressed up as the city's. if mentionsUnknownPlace(u) { return onlyLocalTimeReply } return "сейчас " + ruClock(now) case strings.Contains(u, "день") || strings.Contains(u, "числ"): // "какое число завтра" — answer for the day the user asked about, // not today. Reuses the router's calendar day-word parser. day := now prefix := "сегодня" if d, ok := router.ParseCalendarDate(u, now); ok { day = d prefix = dayPrefix(now, d) } else if mentionsUnknownDay(u) { // He named a day she cannot work out ("в пятницу", "через неделю"). // Answering today's date here would be the same silent wrong answer // this arm was fixed for, so say what she can do instead. return onlyNearDaysReply } dow := ruWeekdays[day.Weekday()] month := ruMonths[day.Month()-1] return fmt.Sprintf("%s %s, %d %s %d года", prefix, dow, day.Day(), month, day.Year()) case strings.Contains(u, "кто дома") || strings.Contains(u, "человек дома"): return "присутствие пока не подключено к голосовому запросу." case strings.Contains(u, "памят") || strings.Contains(u, "процессор") || strings.Contains(u, "загрузк") || strings.Contains(u, "статус") || strings.Contains(u, "работа") || strings.Contains(u, "сервис") || strings.Contains(u, "диск") || strings.Contains(u, "ip") || strings.Contains(u, "аптайм") || strings.Contains(u, "трафик") || strings.Contains(u, "интернет"): return "системная статистика пока не подключена." default: return "пока не умею отвечать на этот вопрос." } } // chatHistory collects dialogue turns from the session store for the current // conversation. Returns prior user utterances (newest last) up to a depth of // 4 turns. Returns nil when there's no session or no history. func (h *reactiveHandler) chatHistory() []dialogue.Turn { if h.dialogueSessions == nil { return nil } now := h.now() prev := h.dialogueSessions.Get(voiceDialogueID, now) if prev == nil { return nil } // History already includes the immediate prior turn (set by the dialogue // merge at lines 373-395), plus up to 3 more from deeper history. out := make([]dialogue.Turn, 0, 1+len(prev.History)) out = append(out, dialogue.Turn{ Intent: prev.Intent, Slots: prev.Slots, Text: prev.Slots.Text, }) out = append(out, prev.History...) return out } // reply wraps a text reply through TTS to produce a PushToTalkResp. If TTS // fails, the response carries an empty audio + the text — the client can // still display text if it can't play. The routedChannels field is // reserved for a future "the dispatcher also forwarded to ntfy/telegram" // reply (today the reactive path doesn't dispatch nudges; that's the loop // tick's job). func (h *reactiveHandler) reply(ctx context.Context, text string, _ []string) (voice.PushToTalkResp, error) { spoken := ttsnorm.Speakable(text) log.Printf("voice: reply → %q", text) audioOut, err := h.tts.Synthesize(ctx, spoken) if err != nil { log.Printf("voice: tts error: %v", err) return voice.PushToTalkResp{ReplyText: text, ReplyAudio: audio.Audio{}}, nil } return voice.PushToTalkResp{ReplyText: text, ReplyAudio: audioOut}, nil }