diff --git a/CLAUDE.md b/CLAUDE.md index b013500..6fed4ab 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -343,9 +343,36 @@ the possessive agenda rules claim those cases at stage 0 and name nothing, so no label reaches the head. That is the same trade V-660 flagged and it wants the owner's call. -**Nothing of this runs in Go.** The weights are `heads.pt` and `out/body_heads/` -on workpc. Reaching the daemon needs an ONNX export and a caller. The resident -e5-small must not be replaced by the copy, because recall depends on that file. +**The heads run in Go and route every turn, since 08-08-2026** (V-664, +`docs/evals/2026-08-08-routing-heads-in-go.md`). This section used to say +nothing of it ran. `RouterHeads` in `internal/router/heads.go` loads +`router_heads.onnx` and reads intent, destination and clarify off one forward +pass. It is stage 0b: after the grammars, **before** the resident model, and the +classifier is still behind both. Through the cascade it scores intent **96.9%** +and destination **75.8%** at p50 27.9ms. That beats the gemma-4-12b cascade, +84.4% and 72.7%, at a twelfth of its 329ms. The workstation stays the better +phraser and is no longer the better router. + +Three rules around it. The **clarify head decides first**, before the intent +threshold. It answers a different question. A thin utterance scores low +intent by construction, so gating it cost 6 of 8 ambiguous cases. The +**destination head is read on `IntentQuery` only**, since no other intent +reaches `queryWalk`. And `headsThreshold` is 0.6, the measured knee: every value +to 0.85 drops right answers and keeps the same two wrong ones. + +`voice.embedder.heads_path` is the whole switch. Empty, missing or unloadable +means the heads are nil and the cascade is byte-for-byte what shipped before +them. **It must never be pointed at `model_path`.** The resident e5-small must +not be replaced by the fine-tuned copy. Recall depends on that file scoring +what it scored. + +**The hand-written tokenizer read every long word backwards** until this task +(`encodeWord`, `onnxembedder.go`). It cost recall@1 7.4 points and recall@3 11.1. +Nothing caught it because seeds and queries were mangled the same way, so cosine +survived. The heads found it. They are trained through transformers and read +through this. The embedder id now carries a tokenizer revision +(`@384/tok2`), so fixing the tokenizer triggers `ReembedAll` the way swapping the +model file does. Bump `tokenizerRev` on any change to what it emits. `Confidence: 1.0` used to be hardcoded in `llmrouter.go`, so the LLM path could never ask for clarification (6/6 refusal cases missed on the fixture) — Vikunja diff --git a/cmd/mavend/chat_degrade_test.go b/cmd/mavend/chat_degrade_test.go index c376aa0..f127ee0 100644 --- a/cmd/mavend/chat_degrade_test.go +++ b/cmd/mavend/chat_degrade_test.go @@ -19,7 +19,7 @@ func TestChatAnswersWithNoLlamaServer(t *testing.T) { dead := llm.New("http://127.0.0.1:1", 500*time.Millisecond) emb := router.NewHashEmbedder(1024) h.recall.embedder = emb - h.router = buildRouter(emb, h.matcher, 0.55, pickLLMRouter(true, dead)) + h.router = buildRouter(emb, h.matcher, 0.55, pickLLMRouter(true, dead), nil) h.replier = newLLMReplier(dead, nil) ctx := withDialogueID(context.Background(), dialogueIDFor(sourceText, "web")) diff --git a/cmd/mavend/clarify_test.go b/cmd/mavend/clarify_test.go index 946c393..ec7a47a 100644 --- a/cmd/mavend/clarify_test.go +++ b/cmd/mavend/clarify_test.go @@ -317,7 +317,7 @@ func TestClarifyExpiryIsAnnouncedAndWordsStillRoute(t *testing.T) { h, _, now := newClarifyHandler(t) emb := router.NewHashEmbedder(1024) h.recall.embedder = emb - h.router = buildRouter(emb, h.matcher, 0.55, nil) + h.router = buildRouter(emb, h.matcher, 0.55, nil, nil) if _, asked := h.askClarify(ctx, clarifyDec(router.IntentReminder, router.Slots{Text: "напомни"}, "напомни")); !asked { t.Fatal("expected a question") @@ -671,7 +671,7 @@ func TestUnresolvedActSaysItDoesNotKnowTheCommand(t *testing.T) { func newRoutingClarifyHandler(t *testing.T) (*reactiveHandler, *store.Store) { t.Helper() h, st, _ := newClarifyHandler(t) - h.router = buildRouter(router.NewHashEmbedder(1024), h.matcher, 0.55, nil) + h.router = buildRouter(router.NewHashEmbedder(1024), h.matcher, 0.55, nil, nil) h.recall = recallWiring{embedder: router.NewHashEmbedder(1024), memStore: memory.NewInMemoryStore()} return h, st } diff --git a/cmd/mavend/decisiontrace_test.go b/cmd/mavend/decisiontrace_test.go index 813d34f..eac9f2e 100644 --- a/cmd/mavend/decisiontrace_test.go +++ b/cmd/mavend/decisiontrace_test.go @@ -25,7 +25,7 @@ func traceHandler(t *testing.T, ring *decision.Ring) *reactiveHandler { return &reactiveHandler{ api: api, recall: recallWiring{embedder: emb, memStore: memory.NewInMemoryStore()}, - router: buildRouter(emb, tool.NewMatcher(api), 0.55, nil), + router: buildRouter(emb, tool.NewMatcher(api), 0.55, nil, nil), replier: voice.NewStubReplier(), now: func() time.Time { return now }, dataStore: st, diff --git a/cmd/mavend/dialogue_contract_test.go b/cmd/mavend/dialogue_contract_test.go index 7db4ced..b4557d9 100644 --- a/cmd/mavend/dialogue_contract_test.go +++ b/cmd/mavend/dialogue_contract_test.go @@ -171,7 +171,7 @@ func newDialogueHandler(t *testing.T) (*reactiveHandler, *store.Store, *time.Tim // and never a coincidence (V-577, V-579). checkEnd refuses any reminder // landing on it, and at 09:00 the row that answers "на 9" would trip that. *now = time.Date(2026, 7, 31, 9, 17, 0, 0, time.UTC) - h.router = buildRouter(router.NewHashEmbedder(1024), h.matcher, 0.55, nil) + h.router = buildRouter(router.NewHashEmbedder(1024), h.matcher, 0.55, nil, nil) h.recall = recallWiring{embedder: router.NewHashEmbedder(1024), memStore: memory.NewInMemoryStore()} return h, st, now } diff --git a/cmd/mavend/fact_subject_test.go b/cmd/mavend/fact_subject_test.go index 885f067..34aaed7 100644 --- a/cmd/mavend/fact_subject_test.go +++ b/cmd/mavend/fact_subject_test.go @@ -24,7 +24,7 @@ func TestApplyAction_FactCapture_QueuesEntityResolution(t *testing.T) { emb := router.NewHashEmbedder(1024) matcher := tool.NewMatcher(api) - rtr := buildRouter(emb, matcher, 0.55, nil) + rtr := buildRouter(emb, matcher, 0.55, nil, nil) h := &reactiveHandler{ api: api, diff --git a/cmd/mavend/factgate_test.go b/cmd/mavend/factgate_test.go index 842bb2c..72ecb87 100644 --- a/cmd/mavend/factgate_test.go +++ b/cmd/mavend/factgate_test.go @@ -20,7 +20,7 @@ func newFactGateHandler(t *testing.T, now time.Time) (*reactiveHandler, ipc.Core h := &reactiveHandler{ api: api, recall: recallWiring{embedder: emb, memStore: memory.NewInMemoryStore()}, - router: buildRouter(emb, tool.NewMatcher(api), 0.55, nil), + router: buildRouter(emb, tool.NewMatcher(api), 0.55, nil, nil), replier: voice.NewStubReplier(), now: func() time.Time { return now }, dataStore: st, diff --git a/cmd/mavend/notefragment_test.go b/cmd/mavend/notefragment_test.go index cf9becc..7e2f50d 100644 --- a/cmd/mavend/notefragment_test.go +++ b/cmd/mavend/notefragment_test.go @@ -45,7 +45,7 @@ func newNoteHandler(t *testing.T) (*reactiveHandler, *store.Store) { h := &reactiveHandler{ api: api, recall: recallWiring{embedder: emb, memStore: memory.NewInMemoryStore()}, - router: buildRouter(emb, tool.NewMatcher(api), 0.55, nil), + router: buildRouter(emb, tool.NewMatcher(api), 0.55, nil, nil), replier: voice.NewStubReplier(), now: func() time.Time { return now }, dataStore: st, diff --git a/cmd/mavend/reactive_notes_test.go b/cmd/mavend/reactive_notes_test.go index 811e706..0449b88 100644 --- a/cmd/mavend/reactive_notes_test.go +++ b/cmd/mavend/reactive_notes_test.go @@ -22,7 +22,7 @@ func TestReactiveNotesReminders(t *testing.T) { emb := router.NewHashEmbedder(1024) matcher := tool.NewMatcher(api) - rtr := buildRouter(emb, matcher, 0.55, nil) + rtr := buildRouter(emb, matcher, 0.55, nil, nil) h := &reactiveHandler{ api: api, @@ -104,7 +104,7 @@ func TestSpokenTaskCaptureFilesATask(t *testing.T) { h := &reactiveHandler{ api: api, recall: recallWiring{embedder: emb, memStore: memory.NewInMemoryStore()}, - router: buildRouter(emb, matcher, 0.55, nil), + router: buildRouter(emb, matcher, 0.55, nil, nil), replier: voice.NewStubReplier(), now: func() time.Time { return now }, dataStore: st, diff --git a/cmd/mavend/simulator_test.go b/cmd/mavend/simulator_test.go index bf347a0..23b5957 100644 --- a/cmd/mavend/simulator_test.go +++ b/cmd/mavend/simulator_test.go @@ -474,7 +474,7 @@ func newSimWorld(t *testing.T, sc scenario) *simWorld { // used to be built on a nil API, which meant any scenario that produced an // act panicked the moment the matcher was consulted. matcher := tool.NewMatcher(api) - rtr := buildRouter(emb, matcher, config.DefaultRouterThreshold, router.NewLLMRouter(scripted)) + rtr := buildRouter(emb, matcher, config.DefaultRouterThreshold, router.NewLLMRouter(scripted), nil) w.handler = &reactiveHandler{ stt: simTranscriber{}, diff --git a/cmd/mavend/sttseam_test.go b/cmd/mavend/sttseam_test.go new file mode 100644 index 0000000..007e3cb --- /dev/null +++ b/cmd/mavend/sttseam_test.go @@ -0,0 +1,43 @@ +package main + +import ( + "testing" + + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/stt" +) + +// A box with no workstation.stt block transcribes exactly as it did before the +// seam existed: the floor is handed back untouched, and nothing probes. +func TestSttSeamWithNoBlockIsTheFloor(t *testing.T) { + floor := stt.NewStub() + got, pair := sttSeam(&config.Config{}, floor) + if pair != nil { + t.Fatal("no block must build no pair") + } + if got != stt.Transcriber(floor) { + t.Fatal("no block must hand back the floor itself") + } +} + +func TestSttSeamPrefersTheWorkstation(t *testing.T) { + cfg := &config.Config{Workstation: &config.WorkstationConfig{ + URL: "http://127.0.0.1:1", + Stt: &config.WorkstationSttConfig{ + URL: "http://127.0.0.1:2/transcribe", + Health: "http://127.0.0.1:2/health", + }, + }} + got, pair := sttSeam(cfg, stt.NewStub()) + if pair == nil { + t.Fatal("a configured block must build a pair") + } + defer pair.Stop() + if got != stt.Transcriber(pair) { + t.Fatal("the pair is what callers must transcribe through") + } + // Nothing answers on port 2, so the seam is the floor until it does. + if pair.Available() { + t.Fatal("an unreachable workstation must not be available") + } +} diff --git a/cmd/mavend/voicewire.go b/cmd/mavend/voicewire.go index 9e308e9..f3a2f7f 100644 --- a/cmd/mavend/voicewire.go +++ b/cmd/mavend/voicewire.go @@ -36,7 +36,9 @@ type voiceWiring struct { sessions *voice.Sessions voiceSink delivery.Sink embedder router.Embedder - handler *reactiveHandler // the reactive handler for IPC Chat + // heads — the routing heads, nil unless embedder.heads_path is set. + heads *router.RouterHeads + handler *reactiveHandler // the reactive handler for IPC Chat // worker clients (set when configured as Remote): closed on shutdown so // mavsttd / mavttsd don't keep a stale conn into a restarting daemon. sttClient *worker.Client @@ -53,7 +55,11 @@ type voiceWiring struct { // unless a `workstation` block names an address. Held here only so the // prober is stopped on shutdown; callers were handed it at build time. pair *llm.Pair - mcp *mcpWiring + // sttPair — CrisperWhisper 2.0 on the workstation with mavsttd as the + // floor, nil unless the `workstation.stt` block names an address. Held for + // the same reason as pair: to stop its prober on shutdown. + sttPair *stt.Pair + mcp *mcpWiring // home — the Home Assistant client, nil unless the `smarthome` block is // enabled (Vikunja #256). Its devices land in the same allowlist as every // other act, so nothing else here has to know about it. @@ -72,6 +78,9 @@ func (w *voiceWiring) close() { if w.embedder != nil { _ = w.embedder.Close() } + if w.heads != nil { + _ = w.heads.Close() + } if w.server != nil { _ = w.server.Close() } @@ -84,6 +93,9 @@ func (w *voiceWiring) close() { if w.pair != nil { w.pair.Stop() } + if w.sttPair != nil { + w.sttPair.Stop() + } w.mcp.close() } @@ -112,6 +124,7 @@ func wireVoice(cfg *config.Config, coreAPI ipc.CoreAPI, phr phraser.Phraser, mem } else { transcriber = stt.NewStub() } + transcriber, w.sttPair = sttSeam(cfg, transcriber) w.transcriber = transcriber // ----- tts (Stub in-process OR Remote) ----- @@ -147,6 +160,24 @@ func wireVoice(cfg *config.Config, coreAPI ipc.CoreAPI, phr phraser.Phraser, mem emb = router.NewHashEmbedder(1024) } w.embedder = emb + + // ----- router: routing heads (only when configured, and never fatal) ----- + // A missing or broken weights file logs and leaves w.heads nil, which is + // byte-for-byte the cascade that shipped before V-664. Refusing to start + // over a routing accelerator would trade a working box for a better one. + if cfg.Voice.Embedder != nil && cfg.Voice.Embedder.HeadsPath != "" { + h, err := router.NewRouterHeads( + cfg.Voice.Embedder.HeadsPath, + cfg.Voice.Embedder.TokenizerPath, + ) + if err != nil { + log.Printf("voice: routing heads unavailable, cascade unchanged: %v", err) + } else { + log.Printf("voice: routing heads loaded from %s", cfg.Voice.Embedder.HeadsPath) + w.heads = h + } + } + repairFactVectors(dataStore, emb) checkStoredEmbedder(dataStore, emb) // Retention is enforced on write, which is not enough on its own: a box that @@ -223,7 +254,8 @@ func wireVoice(cfg *config.Config, coreAPI ipc.CoreAPI, phr phraser.Phraser, mem // against the classifier's 50.0%, at about 1s a turn instead of 30ms (see // config.VoiceConfig.LLMRouter). The classifier always stays wired as the // fallback, so a model error never breaks a turn. - rtr := buildRouter(emb, matcher, threshold, pickLLMRouter(cfg.Voice.UseLLMRouter(), hot)) + rtr := buildRouter(emb, matcher, threshold, + pickLLMRouter(cfg.Voice.UseLLMRouter(), hot), w.heads) // ----- sessions registry (shared with voicesink) ----- sessions := voice.NewSessions() @@ -366,6 +398,44 @@ func modelSeam(cfg *config.Config, resident *llm.Client) (router.Completer, *llm return pair, pair } +// sttSeam builds the transcription seam the voice path and the meeting +// recorder share. It is modelSeam for audio and follows the same rule. +// +// With no `workstation.stt` block it hands back the floor untouched, which is +// today's deploy exactly. With one, it is an stt.Pair preferring CrisperWhisper +// 2.0 on workpc, which scores 10.4% WER in Russian against the floor's 27.5% +// (docs/evals/2026-08-09-crisperwhisper2-russian-wer.md). +// +// Only the silent half of the degradation rule applies here. A worse transcript +// is still a turn, so there is nothing to name a gap about and the fallback is +// never spoken. That is why stt.Pair has no TranscribeRemote. +func sttSeam(cfg *config.Config, floor stt.Transcriber) (stt.Transcriber, *stt.Pair) { + if cfg.Workstation == nil || cfg.Workstation.Stt == nil { + return floor, nil + } + s := cfg.Workstation.Stt + lang := "" + if cfg.Voice != nil { + lang = cfg.Voice.Lang + if cfg.Voice.Stt != nil && cfg.Voice.Stt.Lang != "" { + lang = cfg.Voice.Stt.Lang + } + } + pair := stt.NewPair( + stt.NewHTTPTranscriber(s.URL, s.Token, lang, time.Duration(s.Timeout)), + floor, + s.Health, + time.Duration(s.Probe), + ) + pair.Start(context.Background()) + if s.Token == "" { + log.Print("voice: the workstation transcriber has no token, so anything on the LAN can post audio to it") + } + log.Printf("voice: workstation transcriber at %s, probed every %s, mavsttd as the floor", + s.URL, time.Duration(s.Probe)) + return pair, pair +} + func pickLLMRouter(enabled bool, c router.Completer) *router.LLMRouter { if !enabled { return nil @@ -390,7 +460,8 @@ func pickLLMRouter(enabled bool, c router.Completer) *router.LLMRouter { // intent from seedDir (models/seeds/.txt) — see seedClassifier // below for the current intent list and file names. // - Threshold is from voice.router_threshold config (default 0.55). -func buildRouter(emb router.Embedder, acts router.ActMatcher, threshold float64, llmR *router.LLMRouter) *router.Router { +func buildRouter(emb router.Embedder, acts router.ActMatcher, threshold float64, + llmR *router.LLMRouter, heads *router.RouterHeads) *router.Router { cls := router.NewClassifier(emb) seedClassifier(cls) grammars := router.DefaultGrammars(acts) @@ -442,6 +513,7 @@ func buildRouter(emb router.Embedder, acts router.ActMatcher, threshold float64, }, Threshold: threshold, LLM: llmR, + Heads: heads, }) } diff --git a/deploy/mavend.json b/deploy/mavend.json index 2881cb6..aa0d95b 100644 --- a/deploy/mavend.json +++ b/deploy/mavend.json @@ -251,7 +251,8 @@ "embedder": { "model_path": "/opt/maven/models/embedder/multilingual-e5-small/model_quantized.onnx", "tokenizer_path": "/opt/maven/models/embedder/multilingual-e5-small/tokenizer.json", - "lib_path": "/opt/maven/lib/libonnxruntime.so" + "lib_path": "/opt/maven/lib/libonnxruntime.so", + "heads_path": "/opt/maven/models/embedder/router-heads/router_heads.onnx" }, "llm_router": true, "query_min_score": 0.55, diff --git a/docs/evals/2026-08-08-routing-heads-in-go.md b/docs/evals/2026-08-08-routing-heads-in-go.md new file mode 100644 index 0000000..aea3a6a --- /dev/null +++ b/docs/evals/2026-08-08-routing-heads-in-go.md @@ -0,0 +1,146 @@ +# The routing heads, running in Go + +Date: 2026-08-08. Vikunja V-664. +Weights: `router_heads.onnx`, fp32, exported from `heads.pt` on workpc. +Fixture: `internal/router/eval/ru_routing_v1.json`, 96 cases, 33 carrying a destination. +Runner: `make t PKG=./internal/router/eval/ RUN=TestONNXRoutingHeads`. + +The four heads of V-661 ran nowhere. This is the number they score through the +Go cascade. Same fixture and same grammars as `TestONNXBaseline`, and only the +middle stage varies. + +## Headline + +| | classifier + ONNX | heads + classifier | gemma-4-12b cascade | +|---|---|---|---| +| intent | 75.0% (72/96) | **96.9% (93/96)** | 84.4% | +| destination | 33.3% (11/33) | **75.8% (25/33)** | 72.7% | +| false clarify | 0 | 1 | 2 | +| missed clarify | 8 | 1 | 1 | +| p50 | 24.5ms | 27.9ms | 329ms | + +A 118M encoder beats the 12B teacher it was distilled from. It wins on both +halves of the route, at a twelfth of the latency. The workstation stays the +better phraser and is no longer the better router. + +The p50 is not the heads. Most of it is the classifier's own embedder pass on +the turns the heads decline, plus process warm-up on the first case. The heads' +own forward pass measures 7.3ms on workpc. + +## Two defects were in the way, and the first was not in the heads + +**The tokenizer read every long word backwards.** `encodeWord` backtracks the +Viterbi path from the end of a word and prepends each piece. That puts them back +in reading order, and a second reverse after the loop undid it. So +`query: вода` tokenized to `[0 12 1294 41 12489 2]` where the reference +tokenizer gives `[0 41 1294 12 12489 2]`. + +It was found here and only here. The heads were trained through transformers and +are read through the hand-written tokenizer. So a mismatch shows up as a score +far below what Python measured on the same weights. Nothing else in the suite +compares the two. + +Measured on the recall fixture, same 27 cases either way: + +| | reversed | fixed | +|---|---|---| +| recall@1 | 70.4% (19/27) | **77.8% (21/27)** | +| recall@3 | 85.2% (23/27) | **96.3% (26/27)** | +| answered after gate | 63.0% | 66.7% | +| wrong note on top | 8 | 6 | +| false recall | 0/5 | 1/5 | + +The classifier barely moved, 76.0% to 75.0%, and destination 36.4% to 33.3%. +Both are one case on 96 and neither is a finding. Seeds and queries were mangled +the same way, so cosine survived it. Recall is where it cost, because a stored +passage and a live query are different lengths and break differently. + +The one new false recall is the honest cost and it is not being hidden. A +sharper embedder scores every candidate higher, including the ones that should +have stayed under the gate. That is the same trade `2026-08-04-recall-e5-small.md` +recorded when e5-small replaced MiniLM. + +The embedder id now carries a tokenizer revision, `model_quantized@384/tok2`. +Stored vectors were written under rev 1 and no longer sit in the same space as a +query embedded now. The model file's name never moved, so nothing would have +triggered `ReembedAll`. On the box the marker fired on start, and the re-embed +rewrote 65 notes and 19 facts in 5 seconds. + +**The clarify head was being thrown away.** It was read only when the intent head +cleared its own threshold. That cost 6 of the 8 ambiguous cases. `вода` reads as intent +`act` at 0.233 and clarify at 0.983. Burying that handed the turn to the +classifier, which routed it confidently and never asked. The clarify head answers +a different question, which is whether there is enough here to act on at all. So +it decides on its own and decides first. + +| | intent-gated | clarify decides first | +|---|---|---| +| intent | 90.6% | 96.9% | +| missed clarify | 7 | 1 | +| false clarify | 0 | 1 | + +## The threshold is measured, not chosen + +Max softmax over the intent head, on the 88 cases carrying an intent: + +| threshold | kept | accuracy kept | wrong kept | right dropped | +|---|---|---|---|---| +| 0.5 | 84 | 96.4% | 3 | 2 | +| **0.6** | **81** | **97.5%** | **2** | **4** | +| 0.7 | 75 | 97.3% | 2 | 10 | +| 0.8 | 64 | 96.9% | 2 | 21 | +| 0.9 | 46 | 100.0% | 0 | 37 | + +0.6 is the knee. Every value from 0.7 to 0.85 drops right answers and keeps the +same two wrong ones. 0.9 is the only value that clears them, and it costs 37 +correct routes to do it. + +## Quantization was measured and rejected + +| build | size | intent | destination | p50 | +|---|---|---|---|---| +| fp32 | 470MB | 83/88 (94.3%) | 28/33 (84.8%) | 7.3ms | +| int8 | 118MB | 79/88 (89.8%) | 26/33 (78.8%) | 4.0ms | +| fp16 | 235MB | will not load | — | — | + +Python numbers, on the heads alone rather than through the cascade. int8 costs +4.5 points of intent and 6 of destination to save 3ms. The cascade around it has +a p50 over a second when the resident model answers. The fp16 graph is broken: +`convert_float_to_float16` leaves a Cast node emitting float16 where the graph +expects float, and onnxruntime refuses the session. It was not worth fixing. + +The exporter also had to be told to write one file. It splits weights into a +`.onnx.data` sidecar by default. This onnxruntime resolves that path against the +process working directory rather than the model. A split graph loads from one +directory only. + +## What is still wrong + +**Four of the eight destination misses are calendar.** Training cannot move them. +The possessive agenda rules claim those cases at stage 0 and name nothing on +purpose. That caution was free while nothing downstream could name anything +either. It has now cost four points in three separate measurements. The call is +the owner's and it is still open. + +**The slot head is exported and not read.** Slots come from the stage-2 +extractor. Mapping BIO tags back to text needs character offsets the unigram +tokenizer does not keep, which is its own piece of work. + +**`поужинал` is a false clarify**, which is the same defect `thinSingleToken` +was narrowed for on 2026-08-01, arriving now from a different direction. + +## On the box + +Deployed to homesrv the same day. `voice: routing heads loaded` on start, and +`/trace` shows `routing-heads` winning or thinning every turn. The resident model +and the classifier are both marked never asked. Live probes: + +```text +что такое TCP? -> kiwix a real definition +кто такой Линус Торвальдс? -> kiwix a real answer +во сколько я лёг вчера -> personal не нашла у тебя такой записи +вода -> thinned to clarify at 0.233 / 0.983 +``` + +A missing or broken weights file logs and leaves the heads nil, which is +byte-for-byte the cascade that shipped before this. diff --git a/docs/evals/2026-08-09-crisperwhisper2-russian-wer.md b/docs/evals/2026-08-09-crisperwhisper2-russian-wer.md new file mode 100644 index 0000000..0a6603b --- /dev/null +++ b/docs/evals/2026-08-09-crisperwhisper2-russian-wer.md @@ -0,0 +1,110 @@ +# CrisperWhisper 2.0 in Russian, measured + +Date: 2026-08-09. Vikunja V-665. +Corpus: `bond005/sberdevices_golos_10h_crowd`, test split, first 200 clips. +Harness: `~/Programs/cw2-eval` on workpc, not in this repo. +Runner: `./.venv/bin/python run_asr.py ...` then `score.py`. + +The model card benchmarks disfluency F1 in German and English. It never names +Russian and publishes no per-language WER. So the measurement came before the +wiring. + +## The corpus + +200 clips, 13.7 minutes, 1001 reference words. Median clip 3.91s, range 1.04s +to 13.5s. Golos crowd is short crowd-sourced Russian spoken close to the +microphone, which is the nearest public thing to someone talking to Maven. The +alternatives are read speech, which flatters every model equally. + +Two rows carry a null transcription and are skipped. + +Scoring normalizes both sides: lowercase, `ё` to `е`, punctuation stripped, and +digits expanded to Russian words through num2words. Without that last step a +model is penalized for writing `60000` where the reference says +`шестьдесят тысяч`. Thousands separators are joined before expansion, or +`60 000` expands to `шестьдесят ноль`. + +## Headline + +| arm | WER | CER | exact | empty | RTF | +|---|---|---|---|---|---| +| cw2-turbo-intended | **10.4%** | 3.4% | 65.5% | 0 | 0.065 | +| cw2-turbo-verbatim | 10.8% | **3.1%** | **66.5%** | 0 | 0.065 | +| whisper-turbo | 11.8% | 4.1% | 64.0% | 0 | 0.031 | +| cw2-large-intended | 12.3% | 3.8% | 63.5% | 0 | 0.107 | +| whisper-small | 27.5% | 9.8% | 35.0% | 0 | 0.026 | + +`whisper-small` is the floor, because `ggml-small.bin` is what mavsttd loads on +homesrv today. CW2 turbo beats it by 17 points of WER and takes exact matches +from 35.0% to 65.5%. + +Two results are worth naming beyond the winner. CW2 turbo beats its own base +model, whisper-large-v3-turbo, by 1.4 points. And it beats CW2 large by 1.9 +points, which inverts what the card implies by calling turbo a degraded draft. +No arm returned an empty transcript. + +## Intended and verbatim are closer than the mode names suggest + +The two modes disagree on 70 of the 200 clips before normalization and on 29 +after it. So the raw difference is mostly casing and punctuation, which +normalization removes and which Maven does not read either. + +Verbatim scores worse on WER and better on CER and exact matches. The reason is +script, not disfluency: + +```text +ref: футбольный матч челси брайтон +int: Футбольный матч Chelsea-Брайтон. +ver: Футбольный матч Челси Брайтон. +``` + +Intended writes foreign entity names in Latin script and verbatim +transliterates them. Golos references are Cyrillic throughout, so verbatim +collects the exact matches. That is a property of this corpus rather than a +quality difference. + +**This corpus cannot settle the mode choice.** Golos crowd is clean short +commands with almost no disfluency. The two modes have nothing to disagree +about here. They separate on spontaneous speech with fillers, restarts and +repairs, which is what the owner speaks. Intended stays the choice for the +reason it was always the choice. Maven wants what was meant, not every stumble +on the way there. + +The Latin-script habit is the one finding here that touches routing. The +routing heads were trained on Cyrillic utterances, so an entity name arriving +in Latin script is out of distribution for them. Nothing measures that yet. + +## The runtime is workpc, because whisper.cpp cannot load CW2 + +`num_languages()` in `deps/whisper.cpp/src/whisper.cpp` derives the language +count from the vocabulary size: + +```cpp +return n_vocab - 51765 - (is_multilingual() ? 1 : 0); +``` + +CW2 carries 31 extra tokens, so `n_vocab` is 51897 and this yields 131 +languages. The derived `dt` offset becomes 33 and shifts seven special token +ids, including `token_beg` and `token_transcribe`. The architecture is +otherwise byte-identical to whisper-large-v3-turbo, and the new tokens sit +above every whisper special id. + +So loading CW2 in whisper.cpp is a patch to a vendored dependency, not a port. +It was not taken, because STT is moving to workpc anyway under V-486. CW2 turbo +becomes the preferred remote and `ggml-small.bin` on homesrv stays the floor, +which is the shape `modelSeam` already uses for routing and replies. The 27.5% +floor is what a turn falls back to when the workstation is down, and this table +is what that costs. + +## License + +Standard CW2 weights carry `nyra-health-non-commercial-research`. The Pro +variants are commercial-license only. Maven is personal and self-hosted, so the +standard weights are usable and the Pro ones are not free to take. + +## What is not measured + +Disfluent spontaneous speech, which is the whole reason to prefer Intended. +Long-form audio beyond 13.5s. Far-field or noisy microphones. English, which +Maven also speaks. The ONNX turbo export, which was never run, since the +transformers path already meets the latency budget at RTF 0.065. diff --git a/internal/config/voice.go b/internal/config/voice.go index 752a8f6..1e9a9e3 100644 --- a/internal/config/voice.go +++ b/internal/config/voice.go @@ -37,6 +37,16 @@ type EmbedderConfig struct { ModelPath string `json:"model_path,omitempty"` TokenizerPath string `json:"tokenizer_path,omitempty"` LibPath string `json:"lib_path,omitempty"` + + // HeadsPath — the routing heads graph, which is a fine-tuned COPY of the + // model above with four linear heads on its pooled output (V-664). Empty + // means no heads, and the cascade runs exactly as it did before they + // existed. It shares LibPath and TokenizerPath, and router_heads.json is + // read from the same directory. + // + // It must never be pointed at ModelPath. Memory recall depends on the + // resident copy scoring what it scored, and the fine-tuned one does not. + HeadsPath string `json:"heads_path,omitempty"` } // WeatherConfig configures the weather provider for voice queries. diff --git a/internal/config/workstation.go b/internal/config/workstation.go index 7a79813..6fab688 100644 --- a/internal/config/workstation.go +++ b/internal/config/workstation.go @@ -1,6 +1,7 @@ package config import ( + "net/url" "strings" "time" ) @@ -35,12 +36,54 @@ type WorkstationConfig struct { // 0 ⇒ DefaultWorkstationTimeout. A big model on a LAN host is slower than // the resident one, and a request that overruns falls back to the floor. Timeout Duration `json:"timeout,omitempty"` + + // Stt — CrisperWhisper 2.0 on the same machine, a separate service on its + // own port. Absent ⇒ every utterance goes to mavsttd, which is today. + Stt *WorkstationSttConfig `json:"stt,omitempty"` +} + +// WorkstationSttConfig — speech-to-text on the workstation. +// +// It is a second service and not a second endpoint on mavgpud: whisper.cpp +// cannot load CrisperWhisper 2.0 at all, because it derives its language count +// from the vocabulary size and CW2's 51897 tokens shift seven special token +// ids. So CW2 runs under transformers, and this block addresses it. +// +// Worth the trouble: CW2 turbo scores 10.4% WER in Russian against 27.5% for +// the ggml-small.bin homesrv loads +// (docs/evals/2026-08-09-crisperwhisper2-russian-wer.md). +type WorkstationSttConfig struct { + // URL — the transcribe endpoint, e.g. + // "http://192.168.1.105:8081/transcribe". Empty ⇒ the block is normalised + // to nil and mavsttd takes every turn. + URL string `json:"url,omitempty"` + + // Health — the admission endpoint. Empty ⇒ the URL's origin + "/health". + // It answers 503 while the card is held, and that is the signal. + Health string `json:"health,omitempty"` + + // Token — the bearer token the service checks. Audio is the most sensitive + // thing that crosses this seam, so a LAN deployment should set one. Write + // it as ${MAVEN_STT_TOKEN} and keep the value in deploy/telegram.env, the + // way every other secret in this file is written. + Token string `json:"token,omitempty"` + + // Probe — how often admission is re-checked. 0 ⇒ DefaultWorkstationProbe. + Probe Duration `json:"probe,omitempty"` + + // Timeout — the per-request budget for one utterance. 0 ⇒ + // DefaultWorkstationSttTimeout. A request that overruns falls back to + // mavsttd, which costs a worse transcript and not the turn. + Timeout Duration `json:"timeout,omitempty"` } // Workstation defaults, applied in normaliseWorkstation. const ( DefaultWorkstationProbe = 15 * time.Second DefaultWorkstationTimeout = 90 * time.Second + // One utterance, not one completion. A voice turn waits on this, so the + // budget is a few seconds and not a minute and a half. + DefaultWorkstationSttTimeout = 10 * time.Second ) // normaliseWorkstation applies the block's defaults. No address, no preferred @@ -63,4 +106,36 @@ func (c *Config) normaliseWorkstation() { if w.Timeout <= 0 { w.Timeout = Duration(DefaultWorkstationTimeout) } + normaliseWorkstationStt(w) +} + +// normaliseWorkstationStt applies the speech-to-text block's defaults. No +// address, no remote: mavsttd then takes every utterance, which is today. +func normaliseWorkstationStt(w *WorkstationConfig) { + if w.Stt != nil && strings.TrimSpace(w.Stt.URL) == "" { + w.Stt = nil + } + if w.Stt == nil { + return + } + s := w.Stt + if strings.TrimSpace(s.Health) == "" { + s.Health = healthOrigin(s.URL) + } + if s.Probe <= 0 { + s.Probe = Duration(DefaultWorkstationProbe) + } + if s.Timeout <= 0 { + s.Timeout = Duration(DefaultWorkstationSttTimeout) + } +} + +// healthOrigin derives the admission endpoint from the transcribe endpoint. +// The URL names a path, so appending to it would ask for /transcribe/health. +func healthOrigin(raw string) string { + u, err := url.Parse(raw) + if err != nil || u.Host == "" { + return strings.TrimRight(raw, "/") + "/health" + } + return u.Scheme + "://" + u.Host + "/health" } diff --git a/internal/router/decisiontrace.go b/internal/router/decisiontrace.go index f66ac2a..140ec20 100644 --- a/internal/router/decisiontrace.go +++ b/internal/router/decisiontrace.go @@ -14,12 +14,15 @@ import ( "github.com/kami/maven/internal/decision" ) -// The two routing engines, named as claimants. They are one stage and not two, -// because only one of them ever runs: the classifier is reached when the model -// is absent or errored, never alongside it. +// The three routing engines, named as claimants. The model and the classifier +// are one stage and not two, because only one of them ever runs: the classifier +// is reached when the model is absent or errored, never alongside it. The heads +// run before both and decline on low confidence, so they can appear beside +// either one in a record. const ( claimantLLM = "llm-router" claimantClassifier = "classifier" + claimantHeads = "routing-heads" ) // thinReason names which arm of gateLLMDecision cut the confidence. The gate diff --git a/internal/router/embedderid_test.go b/internal/router/embedderid_test.go index 03f1e16..6b2ecfd 100644 --- a/internal/router/embedderid_test.go +++ b/internal/router/embedderid_test.go @@ -1,10 +1,13 @@ package router -import "testing" +import ( + "strings" + "testing" +) func TestEmbedderIDFromModelPath(t *testing.T) { got := modelIDFromPath("/opt/maven/models/embedder/multilingual-e5-small.onnx") - if got != "multilingual-e5-small@384" { + if got != "multilingual-e5-small@384/tok2" { t.Fatalf("modelIDFromPath = %q", got) } // A different model file must produce a different id, even at 384 dim. @@ -12,6 +15,13 @@ func TestEmbedderIDFromModelPath(t *testing.T) { if old == got { t.Fatal("two different models share one id") } + // The tokenizer is half of what makes a vector, and it changes under a + // model file whose name never moves (V-664). An id that ignored it would + // leave stored passages in one space and every new query in another, with + // nothing to trigger the re-embed. + if !strings.Contains(got, "/tok") { + t.Fatalf("id %q does not name the tokenizer revision", got) + } } func TestEmbedderIDIncludesDim(t *testing.T) { diff --git a/internal/router/eval/heads_test.go b/internal/router/eval/heads_test.go new file mode 100644 index 0000000..c65fd36 --- /dev/null +++ b/internal/router/eval/heads_test.go @@ -0,0 +1,86 @@ +package eval + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/router" +) + +// TestONNXRoutingHeads — the cascade with the routing heads wired, which is +// what V-664 deploys. Opt-in via MAVEN_ONNX_LIB, same as TestONNXBaseline, and +// one TestONNX* per process. +// +// The comparison worth reading is against TestONNXBaseline, which is the same +// cascade with the same grammars and the same classifier floor and no heads. +// Only the middle arm varies. +// +// It also checks the Go unigram tokenizer against the Python one, because the +// heads were trained through transformers and are read through a hand-written +// tokenizer. A mismatch shows up here as a score below what Python measured on +// the same weights, and nowhere else. +func TestONNXRoutingHeads(t *testing.T) { + lib := os.Getenv("MAVEN_ONNX_LIB") + if lib == "" { + t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing") + } + // Absolute, because onnxruntime resolves a graph's external weights file + // against the model path it was given, and a relative one lands in the + // test's working directory. + root, err := filepath.Abs("../../..") + if err != nil { + t.Fatal(err) + } + model := filepath.Join(root, "models/embedder/multilingual-e5-small/model_quantized.onnx") + tok := filepath.Join(root, "models/embedder/multilingual-e5-small/tokenizer.json") + heads := filepath.Join(root, "models/embedder/router-heads/router_heads.onnx") + for _, p := range []string{lib, model, tok, heads} { + if _, err := os.Stat(p); err != nil { + t.Skipf("missing %s: %v", p, err) + } + } + emb, err2 := router.NewONNXEmbedder(model, tok, lib) + if err2 != nil { + t.Skipf("onnx embedder unavailable: %v", err2) + } + err = nil + defer emb.Close() + + h, err := router.NewRouterHeads(heads, tok) + if err != nil { + t.Skipf("routing heads unavailable: %v", err) + } + defer h.Close() + + f, err := Load() + if err != nil { + t.Fatalf("Load: %v", err) + } + rep, err := Score(context.Background(), "heads+classifier", withHeads(t, emb, h), f) + if err != nil { + t.Fatalf("Score: %v", err) + } + t.Log("\n" + rep.String() + rep.Failures()) +} + +// withHeads mirrors newBaselineRouter and adds the one arm under test. It is a +// separate function rather than a parameter so the baseline's signature stays +// the shape every other test calls it with. +func withHeads(t *testing.T, emb router.Embedder, h *router.RouterHeads) *router.Router { + t.Helper() + acts := router.DefaultActMatcher{Fns: actFns} + return router.New(router.Config{ + Grammars: baselineGrammars(acts), + Classifier: newBaselineClassifier(t, emb), + Extractor: router.Extractor{ + Time: router.StubDateTimeParser{}, + Acts: acts, + Facts: router.DefaultFactParser{}, + }, + Threshold: config.DefaultRouterThreshold, + Heads: h, + }) +} diff --git a/internal/router/heads.go b/internal/router/heads.go new file mode 100644 index 0000000..acb2b99 --- /dev/null +++ b/internal/router/heads.go @@ -0,0 +1,243 @@ +package router + +import ( + "context" + "encoding/json" + "fmt" + "math" + "os" + "path/filepath" + + ort "github.com/yalue/onnxruntime_go" +) + +// The routing heads (V-546, V-661, V-664). Four linear heads over one masked +// mean pool of a fine-tuned copy of multilingual-e5-small: intent, +// destination, BIO slot tags and clarify. Trained on workpc, exported to ONNX, +// and read here. +// +// Why this is not the classifier. The classifier compares one utterance to +// frozen seed phrases by cosine. A head is a softmax over the label set, so it +// cannot name a value that does not exist, and its max is a calibratable +// confidence where Confidence: 1.0 was a hardcode. +// +// Why it is not the resident model either. It answers in single-digit +// milliseconds against the model's p50 of 1.19s, and it names a destination +// the classifier arm never names at all. +// +// The body is a COPY of the embedder weights, fine-tuned. It must never +// replace models/embedder/multilingual-e5-small — memory recall depends on +// that file scoring what it scored. +// +// The slot head is exported and deliberately not read. Slots already come from +// the stage-2 extractor, and mapping BIO tags back to text needs character +// offsets the unigram tokenizer does not keep. Reading it is separate work. +const ( + // headsSeq — the sequence length the heads were trained at. Padding is + // masked out of both attention and the pool, so this changes nothing but + // truncation, and truncation is what training did at 64. + headsSeq = 64 + + // headsThreshold — max softmax over the intent head, below which the heads + // decline and the cascade carries on to the resident model. + // + // 0.6 is the knee measured on the 88-case intent fixture + // (docs/evals/2026-08-08-routing-heads-in-go.md). It keeps 81 of 88 cases + // at 97.5% accuracy. Every higher value up to 0.9 drops right answers and + // keeps the same two wrong ones, so it buys nothing. + headsThreshold = 0.6 +) + +// RouterHeads runs the exported graph. Nil is a working value everywhere: a +// deployment with no weights file routes exactly as it did before this +// existed. +type RouterHeads struct { + tokenizer *unigramTokenizer + session *ort.DynamicSession[int64, float32] + intents []Intent + sources []Source + threshold float64 +} + +// headsMeta — router_heads.json, written beside the weights by the exporter. +// The label order is the head's output order and cannot be inferred from Go. +type headsMeta struct { + Intents []string `json:"intents"` + Sources []string `json:"sources"` + Prefix string `json:"prefix"` +} + +// NewRouterHeads loads the graph and its label order. modelPath points at the +// .onnx; the external weights and router_heads.json sit beside it. +// +// It assumes the ONNX environment is already initialised, because the embedder +// does that at startup and the runtime allows it once. +func NewRouterHeads(modelPath, tokenizerPath string) (*RouterHeads, error) { + metaPath := filepath.Join(filepath.Dir(modelPath), "router_heads.json") + raw, err := os.ReadFile(metaPath) + if err != nil { + return nil, fmt.Errorf("heads: read %s: %w", metaPath, err) + } + var meta headsMeta + if err := json.Unmarshal(raw, &meta); err != nil { + return nil, fmt.Errorf("heads: parse %s: %w", metaPath, err) + } + if meta.Prefix != queryPrefix { + return nil, fmt.Errorf("heads: trained with prefix %q, this build uses %q", + meta.Prefix, queryPrefix) + } + + intents := make([]Intent, len(meta.Intents)) + for i, s := range meta.Intents { + intents[i] = Intent(s) + } + sources := make([]Source, len(meta.Sources)) + for i, s := range meta.Sources { + // SourceUnknown is not in Sources, because it is the absence of a + // choice. It is a class the head can emit, and the one it should emit + // often, so it is allowed here and nowhere else. + if s != string(SourceUnknown) && !ValidSource(Source(s)) { + return nil, fmt.Errorf("heads: unknown destination %q in %s", s, metaPath) + } + sources[i] = Source(s) + } + + tok, err := newUnigramTokenizer(tokenizerPath) + if err != nil { + return nil, fmt.Errorf("heads: tokenizer: %w", err) + } + session, err := ort.NewDynamicSession[int64, float32]( + modelPath, + []string{"input_ids", "attention_mask"}, + []string{"intent", "source", "slots", "clarify"}, + ) + if err != nil { + return nil, fmt.Errorf("heads: create session: %w", err) + } + + return &RouterHeads{ + tokenizer: tok, + session: session, + intents: intents, + sources: sources, + threshold: headsThreshold, + }, nil +} + +func (h *RouterHeads) Close() error { + if h == nil { + return nil + } + h.session.Destroy() + return nil +} + +// headsResult — one forward pass, read back. +type headsResult struct { + Intent Intent + Source Source + Confidence float64 + Clarify bool +} + +// Route runs the heads and reports whether they are confident enough to answer. +// A false second return is a decline, not an error: the cascade goes on to the +// resident model, which is what happens today. +func (h *RouterHeads) Route(ctx context.Context, utterance string) (headsResult, bool, error) { + if h == nil { + return headsResult{}, false, nil + } + ids, mask, _ := h.tokenizer.Encode(queryPrefix + utterance) + ids, mask = ids[:headsSeq], mask[:headsSeq] + // The tokenizer pads and truncates to its own length, which is longer than + // this one. Cutting the tail can cut the separator with it, so put it back. + if mask[headsSeq-1] == 1 { + ids[headsSeq-1] = sepTokenID + } + + shape := ort.NewShape(1, headsSeq) + idsT, err := ort.NewTensor(shape, ids) + if err != nil { + return headsResult{}, false, fmt.Errorf("heads: ids tensor: %w", err) + } + defer idsT.Destroy() + maskT, err := ort.NewTensor(shape, mask) + if err != nil { + return headsResult{}, false, fmt.Errorf("heads: mask tensor: %w", err) + } + defer maskT.Destroy() + + intentT, err := ort.NewEmptyTensor[float32](ort.NewShape(1, int64(len(h.intents)))) + if err != nil { + return headsResult{}, false, fmt.Errorf("heads: intent tensor: %w", err) + } + defer intentT.Destroy() + sourceT, err := ort.NewEmptyTensor[float32](ort.NewShape(1, int64(len(h.sources)))) + if err != nil { + return headsResult{}, false, fmt.Errorf("heads: source tensor: %w", err) + } + defer sourceT.Destroy() + slotsT, err := ort.NewEmptyTensor[float32](ort.NewShape(1, headsSeq, int64(numBIOTags))) + if err != nil { + return headsResult{}, false, fmt.Errorf("heads: slots tensor: %w", err) + } + defer slotsT.Destroy() + clarifyT, err := ort.NewEmptyTensor[float32](ort.NewShape(1, 2)) + if err != nil { + return headsResult{}, false, fmt.Errorf("heads: clarify tensor: %w", err) + } + defer clarifyT.Destroy() + + if err := h.session.Run( + []*ort.Tensor[int64]{idsT, maskT}, + []*ort.Tensor[float32]{intentT, sourceT, slotsT, clarifyT}, + ); err != nil { + return headsResult{}, false, fmt.Errorf("heads: run: %w", err) + } + + // The graph applies its own softmax, so these are probabilities and the max + // is the same number the eval calibrated the threshold against. + i, conf := argmax(intentT.GetData()) + res := headsResult{ + Intent: h.intents[i], + Confidence: conf, + } + cl := clarifyT.GetData() + res.Clarify = len(cl) == 2 && cl[1] > cl[0] + + // The destination head is trained on query rows and is meaningless on any + // other intent, the same way queryWalk is never reached by one. + if res.Intent == IntentQuery { + s, _ := argmax(sourceT.GetData()) + res.Source = h.sources[s] + } + + // The clarify head decides on its own, and it decides first. It answers a + // different question from the intent head — not which intent, but whether + // there is enough here to act on at all — so a low intent confidence is no + // reason to discard it. It is usually the same turns: "вода" reads as + // intent act at 0.23 and clarify at 0.98, and letting the intent threshold + // bury that hands the turn to the classifier, which routes it confidently + // and never asks. + if res.Clarify { + return res, true, nil + } + if conf < h.threshold { + return res, false, nil + } + return res, true, nil +} + +// numBIOTags — O plus B- and I- for each of Maven's five slots. The head is not +// read, but the graph writes it and the output tensor has to be the right size. +const numBIOTags = 11 + +func argmax(v []float32) (int, float64) { + best, bestV := 0, math.Inf(-1) + for i, x := range v { + if float64(x) > bestV { + best, bestV = i, float64(x) + } + } + return best, bestV +} diff --git a/internal/router/onnxembedder.go b/internal/router/onnxembedder.go index cbbf63c..005255f 100644 --- a/internal/router/onnxembedder.go +++ b/internal/router/onnxembedder.go @@ -66,12 +66,19 @@ func NewONNXEmbedder(modelPath, tokenizerPath, libPath string) (*onnxEmbedder, e func (e *onnxEmbedder) Dim() int { return embedDim } // ID names the loaded model for the DB marker (Vikunja #378): the model file's -// own name plus the dimension, so pointing the config at another model changes -// the string on its own. +// own name, the dimension, and the tokenizer revision, so pointing the config +// at another model changes the string on its own. func (e *onnxEmbedder) ID() string { return e.id } +// tokenizerRev — bumped whenever the tokenizer changes what it emits for the +// same text, because that changes every vector while the model file's name +// stays put. Rev 2 is the fix for the reversed word pieces (V-664): stored +// passages embedded under rev 1 no longer sit in the same space as a query +// embedded now, and ReembedAll rewrites them because this string moved. +const tokenizerRev = 2 + // modelIDFromPath turns /opt/.../multilingual-e5-small.onnx into -// "multilingual-e5-small@384". +// "multilingual-e5-small@384/tok2". func modelIDFromPath(modelPath string) string { name := modelPath if i := strings.LastIndexAny(name, "/\\"); i >= 0 { @@ -81,7 +88,7 @@ func modelIDFromPath(modelPath string) string { if name == "" { name = "onnx" } - return fmt.Sprintf("%s@%d", name, embedDim) + return fmt.Sprintf("%s@%d/tok%d", name, embedDim, tokenizerRev) } // Embed treats the text as a query. The classifier compares one short @@ -339,14 +346,17 @@ func (t *unigramTokenizer) encodeWord(word string) []int64 { } } + // Backtracking walks the word from its end, and prepending each piece puts + // it back in reading order. There used to be a second reverse after this + // loop, which undid it: every multi-piece word came out backwards, and + // "query: вода" tokenized to [0 12 1294 41 12489 2] where the reference + // tokenizer gives [0 41 1294 12 12489 2] (V-664). A transformer reads + // position, so the pieces of a long Russian word were being read in the + // wrong order on every turn. var result []int64 for i := n; i > 0; i = prev[i] { result = append([]int64{bestID[i]}, result...) } - // Reverse - for l, r := 0, len(result)-1; l < r; l, r = l+1, r-1 { - result[l], result[r] = result[r], result[l] - } return result } diff --git a/internal/router/router.go b/internal/router/router.go index 2a6415d..3ad9653 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -31,6 +31,12 @@ type Config struct { // error/parse failure, falls through to the classifier (never fails the // turn on the model). LLM *LLMRouter + // Heads — optional routing heads over the fine-tuned embedder copy. When + // set, Route consults them after stage 0 and before the LLM router. They + // decline below their own confidence threshold, so a low-confidence turn + // reaches the model exactly as it does today. Nil is the shipped-before + // behaviour and costs nothing. + Heads *RouterHeads } // Router — the deterministic cascade. Route never guesses: stage 0 wins @@ -42,6 +48,7 @@ type Router struct { extractor Extractor threshold float64 llm *LLMRouter + heads *RouterHeads } func New(cfg Config) *Router { @@ -51,6 +58,7 @@ func New(cfg Config) *Router { extractor: cfg.Extractor, threshold: cfg.Threshold, llm: cfg.LLM, + heads: cfg.Heads, } } @@ -98,6 +106,60 @@ func (r *Router) Route(ctx context.Context, utterance string, now time.Time) (De } r.noteGrammarOutcomes(ctx, len(r.grammars), declinedBuild, "", "") + // stage 0b — routing heads (when wired). A softmax over the label set, so + // it cannot name an intent or a destination that does not exist, and its + // max is a real confidence. It runs before the model because it is three + // orders of magnitude faster and scores better on both halves of the route. + // + // It declines below its threshold rather than clarifying. A declined turn + // carries on to the model and then the classifier, which is what a box with + // no weights file does on every turn. + if r.heads != nil { + res, ok, err := r.heads.Route(ctx, utterance) + switch { + case err != nil: + log.Printf("router: heads fell through to the rest of the cascade: %v", err) + decision.Note(ctx, decision.Claim{ + Stage: decision.StageRoute, Claimant: claimantHeads, + Outcome: decision.Declined, Reason: "error: " + err.Error(), + }) + case !ok: + decision.Note(ctx, decision.Scored(decision.StageRoute, claimantHeads, + string(res.Intent), res.Confidence, decision.Declined, + "below the heads confidence threshold")) + default: + d := Decision{ + Utterance: utterance, + Stage: 2, + Intent: res.Intent, + Confidence: res.Confidence, + Source: res.Source, + Clarify: res.Clarify, + } + r.fillSlots(ctx, &d, now) + decision.Note(ctx, decision.Claim{ + Stage: decision.StageRoute, Claimant: claimantLLM, + Outcome: decision.NeverAsked, Reason: "the routing heads answered", + }) + decision.Note(ctx, decision.Claim{ + Stage: decision.StageRoute, Claimant: claimantClassifier, + Outcome: decision.NeverAsked, Reason: "the routing heads answered", + }) + outcome, reason := decision.Won, "" + if d.Clarify { + outcome, reason = decision.Thinned, "the clarify head says there is too little here to act on" + } + decision.Note(ctx, decision.Scored(decision.StageRoute, claimantHeads, + string(d.Intent), d.Confidence, outcome, reason)) + return d, nil + } + } else { + decision.Note(ctx, decision.Claim{ + Stage: decision.StageRoute, Claimant: claimantHeads, + Outcome: decision.NeverAsked, Reason: "no routing heads are wired", + }) + } + // stage 1a — LLM router (when wired). It reasons over the utterance instead // of nearest-centroid guessing. On any error/parse-fail, fall through to the // classifier cascade (never fail the turn on the model). diff --git a/internal/stt/http.go b/internal/stt/http.go new file mode 100644 index 0000000..adf04fd --- /dev/null +++ b/internal/stt/http.go @@ -0,0 +1,92 @@ +package stt + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "strconv" + "time" + + "github.com/kami/maven/internal/audio" +) + +// HTTPTranscriber — speech-to-text on another host, over HTTP. +// +// mavsttd is whisper.cpp linked into a Go daemon and reached over a unix +// socket. CrisperWhisper 2.0 cannot be reached that way: whisper.cpp derives +// its language count from the vocabulary size, and CW2's 51897 tokens shift +// seven special token ids. It runs under transformers instead, as a service +// beside the model on workpc. See docs/evals/2026-08-09-crisperwhisper2-russian-wer.md. +// +// So this is the second transport for the same seam, not a second seam. The +// caller still sees stt.Transcriber and one method. +type HTTPTranscriber struct { + url string + token string + lang string + http *http.Client +} + +// NewHTTPTranscriber builds the remote client. token may be empty for a +// service on a trusted socket, but audio is the most sensitive thing that +// crosses this seam, so a LAN deployment should always set one. +func NewHTTPTranscriber(url, token, lang string, timeout time.Duration) *HTTPTranscriber { + return &HTTPTranscriber{ + url: url, + token: token, + lang: lang, + http: &http.Client{Timeout: timeout}, + } +} + +// ErrFormat — the audio is not the one canonical shape. Refused at the seam +// rather than sent to a model that expects something else. +var ErrFormat = errors.New("stt: audio is not 16kHz mono pcm_s16le") + +type httpTranscript struct { + Text string `json:"text"` + Confidence float64 `json:"confidence"` +} + +// Transcribe posts the raw PCM and reads back the text. +// +// The body is the PCM bytes themselves rather than JSON. A minute of 16kHz +// mono is under 2MB raw and about 2.6MB base64, and the format is fixed by +// audio.PCM16kMono, so a header carries it more cheaply than an envelope. +func (t *HTTPTranscriber) Transcribe(ctx context.Context, a audio.Audio) (string, float64, error) { + if !a.Format.IsValid() { + return "", 0, ErrFormat + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, t.url, bytes.NewReader(a.Bytes)) + if err != nil { + return "", 0, fmt.Errorf("stt: build request: %w", err) + } + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("X-Sample-Rate", strconv.Itoa(a.Format.SampleRate)) + req.Header.Set("X-Channels", strconv.Itoa(a.Format.Channels)) + req.Header.Set("X-Sample-Bits", strconv.Itoa(a.Format.SampleBits)) + req.Header.Set("X-Language", t.lang) + if t.token != "" { + req.Header.Set("Authorization", "Bearer "+t.token) + } + + resp, err := t.http.Do(req) + if err != nil { + return "", 0, fmt.Errorf("stt: post audio: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", 0, fmt.Errorf("stt: remote returned %d", resp.StatusCode) + } + + var out httpTranscript + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return "", 0, fmt.Errorf("stt: decode transcript: %w", err) + } + return out.Text, out.Confidence, nil +} + +var _ Transcriber = (*HTTPTranscriber)(nil) diff --git a/internal/stt/http_test.go b/internal/stt/http_test.go new file mode 100644 index 0000000..85d3289 --- /dev/null +++ b/internal/stt/http_test.go @@ -0,0 +1,89 @@ +package stt + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strconv" + "testing" + "time" + + "github.com/kami/maven/internal/audio" +) + +func TestHTTPTranscriberSendsRawPCM(t *testing.T) { + t.Parallel() + var gotBody []byte + var gotHeader http.Header + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotBody, _ = io.ReadAll(r.Body) + gotHeader = r.Header.Clone() + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"text":"привет","confidence":0.82}`) + })) + defer srv.Close() + + a := audio.Audio{Format: audio.PCM16kMono, Bytes: []byte("pcm-bytes")} + tr := NewHTTPTranscriber(srv.URL, "s3cret", "ru", 2*time.Second) + text, conf, err := tr.Transcribe(context.Background(), a) + if err != nil { + t.Fatalf("Transcribe: %v", err) + } + if text != "привет" || conf != 0.82 { + t.Fatalf("got %q %v", text, conf) + } + if string(gotBody) != "pcm-bytes" { + t.Fatalf("body should be the PCM itself, got %q", gotBody) + } + if got := gotHeader.Get("X-Sample-Rate"); got != strconv.Itoa(audio.PCM16kMono.SampleRate) { + t.Fatalf("X-Sample-Rate = %q", got) + } + if got := gotHeader.Get("X-Language"); got != "ru" { + t.Fatalf("X-Language = %q", got) + } + // Audio is the most sensitive thing crossing this seam. + if got := gotHeader.Get("Authorization"); got != "Bearer s3cret" { + t.Fatalf("Authorization = %q", got) + } +} + +func TestHTTPTranscriberOmitsEmptyToken(t *testing.T) { + t.Parallel() + var auth string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + auth = r.Header.Get("Authorization") + _, _ = io.WriteString(w, `{"text":"x"}`) + })) + defer srv.Close() + a := audio.Audio{Format: audio.PCM16kMono, Bytes: []byte("x")} + if _, _, err := NewHTTPTranscriber(srv.URL, "", "ru", time.Second).Transcribe(context.Background(), a); err != nil { + t.Fatalf("Transcribe: %v", err) + } + if auth != "" { + t.Fatalf("Authorization should be absent, got %q", auth) + } +} + +func TestHTTPTranscriberRefusesWrongFormat(t *testing.T) { + t.Parallel() + a := audio.Audio{Format: audio.Format{SampleRate: 44100, Channels: 2, SampleBits: 16, Encoding: "pcm_s16le"}} + _, _, err := NewHTTPTranscriber("http://example.invalid", "", "ru", time.Second).Transcribe(context.Background(), a) + if !errors.Is(err, ErrFormat) { + t.Fatalf("want ErrFormat, got %v", err) + } +} + +func TestHTTPTranscriberErrorsOnBadStatus(t *testing.T) { + t.Parallel() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + })) + defer srv.Close() + a := audio.Audio{Format: audio.PCM16kMono, Bytes: []byte("x")} + _, _, err := NewHTTPTranscriber(srv.URL, "", "ru", time.Second).Transcribe(context.Background(), a) + if err == nil { + t.Fatal("a 401 must be an error, so the Pair falls back") + } +} diff --git a/internal/stt/pair.go b/internal/stt/pair.go new file mode 100644 index 0000000..2001ed6 --- /dev/null +++ b/internal/stt/pair.go @@ -0,0 +1,157 @@ +package stt + +import ( + "context" + "errors" + "log" + "net/http" + "sync" + "sync/atomic" + "time" + + "github.com/kami/maven/internal/audio" +) + +// Pair — a preferred transcriber on the workstation, with mavsttd as the floor. +// +// Same arrangement as llm.Pair and for the same reason. The microphone is at +// workpc, the card there has 16GB, and CrisperWhisper 2.0 turbo scores 10.4% +// WER in Russian against 27.5% for the ggml-small.bin homesrv loads +// (docs/evals/2026-08-09-crisperwhisper2-russian-wer.md). The workstation is +// never assumed up: it sleeps, and the card is often held by a training run. +// +// Speech-to-text has only the silent half of the degradation rule. A worse +// transcript is still a turn, and there is nothing to name a gap about, so +// Transcribe always falls back. That is the whole difference from llm.Pair, +// which also carries CompleteRemote for callers that must refuse instead. +type Pair struct { + remote Transcriber + floor Transcriber + + // up — the cached admission answer, written only by the prober and read by + // every turn. A voice turn must never wait on a machine that may be asleep. + up atomic.Bool + + health string + interval time.Duration + http *http.Client + stop chan struct{} + stopOnce sync.Once +} + +const ( + probeTimeout = 2 * time.Second + defaultProbeInterval = 15 * time.Second +) + +// ErrNoFloor — a Pair was built with no local transcriber to fall back to. A +// configuration mistake: the floor is what makes the remote optional. +var ErrNoFloor = errors.New("stt: no floor transcriber") + +// NewPair builds the two-transcriber arrangement. remote may be nil, which is +// the unconfigured deploy: every turn goes to the floor and nothing probes. +func NewPair(remote, floor Transcriber, health string, interval time.Duration) *Pair { + if interval <= 0 { + // The config normalises this, so a zero here is a caller that built the + // Pair directly. Panicking in a ticker is the wrong way to say so. + interval = defaultProbeInterval + } + return &Pair{ + remote: remote, + floor: floor, + health: health, + interval: interval, + http: &http.Client{Timeout: probeTimeout}, + stop: make(chan struct{}), + } +} + +// Start begins probing. The first probe runs before the first tick, so a +// workstation that is already up serves the first utterance rather than the +// second. Safe with a nil remote. +func (p *Pair) Start(ctx context.Context) { + if p.remote == nil || p.health == "" { + return + } + go func() { + p.probe(ctx) + t := time.NewTicker(p.interval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-p.stop: + return + case <-t.C: + p.probe(ctx) + } + } + }() +} + +// Stop ends the prober. Idempotent and safe from two goroutines. +func (p *Pair) Stop() { + p.stopOnce.Do(func() { close(p.stop) }) +} + +// Available reports whether the workstation will transcribe right now. +func (p *Pair) Available() bool { + return p.remote != nil && p.up.Load() +} + +func (p *Pair) probe(ctx context.Context) { + ctx, cancel := context.WithTimeout(ctx, probeTimeout) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.health, nil) + if err != nil { + p.set(false) + return + } + resp, err := p.http.Do(req) + if err != nil { + p.set(false) + return + } + defer resp.Body.Close() + p.set(resp.StatusCode == http.StatusOK) +} + +// set records the admission answer and logs only transitions. A machine that +// sleeps nightly would otherwise write one line per interval forever. +func (p *Pair) set(up bool) { + if p.up.Swap(up) == up { + return + } + if up { + log.Printf("stt: workstation transcriber available at %s", p.health) + } else { + log.Print("stt: workstation transcriber unavailable, falling back to mavsttd") + } +} + +// Transcribe sends the audio to the workstation when it will take work, and to +// mavsttd otherwise. A remote that fails mid-request falls back too, because +// the admission answer is a cache and can be one interval out of date. +// +// Killing the remote mid-session must not drop the turn. That is the whole +// point of the floor, and it is what TestPairFallsBackWhenRemoteFails pins. +func (p *Pair) Transcribe(ctx context.Context, a audio.Audio) (string, float64, error) { + if p.floor == nil { + return "", 0, ErrNoFloor + } + if p.Available() { + text, conf, err := p.remote.Transcribe(ctx, a) + if err == nil { + log.Print("stt: transcribed on the workstation") + return text, conf, nil + } + // The cached answer was wrong. Correct it now rather than sending the + // next utterance into the same hole, then fall back. + p.set(false) + log.Printf("stt: workstation failed mid-request, falling back: %v", err) + } + return p.floor.Transcribe(ctx, a) +} + +var _ Transcriber = (*Pair)(nil) diff --git a/internal/stt/pair_test.go b/internal/stt/pair_test.go new file mode 100644 index 0000000..b14b241 --- /dev/null +++ b/internal/stt/pair_test.go @@ -0,0 +1,143 @@ +package stt + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/kami/maven/internal/audio" +) + +// scripted — a Transcriber that answers with a fixed text, or fails. +type scripted struct { + text string + err error + calls atomic.Int32 +} + +func (s *scripted) Transcribe(_ context.Context, _ audio.Audio) (string, float64, error) { + s.calls.Add(1) + if s.err != nil { + return "", 0, s.err + } + return s.text, 0.9, nil +} + +func sample() audio.Audio { + return audio.Audio{Format: audio.PCM16kMono, Bytes: make([]byte, 3200)} +} + +// up builds a Pair whose admission answer is already true, without probing. +func up(remote, floor Transcriber) *Pair { + p := NewPair(remote, floor, "", time.Minute) + p.up.Store(true) + return p +} + +func TestPairPrefersTheWorkstation(t *testing.T) { + t.Parallel() + remote := &scripted{text: "с рабочей станции"} + floor := &scripted{text: "с homesrv"} + text, _, err := up(remote, floor).Transcribe(context.Background(), sample()) + if err != nil { + t.Fatalf("Transcribe: %v", err) + } + if text != "с рабочей станции" { + t.Fatalf("want the remote transcript, got %q", text) + } + if floor.calls.Load() != 0 { + t.Fatalf("floor was called %d times, want 0", floor.calls.Load()) + } +} + +// The turn is what matters. A remote that dies mid-session must cost a worse +// transcript and nothing else. This is the V-486 bar. +func TestPairFallsBackWhenRemoteFails(t *testing.T) { + t.Parallel() + remote := &scripted{err: errors.New("connection refused")} + floor := &scripted{text: "с homesrv"} + p := up(remote, floor) + + text, conf, err := p.Transcribe(context.Background(), sample()) + if err != nil { + t.Fatalf("a failed remote must not fail the turn: %v", err) + } + if text != "с homesrv" { + t.Fatalf("want the floor transcript, got %q", text) + } + if conf != 0.9 { + t.Fatalf("want the floor confidence, got %v", conf) + } + if p.Available() { + t.Fatal("a failed request must correct the cached admission answer") + } + + // The next utterance goes straight to the floor rather than into the + // same hole. + if _, _, err := p.Transcribe(context.Background(), sample()); err != nil { + t.Fatalf("second turn: %v", err) + } + if remote.calls.Load() != 1 { + t.Fatalf("remote called %d times, want 1", remote.calls.Load()) + } +} + +func TestPairWithNoRemoteIsTheFloor(t *testing.T) { + t.Parallel() + floor := &scripted{text: "с homesrv"} + p := NewPair(nil, floor, "", time.Minute) + p.Start(context.Background()) // no health url, so this is a no-op + if p.Available() { + t.Fatal("an unconfigured remote is never available") + } + text, _, err := p.Transcribe(context.Background(), sample()) + if err != nil { + t.Fatalf("Transcribe: %v", err) + } + if text != "с homesrv" { + t.Fatalf("want the floor transcript, got %q", text) + } +} + +func TestPairWithNoFloorRefuses(t *testing.T) { + t.Parallel() + _, _, err := NewPair(nil, nil, "", time.Minute).Transcribe(context.Background(), sample()) + if !errors.Is(err, ErrNoFloor) { + t.Fatalf("want ErrNoFloor, got %v", err) + } +} + +func TestPairProbeReadsHealth(t *testing.T) { + t.Parallel() + var ok atomic.Bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if !ok.Load() { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + p := NewPair(&scripted{text: "remote"}, &scripted{text: "floor"}, srv.URL, time.Minute) + p.probe(context.Background()) + if p.Available() { + t.Fatal("a 503 means the card is busy, so the workstation is not available") + } + ok.Store(true) + p.probe(context.Background()) + if !p.Available() { + t.Fatal("a 200 means the workstation will take work") + } +} + +func TestPairStopIsIdempotent(t *testing.T) { + t.Parallel() + p := NewPair(nil, &scripted{}, "", time.Minute) + p.Stop() + p.Stop() +}