Harden semantic boundaries and repair dialogue state
Replace nearest-neighbour personal routing with a frozen class-balanced linear head measured on historical, stratified, cross-validation, holdout, and fresh challenge gates (V-702). Close the four repair handoff holes, preserve nested clarification flows, and route Russian possession statements through structural grammar rather than lexical exceptions (V-573). Owner explicitly requested direct commits to master.
This commit is contained in:
+5
-20
@@ -40,34 +40,19 @@ type label struct {
|
||||
Labeled bool `json:"labeled"`
|
||||
}
|
||||
|
||||
// grammars mirrors buildRouter's order in cmd/mavend/voicewire.go. Order is
|
||||
// load-bearing there and so it is here: the agenda rules must sit after the
|
||||
// clock rules, Praxis before the capture marker, the narrative rules last.
|
||||
// grammars is the daemon's canonical ordered stage-zero set. Label generation
|
||||
// must not maintain a second copy: that drift was the defect fixed by V-693.
|
||||
func grammars() []router.Grammar {
|
||||
var g []router.Grammar
|
||||
g = append(g, router.SystemTimeDateGrammars()...)
|
||||
g = append(g, router.AgendaQueryGrammars()...)
|
||||
g = append(g, router.FeedQueryGrammar())
|
||||
g = append(g, router.TaskListGrammar())
|
||||
g = append(g, router.ListGrammars()...)
|
||||
g = append(g, router.ReminderGrammar())
|
||||
g = append(g, router.PraxisGrammars()...)
|
||||
g = append(g, router.TaskCaptureGrammar())
|
||||
g = append(g, router.NarrativeQueryGrammars()...)
|
||||
return g
|
||||
return router.StageZeroGrammars(router.DefaultActMatcher{})
|
||||
}
|
||||
|
||||
func match(gs []router.Grammar, utterance string) label {
|
||||
out := label{Utterance: utterance}
|
||||
for _, g := range gs {
|
||||
m := g.Pattern.FindStringSubmatch(utterance)
|
||||
if m == nil {
|
||||
d, matched, ok := g.Evaluate(utterance)
|
||||
if !matched || !ok {
|
||||
continue
|
||||
}
|
||||
d, ok := g.Build(m)
|
||||
if !ok {
|
||||
continue // the rule saw its shape and declined it
|
||||
}
|
||||
out.Intent = string(d.Intent)
|
||||
out.Grammar = g.Name
|
||||
out.Key = d.Slots.Key
|
||||
|
||||
+50
-7
@@ -342,7 +342,7 @@ func (h *reactiveHandler) resolveClarifyAnswer(ctx context.Context, text string)
|
||||
|
||||
switch role {
|
||||
case roleCancel:
|
||||
h.clarifyStore.Delete(dialogueIDOf(ctx))
|
||||
h.completeClarifyTop(ctx)
|
||||
return clarifyCancelled, true
|
||||
case roleSideQuery:
|
||||
// He asked something of his own WITHOUT leaving the flow. The question
|
||||
@@ -419,8 +419,6 @@ func (h *reactiveHandler) resolveClarifyAnswer(ctx context.Context, text string)
|
||||
if stillOpen(q.Missing, whenTextOf(q), merged) {
|
||||
return h.reaskOrGiveUp(ctx, q, merged, text, taken), true
|
||||
}
|
||||
h.clarifyStore.Delete(dialogueIDOf(ctx))
|
||||
|
||||
// One gap filled is not the same as a complete request. askClarify parks
|
||||
// only the first gap, because one question per turn is the rule, but a
|
||||
// reminder wants both a subject and a time. "напомни" with neither used to
|
||||
@@ -431,6 +429,7 @@ func (h *reactiveHandler) resolveClarifyAnswer(ctx context.Context, text string)
|
||||
if reply, asked := h.askRemainingGap(ctx, q, intent, merged); asked {
|
||||
return reply, true
|
||||
}
|
||||
h.completeClarifyTop(ctx)
|
||||
|
||||
// Rebuild the decision as if it had routed cleanly, then run it down the
|
||||
// normal path. Clarify is deliberately false and the intent is unchanged:
|
||||
@@ -446,6 +445,27 @@ func (h *reactiveHandler) resolveClarifyAnswer(ctx context.Context, text string)
|
||||
return h.finishClarified(ctx, dec), true
|
||||
}
|
||||
|
||||
// completeClarifyTop finishes only the active question. A nested question can
|
||||
// sit above a flow that was suspended by a side request or repair; deleting the
|
||||
// dialogue id here erased both. If one survives underneath, restart its clock
|
||||
// from the moment it is spoken again and attach its question to this turn.
|
||||
func (h *reactiveHandler) completeClarifyTop(ctx context.Context) {
|
||||
if h.clarifyStore == nil {
|
||||
return
|
||||
}
|
||||
_, resumed := h.clarifyStore.CompleteTop(dialogueIDOf(ctx), h.now())
|
||||
if resumed == nil || len(resumed.Missing) == 0 {
|
||||
return
|
||||
}
|
||||
question, ok := clarifyResumedFor(resumed.Missing[0])
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if rt := turnRouteFrom(ctx); rt != nil {
|
||||
rt.resume = question
|
||||
}
|
||||
}
|
||||
|
||||
// noteDropped records that the parked request was let go this turn, so runTurn
|
||||
// can say it in front of whatever these words are answered with. Nothing to
|
||||
// record outside runTurn — a unit test calling one resolver has no turn to glue
|
||||
@@ -487,7 +507,7 @@ func (h *reactiveHandler) noteSuspended(ctx context.Context, q *dialogue.Pending
|
||||
return
|
||||
}
|
||||
if !q.CanResume() {
|
||||
h.clarifyStore.Delete(dialogueIDOf(ctx))
|
||||
h.completeClarifyTop(ctx)
|
||||
h.noteDropped(ctx)
|
||||
log.Printf("voice: clarify — letting the question about %s go: %d asides in a row, %d rides in all", q.Missing[0], q.Suspends, q.Rides)
|
||||
return
|
||||
@@ -576,7 +596,7 @@ func (h *reactiveHandler) reaskOrGiveUp(ctx context.Context, q *dialogue.Pending
|
||||
question, _ = h.questionFor(q.Missing[0], q.Attempts+1, whenTextOf(q), merged, taken)
|
||||
}
|
||||
if question == "" || !q.CanAsk() {
|
||||
h.clarifyStore.Delete(dialogueIDOf(ctx))
|
||||
h.completeClarifyTop(ctx)
|
||||
log.Printf("voice: clarify — gave up on %v after %d question(s), answer was %q", q.Missing, q.Attempts, text)
|
||||
return clarifyGaveUp
|
||||
}
|
||||
@@ -590,15 +610,38 @@ func (h *reactiveHandler) reaskOrGiveUp(ctx context.Context, q *dialogue.Pending
|
||||
return question
|
||||
}
|
||||
|
||||
// finishClarified runs a completed decision through the same steps a freshly
|
||||
// routed one takes: remember the turn, act, then phrase.
|
||||
// finishClarified completes a decision whose parked gaps were already checked
|
||||
// by resolveClarifyAnswer. It still records the turn for a later correction;
|
||||
// the old path made anything completed through dialogue uncorrectable (V-573).
|
||||
func (h *reactiveHandler) finishClarified(ctx context.Context, dec router.Decision) string {
|
||||
return h.finishRebuilt(ctx, dec, false)
|
||||
}
|
||||
|
||||
// finishRepaired validates a decision rebuilt from an older utterance. Unlike
|
||||
// resolveClarifyAnswer, repair has not passed the current slot gate, so it must
|
||||
// ask about any missing argument before acting (V-573).
|
||||
func (h *reactiveHandler) finishRepaired(ctx context.Context, dec router.Decision) string {
|
||||
return h.finishRebuilt(ctx, dec, true)
|
||||
}
|
||||
|
||||
func (h *reactiveHandler) finishRebuilt(ctx context.Context, dec router.Decision, validate bool) string {
|
||||
if validate && (dec.Clarify || len(missingFor(dec)) > 0) {
|
||||
if reply := h.hexisBeforeClarify(ctx, dec); reply != "" {
|
||||
return reply
|
||||
}
|
||||
if question, asked := h.askClarify(ctx, dec); asked {
|
||||
return question
|
||||
}
|
||||
}
|
||||
if h.dialogueSessions != nil {
|
||||
now := h.now()
|
||||
prev := h.dialogueSessions.Get(dialogueIDOf(ctx), now)
|
||||
dec = followUpMerge(prev, dec, now)
|
||||
h.rememberTurn(ctx, prev, dec, now)
|
||||
}
|
||||
if !dec.Clarify {
|
||||
h.recordTurn(dec.Utterance, dec.Intent)
|
||||
}
|
||||
reply := h.applyAction(ctx, dec)
|
||||
if reply == "" {
|
||||
reply = h.replier.Reply(ctx, dec)
|
||||
|
||||
@@ -31,8 +31,8 @@ import (
|
||||
// and nothing should: a missing name costs one line of the record, while a
|
||||
// check that walks the ladder would have to run the ladder.
|
||||
var preRouteLadder = []string{
|
||||
"confirm", "clarify-answer", "quiet-toggle", "snooze", "ack", "repair",
|
||||
"repair-negative", "ordinal",
|
||||
"confirm", "repair", "repair-negative", "clarify-answer", "quiet-toggle",
|
||||
"snooze", "ack", "ordinal",
|
||||
}
|
||||
|
||||
// notePreRoute records one rung of that ladder and passes its verdict through
|
||||
|
||||
@@ -651,7 +651,7 @@ func dialogueTraces() []trace {
|
||||
end: endState{},
|
||||
},
|
||||
|
||||
// ---- rows below carry the CORRECT expectation and fail today ----
|
||||
// ---- formerly failing interleavings; kept as permanent contracts ----
|
||||
|
||||
// The owner's own sentence from V-577 shape 2, in his words. It needs
|
||||
// an engine that can route it: the hash embedder marks it note with
|
||||
@@ -660,7 +660,6 @@ func dialogueTraces() []trace {
|
||||
// floor's deterministic fact parser reads.
|
||||
{
|
||||
name: "a note stated mid-flow is stored, not dropped",
|
||||
skip: "the offline floor cannot route «у меня новый ноутбук» confidently; needs the resident model",
|
||||
turns: []turn{
|
||||
{say: "напомни позвонить врачу", question: dialogue.SlotTime, attempt: 1,
|
||||
parked: &parkedWant{slot: dialogue.SlotTime, attempt: 1, carries: "врачу"}},
|
||||
@@ -747,7 +746,6 @@ func dialogueTraces() []trace {
|
||||
// written yet and is not this task's to invent.
|
||||
{
|
||||
name: "cancel: a parked question, then never mind",
|
||||
skip: "V-560: a cancel is scored as a failed answer, not as a cancel",
|
||||
turns: []turn{
|
||||
{say: "напомни позвонить маме", question: dialogue.SlotTime, attempt: 1,
|
||||
parked: &parkedWant{slot: dialogue.SlotTime, attempt: 1}},
|
||||
@@ -761,7 +759,6 @@ func dialogueTraces() []trace {
|
||||
// scores "нет, это была заметка" as a bad time answer and asks again.
|
||||
{
|
||||
name: "correction while a question is parked",
|
||||
skip: "V-560: clarify pre-empts the repair marker, so a correction cannot be spoken mid-flow",
|
||||
turns: []turn{
|
||||
{say: "добавь в задачи купить молоко", contains: []string{"купить молоко"}},
|
||||
{say: "напомни позвонить маме", question: dialogue.SlotTime, attempt: 1,
|
||||
@@ -771,21 +768,17 @@ func dialogueTraces() []trace {
|
||||
},
|
||||
end: endState{tasks: []string{"купить молоко"}},
|
||||
},
|
||||
// A reminder said whole, in one breath, with the hour in it — and she
|
||||
// asks when. ReminderGrammar (stage0.go) builds its slots by hand and
|
||||
// never runs the extractor, so a stage-0 reminder carries no time
|
||||
// whatever the sentence says, and the clarify gate reads the gap as
|
||||
// real. It costs a turn on the commonest reminder shape there is.
|
||||
//
|
||||
// Hermetic despite the date parser: stage 0 calls no parser at all, so
|
||||
// this fails the same way with or without python dateparser installed.
|
||||
// Stage 0 has extracted the hour since V-572. The day remains genuinely
|
||||
// absent, and V-579 deliberately refuses to invent it even when 11:00 is
|
||||
// still ahead on today's clock. This stale skipped row used to expect a
|
||||
// commit and contradicted every neighbouring time-contract row.
|
||||
{
|
||||
name: "a reminder said whole is not asked about",
|
||||
skip: "V-562: a stage-0 decision never meets the extractor, so its slots are never validated",
|
||||
name: "a stage-zero reminder keeps its hour and asks for the missing day",
|
||||
turns: []turn{
|
||||
{say: "напомни в 11:00 позвонить маме", contains: []string{"11:00"}, noQuestion: true},
|
||||
{say: "напомни в 11:00 позвонить маме", question: dialogue.SlotTime, attempt: 1,
|
||||
gap: whenNoDay, parked: &parkedWant{slot: dialogue.SlotTime, attempt: 1, carries: "маме"}},
|
||||
},
|
||||
end: endState{reminders: []reminderWant{{payload: "позвонить маме", fireAt: "2026-07-31 11:00"}}},
|
||||
end: endState{},
|
||||
},
|
||||
// The same gap on the repair path. A correction redoes the request
|
||||
// through finishClarified, which goes straight to applyAction — it never
|
||||
@@ -795,7 +788,6 @@ func dialogueTraces() []trace {
|
||||
// with no time.
|
||||
{
|
||||
name: "a correction that lands short asks rather than failing",
|
||||
skip: "V-562: finishClarified skips the clarify gate, so a repaired decision is never checked for gaps",
|
||||
turns: []turn{
|
||||
{say: "добавь в задачи купить молоко", contains: []string{"купить молоко"}},
|
||||
{say: "нет, это было напоминание", contains: []string{"поняла, это напоминание"},
|
||||
|
||||
+303
-47
@@ -2,6 +2,8 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"log"
|
||||
"math"
|
||||
"sync"
|
||||
@@ -23,31 +25,40 @@ import (
|
||||
// every utterance the list misses is one that reaches the world. It also drifts
|
||||
// silently — a missing verb looks exactly like no bug.
|
||||
//
|
||||
// So the boundary asks the embedder instead. Two frozen seed sets — questions
|
||||
// about him, questions about the world — are embedded once, and the turn's own
|
||||
// query vector, already computed by queryEmbed upstream, is scored against
|
||||
// both. Nearest side wins. Word order, verb form and unseen phrasing stop
|
||||
// mattering, which is exactly what a lexicon could not do.
|
||||
// So the boundary asks the embedder instead. A frozen bilingual corpus is
|
||||
// embedded at model-fit time, then a class-balanced logistic head is fitted
|
||||
// over those vectors. The head learns a direction in semantic space instead
|
||||
// of choosing whichever single example happens to share the most words. That
|
||||
// matters for a public noun inside a private question and for advice about an
|
||||
// owned object: nearest-neighbour scoring confuses both, while a trained head
|
||||
// combines the evidence across the whole sentence.
|
||||
//
|
||||
// Measured 03-08-2026 against multilingual-e5-small on 19 held-out utterances,
|
||||
// none of them a seed: 19 right (TestONNXPersonalBoundary). A 20th, "as i said,
|
||||
// what is the population of india", missed by +0.008 during the first pass and
|
||||
// is a world seed now, which is why it is not in the held-out set. True
|
||||
// positives clear the world side by +0.014 to +0.089 and the nearest true
|
||||
// negative sits at -0.005, so the gate is the sign of the difference and
|
||||
// nothing tighter: the margins are too thin to justify a threshold, and the
|
||||
// asymmetry favours claiming anyway. A false claim costs one honest "не знаю";
|
||||
// a false pass sends his life to an upstream engine.
|
||||
// The corpus covers six sentence shapes on both sides: remembered speech,
|
||||
// possession, narrative, first-person preambles, current advice/information,
|
||||
// and public proper nouns. Training weights each class equally, so the larger
|
||||
// world corpus cannot move the prior merely by containing more examples. A
|
||||
// small L2 term makes the solution stable; its value and the fixed optimiser
|
||||
// are measured by model-backed cross-validation, not adjusted at runtime.
|
||||
//
|
||||
// The embedder is the one model CLAUDE.md pins to homesrv permanently, and it
|
||||
// is what makes this affordable: no llama-server call, no network, one cosine
|
||||
// per seed against a vector the turn already has.
|
||||
// This linear head measures 29/29 on the historical regression suite and
|
||||
// 72/72 on the separate stratified fixture (V-702, 13-08-2026). The gate is
|
||||
// still probability 0.5: a false claim costs one honest "не знаю", while a
|
||||
// false pass can send his life to an upstream engine.
|
||||
//
|
||||
// The embedder is the one model CLAUDE.md pins to homesrv permanently. Its head
|
||||
// is fitted and verified by the model-backed gate, then frozen into the binary;
|
||||
// inference is one dot product against a vector the turn already has. Unknown
|
||||
// embedding spaces fit their own head once per process instead of applying
|
||||
// foreign weights. Neither path calls llama-server or the network.
|
||||
|
||||
// personalSeeds — questions about him. Frozen: they are scoring data, so
|
||||
// editing one moves the boundary and must be re-measured, not eyeballed. Cover
|
||||
// both classes the boundary owns, possession and first-person speech, in both
|
||||
// languages.
|
||||
// personalSeeds and worldSeeds are the frozen training corpus for the linear
|
||||
// boundary head. Editing either changes a model, not a phrase list: every edit
|
||||
// therefore needs the model-backed regression, stratified evaluation and
|
||||
// training-corpus cross-validation. The examples describe where an answer can
|
||||
// come from, in both languages. None is a special case copied from an eval.
|
||||
var personalSeeds = []string{
|
||||
// The original compact corpus. It remains here both as training signal and
|
||||
// as provenance for the regressions that introduced the semantic boundary.
|
||||
"что я говорил про это",
|
||||
"я тебе рассказывал об этом?",
|
||||
"что я записал про врача",
|
||||
@@ -56,21 +67,79 @@ var personalSeeds = []string{
|
||||
"когда моя встреча",
|
||||
"what did i say about this",
|
||||
"did i mention this to you",
|
||||
|
||||
// Remembered speech.
|
||||
"какой адрес я тебе сообщал?",
|
||||
"что я говорил о своём самочувствии?",
|
||||
"какое решение по ремонту я озвучил?",
|
||||
"что я обещал сделать после отпуска?",
|
||||
"what reason did I give for declining the offer?",
|
||||
"did I tell you where I grew up?",
|
||||
"which restaurant did I say I wanted to visit?",
|
||||
"what explanation did I give for missing the meeting?",
|
||||
|
||||
// Stored attributes of his possessions and records.
|
||||
"где лежит мой договор аренды?",
|
||||
"когда заканчивается моя подписка на спортзал?",
|
||||
"какой размер у моей запасной куртки?",
|
||||
"до какой даты действует мой пропуск?",
|
||||
"какой размер у моего велосипедного шлема?",
|
||||
"where is my vehicle registration document?",
|
||||
"when is my museum membership renewal?",
|
||||
"what number is on my travel insurance policy?",
|
||||
"which shelf did I put my tax folder on?",
|
||||
"what size is my waterproof coat?",
|
||||
|
||||
// Narratives that only his memories or records can supply.
|
||||
"собери по моим записям рассказ о поездке в Самару",
|
||||
"напомни, как прошёл мой первый урок вождения",
|
||||
"восстанови из дневника, как я искал первую квартиру",
|
||||
"перескажи по моим словам, как прошла встреча выпускников",
|
||||
"summarize my account of moving into this apartment",
|
||||
"tell me what happened during my first week at the new job",
|
||||
"recreate the story of my graduation from my journal",
|
||||
"piece together my account of adopting the dog",
|
||||
|
||||
// First-person framing around a private answer.
|
||||
"возвращаясь к нашей беседе, какой банк я выбрал?",
|
||||
"кажется, я уже говорил: на какую дату записался к врачу?",
|
||||
"если мы это обсуждали, какую школу вождения я предпочёл?",
|
||||
"напомню наш разговор: когда я решил менять работу?",
|
||||
"as I mentioned before, which contractor did I hire?",
|
||||
"coming back to our chat, what date did I book the inspection for?",
|
||||
"if we covered this already, which course did I enroll in?",
|
||||
"back to what I told you: where did I plan to stay in Oslo?",
|
||||
|
||||
// Current information that lives only in his records.
|
||||
"какой счёт мне нужно оплатить на этой неделе?",
|
||||
"сколько часов я работал в прошлом месяце?",
|
||||
"какую процедуру мастер советовал выполнить утром?",
|
||||
"какая из моих заявок всё ещё не закрыта?",
|
||||
"which appointment do I have tomorrow morning?",
|
||||
"how many kilometres did I run last week?",
|
||||
"what maintenance did the mechanic tell me to schedule?",
|
||||
"which item on my project list is overdue?",
|
||||
|
||||
// Public names inside questions that still require his records.
|
||||
"какую цитату из Набокова я сохранил?",
|
||||
"когда у меня созвон с Ириной Петровой?",
|
||||
"что я думал о романе Умберто Эко?",
|
||||
"какую оценку я дал выставке Айвазовского?",
|
||||
"какую фотографию Эрмитажа я отметил для печати?",
|
||||
"what did I note down after Margaret Hamilton's lecture?",
|
||||
"when is my booking at the Royal Albert Hall?",
|
||||
"which Nina Simone song did I call my favourite?",
|
||||
"what opinion did I share about Zadie Smith's new novel?",
|
||||
"what reminder did I attach to the Jira migration?",
|
||||
}
|
||||
|
||||
// worldSeeds — questions the world can answer, including the two shapes that
|
||||
// look personal and are not: a first-person preamble on a world question ("как
|
||||
// я говорил, ..."), and first person without possession ("что я могу
|
||||
// посмотреть вечером"). Refusing those is the opposite mistake and the older
|
||||
// comment on personalMarkers already named it.
|
||||
var worldSeeds = []string{
|
||||
// The original compact corpus, retained as above.
|
||||
"почему небо синее",
|
||||
"какая столица франции",
|
||||
"как сварить борщ",
|
||||
"кто написал эту книгу",
|
||||
"what is the capital of france",
|
||||
"how do i boil an egg",
|
||||
"как я говорил, почему небо синее",
|
||||
"as i said, why is the sky blue",
|
||||
"as i said, what is the population of india",
|
||||
"что я могу посмотреть вечером",
|
||||
@@ -103,21 +172,89 @@ var worldSeeds = []string{
|
||||
"расскажи про древний рим",
|
||||
"объясни как работает двигатель",
|
||||
"tell me about the roman empire",
|
||||
|
||||
// Speech and reports by somebody other than the owner.
|
||||
"что Александр Пушкин писал о Москве?",
|
||||
"как учёные объясняли исчезновение динозавров?",
|
||||
"что Менделеев говорил о будущем химии?",
|
||||
"какие выводы сделал Амундсен после экспедиции?",
|
||||
"what did Virginia Woolf write about fiction?",
|
||||
"how did researchers describe the Tunguska event?",
|
||||
"what did witnesses report after the Lisbon earthquake?",
|
||||
"which ideas did Ada Lovelace describe in her notes?",
|
||||
|
||||
// General advice about an owned object. Ownership supplies context, but an
|
||||
// outside source can still supply the answer.
|
||||
"как починить мой скрипящий стул?",
|
||||
"почему мой роутер теряет соединение?",
|
||||
"чем очистить мой велосипед от ржавчины?",
|
||||
"какой бензин подходит для моего генератора?",
|
||||
"какой чехол подобрать для моего планшета?",
|
||||
"как защитить мой деревянный стол от влаги?",
|
||||
"how do I remove a stain from my jacket?",
|
||||
"why is my freezer building up ice?",
|
||||
"which oil should I use in my lawn mower?",
|
||||
"what detergent is safe for my washing machine?",
|
||||
"which replacement blade should I buy for my circular saw?",
|
||||
"how can I keep my garden tools from rusting?",
|
||||
|
||||
// Public narratives.
|
||||
"расскажи историю строительства Транссибирской магистрали",
|
||||
"опиши, как развивалась письменность",
|
||||
"объясни, как появился периодический закон",
|
||||
"опиши первую успешную зимовку в Антарктиде",
|
||||
"tell the story of the discovery of penicillin",
|
||||
"describe how the first transatlantic cable was laid",
|
||||
"explain how the Olympic Games were revived",
|
||||
"describe the expedition that first reached the South Pole",
|
||||
|
||||
// First-person framing around a public answer.
|
||||
"как я уже спрашивал, почему звёзды мерцают?",
|
||||
"повторю свой вопрос: как образуются коралловые рифы?",
|
||||
"возможно, я повторяюсь: когда возвели собор Святого Петра?",
|
||||
"я мог уже спрашивать: из чего делают фарфор?",
|
||||
"as I asked earlier, why do leaves change colour?",
|
||||
"to repeat my question, how are fjords formed?",
|
||||
"I might be asking twice, when was Angkor Wat constructed?",
|
||||
"I may have asked before, what causes bioluminescence?",
|
||||
|
||||
// Public current information and generally applicable advice.
|
||||
"какие поезда сегодня идут из Москвы в Тверь?",
|
||||
"как правильно хранить чугунную сковороду?",
|
||||
"какие выставки проходят в Петербурге в этом месяце?",
|
||||
"какой сейчас уровень воды в Волге?",
|
||||
"what is the latest supported version of Ubuntu?",
|
||||
"how should I prepare a wooden deck for winter?",
|
||||
"which film festivals are taking place this season?",
|
||||
"what is the current exchange rate for the Norwegian krone?",
|
||||
|
||||
// Public facts about named people, places and organisations.
|
||||
"кто такая Софья Ковалевская?",
|
||||
"когда была основана компания Nintendo?",
|
||||
"чем прославился архитектор Фрэнк Ллойд Райт?",
|
||||
"где находится музей Прадо?",
|
||||
"who was James Baldwin?",
|
||||
"what is the city of Petra known for?",
|
||||
"when was the composer Philip Glass born?",
|
||||
"where is the Uffizi Gallery located?",
|
||||
}
|
||||
|
||||
// personalBoundary holds the embedded seeds. Zero value is usable and means
|
||||
// "not loaded yet"; a handler built without an embedder never loads and the
|
||||
// boundary falls back to personalMarkers.
|
||||
// personalBoundary holds the frozen or locally fitted head and, when fitting
|
||||
// was necessary, its embedded corpus. Zero value is usable and means "not
|
||||
// loaded yet"; a handler built without an embedder never loads and the boundary
|
||||
// falls back to personalMarkers.
|
||||
type personalBoundary struct {
|
||||
once sync.Once
|
||||
personal [][]float32
|
||||
world [][]float32
|
||||
head personalBoundaryLinearHead
|
||||
loaded bool
|
||||
}
|
||||
|
||||
// load embeds both seed sets, once per process. Seeds are embedded on the QUERY
|
||||
// side, like the utterance they are compared with — a question against a
|
||||
// question. Mixing sides would measure the e5 prefix, not the meaning.
|
||||
// load selects the pinned frozen head or embeds and fits the seed sets once per
|
||||
// process for another embedding space. Seeds are embedded on the QUERY side,
|
||||
// like the utterance they classify. Mixing sides would measure the e5 prefix,
|
||||
// not the meaning.
|
||||
func (b *personalBoundary) load(ctx context.Context, emb router.Embedder) {
|
||||
b.once.Do(func() {
|
||||
if emb == nil {
|
||||
@@ -135,30 +272,149 @@ func (b *personalBoundary) load(ctx context.Context, emb router.Embedder) {
|
||||
}
|
||||
return out
|
||||
}
|
||||
// The deployed e5-small head is fitted offline from the corpus below and
|
||||
// checked back against it by TestONNXPersonalBoundaryFrozenHead. Loading
|
||||
// it directly keeps the first personal query from embedding 132 examples.
|
||||
if router.EmbedderID(emb) == personalBoundaryHeadModelID {
|
||||
head, ok := frozenPersonalBoundaryHead()
|
||||
if ok && len(head.weights) == emb.Dim() {
|
||||
b.head, b.loaded = head, true
|
||||
return
|
||||
}
|
||||
log.Printf("voice: frozen personal boundary head is corrupt; rebuilding from its corpus")
|
||||
}
|
||||
|
||||
p, w := embedAll(personalSeeds), embedAll(worldSeeds)
|
||||
if p == nil || w == nil {
|
||||
return
|
||||
}
|
||||
b.personal, b.world, b.loaded = p, w, true
|
||||
epochs := personalBoundaryTrainingEpochs
|
||||
if router.EmbedderID(emb) == personalBoundaryHashModelID {
|
||||
epochs = personalBoundaryHashTrainingEpochs
|
||||
}
|
||||
head, ok := trainPersonalBoundaryLinearHeadEpochs(p, w, epochs)
|
||||
if !ok {
|
||||
log.Printf("voice: personal boundary training examples have inconsistent dimensions; falling back to possession markers")
|
||||
return
|
||||
}
|
||||
b.personal, b.world, b.head, b.loaded = p, w, head, true
|
||||
})
|
||||
}
|
||||
|
||||
// score returns the best similarity to each side. ok is false when the seeds
|
||||
// are not loaded, which is the caller's signal to use the markers instead.
|
||||
func (b *personalBoundary) score(vec []float32) (personal, world float64, ok bool) {
|
||||
if !b.loaded || len(vec) == 0 {
|
||||
return 0, 0, false
|
||||
const (
|
||||
personalBoundaryTrainingEpochs = 5000
|
||||
personalBoundaryLearningRate = 10.0
|
||||
personalBoundaryL2 = 0.0003
|
||||
)
|
||||
|
||||
const personalBoundaryHeadModelID = "model_quantized@384/tok2"
|
||||
|
||||
const personalBoundaryHashModelID = "hash@1024"
|
||||
|
||||
const personalBoundaryHeadWeights = "a3q5vmod2L7msrs+1RE8Pp5HDEBlv609AC9cvzm4D0CL7Fc/FU9cvxLmAMA638c/BgDBP6Is1r7PzBO/6MVAPsmEWT6XowjAouT0v8jMN79d2Sk+7XLlPX2akD+lmKi/q922vvLSFcBb0ma/cN3QP27zBMDl45i/iuE0P4KIJb+7dua+gTePP5unVz9H3q29Sxsev7YJe7+SvoQ+r6jyPxW2DL8sMQc/+iExQM5y8D/qJSZAtFyKP3PbyD8OK0dAHD+0v056qj4AbOS+AFHzP1KPeT9+cqu/aMIQv9wCqL8WbYe/xED1vu7pHMCPlxe/ZUGLPqFoDb8GPQ6/XE6cvqPVi7xKdr0/CE1PP4dPrj6TxoK+KokGP7xxu73h6DW/Lw8APsjd1D43aci/ZBMoQPyy8D8G6w/AMT1tPSEUU7/Sp+c+sjpRvyfl2L4KDs8/q/Ibv3urHj/+7ls/yxjaP8WS8jy8cd6+BO+4P/IcJkBTEPo/q2VGvqvsUD9anuk8UiO/PSw707+5+oY+zBpHP6e+UT4qaEe/zqjGvypN1j45TFY+nZ36v9rP8L9bmyE/Rn8UwONI0D5Yhs6/InCYv4kGgz/LNXO/rhK+Pu2Qdz/W8ijAdi3hv5qT5D9383k8Ir2wP0MRD0AxCCQ/0CUDP5kWoz+TQjdAOxI0vSbxDb/xj54/N/G6v86Ixr932Lk/jQ2jvqn2nr9y3JC96jDDPsyPlj9q/OQ/cOcCQJ+15z9747s/8Zh8PoS4oL0GKma/lfuPv/Clgb9GPKW+2OR3vimzAUBVYxXARcw0vynpsr/IUqe/bsUhv5kwWcCZtnE/fr87vjvfdr4mHis/xMpzvn20HL4SHFu/1DFXvVgOg76GXEq/pB2QP2u6e71q7w0+7F3APlte1j9YKXK/1cljPkFx0L/CndS9b4CeP4BIvj/fP5Q99jbZvL1h778WhC0/pNhov4+x1r+lYeE/9Y6gP9gtqr75dIe/wGiKv4q56D10ckY+UuvDvoIUnz/3TVM/moHcP6FkUz6//pY+FYhcwFEkD8B2a2c9mC+UP/ZeTb5FgIq+rgEOvylj8D9dvx8/OngmPyiplT9oiLy/AJwswKOJdL+i8/m9GPNfvyyWk77jVPC/0u+IPpx/Fz/QdvG/Ag9gP41l2rxmXUo/hdL0vx1XX0BUp+w9hmYyPk21dT6UJmK/zajGP7gBSD0FqoXAkis4P7kehz94wNa//nfZvxA0Fz8b9ze/IETPv3xEb76BG8k/SpyVP9xkEUC2/jlAcv8/wKKxU75E0xM+9BItPzlQKr6S0wdAMa39v0GKA8AMB3G/IeKvvyTZkz+es62/UEYTP3j+lj4SRM+/Dbfgvupdsj/wcUbAbjqRv/WV/r5WRaO/iB67P3/UyD8AK5Q+LzvJPsjPPL/fwkS/atd9P56MHz9CIJu9ugjgvp7J2D8otQC/YYoowKGEFD4eMVC/xy3UP2UEND9nU0i/ol4GQJuwfb+xeaa/B3IjwDK6Gz8dVv8/2wbLPlUo6j+FDCk/4Q/VP/J8JkCYVd0/gMS/P9Bwhj9R94a9M0Mjv/hKdL8cl6Y/lD73vwgior9+56Q/YI+1v9Wd0j8ltAjAmD5dP56Hnb+rdrA+gn2jP0bFA7/lkZU/tK6VP63ItT5Oi7O+YjfUv5iUzT+n5H8/zXMpvjefvj67z66/GA71Pj2h2T5bXxW/EyfLP1LZxr/B758/iCd2v0jnoT8twoG/oAO9vjpYDr61q6I+AEVFv1OP2b1VQpO/5FYdP5vgaz/4Lbm9CMCjvhbWlL9pYQk/1l5hPjCTYj8dtiJATXjavb6SlL7rp0E/cMBgP9UIXLwVYXC+rFS2v9yeFUD88JBAbwWcvt7s1D/bsuU/BCv0PzSdQEA7l36/FULEvmxlo79jjzc+gFvav1vptb/YjkS/Zo76vqK+3j+qvqi/qyfpPj1BLj+ehSzA4Z8nPyS/1b8kz5a9NIuZv31beL/k0oXAXFO/P8cCh8BSPzS+N7agvhjPUD6/G24/GIP0PYlNOsAFe6q+"
|
||||
|
||||
// HashEmbedder is a deterministic offline floor. Its 1024-dimensional head is
|
||||
// trained on first use instead of embedded here because the binary form is
|
||||
// still tiny but not meaningful as a production quality claim. The floor's
|
||||
// optimizer uses fewer steps: the hash vectors are sparse and converge long
|
||||
// before the semantic head, keeping an unconfigured box responsive.
|
||||
const personalBoundaryHashTrainingEpochs = 400
|
||||
|
||||
type personalBoundaryLinearHead struct {
|
||||
weights []float64
|
||||
bias float64
|
||||
}
|
||||
|
||||
func frozenPersonalBoundaryHead() (personalBoundaryLinearHead, bool) {
|
||||
raw, err := base64.StdEncoding.DecodeString(personalBoundaryHeadWeights)
|
||||
if err != nil || len(raw)%4 != 0 {
|
||||
return personalBoundaryLinearHead{}, false
|
||||
}
|
||||
best := func(seeds [][]float32) float64 {
|
||||
m := -1.0
|
||||
for _, s := range seeds {
|
||||
if c := cosine(vec, s); c > m {
|
||||
m = c
|
||||
weights := make([]float64, len(raw)/4)
|
||||
for i := range weights {
|
||||
weights[i] = float64(math.Float32frombits(binary.LittleEndian.Uint32(raw[4*i:])))
|
||||
}
|
||||
return personalBoundaryLinearHead{weights: weights, bias: -3.122734201373742}, true
|
||||
}
|
||||
|
||||
// trainPersonalBoundaryLinearHead fits binary logistic regression with full
|
||||
// batch gradient descent. Each side contributes total weight 0.5 regardless
|
||||
// of its number of examples. The optimiser is intentionally tiny and local:
|
||||
// the embedder supplies all learned language knowledge; this only learns one
|
||||
// separating hyperplane over its 384-dimensional vectors.
|
||||
func trainPersonalBoundaryLinearHead(personal, world [][]float32) (personalBoundaryLinearHead, bool) {
|
||||
return trainPersonalBoundaryLinearHeadEpochs(personal, world, personalBoundaryTrainingEpochs)
|
||||
}
|
||||
|
||||
func trainPersonalBoundaryLinearHeadEpochs(personal, world [][]float32, epochs int) (personalBoundaryLinearHead, bool) {
|
||||
if len(personal) == 0 || len(world) == 0 || len(personal[0]) == 0 {
|
||||
return personalBoundaryLinearHead{}, false
|
||||
}
|
||||
dim := len(personal[0])
|
||||
for _, vectors := range [][][]float32{personal, world} {
|
||||
for _, vector := range vectors {
|
||||
if len(vector) != dim {
|
||||
return personalBoundaryLinearHead{}, false
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
return best(b.personal), best(b.world), true
|
||||
|
||||
head := personalBoundaryLinearHead{weights: make([]float64, dim)}
|
||||
personalWeight := 0.5 / float64(len(personal))
|
||||
worldWeight := 0.5 / float64(len(world))
|
||||
for epoch := 0; epoch < epochs; epoch++ {
|
||||
gradient := make([]float64, dim)
|
||||
biasGradient := 0.0
|
||||
accumulate := func(vectors [][]float32, target, sampleWeight float64) {
|
||||
for _, vector := range vectors {
|
||||
probability := logistic(head.logit(vector))
|
||||
error := (probability - target) * sampleWeight
|
||||
biasGradient += error
|
||||
for i, value := range vector {
|
||||
gradient[i] += error * float64(value)
|
||||
}
|
||||
}
|
||||
}
|
||||
accumulate(personal, 1, personalWeight)
|
||||
accumulate(world, 0, worldWeight)
|
||||
|
||||
step := personalBoundaryLearningRate / (1 + float64(epoch)/1000)
|
||||
for i := range head.weights {
|
||||
head.weights[i] -= step * (gradient[i] + personalBoundaryL2*head.weights[i])
|
||||
}
|
||||
head.bias -= step * biasGradient
|
||||
}
|
||||
return head, true
|
||||
}
|
||||
|
||||
func (h personalBoundaryLinearHead) logit(vec []float32) float64 {
|
||||
if len(vec) != len(h.weights) {
|
||||
return 0
|
||||
}
|
||||
score := h.bias
|
||||
for i, value := range vec {
|
||||
score += h.weights[i] * float64(value)
|
||||
}
|
||||
return score
|
||||
}
|
||||
|
||||
func logistic(value float64) float64 {
|
||||
if value >= 0 {
|
||||
return 1 / (1 + math.Exp(-value))
|
||||
}
|
||||
exp := math.Exp(value)
|
||||
return exp / (1 + exp)
|
||||
}
|
||||
|
||||
// score returns complementary class probabilities. ok is false when the
|
||||
// corpus is not loaded or the query vector belongs to another embedding
|
||||
// space, which is the caller's signal to use the offline marker floor.
|
||||
func (b *personalBoundary) score(vec []float32) (personal, world float64, ok bool) {
|
||||
if !b.loaded || len(vec) != len(b.head.weights) {
|
||||
return 0, 0, false
|
||||
}
|
||||
personal = logistic(b.head.logit(vec))
|
||||
return personal, 1 - personal, true
|
||||
}
|
||||
|
||||
// cosine — same math as internal/router and internal/memory, small enough that
|
||||
|
||||
@@ -0,0 +1,405 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
_ "embed"
|
||||
"encoding/json"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode"
|
||||
|
||||
"github.com/kami/maven/internal/router"
|
||||
)
|
||||
|
||||
// This fixture is intentionally separate from personalboundary_test.go. The
|
||||
// small regression table there explains individual fixes; this matrix measures
|
||||
// the boundary as a classifier and prevents a repaired sentence shape from
|
||||
// standing in for language and subject coverage.
|
||||
//
|
||||
//go:embed testdata/personal_boundary_v1.json
|
||||
var personalBoundaryFixtureJSON []byte
|
||||
|
||||
type personalBoundaryEvalCase struct {
|
||||
ID string `json:"id"`
|
||||
Utterance string `json:"utterance"`
|
||||
Lang string `json:"lang"`
|
||||
Want string `json:"want"`
|
||||
Stratum string `json:"stratum"`
|
||||
}
|
||||
|
||||
type personalBoundaryEvalFixture struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
Name string `json:"name"`
|
||||
Notes []string `json:"notes"`
|
||||
Cases []personalBoundaryEvalCase `json:"cases"`
|
||||
}
|
||||
|
||||
var personalBoundaryEvalStrata = []string{
|
||||
"remembered_speech",
|
||||
"possession",
|
||||
"narrative",
|
||||
"first_person_preamble",
|
||||
"advice_current_info",
|
||||
"public_proper_nouns",
|
||||
}
|
||||
|
||||
func loadPersonalBoundaryEvalFixture(t *testing.T) personalBoundaryEvalFixture {
|
||||
t.Helper()
|
||||
var fixture personalBoundaryEvalFixture
|
||||
if err := json.Unmarshal(personalBoundaryFixtureJSON, &fixture); err != nil {
|
||||
t.Fatalf("parse personal boundary fixture: %v", err)
|
||||
}
|
||||
if fixture.SchemaVersion != 1 {
|
||||
t.Fatalf("personal boundary fixture schema_version = %d, want 1", fixture.SchemaVersion)
|
||||
}
|
||||
if fixture.Name != "personal_boundary_v1" {
|
||||
t.Fatalf("personal boundary fixture name = %q, want personal_boundary_v1", fixture.Name)
|
||||
}
|
||||
return fixture
|
||||
}
|
||||
|
||||
// TestPersonalBoundaryEvalFixture enforces the sampling contract separately
|
||||
// from the model measurement. It runs in ordinary CI even when ONNX Runtime is
|
||||
// absent, so a fixture edit cannot silently unbalance a language, side or
|
||||
// sentence shape, or turn a production seed into a held-out case.
|
||||
func TestPersonalBoundaryEvalFixture(t *testing.T) {
|
||||
fixture := loadPersonalBoundaryEvalFixture(t)
|
||||
const wantPerCell = 3
|
||||
const wantTotal = 6 * 2 * 2 * wantPerCell
|
||||
if len(fixture.Cases) != wantTotal {
|
||||
t.Errorf("fixture has %d cases, want %d", len(fixture.Cases), wantTotal)
|
||||
}
|
||||
|
||||
validStrata := make(map[string]bool, len(personalBoundaryEvalStrata))
|
||||
for _, stratum := range personalBoundaryEvalStrata {
|
||||
validStrata[stratum] = true
|
||||
}
|
||||
seedSource := make(map[string]string, len(personalSeeds)+len(worldSeeds))
|
||||
for _, seed := range personalSeeds {
|
||||
seedSource[normalizePersonalBoundaryEval(seed)] = "personalSeeds"
|
||||
}
|
||||
for _, seed := range worldSeeds {
|
||||
seedSource[normalizePersonalBoundaryEval(seed)] = "worldSeeds"
|
||||
}
|
||||
|
||||
seenID := make(map[string]bool, len(fixture.Cases))
|
||||
seenUtterance := make(map[string]string, len(fixture.Cases))
|
||||
cells := make(map[string]int)
|
||||
for _, c := range fixture.Cases {
|
||||
if strings.TrimSpace(c.ID) == "" || seenID[c.ID] {
|
||||
t.Errorf("case %q: empty or duplicate id", c.ID)
|
||||
}
|
||||
seenID[c.ID] = true
|
||||
if c.Lang != "ru" && c.Lang != "en" {
|
||||
t.Errorf("%s: lang = %q, want ru|en", c.ID, c.Lang)
|
||||
}
|
||||
if c.Want != "personal" && c.Want != "world" {
|
||||
t.Errorf("%s: want = %q, want personal|world", c.ID, c.Want)
|
||||
}
|
||||
if !validStrata[c.Stratum] {
|
||||
t.Errorf("%s: stratum = %q, not one of the six declared strata", c.ID, c.Stratum)
|
||||
}
|
||||
|
||||
normalized := normalizePersonalBoundaryEval(c.Utterance)
|
||||
if normalized == "" {
|
||||
t.Errorf("%s: empty utterance", c.ID)
|
||||
}
|
||||
if previous, ok := seenUtterance[normalized]; ok {
|
||||
t.Errorf("%s: utterance duplicates %s after normalization", c.ID, previous)
|
||||
}
|
||||
seenUtterance[normalized] = c.ID
|
||||
if source, ok := seedSource[normalized]; ok {
|
||||
t.Errorf("%s: %q is verbatim in %s, so it is not held out", c.ID, c.Utterance, source)
|
||||
}
|
||||
// The original failure names Baikal. Replacing that sentence's verb or
|
||||
// punctuation would measure an exception, not the boundary. This corpus
|
||||
// instead varies people, places, products and events.
|
||||
if strings.Contains(normalized, "байкал") || strings.Contains(normalized, "baikal") {
|
||||
t.Errorf("%s: the stratified fixture must not copy the Baikal regression", c.ID)
|
||||
}
|
||||
cells[c.Stratum+"/"+c.Lang+"/"+c.Want]++
|
||||
}
|
||||
|
||||
for _, stratum := range personalBoundaryEvalStrata {
|
||||
for _, lang := range []string{"ru", "en"} {
|
||||
for _, want := range []string{"personal", "world"} {
|
||||
cell := stratum + "/" + lang + "/" + want
|
||||
if got := cells[cell]; got != wantPerCell {
|
||||
t.Errorf("fixture cell %s has %d cases, want %d", cell, got, wantPerCell)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// normalizePersonalBoundaryEval compares content rather than typography:
|
||||
// case, punctuation and repeated whitespace cannot disguise a copied seed or
|
||||
// duplicate case. This is fixture hygiene only; it does not participate in the
|
||||
// production boundary.
|
||||
func normalizePersonalBoundaryEval(s string) string {
|
||||
var b strings.Builder
|
||||
space := true
|
||||
for _, r := range strings.ToLower(s) {
|
||||
if unicode.IsLetter(r) || unicode.IsNumber(r) {
|
||||
b.WriteRune(r)
|
||||
space = false
|
||||
continue
|
||||
}
|
||||
if !space {
|
||||
b.WriteByte(' ')
|
||||
space = true
|
||||
}
|
||||
}
|
||||
return strings.TrimSpace(b.String())
|
||||
}
|
||||
|
||||
type personalBoundaryEvalStat struct {
|
||||
Correct int
|
||||
Total int
|
||||
}
|
||||
|
||||
type personalBoundaryEvalReport struct {
|
||||
Name string
|
||||
Correct int
|
||||
Total int
|
||||
MinimumMargin float64
|
||||
ByStratum map[string]personalBoundaryEvalStat
|
||||
ByLanguage map[string]personalBoundaryEvalStat
|
||||
ByExpectedClass map[string]personalBoundaryEvalStat
|
||||
ByCell map[string]personalBoundaryEvalStat
|
||||
}
|
||||
|
||||
func newPersonalBoundaryEvalReport(name string) *personalBoundaryEvalReport {
|
||||
return &personalBoundaryEvalReport{
|
||||
Name: name,
|
||||
MinimumMargin: math.Inf(1),
|
||||
ByStratum: make(map[string]personalBoundaryEvalStat),
|
||||
ByLanguage: make(map[string]personalBoundaryEvalStat),
|
||||
ByExpectedClass: make(map[string]personalBoundaryEvalStat),
|
||||
ByCell: make(map[string]personalBoundaryEvalStat),
|
||||
}
|
||||
}
|
||||
|
||||
func (r *personalBoundaryEvalReport) add(c personalBoundaryEvalCase, gotPersonal bool, personal, world float64) {
|
||||
wantPersonal := c.Want == "personal"
|
||||
correct := gotPersonal == wantPersonal
|
||||
r.Total++
|
||||
if correct {
|
||||
r.Correct++
|
||||
}
|
||||
signedMargin := personal - world
|
||||
if !wantPersonal {
|
||||
signedMargin = -signedMargin
|
||||
}
|
||||
if signedMargin < r.MinimumMargin {
|
||||
r.MinimumMargin = signedMargin
|
||||
}
|
||||
add := func(stats map[string]personalBoundaryEvalStat, key string) {
|
||||
stat := stats[key]
|
||||
stat.Total++
|
||||
if correct {
|
||||
stat.Correct++
|
||||
}
|
||||
stats[key] = stat
|
||||
}
|
||||
add(r.ByStratum, c.Stratum)
|
||||
add(r.ByLanguage, c.Lang)
|
||||
add(r.ByExpectedClass, c.Want)
|
||||
add(r.ByCell, c.Stratum+"/"+c.Lang+"/"+c.Want)
|
||||
}
|
||||
|
||||
// TestONNXPersonalBoundaryStratified scores the model homesrv actually runs.
|
||||
// Production is read from personalBoundary.score; top1, top2, top3 and a
|
||||
// whole-class centroid are diagnostics over the same embedded seeds. Today
|
||||
// production and top3 coincide, but keeping them separate means a later scoring
|
||||
// experiment can be compared without rewriting this evaluation or putting its
|
||||
// candidate math in runtime code. The privacy boundary is a hard contract, so
|
||||
// every production miss is a test failure rather than an accuracy target to
|
||||
// average away.
|
||||
func TestONNXPersonalBoundaryStratified(t *testing.T) {
|
||||
if os.Getenv("MAVEN_EVAL_PERSONAL_BOUNDARY") == "" {
|
||||
t.Skip("set MAVEN_EVAL_PERSONAL_BOUNDARY=1 to run the deliberately strict V-702 matrix")
|
||||
}
|
||||
lib := os.Getenv("MAVEN_ONNX_LIB")
|
||||
if lib == "" {
|
||||
t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing")
|
||||
}
|
||||
modelDir := filepath.Join("../..", "models/embedder/multilingual-e5-small")
|
||||
model := filepath.Join(modelDir, "model_quantized.onnx")
|
||||
tokenizer := filepath.Join(modelDir, "tokenizer.json")
|
||||
for _, path := range []string{lib, model, tokenizer} {
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
t.Skipf("personal boundary eval dependency %s unavailable: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
embedder, err := router.NewONNXEmbedder(model, tokenizer, lib)
|
||||
if err != nil {
|
||||
t.Skipf("onnx embedder unavailable: %v", err)
|
||||
}
|
||||
defer embedder.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
boundary := &personalBoundary{}
|
||||
boundary.load(ctx, embedder)
|
||||
if !boundary.loaded {
|
||||
t.Fatal("personal boundary seeds did not load with a working embedder")
|
||||
}
|
||||
// Production loads its model-ID-pinned frozen head and deliberately skips
|
||||
// the 132 corpus embeddings on a user's first query. This test still needs
|
||||
// those vectors for the historical top-k/centroid diagnostics, so build
|
||||
// them here without putting that latency back in runtime code.
|
||||
embedCorpus := func(values []string) [][]float32 {
|
||||
vectors := make([][]float32, len(values))
|
||||
for i, value := range values {
|
||||
vector, err := router.EmbedQuery(ctx, embedder, value)
|
||||
if err != nil {
|
||||
t.Fatalf("embed diagnostic corpus %q: %v", value, err)
|
||||
}
|
||||
vectors[i] = vector
|
||||
}
|
||||
return vectors
|
||||
}
|
||||
boundary.personal = embedCorpus(personalSeeds)
|
||||
boundary.world = embedCorpus(worldSeeds)
|
||||
personalCentroid := personalBoundaryEvalCentroid(boundary.personal)
|
||||
worldCentroid := personalBoundaryEvalCentroid(boundary.world)
|
||||
if len(personalCentroid) == 0 || len(worldCentroid) == 0 {
|
||||
t.Fatal("personal boundary seed vectors do not share a dimension")
|
||||
}
|
||||
|
||||
type candidate struct {
|
||||
name string
|
||||
score func([]float32) (float64, float64)
|
||||
}
|
||||
candidates := []candidate{
|
||||
{name: "production", score: func(vec []float32) (float64, float64) {
|
||||
personal, world, ok := boundary.score(vec)
|
||||
if !ok {
|
||||
t.Fatal("loaded personal boundary declined to score")
|
||||
}
|
||||
return personal, world
|
||||
}},
|
||||
{name: "top1", score: func(vec []float32) (float64, float64) {
|
||||
return meanNearest(vec, boundary.personal, 1), meanNearest(vec, boundary.world, 1)
|
||||
}},
|
||||
{name: "top2", score: func(vec []float32) (float64, float64) {
|
||||
return meanNearest(vec, boundary.personal, 2), meanNearest(vec, boundary.world, 2)
|
||||
}},
|
||||
{name: "top3", score: func(vec []float32) (float64, float64) {
|
||||
return meanNearest(vec, boundary.personal, 3), meanNearest(vec, boundary.world, 3)
|
||||
}},
|
||||
{name: "centroid", score: func(vec []float32) (float64, float64) {
|
||||
return cosine(vec, personalCentroid), cosine(vec, worldCentroid)
|
||||
}},
|
||||
}
|
||||
reports := make(map[string]*personalBoundaryEvalReport, len(candidates))
|
||||
for _, candidate := range candidates {
|
||||
reports[candidate.name] = newPersonalBoundaryEvalReport(candidate.name)
|
||||
}
|
||||
|
||||
fixture := loadPersonalBoundaryEvalFixture(t)
|
||||
for _, c := range fixture.Cases {
|
||||
vec, err := router.EmbedQuery(ctx, embedder, c.Utterance)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: embed %q: %v", c.ID, c.Utterance, err)
|
||||
}
|
||||
for _, candidate := range candidates {
|
||||
personal, world := candidate.score(vec)
|
||||
gotPersonal := personal > world
|
||||
reports[candidate.name].add(c, gotPersonal, personal, world)
|
||||
if candidate.name == "production" && gotPersonal != (c.Want == "personal") {
|
||||
t.Errorf("%s [%s/%s]: got %s, want %s (personal %.4f world %.4f delta %+.4f): %q",
|
||||
c.ID, c.Lang, c.Stratum, boundaryEvalSide(gotPersonal), c.Want,
|
||||
personal, world, personal-world, c.Utterance)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, candidate := range candidates {
|
||||
report := reports[candidate.name]
|
||||
t.Logf("candidate %-15s %2d/%d (%.1f%%), minimum signed margin %+.4f",
|
||||
report.Name, report.Correct, report.Total,
|
||||
100*float64(report.Correct)/float64(report.Total), report.MinimumMargin)
|
||||
}
|
||||
production := reports["production"]
|
||||
for _, lang := range []string{"ru", "en"} {
|
||||
stat := production.ByLanguage[lang]
|
||||
t.Logf("production language %-2s %2d/%d", lang, stat.Correct, stat.Total)
|
||||
}
|
||||
for _, side := range []string{"personal", "world"} {
|
||||
stat := production.ByExpectedClass[side]
|
||||
t.Logf("production expected %-8s %2d/%d", side, stat.Correct, stat.Total)
|
||||
}
|
||||
strata := append([]string(nil), personalBoundaryEvalStrata...)
|
||||
sort.Strings(strata)
|
||||
for _, stratum := range strata {
|
||||
stat := production.ByStratum[stratum]
|
||||
ruPersonal := production.ByCell[stratum+"/ru/personal"]
|
||||
ruWorld := production.ByCell[stratum+"/ru/world"]
|
||||
enPersonal := production.ByCell[stratum+"/en/personal"]
|
||||
enWorld := production.ByCell[stratum+"/en/world"]
|
||||
t.Logf("production stratum %-21s %2d/%d | ru personal %d/%d world %d/%d | en personal %d/%d world %d/%d",
|
||||
stratum, stat.Correct, stat.Total,
|
||||
ruPersonal.Correct, ruPersonal.Total, ruWorld.Correct, ruWorld.Total,
|
||||
enPersonal.Correct, enPersonal.Total, enWorld.Correct, enWorld.Total)
|
||||
}
|
||||
}
|
||||
|
||||
func personalBoundaryEvalCentroid(vectors [][]float32) []float32 {
|
||||
if len(vectors) == 0 {
|
||||
return nil
|
||||
}
|
||||
centroid := make([]float32, len(vectors[0]))
|
||||
for _, vector := range vectors {
|
||||
if len(vector) != len(centroid) {
|
||||
return nil
|
||||
}
|
||||
for i, value := range vector {
|
||||
centroid[i] += value
|
||||
}
|
||||
}
|
||||
for i := range centroid {
|
||||
centroid[i] /= float32(len(vectors))
|
||||
}
|
||||
return centroid
|
||||
}
|
||||
|
||||
func boundaryEvalSide(personal bool) string {
|
||||
if personal {
|
||||
return "personal"
|
||||
}
|
||||
return "world"
|
||||
}
|
||||
|
||||
// meanNearest is an evaluation baseline retained beside the strict fixture;
|
||||
// production uses the linear head in personalboundary.go.
|
||||
func meanNearest(vec []float32, seeds [][]float32, k int) float64 {
|
||||
if len(seeds) == 0 || k <= 0 {
|
||||
return -1
|
||||
}
|
||||
if k > len(seeds) {
|
||||
k = len(seeds)
|
||||
}
|
||||
top := make([]float64, k)
|
||||
for i := range top {
|
||||
top[i] = -1
|
||||
}
|
||||
for _, seed := range seeds {
|
||||
candidate := cosine(vec, seed)
|
||||
for i := range top {
|
||||
if candidate > top[i] {
|
||||
candidate, top[i] = top[i], candidate
|
||||
}
|
||||
}
|
||||
}
|
||||
var sum float64
|
||||
for _, similarity := range top {
|
||||
sum += similarity
|
||||
}
|
||||
return sum / float64(k)
|
||||
}
|
||||
@@ -2,13 +2,213 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/router"
|
||||
)
|
||||
|
||||
func TestPersonalBoundaryLinearHeadSeparatesSemanticDirections(t *testing.T) {
|
||||
personal := [][]float32{{1, 0}, {0.9, 0.1}, {0.8, -0.1}}
|
||||
world := [][]float32{{-1, 0}, {-0.9, 0.1}, {-0.8, -0.1}}
|
||||
head, ok := trainPersonalBoundaryLinearHead(personal, world)
|
||||
if !ok {
|
||||
t.Fatal("valid training vectors were rejected")
|
||||
}
|
||||
b := personalBoundary{personal: personal, world: world, head: head, loaded: true}
|
||||
for _, tc := range []struct {
|
||||
vector []float32
|
||||
personal bool
|
||||
}{
|
||||
{vector: []float32{0.75, 0.2}, personal: true},
|
||||
{vector: []float32{-0.75, 0.2}, personal: false},
|
||||
} {
|
||||
personalScore, worldScore, ok := b.score(tc.vector)
|
||||
if !ok {
|
||||
t.Fatal("loaded boundary did not score")
|
||||
}
|
||||
if got := personalScore > worldScore; got != tc.personal {
|
||||
t.Fatalf("vector %v classified personal=%v (scores %.4f/%.4f), want %v",
|
||||
tc.vector, got, personalScore, worldScore, tc.personal)
|
||||
}
|
||||
if math.Abs(personalScore+worldScore-1) > 1e-12 {
|
||||
t.Fatalf("scores %.8f and %.8f are not complementary probabilities", personalScore, worldScore)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPersonalBoundaryTrainingBalancesClasses(t *testing.T) {
|
||||
personal := [][]float32{{1, 0}, {0.8, 0.2}}
|
||||
world := [][]float32{{-1, 0}}
|
||||
oneWorld, ok := trainPersonalBoundaryLinearHead(personal, world)
|
||||
if !ok {
|
||||
t.Fatal("valid training vectors were rejected")
|
||||
}
|
||||
repeatedWorld := make([][]float32, 12)
|
||||
for i := range repeatedWorld {
|
||||
repeatedWorld[i] = world[0]
|
||||
}
|
||||
twelveWorld, ok := trainPersonalBoundaryLinearHead(personal, repeatedWorld)
|
||||
if !ok {
|
||||
t.Fatal("valid repeated training vectors were rejected")
|
||||
}
|
||||
if math.Abs(oneWorld.bias-twelveWorld.bias) > 1e-10 {
|
||||
t.Fatalf("duplicating one class moved bias from %.12f to %.12f", oneWorld.bias, twelveWorld.bias)
|
||||
}
|
||||
for i := range oneWorld.weights {
|
||||
if math.Abs(oneWorld.weights[i]-twelveWorld.weights[i]) > 1e-10 {
|
||||
t.Fatalf("duplicating one class moved weight %d from %.12f to %.12f",
|
||||
i, oneWorld.weights[i], twelveWorld.weights[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPersonalBoundaryTrainingRejectsMixedDimensions(t *testing.T) {
|
||||
if _, ok := trainPersonalBoundaryLinearHead(
|
||||
[][]float32{{1, 0}},
|
||||
[][]float32{{-1, 0, 0}},
|
||||
); ok {
|
||||
t.Fatal("mixed embedding dimensions were accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// The corpus is grouped by sentence shape in personalboundary.go. This test
|
||||
// leaves one entire shape out of training at a time, then requires the linear
|
||||
// head to classify the omitted examples from the semantics learned from the
|
||||
// other shapes. It is ordinary deterministic CI: the small axis vectors stand
|
||||
// in for frozen embedding directions, so the test proves the training code
|
||||
// generalises across groups rather than memorising one row at a time.
|
||||
func TestPersonalBoundaryLinearHeadLeaveOneShapeOut(t *testing.T) {
|
||||
type example struct {
|
||||
vector []float32
|
||||
shape int
|
||||
want bool
|
||||
}
|
||||
const shapeCount = 6
|
||||
examples := make([]example, 0, shapeCount*4)
|
||||
for shape := 0; shape < shapeCount; shape++ {
|
||||
for variant := 0; variant < 2; variant++ {
|
||||
personal := make([]float32, shapeCount+1)
|
||||
world := make([]float32, shapeCount+1)
|
||||
personal[0], world[0] = 1, -1
|
||||
personal[shape+1] = float32(0.1 * float64(variant+1))
|
||||
world[shape+1] = float32(-0.1 * float64(variant+1))
|
||||
examples = append(examples,
|
||||
example{vector: personal, shape: shape, want: true},
|
||||
example{vector: world, shape: shape, want: false},
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
for omitted := 0; omitted < shapeCount; omitted++ {
|
||||
var personal, world [][]float32
|
||||
for _, example := range examples {
|
||||
if example.shape == omitted {
|
||||
continue
|
||||
}
|
||||
if example.want {
|
||||
personal = append(personal, example.vector)
|
||||
} else {
|
||||
world = append(world, example.vector)
|
||||
}
|
||||
}
|
||||
head, ok := trainPersonalBoundaryLinearHead(personal, world)
|
||||
if !ok {
|
||||
t.Fatalf("fold %d rejected valid vectors", omitted)
|
||||
}
|
||||
for _, example := range examples {
|
||||
if example.shape != omitted {
|
||||
continue
|
||||
}
|
||||
if got := head.logit(example.vector) > 0; got != example.want {
|
||||
t.Errorf("fold %d classified %v as personal=%v, want %v", omitted, example.vector, got, example.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPersonalBoundaryTrainingCorpusIsIndependent(t *testing.T) {
|
||||
// The strict stratified fixture already enforces this for its 72 rows. The
|
||||
// historical regression table lives here, so protect it here too: a future
|
||||
// seed addition must not copy a regression sentence into training.
|
||||
training := make(map[string]bool, len(personalSeeds)+len(worldSeeds))
|
||||
for _, seed := range append(append([]string(nil), personalSeeds...), worldSeeds...) {
|
||||
training[normalizePersonalBoundaryTraining(seed)] = true
|
||||
}
|
||||
for _, regression := range []string{
|
||||
"что я говорил про бэкапы?",
|
||||
"что я сказал вчера про отпуск",
|
||||
"я писал что-нибудь про сервер",
|
||||
"я упоминал про конференцию?",
|
||||
"что я отмечал по поводу переезда",
|
||||
"я рассказывал тебе про новую работу?",
|
||||
"во сколько у меня встреча",
|
||||
"когда мой следующий отпуск",
|
||||
"what did i say about backups",
|
||||
"did i tell you about the doctor",
|
||||
"как я говорил, почему небо синее",
|
||||
"как уже я говорил, какая столица франции",
|
||||
"почему трава зелёная",
|
||||
"столица франции",
|
||||
"как мне сварить борщ",
|
||||
"что мне посмотреть вечером",
|
||||
"я хочу узнать про рим",
|
||||
"кто такой гагарин",
|
||||
"how do i boil an egg",
|
||||
"во сколько закат сегодня",
|
||||
"когда сегодня заканчивается концерт",
|
||||
"во сколько завтра открывается аптека",
|
||||
"какой сегодня праздник",
|
||||
"что интересного произошло сегодня в мире",
|
||||
"кто выиграл вчера матч",
|
||||
"расскажи про эверест",
|
||||
"расскажи про войну 1812 года",
|
||||
"объясни что такое инфляция",
|
||||
"я рассказывал тебе про байкал?",
|
||||
} {
|
||||
if training[normalizePersonalBoundaryTraining(regression)] {
|
||||
t.Errorf("regression utterance leaked into training: %q", regression)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPersonalBoundaryFrozenHeadDecodes(t *testing.T) {
|
||||
head, ok := frozenPersonalBoundaryHead()
|
||||
if !ok {
|
||||
t.Fatal("frozen head did not decode")
|
||||
}
|
||||
if len(head.weights) != 384 {
|
||||
t.Fatalf("frozen head has %d weights, want 384", len(head.weights))
|
||||
}
|
||||
}
|
||||
|
||||
func TestPersonalBoundaryHashFloorLatency(t *testing.T) {
|
||||
b := &personalBoundary{}
|
||||
embedder := router.NewHashEmbedder(1024)
|
||||
query, err := router.EmbedQuery(context.Background(), embedder, "когда моя встреча")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
started := time.Now()
|
||||
b.load(context.Background(), embedder)
|
||||
if _, _, ok := b.score(query); !ok {
|
||||
t.Fatal("hash-floor boundary declined to score")
|
||||
}
|
||||
elapsed := time.Since(started)
|
||||
t.Logf("hash-floor corpus fit+score: %s", elapsed)
|
||||
if elapsed > 2*time.Second {
|
||||
t.Errorf("hash-floor boundary took %s, exceeds 2s local floor ceiling", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
func normalizePersonalBoundaryTraining(value string) string {
|
||||
return strings.Join(strings.Fields(strings.ToLower(value)), " ")
|
||||
}
|
||||
|
||||
// A handler with no embedder never loads the seeds, so the boundary falls back
|
||||
// to the possession markers. That is the offline floor and it must keep working
|
||||
// — an embedder that fails to load must not open the boundary.
|
||||
@@ -113,3 +313,419 @@ func TestONNXPersonalBoundary(t *testing.T) {
|
||||
}
|
||||
t.Logf("personal boundary: %d/%d held-out utterances correct", len(cases)-wrong, len(cases))
|
||||
}
|
||||
|
||||
func TestONNXPersonalBoundaryFourFold(t *testing.T) {
|
||||
lib := os.Getenv("MAVEN_ONNX_LIB")
|
||||
if lib == "" {
|
||||
t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing")
|
||||
}
|
||||
dir := filepath.Join("../..", "models/embedder/multilingual-e5-small")
|
||||
emb, err := router.NewONNXEmbedder(filepath.Join(dir, "model_quantized.onnx"), filepath.Join(dir, "tokenizer.json"), lib)
|
||||
if err != nil {
|
||||
t.Skipf("onnx embedder unavailable: %v", err)
|
||||
}
|
||||
defer emb.Close()
|
||||
ctx := context.Background()
|
||||
embedAll := func(values []string) [][]float32 {
|
||||
vectors := make([][]float32, len(values))
|
||||
for i, value := range values {
|
||||
vector, err := router.EmbedQuery(ctx, emb, value)
|
||||
if err != nil {
|
||||
t.Fatalf("embed %q: %v", value, err)
|
||||
}
|
||||
vectors[i] = vector
|
||||
}
|
||||
return vectors
|
||||
}
|
||||
personalVectors := embedAll(personalSeeds)
|
||||
worldVectors := embedAll(worldSeeds)
|
||||
|
||||
type group struct {
|
||||
name string
|
||||
personalStart, personalCount int
|
||||
worldStart, worldCount int
|
||||
}
|
||||
groups := []group{
|
||||
{name: "remembered_speech", personalStart: 8, personalCount: 8, worldStart: 20, worldCount: 8},
|
||||
{name: "possession", personalStart: 16, personalCount: 10, worldStart: 28, worldCount: 12},
|
||||
{name: "narrative", personalStart: 26, personalCount: 8, worldStart: 40, worldCount: 8},
|
||||
{name: "first_person_preamble", personalStart: 34, personalCount: 8, worldStart: 48, worldCount: 8},
|
||||
{name: "advice_current_info", personalStart: 42, personalCount: 8, worldStart: 56, worldCount: 8},
|
||||
{name: "public_proper_nouns", personalStart: 50, personalCount: 10, worldStart: 64, worldCount: 8},
|
||||
}
|
||||
|
||||
const foldCount = 4
|
||||
aggregateCorrect, aggregateTotal := 0, 0
|
||||
for omittedFold := 0; omittedFold < foldCount; omittedFold++ {
|
||||
trainingPersonal := append([][]float32(nil), personalVectors[:8]...)
|
||||
trainingWorld := append([][]float32(nil), worldVectors[:20]...)
|
||||
var heldPersonal, heldWorld [][]float32
|
||||
partition := func(vectors [][]float32, start, count int, training, held *[][]float32) {
|
||||
for relative, vector := range vectors[start : start+count] {
|
||||
if relative%foldCount == omittedFold {
|
||||
*held = append(*held, vector)
|
||||
} else {
|
||||
*training = append(*training, vector)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, group := range groups {
|
||||
partition(personalVectors, group.personalStart, group.personalCount, &trainingPersonal, &heldPersonal)
|
||||
partition(worldVectors, group.worldStart, group.worldCount, &trainingWorld, &heldWorld)
|
||||
}
|
||||
head, ok := trainPersonalBoundaryLinearHead(
|
||||
trainingPersonal,
|
||||
trainingWorld,
|
||||
)
|
||||
if !ok {
|
||||
t.Fatalf("fold %d: valid training fold rejected", omittedFold)
|
||||
}
|
||||
correct, total := 0, 0
|
||||
for _, vector := range heldPersonal {
|
||||
total++
|
||||
if head.logit(vector) > 0 {
|
||||
correct++
|
||||
}
|
||||
}
|
||||
for _, vector := range heldWorld {
|
||||
total++
|
||||
if head.logit(vector) <= 0 {
|
||||
correct++
|
||||
}
|
||||
}
|
||||
t.Logf("fold %d: %d/%d held-out training examples", omittedFold+1, correct, total)
|
||||
aggregateCorrect += correct
|
||||
aggregateTotal += total
|
||||
}
|
||||
t.Logf("four-fold aggregate: %d/%d", aggregateCorrect, aggregateTotal)
|
||||
if aggregateCorrect < 99 {
|
||||
t.Errorf("four-fold aggregate %d/%d, want at least 99/104", aggregateCorrect, aggregateTotal)
|
||||
}
|
||||
}
|
||||
|
||||
func TestONNXPersonalBoundarySemanticGroupHoldout(t *testing.T) {
|
||||
lib := os.Getenv("MAVEN_ONNX_LIB")
|
||||
if lib == "" {
|
||||
t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing")
|
||||
}
|
||||
dir := filepath.Join("../..", "models/embedder/multilingual-e5-small")
|
||||
emb, err := router.NewONNXEmbedder(filepath.Join(dir, "model_quantized.onnx"), filepath.Join(dir, "tokenizer.json"), lib)
|
||||
if err != nil {
|
||||
t.Skipf("onnx embedder unavailable: %v", err)
|
||||
}
|
||||
defer emb.Close()
|
||||
ctx := context.Background()
|
||||
embedAll := func(values []string) [][]float32 {
|
||||
vectors := make([][]float32, len(values))
|
||||
for i, value := range values {
|
||||
vector, err := router.EmbedQuery(ctx, emb, value)
|
||||
if err != nil {
|
||||
t.Fatalf("embed %q: %v", value, err)
|
||||
}
|
||||
vectors[i] = vector
|
||||
}
|
||||
return vectors
|
||||
}
|
||||
personalVectors := embedAll(personalSeeds)
|
||||
worldVectors := embedAll(worldSeeds)
|
||||
|
||||
type group struct {
|
||||
name string
|
||||
personalStart, personalCount int
|
||||
worldStart, worldCount int
|
||||
}
|
||||
groups := []group{
|
||||
{name: "remembered_speech", personalStart: 8, personalCount: 8, worldStart: 20, worldCount: 8},
|
||||
{name: "possession", personalStart: 16, personalCount: 10, worldStart: 28, worldCount: 12},
|
||||
{name: "narrative", personalStart: 26, personalCount: 8, worldStart: 40, worldCount: 8},
|
||||
{name: "first_person_preamble", personalStart: 34, personalCount: 8, worldStart: 48, worldCount: 8},
|
||||
{name: "advice_current_info", personalStart: 42, personalCount: 8, worldStart: 56, worldCount: 8},
|
||||
{name: "public_proper_nouns", personalStart: 50, personalCount: 10, worldStart: 64, worldCount: 8},
|
||||
}
|
||||
|
||||
aggregateCorrect, aggregateTotal := 0, 0
|
||||
for _, omitted := range groups {
|
||||
excluding := func(vectors [][]float32, start, count int) [][]float32 {
|
||||
result := make([][]float32, 0, len(vectors)-count)
|
||||
result = append(result, vectors[:start]...)
|
||||
return append(result, vectors[start+count:]...)
|
||||
}
|
||||
head, ok := trainPersonalBoundaryLinearHead(
|
||||
excluding(personalVectors, omitted.personalStart, omitted.personalCount),
|
||||
excluding(worldVectors, omitted.worldStart, omitted.worldCount),
|
||||
)
|
||||
if !ok {
|
||||
t.Fatalf("%s: valid training fold rejected", omitted.name)
|
||||
}
|
||||
correct, total := 0, 0
|
||||
for _, vector := range personalVectors[omitted.personalStart : omitted.personalStart+omitted.personalCount] {
|
||||
total++
|
||||
if head.logit(vector) > 0 {
|
||||
correct++
|
||||
}
|
||||
}
|
||||
for _, vector := range worldVectors[omitted.worldStart : omitted.worldStart+omitted.worldCount] {
|
||||
total++
|
||||
if head.logit(vector) <= 0 {
|
||||
correct++
|
||||
}
|
||||
}
|
||||
t.Logf("leave %-21s out: %d/%d", omitted.name, correct, total)
|
||||
aggregateCorrect += correct
|
||||
aggregateTotal += total
|
||||
// Whole-shape holdout is an honest diagnostic, not a 100% release gate:
|
||||
// some shapes (notably private-vs-general possession) define a distinct
|
||||
// semantic ambiguity. The separately authored challenge set remains the
|
||||
// strict generalisation gate.
|
||||
}
|
||||
if aggregateCorrect < 92 {
|
||||
t.Errorf("whole-shape aggregate %d/%d, want at least 92/104", aggregateCorrect, aggregateTotal)
|
||||
}
|
||||
}
|
||||
|
||||
// This challenge set was originally authored after the six-shape training
|
||||
// corpus and the 72-case matrix were frozen. Its sole miss then informed the
|
||||
// regularisation comparison, so it is now a strict regression gate rather than
|
||||
// independent evidence. It remains outside the production corpus.
|
||||
func TestONNXPersonalBoundaryChallenge(t *testing.T) {
|
||||
lib := os.Getenv("MAVEN_ONNX_LIB")
|
||||
if lib == "" {
|
||||
t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing")
|
||||
}
|
||||
dir := filepath.Join("../..", "models/embedder/multilingual-e5-small")
|
||||
emb, err := router.NewONNXEmbedder(filepath.Join(dir, "model_quantized.onnx"), filepath.Join(dir, "tokenizer.json"), lib)
|
||||
if err != nil {
|
||||
t.Skipf("onnx embedder unavailable: %v", err)
|
||||
}
|
||||
defer emb.Close()
|
||||
|
||||
cases := []struct {
|
||||
utterance string
|
||||
personal bool
|
||||
}{
|
||||
{"какой пароль приложения я задал для почтового клиента?", true},
|
||||
{"на каком порту я решил поднять тестовый сервис?", true},
|
||||
{"какую причину я указал, когда отменил бронь?", true},
|
||||
{"где в гараже я сложил зимние шины?", true},
|
||||
{"какой сериал я бросил после второго сезона?", true},
|
||||
{"о чём мы договорились с Олегом на прошлой неделе?", true},
|
||||
{"почему мой монитор мерцает при частоте 144 герца?", false},
|
||||
{"подойдёт ли кабель Thunderbolt 3 к разъёму USB4?", false},
|
||||
{"как вывести запах дыма из моей куртки?", false},
|
||||
{"что означают кольца на флаге Олимпиады?", false},
|
||||
{"почему после дождя на асфальте видна радуга?", false},
|
||||
{"какой формат файлов поддерживает Kindle Paperwhite?", false},
|
||||
{"which SSH key did I install on the build server?", true},
|
||||
{"what spending limit did I set for the travel card?", true},
|
||||
{"where did I store the spare apartment fob?", true},
|
||||
{"which objection did I raise during the design review?", true},
|
||||
{"what route did I plan for the Sunday hike?", true},
|
||||
{"when did I promise Maya I would send the draft?", true},
|
||||
{"why does my mechanical keyboard sometimes chatter?", false},
|
||||
{"can my USB-C charger safely power a Steam Deck?", false},
|
||||
{"how do I stop condensation inside my camera lens?", false},
|
||||
{"what caused the Tacoma Narrows Bridge to collapse?", false},
|
||||
{"why are some auroras red instead of green?", false},
|
||||
{"which codecs does the current Firefox release support?", false},
|
||||
}
|
||||
|
||||
b := &personalBoundary{}
|
||||
b.load(context.Background(), emb)
|
||||
correct := 0
|
||||
minimumMargin := math.Inf(1)
|
||||
for _, testCase := range cases {
|
||||
vector, err := router.EmbedQuery(context.Background(), emb, testCase.utterance)
|
||||
if err != nil {
|
||||
t.Fatalf("embed %q: %v", testCase.utterance, err)
|
||||
}
|
||||
personal, world, ok := b.score(vector)
|
||||
if !ok {
|
||||
t.Fatal("loaded boundary declined to score")
|
||||
}
|
||||
got := personal > world
|
||||
signedMargin := personal - world
|
||||
if !testCase.personal {
|
||||
signedMargin = -signedMargin
|
||||
}
|
||||
if signedMargin < minimumMargin {
|
||||
minimumMargin = signedMargin
|
||||
}
|
||||
if got == testCase.personal {
|
||||
correct++
|
||||
} else {
|
||||
t.Logf("miss %q: personal=%v want %v (%.4f/%.4f)", testCase.utterance, got, testCase.personal, personal, world)
|
||||
}
|
||||
}
|
||||
t.Logf("regularisation challenge: %d/%d, minimum signed margin %+.4f", correct, len(cases), minimumMargin)
|
||||
if correct != len(cases) {
|
||||
t.Errorf("regularisation challenge %d/%d, want every case correct", correct, len(cases))
|
||||
}
|
||||
}
|
||||
|
||||
// TestONNXPersonalBoundaryPostRetuneChallenge was authored only after the L2
|
||||
// coefficient and frozen head had been selected using corpus cross-validation.
|
||||
// It deliberately returns to private configuration, commitments and stored
|
||||
// choices with new objects, and contrasts them with public technical facts,
|
||||
// compatibility and maintenance. No result from this table may be used to
|
||||
// tune the current head; a miss is evidence for the next independently
|
||||
// evaluated model revision.
|
||||
func TestONNXPersonalBoundaryPostRetuneChallenge(t *testing.T) {
|
||||
lib := os.Getenv("MAVEN_ONNX_LIB")
|
||||
if lib == "" {
|
||||
t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing")
|
||||
}
|
||||
dir := filepath.Join("../..", "models/embedder/multilingual-e5-small")
|
||||
emb, err := router.NewONNXEmbedder(filepath.Join(dir, "model_quantized.onnx"), filepath.Join(dir, "tokenizer.json"), lib)
|
||||
if err != nil {
|
||||
t.Skipf("onnx embedder unavailable: %v", err)
|
||||
}
|
||||
defer emb.Close()
|
||||
|
||||
cases := []struct {
|
||||
utterance string
|
||||
personal bool
|
||||
}{
|
||||
{"какое имя я выбрал для гостевой сети Wi-Fi?", true},
|
||||
{"на какой день я перенёс техосмотр машины?", true},
|
||||
{"какую сумму мы с Мариной согласовали за ремонт кухни?", true},
|
||||
{"где я сохранил резервные коды от GitHub?", true},
|
||||
{"какой из макетов визитки я одобрил?", true},
|
||||
{"что я решил делать со страховкой перед поездкой?", true},
|
||||
{"какой диапазон частот использует Wi-Fi 6E?", false},
|
||||
{"почему OLED-экраны со временем выгорают?", false},
|
||||
{"можно ли подключить монитор DisplayPort к Thunderbolt 4?", false},
|
||||
{"чем безопасно чистить замшевые ботинки?", false},
|
||||
{"когда появился протокол WebSocket?", false},
|
||||
{"почему соль ускоряет таяние льда?", false},
|
||||
{"which hostname did I assign to the home NAS?", true},
|
||||
{"what date did I move the annual checkup to?", true},
|
||||
{"where did I save the recovery phrase for the hardware wallet?", true},
|
||||
{"which catering quote did we accept for the party?", true},
|
||||
{"what did I decide about renewing the domain?", true},
|
||||
{"which paint sample did I approve for the hallway?", true},
|
||||
{"does Wi-Fi 7 work with older wireless clients?", false},
|
||||
{"why can an SSD slow down when it is nearly full?", false},
|
||||
{"how should suede shoes be cleaned?", false},
|
||||
{"when was the WebSocket protocol standardized?", false},
|
||||
{"what does a hardware-wallet recovery phrase do?", false},
|
||||
{"why does road salt damage concrete?", false},
|
||||
}
|
||||
|
||||
b := &personalBoundary{}
|
||||
b.load(context.Background(), emb)
|
||||
correct := 0
|
||||
minimumMargin := math.Inf(1)
|
||||
for _, testCase := range cases {
|
||||
vector, err := router.EmbedQuery(context.Background(), emb, testCase.utterance)
|
||||
if err != nil {
|
||||
t.Fatalf("embed %q: %v", testCase.utterance, err)
|
||||
}
|
||||
personal, world, ok := b.score(vector)
|
||||
if !ok {
|
||||
t.Fatal("loaded boundary declined to score")
|
||||
}
|
||||
got := personal > world
|
||||
signedMargin := personal - world
|
||||
if !testCase.personal {
|
||||
signedMargin = -signedMargin
|
||||
}
|
||||
if signedMargin < minimumMargin {
|
||||
minimumMargin = signedMargin
|
||||
}
|
||||
if got == testCase.personal {
|
||||
correct++
|
||||
} else {
|
||||
t.Logf("miss %q: personal=%v want %v (%.4f/%.4f)", testCase.utterance, got, testCase.personal, personal, world)
|
||||
}
|
||||
}
|
||||
t.Logf("post-retune challenge: %d/%d, minimum signed margin %+.4f", correct, len(cases), minimumMargin)
|
||||
if correct != len(cases) {
|
||||
t.Errorf("post-retune challenge %d/%d, want every case correct", correct, len(cases))
|
||||
}
|
||||
}
|
||||
|
||||
func TestONNXPersonalBoundaryLatency(t *testing.T) {
|
||||
lib := os.Getenv("MAVEN_ONNX_LIB")
|
||||
if lib == "" {
|
||||
t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing")
|
||||
}
|
||||
dir := filepath.Join("../..", "models/embedder/multilingual-e5-small")
|
||||
emb, err := router.NewONNXEmbedder(filepath.Join(dir, "model_quantized.onnx"), filepath.Join(dir, "tokenizer.json"), lib)
|
||||
if err != nil {
|
||||
t.Skipf("onnx embedder unavailable: %v", err)
|
||||
}
|
||||
defer emb.Close()
|
||||
ctx := context.Background()
|
||||
query, err := router.EmbedQuery(ctx, emb, "что я решил насчёт переезда?")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
b := &personalBoundary{}
|
||||
coldStart := time.Now()
|
||||
b.load(ctx, emb)
|
||||
if _, _, ok := b.score(query); !ok {
|
||||
t.Fatal("loaded boundary declined to score")
|
||||
}
|
||||
cold := time.Since(coldStart)
|
||||
|
||||
const iterations = 100000
|
||||
steadyStart := time.Now()
|
||||
for i := 0; i < iterations; i++ {
|
||||
if _, _, ok := b.score(query); !ok {
|
||||
t.Fatal("loaded boundary declined to score")
|
||||
}
|
||||
}
|
||||
steady := time.Since(steadyStart) / iterations
|
||||
t.Logf("boundary cold load+train+score: %s; steady score: %s/op", cold, steady)
|
||||
// This is a user-visible first-turn path. Keep a generous ceiling to avoid
|
||||
// noisy CI while making an accidental per-turn training/load regression
|
||||
// unmistakable.
|
||||
if cold > 5*time.Second {
|
||||
t.Errorf("cold boundary load %s exceeds 5s local usability ceiling", cold)
|
||||
}
|
||||
if steady > 100*time.Microsecond {
|
||||
t.Errorf("steady boundary score %s exceeds 100µs ceiling", steady)
|
||||
}
|
||||
}
|
||||
|
||||
func TestONNXPersonalBoundaryFrozenHeadMatchesCorpusFit(t *testing.T) {
|
||||
lib := os.Getenv("MAVEN_ONNX_LIB")
|
||||
if lib == "" {
|
||||
t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing")
|
||||
}
|
||||
dir := filepath.Join("../..", "models/embedder/multilingual-e5-small")
|
||||
emb, err := router.NewONNXEmbedder(filepath.Join(dir, "model_quantized.onnx"), filepath.Join(dir, "tokenizer.json"), lib)
|
||||
if err != nil {
|
||||
t.Skipf("onnx embedder unavailable: %v", err)
|
||||
}
|
||||
defer emb.Close()
|
||||
ctx := context.Background()
|
||||
embedAll := func(values []string) [][]float32 {
|
||||
vectors := make([][]float32, len(values))
|
||||
for i, value := range values {
|
||||
vector, err := router.EmbedQuery(ctx, emb, value)
|
||||
if err != nil {
|
||||
t.Fatalf("embed %q: %v", value, err)
|
||||
}
|
||||
vectors[i] = vector
|
||||
}
|
||||
return vectors
|
||||
}
|
||||
fitted, ok := trainPersonalBoundaryLinearHead(embedAll(personalSeeds), embedAll(worldSeeds))
|
||||
if !ok {
|
||||
t.Fatal("corpus fit failed")
|
||||
}
|
||||
frozen, ok := frozenPersonalBoundaryHead()
|
||||
if !ok {
|
||||
t.Fatal("frozen head did not decode")
|
||||
}
|
||||
if math.Abs(fitted.bias-frozen.bias) > 1e-9 {
|
||||
t.Fatalf("frozen bias %.12f != fitted %.12f", frozen.bias, fitted.bias)
|
||||
}
|
||||
for i := range fitted.weights {
|
||||
if math.Abs(fitted.weights[i]-frozen.weights[i]) > 5e-7 {
|
||||
t.Fatalf("frozen weight %d %.12f != fitted %.12f", i, frozen.weights[i], fitted.weights[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+67
-19
@@ -172,14 +172,57 @@ func (h *reactiveHandler) stampLastTurn(utterance string, traceID int64) {
|
||||
h.lastRouted.traceID = traceID
|
||||
}
|
||||
|
||||
func (h *reactiveHandler) takeLastTurn() *routedTurn {
|
||||
// takeLastTurnIf atomically claims the previous acted turn only when the
|
||||
// caller can actually handle it. A declined repair must not spend the pointer:
|
||||
// "нет, это заметка" may name the intent Maven already chose and be followed
|
||||
// immediately by the real correction. The older read-then-clear helper lost
|
||||
// the original before checking either that case or the repair window (V-573).
|
||||
func (h *reactiveHandler) takeLastTurnIf(accept func(*routedTurn) bool) *routedTurn {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
last := h.lastRouted
|
||||
// Taken, not read: one utterance is corrected once. Saying "нет, не так"
|
||||
// twice would otherwise redo the same request twice.
|
||||
if h.lastRouted == nil || !accept(h.lastRouted) {
|
||||
return nil
|
||||
}
|
||||
last := *h.lastRouted
|
||||
// A handled correction is still spent once. Returning a copy prevents a
|
||||
// later trace stamp from mutating the evidence after this resolver owns it.
|
||||
h.lastRouted = nil
|
||||
return last
|
||||
return &last
|
||||
}
|
||||
|
||||
// takeTargetedRepair atomically distinguishes the three outcomes a targeted
|
||||
// correction needs. A recent, differently-routed turn is claimed and spent; a
|
||||
// recent turn already carrying that intent is retained and reported as
|
||||
// already-correct; everything else declines. Treating the second case as a
|
||||
// generic decline lets runTurn route the correction words as a fresh turn and
|
||||
// record them over the very pointer this helper was meant to preserve.
|
||||
func (h *reactiveHandler) takeTargetedRepair(now time.Time, corrected router.Intent) (last *routedTurn, already bool) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if h.lastRouted == nil || now.Sub(h.lastRouted.at) > repairWindow {
|
||||
return nil, false
|
||||
}
|
||||
if h.lastRouted.intent == corrected {
|
||||
return nil, true
|
||||
}
|
||||
copy := *h.lastRouted
|
||||
h.lastRouted = nil
|
||||
return ©, false
|
||||
}
|
||||
|
||||
// suspendClarifyForRepair makes a correction an aside to any question already
|
||||
// parked in this dialogue. It is called only after a repair has actually found
|
||||
// a target, so an ordinary utterance that merely resembles one changes no
|
||||
// dialogue state. If the redo itself needs a question, askClarify sees the
|
||||
// suspended flag and pushes that question instead of overwriting the older
|
||||
// request.
|
||||
func (h *reactiveHandler) suspendClarifyForRepair(ctx context.Context) {
|
||||
if h.clarifyStore == nil {
|
||||
return
|
||||
}
|
||||
if q := h.clarifyStore.Get(dialogueIDOf(ctx), h.now()); q != nil {
|
||||
h.noteSuspended(ctx, q)
|
||||
}
|
||||
}
|
||||
|
||||
// resolveUntargetedRepair handles the cheap half of a spoken correction: he says
|
||||
@@ -199,15 +242,16 @@ func (h *reactiveHandler) resolveUntargetedRepair(ctx context.Context, text stri
|
||||
if !isRepairNegative(text) {
|
||||
return "", false
|
||||
}
|
||||
last := h.takeLastTurn()
|
||||
if last == nil || h.now().Sub(last.at) > repairWindow {
|
||||
return "", false
|
||||
}
|
||||
if last.traceID == 0 {
|
||||
now := h.now()
|
||||
last := h.takeLastTurnIf(func(last *routedTurn) bool {
|
||||
return now.Sub(last.at) <= repairWindow && last.traceID != 0
|
||||
})
|
||||
if last == nil {
|
||||
// No row to point at, so there is no label to write and nothing this
|
||||
// resolver can do. Routing the words normally is the honest outcome.
|
||||
return "", false
|
||||
}
|
||||
h.suspendClarifyForRepair(ctx)
|
||||
h.labelCorrection(ctx, last, "")
|
||||
log.Printf("voice: repair — %q marked wrong, no target given", last.utterance)
|
||||
return phraser.A(phraser.RepairNoted, nil), true
|
||||
@@ -239,16 +283,20 @@ func (h *reactiveHandler) resolveRepair(ctx context.Context, text string) (strin
|
||||
if !ok || h.router == nil {
|
||||
return "", false
|
||||
}
|
||||
last := h.takeLastTurn()
|
||||
if last == nil || h.now().Sub(last.at) > repairWindow {
|
||||
return "", false
|
||||
}
|
||||
if last.intent == corrected {
|
||||
// She already did what he is asking for. Correcting the classifier
|
||||
// here would teach it the label it produced, and redoing the request
|
||||
// would file it twice.
|
||||
last, already := h.takeTargetedRepair(h.now(), corrected)
|
||||
if already {
|
||||
// This is still a correction turn, not slot material and not a fresh note.
|
||||
// Say why nothing ran, retain the original pointer, and keep any parked
|
||||
// question audible for the next breath.
|
||||
h.suspendClarifyForRepair(ctx)
|
||||
return "это уже " + say + " — ничего не переделываю.", true
|
||||
}
|
||||
if last == nil {
|
||||
// Nothing recent to correct. Routing the words normally is the honest
|
||||
// outcome; an expired pointer cannot become usable again.
|
||||
return "", false
|
||||
}
|
||||
h.suspendClarifyForRepair(ctx)
|
||||
learned := true
|
||||
if err := h.router.CorrectMisroute(ctx, last.utterance, corrected); err != nil {
|
||||
// The redo is still worth doing: he asked for something and it did not
|
||||
@@ -271,7 +319,7 @@ func (h *reactiveHandler) resolveRepair(ctx context.Context, text string) (strin
|
||||
if dec.Slots.Text == "" && corrected != router.IntentReminder {
|
||||
dec.Slots.Text = last.utterance
|
||||
}
|
||||
return repairLine(say, learned) + " " + h.finishClarified(ctx, dec), true
|
||||
return repairLine(say, learned) + " " + h.finishRepaired(ctx, dec), true
|
||||
}
|
||||
|
||||
// repairLine — what she says before redoing it, so the correction is visible
|
||||
|
||||
+125
-4
@@ -6,6 +6,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/dialogue"
|
||||
"github.com/kami/maven/internal/router"
|
||||
"github.com/kami/maven/internal/store"
|
||||
)
|
||||
@@ -93,7 +94,7 @@ func TestRepairNeedsARecentTurnToPointAt(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRepairIsSpentOnce(t *testing.T) {
|
||||
h, _, _ := newClarifyHandler(t)
|
||||
h, st, _ := newClarifyHandler(t)
|
||||
emb := router.NewHashEmbedder(256)
|
||||
h.recall.embedder = emb
|
||||
h.router = router.New(router.Config{Classifier: router.NewClassifier(emb), Extractor: h.extractor})
|
||||
@@ -103,8 +104,17 @@ func TestRepairIsSpentOnce(t *testing.T) {
|
||||
if _, handled := h.resolveRepair(ctx, "нет, это заметка"); !handled {
|
||||
t.Fatal("the first correction was not handled")
|
||||
}
|
||||
if _, handled := h.resolveRepair(ctx, "нет, это заметка"); handled {
|
||||
t.Error("the same turn was corrected twice")
|
||||
before, err := st.RecentNotes(ctx, 10)
|
||||
if err != nil || len(before) != 1 {
|
||||
t.Fatalf("first repair notes=%+v err=%v", before, err)
|
||||
}
|
||||
reply, handled := h.resolveRepair(ctx, "нет, это заметка")
|
||||
if !handled || !strings.Contains(reply, "уже") {
|
||||
t.Fatalf("the repeated correction was not acknowledged as already applied: handled=%v reply=%q", handled, reply)
|
||||
}
|
||||
after, err := st.RecentNotes(ctx, 10)
|
||||
if err != nil || len(after) != 1 {
|
||||
t.Fatalf("the same turn was redone twice: notes=%+v err=%v", after, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -114,8 +124,119 @@ func TestRepairPassesWhenSheAlreadyDidThat(t *testing.T) {
|
||||
h, _, _ := newClarifyHandler(t)
|
||||
h.router = router.New(router.Config{Classifier: router.NewClassifier(router.NewHashEmbedder(256))})
|
||||
h.recordTurn("купить хлеб", router.IntentNote)
|
||||
reply, handled := h.resolveRepair(context.Background(), "нет, это заметка")
|
||||
if !handled || !strings.Contains(reply, "уже") {
|
||||
t.Fatalf("a redundant correction must be acknowledged without redoing it: handled=%v reply=%q", handled, reply)
|
||||
}
|
||||
// Acknowledging the redundant target must not spend the original. If this
|
||||
// resolver declines instead, runTurn routes the correction as a fresh turn
|
||||
// and recordTurn overwrites the pointer even though takeLastTurn retained it.
|
||||
if _, handled := h.resolveRepair(context.Background(), "нет, это факт"); !handled {
|
||||
t.Error("a redundant same-intent repair spent the original turn")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepairResumesQuestionParkedAfterTheCorrectedTurn(t *testing.T) {
|
||||
h, _, _ := newClarifyHandler(t)
|
||||
emb := router.NewHashEmbedder(256)
|
||||
h.recall.embedder = emb
|
||||
h.router = router.New(router.Config{Classifier: router.NewClassifier(emb), Extractor: h.extractor})
|
||||
ctx := context.Background()
|
||||
|
||||
h.recordTurn("купить хлеб", router.IntentFact)
|
||||
if _, asked := h.askClarify(ctx, clarifyDec(router.IntentReminder,
|
||||
router.Slots{Text: "позвонить маме"}, "напомни позвонить маме")); !asked {
|
||||
t.Fatal("expected a parked reminder question")
|
||||
}
|
||||
|
||||
reply := h.runTurn(ctx, "нет, это был вопрос", sourceText)
|
||||
resumed, _ := clarifyResumedFor(dialogue.SlotTime)
|
||||
if !strings.HasSuffix(reply, resumed) {
|
||||
t.Fatalf("the correction hid the still-live question: reply=%q want suffix=%q", reply, resumed)
|
||||
}
|
||||
q := h.clarifyStore.Get(voiceDialogueID, h.now())
|
||||
if q == nil {
|
||||
t.Fatal("the correction dropped the parked question")
|
||||
}
|
||||
if q.Attempts != 1 || q.Suspends != 1 {
|
||||
t.Fatalf("the correction spent a retry instead of suspending the question: %+v", q)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepairedClarifyCompletesWithoutDroppingTheOlderQuestion(t *testing.T) {
|
||||
h, st, _ := newClarifyHandler(t)
|
||||
emb := router.NewHashEmbedder(256)
|
||||
h.recall.embedder = emb
|
||||
h.router = router.New(router.Config{Classifier: router.NewClassifier(emb), Extractor: h.extractor})
|
||||
ctx := context.Background()
|
||||
|
||||
h.recordTurn("купить хлеб", router.IntentFact)
|
||||
if _, asked := h.askClarify(ctx, clarifyDec(router.IntentReminder,
|
||||
router.Slots{Text: "позвонить маме"}, "напомни позвонить маме")); !asked {
|
||||
t.Fatal("expected the older reminder question")
|
||||
}
|
||||
|
||||
if reply := h.runTurn(ctx, "нет, это было напоминание", sourceText); !strings.Contains(reply, "Когда") {
|
||||
t.Fatalf("the repaired reminder did not ask for its missing time: %q", reply)
|
||||
}
|
||||
if depth := h.clarifyStore.Depth(voiceDialogueID); depth != 2 {
|
||||
t.Fatalf("the repaired question overwrote the older one: depth=%d want=2", depth)
|
||||
}
|
||||
|
||||
reply := h.runTurn(ctx, "сегодня в 15:00", sourceText)
|
||||
resumed, _ := clarifyResumedFor(dialogue.SlotTime)
|
||||
if !strings.HasSuffix(reply, resumed) {
|
||||
t.Fatalf("completing the repaired request did not resume the older one: reply=%q", reply)
|
||||
}
|
||||
q := h.clarifyStore.Get(voiceDialogueID, h.now())
|
||||
if q == nil || !strings.Contains(q.Utterance, "маме") {
|
||||
t.Fatalf("the older question was lost after the top one completed: %+v", q)
|
||||
}
|
||||
reminders, err := st.DueReminders(ctx, h.now().Add(48*time.Hour))
|
||||
if err != nil || len(reminders) != 1 || !strings.Contains(reminders[0].Payload, "хлеб") {
|
||||
t.Fatalf("the repaired reminder did not land exactly once: reminders=%+v err=%v", reminders, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepairedClarifyGiveUpKeepsTheOlderQuestion(t *testing.T) {
|
||||
h, _, _ := newClarifyHandler(t)
|
||||
ctx := context.Background()
|
||||
older := &dialogue.PendingQuestion{
|
||||
Intent: dialogue.IntentReminder, Missing: []dialogue.Slot{dialogue.SlotTime},
|
||||
Utterance: "напомни позвонить маме", Asked: h.now(), TTL: clarifyTTL,
|
||||
}
|
||||
top := &dialogue.PendingQuestion{
|
||||
Intent: dialogue.IntentReminder, Missing: []dialogue.Slot{dialogue.SlotTime},
|
||||
Utterance: "напомни купить хлеб", Asked: h.now(), TTL: clarifyTTL,
|
||||
Attempts: dialogue.DefaultMaxAttempts, MaxAttempts: dialogue.DefaultMaxAttempts,
|
||||
}
|
||||
h.clarifyStore.Push(voiceDialogueID, older)
|
||||
h.clarifyStore.Push(voiceDialogueID, top)
|
||||
|
||||
if reply := h.reaskOrGiveUp(ctx, top, top.Slots, "не знаю", ""); reply != clarifyGaveUp {
|
||||
t.Fatalf("reply=%q, want the explicit give-up line", reply)
|
||||
}
|
||||
if depth := h.clarifyStore.Depth(voiceDialogueID); depth != 1 {
|
||||
t.Fatalf("giving up on the top request erased the older flow: depth=%d", depth)
|
||||
}
|
||||
q := h.clarifyStore.Get(voiceDialogueID, h.now())
|
||||
if q != older {
|
||||
t.Fatalf("survivor=%+v, want the older parked question", q)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStaleRepairDoesNotSpendTheOriginal(t *testing.T) {
|
||||
h, _, now := newClarifyHandler(t)
|
||||
h.router = router.New(router.Config{Classifier: router.NewClassifier(router.NewHashEmbedder(256))})
|
||||
h.recordTurn("купить хлеб", router.IntentFact)
|
||||
*now = now.Add(repairWindow + time.Minute)
|
||||
if _, handled := h.resolveRepair(context.Background(), "нет, это заметка"); handled {
|
||||
t.Error("a correction to the intent she already used was handled")
|
||||
t.Fatal("a stale correction was handled")
|
||||
}
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if h.lastRouted == nil || h.lastRouted.utterance != "купить хлеб" {
|
||||
t.Fatal("a stale declined correction spent the original turn")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+521
@@ -0,0 +1,521 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"name": "personal_boundary_v1",
|
||||
"notes": [
|
||||
"Held-out personal-boundary fixture for V-702. Cases are not production seeds and the fixture test enforces that after case folding and punctuation removal.",
|
||||
"The matrix is balanced: three cases for every stratum × language × expected side cell (6 × 2 × 2 × 3 = 72).",
|
||||
"Personal means that only the owner's local data can answer. World means that an outside knowledge source can answer even when the wording uses first person or possession.",
|
||||
"Public subjects are deliberately varied. No case repeats the Baikal regression or changes only its place name."
|
||||
],
|
||||
"cases": [
|
||||
{
|
||||
"id": "pb-ru-remembered-personal-01",
|
||||
"utterance": "что я раньше говорил насчёт бессонницы?",
|
||||
"lang": "ru",
|
||||
"want": "personal",
|
||||
"stratum": "remembered_speech"
|
||||
},
|
||||
{
|
||||
"id": "pb-ru-remembered-personal-02",
|
||||
"utterance": "помнишь, какие причины увольнения я называл?",
|
||||
"lang": "ru",
|
||||
"want": "personal",
|
||||
"stratum": "remembered_speech"
|
||||
},
|
||||
{
|
||||
"id": "pb-ru-remembered-personal-03",
|
||||
"utterance": "упоминал ли я, куда хочу переехать?",
|
||||
"lang": "ru",
|
||||
"want": "personal",
|
||||
"stratum": "remembered_speech"
|
||||
},
|
||||
{
|
||||
"id": "pb-ru-remembered-world-01",
|
||||
"utterance": "что Чехов говорил о краткости?",
|
||||
"lang": "ru",
|
||||
"want": "world",
|
||||
"stratum": "remembered_speech"
|
||||
},
|
||||
{
|
||||
"id": "pb-ru-remembered-world-02",
|
||||
"utterance": "какие причины кризиса называли экономисты?",
|
||||
"lang": "ru",
|
||||
"want": "world",
|
||||
"stratum": "remembered_speech"
|
||||
},
|
||||
{
|
||||
"id": "pb-ru-remembered-world-03",
|
||||
"utterance": "что свидетели рассказывали об извержении Кракатау?",
|
||||
"lang": "ru",
|
||||
"want": "world",
|
||||
"stratum": "remembered_speech"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-remembered-personal-01",
|
||||
"utterance": "what was it I told you about changing careers?",
|
||||
"lang": "en",
|
||||
"want": "personal",
|
||||
"stratum": "remembered_speech"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-remembered-personal-02",
|
||||
"utterance": "have I ever mentioned where I studied?",
|
||||
"lang": "en",
|
||||
"want": "personal",
|
||||
"stratum": "remembered_speech"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-remembered-personal-03",
|
||||
"utterance": "do you remember which camera I said I preferred?",
|
||||
"lang": "en",
|
||||
"want": "personal",
|
||||
"stratum": "remembered_speech"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-remembered-world-01",
|
||||
"utterance": "what did Marie Curie write about radium?",
|
||||
"lang": "en",
|
||||
"want": "world",
|
||||
"stratum": "remembered_speech"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-remembered-world-02",
|
||||
"utterance": "which causes of inflation do economists usually mention?",
|
||||
"lang": "en",
|
||||
"want": "world",
|
||||
"stratum": "remembered_speech"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-remembered-world-03",
|
||||
"utterance": "what did the Apollo astronauts report about lunar dust?",
|
||||
"lang": "en",
|
||||
"want": "world",
|
||||
"stratum": "remembered_speech"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "pb-ru-possession-personal-01",
|
||||
"utterance": "какой номер у моего страхового полиса?",
|
||||
"lang": "ru",
|
||||
"want": "personal",
|
||||
"stratum": "possession"
|
||||
},
|
||||
{
|
||||
"id": "pb-ru-possession-personal-02",
|
||||
"utterance": "где я оставил свои запасные ключи?",
|
||||
"lang": "ru",
|
||||
"want": "personal",
|
||||
"stratum": "possession"
|
||||
},
|
||||
{
|
||||
"id": "pb-ru-possession-personal-03",
|
||||
"utterance": "до какого числа действует мой абонемент?",
|
||||
"lang": "ru",
|
||||
"want": "personal",
|
||||
"stratum": "possession"
|
||||
},
|
||||
{
|
||||
"id": "pb-ru-possession-world-01",
|
||||
"utterance": "как убрать царапину с моего стола?",
|
||||
"lang": "ru",
|
||||
"want": "world",
|
||||
"stratum": "possession"
|
||||
},
|
||||
{
|
||||
"id": "pb-ru-possession-world-02",
|
||||
"utterance": "почему у меня запотевают окна зимой?",
|
||||
"lang": "ru",
|
||||
"want": "world",
|
||||
"stratum": "possession"
|
||||
},
|
||||
{
|
||||
"id": "pb-ru-possession-world-03",
|
||||
"utterance": "чем зарядить мой телефон в поездке?",
|
||||
"lang": "ru",
|
||||
"want": "world",
|
||||
"stratum": "possession"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-possession-personal-01",
|
||||
"utterance": "when does my library card expire?",
|
||||
"lang": "en",
|
||||
"want": "personal",
|
||||
"stratum": "possession"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-possession-personal-02",
|
||||
"utterance": "where did I put my passport copy?",
|
||||
"lang": "en",
|
||||
"want": "personal",
|
||||
"stratum": "possession"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-possession-personal-03",
|
||||
"utterance": "what size are my hiking boots?",
|
||||
"lang": "en",
|
||||
"want": "personal",
|
||||
"stratum": "possession"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-possession-world-01",
|
||||
"utterance": "how can I descale my kettle safely?",
|
||||
"lang": "en",
|
||||
"want": "world",
|
||||
"stratum": "possession"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-possession-world-02",
|
||||
"utterance": "why does my laptop fan get loud under load?",
|
||||
"lang": "en",
|
||||
"want": "world",
|
||||
"stratum": "possession"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-possession-world-03",
|
||||
"utterance": "which adapter should I use for my phone abroad?",
|
||||
"lang": "en",
|
||||
"want": "world",
|
||||
"stratum": "possession"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "pb-ru-narrative-personal-01",
|
||||
"utterance": "напомни историю о том, как я познакомился с Антоном",
|
||||
"lang": "ru",
|
||||
"want": "personal",
|
||||
"stratum": "narrative"
|
||||
},
|
||||
{
|
||||
"id": "pb-ru-narrative-personal-02",
|
||||
"utterance": "расскажи, что со мной случилось в первый день на новой работе",
|
||||
"lang": "ru",
|
||||
"want": "personal",
|
||||
"stratum": "narrative"
|
||||
},
|
||||
{
|
||||
"id": "pb-ru-narrative-personal-03",
|
||||
"utterance": "восстанови по моим заметкам историю поездки в Казань",
|
||||
"lang": "ru",
|
||||
"want": "personal",
|
||||
"stratum": "narrative"
|
||||
},
|
||||
{
|
||||
"id": "pb-ru-narrative-world-01",
|
||||
"utterance": "опиши восхождение на Эверест",
|
||||
"lang": "ru",
|
||||
"want": "world",
|
||||
"stratum": "narrative"
|
||||
},
|
||||
{
|
||||
"id": "pb-ru-narrative-world-02",
|
||||
"utterance": "расскажи историю создания языка Rust",
|
||||
"lang": "ru",
|
||||
"want": "world",
|
||||
"stratum": "narrative"
|
||||
},
|
||||
{
|
||||
"id": "pb-ru-narrative-world-03",
|
||||
"utterance": "объясни, как возникли кольца Сатурна",
|
||||
"lang": "ru",
|
||||
"want": "world",
|
||||
"stratum": "narrative"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-narrative-personal-01",
|
||||
"utterance": "retell the story of how I met Lena from what I told you",
|
||||
"lang": "en",
|
||||
"want": "personal",
|
||||
"stratum": "narrative"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-narrative-personal-02",
|
||||
"utterance": "walk me through what happened on my first day at university",
|
||||
"lang": "en",
|
||||
"want": "personal",
|
||||
"stratum": "narrative"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-narrative-personal-03",
|
||||
"utterance": "reconstruct my Prague trip from my notes",
|
||||
"lang": "en",
|
||||
"want": "personal",
|
||||
"stratum": "narrative"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-narrative-world-01",
|
||||
"utterance": "tell me the story of the first Moon landing",
|
||||
"lang": "en",
|
||||
"want": "world",
|
||||
"stratum": "narrative"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-narrative-world-02",
|
||||
"utterance": "describe how the printing press spread through Europe",
|
||||
"lang": "en",
|
||||
"want": "world",
|
||||
"stratum": "narrative"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-narrative-world-03",
|
||||
"utterance": "explain how the Panama Canal was built",
|
||||
"lang": "en",
|
||||
"want": "world",
|
||||
"stratum": "narrative"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "pb-ru-preamble-personal-01",
|
||||
"utterance": "если помнишь наш разговор, что я решил насчёт переезда?",
|
||||
"lang": "ru",
|
||||
"want": "personal",
|
||||
"stratum": "first_person_preamble"
|
||||
},
|
||||
{
|
||||
"id": "pb-ru-preamble-personal-02",
|
||||
"utterance": "как я уже упоминал, когда мне продлевать страховку?",
|
||||
"lang": "ru",
|
||||
"want": "personal",
|
||||
"stratum": "first_person_preamble"
|
||||
},
|
||||
{
|
||||
"id": "pb-ru-preamble-personal-03",
|
||||
"utterance": "возвращаясь к тому, что я рассказывал, какую модель велосипеда я выбрал?",
|
||||
"lang": "ru",
|
||||
"want": "personal",
|
||||
"stratum": "first_person_preamble"
|
||||
},
|
||||
{
|
||||
"id": "pb-ru-preamble-world-01",
|
||||
"utterance": "как я уже говорил, почему самолёты оставляют белый след?",
|
||||
"lang": "ru",
|
||||
"want": "world",
|
||||
"stratum": "first_person_preamble"
|
||||
},
|
||||
{
|
||||
"id": "pb-ru-preamble-world-02",
|
||||
"utterance": "возвращаясь к моему вопросу, из чего состоит базальт?",
|
||||
"lang": "ru",
|
||||
"want": "world",
|
||||
"stratum": "first_person_preamble"
|
||||
},
|
||||
{
|
||||
"id": "pb-ru-preamble-world-03",
|
||||
"utterance": "я, возможно, повторяюсь, но когда построили Колизей?",
|
||||
"lang": "ru",
|
||||
"want": "world",
|
||||
"stratum": "first_person_preamble"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-preamble-personal-01",
|
||||
"utterance": "as I mentioned earlier, which dentist did I choose?",
|
||||
"lang": "en",
|
||||
"want": "personal",
|
||||
"stratum": "first_person_preamble"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-preamble-personal-02",
|
||||
"utterance": "coming back to what I told you, when am I taking leave?",
|
||||
"lang": "en",
|
||||
"want": "personal",
|
||||
"stratum": "first_person_preamble"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-preamble-personal-03",
|
||||
"utterance": "I may have said this already, which Linux distro did I settle on?",
|
||||
"lang": "en",
|
||||
"want": "personal",
|
||||
"stratum": "first_person_preamble"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-preamble-world-01",
|
||||
"utterance": "as I was saying, why do tides happen?",
|
||||
"lang": "en",
|
||||
"want": "world",
|
||||
"stratum": "first_person_preamble"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-preamble-world-02",
|
||||
"utterance": "coming back to my question, how are auroras formed?",
|
||||
"lang": "en",
|
||||
"want": "world",
|
||||
"stratum": "first_person_preamble"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-preamble-world-03",
|
||||
"utterance": "I may be repeating myself, when was Machu Picchu built?",
|
||||
"lang": "en",
|
||||
"want": "world",
|
||||
"stratum": "first_person_preamble"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "pb-ru-advice-personal-01",
|
||||
"utterance": "что из моих дел нужно закончить до пятницы?",
|
||||
"lang": "ru",
|
||||
"want": "personal",
|
||||
"stratum": "advice_current_info"
|
||||
},
|
||||
{
|
||||
"id": "pb-ru-advice-personal-02",
|
||||
"utterance": "какое лекарство врач велел мне принимать утром?",
|
||||
"lang": "ru",
|
||||
"want": "personal",
|
||||
"stratum": "advice_current_info"
|
||||
},
|
||||
{
|
||||
"id": "pb-ru-advice-personal-03",
|
||||
"utterance": "сколько денег я потратил на продукты в этом месяце?",
|
||||
"lang": "ru",
|
||||
"want": "personal",
|
||||
"stratum": "advice_current_info"
|
||||
},
|
||||
{
|
||||
"id": "pb-ru-advice-world-01",
|
||||
"utterance": "как безопасно заменить розетку?",
|
||||
"lang": "ru",
|
||||
"want": "world",
|
||||
"stratum": "advice_current_info"
|
||||
},
|
||||
{
|
||||
"id": "pb-ru-advice-world-02",
|
||||
"utterance": "какая сейчас версия Debian stable?",
|
||||
"lang": "ru",
|
||||
"want": "world",
|
||||
"stratum": "advice_current_info"
|
||||
},
|
||||
{
|
||||
"id": "pb-ru-advice-world-03",
|
||||
"utterance": "что сегодня происходит на мировых рынках?",
|
||||
"lang": "ru",
|
||||
"want": "world",
|
||||
"stratum": "advice_current_info"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-advice-personal-01",
|
||||
"utterance": "which of my tasks is due before Friday?",
|
||||
"lang": "en",
|
||||
"want": "personal",
|
||||
"stratum": "advice_current_info"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-advice-personal-02",
|
||||
"utterance": "what dosage did my doctor tell me to take at breakfast?",
|
||||
"lang": "en",
|
||||
"want": "personal",
|
||||
"stratum": "advice_current_info"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-advice-personal-03",
|
||||
"utterance": "how much did I spend on groceries this month?",
|
||||
"lang": "en",
|
||||
"want": "personal",
|
||||
"stratum": "advice_current_info"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-advice-world-01",
|
||||
"utterance": "how should I clean a cast-iron pan?",
|
||||
"lang": "en",
|
||||
"want": "world",
|
||||
"stratum": "advice_current_info"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-advice-world-02",
|
||||
"utterance": "what is the current stable release of PostgreSQL?",
|
||||
"lang": "en",
|
||||
"want": "world",
|
||||
"stratum": "advice_current_info"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-advice-world-03",
|
||||
"utterance": "which major elections are happening this month?",
|
||||
"lang": "en",
|
||||
"want": "world",
|
||||
"stratum": "advice_current_info"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "pb-ru-proper-personal-01",
|
||||
"utterance": "что я записал после доклада Линуса Торвальдса?",
|
||||
"lang": "ru",
|
||||
"want": "personal",
|
||||
"stratum": "public_proper_nouns"
|
||||
},
|
||||
{
|
||||
"id": "pb-ru-proper-personal-02",
|
||||
"utterance": "какое мнение я высказал о фильмах Куросавы?",
|
||||
"lang": "ru",
|
||||
"want": "personal",
|
||||
"stratum": "public_proper_nouns"
|
||||
},
|
||||
{
|
||||
"id": "pb-ru-proper-personal-03",
|
||||
"utterance": "когда у меня билеты на концерт Земфиры?",
|
||||
"lang": "ru",
|
||||
"want": "personal",
|
||||
"stratum": "public_proper_nouns"
|
||||
},
|
||||
{
|
||||
"id": "pb-ru-proper-world-01",
|
||||
"utterance": "кто такой Алан Тьюринг?",
|
||||
"lang": "ru",
|
||||
"want": "world",
|
||||
"stratum": "public_proper_nouns"
|
||||
},
|
||||
{
|
||||
"id": "pb-ru-proper-world-02",
|
||||
"utterance": "чем известна Фрида Кало?",
|
||||
"lang": "ru",
|
||||
"want": "world",
|
||||
"stratum": "public_proper_nouns"
|
||||
},
|
||||
{
|
||||
"id": "pb-ru-proper-world-03",
|
||||
"utterance": "когда родился Юрий Гагарин?",
|
||||
"lang": "ru",
|
||||
"want": "world",
|
||||
"stratum": "public_proper_nouns"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-proper-personal-01",
|
||||
"utterance": "what notes did I make after Grace Hopper's talk?",
|
||||
"lang": "en",
|
||||
"want": "personal",
|
||||
"stratum": "public_proper_nouns"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-proper-personal-02",
|
||||
"utterance": "which David Bowie album did I say I liked most?",
|
||||
"lang": "en",
|
||||
"want": "personal",
|
||||
"stratum": "public_proper_nouns"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-proper-personal-03",
|
||||
"utterance": "when are my tickets for the Radiohead show?",
|
||||
"lang": "en",
|
||||
"want": "personal",
|
||||
"stratum": "public_proper_nouns"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-proper-world-01",
|
||||
"utterance": "who was Katherine Johnson?",
|
||||
"lang": "en",
|
||||
"want": "world",
|
||||
"stratum": "public_proper_nouns"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-proper-world-02",
|
||||
"utterance": "what is Antoni Gaudí famous for?",
|
||||
"lang": "en",
|
||||
"want": "world",
|
||||
"stratum": "public_proper_nouns"
|
||||
},
|
||||
{
|
||||
"id": "pb-en-proper-world-03",
|
||||
"utterance": "when was Nelson Mandela born?",
|
||||
"lang": "en",
|
||||
"want": "world",
|
||||
"stratum": "public_proper_nouns"
|
||||
}
|
||||
]
|
||||
}
|
||||
+40
-41
@@ -260,8 +260,9 @@ const (
|
||||
)
|
||||
|
||||
// runTurn — the reactive turn pipeline shared by the voice and text entry
|
||||
// points: expired-clarify notice → confirm answer → clarify answer → quiet
|
||||
// toggle → route → dialogue merge → clarify question → action → replier.
|
||||
// points: expired-clarify notice → confirm answer → explicit correction →
|
||||
// clarify answer → quiet toggle → route → dialogue merge → clarify question →
|
||||
// action → replier.
|
||||
// Takes the already-transcribed utterance, returns the reply text; the voice
|
||||
// path wraps it in stt/tts, the text path returns it as-is.
|
||||
//
|
||||
@@ -290,6 +291,14 @@ func (h *reactiveHandler) runTurn(ctx context.Context, text string, src turnSour
|
||||
now := h.now()
|
||||
rt := h.newTurnRoute(text, now)
|
||||
ctx = withTurnRoute(ctx, rt)
|
||||
// A resolver may suspend an older clarify flow even when it handles this
|
||||
// turn itself. Finalise that state at one choke point so early returns from
|
||||
// confirm/repair/clarify cannot leave a live question parked without saying
|
||||
// it again, or silently drop one when the suspension bound is reached.
|
||||
defer func() {
|
||||
reply = withNotice(rt.dropped, reply)
|
||||
reply = withResumed(reply, rt.resume)
|
||||
}()
|
||||
|
||||
// 1. expired clarify — a question was parked but its TTL ran out, so the
|
||||
// request behind it is gone. Say that out loud (see clarify.go) and carry
|
||||
@@ -310,7 +319,23 @@ func (h *reactiveHandler) runTurn(ctx context.Context, text string, src turnSour
|
||||
return withNotice(expiredNotice, reply)
|
||||
}
|
||||
|
||||
// 3. clarify answer — if she asked a live question last turn, this
|
||||
// 3. spoken correction — an explicit "нет, это был вопрос" names both the
|
||||
// prior mistake and its replacement. It is narrower evidence than a parked
|
||||
// question merely being present, so it gets first refusal. Otherwise the
|
||||
// clarify resolver treats the correction as a bad slot value and spends a
|
||||
// retry on a turn that was never an answer (V-573).
|
||||
if reply, handled := h.resolveRepair(ctx, text); notePreRoute(ctx, "repair", handled) {
|
||||
return withNotice(expiredNotice, reply)
|
||||
}
|
||||
|
||||
// 3b. The same correction without a target — "нет, не так". It cannot redo
|
||||
// the turn, but it can durably label the previous decision as wrong. Like a
|
||||
// targeted repair, it is not an answer to a parked slot question.
|
||||
if reply, handled := h.resolveUntargetedRepair(ctx, text); notePreRoute(ctx, "repair-negative", handled) {
|
||||
return withNotice(expiredNotice, reply)
|
||||
}
|
||||
|
||||
// 4. clarify answer — if she asked a live question last turn, this
|
||||
// utterance is its answer, not a fresh command. After the confirm check: a
|
||||
// y/n gate is armed by her own prompt and is the narrower claim on the
|
||||
// utterance.
|
||||
@@ -321,20 +346,10 @@ func (h *reactiveHandler) runTurn(ctx context.Context, text string, src turnSour
|
||||
if reply, handled := h.resolveClarifyAnswer(ctx, text); notePreRoute(ctx, "clarify-answer", handled) {
|
||||
return withNotice(expiredNotice, reply)
|
||||
}
|
||||
// It did not claim the turn. If it let a parked request go to get out of the
|
||||
// way, that has to be said in front of whatever these words are answered
|
||||
// with — carried on the same notice, so every exit below keeps it.
|
||||
expiredNotice = withNotice(expiredNotice, rt.dropped)
|
||||
// It did not claim the turn. Any drop notice or resumed question recorded on
|
||||
// rt is attached by the turn finaliser above, including on an early return.
|
||||
|
||||
// 3b. and if it SUSPENDED a request instead of letting it go, the question
|
||||
// comes back on the end of whatever these words are answered with (Vikunja
|
||||
// #561). A deferred append rather than a call at each exit: there are eight
|
||||
// returns between here and the replier, and the flow has to survive all of
|
||||
// them — one that forgot would be a request parked for ever, waiting for an
|
||||
// answer to a question he never heard asked.
|
||||
defer func() { reply = withResumed(reply, rt.resume) }()
|
||||
|
||||
// 4. quiet-hours toggle — keyword match, not classifier-dependent.
|
||||
// 5. quiet-hours toggle — keyword match, not classifier-dependent.
|
||||
// "тихий режим" / "quiet on" would route through the classifier
|
||||
// unreliably (it's a command, not a free-form query), so we match it
|
||||
// before routing. Same pattern as the confirm turn above.
|
||||
@@ -342,7 +357,7 @@ func (h *reactiveHandler) runTurn(ctx context.Context, text string, src turnSour
|
||||
return withNotice(expiredNotice, reply)
|
||||
}
|
||||
|
||||
// 4b. spoken snooze — "не сейчас" / "потом" answers the nudge she just
|
||||
// 5b. spoken snooze — "не сейчас" / "потом" answers the nudge she just
|
||||
// sent. Only handled when a pending nudge is actually inside the window
|
||||
// (snooze.go); otherwise the words route normally, because "потом" is an
|
||||
// ordinary word and eating every one of them would break real sentences.
|
||||
@@ -350,30 +365,14 @@ func (h *reactiveHandler) runTurn(ctx context.Context, text string, src turnSour
|
||||
return withNotice(expiredNotice, reply)
|
||||
}
|
||||
|
||||
// 4c. spoken ack — "готово" closes that same nudge as `acted`. Only the
|
||||
// 5c. spoken ack — "готово" closes that same nudge as `acted`. Only the
|
||||
// contentless form is intercepted here; "выпил воды" keeps routing and
|
||||
// closes the nudge after its fact lands (ackFromFact, step 8b).
|
||||
if reply, handled := h.resolveAck(ctx, text, src); notePreRoute(ctx, "ack", handled) {
|
||||
return withNotice(expiredNotice, reply)
|
||||
}
|
||||
|
||||
// 4d. spoken correction — "нет, это была заметка" points at the previous
|
||||
// turn and names what it should have been (repair.go). Before routing,
|
||||
// like the confirm and clarify turns: routing the correction as a fresh
|
||||
// utterance files the correction itself instead of fixing anything.
|
||||
if reply, handled := h.resolveRepair(ctx, text); notePreRoute(ctx, "repair", handled) {
|
||||
return withNotice(expiredNotice, reply)
|
||||
}
|
||||
|
||||
// 4d-ii. and the same correction without a target — "нет, не так" (V-636).
|
||||
// After the targeted one, which is the narrower claim: an utterance that
|
||||
// names an intent is answered by redoing the request, and this rung only
|
||||
// gets the ones that name nothing.
|
||||
if reply, handled := h.resolveUntargetedRepair(ctx, text); notePreRoute(ctx, "repair-negative", handled) {
|
||||
return withNotice(expiredNotice, reply)
|
||||
}
|
||||
|
||||
// 4e. ordinal selection — "второй", "первую сделал" pick from the list she
|
||||
// 5d. ordinal selection — "второй", "первую сделал" pick from the list she
|
||||
// just read (ordinal.go). Before routing, and only when a list is actually
|
||||
// bound to the session: with nothing offered, "второй" is an ordinary word
|
||||
// and keeps routing.
|
||||
@@ -381,7 +380,7 @@ func (h *reactiveHandler) runTurn(ctx context.Context, text string, src turnSour
|
||||
return withNotice(expiredNotice, reply)
|
||||
}
|
||||
|
||||
// 5. route. An elliptical follow-up — "а завтра?" — is answered from the
|
||||
// 6. route. An elliptical follow-up — "а завтра?" — is answered from the
|
||||
// previous turn instead (continuation.go): the intent is the part it is
|
||||
// missing, so no amount of routing recovers it, and the model's guess
|
||||
// costs seconds to obtain and is close to a coin flip. Everything else
|
||||
@@ -398,7 +397,7 @@ func (h *reactiveHandler) runTurn(ctx context.Context, text string, src turnSour
|
||||
}
|
||||
log.Printf("voice: route result: intent=%s slots=%+v", dec.Intent, dec.Slots)
|
||||
|
||||
// 6. dialogue — fill this turn's missing slots from a prior same-intent
|
||||
// 7. dialogue — fill this turn's missing slots from a prior same-intent
|
||||
// turn (follow-ups like «напомни завтра» → «…позвонить маме»), then remember
|
||||
// this turn for the next follow-up. Only same-intent, non-expired, non-
|
||||
// clarify turns carry (see followUpMerge). Best-effort: nil store ⇒ skipped.
|
||||
@@ -416,7 +415,7 @@ func (h *reactiveHandler) runTurn(ctx context.Context, text string, src turnSour
|
||||
}
|
||||
}
|
||||
|
||||
// 7. clarify — something she needs is missing. If one named thing is missing,
|
||||
// 8. clarify — something she needs is missing. If one named thing is missing,
|
||||
// ask about it and park the request (clarify.go); otherwise the replier's
|
||||
// canned reply stands.
|
||||
//
|
||||
@@ -443,7 +442,7 @@ func (h *reactiveHandler) runTurn(ctx context.Context, text string, src turnSour
|
||||
h.recordTurn(text, dec.Intent)
|
||||
}
|
||||
|
||||
// 8. action — execute the decision's intent. errors here surface as
|
||||
// 9. action — execute the decision's intent. errors here surface as
|
||||
// short reply text (the user wants to know the action didn't land);
|
||||
// the round-trip stays alive.
|
||||
replyText := h.applyAction(ctx, dec)
|
||||
@@ -452,11 +451,11 @@ func (h *reactiveHandler) runTurn(ctx context.Context, text string, src turnSour
|
||||
// intent has no chain and no scoreboard, so the handler is the winner.
|
||||
noteTerminal(ctx, "action-handler", dec.Intent, "")
|
||||
|
||||
// 8b. a fact that answers a live nudge closes it as `acted` (ack.go).
|
||||
// 9b. a fact that answers a live nudge closes it as `acted` (ack.go).
|
||||
// Silent: the fact reply stands, she does not congratulate him for it.
|
||||
h.ackFromFact(ctx, dec)
|
||||
|
||||
// 9. replier — phrase the reply across the router decision.
|
||||
// 10. replier — phrase the reply across the router decision.
|
||||
if replyText == "" {
|
||||
replyText = h.replier.Reply(ctx, dec)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user