diff --git a/cmd/mavend/actions_query.go b/cmd/mavend/actions_query.go index a0aaaab..6c3f4e6 100644 --- a/cmd/mavend/actions_query.go +++ b/cmd/mavend/actions_query.go @@ -641,10 +641,11 @@ const ( // небо синее?", because the right-note and must-be-silent score ranges overlap // and no threshold sits between them. func recallOnTopic(utterance, text string) bool { - if memory.RecallAllowed(utterance, text) { + if memory.RecallAllowed(utterance, text, + router.IsOpenQuestionShaped(utterance), router.IsLocativeQuestionShaped(utterance)) { return true } - log.Printf("voice: recall %q rejected for %q: a world question and no shared topic word", text, utterance) + log.Printf("voice: recall %q rejected for %q: no structural ask with a shared topic, or a world/locative question with no shared topic", text, utterance) return false } diff --git a/cmd/mavend/query_recall_test.go b/cmd/mavend/query_recall_test.go index 21e3313..2297653 100644 --- a/cmd/mavend/query_recall_test.go +++ b/cmd/mavend/query_recall_test.go @@ -193,9 +193,9 @@ func TestQueryRecallNoteCanWin(t *testing.T) { t.Run("the better-matching fact answers", func(t *testing.T) { h, _ := buildRecallHandler(t, q, []recallCase{ {text: "молоко стоит в холодильнике", score: 0.80, kind: "note"}, - {text: "купил молоко в среду", score: 0.95, kind: "fact"}, + {text: "молоко было в холодильнике в среду", score: 0.95, kind: "fact"}, }) - if reply := askQuery(t, h, q); reply != "купил молоко в среду" { + if reply := askQuery(t, h, q); reply != "молоко было в холодильнике в среду" { t.Errorf("reply %q, want the fact read back", reply) } }) @@ -212,3 +212,63 @@ func TestQueryRecallNoteCanWin(t *testing.T) { } }) } + +// TestQueryRecallRequiresStructuralOrTopicEvidence — the whole-assistant +// cold-start regression. The routing heads called an ordinary past-tense +// report a query; with one note in the store the margin gate has no runner-up, +// and cosine 0.825 was enough to speak a completely unrelated spare-key note. +// A bare question mark does not turn the proposition into an open information +// question, negation must not weaken the refusal, and a locative question must +// corroborate the target it asks Maven to locate (V-719). +func TestQueryRecallRequiresStructuralOrTopicEvidence(t *testing.T) { + const unrelated = "запомни: запасной ключ лежит в синей коробке" + for _, tc := range []struct { + query string + score float64 + }{ + {"я отменил напоминание про молоко", 0.825031306}, + {"я отменил напоминание про молоко?", 0.825031306}, + {"я не отменил напоминание про молоко", 0.825031306}, + {"я не отменил напоминание про молоко?", 0.825031306}, + {"где мой паспорт?", 0.817210}, + {"где я отменил напоминание про молоко?", 0.805800}, + {"где лежит синяя рубашка?", 0.837694}, + {"где лежит синяя папка?", 0.837472}, + {"где мой запасной паспорт?", 0.831662}, + {"где лежит запасная флешка?", 0.838980}, + {"где находится синяя коробка с документами?", 0.866553}, + {"где лежит ключ от машины?", 0.843853}, + {"где синяя коробка?", 0.90}, + } { + t.Run(tc.query, func(t *testing.T) { + h, phr := buildRecallHandler(t, tc.query, []recallCase{ + {text: unrelated, score: tc.score, kind: "note"}, + }) + reply := askQuery(t, h, tc.query) + if strings.Contains(reply, "запасной ключ") { + t.Fatalf("unrelated note escaped into reply %q", reply) + } + if len(phr.notes) != 0 { + t.Fatalf("unrelated note reached the phraser: %q", phr.notes) + } + }) + } + + // Voice punctuation is optional. A nominal request with no interrogative + // still works when the candidate itself corroborates the named topic. + const nominal = "адрес домашнего сервера" + h, _ := buildRecallHandler(t, nominal, []recallCase{ + {text: "домашний сервер на 192.168.1.104", score: 0.90, kind: "note"}, + }) + if reply := askQuery(t, h, nominal); !strings.Contains(reply, "домашний сервер") { + t.Fatalf("nominal recall lost its shared-topic answer: %q", reply) + } + + const locative = "где лежит запасной ключ?" + h, _ = buildRecallHandler(t, locative, []recallCase{ + {text: "запасной ключ лежит в синей коробке", score: 0.90, kind: "note"}, + }) + if reply := askQuery(t, h, locative); !strings.Contains(reply, "запасной ключ") { + t.Fatalf("locative recall lost its corroborated target: %q", reply) + } +} diff --git a/cmd/mavend/reactive_notes_test.go b/cmd/mavend/reactive_notes_test.go index 0449b88..61cf5c4 100644 --- a/cmd/mavend/reactive_notes_test.go +++ b/cmd/mavend/reactive_notes_test.go @@ -8,6 +8,7 @@ import ( "github.com/kami/maven/internal/ipc" "github.com/kami/maven/internal/memory" + "github.com/kami/maven/internal/phraser" "github.com/kami/maven/internal/router" "github.com/kami/maven/internal/store" "github.com/kami/maven/internal/tool" @@ -84,12 +85,76 @@ func TestReactiveNotesReminders(t *testing.T) { t.Fatal("expected at least one note, got none") } last := notes[0] - if last.Text != "запомни что кофе закончился" { - t.Errorf("note text = %q, want %q", last.Text, "запомни что кофе закончился") + if last.Text != "кофе закончился" { + t.Errorf("note text = %q, want %q", last.Text, "кофе закончился") } }) } +// TestRunTurnExplicitNoteStoresOnlyTheBody pins the live failure end to end: +// a routed text turn reaches actionNote, stores only the dictated body in both +// durable and vector memory, and cannot ask the resident model to choose the +// acknowledgement's grammatical gender (V-721). +func TestRunTurnExplicitNoteStoresOnlyTheBody(t *testing.T) { + ctx := context.Background() + st := newTestStore(t) + api := ipc.NewStoreAPI(st) + now := time.Date(2026, 8, 15, 8, 0, 0, 0, time.FixedZone("+04", 4*60*60)) + emb := router.NewHashEmbedder(1024) + mem := memory.NewInMemoryStore() + rtr := router.New(router.Config{ + Grammars: []router.Grammar{{ + Name: "explicit-note-test", + Decide: func(string) (router.Decision, bool) { + return router.Decision{ + Stage: 0, Intent: router.IntentNote, Confidence: 1, + // Deliberately hostile model slot: neither persistence nor + // acknowledgement may use it. + Slots: router.Slots{Text: "ты поедешь на дачу"}, + }, true + }, + }}, + Threshold: 0.55, + }) + model := &countingCompleter{out: `{"response":"Хорошо, сохранил.","mood":"neutral"}`} + h := &reactiveHandler{ + api: api, router: rtr, + recall: recallWiring{embedder: emb, memStore: mem}, + replier: newLLMReplier(model, nil), + now: func() time.Time { return now }, + dataStore: st, + } + + const utterance = "запомни: запасной ключ лежит в синей коробке" + if reply := h.runTurn(ctx, utterance, sourceText); reply != "сохранила заметку." { + t.Fatalf("reply = %q, want the fixed feminine acknowledgement", reply) + } + if model.calls != 0 { + t.Fatalf("resident model was called %d time(s) for a note acknowledgement", model.calls) + } + notes, err := st.RecentNotes(ctx, 10) + if err != nil { + t.Fatalf("RecentNotes: %v", err) + } + const body = "запасной ключ лежит в синей коробке" + if len(notes) != 1 || notes[0].Text != body || notes[0].Source != "tap:voice" || !notes[0].Ts.Equal(now) { + t.Fatalf("stored notes = %+v, want one exact body at the turn time", notes) + } + records, err := mem.ByPrefix(ctx, "note:") + if err != nil { + t.Fatalf("vector catalog: %v", err) + } + if len(records) != 1 || records[0].Meta["text"] != body { + t.Fatalf("vector records = %+v, want the same extracted body", records) + } + if records[0].Meta["text"] == utterance || records[0].Meta["text"] == "ты поедешь на дачу" { + t.Fatalf("vector metadata used a command or model rewrite: %+v", records[0].Meta) + } + if !phraser.IsAck(phraser.AckNote, nil, "сохранила заметку.") { + t.Fatal("fixed acknowledgement is not registered as the note acknowledgement") + } +} + // TestSpokenTaskCaptureFilesATask — the whole path, from the utterance to the // task table. It went dead when the router started claiming the marker as an // act: capture rides the note intent, so nothing below actionNote was ever diff --git a/cmd/mavend/recall.go b/cmd/mavend/recall.go index d8621cc..70cafdf 100644 --- a/cmd/mavend/recall.go +++ b/cmd/mavend/recall.go @@ -56,7 +56,7 @@ type recallWiring struct { // minScore — the note-recall confidence gate. Top cosine below this ⇒ // "I don't know" instead of a guess. Tuned for the ONNX embedder; a knob, // not load-bearing math (same posture as the presence thresholds). Set by - // wireVoice from VoiceConfig; default 0.55. + // wireVoice from VoiceConfig; default 0.80. minScore float64 // minMargin — the second half of that gate: how far the top hit must beat diff --git a/deploy/mavend.json b/deploy/mavend.json index 8262054..fe9929b 100644 --- a/deploy/mavend.json +++ b/deploy/mavend.json @@ -259,7 +259,7 @@ "heads_path": "/opt/maven/models/embedder/router-heads/router_heads.onnx" }, "llm_router": true, - "query_min_score": 0.55, + "query_min_score": 0.80, "query_min_margin": 0.008, "clarify_max_attempts": 3, "tool_timeout": "30s", diff --git a/internal/config/deployconfig_test.go b/internal/config/deployconfig_test.go index 44416f7..dcdfb7d 100644 --- a/internal/config/deployconfig_test.go +++ b/internal/config/deployconfig_test.go @@ -52,6 +52,14 @@ func TestDeployConfigLoads(t *testing.T) { if cfg.Voice.RouterThreshold <= 0 { t.Error("router threshold did not get its default") } + // The recall thresholds are a measured pair. An explicit deployment value + // silently overriding a retuned default would make the eval and the box run + // different safety gates, so pin both directions here. + if cfg.Voice.QueryMinScore != DefaultQueryMinScore || cfg.Voice.QueryMinMargin != DefaultQueryMinMargin { + t.Errorf("deploy recall gate is %.3f/%.3f, defaults are %.3f/%.3f", + cfg.Voice.QueryMinScore, cfg.Voice.QueryMinMargin, + DefaultQueryMinScore, DefaultQueryMinMargin) + } // The second reach (V-649). The token is a ${VAR} that CI cannot resolve, so // the committed deployment makes the dark state explicit. Removing disabled diff --git a/internal/config/voice.go b/internal/config/voice.go index 4f3b568..d19e4bb 100644 --- a/internal/config/voice.go +++ b/internal/config/voice.go @@ -77,11 +77,15 @@ type ToolConfig struct { // Voice defaults, applied in normaliseVoice. const ( DefaultRouterThreshold = 0.55 - DefaultQueryMinScore = 0.55 - // Read off the margin sweep in internal/memory/recalleval on the e5 - // embedder: 0.008 answers 68% of real questions (down from 72%) and cuts - // false recall from 5/5 to 1/5. Every larger delta costs real recall - // without removing that last one until 0.020, which drops recall to 44%. + // The absolute half of the recall gate, recalibrated after tokenizer rev 2 + // changed the e5 score distribution. At 0.80 with the independent 0.008 + // margin and structural eligibility, the held-out fixture answers 14/27 + // real recalls and 0/14 false ones. The score distributions still overlap + // (true minimum 0.817, silent maximum 0.874), so structure remains decisive. + DefaultQueryMinScore = 0.80 + // The runner-up half is intentionally separate. Keep 0.008 as the ambiguity + // floor measured before the structural gate; a single-hit store has no + // runner-up, which is why score and structure are both required. DefaultQueryMinMargin = 0.008 // DefaultClarifyMaxAttempts — see dialogue.DefaultMaxAttempts. DefaultClarifyMaxAttempts = 3 @@ -258,8 +262,8 @@ type VoiceConfig struct { LLMRouter *bool `json:"llm_router,omitempty"` // QueryMinScore — the note-recall confidence gate. Top cosine below this - // ⇒ "I don't know" instead of a guess. Tuned for the ONNX embedder (0.55); - // the HashEmbedder floor scores lexically and may never clear it. 0.55 + // ⇒ "I don't know" instead of a guess. Tuned for the corrected ONNX + // embedder (0.80); the HashEmbedder floor scores lexically and may never clear it. 0.80 // default if unset. QueryMinScore float64 `json:"query_min_score,omitempty"` diff --git a/internal/config/voice_test.go b/internal/config/voice_test.go index 86c274f..89ad4fd 100644 --- a/internal/config/voice_test.go +++ b/internal/config/voice_test.go @@ -68,3 +68,16 @@ func voiceConfigWith(t *testing.T, model, heads string) error { _, err := Load(writeConfig(t, body)) return err } + +func TestVoiceRecallGateDefaults(t *testing.T) { + cfg, err := Load(writeConfig(t, `{"voice":{"enabled":true,"bind":"127.0.0.1:9100"}}`)) + if err != nil { + t.Fatalf("Load: %v", err) + } + if got := cfg.Voice.QueryMinScore; got != DefaultQueryMinScore { + t.Errorf("query score default = %.3f, want %.3f", got, DefaultQueryMinScore) + } + if got := cfg.Voice.QueryMinMargin; got != DefaultQueryMinMargin { + t.Errorf("query margin default = %.3f, want %.3f", got, DefaultQueryMinMargin) + } +} diff --git a/internal/memory/lexical.go b/internal/memory/lexical.go index fa47e8d..6566d49 100644 --- a/internal/memory/lexical.go +++ b/internal/memory/lexical.go @@ -3,8 +3,19 @@ package memory import ( "strings" "unicode" + + "github.com/kami/maven/internal/lexicon" + "github.com/kami/maven/internal/morph" ) +var recallCaptureVerbs = func() map[string]bool { + out := map[string]bool{} + for _, w := range lexicon.CaptureVerbs() { + out[w] = true + } + return out +}() + // stopwords — words that carry no topic. A question and a note that share only // these share nothing: "почему небо синее" and "сеть какая-то медленная" both // contain "какая"-shaped filler and are about different worlds. @@ -49,18 +60,35 @@ var firstPerson = map[string]bool{ // RecallAllowed is the second half of the recall gate (#470). A hit that // cleared the score and margin gate may still be about something else -// entirely: the held-out fixture puts the right note at 0.791-0.890 and the -// must-be-silent cases at 0.795-0.835, so no threshold sits between them, and +// entirely: the held-out fixture puts the right note at 0.817-0.892 and the +// must-be-silent cases at 0.787-0.874, so no threshold sits between them, and // a note about his slow network answered "почему небо синее?". // -// The veto applies only to a question that mentions nothing of his. That +// The broad veto applies only to a question that mentions nothing of his. That // restriction is what keeps the fix from costing more than it saves: recall // exists to find the note whose words he no longer remembers, and demanding a -// shared word of every recall silenced four true recalls on the fixture to -// kill one false one. A question about his own life keeps the embedder alone +// shared word of every recall silences several right-note paraphrases on the +// fixture. A question about his own life keeps the embedder alone // as its judge. A question about the world has to name something the memory // actually mentions. // +// openQuestion is the caller's structural evidence that the words ask for +// information: an interrogative or narrative request, not punctuation alone. +// It closes a different hole: a model can mis-route an ordinary first-person +// report as a query. With only one stored note there is no runner-up for the +// margin gate, so "я отменил напоминание про молоко" recalled an unrelated note +// at cosine 0.825. A report or polar proposition therefore needs a named topic +// shared with the hit, even when it ends in '?'. Nominal requests still work — +// "адрес домашнего сервера" shares its topic — while a statistical route alone +// cannot make an unrelated personal statement into a recall request. +// +// requireNamedTopic is the stricter locative frame: an answer to "where is X" +// must corroborate every identity term in X. One overlapping modifier or +// predicate is not enough: the spare-key note scores 0.832-0.867 for a spare +// passport, blue shirt, blue document box, and car key, while sharing words +// such as "spare", "blue", "box", or "key" (V-719). Verbs are grammar, not +// identity, and are excluded using the embedded morphology dictionary. +// // The veto's price was re-measured on 2026-08-03 (#496, // docs/evals/2026-08-03-recall-topic-veto.md). It costs one true recall and // buys one false one, and the fixture pass count is the same either way. The @@ -68,11 +96,89 @@ var firstPerson = map[string]bool{ // reported as, and the fixture has no cross-language case at all. Do not add a // script test or a bilingual stem map for it — both are no-ops here. The // separating signal is semantic and belongs in a reranker, not in this file. -func RecallAllowed(query, text string) bool { +func RecallAllowed(query, text string, openQuestion, requireNamedTopic bool) bool { + shared := sharesNamedTopic(query, text) + if requireNamedTopic && !corroboratesNamedIdentity(query, text) { + return false + } + if !openQuestion && !shared { + return false + } if mentionsHim(query) { return true } - return SharesContentWord(query, text) + return shared || len(contentWords(query)) == 0 +} + +// corroboratesNamedIdentity requires every non-frame, non-verb query term to +// occur in the candidate proposition's subject as the same dictionary word. +// Searching the whole candidate is unsafe: in "the spare key lies in the blue +// box", the box is a location, not the thing being located, and cannot answer +// "where is the blue box?". Locative predicates are deliberately excluded, so +// two memories do not become the same target merely because both things "lie" +// somewhere. Requiring all remaining terms preserves qualifiers too: a key is +// not a car key and a box is not a box of documents. +func corroboratesNamedIdentity(query, text string) bool { + q := identityWords(query) + if len(q) == 0 { + return false + } + t := propositionSubjectIdentity(text) + if len(t) == 0 { + return false + } + for _, want := range q { + found := false + for _, got := range t { + if want == got || morph.SameWord(want, got) { + found = true + break + } + } + if !found { + return false + } + } + return true +} + +// propositionSubjectIdentity returns the identity phrase before the first +// dictionary-proven main verb. A leading capture imperative is storage frame, +// not the proposition: "remember: the spare key lies ..." has "spare key" as +// its subject. If no subject/predicate boundary can be proved, it returns nil; +// strict locative recall then abstains instead of treating a location object or +// incidental modifier anywhere in the note as the requested entity. +func propositionSubjectIdentity(s string) []string { + toks := wordTokens(s) + start := 0 + for start < len(toks) && recallCaptureVerbs[toks[start]] { + start++ + } + predicate := -1 + for i := start; i < len(toks); i++ { + if morph.IsVerbForm(toks[i]) { + predicate = i + break + } + } + if predicate <= start { + return nil + } + return identityTokens(toks[start:predicate]) +} + +func identityWords(s string) []string { + return identityTokens(contentWords(s)) +} + +func identityTokens(words []string) []string { + out := make([]string, 0, len(words)) + for _, w := range words { + if !stopwords[w] && !morph.IsVerbForm(w) { + out = append(out, w) + } + } + return out } func mentionsHim(query string) bool { @@ -96,7 +202,21 @@ func SharesContentWord(query, text string) bool { // gate is then the only judge it can have. return true } - t := contentWords(text) + return sharesContentWords(q, contentWords(text)) +} + +// sharesNamedTopic is the stricter form used for a non-question-shaped turn: +// an utterance made entirely of frame words names no topic, so it cannot use +// the score gate as its only evidence that a stored note should be spoken. +func sharesNamedTopic(query, text string) bool { + q := contentWords(query) + if len(q) == 0 { + return false + } + return sharesContentWords(q, contentWords(text)) +} + +func sharesContentWords(q, t []string) bool { for _, a := range q { for _, b := range t { if a == b || sameStem(a, b) { @@ -109,9 +229,7 @@ func SharesContentWord(query, text string) bool { func contentWords(s string) []string { var out []string - for _, w := range strings.FieldsFunc(strings.ToLower(s), func(r rune) bool { - return !unicode.IsLetter(r) && !unicode.IsDigit(r) - }) { + for _, w := range wordTokens(s) { if !stopwords[w] { out = append(out, w) } @@ -119,6 +237,12 @@ func contentWords(s string) []string { return out } +func wordTokens(s string) []string { + return strings.FieldsFunc(strings.ToLower(s), func(r rune) bool { + return !unicode.IsLetter(r) && !unicode.IsDigit(r) + }) +} + // sameStem is inflection and derivation tolerance: Russian marks case and // tense on the ending, and the note and the question rarely use the same form. // "воду" and "вода" are the same water, "кормить" and "корм" the same feeding. diff --git a/internal/memory/lexical_test.go b/internal/memory/lexical_test.go index c6f805c..d17aa9b 100644 --- a/internal/memory/lexical_test.go +++ b/internal/memory/lexical_test.go @@ -4,29 +4,58 @@ import "testing" func TestRecallAllowed(t *testing.T) { cases := []struct { - name string - query, text string - want bool + name string + query, text string + openQuestion, requireNamedTopic bool + want bool }{ // The #470 shape: a world question and a note about his box. - {"world question, unrelated note", "почему небо синее", "сеть какая-то медленная", false}, - {"world question, unrelated fact", "какая столица Франции", "какая последняя версия языка Go", false}, - {"silent fixture case", "во сколько отходит поезд", "бэкап запускается в три ночи", false}, + {"world question, unrelated note", "почему небо синее", "сеть какая-то медленная", true, false, false}, + {"world question, unrelated fact", "какая столица Франции", "какая последняя версия языка Go", true, false, false}, + {"silent fixture case", "во сколько отходит поезд", "бэкап запускается в три ночи", true, false, false}, // A world question that does name the topic keeps its answer. - {"world question, same topic", "какой поезд идёт в Минск", "поезда в Минск ходят утром", true}, + {"world question, same topic", "какой поезд идёт в Минск", "поезда в Минск ходят утром", true, false, true}, // A question about his own life is judged by the embedder alone, // because recall exists for words he no longer remembers. - {"about him, no shared word", "во сколько я обычно засыпаю", "ложусь около одиннадцати", true}, - {"about him, english", "which colour scheme do i like", "тёмная тема везде", true}, + {"about him, no shared word", "во сколько я обычно засыпаю", "ложусь около одиннадцати", true, false, true}, + {"about him, english", "which colour scheme do i like", "тёмная тема везде", true, false, true}, + + // A locative asks for the named object's or event's location. Even a + // personal question must corroborate every identity term in that target; + // an incidental adjective or noun is insufficient (V-719). + {"locative, unrelated high hit", "где мой паспорт?", "запомни: запасной ключ лежит в синей коробке", true, true, false}, + {"locative event, unrelated high hit", "где я отменил напоминание про молоко?", "запомни: запасной ключ лежит в синей коробке", true, true, false}, + {"locative, shared adjective only", "где лежит синяя рубашка?", "запасной ключ лежит в синей коробке", true, true, false}, + {"locative, shared modifier only", "где мой запасной паспорт?", "запасной ключ лежит в синей коробке", true, true, false}, + {"locative, shared head missing qualifier", "где лежит ключ от машины?", "запасной ключ лежит в синей коробке", true, true, false}, + {"locative, shared phrase missing complement", "где находится синяя коробка с документами?", "запасной ключ лежит в синей коробке", true, true, false}, + {"locative, location object is not subject", "где синяя коробка?", "запасной ключ лежит в синей коробке", true, true, false}, + {"locative, shared target", "где лежит запасной ключ?", "запасной ключ лежит в синей коробке", true, true, true}, + {"locative, capture prefix is not subject", "где лежит запасной ключ?", "запомни: запасной ключ лежит в синей коробке", true, true, true}, // Inflection must not break a match. - {"inflected", "чем кормить кота", "корм для кота в шкафу", true}, + {"inflected", "чем кормить кота", "корм для кота в шкафу", true, false, true}, + + // A model-routed query without structural question evidence must still + // name the hit's topic. This is the cold-start false recall seen in the + // whole-assistant E2E, and its negated neighbour. + {"declarative report is not a recall", "я отменил напоминание про молоко", "запомни: запасной ключ лежит в синей коробке", false, false, false}, + {"polar punctuation does not waive the topic", "я отменил напоминание про молоко?", "запомни: запасной ключ лежит в синей коробке", false, false, false}, + {"negated declarative report is not a recall", "я не отменил напоминание про молоко", "запомни: запасной ключ лежит в синей коробке", false, false, false}, + {"negated polar report is not a recall", "я не отменил напоминание про молоко?", "запомни: запасной ключ лежит в синей коробке", false, false, false}, + + // Question marks are optional in voice transcripts. A nominal request + // and an intonational personal query remain eligible when their stored + // answer corroborates the named topic. + {"nominal request with a topic", "адрес домашнего сервера", "домашний сервер на 192.168.1.104", false, false, true}, + {"intonational query with a topic", "я сегодня вообще пил воду", "выпил стакан воды утром", false, false, true}, } for _, c := range cases { - if got := RecallAllowed(c.query, c.text); got != c.want { - t.Errorf("%s: RecallAllowed(%q, %q) = %v, want %v", c.name, c.query, c.text, got, c.want) + if got := RecallAllowed(c.query, c.text, c.openQuestion, c.requireNamedTopic); got != c.want { + t.Errorf("%s: RecallAllowed(%q, %q, open-question=%t, require-topic=%t) = %v, want %v", + c.name, c.query, c.text, c.openQuestion, c.requireNamedTopic, got, c.want) } } } @@ -39,10 +68,10 @@ func TestRecallAllowed(t *testing.T) { // which puts false recall back to 1/5. Anyone loosening the veto has to move // the first line without moving the second. func TestRecallVetoTradeIsPinned(t *testing.T) { - if RecallAllowed("what fixed the screen problem", "the flicker went away once i swapped the display cable") { + if RecallAllowed("what fixed the screen problem", "the flicker went away once i swapped the display cable", true, false) { t.Error("en-hard-024 is expected to stay vetoed — if this passes now, re-measure false recall before celebrating") } - if RecallAllowed("во сколько отходит поезд", "погулял вдоль реки") { + if RecallAllowed("во сколько отходит поезд", "погулял вдоль реки", true, false) { t.Error("ru-silent-029 must stay vetoed — this is the false recall the veto exists to stop") } } @@ -50,7 +79,7 @@ func TestRecallVetoTradeIsPinned(t *testing.T) { // A question made only of filler has no topic word to match on, and the score // gate is then the only judge it can have. func TestRecallAllowedFallsBackWhenNothingToCompare(t *testing.T) { - if !RecallAllowed("что это", "сеть какая-то медленная") { + if !RecallAllowed("что это", "сеть какая-то медленная", true, false) { t.Error("a question with no content word must not be vetoed") } } diff --git a/internal/memory/locativeverify.go b/internal/memory/locativeverify.go new file mode 100644 index 0000000..9bf4eb0 --- /dev/null +++ b/internal/memory/locativeverify.go @@ -0,0 +1,172 @@ +package memory + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "strings" + "time" + "unicode/utf8" + + "github.com/kami/maven/internal/llm" +) + +// LocativeCompleter is the narrow resident-model seam the verifier needs. +// *llm.Client satisfies it. Keeping the interface here lets the safety and +// malformed-output paths run without a server in unit tests. +type LocativeCompleter interface { + Complete(context.Context, llm.Req) (string, error) +} + +// LocativeAnswerVerifier is the model-backed second opinion for candidates the +// deterministic locative identity gate rejected. It never replaces that gate: +// exact structural accepts do not call it, and a missing model, timeout, error, +// or malformed verdict remains an abstention in the daemon. +type LocativeAnswerVerifier struct { + c LocativeCompleter +} + +// LocativeVerdict keeps the two extracted referents as auditable evidence. +// The daemon consumes only Answerable; the live eval records all three fields +// so a yes/no score cannot hide what the model thought it was comparing. +type LocativeVerdict struct { + Target string `json:"target"` + MemorySubject string `json:"memory_subject"` + Answerable bool `json:"-"` + Raw string `json:"-"` +} + +// NewLocativeAnswerVerifier returns nil when no resident completion seam exists. +// That is the ordinary no-model deployment and deliberately means abstain. +func NewLocativeAnswerVerifier(c LocativeCompleter) *LocativeAnswerVerifier { + if c == nil { + return nil + } + return &LocativeAnswerVerifier{c: c} +} + +// locativeVerifierGrammar fixes both the shape and every variable-width field. +// target and memory_subject come before answer deliberately: the small resident +// model must identify the two referents before choosing the verdict instead of +// emitting an unconstrained first-token hunch. +const locativeVerifierGrammar = ` +root ::= "{" ws "\"target\"" ws ":" ws string "," ws "\"memory_subject\"" ws ":" ws string "," ws "\"answer\"" ws ":" ws answer ws "}" +answer ::= "\"no\"" | "\"yes\"" +string ::= "\"" ([^"\\\x00-\x1F] | "\\" ["\\/bfnrt] | "\\u" [0-9a-fA-F]{4}){1,80} "\"" +ws ::= [ \t\n]{0,2} +` + +// locativeVerifierSystem teaches a relation, not the held-out fixture. The +// examples use different entities from the measured nginx/token/disk/key and +// adversarial passport/shirt/box cases. Two negative examples pin the dangerous +// distinctions: a location object is not the proposition subject, and a shared +// generic noun with a conflicting complement is not the same referent. +const locativeVerifierSystem = `Ты — строгий классификатор логического следования для личной памяти. Вход — JSON с одним question и одним memory. +Сначала выдели target: конкретный предмет/событие, чьё место или источник спрашивают. Затем memory_subject: предмет/событие, МЕСТО КОТОРОГО сообщает память. Предмет после слов места (в, на, под, рядом с, inside, at, under) — это место/контейнер, а НЕ memory_subject. +answer=yes только если target и memory_subject — один и тот же конкретный референт и память прямо сообщает запрошенное место/источник. Настоящие синонимы и контекстные названия допустимы. Совпадение цвета, свойства, общего слова, контейнера, места или действия недостаточно. Уточнения принадлежности/состава не должны конфликтовать. Не используй внешние знания. Сомнение => no. + +Примеры: +input: {"question":"где красная тетрадь?","memory":"зарядка лежит на красной тетради"} +output: {"target":"красная тетрадь","memory_subject":"зарядка","answer":"no"} +input: {"question":"где ключ от гаража?","memory":"ключ от офиса лежит под ковриком"} +output: {"target":"ключ от гаража","memory_subject":"ключ от офиса","answer":"no"} +input: {"question":"где дубликат ключа от мастерской?","memory":"запасной ключ мастерской лежит в ящике"} +output: {"target":"дубликат ключа от мастерской","memory_subject":"запасной ключ мастерской","answer":"yes"} +input: {"question":"where are the database settings?","memory":"the database configuration is in /etc/db"} +output: {"target":"database settings","memory_subject":"database configuration","answer":"yes"} + +Верни только JSON требуемой формы.` + +const ( + locativeVerifierTimeout = 8 * time.Second + locativeVerifierMaxTokens = 128 + locativeVerifierMaxField = 80 +) + +// Answerable implements the daemon's deliberately tiny verifier interface. +func (v *LocativeAnswerVerifier) Answerable(ctx context.Context, question, candidate string) (bool, error) { + verdict, err := v.Evaluate(ctx, question, candidate) + return verdict.Answerable, err +} + +// Evaluate returns the bounded model verdict and its extracted referents. +// Callers must treat every error as false; it never manufactures a fallback. +func (v *LocativeAnswerVerifier) Evaluate(ctx context.Context, question, candidate string) (LocativeVerdict, error) { + if v == nil || v.c == nil { + return LocativeVerdict{}, errors.New("locative verifier: resident model unavailable") + } + input, err := json.Marshal(struct { + Question string `json:"question"` + Memory string `json:"memory"` + }{Question: question, Memory: candidate}) + if err != nil { + return LocativeVerdict{}, fmt.Errorf("locative verifier: encode input: %w", err) + } + + callCtx, cancel := context.WithTimeout(ctx, locativeVerifierTimeout) + defer cancel() + raw, err := v.c.Complete(callCtx, llm.Req{ + System: locativeVerifierSystem, + User: string(input), + Grammar: locativeVerifierGrammar, + MaxTokens: locativeVerifierMaxTokens, + RepeatPenalty: 1.1, + }) + if err != nil { + return LocativeVerdict{}, fmt.Errorf("locative verifier: complete: %w", err) + } + return parseLocativeVerdict(raw) +} + +func parseLocativeVerdict(raw string) (LocativeVerdict, error) { + trimmed := strings.TrimSpace(raw) + var wire struct { + Target string `json:"target"` + MemorySubject string `json:"memory_subject"` + Answer string `json:"answer"` + } + dec := json.NewDecoder(strings.NewReader(trimmed)) + dec.DisallowUnknownFields() + if err := dec.Decode(&wire); err != nil { + return LocativeVerdict{}, fmt.Errorf("locative verifier: parse %q: %w", boundedRaw(trimmed), err) + } + var trailing any + if err := dec.Decode(&trailing); !errors.Is(err, io.EOF) { + if err == nil { + return LocativeVerdict{}, fmt.Errorf("locative verifier: trailing JSON in %q", boundedRaw(trimmed)) + } + return LocativeVerdict{}, fmt.Errorf("locative verifier: trailing data in %q: %w", boundedRaw(trimmed), err) + } + wire.Target = strings.TrimSpace(wire.Target) + wire.MemorySubject = strings.TrimSpace(wire.MemorySubject) + if wire.Target == "" || wire.MemorySubject == "" || + utf8.RuneCountInString(wire.Target) > locativeVerifierMaxField || + utf8.RuneCountInString(wire.MemorySubject) > locativeVerifierMaxField { + return LocativeVerdict{}, fmt.Errorf("locative verifier: empty or oversized referent in %q", boundedRaw(trimmed)) + } + var answerable bool + switch wire.Answer { + case "yes": + answerable = true + case "no": + answerable = false + default: + return LocativeVerdict{}, fmt.Errorf("locative verifier: invalid answer %q", wire.Answer) + } + return LocativeVerdict{ + Target: wire.Target, + MemorySubject: wire.MemorySubject, + Answerable: answerable, + Raw: trimmed, + }, nil +} + +func boundedRaw(s string) string { + const max = 240 + if len(s) <= max { + return s + } + return s[:max] + "…" +} diff --git a/internal/memory/locativeverify_test.go b/internal/memory/locativeverify_test.go new file mode 100644 index 0000000..4b32229 --- /dev/null +++ b/internal/memory/locativeverify_test.go @@ -0,0 +1,106 @@ +package memory + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/kami/maven/internal/llm" +) + +type locativeCompleteFunc func(context.Context, llm.Req) (string, error) + +func (f locativeCompleteFunc) Complete(ctx context.Context, req llm.Req) (string, error) { + return f(ctx, req) +} + +func TestLocativeAnswerVerifierRequestAndVerdict(t *testing.T) { + var got llm.Req + v := NewLocativeAnswerVerifier(locativeCompleteFunc(func(_ context.Context, req llm.Req) (string, error) { + got = req + return `{"target":"настройки nginx","memory_subject":"конфиг nginx","answer":"yes"}`, nil + })) + verdict, err := v.Evaluate(context.Background(), "где настройки nginx?", "конфиг nginx лежит в /etc/nginx") + if err != nil { + t.Fatalf("Evaluate: %v", err) + } + if !verdict.Answerable || verdict.Target != "настройки nginx" || verdict.MemorySubject != "конфиг nginx" { + t.Fatalf("verdict = %+v", verdict) + } + if got.Grammar != locativeVerifierGrammar || got.MaxTokens != locativeVerifierMaxTokens || got.RepeatPenalty != 1.1 { + t.Fatalf("request bounds drifted: %+v", got) + } + var input map[string]string + if err := json.Unmarshal([]byte(got.User), &input); err != nil { + t.Fatalf("user input is not JSON: %v", err) + } + if len(input) != 2 || input["question"] != "где настройки nginx?" || input["memory"] != "конфиг nginx лежит в /etc/nginx" { + t.Fatalf("model saw fields outside question+memory: %#v", input) + } + if strings.Contains(got.System, "nginx") { + t.Fatal("held-out entity leaked into the static verifier prompt") + } +} + +func TestLocativeAnswerVerifierNoAndMalformedFailClosed(t *testing.T) { + for _, tc := range []struct { + name string + raw string + want bool + err bool + }{ + {"no", `{"target":"паспорт","memory_subject":"ключ","answer":"no"}`, false, false}, + {"bare yes", `yes`, false, true}, + {"unknown answer", `{"target":"a","memory_subject":"b","answer":"maybe"}`, false, true}, + {"empty target", `{"target":"","memory_subject":"b","answer":"yes"}`, false, true}, + {"extra field", `{"target":"a","memory_subject":"b","answer":"yes","why":"guess"}`, false, true}, + {"trailing object", `{"target":"a","memory_subject":"b","answer":"yes"}{}`, false, true}, + } { + t.Run(tc.name, func(t *testing.T) { + v := NewLocativeAnswerVerifier(locativeCompleteFunc(func(context.Context, llm.Req) (string, error) { + return tc.raw, nil + })) + got, err := v.Answerable(context.Background(), "q", "m") + if got != tc.want || (err != nil) != tc.err { + t.Fatalf("Answerable = %v, %v; want %v, err=%v", got, err, tc.want, tc.err) + } + }) + } +} + +func TestLocativeAnswerVerifierUnavailableErrorAndTimeout(t *testing.T) { + if v := NewLocativeAnswerVerifier(nil); v != nil { + t.Fatal("nil resident model produced a verifier") + } + boom := errors.New("llama down") + v := NewLocativeAnswerVerifier(locativeCompleteFunc(func(context.Context, llm.Req) (string, error) { + return "", boom + })) + if ok, err := v.Answerable(context.Background(), "q", "m"); ok || !errors.Is(err, boom) { + t.Fatalf("model error = %v, %v; want false wrapping %v", ok, err, boom) + } + + v = NewLocativeAnswerVerifier(locativeCompleteFunc(func(ctx context.Context, _ llm.Req) (string, error) { + <-ctx.Done() + return "", ctx.Err() + })) + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond) + defer cancel() + if ok, err := v.Answerable(ctx, "q", "m"); ok || !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("timeout = %v, %v; want false deadline", ok, err) + } +} + +func TestLocativeVerifierGrammarIsFullyBounded(t *testing.T) { + if strings.Contains(locativeVerifierGrammar, "*") || strings.Contains(locativeVerifierGrammar, "+") { + t.Fatalf("grammar contains an unbounded repetition:\n%s", locativeVerifierGrammar) + } + for _, bound := range []string{"{1,80}", "{0,2}"} { + if !strings.Contains(locativeVerifierGrammar, bound) { + t.Errorf("grammar missing bound %s", bound) + } + } +} diff --git a/internal/memory/recalleval/recalleval.go b/internal/memory/recalleval/recalleval.go index 9c70013..1aaf69e 100644 --- a/internal/memory/recalleval/recalleval.go +++ b/internal/memory/recalleval/recalleval.go @@ -473,7 +473,8 @@ func bestRecall(query string, results []memory.Result, minScore, minMargin float return "" } text := results[0].Meta["text"] - if !memory.RecallAllowed(query, text) { + if !memory.RecallAllowed(query, text, + router.IsOpenQuestionShaped(query), router.IsLocativeQuestionShaped(query)) { return "" } return text diff --git a/internal/memory/recalleval/recalleval_test.go b/internal/memory/recalleval/recalleval_test.go index f8b324e..e364118 100644 --- a/internal/memory/recalleval/recalleval_test.go +++ b/internal/memory/recalleval/recalleval_test.go @@ -75,8 +75,8 @@ func TestLoadFixture(t *testing.T) { } } // Both lanes need enough cases that a rate means something. - if silent < 5 { - t.Errorf("%d must-be-silent cases, want >= 5", silent) + if silent < 15 { + t.Errorf("%d must-be-silent cases, want >= 15", silent) } if en < 5 { t.Errorf("%d English cases, want >= 5", en) @@ -176,6 +176,23 @@ func TestBestRecallMatchesDaemon(t *testing.T) { if got := bestRecall("почему небо синее", offTopic, 0.55, 0); got != "" { t.Errorf("off topic: got %q, want silence", got) } + // A statistical query route is not by itself evidence that an ordinary + // first-person report asks for a stored note. This is deliberately one hit: + // a fresh Maven has no runner-up, so only the absolute and structural gates + // can stop the live cold-start false recall. + report := []memory.Result{{ID: "a", Score: 0.825031306, Meta: map[string]string{"text": "запомни: запасной ключ лежит в синей коробке"}}} + if got := bestRecall("я отменил напоминание про молоко", report, 0.55, 0.008); got != "" { + t.Errorf("declarative report: got %q, want silence", got) + } + if got := bestRecall("я отменил напоминание про молоко?", report, 0.55, 0.008); got != "" { + t.Errorf("polar report: got %q, want silence", got) + } + // A voice transcript can omit punctuation. A nominal request that shares + // the answer's topic stays eligible. + nominal := []memory.Result{{ID: "a", Score: 0.90, Meta: map[string]string{"text": "домашний сервер на 192.168.1.104"}}} + if got := bestRecall("адрес домашнего сервера", nominal, 0.80, 0.008); got != "домашний сервер на 192.168.1.104" { + t.Errorf("nominal topic request: got %q, want the server note", got) + } clear := []memory.Result{ {ID: "a", Score: 0.86, Meta: map[string]string{"text": "чай"}}, {ID: "b", Score: 0.70, Meta: map[string]string{"text": "кофе"}}, @@ -265,9 +282,9 @@ func sqliteStores(t *testing.T) NewStore { // like TestONNXBaseline in internal/router/eval. `make eval-recall` points it at // the vendored runtime. // -// Reports rather than asserts. The gate sweep is the point: it prints -// answered-vs-false-recall at a range of query_min_score values, so the right -// threshold is read off data instead of guessed. +// Reports the full distributions and pins only the two operator-facing safety +// ratchets: do not lose the measured answering floor, and never read a note on +// a must-be-silent case. Exact scores stay observable rather than asserted. func TestONNXRecall(t *testing.T) { lib := os.Getenv("MAVEN_ONNX_LIB") if lib == "" { @@ -295,11 +312,36 @@ func TestONNXRecall(t *testing.T) { t.Fatalf("Score: %v", err) } t.Log("\n" + rep.String() + rep.Failures()) + var semanticOnly []string + for _, o := range rep.Outcomes { + if !o.Case.Answerable() && len(o.Hits) > 0 { + t.Logf("silent candidate %s score=%.6f margin=%.6f query=%q memory=%q", + o.Case.ID, o.TopScor, o.Margin, o.Case.Query, o.Hits[0].Meta["text"]) + } + if o.Case.Answerable() && o.Rank1 && + !memory.SharesContentWord(o.Case.Query, o.Hits[0].Meta["text"]) { + semanticOnly = append(semanticOnly, o.Case.ID) + } + } + t.Logf("right-note rank-1 cases with semantic-only (no lexical topic) evidence: %s", + strings.Join(semanticOnly, ", ")) + // V-719 closes the single-hit locative failure class by requiring every + // named identity term to be corroborated. The real e5 distributions overlap: + // four true locative paraphrases and six false locative neighbours cannot be + // separated by score or target-phrase cosine. The deterministic floor is 14; + // recovering those four safely needs a separate answerability verifier, and + // an absent or failed verifier must keep this fail-closed result. + if answered := rep.Rank1 - rep.Gated; answered < 14 { + t.Errorf("answered %d/%d, want at least 14 strict-floor recalls", answered, rep.Answerable) + } + if rep.FalseRecall != 0 { + t.Errorf("false recall %d/%d, want zero:\n%s", rep.FalseRecall, rep.NoAnswer, rep.Failures()) + } // Cached for the sweeps only: the headline run above must pay the real // embedder cost so its latency numbers mean something. cached := Cache(emb) t.Log("\ngate sweep (margin off):\n" + sweep(t, cached, f)) - t.Log("\nmargin sweep (gate 0.55):\n" + marginSweep(t, cached, f)) + t.Logf("\nmargin sweep (gate %.2f):\n%s", config.DefaultQueryMinScore, marginSweep(t, cached, f)) } // sweep scores the fixture at a range of gates and renders one line each. Two @@ -321,9 +363,8 @@ func sweep(t *testing.T, emb router.Embedder, f Fixture) string { } // marginSweep is the same idea for the margin gate (top1 − top2 > delta), with -// the absolute gate held at its default. The absolute score cannot separate a -// real hit from a made-up question under e5 — every score lands in one narrow -// band — so this sweep is the one that picks a number. +// the absolute gate held at its default. Neither axis separates every case on +// its own under e5's narrow score band; the deployed pair is calibrated jointly. func marginSweep(t *testing.T, emb router.Embedder, f Fixture) string { t.Helper() var b strings.Builder diff --git a/internal/memory/recalleval/ru_recall_v1.json b/internal/memory/recalleval/ru_recall_v1.json index f5f9c7d..f2a82f4 100644 --- a/internal/memory/recalleval/ru_recall_v1.json +++ b/internal/memory/recalleval/ru_recall_v1.json @@ -413,6 +413,136 @@ {"id": "n1", "text": "на заправке у моста дешевле бензин", "kind": "note"}, {"id": "n2", "text": "надо поменять зимние шины", "kind": "note"} ] + }, + { + "id": "ru-silent-033", + "lang": "ru", + "tags": ["silent", "hard"], + "note": "Observed through the whole-assistant E2E with only n1 stored: routing called an ordinary past-tense report a query and the single-hit recall gate spoke n1 at cosine 0.825. A report is not a request to read a semantically nearby note.", + "query": "я отменил напоминание про молоко", + "want": "", + "notes": [ + {"id": "n1", "text": "запомни: запасной ключ лежит в синей коробке", "kind": "note"}, + {"id": "n2", "text": "домашний сервер на 192.168.1.104", "kind": "note"}, + {"id": "n3", "text": "чай пью только зелёный", "kind": "note"} + ] + }, + { + "id": "ru-silent-034", + "lang": "ru", + "tags": ["silent", "hard", "locative"], + "note": "V-719 whole-assistant adversarial: with only n1 stored, the open personal question scored 0.817 against it. An explicit location request must not substitute a semantically nearby object's location for the named target.", + "query": "где мой паспорт?", + "want": "", + "notes": [ + {"id": "n1", "text": "запомни: запасной ключ лежит в синей коробке", "kind": "note"}, + {"id": "n2", "text": "домашний сервер на 192.168.1.104", "kind": "note"}, + {"id": "n3", "text": "чай пью только зелёный", "kind": "note"} + ] + }, + { + "id": "ru-silent-035", + "lang": "ru", + "tags": ["silent", "hard", "locative"], + "note": "V-719 adversarial neighbour: a locative interrogative must not reopen the report-shaped false recall; the unrelated single note scored 0.806.", + "query": "где я отменил напоминание про молоко?", + "want": "", + "notes": [ + {"id": "n1", "text": "запомни: запасной ключ лежит в синей коробке", "kind": "note"}, + {"id": "n2", "text": "домашний сервер на 192.168.1.104", "kind": "note"}, + {"id": "n3", "text": "чай пью только зелёный", "kind": "note"} + ] + }, + { + "id": "ru-silent-036", + "lang": "ru", + "tags": ["silent", "hard", "locative", "incidental-overlap"], + "note": "V-719 adversarial: blue is only an incidental shared modifier; shirt and spare key are different targets despite cosine 0.838.", + "query": "где лежит синяя рубашка?", + "want": "", + "notes": [ + {"id": "n1", "text": "запомни: запасной ключ лежит в синей коробке", "kind": "note"}, + {"id": "n2", "text": "домашний сервер на 192.168.1.104", "kind": "note"}, + {"id": "n3", "text": "чай пью только зелёный", "kind": "note"} + ] + }, + { + "id": "ru-silent-037", + "lang": "ru", + "tags": ["silent", "hard", "locative", "incidental-overlap"], + "note": "V-719 adversarial: blue is only an incidental shared modifier; folder and spare key are different targets despite cosine 0.837.", + "query": "где лежит синяя папка?", + "want": "", + "notes": [ + {"id": "n1", "text": "запомни: запасной ключ лежит в синей коробке", "kind": "note"}, + {"id": "n2", "text": "домашний сервер на 192.168.1.104", "kind": "note"}, + {"id": "n3", "text": "чай пью только зелёный", "kind": "note"} + ] + }, + { + "id": "ru-silent-038", + "lang": "ru", + "tags": ["silent", "hard", "locative", "incidental-overlap"], + "note": "V-719 adversarial: spare is only an incidental shared modifier; passport and key are different targets despite cosine 0.832.", + "query": "где мой запасной паспорт?", + "want": "", + "notes": [ + {"id": "n1", "text": "запомни: запасной ключ лежит в синей коробке", "kind": "note"}, + {"id": "n2", "text": "домашний сервер на 192.168.1.104", "kind": "note"}, + {"id": "n3", "text": "чай пью только зелёный", "kind": "note"} + ] + }, + { + "id": "ru-silent-039", + "lang": "ru", + "tags": ["silent", "hard", "locative", "incidental-overlap"], + "note": "V-719 adversarial: spare is only an incidental shared modifier; flash drive and key are different targets despite cosine 0.839.", + "query": "где лежит запасная флешка?", + "want": "", + "notes": [ + {"id": "n1", "text": "запомни: запасной ключ лежит в синей коробке", "kind": "note"}, + {"id": "n2", "text": "домашний сервер на 192.168.1.104", "kind": "note"}, + {"id": "n3", "text": "чай пью только зелёный", "kind": "note"} + ] + }, + { + "id": "ru-silent-040", + "lang": "ru", + "tags": ["silent", "hard", "locative", "incidental-overlap"], + "note": "V-719 adversarial: blue box overlaps, but the candidate does not corroborate the documents qualifier despite cosine 0.867.", + "query": "где находится синяя коробка с документами?", + "want": "", + "notes": [ + {"id": "n1", "text": "запомни: запасной ключ лежит в синей коробке", "kind": "note"}, + {"id": "n2", "text": "домашний сервер на 192.168.1.104", "kind": "note"}, + {"id": "n3", "text": "чай пью только зелёный", "kind": "note"} + ] + }, + { + "id": "ru-silent-041", + "lang": "ru", + "tags": ["silent", "hard", "locative", "incidental-overlap"], + "note": "V-719 adversarial: key overlaps, but the candidate does not corroborate the car qualifier despite cosine 0.844.", + "query": "где лежит ключ от машины?", + "want": "", + "notes": [ + {"id": "n1", "text": "запомни: запасной ключ лежит в синей коробке", "kind": "note"}, + {"id": "n2", "text": "домашний сервер на 192.168.1.104", "kind": "note"}, + {"id": "n3", "text": "чай пью только зелёный", "kind": "note"} + ] + }, + { + "id": "ru-silent-042", + "lang": "ru", + "tags": ["silent", "hard", "locative", "location-object"], + "note": "V-719 adversarial: the requested blue box occurs in n1 only as the spare key's location, not as the proposition subject. Arbitrary whole-note overlap must not turn it into the box's own location.", + "query": "где синяя коробка?", + "want": "", + "notes": [ + {"id": "n1", "text": "запомни: запасной ключ лежит в синей коробке", "kind": "note"}, + {"id": "n2", "text": "домашний сервер на 192.168.1.104", "kind": "note"}, + {"id": "n3", "text": "чай пью только зелёный", "kind": "note"} + ] } ] }