Merge pull request 'Bug: a question writes invented knowledge into memory as a self fact, and recall then serves it back for unrelated questions' (#99) from task/470-bug-a-question-writes-invented-knowledge into master
Reviewed-on: #99
This commit was merged in pull request #99.
This commit is contained in:
@@ -15,14 +15,41 @@ func (h *reactiveHandler) actionFact(ctx context.Context, dec router.Decision) s
|
||||
if !dec.Slots.HasKey {
|
||||
return "не разобрала, что записать — попробуй иначе."
|
||||
}
|
||||
// A question is never a fact about him (#470). "какая последняя версия
|
||||
// языка Go?" used to land here, and the value stored was whatever the
|
||||
// model invented for it, at confidence 1.00, indexed for recall under the
|
||||
// question's own text. Two such rows then claimed seven unrelated world
|
||||
// questions through recall and silently disabled world answering.
|
||||
//
|
||||
// The routing error itself is not fixed here — the answer is to answer.
|
||||
// Sending the turn down the query chain is what he asked for anyway, and
|
||||
// it costs a mis-routed capture nothing: an explicit "запиши ..." is not
|
||||
// question-shaped, so it never takes this branch.
|
||||
if router.IsQuestionShaped(dec.Utterance) {
|
||||
log.Printf("voice: fact write refused, utterance is a question: %q (key %q) — answering as a query",
|
||||
dec.Utterance, dec.Slots.Key)
|
||||
q := dec
|
||||
q.Intent = router.IntentQuery
|
||||
// The key the model extracted is its guess at what to store, not a
|
||||
// fact he has. Left in place, queryFactByKey would read it back and
|
||||
// claim the turn before any real source ran.
|
||||
q.Slots.Key, q.Slots.HasKey = "", false
|
||||
q.Slots.Value = ""
|
||||
return h.actionQuery(ctx, q)
|
||||
}
|
||||
now := h.now()
|
||||
req := ipc.WriteFactReq{
|
||||
Ts: now,
|
||||
Kind: "self",
|
||||
Key: dec.Slots.Key,
|
||||
Value: dec.Slots.Value,
|
||||
Source: "tap:voice",
|
||||
Confidence: 1.0,
|
||||
Ts: now,
|
||||
Kind: "self",
|
||||
Key: dec.Slots.Key,
|
||||
Value: dec.Slots.Value,
|
||||
Source: "tap:voice",
|
||||
// Not 1.00 unconditionally any more (#470). A value he said is
|
||||
// evidence; a value the model supplied for words he never said is a
|
||||
// guess, and writing a guess at full confidence is the same mistake
|
||||
// the act path already refuses under "LLM output is not
|
||||
// authorization".
|
||||
Confidence: factConfidence(dec.Utterance, dec.Slots.Value),
|
||||
// Subject: the key doubles as the entity-resolution candidate —
|
||||
// a voice-tapped fact's key is usually the thing/person it's
|
||||
// about ("espresso_machine", "kate"), so queueing it for Nexus
|
||||
|
||||
@@ -434,6 +434,14 @@ func (h *reactiveHandler) queryMemory(ctx context.Context, t *queryTurn) (string
|
||||
return "", false
|
||||
}
|
||||
text := hit.Meta["text"]
|
||||
// The score cleared the gate and the topic still has to match (#470). A
|
||||
// note about his slow network scored high enough to answer "почему небо
|
||||
// синее?", because the right-note and must-be-silent score ranges overlap
|
||||
// and no threshold sits between them.
|
||||
if !memory.RecallAllowed(t.dec.Utterance, text) {
|
||||
log.Printf("voice: recall %q rejected for %q: a world question and no shared topic word", text, t.dec.Utterance)
|
||||
return "", false
|
||||
}
|
||||
// A note is phrased in Maven's voice; a fact is read back as it was
|
||||
// stored.
|
||||
if hit.Meta["type"] == "note" {
|
||||
@@ -469,6 +477,12 @@ func (h *reactiveHandler) queryNotes(ctx context.Context, t *queryTurn) (string,
|
||||
if !memory.ConfidentScores(noteScores, h.queryMinScore, h.queryMinMargin) {
|
||||
return "", false
|
||||
}
|
||||
// Same topic veto as queryMemory above: the best note must be about what
|
||||
// he asked, not merely the nearest vector in the index.
|
||||
if !memory.RecallAllowed(t.dec.Utterance, notes[0].Text) {
|
||||
log.Printf("voice: note %q rejected for %q: a world question and no shared topic word", notes[0].Text, t.dec.Utterance)
|
||||
return "", false
|
||||
}
|
||||
texts := make([]string, len(notes))
|
||||
for i, n := range notes {
|
||||
texts[i] = n.Text
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// ungroundedConfidence — what a self fact is worth when its value appears
|
||||
// nowhere in what he said. Below `query_min_score` is not the point (recall
|
||||
// gates on vector distance, not on this number); the point is that
|
||||
// `/history` and every future reader can tell a value he said from a value
|
||||
// the model supplied.
|
||||
const ungroundedConfidence = 0.6
|
||||
|
||||
// factConfidence scores a self fact by whether its value is grounded in the
|
||||
// utterance it came from. Grounded stays 1.00, which is what a tapped fact
|
||||
// has always been worth. Ungrounded drops, and says so in the log.
|
||||
//
|
||||
// An empty value is grounded by definition: the key alone carries the fact
|
||||
// ("поужинал"), and there is nothing for the model to have invented.
|
||||
func factConfidence(utterance, value string) float64 {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return 1.0
|
||||
}
|
||||
if valueGrounded(utterance, value) {
|
||||
return 1.0
|
||||
}
|
||||
log.Printf("voice: fact value %q is not in %q — writing at confidence %.2f",
|
||||
value, utterance, ungroundedConfidence)
|
||||
return ungroundedConfidence
|
||||
}
|
||||
|
||||
// valueGrounded reports whether every word of value traces back to a word he
|
||||
// actually said. The comparison is on a 4-rune prefix, so the model's
|
||||
// normalization survives ("пил воду" → "вода") while an invented value
|
||||
// ("1.20" for a question about Go) does not.
|
||||
func valueGrounded(utterance, value string) bool {
|
||||
said := factTokens(utterance)
|
||||
words := factTokens(value)
|
||||
if len(words) == 0 {
|
||||
return true
|
||||
}
|
||||
for _, w := range words {
|
||||
if !anyTokenMatches(said, w) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func anyTokenMatches(said []string, w string) bool {
|
||||
for _, s := range said {
|
||||
if s == w || sameStem(s, w) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// sameStem is inflection tolerance and nothing more: it compares all but the
|
||||
// last rune of the shorter word, and never fewer than three. Russian marks
|
||||
// case on the ending, so "пил воду" and the stored "вода" are the same word he
|
||||
// said, while "1.20" and "версия" are not. A word of three runes or fewer must
|
||||
// match outright, where a shorter prefix would match half the language.
|
||||
func sameStem(a, b string) bool {
|
||||
ar, br := []rune(a), []rune(b)
|
||||
shorter := min(len(ar), len(br))
|
||||
n := shorter - 1
|
||||
if n < 3 || len(ar) < n || len(br) < n {
|
||||
return false
|
||||
}
|
||||
return string(ar[:n]) == string(br[:n])
|
||||
}
|
||||
|
||||
// factTokens lowercases and splits on everything that is not a letter or a
|
||||
// digit, the same shape planTokens uses in the router.
|
||||
func factTokens(s string) []string {
|
||||
return strings.FieldsFunc(strings.ToLower(s), func(r rune) bool {
|
||||
return !unicode.IsLetter(r) && !unicode.IsDigit(r)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/memory"
|
||||
"github.com/kami/maven/internal/router"
|
||||
"github.com/kami/maven/internal/tool"
|
||||
"github.com/kami/maven/internal/voice"
|
||||
)
|
||||
|
||||
func newFactGateHandler(t *testing.T, now time.Time) (*reactiveHandler, ipc.CoreAPI) {
|
||||
t.Helper()
|
||||
st := newTestStore(t)
|
||||
api := ipc.NewStoreAPI(st)
|
||||
emb := router.NewHashEmbedder(1024)
|
||||
h := &reactiveHandler{
|
||||
api: api,
|
||||
embedder: emb,
|
||||
router: buildRouter(emb, tool.NewMatcher(api), 0.55, nil),
|
||||
replier: voice.NewStubReplier(),
|
||||
now: func() time.Time { return now },
|
||||
memStore: memory.NewInMemoryStore(),
|
||||
dataStore: st,
|
||||
}
|
||||
return h, api
|
||||
}
|
||||
|
||||
// The write half of #470: a question routed to IntentFact must not become a
|
||||
// fact about him, and must not leave a vector behind for recall to serve.
|
||||
func TestActionFact_QuestionIsNotWritten(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
h, api := newFactGateHandler(t, time.Now())
|
||||
|
||||
reply := h.actionFact(ctx, router.Decision{
|
||||
Intent: router.IntentFact,
|
||||
Utterance: "какая последняя версия языка Go?",
|
||||
Slots: router.Slots{Key: "go_version", HasKey: true, Value: `"1.20"`},
|
||||
})
|
||||
|
||||
if _, err := api.LatestFact(ctx, "go_version"); err == nil {
|
||||
t.Fatal("a question was stored as a fact about him")
|
||||
}
|
||||
hits, err := h.memStore.Search(ctx, mustEmbedPassage(t, h, "какая последняя версия языка Go?"), 3)
|
||||
if err != nil {
|
||||
t.Fatalf("memory search: %v", err)
|
||||
}
|
||||
if len(hits) != 0 {
|
||||
t.Fatalf("the question was indexed for recall: %+v", hits)
|
||||
}
|
||||
// It went down the query chain instead. Nothing is configured to answer a
|
||||
// world question in this harness, so "не знаю." is the honest outcome —
|
||||
// what matters is that the turn was answered, not stored.
|
||||
if reply == "" {
|
||||
t.Fatal("the turn was neither stored nor answered")
|
||||
}
|
||||
}
|
||||
|
||||
// The capture that must survive the gate: an explicit instruction to record,
|
||||
// even though it contains an interrogative.
|
||||
func TestActionFact_ExplicitCaptureStillWrites(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
h, api := newFactGateHandler(t, time.Now())
|
||||
|
||||
h.actionFact(ctx, router.Decision{
|
||||
Intent: router.IntentFact,
|
||||
Utterance: "запиши что я пил воду",
|
||||
Slots: router.Slots{Key: "water", HasKey: true, Value: `"вода"`},
|
||||
})
|
||||
|
||||
f, err := api.LatestFact(ctx, "water")
|
||||
if err != nil {
|
||||
t.Fatalf("an explicit capture was refused: %v", err)
|
||||
}
|
||||
if f.Confidence != 1.0 {
|
||||
t.Errorf("confidence = %v, want 1.0 for a value he said", f.Confidence)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFactConfidence(t *testing.T) {
|
||||
cases := []struct {
|
||||
utterance, value string
|
||||
want float64
|
||||
}{
|
||||
{"запиши что я пил воду", `"вода"`, 1.0},
|
||||
{"я выпил кофе", `"кофе"`, 1.0},
|
||||
{"поужинал", "", 1.0},
|
||||
{"отметь что я полил кактус", `"полил кактус"`, 1.0},
|
||||
{"какая последняя версия языка Go", `"1.20"`, ungroundedConfidence},
|
||||
{"кто премьер Японии", `"Тонио Озаки"`, ungroundedConfidence},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := factConfidence(c.utterance, c.value); got != c.want {
|
||||
t.Errorf("factConfidence(%q, %q) = %v, want %v", c.utterance, c.value, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func mustEmbedPassage(t *testing.T, h *reactiveHandler, text string) []float32 {
|
||||
t.Helper()
|
||||
vec, err := router.EmbedQuery(context.Background(), h.embedder, text)
|
||||
if err != nil {
|
||||
t.Fatalf("embed %q: %v", text, err)
|
||||
}
|
||||
return vec
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package memory
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// stopwords — words that carry no topic. A question and a note that share only
|
||||
// these share nothing: "почему небо синее" and "сеть какая-то медленная" both
|
||||
// contain "какая"-shaped filler and are about different worlds.
|
||||
var stopwords = map[string]bool{
|
||||
// interrogatives and demonstratives
|
||||
"что": true, "чего": true, "какой": true, "какая": true, "какое": true,
|
||||
"какие": true, "каких": true, "кто": true, "кого": true, "кому": true,
|
||||
"почему": true, "зачем": true, "где": true, "куда": true, "откуда": true,
|
||||
"когда": true, "сколько": true, "как": true, "то": true, "это": true,
|
||||
"этот": true, "тот": true, "там": true, "тут": true, "такой": true,
|
||||
// pronouns — every sentence he says is about him, so "я" is not a topic
|
||||
"я": true, "меня": true, "мне": true, "мой": true, "моя": true, "мои": true,
|
||||
"ты": true, "тебя": true, "тебе": true, "твой": true, "он": true, "она": true,
|
||||
"они": true, "мы": true, "себя": true, "свой": true,
|
||||
// prepositions, conjunctions, particles, copulas
|
||||
"в": true, "во": true, "на": true, "с": true, "со": true, "у": true,
|
||||
"о": true, "об": true, "про": true, "за": true, "из": true, "по": true,
|
||||
"до": true, "от": true, "для": true, "над": true, "под": true, "при": true,
|
||||
"и": true, "а": true, "но": true, "или": true, "же": true, "ли": true,
|
||||
"не": true, "ни": true, "бы": true, "был": true, "была": true, "было": true,
|
||||
"быть": true, "есть": true, "был-ли": true, "уже": true, "ещё": true,
|
||||
"еще": true, "так": true, "вот": true, "там-же": true,
|
||||
// English filler, for the mixed utterances he does say
|
||||
"the": true, "a": true, "an": true, "is": true, "are": true, "was": true,
|
||||
"were": true, "be": true, "of": true, "in": true, "on": true, "at": true,
|
||||
"to": true, "for": true, "about": true, "and": true, "or": true, "not": true,
|
||||
"what": true, "who": true, "why": true, "when": true, "where": true,
|
||||
"which": true, "how": true, "i": true, "my": true, "me": true, "it": true,
|
||||
"this": true, "that": true,
|
||||
}
|
||||
|
||||
// firstPerson — the words that make an utterance a question about his own
|
||||
// life. Not possession only: "как я восстановил конфиги" owns nothing and is
|
||||
// still about him.
|
||||
var firstPerson = map[string]bool{
|
||||
"я": true, "меня": true, "мне": true, "мной": true, "мой": true,
|
||||
"моя": true, "моё": true, "мое": true, "мои": true, "моего": true,
|
||||
"моей": true, "моих": true, "моим": true, "себя": true, "свой": true,
|
||||
"своя": true, "свои": true, "своего": true, "мною": true,
|
||||
"i": true, "me": true, "my": true, "mine": true, "myself": true,
|
||||
}
|
||||
|
||||
// RecallAllowed is the second half of the recall gate (#470). A hit that
|
||||
// cleared the score and margin gate may still be about something else
|
||||
// entirely: the held-out fixture puts the right note at 0.791-0.890 and the
|
||||
// must-be-silent cases at 0.795-0.835, so no threshold sits between them, and
|
||||
// a note about his slow network answered "почему небо синее?".
|
||||
//
|
||||
// The veto applies only to a question that mentions nothing of his. That
|
||||
// restriction is what keeps the fix from costing more than it saves: recall
|
||||
// exists to find the note whose words he no longer remembers, and demanding a
|
||||
// shared word of every recall silenced four true recalls on the fixture to
|
||||
// kill one false one. A question about his own life keeps the embedder alone
|
||||
// as its judge. A question about the world has to name something the memory
|
||||
// actually mentions.
|
||||
func RecallAllowed(query, text string) bool {
|
||||
if mentionsHim(query) {
|
||||
return true
|
||||
}
|
||||
return SharesContentWord(query, text)
|
||||
}
|
||||
|
||||
func mentionsHim(query string) bool {
|
||||
for _, w := range strings.FieldsFunc(strings.ToLower(query), func(r rune) bool {
|
||||
return !unicode.IsLetter(r) && !unicode.IsDigit(r)
|
||||
}) {
|
||||
if firstPerson[w] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// SharesContentWord reports whether query and text have at least one topic
|
||||
// word in common, after dropping the words that carry no topic. Stems are
|
||||
// compared, so the note and the question do not have to inflect alike.
|
||||
func SharesContentWord(query, text string) bool {
|
||||
q := contentWords(query)
|
||||
if len(q) == 0 {
|
||||
// Nothing to compare — a question made entirely of filler. The score
|
||||
// gate is then the only judge it can have.
|
||||
return true
|
||||
}
|
||||
t := contentWords(text)
|
||||
for _, a := range q {
|
||||
for _, b := range t {
|
||||
if a == b || sameStem(a, b) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func contentWords(s string) []string {
|
||||
var out []string
|
||||
for _, w := range strings.FieldsFunc(strings.ToLower(s), func(r rune) bool {
|
||||
return !unicode.IsLetter(r) && !unicode.IsDigit(r)
|
||||
}) {
|
||||
if !stopwords[w] {
|
||||
out = append(out, w)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sameStem is inflection and derivation tolerance: Russian marks case and
|
||||
// tense on the ending, and the note and the question rarely use the same form.
|
||||
// "воду" and "вода" are the same water, "кормить" and "корм" the same feeding.
|
||||
// All but the last rune of the shorter word must match, and never fewer than
|
||||
// three, which is what keeps "сеть" clear of "сеанс".
|
||||
func sameStem(a, b string) bool {
|
||||
ar, br := []rune(a), []rune(b)
|
||||
n := min(len(ar), len(br)) - 1
|
||||
if n < 3 {
|
||||
return false
|
||||
}
|
||||
return string(ar[:n]) == string(br[:n])
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package memory
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestRecallAllowed(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
query, text string
|
||||
want bool
|
||||
}{
|
||||
// The #470 shape: a world question and a note about his box.
|
||||
{"world question, unrelated note", "почему небо синее", "сеть какая-то медленная", false},
|
||||
{"world question, unrelated fact", "какая столица Франции", "какая последняя версия языка Go", false},
|
||||
{"silent fixture case", "во сколько отходит поезд", "бэкап запускается в три ночи", false},
|
||||
|
||||
// A world question that does name the topic keeps its answer.
|
||||
{"world question, same topic", "какой поезд идёт в Минск", "поезда в Минск ходят утром", true},
|
||||
|
||||
// A question about his own life is judged by the embedder alone,
|
||||
// because recall exists for words he no longer remembers.
|
||||
{"about him, no shared word", "во сколько я обычно засыпаю", "ложусь около одиннадцати", true},
|
||||
{"about him, english", "which colour scheme do i like", "тёмная тема везде", true},
|
||||
|
||||
// Inflection must not break a match.
|
||||
{"inflected", "чем кормить кота", "корм для кота в шкафу", true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := RecallAllowed(c.query, c.text); got != c.want {
|
||||
t.Errorf("%s: RecallAllowed(%q, %q) = %v, want %v", c.name, c.query, c.text, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A question made only of filler has no topic word to match on, and the score
|
||||
// gate is then the only judge it can have.
|
||||
func TestRecallAllowedFallsBackWhenNothingToCompare(t *testing.T) {
|
||||
if !RecallAllowed("что это", "сеть какая-то медленная") {
|
||||
t.Error("a question with no content word must not be vetoed")
|
||||
}
|
||||
}
|
||||
@@ -378,7 +378,7 @@ func scoreCase(ctx context.Context, emb router.Embedder, newStore NewStore, minS
|
||||
if len(hits) > 1 {
|
||||
o.Margin = hits[0].Score - hits[1].Score
|
||||
}
|
||||
o.Recalled = bestRecall(hits, minScore, minMargin)
|
||||
o.Recalled = bestRecall(c.Query, hits, minScore, minMargin)
|
||||
}
|
||||
for i, h := range hits {
|
||||
if h.ID != c.Want {
|
||||
@@ -424,11 +424,19 @@ func rankNote(inTop3 bool) string {
|
||||
// is not importable; recalleval_test.go asserts the two agree in behaviour.
|
||||
// The daemon returns the whole hit (a note and a fact are said differently);
|
||||
// the harness only scores what came back, so it keeps returning the text.
|
||||
func bestRecall(results []memory.Result, minScore, minMargin float64) string {
|
||||
// bestRecall mirrors the daemon's gate in cmd/mavend/recall.go, including the
|
||||
// topic veto added for #470: a score that clears the gate still has to be
|
||||
// about what he asked. Keep the two in step — a fixture that measures a
|
||||
// weaker gate than the daemon runs flatters it.
|
||||
func bestRecall(query string, results []memory.Result, minScore, minMargin float64) string {
|
||||
if !memory.Confident(results, minScore, minMargin) {
|
||||
return ""
|
||||
}
|
||||
return results[0].Meta["text"]
|
||||
text := results[0].Meta["text"]
|
||||
if !memory.RecallAllowed(query, text) {
|
||||
return ""
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
func bump(m map[string]TagStat, key string, pass bool) {
|
||||
|
||||
@@ -140,21 +140,22 @@ func words(s string) []string {
|
||||
|
||||
// TestBestRecallMatchesDaemon — the harness duplicates bestRecall from
|
||||
// cmd/mavend/recall.go (package main is not importable). This pins the copy to
|
||||
// the original's three rules: no hits, below the gate, or no text ⇒ silence.
|
||||
// the original's rules: no hits, below the gate, no text, or no shared topic
|
||||
// word ⇒ silence.
|
||||
func TestBestRecallMatchesDaemon(t *testing.T) {
|
||||
if got := bestRecall(nil, 0.55, 0); got != "" {
|
||||
if got := bestRecall("чай", nil, 0.55, 0); got != "" {
|
||||
t.Errorf("no hits: got %q, want silence", got)
|
||||
}
|
||||
low := []memory.Result{{ID: "a", Score: 0.4, Meta: map[string]string{"text": "чай"}}}
|
||||
if got := bestRecall(low, 0.55, 0); got != "" {
|
||||
if got := bestRecall("чай", low, 0.55, 0); got != "" {
|
||||
t.Errorf("below gate: got %q, want silence", got)
|
||||
}
|
||||
noText := []memory.Result{{ID: "a", Score: 0.9, Meta: map[string]string{}}}
|
||||
if got := bestRecall(noText, 0.55, 0); got != "" {
|
||||
if got := bestRecall("чай", noText, 0.55, 0); got != "" {
|
||||
t.Errorf("no text: got %q, want silence", got)
|
||||
}
|
||||
ok := []memory.Result{{ID: "a", Score: 0.9, Meta: map[string]string{"text": "чай"}}}
|
||||
if got := bestRecall(ok, 0.55, 0); got != "чай" {
|
||||
if got := bestRecall("чай", ok, 0.55, 0); got != "чай" {
|
||||
t.Errorf("above gate: got %q, want %q", got, "чай")
|
||||
}
|
||||
// Margin: a close runner-up means the embedder cannot tell the two apart,
|
||||
@@ -163,17 +164,23 @@ func TestBestRecallMatchesDaemon(t *testing.T) {
|
||||
{ID: "a", Score: 0.86, Meta: map[string]string{"text": "чай"}},
|
||||
{ID: "b", Score: 0.85, Meta: map[string]string{"text": "кофе"}},
|
||||
}
|
||||
if got := bestRecall(close, 0.55, 0.03); got != "" {
|
||||
if got := bestRecall("чай", close, 0.55, 0.03); got != "" {
|
||||
t.Errorf("thin margin: got %q, want silence", got)
|
||||
}
|
||||
if got := bestRecall(close, 0.55, 0); got != "чай" {
|
||||
if got := bestRecall("чай", close, 0.55, 0); got != "чай" {
|
||||
t.Errorf("margin off: got %q, want %q", got, "чай")
|
||||
}
|
||||
// The topic veto (#470): the score is fine and the note is about
|
||||
// something else.
|
||||
offTopic := []memory.Result{{ID: "a", Score: 0.9, Meta: map[string]string{"text": "сеть какая-то медленная"}}}
|
||||
if got := bestRecall("почему небо синее", offTopic, 0.55, 0); got != "" {
|
||||
t.Errorf("off topic: got %q, want silence", got)
|
||||
}
|
||||
clear := []memory.Result{
|
||||
{ID: "a", Score: 0.86, Meta: map[string]string{"text": "чай"}},
|
||||
{ID: "b", Score: 0.70, Meta: map[string]string{"text": "кофе"}},
|
||||
}
|
||||
if got := bestRecall(clear, 0.55, 0.03); got != "чай" {
|
||||
if got := bestRecall("чай", clear, 0.55, 0.03); got != "чай" {
|
||||
t.Errorf("wide margin: got %q, want %q", got, "чай")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
package router
|
||||
|
||||
import "strings"
|
||||
|
||||
// interrogatives — the question words that mark an utterance as asking rather
|
||||
// than telling. Tokenized, never substring: "что" inside "чтобы" and "как"
|
||||
// inside "какао" are not questions.
|
||||
var interrogatives = []string{
|
||||
"что", "чего", "какой", "какая", "какое", "какие", "каких",
|
||||
"кто", "кого", "кому", "чей", "почему", "зачем", "отчего",
|
||||
"где", "куда", "откуда", "когда", "сколько", "как",
|
||||
"what", "who", "whom", "why", "when", "where", "which", "how",
|
||||
}
|
||||
|
||||
// narrativeRequests — "tell me about X" asks for knowledge Maven does not
|
||||
// hold about him. It carries no question mark and no interrogative, which is
|
||||
// how "расскажи про битву при Ватерлоо" reached the fact store (#470).
|
||||
var narrativeRequests = []string{
|
||||
"расскажи", "объясни", "опиши", "перечисли",
|
||||
"tell", "explain", "describe",
|
||||
}
|
||||
|
||||
// captureVerbs — an explicit instruction to record something. These win over
|
||||
// every test below, because "запиши что я пил воду" contains an interrogative
|
||||
// and is still a capture: the word he said is "запиши".
|
||||
var captureVerbs = []string{
|
||||
"запиши", "запомни", "отметь", "заметь", "добавь", "сохрани",
|
||||
"note", "remember", "log", "save",
|
||||
}
|
||||
|
||||
// IsQuestionShaped reports whether text asks for something rather than
|
||||
// records it. It is a deterministic offline test over tokens, so it costs
|
||||
// nothing and never depends on the model that produced the routing decision.
|
||||
//
|
||||
// It exists because a mis-routed question used to be persisted as a fact
|
||||
// about the owner, with the model's invented answer as the value (#470). The
|
||||
// predicate is deliberately blunt: refusing to store a question is cheap and
|
||||
// reversible, storing an invented fact about him is neither.
|
||||
func IsQuestionShaped(text string) bool {
|
||||
t := strings.TrimSpace(text)
|
||||
if t == "" {
|
||||
return false
|
||||
}
|
||||
toks := planTokens(t)
|
||||
for _, v := range captureVerbs {
|
||||
if hasTok(toks, v) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if strings.HasSuffix(t, "?") {
|
||||
return true
|
||||
}
|
||||
for _, w := range interrogatives {
|
||||
if hasTok(toks, w) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, w := range narrativeRequests {
|
||||
if hasTok(toks, w) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package router
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestIsQuestionShaped(t *testing.T) {
|
||||
// The seven utterances #470 recorded, plus the captures that must keep
|
||||
// working. A capture misread as a question loses a fact; a question
|
||||
// misread as a capture poisons recall, so the captures are the ones worth
|
||||
// pinning here.
|
||||
cases := []struct {
|
||||
text string
|
||||
want bool
|
||||
}{
|
||||
{"какая последняя версия языка Go?", true},
|
||||
{"что дальше?", true},
|
||||
{"расскажи про битву при Ватерлоо", true},
|
||||
{"почему небо синее?", true},
|
||||
{"какая столица Австралии?", true},
|
||||
{"кто такой Никола Тесла?", true},
|
||||
{"сколько стоит доллар", true},
|
||||
{"who is the premier of Japan", true},
|
||||
{"объясни линии Фраунгофера", true},
|
||||
|
||||
{"запиши что я пил воду", false},
|
||||
{"запомни какая у меня машина", false},
|
||||
{"отметь что я поужинал", false},
|
||||
{"поужинал", false},
|
||||
{"я выпил кофе", false},
|
||||
{"вода", false},
|
||||
{"привет", false},
|
||||
{"", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := IsQuestionShaped(c.text); got != c.want {
|
||||
t.Errorf("IsQuestionShaped(%q) = %v, want %v", c.text, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Substring matching is what made the day-plan predicates wrong before, and
|
||||
// this predicate gates a write, so it gets the same guard.
|
||||
func TestIsQuestionShapedIsTokenized(t *testing.T) {
|
||||
for _, text := range []string{"чтобы не забыть, я полил кактус", "какао выпил"} {
|
||||
if IsQuestionShaped(text) {
|
||||
t.Errorf("IsQuestionShaped(%q) = true; a question word inside a longer word is not a question", text)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -332,6 +333,20 @@ func (s *Store) VoidLatestFact(ctx context.Context, key, source string, ts time.
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("void: last insert id: %w", err)
|
||||
}
|
||||
// The other half of the repair (#470). A fact reaches recall through a
|
||||
// vector keyed `fact:<key>:<unix>`, holding the utterance that wrote it.
|
||||
// Voiding the row alone left that vector answering questions, so revert
|
||||
// reported success on a box that stayed broken. Deleting every vector for
|
||||
// the key covers the earlier rows too: their values are superseded, and a
|
||||
// superseded value has no business claiming a turn.
|
||||
//
|
||||
// Best-effort by design: the audit trail is already committed, and a fact
|
||||
// that is voided but still recallable is better than a void that failed.
|
||||
if n, derr := s.VectorMemory().DeletePrefix(ctx, "fact:"+key+":"); derr != nil {
|
||||
log.Printf("store: void %q: memory vectors survive: %v", key, derr)
|
||||
} else if n > 0 {
|
||||
log.Printf("store: void %q: dropped %d memory vector(s)", key, n)
|
||||
}
|
||||
return oldID, newID, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -148,6 +148,27 @@ func (m *MemoryStore) Delete(ctx context.Context, id string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeletePrefix removes every vector whose id starts with prefix and returns
|
||||
// how many went. Same escaping as ByPrefix, so a key containing % or _ cannot
|
||||
// widen the delete.
|
||||
//
|
||||
// It exists for the repair half of a revert (#470). Voiding a fact row left
|
||||
// its vector in the index, so recall kept serving the voided fact's utterance
|
||||
// and the documented repair did not repair.
|
||||
func (m *MemoryStore) DeletePrefix(ctx context.Context, prefix string) (int64, error) {
|
||||
pattern := escapeLike(prefix) + "%"
|
||||
res, err := m.db.ExecContext(ctx,
|
||||
`DELETE FROM memory_vectors WHERE id LIKE ? ESCAPE '\'`, pattern)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("memory: delete prefix %q: %w", prefix, err)
|
||||
}
|
||||
n, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("memory: delete prefix %q: rows affected: %w", prefix, err)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// escapeLike neutralises the LIKE wildcards in a literal prefix.
|
||||
func escapeLike(s string) string {
|
||||
r := strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`)
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Stage 3 of #470: reverting a fact reported success and left the vector that
|
||||
// was answering questions, so the documented repair did not repair.
|
||||
func TestVoidLatestFactDropsMemoryVectors(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newTestStore(t)
|
||||
now := time.Now()
|
||||
mem := s.VectorMemory()
|
||||
|
||||
if _, err := s.WriteFact(ctx, now, KindSelf, "go_version", `"1.20"`, "tap:voice", 1.0, sql.NullInt64{}); err != nil {
|
||||
t.Fatalf("WriteFact: %v", err)
|
||||
}
|
||||
// The id shape actionFact writes: fact:<key>:<unix>.
|
||||
if err := mem.Insert(ctx, "fact:go_version:1", []float32{1, 0, 0}, map[string]string{
|
||||
"type": "fact", "text": "какая последняя версия языка Go?",
|
||||
}); err != nil {
|
||||
t.Fatalf("Insert: %v", err)
|
||||
}
|
||||
// A vector for another key must survive the void.
|
||||
if err := mem.Insert(ctx, "fact:water:1", []float32{0, 1, 0}, map[string]string{
|
||||
"type": "fact", "text": "запиши что я пил воду",
|
||||
}); err != nil {
|
||||
t.Fatalf("Insert: %v", err)
|
||||
}
|
||||
|
||||
if _, _, err := s.VoidLatestFact(ctx, "go_version", "feedback", now.Add(time.Minute)); err != nil {
|
||||
t.Fatalf("VoidLatestFact: %v", err)
|
||||
}
|
||||
|
||||
got, err := mem.ByPrefix(ctx, "fact:")
|
||||
if err != nil {
|
||||
t.Fatalf("ByPrefix: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].ID != "fact:water:1" {
|
||||
t.Fatalf("after the void the index holds %+v; want only fact:water:1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeletePrefixDoesNotWidenOnWildcards(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newTestStore(t)
|
||||
mem := s.VectorMemory()
|
||||
|
||||
for _, id := range []string{"fact:a_b:1", "fact:axb:1"} {
|
||||
if err := mem.Insert(ctx, id, []float32{1, 0}, map[string]string{"type": "fact"}); err != nil {
|
||||
t.Fatalf("Insert %q: %v", id, err)
|
||||
}
|
||||
}
|
||||
n, err := mem.DeletePrefix(ctx, "fact:a_b:")
|
||||
if err != nil {
|
||||
t.Fatalf("DeletePrefix: %v", err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("deleted %d rows; the _ in the key must not match x", n)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user