claude
9e25f18a3e
#496 asked to skip the veto when the question and the hit are in different scripts, so an English question stops losing a Russian note. Measured first: the fixture has no cross-language case, and en-hard-024 is an English question against an English note. Both proposed fixes are no-ops. What the veto actually does on the fixture, with the real embedder: it costs en-hard-024 and buys ru-silent-029. Pass count is 22/32 either way; false recall is 0/5 with it and 1/5 without. The two cases are one lexical class, so no rule cheap enough for RecallAllowed separates them. Accepts the loss and pins both sides in a test. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
57 lines
2.8 KiB
Go
57 lines
2.8 KiB
Go
package memory
|
|
|
|
import "testing"
|
|
|
|
func TestRecallAllowed(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
query, text string
|
|
want bool
|
|
}{
|
|
// The #470 shape: a world question and a note about his box.
|
|
{"world question, unrelated note", "почему небо синее", "сеть какая-то медленная", false},
|
|
{"world question, unrelated fact", "какая столица Франции", "какая последняя версия языка Go", false},
|
|
{"silent fixture case", "во сколько отходит поезд", "бэкап запускается в три ночи", false},
|
|
|
|
// A world question that does name the topic keeps its answer.
|
|
{"world question, same topic", "какой поезд идёт в Минск", "поезда в Минск ходят утром", true},
|
|
|
|
// A question about his own life is judged by the embedder alone,
|
|
// because recall exists for words he no longer remembers.
|
|
{"about him, no shared word", "во сколько я обычно засыпаю", "ложусь около одиннадцати", true},
|
|
{"about him, english", "which colour scheme do i like", "тёмная тема везде", true},
|
|
|
|
// Inflection must not break a match.
|
|
{"inflected", "чем кормить кота", "корм для кота в шкафу", true},
|
|
}
|
|
for _, c := range cases {
|
|
if got := RecallAllowed(c.query, c.text); got != c.want {
|
|
t.Errorf("%s: RecallAllowed(%q, %q) = %v, want %v", c.name, c.query, c.text, got, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The known cost of the veto and the thing that pays for it, both measured on
|
|
// the held-out fixture with the real embedder (#496,
|
|
// docs/evals/2026-08-03-recall-topic-veto.md). The two are one lexical class:
|
|
// zero shared content words, no first-person marker, scores 0.826 against 0.835
|
|
// and margins 0.023 against 0.019. Recovering the first re-admits the second,
|
|
// which puts false recall back to 1/5. Anyone loosening the veto has to move
|
|
// the first line without moving the second.
|
|
func TestRecallVetoTradeIsPinned(t *testing.T) {
|
|
if RecallAllowed("what fixed the screen problem", "the flicker went away once i swapped the display cable") {
|
|
t.Error("en-hard-024 is expected to stay vetoed — if this passes now, re-measure false recall before celebrating")
|
|
}
|
|
if RecallAllowed("во сколько отходит поезд", "погулял вдоль реки") {
|
|
t.Error("ru-silent-029 must stay vetoed — this is the false recall the veto exists to stop")
|
|
}
|
|
}
|
|
|
|
// A question made only of filler has no topic word to match on, and the score
|
|
// gate is then the only judge it can have.
|
|
func TestRecallAllowedFallsBackWhenNothingToCompare(t *testing.T) {
|
|
if !RecallAllowed("что это", "сеть какая-то медленная") {
|
|
t.Error("a question with no content word must not be vetoed")
|
|
}
|
|
}
|