Give the router prompt a destination to write (V-660)
V-659 measured the destination at 12/33 on the classifier cascade and named the gap: recall 0/15, because nothing anywhere names it. The model could not help, for a structural reason rather than a capability one. Nothing in routeSystem mentioned a Source and routeGrammar could not emit one, so there was no string for it to write. Same shape as the Praxis reach V-517 measured at 0/12. routeGrammar grows a source rule, closed over router.Sources plus the empty floor. A grammar cannot emit a destination that does not exist, which is the guarantee V-546 wants from a softmax and gets here for free. The prompt lists the twelve in Russian, one line each, and says plainly that "" is a normal answer to give often: two sources that can both answer means the chain walks, and guessing is the failure mode this whole field exists to stop. The read-back goes through ValidSource and runs on IntentQuery alone. The grammar already bounds the enum, but it is a request to a server that may be running another build, and only a query reaches queryWalk. Measured against gemma-4-12b on the workstation, same fixture, cascade with a hash fallback: destination 24/33 (72.7%) against the classifier's 12/33, and intent 81/96 (84.4%) which is where it already was. Recall is the whole move, 0/15 to 14/15. The model alone scores 26/33. Four cases the cascade loses and llm-only wins are calendar. The possessive agenda rules claim them at stage 0 and deliberately name nothing, because "что у меня в списке покупок" matches the same rule and naming the calendar would take the list source off the turn. So stage 0's caution now costs four destination points it did not cost before. That is a real trade and it wants its own argument, not a quiet edit here. The resident Qwen3-1.7B is unmeasured: it binds --port 0 inside the container and no host process can reach it. llm/check_prompt_parity.py in the training workspace compares its copy of routeSystem to this one and will fail until that copy gets the same edit. V-362 covers the catch-up. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013ptwopxyo3Z2kwFckHkLvN
This commit is contained in:
@@ -398,3 +398,70 @@ func TestLLMReminderWithSubjectIsNotGated(t *testing.T) {
|
||||
t.Fatalf("a complete reminder was sent back as a question: %+v", d.Slots)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRouteGrammarCoversSources — the grammar enum and router.Sources are two
|
||||
// hand-written lists of the same twelve destinations, and nothing else notices
|
||||
// when one grows. A destination missing from the grammar is a destination the
|
||||
// model is structurally unable to name, which is the exact defect V-517
|
||||
// measured for Praxis: not a weak model, an absent string.
|
||||
func TestRouteGrammarCoversSources(t *testing.T) {
|
||||
for _, s := range Sources {
|
||||
if !strings.Contains(routeGrammar, `"\"`+string(s)+`\""`) {
|
||||
t.Errorf("routeGrammar cannot emit %q — the model can never name it", s)
|
||||
}
|
||||
}
|
||||
// The floor has to be reachable too, or the model is forced to pick one.
|
||||
if !strings.Contains(routeGrammar, `"\"\""`) {
|
||||
t.Error(`routeGrammar cannot emit "" — the model cannot decline a destination`)
|
||||
}
|
||||
// Count the alternatives on the source rule: an extra one is a destination
|
||||
// the daemon would drop to SourceUnknown after the model spent tokens on it.
|
||||
for _, line := range strings.Split(routeGrammar, "\n") {
|
||||
if !strings.HasPrefix(line, "source ") {
|
||||
continue
|
||||
}
|
||||
if got, want := strings.Count(line, "|")+1, len(Sources)+1; got != want {
|
||||
t.Errorf("source rule has %d alternatives, want %d (Sources plus the floor)", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The destination is read back only through ValidSource. A model on an older or
|
||||
// newer build can write a string this binary does not know, and trusting it
|
||||
// would take real query sources off the turn for a name nothing answers.
|
||||
func TestLLMUnknownSourceFallsToTheFloor(t *testing.T) {
|
||||
r := newLLMTestRouter(t, `{"intent":"query","text":"что там с бэкапами","source":"praxis"}`)
|
||||
d, err := r.Route(context.Background(), "что там с бэкапами", refNow())
|
||||
if err != nil {
|
||||
t.Fatalf("route: %v", err)
|
||||
}
|
||||
if d.Source != SourceUnknown {
|
||||
t.Fatalf("invented destination %q was trusted, want the floor", d.Source)
|
||||
}
|
||||
}
|
||||
|
||||
// And a known one survives, or the read-back is just a filter.
|
||||
func TestLLMNamedSourceSurvives(t *testing.T) {
|
||||
r := newLLMTestRouter(t, `{"intent":"query","text":"кто такой Линус Торвальдс","source":"world"}`)
|
||||
d, err := r.Route(context.Background(), "кто такой Линус Торвальдс?", refNow())
|
||||
if err != nil {
|
||||
t.Fatalf("route: %v", err)
|
||||
}
|
||||
if d.Source != SourceWorld {
|
||||
t.Fatalf("source %q, want %q", d.Source, SourceWorld)
|
||||
}
|
||||
}
|
||||
|
||||
// A destination on anything but a query is dropped. Only IntentQuery reaches
|
||||
// queryWalk, so a source elsewhere is a field nobody reads and a claim nobody
|
||||
// checks.
|
||||
func TestLLMSourceIsQueryOnly(t *testing.T) {
|
||||
r := newLLMTestRouter(t, `{"intent":"note","text":"кофе кончился","source":"recall"}`)
|
||||
d, err := r.Route(context.Background(), "запиши что кофе кончился", refNow())
|
||||
if err != nil {
|
||||
t.Fatalf("route: %v", err)
|
||||
}
|
||||
if d.Source != SourceUnknown {
|
||||
t.Fatalf("a note carried destination %q", d.Source)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user