Let the LLM router answer "unknown" when it cannot route

Chose an 8th enum value over a confidence number: the model already picks
one enum token, so it costs nothing in the grammar, while a score from a
0.8B model would be uncalibrated noise. A refusal returns "no decision"
with no error, which is the fall-through the caller already uses for a
bad parse, so the classifier and its clarify gate take the turn.

Reviewers: the prompt's counter-examples matter most — a small model will
over-use any easy escape hatch. The training workspace copy of the prompt
still needs the same edit (Vikunja #362).

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CGeSZxh1DCtRxmFVSYVGvJ
This commit is contained in:
kami
2026-07-31 11:35:37 +04:00
parent c47886c2bc
commit bd16ca69e5
3 changed files with 98 additions and 11 deletions
+58 -1
View File
@@ -100,8 +100,10 @@ func TestLLMRouterReminderMapping(t *testing.T) {
}
}
// An intent name that is not in the contract at all (as opposed to "unknown",
// which is a real refusal) still defaults to chat.
func TestLLMRouterChatFallback(t *testing.T) {
lr := NewLLMRouter(mockLLM{out: `{"intent":"unknown"}`})
lr := NewLLMRouter(mockLLM{out: `{"intent":"banana"}`})
d, ok, err := lr.Route(context.Background(), "как дела?", time.Now())
if err != nil || !ok {
t.Fatalf("ok=%v err=%v", ok, err)
@@ -111,6 +113,61 @@ func TestLLMRouterChatFallback(t *testing.T) {
}
}
// The model must be able to say "I could not route this".
func TestRouteGrammarAllowsUnknown(t *testing.T) {
if !strings.Contains(routeGrammar, `"\"unknown\""`) {
t.Fatal("grammar cannot express a refusal")
}
}
// If the prompt does not tell the model when to refuse, it never will.
func TestRoutePromptExplainsUnknown(t *testing.T) {
if !strings.Contains(routeSystem, "unknown") {
t.Fatal("prompt never mentions the unknown intent")
}
if !strings.Contains(routeSystem, `"сделай это" → {"intent":"unknown"}`) {
t.Fatal("prompt lost its worked refusal example")
}
// A refusal-only router is useless, so the prompt must also show cases that
// look ambiguous but are not.
if !strings.Contains(routeSystem, "здесь unknown не нужен") {
t.Fatal("prompt lost its counter-examples")
}
}
// A refusal is not an error. It reports "no decision" so the cascade moves on.
func TestLLMRouterUnknownRefuses(t *testing.T) {
lr := NewLLMRouter(mockLLM{out: `{"intent":"unknown"}`})
_, ok, err := lr.Route(context.Background(), "сделай это", time.Now())
if ok {
t.Fatal("a refusal must not produce a usable decision")
}
if err != nil {
t.Fatalf("a refusal is not an error, got %v", err)
}
}
// The whole point of the refusal: the turn keeps going on the classifier, the
// same way it does when the model returns garbage.
func TestRouterFallsBackWhenLLMRefuses(t *testing.T) {
c := NewClassifier(NewHashEmbedder(1024))
seedClassifier(t, c)
r := New(Config{
Classifier: c,
Extractor: Extractor{Time: StubDateTimeParser{}, Facts: DefaultFactParser{}},
Threshold: 0.4,
LLM: NewLLMRouter(mockLLM{out: `{"intent":"unknown"}`}),
})
d, err := r.Route(context.Background(), "напомни позвонить маме", refNow())
if err != nil {
t.Fatalf("route: %v", err)
}
// Stage 1 is the LLM's own answer; the classifier lands on stage 2 or 3.
if d.Stage < 2 {
t.Fatalf("want the classifier to decide, got stage %d (%+v)", d.Stage, d)
}
}
func TestLLMRouterLLMError(t *testing.T) {
lr := NewLLMRouter(mockLLM{out: "", err: fmt.Errorf("llm down")})
_, ok, err := lr.Route(context.Background(), "x", time.Now())