Let the LLM router answer "unknown" when it cannot route
Chose an 8th enum value over a confidence number: the model already picks one enum token, so it costs nothing in the grammar, while a score from a 0.8B model would be uncalibrated noise. A refusal returns "no decision" with no error, which is the fall-through the caller already uses for a bad parse, so the classifier and its clarify gate take the turn. Reviewers: the prompt's counter-examples matter most — a small model will over-use any easy escape hatch. The training workspace copy of the prompt still needs the same edit (Vikunja #362). Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CGeSZxh1DCtRxmFVSYVGvJ
This commit is contained in:
@@ -100,8 +100,10 @@ func TestLLMRouterReminderMapping(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// An intent name that is not in the contract at all (as opposed to "unknown",
|
||||
// which is a real refusal) still defaults to chat.
|
||||
func TestLLMRouterChatFallback(t *testing.T) {
|
||||
lr := NewLLMRouter(mockLLM{out: `{"intent":"unknown"}`})
|
||||
lr := NewLLMRouter(mockLLM{out: `{"intent":"banana"}`})
|
||||
d, ok, err := lr.Route(context.Background(), "как дела?", time.Now())
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("ok=%v err=%v", ok, err)
|
||||
@@ -111,6 +113,61 @@ func TestLLMRouterChatFallback(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The model must be able to say "I could not route this".
|
||||
func TestRouteGrammarAllowsUnknown(t *testing.T) {
|
||||
if !strings.Contains(routeGrammar, `"\"unknown\""`) {
|
||||
t.Fatal("grammar cannot express a refusal")
|
||||
}
|
||||
}
|
||||
|
||||
// If the prompt does not tell the model when to refuse, it never will.
|
||||
func TestRoutePromptExplainsUnknown(t *testing.T) {
|
||||
if !strings.Contains(routeSystem, "unknown") {
|
||||
t.Fatal("prompt never mentions the unknown intent")
|
||||
}
|
||||
if !strings.Contains(routeSystem, `"сделай это" → {"intent":"unknown"}`) {
|
||||
t.Fatal("prompt lost its worked refusal example")
|
||||
}
|
||||
// A refusal-only router is useless, so the prompt must also show cases that
|
||||
// look ambiguous but are not.
|
||||
if !strings.Contains(routeSystem, "здесь unknown не нужен") {
|
||||
t.Fatal("prompt lost its counter-examples")
|
||||
}
|
||||
}
|
||||
|
||||
// A refusal is not an error. It reports "no decision" so the cascade moves on.
|
||||
func TestLLMRouterUnknownRefuses(t *testing.T) {
|
||||
lr := NewLLMRouter(mockLLM{out: `{"intent":"unknown"}`})
|
||||
_, ok, err := lr.Route(context.Background(), "сделай это", time.Now())
|
||||
if ok {
|
||||
t.Fatal("a refusal must not produce a usable decision")
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("a refusal is not an error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The whole point of the refusal: the turn keeps going on the classifier, the
|
||||
// same way it does when the model returns garbage.
|
||||
func TestRouterFallsBackWhenLLMRefuses(t *testing.T) {
|
||||
c := NewClassifier(NewHashEmbedder(1024))
|
||||
seedClassifier(t, c)
|
||||
r := New(Config{
|
||||
Classifier: c,
|
||||
Extractor: Extractor{Time: StubDateTimeParser{}, Facts: DefaultFactParser{}},
|
||||
Threshold: 0.4,
|
||||
LLM: NewLLMRouter(mockLLM{out: `{"intent":"unknown"}`}),
|
||||
})
|
||||
d, err := r.Route(context.Background(), "напомни позвонить маме", refNow())
|
||||
if err != nil {
|
||||
t.Fatalf("route: %v", err)
|
||||
}
|
||||
// Stage 1 is the LLM's own answer; the classifier lands on stage 2 or 3.
|
||||
if d.Stage < 2 {
|
||||
t.Fatalf("want the classifier to decide, got stage %d (%+v)", d.Stage, d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMRouterLLMError(t *testing.T) {
|
||||
lr := NewLLMRouter(mockLLM{out: "", err: fmt.Errorf("llm down")})
|
||||
_, ok, err := lr.Route(context.Background(), "x", time.Now())
|
||||
|
||||
Reference in New Issue
Block a user