From bd16ca69e529d5bb73d3727a25eff9dc0ab5a143 Mon Sep 17 00:00:00 2001 From: kami Date: Fri, 31 Jul 2026 11:35:37 +0400 Subject: [PATCH] Let the LLM router answer "unknown" when it cannot route MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Chose an 8th enum value over a confidence number: the model already picks one enum token, so it costs nothing in the grammar, while a score from a 0.8B model would be uncalibrated noise. A refusal returns "no decision" with no error, which is the fall-through the caller already uses for a bad parse, so the classifier and its clarify gate take the turn. Reviewers: the prompt's counter-examples matter most — a small model will over-use any easy escape hatch. The training workspace copy of the prompt still needs the same edit (Vikunja #362). Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CGeSZxh1DCtRxmFVSYVGvJ --- internal/config/config.go | 14 ++++---- internal/router/llmrouter.go | 36 +++++++++++++++++-- internal/router/llmrouter_test.go | 59 ++++++++++++++++++++++++++++++- 3 files changed, 98 insertions(+), 11 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index aa98fd5..5205f4b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -262,13 +262,13 @@ type VoiceConfig struct { // the model gets 50.0% of intents right against the classifier's 36.8%, but // it costs about 800ms per turn instead of 30ms. // - // TODO: the default stays false until two things land. - // 1. The LLM router cannot refuse. LLMRouter.Route hardcodes - // Confidence: 1.0, so the stage-3 clarify gate never fires and an - // unclear utterance becomes a confident wrong action (Vikunja #359). - // 2. Extractor.Extract never runs on an LLM decision, so acts arrive with - // no Fn and reminders with no Time. - // Turning this on today makes routing more accurate and less safe. + // TODO: the default stays false until this lands. + // Extractor.Extract never runs on an LLM decision, so acts arrive with no + // Fn and reminders with no Time. Turning this on today makes routing more + // accurate and less safe. + // + // The router can now refuse: it answers "unknown" when it cannot route, and + // the turn drops to the classifier and its clarify gate (Vikunja #359). LLMRouter bool `json:"llm_router,omitempty"` // QueryMinScore — the note-recall confidence gate. Top cosine below this diff --git a/internal/router/llmrouter.go b/internal/router/llmrouter.go index 5fdef26..36a921a 100644 --- a/internal/router/llmrouter.go +++ b/internal/router/llmrouter.go @@ -29,7 +29,7 @@ func NewLLMRouter(c Completer) *LLMRouter { return &LLMRouter{c: c} } const routeGrammar = ` root ::= "[" ws action ("," ws action)* ws "]" action ::= "{" ws "\"intent\"" ws ":" ws intent ("," ws field)* ws "}" -intent ::= "\"fact\"" | "\"reminder\"" | "\"note\"" | "\"query\"" | "\"act\"" | "\"chat\"" | "\"system\"" +intent ::= "\"fact\"" | "\"reminder\"" | "\"note\"" | "\"query\"" | "\"act\"" | "\"chat\"" | "\"system\"" | "\"unknown\"" field ::= key ws ":" ws string key ::= "\"key\"" | "\"value\"" | "\"text\"" | "\"verb\"" string ::= "\"" ([^"\\] | "\\" .){0,120} "\"" @@ -41,12 +41,17 @@ ws ::= [ \t\n]* // question naming a fact key ("сколько воды я выпил с утра") matched the fact // rule first and was stored as an assertion — 15 of 76 fixture cases. // +// Changed again 31-07-2026: added the "unknown" escape hatch so the model can +// admit it cannot route (Vikunja #359). +// // The training workspace keeps its own copy of this prompt for relabelling, and // `llm/check_prompt_parity.py` there compares the two. That copy is in another -// repo and was not touched, so parity will fail until it gets the same edit. +// repo and was not touched, so parity will fail until it gets the same edits — +// both the rule reorder and the "unknown" wording (Vikunja #362). const routeSystem = `Классифицируй ровно одно сообщение пользователя. Верни ОДИН JSON-массив действий. Ровно одно намерение: fact, reminder, note, query, act, chat, system. +Есть восьмое значение unknown — только для случаев, когда просьбу невозможно понять. Классифицируй по цели пользователя. Порядок решения: 1. Хочет напоминание в будущем → reminder @@ -56,12 +61,14 @@ const routeSystem = `Классифицируй ровно одно сообще 5. Утверждает: сообщает или обновляет текущее состояние/событие → fact 6. Просит выполнить работу → act 7. Про ассистента, настройки или память → system -8. Иначе → chat +8. Реплика — обрывок или указание на неназванное («это», «то», «потом»), и без него непонятно, что именно нужно сделать → unknown +9. Иначе → chat Различия: - note — сохранить информацию, без напоминания. text = суть. - reminder — уведомить позже. text = что напомнить. - fact — неявное обновление: пользователь сообщает, что что-то в мире изменилось (текущее/изменённое состояние, случившееся событие). key/value. +- unknown — редкий случай. Ставь его, только если в самой реплике нет ни предмета, ни действия. Короткая, простая или незнакомая тема — это не причина для unknown: приветствие и болтовня — это chat, вопрос на любую тему — это query, просьба сделать что-то названное — это act. - query против fact — решает форма реплики, а не тема. Вопрос о состоянии — это query, даже если названо то же самое, что бывает в fact. Только утверждение — это fact. Примеры: @@ -76,6 +83,13 @@ const routeSystem = `Классифицируй ровно одно сообще "напиши письмо" → {"intent":"act","verb":"написать письмо"} "очисти память" → {"intent":"system"} "привет" → {"intent":"chat","text":"привет"} +"сделай это" → {"intent":"unknown"} +"ну это" → {"intent":"unknown"} +"потом" → {"intent":"unknown"} +Но не путай — здесь unknown не нужен: +"сделай кофе" → {"intent":"act","verb":"сделать кофе"} +"что такое кватернион?" → {"intent":"query","text":"что такое кватернион"} +"ага" → {"intent":"chat","text":"ага"} Ответ — JSON-массив: по одному объекту на каждую просьбу. Обычно один. Если в реплике несколько просьб — по объекту на каждую. "напомни купить молоко, и запиши что кофе кончился" → [{"intent":"reminder","text":"купить молоко"},{"intent":"note","text":"кофе кончился"}]. Только JSON, без пояснений.` @@ -84,6 +98,11 @@ const routeSystem = `Классифицируй ровно одно сообще // the loop without hurting short slot values. const routeRepeatPenalty = 1.15 +// routeIntentUnknown — the model's way of saying "I could not route this". +// It is a wire value only: it never becomes a router.Intent, it just makes +// Route return ok=false so the caller drops to the classifier cascade. +const routeIntentUnknown = "unknown" + type routeAction struct { Intent string `json:"intent"` Key string `json:"key"` @@ -92,6 +111,10 @@ type routeAction struct { Verb string `json:"verb"` } +// Route asks the model for one decision. The bool is false when there is no +// decision to use: either the model failed (err set) or it refused with the +// "unknown" intent (err nil). Both mean the same thing to the caller — use the +// classifier instead. func (lr *LLMRouter) Route(ctx context.Context, utterance string, now time.Time) (Decision, bool, error) { raw, err := lr.c.Complete(ctx, llm.Req{ System: routeSystem, @@ -115,6 +138,13 @@ func (lr *LLMRouter) Route(ctx context.Context, utterance string, now time.Time) // with the engine turn-on (Router.Route → []Decision, both voice.go handlers // loop). Until then only the first ask is honored. a := acts[0] + // The model refused. Report "no decision" without an error, which is the + // same fall-through the caller already uses for a parse failure — the + // classifier cascade gets the turn and its own confidence gate decides + // whether to ask. Better a slower second opinion than a confident guess. + if a.Intent == routeIntentUnknown { + return Decision{}, false, nil + } d := Decision{Utterance: utterance, Stage: 1, Confidence: 1.0} switch Intent(a.Intent) { case IntentFact: diff --git a/internal/router/llmrouter_test.go b/internal/router/llmrouter_test.go index 7c73852..0680acc 100644 --- a/internal/router/llmrouter_test.go +++ b/internal/router/llmrouter_test.go @@ -100,8 +100,10 @@ func TestLLMRouterReminderMapping(t *testing.T) { } } +// An intent name that is not in the contract at all (as opposed to "unknown", +// which is a real refusal) still defaults to chat. func TestLLMRouterChatFallback(t *testing.T) { - lr := NewLLMRouter(mockLLM{out: `{"intent":"unknown"}`}) + lr := NewLLMRouter(mockLLM{out: `{"intent":"banana"}`}) d, ok, err := lr.Route(context.Background(), "как дела?", time.Now()) if err != nil || !ok { t.Fatalf("ok=%v err=%v", ok, err) @@ -111,6 +113,61 @@ func TestLLMRouterChatFallback(t *testing.T) { } } +// The model must be able to say "I could not route this". +func TestRouteGrammarAllowsUnknown(t *testing.T) { + if !strings.Contains(routeGrammar, `"\"unknown\""`) { + t.Fatal("grammar cannot express a refusal") + } +} + +// If the prompt does not tell the model when to refuse, it never will. +func TestRoutePromptExplainsUnknown(t *testing.T) { + if !strings.Contains(routeSystem, "unknown") { + t.Fatal("prompt never mentions the unknown intent") + } + if !strings.Contains(routeSystem, `"сделай это" → {"intent":"unknown"}`) { + t.Fatal("prompt lost its worked refusal example") + } + // A refusal-only router is useless, so the prompt must also show cases that + // look ambiguous but are not. + if !strings.Contains(routeSystem, "здесь unknown не нужен") { + t.Fatal("prompt lost its counter-examples") + } +} + +// A refusal is not an error. It reports "no decision" so the cascade moves on. +func TestLLMRouterUnknownRefuses(t *testing.T) { + lr := NewLLMRouter(mockLLM{out: `{"intent":"unknown"}`}) + _, ok, err := lr.Route(context.Background(), "сделай это", time.Now()) + if ok { + t.Fatal("a refusal must not produce a usable decision") + } + if err != nil { + t.Fatalf("a refusal is not an error, got %v", err) + } +} + +// The whole point of the refusal: the turn keeps going on the classifier, the +// same way it does when the model returns garbage. +func TestRouterFallsBackWhenLLMRefuses(t *testing.T) { + c := NewClassifier(NewHashEmbedder(1024)) + seedClassifier(t, c) + r := New(Config{ + Classifier: c, + Extractor: Extractor{Time: StubDateTimeParser{}, Facts: DefaultFactParser{}}, + Threshold: 0.4, + LLM: NewLLMRouter(mockLLM{out: `{"intent":"unknown"}`}), + }) + d, err := r.Route(context.Background(), "напомни позвонить маме", refNow()) + if err != nil { + t.Fatalf("route: %v", err) + } + // Stage 1 is the LLM's own answer; the classifier lands on stage 2 or 3. + if d.Stage < 2 { + t.Fatalf("want the classifier to decide, got stage %d (%+v)", d.Stage, d) + } +} + func TestLLMRouterLLMError(t *testing.T) { lr := NewLLMRouter(mockLLM{out: "", err: fmt.Errorf("llm down")}) _, ok, err := lr.Route(context.Background(), "x", time.Now())