From c31f0d10011701c741995093eedd269908a16364 Mon Sep 17 00:00:00 2001 From: kami Date: Fri, 31 Jul 2026 11:44:06 +0400 Subject: [PATCH] Extract slots for LLM router decisions too An LLM-routed reminder came back with no parsed time and an act with no fn, because only the classifier path ran the extractor. Now the router runs the same extraction after an LLM decision and fills only the empty slots. No time in the utterance still means no time. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CGeSZxh1DCtRxmFVSYVGvJ --- internal/router/llmrouter_test.go | 84 +++++++++++++++++++++++++++++++ internal/router/router.go | 34 +++++++++++++ 2 files changed, 118 insertions(+) diff --git a/internal/router/llmrouter_test.go b/internal/router/llmrouter_test.go index 0680acc..658dab7 100644 --- a/internal/router/llmrouter_test.go +++ b/internal/router/llmrouter_test.go @@ -175,3 +175,87 @@ func TestLLMRouterLLMError(t *testing.T) { t.Fatal("want ok=false, err!=nil on llm error") } } + +// --- slot extraction on top of an LLM decision -------------------------------- + +// newLLMTestRouter — a router whose route always comes from the mock model. +func newLLMTestRouter(t *testing.T, out string) *Router { + t.Helper() + c := NewClassifier(NewHashEmbedder(1024)) + seedClassifier(t, c) + acts := DefaultActMatcher{Fns: []string{"restart", "stop", "run", "backup"}} + return New(Config{ + Classifier: c, + Extractor: Extractor{Time: StubDateTimeParser{}, Acts: acts, Facts: DefaultFactParser{}}, + Threshold: 0.4, + LLM: NewLLMRouter(mockLLM{out: out}), + }) +} + +// The model cannot produce a fire time, so without extraction every LLM-routed +// reminder was dropped as "no time". +func TestLLMDecisionGetsReminderTime(t *testing.T) { + r := newLLMTestRouter(t, `{"intent":"reminder","text":"позвонить маме"}`) + d, err := r.Route(context.Background(), "напомни позвонить маме через 2 часа", refNow()) + if err != nil { + t.Fatalf("route: %v", err) + } + if d.Intent != IntentReminder { + t.Fatalf("want reminder, got %v", d.Intent) + } + if !d.Slots.HasTime || !d.Slots.Time.Equal(refNow().Add(2*time.Hour)) { + t.Fatalf("want time now+2h, got %+v", d.Slots) + } + if d.Slots.Text != "позвонить маме" { + t.Fatalf("extraction overwrote the model's text: %q", d.Slots.Text) + } +} + +// No time in the utterance ⇒ no time in the slots. Do not invent one; the +// daemon says it could not read the time. +func TestLLMReminderWithoutTimeStaysEmpty(t *testing.T) { + r := newLLMTestRouter(t, `{"intent":"reminder","text":"позвонить маме"}`) + d, err := r.Route(context.Background(), "напомни позвонить маме", refNow()) + if err != nil { + t.Fatalf("route: %v", err) + } + if d.Slots.HasTime { + t.Fatalf("invented a time: %v", d.Slots.Time) + } +} + +// An act decision arrived with no Fn, so the tool never ran. +func TestLLMDecisionGetsActFn(t *testing.T) { + r := newLLMTestRouter(t, `{"intent":"act","verb":"restart nginx"}`) + d, err := r.Route(context.Background(), "слушай, restart nginx пожалуйста", refNow()) + if err != nil { + t.Fatalf("route: %v", err) + } + if !d.Slots.HasFn || d.Slots.Fn != "restart" || len(d.Slots.Args) != 1 || d.Slots.Args[0] != "nginx" { + t.Fatalf("want fn=restart args=[nginx], got %+v", d.Slots) + } +} + +// The model's own slots win; extraction only fills gaps. +func TestLLMSlotsWinOverExtraction(t *testing.T) { + r := newLLMTestRouter(t, `{"intent":"fact","key":"hydration","value":"выпил"}`) + d, err := r.Route(context.Background(), "я выпил воду", refNow()) + if err != nil { + t.Fatalf("route: %v", err) + } + if d.Slots.Key != "hydration" { + t.Fatalf("extraction overwrote the model's key: %q", d.Slots.Key) + } +} + +// A fact the model left keyless still gets one from the parser. +func TestLLMFactGetsKeyFromParser(t *testing.T) { + r := newLLMTestRouter(t, `{"intent":"fact","text":"я выпил воду"}`) + d, err := r.Route(context.Background(), "я выпил воду", refNow()) + if err != nil { + t.Fatalf("route: %v", err) + } + if !d.Slots.HasKey || d.Slots.Key != "water" { + t.Fatalf("want key=water, got %+v", d.Slots) + } +} diff --git a/internal/router/router.go b/internal/router/router.go index 349534d..4b8fd18 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -88,6 +88,7 @@ func (r *Router) Route(ctx context.Context, utterance string, now time.Time) (De if r.llm != nil { if d, ok, err := r.llm.Route(ctx, utterance, now); err == nil && ok { d.Utterance = utterance + r.fillSlots(ctx, &d, now) return d, nil } else if err != nil { log.Printf("router: llm route fell back to classifier: %v", err) @@ -118,6 +119,39 @@ func (r *Router) Route(ctx context.Context, utterance string, now time.Time) (De return d, nil } +// fillSlots — run stage-2 extraction on an LLM decision and fill only the slots +// the model left empty. The LLM wins where it answered: it saw the sentence, the +// parsers are keyword tables. Extraction covers what the model cannot produce at +// all — a parsed reminder time and an allowlist fn. +// +// If a reminder still has no time, leave it missing. The daemon then says it +// could not read the time; inventing one would set a wrong alarm. +func (r *Router) fillSlots(ctx context.Context, d *Decision, now time.Time) { + ex := r.extractor.Extract(ctx, d.Intent, d.Utterance, now) + if !d.Slots.HasTime && ex.HasTime { + d.Slots.Time, d.Slots.HasTime = ex.Time, ex.HasTime + } + if !d.Slots.HasKey && ex.HasKey { + d.Slots.Key, d.Slots.Value, d.Slots.HasKey = ex.Key, ex.Value, ex.HasKey + } + if !d.Slots.HasFn && ex.HasFn { + d.Slots.Fn, d.Slots.Args, d.Slots.HasFn = ex.Fn, ex.Args, ex.HasFn + } + // For an act the model returns the verb in Text ("restart nginx"), which is + // often cleaner than the raw utterance ("maven, could you restart nginx"). + // Try it too when the utterance did not match the allowlist. + if d.Intent == IntentAct && !d.Slots.HasFn && r.extractor.Acts != nil && + d.Slots.Text != "" && d.Slots.Text != d.Utterance { + if fn, args, ok := r.extractor.Acts.Match(d.Slots.Text); ok { + d.Slots.Fn, d.Slots.Args, d.Slots.HasFn = fn, args, true + } + } + if d.Slots.Text == "" { + d.Slots.Text = ex.Text + } + // Stage stays 1: it says who decided the route, and that was the LLM. +} + // CorrectMisroute — the user corrected a bad classification. Appends a new // example for the corrected intent (append-only — grows the classifier, no // retrain). Same shape as nudges.outcome tuning cooldowns: more reliable over