diff --git a/internal/router/llmrouter_test.go b/internal/router/llmrouter_test.go index 0680acc..658dab7 100644 --- a/internal/router/llmrouter_test.go +++ b/internal/router/llmrouter_test.go @@ -175,3 +175,87 @@ func TestLLMRouterLLMError(t *testing.T) { t.Fatal("want ok=false, err!=nil on llm error") } } + +// --- slot extraction on top of an LLM decision -------------------------------- + +// newLLMTestRouter — a router whose route always comes from the mock model. +func newLLMTestRouter(t *testing.T, out string) *Router { + t.Helper() + c := NewClassifier(NewHashEmbedder(1024)) + seedClassifier(t, c) + acts := DefaultActMatcher{Fns: []string{"restart", "stop", "run", "backup"}} + return New(Config{ + Classifier: c, + Extractor: Extractor{Time: StubDateTimeParser{}, Acts: acts, Facts: DefaultFactParser{}}, + Threshold: 0.4, + LLM: NewLLMRouter(mockLLM{out: out}), + }) +} + +// The model cannot produce a fire time, so without extraction every LLM-routed +// reminder was dropped as "no time". +func TestLLMDecisionGetsReminderTime(t *testing.T) { + r := newLLMTestRouter(t, `{"intent":"reminder","text":"позвонить маме"}`) + d, err := r.Route(context.Background(), "напомни позвонить маме через 2 часа", refNow()) + if err != nil { + t.Fatalf("route: %v", err) + } + if d.Intent != IntentReminder { + t.Fatalf("want reminder, got %v", d.Intent) + } + if !d.Slots.HasTime || !d.Slots.Time.Equal(refNow().Add(2*time.Hour)) { + t.Fatalf("want time now+2h, got %+v", d.Slots) + } + if d.Slots.Text != "позвонить маме" { + t.Fatalf("extraction overwrote the model's text: %q", d.Slots.Text) + } +} + +// No time in the utterance ⇒ no time in the slots. Do not invent one; the +// daemon says it could not read the time. +func TestLLMReminderWithoutTimeStaysEmpty(t *testing.T) { + r := newLLMTestRouter(t, `{"intent":"reminder","text":"позвонить маме"}`) + d, err := r.Route(context.Background(), "напомни позвонить маме", refNow()) + if err != nil { + t.Fatalf("route: %v", err) + } + if d.Slots.HasTime { + t.Fatalf("invented a time: %v", d.Slots.Time) + } +} + +// An act decision arrived with no Fn, so the tool never ran. +func TestLLMDecisionGetsActFn(t *testing.T) { + r := newLLMTestRouter(t, `{"intent":"act","verb":"restart nginx"}`) + d, err := r.Route(context.Background(), "слушай, restart nginx пожалуйста", refNow()) + if err != nil { + t.Fatalf("route: %v", err) + } + if !d.Slots.HasFn || d.Slots.Fn != "restart" || len(d.Slots.Args) != 1 || d.Slots.Args[0] != "nginx" { + t.Fatalf("want fn=restart args=[nginx], got %+v", d.Slots) + } +} + +// The model's own slots win; extraction only fills gaps. +func TestLLMSlotsWinOverExtraction(t *testing.T) { + r := newLLMTestRouter(t, `{"intent":"fact","key":"hydration","value":"выпил"}`) + d, err := r.Route(context.Background(), "я выпил воду", refNow()) + if err != nil { + t.Fatalf("route: %v", err) + } + if d.Slots.Key != "hydration" { + t.Fatalf("extraction overwrote the model's key: %q", d.Slots.Key) + } +} + +// A fact the model left keyless still gets one from the parser. +func TestLLMFactGetsKeyFromParser(t *testing.T) { + r := newLLMTestRouter(t, `{"intent":"fact","text":"я выпил воду"}`) + d, err := r.Route(context.Background(), "я выпил воду", refNow()) + if err != nil { + t.Fatalf("route: %v", err) + } + if !d.Slots.HasKey || d.Slots.Key != "water" { + t.Fatalf("want key=water, got %+v", d.Slots) + } +} diff --git a/internal/router/router.go b/internal/router/router.go index 349534d..4b8fd18 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -88,6 +88,7 @@ func (r *Router) Route(ctx context.Context, utterance string, now time.Time) (De if r.llm != nil { if d, ok, err := r.llm.Route(ctx, utterance, now); err == nil && ok { d.Utterance = utterance + r.fillSlots(ctx, &d, now) return d, nil } else if err != nil { log.Printf("router: llm route fell back to classifier: %v", err) @@ -118,6 +119,39 @@ func (r *Router) Route(ctx context.Context, utterance string, now time.Time) (De return d, nil } +// fillSlots — run stage-2 extraction on an LLM decision and fill only the slots +// the model left empty. The LLM wins where it answered: it saw the sentence, the +// parsers are keyword tables. Extraction covers what the model cannot produce at +// all — a parsed reminder time and an allowlist fn. +// +// If a reminder still has no time, leave it missing. The daemon then says it +// could not read the time; inventing one would set a wrong alarm. +func (r *Router) fillSlots(ctx context.Context, d *Decision, now time.Time) { + ex := r.extractor.Extract(ctx, d.Intent, d.Utterance, now) + if !d.Slots.HasTime && ex.HasTime { + d.Slots.Time, d.Slots.HasTime = ex.Time, ex.HasTime + } + if !d.Slots.HasKey && ex.HasKey { + d.Slots.Key, d.Slots.Value, d.Slots.HasKey = ex.Key, ex.Value, ex.HasKey + } + if !d.Slots.HasFn && ex.HasFn { + d.Slots.Fn, d.Slots.Args, d.Slots.HasFn = ex.Fn, ex.Args, ex.HasFn + } + // For an act the model returns the verb in Text ("restart nginx"), which is + // often cleaner than the raw utterance ("maven, could you restart nginx"). + // Try it too when the utterance did not match the allowlist. + if d.Intent == IntentAct && !d.Slots.HasFn && r.extractor.Acts != nil && + d.Slots.Text != "" && d.Slots.Text != d.Utterance { + if fn, args, ok := r.extractor.Acts.Match(d.Slots.Text); ok { + d.Slots.Fn, d.Slots.Args, d.Slots.HasFn = fn, args, true + } + } + if d.Slots.Text == "" { + d.Slots.Text = ex.Text + } + // Stage stays 1: it says who decided the route, and that was the LLM. +} + // CorrectMisroute — the user corrected a bad classification. Appends a new // example for the corrected intent (append-only — grows the classifier, no // retrain). Same shape as nudges.outcome tuning cooldowns: more reliable over