package router import ( "context" "fmt" "strings" "testing" "time" "github.com/kami/maven/internal/llm" ) type mockLLM struct { out string err error got *llm.Req // last request, when the test wants to inspect it } func (m mockLLM) Complete(_ context.Context, r llm.Req) (string, error) { if m.got != nil { *m.got = r } return m.out, m.err } // Without a repeat penalty the model loops inside the text field until MaxTokens // and the truncated JSON fails to parse. func TestLLMRouterSetsRepeatPenalty(t *testing.T) { var got llm.Req lr := NewLLMRouter(mockLLM{out: `{"intent":"chat","text":"привет"}`, got: &got}) if _, _, err := lr.Route(context.Background(), "привет", time.Now()); err != nil { t.Fatalf("route: %v", err) } if got.RepeatPenalty <= 1 { t.Fatalf("want repeat penalty above 1, got %v", got.RepeatPenalty) } } // An unbounded string rule lets one field eat the whole token budget. func TestRouteGrammarBoundsStrings(t *testing.T) { if !strings.Contains(routeGrammar, `{0,120} "\""`) { t.Fatal("grammar string rule lost its length bound") } // And it must not admit a raw newline or a made-up escape, either of which // makes the route unparseable and costs the turn its router (Vikunja #537). if !strings.Contains(routeGrammar, `[^"\\\x00-\x1F]`) { t.Error("string rule admits raw control characters") } if strings.Contains(routeGrammar, `"\\" .`) { t.Error(`string rule still admits "\\" . — \q satisfies the grammar and fails to parse`) } } // A question naming a fact key used to be stored as a fact because the fact rule // was tested first. Keep the query rule above it. func TestRoutePromptTestsQueryBeforeFact(t *testing.T) { query := strings.Index(routeSystem, "→ query") fact := strings.Index(routeSystem, "состояние/событие → fact") if query < 0 || fact < 0 { t.Fatalf("prompt lost a rule: query=%d fact=%d", query, fact) } if query > fact { t.Fatal("query rule must come before the fact rule") } if !strings.Contains(routeSystem, "Задаёт вопрос") { t.Fatal("prompt lost the explicit question test") } } func TestLLMRouterFactMapping(t *testing.T) { lr := NewLLMRouter(mockLLM{out: `{"intent":"fact","key":"water","value":"выпил"}`}) d, ok, err := lr.Route(context.Background(), "я выпил воду", time.Now()) if err != nil || !ok { t.Fatalf("ok=%v err=%v", ok, err) } if d.Intent != IntentFact || d.Slots.Key != "water" || !d.Slots.HasKey { t.Fatalf("bad decision %+v", d) } } func TestLLMRouterNoteMapping(t *testing.T) { lr := NewLLMRouter(mockLLM{out: `{"intent":"note","text":"кофе закончился"}`}) d, ok, err := lr.Route(context.Background(), "запомни что кофе закончился", time.Now()) if err != nil || !ok { t.Fatalf("ok=%v err=%v", ok, err) } if d.Intent != IntentNote || d.Slots.Text != "кофе закончился" { t.Fatalf("bad decision %+v", d) } } func TestLLMRouterBadJSONFallsBack(t *testing.T) { lr := NewLLMRouter(mockLLM{out: `garbage`}) _, ok, err := lr.Route(context.Background(), "x", time.Now()) if ok || err == nil { t.Fatal("want ok=false, err!=nil on bad json") } } func TestLLMRouterReminderMapping(t *testing.T) { lr := NewLLMRouter(mockLLM{out: `{"intent":"reminder","text":"позвонить маме"}`}) d, ok, err := lr.Route(context.Background(), "напомни позвонить маме", time.Now()) if err != nil || !ok { t.Fatalf("ok=%v err=%v", ok, err) } if d.Intent != IntentReminder || d.Slots.Text != "позвонить маме" { t.Fatalf("bad decision %+v", d) } } // An intent name that is not in the contract at all (as opposed to "unknown", // which is a real refusal) still defaults to chat. func TestLLMRouterChatFallback(t *testing.T) { lr := NewLLMRouter(mockLLM{out: `{"intent":"banana"}`}) d, ok, err := lr.Route(context.Background(), "как дела?", time.Now()) if err != nil || !ok { t.Fatalf("ok=%v err=%v", ok, err) } if d.Intent != IntentChat { t.Fatalf("unknown intent should default to chat, got %s", d.Intent) } } // The model must be able to say "I could not route this". func TestRouteGrammarAllowsUnknown(t *testing.T) { if !strings.Contains(routeGrammar, `"\"unknown\""`) { t.Fatal("grammar cannot express a refusal") } } // If the prompt does not tell the model when to refuse, it never will. func TestRoutePromptExplainsUnknown(t *testing.T) { if !strings.Contains(routeSystem, "unknown") { t.Fatal("prompt never mentions the unknown intent") } if !strings.Contains(routeSystem, `"сделай это" → {"intent":"unknown"}`) { t.Fatal("prompt lost its worked refusal example") } // A refusal-only router is useless, so the prompt must also show cases that // look ambiguous but are not. if !strings.Contains(routeSystem, "здесь unknown не нужен") { t.Fatal("prompt lost its counter-examples") } } // A refusal is not an error. It reports "no decision" so the cascade moves on. func TestLLMRouterUnknownRefuses(t *testing.T) { lr := NewLLMRouter(mockLLM{out: `{"intent":"unknown"}`}) _, ok, err := lr.Route(context.Background(), "сделай это", time.Now()) if ok { t.Fatal("a refusal must not produce a usable decision") } if err != nil { t.Fatalf("a refusal is not an error, got %v", err) } } // The whole point of the refusal: the turn keeps going on the classifier, the // same way it does when the model returns garbage. func TestRouterFallsBackWhenLLMRefuses(t *testing.T) { c := NewClassifier(NewHashEmbedder(1024)) seedClassifier(t, c) r := New(Config{ Classifier: c, Extractor: Extractor{Time: StubDateTimeParser{}, Facts: DefaultFactParser{}}, Threshold: 0.4, LLM: NewLLMRouter(mockLLM{out: `{"intent":"unknown"}`}), }) d, err := r.Route(context.Background(), "напомни позвонить маме", refNow()) if err != nil { t.Fatalf("route: %v", err) } // Stage 1 is the LLM's own answer; the classifier lands on stage 2 or 3. if d.Stage < 2 { t.Fatalf("want the classifier to decide, got stage %d (%+v)", d.Stage, d) } } func TestLLMRouterLLMError(t *testing.T) { lr := NewLLMRouter(mockLLM{out: "", err: fmt.Errorf("llm down")}) _, ok, err := lr.Route(context.Background(), "x", time.Now()) if ok || err == nil { t.Fatal("want ok=false, err!=nil on llm error") } } // --- slot extraction on top of an LLM decision -------------------------------- // newLLMTestRouter — a router whose route always comes from the mock model. func newLLMTestRouter(t *testing.T, out string) *Router { t.Helper() c := NewClassifier(NewHashEmbedder(1024)) seedClassifier(t, c) acts := DefaultActMatcher{Fns: []string{"restart", "stop", "run", "backup"}} return New(Config{ Classifier: c, Extractor: Extractor{Time: StubDateTimeParser{}, Acts: acts, Facts: DefaultFactParser{}}, Threshold: 0.4, LLM: NewLLMRouter(mockLLM{out: out}), }) } // The model cannot produce a fire time, so without extraction every LLM-routed // reminder was dropped as "no time". func TestLLMDecisionGetsReminderTime(t *testing.T) { r := newLLMTestRouter(t, `{"intent":"reminder","text":"позвонить маме"}`) d, err := r.Route(context.Background(), "напомни позвонить маме через 2 часа", refNow()) if err != nil { t.Fatalf("route: %v", err) } if d.Intent != IntentReminder { t.Fatalf("want reminder, got %v", d.Intent) } if !d.Slots.HasTime || !d.Slots.Time.Equal(refNow().Add(2*time.Hour)) { t.Fatalf("want time now+2h, got %+v", d.Slots) } if d.Slots.Text != "позвонить маме" { t.Fatalf("extraction overwrote the model's text: %q", d.Slots.Text) } } // No time in the utterance ⇒ no time in the slots. Do not invent one; the // daemon says it could not read the time. func TestLLMReminderWithoutTimeStaysEmpty(t *testing.T) { r := newLLMTestRouter(t, `{"intent":"reminder","text":"позвонить маме"}`) d, err := r.Route(context.Background(), "напомни позвонить маме", refNow()) if err != nil { t.Fatalf("route: %v", err) } if d.Slots.HasTime { t.Fatalf("invented a time: %v", d.Slots.Time) } } // An act decision arrived with no Fn, so the tool never ran. func TestLLMDecisionGetsActFn(t *testing.T) { r := newLLMTestRouter(t, `{"intent":"act","verb":"restart nginx"}`) d, err := r.Route(context.Background(), "слушай, restart nginx пожалуйста", refNow()) if err != nil { t.Fatalf("route: %v", err) } if !d.Slots.HasFn || d.Slots.Fn != "restart" || len(d.Slots.Args) != 1 || d.Slots.Args[0] != "nginx" { t.Fatalf("want fn=restart args=[nginx], got %+v", d.Slots) } } // The model's own slots win; extraction only fills gaps. func TestLLMSlotsWinOverExtraction(t *testing.T) { r := newLLMTestRouter(t, `{"intent":"fact","key":"hydration","value":"выпил"}`) d, err := r.Route(context.Background(), "я выпил воду", refNow()) if err != nil { t.Fatalf("route: %v", err) } if d.Slots.Key != "hydration" { t.Fatalf("extraction overwrote the model's key: %q", d.Slots.Key) } } // A fact the model left keyless still gets one from the parser. func TestLLMFactGetsKeyFromParser(t *testing.T) { r := newLLMTestRouter(t, `{"intent":"fact","text":"я выпил воду"}`) d, err := r.Route(context.Background(), "я выпил воду", refNow()) if err != nil { t.Fatalf("route: %v", err) } if !d.Slots.HasKey || d.Slots.Key != "water" { t.Fatalf("want key=water, got %+v", d.Slots) } if d.Clarify { t.Fatalf("the parser resolved the key, this must not clarify: %+v", d) } } // --- confidence / stage-3 gate on the LLM path (Vikunja #359) ----------------- // A single-token utterance is thin evidence on its own — "вода" alone is a // fact/query coin flip. The gate must ask rather than guess confidently. func TestLLMRouterSingleTokenTripsClarify(t *testing.T) { r := newLLMTestRouter(t, `{"intent":"query","text":"вода"}`) d, err := r.Route(context.Background(), "вода", refNow()) if err != nil { t.Fatalf("route: %v", err) } if !d.Clarify { t.Fatalf("a bare single-token decision must clarify, got %+v", d) } } // A multi-word utterance with a clean answer must not be punished — the // whole point is not trading the confident cases away for clarify coverage. func TestLLMRouterMultiWordStaysConfident(t *testing.T) { r := newLLMTestRouter(t, `{"intent":"reminder","text":"позвонить маме"}`) d, err := r.Route(context.Background(), "напомни позвонить маме", refNow()) if err != nil { t.Fatalf("route: %v", err) } if d.Clarify { t.Fatalf("a clean multi-word decision must not clarify: %+v", d) } if d.Confidence != llmFullConfidence { t.Fatalf("want full confidence, got %v", d.Confidence) } } // "бэкап" alone: the model guesses act, but nothing on the allowlist matches // "бэкап" as a verb — that must not fire a tool blind. func TestLLMRouterActWithoutFnTripsClarify(t *testing.T) { r := newLLMTestRouter(t, `{"intent":"act","verb":"бэкап"}`) d, err := r.Route(context.Background(), "бэкап сделай пожалуйста расписание", refNow()) if err != nil { t.Fatalf("route: %v", err) } if d.Slots.HasFn { t.Fatalf("test setup drifted: %q now resolves to an fn", d.Slots.Fn) } if !d.Clarify { t.Fatalf("an unresolved act must clarify rather than guess: %+v", d) } } // An act that DOES resolve to an allowlisted fn must stay confident even // though its own verb is single-word-ish in spirit — guard against the fn // check firing on the happy path. func TestLLMRouterActWithFnStaysConfident(t *testing.T) { r := newLLMTestRouter(t, `{"intent":"act","verb":"restart nginx"}`) d, err := r.Route(context.Background(), "слушай, restart nginx пожалуйста", refNow()) if err != nil { t.Fatalf("route: %v", err) } if !d.Slots.HasFn { t.Fatalf("test setup drifted, want fn resolved: %+v", d.Slots) } if d.Clarify { t.Fatalf("a resolved act must not clarify: %+v", d) } } // A fact where NEITHER the model NOR the deterministic parser can name a key // must clarify instead of silently writing under an empty/guessed key. func TestLLMRouterFactWithoutKeyTripsClarify(t *testing.T) { r := newLLMTestRouter(t, `{"intent":"fact","value":"что-то"}`) d, err := r.Route(context.Background(), "у меня какая-то фигня случилась вот прямо только что", refNow()) if err != nil { t.Fatalf("route: %v", err) } if d.Slots.HasKey { t.Fatalf("test setup drifted: parser now resolves a key for this utterance") } if !d.Clarify { t.Fatalf("a keyless fact must clarify rather than guess: %+v", d) } } // The whole point of #359: the classifier cascade cannot be traded away for // clarify coverage. A multi-word fact the parser CAN key must stay confident // through the full Router.Route path, not just the raw LLMRouter. func TestRouterLLMFactWithResolvedKeyStaysConfident(t *testing.T) { r := newLLMTestRouter(t, `{"intent":"fact","text":"я выпил воду"}`) d, err := r.Route(context.Background(), "я выпил воду", refNow()) if err != nil { t.Fatalf("route: %v", err) } if d.Clarify { t.Fatalf("a fact the parser could key must not clarify: %+v", d) } } // A reminder with a time and no subject must come back empty and gated, not // backfilled with the raw words. "напомни в 11" carries an hour and nothing to // say at that hour; parking the utterance in Text made the request look // complete, so the daemon set a reminder that fires saying "напомни в 11" // (Vikunja #383). func TestLLMReminderWithoutSubjectAsksInsteadOfGuessing(t *testing.T) { r := newLLMTestRouter(t, `{"intent":"reminder"}`) d, err := r.Route(context.Background(), "напомни в 11", refNow()) if err != nil { t.Fatalf("route: %v", err) } if d.Slots.Text != "" { t.Fatalf("subject backfilled from the utterance: %q", d.Slots.Text) } if !d.Clarify { t.Fatalf("a subjectless reminder was accepted, confidence %v", d.Confidence) } } // The gate is about the subject, not about reminders in general: one that has // both halves still runs without a question. func TestLLMReminderWithSubjectIsNotGated(t *testing.T) { r := newLLMTestRouter(t, `{"intent":"reminder","text":"позвонить маме"}`) d, err := r.Route(context.Background(), "напомни в 11 позвонить маме", refNow()) if err != nil { t.Fatalf("route: %v", err) } if d.Clarify { t.Fatalf("a complete reminder was sent back as a question: %+v", d.Slots) } }