kami
d30618ecb7
Route now sets RepeatPenalty on the request, and the grammar's string rule is capped at 120 characters. Two of 76 fixture cases looped one sentence inside the text field until MaxTokens, which cut the JSON in half. Reviewers: the new constant and the grammar string rule. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CGeSZxh1DCtRxmFVSYVGvJ
121 lines
3.8 KiB
Go
121 lines
3.8 KiB
Go
package router
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/kami/maven/internal/llm"
|
|
)
|
|
|
|
type mockLLM struct {
|
|
out string
|
|
err error
|
|
got *llm.Req // last request, when the test wants to inspect it
|
|
}
|
|
|
|
func (m mockLLM) Complete(_ context.Context, r llm.Req) (string, error) {
|
|
if m.got != nil {
|
|
*m.got = r
|
|
}
|
|
return m.out, m.err
|
|
}
|
|
|
|
// Without a repeat penalty the model loops inside the text field until MaxTokens
|
|
// and the truncated JSON fails to parse.
|
|
func TestLLMRouterSetsRepeatPenalty(t *testing.T) {
|
|
var got llm.Req
|
|
lr := NewLLMRouter(mockLLM{out: `{"intent":"chat","text":"привет"}`, got: &got})
|
|
if _, _, err := lr.Route(context.Background(), "привет", time.Now()); err != nil {
|
|
t.Fatalf("route: %v", err)
|
|
}
|
|
if got.RepeatPenalty <= 1 {
|
|
t.Fatalf("want repeat penalty above 1, got %v", got.RepeatPenalty)
|
|
}
|
|
}
|
|
|
|
// An unbounded string rule lets one field eat the whole token budget.
|
|
func TestRouteGrammarBoundsStrings(t *testing.T) {
|
|
if !strings.Contains(routeGrammar, `string ::= "\"" ([^"\\] | "\\" .){0,120} "\""`) {
|
|
t.Fatal("grammar string rule lost its length bound")
|
|
}
|
|
}
|
|
|
|
// A question naming a fact key used to be stored as a fact because the fact rule
|
|
// was tested first. Keep the query rule above it.
|
|
func TestRoutePromptTestsQueryBeforeFact(t *testing.T) {
|
|
query := strings.Index(routeSystem, "→ query")
|
|
fact := strings.Index(routeSystem, "состояние/событие → fact")
|
|
if query < 0 || fact < 0 {
|
|
t.Fatalf("prompt lost a rule: query=%d fact=%d", query, fact)
|
|
}
|
|
if query > fact {
|
|
t.Fatal("query rule must come before the fact rule")
|
|
}
|
|
if !strings.Contains(routeSystem, "Задаёт вопрос") {
|
|
t.Fatal("prompt lost the explicit question test")
|
|
}
|
|
}
|
|
|
|
func TestLLMRouterFactMapping(t *testing.T) {
|
|
lr := NewLLMRouter(mockLLM{out: `{"intent":"fact","key":"water","value":"выпил"}`})
|
|
d, ok, err := lr.Route(context.Background(), "я выпил воду", time.Now())
|
|
if err != nil || !ok {
|
|
t.Fatalf("ok=%v err=%v", ok, err)
|
|
}
|
|
if d.Intent != IntentFact || d.Slots.Key != "water" || !d.Slots.HasKey {
|
|
t.Fatalf("bad decision %+v", d)
|
|
}
|
|
}
|
|
|
|
func TestLLMRouterNoteMapping(t *testing.T) {
|
|
lr := NewLLMRouter(mockLLM{out: `{"intent":"note","text":"кофе закончился"}`})
|
|
d, ok, err := lr.Route(context.Background(), "запомни что кофе закончился", time.Now())
|
|
if err != nil || !ok {
|
|
t.Fatalf("ok=%v err=%v", ok, err)
|
|
}
|
|
if d.Intent != IntentNote || d.Slots.Text != "кофе закончился" {
|
|
t.Fatalf("bad decision %+v", d)
|
|
}
|
|
}
|
|
|
|
func TestLLMRouterBadJSONFallsBack(t *testing.T) {
|
|
lr := NewLLMRouter(mockLLM{out: `garbage`})
|
|
_, ok, err := lr.Route(context.Background(), "x", time.Now())
|
|
if ok || err == nil {
|
|
t.Fatal("want ok=false, err!=nil on bad json")
|
|
}
|
|
}
|
|
|
|
func TestLLMRouterReminderMapping(t *testing.T) {
|
|
lr := NewLLMRouter(mockLLM{out: `{"intent":"reminder","text":"позвонить маме"}`})
|
|
d, ok, err := lr.Route(context.Background(), "напомни позвонить маме", time.Now())
|
|
if err != nil || !ok {
|
|
t.Fatalf("ok=%v err=%v", ok, err)
|
|
}
|
|
if d.Intent != IntentReminder || d.Slots.Text != "позвонить маме" {
|
|
t.Fatalf("bad decision %+v", d)
|
|
}
|
|
}
|
|
|
|
func TestLLMRouterChatFallback(t *testing.T) {
|
|
lr := NewLLMRouter(mockLLM{out: `{"intent":"unknown"}`})
|
|
d, ok, err := lr.Route(context.Background(), "как дела?", time.Now())
|
|
if err != nil || !ok {
|
|
t.Fatalf("ok=%v err=%v", ok, err)
|
|
}
|
|
if d.Intent != IntentChat {
|
|
t.Fatalf("unknown intent should default to chat, got %s", d.Intent)
|
|
}
|
|
}
|
|
|
|
func TestLLMRouterLLMError(t *testing.T) {
|
|
lr := NewLLMRouter(mockLLM{out: "", err: fmt.Errorf("llm down")})
|
|
_, ok, err := lr.Route(context.Background(), "x", time.Now())
|
|
if ok || err == nil {
|
|
t.Fatal("want ok=false, err!=nil on llm error")
|
|
}
|
|
}
|