diff --git a/internal/router/llmrouter.go b/internal/router/llmrouter.go index 40aeb18..5fdef26 100644 --- a/internal/router/llmrouter.go +++ b/internal/router/llmrouter.go @@ -23,14 +23,16 @@ func NewLLMRouter(c Completer) *LLMRouter { return &LLMRouter{c: c} } // routeGrammar — GBNF constraining the model to a JSON ARRAY of fixed-shape // action objects (one per ask; compound utterances → multiple). Enum + key set -// prevent free-form drift from a sub-1B model. +// prevent free-form drift from a sub-1B model. The string rule is length-bounded +// so a repetition loop cannot fill the whole token budget with one field and +// truncate the JSON. const routeGrammar = ` root ::= "[" ws action ("," ws action)* ws "]" action ::= "{" ws "\"intent\"" ws ":" ws intent ("," ws field)* ws "}" intent ::= "\"fact\"" | "\"reminder\"" | "\"note\"" | "\"query\"" | "\"act\"" | "\"chat\"" | "\"system\"" field ::= key ws ":" ws string key ::= "\"key\"" | "\"value\"" | "\"text\"" | "\"verb\"" -string ::= "\"" ([^"\\] | "\\" .)* "\"" +string ::= "\"" ([^"\\] | "\\" .){0,120} "\"" ws ::= [ \t\n]* ` @@ -77,6 +79,11 @@ const routeSystem = `Классифицируй ровно одно сообще Ответ — JSON-массив: по одному объекту на каждую просьбу. Обычно один. Если в реплике несколько просьб — по объекту на каждую. "напомни купить молоко, и запиши что кофе кончился" → [{"intent":"reminder","text":"купить молоко"},{"intent":"note","text":"кофе кончился"}]. Только JSON, без пояснений.` +// routeRepeatPenalty — the sub-1B model loops one sentence inside the text field +// until it runs out of tokens, which truncates the JSON. 1.15 is enough to break +// the loop without hurting short slot values. +const routeRepeatPenalty = 1.15 + type routeAction struct { Intent string `json:"intent"` Key string `json:"key"` @@ -86,7 +93,13 @@ type routeAction struct { } func (lr *LLMRouter) Route(ctx context.Context, utterance string, now time.Time) (Decision, bool, error) { - raw, err := lr.c.Complete(ctx, llm.Req{System: routeSystem, User: utterance, Grammar: routeGrammar, MaxTokens: 128}) + raw, err := lr.c.Complete(ctx, llm.Req{ + System: routeSystem, + User: utterance, + Grammar: routeGrammar, + MaxTokens: 128, + RepeatPenalty: routeRepeatPenalty, + }) if err != nil { return Decision{}, false, err } diff --git a/internal/router/llmrouter_test.go b/internal/router/llmrouter_test.go index de8cde5..7c73852 100644 --- a/internal/router/llmrouter_test.go +++ b/internal/router/llmrouter_test.go @@ -13,9 +13,35 @@ import ( type mockLLM struct { out string err error + got *llm.Req // last request, when the test wants to inspect it } -func (m mockLLM) Complete(_ context.Context, _ llm.Req) (string, error) { return m.out, m.err } +func (m mockLLM) Complete(_ context.Context, r llm.Req) (string, error) { + if m.got != nil { + *m.got = r + } + return m.out, m.err +} + +// Without a repeat penalty the model loops inside the text field until MaxTokens +// and the truncated JSON fails to parse. +func TestLLMRouterSetsRepeatPenalty(t *testing.T) { + var got llm.Req + lr := NewLLMRouter(mockLLM{out: `{"intent":"chat","text":"привет"}`, got: &got}) + if _, _, err := lr.Route(context.Background(), "привет", time.Now()); err != nil { + t.Fatalf("route: %v", err) + } + if got.RepeatPenalty <= 1 { + t.Fatalf("want repeat penalty above 1, got %v", got.RepeatPenalty) + } +} + +// An unbounded string rule lets one field eat the whole token budget. +func TestRouteGrammarBoundsStrings(t *testing.T) { + if !strings.Contains(routeGrammar, `string ::= "\"" ([^"\\] | "\\" .){0,120} "\""`) { + t.Fatal("grammar string rule lost its length bound") + } +} // A question naming a fact key used to be stored as a fact because the fact rule // was tested first. Keep the query rule above it.