claude
4cfef41541
A spoken turn took 25-34 seconds and effectively all of it was one phrasing
call generating whitespace. Both interactive turns measured on 2026-08-04
decoded exactly 512 tokens, which is the phrasing MaxTokens, and both ran to
the cap. Background phrasing on the same server in the same window stopped at
32-36 tokens in 4.3s, so it was never the server and never contention.
`ws ::= [ \t\n]*` is a licence to emit whitespace until max_tokens. The model
opens the object, satisfies ws forever, and only the cap stops it. Bounding
the rule fixes it outright with no repeat penalty at all: three runs, three
clean stops at 33 tokens. routeGrammar carried the same rule and is bounded
too — it never ran away only because that path sends routeRepeatPenalty, which
is an accident rather than a defence.
chatReq had no repeat-penalty field at all, so every caller through
chatWithSystem ran at the server default of 1.0 while Replier.PhraseReply sent
1.3 through internal/llm and was protected by accident. Adding it is defence
in depth, not the fix. Two wire structs disagreeing about the sampler is not a
decision anybody made.
finish_reason is parsed on both transports now and a cap hit logs. Both replies
that ran away happened to parse — the grammar had already closed the JSON — so
a truncated generation was indistinguishable from a whole one at every layer
above the response struct.
The phraser test rejects unbounded repetition anywhere in responseGrammar
rather than checking ws by name. A grammar is a budget: every repetition in it
is something the model may do until the token cap, and the cap is not a design.
routeGrammar keeps one, `("," ws action)*`, because a compound utterance is any
number of actions and capping it would drop the last ask.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011x5DgnExQ5XZy8TZPs5bot
82 lines
3.3 KiB
Go
82 lines
3.3 KiB
Go
package phraser
|
|
|
|
import (
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"regexp"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// unboundedRepeat finds a `*` or `+` applied to a character class or a group.
|
|
//
|
|
// Those are the two forms that let the model generate without limit. A literal
|
|
// `*` inside a quoted terminal is not one, which is why the pattern anchors on
|
|
// `]` and `)`.
|
|
var unboundedRepeat = regexp.MustCompile(`[\]\)]\s*[*+]`)
|
|
|
|
// An unbounded ws rule cost 24-30 seconds a turn (Vikunja #531). The model
|
|
// opened the JSON object, satisfied `ws ::= [ \t\n]*` with whitespace, and ran
|
|
// to the 512-token cap doing it — both interactive turns measured on
|
|
// 2026-08-04 decoded exactly 512 tokens of mostly whitespace.
|
|
//
|
|
// The rule this test enforces is stronger than "ws is bounded", on purpose. A
|
|
// grammar is a budget: every repetition in it is something the model is allowed
|
|
// to do until the token cap, and the cap is not a design. A new rule with a
|
|
// bare `*` is the same defect wearing a different name, and it would cost
|
|
// another QA sitting to find.
|
|
func TestResponseGrammarHasNoUnboundedRepetition(t *testing.T) {
|
|
for i, line := range strings.Split(responseGrammar, "\n") {
|
|
if m := unboundedRepeat.FindString(line); m != "" {
|
|
t.Errorf("responseGrammar line %d has unbounded repetition %q — bound it, a cap is not a design:\n\t%s",
|
|
i+1, m, strings.TrimSpace(line))
|
|
}
|
|
}
|
|
}
|
|
|
|
// The measured bound. 400 was too tight for the string rule and the comment on
|
|
// responseGrammar records why; {0,4} for whitespace was measured the same way,
|
|
// three runs stopping cleanly at 33 tokens with no repeat penalty at all.
|
|
func TestResponseGrammarWhitespaceIsBounded(t *testing.T) {
|
|
if !strings.Contains(responseGrammar, `ws ::= [ \t\n]{0,4}`) {
|
|
t.Errorf("the ws rule is not the measured {0,4} bound:\n%s", responseGrammar)
|
|
}
|
|
}
|
|
|
|
// The phrasing request must carry a repeat penalty. It is not what fixes #531 —
|
|
// the bounded grammar is — but chatReq having no such field while
|
|
// Replier.PhraseReply sent 1.3 is how one phrasing path ran away and the other
|
|
// did not. Two wire structs disagreeing about the sampler is not a decision
|
|
// anybody made.
|
|
func TestPhrasingRequestCarriesARepeatPenalty(t *testing.T) {
|
|
if phraseRepeatPenalty <= 1.0 {
|
|
t.Fatalf("phraseRepeatPenalty is %v, which is the server default and no penalty at all", phraseRepeatPenalty)
|
|
}
|
|
var got []float64
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
var req chatReq
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
t.Errorf("decode request: %v", err)
|
|
}
|
|
got = append(got, req.RepeatPenalty)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.Write([]byte(`{"choices":[{"message":{"content":"{\"response\": \"ага\", \"mood\": \"neutral\"}"},"finish_reason":"stop"}]}`))
|
|
}))
|
|
defer srv.Close()
|
|
|
|
// Every path, not just one: the defect was a whole transport missing the
|
|
// field, so a test that checked a single caller would have passed before
|
|
// the fix as easily as after it.
|
|
callAllPhrasingPaths(t, NewLLMPhraserAt(srv.URL, Config{LLMNudges: true}))
|
|
|
|
if len(got) == 0 {
|
|
t.Fatal("no request captured")
|
|
}
|
|
for i, p := range got {
|
|
if p != phraseRepeatPenalty {
|
|
t.Errorf("request %d sent repeat_penalty %v, want %v", i, p, phraseRepeatPenalty)
|
|
}
|
|
}
|
|
}
|