From c82dbd1e656604c2405c5b879633c069ae9c6775 Mon Sep 17 00:00:00 2001 From: claude Date: Wed, 5 Aug 2026 01:51:43 +0400 Subject: [PATCH] phrasing temperature is a config field, and a sweep to measure it (V-402) Both chatReq sites sent a hardcoded 0.7 and the remote path had its own const, so the one dial that governs how much a 1.7B invents could not be turned from outside the package. Config.Temperature now feeds both, 0 still means 0.7, and world.go reads the same accessor so resident and remote cannot drift. TestTalkTemperatureSweep scores the talk fixture at 0.7, 0.4, 0.2 and near greedy, three runs each so the noise band is visible. Opt-in twice (MAVEN_LLM_URL and MAVEN_TEMP_SWEEP) because it costs upwards of twenty minutes on the CPU floor. It reports and asserts nothing: the composite is not the number to read. Co-Authored-By: Claude Opus 5 --- internal/phraser/eval/temperature_test.go | 87 +++++++++++++++++++++++ internal/phraser/llmphraser.go | 12 +++- internal/phraser/world.go | 22 ++++-- internal/phraser/world_test.go | 4 +- 4 files changed, 114 insertions(+), 11 deletions(-) create mode 100644 internal/phraser/eval/temperature_test.go diff --git a/internal/phraser/eval/temperature_test.go b/internal/phraser/eval/temperature_test.go new file mode 100644 index 0000000..4415fdf --- /dev/null +++ b/internal/phraser/eval/temperature_test.go @@ -0,0 +1,87 @@ +package eval + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/kami/maven/internal/llm" + "github.com/kami/maven/internal/persona" + "github.com/kami/maven/internal/phraser" +) + +// sweepTemperatures — the dial positions worth comparing (Vikunja #402). +// 0.7 is what the transport has always sent; 0.05 stands in for near-greedy, +// since 0 means "use the default" to the phraser. +var sweepTemperatures = []float64{0.7, 0.4, 0.2, 0.05} + +// sweepRuns — how many runs per position. Three, because one run of a sampled +// model tells you nothing about whether a two-point difference is real. +const sweepRuns = 3 + +// TestTalkTemperatureSweep scores the talk fixture at each temperature. +// +// Opt-in twice over: it needs a llama-server AND it costs roughly +// len(sweepTemperatures) * sweepRuns * the baseline run time, which is upwards +// of twenty minutes on the CPU floor. +// +// MAVEN_LLM_URL=http://127.0.0.1:18099 MAVEN_TEMP_SWEEP=1 \ +// go test -v -timeout 90m -run TestTalkTemperatureSweep ./internal/phraser/eval/ +// +// Reports, asserts nothing. The composite is not the number to read — the task +// says to watch ontopic and invented content against how flat the replies get, +// and the replies are logged for exactly that reason. +// +// Note that only the chat/query/world paths move: the reply path is a Replier +// over llm.Client, which samples greedily and does not read this dial. +func TestTalkTemperatureSweep(t *testing.T) { + base := os.Getenv("MAVEN_LLM_URL") + if base == "" { + t.Skip("MAVEN_LLM_URL unset — point it at a running llama-server") + } + if os.Getenv("MAVEN_TEMP_SWEEP") == "" { + t.Skip("MAVEN_TEMP_SWEEP unset — this sweep costs many minutes, see the doc comment") + } + noProxyLoopback(t) + + ctx := context.Background() + f, err := LoadTalk() + if err != nil { + t.Fatalf("LoadTalk: %v", err) + } + model, err := llm.ModelID(ctx, base) + if err != nil { + t.Fatalf("no model at %s: %v", base, err) + } + + block := func() string { return persona.Facts{}.Block(time.Now()) } + summary := fmt.Sprintf("temperature sweep, %s, %d runs each\n", model, sweepRuns) + + for _, temp := range sweepTemperatures { + for run := 1; run <= sweepRuns; run++ { + cfg := phraser.DefaultConfig("") + cfg.Timeout = 5 * time.Minute + cfg.ContextBlock = block + cfg.Temperature = temp + p := phraser.NewLLMPhraserAt(base, cfg) + + name := fmt.Sprintf("temp %.2f run %d", temp, run) + target := Pair{Talker: p, Confirmer: phraser.NewReplier(llm.New(base, cfg.Timeout), block)} + rep, err := ScoreTalk(ctx, name, target, f) + p.Close() + if err != nil { + t.Fatalf("ScoreTalk at %.2f: %v", temp, err) + } + if rep.Errors == rep.Total { + t.Fatalf("every case errored at %.2f — nothing was measured", temp) + } + t.Log("\n" + rep.String() + "\nreplies:\n" + rep.Replies() + "\nfailures:\n" + rep.Failures()) + summary += fmt.Sprintf(" %-18s %2d/%2d (%.1f%%) ontopic %d/%d errors %d\n", + name, rep.Passed, rep.Total, 100*rep.Accuracy(), + rep.ByCheck[CheckOnTopic], rep.Total, rep.Errors) + } + } + t.Log("\n" + summary) +} diff --git a/internal/phraser/llmphraser.go b/internal/phraser/llmphraser.go index ddbaa64..37e0020 100644 --- a/internal/phraser/llmphraser.go +++ b/internal/phraser/llmphraser.go @@ -131,6 +131,14 @@ type Config struct { // query and reminder phrasing are untouched and still go through the model. LLMNudges bool + // Temperature — what every phrasing call samples at. 0 ⇒ 0.7, which is + // what this transport has always sent. + // + // A field rather than a constant so the talk fixture can sweep it + // (Vikunja #402). Sampling is a dial, and a dial nobody can turn from + // outside the package cannot be measured, only argued about. + Temperature float64 + // NoGrammar turns the GBNF constraint off (zero value ⇒ grammar ON). // The escape hatch exists because the target resident model — the // locally CPT'd Qwen3-1.7B — does not exist yet: if its chat template @@ -594,7 +602,7 @@ func (p *LLMPhraser) chatWithMessages(ctx context.Context, msgs []chatMsg, maxTo defer release() req := chatReq{ Messages: msgs, - Temperature: 0.7, + Temperature: p.temperature(), MaxTokens: maxTokens, Grammar: p.grammar(), } @@ -762,7 +770,7 @@ func (p *LLMPhraser) chatWithSystem(ctx context.Context, system, user string, ma {Role: "system", Content: system}, {Role: "user", Content: user}, }, - Temperature: 0.7, + Temperature: p.temperature(), MaxTokens: maxTokens, Grammar: p.grammar(), } diff --git a/internal/phraser/world.go b/internal/phraser/world.go index a021a4b..08d18d3 100644 --- a/internal/phraser/world.go +++ b/internal/phraser/world.go @@ -37,11 +37,19 @@ type Remote interface { // (docs/evals/2026-08-02-workstation-gemma4-12b.md). var ErrNoWorldModel = errors.New("phraser: no world model available") -// chatTemperature — what the phraser's own transport has always sampled at. -// Named so the remote path cannot drift from it silently. Whether 0.7 is right -// at all is Vikunja #402, and answering that here would hide a phrasing change -// inside a routing change. -const chatTemperature = 0.7 +// defaultChatTemperature — what the phraser's own transport has always sampled +// at, and what Config.Temperature falls back to. Named so the remote path +// cannot drift from the resident one silently. +const defaultChatTemperature = 0.7 + +// temperature — the sampling temperature for every phrasing call, resident or +// remote. Both paths read this, so a sweep moves them together. +func (p *LLMPhraser) temperature() float64 { + if p.cfg.Temperature > 0 { + return p.cfg.Temperature + } + return defaultChatTemperature +} // UseRemote points the phraser at the workstation model. Wiring time only, once, // before anything phrases: the field is read without a lock on every call @@ -85,7 +93,7 @@ func (p *LLMPhraser) PhraseWorld(ctx context.Context, utterance string, sources User: user, Grammar: p.grammar(), MaxTokens: 768, - Temperature: chatTemperature, + Temperature: p.temperature(), }) if err != nil { // The cached probe was one interval stale, or the card went away @@ -124,7 +132,7 @@ func (p *LLMPhraser) remoteChat(ctx context.Context, system, user string, maxTok User: user, Grammar: p.grammar(), MaxTokens: maxTokens, - Temperature: chatTemperature, + Temperature: p.temperature(), }) if err != nil { log.Printf("phraser: workstation model declined, phrasing here instead: %v", err) diff --git a/internal/phraser/world_test.go b/internal/phraser/world_test.go index 1cdb6ba..285c9d6 100644 --- a/internal/phraser/world_test.go +++ b/internal/phraser/world_test.go @@ -141,9 +141,9 @@ func TestNudgePhrasingPrefersTheWorkstationSilently(t *testing.T) { if len(remote.got) != 1 { t.Fatalf("the workstation saw %d requests, want 1", len(remote.got)) } - if remote.got[0].Temperature != chatTemperature { + if remote.got[0].Temperature != defaultChatTemperature { t.Errorf("temperature = %v, want %v (what the resident transport samples at)", - remote.got[0].Temperature, chatTemperature) + remote.got[0].Temperature, defaultChatTemperature) } if len(spy.user) != 0 { t.Errorf("the resident model phrased %d nudges, want 0", len(spy.user))