diff --git a/internal/phraser/eval/temperature_test.go b/internal/phraser/eval/temperature_test.go new file mode 100644 index 0000000..4415fdf --- /dev/null +++ b/internal/phraser/eval/temperature_test.go @@ -0,0 +1,87 @@ +package eval + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/kami/maven/internal/llm" + "github.com/kami/maven/internal/persona" + "github.com/kami/maven/internal/phraser" +) + +// sweepTemperatures — the dial positions worth comparing (Vikunja #402). +// 0.7 is what the transport has always sent; 0.05 stands in for near-greedy, +// since 0 means "use the default" to the phraser. +var sweepTemperatures = []float64{0.7, 0.4, 0.2, 0.05} + +// sweepRuns — how many runs per position. Three, because one run of a sampled +// model tells you nothing about whether a two-point difference is real. +const sweepRuns = 3 + +// TestTalkTemperatureSweep scores the talk fixture at each temperature. +// +// Opt-in twice over: it needs a llama-server AND it costs roughly +// len(sweepTemperatures) * sweepRuns * the baseline run time, which is upwards +// of twenty minutes on the CPU floor. +// +// MAVEN_LLM_URL=http://127.0.0.1:18099 MAVEN_TEMP_SWEEP=1 \ +// go test -v -timeout 90m -run TestTalkTemperatureSweep ./internal/phraser/eval/ +// +// Reports, asserts nothing. The composite is not the number to read — the task +// says to watch ontopic and invented content against how flat the replies get, +// and the replies are logged for exactly that reason. +// +// Note that only the chat/query/world paths move: the reply path is a Replier +// over llm.Client, which samples greedily and does not read this dial. +func TestTalkTemperatureSweep(t *testing.T) { + base := os.Getenv("MAVEN_LLM_URL") + if base == "" { + t.Skip("MAVEN_LLM_URL unset — point it at a running llama-server") + } + if os.Getenv("MAVEN_TEMP_SWEEP") == "" { + t.Skip("MAVEN_TEMP_SWEEP unset — this sweep costs many minutes, see the doc comment") + } + noProxyLoopback(t) + + ctx := context.Background() + f, err := LoadTalk() + if err != nil { + t.Fatalf("LoadTalk: %v", err) + } + model, err := llm.ModelID(ctx, base) + if err != nil { + t.Fatalf("no model at %s: %v", base, err) + } + + block := func() string { return persona.Facts{}.Block(time.Now()) } + summary := fmt.Sprintf("temperature sweep, %s, %d runs each\n", model, sweepRuns) + + for _, temp := range sweepTemperatures { + for run := 1; run <= sweepRuns; run++ { + cfg := phraser.DefaultConfig("") + cfg.Timeout = 5 * time.Minute + cfg.ContextBlock = block + cfg.Temperature = temp + p := phraser.NewLLMPhraserAt(base, cfg) + + name := fmt.Sprintf("temp %.2f run %d", temp, run) + target := Pair{Talker: p, Confirmer: phraser.NewReplier(llm.New(base, cfg.Timeout), block)} + rep, err := ScoreTalk(ctx, name, target, f) + p.Close() + if err != nil { + t.Fatalf("ScoreTalk at %.2f: %v", temp, err) + } + if rep.Errors == rep.Total { + t.Fatalf("every case errored at %.2f — nothing was measured", temp) + } + t.Log("\n" + rep.String() + "\nreplies:\n" + rep.Replies() + "\nfailures:\n" + rep.Failures()) + summary += fmt.Sprintf(" %-18s %2d/%2d (%.1f%%) ontopic %d/%d errors %d\n", + name, rep.Passed, rep.Total, 100*rep.Accuracy(), + rep.ByCheck[CheckOnTopic], rep.Total, rep.Errors) + } + } + t.Log("\n" + summary) +} diff --git a/internal/phraser/llmphraser.go b/internal/phraser/llmphraser.go index ddbaa64..37e0020 100644 --- a/internal/phraser/llmphraser.go +++ b/internal/phraser/llmphraser.go @@ -131,6 +131,14 @@ type Config struct { // query and reminder phrasing are untouched and still go through the model. LLMNudges bool + // Temperature — what every phrasing call samples at. 0 ⇒ 0.7, which is + // what this transport has always sent. + // + // A field rather than a constant so the talk fixture can sweep it + // (Vikunja #402). Sampling is a dial, and a dial nobody can turn from + // outside the package cannot be measured, only argued about. + Temperature float64 + // NoGrammar turns the GBNF constraint off (zero value ⇒ grammar ON). // The escape hatch exists because the target resident model — the // locally CPT'd Qwen3-1.7B — does not exist yet: if its chat template @@ -594,7 +602,7 @@ func (p *LLMPhraser) chatWithMessages(ctx context.Context, msgs []chatMsg, maxTo defer release() req := chatReq{ Messages: msgs, - Temperature: 0.7, + Temperature: p.temperature(), MaxTokens: maxTokens, Grammar: p.grammar(), } @@ -762,7 +770,7 @@ func (p *LLMPhraser) chatWithSystem(ctx context.Context, system, user string, ma {Role: "system", Content: system}, {Role: "user", Content: user}, }, - Temperature: 0.7, + Temperature: p.temperature(), MaxTokens: maxTokens, Grammar: p.grammar(), } diff --git a/internal/phraser/world.go b/internal/phraser/world.go index a021a4b..08d18d3 100644 --- a/internal/phraser/world.go +++ b/internal/phraser/world.go @@ -37,11 +37,19 @@ type Remote interface { // (docs/evals/2026-08-02-workstation-gemma4-12b.md). var ErrNoWorldModel = errors.New("phraser: no world model available") -// chatTemperature — what the phraser's own transport has always sampled at. -// Named so the remote path cannot drift from it silently. Whether 0.7 is right -// at all is Vikunja #402, and answering that here would hide a phrasing change -// inside a routing change. -const chatTemperature = 0.7 +// defaultChatTemperature — what the phraser's own transport has always sampled +// at, and what Config.Temperature falls back to. Named so the remote path +// cannot drift from the resident one silently. +const defaultChatTemperature = 0.7 + +// temperature — the sampling temperature for every phrasing call, resident or +// remote. Both paths read this, so a sweep moves them together. +func (p *LLMPhraser) temperature() float64 { + if p.cfg.Temperature > 0 { + return p.cfg.Temperature + } + return defaultChatTemperature +} // UseRemote points the phraser at the workstation model. Wiring time only, once, // before anything phrases: the field is read without a lock on every call @@ -85,7 +93,7 @@ func (p *LLMPhraser) PhraseWorld(ctx context.Context, utterance string, sources User: user, Grammar: p.grammar(), MaxTokens: 768, - Temperature: chatTemperature, + Temperature: p.temperature(), }) if err != nil { // The cached probe was one interval stale, or the card went away @@ -124,7 +132,7 @@ func (p *LLMPhraser) remoteChat(ctx context.Context, system, user string, maxTok User: user, Grammar: p.grammar(), MaxTokens: maxTokens, - Temperature: chatTemperature, + Temperature: p.temperature(), }) if err != nil { log.Printf("phraser: workstation model declined, phrasing here instead: %v", err) diff --git a/internal/phraser/world_test.go b/internal/phraser/world_test.go index 1cdb6ba..285c9d6 100644 --- a/internal/phraser/world_test.go +++ b/internal/phraser/world_test.go @@ -141,9 +141,9 @@ func TestNudgePhrasingPrefersTheWorkstationSilently(t *testing.T) { if len(remote.got) != 1 { t.Fatalf("the workstation saw %d requests, want 1", len(remote.got)) } - if remote.got[0].Temperature != chatTemperature { + if remote.got[0].Temperature != defaultChatTemperature { t.Errorf("temperature = %v, want %v (what the resident transport samples at)", - remote.got[0].Temperature, chatTemperature) + remote.got[0].Temperature, defaultChatTemperature) } if len(spy.user) != 0 { t.Errorf("the resident model phrased %d nudges, want 0", len(spy.user))