phrasing temperature is a config field, and a sweep to measure it (V-402)
Both chatReq sites sent a hardcoded 0.7 and the remote path had its own const, so the one dial that governs how much a 1.7B invents could not be turned from outside the package. Config.Temperature now feeds both, 0 still means 0.7, and world.go reads the same accessor so resident and remote cannot drift. TestTalkTemperatureSweep scores the talk fixture at 0.7, 0.4, 0.2 and near greedy, three runs each so the noise band is visible. Opt-in twice (MAVEN_LLM_URL and MAVEN_TEMP_SWEEP) because it costs upwards of twenty minutes on the CPU floor. It reports and asserts nothing: the composite is not the number to read. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,87 @@
|
||||
package eval
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/llm"
|
||||
"github.com/kami/maven/internal/persona"
|
||||
"github.com/kami/maven/internal/phraser"
|
||||
)
|
||||
|
||||
// sweepTemperatures — the dial positions worth comparing (Vikunja #402).
|
||||
// 0.7 is what the transport has always sent; 0.05 stands in for near-greedy,
|
||||
// since 0 means "use the default" to the phraser.
|
||||
var sweepTemperatures = []float64{0.7, 0.4, 0.2, 0.05}
|
||||
|
||||
// sweepRuns — how many runs per position. Three, because one run of a sampled
|
||||
// model tells you nothing about whether a two-point difference is real.
|
||||
const sweepRuns = 3
|
||||
|
||||
// TestTalkTemperatureSweep scores the talk fixture at each temperature.
|
||||
//
|
||||
// Opt-in twice over: it needs a llama-server AND it costs roughly
|
||||
// len(sweepTemperatures) * sweepRuns * the baseline run time, which is upwards
|
||||
// of twenty minutes on the CPU floor.
|
||||
//
|
||||
// MAVEN_LLM_URL=http://127.0.0.1:18099 MAVEN_TEMP_SWEEP=1 \
|
||||
// go test -v -timeout 90m -run TestTalkTemperatureSweep ./internal/phraser/eval/
|
||||
//
|
||||
// Reports, asserts nothing. The composite is not the number to read — the task
|
||||
// says to watch ontopic and invented content against how flat the replies get,
|
||||
// and the replies are logged for exactly that reason.
|
||||
//
|
||||
// Note that only the chat/query/world paths move: the reply path is a Replier
|
||||
// over llm.Client, which samples greedily and does not read this dial.
|
||||
func TestTalkTemperatureSweep(t *testing.T) {
|
||||
base := os.Getenv("MAVEN_LLM_URL")
|
||||
if base == "" {
|
||||
t.Skip("MAVEN_LLM_URL unset — point it at a running llama-server")
|
||||
}
|
||||
if os.Getenv("MAVEN_TEMP_SWEEP") == "" {
|
||||
t.Skip("MAVEN_TEMP_SWEEP unset — this sweep costs many minutes, see the doc comment")
|
||||
}
|
||||
noProxyLoopback(t)
|
||||
|
||||
ctx := context.Background()
|
||||
f, err := LoadTalk()
|
||||
if err != nil {
|
||||
t.Fatalf("LoadTalk: %v", err)
|
||||
}
|
||||
model, err := llm.ModelID(ctx, base)
|
||||
if err != nil {
|
||||
t.Fatalf("no model at %s: %v", base, err)
|
||||
}
|
||||
|
||||
block := func() string { return persona.Facts{}.Block(time.Now()) }
|
||||
summary := fmt.Sprintf("temperature sweep, %s, %d runs each\n", model, sweepRuns)
|
||||
|
||||
for _, temp := range sweepTemperatures {
|
||||
for run := 1; run <= sweepRuns; run++ {
|
||||
cfg := phraser.DefaultConfig("")
|
||||
cfg.Timeout = 5 * time.Minute
|
||||
cfg.ContextBlock = block
|
||||
cfg.Temperature = temp
|
||||
p := phraser.NewLLMPhraserAt(base, cfg)
|
||||
|
||||
name := fmt.Sprintf("temp %.2f run %d", temp, run)
|
||||
target := Pair{Talker: p, Confirmer: phraser.NewReplier(llm.New(base, cfg.Timeout), block)}
|
||||
rep, err := ScoreTalk(ctx, name, target, f)
|
||||
p.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("ScoreTalk at %.2f: %v", temp, err)
|
||||
}
|
||||
if rep.Errors == rep.Total {
|
||||
t.Fatalf("every case errored at %.2f — nothing was measured", temp)
|
||||
}
|
||||
t.Log("\n" + rep.String() + "\nreplies:\n" + rep.Replies() + "\nfailures:\n" + rep.Failures())
|
||||
summary += fmt.Sprintf(" %-18s %2d/%2d (%.1f%%) ontopic %d/%d errors %d\n",
|
||||
name, rep.Passed, rep.Total, 100*rep.Accuracy(),
|
||||
rep.ByCheck[CheckOnTopic], rep.Total, rep.Errors)
|
||||
}
|
||||
}
|
||||
t.Log("\n" + summary)
|
||||
}
|
||||
@@ -131,6 +131,14 @@ type Config struct {
|
||||
// query and reminder phrasing are untouched and still go through the model.
|
||||
LLMNudges bool
|
||||
|
||||
// Temperature — what every phrasing call samples at. 0 ⇒ 0.7, which is
|
||||
// what this transport has always sent.
|
||||
//
|
||||
// A field rather than a constant so the talk fixture can sweep it
|
||||
// (Vikunja #402). Sampling is a dial, and a dial nobody can turn from
|
||||
// outside the package cannot be measured, only argued about.
|
||||
Temperature float64
|
||||
|
||||
// NoGrammar turns the GBNF constraint off (zero value ⇒ grammar ON).
|
||||
// The escape hatch exists because the target resident model — the
|
||||
// locally CPT'd Qwen3-1.7B — does not exist yet: if its chat template
|
||||
@@ -594,7 +602,7 @@ func (p *LLMPhraser) chatWithMessages(ctx context.Context, msgs []chatMsg, maxTo
|
||||
defer release()
|
||||
req := chatReq{
|
||||
Messages: msgs,
|
||||
Temperature: 0.7,
|
||||
Temperature: p.temperature(),
|
||||
MaxTokens: maxTokens,
|
||||
Grammar: p.grammar(),
|
||||
}
|
||||
@@ -762,7 +770,7 @@ func (p *LLMPhraser) chatWithSystem(ctx context.Context, system, user string, ma
|
||||
{Role: "system", Content: system},
|
||||
{Role: "user", Content: user},
|
||||
},
|
||||
Temperature: 0.7,
|
||||
Temperature: p.temperature(),
|
||||
MaxTokens: maxTokens,
|
||||
Grammar: p.grammar(),
|
||||
}
|
||||
|
||||
@@ -37,11 +37,19 @@ type Remote interface {
|
||||
// (docs/evals/2026-08-02-workstation-gemma4-12b.md).
|
||||
var ErrNoWorldModel = errors.New("phraser: no world model available")
|
||||
|
||||
// chatTemperature — what the phraser's own transport has always sampled at.
|
||||
// Named so the remote path cannot drift from it silently. Whether 0.7 is right
|
||||
// at all is Vikunja #402, and answering that here would hide a phrasing change
|
||||
// inside a routing change.
|
||||
const chatTemperature = 0.7
|
||||
// defaultChatTemperature — what the phraser's own transport has always sampled
|
||||
// at, and what Config.Temperature falls back to. Named so the remote path
|
||||
// cannot drift from the resident one silently.
|
||||
const defaultChatTemperature = 0.7
|
||||
|
||||
// temperature — the sampling temperature for every phrasing call, resident or
|
||||
// remote. Both paths read this, so a sweep moves them together.
|
||||
func (p *LLMPhraser) temperature() float64 {
|
||||
if p.cfg.Temperature > 0 {
|
||||
return p.cfg.Temperature
|
||||
}
|
||||
return defaultChatTemperature
|
||||
}
|
||||
|
||||
// UseRemote points the phraser at the workstation model. Wiring time only, once,
|
||||
// before anything phrases: the field is read without a lock on every call
|
||||
@@ -85,7 +93,7 @@ func (p *LLMPhraser) PhraseWorld(ctx context.Context, utterance string, sources
|
||||
User: user,
|
||||
Grammar: p.grammar(),
|
||||
MaxTokens: 768,
|
||||
Temperature: chatTemperature,
|
||||
Temperature: p.temperature(),
|
||||
})
|
||||
if err != nil {
|
||||
// The cached probe was one interval stale, or the card went away
|
||||
@@ -124,7 +132,7 @@ func (p *LLMPhraser) remoteChat(ctx context.Context, system, user string, maxTok
|
||||
User: user,
|
||||
Grammar: p.grammar(),
|
||||
MaxTokens: maxTokens,
|
||||
Temperature: chatTemperature,
|
||||
Temperature: p.temperature(),
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("phraser: workstation model declined, phrasing here instead: %v", err)
|
||||
|
||||
@@ -141,9 +141,9 @@ func TestNudgePhrasingPrefersTheWorkstationSilently(t *testing.T) {
|
||||
if len(remote.got) != 1 {
|
||||
t.Fatalf("the workstation saw %d requests, want 1", len(remote.got))
|
||||
}
|
||||
if remote.got[0].Temperature != chatTemperature {
|
||||
if remote.got[0].Temperature != defaultChatTemperature {
|
||||
t.Errorf("temperature = %v, want %v (what the resident transport samples at)",
|
||||
remote.got[0].Temperature, chatTemperature)
|
||||
remote.got[0].Temperature, defaultChatTemperature)
|
||||
}
|
||||
if len(spy.user) != 0 {
|
||||
t.Errorf("the resident model phrased %d nudges, want 0", len(spy.user))
|
||||
|
||||
Reference in New Issue
Block a user