Label eval runs with the model the server actually loaded (#379)
The phrasing eval printed "llm (0.8B, ...)" no matter which gguf llama-server had loaded, so two runs of two different models came out named the same and were easy to mix up when comparing. It now asks llama-server over /v1/models, same as the router eval already did. The helper moved to internal/llm so both share it, and it now errors instead of returning a blank name when the id field is missing — an unreachable server gets labelled "unknown-model", never a plausible-looking guess. Both eval paths stay opt-in behind MAVEN_LLM_URL; no server needed for go test. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CGeSZxh1DCtRxmFVSYVGvJ
This commit is contained in:
@@ -7,6 +7,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/llm"
|
||||
"github.com/kami/maven/internal/phraser"
|
||||
)
|
||||
|
||||
@@ -32,6 +33,7 @@ func TestLLMPhrasingBaseline(t *testing.T) {
|
||||
// case as a phrasing error and read as "the model cannot phrase".
|
||||
noProxyLoopback(t)
|
||||
|
||||
ctx := context.Background()
|
||||
f, err := Load()
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
@@ -44,7 +46,19 @@ func TestLLMPhrasingBaseline(t *testing.T) {
|
||||
p := phraser.NewLLMPhraserAt(base, cfg)
|
||||
defer p.Close()
|
||||
|
||||
rep, err := Score(context.Background(), "llm (0.8B, built-in persona)", p, f)
|
||||
// Label the run with whatever gguf the server actually has loaded. It used
|
||||
// to say "0.8B" no matter what, so two runs of two different models came
|
||||
// out named the same and were easy to mix up when comparing.
|
||||
model, err := llm.ModelID(ctx, base)
|
||||
if err != nil {
|
||||
// An unlabelled score is still a score, but say so loudly — a made-up
|
||||
// name in a bake-off table is worse than no name.
|
||||
t.Logf("could not read model id from %s: %v — report will say %q", base, err, llm.UnknownModel)
|
||||
model = llm.UnknownModel
|
||||
}
|
||||
t.Logf("scoring model %s at %s", model, base)
|
||||
|
||||
rep, err := Score(ctx, "llm ("+model+", built-in persona)", p, f)
|
||||
if err != nil {
|
||||
t.Fatalf("Score: %v", err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user