Merge the eval label and kill script fixes

This commit is contained in:
kami
2026-07-31 14:32:45 +04:00
5 changed files with 184 additions and 12 deletions
+15 -1
View File
@@ -7,6 +7,7 @@ import (
"testing"
"time"
"github.com/kami/maven/internal/llm"
"github.com/kami/maven/internal/phraser"
)
@@ -32,6 +33,7 @@ func TestLLMPhrasingBaseline(t *testing.T) {
// case as a phrasing error and read as "the model cannot phrase".
noProxyLoopback(t)
ctx := context.Background()
f, err := Load()
if err != nil {
t.Fatalf("Load: %v", err)
@@ -44,7 +46,19 @@ func TestLLMPhrasingBaseline(t *testing.T) {
p := phraser.NewLLMPhraserAt(base, cfg)
defer p.Close()
rep, err := Score(context.Background(), "llm (0.8B, built-in persona)", p, f)
// Label the run with whatever gguf the server actually has loaded. It used
// to say "0.8B" no matter what, so two runs of two different models came
// out named the same and were easy to mix up when comparing.
model, err := llm.ModelID(ctx, base)
if err != nil {
// An unlabelled score is still a score, but say so loudly — a made-up
// name in a bake-off table is worse than no name.
t.Logf("could not read model id from %s: %v — report will say %q", base, err, llm.UnknownModel)
model = llm.UnknownModel
}
t.Logf("scoring model %s at %s", model, base)
rep, err := Score(ctx, "llm ("+model+", built-in persona)", p, f)
if err != nil {
t.Fatalf("Score: %v", err)
}