kami
94eb92fb15
The bake-off in #278/#250 needs two models' scores side by side, and the report names only carried the config, so the rows were indistinguishable. ModelID reads /v1/models instead of taking a string that goes stale. New target: make eval-models MAVEN_LLM_URL=... Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CGeSZxh1DCtRxmFVSYVGvJ
52 lines
1.4 KiB
Go
52 lines
1.4 KiB
Go
package eval
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"strings"
|
|
)
|
|
|
|
// ModelID asks llama-server which model it has loaded, so a scoring run can
|
|
// label itself. Without this a bake-off between two models produces two tables
|
|
// that look identical, and the operator has to remember which server was up.
|
|
//
|
|
// Read from the server rather than passed in on purpose: a hand-typed label
|
|
// goes stale the moment someone restarts the server with a different -m.
|
|
func ModelID(ctx context.Context, base string) (string, error) {
|
|
req, err := http.NewRequestWithContext(ctx, "GET", strings.TrimSuffix(base, "/")+"/v1/models", nil)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != 200 {
|
|
return "", fmt.Errorf("models: status %d", resp.StatusCode)
|
|
}
|
|
var out struct {
|
|
Data []struct {
|
|
ID string `json:"id"`
|
|
} `json:"data"`
|
|
}
|
|
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
|
|
return "", err
|
|
}
|
|
if len(out.Data) == 0 {
|
|
return "", fmt.Errorf("models: empty list")
|
|
}
|
|
return shortModelID(out.Data[0].ID), nil
|
|
}
|
|
|
|
// shortModelID trims the path and the .gguf suffix — llama-server reports the
|
|
// file name it was started with, which is too long for a table header.
|
|
func shortModelID(id string) string {
|
|
if i := strings.LastIndexAny(id, "/\\"); i >= 0 {
|
|
id = id[i+1:]
|
|
}
|
|
return strings.TrimSuffix(id, ".gguf")
|
|
}
|