kami
742b2ad1d7
Same 9 cases as the retrieval eval, so the numbers compare directly: hand-written keywords hit 8 of 8, this is what the model reaches on its own. Reports the hand-written query next to the model's for every case, because where the phrasing differs is the useful part. Opt-in on MAVEN_KIWIX_URL + MAVEN_LLM_URL, like the other evals. Result on Qwen3.5-0.8B: 3 of 8, identical on all three runs. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CGeSZxh1DCtRxmFVSYVGvJ
34 lines
908 B
Go
34 lines
908 B
Go
package kiwix
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/kami/maven/internal/llm"
|
|
)
|
|
|
|
// Opt-in: needs a live Kiwix server AND a live llama-server.
|
|
// MAVEN_KIWIX_URL=http://127.0.0.1:8034 MAVEN_LLM_URL=http://127.0.0.1:18099 \
|
|
//
|
|
// no_proxy=127.0.0.1,localhost go test -run RewriteEval -v ./internal/kiwix/
|
|
func TestRewriteEval(t *testing.T) {
|
|
kbase, lbase := os.Getenv("MAVEN_KIWIX_URL"), os.Getenv("MAVEN_LLM_URL")
|
|
if kbase == "" || lbase == "" {
|
|
t.Skip("set MAVEN_KIWIX_URL and MAVEN_LLM_URL to run the rewrite eval")
|
|
}
|
|
noProxyLoopback(t)
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
|
|
defer cancel()
|
|
|
|
rw := NewRewriter(llm.New(lbase, 3*time.Minute))
|
|
rep, err := RunRewriteEval(ctx, New(kbase), rw, 5)
|
|
if err != nil {
|
|
t.Fatalf("eval: %v", err)
|
|
}
|
|
// No pass bar on purpose: the number is the finding.
|
|
t.Log("\n" + rep.String() + rep.Detail())
|
|
}
|