kami
742b2ad1d7
Same 9 cases as the retrieval eval, so the numbers compare directly: hand-written keywords hit 8 of 8, this is what the model reaches on its own. Reports the hand-written query next to the model's for every case, because where the phrasing differs is the useful part. Opt-in on MAVEN_KIWIX_URL + MAVEN_LLM_URL, like the other evals. Result on Qwen3.5-0.8B: 3 of 8, identical on all three runs. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CGeSZxh1DCtRxmFVSYVGvJ
97 lines
2.8 KiB
Go
97 lines
2.8 KiB
Go
package kiwix
|
|
|
|
// End-to-end score: Russian question -> model rewrite -> Kiwix search -> did a
|
|
// wanted article come back. Same 9 cases as the retrieval eval, so the two
|
|
// numbers are directly comparable: retrieval with hand-written keywords is the
|
|
// ceiling, this is what the model actually reaches.
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
)
|
|
|
|
// RewriteOutcome — one case, end to end.
|
|
type RewriteOutcome struct {
|
|
Outcome
|
|
ModelQuery string // what the model asked for ("" if it failed)
|
|
RewriteErr error
|
|
}
|
|
|
|
// RunRewriteEval rewrites every question with the model, then searches.
|
|
func RunRewriteEval(ctx context.Context, c *Client, rw *Rewriter, topN int) (RewriteReport, error) {
|
|
var f fixture
|
|
if err := json.Unmarshal(knowledgeFixtureJSON, &f); err != nil {
|
|
return RewriteReport{}, err
|
|
}
|
|
rep := RewriteReport{Report: Report{Name: f.Name + "-rewrite", Book: f.Book, TopN: topN}}
|
|
for _, cs := range f.Cases {
|
|
out := RewriteOutcome{Outcome: Outcome{Case: cs}}
|
|
q, err := rw.Rewrite(ctx, cs.Question)
|
|
out.ModelQuery, out.RewriteErr = q, err
|
|
if err == nil {
|
|
res, serr := c.Search(ctx, q, f.Book, topN)
|
|
out.Err = serr
|
|
for i, hit := range res {
|
|
out.Titles = append(out.Titles, hit.Title)
|
|
if out.Rank == 0 && matches(cs.WantTitles, hit.Title) {
|
|
out.Rank = i + 1
|
|
}
|
|
}
|
|
}
|
|
if out.RewriteErr != nil || out.Err != nil {
|
|
rep.Errors++
|
|
}
|
|
if !cs.ExpectMiss {
|
|
rep.Scored++
|
|
if out.Hit() {
|
|
rep.Hits++
|
|
}
|
|
}
|
|
rep.Cases = append(rep.Cases, out)
|
|
}
|
|
return rep, nil
|
|
}
|
|
|
|
// RewriteReport — the score plus per-case detail.
|
|
type RewriteReport struct {
|
|
Report
|
|
Cases []RewriteOutcome
|
|
}
|
|
|
|
// String — the headline number.
|
|
func (r RewriteReport) String() string {
|
|
return fmt.Sprintf("%s: %d/%d answerable questions retrieve a wanted article in top %d (%.1f%%), %d errors\n book: %s\n",
|
|
r.Name, r.Hits, r.Scored, r.TopN, 100*r.Accuracy(), r.Errors, r.Book)
|
|
}
|
|
|
|
// Detail — per case: hand-written query next to the model's, and what came back.
|
|
// The point is seeing WHERE the model's phrasing differs, not just the score.
|
|
func (r RewriteReport) Detail() string {
|
|
var b strings.Builder
|
|
for _, o := range r.Cases {
|
|
mark := "MISS"
|
|
switch {
|
|
case o.Case.ExpectMiss:
|
|
mark = "n/a "
|
|
case o.Hit():
|
|
mark = fmt.Sprintf("hit@%d", o.Rank)
|
|
}
|
|
fmt.Fprintf(&b, " %-6s %-20s\n", mark, o.Case.ID)
|
|
fmt.Fprintf(&b, " asked: %s\n", o.Case.Question)
|
|
fmt.Fprintf(&b, " hand: %q\n", o.Case.Query)
|
|
fmt.Fprintf(&b, " model: %q\n", o.ModelQuery)
|
|
if o.RewriteErr != nil {
|
|
fmt.Fprintf(&b, " rewrite rejected: %v\n", o.RewriteErr)
|
|
continue
|
|
}
|
|
if o.Err != nil {
|
|
fmt.Fprintf(&b, " search error: %v\n", o.Err)
|
|
continue
|
|
}
|
|
fmt.Fprintf(&b, " got: %s\n", strings.Join(o.Titles, " | "))
|
|
}
|
|
return b.String()
|
|
}
|