Maven/internal/kiwix/retrieval_eval.go

package kiwix

// This scores retrieval alone: no LLM. For each general-knowledge question we
// hand-write English keywords and ask whether the article that would answer it
// comes back in the top N hits. If this score is low, reading Wikipedia cannot
// help the model no matter how good the prompt is.
//
// The unanswerable case (know-dont-know) is not scored. Whether the junk it
// returns is "nothing useful" is a human judgement, so the report just prints
// the titles and leaves the score to the 8 answerable cases.

import (
	"context"
	_ "embed"
	"encoding/json"
	"fmt"
	"strings"
)

//go:embed knowledge_v1.json
var knowledgeFixtureJSON []byte

// EvalCase — one question with hand-written keywords.
type EvalCase struct {
	ID         string   `json:"id"`
	Question   string   `json:"question"`
	Query      string   `json:"query"`
	WantTitles []string `json:"want_titles"`
	ExpectMiss bool     `json:"expect_miss"`
}

type fixture struct {
	Name  string     `json:"name"`
	Book  string     `json:"book"`
	Cases []EvalCase `json:"cases"`
}

// Outcome — what one case retrieved.
type Outcome struct {
	Case   EvalCase
	Titles []string // titles of the top N hits, in rank order
	Rank   int      // 1-based rank of the first wanted title, 0 if none
	Err    error
}

// Hit is true when a wanted title came back.
func (o Outcome) Hit() bool { return o.Rank > 0 }

// Report — the score plus per-case detail.
type Report struct {
	Name     string
	Book     string
	TopN     int
	Scored   int // answerable cases
	Hits     int
	Errors   int
	Outcomes []Outcome
}

// Accuracy over the answerable cases.
func (r Report) Accuracy() float64 {
	if r.Scored == 0 {
		return 0
	}
	return float64(r.Hits) / float64(r.Scored)
}

// RunRetrievalEval searches for every fixture case.
func RunRetrievalEval(ctx context.Context, c *Client, topN int) (Report, error) {
	var f fixture
	if err := json.Unmarshal(knowledgeFixtureJSON, &f); err != nil {
		return Report{}, err
	}
	rep := Report{Name: f.Name, Book: f.Book, TopN: topN}
	for _, cs := range f.Cases {
		res, err := c.Search(ctx, cs.Query, f.Book, topN)
		o := Outcome{Case: cs, Err: err}
		if err != nil {
			rep.Errors++
		}
		for i, hit := range res {
			o.Titles = append(o.Titles, hit.Title)
			if o.Rank == 0 && matches(cs.WantTitles, hit.Title) {
				o.Rank = i + 1
			}
		}
		if !cs.ExpectMiss {
			rep.Scored++
			if o.Hit() {
				rep.Hits++
			}
		}
		rep.Outcomes = append(rep.Outcomes, o)
	}
	return rep, nil
}

func matches(want []string, title string) bool {
	for _, w := range want {
		if strings.EqualFold(strings.TrimSpace(title), w) {
			return true
		}
	}
	return false
}

// String — the headline number.
func (r Report) String() string {
	var b strings.Builder
	fmt.Fprintf(&b, "%s: %d/%d answerable questions retrieve a wanted article in top %d (%.1f%%), %d errors\n",
		r.Name, r.Hits, r.Scored, r.TopN, 100*r.Accuracy(), r.Errors)
	fmt.Fprintf(&b, "  book: %s\n", r.Book)
	return b.String()
}

// Detail — per case: what was asked, what was searched, what came back.
func (r Report) Detail() string {
	var b strings.Builder
	for _, o := range r.Outcomes {
		mark := "MISS"
		switch {
		case o.Case.ExpectMiss:
			mark = "n/a "
		case o.Hit():
			mark = fmt.Sprintf("hit@%d", o.Rank)
		}
		fmt.Fprintf(&b, "  %-6s %-20s q=%q\n", mark, o.Case.ID, o.Case.Query)
		if o.Err != nil {
			fmt.Fprintf(&b, "         error: %v\n", o.Err)
			continue
		}
		fmt.Fprintf(&b, "         got: %s\n", strings.Join(o.Titles, " | "))
	}
	return b.String()
}