From 742b2ad1d7ee115c26e1687d3377581455a4fa69 Mon Sep 17 00:00:00 2001
From: kami <kami@kvmx.ru>
Date: Fri, 31 Jul 2026 18:19:56 +0400
Subject: [PATCH] Score the Russian-to-keywords rewrite end to end (#403)

Same 9 cases as the retrieval eval, so the numbers compare directly:
hand-written keywords hit 8 of 8, this is what the model reaches on its
own. Reports the hand-written query next to the model's for every case,
because where the phrasing differs is the useful part.

Opt-in on MAVEN_KIWIX_URL + MAVEN_LLM_URL, like the other evals.

Result on Qwen3.5-0.8B: 3 of 8, identical on all three runs.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CGeSZxh1DCtRxmFVSYVGvJ
---
 internal/kiwix/rewrite_eval.go      | 96 +++++++++++++++++++++++++++++
 internal/kiwix/rewrite_eval_test.go | 33 ++++++++++
 2 files changed, 129 insertions(+)
 create mode 100644 internal/kiwix/rewrite_eval.go
 create mode 100644 internal/kiwix/rewrite_eval_test.go

diff --git a/internal/kiwix/rewrite_eval.go b/internal/kiwix/rewrite_eval.go
new file mode 100644
index 0000000..6a74952
--- /dev/null
+++ b/internal/kiwix/rewrite_eval.go
@@ -0,0 +1,96 @@
+package kiwix
+
+// End-to-end score: Russian question -> model rewrite -> Kiwix search -> did a
+// wanted article come back. Same 9 cases as the retrieval eval, so the two
+// numbers are directly comparable: retrieval with hand-written keywords is the
+// ceiling, this is what the model actually reaches.
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+)
+
+// RewriteOutcome — one case, end to end.
+type RewriteOutcome struct {
+	Outcome
+	ModelQuery string // what the model asked for ("" if it failed)
+	RewriteErr error
+}
+
+// RunRewriteEval rewrites every question with the model, then searches.
+func RunRewriteEval(ctx context.Context, c *Client, rw *Rewriter, topN int) (RewriteReport, error) {
+	var f fixture
+	if err := json.Unmarshal(knowledgeFixtureJSON, &f); err != nil {
+		return RewriteReport{}, err
+	}
+	rep := RewriteReport{Report: Report{Name: f.Name + "-rewrite", Book: f.Book, TopN: topN}}
+	for _, cs := range f.Cases {
+		out := RewriteOutcome{Outcome: Outcome{Case: cs}}
+		q, err := rw.Rewrite(ctx, cs.Question)
+		out.ModelQuery, out.RewriteErr = q, err
+		if err == nil {
+			res, serr := c.Search(ctx, q, f.Book, topN)
+			out.Err = serr
+			for i, hit := range res {
+				out.Titles = append(out.Titles, hit.Title)
+				if out.Rank == 0 && matches(cs.WantTitles, hit.Title) {
+					out.Rank = i + 1
+				}
+			}
+		}
+		if out.RewriteErr != nil || out.Err != nil {
+			rep.Errors++
+		}
+		if !cs.ExpectMiss {
+			rep.Scored++
+			if out.Hit() {
+				rep.Hits++
+			}
+		}
+		rep.Cases = append(rep.Cases, out)
+	}
+	return rep, nil
+}
+
+// RewriteReport — the score plus per-case detail.
+type RewriteReport struct {
+	Report
+	Cases []RewriteOutcome
+}
+
+// String — the headline number.
+func (r RewriteReport) String() string {
+	return fmt.Sprintf("%s: %d/%d answerable questions retrieve a wanted article in top %d (%.1f%%), %d errors\n  book: %s\n",
+		r.Name, r.Hits, r.Scored, r.TopN, 100*r.Accuracy(), r.Errors, r.Book)
+}
+
+// Detail — per case: hand-written query next to the model's, and what came back.
+// The point is seeing WHERE the model's phrasing differs, not just the score.
+func (r RewriteReport) Detail() string {
+	var b strings.Builder
+	for _, o := range r.Cases {
+		mark := "MISS"
+		switch {
+		case o.Case.ExpectMiss:
+			mark = "n/a "
+		case o.Hit():
+			mark = fmt.Sprintf("hit@%d", o.Rank)
+		}
+		fmt.Fprintf(&b, "  %-6s %-20s\n", mark, o.Case.ID)
+		fmt.Fprintf(&b, "         asked:  %s\n", o.Case.Question)
+		fmt.Fprintf(&b, "         hand:   %q\n", o.Case.Query)
+		fmt.Fprintf(&b, "         model:  %q\n", o.ModelQuery)
+		if o.RewriteErr != nil {
+			fmt.Fprintf(&b, "         rewrite rejected: %v\n", o.RewriteErr)
+			continue
+		}
+		if o.Err != nil {
+			fmt.Fprintf(&b, "         search error: %v\n", o.Err)
+			continue
+		}
+		fmt.Fprintf(&b, "         got:    %s\n", strings.Join(o.Titles, " | "))
+	}
+	return b.String()
+}
diff --git a/internal/kiwix/rewrite_eval_test.go b/internal/kiwix/rewrite_eval_test.go
new file mode 100644
index 0000000..aa4ee2d
--- /dev/null
+++ b/internal/kiwix/rewrite_eval_test.go
@@ -0,0 +1,33 @@
+package kiwix
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/kami/maven/internal/llm"
+)
+
+// Opt-in: needs a live Kiwix server AND a live llama-server.
+// MAVEN_KIWIX_URL=http://127.0.0.1:8034 MAVEN_LLM_URL=http://127.0.0.1:18099 \
+//
+//	no_proxy=127.0.0.1,localhost go test -run RewriteEval -v ./internal/kiwix/
+func TestRewriteEval(t *testing.T) {
+	kbase, lbase := os.Getenv("MAVEN_KIWIX_URL"), os.Getenv("MAVEN_LLM_URL")
+	if kbase == "" || lbase == "" {
+		t.Skip("set MAVEN_KIWIX_URL and MAVEN_LLM_URL to run the rewrite eval")
+	}
+	noProxyLoopback(t)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
+	defer cancel()
+
+	rw := NewRewriter(llm.New(lbase, 3*time.Minute))
+	rep, err := RunRewriteEval(ctx, New(kbase), rw, 5)
+	if err != nil {
+		t.Fatalf("eval: %v", err)
+	}
+	// No pass bar on purpose: the number is the finding.
+	t.Log("\n" + rep.String() + rep.Detail())
+}