Search Kiwix for the topic, not the whole sentence (V-668)

Kiwix ranks by keyword overlap, which the package doc has said since it was
written: "why is the sky blue" finds a TV episode. queryKiwix sent the whole
Russian sentence, because the verbatim path added by V-508 skips the rewriter
that would have reduced it.

Measured against the Russian ZIM on 2026-08-09, over eight questions. Four
reach the right article where they did not: TCP was "Перехват TCP-соединения"
and is TCP, фотосинтез was "C4-фотосинтез" and is Фотосинтез, Линус Торвальдс
was "Tux", and "кто написал Войну и мир" was "Радуйся, мир (Доктор Кто)".
Two were already right and stay right. Two are still wrong and were wrong
before. Nothing regressed.

kiwix.Topic drops the narrative request, the interrogative and a verb behind
one, and keeps everything else. A word it cannot classify is more likely the
topic than noise. TitlePath tries the exact article first, since a ZIM is
addressable by title and a wrong title is a 404.

The gate this task set out to build does not exist. Query-to-passage cosine
scored 0.79-0.91 on answerable questions and 0.75-0.84 on unanswerable ones,
and the sets overlap. The wrong TCP article scored 0.8653, above five of six
unanswerable rows. e5 measures topic, not whether the passage answers.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013ptwopxyo3Z2kwFckHkLvN

internal/kiwix/client.go

@@ -127,6 +127,22 @@ func (c *Client) Article(ctx context.Context, path string, maxRunes int) (crawl.
 	return crawl.Extract(u, body, maxRunes), nil
 }
 
+// TitlePath is the article path for an exact title, for Article to fetch.
+//
+// It exists because a ZIM is addressable by title and the full-text index is
+// not the only way in. "Франция", "TCP" and "Небо" resolve; "Трюмбальная
+// нидроскопия" is a 404, which is the honest answer and the reason this is
+// safe to try first. Measured on 2026-08-09, keyword search on the same terms
+// returns "Список пэров Франции" and "Список портов TCP и UDP" instead.
+//
+// A miss is normal rather than a failure. An article whose title inverts a name
+// ("Торвальдс, Линус") is a 404 here and the first hit in search, so the caller
+// falls through and loses nothing.
+func TitlePath(book, title string) string {
+	t := strings.ReplaceAll(strings.TrimSpace(title), " ", "_")
+	return "/content/" + url.PathEscape(book) + "/A/" + url.PathEscape(t)
+}
+
 // rss mirrors just the bits of the RSS 2.0 reply we use.
 type rss struct {
 	Items []struct {

internal/kiwix/live_test.go

@@ -0,0 +1,65 @@
+package kiwix
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+)
+
+// TestLiveTopicBeatsTheSentence — the measurement V-668 turned on, kept as a
+// test so the claim can be re-run rather than believed.
+//
+// It prints the article the old path returned and the article the new one
+// returns, for the same question. It asserts nothing about which is better,
+// because "is this the right article" is a human's call. It fails only if the
+// two paths agree on every case, which would mean the change does nothing.
+//
+//	MAVEN_KIWIX_URL=http://127.0.0.1:8034 make t PKG=./internal/kiwix/ RUN=TestLive V=1
+func TestLiveTopicBeatsTheSentence(t *testing.T) {
+	base := os.Getenv("MAVEN_KIWIX_URL")
+	if base == "" {
+		t.Skip("MAVEN_KIWIX_URL unset — point it at the kiwix-server host port")
+	}
+	const book = "wikipedia_ru_all_maxi_2026-02"
+	c := New(base)
+	questions := []string{
+		"что такое TCP?",
+		"что такое фотосинтез",
+		"кто такой Линус Торвальдс?",
+		"кто написал Войну и мир",
+		"что такое чёрная дыра",
+		"почему небо голубое",
+		"почему трава зелёная",
+		"столица Франции",
+	}
+	moved := 0
+	for _, q := range questions {
+		ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+		before := firstTitle(ctx, c, q, book)
+		topic := Topic(q)
+		after := ""
+		if page, err := c.Article(ctx, TitlePath(book, topic), 400); err == nil && page.Text != "" {
+			after = page.Title + " (by title)"
+		} else {
+			after = firstTitle(ctx, c, topic, book)
+		}
+		cancel()
+		if before != after {
+			moved++
+		}
+		t.Logf("%-30s  before=%-34q after=%q", q, before, after)
+	}
+	t.Logf("%d of %d questions reach a different article", moved, len(questions))
+	if moved == 0 {
+		t.Error("the topic path returns exactly what the sentence path returned")
+	}
+}
+
+func firstTitle(ctx context.Context, c *Client, pattern, book string) string {
+	hits, err := c.Search(ctx, pattern, book, 3)
+	if err != nil || len(hits) == 0 {
+		return "(nothing)"
+	}
+	return hits[0].Title
+}

internal/kiwix/topic.go

@@ -0,0 +1,98 @@
+package kiwix
+
+import (
+	"strings"
+	"unicode"
+
+	"github.com/kami/maven/internal/lexicon"
+	"github.com/kami/maven/internal/morph"
+)
+
+// Topic reduces a question to the thing it is about, because Kiwix ranks by
+// keyword overlap and a whole sentence buries the keyword that matters.
+//
+// This package's own doc says it: "why is the sky blue" finds a TV episode.
+// Measured against the Russian ZIM on 2026-08-09, the sentence and the topic
+// return different articles for the same question. "кто написал Войну и мир"
+// returns "Радуйся, мир (Доктор Кто)"; "Войну и мир" returns the novel first.
+// "что такое TCP" returns "Перехват TCP-соединения"; "TCP" returns TCP. The
+// English path had a rewriter doing this with a model call. The Russian path
+// reads the book verbatim (V-508) and had nothing.
+//
+// It drops three things off the front and stops: the narrative request, the
+// interrogative, and a verb sitting between them and the noun. Everything else
+// is kept, because a word this cannot classify is more likely the topic than
+// noise. An empty return means the utterance was question words alone, and the
+// caller searches the sentence as before.
+func Topic(utterance string) string {
+	words := strings.Fields(strings.TrimSpace(utterance))
+	cut := 0
+	for cut < len(words) {
+		w := strings.Trim(strings.ToLower(words[cut]), ".,!?…:;\"'«»")
+		if w == "" {
+			cut++
+			continue
+		}
+		switch {
+		case inList(lexicon.NarrativeRequests(), w),
+			inList(lexicon.Interrogatives(), w),
+			inList(lexicon.FirstPerson(), w),
+			// "что ТАКОЕ x", "кто ТАКОЙ x" — the copula that only ever follows
+			// an interrogative, and never a topic on its own.
+			cut > 0 && isCopula(w),
+			// "расскажи ПРО x", "о x". One-letter and two-letter prepositions
+			// are not a closed class worth a lexicon set of their own.
+			cut > 0 && isLeadingPreposition(w),
+			// "кто НАПИСАЛ Войну и мир". A verb here is the question's own
+			// verb, not part of the title. Only after something was already
+			// dropped, so "написал отчёт" as a topic survives intact.
+			cut > 0 && morph.IsVerbForm(w):
+			cut++
+		default:
+			// The question mark is the sentence's, not the title's, and Kiwix
+			// carries it into the keyword match.
+			topic := strings.TrimRight(strings.Join(words[cut:], " "), " .,!?…:;\"'«»")
+			if !hasLetter(topic) {
+				return ""
+			}
+			return topic
+		}
+	}
+	return ""
+}
+
+func isCopula(w string) bool {
+	switch w {
+	case "такое", "такой", "такая", "такие", "is", "are", "was", "were":
+		return true
+	}
+	return false
+}
+
+func isLeadingPreposition(w string) bool {
+	switch w {
+	case "про", "о", "об", "обо", "по", "about", "of", "on":
+		return true
+	}
+	return false
+}
+
+func inList(list []string, w string) bool {
+	for _, x := range list {
+		if x == w {
+			return true
+		}
+	}
+	return false
+}
+
+// hasLetter is the guard against a topic that reduced to punctuation or digits
+// alone, which no ZIM title matches.
+func hasLetter(s string) bool {
+	for _, r := range s {
+		if unicode.IsLetter(r) {
+			return true
+		}
+	}
+	return false
+}

internal/kiwix/topic_test.go

@@ -0,0 +1,51 @@
+package kiwix
+
+import "testing"
+
+// The cases the 2026-08-09 measurement turned on, plus the ones a topic must
+// not damage. Each left column returned a wrong article when it was sent whole.
+func TestTopicKeepsTheThingTheQuestionIsAbout(t *testing.T) {
+	cases := []struct{ utterance, want string }{
+		{"что такое TCP?", "TCP"},
+		{"что такое фотосинтез", "фотосинтез"},
+		{"кто такой Линус Торвальдс?", "Линус Торвальдс"},
+		{"кто написал Войну и мир", "Войну и мир"},
+		{"расскажи про битву при Ватерлоо", "битву при Ватерлоо"},
+		{"what is photosynthesis", "photosynthesis"},
+		// No question word, so there is nothing to drop. The topic is the
+		// whole utterance and the search is what it was before.
+		{"столица Франции", "столица Франции"},
+		{"почему небо голубое", "небо голубое"},
+	}
+	for _, c := range cases {
+		if got := Topic(c.utterance); got != c.want {
+			t.Errorf("Topic(%q) = %q, want %q", c.utterance, got, c.want)
+		}
+	}
+}
+
+// A verb only goes when a question word already went. Otherwise "написал
+// отчёт" loses the verb that names what he means.
+func TestTopicDropsAVerbOnlyBehindAQuestionWord(t *testing.T) {
+	if got := Topic("написал отчёт"); got != "написал отчёт" {
+		t.Errorf("Topic dropped a leading verb with no question word: %q", got)
+	}
+}
+
+// Question words alone reduce to nothing, and the caller reads that as "no
+// topic" and searches the sentence rather than searching the empty string.
+func TestTopicIsEmptyWhenNothingIsLeft(t *testing.T) {
+	for _, q := range []string{"что такое?", "кто?", "почему", "???"} {
+		if got := Topic(q); got != "" {
+			t.Errorf("Topic(%q) = %q, want empty", q, got)
+		}
+	}
+}
+
+func TestTitlePathEscapesAndUnderscores(t *testing.T) {
+	got := TitlePath("wikipedia_ru_all_maxi_2026-02", "Чёрная дыра")
+	want := "/content/wikipedia_ru_all_maxi_2026-02/A/%D0%A7%D1%91%D1%80%D0%BD%D0%B0%D1%8F_%D0%B4%D1%8B%D1%80%D0%B0"
+	if got != want {
+		t.Errorf("TitlePath = %q, want %q", got, want)
+	}
+}