From 2ea39a3d419404979109f0d012665f639b8f8cd7 Mon Sep 17 00:00:00 2001 From: claude Date: Sun, 9 Aug 2026 10:42:45 +0400 Subject: [PATCH] Search Kiwix for the topic, not the whole sentence (V-668) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kiwix ranks by keyword overlap, which the package doc has said since it was written: "why is the sky blue" finds a TV episode. queryKiwix sent the whole Russian sentence, because the verbatim path added by V-508 skips the rewriter that would have reduced it. Measured against the Russian ZIM on 2026-08-09, over eight questions. Four reach the right article where they did not: TCP was "Перехват TCP-соединения" and is TCP, фотосинтез was "C4-фотосинтез" and is Фотосинтез, Линус Торвальдс was "Tux", and "кто написал Войну и мир" was "Радуйся, мир (Доктор Кто)". Two were already right and stay right. Two are still wrong and were wrong before. Nothing regressed. kiwix.Topic drops the narrative request, the interrogative and a verb behind one, and keeps everything else. A word it cannot classify is more likely the topic than noise. TitlePath tries the exact article first, since a ZIM is addressable by title and a wrong title is a 404. The gate this task set out to build does not exist. Query-to-passage cosine scored 0.79-0.91 on answerable questions and 0.75-0.84 on unanswerable ones, and the sets overlap. The wrong TCP article scored 0.8653, above five of six unanswerable rows. e5 measures topic, not whether the passage answers. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013ptwopxyo3Z2kwFckHkLvN --- cmd/mavend/actions_query.go | 33 ++++++++++-- internal/kiwix/client.go | 16 ++++++ internal/kiwix/live_test.go | 65 ++++++++++++++++++++++++ internal/kiwix/topic.go | 98 ++++++++++++++++++++++++++++++++++++ internal/kiwix/topic_test.go | 51 +++++++++++++++++++ 5 files changed, 259 insertions(+), 4 deletions(-) create mode 100644 internal/kiwix/live_test.go create mode 100644 internal/kiwix/topic.go create mode 100644 internal/kiwix/topic_test.go diff --git a/cmd/mavend/actions_query.go b/cmd/mavend/actions_query.go index a945805..4a79406 100644 --- a/cmd/mavend/actions_query.go +++ b/cmd/mavend/actions_query.go @@ -13,6 +13,7 @@ import ( "github.com/kami/maven/internal/crawl" "github.com/kami/maven/internal/decision" "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/kiwix" "github.com/kami/maven/internal/memory" "github.com/kami/maven/internal/morning" "github.com/kami/maven/internal/phraser" @@ -926,6 +927,26 @@ func (h *reactiveHandler) queryKiwix(ctx context.Context, t *queryTurn) (string, } } + // The topic, not the sentence (V-668). Kiwix ranks by keyword overlap, so + // the question words outrank the one word that names the article: measured + // on 2026-08-09, "что такое TCP" returns "Перехват TCP-соединения" and + // "TCP" returns TCP. Only the verbatim path needs this. The rewriter + // already reduces a question to English keywords, and reducing twice would + // take the topic off the input it reads. + if verbatim { + if topic := kiwix.Topic(pattern); topic != "" { + // The article named exactly, before any ranking runs. A ZIM is + // addressable by title and a wrong title is a 404, so this either + // answers or costs one request that says nothing. + page, err := h.kiwix.client.Article(ctxK, kiwix.TitlePath(book, topic), h.kiwix.runes) + if err == nil && page.Text != "" { + log.Printf("voice: kiwix: %q in %q → title hit %q", topic, book, page.Title) + return h.kiwixReply(ctx, t, page.Title, page.Text) + } + pattern = topic + } + } + hits, err := h.kiwix.client.Search(ctxK, pattern, book, h.kiwix.max) if err != nil { log.Printf("voice: kiwix: search %q: %v", pattern, err) @@ -958,14 +979,18 @@ func (h *reactiveHandler) queryKiwix(ctx context.Context, t *queryTurn) (string, } page = crawl.Page{Title: top.Title, Text: top.Snippet} } - // Handed over the same way a note or a page is: context for the question he - // asked, not something to recite. - snippet := top.Title + "\n" + crawl.TrimRunes(page.Text, h.kiwix.runes) + return h.kiwixReply(ctx, t, top.Title, page.Text) +} + +// kiwixReply hands one article over the same way a note or a page is handed +// over: context for the question he asked, not something to recite. +func (h *reactiveHandler) kiwixReply(ctx context.Context, t *queryTurn, title, text string) (string, bool) { + snippet := title + "\n" + crawl.TrimRunes(text, h.kiwix.runes) reply := h.phraseSource(ctx, "kiwix", t.dec.Utterance, []string{snippet}) if reply == "" { // No phraser, or it failed. Read back the best hit rather than pretend // the search did not happen. - return readBack(top.Title + " — " + page.Text), true + return readBack(title + " — " + text), true } return reply, true } diff --git a/internal/kiwix/client.go b/internal/kiwix/client.go index 52a218d..f057046 100644 --- a/internal/kiwix/client.go +++ b/internal/kiwix/client.go @@ -127,6 +127,22 @@ func (c *Client) Article(ctx context.Context, path string, maxRunes int) (crawl. return crawl.Extract(u, body, maxRunes), nil } +// TitlePath is the article path for an exact title, for Article to fetch. +// +// It exists because a ZIM is addressable by title and the full-text index is +// not the only way in. "Франция", "TCP" and "Небо" resolve; "Трюмбальная +// нидроскопия" is a 404, which is the honest answer and the reason this is +// safe to try first. Measured on 2026-08-09, keyword search on the same terms +// returns "Список пэров Франции" and "Список портов TCP и UDP" instead. +// +// A miss is normal rather than a failure. An article whose title inverts a name +// ("Торвальдс, Линус") is a 404 here and the first hit in search, so the caller +// falls through and loses nothing. +func TitlePath(book, title string) string { + t := strings.ReplaceAll(strings.TrimSpace(title), " ", "_") + return "/content/" + url.PathEscape(book) + "/A/" + url.PathEscape(t) +} + // rss mirrors just the bits of the RSS 2.0 reply we use. type rss struct { Items []struct { diff --git a/internal/kiwix/live_test.go b/internal/kiwix/live_test.go new file mode 100644 index 0000000..95343de --- /dev/null +++ b/internal/kiwix/live_test.go @@ -0,0 +1,65 @@ +package kiwix + +import ( + "context" + "os" + "testing" + "time" +) + +// TestLiveTopicBeatsTheSentence — the measurement V-668 turned on, kept as a +// test so the claim can be re-run rather than believed. +// +// It prints the article the old path returned and the article the new one +// returns, for the same question. It asserts nothing about which is better, +// because "is this the right article" is a human's call. It fails only if the +// two paths agree on every case, which would mean the change does nothing. +// +// MAVEN_KIWIX_URL=http://127.0.0.1:8034 make t PKG=./internal/kiwix/ RUN=TestLive V=1 +func TestLiveTopicBeatsTheSentence(t *testing.T) { + base := os.Getenv("MAVEN_KIWIX_URL") + if base == "" { + t.Skip("MAVEN_KIWIX_URL unset — point it at the kiwix-server host port") + } + const book = "wikipedia_ru_all_maxi_2026-02" + c := New(base) + questions := []string{ + "что такое TCP?", + "что такое фотосинтез", + "кто такой Линус Торвальдс?", + "кто написал Войну и мир", + "что такое чёрная дыра", + "почему небо голубое", + "почему трава зелёная", + "столица Франции", + } + moved := 0 + for _, q := range questions { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + before := firstTitle(ctx, c, q, book) + topic := Topic(q) + after := "" + if page, err := c.Article(ctx, TitlePath(book, topic), 400); err == nil && page.Text != "" { + after = page.Title + " (by title)" + } else { + after = firstTitle(ctx, c, topic, book) + } + cancel() + if before != after { + moved++ + } + t.Logf("%-30s before=%-34q after=%q", q, before, after) + } + t.Logf("%d of %d questions reach a different article", moved, len(questions)) + if moved == 0 { + t.Error("the topic path returns exactly what the sentence path returned") + } +} + +func firstTitle(ctx context.Context, c *Client, pattern, book string) string { + hits, err := c.Search(ctx, pattern, book, 3) + if err != nil || len(hits) == 0 { + return "(nothing)" + } + return hits[0].Title +} diff --git a/internal/kiwix/topic.go b/internal/kiwix/topic.go new file mode 100644 index 0000000..2237070 --- /dev/null +++ b/internal/kiwix/topic.go @@ -0,0 +1,98 @@ +package kiwix + +import ( + "strings" + "unicode" + + "github.com/kami/maven/internal/lexicon" + "github.com/kami/maven/internal/morph" +) + +// Topic reduces a question to the thing it is about, because Kiwix ranks by +// keyword overlap and a whole sentence buries the keyword that matters. +// +// This package's own doc says it: "why is the sky blue" finds a TV episode. +// Measured against the Russian ZIM on 2026-08-09, the sentence and the topic +// return different articles for the same question. "кто написал Войну и мир" +// returns "Радуйся, мир (Доктор Кто)"; "Войну и мир" returns the novel first. +// "что такое TCP" returns "Перехват TCP-соединения"; "TCP" returns TCP. The +// English path had a rewriter doing this with a model call. The Russian path +// reads the book verbatim (V-508) and had nothing. +// +// It drops three things off the front and stops: the narrative request, the +// interrogative, and a verb sitting between them and the noun. Everything else +// is kept, because a word this cannot classify is more likely the topic than +// noise. An empty return means the utterance was question words alone, and the +// caller searches the sentence as before. +func Topic(utterance string) string { + words := strings.Fields(strings.TrimSpace(utterance)) + cut := 0 + for cut < len(words) { + w := strings.Trim(strings.ToLower(words[cut]), ".,!?…:;\"'«»") + if w == "" { + cut++ + continue + } + switch { + case inList(lexicon.NarrativeRequests(), w), + inList(lexicon.Interrogatives(), w), + inList(lexicon.FirstPerson(), w), + // "что ТАКОЕ x", "кто ТАКОЙ x" — the copula that only ever follows + // an interrogative, and never a topic on its own. + cut > 0 && isCopula(w), + // "расскажи ПРО x", "о x". One-letter and two-letter prepositions + // are not a closed class worth a lexicon set of their own. + cut > 0 && isLeadingPreposition(w), + // "кто НАПИСАЛ Войну и мир". A verb here is the question's own + // verb, not part of the title. Only after something was already + // dropped, so "написал отчёт" as a topic survives intact. + cut > 0 && morph.IsVerbForm(w): + cut++ + default: + // The question mark is the sentence's, not the title's, and Kiwix + // carries it into the keyword match. + topic := strings.TrimRight(strings.Join(words[cut:], " "), " .,!?…:;\"'«»") + if !hasLetter(topic) { + return "" + } + return topic + } + } + return "" +} + +func isCopula(w string) bool { + switch w { + case "такое", "такой", "такая", "такие", "is", "are", "was", "were": + return true + } + return false +} + +func isLeadingPreposition(w string) bool { + switch w { + case "про", "о", "об", "обо", "по", "about", "of", "on": + return true + } + return false +} + +func inList(list []string, w string) bool { + for _, x := range list { + if x == w { + return true + } + } + return false +} + +// hasLetter is the guard against a topic that reduced to punctuation or digits +// alone, which no ZIM title matches. +func hasLetter(s string) bool { + for _, r := range s { + if unicode.IsLetter(r) { + return true + } + } + return false +} diff --git a/internal/kiwix/topic_test.go b/internal/kiwix/topic_test.go new file mode 100644 index 0000000..bc01683 --- /dev/null +++ b/internal/kiwix/topic_test.go @@ -0,0 +1,51 @@ +package kiwix + +import "testing" + +// The cases the 2026-08-09 measurement turned on, plus the ones a topic must +// not damage. Each left column returned a wrong article when it was sent whole. +func TestTopicKeepsTheThingTheQuestionIsAbout(t *testing.T) { + cases := []struct{ utterance, want string }{ + {"что такое TCP?", "TCP"}, + {"что такое фотосинтез", "фотосинтез"}, + {"кто такой Линус Торвальдс?", "Линус Торвальдс"}, + {"кто написал Войну и мир", "Войну и мир"}, + {"расскажи про битву при Ватерлоо", "битву при Ватерлоо"}, + {"what is photosynthesis", "photosynthesis"}, + // No question word, so there is nothing to drop. The topic is the + // whole utterance and the search is what it was before. + {"столица Франции", "столица Франции"}, + {"почему небо голубое", "небо голубое"}, + } + for _, c := range cases { + if got := Topic(c.utterance); got != c.want { + t.Errorf("Topic(%q) = %q, want %q", c.utterance, got, c.want) + } + } +} + +// A verb only goes when a question word already went. Otherwise "написал +// отчёт" loses the verb that names what he means. +func TestTopicDropsAVerbOnlyBehindAQuestionWord(t *testing.T) { + if got := Topic("написал отчёт"); got != "написал отчёт" { + t.Errorf("Topic dropped a leading verb with no question word: %q", got) + } +} + +// Question words alone reduce to nothing, and the caller reads that as "no +// topic" and searches the sentence rather than searching the empty string. +func TestTopicIsEmptyWhenNothingIsLeft(t *testing.T) { + for _, q := range []string{"что такое?", "кто?", "почему", "???"} { + if got := Topic(q); got != "" { + t.Errorf("Topic(%q) = %q, want empty", q, got) + } + } +} + +func TestTitlePathEscapesAndUnderscores(t *testing.T) { + got := TitlePath("wikipedia_ru_all_maxi_2026-02", "Чёрная дыра") + want := "/content/wikipedia_ru_all_maxi_2026-02/A/%D0%A7%D1%91%D1%80%D0%BD%D0%B0%D1%8F_%D0%B4%D1%8B%D1%80%D0%B0" + if got != want { + t.Errorf("TitlePath = %q, want %q", got, want) + } +}