Merge pull request 'Kiwix answers a question it cannot answer, and nothing gates it' (#212) from task/668-kiwix-answers-a-question-it-cannot-answe into master

CLAUDE.md

@@ -675,11 +675,25 @@ world questions, so she needs to read external sources. What replaces it:
   `wikipedia_ru_all_maxi_2026-02` verbatim** through `kiwix.book_ru`. The RU→EN rewriter
   is the workaround for an English book and is skipped there. Kiwix catalog names come
   from the filename, not the `<name>` field.
+  **That verbatim path sent the whole sentence to a keyword engine until 09-08-2026**
+  (V-668, `docs/evals/2026-08-09-kiwix-topic-retrieval.md`). Kiwix ranks by keyword
+  overlap, so the question words outrank the one word naming the article. "что такое TCP"
+  returned "Перехват TCP-соединения". "кто написал Войну и мир" returned an episode of
+  Doctor Who. `kiwix.Topic` drops the narrative request, the interrogative and a verb
+  behind one. `kiwix.TitlePath` tries the exact article first, since a ZIM is addressable
+  by title and a wrong title is a 404. Four of eight questions reach the right article
+  where they did not, two were already right, and nothing regressed. Both apply on the
+  verbatim path alone. The rewriter already reduces a question, and reducing twice takes
+  the topic off its input.
   `Response.Empty()` is the whole gate and there is no quality threshold in front of it:
   the three signals one could read were measured on 2026-08-05 and none of them separate a
   real question from an invented one. Token overlap would cost "столица Франции" its
   answer, because the answer is Париж and that word is not in the question. See
-  `docs/evals/2026-08-05-search-quality-signals.md` (V-539). **Which query source claimed
+  `docs/evals/2026-08-05-search-quality-signals.md` (V-539). **The embedder is not a
+  fourth signal**, measured 2026-08-09 (V-668). Query-to-passage cosine scores 0.79 to
+  0.91 on answerable questions and 0.75 to 0.84 on unanswerable ones, and the sets
+  overlap. The wrong TCP article scored 0.8653, above five of six unanswerable rows. It
+  measures topic and not whether the passage answers, so no threshold splits them. **Which query source claimed
   a turn is readable on `/chat`** as a badge beside the reply, carried on
   `ipc.ChatReply.Source` and noted by `noteQuerySource` in `cmd/mavend/querysource.go`. It
   rides the context, so `handleText` keeps the one string signature the mic, telegram and

cmd/mavend/actions_query.go

@@ -13,6 +13,7 @@ import (
 	"github.com/kami/maven/internal/crawl"
 	"github.com/kami/maven/internal/decision"
 	"github.com/kami/maven/internal/ipc"
+	"github.com/kami/maven/internal/kiwix"
 	"github.com/kami/maven/internal/memory"
 	"github.com/kami/maven/internal/morning"
 	"github.com/kami/maven/internal/phraser"
@@ -926,6 +927,26 @@ func (h *reactiveHandler) queryKiwix(ctx context.Context, t *queryTurn) (string,
 		}
 	}
 
+	// The topic, not the sentence (V-668). Kiwix ranks by keyword overlap, so
+	// the question words outrank the one word that names the article: measured
+	// on 2026-08-09, "что такое TCP" returns "Перехват TCP-соединения" and
+	// "TCP" returns TCP. Only the verbatim path needs this. The rewriter
+	// already reduces a question to English keywords, and reducing twice would
+	// take the topic off the input it reads.
+	if verbatim {
+		if topic := kiwix.Topic(pattern); topic != "" {
+			// The article named exactly, before any ranking runs. A ZIM is
+			// addressable by title and a wrong title is a 404, so this either
+			// answers or costs one request that says nothing.
+			page, err := h.kiwix.client.Article(ctxK, kiwix.TitlePath(book, topic), h.kiwix.runes)
+			if err == nil && page.Text != "" {
+				log.Printf("voice: kiwix: %q in %q → title hit %q", topic, book, page.Title)
+				return h.kiwixReply(ctx, t, page.Title, page.Text)
+			}
+			pattern = topic
+		}
+	}
+
 	hits, err := h.kiwix.client.Search(ctxK, pattern, book, h.kiwix.max)
 	if err != nil {
 		log.Printf("voice: kiwix: search %q: %v", pattern, err)
@@ -958,14 +979,18 @@ func (h *reactiveHandler) queryKiwix(ctx context.Context, t *queryTurn) (string,
 		}
 		page = crawl.Page{Title: top.Title, Text: top.Snippet}
 	}
-	// Handed over the same way a note or a page is: context for the question he
-	// asked, not something to recite.
-	snippet := top.Title + "\n" + crawl.TrimRunes(page.Text, h.kiwix.runes)
+	return h.kiwixReply(ctx, t, top.Title, page.Text)
+}
+
+// kiwixReply hands one article over the same way a note or a page is handed
+// over: context for the question he asked, not something to recite.
+func (h *reactiveHandler) kiwixReply(ctx context.Context, t *queryTurn, title, text string) (string, bool) {
+	snippet := title + "\n" + crawl.TrimRunes(text, h.kiwix.runes)
 	reply := h.phraseSource(ctx, "kiwix", t.dec.Utterance, []string{snippet})
 	if reply == "" {
 		// No phraser, or it failed. Read back the best hit rather than pretend
 		// the search did not happen.
-		return readBack(top.Title + " — " + page.Text), true
+		return readBack(title + " — " + text), true
 	}
 	return reply, true
 }

cmd/mavwaked/silero.go

@@ -52,10 +52,10 @@ type sileroVAD struct {
 	mu      sync.Mutex
 	session *ort.DynamicAdvancedSession
 
-	pending []float32               // samples not yet part of a full window
-	context [sileroContext]float32  // tail of the previous window
-	state   []float32               // [2][1][128], carried between windows
-	last    float64                 // most recent probability, held between windows
+	pending []float32              // samples not yet part of a full window
+	context [sileroContext]float32 // tail of the previous window
+	state   []float32              // [2][1][128], carried between windows
+	last    float64                // most recent probability, held between windows
 	sr      []int64
 }
 

docs/evals/2026-08-09-kiwix-topic-retrieval.md

@@ -0,0 +1,103 @@
+# Kiwix answered the wrong question, and the fix was not a relevance gate
+
+Date: 2026-08-09. Task: V-668. Box: homesrv, workstation off.
+Book: `wikipedia_ru_all_maxi_2026-02` on `127.0.0.1:8034`.
+
+## What started it
+
+Two turns on 2026-08-09 came back wrong from the offline encyclopedia.
+"почему небо голубое" was answered off the song "Город золотой". "что такое
+TCP?" was answered off "Перехват TCP-соединения". Both were phrased
+confidently, because `queryKiwix` claims a turn whenever the search returns
+anything and `len(hits) == 0` is its only gate.
+
+The plan was a relevance gate. multilingual-e5-small is asymmetric and trained
+for exactly this, `query:` against `passage:`, and the query vector is already
+held on the turn. The 2026-08-05 measurement that killed a search-quality gate
+killed three lexical signals. It says in its own words that it never probed
+Kiwix.
+
+## The gate does not exist
+
+Fourteen Russian questions, eight the encyclopedia can answer and six it
+cannot. Each question was searched, the top article read, and the cosine of
+`EmbedQuery(question)` against `EmbedPassage(article)` recorded.
+
+| set | n | min | mean | max |
+|---|---|---|---|---|
+| answerable | 8 | 0.7934 | 0.8400 | 0.9087 |
+| not answerable | 6 | 0.7480 | 0.7852 | 0.8367 |
+
+Two of the six unanswerable score above the weakest answerable one. That alone
+would be a poor threshold. The log killed it outright: seven of the eight
+answerable questions got a **wrong** article back, and those wrong articles
+scored high. The TCP hijacking article scored 0.8653, above five of the six
+unanswerable rows.
+
+The finding is that this cosine measures topic and not answerhood. A page about
+hijacking TCP sessions is about TCP. No threshold separates it from a page that
+defines TCP, and one that tried would take the definition with it.
+
+## The defect is retrieval
+
+`internal/kiwix/client.go` has said it since it was written: ranking is keyword
+based, "why is the sky blue" finds a TV episode. `queryKiwix` sends the whole
+sentence. The English path has a rewriter that reduces a question to keywords
+with a model call. The Russian path reads the book verbatim (V-508) and had
+nothing. So the question words compete with the one word that names the article.
+
+Dropping the question words changes the answer:
+
+| sent | first hit |
+|---|---|
+| `кто написал Войну и мир` | Радуйся, мир (Доктор Кто) |
+| `Война и мир` | Война и мир |
+| `что такое TCP` | Перехват TCP-соединения |
+| `TCP` | TCP |
+
+A ZIM is also addressable by title, which nothing here used. `/A/Франция`,
+`/A/TCP` and `/A/Небо` are 200. `/A/Трюмбальная_нидроскопия` is 404. So an
+exact title is safe to try first: it either answers or costs one request that
+says nothing.
+
+## What shipped, measured
+
+`kiwix.Topic` drops the narrative request, the interrogative and a verb sitting
+behind one. It keeps everything else, because a word it cannot classify is more
+likely the topic than noise. `kiwix.TitlePath` tries the exact article before
+any ranking runs. Both apply on the verbatim path only, since reducing twice
+would take the topic off the rewriter's input.
+
+| question | before | after |
+|---|---|---|
+| что такое TCP? | Перехват TCP-соединения | **TCP** (by title) |
+| что такое фотосинтез | C4-фотосинтез | **Фотосинтез** |
+| кто такой Линус Торвальдс? | Tux | **Торвальдс, Линус** (by title) |
+| кто написал Войну и мир | Радуйся, мир (Доктор Кто) | **Война и мир** |
+| что такое чёрная дыра | Чёрная дыра | Чёрная дыра |
+| почему небо голубое | Город золотой | Под небом голубым… (фильм) |
+| почему трава зелёная | Сено | Зелень |
+| столица Франции | Список столиц Олимпийских игр | Список столиц Олимпийских игр |
+
+Four questions reach the right article where they did not. Two were already
+right and stay right. Nothing regressed.
+
+## What is still wrong
+
+Two of the eight are still not answered, and both are the same shape. The
+question names no article. "почему небо голубое" is answered by Rayleigh
+scattering and "столица Франции" by the lead of Франция. Neither title is in
+the question. Keyword retrieval cannot bridge that and neither can a
+threshold. The candidates are a semantic index over titles, or asking the
+resident model for the article title rather than for keywords.
+
+`Response.Empty()` is still the whole gate. A wrong article that the search
+does return is still spoken. What this change buys is that the article is
+usually right, not that a wrong one is caught.
+
+## Not measured here
+
+The English path, which still goes through the rewriter and was not touched.
+SearXNG, where the same question about answerhood is open and the 2026-08-05
+result stands. The cascade end to end, since the workstation is off and the
+phrasing arm is the resident model.

internal/kiwix/client.go

@@ -127,6 +127,22 @@ func (c *Client) Article(ctx context.Context, path string, maxRunes int) (crawl.
 	return crawl.Extract(u, body, maxRunes), nil
 }
 
+// TitlePath is the article path for an exact title, for Article to fetch.
+//
+// It exists because a ZIM is addressable by title and the full-text index is
+// not the only way in. "Франция", "TCP" and "Небо" resolve; "Трюмбальная
+// нидроскопия" is a 404, which is the honest answer and the reason this is
+// safe to try first. Measured on 2026-08-09, keyword search on the same terms
+// returns "Список пэров Франции" and "Список портов TCP и UDP" instead.
+//
+// A miss is normal rather than a failure. An article whose title inverts a name
+// ("Торвальдс, Линус") is a 404 here and the first hit in search, so the caller
+// falls through and loses nothing.
+func TitlePath(book, title string) string {
+	t := strings.ReplaceAll(strings.TrimSpace(title), " ", "_")
+	return "/content/" + url.PathEscape(book) + "/A/" + url.PathEscape(t)
+}
+
 // rss mirrors just the bits of the RSS 2.0 reply we use.
 type rss struct {
 	Items []struct {

internal/kiwix/live_test.go

@@ -0,0 +1,65 @@
+package kiwix
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+)
+
+// TestLiveTopicBeatsTheSentence — the measurement V-668 turned on, kept as a
+// test so the claim can be re-run rather than believed.
+//
+// It prints the article the old path returned and the article the new one
+// returns, for the same question. It asserts nothing about which is better,
+// because "is this the right article" is a human's call. It fails only if the
+// two paths agree on every case, which would mean the change does nothing.
+//
+//	MAVEN_KIWIX_URL=http://127.0.0.1:8034 make t PKG=./internal/kiwix/ RUN=TestLive V=1
+func TestLiveTopicBeatsTheSentence(t *testing.T) {
+	base := os.Getenv("MAVEN_KIWIX_URL")
+	if base == "" {
+		t.Skip("MAVEN_KIWIX_URL unset — point it at the kiwix-server host port")
+	}
+	const book = "wikipedia_ru_all_maxi_2026-02"
+	c := New(base)
+	questions := []string{
+		"что такое TCP?",
+		"что такое фотосинтез",
+		"кто такой Линус Торвальдс?",
+		"кто написал Войну и мир",
+		"что такое чёрная дыра",
+		"почему небо голубое",
+		"почему трава зелёная",
+		"столица Франции",
+	}
+	moved := 0
+	for _, q := range questions {
+		ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+		before := firstTitle(ctx, c, q, book)
+		topic := Topic(q)
+		after := ""
+		if page, err := c.Article(ctx, TitlePath(book, topic), 400); err == nil && page.Text != "" {
+			after = page.Title + " (by title)"
+		} else {
+			after = firstTitle(ctx, c, topic, book)
+		}
+		cancel()
+		if before != after {
+			moved++
+		}
+		t.Logf("%-30s  before=%-34q after=%q", q, before, after)
+	}
+	t.Logf("%d of %d questions reach a different article", moved, len(questions))
+	if moved == 0 {
+		t.Error("the topic path returns exactly what the sentence path returned")
+	}
+}
+
+func firstTitle(ctx context.Context, c *Client, pattern, book string) string {
+	hits, err := c.Search(ctx, pattern, book, 3)
+	if err != nil || len(hits) == 0 {
+		return "(nothing)"
+	}
+	return hits[0].Title
+}

internal/kiwix/topic.go

@@ -0,0 +1,98 @@
+package kiwix
+
+import (
+	"strings"
+	"unicode"
+
+	"github.com/kami/maven/internal/lexicon"
+	"github.com/kami/maven/internal/morph"
+)
+
+// Topic reduces a question to the thing it is about, because Kiwix ranks by
+// keyword overlap and a whole sentence buries the keyword that matters.
+//
+// This package's own doc says it: "why is the sky blue" finds a TV episode.
+// Measured against the Russian ZIM on 2026-08-09, the sentence and the topic
+// return different articles for the same question. "кто написал Войну и мир"
+// returns "Радуйся, мир (Доктор Кто)"; "Войну и мир" returns the novel first.
+// "что такое TCP" returns "Перехват TCP-соединения"; "TCP" returns TCP. The
+// English path had a rewriter doing this with a model call. The Russian path
+// reads the book verbatim (V-508) and had nothing.
+//
+// It drops three things off the front and stops: the narrative request, the
+// interrogative, and a verb sitting between them and the noun. Everything else
+// is kept, because a word this cannot classify is more likely the topic than
+// noise. An empty return means the utterance was question words alone, and the
+// caller searches the sentence as before.
+func Topic(utterance string) string {
+	words := strings.Fields(strings.TrimSpace(utterance))
+	cut := 0
+	for cut < len(words) {
+		w := strings.Trim(strings.ToLower(words[cut]), ".,!?…:;\"'«»")
+		if w == "" {
+			cut++
+			continue
+		}
+		switch {
+		case inList(lexicon.NarrativeRequests(), w),
+			inList(lexicon.Interrogatives(), w),
+			inList(lexicon.FirstPerson(), w),
+			// "что ТАКОЕ x", "кто ТАКОЙ x" — the copula that only ever follows
+			// an interrogative, and never a topic on its own.
+			cut > 0 && isCopula(w),
+			// "расскажи ПРО x", "о x". One-letter and two-letter prepositions
+			// are not a closed class worth a lexicon set of their own.
+			cut > 0 && isLeadingPreposition(w),
+			// "кто НАПИСАЛ Войну и мир". A verb here is the question's own
+			// verb, not part of the title. Only after something was already
+			// dropped, so "написал отчёт" as a topic survives intact.
+			cut > 0 && morph.IsVerbForm(w):
+			cut++
+		default:
+			// The question mark is the sentence's, not the title's, and Kiwix
+			// carries it into the keyword match.
+			topic := strings.TrimRight(strings.Join(words[cut:], " "), " .,!?…:;\"'«»")
+			if !hasLetter(topic) {
+				return ""
+			}
+			return topic
+		}
+	}
+	return ""
+}
+
+func isCopula(w string) bool {
+	switch w {
+	case "такое", "такой", "такая", "такие", "is", "are", "was", "were":
+		return true
+	}
+	return false
+}
+
+func isLeadingPreposition(w string) bool {
+	switch w {
+	case "про", "о", "об", "обо", "по", "about", "of", "on":
+		return true
+	}
+	return false
+}
+
+func inList(list []string, w string) bool {
+	for _, x := range list {
+		if x == w {
+			return true
+		}
+	}
+	return false
+}
+
+// hasLetter is the guard against a topic that reduced to punctuation or digits
+// alone, which no ZIM title matches.
+func hasLetter(s string) bool {
+	for _, r := range s {
+		if unicode.IsLetter(r) {
+			return true
+		}
+	}
+	return false
+}

internal/kiwix/topic_test.go

@@ -0,0 +1,51 @@
+package kiwix
+
+import "testing"
+
+// The cases the 2026-08-09 measurement turned on, plus the ones a topic must
+// not damage. Each left column returned a wrong article when it was sent whole.
+func TestTopicKeepsTheThingTheQuestionIsAbout(t *testing.T) {
+	cases := []struct{ utterance, want string }{
+		{"что такое TCP?", "TCP"},
+		{"что такое фотосинтез", "фотосинтез"},
+		{"кто такой Линус Торвальдс?", "Линус Торвальдс"},
+		{"кто написал Войну и мир", "Войну и мир"},
+		{"расскажи про битву при Ватерлоо", "битву при Ватерлоо"},
+		{"what is photosynthesis", "photosynthesis"},
+		// No question word, so there is nothing to drop. The topic is the
+		// whole utterance and the search is what it was before.
+		{"столица Франции", "столица Франции"},
+		{"почему небо голубое", "небо голубое"},
+	}
+	for _, c := range cases {
+		if got := Topic(c.utterance); got != c.want {
+			t.Errorf("Topic(%q) = %q, want %q", c.utterance, got, c.want)
+		}
+	}
+}
+
+// A verb only goes when a question word already went. Otherwise "написал
+// отчёт" loses the verb that names what he means.
+func TestTopicDropsAVerbOnlyBehindAQuestionWord(t *testing.T) {
+	if got := Topic("написал отчёт"); got != "написал отчёт" {
+		t.Errorf("Topic dropped a leading verb with no question word: %q", got)
+	}
+}
+
+// Question words alone reduce to nothing, and the caller reads that as "no
+// topic" and searches the sentence rather than searching the empty string.
+func TestTopicIsEmptyWhenNothingIsLeft(t *testing.T) {
+	for _, q := range []string{"что такое?", "кто?", "почему", "???"} {
+		if got := Topic(q); got != "" {
+			t.Errorf("Topic(%q) = %q, want empty", q, got)
+		}
+	}
+}
+
+func TestTitlePathEscapesAndUnderscores(t *testing.T) {
+	got := TitlePath("wikipedia_ru_all_maxi_2026-02", "Чёрная дыра")
+	want := "/content/wikipedia_ru_all_maxi_2026-02/A/%D0%A7%D1%91%D1%80%D0%BD%D0%B0%D1%8F_%D0%B4%D1%8B%D1%80%D0%B0"
+	if got != want {
+		t.Errorf("TitlePath = %q, want %q", got, want)
+	}
+}