diff --git a/CLAUDE.md b/CLAUDE.md index e4806f9..502639b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -675,11 +675,25 @@ world questions, so she needs to read external sources. What replaces it: `wikipedia_ru_all_maxi_2026-02` verbatim** through `kiwix.book_ru`. The RU→EN rewriter is the workaround for an English book and is skipped there. Kiwix catalog names come from the filename, not the `` field. + **That verbatim path sent the whole sentence to a keyword engine until 09-08-2026** + (V-668, `docs/evals/2026-08-09-kiwix-topic-retrieval.md`). Kiwix ranks by keyword + overlap, so the question words outrank the one word naming the article. "что такое TCP" + returned "Перехват TCP-соединения". "кто написал Войну и мир" returned an episode of + Doctor Who. `kiwix.Topic` drops the narrative request, the interrogative and a verb + behind one. `kiwix.TitlePath` tries the exact article first, since a ZIM is addressable + by title and a wrong title is a 404. Four of eight questions reach the right article + where they did not, two were already right, and nothing regressed. Both apply on the + verbatim path alone. The rewriter already reduces a question, and reducing twice takes + the topic off its input. `Response.Empty()` is the whole gate and there is no quality threshold in front of it: the three signals one could read were measured on 2026-08-05 and none of them separate a real question from an invented one. Token overlap would cost "столица Франции" its answer, because the answer is Париж and that word is not in the question. See - `docs/evals/2026-08-05-search-quality-signals.md` (V-539). **Which query source claimed + `docs/evals/2026-08-05-search-quality-signals.md` (V-539). **The embedder is not a + fourth signal**, measured 2026-08-09 (V-668). Query-to-passage cosine scores 0.79 to + 0.91 on answerable questions and 0.75 to 0.84 on unanswerable ones, and the sets + overlap. The wrong TCP article scored 0.8653, above five of six unanswerable rows. It + measures topic and not whether the passage answers, so no threshold splits them. **Which query source claimed a turn is readable on `/chat`** as a badge beside the reply, carried on `ipc.ChatReply.Source` and noted by `noteQuerySource` in `cmd/mavend/querysource.go`. It rides the context, so `handleText` keeps the one string signature the mic, telegram and diff --git a/cmd/mavend/actions_query.go b/cmd/mavend/actions_query.go index a945805..4a79406 100644 --- a/cmd/mavend/actions_query.go +++ b/cmd/mavend/actions_query.go @@ -13,6 +13,7 @@ import ( "github.com/kami/maven/internal/crawl" "github.com/kami/maven/internal/decision" "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/kiwix" "github.com/kami/maven/internal/memory" "github.com/kami/maven/internal/morning" "github.com/kami/maven/internal/phraser" @@ -926,6 +927,26 @@ func (h *reactiveHandler) queryKiwix(ctx context.Context, t *queryTurn) (string, } } + // The topic, not the sentence (V-668). Kiwix ranks by keyword overlap, so + // the question words outrank the one word that names the article: measured + // on 2026-08-09, "что такое TCP" returns "Перехват TCP-соединения" and + // "TCP" returns TCP. Only the verbatim path needs this. The rewriter + // already reduces a question to English keywords, and reducing twice would + // take the topic off the input it reads. + if verbatim { + if topic := kiwix.Topic(pattern); topic != "" { + // The article named exactly, before any ranking runs. A ZIM is + // addressable by title and a wrong title is a 404, so this either + // answers or costs one request that says nothing. + page, err := h.kiwix.client.Article(ctxK, kiwix.TitlePath(book, topic), h.kiwix.runes) + if err == nil && page.Text != "" { + log.Printf("voice: kiwix: %q in %q → title hit %q", topic, book, page.Title) + return h.kiwixReply(ctx, t, page.Title, page.Text) + } + pattern = topic + } + } + hits, err := h.kiwix.client.Search(ctxK, pattern, book, h.kiwix.max) if err != nil { log.Printf("voice: kiwix: search %q: %v", pattern, err) @@ -958,14 +979,18 @@ func (h *reactiveHandler) queryKiwix(ctx context.Context, t *queryTurn) (string, } page = crawl.Page{Title: top.Title, Text: top.Snippet} } - // Handed over the same way a note or a page is: context for the question he - // asked, not something to recite. - snippet := top.Title + "\n" + crawl.TrimRunes(page.Text, h.kiwix.runes) + return h.kiwixReply(ctx, t, top.Title, page.Text) +} + +// kiwixReply hands one article over the same way a note or a page is handed +// over: context for the question he asked, not something to recite. +func (h *reactiveHandler) kiwixReply(ctx context.Context, t *queryTurn, title, text string) (string, bool) { + snippet := title + "\n" + crawl.TrimRunes(text, h.kiwix.runes) reply := h.phraseSource(ctx, "kiwix", t.dec.Utterance, []string{snippet}) if reply == "" { // No phraser, or it failed. Read back the best hit rather than pretend // the search did not happen. - return readBack(top.Title + " — " + page.Text), true + return readBack(title + " — " + text), true } return reply, true } diff --git a/cmd/mavwaked/silero.go b/cmd/mavwaked/silero.go index 6c015fb..ed8fa0c 100644 --- a/cmd/mavwaked/silero.go +++ b/cmd/mavwaked/silero.go @@ -52,10 +52,10 @@ type sileroVAD struct { mu sync.Mutex session *ort.DynamicAdvancedSession - pending []float32 // samples not yet part of a full window - context [sileroContext]float32 // tail of the previous window - state []float32 // [2][1][128], carried between windows - last float64 // most recent probability, held between windows + pending []float32 // samples not yet part of a full window + context [sileroContext]float32 // tail of the previous window + state []float32 // [2][1][128], carried between windows + last float64 // most recent probability, held between windows sr []int64 } diff --git a/docs/evals/2026-08-09-kiwix-topic-retrieval.md b/docs/evals/2026-08-09-kiwix-topic-retrieval.md new file mode 100644 index 0000000..115605d --- /dev/null +++ b/docs/evals/2026-08-09-kiwix-topic-retrieval.md @@ -0,0 +1,103 @@ +# Kiwix answered the wrong question, and the fix was not a relevance gate + +Date: 2026-08-09. Task: V-668. Box: homesrv, workstation off. +Book: `wikipedia_ru_all_maxi_2026-02` on `127.0.0.1:8034`. + +## What started it + +Two turns on 2026-08-09 came back wrong from the offline encyclopedia. +"почему небо голубое" was answered off the song "Город золотой". "что такое +TCP?" was answered off "Перехват TCP-соединения". Both were phrased +confidently, because `queryKiwix` claims a turn whenever the search returns +anything and `len(hits) == 0` is its only gate. + +The plan was a relevance gate. multilingual-e5-small is asymmetric and trained +for exactly this, `query:` against `passage:`, and the query vector is already +held on the turn. The 2026-08-05 measurement that killed a search-quality gate +killed three lexical signals. It says in its own words that it never probed +Kiwix. + +## The gate does not exist + +Fourteen Russian questions, eight the encyclopedia can answer and six it +cannot. Each question was searched, the top article read, and the cosine of +`EmbedQuery(question)` against `EmbedPassage(article)` recorded. + +| set | n | min | mean | max | +|---|---|---|---|---| +| answerable | 8 | 0.7934 | 0.8400 | 0.9087 | +| not answerable | 6 | 0.7480 | 0.7852 | 0.8367 | + +Two of the six unanswerable score above the weakest answerable one. That alone +would be a poor threshold. The log killed it outright: seven of the eight +answerable questions got a **wrong** article back, and those wrong articles +scored high. The TCP hijacking article scored 0.8653, above five of the six +unanswerable rows. + +The finding is that this cosine measures topic and not answerhood. A page about +hijacking TCP sessions is about TCP. No threshold separates it from a page that +defines TCP, and one that tried would take the definition with it. + +## The defect is retrieval + +`internal/kiwix/client.go` has said it since it was written: ranking is keyword +based, "why is the sky blue" finds a TV episode. `queryKiwix` sends the whole +sentence. The English path has a rewriter that reduces a question to keywords +with a model call. The Russian path reads the book verbatim (V-508) and had +nothing. So the question words compete with the one word that names the article. + +Dropping the question words changes the answer: + +| sent | first hit | +|---|---| +| `кто написал Войну и мир` | Радуйся, мир (Доктор Кто) | +| `Война и мир` | Война и мир | +| `что такое TCP` | Перехват TCP-соединения | +| `TCP` | TCP | + +A ZIM is also addressable by title, which nothing here used. `/A/Франция`, +`/A/TCP` and `/A/Небо` are 200. `/A/Трюмбальная_нидроскопия` is 404. So an +exact title is safe to try first: it either answers or costs one request that +says nothing. + +## What shipped, measured + +`kiwix.Topic` drops the narrative request, the interrogative and a verb sitting +behind one. It keeps everything else, because a word it cannot classify is more +likely the topic than noise. `kiwix.TitlePath` tries the exact article before +any ranking runs. Both apply on the verbatim path only, since reducing twice +would take the topic off the rewriter's input. + +| question | before | after | +|---|---|---| +| что такое TCP? | Перехват TCP-соединения | **TCP** (by title) | +| что такое фотосинтез | C4-фотосинтез | **Фотосинтез** | +| кто такой Линус Торвальдс? | Tux | **Торвальдс, Линус** (by title) | +| кто написал Войну и мир | Радуйся, мир (Доктор Кто) | **Война и мир** | +| что такое чёрная дыра | Чёрная дыра | Чёрная дыра | +| почему небо голубое | Город золотой | Под небом голубым… (фильм) | +| почему трава зелёная | Сено | Зелень | +| столица Франции | Список столиц Олимпийских игр | Список столиц Олимпийских игр | + +Four questions reach the right article where they did not. Two were already +right and stay right. Nothing regressed. + +## What is still wrong + +Two of the eight are still not answered, and both are the same shape. The +question names no article. "почему небо голубое" is answered by Rayleigh +scattering and "столица Франции" by the lead of Франция. Neither title is in +the question. Keyword retrieval cannot bridge that and neither can a +threshold. The candidates are a semantic index over titles, or asking the +resident model for the article title rather than for keywords. + +`Response.Empty()` is still the whole gate. A wrong article that the search +does return is still spoken. What this change buys is that the article is +usually right, not that a wrong one is caught. + +## Not measured here + +The English path, which still goes through the rewriter and was not touched. +SearXNG, where the same question about answerhood is open and the 2026-08-05 +result stands. The cascade end to end, since the workstation is off and the +phrasing arm is the resident model. diff --git a/internal/kiwix/client.go b/internal/kiwix/client.go index 52a218d..f057046 100644 --- a/internal/kiwix/client.go +++ b/internal/kiwix/client.go @@ -127,6 +127,22 @@ func (c *Client) Article(ctx context.Context, path string, maxRunes int) (crawl. return crawl.Extract(u, body, maxRunes), nil } +// TitlePath is the article path for an exact title, for Article to fetch. +// +// It exists because a ZIM is addressable by title and the full-text index is +// not the only way in. "Франция", "TCP" and "Небо" resolve; "Трюмбальная +// нидроскопия" is a 404, which is the honest answer and the reason this is +// safe to try first. Measured on 2026-08-09, keyword search on the same terms +// returns "Список пэров Франции" and "Список портов TCP и UDP" instead. +// +// A miss is normal rather than a failure. An article whose title inverts a name +// ("Торвальдс, Линус") is a 404 here and the first hit in search, so the caller +// falls through and loses nothing. +func TitlePath(book, title string) string { + t := strings.ReplaceAll(strings.TrimSpace(title), " ", "_") + return "/content/" + url.PathEscape(book) + "/A/" + url.PathEscape(t) +} + // rss mirrors just the bits of the RSS 2.0 reply we use. type rss struct { Items []struct { diff --git a/internal/kiwix/live_test.go b/internal/kiwix/live_test.go new file mode 100644 index 0000000..95343de --- /dev/null +++ b/internal/kiwix/live_test.go @@ -0,0 +1,65 @@ +package kiwix + +import ( + "context" + "os" + "testing" + "time" +) + +// TestLiveTopicBeatsTheSentence — the measurement V-668 turned on, kept as a +// test so the claim can be re-run rather than believed. +// +// It prints the article the old path returned and the article the new one +// returns, for the same question. It asserts nothing about which is better, +// because "is this the right article" is a human's call. It fails only if the +// two paths agree on every case, which would mean the change does nothing. +// +// MAVEN_KIWIX_URL=http://127.0.0.1:8034 make t PKG=./internal/kiwix/ RUN=TestLive V=1 +func TestLiveTopicBeatsTheSentence(t *testing.T) { + base := os.Getenv("MAVEN_KIWIX_URL") + if base == "" { + t.Skip("MAVEN_KIWIX_URL unset — point it at the kiwix-server host port") + } + const book = "wikipedia_ru_all_maxi_2026-02" + c := New(base) + questions := []string{ + "что такое TCP?", + "что такое фотосинтез", + "кто такой Линус Торвальдс?", + "кто написал Войну и мир", + "что такое чёрная дыра", + "почему небо голубое", + "почему трава зелёная", + "столица Франции", + } + moved := 0 + for _, q := range questions { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + before := firstTitle(ctx, c, q, book) + topic := Topic(q) + after := "" + if page, err := c.Article(ctx, TitlePath(book, topic), 400); err == nil && page.Text != "" { + after = page.Title + " (by title)" + } else { + after = firstTitle(ctx, c, topic, book) + } + cancel() + if before != after { + moved++ + } + t.Logf("%-30s before=%-34q after=%q", q, before, after) + } + t.Logf("%d of %d questions reach a different article", moved, len(questions)) + if moved == 0 { + t.Error("the topic path returns exactly what the sentence path returned") + } +} + +func firstTitle(ctx context.Context, c *Client, pattern, book string) string { + hits, err := c.Search(ctx, pattern, book, 3) + if err != nil || len(hits) == 0 { + return "(nothing)" + } + return hits[0].Title +} diff --git a/internal/kiwix/topic.go b/internal/kiwix/topic.go new file mode 100644 index 0000000..2237070 --- /dev/null +++ b/internal/kiwix/topic.go @@ -0,0 +1,98 @@ +package kiwix + +import ( + "strings" + "unicode" + + "github.com/kami/maven/internal/lexicon" + "github.com/kami/maven/internal/morph" +) + +// Topic reduces a question to the thing it is about, because Kiwix ranks by +// keyword overlap and a whole sentence buries the keyword that matters. +// +// This package's own doc says it: "why is the sky blue" finds a TV episode. +// Measured against the Russian ZIM on 2026-08-09, the sentence and the topic +// return different articles for the same question. "кто написал Войну и мир" +// returns "Радуйся, мир (Доктор Кто)"; "Войну и мир" returns the novel first. +// "что такое TCP" returns "Перехват TCP-соединения"; "TCP" returns TCP. The +// English path had a rewriter doing this with a model call. The Russian path +// reads the book verbatim (V-508) and had nothing. +// +// It drops three things off the front and stops: the narrative request, the +// interrogative, and a verb sitting between them and the noun. Everything else +// is kept, because a word this cannot classify is more likely the topic than +// noise. An empty return means the utterance was question words alone, and the +// caller searches the sentence as before. +func Topic(utterance string) string { + words := strings.Fields(strings.TrimSpace(utterance)) + cut := 0 + for cut < len(words) { + w := strings.Trim(strings.ToLower(words[cut]), ".,!?…:;\"'«»") + if w == "" { + cut++ + continue + } + switch { + case inList(lexicon.NarrativeRequests(), w), + inList(lexicon.Interrogatives(), w), + inList(lexicon.FirstPerson(), w), + // "что ТАКОЕ x", "кто ТАКОЙ x" — the copula that only ever follows + // an interrogative, and never a topic on its own. + cut > 0 && isCopula(w), + // "расскажи ПРО x", "о x". One-letter and two-letter prepositions + // are not a closed class worth a lexicon set of their own. + cut > 0 && isLeadingPreposition(w), + // "кто НАПИСАЛ Войну и мир". A verb here is the question's own + // verb, not part of the title. Only after something was already + // dropped, so "написал отчёт" as a topic survives intact. + cut > 0 && morph.IsVerbForm(w): + cut++ + default: + // The question mark is the sentence's, not the title's, and Kiwix + // carries it into the keyword match. + topic := strings.TrimRight(strings.Join(words[cut:], " "), " .,!?…:;\"'«»") + if !hasLetter(topic) { + return "" + } + return topic + } + } + return "" +} + +func isCopula(w string) bool { + switch w { + case "такое", "такой", "такая", "такие", "is", "are", "was", "were": + return true + } + return false +} + +func isLeadingPreposition(w string) bool { + switch w { + case "про", "о", "об", "обо", "по", "about", "of", "on": + return true + } + return false +} + +func inList(list []string, w string) bool { + for _, x := range list { + if x == w { + return true + } + } + return false +} + +// hasLetter is the guard against a topic that reduced to punctuation or digits +// alone, which no ZIM title matches. +func hasLetter(s string) bool { + for _, r := range s { + if unicode.IsLetter(r) { + return true + } + } + return false +} diff --git a/internal/kiwix/topic_test.go b/internal/kiwix/topic_test.go new file mode 100644 index 0000000..bc01683 --- /dev/null +++ b/internal/kiwix/topic_test.go @@ -0,0 +1,51 @@ +package kiwix + +import "testing" + +// The cases the 2026-08-09 measurement turned on, plus the ones a topic must +// not damage. Each left column returned a wrong article when it was sent whole. +func TestTopicKeepsTheThingTheQuestionIsAbout(t *testing.T) { + cases := []struct{ utterance, want string }{ + {"что такое TCP?", "TCP"}, + {"что такое фотосинтез", "фотосинтез"}, + {"кто такой Линус Торвальдс?", "Линус Торвальдс"}, + {"кто написал Войну и мир", "Войну и мир"}, + {"расскажи про битву при Ватерлоо", "битву при Ватерлоо"}, + {"what is photosynthesis", "photosynthesis"}, + // No question word, so there is nothing to drop. The topic is the + // whole utterance and the search is what it was before. + {"столица Франции", "столица Франции"}, + {"почему небо голубое", "небо голубое"}, + } + for _, c := range cases { + if got := Topic(c.utterance); got != c.want { + t.Errorf("Topic(%q) = %q, want %q", c.utterance, got, c.want) + } + } +} + +// A verb only goes when a question word already went. Otherwise "написал +// отчёт" loses the verb that names what he means. +func TestTopicDropsAVerbOnlyBehindAQuestionWord(t *testing.T) { + if got := Topic("написал отчёт"); got != "написал отчёт" { + t.Errorf("Topic dropped a leading verb with no question word: %q", got) + } +} + +// Question words alone reduce to nothing, and the caller reads that as "no +// topic" and searches the sentence rather than searching the empty string. +func TestTopicIsEmptyWhenNothingIsLeft(t *testing.T) { + for _, q := range []string{"что такое?", "кто?", "почему", "???"} { + if got := Topic(q); got != "" { + t.Errorf("Topic(%q) = %q, want empty", q, got) + } + } +} + +func TestTitlePathEscapesAndUnderscores(t *testing.T) { + got := TitlePath("wikipedia_ru_all_maxi_2026-02", "Чёрная дыра") + want := "/content/wikipedia_ru_all_maxi_2026-02/A/%D0%A7%D1%91%D1%80%D0%BD%D0%B0%D1%8F_%D0%B4%D1%8B%D1%80%D0%B0" + if got != want { + t.Errorf("TitlePath = %q, want %q", got, want) + } +}