From 1f38e71d1af44c8a8d06c343dbe2f3e93026d2bd Mon Sep 17 00:00:00 2001 From: claude Date: Wed, 5 Aug 2026 15:52:54 +0400 Subject: [PATCH] the ZIM fallback fires fast, and reads Russian in Russian (V-508) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verification, as the task asked. Drove что такое фотосинтез through /api/chat with the search reachable, with the container stopped, and with the host blackholed. Kiwix claims the turn in both failure cases, and a stopped container costs nothing: DNS fails and the ZIM answers inside the same second. The blackhole is the case that hurts. The search waited its full 8-second budget before the ZIM was asked and the turn took 15.4s against 3.5, which he sits through with nothing being said. So the connect phase alone is now capped at 1.5s. A reachable instance that is merely slow keeps the whole budget, because it is fanning out to real engines. The RU Wikipedia ZIM is on the box (owner moved it into the kiwix zims dir), and kiwix-serve picked it up. A Cyrillic question now searches book_ru verbatim and skips the RU->EN rewrite: that rewriter is the workaround for an English book, and against a Russian one it is a translation of his own words back at him. Catalog names come from the filename, not the field — books.name=wikipedia_ru_all returns nothing. Measurement in docs/evals/2026-08-05-kiwix-offline-fallback.md. The RU book answering a driven turn needs a rebuild and is not verified yet. --- CLAUDE.md | 8 +++ cmd/mavend/actions_query.go | 33 ++++++++-- cmd/mavend/kiwixbook_test.go | 44 +++++++++++++ cmd/mavend/kiwixwire.go | 12 +++- deploy/mavend.json | 1 + .../2026-08-05-kiwix-offline-fallback.md | 65 +++++++++++++++++++ internal/config/config.go | 11 ++++ internal/websearch/searxng.go | 32 ++++++++- internal/websearch/searxng_test.go | 37 +++++++++++ 9 files changed, 234 insertions(+), 9 deletions(-) create mode 100644 cmd/mavend/kiwixbook_test.go create mode 100644 docs/evals/2026-08-05-kiwix-offline-fallback.md diff --git a/CLAUDE.md b/CLAUDE.md index 90141a9..5bec859 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -286,6 +286,14 @@ world questions, so she needs to read external sources. What replaces it: - **In the world, live search leads and the ZIMs are the fallback** (owner's call, 2026-08-02). A self-hosted SearXNG (`search` block) answers first; the Kiwix ZIMs on homesrv answer when the search is empty, unreachable, or the line is down. + **Verified with the line down on 2026-08-05** (V-508, + `docs/evals/2026-08-05-kiwix-offline-fallback.md`): a stopped SearXNG costs nothing, + the ZIM answers in the same turn budget. A blackholed host cost 8 seconds he waited + through. So the connect phase alone is capped at `dialTimeout` (1.5s), while a slow + instance that did connect keeps the full 8. **A Russian question reads + `wikipedia_ru_all_maxi_2026-02` verbatim** through `kiwix.book_ru`. The RU→EN rewriter + is the workaround for an English book and is skipped there. Kiwix catalog names come + from the filename, not the `` field. `Response.Empty()` is the whole gate and there is no quality threshold in front of it: the three signals one could read were measured on 2026-08-05 and none of them separate a real question from an invented one. Token overlap would cost "столица Франции" its diff --git a/cmd/mavend/actions_query.go b/cmd/mavend/actions_query.go index d0a35fb..03fdda9 100644 --- a/cmd/mavend/actions_query.go +++ b/cmd/mavend/actions_query.go @@ -8,6 +8,7 @@ import ( "regexp" "strings" "time" + "unicode" "github.com/kami/maven/internal/crawl" "github.com/kami/maven/internal/ipc" @@ -695,11 +696,19 @@ func (h *reactiveHandler) queryKiwix(ctx context.Context, t *queryTurn) (string, ctxK, cancel := context.WithTimeout(ctx, kiwixTimeout) defer cancel() - // The ZIMs are English and kiwix ranks by keyword overlap, not meaning, so - // a Russian sentence matches nothing at all. The rewriter turns it into a - // handful of English keywords with the resident model. + // A Russian question reads the Russian ZIM verbatim when there is one + // (V-508). Kiwix ranks by keyword overlap rather than meaning, so an English + // book matches a Russian sentence not at all, and the rewriter exists to + // turn the question into English keywords with the resident model. Against a + // Russian book that is a translation of his own words back at him: it costs + // a model call and drops whatever the keywords do not carry. + book, verbatim := h.kiwix.book, false + if h.kiwix.bookRU != "" && hasCyrillic(t.dec.Utterance) { + book, verbatim = h.kiwix.bookRU, true + } + pattern := t.dec.Utterance - if h.kiwix.rewriter != nil { + if h.kiwix.rewriter != nil && !verbatim { q, err := h.kiwix.rewriter.Rewrite(ctxK, t.dec.Utterance) if err != nil { // Fall through to the verbatim question rather than give up. It @@ -710,7 +719,7 @@ func (h *reactiveHandler) queryKiwix(ctx context.Context, t *queryTurn) (string, } } - hits, err := h.kiwix.client.Search(ctxK, pattern, h.kiwix.book, h.kiwix.max) + hits, err := h.kiwix.client.Search(ctxK, pattern, book, h.kiwix.max) if err != nil { log.Printf("voice: kiwix: search %q: %v", pattern, err) return "", false @@ -722,7 +731,7 @@ func (h *reactiveHandler) queryKiwix(ctx context.Context, t *queryTurn) (string, // Logged on the way through, not only on failure. Without this there is no // way to tell from the outside whether an answer came off a ZIM or out of // the model's weights, and those are the two cases worth telling apart. - log.Printf("voice: kiwix: %q → %d hits, top %q", pattern, len(hits), top.Title) + log.Printf("voice: kiwix: %q in %q → %d hits, top %q", pattern, book, len(hits), top.Title) // The top hit only, read as an article rather than as a snippet. Kiwix // builds its snippet from wherever the keyword matched, which on Wikipedia @@ -754,6 +763,18 @@ func (h *reactiveHandler) queryKiwix(ctx context.Context, t *queryTurn) (string, return reply, true } +// hasCyrillic reports whether the text carries a Cyrillic letter, which is the +// whole test for "he asked this in Russian". A question mixing a Latin proper +// noun into a Russian sentence is still Russian, so one letter is enough. +func hasCyrillic(s string) bool { + for _, r := range s { + if unicode.Is(unicode.Cyrillic, r) { + return true + } + } + return false +} + // queryPersonal — stop the walk on a question about him that his own data did // not answer. // diff --git a/cmd/mavend/kiwixbook_test.go b/cmd/mavend/kiwixbook_test.go new file mode 100644 index 0000000..3ec8049 --- /dev/null +++ b/cmd/mavend/kiwixbook_test.go @@ -0,0 +1,44 @@ +package main + +import "testing" + +func TestHasCyrillic(t *testing.T) { + for _, s := range []string{"что такое фотосинтез", "кто такой Elon Musk", "фотосинтез"} { + if !hasCyrillic(s) { + t.Errorf("hasCyrillic(%q) = false; it is a Russian question", s) + } + } + for _, s := range []string{"what is photosynthesis", "", "3:2"} { + if hasCyrillic(s) { + t.Errorf("hasCyrillic(%q) = true; there is no Cyrillic in it", s) + } + } +} + +// The book choice and the rewrite decision are the same decision: a Russian +// book reads his question as he asked it, an English one needs it translated +// into keywords first (V-508). +func TestKiwixBookChoice(t *testing.T) { + for _, tc := range []struct { + name string + wiring kiwixWiring + utterance string + wantBook string + wantVerb bool + }{ + {"a russian question reads the russian book verbatim", + kiwixWiring{book: "en", bookRU: "ru"}, "что такое фотосинтез", "ru", true}, + {"an english question reads the english book", + kiwixWiring{book: "en", bookRU: "ru"}, "what is photosynthesis", "en", false}, + {"no russian book configured leaves every question on the english one", + kiwixWiring{book: "en"}, "что такое фотосинтез", "en", false}, + } { + book, verbatim := tc.wiring.book, false + if tc.wiring.bookRU != "" && hasCyrillic(tc.utterance) { + book, verbatim = tc.wiring.bookRU, true + } + if book != tc.wantBook || verbatim != tc.wantVerb { + t.Errorf("%s: book=%q verbatim=%v, want %q/%v", tc.name, book, verbatim, tc.wantBook, tc.wantVerb) + } + } +} diff --git a/cmd/mavend/kiwixwire.go b/cmd/mavend/kiwixwire.go index 84e3a5d..bcef387 100644 --- a/cmd/mavend/kiwixwire.go +++ b/cmd/mavend/kiwixwire.go @@ -19,8 +19,12 @@ type kiwixWiring struct { client *kiwix.Client rewriter *kiwix.Rewriter // nil ⇒ the question is searched verbatim book string - max int - runes int + // bookRU — searched instead of book when the question is Cyrillic, and + // searched verbatim because it is in his language already (V-508). Empty ⇒ + // every question goes to book. + bookRU string + max int + runes int } // wireKiwix builds the ZIM reader from the `kiwix` block, or returns nil when @@ -38,9 +42,13 @@ func wireKiwix(cfg *config.Config, c *llm.Client) *kiwixWiring { w := &kiwixWiring{ client: kiwix.New(kc.URL), book: kc.Book, + bookRU: kc.BookRU, max: kc.MaxResults, runes: kc.SnippetRunes, } + if kc.BookRU != "" { + log.Printf("voice: kiwix: russian questions read %q verbatim", kc.BookRU) + } switch { case !kc.RewriteEnabled(): log.Printf("voice: kiwix at %s (book %q, query rewriting off by config)", kc.URL, kc.Book) diff --git a/deploy/mavend.json b/deploy/mavend.json index e1d1a9e..ef0785d 100644 --- a/deploy/mavend.json +++ b/deploy/mavend.json @@ -90,6 +90,7 @@ "kiwix": { "url": "http://kiwix-server:8080", "book": "wikipedia_en_all_maxi_2026-02", + "book_ru": "wikipedia_ru_all_maxi_2026-02", "max_results": 5, "snippet_runes": 1500 }, diff --git a/docs/evals/2026-08-05-kiwix-offline-fallback.md b/docs/evals/2026-08-05-kiwix-offline-fallback.md new file mode 100644 index 0000000..7f91d03 --- /dev/null +++ b/docs/evals/2026-08-05-kiwix-offline-fallback.md @@ -0,0 +1,65 @@ +# Does the ZIM answer when the line is down? (V-508) + +Measured 2026-08-05 on the deploy, through `POST /api/chat`. The question was +`что такое фотосинтез` in every run. Which source claimed is read off +`voice: query claimed by source` and off the badge V-539 added. + +## It fires, and it is fast when the host is gone + +`docker stop searxng`, then one question: + +| | Claimed by | Turn | +|---|---|---| +| Search reachable | search | 3.5 s | +| Container stopped | kiwix | 3.5 s | +| Host blackholed | kiwix | 15.4 s | + +With the container stopped, DNS failed and the ZIM answered inside the same +second: + +``` +15:40:51 voice: search "что такое фотосинтез": ... lookup searxng: no such host +15:40:51 voice: kiwix: "photosynthesis" → 5 hits, top "Photosynthesis" +15:40:53 voice: query claimed by source "kiwix" +``` + +The rewrite, the search and the reply all fit in the same turn budget as a live +search. The fallback works. + +## The blackhole is the case that hurts + +192.0.2.1 is reserved and routed nowhere. Pointing `search.url` at it is the +shape of a real outage: the router drops the packet instead of refusing it. The +search sat for its full 8-second budget before the ZIM was asked. The turn took +15.4 seconds against 3.5. He waits through all of it with nothing +being said. + +Fixed by capping the connect phase alone at 1.5 s (`dialTimeout` in +`internal/websearch/searxng.go`). The instance is on the LAN, so a connection it +will ever accept is accepted in milliseconds. A reachable instance that is +merely slow still gets the whole 8 seconds. It is fanning out to real engines, +which is worth waiting for. + +## The Russian ZIM is now on the box and is read directly + +`wikipedia_ru_all_maxi_2026-02` (41 GB) was copied to the kiwix zims directory +and kiwix-serve picked it up. Note that the catalog name is derived from the +filename. `books.name=wikipedia_ru_all_maxi_2026-02` returns Фотосинтез, +С4-фотосинтез and Википедия. The `` field in the catalog says +`wikipedia_ru_all`, which returns nothing. + +A Cyrillic question now searches that book verbatim (`book_ru` in the `kiwix` +block). The rewriter was never a feature. An English ZIM cannot match a Russian +sentence, so the resident model translated the question into English keywords +first. That costs a model call. It also drops whatever the keywords do not carry. +Against a Russian book it is a translation of his own words back at him. + +## Not measured here + +- The Russian book answering a driven turn. The `book_ru` field is a binary + change, so it needs a rebuild the owner runs. The book itself was verified by + querying kiwix-serve directly. +- Recall against the Russian book compared with the rewrite path. Reading his + own language directly should win, and it was not scored. +- `ru.stackoverflow.com_mul_all_2026-02.zim` is still in the staging directory + and is wired to nothing. diff --git a/internal/config/config.go b/internal/config/config.go index 51297bb..14055ac 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1085,6 +1085,17 @@ type KiwixConfig struct { // query at a time. Book string `json:"book,omitempty"` + // BookRU — the ZIM to search when the question is in Russian, by the same + // catalog name. Empty ⇒ every question goes to Book. + // + // It exists because the rewriter is a workaround, not a feature (V-508). An + // English ZIM cannot match a Russian sentence, so the resident model turns + // the question into English keywords first, and that costs a model call and + // loses whatever the keywords drop. A Russian ZIM matches the question as he + // asked it. So a Cyrillic question searches this book verbatim and skips the + // rewrite, and the English book keeps answering English ones. + BookRU string `json:"book_ru,omitempty"` + // MaxResults — how many hits are asked for. 0 ⇒ DefaultKiwixResults. // Only the top few reach the phraser regardless; the rest are context the // snippet ranking throws away. diff --git a/internal/websearch/searxng.go b/internal/websearch/searxng.go index 21c5ffb..c6c3ced 100644 --- a/internal/websearch/searxng.go +++ b/internal/websearch/searxng.go @@ -25,6 +25,7 @@ import ( "encoding/json" "fmt" "io" + "net" "net/http" "net/url" "strings" @@ -56,6 +57,21 @@ func (r Response) Empty() bool { return len(r.Answers) == 0 && len(r.Results) == // a dead engine does not hold a voice turn open. const DefaultTimeout = 8 * time.Second +// dialTimeout — how long a connection to the instance may take before the turn +// gives up on it and falls through to the ZIM. +// +// It is separate from DefaultTimeout because the two failures are different +// (V-508). A reachable instance that is merely slow deserves the full budget: +// it is fanning out to real engines. A host that never answers a SYN deserves +// almost none, and the difference was measured. With the container stopped, DNS +// failed and the ZIM answered inside the same second. With the host blackholed, +// the search sat for the whole 8 seconds and the turn took 15.4 seconds instead +// of 3.5, which is a wait he sits through with nothing being said. +// +// The instance is on the LAN or the same host, so a connection it will ever +// accept is accepted in milliseconds. +const dialTimeout = 1500 * time.Millisecond + // maxBodyBytes caps the JSON read. A 20-result reply is tens of kilobytes; this // is slack for a wide one and a hard stop against a misconfigured endpoint. const maxBodyBytes = 4 << 20 @@ -94,7 +110,13 @@ func New(baseURL string, opt Options) *Client { base: strings.TrimRight(baseURL, "/"), language: strings.TrimSpace(opt.Language), engines: strings.TrimSpace(opt.Engines), - http: &http.Client{Timeout: t}, + http: &http.Client{ + Timeout: t, + // Cloned from the default so the rest of the transport (proxy, + // keep-alives, HTTP/2) keeps stock behaviour and only the dial + // budget changes. + Transport: dialCappedTransport(), + }, } } @@ -134,6 +156,14 @@ func (c *Client) Search(ctx context.Context, query string, limit int) (Response, return ParseResponse(body, limit) } +// dialCappedTransport is http.DefaultTransport with dialTimeout on the connect +// phase. A read that has already connected still gets the full request budget. +func dialCappedTransport() *http.Transport { + tr := http.DefaultTransport.(*http.Transport).Clone() + tr.DialContext = (&net.Dialer{Timeout: dialTimeout, KeepAlive: 30 * time.Second}).DialContext + return tr +} + // wire mirrors just the fields of the SearXNG JSON reply we read. type wire struct { Answers []json.RawMessage `json:"answers"` diff --git a/internal/websearch/searxng_test.go b/internal/websearch/searxng_test.go index 144f21b..206ecf7 100644 --- a/internal/websearch/searxng_test.go +++ b/internal/websearch/searxng_test.go @@ -6,6 +6,7 @@ import ( "net/http/httptest" "strings" "testing" + "time" ) const sampleJSON = `{ @@ -143,3 +144,39 @@ func TestSearchEmptyQuery(t *testing.T) { t.Fatal("empty query accepted") } } + +// A host that never answers a SYN must not hold the turn open for the whole +// request budget: the ZIM behind this search is the answer, and he waits +// through every second of the delay (V-508). 192.0.2.1 is TEST-NET-1, which is +// reserved for documentation and routed nowhere. +func TestSearchGivesUpOnAnUnreachableHostFast(t *testing.T) { + c := New("http://192.0.2.1:9563", Options{Timeout: 8 * time.Second}) + start := time.Now() + _, err := c.Search(context.Background(), "фотосинтез", 4) + elapsed := time.Since(start) + if err == nil { + t.Fatal("Search reached 192.0.2.1; the address is routed nowhere") + } + if elapsed > 4*time.Second { + t.Errorf("Search took %v to give up; the dial cap is %v", elapsed, dialTimeout) + } +} + +// The dial cap must not shorten a request to an instance that did connect. A +// slow SearXNG is fanning out to real engines, which is worth waiting for. +func TestSlowButReachableInstanceKeepsTheFullBudget(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(2 * dialTimeout) + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"answers":[],"results":[{"title":"Фотосинтез","url":"http://x","content":"процесс","engine":"test"}]}`)) + })) + defer srv.Close() + c := New(srv.URL, Options{Timeout: 8 * time.Second}) + resp, err := c.Search(context.Background(), "фотосинтез", 4) + if err != nil { + t.Fatalf("Search: %v", err) + } + if len(resp.Results) != 1 { + t.Errorf("results = %d, want 1", len(resp.Results)) + } +}