diff --git a/CLAUDE.md b/CLAUDE.md index 90141a9..5bec859 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -286,6 +286,14 @@ world questions, so she needs to read external sources. What replaces it: - **In the world, live search leads and the ZIMs are the fallback** (owner's call, 2026-08-02). A self-hosted SearXNG (`search` block) answers first; the Kiwix ZIMs on homesrv answer when the search is empty, unreachable, or the line is down. + **Verified with the line down on 2026-08-05** (V-508, + `docs/evals/2026-08-05-kiwix-offline-fallback.md`): a stopped SearXNG costs nothing, + the ZIM answers in the same turn budget. A blackholed host cost 8 seconds he waited + through. So the connect phase alone is capped at `dialTimeout` (1.5s), while a slow + instance that did connect keeps the full 8. **A Russian question reads + `wikipedia_ru_all_maxi_2026-02` verbatim** through `kiwix.book_ru`. The RU→EN rewriter + is the workaround for an English book and is skipped there. Kiwix catalog names come + from the filename, not the `` field. `Response.Empty()` is the whole gate and there is no quality threshold in front of it: the three signals one could read were measured on 2026-08-05 and none of them separate a real question from an invented one. Token overlap would cost "столица Франции" its diff --git a/cmd/mavend/actions_query.go b/cmd/mavend/actions_query.go index d0a35fb..03fdda9 100644 --- a/cmd/mavend/actions_query.go +++ b/cmd/mavend/actions_query.go @@ -8,6 +8,7 @@ import ( "regexp" "strings" "time" + "unicode" "github.com/kami/maven/internal/crawl" "github.com/kami/maven/internal/ipc" @@ -695,11 +696,19 @@ func (h *reactiveHandler) queryKiwix(ctx context.Context, t *queryTurn) (string, ctxK, cancel := context.WithTimeout(ctx, kiwixTimeout) defer cancel() - // The ZIMs are English and kiwix ranks by keyword overlap, not meaning, so - // a Russian sentence matches nothing at all. The rewriter turns it into a - // handful of English keywords with the resident model. + // A Russian question reads the Russian ZIM verbatim when there is one + // (V-508). Kiwix ranks by keyword overlap rather than meaning, so an English + // book matches a Russian sentence not at all, and the rewriter exists to + // turn the question into English keywords with the resident model. Against a + // Russian book that is a translation of his own words back at him: it costs + // a model call and drops whatever the keywords do not carry. + book, verbatim := h.kiwix.book, false + if h.kiwix.bookRU != "" && hasCyrillic(t.dec.Utterance) { + book, verbatim = h.kiwix.bookRU, true + } + pattern := t.dec.Utterance - if h.kiwix.rewriter != nil { + if h.kiwix.rewriter != nil && !verbatim { q, err := h.kiwix.rewriter.Rewrite(ctxK, t.dec.Utterance) if err != nil { // Fall through to the verbatim question rather than give up. It @@ -710,7 +719,7 @@ func (h *reactiveHandler) queryKiwix(ctx context.Context, t *queryTurn) (string, } } - hits, err := h.kiwix.client.Search(ctxK, pattern, h.kiwix.book, h.kiwix.max) + hits, err := h.kiwix.client.Search(ctxK, pattern, book, h.kiwix.max) if err != nil { log.Printf("voice: kiwix: search %q: %v", pattern, err) return "", false @@ -722,7 +731,7 @@ func (h *reactiveHandler) queryKiwix(ctx context.Context, t *queryTurn) (string, // Logged on the way through, not only on failure. Without this there is no // way to tell from the outside whether an answer came off a ZIM or out of // the model's weights, and those are the two cases worth telling apart. - log.Printf("voice: kiwix: %q → %d hits, top %q", pattern, len(hits), top.Title) + log.Printf("voice: kiwix: %q in %q → %d hits, top %q", pattern, book, len(hits), top.Title) // The top hit only, read as an article rather than as a snippet. Kiwix // builds its snippet from wherever the keyword matched, which on Wikipedia @@ -754,6 +763,18 @@ func (h *reactiveHandler) queryKiwix(ctx context.Context, t *queryTurn) (string, return reply, true } +// hasCyrillic reports whether the text carries a Cyrillic letter, which is the +// whole test for "he asked this in Russian". A question mixing a Latin proper +// noun into a Russian sentence is still Russian, so one letter is enough. +func hasCyrillic(s string) bool { + for _, r := range s { + if unicode.Is(unicode.Cyrillic, r) { + return true + } + } + return false +} + // queryPersonal — stop the walk on a question about him that his own data did // not answer. // diff --git a/cmd/mavend/kiwixbook_test.go b/cmd/mavend/kiwixbook_test.go new file mode 100644 index 0000000..3ec8049 --- /dev/null +++ b/cmd/mavend/kiwixbook_test.go @@ -0,0 +1,44 @@ +package main + +import "testing" + +func TestHasCyrillic(t *testing.T) { + for _, s := range []string{"что такое фотосинтез", "кто такой Elon Musk", "фотосинтез"} { + if !hasCyrillic(s) { + t.Errorf("hasCyrillic(%q) = false; it is a Russian question", s) + } + } + for _, s := range []string{"what is photosynthesis", "", "3:2"} { + if hasCyrillic(s) { + t.Errorf("hasCyrillic(%q) = true; there is no Cyrillic in it", s) + } + } +} + +// The book choice and the rewrite decision are the same decision: a Russian +// book reads his question as he asked it, an English one needs it translated +// into keywords first (V-508). +func TestKiwixBookChoice(t *testing.T) { + for _, tc := range []struct { + name string + wiring kiwixWiring + utterance string + wantBook string + wantVerb bool + }{ + {"a russian question reads the russian book verbatim", + kiwixWiring{book: "en", bookRU: "ru"}, "что такое фотосинтез", "ru", true}, + {"an english question reads the english book", + kiwixWiring{book: "en", bookRU: "ru"}, "what is photosynthesis", "en", false}, + {"no russian book configured leaves every question on the english one", + kiwixWiring{book: "en"}, "что такое фотосинтез", "en", false}, + } { + book, verbatim := tc.wiring.book, false + if tc.wiring.bookRU != "" && hasCyrillic(tc.utterance) { + book, verbatim = tc.wiring.bookRU, true + } + if book != tc.wantBook || verbatim != tc.wantVerb { + t.Errorf("%s: book=%q verbatim=%v, want %q/%v", tc.name, book, verbatim, tc.wantBook, tc.wantVerb) + } + } +} diff --git a/cmd/mavend/kiwixwire.go b/cmd/mavend/kiwixwire.go index 84e3a5d..bcef387 100644 --- a/cmd/mavend/kiwixwire.go +++ b/cmd/mavend/kiwixwire.go @@ -19,8 +19,12 @@ type kiwixWiring struct { client *kiwix.Client rewriter *kiwix.Rewriter // nil ⇒ the question is searched verbatim book string - max int - runes int + // bookRU — searched instead of book when the question is Cyrillic, and + // searched verbatim because it is in his language already (V-508). Empty ⇒ + // every question goes to book. + bookRU string + max int + runes int } // wireKiwix builds the ZIM reader from the `kiwix` block, or returns nil when @@ -38,9 +42,13 @@ func wireKiwix(cfg *config.Config, c *llm.Client) *kiwixWiring { w := &kiwixWiring{ client: kiwix.New(kc.URL), book: kc.Book, + bookRU: kc.BookRU, max: kc.MaxResults, runes: kc.SnippetRunes, } + if kc.BookRU != "" { + log.Printf("voice: kiwix: russian questions read %q verbatim", kc.BookRU) + } switch { case !kc.RewriteEnabled(): log.Printf("voice: kiwix at %s (book %q, query rewriting off by config)", kc.URL, kc.Book) diff --git a/deploy/mavend.json b/deploy/mavend.json index e1d1a9e..ef0785d 100644 --- a/deploy/mavend.json +++ b/deploy/mavend.json @@ -90,6 +90,7 @@ "kiwix": { "url": "http://kiwix-server:8080", "book": "wikipedia_en_all_maxi_2026-02", + "book_ru": "wikipedia_ru_all_maxi_2026-02", "max_results": 5, "snippet_runes": 1500 }, diff --git a/docs/evals/2026-08-05-kiwix-offline-fallback.md b/docs/evals/2026-08-05-kiwix-offline-fallback.md new file mode 100644 index 0000000..7f91d03 --- /dev/null +++ b/docs/evals/2026-08-05-kiwix-offline-fallback.md @@ -0,0 +1,65 @@ +# Does the ZIM answer when the line is down? (V-508) + +Measured 2026-08-05 on the deploy, through `POST /api/chat`. The question was +`что такое фотосинтез` in every run. Which source claimed is read off +`voice: query claimed by source` and off the badge V-539 added. + +## It fires, and it is fast when the host is gone + +`docker stop searxng`, then one question: + +| | Claimed by | Turn | +|---|---|---| +| Search reachable | search | 3.5 s | +| Container stopped | kiwix | 3.5 s | +| Host blackholed | kiwix | 15.4 s | + +With the container stopped, DNS failed and the ZIM answered inside the same +second: + +``` +15:40:51 voice: search "что такое фотосинтез": ... lookup searxng: no such host +15:40:51 voice: kiwix: "photosynthesis" → 5 hits, top "Photosynthesis" +15:40:53 voice: query claimed by source "kiwix" +``` + +The rewrite, the search and the reply all fit in the same turn budget as a live +search. The fallback works. + +## The blackhole is the case that hurts + +192.0.2.1 is reserved and routed nowhere. Pointing `search.url` at it is the +shape of a real outage: the router drops the packet instead of refusing it. The +search sat for its full 8-second budget before the ZIM was asked. The turn took +15.4 seconds against 3.5. He waits through all of it with nothing +being said. + +Fixed by capping the connect phase alone at 1.5 s (`dialTimeout` in +`internal/websearch/searxng.go`). The instance is on the LAN, so a connection it +will ever accept is accepted in milliseconds. A reachable instance that is +merely slow still gets the whole 8 seconds. It is fanning out to real engines, +which is worth waiting for. + +## The Russian ZIM is now on the box and is read directly + +`wikipedia_ru_all_maxi_2026-02` (41 GB) was copied to the kiwix zims directory +and kiwix-serve picked it up. Note that the catalog name is derived from the +filename. `books.name=wikipedia_ru_all_maxi_2026-02` returns Фотосинтез, +С4-фотосинтез and Википедия. The `` field in the catalog says +`wikipedia_ru_all`, which returns nothing. + +A Cyrillic question now searches that book verbatim (`book_ru` in the `kiwix` +block). The rewriter was never a feature. An English ZIM cannot match a Russian +sentence, so the resident model translated the question into English keywords +first. That costs a model call. It also drops whatever the keywords do not carry. +Against a Russian book it is a translation of his own words back at him. + +## Not measured here + +- The Russian book answering a driven turn. The `book_ru` field is a binary + change, so it needs a rebuild the owner runs. The book itself was verified by + querying kiwix-serve directly. +- Recall against the Russian book compared with the rewrite path. Reading his + own language directly should win, and it was not scored. +- `ru.stackoverflow.com_mul_all_2026-02.zim` is still in the staging directory + and is wired to nothing. diff --git a/internal/config/config.go b/internal/config/config.go index 51297bb..14055ac 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1085,6 +1085,17 @@ type KiwixConfig struct { // query at a time. Book string `json:"book,omitempty"` + // BookRU — the ZIM to search when the question is in Russian, by the same + // catalog name. Empty ⇒ every question goes to Book. + // + // It exists because the rewriter is a workaround, not a feature (V-508). An + // English ZIM cannot match a Russian sentence, so the resident model turns + // the question into English keywords first, and that costs a model call and + // loses whatever the keywords drop. A Russian ZIM matches the question as he + // asked it. So a Cyrillic question searches this book verbatim and skips the + // rewrite, and the English book keeps answering English ones. + BookRU string `json:"book_ru,omitempty"` + // MaxResults — how many hits are asked for. 0 ⇒ DefaultKiwixResults. // Only the top few reach the phraser regardless; the rest are context the // snippet ranking throws away. diff --git a/internal/websearch/searxng.go b/internal/websearch/searxng.go index 21c5ffb..c6c3ced 100644 --- a/internal/websearch/searxng.go +++ b/internal/websearch/searxng.go @@ -25,6 +25,7 @@ import ( "encoding/json" "fmt" "io" + "net" "net/http" "net/url" "strings" @@ -56,6 +57,21 @@ func (r Response) Empty() bool { return len(r.Answers) == 0 && len(r.Results) == // a dead engine does not hold a voice turn open. const DefaultTimeout = 8 * time.Second +// dialTimeout — how long a connection to the instance may take before the turn +// gives up on it and falls through to the ZIM. +// +// It is separate from DefaultTimeout because the two failures are different +// (V-508). A reachable instance that is merely slow deserves the full budget: +// it is fanning out to real engines. A host that never answers a SYN deserves +// almost none, and the difference was measured. With the container stopped, DNS +// failed and the ZIM answered inside the same second. With the host blackholed, +// the search sat for the whole 8 seconds and the turn took 15.4 seconds instead +// of 3.5, which is a wait he sits through with nothing being said. +// +// The instance is on the LAN or the same host, so a connection it will ever +// accept is accepted in milliseconds. +const dialTimeout = 1500 * time.Millisecond + // maxBodyBytes caps the JSON read. A 20-result reply is tens of kilobytes; this // is slack for a wide one and a hard stop against a misconfigured endpoint. const maxBodyBytes = 4 << 20 @@ -94,7 +110,13 @@ func New(baseURL string, opt Options) *Client { base: strings.TrimRight(baseURL, "/"), language: strings.TrimSpace(opt.Language), engines: strings.TrimSpace(opt.Engines), - http: &http.Client{Timeout: t}, + http: &http.Client{ + Timeout: t, + // Cloned from the default so the rest of the transport (proxy, + // keep-alives, HTTP/2) keeps stock behaviour and only the dial + // budget changes. + Transport: dialCappedTransport(), + }, } } @@ -134,6 +156,14 @@ func (c *Client) Search(ctx context.Context, query string, limit int) (Response, return ParseResponse(body, limit) } +// dialCappedTransport is http.DefaultTransport with dialTimeout on the connect +// phase. A read that has already connected still gets the full request budget. +func dialCappedTransport() *http.Transport { + tr := http.DefaultTransport.(*http.Transport).Clone() + tr.DialContext = (&net.Dialer{Timeout: dialTimeout, KeepAlive: 30 * time.Second}).DialContext + return tr +} + // wire mirrors just the fields of the SearXNG JSON reply we read. type wire struct { Answers []json.RawMessage `json:"answers"` diff --git a/internal/websearch/searxng_test.go b/internal/websearch/searxng_test.go index 144f21b..206ecf7 100644 --- a/internal/websearch/searxng_test.go +++ b/internal/websearch/searxng_test.go @@ -6,6 +6,7 @@ import ( "net/http/httptest" "strings" "testing" + "time" ) const sampleJSON = `{ @@ -143,3 +144,39 @@ func TestSearchEmptyQuery(t *testing.T) { t.Fatal("empty query accepted") } } + +// A host that never answers a SYN must not hold the turn open for the whole +// request budget: the ZIM behind this search is the answer, and he waits +// through every second of the delay (V-508). 192.0.2.1 is TEST-NET-1, which is +// reserved for documentation and routed nowhere. +func TestSearchGivesUpOnAnUnreachableHostFast(t *testing.T) { + c := New("http://192.0.2.1:9563", Options{Timeout: 8 * time.Second}) + start := time.Now() + _, err := c.Search(context.Background(), "фотосинтез", 4) + elapsed := time.Since(start) + if err == nil { + t.Fatal("Search reached 192.0.2.1; the address is routed nowhere") + } + if elapsed > 4*time.Second { + t.Errorf("Search took %v to give up; the dial cap is %v", elapsed, dialTimeout) + } +} + +// The dial cap must not shorten a request to an instance that did connect. A +// slow SearXNG is fanning out to real engines, which is worth waiting for. +func TestSlowButReachableInstanceKeepsTheFullBudget(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(2 * dialTimeout) + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"answers":[],"results":[{"title":"Фотосинтез","url":"http://x","content":"процесс","engine":"test"}]}`)) + })) + defer srv.Close() + c := New(srv.URL, Options{Timeout: 8 * time.Second}) + resp, err := c.Search(context.Background(), "фотосинтез", 4) + if err != nil { + t.Fatalf("Search: %v", err) + } + if len(resp.Results) != 1 { + t.Errorf("results = %d, want 1", len(resp.Results)) + } +}