Merge the ZIM fallback verification and the Russian book (#173)

CLAUDE.md

@@ -286,6 +286,14 @@ world questions, so she needs to read external sources. What replaces it:
 - **In the world, live search leads and the ZIMs are the fallback** (owner's call,
   2026-08-02). A self-hosted SearXNG (`search` block) answers first; the Kiwix ZIMs on
   homesrv answer when the search is empty, unreachable, or the line is down.
+  **Verified with the line down on 2026-08-05** (V-508,
+  `docs/evals/2026-08-05-kiwix-offline-fallback.md`): a stopped SearXNG costs nothing,
+  the ZIM answers in the same turn budget. A blackholed host cost 8 seconds he waited
+  through. So the connect phase alone is capped at `dialTimeout` (1.5s), while a slow
+  instance that did connect keeps the full 8. **A Russian question reads
+  `wikipedia_ru_all_maxi_2026-02` verbatim** through `kiwix.book_ru`. The RU→EN rewriter
+  is the workaround for an English book and is skipped there. Kiwix catalog names come
+  from the filename, not the `<name>` field.
   `Response.Empty()` is the whole gate and there is no quality threshold in front of it:
   the three signals one could read were measured on 2026-08-05 and none of them separate a
   real question from an invented one. Token overlap would cost "столица Франции" its

cmd/mavend/actions_query.go

@@ -8,6 +8,7 @@ import (
 	"regexp"
 	"strings"
 	"time"
+	"unicode"
 
 	"github.com/kami/maven/internal/crawl"
 	"github.com/kami/maven/internal/ipc"
@@ -695,11 +696,19 @@ func (h *reactiveHandler) queryKiwix(ctx context.Context, t *queryTurn) (string,
 	ctxK, cancel := context.WithTimeout(ctx, kiwixTimeout)
 	defer cancel()
 
-	// The ZIMs are English and kiwix ranks by keyword overlap, not meaning, so
-	// a Russian sentence matches nothing at all. The rewriter turns it into a
-	// handful of English keywords with the resident model.
+	// A Russian question reads the Russian ZIM verbatim when there is one
+	// (V-508). Kiwix ranks by keyword overlap rather than meaning, so an English
+	// book matches a Russian sentence not at all, and the rewriter exists to
+	// turn the question into English keywords with the resident model. Against a
+	// Russian book that is a translation of his own words back at him: it costs
+	// a model call and drops whatever the keywords do not carry.
+	book, verbatim := h.kiwix.book, false
+	if h.kiwix.bookRU != "" && hasCyrillic(t.dec.Utterance) {
+		book, verbatim = h.kiwix.bookRU, true
+	}
+
 	pattern := t.dec.Utterance
-	if h.kiwix.rewriter != nil {
+	if h.kiwix.rewriter != nil && !verbatim {
 		q, err := h.kiwix.rewriter.Rewrite(ctxK, t.dec.Utterance)
 		if err != nil {
 			// Fall through to the verbatim question rather than give up. It
@@ -710,7 +719,7 @@ func (h *reactiveHandler) queryKiwix(ctx context.Context, t *queryTurn) (string,
 		}
 	}
 
-	hits, err := h.kiwix.client.Search(ctxK, pattern, h.kiwix.book, h.kiwix.max)
+	hits, err := h.kiwix.client.Search(ctxK, pattern, book, h.kiwix.max)
 	if err != nil {
 		log.Printf("voice: kiwix: search %q: %v", pattern, err)
 		return "", false
@@ -722,7 +731,7 @@ func (h *reactiveHandler) queryKiwix(ctx context.Context, t *queryTurn) (string,
 	// Logged on the way through, not only on failure. Without this there is no
 	// way to tell from the outside whether an answer came off a ZIM or out of
 	// the model's weights, and those are the two cases worth telling apart.
-	log.Printf("voice: kiwix: %q → %d hits, top %q", pattern, len(hits), top.Title)
+	log.Printf("voice: kiwix: %q in %q → %d hits, top %q", pattern, book, len(hits), top.Title)
 
 	// The top hit only, read as an article rather than as a snippet. Kiwix
 	// builds its snippet from wherever the keyword matched, which on Wikipedia
@@ -754,6 +763,18 @@ func (h *reactiveHandler) queryKiwix(ctx context.Context, t *queryTurn) (string,
 	return reply, true
 }
 
+// hasCyrillic reports whether the text carries a Cyrillic letter, which is the
+// whole test for "he asked this in Russian". A question mixing a Latin proper
+// noun into a Russian sentence is still Russian, so one letter is enough.
+func hasCyrillic(s string) bool {
+	for _, r := range s {
+		if unicode.Is(unicode.Cyrillic, r) {
+			return true
+		}
+	}
+	return false
+}
+
 // queryPersonal — stop the walk on a question about him that his own data did
 // not answer.
 //

cmd/mavend/kiwixbook_test.go

@@ -0,0 +1,44 @@
+package main
+
+import "testing"
+
+func TestHasCyrillic(t *testing.T) {
+	for _, s := range []string{"что такое фотосинтез", "кто такой Elon Musk", "фотосинтез"} {
+		if !hasCyrillic(s) {
+			t.Errorf("hasCyrillic(%q) = false; it is a Russian question", s)
+		}
+	}
+	for _, s := range []string{"what is photosynthesis", "", "3:2"} {
+		if hasCyrillic(s) {
+			t.Errorf("hasCyrillic(%q) = true; there is no Cyrillic in it", s)
+		}
+	}
+}
+
+// The book choice and the rewrite decision are the same decision: a Russian
+// book reads his question as he asked it, an English one needs it translated
+// into keywords first (V-508).
+func TestKiwixBookChoice(t *testing.T) {
+	for _, tc := range []struct {
+		name      string
+		wiring    kiwixWiring
+		utterance string
+		wantBook  string
+		wantVerb  bool
+	}{
+		{"a russian question reads the russian book verbatim",
+			kiwixWiring{book: "en", bookRU: "ru"}, "что такое фотосинтез", "ru", true},
+		{"an english question reads the english book",
+			kiwixWiring{book: "en", bookRU: "ru"}, "what is photosynthesis", "en", false},
+		{"no russian book configured leaves every question on the english one",
+			kiwixWiring{book: "en"}, "что такое фотосинтез", "en", false},
+	} {
+		book, verbatim := tc.wiring.book, false
+		if tc.wiring.bookRU != "" && hasCyrillic(tc.utterance) {
+			book, verbatim = tc.wiring.bookRU, true
+		}
+		if book != tc.wantBook || verbatim != tc.wantVerb {
+			t.Errorf("%s: book=%q verbatim=%v, want %q/%v", tc.name, book, verbatim, tc.wantBook, tc.wantVerb)
+		}
+	}
+}

cmd/mavend/kiwixwire.go

@@ -19,8 +19,12 @@ type kiwixWiring struct {
 	client   *kiwix.Client
 	rewriter *kiwix.Rewriter // nil ⇒ the question is searched verbatim
 	book     string
-	max      int
-	runes    int
+	// bookRU — searched instead of book when the question is Cyrillic, and
+	// searched verbatim because it is in his language already (V-508). Empty ⇒
+	// every question goes to book.
+	bookRU string
+	max    int
+	runes  int
 }
 
 // wireKiwix builds the ZIM reader from the `kiwix` block, or returns nil when
@@ -38,9 +42,13 @@ func wireKiwix(cfg *config.Config, c *llm.Client) *kiwixWiring {
 	w := &kiwixWiring{
 		client: kiwix.New(kc.URL),
 		book:   kc.Book,
+		bookRU: kc.BookRU,
 		max:    kc.MaxResults,
 		runes:  kc.SnippetRunes,
 	}
+	if kc.BookRU != "" {
+		log.Printf("voice: kiwix: russian questions read %q verbatim", kc.BookRU)
+	}
 	switch {
 	case !kc.RewriteEnabled():
 		log.Printf("voice: kiwix at %s (book %q, query rewriting off by config)", kc.URL, kc.Book)

deploy/mavend.json

@@ -90,6 +90,7 @@
   "kiwix": {
     "url": "http://kiwix-server:8080",
     "book": "wikipedia_en_all_maxi_2026-02",
+    "book_ru": "wikipedia_ru_all_maxi_2026-02",
     "max_results": 5,
     "snippet_runes": 1500
   },

docs/evals/2026-08-05-kiwix-offline-fallback.md

@@ -0,0 +1,65 @@
+# Does the ZIM answer when the line is down? (V-508)
+
+Measured 2026-08-05 on the deploy, through `POST /api/chat`. The question was
+`что такое фотосинтез` in every run. Which source claimed is read off
+`voice: query claimed by source` and off the badge V-539 added.
+
+## It fires, and it is fast when the host is gone
+
+`docker stop searxng`, then one question:
+
+| | Claimed by | Turn |
+|---|---|---|
+| Search reachable | search | 3.5 s |
+| Container stopped | kiwix | 3.5 s |
+| Host blackholed | kiwix | 15.4 s |
+
+With the container stopped, DNS failed and the ZIM answered inside the same
+second:
+
+```
+15:40:51 voice: search "что такое фотосинтез": ... lookup searxng: no such host
+15:40:51 voice: kiwix: "photosynthesis" → 5 hits, top "Photosynthesis"
+15:40:53 voice: query claimed by source "kiwix"
+```
+
+The rewrite, the search and the reply all fit in the same turn budget as a live
+search. The fallback works.
+
+## The blackhole is the case that hurts
+
+192.0.2.1 is reserved and routed nowhere. Pointing `search.url` at it is the
+shape of a real outage: the router drops the packet instead of refusing it. The
+search sat for its full 8-second budget before the ZIM was asked. The turn took
+15.4 seconds against 3.5. He waits through all of it with nothing
+being said.
+
+Fixed by capping the connect phase alone at 1.5 s (`dialTimeout` in
+`internal/websearch/searxng.go`). The instance is on the LAN, so a connection it
+will ever accept is accepted in milliseconds. A reachable instance that is
+merely slow still gets the whole 8 seconds. It is fanning out to real engines,
+which is worth waiting for.
+
+## The Russian ZIM is now on the box and is read directly
+
+`wikipedia_ru_all_maxi_2026-02` (41 GB) was copied to the kiwix zims directory
+and kiwix-serve picked it up. Note that the catalog name is derived from the
+filename. `books.name=wikipedia_ru_all_maxi_2026-02` returns Фотосинтез,
+С4-фотосинтез and Википедия. The `<name>` field in the catalog says
+`wikipedia_ru_all`, which returns nothing.
+
+A Cyrillic question now searches that book verbatim (`book_ru` in the `kiwix`
+block). The rewriter was never a feature. An English ZIM cannot match a Russian
+sentence, so the resident model translated the question into English keywords
+first. That costs a model call. It also drops whatever the keywords do not carry.
+Against a Russian book it is a translation of his own words back at him.
+
+## Not measured here
+
+- The Russian book answering a driven turn. The `book_ru` field is a binary
+  change, so it needs a rebuild the owner runs. The book itself was verified by
+  querying kiwix-serve directly.
+- Recall against the Russian book compared with the rewrite path. Reading his
+  own language directly should win, and it was not scored.
+- `ru.stackoverflow.com_mul_all_2026-02.zim` is still in the staging directory
+  and is wired to nothing.

internal/config/config.go

@@ -1085,6 +1085,17 @@ type KiwixConfig struct {
 	// query at a time.
 	Book string `json:"book,omitempty"`
 
+	// BookRU — the ZIM to search when the question is in Russian, by the same
+	// catalog name. Empty ⇒ every question goes to Book.
+	//
+	// It exists because the rewriter is a workaround, not a feature (V-508). An
+	// English ZIM cannot match a Russian sentence, so the resident model turns
+	// the question into English keywords first, and that costs a model call and
+	// loses whatever the keywords drop. A Russian ZIM matches the question as he
+	// asked it. So a Cyrillic question searches this book verbatim and skips the
+	// rewrite, and the English book keeps answering English ones.
+	BookRU string `json:"book_ru,omitempty"`
+
 	// MaxResults — how many hits are asked for. 0 ⇒ DefaultKiwixResults.
 	// Only the top few reach the phraser regardless; the rest are context the
 	// snippet ranking throws away.

internal/websearch/searxng.go

@@ -25,6 +25,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"net/url"
 	"strings"
@@ -56,6 +57,21 @@ func (r Response) Empty() bool { return len(r.Answers) == 0 && len(r.Results) ==
 // a dead engine does not hold a voice turn open.
 const DefaultTimeout = 8 * time.Second
 
+// dialTimeout — how long a connection to the instance may take before the turn
+// gives up on it and falls through to the ZIM.
+//
+// It is separate from DefaultTimeout because the two failures are different
+// (V-508). A reachable instance that is merely slow deserves the full budget:
+// it is fanning out to real engines. A host that never answers a SYN deserves
+// almost none, and the difference was measured. With the container stopped, DNS
+// failed and the ZIM answered inside the same second. With the host blackholed,
+// the search sat for the whole 8 seconds and the turn took 15.4 seconds instead
+// of 3.5, which is a wait he sits through with nothing being said.
+//
+// The instance is on the LAN or the same host, so a connection it will ever
+// accept is accepted in milliseconds.
+const dialTimeout = 1500 * time.Millisecond
+
 // maxBodyBytes caps the JSON read. A 20-result reply is tens of kilobytes; this
 // is slack for a wide one and a hard stop against a misconfigured endpoint.
 const maxBodyBytes = 4 << 20
@@ -94,7 +110,13 @@ func New(baseURL string, opt Options) *Client {
 		base:     strings.TrimRight(baseURL, "/"),
 		language: strings.TrimSpace(opt.Language),
 		engines:  strings.TrimSpace(opt.Engines),
-		http:     &http.Client{Timeout: t},
+		http: &http.Client{
+			Timeout: t,
+			// Cloned from the default so the rest of the transport (proxy,
+			// keep-alives, HTTP/2) keeps stock behaviour and only the dial
+			// budget changes.
+			Transport: dialCappedTransport(),
+		},
 	}
 }
 
@@ -134,6 +156,14 @@ func (c *Client) Search(ctx context.Context, query string, limit int) (Response,
 	return ParseResponse(body, limit)
 }
 
+// dialCappedTransport is http.DefaultTransport with dialTimeout on the connect
+// phase. A read that has already connected still gets the full request budget.
+func dialCappedTransport() *http.Transport {
+	tr := http.DefaultTransport.(*http.Transport).Clone()
+	tr.DialContext = (&net.Dialer{Timeout: dialTimeout, KeepAlive: 30 * time.Second}).DialContext
+	return tr
+}
+
 // wire mirrors just the fields of the SearXNG JSON reply we read.
 type wire struct {
 	Answers []json.RawMessage `json:"answers"`

internal/websearch/searxng_test.go

@@ -6,6 +6,7 @@ import (
 	"net/http/httptest"
 	"strings"
 	"testing"
+	"time"
 )
 
 const sampleJSON = `{
@@ -143,3 +144,39 @@ func TestSearchEmptyQuery(t *testing.T) {
 		t.Fatal("empty query accepted")
 	}
 }
+
+// A host that never answers a SYN must not hold the turn open for the whole
+// request budget: the ZIM behind this search is the answer, and he waits
+// through every second of the delay (V-508). 192.0.2.1 is TEST-NET-1, which is
+// reserved for documentation and routed nowhere.
+func TestSearchGivesUpOnAnUnreachableHostFast(t *testing.T) {
+	c := New("http://192.0.2.1:9563", Options{Timeout: 8 * time.Second})
+	start := time.Now()
+	_, err := c.Search(context.Background(), "фотосинтез", 4)
+	elapsed := time.Since(start)
+	if err == nil {
+		t.Fatal("Search reached 192.0.2.1; the address is routed nowhere")
+	}
+	if elapsed > 4*time.Second {
+		t.Errorf("Search took %v to give up; the dial cap is %v", elapsed, dialTimeout)
+	}
+}
+
+// The dial cap must not shorten a request to an instance that did connect. A
+// slow SearXNG is fanning out to real engines, which is worth waiting for.
+func TestSlowButReachableInstanceKeepsTheFullBudget(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		time.Sleep(2 * dialTimeout)
+		w.Header().Set("Content-Type", "application/json")
+		w.Write([]byte(`{"answers":[],"results":[{"title":"Фотосинтез","url":"http://x","content":"процесс","engine":"test"}]}`))
+	}))
+	defer srv.Close()
+	c := New(srv.URL, Options{Timeout: 8 * time.Second})
+	resp, err := c.Search(context.Background(), "фотосинтез", 4)
+	if err != nil {
+		t.Fatalf("Search: %v", err)
+	}
+	if len(resp.Results) != 1 {
+		t.Errorf("results = %d, want 1", len(resp.Results))
+	}
+}