the ZIM fallback fires fast, and reads Russian in Russian (V-508)
Verification, as the task asked. Drove что такое фотосинтез through /api/chat with the search reachable, with the container stopped, and with the host blackholed. Kiwix claims the turn in both failure cases, and a stopped container costs nothing: DNS fails and the ZIM answers inside the same second. The blackhole is the case that hurts. The search waited its full 8-second budget before the ZIM was asked and the turn took 15.4s against 3.5, which he sits through with nothing being said. So the connect phase alone is now capped at 1.5s. A reachable instance that is merely slow keeps the whole budget, because it is fanning out to real engines. The RU Wikipedia ZIM is on the box (owner moved it into the kiwix zims dir), and kiwix-serve picked it up. A Cyrillic question now searches book_ru verbatim and skips the RU->EN rewrite: that rewriter is the workaround for an English book, and against a Russian one it is a translation of his own words back at him. Catalog names come from the filename, not the <name> field — books.name=wikipedia_ru_all returns nothing. Measurement in docs/evals/2026-08-05-kiwix-offline-fallback.md. The RU book answering a driven turn needs a rebuild and is not verified yet.
This commit is contained in:
@@ -1085,6 +1085,17 @@ type KiwixConfig struct {
|
||||
// query at a time.
|
||||
Book string `json:"book,omitempty"`
|
||||
|
||||
// BookRU — the ZIM to search when the question is in Russian, by the same
|
||||
// catalog name. Empty ⇒ every question goes to Book.
|
||||
//
|
||||
// It exists because the rewriter is a workaround, not a feature (V-508). An
|
||||
// English ZIM cannot match a Russian sentence, so the resident model turns
|
||||
// the question into English keywords first, and that costs a model call and
|
||||
// loses whatever the keywords drop. A Russian ZIM matches the question as he
|
||||
// asked it. So a Cyrillic question searches this book verbatim and skips the
|
||||
// rewrite, and the English book keeps answering English ones.
|
||||
BookRU string `json:"book_ru,omitempty"`
|
||||
|
||||
// MaxResults — how many hits are asked for. 0 ⇒ DefaultKiwixResults.
|
||||
// Only the top few reach the phraser regardless; the rest are context the
|
||||
// snippet ranking throws away.
|
||||
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
@@ -56,6 +57,21 @@ func (r Response) Empty() bool { return len(r.Answers) == 0 && len(r.Results) ==
|
||||
// a dead engine does not hold a voice turn open.
|
||||
const DefaultTimeout = 8 * time.Second
|
||||
|
||||
// dialTimeout — how long a connection to the instance may take before the turn
|
||||
// gives up on it and falls through to the ZIM.
|
||||
//
|
||||
// It is separate from DefaultTimeout because the two failures are different
|
||||
// (V-508). A reachable instance that is merely slow deserves the full budget:
|
||||
// it is fanning out to real engines. A host that never answers a SYN deserves
|
||||
// almost none, and the difference was measured. With the container stopped, DNS
|
||||
// failed and the ZIM answered inside the same second. With the host blackholed,
|
||||
// the search sat for the whole 8 seconds and the turn took 15.4 seconds instead
|
||||
// of 3.5, which is a wait he sits through with nothing being said.
|
||||
//
|
||||
// The instance is on the LAN or the same host, so a connection it will ever
|
||||
// accept is accepted in milliseconds.
|
||||
const dialTimeout = 1500 * time.Millisecond
|
||||
|
||||
// maxBodyBytes caps the JSON read. A 20-result reply is tens of kilobytes; this
|
||||
// is slack for a wide one and a hard stop against a misconfigured endpoint.
|
||||
const maxBodyBytes = 4 << 20
|
||||
@@ -94,7 +110,13 @@ func New(baseURL string, opt Options) *Client {
|
||||
base: strings.TrimRight(baseURL, "/"),
|
||||
language: strings.TrimSpace(opt.Language),
|
||||
engines: strings.TrimSpace(opt.Engines),
|
||||
http: &http.Client{Timeout: t},
|
||||
http: &http.Client{
|
||||
Timeout: t,
|
||||
// Cloned from the default so the rest of the transport (proxy,
|
||||
// keep-alives, HTTP/2) keeps stock behaviour and only the dial
|
||||
// budget changes.
|
||||
Transport: dialCappedTransport(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -134,6 +156,14 @@ func (c *Client) Search(ctx context.Context, query string, limit int) (Response,
|
||||
return ParseResponse(body, limit)
|
||||
}
|
||||
|
||||
// dialCappedTransport is http.DefaultTransport with dialTimeout on the connect
|
||||
// phase. A read that has already connected still gets the full request budget.
|
||||
func dialCappedTransport() *http.Transport {
|
||||
tr := http.DefaultTransport.(*http.Transport).Clone()
|
||||
tr.DialContext = (&net.Dialer{Timeout: dialTimeout, KeepAlive: 30 * time.Second}).DialContext
|
||||
return tr
|
||||
}
|
||||
|
||||
// wire mirrors just the fields of the SearXNG JSON reply we read.
|
||||
type wire struct {
|
||||
Answers []json.RawMessage `json:"answers"`
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const sampleJSON = `{
|
||||
@@ -143,3 +144,39 @@ func TestSearchEmptyQuery(t *testing.T) {
|
||||
t.Fatal("empty query accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// A host that never answers a SYN must not hold the turn open for the whole
|
||||
// request budget: the ZIM behind this search is the answer, and he waits
|
||||
// through every second of the delay (V-508). 192.0.2.1 is TEST-NET-1, which is
|
||||
// reserved for documentation and routed nowhere.
|
||||
func TestSearchGivesUpOnAnUnreachableHostFast(t *testing.T) {
|
||||
c := New("http://192.0.2.1:9563", Options{Timeout: 8 * time.Second})
|
||||
start := time.Now()
|
||||
_, err := c.Search(context.Background(), "фотосинтез", 4)
|
||||
elapsed := time.Since(start)
|
||||
if err == nil {
|
||||
t.Fatal("Search reached 192.0.2.1; the address is routed nowhere")
|
||||
}
|
||||
if elapsed > 4*time.Second {
|
||||
t.Errorf("Search took %v to give up; the dial cap is %v", elapsed, dialTimeout)
|
||||
}
|
||||
}
|
||||
|
||||
// The dial cap must not shorten a request to an instance that did connect. A
|
||||
// slow SearXNG is fanning out to real engines, which is worth waiting for.
|
||||
func TestSlowButReachableInstanceKeepsTheFullBudget(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
time.Sleep(2 * dialTimeout)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte(`{"answers":[],"results":[{"title":"Фотосинтез","url":"http://x","content":"процесс","engine":"test"}]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
c := New(srv.URL, Options{Timeout: 8 * time.Second})
|
||||
resp, err := c.Search(context.Background(), "фотосинтез", 4)
|
||||
if err != nil {
|
||||
t.Fatalf("Search: %v", err)
|
||||
}
|
||||
if len(resp.Results) != 1 {
|
||||
t.Errorf("results = %d, want 1", len(resp.Results))
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user