query: let her search the live web before she reads the ZIMs
The offline encyclopedia was the only world source, and it reads what was true when the ZIM was built. A self-hosted SearXNG now asks first and Kiwix is the fallback for an empty result, an unreachable instance or no line out. Owner's ruling, 2026-08-02. internal/websearch is deliberately thin: no rewriter (SearXNG ranks through real engines, so the Russian question goes out as he asked it), no page fetch, no cache. It cannot read the store, so only the query string can leave the box. The personal boundary is unchanged and still sits above this source, so a question about him is never searched. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BFaeSbLMEVG5ey8tejU3y2
This commit is contained in:
kami
@@ -219,6 +219,11 @@ type Config struct {
|
||||
// / url empty ⇒ the query chain has no ZIM source. See KiwixConfig.
|
||||
Kiwix *KiwixConfig `json:"kiwix,omitempty"`
|
||||
|
||||
// Search — the SearXNG metasearch instance. nil / absent / url empty ⇒ the
|
||||
// query chain has no web-search source and Kiwix is the only encyclopedia.
|
||||
// See SearchConfig.
|
||||
Search *SearchConfig `json:"search,omitempty"`
|
||||
|
||||
// Praxis — the ecosystem attention-state service. When configured, maven
|
||||
// calls the Praxis HTTP tools API for attention listing and item lifecycle.
|
||||
// Maven never touches Praxis's database directly (ecosystem invariant: no
|
||||
@@ -1100,6 +1105,57 @@ const (
|
||||
DefaultKiwixSnippetRunes = 1500
|
||||
)
|
||||
|
||||
// SearchConfig — the self-hosted SearXNG instance she searches with.
|
||||
//
|
||||
// External search is allowed and off unless configured (CLAUDE.md). Configuring
|
||||
// it is the whole opt-in: no `search` block, no query ever leaves the LAN.
|
||||
//
|
||||
// It sits AHEAD of Kiwix in the query chain, and that is the owner's ruling of
|
||||
// 2026-08-02: a live search answers better than a frozen ZIM, and the ZIM is
|
||||
// what she falls back to when the line is down. Everything of HIS still comes
|
||||
// first — the personal boundary runs above both, so a question about him is
|
||||
// never searched.
|
||||
//
|
||||
// Only the query string leaves the box. Notes, facts, the persona block and the
|
||||
// history are never part of a request; internal/websearch cannot read the store.
|
||||
type SearchConfig struct {
|
||||
// URL — base address of the SearXNG instance, e.g. "http://searxng:8080".
|
||||
// Empty ⇒ the whole block is normalised to nil and the source stays off.
|
||||
//
|
||||
// The instance needs `search.formats` to include `json` in its settings.yml.
|
||||
// A stock install answers 403 to format=json, and then every search fails.
|
||||
URL string `json:"url,omitempty"`
|
||||
|
||||
// MaxResults — how many hits are kept as evidence. 0 ⇒ DefaultSearchResults.
|
||||
// Small on purpose: the snippets share a 4096-token context with the persona
|
||||
// block and the prompt.
|
||||
MaxResults int `json:"max_results,omitempty"`
|
||||
|
||||
// SnippetRunes — how much of the joined evidence reaches the phraser.
|
||||
// 0 ⇒ DefaultSearchSnippetRunes.
|
||||
SnippetRunes int `json:"snippet_runes,omitempty"`
|
||||
|
||||
// Language — SearXNG's `language` parameter, e.g. "ru", "en" or "auto".
|
||||
// Empty ⇒ the instance default. He asks in Russian and in English, so
|
||||
// pinning one language here is usually the wrong call.
|
||||
Language string `json:"language,omitempty"`
|
||||
|
||||
// Engines — comma-separated engine names to restrict the search to, e.g.
|
||||
// "duckduckgo,wikipedia". Empty ⇒ whatever the instance has enabled.
|
||||
Engines string `json:"engines,omitempty"`
|
||||
|
||||
// Timeout — per-search budget. 0 ⇒ websearch.DefaultTimeout. SearXNG waits
|
||||
// on the slowest upstream engine, so this is the knob that decides how long
|
||||
// a voice turn can stall on a bad network.
|
||||
Timeout Duration `json:"timeout,omitempty"`
|
||||
}
|
||||
|
||||
// Search defaults, applied in Normalise.
|
||||
const (
|
||||
DefaultSearchResults = 4
|
||||
DefaultSearchSnippetRunes = 1500
|
||||
)
|
||||
|
||||
// CrawlWatchConfig — one page kept an eye on.
|
||||
type CrawlWatchConfig struct {
|
||||
Name string `json:"name"` // note source is "crawl:<name>"
|
||||
@@ -1431,6 +1487,19 @@ func (c *Config) applyDefaults() {
|
||||
}
|
||||
}
|
||||
|
||||
// Same rule for the metasearch instance: no address, nothing to search.
|
||||
if c.Search != nil && strings.TrimSpace(c.Search.URL) == "" {
|
||||
c.Search = nil
|
||||
}
|
||||
if c.Search != nil {
|
||||
if c.Search.MaxResults <= 0 {
|
||||
c.Search.MaxResults = DefaultSearchResults
|
||||
}
|
||||
if c.Search.SnippetRunes <= 0 {
|
||||
c.Search.SnippetRunes = DefaultSearchSnippetRunes
|
||||
}
|
||||
}
|
||||
|
||||
if c.Voice != nil {
|
||||
if c.Voice.RouterThreshold <= 0 {
|
||||
c.Voice.RouterThreshold = DefaultRouterThreshold
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
// Package websearch reads a self-hosted SearXNG instance.
|
||||
//
|
||||
// Why this exists at all: "never phones home" stopped being a hard constraint
|
||||
// on 2026-07-31. A 1.7B does not know enough to answer a world question, and
|
||||
// reading beats recalling at that size. SearXNG is the reading surface for
|
||||
// anything the offline ZIMs do not hold, and it is off unless configured.
|
||||
//
|
||||
// What is NOT here, on purpose:
|
||||
//
|
||||
// - No query rewriting. SearXNG ranks with real engines, so the Russian
|
||||
// question goes out as he asked it. That is the whole reason it sits ahead
|
||||
// of Kiwix, whose keyword ranker needs kiwix.Rewriter to see anything.
|
||||
// - No page fetching. A snippet per result is the evidence; following a link
|
||||
// is crawl.Crawler's job and carries robots and allowlist rules with it.
|
||||
// - No cache and no retries. Boring on purpose, same posture as kiwix.Client.
|
||||
//
|
||||
// Only the query string leaves this process. This package cannot read the
|
||||
// store, so his notes, facts, persona block and history cannot travel with a
|
||||
// search even by accident. The personal boundary in the query chain is what
|
||||
// keeps a question ABOUT him from becoming a query at all.
|
||||
package websearch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Result is one search hit, already reduced to what a phraser can read.
|
||||
type Result struct {
|
||||
Title string
|
||||
URL string
|
||||
Content string // the engine's snippet, plain text
|
||||
Engine string // which upstream engine produced it, e.g. "duckduckgo"
|
||||
}
|
||||
|
||||
// Response is one search. Answers comes from SearXNG's answerer plugins and
|
||||
// from instant answers upstream; it is a direct reply to the question and is
|
||||
// worth more than any snippet, so it is kept separate rather than mixed in.
|
||||
type Response struct {
|
||||
Answers []string
|
||||
Results []Result
|
||||
}
|
||||
|
||||
// Empty reports whether the search found nothing usable. The caller passes the
|
||||
// turn on when it does — an empty search is not a failure worth announcing.
|
||||
func (r Response) Empty() bool { return len(r.Answers) == 0 && len(r.Results) == 0 }
|
||||
|
||||
// DefaultTimeout — the whole request. SearXNG fans out to upstream engines and
|
||||
// waits on the slowest, so this is longer than a LAN call but short enough that
|
||||
// a dead engine does not hold a voice turn open.
|
||||
const DefaultTimeout = 8 * time.Second
|
||||
|
||||
// maxBodyBytes caps the JSON read. A 20-result reply is tens of kilobytes; this
|
||||
// is slack for a wide one and a hard stop against a misconfigured endpoint.
|
||||
const maxBodyBytes = 4 << 20
|
||||
|
||||
// Client is a SearXNG HTTP client.
|
||||
type Client struct {
|
||||
base string
|
||||
language string
|
||||
engines string
|
||||
http *http.Client
|
||||
}
|
||||
|
||||
// Options are the per-instance knobs, all optional.
|
||||
type Options struct {
|
||||
// Language — SearXNG's `language` parameter, e.g. "ru" or "auto". Empty ⇒
|
||||
// the instance default.
|
||||
Language string
|
||||
// Engines — comma-separated engine names to restrict the search to. Empty ⇒
|
||||
// whatever the instance has enabled.
|
||||
Engines string
|
||||
// Timeout — per-request budget. 0 ⇒ DefaultTimeout.
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// New makes a client for a SearXNG base URL like http://searxng:8080.
|
||||
//
|
||||
// The instance must have the JSON format enabled (`search.formats: [html,
|
||||
// json]` in its settings.yml); a stock install answers 403 to format=json and
|
||||
// every search will fail with that status.
|
||||
func New(baseURL string, opt Options) *Client {
|
||||
t := opt.Timeout
|
||||
if t <= 0 {
|
||||
t = DefaultTimeout
|
||||
}
|
||||
return &Client{
|
||||
base: strings.TrimRight(baseURL, "/"),
|
||||
language: strings.TrimSpace(opt.Language),
|
||||
engines: strings.TrimSpace(opt.Engines),
|
||||
http: &http.Client{Timeout: t},
|
||||
}
|
||||
}
|
||||
|
||||
// Search runs one query and returns up to limit results plus any instant
|
||||
// answers. The query goes out verbatim.
|
||||
func (c *Client) Search(ctx context.Context, query string, limit int) (Response, error) {
|
||||
query = strings.TrimSpace(query)
|
||||
if query == "" {
|
||||
return Response{}, fmt.Errorf("websearch: empty query")
|
||||
}
|
||||
q := url.Values{}
|
||||
q.Set("q", query)
|
||||
q.Set("format", "json")
|
||||
if c.language != "" {
|
||||
q.Set("language", c.language)
|
||||
}
|
||||
if c.engines != "" {
|
||||
q.Set("engines", c.engines)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.base+"/search?"+q.Encode(), nil)
|
||||
if err != nil {
|
||||
return Response{}, err
|
||||
}
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return Response{}, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return Response{}, fmt.Errorf("websearch: http %d (json format enabled in searxng?)", resp.StatusCode)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, maxBodyBytes))
|
||||
if err != nil {
|
||||
return Response{}, err
|
||||
}
|
||||
return ParseResponse(body, limit)
|
||||
}
|
||||
|
||||
// wire mirrors just the fields of the SearXNG JSON reply we read.
|
||||
type wire struct {
|
||||
Answers []json.RawMessage `json:"answers"`
|
||||
Results []struct {
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
Content string `json:"content"`
|
||||
Engine string `json:"engine"`
|
||||
} `json:"results"`
|
||||
}
|
||||
|
||||
// ParseResponse turns a SearXNG JSON reply into a Response, keeping at most
|
||||
// limit results. Exported so the parser is testable from a captured reply with
|
||||
// no instance running.
|
||||
func ParseResponse(body []byte, limit int) (Response, error) {
|
||||
var doc wire
|
||||
if err := json.Unmarshal(body, &doc); err != nil {
|
||||
return Response{}, fmt.Errorf("websearch: bad json: %w", err)
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = 5
|
||||
}
|
||||
out := Response{}
|
||||
for _, raw := range doc.Answers {
|
||||
if s := answerText(raw); s != "" {
|
||||
out.Answers = append(out.Answers, s)
|
||||
}
|
||||
}
|
||||
for _, r := range doc.Results {
|
||||
title := clean(r.Title)
|
||||
content := clean(r.Content)
|
||||
if title == "" && content == "" {
|
||||
// A hit with no text is a link with nothing to read. It cannot be
|
||||
// evidence, and counting it toward the limit would push a usable
|
||||
// snippet out of the reply.
|
||||
continue
|
||||
}
|
||||
out.Results = append(out.Results, Result{
|
||||
Title: title,
|
||||
URL: strings.TrimSpace(r.URL),
|
||||
Content: content,
|
||||
Engine: strings.TrimSpace(r.Engine),
|
||||
})
|
||||
if len(out.Results) == limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// answerText reads one entry of `answers`. SearXNG changed its shape: older
|
||||
// versions emit a bare string, newer ones an object with an `answer` field.
|
||||
// Both are in the wild depending on when the instance was pulled, so both are
|
||||
// read rather than pinning a version we do not control.
|
||||
func answerText(raw json.RawMessage) string {
|
||||
var s string
|
||||
if err := json.Unmarshal(raw, &s); err == nil {
|
||||
return clean(s)
|
||||
}
|
||||
var obj struct {
|
||||
Answer string `json:"answer"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &obj); err == nil {
|
||||
return clean(obj.Answer)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Snippets renders the response as evidence lines for a phraser: instant
|
||||
// answers first, then "Title — snippet" per result.
|
||||
//
|
||||
// Answers lead because they are a reply to the question, where a result is a
|
||||
// page that might contain one. The URL is deliberately left out: it is not
|
||||
// evidence, and piper reads one out character by character.
|
||||
func (r Response) Snippets() []string {
|
||||
out := make([]string, 0, len(r.Answers)+len(r.Results))
|
||||
out = append(out, r.Answers...)
|
||||
for _, res := range r.Results {
|
||||
switch {
|
||||
case res.Content == "":
|
||||
out = append(out, res.Title)
|
||||
case res.Title == "":
|
||||
out = append(out, res.Content)
|
||||
default:
|
||||
out = append(out, res.Title+" — "+res.Content)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// clean collapses whitespace. Snippets arrive with newlines and runs of spaces
|
||||
// from the upstream page, and piper reads a reply built out of them badly.
|
||||
func clean(s string) string { return strings.Join(strings.Fields(s), " ") }
|
||||
@@ -0,0 +1,145 @@
|
||||
package websearch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const sampleJSON = `{
|
||||
"query": "почему небо голубое",
|
||||
"answers": ["Rayleigh scattering makes the sky blue."],
|
||||
"results": [
|
||||
{"title": "Рэлеевское рассеяние", "url": "https://ru.wikipedia.org/x", "content": "Рассеяние\n света на молекулах.", "engine": "wikipedia"},
|
||||
{"title": "", "url": "https://example.org/empty", "content": "", "engine": "duckduckgo"},
|
||||
{"title": "Why is the sky blue", "url": "https://example.org/2", "content": "Short answer.", "engine": "duckduckgo"}
|
||||
]
|
||||
}`
|
||||
|
||||
func TestParseResponse(t *testing.T) {
|
||||
got, err := ParseResponse([]byte(sampleJSON), 5)
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if len(got.Answers) != 1 || got.Answers[0] != "Rayleigh scattering makes the sky blue." {
|
||||
t.Fatalf("answers = %#v", got.Answers)
|
||||
}
|
||||
// The textless middle hit is dropped: it is a link with nothing to read.
|
||||
if len(got.Results) != 2 {
|
||||
t.Fatalf("results = %#v", got.Results)
|
||||
}
|
||||
if got.Results[0].Content != "Рассеяние света на молекулах." {
|
||||
t.Fatalf("whitespace not collapsed: %q", got.Results[0].Content)
|
||||
}
|
||||
if got.Empty() {
|
||||
t.Fatal("Empty() on a response with hits")
|
||||
}
|
||||
}
|
||||
|
||||
// The limit counts usable hits, not raw ones — a textless entry must not push a
|
||||
// real snippet out of the reply.
|
||||
func TestParseResponseLimitSkipsEmpty(t *testing.T) {
|
||||
got, err := ParseResponse([]byte(sampleJSON), 2)
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if len(got.Results) != 2 {
|
||||
t.Fatalf("results = %d, want 2", len(got.Results))
|
||||
}
|
||||
if got.Results[1].Title != "Why is the sky blue" {
|
||||
t.Fatalf("second hit = %q", got.Results[1].Title)
|
||||
}
|
||||
}
|
||||
|
||||
// Newer SearXNG emits answers as objects; older ones as bare strings. Both are
|
||||
// in the wild and both must read.
|
||||
func TestParseResponseObjectAnswers(t *testing.T) {
|
||||
got, err := ParseResponse([]byte(`{"answers":[{"answer":"42","url":"x"}],"results":[]}`), 5)
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if len(got.Answers) != 1 || got.Answers[0] != "42" {
|
||||
t.Fatalf("answers = %#v", got.Answers)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResponseEmpty(t *testing.T) {
|
||||
got, err := ParseResponse([]byte(`{"answers":[],"results":[]}`), 5)
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if !got.Empty() {
|
||||
t.Fatal("Empty() = false on a reply with nothing in it")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSnippetsAnswersFirst(t *testing.T) {
|
||||
got, _ := ParseResponse([]byte(sampleJSON), 5)
|
||||
lines := got.Snippets()
|
||||
if len(lines) != 3 {
|
||||
t.Fatalf("lines = %#v", lines)
|
||||
}
|
||||
if lines[0] != "Rayleigh scattering makes the sky blue." {
|
||||
t.Fatalf("answer did not lead: %q", lines[0])
|
||||
}
|
||||
if !strings.Contains(lines[1], " — ") {
|
||||
t.Fatalf("result line = %q", lines[1])
|
||||
}
|
||||
// No URL travels into the evidence: piper reads one out character by
|
||||
// character and it is not evidence anyway.
|
||||
for _, l := range lines {
|
||||
if strings.Contains(l, "http") {
|
||||
t.Fatalf("url leaked into evidence: %q", l)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The query goes out verbatim, and the JSON format is always asked for.
|
||||
func TestSearchRequest(t *testing.T) {
|
||||
var gotQuery, gotFormat, gotLang, gotEngines string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotQuery = r.URL.Query().Get("q")
|
||||
gotFormat = r.URL.Query().Get("format")
|
||||
gotLang = r.URL.Query().Get("language")
|
||||
gotEngines = r.URL.Query().Get("engines")
|
||||
w.Write([]byte(sampleJSON))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := New(srv.URL, Options{Language: "ru", Engines: "duckduckgo"})
|
||||
got, err := c.Search(context.Background(), "почему небо голубое", 3)
|
||||
if err != nil {
|
||||
t.Fatalf("search: %v", err)
|
||||
}
|
||||
if gotQuery != "почему небо голубое" {
|
||||
t.Fatalf("query was rewritten: %q", gotQuery)
|
||||
}
|
||||
if gotFormat != "json" || gotLang != "ru" || gotEngines != "duckduckgo" {
|
||||
t.Fatalf("format=%q language=%q engines=%q", gotFormat, gotLang, gotEngines)
|
||||
}
|
||||
if len(got.Results) != 2 {
|
||||
t.Fatalf("results = %#v", got.Results)
|
||||
}
|
||||
}
|
||||
|
||||
// A stock SearXNG answers 403 to format=json. The error must say so, because
|
||||
// that is the one misconfiguration this client cannot work around.
|
||||
func TestSearchHTTPError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "forbidden", http.StatusForbidden)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
_, err := New(srv.URL, Options{}).Search(context.Background(), "x", 3)
|
||||
if err == nil || !strings.Contains(err.Error(), "403") {
|
||||
t.Fatalf("err = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchEmptyQuery(t *testing.T) {
|
||||
if _, err := New("http://example.invalid", Options{}).Search(context.Background(), " ", 3); err == nil {
|
||||
t.Fatal("empty query accepted")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user