diff --git a/CLAUDE.md b/CLAUDE.md index 5fcbe2a..dab7366 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -188,8 +188,12 @@ world questions, so she needs to read external sources. What replaces it: - **No telemetry, no cloud model, no third-party account.** That part never changes. Nothing about Maven is reported to anyone, and inference stays on the box. -- **Local sources first.** Kiwix ZIMs on homesrv (Wikipedia, ifixit) before anything on the - network. Reading beats recalling for a small model, and a local read costs nothing. +- **His data first, then the world.** Every source that reads his facts, notes, calendar, + tasks or house runs before anything outside, and the personal boundary sits between them. + Reading beats recalling for a small model. +- **In the world, live search leads and the ZIMs are the fallback** (owner's call, + 2026-08-02). A self-hosted SearXNG (`search` block) answers first; the Kiwix ZIMs on + homesrv answer when the search is empty, unreachable, or the line is down. - **External search is allowed and off unless configured**, like the weather and telegram capabilities. - **His notes and facts are never search input.** Looking up why the sky is blue and sending diff --git a/cmd/mavend/actions_query.go b/cmd/mavend/actions_query.go index e7888b0..9ce82f2 100644 --- a/cmd/mavend/actions_query.go +++ b/cmd/mavend/actions_query.go @@ -112,14 +112,18 @@ var querySources = []querySource{ // has no answer in his data, and no outside source can supply one, so this // stops the walk rather than let the encyclopedia and the model guess. {name: "personal", answer: (*reactiveHandler).queryPersonal}, - // The offline encyclopedia, after everything of his and before anything on - // the network. A question he can be answered from his own notes is answered - // from his own notes; only what is left over is looked up. + // The world, read live. Owner's ruling of 2026-08-02: a metasearch hit beats + // a frozen ZIM, so SearXNG asks before Kiwix does. Nothing of his is at + // stake by this point — the boundary above already stopped every question + // about him, and only the query string leaves the box. + {name: "search", answer: (*reactiveHandler).querySearch}, + // The offline encyclopedia, now the fallback for when the line is down or + // the search comes back empty. It reads the way it always did; what changed + // is that it no longer gets first refusal on a world question. {name: "kiwix", answer: (*reactiveHandler).queryKiwix}, // LAST before the model answers from memory, and that position is the whole - // design (Vikunja #259): local sources first. His memory, his notes and the - // offline ZIMs all get their turn before anything touches the network. - // The model does NOT: it + // design (Vikunja #259): everything of his, then the search, then the ZIMs, + // and only then a page he named. The model does NOT come first: it // answers after this, because a URL he said out loud is an instruction and // a 1.7B guessing at a page it cannot read is how contents get invented. // This source only claims a turn where he named a URL, so it never competes @@ -537,9 +541,76 @@ func (h *reactiveHandler) queryWeb(ctx context.Context, t *queryTurn) (string, b // model's own answer than by more waiting. const kiwixTimeout = 20 * time.Second -// queryKiwix — the offline encyclopedia. The last local source: everything of -// his has already had its turn and found nothing, so the question is a world -// question, and reading beats recalling for a 1.7B. +// searchTimeout — the whole metasearch source. websearch.Client already holds a +// per-request timeout from config; this is the outer bound on the turn, so a +// hung dial cannot outlive it either. Shorter than kiwixTimeout because there +// is no rewrite call in front of it: the question goes out verbatim. +const searchTimeout = 12 * time.Second + +// querySearch — the live web, through a self-hosted SearXNG. +// +// Ahead of Kiwix by the owner's ruling of 2026-08-02: a search reads what is +// true today, a ZIM reads what was true when it was built, and the ZIM is the +// fallback for a box with no line out. Everything of his still answers first — +// the personal boundary is directly above this source, so a question ABOUT him +// never becomes a query. +// +// What leaves this process is the query string and nothing else. His notes, his +// facts, the persona block and the history do not travel with it: the websearch +// package cannot read the store. That is the CLAUDE.md rule made mechanical, +// not a promise about how the prompt is assembled. +// +// It claims the turn only when the search returns something. An empty result, +// an unreachable instance and a 403 from an instance without the JSON format +// all fall through to Kiwix, which is the point of the ordering. +func (h *reactiveHandler) querySearch(ctx context.Context, t *queryTurn) (string, bool) { + if h.search == nil { + // Off unless configured, same as the crawler and the ZIMs. Nothing is + // said about it: he never asked for a capability he did not enable. + return "", false + } + ctxS, cancel := context.WithTimeout(ctx, searchTimeout) + defer cancel() + + // Verbatim. No rewriter: SearXNG ranks by meaning through real engines, and + // reducing "почему небо голубое" to English keywords would throw away the + // language he asked in along with the ranking that handles it. + resp, err := h.search.client.Search(ctxS, t.dec.Utterance, h.search.max) + if err != nil { + log.Printf("voice: search %q: %v", t.dec.Utterance, err) + return "", false + } + if resp.Empty() { + return "", false + } + // Logged on the way through, not only on failure. Without this there is no + // telling from the outside whether an answer came off the web, off a ZIM or + // out of the model's weights, and those are the cases worth telling apart. + log.Printf("voice: search: %q → %d answers, %d results", t.dec.Utterance, len(resp.Answers), len(resp.Results)) + + // Handed over the same way a note, a page or an article is: evidence for the + // question he asked, not something to recite. The trim is one budget over the + // joined block, so a long first snippet cannot crowd out the rest. + evidence := crawl.TrimRunes(strings.Join(resp.Snippets(), "\n"), h.search.runes) + var reply string + if h.phraser != nil { + var perr error + reply, perr = h.phraser.PhraseQuery(ctx, t.dec.Utterance, []string{evidence}) + if perr != nil { + log.Printf("voice: search: phrase: %v", perr) + } + } + if reply == "" { + // No phraser, or it failed. Read back the best evidence rather than + // pretend the search did not happen. + return "вот что я нашла: " + crawl.TrimRunes(resp.Snippets()[0], 300), true + } + return reply, true +} + +// queryKiwix — the offline encyclopedia, and the fallback behind querySearch: +// everything of his has already had its turn and the live search found nothing +// or could not be reached. Reading beats recalling for a 1.7B either way. // // What leaves this process is the search query and nothing else. His notes, // his facts, the persona block and the history do not travel with it — the diff --git a/cmd/mavend/actions_query_personal_test.go b/cmd/mavend/actions_query_personal_test.go index e2547d9..23d01e9 100644 --- a/cmd/mavend/actions_query_personal_test.go +++ b/cmd/mavend/actions_query_personal_test.go @@ -101,7 +101,7 @@ func TestPersonalBoundarySitsBetweenHisDataAndTheWorld(t *testing.T) { t.Errorf("%q reads his own data and must run before the personal boundary", his) } } - for _, world := range []string{"kiwix", "web", "general-knowledge"} { + for _, world := range []string{"search", "kiwix", "web", "general-knowledge"} { if i, ok := idx[world]; !ok || i < boundary { t.Errorf("%q reads the world and must run after the personal boundary", world) } diff --git a/cmd/mavend/actions_query_search_test.go b/cmd/mavend/actions_query_search_test.go new file mode 100644 index 0000000..f406c14 --- /dev/null +++ b/cmd/mavend/actions_query_search_test.go @@ -0,0 +1,105 @@ +package main + +import ( + "context" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/kami/maven/internal/router" + "github.com/kami/maven/internal/websearch" +) + +func searchHandler(t *testing.T, body string, status int) (*reactiveHandler, *string) { + t.Helper() + var seen string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + seen = r.URL.RawQuery + if status != http.StatusOK { + http.Error(w, "no", status) + return + } + w.Write([]byte(body)) + })) + t.Cleanup(srv.Close) + return &reactiveHandler{ + // No phraser: querySearch then reads back the best evidence, which is + // what makes the claim visible without a llama-server in the test. + search: &searchWiring{client: websearch.New(srv.URL, websearch.Options{}), max: 3, runes: 1500}, + }, &seen +} + +const searchBody = `{"answers":["Небо голубое из-за рэлеевского рассеяния."], +"results":[{"title":"Рэлеевское рассеяние","url":"https://ru.wikipedia.org/x","content":"Рассеяние света."}]}` + +func TestQuerySearchClaimsAndReadsBack(t *testing.T) { + h, _ := searchHandler(t, searchBody, http.StatusOK) + reply, ok := h.querySearch(context.Background(), &queryTurn{ + dec: router.Decision{Intent: router.IntentQuery, Utterance: "почему небо голубое"}, + }) + if !ok { + t.Fatal("querySearch passed on a search with hits") + } + if !strings.Contains(reply, "рэлеевского рассеяния") { + t.Fatalf("reply = %q", reply) + } +} + +// No rewriter in front of this source: SearXNG ranks by meaning, and reducing +// the question to English keywords would throw away the language he asked in. +func TestQuerySearchSendsTheQuestionVerbatim(t *testing.T) { + h, seen := searchHandler(t, searchBody, http.StatusOK) + h.querySearch(context.Background(), &queryTurn{ + dec: router.Decision{Intent: router.IntentQuery, Utterance: "почему небо голубое"}, + }) + if !strings.Contains(*seen, "q="+url.QueryEscape("почему небо голубое")) { + t.Fatalf("query string = %q", *seen) + } +} + +// The whole reason the ordering is safe: an unreachable or empty instance +// passes the turn to Kiwix instead of claiming it with an apology. +func TestQuerySearchFallsThroughWhenItFails(t *testing.T) { + for _, tc := range []struct { + name string + body string + status int + }{ + {"http error", "", http.StatusForbidden}, + {"no hits", `{"answers":[],"results":[]}`, http.StatusOK}, + } { + t.Run(tc.name, func(t *testing.T) { + h, _ := searchHandler(t, tc.body, tc.status) + if _, ok := h.querySearch(context.Background(), &queryTurn{ + dec: router.Decision{Intent: router.IntentQuery, Utterance: "почему небо голубое"}, + }); ok { + t.Fatal("querySearch claimed the turn; Kiwix never got its fallback") + } + }) + } +} + +// Off unless configured, and silent about it: he never asked for a capability +// he did not enable. +func TestQuerySearchOffWithoutConfig(t *testing.T) { + h := &reactiveHandler{} + if _, ok := h.querySearch(context.Background(), &queryTurn{ + dec: router.Decision{Intent: router.IntentQuery, Utterance: "почему небо голубое"}, + }); ok { + t.Fatal("querySearch claimed a turn with no search block") + } +} + +// The owner's ruling of 2026-08-02: the live search asks first, the ZIM is the +// fallback for a box with no line out. +func TestSearchRunsBeforeKiwix(t *testing.T) { + idx := map[string]int{} + for i, s := range querySources { + idx[s.name] = i + } + if idx["search"] > idx["kiwix"] { + t.Fatalf("search at %d, kiwix at %d: the ZIM is the fallback, not the first read", idx["search"], idx["kiwix"]) + } +} diff --git a/cmd/mavend/searchwire.go b/cmd/mavend/searchwire.go new file mode 100644 index 0000000..d81bb34 --- /dev/null +++ b/cmd/mavend/searchwire.go @@ -0,0 +1,41 @@ +package main + +import ( + "log" + "time" + + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/websearch" +) + +// searchWiring — the metasearch source, assembled. nil ⇒ off, which is the +// default: no `search` block, no query ever leaves the LAN. +// +// Thinner than kiwixWiring because there is nothing to rewrite. SearXNG ranks +// with real engines, so the question goes out as he asked it, and that is the +// reason this source sits ahead of the ZIMs rather than behind them. +type searchWiring struct { + client *websearch.Client + max int + runes int +} + +// wireSearch builds the search client from the `search` block, or returns nil +// when there is none. config.Normalise has already dropped a block with no URL +// and filled the two size defaults, so this does no validation of its own. +func wireSearch(cfg *config.Config) *searchWiring { + if cfg.Search == nil { + return nil + } + sc := cfg.Search + log.Printf("voice: web search at %s (language %q, engines %q)", sc.URL, sc.Language, sc.Engines) + return &searchWiring{ + client: websearch.New(sc.URL, websearch.Options{ + Language: sc.Language, + Engines: sc.Engines, + Timeout: time.Duration(sc.Timeout), + }), + max: sc.MaxResults, + runes: sc.SnippetRunes, + } +} diff --git a/cmd/mavend/voice.go b/cmd/mavend/voice.go index 3e697d9..f083e50 100644 --- a/cmd/mavend/voice.go +++ b/cmd/mavend/voice.go @@ -90,8 +90,14 @@ type reactiveHandler struct { // page reading is off, which is the default: no `crawl` block, no fetch. crawler *crawl.Crawler - // kiwix searches the offline ZIMs (queryKiwix), the last local source - // before the model answers from its own weights. nil ⇒ off, the default. + // search asks a self-hosted SearXNG (querySearch), the first world source + // once his own data has had its turn. nil ⇒ off, the default: no `search` + // block, no query ever leaves the LAN. + search *searchWiring + + // kiwix searches the offline ZIMs (queryKiwix), the fallback behind the + // live search and the last source before the model answers from its own + // weights. nil ⇒ off, the default. kiwix *kiwixWiring // feedsOn — whether any RSS feed is configured (config.Feeds). It changes diff --git a/cmd/mavend/voicewire.go b/cmd/mavend/voicewire.go index ddbb722..f382829 100644 --- a/cmd/mavend/voicewire.go +++ b/cmd/mavend/voicewire.go @@ -255,6 +255,9 @@ func wireVoice(cfg *config.Config, coreAPI ipc.CoreAPI, phr phraser.Phraser, mem // nil unless `crawl.on_demand` is on: reading a page he names is a // capability, and capabilities are off unless configured. crawler: onDemandCrawler(cfg), + // nil unless a `search` block names a SearXNG instance. External search + // is off unless configured, and configuring it is the whole opt-in. + search: wireSearch(cfg), // nil unless a `kiwix` block names a server. Same swap-aware client the // router and replier use, so the rewriter follows a model swap. kiwix: wireKiwix(cfg, llmClient), diff --git a/deploy/README.md b/deploy/README.md index fec729b..1fca50c 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -73,6 +73,36 @@ What it does and does not do: - feed notes are **not** part of recall. "что я говорил про X" searches what he said; headlines are read back only by asking about the feeds. +### Searching the web (`search`, also off by default) + +There is no `search` block either, so no query leaves the LAN. Switching it on +points her at a self-hosted SearXNG: + +```json +"search": { + "url": "http://searxng:8080", + "max_results": 4, + "snippet_runes": 1500, + "language": "auto", + "timeout": "8s" +} +``` + +- the instance needs `json` in its `search.formats` (settings.yml). A stock + SearXNG answers 403 to `format=json`, and then every search fails; +- the question goes out **verbatim**, in the language he asked it. There is no + rewriter here, unlike Kiwix: SearXNG ranks through real engines; +- only the query string leaves the box. `internal/websearch` cannot read the + store, so no note, fact, persona block or history can travel with a search; +- a question about him never becomes a query. The personal boundary in the + query chain stops the walk above this source; +- this runs **before** Kiwix. A live search reads what is true today and the + ZIMs read what was true when they were built, so the ZIMs are the fallback: + an empty result, an unreachable instance or a dead line falls through to + them and she never says the search failed; +- `engines` narrows the search to named engines, e.g. `"duckduckgo,wikipedia"`. + Empty means whatever the instance has enabled. + ### Reading a page (`crawl`, also off by default) There is no `crawl` block either, so no page is fetched. Two halves, separately @@ -95,9 +125,9 @@ switched: a fallback and not a habit; - `watches` re-reads a fixed list on its interval and writes a note when the text changed. Like the feeds, it announces nothing; -- the answer path sits behind his memory and his notes, and ahead of the model - answering from what it remembers. Kiwix is not wired into the chain yet. A - local read costs nothing, so anything local goes first; +- the answer path sits behind his memory, his notes, the web search and the + ZIMs, and ahead of the model answering from what it remembers. A page he + named is an instruction, so it is read last and only when he named one; - `robots.txt` is fetched first and obeyed with no override; a `Disallow` is a refusal she says out loud. `Crawl-delay` is waited out before the page is fetched, and a delay longer than the turn fails the read instead of hanging diff --git a/internal/config/config.go b/internal/config/config.go index 745a4c8..7f8bade 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -219,6 +219,11 @@ type Config struct { // / url empty ⇒ the query chain has no ZIM source. See KiwixConfig. Kiwix *KiwixConfig `json:"kiwix,omitempty"` + // Search — the SearXNG metasearch instance. nil / absent / url empty ⇒ the + // query chain has no web-search source and Kiwix is the only encyclopedia. + // See SearchConfig. + Search *SearchConfig `json:"search,omitempty"` + // Praxis — the ecosystem attention-state service. When configured, maven // calls the Praxis HTTP tools API for attention listing and item lifecycle. // Maven never touches Praxis's database directly (ecosystem invariant: no @@ -1100,6 +1105,57 @@ const ( DefaultKiwixSnippetRunes = 1500 ) +// SearchConfig — the self-hosted SearXNG instance she searches with. +// +// External search is allowed and off unless configured (CLAUDE.md). Configuring +// it is the whole opt-in: no `search` block, no query ever leaves the LAN. +// +// It sits AHEAD of Kiwix in the query chain, and that is the owner's ruling of +// 2026-08-02: a live search answers better than a frozen ZIM, and the ZIM is +// what she falls back to when the line is down. Everything of HIS still comes +// first — the personal boundary runs above both, so a question about him is +// never searched. +// +// Only the query string leaves the box. Notes, facts, the persona block and the +// history are never part of a request; internal/websearch cannot read the store. +type SearchConfig struct { + // URL — base address of the SearXNG instance, e.g. "http://searxng:8080". + // Empty ⇒ the whole block is normalised to nil and the source stays off. + // + // The instance needs `search.formats` to include `json` in its settings.yml. + // A stock install answers 403 to format=json, and then every search fails. + URL string `json:"url,omitempty"` + + // MaxResults — how many hits are kept as evidence. 0 ⇒ DefaultSearchResults. + // Small on purpose: the snippets share a 4096-token context with the persona + // block and the prompt. + MaxResults int `json:"max_results,omitempty"` + + // SnippetRunes — how much of the joined evidence reaches the phraser. + // 0 ⇒ DefaultSearchSnippetRunes. + SnippetRunes int `json:"snippet_runes,omitempty"` + + // Language — SearXNG's `language` parameter, e.g. "ru", "en" or "auto". + // Empty ⇒ the instance default. He asks in Russian and in English, so + // pinning one language here is usually the wrong call. + Language string `json:"language,omitempty"` + + // Engines — comma-separated engine names to restrict the search to, e.g. + // "duckduckgo,wikipedia". Empty ⇒ whatever the instance has enabled. + Engines string `json:"engines,omitempty"` + + // Timeout — per-search budget. 0 ⇒ websearch.DefaultTimeout. SearXNG waits + // on the slowest upstream engine, so this is the knob that decides how long + // a voice turn can stall on a bad network. + Timeout Duration `json:"timeout,omitempty"` +} + +// Search defaults, applied in Normalise. +const ( + DefaultSearchResults = 4 + DefaultSearchSnippetRunes = 1500 +) + // CrawlWatchConfig — one page kept an eye on. type CrawlWatchConfig struct { Name string `json:"name"` // note source is "crawl:" @@ -1431,6 +1487,19 @@ func (c *Config) applyDefaults() { } } + // Same rule for the metasearch instance: no address, nothing to search. + if c.Search != nil && strings.TrimSpace(c.Search.URL) == "" { + c.Search = nil + } + if c.Search != nil { + if c.Search.MaxResults <= 0 { + c.Search.MaxResults = DefaultSearchResults + } + if c.Search.SnippetRunes <= 0 { + c.Search.SnippetRunes = DefaultSearchSnippetRunes + } + } + if c.Voice != nil { if c.Voice.RouterThreshold <= 0 { c.Voice.RouterThreshold = DefaultRouterThreshold diff --git a/internal/websearch/searxng.go b/internal/websearch/searxng.go new file mode 100644 index 0000000..b5e5f92 --- /dev/null +++ b/internal/websearch/searxng.go @@ -0,0 +1,229 @@ +// Package websearch reads a self-hosted SearXNG instance. +// +// Why this exists at all: "never phones home" stopped being a hard constraint +// on 2026-07-31. A 1.7B does not know enough to answer a world question, and +// reading beats recalling at that size. SearXNG is the reading surface for +// anything the offline ZIMs do not hold, and it is off unless configured. +// +// What is NOT here, on purpose: +// +// - No query rewriting. SearXNG ranks with real engines, so the Russian +// question goes out as he asked it. That is the whole reason it sits ahead +// of Kiwix, whose keyword ranker needs kiwix.Rewriter to see anything. +// - No page fetching. A snippet per result is the evidence; following a link +// is crawl.Crawler's job and carries robots and allowlist rules with it. +// - No cache and no retries. Boring on purpose, same posture as kiwix.Client. +// +// Only the query string leaves this process. This package cannot read the +// store, so his notes, facts, persona block and history cannot travel with a +// search even by accident. The personal boundary in the query chain is what +// keeps a question ABOUT him from becoming a query at all. +package websearch + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" +) + +// Result is one search hit, already reduced to what a phraser can read. +type Result struct { + Title string + URL string + Content string // the engine's snippet, plain text + Engine string // which upstream engine produced it, e.g. "duckduckgo" +} + +// Response is one search. Answers comes from SearXNG's answerer plugins and +// from instant answers upstream; it is a direct reply to the question and is +// worth more than any snippet, so it is kept separate rather than mixed in. +type Response struct { + Answers []string + Results []Result +} + +// Empty reports whether the search found nothing usable. The caller passes the +// turn on when it does — an empty search is not a failure worth announcing. +func (r Response) Empty() bool { return len(r.Answers) == 0 && len(r.Results) == 0 } + +// DefaultTimeout — the whole request. SearXNG fans out to upstream engines and +// waits on the slowest, so this is longer than a LAN call but short enough that +// a dead engine does not hold a voice turn open. +const DefaultTimeout = 8 * time.Second + +// maxBodyBytes caps the JSON read. A 20-result reply is tens of kilobytes; this +// is slack for a wide one and a hard stop against a misconfigured endpoint. +const maxBodyBytes = 4 << 20 + +// Client is a SearXNG HTTP client. +type Client struct { + base string + language string + engines string + http *http.Client +} + +// Options are the per-instance knobs, all optional. +type Options struct { + // Language — SearXNG's `language` parameter, e.g. "ru" or "auto". Empty ⇒ + // the instance default. + Language string + // Engines — comma-separated engine names to restrict the search to. Empty ⇒ + // whatever the instance has enabled. + Engines string + // Timeout — per-request budget. 0 ⇒ DefaultTimeout. + Timeout time.Duration +} + +// New makes a client for a SearXNG base URL like http://searxng:8080. +// +// The instance must have the JSON format enabled (`search.formats: [html, +// json]` in its settings.yml); a stock install answers 403 to format=json and +// every search will fail with that status. +func New(baseURL string, opt Options) *Client { + t := opt.Timeout + if t <= 0 { + t = DefaultTimeout + } + return &Client{ + base: strings.TrimRight(baseURL, "/"), + language: strings.TrimSpace(opt.Language), + engines: strings.TrimSpace(opt.Engines), + http: &http.Client{Timeout: t}, + } +} + +// Search runs one query and returns up to limit results plus any instant +// answers. The query goes out verbatim. +func (c *Client) Search(ctx context.Context, query string, limit int) (Response, error) { + query = strings.TrimSpace(query) + if query == "" { + return Response{}, fmt.Errorf("websearch: empty query") + } + q := url.Values{} + q.Set("q", query) + q.Set("format", "json") + if c.language != "" { + q.Set("language", c.language) + } + if c.engines != "" { + q.Set("engines", c.engines) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.base+"/search?"+q.Encode(), nil) + if err != nil { + return Response{}, err + } + resp, err := c.http.Do(req) + if err != nil { + return Response{}, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return Response{}, fmt.Errorf("websearch: http %d (json format enabled in searxng?)", resp.StatusCode) + } + body, err := io.ReadAll(io.LimitReader(resp.Body, maxBodyBytes)) + if err != nil { + return Response{}, err + } + return ParseResponse(body, limit) +} + +// wire mirrors just the fields of the SearXNG JSON reply we read. +type wire struct { + Answers []json.RawMessage `json:"answers"` + Results []struct { + Title string `json:"title"` + URL string `json:"url"` + Content string `json:"content"` + Engine string `json:"engine"` + } `json:"results"` +} + +// ParseResponse turns a SearXNG JSON reply into a Response, keeping at most +// limit results. Exported so the parser is testable from a captured reply with +// no instance running. +func ParseResponse(body []byte, limit int) (Response, error) { + var doc wire + if err := json.Unmarshal(body, &doc); err != nil { + return Response{}, fmt.Errorf("websearch: bad json: %w", err) + } + if limit <= 0 { + limit = 5 + } + out := Response{} + for _, raw := range doc.Answers { + if s := answerText(raw); s != "" { + out.Answers = append(out.Answers, s) + } + } + for _, r := range doc.Results { + title := clean(r.Title) + content := clean(r.Content) + if title == "" && content == "" { + // A hit with no text is a link with nothing to read. It cannot be + // evidence, and counting it toward the limit would push a usable + // snippet out of the reply. + continue + } + out.Results = append(out.Results, Result{ + Title: title, + URL: strings.TrimSpace(r.URL), + Content: content, + Engine: strings.TrimSpace(r.Engine), + }) + if len(out.Results) == limit { + break + } + } + return out, nil +} + +// answerText reads one entry of `answers`. SearXNG changed its shape: older +// versions emit a bare string, newer ones an object with an `answer` field. +// Both are in the wild depending on when the instance was pulled, so both are +// read rather than pinning a version we do not control. +func answerText(raw json.RawMessage) string { + var s string + if err := json.Unmarshal(raw, &s); err == nil { + return clean(s) + } + var obj struct { + Answer string `json:"answer"` + } + if err := json.Unmarshal(raw, &obj); err == nil { + return clean(obj.Answer) + } + return "" +} + +// Snippets renders the response as evidence lines for a phraser: instant +// answers first, then "Title — snippet" per result. +// +// Answers lead because they are a reply to the question, where a result is a +// page that might contain one. The URL is deliberately left out: it is not +// evidence, and piper reads one out character by character. +func (r Response) Snippets() []string { + out := make([]string, 0, len(r.Answers)+len(r.Results)) + out = append(out, r.Answers...) + for _, res := range r.Results { + switch { + case res.Content == "": + out = append(out, res.Title) + case res.Title == "": + out = append(out, res.Content) + default: + out = append(out, res.Title+" — "+res.Content) + } + } + return out +} + +// clean collapses whitespace. Snippets arrive with newlines and runs of spaces +// from the upstream page, and piper reads a reply built out of them badly. +func clean(s string) string { return strings.Join(strings.Fields(s), " ") } diff --git a/internal/websearch/searxng_test.go b/internal/websearch/searxng_test.go new file mode 100644 index 0000000..144f21b --- /dev/null +++ b/internal/websearch/searxng_test.go @@ -0,0 +1,145 @@ +package websearch + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +const sampleJSON = `{ + "query": "почему небо голубое", + "answers": ["Rayleigh scattering makes the sky blue."], + "results": [ + {"title": "Рэлеевское рассеяние", "url": "https://ru.wikipedia.org/x", "content": "Рассеяние\n света на молекулах.", "engine": "wikipedia"}, + {"title": "", "url": "https://example.org/empty", "content": "", "engine": "duckduckgo"}, + {"title": "Why is the sky blue", "url": "https://example.org/2", "content": "Short answer.", "engine": "duckduckgo"} + ] +}` + +func TestParseResponse(t *testing.T) { + got, err := ParseResponse([]byte(sampleJSON), 5) + if err != nil { + t.Fatalf("parse: %v", err) + } + if len(got.Answers) != 1 || got.Answers[0] != "Rayleigh scattering makes the sky blue." { + t.Fatalf("answers = %#v", got.Answers) + } + // The textless middle hit is dropped: it is a link with nothing to read. + if len(got.Results) != 2 { + t.Fatalf("results = %#v", got.Results) + } + if got.Results[0].Content != "Рассеяние света на молекулах." { + t.Fatalf("whitespace not collapsed: %q", got.Results[0].Content) + } + if got.Empty() { + t.Fatal("Empty() on a response with hits") + } +} + +// The limit counts usable hits, not raw ones — a textless entry must not push a +// real snippet out of the reply. +func TestParseResponseLimitSkipsEmpty(t *testing.T) { + got, err := ParseResponse([]byte(sampleJSON), 2) + if err != nil { + t.Fatalf("parse: %v", err) + } + if len(got.Results) != 2 { + t.Fatalf("results = %d, want 2", len(got.Results)) + } + if got.Results[1].Title != "Why is the sky blue" { + t.Fatalf("second hit = %q", got.Results[1].Title) + } +} + +// Newer SearXNG emits answers as objects; older ones as bare strings. Both are +// in the wild and both must read. +func TestParseResponseObjectAnswers(t *testing.T) { + got, err := ParseResponse([]byte(`{"answers":[{"answer":"42","url":"x"}],"results":[]}`), 5) + if err != nil { + t.Fatalf("parse: %v", err) + } + if len(got.Answers) != 1 || got.Answers[0] != "42" { + t.Fatalf("answers = %#v", got.Answers) + } +} + +func TestResponseEmpty(t *testing.T) { + got, err := ParseResponse([]byte(`{"answers":[],"results":[]}`), 5) + if err != nil { + t.Fatalf("parse: %v", err) + } + if !got.Empty() { + t.Fatal("Empty() = false on a reply with nothing in it") + } +} + +func TestSnippetsAnswersFirst(t *testing.T) { + got, _ := ParseResponse([]byte(sampleJSON), 5) + lines := got.Snippets() + if len(lines) != 3 { + t.Fatalf("lines = %#v", lines) + } + if lines[0] != "Rayleigh scattering makes the sky blue." { + t.Fatalf("answer did not lead: %q", lines[0]) + } + if !strings.Contains(lines[1], " — ") { + t.Fatalf("result line = %q", lines[1]) + } + // No URL travels into the evidence: piper reads one out character by + // character and it is not evidence anyway. + for _, l := range lines { + if strings.Contains(l, "http") { + t.Fatalf("url leaked into evidence: %q", l) + } + } +} + +// The query goes out verbatim, and the JSON format is always asked for. +func TestSearchRequest(t *testing.T) { + var gotQuery, gotFormat, gotLang, gotEngines string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotQuery = r.URL.Query().Get("q") + gotFormat = r.URL.Query().Get("format") + gotLang = r.URL.Query().Get("language") + gotEngines = r.URL.Query().Get("engines") + w.Write([]byte(sampleJSON)) + })) + defer srv.Close() + + c := New(srv.URL, Options{Language: "ru", Engines: "duckduckgo"}) + got, err := c.Search(context.Background(), "почему небо голубое", 3) + if err != nil { + t.Fatalf("search: %v", err) + } + if gotQuery != "почему небо голубое" { + t.Fatalf("query was rewritten: %q", gotQuery) + } + if gotFormat != "json" || gotLang != "ru" || gotEngines != "duckduckgo" { + t.Fatalf("format=%q language=%q engines=%q", gotFormat, gotLang, gotEngines) + } + if len(got.Results) != 2 { + t.Fatalf("results = %#v", got.Results) + } +} + +// A stock SearXNG answers 403 to format=json. The error must say so, because +// that is the one misconfiguration this client cannot work around. +func TestSearchHTTPError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "forbidden", http.StatusForbidden) + })) + defer srv.Close() + + _, err := New(srv.URL, Options{}).Search(context.Background(), "x", 3) + if err == nil || !strings.Contains(err.Error(), "403") { + t.Fatalf("err = %v", err) + } +} + +func TestSearchEmptyQuery(t *testing.T) { + if _, err := New("http://example.invalid", Options{}).Search(context.Background(), " ", 3); err == nil { + t.Fatal("empty query accepted") + } +}