Maven/internal/websearch/searxng.go

// Package websearch reads a self-hosted SearXNG instance.
//
// Why this exists at all: "never phones home" stopped being a hard constraint
// on 2026-07-31. A 1.7B does not know enough to answer a world question, and
// reading beats recalling at that size. SearXNG is the reading surface for
// anything the offline ZIMs do not hold, and it is off unless configured.
//
// What is NOT here, on purpose:
//
//   - No query rewriting. SearXNG ranks with real engines, so the Russian
//     question goes out as he asked it. That is the whole reason it sits ahead
//     of Kiwix, whose keyword ranker needs kiwix.Rewriter to see anything.
//   - No page fetching. A snippet per result is the evidence; following a link
//     is crawl.Crawler's job and carries robots and allowlist rules with it.
//   - No cache and no retries. Boring on purpose, same posture as kiwix.Client.
//
// Only the query string leaves this process. This package cannot read the
// store, so his notes, facts, persona block and history cannot travel with a
// search even by accident. The personal boundary in the query chain is what
// keeps a question ABOUT him from becoming a query at all.
package websearch

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"
)

// Result is one search hit, already reduced to what a phraser can read.
type Result struct {
	Title   string
	URL     string
	Content string // the engine's snippet, plain text
	Engine  string // which upstream engine produced it, e.g. "duckduckgo"
}

// Response is one search. Answers comes from SearXNG's answerer plugins and
// from instant answers upstream; it is a direct reply to the question and is
// worth more than any snippet, so it is kept separate rather than mixed in.
type Response struct {
	Answers []string
	Results []Result
}

// Empty reports whether the search found nothing usable. The caller passes the
// turn on when it does — an empty search is not a failure worth announcing.
func (r Response) Empty() bool { return len(r.Answers) == 0 && len(r.Results) == 0 }

// DefaultTimeout — the whole request. SearXNG fans out to upstream engines and
// waits on the slowest, so this is longer than a LAN call but short enough that
// a dead engine does not hold a voice turn open.
const DefaultTimeout = 8 * time.Second

// dialTimeout — how long a connection to the instance may take before the turn
// gives up on it and falls through to the ZIM.
//
// It is separate from DefaultTimeout because the two failures are different
// (V-508). A reachable instance that is merely slow deserves the full budget:
// it is fanning out to real engines. A host that never answers a SYN deserves
// almost none, and the difference was measured. With the container stopped, DNS
// failed and the ZIM answered inside the same second. With the host blackholed,
// the search sat for the whole 8 seconds and the turn took 15.4 seconds instead
// of 3.5, which is a wait he sits through with nothing being said.
//
// The instance is on the LAN or the same host, so a connection it will ever
// accept is accepted in milliseconds.
const dialTimeout = 1500 * time.Millisecond

// maxBodyBytes caps the JSON read. A 20-result reply is tens of kilobytes; this
// is slack for a wide one and a hard stop against a misconfigured endpoint.
const maxBodyBytes = 4 << 20

// Client is a SearXNG HTTP client.
type Client struct {
	base     string
	language string
	engines  string
	http     *http.Client
}

// Options are the per-instance knobs, all optional.
type Options struct {
	// Language — SearXNG's `language` parameter, e.g. "ru" or "auto". Empty ⇒
	// the instance default.
	Language string
	// Engines — comma-separated engine names to restrict the search to. Empty ⇒
	// whatever the instance has enabled.
	Engines string
	// Timeout — per-request budget. 0 ⇒ DefaultTimeout.
	Timeout time.Duration
}

// New makes a client for a SearXNG base URL like http://searxng:9563.
//
// The instance must have the JSON format enabled (`search.formats: [html,
// json]` in its settings.yml); a stock install answers 403 to format=json and
// every search will fail with that status.
func New(baseURL string, opt Options) *Client {
	t := opt.Timeout
	if t <= 0 {
		t = DefaultTimeout
	}
	return &Client{
		base:     strings.TrimRight(baseURL, "/"),
		language: strings.TrimSpace(opt.Language),
		engines:  strings.TrimSpace(opt.Engines),
		http: &http.Client{
			Timeout: t,
			// Cloned from the default so the rest of the transport (proxy,
			// keep-alives, HTTP/2) keeps stock behaviour and only the dial
			// budget changes.
			Transport: dialCappedTransport(),
		},
	}
}

// Search runs one query and returns up to limit results plus any instant
// answers. The query goes out verbatim.
func (c *Client) Search(ctx context.Context, query string, limit int) (Response, error) {
	query = strings.TrimSpace(query)
	if query == "" {
		return Response{}, fmt.Errorf("websearch: empty query")
	}
	q := url.Values{}
	q.Set("q", query)
	q.Set("format", "json")
	if c.language != "" {
		q.Set("language", c.language)
	}
	if c.engines != "" {
		q.Set("engines", c.engines)
	}

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.base+"/search?"+q.Encode(), nil)
	if err != nil {
		return Response{}, err
	}
	resp, err := c.http.Do(req)
	if err != nil {
		return Response{}, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return Response{}, fmt.Errorf("websearch: http %d (json format enabled in searxng?)", resp.StatusCode)
	}
	body, err := io.ReadAll(io.LimitReader(resp.Body, maxBodyBytes))
	if err != nil {
		return Response{}, err
	}
	return ParseResponse(body, limit)
}

// dialCappedTransport is http.DefaultTransport with dialTimeout on the connect
// phase. A read that has already connected still gets the full request budget.
func dialCappedTransport() *http.Transport {
	tr := http.DefaultTransport.(*http.Transport).Clone()
	tr.DialContext = (&net.Dialer{Timeout: dialTimeout, KeepAlive: 30 * time.Second}).DialContext
	return tr
}

// wire mirrors just the fields of the SearXNG JSON reply we read.
type wire struct {
	Answers []json.RawMessage `json:"answers"`
	Results []struct {
		Title   string `json:"title"`
		URL     string `json:"url"`
		Content string `json:"content"`
		Engine  string `json:"engine"`
	} `json:"results"`
}

// ParseResponse turns a SearXNG JSON reply into a Response, keeping at most
// limit results. Exported so the parser is testable from a captured reply with
// no instance running.
func ParseResponse(body []byte, limit int) (Response, error) {
	var doc wire
	if err := json.Unmarshal(body, &doc); err != nil {
		return Response{}, fmt.Errorf("websearch: bad json: %w", err)
	}
	if limit <= 0 {
		limit = 5
	}
	out := Response{}
	for _, raw := range doc.Answers {
		if s := answerText(raw); s != "" {
			out.Answers = append(out.Answers, s)
		}
	}
	for _, r := range doc.Results {
		title := clean(r.Title)
		content := clean(r.Content)
		if title == "" && content == "" {
			// A hit with no text is a link with nothing to read. It cannot be
			// evidence, and counting it toward the limit would push a usable
			// snippet out of the reply.
			continue
		}
		out.Results = append(out.Results, Result{
			Title:   title,
			URL:     strings.TrimSpace(r.URL),
			Content: content,
			Engine:  strings.TrimSpace(r.Engine),
		})
		if len(out.Results) == limit {
			break
		}
	}
	return out, nil
}

// answerText reads one entry of `answers`. SearXNG changed its shape: older
// versions emit a bare string, newer ones an object with an `answer` field.
// Both are in the wild depending on when the instance was pulled, so both are
// read rather than pinning a version we do not control.
func answerText(raw json.RawMessage) string {
	var s string
	if err := json.Unmarshal(raw, &s); err == nil {
		return clean(s)
	}
	var obj struct {
		Answer string `json:"answer"`
	}
	if err := json.Unmarshal(raw, &obj); err == nil {
		return clean(obj.Answer)
	}
	return ""
}

// Snippets renders the response as evidence lines for a phraser: instant
// answers first, then "Title — snippet" per result.
//
// Answers lead because they are a reply to the question, where a result is a
// page that might contain one. The URL is deliberately left out: it is not
// evidence, and piper reads one out character by character.
func (r Response) Snippets() []string {
	out := make([]string, 0, len(r.Answers)+len(r.Results))
	out = append(out, r.Answers...)
	for _, res := range r.Results {
		switch {
		case res.Content == "":
			out = append(out, res.Title)
		case res.Title == "":
			out = append(out, res.Content)
		default:
			out = append(out, res.Title+" — "+res.Content)
		}
	}
	return out
}

// clean collapses whitespace. Snippets arrive with newlines and runs of spaces
// from the upstream page, and piper reads a reply built out of them badly.
func clean(s string) string { return strings.Join(strings.Fields(s), " ") }