// Package kiwix reads a local Kiwix server (offline Wikipedia and friends). // // Why: the resident model is a 0.8B and invents facts. Letting her read a local // article snippet beats letting her recall. Nothing here talks to the internet; // the Kiwix server is on the same box. // // This is search only. Full articles are ~100KB of HTML, far too big for a 4096 // token context, so the unit of context is the search snippet (~500 chars). package kiwix import ( "context" "encoding/xml" "fmt" "html" "io" "net/http" "net/url" "regexp" "strconv" "strings" "time" ) // Result is one search hit. type Result struct { Title string // article title, e.g. "Rayleigh scattering" Path string // e.g. /content/wikipedia_en_all_maxi_2026-02/Rayleigh_scattering Snippet string // plain text, tags stripped, entities decoded WordCount int // 0 if the server did not say } // Client is a Kiwix HTTP client. Boring on purpose: no retries, no cache. type Client struct { base string http *http.Client } // New makes a client for a Kiwix base URL like http://127.0.0.1:8034. func New(baseURL string) *Client { return &Client{ base: strings.TrimRight(baseURL, "/"), http: &http.Client{Timeout: 10 * time.Second}, } } // Search runs a keyword search in one ZIM (book) and returns up to limit hits. // // Ranking is keyword based, not semantic: "Rayleigh scattering" finds the right // article, "why is the sky blue" finds a TV episode. Pass keywords, not questions. func (c *Client) Search(ctx context.Context, pattern, book string, limit int) ([]Result, error) { if limit <= 0 { limit = 5 } q := url.Values{} q.Set("pattern", pattern) q.Set("books.name", book) q.Set("format", "xml") q.Set("pageLength", strconv.Itoa(limit)) req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.base+"/search?"+q.Encode(), nil) if err != nil { return nil, err } resp, err := c.http.Do(req) if err != nil { return nil, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("kiwix search: http %d", resp.StatusCode) } return ParseSearchRSS(resp.Body) } // rss mirrors just the bits of the RSS 2.0 reply we use. type rss struct { Items []struct { Title string `xml:"title"` Link string `xml:"link"` // innerxml keeps the match markers so we can strip them ourselves. Description struct { Inner string `xml:",innerxml"` } `xml:"description"` WordCount string `xml:"wordCount"` } `xml:"channel>item"` } var tagRE = regexp.MustCompile(`<[^>]*>`) // ParseSearchRSS turns a Kiwix search reply into results. Exported so the parser // is testable from a captured response, with no server running. func ParseSearchRSS(r io.Reader) ([]Result, error) { var doc rss if err := xml.NewDecoder(r).Decode(&doc); err != nil { return nil, fmt.Errorf("kiwix search: bad xml: %w", err) } out := make([]Result, 0, len(doc.Items)) for _, it := range doc.Items { n, _ := strconv.Atoi(strings.ReplaceAll(it.WordCount, ",", "")) out = append(out, Result{ Title: strings.TrimSpace(it.Title), Path: strings.TrimSpace(it.Link), Snippet: plainText(it.Description.Inner), WordCount: n, }) } return out, nil } // plainText drops markup and decodes entities, leaving text a model can read. func plainText(s string) string { s = tagRE.ReplaceAllString(s, "") s = html.UnescapeString(s) return strings.TrimSpace(strings.Join(strings.Fields(s), " ")) }