Maven/internal/kiwix/client.go

// Package kiwix reads a local Kiwix server (offline Wikipedia and friends).
//
// Why: the resident model is a 0.8B and invents facts. Letting her read a local
// article snippet beats letting her recall. Nothing here talks to the internet;
// the Kiwix server is on the same box.
//
// This is search only. Full articles are ~100KB of HTML, far too big for a 4096
// token context, so the unit of context is the search snippet (~500 chars).
package kiwix

import (
	"context"
	"encoding/xml"
	"fmt"
	"html"
	"io"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// Result is one search hit.
type Result struct {
	Title     string // article title, e.g. "Rayleigh scattering"
	Path      string // e.g. /content/wikipedia_en_all_maxi_2026-02/Rayleigh_scattering
	Snippet   string // plain text, tags stripped, entities decoded
	WordCount int    // 0 if the server did not say
}

// Client is a Kiwix HTTP client. Boring on purpose: no retries, no cache.
type Client struct {
	base string
	http *http.Client
}

// New makes a client for a Kiwix base URL like http://127.0.0.1:8034.
func New(baseURL string) *Client {
	return &Client{
		base: strings.TrimRight(baseURL, "/"),
		http: &http.Client{Timeout: 10 * time.Second},
	}
}

// Search runs a keyword search in one ZIM (book) and returns up to limit hits.
//
// Ranking is keyword based, not semantic: "Rayleigh scattering" finds the right
// article, "why is the sky blue" finds a TV episode. Pass keywords, not questions.
func (c *Client) Search(ctx context.Context, pattern, book string, limit int) ([]Result, error) {
	if limit <= 0 {
		limit = 5
	}
	q := url.Values{}
	q.Set("pattern", pattern)
	q.Set("books.name", book)
	q.Set("format", "xml")
	q.Set("pageLength", strconv.Itoa(limit))

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.base+"/search?"+q.Encode(), nil)
	if err != nil {
		return nil, err
	}
	resp, err := c.http.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("kiwix search: http %d", resp.StatusCode)
	}
	return ParseSearchRSS(resp.Body)
}

// rss mirrors just the bits of the RSS 2.0 reply we use.
type rss struct {
	Items []struct {
		Title string `xml:"title"`
		Link  string `xml:"link"`
		// innerxml keeps the <b> match markers so we can strip them ourselves.
		Description struct {
			Inner string `xml:",innerxml"`
		} `xml:"description"`
		WordCount string `xml:"wordCount"`
	} `xml:"channel>item"`
}

var tagRE = regexp.MustCompile(`<[^>]*>`)

// ParseSearchRSS turns a Kiwix search reply into results. Exported so the parser
// is testable from a captured response, with no server running.
func ParseSearchRSS(r io.Reader) ([]Result, error) {
	var doc rss
	if err := xml.NewDecoder(r).Decode(&doc); err != nil {
		return nil, fmt.Errorf("kiwix search: bad xml: %w", err)
	}
	out := make([]Result, 0, len(doc.Items))
	for _, it := range doc.Items {
		n, _ := strconv.Atoi(strings.ReplaceAll(it.WordCount, ",", ""))
		out = append(out, Result{
			Title:     strings.TrimSpace(it.Title),
			Path:      strings.TrimSpace(it.Link),
			Snippet:   plainText(it.Description.Inner),
			WordCount: n,
		})
	}
	return out, nil
}

// plainText drops markup and decodes entities, leaving text a model can read.
func plainText(s string) string {
	s = tagRE.ReplaceAllString(s, "")
	s = html.UnescapeString(s)
	return strings.TrimSpace(strings.Join(strings.Fields(s), " "))
}