Maven/internal/kiwix/client.go

// Package kiwix reads a local Kiwix server (offline Wikipedia and friends).
//
// Why: the resident model is a 0.8B and invents facts. Letting her read a local
// article snippet beats letting her recall. Nothing here talks to the internet;
// the Kiwix server is on the same box.
//
// Search finds the article; Article reads it. The snippet a search returns is
// NOT usable context on its own — see the comment on Article — so the unit of
// context is the head of the article, truncated to fit a 4096 token window.
package kiwix

import (
	"context"
	"encoding/xml"
	"fmt"
	"html"
	"io"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/kami/maven/internal/crawl"
)

// Result is one search hit.
type Result struct {
	Title     string // article title, e.g. "Rayleigh scattering"
	Path      string // e.g. /content/wikipedia_en_all_maxi_2026-02/Rayleigh_scattering
	Snippet   string // plain text, tags stripped, entities decoded
	WordCount int    // 0 if the server did not say
}

// Client is a Kiwix HTTP client. Boring on purpose: no retries, no cache.
type Client struct {
	base string
	http *http.Client
}

// New makes a client for a Kiwix base URL like http://127.0.0.1:8034.
func New(baseURL string) *Client {
	return &Client{
		base: strings.TrimRight(baseURL, "/"),
		http: &http.Client{Timeout: 10 * time.Second},
	}
}

// Search runs a keyword search in one ZIM (book) and returns up to limit hits.
//
// Ranking is keyword based, not semantic: "Rayleigh scattering" finds the right
// article, "why is the sky blue" finds a TV episode. Pass keywords, not questions.
func (c *Client) Search(ctx context.Context, pattern, book string, limit int) ([]Result, error) {
	if limit <= 0 {
		limit = 5
	}
	q := url.Values{}
	q.Set("pattern", pattern)
	q.Set("books.name", book)
	q.Set("format", "xml")
	q.Set("pageLength", strconv.Itoa(limit))

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.base+"/search?"+q.Encode(), nil)
	if err != nil {
		return nil, err
	}
	resp, err := c.http.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("kiwix search: http %d", resp.StatusCode)
	}
	return ParseSearchRSS(resp.Body)
}

// articleMaxBytes — how much of an article HTML document is read before the
// rest is discarded. A maxi Wikipedia page is around 100KB; 512KB is slack for
// the long ones and a hard stop against a ZIM entry that is really a binary.
const articleMaxBytes = 512 << 10

// Article fetches one article by the Path a search hit carries and returns it
// as extracted plain text, capped at maxRunes (0 ⇒ crawl.DefaultMaxRunes).
//
// This exists because the search snippet is not usable context. Kiwix builds
// the snippet from wherever the keyword matched, and on a Wikipedia ZIM that is
// routinely the "see also" navigation box at the foot of the page: a search for
// "photosynthesis" comes back with "Ecological economics Ecological footprint
// Ecological forecasting …" and a model handed that writes nothing worth
// hearing. The lead paragraphs are at the top of the document, so truncating an
// article from the front gets the definition the snippet was supposed to be.
//
// Nothing here reaches the internet: the path is resolved against the same
// server the search went to.
func (c *Client) Article(ctx context.Context, path string, maxRunes int) (crawl.Page, error) {
	path = strings.TrimSpace(path)
	if path == "" {
		return crawl.Page{}, fmt.Errorf("kiwix article: empty path")
	}
	if !strings.HasPrefix(path, "/") {
		path = "/" + path
	}
	u := c.base + path
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
	if err != nil {
		return crawl.Page{}, err
	}
	resp, err := c.http.Do(req)
	if err != nil {
		return crawl.Page{}, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return crawl.Page{}, fmt.Errorf("kiwix article %s: http %d", path, resp.StatusCode)
	}
	body, err := io.ReadAll(io.LimitReader(resp.Body, articleMaxBytes))
	if err != nil {
		return crawl.Page{}, err
	}
	return crawl.Extract(u, body, maxRunes), nil
}

// rss mirrors just the bits of the RSS 2.0 reply we use.
type rss struct {
	Items []struct {
		Title string `xml:"title"`
		Link  string `xml:"link"`
		// innerxml keeps the <b> match markers so we can strip them ourselves.
		Description struct {
			Inner string `xml:",innerxml"`
		} `xml:"description"`
		WordCount string `xml:"wordCount"`
	} `xml:"channel>item"`
}

var tagRE = regexp.MustCompile(`<[^>]*>`)

// ParseSearchRSS turns a Kiwix search reply into results. Exported so the parser
// is testable from a captured response, with no server running.
func ParseSearchRSS(r io.Reader) ([]Result, error) {
	var doc rss
	if err := xml.NewDecoder(r).Decode(&doc); err != nil {
		return nil, fmt.Errorf("kiwix search: bad xml: %w", err)
	}
	out := make([]Result, 0, len(doc.Items))
	for _, it := range doc.Items {
		n, _ := strconv.Atoi(strings.ReplaceAll(it.WordCount, ",", ""))
		out = append(out, Result{
			Title:     strings.TrimSpace(it.Title),
			Path:      strings.TrimSpace(it.Link),
			Snippet:   plainText(it.Description.Inner),
			WordCount: n,
		})
	}
	return out, nil
}

// plainText drops markup and decodes entities, leaving text a model can read.
func plainText(s string) string {
	s = tagRE.ReplaceAllString(s, "")
	s = html.UnescapeString(s)
	return strings.TrimSpace(strings.Join(strings.Fields(s), " "))
}