diff --git a/cmd/mavend/actions_query.go b/cmd/mavend/actions_query.go index 5c3ef61..77d7609 100644 --- a/cmd/mavend/actions_query.go +++ b/cmd/mavend/actions_query.go @@ -92,10 +92,14 @@ var querySources = []querySource{ {"embed", (*reactiveHandler).queryEmbed}, {"memory", (*reactiveHandler).queryMemory}, {"notes", (*reactiveHandler).queryNotes}, + // The offline encyclopedia, after everything of his and before anything on + // the network. A question he can be answered from his own notes is answered + // from his own notes; only what is left over is looked up. + {"kiwix", (*reactiveHandler).queryKiwix}, // LAST before the model answers from memory, and that position is the whole - // design (Vikunja #259): local sources first. His memory, his notes and — - // once internal/kiwix is wired into this chain — the offline ZIMs all get - // their turn before anything touches the network. The model does NOT: it + // design (Vikunja #259): local sources first. His memory, his notes and the + // offline ZIMs all get their turn before anything touches the network. + // The model does NOT: it // answers after this, because a URL he said out loud is an instruction and // a 1.7B guessing at a page it cannot read is how contents get invented. // This source only claims a turn where he named a URL, so it never competes @@ -498,6 +502,101 @@ func (h *reactiveHandler) queryWeb(ctx context.Context, t *queryTurn) (string, b return reply, true } +// kiwixTimeout — the whole ZIM source, rewrite included. The rewrite is one +// short constrained completion and the search is a LAN request; if the pair +// takes longer than this something is wrong and he is better served by the +// model's own answer than by more waiting. +const kiwixTimeout = 20 * time.Second + + +// queryKiwix — the offline encyclopedia. The last local source: everything of +// his has already had its turn and found nothing, so the question is a world +// question, and reading beats recalling for a 1.7B. +// +// What leaves this process is the search query and nothing else. His notes, +// his facts, the persona block and the history do not travel with it — the +// kiwix package cannot read the store. That holds even though the server is on +// the LAN, because "local sources first" is not a licence to widen what a +// lookup is allowed to see. +// +// It claims the turn only when the search returns something. No results is not +// a failure worth announcing: it means the ZIM does not cover this, and the +// model answering next is the better outcome than "ничего не нашла". +func (h *reactiveHandler) queryKiwix(ctx context.Context, t *queryTurn) (string, bool) { + if h.kiwix == nil { + // Off unless configured, same as the crawler and the weather. Nothing + // is said about it: he never asked for a capability he did not enable. + return "", false + } + ctxK, cancel := context.WithTimeout(ctx, kiwixTimeout) + defer cancel() + + // The ZIMs are English and kiwix ranks by keyword overlap, not meaning, so + // a Russian sentence matches nothing at all. The rewriter turns it into a + // handful of English keywords with the resident model. + pattern := t.dec.Utterance + if h.kiwix.rewriter != nil { + q, err := h.kiwix.rewriter.Rewrite(ctxK, t.dec.Utterance) + if err != nil { + // Fall through to the verbatim question rather than give up. It + // will usually miss, and missing is a fall-through too. + log.Printf("voice: kiwix: rewrite: %v", err) + } else if q != "" { + pattern = q + } + } + + hits, err := h.kiwix.client.Search(ctxK, pattern, h.kiwix.book, h.kiwix.max) + if err != nil { + log.Printf("voice: kiwix: search %q: %v", pattern, err) + return "", false + } + if len(hits) == 0 { + return "", false + } + top := hits[0] + // Logged on the way through, not only on failure. Without this there is no + // way to tell from the outside whether an answer came off a ZIM or out of + // the model's weights, and those are the two cases worth telling apart. + log.Printf("voice: kiwix: %q → %d hits, top %q", pattern, len(hits), top.Title) + + // The top hit only, read as an article rather than as a snippet. Kiwix + // builds its snippet from wherever the keyword matched, which on Wikipedia + // is usually the navigation box at the foot of the page — the first version + // of this joined three of those and she recited "Ecological economics + // Ecological footprint …" at him. The head of the article is the lead + // paragraph, which is the definition the snippet was meant to be. + page, aerr := h.kiwix.client.Article(ctxK, top.Path, h.kiwix.runes) + if aerr != nil || page.Text == "" { + if aerr != nil { + log.Printf("voice: kiwix: article %s: %v", top.Path, aerr) + } + // The search did find something, so fall back to its snippet rather + // than throw the hit away. + if top.Snippet == "" { + return "", false + } + page = crawl.Page{Title: top.Title, Text: top.Snippet} + } + // Handed over the same way a note or a page is: context for the question he + // asked, not something to recite. + snippet := top.Title + "\n" + crawl.TrimRunes(page.Text, h.kiwix.runes) + var reply string + if h.phraser != nil { + var perr error + reply, perr = h.phraser.PhraseQuery(ctx, t.dec.Utterance, []string{snippet}) + if perr != nil { + log.Printf("voice: kiwix: phrase: %v", perr) + } + } + if reply == "" { + // No phraser, or it failed. Read back the best hit rather than pretend + // the search did not happen. + return "вот что я нашла: " + crawl.TrimRunes(top.Title+" — "+page.Text, 300), true + } + return reply, true +} + // queryGeneral — general knowledge from the phraser, the last source before // giving up. It always claims: either the model answers or Maven says she // doesn't know. diff --git a/cmd/mavend/kiwixwire.go b/cmd/mavend/kiwixwire.go new file mode 100644 index 0000000..84e3a5d --- /dev/null +++ b/cmd/mavend/kiwixwire.go @@ -0,0 +1,54 @@ +package main + +import ( + "log" + + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/kiwix" + "github.com/kami/maven/internal/llm" +) + +// kiwixWiring — the offline encyclopedia, assembled. nil ⇒ off, which is the +// default: the query chain simply has no ZIM source. +// +// The rewriter is separately optional. Searching without one is legal and +// mostly useless against English ZIMs, but it is the honest degraded mode when +// there is no llama-server to rewrite with, and it is what `rewrite: false` +// asks for. +type kiwixWiring struct { + client *kiwix.Client + rewriter *kiwix.Rewriter // nil ⇒ the question is searched verbatim + book string + max int + runes int +} + +// wireKiwix builds the ZIM reader from the `kiwix` block, or returns nil when +// there is none. config.Normalise has already dropped a block with no URL and +// filled the two size defaults, so this does no validation of its own. +// +// The llm client is the phraser's swap-aware one (llmClientFor), so a model +// swap re-points the rewriter with everything else. A nil client means there is +// no resident model at all; that degrades the rewriter, not the source. +func wireKiwix(cfg *config.Config, c *llm.Client) *kiwixWiring { + if cfg.Kiwix == nil { + return nil + } + kc := cfg.Kiwix + w := &kiwixWiring{ + client: kiwix.New(kc.URL), + book: kc.Book, + max: kc.MaxResults, + runes: kc.SnippetRunes, + } + switch { + case !kc.RewriteEnabled(): + log.Printf("voice: kiwix at %s (book %q, query rewriting off by config)", kc.URL, kc.Book) + case c == nil: + log.Printf("voice: kiwix at %s (book %q, no llama-server: searching questions verbatim)", kc.URL, kc.Book) + default: + w.rewriter = kiwix.NewRewriter(c) + log.Printf("voice: kiwix at %s (book %q)", kc.URL, kc.Book) + } + return w +} diff --git a/cmd/mavend/kiwixwire_test.go b/cmd/mavend/kiwixwire_test.go new file mode 100644 index 0000000..c8ca38d --- /dev/null +++ b/cmd/mavend/kiwixwire_test.go @@ -0,0 +1,187 @@ +package main + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/kiwix" + "github.com/kami/maven/internal/phraser" + "github.com/kami/maven/internal/router" + "github.com/kami/maven/internal/voice" +) + +// searchRSS is what kiwix-serve answers a /search with, trimmed to the fields +// ParseSearchRSS reads. +func searchRSS(items ...string) string { + return `` + + strings.Join(items, "") + `` +} + +func rssItem(title, snippet string) string { + return "" + title + "/x" + snippet + "" +} + +// stubKiwixServer answers every search with the given body and records the +// pattern it was asked for, so a test can assert on what left the process. +type stubKiwixServer struct { + *httptest.Server + lastPattern string + lastBook string +} + +func newStubKiwix(t *testing.T, body string, status int) *stubKiwixServer { + t.Helper() + s := &stubKiwixServer{} + s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if status != 0 && status != http.StatusOK { + w.WriteHeader(status) + return + } + // Two endpoints on one server: /search answers the RSS, everything else + // is an article read. Only the search is recorded — an article fetch + // carries no query string and would blank the assertions. + if r.URL.Path != "/search" { + w.Header().Set("Content-Type", "text/html") + _, _ = w.Write([]byte("Article

the lead paragraph

")) + return + } + s.lastPattern = r.URL.Query().Get("pattern") + s.lastBook = r.URL.Query().Get("books.name") + w.Header().Set("Content-Type", "application/xml") + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(s.Close) + return s +} + +// buildKiwixHandler wires the source with no rewriter: the question is searched +// verbatim, which keeps the assertion about what was sent unambiguous. +func buildKiwixHandler(base string) *reactiveHandler { + return &reactiveHandler{ + replier: voice.NewStubReplier(), + phraser: phraser.NewStub(), + kiwix: &kiwixWiring{ + client: kiwix.New(base), + book: "wikipedia_en_all_maxi", + max: config.DefaultKiwixResults, + runes: config.DefaultKiwixSnippetRunes, + }, + } +} + +func askKiwix(h *reactiveHandler, q string) (string, bool) { + return h.queryKiwix(context.Background(), &queryTurn{ + dec: router.Decision{Intent: router.IntentQuery, Utterance: q}, + }) +} + +// The default daemon has no `kiwix` block, and a source that is off must not +// claim the turn — the model answers next, exactly as it did before. +func TestQueryKiwixOffPassesThrough(t *testing.T) { + h := &reactiveHandler{replier: voice.NewStubReplier(), phraser: phraser.NewStub()} + if reply, ok := askKiwix(h, "почему небо синее?"); ok { + t.Errorf("an unconfigured kiwix claimed the turn: %q", reply) + } +} + +func TestQueryKiwixAnswersFromSnippets(t *testing.T) { + s := newStubKiwix(t, searchRSS(rssItem("Rayleigh scattering", "shorter wavelengths scatter more")), 0) + h := buildKiwixHandler(s.URL) + + reply, ok := askKiwix(h, "почему небо синее?") + if !ok { + t.Fatal("kiwix found a hit and did not claim the turn") + } + if reply == "" { + t.Error("claimed the turn with an empty reply") + } + if s.lastBook != "wikipedia_en_all_maxi" { + t.Errorf("books.name = %q, want the configured book", s.lastBook) + } +} + +// No hit is not a failure worth announcing: the ZIM does not cover it, and the +// model answering next beats "ничего не нашла". +func TestQueryKiwixNoHitsPassesThrough(t *testing.T) { + s := newStubKiwix(t, searchRSS(), 0) + if reply, ok := askKiwix(buildKiwixHandler(s.URL), "почему небо синее?"); ok { + t.Errorf("an empty result set claimed the turn: %q", reply) + } +} + +// A dead or misconfigured server must degrade to the model, not to an error +// spoken out loud. A turn never breaks on a capability. +func TestQueryKiwixServerErrorPassesThrough(t *testing.T) { + s := newStubKiwix(t, "", http.StatusBadRequest) + if reply, ok := askKiwix(buildKiwixHandler(s.URL), "почему небо синее?"); ok { + t.Errorf("a 400 claimed the turn: %q", reply) + } +} + +// The privacy rule in CLAUDE.md, asserted rather than assumed: only the +// utterance is searched. No note, no fact, no persona block travels with it. +func TestQueryKiwixSendsOnlyTheQuestion(t *testing.T) { + s := newStubKiwix(t, searchRSS(rssItem("X", "y")), 0) + h := buildKiwixHandler(s.URL) + // A turn carrying notes an earlier source already pulled. They must not + // reach the query string. + _, _ = h.queryKiwix(context.Background(), &queryTurn{ + dec: router.Decision{Intent: router.IntentQuery, Utterance: "почему небо синее?"}, + notes: []ipc.Note{{Text: "пароль от роутера hunter2"}}, + }) + if strings.Contains(s.lastPattern, "hunter2") { + t.Fatalf("a stored note leaked into the search query: %q", s.lastPattern) + } + if s.lastPattern != "почему небо синее?" { + t.Errorf("pattern = %q, want the utterance verbatim", s.lastPattern) + } +} + +func TestWireKiwixOffWithoutABlock(t *testing.T) { + if w := wireKiwix(&config.Config{}, nil); w != nil { + t.Error("wireKiwix built a source with no config block") + } +} + +// No llama-server means no rewriter, but the source still works: searching the +// question verbatim is the honest degraded mode, not a reason to stay dark. +func TestWireKiwixWithoutAnLLMHasNoRewriter(t *testing.T) { + w := wireKiwix(&config.Config{Kiwix: &config.KiwixConfig{ + URL: "http://kiwix:8080", Book: "b", MaxResults: 5, SnippetRunes: 1500, + }}, nil) + if w == nil { + t.Fatal("wireKiwix returned nil for a configured block") + } + if w.rewriter != nil { + t.Error("built a rewriter with no llm client") + } + if w.book != "b" { + t.Errorf("book = %q", w.book) + } +} + +// The whole point of reading the article: kiwix's own snippet is usually the +// navigation box at the foot of the page, so the lead paragraph must be what +// reaches the phraser. +func TestQueryKiwixReadsTheArticleNotTheSnippet(t *testing.T) { + junk := "Ecological economics Ecological footprint Ecological forecasting" + s := newStubKiwix(t, searchRSS(rssItem("Photosynthesis", junk)), 0) + h := buildKiwixHandler(s.URL) + h.phraser = nil // no phraser ⇒ the fallback reads back what it was given + + reply, ok := askKiwix(h, "что такое фотосинтез?") + if !ok { + t.Fatal("did not claim the turn") + } + if !strings.Contains(reply, "the lead paragraph") { + t.Errorf("reply did not come from the article: %q", reply) + } + if strings.Contains(reply, "Ecological economics") { + t.Errorf("recited the navigation-box snippet: %q", reply) + } +} diff --git a/cmd/mavend/voice.go b/cmd/mavend/voice.go index 7bc2190..5588332 100644 --- a/cmd/mavend/voice.go +++ b/cmd/mavend/voice.go @@ -87,6 +87,10 @@ type reactiveHandler struct { // page reading is off, which is the default: no `crawl` block, no fetch. crawler *crawl.Crawler + // kiwix searches the offline ZIMs (queryKiwix), the last local source + // before the model answers from its own weights. nil ⇒ off, the default. + kiwix *kiwixWiring + // feedsOn — whether any RSS feed is configured (config.Feeds). It changes // only what she SAYS when asked and nothing is there: "ленты не настроены" // instead of "ничего нового", which are different truths. diff --git a/cmd/mavend/voicewire.go b/cmd/mavend/voicewire.go index 1adf273..b5657e3 100644 --- a/cmd/mavend/voicewire.go +++ b/cmd/mavend/voicewire.go @@ -254,7 +254,10 @@ func wireVoice(cfg *config.Config, coreAPI ipc.CoreAPI, phr phraser.Phraser, mem netscan: w.netscan, // nil unless `crawl.on_demand` is on: reading a page he names is a // capability, and capabilities are off unless configured. - crawler: onDemandCrawler(cfg), + crawler: onDemandCrawler(cfg), + // nil unless a `kiwix` block names a server. Same swap-aware client the + // router and replier use, so the rewriter follows a model swap. + kiwix: wireKiwix(cfg, llmClient), weatherProvider: weatherProvider, weatherLocation: weatherLocation, memStore: memStore, diff --git a/deploy/mavend.json b/deploy/mavend.json index 79875df..372ff78 100644 --- a/deploy/mavend.json +++ b/deploy/mavend.json @@ -29,6 +29,24 @@ "proxy": "socks5://192.168.240.1:10808" }, + "//kiwix": [ + "The offline encyclopedia, searched after his own notes and before anything", + "on the network. kiwix-server publishes 8034 on loopback only, so a container", + "cannot reach it by address; it is attached to the maven_default network", + "instead and addressed by container name. That attachment is imperative and", + "does not survive recreating the kiwix stack — make it declarative there:", + " networks: [default, maven_default] # maven_default: external: true", + "The book is the catalog name from the /content/... href in", + "/catalog/v2/entries, not the display title. Others on the box:", + "ifixit_en_all_2025-06, devdocs_en_ansible_2025-10." + ], + "kiwix": { + "url": "http://kiwix-server:8080", + "book": "wikipedia_en_all_maxi_2026-02", + "max_results": 5, + "snippet_runes": 1500 + }, + "digest": { "enabled": true, "window": "30m", diff --git a/internal/config/config.go b/internal/config/config.go index 50da777..d04437d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -199,6 +199,10 @@ type Config struct { // fetches a page: not on request, not on a schedule. See CrawlConfig. Crawl *CrawlConfig `json:"crawl,omitempty"` + // Kiwix — the offline ZIM reader (Vikunja #122 neighbourhood). nil / absent + // / url empty ⇒ the query chain has no ZIM source. See KiwixConfig. + Kiwix *KiwixConfig `json:"kiwix,omitempty"` + // Praxis — the ecosystem attention-state service. When configured, maven // calls the Praxis HTTP tools API for attention listing and item lifecycle. // Maven never touches Praxis's database directly (ecosystem invariant: no @@ -1027,6 +1031,59 @@ type CrawlConfig struct { MaxRunes int `json:"max_runes,omitempty"` } +// KiwixConfig — the offline encyclopedia. A kiwix-serve instance holding ZIM +// archives (Wikipedia, ifixit, devdocs) on the LAN, searched before anything +// touches the network. Dark until configured, same as every other reach. +// +// This is the "local sources first" rule in CLAUDE.md made concrete: a 1.7B +// does not know enough to answer a world question, but it can read. A local +// read costs nothing and leaves the box only as far as the LAN. +// +// Only the rewritten search query leaves this process. His notes, facts, +// persona block and history are never part of a request. +type KiwixConfig struct { + // URL — base address of kiwix-serve, e.g. "http://kiwix:8080". Empty ⇒ the + // whole block is normalised to nil and the source stays off. + URL string `json:"url,omitempty"` + + // Book — the ZIM to search, by its catalog name, e.g. + // "wikipedia_en_all_maxi_2026-02". Take it from the /content/… href in + // /catalog/v2/entries; the display title is not the name. + // + // Required. kiwix-serve answers 400 to a search with an empty books.name, + // so a block without one is normalised to nil rather than left to fail one + // query at a time. + Book string `json:"book,omitempty"` + + // MaxResults — how many hits are asked for. 0 ⇒ DefaultKiwixResults. + // Only the top few reach the phraser regardless; the rest are context the + // snippet ranking throws away. + MaxResults int `json:"max_results,omitempty"` + + // SnippetRunes — how much of the joined snippets is handed to the phraser. + // 0 ⇒ DefaultKiwixSnippetRunes. Sized against the 4096-token context, which + // also holds the persona block and the prompt. + SnippetRunes int `json:"snippet_runes,omitempty"` + + // Rewrite — turn the Russian question into English keywords with the + // resident model before searching. The ZIMs are English and kiwix ranks by + // keyword, not meaning, so a Russian sentence matches nothing. Costs one + // short LLM call per query. Default true; set false only to measure the + // difference or when the books are Russian. + Rewrite *bool `json:"rewrite,omitempty"` +} + +// RewriteEnabled — Rewrite with its default applied. Absent ⇒ on. +func (k *KiwixConfig) RewriteEnabled() bool { + return k.Rewrite == nil || *k.Rewrite +} + +// Kiwix defaults, applied in Normalise. +const ( + DefaultKiwixResults = 5 + DefaultKiwixSnippetRunes = 1500 +) + // CrawlWatchConfig — one page kept an eye on. type CrawlWatchConfig struct { Name string `json:"name"` // note source is "crawl:" @@ -1345,6 +1402,19 @@ func (c *Config) applyDefaults() { c.Crawl = nil } + // Same rule for the ZIM reader: no address or no book, nothing to search. + if c.Kiwix != nil && (strings.TrimSpace(c.Kiwix.URL) == "" || strings.TrimSpace(c.Kiwix.Book) == "") { + c.Kiwix = nil + } + if c.Kiwix != nil { + if c.Kiwix.MaxResults <= 0 { + c.Kiwix.MaxResults = DefaultKiwixResults + } + if c.Kiwix.SnippetRunes <= 0 { + c.Kiwix.SnippetRunes = DefaultKiwixSnippetRunes + } + } + if c.Voice != nil { if c.Voice.RouterThreshold <= 0 { c.Voice.RouterThreshold = DefaultRouterThreshold diff --git a/internal/config/config_test.go b/internal/config/config_test.go index d1b10d0..fe275dd 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -367,3 +367,49 @@ func TestUpdateBlockValidatedAtStartup(t *testing.T) { t.Error("Load accepted an update block with no health_socket") } } + +// A kiwix block with no address, or no book, has nothing to search. +// kiwix-serve answers 400 to an empty books.name, so the block is dropped here +// rather than left to fail one query at a time. +func TestNormaliseDropsIncompleteKiwix(t *testing.T) { + for _, tc := range []struct { + name string + in *KiwixConfig + }{ + {"no url", &KiwixConfig{Book: "wikipedia_en_all_maxi"}}, + {"no book", &KiwixConfig{URL: "http://kiwix:8080"}}, + {"blank url", &KiwixConfig{URL: " ", Book: "b"}}, + } { + t.Run(tc.name, func(t *testing.T) { + c := &Config{Kiwix: tc.in} + c.applyDefaults() + if c.Kiwix != nil { + t.Errorf("kept an unusable kiwix block: %+v", c.Kiwix) + } + }) + } +} + +func TestNormaliseFillsKiwixDefaults(t *testing.T) { + c := &Config{Kiwix: &KiwixConfig{URL: "http://kiwix:8080", Book: "b"}} + c.applyDefaults() + if c.Kiwix == nil { + t.Fatal("dropped a complete kiwix block") + } + if c.Kiwix.MaxResults != DefaultKiwixResults { + t.Errorf("MaxResults = %d, want %d", c.Kiwix.MaxResults, DefaultKiwixResults) + } + if c.Kiwix.SnippetRunes != DefaultKiwixSnippetRunes { + t.Errorf("SnippetRunes = %d, want %d", c.Kiwix.SnippetRunes, DefaultKiwixSnippetRunes) + } + // Rewriting is on unless it is turned off: an English ZIM searched with a + // Russian sentence matches nothing, so the useful default is the on one. + if !c.Kiwix.RewriteEnabled() { + t.Error("rewriting defaulted to off") + } + off := false + c.Kiwix.Rewrite = &off + if c.Kiwix.RewriteEnabled() { + t.Error("rewrite: false was not honoured") + } +} diff --git a/internal/kiwix/client.go b/internal/kiwix/client.go index d32f264..f9962b7 100644 --- a/internal/kiwix/client.go +++ b/internal/kiwix/client.go @@ -4,8 +4,9 @@ // article snippet beats letting her recall. Nothing here talks to the internet; // the Kiwix server is on the same box. // -// This is search only. Full articles are ~100KB of HTML, far too big for a 4096 -// token context, so the unit of context is the search snippet (~500 chars). +// Search finds the article; Article reads it. The snippet a search returns is +// NOT usable context on its own — see the comment on Article — so the unit of +// context is the head of the article, truncated to fit a 4096 token window. package kiwix import ( @@ -20,6 +21,8 @@ import ( "strconv" "strings" "time" + + "github.com/kami/maven/internal/crawl" ) // Result is one search hit. @@ -73,6 +76,52 @@ func (c *Client) Search(ctx context.Context, pattern, book string, limit int) ([ return ParseSearchRSS(resp.Body) } +// articleMaxBytes — how much of an article HTML document is read before the +// rest is discarded. A maxi Wikipedia page is around 100KB; 512KB is slack for +// the long ones and a hard stop against a ZIM entry that is really a binary. +const articleMaxBytes = 512 << 10 + +// Article fetches one article by the Path a search hit carries and returns it +// as extracted plain text, capped at maxRunes (0 ⇒ crawl.DefaultMaxRunes). +// +// This exists because the search snippet is not usable context. Kiwix builds +// the snippet from wherever the keyword matched, and on a Wikipedia ZIM that is +// routinely the "see also" navigation box at the foot of the page: a search for +// "photosynthesis" comes back with "Ecological economics Ecological footprint +// Ecological forecasting …" and a model handed that writes nothing worth +// hearing. The lead paragraphs are at the top of the document, so truncating an +// article from the front gets the definition the snippet was supposed to be. +// +// Nothing here reaches the internet: the path is resolved against the same +// server the search went to. +func (c *Client) Article(ctx context.Context, path string, maxRunes int) (crawl.Page, error) { + path = strings.TrimSpace(path) + if path == "" { + return crawl.Page{}, fmt.Errorf("kiwix article: empty path") + } + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + u := c.base + path + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return crawl.Page{}, err + } + resp, err := c.http.Do(req) + if err != nil { + return crawl.Page{}, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return crawl.Page{}, fmt.Errorf("kiwix article %s: http %d", path, resp.StatusCode) + } + body, err := io.ReadAll(io.LimitReader(resp.Body, articleMaxBytes)) + if err != nil { + return crawl.Page{}, err + } + return crawl.Extract(u, body, maxRunes), nil +} + // rss mirrors just the bits of the RSS 2.0 reply we use. type rss struct { Items []struct { diff --git a/internal/phraser/llmphraser.go b/internal/phraser/llmphraser.go index 9b32055..bd8855d 100644 --- a/internal/phraser/llmphraser.go +++ b/internal/phraser/llmphraser.go @@ -458,9 +458,16 @@ func chatSystemPrompt(block func() string) string { // "ты" instruction carries that on its own and those two produced the // worst output. The last line says outright not to echo the instructions, // because a small model will otherwise treat any quoted string as licence. + // + // Amended the same day: with the openers gone the tic went with them, but + // "не забыл ли я" appeared — masculine, about herself. The old "я подумала" + // had been suppressing that by accident, being a feminine past tense the + // model could copy. Two short predicatives are not enough signal on their + // own, so the rule is now stated as morphology (-ла) rather than as a pair + // of words. A suffix rule generalises where an example only gets copied. base := `Ты разговариваешь с хозяином. -О себе — в женском роде: "рада", не "рад"; "поняла", не "понял". Он мужчина: обращайся к нему на "ты", в мужском роде. Никогда не "вы"/"ваш" и никогда "он"/"его" — ты говоришь ему, а не о нём. +О себе — в женском роде: "рада", не "рад"; "поняла", не "понял". Все свои глаголы в прошедшем времени оканчивай на -ла: сделала, забыла, записала, подумала. Он мужчина: обращайся к нему на "ты", в мужском роде. Никогда не "вы"/"ваш" и никогда "он"/"его" — ты говоришь ему, а не о нём. Отвечай по-русски, коротко: одна-три фразы, живым языком. Ты доброжелательная, тебе интересно, но чувства не изображай. Не повторяй формулировки из этой инструкции — отвечай своими словами. @@ -718,7 +725,13 @@ func (p *LLMPhraser) systemPrompt() string { func (p *LLMPhraser) querySystemPrompt() string { // No self-introduction here: the persona block prepended one line above // already says who she is, same as router.KnowledgePrompt. - base := "Ты отвечаешь ему по своим заметкам. Отвечай по-русски, коротко и своими словами, начинай с \"вот что я нашла: \". О себе — в женском роде (\"нашла\", \"записала\"). Он мужчина, обращайся к нему на \"ты\". Respond ONLY with valid JSON: {\"response\": \"...\", \"mood\": \"neutral\"}." + // + // The opener is deliberate and stays: this path answers from his own notes + // and the fixed prefix is what marks the answer as a lookup rather than as + // something she knows. The grammar examples are not deliberate — same + // defect chatSystemPrompt had, where a 1.7B copies a quoted word instead of + // generalising from it. Stated as morphology instead. + base := "Ты отвечаешь ему по своим заметкам. Отвечай по-русски, коротко и своими словами, начинай с \"вот что я нашла: \". О себе — в женском роде, глаголы в прошедшем времени с окончанием -ла. Он мужчина, обращайся к нему на \"ты\". Respond ONLY with valid JSON: {\"response\": \"...\", \"mood\": \"neutral\"}." return persona.Prepend(p.cfg.ContextBlock, base) }