Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| be066a4b04 | |||
| ad074cea31 | |||
| 2c1b0eede0 | |||
| cb3641e7bb | |||
| ee7bec11e3 | |||
| f42d1594ef | |||
| b4646155b4 |
@@ -7,6 +7,8 @@
|
||||
/mavpoll
|
||||
/mavcaldav
|
||||
/mavwaked
|
||||
/mavmaild
|
||||
/mavupdate
|
||||
|
||||
# Certs (private keys, don't commit)
|
||||
certs/
|
||||
@@ -36,6 +38,8 @@ deploy/db_key.env
|
||||
deploy/telegram.env
|
||||
# zenmoney API token, read by mavpoll (never in argv, never committed)
|
||||
deploy/zenmoney.token
|
||||
# IMAP password, read by mavmaild (never in argv, never committed)
|
||||
deploy/imap.password
|
||||
|
||||
# Temp files
|
||||
/tmp/
|
||||
|
||||
@@ -34,7 +34,7 @@ CGO daemons (`mavend`, `mavsttd`, `mavttsd`, `mavenclient`) need the vendored to
|
||||
and libs wired through the Makefile — **do not** call `go build` on them bare, use `make`:
|
||||
|
||||
```sh
|
||||
make build # all 8 binaries
|
||||
make build # all 9 binaries
|
||||
make build-web # single daemon (pure-Go ones: web/waked/poll/caldav build without CGO)
|
||||
make test # go test -race across ./internal/... ./cmd/... with CGO env set
|
||||
```
|
||||
@@ -62,6 +62,7 @@ Pure-Go packages (`router`, `memory`, `mavweb`, …) run under a plain `go test
|
||||
| `mavenclient` | Voice loop client (mic → stt → core → tts). |
|
||||
| `mavpoll` | Telegram long-poll reach. |
|
||||
| `mavcaldav` | CalDAV calendar sync. |
|
||||
| `mavmaild` | Mail reader (IMAP, read-only). Holds the IMAP password; core never sees it. |
|
||||
|
||||
Daemons are wired socket-to-socket, not linked. `internal/ipc` is the client/server wire
|
||||
protocol; the config in `deploy/mavend.json` (with `${VAR}` env expansion from gitignored
|
||||
|
||||
+2
-1
@@ -51,7 +51,8 @@ RUN go build -o /out/mavend ./cmd/mavend && \
|
||||
go build -o /out/mavttsd ./cmd/mavttsd && \
|
||||
go build -o /out/mavweb ./cmd/mavweb && \
|
||||
go build -o /out/mavpoll ./cmd/mavpoll && \
|
||||
go build -o /out/mavcaldav ./cmd/mavcaldav
|
||||
go build -o /out/mavcaldav ./cmd/mavcaldav && \
|
||||
go build -o /out/mavmaild ./cmd/mavmaild
|
||||
|
||||
# llama.cpp Vulkan build — the phraser/router LFM engine (llama-server). Built
|
||||
# from source (not a prebuilt vendored blob) so the binary's glibc/GLIBCXX match
|
||||
|
||||
@@ -20,7 +20,7 @@ PIPER_ESPEAK := $(shell pwd)/deps/piper/espeak-ng-data
|
||||
|
||||
all: build
|
||||
|
||||
build: build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav
|
||||
build: build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav build-mail build-update
|
||||
|
||||
build-stt:
|
||||
CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
|
||||
@@ -50,6 +50,15 @@ build-poll:
|
||||
build-caldav:
|
||||
$(GO) build $(GOFLAGS) -o mavcaldav ./cmd/mavcaldav/
|
||||
|
||||
build-mail:
|
||||
$(GO) build $(GOFLAGS) -o mavmaild ./cmd/mavmaild/
|
||||
|
||||
# mavupdate is an operator CLI, not a daemon: nothing runs it but a human on the
|
||||
# box. It is built with the rest so a broken update path is caught by `make
|
||||
# build` rather than the first time it is needed.
|
||||
build-update:
|
||||
$(GO) build $(GOFLAGS) -o mavupdate ./cmd/mavupdate/
|
||||
|
||||
run-web: build-web
|
||||
./mavweb -addr :9200 -voice 127.0.0.1:9100
|
||||
|
||||
@@ -185,4 +194,4 @@ download-embedder:
|
||||
@echo ' sudo cp onnxruntime-linux-x64-1.15.1/lib/libonnxruntime.so* /usr/local/lib/'
|
||||
|
||||
clean:
|
||||
rm -f mavend mavenclient mavsttd mavttsd mavweb mavpoll mavcaldav mavwaked
|
||||
rm -f mavend mavenclient mavsttd mavttsd mavweb mavpoll mavcaldav mavwaked mavmaild
|
||||
|
||||
@@ -8,10 +8,12 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/crawl"
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/memory"
|
||||
"github.com/kami/maven/internal/morning"
|
||||
"github.com/kami/maven/internal/router"
|
||||
"github.com/kami/maven/internal/rss"
|
||||
"github.com/kami/maven/internal/weather"
|
||||
)
|
||||
|
||||
@@ -67,11 +69,23 @@ var querySources = []querySource{
|
||||
// answer it from whatever he once said about spending. Its matcher needs a
|
||||
// money noun plus an actual ask, so "я потратил весь день" is untouched.
|
||||
{"money", (*reactiveHandler).queryMoney},
|
||||
// Before the recall sources and before general knowledge: "что нового?" is
|
||||
// a question about the feeds she reads, and general knowledge would answer
|
||||
// it by inventing news. Its matcher needs a feed noun plus an ask, so
|
||||
// "у меня новая лента в инстаграме" is untouched.
|
||||
{"feeds", (*reactiveHandler).queryFeeds},
|
||||
{"calendar", (*reactiveHandler).queryCalendar},
|
||||
{"weather", (*reactiveHandler).queryWeather},
|
||||
{"embed", (*reactiveHandler).queryEmbed},
|
||||
{"memory", (*reactiveHandler).queryMemory},
|
||||
{"notes", (*reactiveHandler).queryNotes},
|
||||
// LAST before the model answers from memory, and that position is the whole
|
||||
// design (Vikunja #259): local sources first. The model, his own notes and
|
||||
// facts, and — once internal/kiwix is wired into this chain — the offline
|
||||
// ZIMs all get their turn before anything touches the network. This source
|
||||
// only claims a turn where he named a URL out loud, so it never competes
|
||||
// with a local answer.
|
||||
{"web", (*reactiveHandler).queryWeb},
|
||||
{"general-knowledge", (*reactiveHandler).queryGeneral},
|
||||
}
|
||||
|
||||
@@ -180,6 +194,63 @@ func (h *reactiveHandler) queryHabits(ctx context.Context, t *queryTurn) (string
|
||||
return profile.FormatOverallRU(), true
|
||||
}
|
||||
|
||||
// feedNoteWindow — how many recent notes are scanned for feed items, and
|
||||
// feedReadOut — how many headlines she actually reads back. She summarises the
|
||||
// top of the pile, she does not recite a river.
|
||||
const (
|
||||
feedNoteWindow = 200
|
||||
feedReadOut = 3
|
||||
)
|
||||
|
||||
// queryFeeds — "что нового в лентах?", "что нового по технологиям?"
|
||||
// (Vikunja #258).
|
||||
//
|
||||
// This is the ONLY way a feed item reaches him. The poller writes notes and
|
||||
// never speaks; asking is the trigger. If that ever changes, the thing that
|
||||
// changed is "Maven is not a nag", not a detail of this file.
|
||||
func (h *reactiveHandler) queryFeeds(ctx context.Context, t *queryTurn) (string, bool) {
|
||||
q, ok := router.ParseFeedQuery(t.dec.Utterance)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
if !h.feedsOn {
|
||||
// Claim the turn rather than fall through: "не читаю ленты" is true, and
|
||||
// letting general knowledge answer "что нового?" would be an invented
|
||||
// news bulletin.
|
||||
return "я пока не читаю ленты — они не настроены.", true
|
||||
}
|
||||
notes, err := h.api.RecentNotes(ctx, feedNoteWindow)
|
||||
if err != nil {
|
||||
log.Printf("voice: feeds: recent notes: %v", err)
|
||||
return "не получилось посмотреть ленты.", true
|
||||
}
|
||||
var picked []string
|
||||
for _, n := range notes {
|
||||
if !strings.HasPrefix(n.Source, rss.SourcePrefix) {
|
||||
continue
|
||||
}
|
||||
if !router.CategoryMatches(n.Text, q.Category) {
|
||||
continue
|
||||
}
|
||||
// The note carries title, summary and link; she reads the title.
|
||||
title := n.Text
|
||||
if i := strings.IndexByte(title, '\n'); i > 0 {
|
||||
title = title[:i]
|
||||
}
|
||||
picked = append(picked, strings.TrimSpace(title))
|
||||
if len(picked) == feedReadOut {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(picked) == 0 {
|
||||
if q.Category != "" {
|
||||
return "по этой теме в лентах пока ничего.", true
|
||||
}
|
||||
return "в лентах пока ничего нового.", true
|
||||
}
|
||||
return "вот что нового: " + strings.Join(picked, "; "), true
|
||||
}
|
||||
|
||||
// queryCalendar — "что у меня сегодня?", "планы на завтра?"
|
||||
// h.now(), not time.Now(): the handler's clock is the injected one, so this
|
||||
// source can be tested at a fixed time like the rest.
|
||||
@@ -308,6 +379,57 @@ func (h *reactiveHandler) queryNotes(ctx context.Context, t *queryTurn) (string,
|
||||
return reply, true
|
||||
}
|
||||
|
||||
// webPageContextRunes — how much of a fetched page is handed to the phraser.
|
||||
// Less than the crawler keeps: the rest of the 4096-token window belongs to the
|
||||
// prompt, the persona block and the reply.
|
||||
const webPageContextRunes = 1500
|
||||
|
||||
// queryWeb — "посмотри https://example.org/x — что там?" (Vikunja #259).
|
||||
//
|
||||
// It claims a turn ONLY when he named a URL, which is what keeps a fallback from
|
||||
// becoming a habit: no URL, no fetch, and the model answers from what is local.
|
||||
// What leaves the box is the URL and nothing else — no note, no fact, no history
|
||||
// travels with it.
|
||||
func (h *reactiveHandler) queryWeb(ctx context.Context, t *queryTurn) (string, bool) {
|
||||
link, ok := router.FirstURL(t.dec.Utterance)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
if h.crawler == nil {
|
||||
// Claim rather than fall through: he asked about a specific page, and
|
||||
// letting the model answer from the URL's spelling alone is how a small
|
||||
// model invents a page's contents.
|
||||
return "я не читаю страницы — это не настроено.", true
|
||||
}
|
||||
ctxFetch, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
page, err := h.crawler.Page(ctxFetch, link)
|
||||
if err != nil {
|
||||
if errors.Is(err, crawl.ErrRobots) {
|
||||
return "эта страница закрыта для чтения — robots.txt не разрешает.", true
|
||||
}
|
||||
log.Printf("voice: web: %v", err)
|
||||
return "не получилось прочитать страницу.", true
|
||||
}
|
||||
if page.Text == "" {
|
||||
return "страница открылась, но читать там нечего.", true
|
||||
}
|
||||
// The page is handed to the phraser the same way a note is: as context for
|
||||
// the question he actually asked. She answers the question, she does not
|
||||
// recite the page.
|
||||
snippet := page.Title + "\n" + crawl.TrimRunes(page.Text, webPageContextRunes)
|
||||
reply, perr := h.phraser.PhraseQuery(ctx, t.dec.Utterance, []string{snippet})
|
||||
if perr != nil {
|
||||
log.Printf("voice: web: phrase: %v", perr)
|
||||
}
|
||||
if reply == "" {
|
||||
// No phraser (or it failed): read back the top of the page rather than
|
||||
// pretend the fetch did not happen.
|
||||
return "вот что на странице: " + crawl.TrimRunes(page.Text, 300), true
|
||||
}
|
||||
return reply, true
|
||||
}
|
||||
|
||||
// queryGeneral — general knowledge from the phraser, the last source before
|
||||
// giving up. It always claims: either the model answers or Maven says she
|
||||
// doesn't know.
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
// mavend/crawls.go — the driver for reading web pages (Vikunja #259,
|
||||
// docs/plans/14-web-crawler.md). The crawler is pure and lives in
|
||||
// internal/crawl; this is the impure half: the guarded fetcher, a ticker for the
|
||||
// scheduled watches, and the fact-backed dedup hashes.
|
||||
//
|
||||
// Two paths, one config block, both off unless configured:
|
||||
//
|
||||
// - ON DEMAND — he names a URL out loud and she reads it. That is the
|
||||
// `queryWeb` source in actions_query.go, LAST in the chain: after his
|
||||
// memory, after the notes, and (once Kiwix is wired into the chain) after
|
||||
// the local ZIMs. A local read costs nothing and leaks nothing; a fetch puts
|
||||
// a URL in someone's log, so it goes last.
|
||||
// - SCHEDULED — a watched page is re-read on its interval, and a page whose
|
||||
// text changed is written as a note. It does NOT announce itself. Same rule
|
||||
// as the feed poller: notes, never nudges.
|
||||
//
|
||||
// Only the URL goes out. Nothing here reads a note, a fact, the persona block or
|
||||
// the history, and internal/crawl has no access to the store at all.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/config"
|
||||
"github.com/kami/maven/internal/crawl"
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/router"
|
||||
"github.com/kami/maven/internal/webfetch"
|
||||
)
|
||||
|
||||
// newCrawler builds the crawler from the `crawl` block, or returns nil when
|
||||
// there is none. Every caller checks for nil, and nil means no page is ever
|
||||
// fetched.
|
||||
func newCrawler(cfg *config.Config) *crawl.Crawler {
|
||||
if cfg.Crawl == nil {
|
||||
return nil
|
||||
}
|
||||
cc := cfg.Crawl
|
||||
|
||||
hosts := append([]string(nil), cc.AllowHosts...)
|
||||
// A watched page's own host is always reachable; otherwise an allowlist and
|
||||
// a watch list would have to be kept in sync by hand.
|
||||
for _, w := range cc.Watches {
|
||||
if u, err := url.Parse(w.URL); err == nil && u.Hostname() != "" {
|
||||
hosts = append(hosts, u.Hostname())
|
||||
}
|
||||
}
|
||||
// An allowlist plus on-demand is a contradiction worth logging rather than
|
||||
// silently resolving: he asked for arbitrary pages AND for a fixed list.
|
||||
// The allowlist wins, because it is the narrower instruction.
|
||||
if len(hosts) > 0 && cc.OnDemand && len(cc.AllowHosts) > 0 {
|
||||
log.Printf("crawl: allow_hosts is set, so on-demand reading is limited to those hosts")
|
||||
}
|
||||
ua := cc.UserAgent
|
||||
if ua == "" {
|
||||
ua = webfetch.DefaultUserAgent
|
||||
}
|
||||
fetcher := webfetch.New(webfetch.Config{
|
||||
AllowHosts: hosts,
|
||||
DenyHosts: cc.DenyHosts,
|
||||
Timeout: time.Duration(cc.Timeout),
|
||||
MaxBytes: cc.MaxBytes,
|
||||
UserAgent: ua,
|
||||
})
|
||||
// The user-agent handed to the crawler is the one the fetcher sends: obeying
|
||||
// robots rules written for a different name would be a lie.
|
||||
return crawl.New(&crawlFetcher{f: fetcher}, crawl.Config{
|
||||
UserAgent: ua,
|
||||
MaxRunes: cc.MaxRunes,
|
||||
})
|
||||
}
|
||||
|
||||
// onDemandCrawler returns a crawler for the answer path, or nil when on-demand
|
||||
// reading is off. The scheduled watches can be on while this is off: reading a
|
||||
// fixed list of pages on a timer and reading whatever URL is in an utterance are
|
||||
// different permissions, and the config keeps them separate.
|
||||
func onDemandCrawler(cfg *config.Config) *crawl.Crawler {
|
||||
if cfg.Crawl == nil || !cfg.Crawl.OnDemand {
|
||||
return nil
|
||||
}
|
||||
return newCrawler(cfg)
|
||||
}
|
||||
|
||||
// crawlWorker — ticker + watcher for the scheduled half.
|
||||
type crawlWorker struct {
|
||||
watcher *crawl.Watcher
|
||||
interval time.Duration
|
||||
}
|
||||
|
||||
// crawlTickInterval — how often the worker asks what is due. Per-watch cadence
|
||||
// is the watcher's business.
|
||||
const crawlTickInterval = 15 * time.Minute
|
||||
|
||||
// newCrawlWorker wires the scheduled crawls, or nil when nothing is watched.
|
||||
func newCrawlWorker(c *crawl.Crawler, api ipc.CoreAPI, emb router.Embedder, cfg *config.Config) *crawlWorker {
|
||||
if c == nil || cfg.Crawl == nil || len(cfg.Crawl.Watches) == 0 {
|
||||
return nil
|
||||
}
|
||||
watches := make([]crawl.WatchConfig, 0, len(cfg.Crawl.Watches))
|
||||
for _, w := range cfg.Crawl.Watches {
|
||||
watches = append(watches, crawl.WatchConfig{
|
||||
Name: w.Name,
|
||||
URL: w.URL,
|
||||
Interval: time.Duration(w.Interval),
|
||||
})
|
||||
}
|
||||
watcher := crawl.NewWatcher(c, watches, api, &factHashes{api: api},
|
||||
crawlEmbedder(emb), time.Duration(cfg.Crawl.Interval))
|
||||
if watcher == nil {
|
||||
log.Printf("crawl: configured but nothing watchable — scheduled crawls disabled")
|
||||
return nil
|
||||
}
|
||||
log.Printf("crawl: watching %d page(s), checking what is due every %s", len(watches), crawlTickInterval)
|
||||
return &crawlWorker{watcher: watcher, interval: crawlTickInterval}
|
||||
}
|
||||
|
||||
// run checks what is due until ctx is canceled. The first round runs
|
||||
// immediately; it writes notes only, so an early round startles nobody.
|
||||
func (w *crawlWorker) run(ctx context.Context) {
|
||||
w.watcher.CheckDue(ctx, time.Now())
|
||||
t := time.NewTicker(w.interval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case now := <-t.C:
|
||||
w.watcher.CheckDue(ctx, now)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// crawlFetcher adapts webfetch to crawl.Fetcher, which is the seam that keeps
|
||||
// net/http out of the crawler package.
|
||||
type crawlFetcher struct{ f *webfetch.Fetcher }
|
||||
|
||||
func (a *crawlFetcher) Get(ctx context.Context, u string) (*crawl.Response, error) {
|
||||
resp, err := a.f.Get(ctx, u)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &crawl.Response{URL: resp.URL, ContentType: resp.ContentType, Body: resp.Body}, nil
|
||||
}
|
||||
|
||||
// factHashes stores each watch's last content hash as a config fact, so a
|
||||
// restart does not re-note an unchanged page. Same mechanism the feed reader
|
||||
// uses for its marks, and inspectable on /dash.
|
||||
type factHashes struct{ api ipc.CoreAPI }
|
||||
|
||||
func hashKey(name string) string { return "crawl:hash:" + name }
|
||||
|
||||
func (h *factHashes) LastHash(ctx context.Context, name string) (string, error) {
|
||||
f, err := h.api.LatestFact(ctx, hashKey(name))
|
||||
if err != nil {
|
||||
// No hash yet is not an error: the watcher treats "" as "never read".
|
||||
return "", nil
|
||||
}
|
||||
return f.Value, nil
|
||||
}
|
||||
|
||||
func (h *factHashes) SetHash(ctx context.Context, name, hash string) error {
|
||||
_, err := h.api.WriteFact(ctx, ipc.WriteFactReq{
|
||||
Ts: time.Now(),
|
||||
Kind: "config",
|
||||
Key: hashKey(name),
|
||||
Value: hash,
|
||||
Source: "poll:crawl",
|
||||
Confidence: 1.0,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// crawlEmbedder adapts router.Embedder for the watcher, embedding with
|
||||
// EmbedPassage (a page is text being searched FOR, and the e5 embedder is
|
||||
// asymmetric).
|
||||
func crawlEmbedder(emb router.Embedder) crawl.Embedder {
|
||||
if emb == nil {
|
||||
return nil
|
||||
}
|
||||
return passageEmbedder{emb}
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/kami/maven/internal/config"
|
||||
"github.com/kami/maven/internal/crawl"
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/phraser"
|
||||
"github.com/kami/maven/internal/router"
|
||||
"github.com/kami/maven/internal/voice"
|
||||
)
|
||||
|
||||
// The default config reads nothing. This is the whole "off unless configured"
|
||||
// contract for the crawler, asserted at the wiring level rather than trusted.
|
||||
func TestCrawlOffByDefault(t *testing.T) {
|
||||
cfg := &config.Config{}
|
||||
if c := newCrawler(cfg); c != nil {
|
||||
t.Error("newCrawler with no crawl block returned a crawler")
|
||||
}
|
||||
if c := onDemandCrawler(cfg); c != nil {
|
||||
t.Error("onDemandCrawler with no crawl block returned a crawler")
|
||||
}
|
||||
if w := newCrawlWorker(nil, nil, nil, cfg); w != nil {
|
||||
t.Error("newCrawlWorker with no crawl block returned a worker")
|
||||
}
|
||||
// Watches configured but on_demand off ⇒ the answer path still reads
|
||||
// nothing: a timer over a fixed list is not permission for arbitrary URLs.
|
||||
withWatch := &config.Config{Crawl: &config.CrawlConfig{
|
||||
Watches: []config.CrawlWatchConfig{{Name: "p", URL: "https://example.org/p"}},
|
||||
}}
|
||||
if c := onDemandCrawler(withWatch); c != nil {
|
||||
t.Error("onDemandCrawler honoured a watch list as on-demand permission")
|
||||
}
|
||||
if c := newCrawler(withWatch); c == nil {
|
||||
t.Error("newCrawler returned nil for a configured watch")
|
||||
}
|
||||
}
|
||||
|
||||
// The wired fetcher must refuse a private address, because the crawler on this
|
||||
// box sits one hop from the whole homelab. Same guard the webfetch tests cover;
|
||||
// this asserts the daemon actually wires it.
|
||||
func TestCrawlerRefusesPrivateAddress(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
w.Write([]byte("<html><body>secret</body></html>"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := newCrawler(&config.Config{Crawl: &config.CrawlConfig{OnDemand: true}})
|
||||
if c == nil {
|
||||
t.Fatal("newCrawler returned nil for an on-demand config")
|
||||
}
|
||||
if _, err := c.Page(context.Background(), srv.URL); err == nil {
|
||||
t.Fatalf("reading %s succeeded; a loopback address must be refused", srv.URL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFactHashesRoundTrip(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
st := newTestStore(t)
|
||||
h := &factHashes{api: ipc.NewStoreAPI(st)}
|
||||
|
||||
got, err := h.LastHash(ctx, "page")
|
||||
if err != nil {
|
||||
t.Fatalf("LastHash on a fresh store: %v", err)
|
||||
}
|
||||
if got != "" {
|
||||
t.Errorf("LastHash = %q, want empty for a never-read page", got)
|
||||
}
|
||||
if err := h.SetHash(ctx, "page", "deadbeef"); err != nil {
|
||||
t.Fatalf("SetHash: %v", err)
|
||||
}
|
||||
got, err = h.LastHash(ctx, "page")
|
||||
if err != nil {
|
||||
t.Fatalf("LastHash: %v", err)
|
||||
}
|
||||
if got != "deadbeef" {
|
||||
t.Errorf("LastHash = %q, want deadbeef", got)
|
||||
}
|
||||
if key := hashKey("page"); key != "crawl:hash:page" {
|
||||
t.Errorf("hashKey = %q", key)
|
||||
}
|
||||
}
|
||||
|
||||
// stubCrawlFetcher serves one fixed page to every URL, so queryWeb can be
|
||||
// exercised without a network or an allowlist.
|
||||
type stubCrawlFetcher struct{ body, ctype string }
|
||||
|
||||
func (s *stubCrawlFetcher) Get(_ context.Context, u string) (*crawl.Response, error) {
|
||||
ct := s.ctype
|
||||
if ct == "" {
|
||||
ct = "text/html"
|
||||
}
|
||||
if strings.HasSuffix(u, "/robots.txt") {
|
||||
return &crawl.Response{URL: u, ContentType: "text/plain", Body: []byte("")}, nil
|
||||
}
|
||||
return &crawl.Response{URL: u, ContentType: ct, Body: []byte(s.body)}, nil
|
||||
}
|
||||
|
||||
func buildWebHandler(c *crawl.Crawler) *reactiveHandler {
|
||||
return &reactiveHandler{
|
||||
replier: voice.NewStubReplier(),
|
||||
phraser: phraser.NewStub(),
|
||||
crawler: c,
|
||||
}
|
||||
}
|
||||
|
||||
func askWeb(h *reactiveHandler, q string) (string, bool) {
|
||||
return h.queryWeb(context.Background(), &queryTurn{
|
||||
dec: router.Decision{Intent: router.IntentQuery, Utterance: q},
|
||||
})
|
||||
}
|
||||
|
||||
func TestQueryWebPassesWithoutAURL(t *testing.T) {
|
||||
h := buildWebHandler(crawl.New(&stubCrawlFetcher{body: "<html><body>x</body></html>"}, crawl.Config{}))
|
||||
if reply, ok := askWeb(h, "почему небо синее?"); ok {
|
||||
t.Errorf("the web source claimed a question with no URL: %q", reply)
|
||||
}
|
||||
}
|
||||
|
||||
// Not configured is said out loud rather than falling through, so a small model
|
||||
// never invents a page's contents from its URL.
|
||||
func TestQueryWebSaysWhenNotConfigured(t *testing.T) {
|
||||
h := buildWebHandler(nil)
|
||||
reply, ok := askWeb(h, "посмотри https://example.org/page")
|
||||
if !ok {
|
||||
t.Fatal("the web source did not claim a question with a URL")
|
||||
}
|
||||
if !strings.Contains(reply, "не настроено") {
|
||||
t.Errorf("reply = %q, want the not-configured answer", reply)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryWebReadsThePage(t *testing.T) {
|
||||
h := buildWebHandler(crawl.New(&stubCrawlFetcher{
|
||||
body: "<html><head><title>Заголовок</title></head><body><p>текст страницы</p></body></html>",
|
||||
}, crawl.Config{}))
|
||||
reply, ok := askWeb(h, "посмотри https://example.org/page — что там?")
|
||||
if !ok {
|
||||
t.Fatal("the web source did not claim a question with a URL")
|
||||
}
|
||||
if !strings.Contains(reply, "текст страницы") {
|
||||
t.Errorf("reply = %q, want the page text read back", reply)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryWebRefusesNonHTML(t *testing.T) {
|
||||
h := buildWebHandler(crawl.New(&stubCrawlFetcher{
|
||||
body: "\x00\x01binary", ctype: "application/octet-stream",
|
||||
}, crawl.Config{}))
|
||||
reply, ok := askWeb(h, "почитай https://example.org/blob.bin")
|
||||
if !ok {
|
||||
t.Fatal("the web source did not claim a question with a URL")
|
||||
}
|
||||
if !strings.Contains(reply, "не получилось") {
|
||||
t.Errorf("reply = %q, want the read-failed answer", reply)
|
||||
}
|
||||
}
|
||||
|
||||
// robots.txt is honoured on the answer path too, and she says so instead of
|
||||
// reporting a generic failure.
|
||||
func TestQueryWebObeysRobots(t *testing.T) {
|
||||
h := buildWebHandler(crawl.New(&robotsDenyFetcher{}, crawl.Config{}))
|
||||
reply, ok := askWeb(h, "посмотри https://example.org/private")
|
||||
if !ok {
|
||||
t.Fatal("the web source did not claim a question with a URL")
|
||||
}
|
||||
if !strings.Contains(reply, "robots.txt") {
|
||||
t.Errorf("reply = %q, want the robots answer", reply)
|
||||
}
|
||||
}
|
||||
|
||||
type robotsDenyFetcher struct{}
|
||||
|
||||
func (robotsDenyFetcher) Get(_ context.Context, u string) (*crawl.Response, error) {
|
||||
if strings.HasSuffix(u, "/robots.txt") {
|
||||
return &crawl.Response{URL: u, ContentType: "text/plain",
|
||||
Body: []byte("User-agent: *\nDisallow: /private\n")}, nil
|
||||
}
|
||||
return &crawl.Response{URL: u, ContentType: "text/html", Body: []byte("<html>nope</html>")}, nil
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
// mavend/feeds.go — the driver for RSS/Atom reading (Vikunja #258,
|
||||
// docs/plans/13-rss-news-feeds.md). The reader itself is pure and lives in
|
||||
// internal/rss; this is the impure half: a ticker, the guarded fetcher, and the
|
||||
// two adapters that let a pure package talk to the store.
|
||||
//
|
||||
// Why in-core rather than its own daemon like mavmaild and mavpoll: those two
|
||||
// hold a CREDENTIAL (an IMAP password, a zenmoney token), and the reason they
|
||||
// are separate processes is that core must never see it. A feed URL is public,
|
||||
// there is no secret to isolate, and a whole extra binary and compose service
|
||||
// would buy nothing. The other half of the mavpoll precedent — off unless
|
||||
// configured — is kept: no `feeds` block, no poller, no outbound request.
|
||||
//
|
||||
// It is its own goroutine, not a step on the tick: the tick has a delivery
|
||||
// deadline behind it, and a feed read is a network round-trip that nobody is
|
||||
// waiting on.
|
||||
//
|
||||
// Nothing here dispatches. A feed that announced itself would be a nag, so the
|
||||
// only output is notes with source "rss:<feed>", which the answer path reads
|
||||
// when he asks ("что нового в лентах?" — see queryFeeds in actions_query.go).
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/config"
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/router"
|
||||
"github.com/kami/maven/internal/rss"
|
||||
"github.com/kami/maven/internal/webfetch"
|
||||
)
|
||||
|
||||
// feedWorker — ticker + poller.
|
||||
type feedWorker struct {
|
||||
poller *rss.Poller
|
||||
interval time.Duration
|
||||
}
|
||||
|
||||
// feedTickInterval — how often the worker asks the poller what is due. Per-feed
|
||||
// cadence is the poller's business; this is just the granularity.
|
||||
const feedTickInterval = 5 * time.Minute
|
||||
|
||||
// newFeedWorker wires feed reading, or returns nil when it must not run:
|
||||
// no `feeds` block (the normal case), or nothing valid in it. Every caller
|
||||
// checks for nil.
|
||||
func newFeedWorker(api ipc.CoreAPI, emb router.Embedder, cfg *config.Config) *feedWorker {
|
||||
if cfg.Feeds == nil {
|
||||
return nil
|
||||
}
|
||||
fc := cfg.Feeds
|
||||
|
||||
feeds := make([]rss.FeedConfig, 0, len(fc.Sources))
|
||||
hosts := append([]string(nil), fc.AllowHosts...)
|
||||
for _, s := range fc.Sources {
|
||||
feeds = append(feeds, rss.FeedConfig{
|
||||
Name: s.Name,
|
||||
URL: s.URL,
|
||||
Category: s.Category,
|
||||
Interval: time.Duration(s.Interval),
|
||||
Include: s.Include,
|
||||
Exclude: s.Exclude,
|
||||
})
|
||||
// Each configured feed's own host is allowed. The allowlist is then
|
||||
// exactly "the feeds he asked for", so a redirect off to somewhere else
|
||||
// is refused by the fetcher rather than followed.
|
||||
if u, err := url.Parse(s.URL); err == nil && u.Hostname() != "" {
|
||||
hosts = append(hosts, u.Hostname())
|
||||
}
|
||||
}
|
||||
|
||||
fetcher := webfetch.New(webfetch.Config{
|
||||
AllowHosts: hosts,
|
||||
Timeout: time.Duration(fc.Timeout),
|
||||
MaxBytes: fc.MaxBytes,
|
||||
})
|
||||
poller := rss.NewPoller(feeds, &feedFetcher{f: fetcher}, api, &factMarks{api: api},
|
||||
embedderFor(emb), nil, rss.Config{
|
||||
DefaultInterval: time.Duration(fc.PollInterval),
|
||||
MaxItems: fc.MaxItems,
|
||||
MaxAge: time.Duration(fc.MaxAge),
|
||||
})
|
||||
if poller == nil {
|
||||
log.Printf("feeds: configured but nothing pollable — feed reading disabled")
|
||||
return nil
|
||||
}
|
||||
log.Printf("feeds: reading %d feed(s), checking what is due every %s", len(feeds), feedTickInterval)
|
||||
return &feedWorker{poller: poller, interval: feedTickInterval}
|
||||
}
|
||||
|
||||
// run polls what is due until ctx is canceled. The first round runs immediately
|
||||
// so a restart does not blind her for the first interval; it writes notes only,
|
||||
// so an early round cannot startle anyone.
|
||||
func (w *feedWorker) run(ctx context.Context) {
|
||||
w.poller.PollDue(ctx, time.Now())
|
||||
t := time.NewTicker(w.interval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case now := <-t.C:
|
||||
w.poller.PollDue(ctx, now)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// embedderOf — the voice wiring's embedder, or nil when voice is not wired.
|
||||
// Feed notes are embedded with the SAME model the rest of the store uses, or not
|
||||
// at all; a second embedder would write vectors nothing can search.
|
||||
func embedderOf(w *voiceWiring) router.Embedder {
|
||||
if w == nil {
|
||||
return nil
|
||||
}
|
||||
return w.embedder
|
||||
}
|
||||
|
||||
// feedFetcher adapts webfetch to rss.Fetcher — the pure package names the two
|
||||
// fields it needs and stays free of net/http.
|
||||
type feedFetcher struct{ f *webfetch.Fetcher }
|
||||
|
||||
func (a *feedFetcher) Get(ctx context.Context, u string) (*rss.Body, error) {
|
||||
resp, err := a.f.Get(ctx, u)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &rss.Body{Bytes: resp.Body}, nil
|
||||
}
|
||||
|
||||
// factMarks stores "how far this feed was read" as a config fact, the same
|
||||
// mechanism the plan named and the same one the pattern tick uses for its own
|
||||
// bookkeeping. Durable, inspectable on /dash, and cheap.
|
||||
type factMarks struct{ api ipc.CoreAPI }
|
||||
|
||||
func markKey(feed string) string { return "rss:latest:" + feed }
|
||||
|
||||
func (m *factMarks) LastMark(ctx context.Context, feed string) (time.Time, error) {
|
||||
f, err := m.api.LatestFact(ctx, markKey(feed))
|
||||
if err != nil {
|
||||
// No mark yet is not an error worth propagating: the poller treats a
|
||||
// zero time as a cold start.
|
||||
return time.Time{}, nil
|
||||
}
|
||||
t, err := time.Parse(time.RFC3339, f.Value)
|
||||
if err != nil {
|
||||
return time.Time{}, nil
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
func (m *factMarks) SetMark(ctx context.Context, feed string, at time.Time) error {
|
||||
_, err := m.api.WriteFact(ctx, ipc.WriteFactReq{
|
||||
Ts: time.Now(),
|
||||
Kind: "config",
|
||||
Key: markKey(feed),
|
||||
Value: at.UTC().Format(time.RFC3339),
|
||||
Source: "poll:rss",
|
||||
Confidence: 1.0,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// embedderFor adapts router.Embedder to rss.Embedder, and returns nil when
|
||||
// there is none — a note without a vector is still a note the recent-notes path
|
||||
// can read.
|
||||
//
|
||||
// EmbedPassage, not Embed: a feed item is text being searched FOR, and the e5
|
||||
// embedder is asymmetric. Getting this backwards makes the item unfindable by
|
||||
// the question that should have matched it.
|
||||
func embedderFor(emb router.Embedder) rss.Embedder {
|
||||
if emb == nil {
|
||||
return nil
|
||||
}
|
||||
return passageEmbedder{emb}
|
||||
}
|
||||
|
||||
type passageEmbedder struct{ e router.Embedder }
|
||||
|
||||
func (p passageEmbedder) Embed(ctx context.Context, text string) ([]float32, error) {
|
||||
return router.EmbedPassage(ctx, p.e, text)
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/config"
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/phraser"
|
||||
"github.com/kami/maven/internal/router"
|
||||
"github.com/kami/maven/internal/rss"
|
||||
"github.com/kami/maven/internal/voice"
|
||||
)
|
||||
|
||||
// buildFeedHandler — a handler with the given feed notes already stored. No
|
||||
// embedder: the feed source answers from recent notes by source, which is what
|
||||
// makes it work for notes written before an embedder existed.
|
||||
func buildFeedHandler(t *testing.T, feedsOn bool, notes ...ipc.Note) *reactiveHandler {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
st := newTestStore(t)
|
||||
now := time.Now()
|
||||
for i, n := range notes {
|
||||
ts := now.Add(time.Duration(i) * time.Minute)
|
||||
if _, err := st.WriteNote(ctx, ts, n.Text, nil, n.Source); err != nil {
|
||||
t.Fatalf("WriteNote: %v", err)
|
||||
}
|
||||
}
|
||||
return &reactiveHandler{
|
||||
api: ipc.NewStoreAPI(st),
|
||||
replier: voice.NewStubReplier(),
|
||||
phraser: phraser.NewStub(),
|
||||
now: func() time.Time { return now },
|
||||
feedsOn: feedsOn,
|
||||
embedder: nil,
|
||||
}
|
||||
}
|
||||
|
||||
func askFeeds(t *testing.T, h *reactiveHandler, q string) (string, bool) {
|
||||
t.Helper()
|
||||
return h.queryFeeds(context.Background(), &queryTurn{
|
||||
dec: router.Decision{Intent: router.IntentQuery, Utterance: q},
|
||||
})
|
||||
}
|
||||
|
||||
func TestQueryFeedsReadsFeedNotes(t *testing.T) {
|
||||
h := buildFeedHandler(t, true,
|
||||
ipc.Note{Text: "Новая уязвимость в ядре [технологии]\nпатч вышел\nhttps://example.org/a", Source: "rss:habr"},
|
||||
ipc.Note{Text: "что-то он сам сказал", Source: "tap:voice"},
|
||||
)
|
||||
reply, ok := askFeeds(t, h, "что нового в лентах?")
|
||||
if !ok {
|
||||
t.Fatal("the feed source did not claim the question")
|
||||
}
|
||||
if !strings.Contains(reply, "уязвимость") {
|
||||
t.Errorf("reply = %q, want the headline", reply)
|
||||
}
|
||||
if strings.Contains(reply, "он сам сказал") {
|
||||
t.Errorf("a note he dictated leaked into the feed answer: %q", reply)
|
||||
}
|
||||
// She reads the headline, not the summary and not the URL.
|
||||
if strings.Contains(reply, "https://") || strings.Contains(reply, "патч вышел") {
|
||||
t.Errorf("reply = %q, want the title line only", reply)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryFeedsByCategory(t *testing.T) {
|
||||
h := buildFeedHandler(t, true,
|
||||
ipc.Note{Text: "Релиз ядра [технологии]", Source: "rss:habr"},
|
||||
ipc.Note{Text: "Выборы отложены [политика]", Source: "rss:news"},
|
||||
)
|
||||
reply, ok := askFeeds(t, h, "что нового по технологиям?")
|
||||
if !ok {
|
||||
t.Fatal("not claimed")
|
||||
}
|
||||
if !strings.Contains(reply, "ядра") || strings.Contains(reply, "Выборы") {
|
||||
t.Fatalf("reply = %q, want only the технологии item", reply)
|
||||
}
|
||||
reply, _ = askFeeds(t, h, "что нового по спорту?")
|
||||
if !strings.Contains(reply, "ничего") {
|
||||
t.Fatalf("reply = %q, want an honest empty answer for an unread category", reply)
|
||||
}
|
||||
}
|
||||
|
||||
// "не настроены" and "ничего нового" are different truths, and neither may be
|
||||
// answered by the model inventing a bulletin.
|
||||
func TestQueryFeedsOffAndEmptyDiffer(t *testing.T) {
|
||||
off := buildFeedHandler(t, false)
|
||||
reply, ok := askFeeds(t, off, "что нового?")
|
||||
if !ok || !strings.Contains(reply, "не настроены") {
|
||||
t.Fatalf("feeds off: reply = %q, ok = %v", reply, ok)
|
||||
}
|
||||
on := buildFeedHandler(t, true)
|
||||
reply, ok = askFeeds(t, on, "что нового?")
|
||||
if !ok || !strings.Contains(reply, "ничего нового") {
|
||||
t.Fatalf("feeds on but empty: reply = %q, ok = %v", reply, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryFeedsPassesOnANonFeedQuestion(t *testing.T) {
|
||||
h := buildFeedHandler(t, true)
|
||||
if reply, ok := askFeeds(t, h, "напомни полить цветы"); ok {
|
||||
t.Fatalf("claimed an unrelated question with %q", reply)
|
||||
}
|
||||
}
|
||||
|
||||
// The mark is what stops a restart from re-noting yesterday's headlines, so the
|
||||
// fact round-trip is worth a test of its own.
|
||||
func TestFactMarksRoundTrip(t *testing.T) {
|
||||
st := newTestStore(t)
|
||||
m := &factMarks{api: ipc.NewStoreAPI(st)}
|
||||
ctx := context.Background()
|
||||
|
||||
at, err := m.LastMark(ctx, "habr")
|
||||
if err != nil || !at.IsZero() {
|
||||
t.Fatalf("no mark yet: got %v, %v — want zero time and no error", at, err)
|
||||
}
|
||||
want := time.Date(2026, 7, 28, 10, 0, 0, 0, time.UTC)
|
||||
if err := m.SetMark(ctx, "habr", want); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := m.LastMark(ctx, "habr")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !got.Equal(want) {
|
||||
t.Fatalf("mark = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// Off unless configured, checked at the wiring seam: no `feeds` block ⇒ no
|
||||
// worker ⇒ no outbound request is possible.
|
||||
func TestNewFeedWorkerOffByDefault(t *testing.T) {
|
||||
st := newTestStore(t)
|
||||
api := ipc.NewStoreAPI(st)
|
||||
if w := newFeedWorker(api, nil, &config.Config{}); w != nil {
|
||||
t.Fatal("a config with no feeds block wired a feed worker")
|
||||
}
|
||||
// An empty sources list is normalised to "off" by config.Load; the worker
|
||||
// refuses it too, so a hand-built Config cannot switch it on by accident.
|
||||
if w := newFeedWorker(api, nil, &config.Config{Feeds: &config.FeedsConfig{}}); w != nil {
|
||||
t.Fatal("an empty sources list wired a feed worker")
|
||||
}
|
||||
cfg := &config.Config{Feeds: &config.FeedsConfig{Sources: []config.FeedSourceConfig{
|
||||
{Name: "habr", URL: "https://example.org/rss"},
|
||||
}}}
|
||||
w := newFeedWorker(api, nil, cfg)
|
||||
if w == nil {
|
||||
t.Fatal("a configured feed did not wire a worker")
|
||||
}
|
||||
if got := w.poller.Feeds(); len(got) != 1 || got[0].Name != "habr" {
|
||||
t.Fatalf("feeds = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The fetcher the worker builds must be allowlisted to the configured feeds and
|
||||
// nothing else — the crawler's SSRF guards are only worth as much as the
|
||||
// allowlist handed to them.
|
||||
func TestFeedWorkerFetcherIsAllowlisted(t *testing.T) {
|
||||
cfg := &config.Config{Feeds: &config.FeedsConfig{Sources: []config.FeedSourceConfig{
|
||||
{Name: "habr", URL: "https://feeds.example.org/rss"},
|
||||
}}}
|
||||
w := newFeedWorker(ipc.NewStoreAPI(newTestStore(t)), nil, cfg)
|
||||
if w == nil {
|
||||
t.Fatal("no worker")
|
||||
}
|
||||
// PollFeed goes through the guarded fetcher; a feed URL pointing at the box
|
||||
// itself must fail rather than be read.
|
||||
_, err := w.poller.PollFeed(context.Background(), rss.FeedConfig{
|
||||
Name: "evil", URL: "http://127.0.0.1:9100/mcp",
|
||||
}, time.Now())
|
||||
if err == nil {
|
||||
t.Fatal("the poller fetched a private address")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
// mavend/mail.go — core's half of the email reader (Vikunja #246,
|
||||
// docs/plans/01-email-reader.md).
|
||||
//
|
||||
// The split: cmd/mavmaild holds the IMAP credential, connects to the mailbox
|
||||
// and converts messages to plaintext; it hands each message to core over
|
||||
// ipc.MethodIngestMail. Core runs the extraction on the resident model —
|
||||
// llama-server lives in this process, spawned by the phraser — and writes what
|
||||
// comes back through the one task intake seam.
|
||||
//
|
||||
// What this file may produce is exactly one thing: rows in `tasks` with status
|
||||
// "candidate". No fact, no reminder, no note, no nudge, no calendar event. A
|
||||
// 1.7B misreading a mail can therefore put a wrong line on a review page and
|
||||
// nothing else; it can never make Maven speak, and it can never make her
|
||||
// recite something out of an advert as true.
|
||||
//
|
||||
// Off unless configured twice over: no `email` block in mavend.json ⇒ the IPC
|
||||
// method does not exist; no llama-server phraser ⇒ same. A reader pointed at a
|
||||
// core that is not set up for mail gets ErrUnknownMethod rather than silence.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/config"
|
||||
"github.com/kami/maven/internal/email"
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/phraser"
|
||||
"github.com/kami/maven/internal/store"
|
||||
)
|
||||
|
||||
// evidenceMaxChars — how much of the subject line is kept as a candidate's
|
||||
// evidence. Enough to recognise the mail on /tasks, not enough to turn the task
|
||||
// list into a copy of his mailbox.
|
||||
const evidenceMaxChars = 160
|
||||
|
||||
// mailIntake — extraction + capture for one message at a time.
|
||||
type mailIntake struct {
|
||||
st *store.Store
|
||||
ex *email.Extractor
|
||||
timeout time.Duration
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// newMailIntake returns nil when mail ingestion must not be available, which is
|
||||
// the default. Both preconditions are real:
|
||||
//
|
||||
// - no cfg.Email ⇒ not configured, and a capability is off unless configured;
|
||||
// - no llama-server phraser ⇒ nothing to extract with. There is deliberately
|
||||
// no keyword fallback: "the subject line became a task" is not extraction,
|
||||
// it is a mailbox rendered as a to-do list, and it would fill the review
|
||||
// page faster than he could clear it.
|
||||
func newMailIntake(st *store.Store, phr phraser.Phraser, cfg *config.Config) *mailIntake {
|
||||
if cfg.Email == nil {
|
||||
return nil
|
||||
}
|
||||
lp, ok := phr.(*phraser.LLMPhraser)
|
||||
if !ok {
|
||||
log.Printf("mail intake: configured but no llama-server phraser — mail ingestion disabled")
|
||||
return nil
|
||||
}
|
||||
timeout := time.Duration(cfg.Email.Timeout)
|
||||
if timeout <= 0 {
|
||||
timeout = config.DefaultEmailTimeout
|
||||
}
|
||||
ex := email.NewExtractor(llmClientFor(lp, timeout), cfg.Email.MaxTasks, contextBlockFn(cfg, time.Now))
|
||||
log.Printf("mail intake: enabled (max %d candidates per message, timeout %s)", cfg.Email.MaxTasks, timeout)
|
||||
return &mailIntake{st: st, ex: ex, timeout: timeout, now: time.Now}
|
||||
}
|
||||
|
||||
// ingest handles one ipc.MethodIngestMail call.
|
||||
//
|
||||
// Junk and empty messages are answered Skipped without touching the model — the
|
||||
// reader's header filter is what keeps the resident model off newsletters.
|
||||
//
|
||||
// Every candidate is captured with Status "candidate", Source "email:<mailbox>"
|
||||
// and the subject as Evidence. CaptureTask dedupes on normalised text among
|
||||
// live rows, so a mailbox re-read after a restart produces Created=0 rather
|
||||
// than a second copy of every task.
|
||||
func (m *mailIntake) ingest(ctx context.Context, req ipc.IngestMailReq) (ipc.IngestMailResp, error) {
|
||||
msg := email.Message{
|
||||
UID: req.UID,
|
||||
From: req.From,
|
||||
Subject: req.Subject,
|
||||
Date: req.Date,
|
||||
Body: req.Body,
|
||||
Junk: req.Junk,
|
||||
}
|
||||
if msg.Junk || (msg.Subject == "" && msg.Body == "") {
|
||||
return ipc.IngestMailResp{Skipped: true}, nil
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, m.timeout)
|
||||
defer cancel()
|
||||
cands, err := m.ex.Extract(ctx, msg)
|
||||
if err != nil {
|
||||
// The error from internal/email never carries mail text; keep it that way
|
||||
// by not adding the subject here.
|
||||
return ipc.IngestMailResp{}, fmt.Errorf("mail intake: uid %d: %w", req.UID, err)
|
||||
}
|
||||
if len(cands) == 0 {
|
||||
return ipc.IngestMailResp{}, nil
|
||||
}
|
||||
|
||||
source := email.SourcePrefix + req.Mailbox
|
||||
evidence := truncateRunes(req.Subject, evidenceMaxChars)
|
||||
now := m.now()
|
||||
var resp ipc.IngestMailResp
|
||||
for _, c := range cands {
|
||||
t := store.Task{
|
||||
CreatedTs: now,
|
||||
Text: c.Text,
|
||||
Source: source,
|
||||
Evidence: evidence,
|
||||
// The one status this path may ever write. Anything Maven derived from
|
||||
// something she read is a suggestion until he confirms it on /tasks.
|
||||
Status: store.TaskCandidate,
|
||||
}
|
||||
if due, ok := email.ParseDue(c.Due); ok {
|
||||
t.Due = &due
|
||||
}
|
||||
id, created, err := m.st.CaptureTask(ctx, t)
|
||||
if err != nil {
|
||||
return resp, fmt.Errorf("mail intake: capture: %w", err)
|
||||
}
|
||||
resp.TaskIDs = append(resp.TaskIDs, id)
|
||||
if created {
|
||||
resp.Created++
|
||||
}
|
||||
}
|
||||
// Counts only: the log line names the mailbox and the UID, never the subject,
|
||||
// the sender or the task text. Reviewing a candidate is what /tasks is for.
|
||||
log.Printf("mail intake: %s uid %d → %d candidate(s), %d new", source, req.UID, len(cands), resp.Created)
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// wireMailIntake installs the IPC hook, or leaves it nil so the method reports
|
||||
// ErrUnknownMethod. Called on both startup paths (unlocked boot and passkey
|
||||
// unlock) so mail behaves the same either way.
|
||||
func wireMailIntake(srv *ipc.Server, st *store.Store, phr phraser.Phraser, cfg *config.Config) {
|
||||
mi := newMailIntake(st, phr, cfg)
|
||||
if mi == nil {
|
||||
return
|
||||
}
|
||||
srv.IngestMailFn = mi.ingest
|
||||
}
|
||||
|
||||
// truncateRunes cuts a string to n runes, marking the cut.
|
||||
func truncateRunes(s string, n int) string {
|
||||
r := []rune(s)
|
||||
if len(r) <= n {
|
||||
return s
|
||||
}
|
||||
return string(r[:n]) + "…"
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/config"
|
||||
"github.com/kami/maven/internal/email"
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/llm"
|
||||
"github.com/kami/maven/internal/store"
|
||||
)
|
||||
|
||||
// mailLLM — a canned extraction reply.
|
||||
type mailLLM struct {
|
||||
reply string
|
||||
calls int
|
||||
}
|
||||
|
||||
func (m *mailLLM) Complete(_ context.Context, _ llm.Req) (string, error) {
|
||||
m.calls++
|
||||
return m.reply, nil
|
||||
}
|
||||
|
||||
func newTestIntake(t *testing.T, reply string) (*mailIntake, *store.Store, *mailLLM) {
|
||||
t.Helper()
|
||||
st := newTestStore(t)
|
||||
fake := &mailLLM{reply: reply}
|
||||
return &mailIntake{
|
||||
st: st,
|
||||
ex: email.NewExtractor(fake, 0, nil),
|
||||
timeout: 5 * time.Second,
|
||||
now: func() time.Time { return time.Date(2026, 8, 1, 10, 0, 0, 0, time.UTC) },
|
||||
}, st, fake
|
||||
}
|
||||
|
||||
func ingestReq() ipc.IngestMailReq {
|
||||
return ipc.IngestMailReq{
|
||||
Mailbox: "INBOX", UID: 42,
|
||||
From: "billing@isp.example",
|
||||
Subject: "Счёт за интернет",
|
||||
Body: "Оплатите счёт до 5 августа.",
|
||||
}
|
||||
}
|
||||
|
||||
// The one property that matters: a mail-derived task is a candidate, attributed
|
||||
// to the mailbox, with the subject as reviewable evidence — and nothing else is
|
||||
// written.
|
||||
func TestIngestCapturesCandidates(t *testing.T) {
|
||||
mi, st, _ := newTestIntake(t, `[{"text":"оплатить счёт за интернет","due":"2026-08-05"}]`)
|
||||
resp, err := mi.ingest(context.Background(), ingestReq())
|
||||
if err != nil {
|
||||
t.Fatalf("ingest: %v", err)
|
||||
}
|
||||
if resp.Created != 1 || len(resp.TaskIDs) != 1 {
|
||||
t.Fatalf("resp = %+v, want one created task", resp)
|
||||
}
|
||||
tasks, err := st.ListTasks(context.Background(), "")
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if len(tasks) != 1 {
|
||||
t.Fatalf("got %d tasks, want 1", len(tasks))
|
||||
}
|
||||
got := tasks[0]
|
||||
if got.Status != store.TaskCandidate {
|
||||
t.Errorf("status = %q, want %q — mail may only produce candidates", got.Status, store.TaskCandidate)
|
||||
}
|
||||
if got.Source != "email:INBOX" {
|
||||
t.Errorf("source = %q, want email:INBOX", got.Source)
|
||||
}
|
||||
if got.Evidence != "Счёт за интернет" {
|
||||
t.Errorf("evidence = %q, want the subject line", got.Evidence)
|
||||
}
|
||||
if got.Due == nil || got.Due.Format("2006-01-02") != "2026-08-05" {
|
||||
t.Errorf("due = %v, want 2026-08-05", got.Due)
|
||||
}
|
||||
// Nothing else may have been written: no reminder, no fact.
|
||||
rem, err := st.ListReminders(context.Background(), 10)
|
||||
if err != nil {
|
||||
t.Fatalf("list reminders: %v", err)
|
||||
}
|
||||
if len(rem) != 0 {
|
||||
t.Errorf("mail created %d reminders; a misread mail must never be able to fire", len(rem))
|
||||
}
|
||||
}
|
||||
|
||||
// Re-reading a mailbox must not grow the list — CaptureTask dedupes among live
|
||||
// rows, and the intake relies on exactly that.
|
||||
func TestIngestSameMailTwiceIsIdempotent(t *testing.T) {
|
||||
mi, st, _ := newTestIntake(t, `[{"text":"оплатить счёт","due":""}]`)
|
||||
if _, err := mi.ingest(context.Background(), ingestReq()); err != nil {
|
||||
t.Fatalf("first ingest: %v", err)
|
||||
}
|
||||
resp, err := mi.ingest(context.Background(), ingestReq())
|
||||
if err != nil {
|
||||
t.Fatalf("second ingest: %v", err)
|
||||
}
|
||||
if resp.Created != 0 || len(resp.TaskIDs) != 1 {
|
||||
t.Errorf("resp = %+v, want the existing row and Created=0", resp)
|
||||
}
|
||||
tasks, _ := st.ListTasks(context.Background(), "")
|
||||
if len(tasks) != 1 {
|
||||
t.Errorf("got %d tasks after two reads, want 1", len(tasks))
|
||||
}
|
||||
}
|
||||
|
||||
func TestIngestJunkSkipsTheModel(t *testing.T) {
|
||||
mi, st, fake := newTestIntake(t, `[{"text":"купить со скидкой","due":""}]`)
|
||||
req := ingestReq()
|
||||
req.Junk = true
|
||||
resp, err := mi.ingest(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("ingest: %v", err)
|
||||
}
|
||||
if !resp.Skipped || resp.Created != 0 {
|
||||
t.Errorf("resp = %+v, want skipped", resp)
|
||||
}
|
||||
if fake.calls != 0 {
|
||||
t.Errorf("model called %d times for junk, want 0", fake.calls)
|
||||
}
|
||||
if tasks, _ := st.ListTasks(context.Background(), ""); len(tasks) != 0 {
|
||||
t.Errorf("junk produced %d tasks, want 0", len(tasks))
|
||||
}
|
||||
}
|
||||
|
||||
func TestIngestEmptyMessageSkipped(t *testing.T) {
|
||||
mi, _, fake := newTestIntake(t, "[]")
|
||||
resp, err := mi.ingest(context.Background(), ipc.IngestMailReq{Mailbox: "INBOX", UID: 1})
|
||||
if err != nil || !resp.Skipped {
|
||||
t.Fatalf("resp = %+v, err = %v; want skipped", resp, err)
|
||||
}
|
||||
if fake.calls != 0 {
|
||||
t.Errorf("model called %d times for an empty message, want 0", fake.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIngestNoTasksWritesNothing(t *testing.T) {
|
||||
mi, st, _ := newTestIntake(t, "[]")
|
||||
resp, err := mi.ingest(context.Background(), ingestReq())
|
||||
if err != nil {
|
||||
t.Fatalf("ingest: %v", err)
|
||||
}
|
||||
if resp.Created != 0 || len(resp.TaskIDs) != 0 || resp.Skipped {
|
||||
t.Errorf("resp = %+v, want nothing captured and not skipped", resp)
|
||||
}
|
||||
if tasks, _ := st.ListTasks(context.Background(), ""); len(tasks) != 0 {
|
||||
t.Errorf("got %d tasks, want 0", len(tasks))
|
||||
}
|
||||
}
|
||||
|
||||
func TestIngestTruncatesEvidence(t *testing.T) {
|
||||
mi, st, _ := newTestIntake(t, `[{"text":"дело","due":""}]`)
|
||||
req := ingestReq()
|
||||
req.Subject = strings.Repeat("щ", 400)
|
||||
if _, err := mi.ingest(context.Background(), req); err != nil {
|
||||
t.Fatalf("ingest: %v", err)
|
||||
}
|
||||
tasks, _ := st.ListTasks(context.Background(), "")
|
||||
if len(tasks) != 1 {
|
||||
t.Fatalf("got %d tasks, want 1", len(tasks))
|
||||
}
|
||||
if n := len([]rune(tasks[0].Evidence)); n > evidenceMaxChars+1 {
|
||||
t.Errorf("evidence kept %d runes, want ≤ %d", n, evidenceMaxChars)
|
||||
}
|
||||
}
|
||||
|
||||
// Off unless configured: no email block ⇒ no intake, so the IPC method does not
|
||||
// exist at all.
|
||||
func TestNewMailIntakeOffWithoutConfig(t *testing.T) {
|
||||
st := newTestStore(t)
|
||||
if mi := newMailIntake(st, nil, &config.Config{}); mi != nil {
|
||||
t.Error("no email block must mean no mail intake")
|
||||
}
|
||||
// Configured but with a non-LLM phraser: still off — there is no fallback
|
||||
// extraction, by design.
|
||||
if mi := newMailIntake(st, nil, &config.Config{Email: &config.EmailConfig{}}); mi != nil {
|
||||
t.Error("without a llama-server phraser there is nothing to extract with")
|
||||
}
|
||||
}
|
||||
@@ -170,6 +170,8 @@ func run(args []string) error {
|
||||
eco *ecosystemWiring
|
||||
factWorker *factEnrichmentWorker
|
||||
evalWorker *memoryEvalWorker // nil ⇒ memory evaluation off (the default)
|
||||
feedWkr *feedWorker // nil ⇒ no feed is read (the default)
|
||||
crawlWkr *crawlWorker // nil ⇒ no page is watched (the default)
|
||||
)
|
||||
|
||||
if !locked {
|
||||
@@ -265,6 +267,8 @@ func run(args []string) error {
|
||||
tl = newTickLoop(st, gatherer, dispatcher, phr, rules, tickInterval, repeatInterval, autotuneInterval, cfg.Digest, routinesFromConfig(cfg.Routines), config.MorningRoutinesFromConfig(cfg.MorningRoutines), cfg.PatternProposals)
|
||||
factWorker = newFactEnrichmentWorker(st, eco, time.Duration(cfg.FactEnrichmentInterval))
|
||||
evalWorker = newMemoryEvalWorker(st, phr, cfg)
|
||||
feedWkr = newFeedWorker(ipc.NewStoreAPI(st), embedderOf(voiceW), cfg)
|
||||
crawlWkr = newCrawlWorker(newCrawler(cfg), ipc.NewStoreAPI(st), embedderOf(voiceW), cfg)
|
||||
|
||||
coreAPI = &daemonAPI{
|
||||
CoreAPI: ipc.NewStoreAPI(st),
|
||||
@@ -320,6 +324,14 @@ func run(args []string) error {
|
||||
|
||||
srv.StepUp = func(ctx context.Context) error { return passkeySess.Assert(ctx, auth.Scope{}) }
|
||||
|
||||
// Mail ingestion (Vikunja #246): the hook stays nil unless an email block is
|
||||
// configured and there is a llama-server to extract with, in which case
|
||||
// ipc.MethodIngestMail reports ErrUnknownMethod.
|
||||
if !locked {
|
||||
wireMailIntake(srv, st, phr, cfg)
|
||||
wireModelSwap(srv, phr, cfg)
|
||||
}
|
||||
|
||||
// WrapKeyFn — wraps the env key with a passkey credential public key and
|
||||
// persists the wrapped blob. Only wired when the daemon has the key in
|
||||
// memory (env key mode). Called by mavweb after passkey enrollment.
|
||||
@@ -444,6 +456,8 @@ func run(args []string) error {
|
||||
tl = newTickLoop(st, gatherer, dispatcher, phr, rules, tickInterval, repeatInterval, autotuneInterval, cfg.Digest, routinesFromConfig(cfg.Routines), config.MorningRoutinesFromConfig(cfg.MorningRoutines), cfg.PatternProposals)
|
||||
factWorker = newFactEnrichmentWorker(st, eco, time.Duration(cfg.FactEnrichmentInterval))
|
||||
evalWorker = newMemoryEvalWorker(st, phr, cfg)
|
||||
feedWkr = newFeedWorker(ipc.NewStoreAPI(st), embedderOf(voiceW), cfg)
|
||||
crawlWkr = newCrawlWorker(newCrawler(cfg), ipc.NewStoreAPI(st), embedderOf(voiceW), cfg)
|
||||
|
||||
// Swap the CoreAPI from the locked placeholder to the real store adapter.
|
||||
newAPI := &daemonAPI{
|
||||
@@ -457,6 +471,8 @@ func run(args []string) error {
|
||||
}
|
||||
srv.SetAPI(newAPI)
|
||||
srv.Check = (&auth.Gate{Enrollment: auth.NewFloorEnrollment(), Session: passkeySess}).Check
|
||||
wireMailIntake(srv, st, phr, cfg)
|
||||
wireModelSwap(srv, phr, cfg)
|
||||
|
||||
// Start voice server.
|
||||
if voiceW != nil {
|
||||
@@ -488,6 +504,20 @@ func run(args []string) error {
|
||||
}()
|
||||
}
|
||||
|
||||
// Start feed reading (nil unless configured).
|
||||
if feedWkr != nil {
|
||||
go func() {
|
||||
feedWkr.run(ctx)
|
||||
}()
|
||||
}
|
||||
|
||||
// Start the watched-page crawls (nil unless configured).
|
||||
if crawlWkr != nil {
|
||||
go func() {
|
||||
crawlWkr.run(ctx)
|
||||
}()
|
||||
}
|
||||
|
||||
dl.unlock()
|
||||
log.Printf("mavend: unlocked via passkey assertion")
|
||||
return nil
|
||||
@@ -533,6 +563,20 @@ func run(args []string) error {
|
||||
evalWorker.run(ctx)
|
||||
}()
|
||||
}
|
||||
if feedWkr != nil {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
feedWkr.run(ctx)
|
||||
}()
|
||||
}
|
||||
if crawlWkr != nil {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
crawlWkr.run(ctx)
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
<-ctx.Done()
|
||||
|
||||
@@ -16,7 +16,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/config"
|
||||
"github.com/kami/maven/internal/llm"
|
||||
"github.com/kami/maven/internal/memeval"
|
||||
"github.com/kami/maven/internal/phraser"
|
||||
"github.com/kami/maven/internal/store"
|
||||
@@ -50,7 +49,7 @@ func newMemoryEvalWorker(st *store.Store, phr phraser.Phraser, cfg *config.Confi
|
||||
}
|
||||
// A generous per-request timeout: this is a long prompt to a Thinking model
|
||||
// and nobody is waiting on the answer.
|
||||
client := llm.New(lp.BaseURL(), 5*time.Minute)
|
||||
client := llmClientFor(lp, 5*time.Minute)
|
||||
ev := memeval.NewEvaluator(st, st, client, memeval.Config{
|
||||
MaxItems: cfg.MemoryEval.MaxItems,
|
||||
MinConfidence: cfg.MemoryEval.MinConfidence,
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/config"
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/llm"
|
||||
"github.com/kami/maven/internal/phraser"
|
||||
)
|
||||
|
||||
// Swapping the resident model while the daemon runs (Vikunja #250).
|
||||
//
|
||||
// Off unless configured: with no phraser.swap_models allowlist the two IPC
|
||||
// methods are never wired, so they answer ErrUnknownMethod. When it is wired the
|
||||
// swap method is AuthStepUp (internal/auth), which means an authed human surface
|
||||
// only — there is no act, no intent and no timer that reaches it. The daemon
|
||||
// never decides to change its own brain.
|
||||
//
|
||||
// The allowlist is exact-match against paths a human wrote in mavend.json. The
|
||||
// request carries a path and llama-server is started with it as `-m`, so
|
||||
// anything looser would turn "swap the model" into "load any file on my disk".
|
||||
func wireModelSwap(srv *ipc.Server, phr phraser.Phraser, cfg *config.Config) {
|
||||
if cfg.Phraser == nil || len(cfg.Phraser.SwapModels) == 0 {
|
||||
return
|
||||
}
|
||||
lp, ok := phr.(*phraser.LLMPhraser)
|
||||
if !ok {
|
||||
log.Printf("model swap: phraser.swap_models is set but there is no llama-server phraser — swap disabled")
|
||||
return
|
||||
}
|
||||
allowed := map[string]bool{}
|
||||
for _, m := range cfg.Phraser.SwapModels {
|
||||
allowed[filepath.Clean(m)] = true
|
||||
}
|
||||
// The configured model is always swappable back to, listed or not: the way
|
||||
// out of a bad swap must not depend on remembering to allowlist the model
|
||||
// you are already running.
|
||||
allowed[filepath.Clean(cfg.Phraser.ModelPath)] = true
|
||||
|
||||
srv.SwapModelFn = func(ctx context.Context, req ipc.SwapModelReq) (ipc.SwapModelResp, error) {
|
||||
path := filepath.Clean(req.ModelPath)
|
||||
if !allowed[path] {
|
||||
log.Printf("model swap: REFUSED %q — not in phraser.swap_models", req.ModelPath)
|
||||
return ipc.SwapModelResp{}, fmt.Errorf("%w: %q is not in phraser.swap_models", ipc.ErrForbidden, req.ModelPath)
|
||||
}
|
||||
res, err := lp.Swap(ctx, phraser.SwapSpec{
|
||||
ModelPath: path,
|
||||
NGpuLayers: req.NGpuLayers,
|
||||
NCtx: req.NCtx,
|
||||
})
|
||||
resp := ipc.SwapModelResp{
|
||||
Model: res.Model,
|
||||
ModelPath: res.ModelPath,
|
||||
BaseURL: res.BaseURL,
|
||||
RolledBack: res.RolledBack,
|
||||
TookMs: res.Took.Milliseconds(),
|
||||
}
|
||||
if err != nil {
|
||||
// A rolled-back swap is a failure that left a working daemon behind.
|
||||
// Both halves matter to the caller, so the response is filled in even
|
||||
// though the error is returned.
|
||||
log.Printf("model swap: %v", err)
|
||||
return resp, err
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
srv.ModelStatusFn = func(ctx context.Context) (ipc.ModelStatusResp, error) {
|
||||
path, ngl, nctx := lp.LiveModel()
|
||||
base := lp.BaseURL()
|
||||
resp := ipc.ModelStatusResp{
|
||||
ModelPath: path,
|
||||
BaseURL: base,
|
||||
NGpuLayers: ngl,
|
||||
NCtx: nctx,
|
||||
Swappable: cfg.Phraser.SwapModels,
|
||||
}
|
||||
if base == "" {
|
||||
resp.Model = llm.UnknownModel
|
||||
return resp, nil
|
||||
}
|
||||
id, err := llm.ModelID(ctx, base)
|
||||
if err != nil {
|
||||
// Report the honest "I could not confirm it" rather than echoing the
|
||||
// configured filename as if the server had said it.
|
||||
resp.Model = llm.UnknownModel
|
||||
return resp, nil
|
||||
}
|
||||
resp.Model = id
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
log.Printf("model swap: enabled, %d allowlisted model(s) — step-up required", len(cfg.Phraser.SwapModels))
|
||||
}
|
||||
|
||||
// llmClientFor builds a completion client on the phraser's llama-server and
|
||||
// keeps it pointed at the right one across a model swap.
|
||||
//
|
||||
// Without the OnSwap registration every holder of a base URL — the LLM router,
|
||||
// the replier, the mail extractor, the memory evaluator — would keep talking to
|
||||
// the port of a server that no longer exists, and the daemon would degrade to
|
||||
// the classifier permanently after the first swap. The client is re-pointed, not
|
||||
// rebuilt, so nothing that holds it has to know a swap happened.
|
||||
func llmClientFor(lp *phraser.LLMPhraser, timeout time.Duration) *llm.Client {
|
||||
c := llm.New(lp.BaseURL(), timeout)
|
||||
lp.OnSwap(func(base string) { c.SetBaseURL(base) })
|
||||
return c
|
||||
}
|
||||
@@ -52,6 +52,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/audio"
|
||||
"github.com/kami/maven/internal/crawl"
|
||||
"github.com/kami/maven/internal/dialogue"
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/memory"
|
||||
@@ -82,6 +83,15 @@ type reactiveHandler struct {
|
||||
replier voice.Replier
|
||||
now func() time.Time
|
||||
|
||||
// crawler reads a web page he names out loud (queryWeb). nil ⇒ on-demand
|
||||
// page reading is off, which is the default: no `crawl` block, no fetch.
|
||||
crawler *crawl.Crawler
|
||||
|
||||
// feedsOn — whether any RSS feed is configured (config.Feeds). It changes
|
||||
// only what she SAYS when asked and nothing is there: "ленты не настроены"
|
||||
// instead of "ничего нового", which are different truths.
|
||||
feedsOn bool
|
||||
|
||||
weatherProvider weather.Provider
|
||||
weatherLocation string // default location for weather queries
|
||||
|
||||
|
||||
+17
-11
@@ -148,7 +148,9 @@ func wireVoice(cfg *config.Config, coreAPI ipc.CoreAPI, phr phraser.Phraser, mem
|
||||
// The replier uses the same llama-server as the phraser.
|
||||
var llmClient *llm.Client
|
||||
if lp, ok := phr.(*phraser.LLMPhraser); ok {
|
||||
llmClient = llm.New(lp.BaseURL(), 60*time.Second)
|
||||
// llmClientFor, not llm.New: this client must follow the phraser onto
|
||||
// the new llama-server when the resident model is swapped (Vikunja #250).
|
||||
llmClient = llmClientFor(lp, 60*time.Second)
|
||||
}
|
||||
// ----- router (the cascade; floor examples seed the classifier) -----
|
||||
// The act matcher's allowlist is exactly the enabled tool names — the
|
||||
@@ -201,16 +203,20 @@ func wireVoice(cfg *config.Config, coreAPI ipc.CoreAPI, phr phraser.Phraser, mem
|
||||
|
||||
// ----- the handler (the reactive path; closes over stt / tts / router / coreAPI / memory) -----
|
||||
h := &reactiveHandler{
|
||||
stt: transcriber,
|
||||
tts: synthesizer,
|
||||
router: rtr,
|
||||
embedder: emb,
|
||||
api: coreAPI,
|
||||
tools: exec,
|
||||
matcher: matcher,
|
||||
replier: replier,
|
||||
phraser: phr,
|
||||
now: time.Now,
|
||||
stt: transcriber,
|
||||
tts: synthesizer,
|
||||
router: rtr,
|
||||
embedder: emb,
|
||||
api: coreAPI,
|
||||
tools: exec,
|
||||
matcher: matcher,
|
||||
replier: replier,
|
||||
phraser: phr,
|
||||
now: time.Now,
|
||||
feedsOn: cfg.Feeds != nil,
|
||||
// nil unless `crawl.on_demand` is on: reading a page he names is a
|
||||
// capability, and capabilities are off unless configured.
|
||||
crawler: onDemandCrawler(cfg),
|
||||
weatherProvider: weatherProvider,
|
||||
weatherLocation: weatherLocation,
|
||||
memStore: memStore,
|
||||
|
||||
@@ -0,0 +1,332 @@
|
||||
// mavmaild — the mail reader module (Vikunja #246,
|
||||
// docs/plans/01-email-reader.md).
|
||||
//
|
||||
// Every so often it opens one IMAP mailbox read-only, fetches the messages it
|
||||
// has not read yet, and hands each one to core over ipc.MethodIngestMail. Core
|
||||
// runs the extraction on the resident model and writes what comes back as task
|
||||
// CANDIDATES he reviews on /tasks. Nothing here writes to the store, nothing
|
||||
// here can create a reminder, and nothing here speaks.
|
||||
//
|
||||
// Why a separate daemon rather than a loop inside mavend, when extraction has
|
||||
// to happen in mavend anyway: the credential. mavpoll set the precedent with the
|
||||
// zenmoney token (#125) — the module that talks to a third party holds the
|
||||
// secret, reads it from a FILE so it never appears in `ps`, in
|
||||
// docker-compose.yml or in shell history, and core never sees it. Core learns
|
||||
// that mail exists only as message text on one IPC method; it cannot connect to
|
||||
// the mailbox even if it wanted to, and a compromised core yields no mail
|
||||
// password.
|
||||
//
|
||||
// Off unless configured: without -password-file there is nothing to run, and
|
||||
// the daemon says so and exits. If core has no `email` block the very first
|
||||
// ingest comes back ErrUnknownMethod and this daemon stops polling instead of
|
||||
// hammering a socket that will keep refusing.
|
||||
//
|
||||
// Mail is personal, so the log is counts and UIDs: how many messages were
|
||||
// fetched, how many were bulk, how many candidates came back. No subject, no
|
||||
// sender, no body, ever — reviewing a candidate is what /tasks is for.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/email"
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := run(os.Args[1:]); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "mavmaild:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(args []string) error {
|
||||
fs := flag.NewFlagSet("mavmaild", flag.ContinueOnError)
|
||||
socket := fs.String("socket", "", "core IPC socket path (required)")
|
||||
server := fs.String("imap", "", "IMAP server, host or host:993 (required)")
|
||||
user := fs.String("user", "", "IMAP username (required)")
|
||||
passFile := fs.String("password-file", "", "file holding the IMAP password (required — never passed as a flag value)")
|
||||
mailbox := fs.String("mailbox", "INBOX", "mailbox to read, read-only")
|
||||
interval := fs.Duration("interval", 15*time.Minute, "how often to read the mailbox")
|
||||
lookback := fs.Duration("lookback", 72*time.Hour, "how far back to search on each poll")
|
||||
max := fs.Int("max", 25, "most messages to fetch in one poll")
|
||||
timeout := fs.Duration("timeout", 30*time.Second, "IMAP network timeout")
|
||||
statePath := fs.String("state", "", "file remembering which UIDs were read (default: none — every poll re-reads the window)")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
if *socket == "" {
|
||||
return fmt.Errorf("-socket is required")
|
||||
}
|
||||
if *server == "" || *user == "" || *passFile == "" {
|
||||
return fmt.Errorf("mail reading is off unless configured: set -imap, -user and -password-file")
|
||||
}
|
||||
|
||||
// The password is read from a file, never taken as a flag value: an argv
|
||||
// secret is visible in `ps` to every user on the box and lands in the compose
|
||||
// file and the shell history. Read once at start — a rotated password means a
|
||||
// restart, which is cheaper than re-reading his credential every quarter hour.
|
||||
raw, err := os.ReadFile(*passFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read password file: %w", err)
|
||||
}
|
||||
password := strings.TrimSpace(string(raw))
|
||||
if password == "" {
|
||||
return fmt.Errorf("password file %s is empty", *passFile)
|
||||
}
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
core, err := ipc.DialWait(*socket, 60*time.Second)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer core.Close()
|
||||
|
||||
r := &reader{
|
||||
core: core,
|
||||
addr: *server,
|
||||
user: *user,
|
||||
mailbox: *mailbox,
|
||||
lookback: *lookback,
|
||||
max: *max,
|
||||
timeout: *timeout,
|
||||
state: newSeenState(*statePath),
|
||||
}
|
||||
if err := r.state.load(); err != nil {
|
||||
// A missing or corrupt state file must not stop mail from being read: the
|
||||
// worst case is re-reading the window, and capture dedupes on text.
|
||||
log.Printf("mavmaild: state: %v (starting from an empty seen-set)", err)
|
||||
}
|
||||
|
||||
// The password is never logged, not even its length.
|
||||
log.Printf("mavmaild: reading %s on %s every %s (lookback %s, max %d/poll)",
|
||||
*mailbox, *server, *interval, *lookback, *max)
|
||||
|
||||
r.pollOnce(ctx, password) // don't idle a full interval on start
|
||||
t := time.NewTicker(*interval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Printf("mavmaild: bye")
|
||||
return nil
|
||||
case <-t.C:
|
||||
if r.disabled {
|
||||
// Core told us mail ingestion is not configured. Nothing will change
|
||||
// without a core restart, and a restart restarts us too.
|
||||
log.Printf("mavmaild: core does not accept mail — idling")
|
||||
return nil
|
||||
}
|
||||
r.pollOnce(ctx, password)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// mailIngester — the slice of core this daemon uses. One method: hand over a
|
||||
// message. It cannot write a fact, create a reminder or read the store, and the
|
||||
// interface says so.
|
||||
type mailIngester interface {
|
||||
IngestMail(ctx context.Context, req ipc.IngestMailReq) (ipc.IngestMailResp, error)
|
||||
}
|
||||
|
||||
type reader struct {
|
||||
core mailIngester
|
||||
addr string
|
||||
user string
|
||||
mailbox string
|
||||
lookback time.Duration
|
||||
max int
|
||||
timeout time.Duration
|
||||
state *seenState
|
||||
|
||||
// dial — connection seam for the tests; nil ⇒ implicit TLS.
|
||||
dial func(addr string, timeout time.Duration) (*email.Conn, error)
|
||||
|
||||
// disabled — core answered ErrUnknownMethod, i.e. it has no email block.
|
||||
disabled bool
|
||||
}
|
||||
|
||||
// pollOnce — one read of the mailbox, then one ingest per message.
|
||||
//
|
||||
// A fetch error aborts this poll and nothing else; the next tick tries again.
|
||||
// An ingest error for one message does not skip the rest — one mail the model
|
||||
// choked on should not hide the four behind it.
|
||||
func (r *reader) pollOnce(ctx context.Context, password string) {
|
||||
msgs, err := r.fetch(password)
|
||||
if err != nil {
|
||||
// The error may name a UID; it never names a subject or a sender.
|
||||
log.Printf("mavmaild: fetch: %v", err)
|
||||
if len(msgs) == 0 {
|
||||
return
|
||||
}
|
||||
}
|
||||
var junk, candidates, created int
|
||||
for _, m := range msgs {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if m.Junk {
|
||||
junk++
|
||||
// Marked seen without a model call: the header filter already decided,
|
||||
// and re-classifying it every quarter hour would be pure waste.
|
||||
r.state.mark(m.UID)
|
||||
continue
|
||||
}
|
||||
resp, err := r.core.IngestMail(ctx, ipc.IngestMailReq{
|
||||
Mailbox: r.mailbox,
|
||||
UID: m.UID,
|
||||
From: m.From,
|
||||
Subject: m.Subject,
|
||||
Date: m.Date,
|
||||
Body: m.Body,
|
||||
})
|
||||
if errors.Is(err, ipc.ErrUnknownMethod) {
|
||||
log.Printf("mavmaild: core has no email block configured — mail ingestion is off; stopping")
|
||||
r.disabled = true
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
// Not marked seen: an ingest that failed should be retried next poll.
|
||||
log.Printf("mavmaild: ingest uid %d: %v", m.UID, err)
|
||||
continue
|
||||
}
|
||||
r.state.mark(m.UID)
|
||||
candidates += len(resp.TaskIDs)
|
||||
created += resp.Created
|
||||
}
|
||||
if err := r.state.save(); err != nil {
|
||||
log.Printf("mavmaild: state: %v", err)
|
||||
}
|
||||
log.Printf("mavmaild: %s: %d read, %d bulk, %d candidate(s), %d new", r.mailbox, len(msgs), junk, candidates, created)
|
||||
}
|
||||
|
||||
// fetch reads the mailbox. Messages already in the seen-set are not fetched at
|
||||
// all, so a steady mailbox costs one SEARCH per poll and nothing else.
|
||||
func (r *reader) fetch(password string) ([]email.Message, error) {
|
||||
f := email.FetchSince{
|
||||
Addr: r.addr,
|
||||
User: r.user,
|
||||
Mailbox: r.mailbox,
|
||||
Timeout: r.timeout,
|
||||
Since: time.Now().Add(-r.lookback),
|
||||
Max: r.max,
|
||||
Skip: r.state.seen,
|
||||
}
|
||||
return f.RunWith(password, r.dial)
|
||||
}
|
||||
|
||||
// ---- seen state ------------------------------------------------------------
|
||||
|
||||
// seenState — the UIDs already handed to core, persisted so a restart does not
|
||||
// re-read (and re-extract, at multi-second LLM cost) the whole lookback window.
|
||||
//
|
||||
// Correctness does not depend on it: ipc.CaptureTask dedupes on normalised text
|
||||
// among live tasks, so a re-read produces no duplicate rows. This exists to save
|
||||
// the model's time, which is why a broken state file is a log line rather than a
|
||||
// failure.
|
||||
//
|
||||
// UIDs are per-mailbox and monotonic, so the set is kept as a high-water mark
|
||||
// plus the stragglers above it. If the server ever changes UIDVALIDITY, UIDs
|
||||
// reset and the window is simply re-read once — dedupe absorbs it.
|
||||
type seenState struct {
|
||||
path string
|
||||
high uint32
|
||||
set map[uint32]bool
|
||||
dirty bool
|
||||
}
|
||||
|
||||
func newSeenState(path string) *seenState {
|
||||
return &seenState{path: path, set: map[uint32]bool{}}
|
||||
}
|
||||
|
||||
type seenFile struct {
|
||||
High uint32 `json:"high"`
|
||||
UIDs []uint32 `json:"uids,omitempty"`
|
||||
}
|
||||
|
||||
func (s *seenState) seen(uid uint32) bool {
|
||||
return uid <= s.high || s.set[uid]
|
||||
}
|
||||
|
||||
func (s *seenState) mark(uid uint32) {
|
||||
if s.seen(uid) {
|
||||
return
|
||||
}
|
||||
s.set[uid] = true
|
||||
s.dirty = true
|
||||
// Advance the high-water mark through any contiguous run, so the explicit set
|
||||
// stays small on a mailbox read in order.
|
||||
for {
|
||||
next := s.high + 1
|
||||
if !s.set[next] {
|
||||
break
|
||||
}
|
||||
delete(s.set, next)
|
||||
s.high = next
|
||||
}
|
||||
}
|
||||
|
||||
func (s *seenState) load() error {
|
||||
if s.path == "" {
|
||||
return nil
|
||||
}
|
||||
b, err := os.ReadFile(s.path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil // first run
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var f seenFile
|
||||
if err := json.Unmarshal(b, &f); err != nil {
|
||||
return fmt.Errorf("parse %s: %w", s.path, err)
|
||||
}
|
||||
s.high = f.High
|
||||
for _, u := range f.UIDs {
|
||||
s.set[u] = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// save writes the state atomically (temp file + rename), 0600: it is a list of
|
||||
// message ids from his mailbox, which is metadata about his mail.
|
||||
func (s *seenState) save() error {
|
||||
if s.path == "" || !s.dirty {
|
||||
return nil
|
||||
}
|
||||
uids := make([]uint32, 0, len(s.set))
|
||||
for u := range s.set {
|
||||
uids = append(uids, u)
|
||||
}
|
||||
sort.Slice(uids, func(i, j int) bool { return uids[i] < uids[j] })
|
||||
b, err := json.Marshal(seenFile{High: s.high, UIDs: uids})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := s.path + ".tmp"
|
||||
if err := os.MkdirAll(filepath.Dir(s.path), 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(tmp, b, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(tmp, s.path); err != nil {
|
||||
return err
|
||||
}
|
||||
s.dirty = false
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/email"
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
)
|
||||
|
||||
// ---- a scripted IMAP server, same shape internal/email's tests use ---------
|
||||
|
||||
type fakeIMAP struct {
|
||||
msgs map[uint32]string
|
||||
uids []uint32
|
||||
cmds []string
|
||||
}
|
||||
|
||||
func (f *fakeIMAP) serve(c net.Conn) {
|
||||
defer c.Close()
|
||||
fmt.Fprint(c, "* OK fake ready\r\n")
|
||||
r := bufio.NewReader(c)
|
||||
for {
|
||||
line, err := r.ReadString('\n')
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
parts := strings.SplitN(strings.TrimRight(line, "\r\n"), " ", 2)
|
||||
if len(parts) != 2 {
|
||||
return
|
||||
}
|
||||
tag, cmd := parts[0], parts[1]
|
||||
f.cmds = append(f.cmds, cmd)
|
||||
upper := strings.ToUpper(cmd)
|
||||
switch {
|
||||
case strings.HasPrefix(upper, "LOGIN"), strings.HasPrefix(upper, "EXAMINE"):
|
||||
fmt.Fprintf(c, "%s OK\r\n", tag)
|
||||
case strings.HasPrefix(upper, "UID SEARCH"):
|
||||
var ids []string
|
||||
for _, u := range f.uids {
|
||||
ids = append(ids, strconv.FormatUint(uint64(u), 10))
|
||||
}
|
||||
fmt.Fprintf(c, "* SEARCH %s\r\n%s OK\r\n", strings.Join(ids, " "), tag)
|
||||
case strings.HasPrefix(upper, "UID FETCH"):
|
||||
uid, _ := strconv.ParseUint(strings.Fields(cmd)[2], 10, 32)
|
||||
if raw, ok := f.msgs[uint32(uid)]; ok {
|
||||
fmt.Fprintf(c, "* 1 FETCH (UID %d BODY[] {%d}\r\n%s)\r\n", uid, len(raw), raw)
|
||||
}
|
||||
fmt.Fprintf(c, "%s OK\r\n", tag)
|
||||
case strings.HasPrefix(upper, "LOGOUT"):
|
||||
fmt.Fprintf(c, "* BYE\r\n%s OK\r\n", tag)
|
||||
return
|
||||
default:
|
||||
fmt.Fprintf(c, "%s BAD\r\n", tag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeIMAP) dial(_ string, timeout time.Duration) (*email.Conn, error) {
|
||||
cli, srv := net.Pipe()
|
||||
go f.serve(srv)
|
||||
return email.NewConn(cli, timeout)
|
||||
}
|
||||
|
||||
// ---- a fake core -----------------------------------------------------------
|
||||
|
||||
type fakeCore struct {
|
||||
got []ipc.IngestMailReq
|
||||
resp ipc.IngestMailResp
|
||||
err error
|
||||
}
|
||||
|
||||
func (c *fakeCore) IngestMail(_ context.Context, req ipc.IngestMailReq) (ipc.IngestMailResp, error) {
|
||||
c.got = append(c.got, req)
|
||||
if c.err != nil {
|
||||
return ipc.IngestMailResp{}, c.err
|
||||
}
|
||||
return c.resp, nil
|
||||
}
|
||||
|
||||
func mail(subject, body string, extraHeaders ...string) string {
|
||||
h := "Subject: " + subject + "\r\nFrom: a@b.c\r\nContent-Type: text/plain; charset=utf-8\r\n"
|
||||
for _, e := range extraHeaders {
|
||||
h += e + "\r\n"
|
||||
}
|
||||
return h + "\r\n" + body + "\r\n"
|
||||
}
|
||||
|
||||
func newTestReader(t *testing.T, f *fakeIMAP, core *fakeCore, statePath string) *reader {
|
||||
t.Helper()
|
||||
return &reader{
|
||||
core: core, addr: "mail.example:993", user: "kami", mailbox: "INBOX",
|
||||
lookback: 72 * time.Hour, max: 25, timeout: 5 * time.Second,
|
||||
state: newSeenState(statePath),
|
||||
dial: f.dial,
|
||||
}
|
||||
}
|
||||
|
||||
func TestPollHandsMessagesToCore(t *testing.T) {
|
||||
f := &fakeIMAP{
|
||||
uids: []uint32{1, 2},
|
||||
msgs: map[uint32]string{
|
||||
1: mail("Счёт", "Оплатить до 5 августа."),
|
||||
2: mail("Скидки", "Sale!", "List-Unsubscribe: <mailto:u@x>"),
|
||||
},
|
||||
}
|
||||
core := &fakeCore{resp: ipc.IngestMailResp{TaskIDs: []int64{1}, Created: 1}}
|
||||
r := newTestReader(t, f, core, "")
|
||||
r.pollOnce(context.Background(), "secret")
|
||||
|
||||
// The newsletter is filtered before core is asked: only the real mail crosses.
|
||||
if len(core.got) != 1 {
|
||||
t.Fatalf("core saw %d messages, want 1 (the bulk one must not cross): %+v", len(core.got), core.got)
|
||||
}
|
||||
got := core.got[0]
|
||||
if got.UID != 1 || got.Mailbox != "INBOX" || got.Subject != "Счёт" {
|
||||
t.Errorf("ingest req = %+v", got)
|
||||
}
|
||||
if !strings.Contains(got.Body, "Оплатить") {
|
||||
t.Errorf("body = %q", got.Body)
|
||||
}
|
||||
}
|
||||
|
||||
// A second poll must not re-send what core already saw — extraction is a
|
||||
// multi-second LLM call per message.
|
||||
func TestPollSkipsSeenUIDs(t *testing.T) {
|
||||
f := &fakeIMAP{uids: []uint32{5}, msgs: map[uint32]string{5: mail("Счёт", "текст")}}
|
||||
core := &fakeCore{}
|
||||
r := newTestReader(t, f, core, "")
|
||||
r.pollOnce(context.Background(), "secret")
|
||||
r.pollOnce(context.Background(), "secret")
|
||||
if len(core.got) != 1 {
|
||||
t.Errorf("core saw %d messages over two polls, want 1", len(core.got))
|
||||
}
|
||||
}
|
||||
|
||||
// An ingest that failed is NOT marked seen: the next poll retries it.
|
||||
func TestPollRetriesFailedIngest(t *testing.T) {
|
||||
f := &fakeIMAP{uids: []uint32{5}, msgs: map[uint32]string{5: mail("Счёт", "текст")}}
|
||||
core := &fakeCore{err: fmt.Errorf("llama-server is warming up")}
|
||||
r := newTestReader(t, f, core, "")
|
||||
r.pollOnce(context.Background(), "secret")
|
||||
core.err = nil
|
||||
r.pollOnce(context.Background(), "secret")
|
||||
if len(core.got) != 2 {
|
||||
t.Errorf("core saw %d attempts, want 2 (a failed ingest is retried)", len(core.got))
|
||||
}
|
||||
}
|
||||
|
||||
// Core without an email block ⇒ stop, don't hammer the socket.
|
||||
func TestPollStopsWhenCoreRefusesMail(t *testing.T) {
|
||||
f := &fakeIMAP{uids: []uint32{1, 2}, msgs: map[uint32]string{1: mail("a", "b"), 2: mail("c", "d")}}
|
||||
core := &fakeCore{err: fmt.Errorf("call: %w", ipc.ErrUnknownMethod)}
|
||||
r := newTestReader(t, f, core, "")
|
||||
r.pollOnce(context.Background(), "secret")
|
||||
if !r.disabled {
|
||||
t.Error("ErrUnknownMethod must disable the reader")
|
||||
}
|
||||
if len(core.got) != 1 {
|
||||
t.Errorf("core saw %d messages, want 1 — stop at the first refusal", len(core.got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeenStatePersists(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "state", "seen.json")
|
||||
f := &fakeIMAP{uids: []uint32{9}, msgs: map[uint32]string{9: mail("Счёт", "текст")}}
|
||||
core := &fakeCore{}
|
||||
r := newTestReader(t, f, core, path)
|
||||
r.pollOnce(context.Background(), "secret")
|
||||
|
||||
fi, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatalf("state file: %v", err)
|
||||
}
|
||||
// A list of message ids from his mailbox is metadata about his mail.
|
||||
if perm := fi.Mode().Perm(); perm != 0o600 {
|
||||
t.Errorf("state file mode = %v, want 0600", perm)
|
||||
}
|
||||
|
||||
// A fresh reader with the same state file must not re-read the message.
|
||||
core2 := &fakeCore{}
|
||||
r2 := newTestReader(t, f, core2, path)
|
||||
if err := r2.state.load(); err != nil {
|
||||
t.Fatalf("load: %v", err)
|
||||
}
|
||||
r2.pollOnce(context.Background(), "secret")
|
||||
if len(core2.got) != 0 {
|
||||
t.Errorf("after a restart core saw %d messages, want 0", len(core2.got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeenStateHighWaterMark(t *testing.T) {
|
||||
s := newSeenState("")
|
||||
s.mark(1)
|
||||
s.mark(3)
|
||||
s.mark(2)
|
||||
if s.high != 3 {
|
||||
t.Errorf("high = %d, want 3 (contiguous run collapses)", s.high)
|
||||
}
|
||||
if len(s.set) != 0 {
|
||||
t.Errorf("explicit set = %v, want empty", s.set)
|
||||
}
|
||||
if !s.seen(2) || s.seen(4) {
|
||||
t.Errorf("seen(2)=%v seen(4)=%v", s.seen(2), s.seen(4))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeenStateCorruptFileIsNotFatal(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "seen.json")
|
||||
if err := os.WriteFile(path, []byte("{not json"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := newSeenState(path)
|
||||
if err := s.load(); err == nil {
|
||||
t.Error("a corrupt state file should report an error the caller logs")
|
||||
}
|
||||
if s.seen(1) {
|
||||
t.Error("a corrupt state file must leave an empty seen-set, not a poisoned one")
|
||||
}
|
||||
}
|
||||
|
||||
// Off unless configured, and the credential is never a flag value.
|
||||
func TestRunRequiresConfig(t *testing.T) {
|
||||
if err := run([]string{}); err == nil {
|
||||
t.Error("no -socket must be an error")
|
||||
}
|
||||
if err := run([]string{"-socket", "/tmp/nope.sock"}); err == nil {
|
||||
t.Error("no mailbox configuration must be an error, not a default mailbox")
|
||||
}
|
||||
// There is no -password flag at all: only -password-file.
|
||||
if err := run([]string{"-socket", "/x", "-imap", "h", "-user", "u", "-password", "p"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "flag provided but not defined") {
|
||||
t.Errorf("a -password flag must not exist; err = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRejectsEmptyPasswordFile(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "pass")
|
||||
if err := os.WriteFile(path, []byte(" \n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err := run([]string{"-socket", "/x/y.sock", "-imap", "h", "-user", "u", "-password-file", path})
|
||||
if err == nil || !strings.Contains(err.Error(), "empty") {
|
||||
t.Errorf("an empty password file must be refused before dialling; err = %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
// Command mavupdate deploys a new build of Maven to the box she runs on, with
|
||||
// an automatic rollback when the new build does not come up (Vikunja #249).
|
||||
//
|
||||
// It is a CLI on purpose, and it is the ONLY trigger for the update path.
|
||||
//
|
||||
// The obvious design — an IPC method plus a button on the web UI behind the
|
||||
// step-up passkey gate, the way /tools works — was considered and refused. A
|
||||
// step-up gate protects against the wrong person clicking; it does not change
|
||||
// the fact that anything reachable over the network becomes, in the event of a
|
||||
// mavweb bug, a remote arbitrary-code path with a build system attached. An
|
||||
// update needs shell access on the host, which is a strictly higher bar than
|
||||
// the gate that guards the tool allowlist. That is deliberate and it is the
|
||||
// reason there is no MethodApplyUpdate anywhere in internal/ipc.
|
||||
//
|
||||
// Consequently: mavend does not import internal/update, nothing runs on a timer,
|
||||
// nothing checks a release server, and no act, intent, tool or LLM output can
|
||||
// reach any of this. She cannot update herself. She can be updated, by him.
|
||||
//
|
||||
// mavupdate -config deploy/mavend.json list # snapshots available to roll back to
|
||||
// mavupdate -config deploy/mavend.json verify # make build + make test, deploys nothing
|
||||
// mavupdate -config deploy/mavend.json apply -yes # the whole thing
|
||||
// mavupdate -config deploy/mavend.json rollback [id] # restore + restart (default: newest)
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/config"
|
||||
"github.com/kami/maven/internal/update"
|
||||
)
|
||||
|
||||
func main() {
|
||||
cfgPath := flag.String("config", "deploy/mavend.json", "path to mavend.json (the update block is read from it)")
|
||||
yes := flag.Bool("yes", false, "required by `apply` and `rollback`: yes, restart the daemon")
|
||||
flag.Usage = usage
|
||||
flag.Parse()
|
||||
|
||||
// The stdlib flag package stops parsing at the first non-flag argument, so a
|
||||
// `-yes` written after the subcommand (which is how anyone would type it, and
|
||||
// how the usage text shows it) lands in Args instead of the flag. Pick it out
|
||||
// by hand rather than silently treating "apply -yes" as an unconfirmed apply.
|
||||
var args []string
|
||||
for _, a := range flag.Args() {
|
||||
if a == "-yes" || a == "--yes" {
|
||||
*yes = true
|
||||
continue
|
||||
}
|
||||
args = append(args, a)
|
||||
}
|
||||
if len(args) == 0 {
|
||||
usage()
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
cfg, err := config.Load(*cfgPath)
|
||||
if err != nil {
|
||||
die("config: %v", err)
|
||||
}
|
||||
if cfg.Update == nil {
|
||||
die("no `update` block in %s — the update capability is off unless configured.\nSee the package comment in internal/update for what it does and does not do.", *cfgPath)
|
||||
}
|
||||
|
||||
logf := func(format string, a ...any) {
|
||||
fmt.Fprintf(os.Stderr, "%s %s\n", time.Now().Format("15:04:05"), fmt.Sprintf(format, a...))
|
||||
}
|
||||
u, err := update.New(*cfg.Update, update.WithLogger(logf))
|
||||
if err != nil {
|
||||
die("%v", err)
|
||||
}
|
||||
|
||||
// Ctrl-C cancels the build or the health wait. It cannot cancel a rollback
|
||||
// midway into leaving the box in an unknown state, because the rollback runs
|
||||
// on its own context — see cmdApply.
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
switch args[0] {
|
||||
case "list":
|
||||
cmdList(u)
|
||||
case "verify":
|
||||
cmdVerify(ctx, u)
|
||||
case "apply":
|
||||
if !*yes {
|
||||
die("apply restarts mavend and can roll her back. Re-run with -yes if that is what you want.")
|
||||
}
|
||||
cmdApply(ctx, u)
|
||||
case "rollback":
|
||||
if !*yes {
|
||||
die("rollback restores the previous artifacts and restarts mavend. Re-run with -yes.")
|
||||
}
|
||||
id := ""
|
||||
if len(args) > 1 {
|
||||
id = args[1]
|
||||
}
|
||||
cmdRollback(ctx, u, id)
|
||||
default:
|
||||
usage()
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
|
||||
func cmdList(u *update.Updater) {
|
||||
snaps, err := u.Snapshots()
|
||||
if err != nil {
|
||||
die("snapshots: %v", err)
|
||||
}
|
||||
if len(snaps) == 0 {
|
||||
fmt.Println("no snapshots yet — the first `apply` takes one before it builds anything")
|
||||
return
|
||||
}
|
||||
fmt.Printf("%-18s %-12s %s\n", "SNAPSHOT", "COMMIT", "FILES")
|
||||
for _, s := range snaps {
|
||||
commit := s.Commit
|
||||
if len(commit) > 12 {
|
||||
commit = commit[:12]
|
||||
}
|
||||
if commit == "" {
|
||||
commit = "-"
|
||||
}
|
||||
fmt.Printf("%-18s %-12s %d\n", s.ID, commit, len(s.Files))
|
||||
}
|
||||
fmt.Printf("\nrollback to the newest with: mavupdate rollback -yes\n")
|
||||
}
|
||||
|
||||
func cmdVerify(ctx context.Context, u *update.Updater) {
|
||||
steps, err := u.Verify(ctx)
|
||||
report(steps)
|
||||
if err != nil {
|
||||
die("%v", err)
|
||||
}
|
||||
fmt.Println("verified: the tree builds and passes its own tests. Nothing was deployed — run `apply -yes` for that.")
|
||||
}
|
||||
|
||||
func cmdApply(ctx context.Context, u *update.Updater) {
|
||||
res, err := u.Apply(ctx)
|
||||
report(res.Steps)
|
||||
summarize(res)
|
||||
switch {
|
||||
case err == nil:
|
||||
fmt.Println("\nupdate committed: she answers on the new build.")
|
||||
case errors.Is(err, update.ErrRollbackFailed):
|
||||
die("\n%v\n\nSHE IS PROBABLY DOWN. The previous artifacts are in the snapshot dir; copy them\nover the install dir and restart by hand.", err)
|
||||
case errors.Is(err, update.ErrRolledBack):
|
||||
die("\n%v\n\nShe is answering again on the previous build. Nothing was lost; fix the change and retry.", err)
|
||||
default:
|
||||
die("\n%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func cmdRollback(ctx context.Context, u *update.Updater, id string) {
|
||||
res, err := u.Rollback(ctx, id)
|
||||
report(res.Steps)
|
||||
summarize(res)
|
||||
if err != nil && !errors.Is(err, update.ErrRolledBack) {
|
||||
die("\n%v", err)
|
||||
}
|
||||
fmt.Printf("\nrolled back to %s; she answers on it.\n", res.SnapshotID)
|
||||
}
|
||||
|
||||
func report(steps []update.Step) {
|
||||
for _, s := range steps {
|
||||
status := "ok"
|
||||
if s.Err != nil {
|
||||
status = "FAILED: " + s.Err.Error()
|
||||
}
|
||||
fmt.Printf(" %-8s %-8s %s\n", s.Name, s.Took.Round(time.Second), status)
|
||||
if s.Output != "" {
|
||||
fmt.Printf("---- %s output ----\n%s\n-------------------\n", s.Name, s.Output)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func summarize(res update.Result) {
|
||||
fmt.Printf("\nverified=%v snapshot=%s installed=%d restarted=%v healthy=%v rolled_back=%v rollback_healthy=%v took=%s\n",
|
||||
res.Verified, res.SnapshotID, len(res.Installed), res.Restarted, res.Healthy, res.RolledBack, res.RollbackHealthy, res.Took.Round(time.Second))
|
||||
}
|
||||
|
||||
func usage() {
|
||||
fmt.Fprint(os.Stderr, `mavupdate — deploy a new build of Maven, with rollback.
|
||||
|
||||
mavupdate [-config path] list
|
||||
mavupdate [-config path] verify
|
||||
mavupdate [-config path] apply -yes
|
||||
mavupdate [-config path] rollback [snapshot-id] -yes
|
||||
|
||||
apply is: health-check the running daemon, snapshot the deployed artifacts,
|
||||
make build, make test, install, restart, health-check — and restore the
|
||||
snapshot if any of that fails. It never fetches code and never runs by itself.
|
||||
|
||||
`)
|
||||
flag.PrintDefaults()
|
||||
}
|
||||
|
||||
func die(format string, a ...any) {
|
||||
fmt.Fprintf(os.Stderr, format+"\n", a...)
|
||||
os.Exit(1)
|
||||
}
|
||||
@@ -124,6 +124,7 @@ var sidebarSections = []struct {
|
||||
Label: "Settings",
|
||||
Pages: []struct{ Label, URL, Key string }{
|
||||
{Label: "Tools", URL: "/tools", Key: "tools"},
|
||||
{Label: "Model", URL: "/models", Key: "models"},
|
||||
{Label: "Passkey", URL: "/auth/passkey", Key: "passkey"},
|
||||
},
|
||||
},
|
||||
@@ -191,6 +192,8 @@ func pageIcon(key string) string {
|
||||
return `<svg class=icon width="14" height="14"><use href="/ethos-icons.svg#i-grid"/></svg>`
|
||||
case "tools":
|
||||
return `<svg class=icon width="14" height="14"><use href="/ethos-icons.svg#i-settings"/></svg>`
|
||||
case "models":
|
||||
return `<svg class=icon width="14" height="14"><use href="/ethos-icons.svg#i-wave"/></svg>`
|
||||
case "passkey":
|
||||
return `<svg class=icon width="14" height="14"><use href="/ethos-icons.svg#i-lock"/></svg>`
|
||||
default:
|
||||
@@ -225,6 +228,8 @@ func pageTitle(key string) string {
|
||||
return "Ecosystem"
|
||||
case "tools":
|
||||
return "Tools"
|
||||
case "models":
|
||||
return "Resident Model"
|
||||
case "passkey":
|
||||
return "Passkey"
|
||||
default:
|
||||
@@ -479,6 +484,12 @@ func main() {
|
||||
mux.HandleFunc("/routines", func(w http.ResponseWriter, r *http.Request) {
|
||||
handleRoutines(w, r, core, stepUpSession, *requireStepUp)
|
||||
})
|
||||
// /models — the resident-model surface (Vikunja #250). Same step-up gate as
|
||||
// /tools, and for a comparable reason: which model is loaded decides how every
|
||||
// utterance is routed and how every reply is worded. GET is read-only.
|
||||
mux.HandleFunc("/models", func(w http.ResponseWriter, r *http.Request) {
|
||||
handleModels(w, r, core, stepUpSession, *requireStepUp)
|
||||
})
|
||||
|
||||
// State-changing routes on this server, and their gate (Vikunja #317):
|
||||
//
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"html/template"
|
||||
"log"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/webauthn"
|
||||
)
|
||||
|
||||
// The resident-model surface (Vikunja #250).
|
||||
//
|
||||
// GET shows which model llama-server actually has loaded and which files the
|
||||
// daemon is configured to allow. POST swaps to one of them, behind the same
|
||||
// step-up gate as POST /tools: the loaded model decides how every utterance is
|
||||
// routed and how every reply is worded, so it is an owner action.
|
||||
//
|
||||
// There is nothing on this page Maven can press. The swap is an IPC method rated
|
||||
// AuthStepUp in internal/auth, unreachable from an act, an intent or a timer.
|
||||
|
||||
// modelController — the two non-CoreAPI methods this page needs. *ipc.Client
|
||||
// satisfies it; a core without a swap allowlist answers ErrUnknownMethod, which
|
||||
// the page renders as "not configured" rather than an error.
|
||||
type modelController interface {
|
||||
ModelStatus(ctx context.Context) (ipc.ModelStatusResp, error)
|
||||
SwapModel(ctx context.Context, req ipc.SwapModelReq) (ipc.SwapModelResp, error)
|
||||
}
|
||||
|
||||
var modelsTmpl = template.Must(template.New("models").Funcs(shellFuncs()).Parse(shellTopHTML + modelsHTML + shellBottomHTML))
|
||||
|
||||
const modelsHTML = `{{template "shellTop" "models"}}
|
||||
<h1>Resident model</h1>
|
||||
<p class=hint>swapping requires step-up — <a href=/auth/passkey>assert a passkey</a> first. The old model is unloaded before the new one is loaded (one model fits the iGPU at a time), so turns during the load are refused and fall back to the classifier.</p>
|
||||
{{if .Msg}}<div class="msg msg-ok">{{.Msg}}</div>{{end}}
|
||||
{{if .Err}}<div class="msg msg-err">{{.Err}}</div>{{end}}
|
||||
{{if .Off}}
|
||||
<section class=card>
|
||||
<h2 class=card-title>swap not configured</h2>
|
||||
<p class=hint>this core has no <code>phraser.swap_models</code> allowlist, so there is nothing to swap to. Add the gguf paths you allow to <code>deploy/mavend.json</code> and restart once.</p>
|
||||
</section>
|
||||
{{else}}
|
||||
<section class=card>
|
||||
<h2 class=card-title>loaded now</h2>
|
||||
<div class=scroll><table>
|
||||
<tr><th>model</th><td><code>{{.Status.Model}}</code></td></tr>
|
||||
<tr><th>file</th><td><code>{{.Status.ModelPath}}</code></td></tr>
|
||||
<tr><th>server</th><td><code>{{.Status.BaseURL}}</code></td></tr>
|
||||
<tr><th>n_ctx</th><td>{{.Status.NCtx}}</td></tr>
|
||||
<tr><th>n_gpu_layers</th><td>{{.Status.NGpuLayers}}</td></tr>
|
||||
</table></div>
|
||||
<p class=hint>the model name is what llama-server reports for itself, not what the config says it should be.</p>
|
||||
</section>
|
||||
<section class=card>
|
||||
<h2 class=card-title>allowed models <span class=badge>{{len .Status.Swappable}}</span></h2>
|
||||
{{if .Status.Swappable}}<div class=scroll><table><tr><th>file</th><th></th></tr>
|
||||
{{range .Status.Swappable}}<tr><td><code>{{.}}</code></td>
|
||||
<td><form method=post action=/models class=inline-form>
|
||||
<input type=hidden name=model_path value="{{.}}">
|
||||
<button class=btn>load this one</button></form></td></tr>{{end}}
|
||||
</table></div>
|
||||
{{else}}<div class=empty><div>no models allowlisted</div></div>{{end}}
|
||||
</section>
|
||||
{{end}}
|
||||
{{template "shellBottom"}}`
|
||||
|
||||
type modelsPage struct {
|
||||
Msg string
|
||||
Err string
|
||||
Off bool
|
||||
Status ipc.ModelStatusResp
|
||||
}
|
||||
|
||||
// handleModels renders the model surface (GET) and applies a swap (POST).
|
||||
//
|
||||
// A failed swap is reported as a failure with the model that is still serving
|
||||
// named, because that is the state the operator needs: the daemon rolled back
|
||||
// and is answering turns, it just is not answering them with what he asked for.
|
||||
func handleModels(w http.ResponseWriter, r *http.Request, core ipc.CoreAPI, session *webauthn.PasskeySession, requireStepUp bool) {
|
||||
if core == nil {
|
||||
http.Error(w, "models disabled (no -core)", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
mc, ok := core.(modelController)
|
||||
if !ok {
|
||||
http.Error(w, "models unavailable: core connection does not support model swap", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
ctx := r.Context()
|
||||
page := modelsPage{}
|
||||
|
||||
if r.Method == http.MethodPost {
|
||||
if !stepUpOK(session, requireStepUp) {
|
||||
http.Error(w, "step-up required: assert a passkey first", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
path := strings.TrimSpace(r.FormValue("model_path"))
|
||||
if path == "" {
|
||||
http.Error(w, "model_path required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
req := ipc.SwapModelReq{ModelPath: path}
|
||||
if v, err := strconv.Atoi(r.FormValue("n_ctx")); err == nil {
|
||||
req.NCtx = v
|
||||
}
|
||||
res, err := mc.SwapModel(ctx, req)
|
||||
switch {
|
||||
case err == nil:
|
||||
page.Msg = "loaded " + res.Model + " (" + strconv.FormatInt(res.TookMs, 10) + "ms)"
|
||||
log.Printf("models: swapped to %s (%s) in %dms", res.ModelPath, res.Model, res.TookMs)
|
||||
case errors.Is(err, ipc.ErrForbidden):
|
||||
http.Error(w, "refused: that model is not in phraser.swap_models, or step-up was not asserted", http.StatusForbidden)
|
||||
return
|
||||
case errors.Is(err, ipc.ErrUnknownMethod):
|
||||
http.Error(w, "swap not configured on this core", http.StatusServiceUnavailable)
|
||||
return
|
||||
case res.RolledBack:
|
||||
page.Err = "swap failed, rolled back to " + res.Model + " — she is still answering, with the old model"
|
||||
log.Printf("models: swap to %s failed, rolled back: %v", path, err)
|
||||
default:
|
||||
page.Err = "swap failed: " + err.Error()
|
||||
log.Printf("models: swap to %s failed: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
st, err := mc.ModelStatus(ctx)
|
||||
if err != nil {
|
||||
if errors.Is(err, ipc.ErrUnknownMethod) {
|
||||
page.Off = true
|
||||
} else {
|
||||
log.Printf("models: status: %v", err)
|
||||
http.Error(w, "core read failed", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
}
|
||||
page.Status = st
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
if err := modelsTmpl.Execute(w, page); err != nil {
|
||||
log.Printf("models render: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/webauthn"
|
||||
)
|
||||
|
||||
// fakeModelCore is a core that supports the two model methods. It records what
|
||||
// the page asked for, so the tests can assert the gate rather than the HTML.
|
||||
type fakeModelCore struct {
|
||||
ipc.UnimplementedCoreAPI
|
||||
|
||||
status ipc.ModelStatusResp
|
||||
statusErr error
|
||||
|
||||
swapResp ipc.SwapModelResp
|
||||
swapErr error
|
||||
swapped []ipc.SwapModelReq
|
||||
}
|
||||
|
||||
func (f *fakeModelCore) ModelStatus(ctx context.Context) (ipc.ModelStatusResp, error) {
|
||||
return f.status, f.statusErr
|
||||
}
|
||||
|
||||
func (f *fakeModelCore) SwapModel(ctx context.Context, req ipc.SwapModelReq) (ipc.SwapModelResp, error) {
|
||||
f.swapped = append(f.swapped, req)
|
||||
return f.swapResp, f.swapErr
|
||||
}
|
||||
|
||||
func modelsGET(t *testing.T, core ipc.CoreAPI) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
w := httptest.NewRecorder()
|
||||
handleModels(w, httptest.NewRequest(http.MethodGet, "/models", nil), core, nil, false)
|
||||
return w
|
||||
}
|
||||
|
||||
func modelsPOST(t *testing.T, core ipc.CoreAPI, session *webauthn.PasskeySession, requireStepUp bool, path string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
r := httptest.NewRequest(http.MethodPost, "/models", strings.NewReader("model_path="+path))
|
||||
r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
w := httptest.NewRecorder()
|
||||
handleModels(w, r, core, session, requireStepUp)
|
||||
return w
|
||||
}
|
||||
|
||||
func TestModels_GETShowsTheLoadedModelAndTheAllowlist(t *testing.T) {
|
||||
core := &fakeModelCore{status: ipc.ModelStatusResp{
|
||||
Model: "Qwen3-1.7B-UD-Q4_K_XL",
|
||||
ModelPath: "/opt/maven/models/llm/qwen3.gguf",
|
||||
BaseURL: "http://127.0.0.1:18099",
|
||||
NCtx: 4096,
|
||||
Swappable: []string{"/opt/maven/models/llm/qwen3.gguf", "/opt/maven/models/llm/qwen3-cpt.gguf"},
|
||||
}}
|
||||
w := modelsGET(t, core)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("GET /models = %d; want 200", w.Code)
|
||||
}
|
||||
body := w.Body.String()
|
||||
for _, want := range []string{"Qwen3-1.7B-UD-Q4_K_XL", "qwen3-cpt.gguf", "4096"} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("page does not mention %q", want)
|
||||
}
|
||||
}
|
||||
if len(core.swapped) != 0 {
|
||||
t.Errorf("a GET swapped the model: %v", core.swapped)
|
||||
}
|
||||
}
|
||||
|
||||
func TestModels_POSTRequiresStepUpWhenFailingClosed(t *testing.T) {
|
||||
// No WebAuthn configured (nil session) + -require-stepup ⇒ deny, exactly
|
||||
// like POST /tools. Nothing reaches core.
|
||||
core := &fakeModelCore{}
|
||||
w := modelsPOST(t, core, nil, true, "/opt/maven/models/llm/qwen3.gguf")
|
||||
if w.Code != http.StatusForbidden {
|
||||
t.Fatalf("POST /models without assertable step-up = %d; want 403", w.Code)
|
||||
}
|
||||
if len(core.swapped) != 0 {
|
||||
t.Fatalf("a denied POST still called SwapModel: %v", core.swapped)
|
||||
}
|
||||
}
|
||||
|
||||
func TestModels_POSTSwapsAndReportsTheModelThatAnswered(t *testing.T) {
|
||||
core := &fakeModelCore{
|
||||
swapResp: ipc.SwapModelResp{Model: "qwen3-cpt", ModelPath: "/m/cpt.gguf", TookMs: 4200},
|
||||
status: ipc.ModelStatusResp{Model: "qwen3-cpt", ModelPath: "/m/cpt.gguf"},
|
||||
}
|
||||
w := modelsPOST(t, core, nil, false, "/m/cpt.gguf")
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("POST /models = %d; want 200", w.Code)
|
||||
}
|
||||
if len(core.swapped) != 1 || core.swapped[0].ModelPath != "/m/cpt.gguf" {
|
||||
t.Fatalf("SwapModel calls = %v; want one for /m/cpt.gguf", core.swapped)
|
||||
}
|
||||
if !strings.Contains(w.Body.String(), "loaded qwen3-cpt") {
|
||||
t.Errorf("page does not report which model was loaded:\n%s", w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestModels_RolledBackSwapSaysSheIsStillAnswering(t *testing.T) {
|
||||
core := &fakeModelCore{
|
||||
swapResp: ipc.SwapModelResp{Model: "qwen3", ModelPath: "/m/old.gguf", RolledBack: true},
|
||||
swapErr: errBrokenModel{},
|
||||
status: ipc.ModelStatusResp{Model: "qwen3", ModelPath: "/m/old.gguf"},
|
||||
}
|
||||
w := modelsPOST(t, core, nil, false, "/m/cpt.gguf")
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("POST /models after a rollback = %d; want 200 with the failure rendered", w.Code)
|
||||
}
|
||||
body := w.Body.String()
|
||||
if !strings.Contains(body, "rolled back to qwen3") {
|
||||
t.Errorf("page does not say it rolled back:\n%s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestModels_RefusedPathIs403(t *testing.T) {
|
||||
core := &fakeModelCore{swapErr: ipc.ErrForbidden}
|
||||
w := modelsPOST(t, core, nil, false, "/etc/passwd")
|
||||
if w.Code != http.StatusForbidden {
|
||||
t.Fatalf("POST /models with a non-allowlisted path = %d; want 403", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestModels_UnconfiguredCoreRendersOff(t *testing.T) {
|
||||
core := &fakeModelCore{statusErr: ipc.ErrUnknownMethod}
|
||||
w := modelsGET(t, core)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("GET /models against a core without the swap = %d; want 200", w.Code)
|
||||
}
|
||||
if !strings.Contains(w.Body.String(), "swap not configured") {
|
||||
t.Errorf("page does not say the capability is off:\n%s", w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestModels_CoreWithoutTheMethodsIs503(t *testing.T) {
|
||||
// An in-process CoreAPI (no swap methods) must not 500 the page.
|
||||
w := modelsGET(t, ipc.UnimplementedCoreAPI{})
|
||||
if w.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("GET /models on a core without the methods = %d; want 503", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
type errBrokenModel struct{}
|
||||
|
||||
func (errBrokenModel) Error() string { return "llm: server did not start" }
|
||||
@@ -36,6 +36,67 @@ present (see `.dockerignore`).
|
||||
| `/var/lib/maven` (volume) | encrypted db at rest |
|
||||
| `/dev/shm` (tmpfs) | decrypted db working copy (RAM only) |
|
||||
|
||||
## Reading the outside world (off by default)
|
||||
|
||||
`mavend.json` ships without a `feeds` block, which means no RSS/Atom feed is
|
||||
fetched and no outbound request is made. Switching it on is adding the block:
|
||||
|
||||
```json
|
||||
"feeds": {
|
||||
"poll_interval": "30m",
|
||||
"max_items": 5,
|
||||
"max_age": "24h",
|
||||
"sources": [
|
||||
{ "name": "habr", "url": "https://habr.com/ru/rss/best/daily/",
|
||||
"category": "технологии", "exclude": ["реклама"] }
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
What it does and does not do:
|
||||
|
||||
- items are written as notes with source `rss:<name>`, visible on `/dash`;
|
||||
- **nothing is announced.** She reads them back when asked — "что нового в
|
||||
лентах?", "что нового по технологиям?" — and never on arrival. There is no
|
||||
severity or channel knob here on purpose;
|
||||
- the fetcher is allowlisted to the hosts of the configured feeds, plus any
|
||||
`allow_hosts`. It refuses non-http(s) schemes and every private address
|
||||
(loopback, the LAN, the `10.42.0.0/24` wg range, cloud metadata). It caps the
|
||||
response at 2 MiB and redirects at 3, and makes at most one request per host
|
||||
per second. See `internal/webfetch`;
|
||||
- how far each feed was read is stored as a config fact `rss:latest:<name>`, so
|
||||
a restart does not re-note yesterday's headlines.
|
||||
|
||||
### Reading a page (`crawl`, also off by default)
|
||||
|
||||
There is no `crawl` block either, so no page is fetched. Two halves, separately
|
||||
switched:
|
||||
|
||||
```json
|
||||
"crawl": {
|
||||
"on_demand": true,
|
||||
"interval": "6h",
|
||||
"max_runes": 4000,
|
||||
"watches": [
|
||||
{ "name": "changelog", "url": "https://example.org/changelog", "interval": "12h" }
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
- `on_demand` lets her read a page he names in the utterance: "посмотри
|
||||
https://example.org/x — что там?". The page becomes context for his question,
|
||||
and only the URL leaves the box. Without a URL nothing is fetched, so this is
|
||||
a fallback and not a habit;
|
||||
- `watches` re-reads a fixed list on its interval and writes a note when the
|
||||
text changed. Like the feeds, it announces nothing;
|
||||
- the answer path sits **last** in the query chain, behind his memory, his notes
|
||||
and (once wired) the local Kiwix ZIMs. A local read costs nothing;
|
||||
- `robots.txt` is fetched first and obeyed with no override; a `Disallow` is a
|
||||
refusal she says out loud. `Crawl-delay` is honoured;
|
||||
- same guarded fetcher as the feeds: allowlist/denylist, no private addresses,
|
||||
size cap, redirect cap, timeout, one request per host per second;
|
||||
- dedup state is the config fact `crawl:hash:<name>`.
|
||||
|
||||
## Not yet verified / host-dependent
|
||||
|
||||
This stack is correct-by-construction but has **not been build-tested here**
|
||||
@@ -53,3 +114,49 @@ build on the target host, most likely in one of these:
|
||||
work fine over the core socket.
|
||||
- **netdata** — `mavpoll` reaches it via `host.docker.internal`; adjust if
|
||||
netdata runs elsewhere.
|
||||
|
||||
## Updating her (`mavupdate`, Vikunja #249)
|
||||
|
||||
Off unless configured, and there is deliberately no button for it. There is no
|
||||
IPC method, no web route, no timer and no act that starts an update — the trigger
|
||||
is a human running `mavupdate` on the host, which needs shell access, a strictly
|
||||
higher bar than the step-up passkey gate that guards `/tools`. She cannot update
|
||||
herself; she can be updated. Nothing here ever fetches code: the new version is
|
||||
whatever you pulled into the working tree yourself.
|
||||
|
||||
Add an `update` block to `mavend.json` (mavend ignores it — only the CLI reads
|
||||
it), with paths as they exist **on the host**, not inside a container:
|
||||
|
||||
```json
|
||||
"update": {
|
||||
"source_dir": "/home/kami/apps/Maven",
|
||||
"install_dir": "/home/kami/apps/Maven",
|
||||
"snapshot_dir": "/var/lib/maven-snapshots",
|
||||
"binaries": ["mavend", "mavweb", "mavsttd", "mavttsd", "mavwaked",
|
||||
"mavenclient", "mavpoll", "mavcaldav", "mavmaild"],
|
||||
"config_files": ["deploy/mavend.json"],
|
||||
"restart_cmd": ["docker", "compose", "up", "-d", "--build"],
|
||||
"health_socket": "/var/lib/docker/volumes/maven_sockets/_data/mavend.sock",
|
||||
"health_timeout_sec": 120
|
||||
}
|
||||
```
|
||||
|
||||
`snapshot_dir` must be outside `install_dir` (a restore must not read from what
|
||||
the install writes) and `health_socket` is required: an update that cannot check
|
||||
its own result cannot roll itself back, so the config is refused without one.
|
||||
|
||||
Then:
|
||||
|
||||
```sh
|
||||
mavupdate -config deploy/mavend.json verify # make build + make test, deploys nothing
|
||||
mavupdate -config deploy/mavend.json apply -yes # snapshot, verify, install, restart, health-check
|
||||
mavupdate -config deploy/mavend.json list # what you can roll back to
|
||||
mavupdate -config deploy/mavend.json rollback -yes # restore the previous artifacts and restart
|
||||
```
|
||||
|
||||
`apply` refuses to start if she is not already answering — otherwise a failed
|
||||
update and a box that was already broken are indistinguishable afterwards. On any
|
||||
failure after the install it restores the snapshot, restarts, and checks again;
|
||||
if that also fails it says so loudly and names the directory to copy back by hand.
|
||||
The database is never snapshotted or rolled back (see the package comment in
|
||||
`internal/update`); schema compatibility is `store.Migrate`'s job.
|
||||
|
||||
@@ -113,6 +113,31 @@ services:
|
||||
- sockets:/run/maven
|
||||
# - ./deploy/zenmoney.token:/run/secrets/zenmoney.token:ro
|
||||
|
||||
# The mail reader (Vikunja #246) is OFF and commented out: it needs an IMAP
|
||||
# account, and there is none on this box. mavmaild reads the password from a
|
||||
# FILE so it never appears in `ps`, in this file, or in shell history — the
|
||||
# same rule mavpoll follows for the zenmoney token. Core never sees the
|
||||
# password: the reader hands core message text on one IPC method, and core
|
||||
# writes what the model extracts as task CANDIDATES he reviews on /tasks.
|
||||
# Nothing here can create a reminder, so a misread mail cannot fire.
|
||||
#
|
||||
# To enable: write the password to deploy/imap.password (0600, gitignored),
|
||||
# add an "email": {} block to deploy/mavend.json, and uncomment this service.
|
||||
# mavmaild:
|
||||
# <<: *image
|
||||
# command: ["mavmaild", "-socket", "/run/maven/mavend.sock",
|
||||
# "-imap", "imap.example.org:993",
|
||||
# "-user", "kami@example.org",
|
||||
# "-password-file", "/run/secrets/imap.password",
|
||||
# "-mailbox", "INBOX",
|
||||
# "-interval", "15m",
|
||||
# "-state", "/var/lib/maven/mail-seen.json"]
|
||||
# depends_on: [mavend]
|
||||
# volumes:
|
||||
# - sockets:/run/maven
|
||||
# - dbdata:/var/lib/maven
|
||||
# - ./deploy/imap.password:/run/secrets/imap.password:ro
|
||||
|
||||
volumes:
|
||||
dbdata:
|
||||
sockets:
|
||||
|
||||
@@ -27,3 +27,28 @@
|
||||
7. Add voice query handler — `"что нового?"` queries `RecentNotes` filtered by source prefix `rss:` and phrases via `phraser.PhraseQuery`
|
||||
8. Add `feeds` block to `config.Config` and `deploy/mavend.json`
|
||||
9. Test with a live RSS feed (e.g., `https://news.ycombinator.com/rss`) — verify items appear in notes table
|
||||
|
||||
---
|
||||
|
||||
## Shipped 2026-08-01 (#258)
|
||||
|
||||
`internal/webfetch` (the guarded HTTP door: scheme, allow/deny hosts, private-address
|
||||
refusal in the dialer, size cap, redirect cap, per-host rate limit), `internal/rss`
|
||||
(RSS 2.0 + Atom parser, poller with durable marks and a keyword filter),
|
||||
`cmd/mavend/feeds.go` (ticker, fetcher adapter, `rss:latest:<feed>` fact marks),
|
||||
config block `feeds`, and the `feeds` query source with `router.ParseFeedQuery`.
|
||||
|
||||
Deviations from the plan above, both deliberate:
|
||||
|
||||
- **Step 5 (breaking-news nudges) was not built.** A feed that dispatches is a nag,
|
||||
and the one thing Maven is not is a nag. Items are read when asked and nowhere else.
|
||||
If breaking news is ever wanted, it belongs behind the existing delivery policy
|
||||
(severity, quiet hours, digest), not in the poller.
|
||||
- **Step 3 (embedder relevance) is a seam, not an implementation.** `rss.Ranker`
|
||||
exists and is wired nil. Scoring items against an "interest profile" needs a
|
||||
profile, and there is none yet; a threshold with nothing to compare against is a
|
||||
random filter with a confident name. The filter that runs is the per-feed
|
||||
include/exclude keyword list, which he can read and predict.
|
||||
|
||||
No new dependency: stdlib `encoding/xml`, no gofeed. Stock deploy config has no
|
||||
`feeds` block, so the capability is off.
|
||||
|
||||
@@ -28,3 +28,41 @@
|
||||
7. Add IPC methods `MethodTriggerCrawl(name)`, `MethodListCrawls`, `MethodGetCrawlResult(name)`
|
||||
8. Add `crawls` block to `config.Config` and `deploy/mavend.json`
|
||||
9. Test with a static HTML page — verify extraction matches expected values, verify scheduling fires correctly
|
||||
|
||||
## Shipped 2026-08-01 (#259)
|
||||
|
||||
Built as `internal/crawl` (pure: robots, extraction, watcher) plus
|
||||
`cmd/mavend/crawls.go` (fetcher, ticker, dedup facts), on top of the guarded
|
||||
`internal/webfetch` door added with the feed reader (#258). Off unless
|
||||
configured, in two separately-switched halves: `crawl.on_demand` for a URL he
|
||||
names, `crawl.watches` for a scheduled re-read.
|
||||
|
||||
**Limits are code, not documentation** (`internal/webfetch`, tested one test per
|
||||
limit): host allowlist/denylist, no private addresses (loopback, RFC1918 —
|
||||
hence the LAN and the `10.42.0.0/24` wg range —, link-local incl. cloud
|
||||
metadata, CGNAT, v6 ULA) enforced in the dialer's `Control` hook so DNS
|
||||
rebinding and every redirect hop are covered, response size cap, redirect cap,
|
||||
timeout, one request per host per second. `robots.txt` is fetched first, cached
|
||||
per host, and a `Disallow` is refused with no override.
|
||||
|
||||
Deliberate deviations from the plan above:
|
||||
|
||||
- **No CSS selectors and no LLM structured extraction** (steps 2). The output is
|
||||
plaintext handed to the phraser as context for the question he asked. A 1.7B
|
||||
extracting a JSON price table from 4000 runes is a worse bet than reading, and
|
||||
`goquery` is not vendored.
|
||||
- **No `crawl` act verb and no new IPC methods** (steps 5, 7). Reading a page is
|
||||
a query source (`queryWeb` in `actions_query.go`, last in the chain, behind
|
||||
Kiwix once that is wired), not an action he commands. Nothing needs a new wire
|
||||
method to work.
|
||||
- **Notes, not facts.** A page's text is not a fact about him. Only the dedup
|
||||
hash is a fact (`crawl:hash:<name>`, kind `config`, source `poll:crawl`).
|
||||
- **Nothing is dispatched.** A changed page writes a note; it does not nudge.
|
||||
Not a nag.
|
||||
- **No `/tools` crawl history page.** The notes and the hash facts are already
|
||||
visible on `/dash`.
|
||||
|
||||
**No new dependency.** The vendored tree has no `x/net/html`, no `goquery` and
|
||||
no `temoto/robotstxt`, so robots parsing and HTML-to-text are stdlib
|
||||
(`regexp`, `html`) — RE2 has no backreferences, hence the `pairsRE` builder in
|
||||
`extract.go`.
|
||||
|
||||
@@ -393,3 +393,26 @@ func mustWriteFactParams(source string) []byte {
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// TestRequirement_SwapModel — loading a different resident model is an owner
|
||||
// action at the same rung as mutating the tool allowlist: it decides how every
|
||||
// utterance is routed and how every reply is worded. The read side is not.
|
||||
func TestRequirement_SwapModel(t *testing.T) {
|
||||
if got := Requirement(ipc.MethodSwapModel); got != AuthStepUp {
|
||||
t.Errorf("SwapModel authority = %v; want AuthStepUp", got)
|
||||
}
|
||||
if got := Requirement(ipc.MethodModelStatus); got != AuthRead {
|
||||
t.Errorf("ModelStatus authority = %v; want AuthRead", got)
|
||||
}
|
||||
// A surface that cannot carry a passkey gesture cannot swap the model, no
|
||||
// matter what it is enrolled as — this is the "never through voice" property.
|
||||
voice := Scope{Surface: SurfaceVoice, Module: "voice", SourceScope: []string{"*"}}
|
||||
if err := Can(ipc.MethodSwapModel, voice, nil); !errors.Is(err, ErrForbidden) {
|
||||
t.Errorf("voice swapping the model = %v; want ErrForbidden", err)
|
||||
}
|
||||
// And with no step-up session asserted, the gate refuses even a capable surface.
|
||||
noSession := &Gate{Enrollment: NewFloorEnrollment()}
|
||||
if err := noSession.Check(context.Background(), ipc.MethodSwapModel, nil); !errors.Is(err, ipc.ErrForbidden) {
|
||||
t.Errorf("SwapModel with no asserted step-up = %v; want ErrForbidden", err)
|
||||
}
|
||||
}
|
||||
|
||||
+16
-1
@@ -53,6 +53,13 @@ func Requirement(m ipc.Method) Authority {
|
||||
// asserted — never a module or the voice/chat path. maven can propose
|
||||
// (MethodProposeTool, no step-up: she has no passkey) but never en/disable.
|
||||
return AuthStepUp
|
||||
case ipc.MethodSwapModel:
|
||||
// Swapping the resident model changes what routes every utterance and
|
||||
// what words every reply. It is the owner's call, from a surface that can
|
||||
// carry a passkey gesture — the same rung as mutating the tool allowlist,
|
||||
// and for the same reason: nothing Maven says or does may reach it.
|
||||
// MethodModelStatus is only the read side, so it stays at AuthRead.
|
||||
return AuthStepUp
|
||||
case ipc.MethodWriteFact:
|
||||
return AuthWrite
|
||||
case ipc.MethodAssertStepUp:
|
||||
@@ -74,7 +81,15 @@ func Requirement(m ipc.Method) Authority {
|
||||
// existing analogue.
|
||||
ipc.MethodCaptureTask,
|
||||
ipc.MethodListTasks,
|
||||
ipc.MethodSetTaskStatus:
|
||||
ipc.MethodSetTaskStatus,
|
||||
// Mail ingestion (Vikunja #246). AuthRead because of what the method can
|
||||
// produce: candidate tasks and nothing else. It cannot write a fact, set a
|
||||
// reminder, or touch the tool allowlist, so a compromised mail reader can
|
||||
// at worst put junk on a review page he clears in one click.
|
||||
ipc.MethodIngestMail,
|
||||
// The read side of the model swap: which model is resident, which ones are
|
||||
// allowlisted. It loads nothing and changes nothing.
|
||||
ipc.MethodModelStatus:
|
||||
return AuthRead
|
||||
}
|
||||
// Unknown method ⇒ AuthRead, but ipc.dispatch returns ErrUnknownMethod
|
||||
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
"github.com/kami/maven/internal/delivery/ntfysink"
|
||||
"github.com/kami/maven/internal/delivery/telegramsink"
|
||||
"github.com/kami/maven/internal/morning"
|
||||
"github.com/kami/maven/internal/update"
|
||||
"github.com/robfig/cron/v3"
|
||||
)
|
||||
|
||||
@@ -108,6 +109,16 @@ type Config struct {
|
||||
// calls its /v1/chat/completions endpoint to phrase nudges and reminders.
|
||||
Phraser *PhraserConfig `json:"phraser,omitempty"`
|
||||
|
||||
// Update — how THIS box deploys a new build of Maven (Vikunja #249). nil ⇒
|
||||
// the update capability does not exist, which is the state to leave it in
|
||||
// unless the operator has read internal/update's package comment.
|
||||
//
|
||||
// mavend never reads this block: the daemon does not import internal/update
|
||||
// and cannot update itself. It lives here because cmd/mavupdate — a CLI the
|
||||
// owner runs on the host, the only trigger there is — reads the same config
|
||||
// file to find the socket it health-checks.
|
||||
Update *update.Config `json:"update,omitempty"`
|
||||
|
||||
// Voice — the client↔core surface + the stt/tts modules the daemon
|
||||
// wires. nil ⇒ the daemon doesn't wire voice: the TCP listener stays
|
||||
// down, the dispatcher's Voice slot stays nil (the routing table's
|
||||
@@ -150,6 +161,21 @@ type Config struct {
|
||||
// absent ⇒ no evaluation loop at all. See MemoryEvalConfig.
|
||||
MemoryEval *MemoryEvalConfig `json:"memory_eval,omitempty"`
|
||||
|
||||
// Email — mail ingestion (Vikunja #246). nil / absent ⇒ core refuses
|
||||
// ipc.MethodIngestMail outright, so a mail reader cannot make Maven read a
|
||||
// mailbox by merely existing. See EmailConfig; the IMAP host and credential
|
||||
// live in the reader (cmd/mavmaild), never here.
|
||||
Email *EmailConfig `json:"email,omitempty"`
|
||||
|
||||
// Feeds — RSS/Atom feed reading (Vikunja #258). nil / absent ⇒ no feed is
|
||||
// ever fetched: reading the outside world is off unless configured, like
|
||||
// the weather and telegram. See FeedsConfig.
|
||||
Feeds *FeedsConfig `json:"feeds,omitempty"`
|
||||
|
||||
// Crawl — reading a web page (Vikunja #259). nil / absent ⇒ Maven never
|
||||
// fetches a page: not on request, not on a schedule. See CrawlConfig.
|
||||
Crawl *CrawlConfig `json:"crawl,omitempty"`
|
||||
|
||||
// Praxis — the ecosystem attention-state service. When configured, maven
|
||||
// calls the Praxis HTTP tools API for attention listing and item lifecycle.
|
||||
// Maven never touches Praxis's database directly (ecosystem invariant: no
|
||||
@@ -396,6 +422,109 @@ func (p *PatternProposalConfig) AnnounceProposals() bool {
|
||||
return p != nil && p.Notify
|
||||
}
|
||||
|
||||
// FeedsConfig — the RSS/Atom reader (Vikunja #258, docs/plans/13-rss-news-feeds.md).
|
||||
//
|
||||
// Absent ⇒ off, and off means no outbound request at all. Present with an empty
|
||||
// `sources` list is also off — a poller with nothing to poll is not wired.
|
||||
//
|
||||
// What a feed may NOT do here: speak. Items are written as notes with source
|
||||
// "rss:<name>" and read back when he asks; nothing is dispatched, nudged or
|
||||
// announced on arrival. That is the "not a nag" constraint, and it is why there
|
||||
// is no severity or channel field in this block to reach for.
|
||||
type FeedsConfig struct {
|
||||
// Sources — the feeds to read. Empty ⇒ the reader stays down.
|
||||
Sources []FeedSourceConfig `json:"sources,omitempty"`
|
||||
|
||||
// PollInterval — default per-feed cadence. 0 ⇒ rss.DefaultPollInterval (30m).
|
||||
PollInterval Duration `json:"poll_interval,omitempty"`
|
||||
|
||||
// MaxItems — most items kept from one feed in one poll. 0 ⇒
|
||||
// rss.DefaultMaxItems (5). This is the "не завали мне /dash" knob.
|
||||
MaxItems int `json:"max_items,omitempty"`
|
||||
|
||||
// MaxAge — on a first poll (no saved mark), how far back to take items.
|
||||
// 0 ⇒ rss.DefaultMaxAge (24h), so switching a feed on imports today, not
|
||||
// the archive.
|
||||
MaxAge Duration `json:"max_age,omitempty"`
|
||||
|
||||
// AllowHosts — when set, the reader may only connect to these hosts (and
|
||||
// their subdomains). The feed URLs' own hosts are added automatically, so
|
||||
// this is only needed to be stricter than that.
|
||||
AllowHosts []string `json:"allow_hosts,omitempty"`
|
||||
|
||||
// Timeout — per-request budget. 0 ⇒ webfetch.DefaultTimeout.
|
||||
Timeout Duration `json:"timeout,omitempty"`
|
||||
|
||||
// MaxBytes — response size cap. 0 ⇒ webfetch.DefaultMaxBytes (2 MiB).
|
||||
MaxBytes int64 `json:"max_bytes,omitempty"`
|
||||
}
|
||||
|
||||
// FeedSourceConfig — one feed.
|
||||
type FeedSourceConfig struct {
|
||||
Name string `json:"name"` // note source is "rss:<name>"
|
||||
URL string `json:"url"` // http(s) only
|
||||
Category string `json:"category,omitempty"` // "технологии" — what "что нового по X?" matches
|
||||
Interval Duration `json:"interval,omitempty"` // 0 ⇒ FeedsConfig.PollInterval
|
||||
Include []string `json:"include,omitempty"` // keep only items containing one of these
|
||||
Exclude []string `json:"exclude,omitempty"` // drop items containing any of these
|
||||
}
|
||||
|
||||
// CrawlConfig — the web crawler (Vikunja #259, docs/plans/14-web-crawler.md).
|
||||
//
|
||||
// Absent ⇒ off, and off means no page is ever fetched. Present with neither
|
||||
// `on_demand` nor a `watches` entry is also off: there would be nothing to do.
|
||||
//
|
||||
// The crawler is the LAST place an answer is looked for, behind the model, his
|
||||
// own memory and the local Kiwix ZIMs. That ordering lives in the query-source
|
||||
// chain (cmd/mavend/actions_query.go), not here, but it is the reason this block
|
||||
// is small: it is a fallback, not a search engine.
|
||||
//
|
||||
// Only the URL leaves the box. His notes, facts, persona block and history are
|
||||
// never part of a request — the crawler package cannot even read the store.
|
||||
type CrawlConfig struct {
|
||||
// OnDemand — may he ask her to read a page he names out loud
|
||||
// ("посмотри https://… — что там пишут?"). false ⇒ the on-demand answer
|
||||
// source stays off and only the watches below run.
|
||||
OnDemand bool `json:"on_demand,omitempty"`
|
||||
|
||||
// Watches — pages re-read on a schedule. A page whose text changed is
|
||||
// written as a note (source "crawl:<name>"); nothing is announced.
|
||||
Watches []CrawlWatchConfig `json:"watches,omitempty"`
|
||||
|
||||
// Interval — default watch cadence. 0 ⇒ crawl.DefaultWatchInterval (6h).
|
||||
Interval Duration `json:"interval,omitempty"`
|
||||
|
||||
// AllowHosts — when set, the ONLY hosts the crawler may reach (subdomains
|
||||
// included). Watched pages' own hosts are added automatically. Setting this
|
||||
// is how "she may read the arch wiki and nothing else" is expressed.
|
||||
AllowHosts []string `json:"allow_hosts,omitempty"`
|
||||
|
||||
// DenyHosts — never reachable, checked first. Private addresses do not need
|
||||
// to be listed: they are refused unconditionally (see internal/webfetch).
|
||||
DenyHosts []string `json:"deny_hosts,omitempty"`
|
||||
|
||||
// UserAgent — sent on every request AND matched against robots.txt groups.
|
||||
// Empty ⇒ webfetch.DefaultUserAgent.
|
||||
UserAgent string `json:"user_agent,omitempty"`
|
||||
|
||||
// Timeout — per-request budget. 0 ⇒ webfetch.DefaultTimeout.
|
||||
Timeout Duration `json:"timeout,omitempty"`
|
||||
|
||||
// MaxBytes — response size cap. 0 ⇒ webfetch.DefaultMaxBytes (2 MiB).
|
||||
MaxBytes int64 `json:"max_bytes,omitempty"`
|
||||
|
||||
// MaxRunes — how much extracted text is kept. 0 ⇒ crawl.DefaultMaxRunes
|
||||
// (4000), which is what fits a 4096-token context alongside a prompt.
|
||||
MaxRunes int `json:"max_runes,omitempty"`
|
||||
}
|
||||
|
||||
// CrawlWatchConfig — one page kept an eye on.
|
||||
type CrawlWatchConfig struct {
|
||||
Name string `json:"name"` // note source is "crawl:<name>"
|
||||
URL string `json:"url"`
|
||||
Interval Duration `json:"interval,omitempty"` // 0 ⇒ CrawlConfig.Interval
|
||||
}
|
||||
|
||||
// MemoryEvalConfig — the background memory-evaluation loop (Vikunja #248).
|
||||
// Absent ⇒ off, like every other capability that costs something the owner did
|
||||
// not ask for. Each evaluation is a full LLM round-trip on the one resident
|
||||
@@ -418,6 +547,26 @@ type MemoryEvalConfig struct {
|
||||
MinConfidence float64 `json:"min_confidence,omitempty"`
|
||||
}
|
||||
|
||||
// EmailConfig — core's half of the email reader: how many task candidates one
|
||||
// message may produce, and how long the extraction call may take.
|
||||
//
|
||||
// There is deliberately nothing about a mailbox here. Core does not connect to
|
||||
// IMAP, does not know an account exists, and holds no mail credential — the
|
||||
// reader daemon does, the same split mavpoll uses for the zenmoney token. This
|
||||
// block only says "extraction is allowed, with these bounds".
|
||||
type EmailConfig struct {
|
||||
// MaxTasks — candidates per message. 0 ⇒ email.MaxCandidates (3).
|
||||
MaxTasks int `json:"max_tasks,omitempty"`
|
||||
|
||||
// Timeout — per-message extraction budget. 0 ⇒ DefaultEmailTimeout. This is
|
||||
// a Thinking model reading a mail; nobody is waiting on the answer, but a
|
||||
// hung llama-server must not pin the reader's connection forever.
|
||||
Timeout Duration `json:"timeout,omitempty"`
|
||||
}
|
||||
|
||||
// DefaultEmailTimeout — extraction budget per message.
|
||||
const DefaultEmailTimeout = 2 * time.Minute
|
||||
|
||||
// PhraserConfig — the LLM-backed phraser seam. The daemon spawns llama-server
|
||||
// as a managed subprocess and sends chat-completion requests to phrase nudge
|
||||
// and reminder messages. nil ⇒ the template-based Stub is used instead.
|
||||
@@ -441,6 +590,21 @@ type PhraserConfig struct {
|
||||
// persona and invented units). Chat, query and reminder phrasing always go
|
||||
// through the model regardless. See phraser.Config.LLMNudges.
|
||||
LLMNudges bool `json:"llm_nudges,omitempty"`
|
||||
|
||||
// SwapModels — the gguf files the running daemon is allowed to swap to
|
||||
// without a restart (Vikunja #250). Empty (the default) means the swap
|
||||
// capability does not exist: ipc.MethodSwapModel answers ErrUnknownMethod,
|
||||
// exactly like an unconfigured weather or telegram block.
|
||||
//
|
||||
// It is an allowlist and not a directory on purpose. The request carries a
|
||||
// path, and llama-server is started with it as `-m`; anything short of an
|
||||
// exact match against a list a human wrote in this file would make "swap the
|
||||
// model" mean "load a file of your choosing off my disk". ModelPath is
|
||||
// always swappable back to whether or not it is listed.
|
||||
//
|
||||
// Paths must be absolute — the daemon's working directory is not the
|
||||
// operator's, and a relative path here would resolve somewhere surprising.
|
||||
SwapModels []string `json:"swap_models,omitempty"`
|
||||
}
|
||||
|
||||
// EmbedderConfig — paths for the ONNX multilingual embedder. The daemon
|
||||
@@ -607,6 +771,24 @@ func (c *Config) applyDefaults() {
|
||||
c.MemoryEval.Interval = Duration(DefaultMemoryEvalInterval)
|
||||
}
|
||||
|
||||
// Same rule again: absent stays nil (⇒ mail ingestion refused), present gets
|
||||
// the timeout default so `{}` is a valid "on with the defaults".
|
||||
if c.Email != nil && c.Email.Timeout <= 0 {
|
||||
c.Email.Timeout = Duration(DefaultEmailTimeout)
|
||||
}
|
||||
|
||||
// A feeds block with no sources is the same as no block: nothing to poll,
|
||||
// nothing wired. Normalising it to nil keeps that "off" in one place.
|
||||
if c.Feeds != nil && len(c.Feeds.Sources) == 0 {
|
||||
c.Feeds = nil
|
||||
}
|
||||
|
||||
// Same rule for the crawler: a block that neither answers on demand nor
|
||||
// watches anything has nothing to do, so it is normalised to "off".
|
||||
if c.Crawl != nil && !c.Crawl.OnDemand && len(c.Crawl.Watches) == 0 {
|
||||
c.Crawl = nil
|
||||
}
|
||||
|
||||
if c.Voice != nil {
|
||||
if c.Voice.RouterThreshold <= 0 {
|
||||
c.Voice.RouterThreshold = DefaultRouterThreshold
|
||||
@@ -664,6 +846,22 @@ func (c *Config) validate() error {
|
||||
if c.Phraser.ModelPath == "" {
|
||||
return errors.New("phraser.model_path is required")
|
||||
}
|
||||
// A relative entry in the swap allowlist would resolve against the
|
||||
// daemon's working directory, so the path a human reads in this file
|
||||
// would not be the path llama-server is handed. Fail at startup.
|
||||
for _, m := range c.Phraser.SwapModels {
|
||||
if !filepath.IsAbs(m) {
|
||||
return fmt.Errorf("phraser.swap_models: %q must be an absolute path", m)
|
||||
}
|
||||
}
|
||||
}
|
||||
// The update block is validated here even though mavend never acts on it: a
|
||||
// half-written update config that is only noticed by cmd/mavupdate is noticed
|
||||
// at the worst possible moment, halfway through deploying a new build.
|
||||
if c.Update != nil {
|
||||
if err := c.Update.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if c.Voice != nil && c.Voice.Enabled {
|
||||
if c.Voice.Bind == "" {
|
||||
|
||||
@@ -293,3 +293,76 @@ func TestPatternProposalNotifyDefaultsOff(t *testing.T) {
|
||||
t.Errorf("cooldown = %v, want 6h", c.PatternProposals.Cooldown)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSwapModelsAbsentMeansOff — the swap capability does not exist unless the
|
||||
// operator lists the models he allows (Vikunja #250).
|
||||
func TestSwapModelsAbsentMeansOff(t *testing.T) {
|
||||
c, err := Load(writeConfig(t, `{"phraser": {"model_path": "/m/qwen.gguf"}}`))
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
if len(c.Phraser.SwapModels) != 0 {
|
||||
t.Errorf("swap_models = %v; want empty when unconfigured", c.Phraser.SwapModels)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSwapModelsParsedAndMustBeAbsolute(t *testing.T) {
|
||||
c, err := Load(writeConfig(t, `{"phraser": {
|
||||
"model_path": "/m/qwen.gguf",
|
||||
"swap_models": ["/m/qwen.gguf", "/m/qwen-cpt.gguf"]
|
||||
}}`))
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
if len(c.Phraser.SwapModels) != 2 {
|
||||
t.Fatalf("swap_models = %v; want 2 entries", c.Phraser.SwapModels)
|
||||
}
|
||||
// A relative entry would resolve against the daemon's cwd, not the operator's.
|
||||
if _, err := Load(writeConfig(t, `{"phraser": {
|
||||
"model_path": "/m/qwen.gguf",
|
||||
"swap_models": ["models/llm/qwen.gguf"]
|
||||
}}`)); err == nil {
|
||||
t.Error("Load accepted a relative swap_models entry; want a startup failure")
|
||||
}
|
||||
}
|
||||
|
||||
// TestUpdateBlockAbsentMeansOff — mavend never updates itself; the block only
|
||||
// exists so cmd/mavupdate can find the deployment it is asked to update
|
||||
// (Vikunja #249). Absent is the normal state.
|
||||
func TestUpdateBlockAbsentMeansOff(t *testing.T) {
|
||||
c, err := Load(writeConfig(t, `{}`))
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
if c.Update != nil {
|
||||
t.Errorf("update = %+v; want nil when unconfigured", c.Update)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateBlockValidatedAtStartup(t *testing.T) {
|
||||
good := `{"update": {
|
||||
"source_dir": "/srv/maven",
|
||||
"install_dir": "/srv/maven",
|
||||
"snapshot_dir": "/var/lib/maven/snapshots",
|
||||
"binaries": ["mavend", "mavweb"],
|
||||
"restart_cmd": ["docker", "compose", "up", "-d", "--build", "mavend"],
|
||||
"health_socket": "/run/maven/mavend.sock"
|
||||
}}`
|
||||
c, err := Load(writeConfig(t, good))
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
if c.Update == nil || len(c.Update.Binaries) != 2 {
|
||||
t.Fatalf("update block = %+v; want it parsed", c.Update)
|
||||
}
|
||||
// A block with no health check cannot detect its own failure, so it cannot
|
||||
// roll back — refused at load, not halfway through a deploy.
|
||||
noHealth := `{"update": {
|
||||
"source_dir": "/srv/maven", "install_dir": "/srv/maven",
|
||||
"snapshot_dir": "/var/lib/maven/snapshots",
|
||||
"binaries": ["mavend"], "restart_cmd": ["true"]
|
||||
}}`
|
||||
if _, err := Load(writeConfig(t, noHealth)); err == nil {
|
||||
t.Error("Load accepted an update block with no health_socket")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
// Package crawl reads a web page: fetch, robots check, HTML to text.
|
||||
//
|
||||
// It is the LAST place Maven looks for an answer, and that ordering is the whole
|
||||
// design. "Never phones home" is deprecated, but what replaced it puts local
|
||||
// sources first: the resident model, then his own memory, then the Kiwix ZIMs on
|
||||
// the box (internal/kiwix), and only then the network. A local read costs
|
||||
// nothing and leaks nothing; a fetch costs a round-trip and puts a URL in
|
||||
// someone's access log. So this package exists to be the fallback, not the
|
||||
// front door — see the querySources chain in cmd/mavend/actions_query.go for
|
||||
// where it actually sits.
|
||||
//
|
||||
// What never leaves the box: his notes, his facts, the persona block, the
|
||||
// conversation history. Only the URL is requested and, for the on-demand path,
|
||||
// only because he said it out loud. Nothing here reads the store.
|
||||
//
|
||||
// The limits are not in this package — they are in internal/webfetch, which is
|
||||
// the only way anything here touches a socket: http(s) only, host allow/deny,
|
||||
// private-address refusal, size cap, redirect cap, per-host rate limit. What
|
||||
// this package adds is politeness (robots.txt) and dedup.
|
||||
package crawl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Errors callers distinguish.
|
||||
var (
|
||||
ErrRobots = errors.New("crawl: robots.txt disallows this path")
|
||||
ErrNotHTML = errors.New("crawl: response is not html or text")
|
||||
)
|
||||
|
||||
// Fetcher is the guarded HTTP door (internal/webfetch adapted by the daemon). An
|
||||
// interface so this package constructs no http.Client of its own and can be
|
||||
// tested without a network.
|
||||
type Fetcher interface {
|
||||
Get(ctx context.Context, url string) (*Response, error)
|
||||
}
|
||||
|
||||
// Response is the minimum a crawl needs from a fetch.
|
||||
type Response struct {
|
||||
URL string
|
||||
ContentType string
|
||||
Body []byte
|
||||
}
|
||||
|
||||
// Config — crawler knobs.
|
||||
type Config struct {
|
||||
// UserAgent is the name matched against robots.txt groups. It must be the
|
||||
// same string the fetcher sends, or Maven would be claiming one identity
|
||||
// and obeying the rules for another.
|
||||
UserAgent string
|
||||
// MaxRunes caps extracted text. 0 ⇒ DefaultMaxRunes.
|
||||
MaxRunes int
|
||||
// RobotsTTL — how long a parsed robots.txt is trusted. 0 ⇒ 1h.
|
||||
RobotsTTL time.Duration
|
||||
// Now is injectable for tests. nil ⇒ time.Now.
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// Crawler fetches and extracts pages. Safe for concurrent use.
|
||||
type Crawler struct {
|
||||
fetch Fetcher
|
||||
cfg Config
|
||||
robots *robotsCache
|
||||
}
|
||||
|
||||
// New builds a crawler. Returns nil when there is no fetcher, which is how the
|
||||
// daemon expresses "crawling is off unless configured".
|
||||
func New(fetch Fetcher, cfg Config) *Crawler {
|
||||
if fetch == nil {
|
||||
return nil
|
||||
}
|
||||
if cfg.UserAgent == "" {
|
||||
cfg.UserAgent = "Maven"
|
||||
}
|
||||
if cfg.MaxRunes <= 0 {
|
||||
cfg.MaxRunes = DefaultMaxRunes
|
||||
}
|
||||
if cfg.RobotsTTL <= 0 {
|
||||
cfg.RobotsTTL = time.Hour
|
||||
}
|
||||
if cfg.Now == nil {
|
||||
cfg.Now = time.Now
|
||||
}
|
||||
return &Crawler{fetch: fetch, cfg: cfg, robots: newRobotsCache(cfg.RobotsTTL)}
|
||||
}
|
||||
|
||||
// Page fetches rawURL and returns its text. It checks robots.txt first and
|
||||
// refuses a disallowed path with ErrRobots — there is no override.
|
||||
func (c *Crawler) Page(ctx context.Context, rawURL string) (Page, error) {
|
||||
u, err := url.Parse(strings.TrimSpace(rawURL))
|
||||
if err != nil {
|
||||
return Page{}, fmt.Errorf("crawl: bad url %q: %w", rawURL, err)
|
||||
}
|
||||
ok, err := c.allowed(ctx, u)
|
||||
if err != nil {
|
||||
return Page{}, err
|
||||
}
|
||||
if !ok {
|
||||
return Page{}, fmt.Errorf("%w: %s", ErrRobots, u.Path)
|
||||
}
|
||||
resp, err := c.fetch.Get(ctx, u.String())
|
||||
if err != nil {
|
||||
return Page{}, err
|
||||
}
|
||||
// A PDF or an image is bytes Maven cannot read; saying so beats storing
|
||||
// binary garbage as a "note".
|
||||
ct := strings.ToLower(resp.ContentType)
|
||||
if ct != "" && !strings.Contains(ct, "html") && !strings.Contains(ct, "text/") &&
|
||||
!strings.Contains(ct, "xml") && !strings.Contains(ct, "json") {
|
||||
return Page{}, fmt.Errorf("%w: %s", ErrNotHTML, resp.ContentType)
|
||||
}
|
||||
return Extract(resp.URL, resp.Body, c.cfg.MaxRunes), nil
|
||||
}
|
||||
|
||||
// allowed consults robots.txt for u's host, reading it at most once per TTL.
|
||||
//
|
||||
// A robots.txt that cannot be fetched (404, a timeout, a blocked host) means
|
||||
// allow, per the standard. The one thing that is NOT fail-open is an explicit
|
||||
// Disallow.
|
||||
func (c *Crawler) allowed(ctx context.Context, u *url.URL) (bool, error) {
|
||||
host := u.Host
|
||||
now := c.cfg.Now()
|
||||
rules, ok := c.robots.get(host, now)
|
||||
if !ok {
|
||||
robotsURL := u.Scheme + "://" + host + "/robots.txt"
|
||||
resp, err := c.fetch.Get(ctx, robotsURL)
|
||||
switch {
|
||||
case err != nil:
|
||||
// Note what is NOT swallowed: a refusal from the guarded fetcher.
|
||||
// If webfetch says this host is denied or private, the page fetch
|
||||
// would fail the same way, and reporting the real reason beats
|
||||
// reporting a robots verdict we never got.
|
||||
if isFatalFetchError(err) {
|
||||
return false, err
|
||||
}
|
||||
rules = Rules{}
|
||||
default:
|
||||
rules = ParseRobots(string(resp.Body), c.cfg.UserAgent)
|
||||
}
|
||||
c.robots.put(host, rules, now)
|
||||
}
|
||||
path := u.EscapedPath()
|
||||
if u.RawQuery != "" {
|
||||
path += "?" + u.RawQuery
|
||||
}
|
||||
return rules.Allowed(path), nil
|
||||
}
|
||||
|
||||
// isFatalFetchError — a fetch failure that means "this host is off limits"
|
||||
// rather than "there is no robots.txt here". The sentinel set is webfetch's, but
|
||||
// this package must not import it (the interface exists precisely so it does
|
||||
// not), so the check is on the message. Ugly and honest: the alternative is a
|
||||
// dependency inversion for two strings.
|
||||
func isFatalFetchError(err error) bool {
|
||||
s := err.Error()
|
||||
return strings.Contains(s, "not allowed") || strings.Contains(s, "private address") ||
|
||||
strings.Contains(s, "only http and https")
|
||||
}
|
||||
|
||||
// Hash is the dedup key for a crawl result: the sha256 of the extracted text,
|
||||
// hex, first 16 chars. Text and not raw HTML, because a page whose only change
|
||||
// is a rotating ad slot or a CSRF token has not changed.
|
||||
func Hash(text string) string {
|
||||
sum := sha256.Sum256([]byte(strings.TrimSpace(text)))
|
||||
return hex.EncodeToString(sum[:])[:16]
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package crawl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fakeFetcher serves canned pages by URL and counts requests, so a test can
|
||||
// assert that robots.txt was read once and that a refusal never reached the page.
|
||||
type fakeFetcher struct {
|
||||
pages map[string]Response
|
||||
err error
|
||||
calls []string
|
||||
}
|
||||
|
||||
func (f *fakeFetcher) Get(_ context.Context, u string) (*Response, error) {
|
||||
f.calls = append(f.calls, u)
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
r, ok := f.pages[u]
|
||||
if !ok {
|
||||
return nil, errors.New("http 404")
|
||||
}
|
||||
if r.URL == "" {
|
||||
r.URL = u
|
||||
}
|
||||
if r.ContentType == "" {
|
||||
r.ContentType = "text/html; charset=utf-8"
|
||||
}
|
||||
return &r, nil
|
||||
}
|
||||
|
||||
const htmlPage = `<html><head><title>Почему небо синее</title>
|
||||
<style>body{color:red}</style><script>track()</script></head>
|
||||
<body><nav>меню</nav><h1>Небо</h1>
|
||||
<p>Свет рассеивается на молекулах воздуха.</p>
|
||||
<p>Короткие волны рассеиваются сильнее.</p>
|
||||
<footer>© 2026</footer></body></html>`
|
||||
|
||||
func newTestCrawler(f *fakeFetcher) *Crawler {
|
||||
return New(f, Config{UserAgent: "Maven/1.0", Now: func() time.Time { return time.Unix(0, 0) }})
|
||||
}
|
||||
|
||||
func TestPageExtractsText(t *testing.T) {
|
||||
f := &fakeFetcher{pages: map[string]Response{
|
||||
"https://example.org/sky": {Body: []byte(htmlPage)},
|
||||
}}
|
||||
page, err := newTestCrawler(f).Page(context.Background(), "https://example.org/sky")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if page.Title != "Почему небо синее" {
|
||||
t.Errorf("title = %q", page.Title)
|
||||
}
|
||||
if !strings.Contains(page.Text, "Свет рассеивается") {
|
||||
t.Errorf("body text missing: %q", page.Text)
|
||||
}
|
||||
for _, junk := range []string{"track()", "color:red", "меню", "© 2026"} {
|
||||
if strings.Contains(page.Text, junk) {
|
||||
t.Errorf("%q survived extraction: %q", junk, page.Text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRobotsIsCheckedAndObeyed(t *testing.T) {
|
||||
f := &fakeFetcher{pages: map[string]Response{
|
||||
"https://example.org/robots.txt": {Body: []byte("User-agent: *\nDisallow: /secret\n"), ContentType: "text/plain"},
|
||||
"https://example.org/secret/x": {Body: []byte(htmlPage)},
|
||||
"https://example.org/open": {Body: []byte(htmlPage)},
|
||||
}}
|
||||
c := newTestCrawler(f)
|
||||
if _, err := c.Page(context.Background(), "https://example.org/secret/x"); !errors.Is(err, ErrRobots) {
|
||||
t.Fatalf("error = %v, want ErrRobots", err)
|
||||
}
|
||||
for _, u := range f.calls {
|
||||
if strings.Contains(u, "/secret") {
|
||||
t.Fatal("the disallowed page was fetched anyway")
|
||||
}
|
||||
}
|
||||
if _, err := c.Page(context.Background(), "https://example.org/open"); err != nil {
|
||||
t.Fatalf("allowed page: %v", err)
|
||||
}
|
||||
// robots.txt was read once for the host, not once per page.
|
||||
robotsReads := 0
|
||||
for _, u := range f.calls {
|
||||
if strings.HasSuffix(u, "/robots.txt") {
|
||||
robotsReads++
|
||||
}
|
||||
}
|
||||
if robotsReads != 1 {
|
||||
t.Fatalf("robots.txt read %d times, want 1", robotsReads)
|
||||
}
|
||||
}
|
||||
|
||||
// No robots.txt means allow — that is the standard, and the alternative makes
|
||||
// most of the web unreadable.
|
||||
func TestMissingRobotsAllows(t *testing.T) {
|
||||
f := &fakeFetcher{pages: map[string]Response{
|
||||
"https://example.org/page": {Body: []byte(htmlPage)},
|
||||
}}
|
||||
if _, err := newTestCrawler(f).Page(context.Background(), "https://example.org/page"); err != nil {
|
||||
t.Fatalf("err = %v, want the page", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A refusal from the guarded fetcher must surface as itself, not be laundered
|
||||
// into "no robots.txt, go ahead".
|
||||
func TestFetcherRefusalIsNotSwallowed(t *testing.T) {
|
||||
f := &fakeFetcher{err: errors.New("webfetch: refusing to connect to a private address: 127.0.0.1")}
|
||||
_, err := newTestCrawler(f).Page(context.Background(), "http://127.0.0.1:9100/mcp")
|
||||
if err == nil || !strings.Contains(err.Error(), "private address") {
|
||||
t.Fatalf("error = %v, want the fetcher's refusal", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNonTextIsRefused(t *testing.T) {
|
||||
f := &fakeFetcher{pages: map[string]Response{
|
||||
"https://example.org/f.pdf": {Body: []byte("%PDF-1.7"), ContentType: "application/pdf"},
|
||||
}}
|
||||
if _, err := newTestCrawler(f).Page(context.Background(), "https://example.org/f.pdf"); !errors.Is(err, ErrNotHTML) {
|
||||
t.Fatalf("error = %v, want ErrNotHTML", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaxRunesCapsText(t *testing.T) {
|
||||
long := "<html><body><p>" + strings.Repeat("привет ", 2000) + "</p></body></html>"
|
||||
f := &fakeFetcher{pages: map[string]Response{"https://example.org/l": {Body: []byte(long)}}}
|
||||
c := New(f, Config{MaxRunes: 50})
|
||||
page, err := c.Page(context.Background(), "https://example.org/l")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := len([]rune(page.Text)); n > 51 {
|
||||
t.Fatalf("text = %d runes, want the 50-rune cap", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewWithoutFetcherIsNil(t *testing.T) {
|
||||
if New(nil, Config{}) != nil {
|
||||
t.Fatal("a crawler with no fetcher must be nil — crawling is off unless configured")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHashIgnoresNothingButText(t *testing.T) {
|
||||
if Hash("a") == Hash("b") {
|
||||
t.Fatal("different text hashed the same")
|
||||
}
|
||||
if Hash(" same \n") != Hash("same") {
|
||||
t.Fatal("surrounding whitespace changed the hash")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package crawl
|
||||
|
||||
import (
|
||||
"html"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// HTML → text, with a regexp and no tokenizer.
|
||||
//
|
||||
// golang.org/x/net/html is not vendored and the network is not assumed, so this
|
||||
// is stdlib. That is less of a compromise than it sounds: the unit of context
|
||||
// here is a few hundred words for a 4096-token model to read, exactly like the
|
||||
// Kiwix snippet, so what matters is dropping script/style/nav noise and keeping
|
||||
// paragraph boundaries. A DOM would buy correctness on malformed markup that is
|
||||
// then thrown away by truncation anyway.
|
||||
//
|
||||
// What this deliberately does NOT do: run JavaScript, follow links, or extract
|
||||
// structured fields with CSS selectors or an LLM prompt. The plan's step 2 asked
|
||||
// for the last of those; see docs/plans/14-web-crawler.md for why it was left
|
||||
// out for now.
|
||||
|
||||
var (
|
||||
// RE2 has no backreferences, so each tag pair is spelled out rather than
|
||||
// captured and matched against itself.
|
||||
dropRE = regexp.MustCompile(pairsRE("script", "style", "noscript", "svg", "head", "nav", "footer", "form"))
|
||||
titleRE = regexp.MustCompile(`(?is)<title\b[^>]*>(.*?)</title>`)
|
||||
h1RE = regexp.MustCompile(`(?is)<h1\b[^>]*>(.*?)</h1>`)
|
||||
// Block-level tags become newlines so paragraphs survive as paragraphs.
|
||||
blockRE = regexp.MustCompile(`(?is)</?(p|div|br|li|tr|h[1-6]|section|article|blockquote|pre)\b[^>]*>`)
|
||||
tagRE = regexp.MustCompile(`(?s)<[^>]*>`)
|
||||
commentRE = regexp.MustCompile(`(?s)<!--.*?-->`)
|
||||
spaceRE = regexp.MustCompile(`[ \t\f\v]+`)
|
||||
blankRE = regexp.MustCompile(`\n{2,}`)
|
||||
)
|
||||
|
||||
// pairsRE builds `(?is)<tag …>…</tag>|…` for the given tags.
|
||||
func pairsRE(tags ...string) string {
|
||||
parts := make([]string, 0, len(tags))
|
||||
for _, t := range tags {
|
||||
parts = append(parts, `<`+t+`\b[^>]*>.*?</`+t+`>`)
|
||||
}
|
||||
return `(?is)` + strings.Join(parts, "|")
|
||||
}
|
||||
|
||||
// Page is an extracted page.
|
||||
type Page struct {
|
||||
URL string
|
||||
Title string
|
||||
Text string // plain text, paragraphs separated by single newlines
|
||||
}
|
||||
|
||||
// Extract turns a fetched HTML document into a Page. maxRunes caps the text (0 ⇒
|
||||
// DefaultMaxRunes); the cap is on runes, not bytes, because a Russian page cut
|
||||
// at a byte boundary ends in half a letter.
|
||||
func Extract(url string, body []byte, maxRunes int) Page {
|
||||
if maxRunes <= 0 {
|
||||
maxRunes = DefaultMaxRunes
|
||||
}
|
||||
s := string(body)
|
||||
s = commentRE.ReplaceAllString(s, " ")
|
||||
|
||||
title := firstGroup(titleRE, s)
|
||||
if title == "" {
|
||||
title = firstGroup(h1RE, s)
|
||||
}
|
||||
|
||||
s = dropRE.ReplaceAllString(s, "\n")
|
||||
s = blockRE.ReplaceAllString(s, "\n")
|
||||
s = tagRE.ReplaceAllString(s, " ")
|
||||
s = html.UnescapeString(s)
|
||||
s = spaceRE.ReplaceAllString(s, " ")
|
||||
|
||||
var lines []string
|
||||
for _, l := range strings.Split(s, "\n") {
|
||||
if l = strings.TrimSpace(l); l != "" {
|
||||
lines = append(lines, l)
|
||||
}
|
||||
}
|
||||
text := blankRE.ReplaceAllString(strings.Join(lines, "\n"), "\n")
|
||||
|
||||
return Page{URL: url, Title: title, Text: TrimRunes(text, maxRunes)}
|
||||
}
|
||||
|
||||
// DefaultMaxRunes — how much of a page is kept. ~4000 runes is a long answer's
|
||||
// worth of context and still leaves room in a 4096-token window for the prompt
|
||||
// and the reply.
|
||||
const DefaultMaxRunes = 4000
|
||||
|
||||
func firstGroup(re *regexp.Regexp, s string) string {
|
||||
m := re.FindStringSubmatch(s)
|
||||
if len(m) < 2 {
|
||||
return ""
|
||||
}
|
||||
t := tagRE.ReplaceAllString(m[1], " ")
|
||||
return strings.TrimSpace(strings.Join(strings.Fields(html.UnescapeString(t)), " "))
|
||||
}
|
||||
|
||||
// TrimRunes cuts s to at most max runes, on a rune boundary.
|
||||
func TrimRunes(s string, max int) string {
|
||||
r := []rune(s)
|
||||
if len(r) <= max {
|
||||
return s
|
||||
}
|
||||
return strings.TrimSpace(string(r[:max])) + "…"
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
package crawl
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// robots.txt, parsed the small way: no wildcards beyond the two the standard
|
||||
// actually defines (`*` inside a path and `$` at the end), no sitemaps, no
|
||||
// crawl-delay-per-agent gymnastics. A personal assistant reading a handful of
|
||||
// pages does not need a spec-complete implementation; it needs to not be rude,
|
||||
// and to be auditable in one sitting.
|
||||
//
|
||||
// Two rules worth stating because they are choices, not accidents:
|
||||
//
|
||||
// - a missing or unreadable robots.txt means ALLOW. That is what the standard
|
||||
// says (404 ⇒ unrestricted), and the alternative would make a site that
|
||||
// simply has no robots.txt unreadable;
|
||||
// - an explicit Disallow means REFUSE, and Maven does not offer an override.
|
||||
// There is no "but he asked me to" flag: the page is not read.
|
||||
|
||||
// Rules is a parsed robots.txt for one user-agent.
|
||||
type Rules struct {
|
||||
allow []string
|
||||
disallow []string
|
||||
// Delay is Crawl-delay in seconds when the group named one, 0 otherwise.
|
||||
// The fetcher's own per-host rate limit is the floor; this can only make
|
||||
// Maven slower, never faster.
|
||||
Delay time.Duration
|
||||
}
|
||||
|
||||
// ParseRobots reads robots.txt and returns the rules that apply to agent.
|
||||
//
|
||||
// Group selection follows the standard: the most specific matching group wins,
|
||||
// which here means an exact user-agent match beats `*`. Lines that are neither
|
||||
// are ignored rather than guessed at.
|
||||
func ParseRobots(body string, agent string) Rules {
|
||||
agent = strings.ToLower(agent)
|
||||
|
||||
type group struct {
|
||||
agents []string
|
||||
allow []string
|
||||
disallow []string
|
||||
delay time.Duration
|
||||
}
|
||||
var groups []group
|
||||
var cur *group
|
||||
// startNew tracks whether the next User-agent line opens a new group or
|
||||
// joins the current one: consecutive User-agent lines share their rules.
|
||||
startNew := true
|
||||
|
||||
for _, raw := range strings.Split(body, "\n") {
|
||||
line := raw
|
||||
if i := strings.IndexByte(line, '#'); i >= 0 {
|
||||
line = line[:i]
|
||||
}
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
key, val, ok := strings.Cut(line, ":")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
key = strings.ToLower(strings.TrimSpace(key))
|
||||
val = strings.TrimSpace(val)
|
||||
|
||||
switch key {
|
||||
case "user-agent":
|
||||
if startNew || cur == nil {
|
||||
groups = append(groups, group{})
|
||||
cur = &groups[len(groups)-1]
|
||||
startNew = false
|
||||
}
|
||||
cur.agents = append(cur.agents, strings.ToLower(val))
|
||||
case "disallow":
|
||||
if cur == nil {
|
||||
continue
|
||||
}
|
||||
startNew = true
|
||||
// "Disallow:" with an empty value allows everything, and is not a
|
||||
// path rule at all.
|
||||
if val != "" {
|
||||
cur.disallow = append(cur.disallow, val)
|
||||
}
|
||||
case "allow":
|
||||
if cur == nil {
|
||||
continue
|
||||
}
|
||||
startNew = true
|
||||
if val != "" {
|
||||
cur.allow = append(cur.allow, val)
|
||||
}
|
||||
case "crawl-delay":
|
||||
if cur == nil {
|
||||
continue
|
||||
}
|
||||
startNew = true
|
||||
if d, err := time.ParseDuration(val + "s"); err == nil && d > 0 {
|
||||
cur.delay = d
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var star, exact *group
|
||||
for i := range groups {
|
||||
for _, a := range groups[i].agents {
|
||||
if a == "*" && star == nil {
|
||||
star = &groups[i]
|
||||
}
|
||||
// A robots.txt names "maven", we send "Maven/1.0 (…)": match on
|
||||
// prefix, which is how every crawler reads this field.
|
||||
if a != "*" && a != "" && strings.HasPrefix(agent, a) {
|
||||
exact = &groups[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
g := exact
|
||||
if g == nil {
|
||||
g = star
|
||||
}
|
||||
if g == nil {
|
||||
return Rules{}
|
||||
}
|
||||
return Rules{allow: g.allow, disallow: g.disallow, Delay: g.delay}
|
||||
}
|
||||
|
||||
// Allowed reports whether path may be fetched. Longest matching rule wins, and
|
||||
// Allow beats Disallow at equal length — the standard's tie-break, and the one
|
||||
// that makes "Disallow: /" plus "Allow: /public" mean what it looks like.
|
||||
func (r Rules) Allowed(path string) bool {
|
||||
if path == "" {
|
||||
path = "/"
|
||||
}
|
||||
best, allowed := -1, true
|
||||
for _, p := range r.disallow {
|
||||
if n, ok := matchPath(p, path); ok && n > best {
|
||||
best, allowed = n, false
|
||||
}
|
||||
}
|
||||
for _, p := range r.allow {
|
||||
if n, ok := matchPath(p, path); ok && n >= best {
|
||||
best, allowed = n, true
|
||||
}
|
||||
}
|
||||
return allowed
|
||||
}
|
||||
|
||||
// matchPath applies a robots path pattern and returns the pattern's length as
|
||||
// the specificity score. `*` matches any run of characters, `$` anchors the end.
|
||||
// A pattern is a PREFIX match otherwise, which is what "Disallow: /admin" means.
|
||||
func matchPath(pattern, path string) (int, bool) {
|
||||
score := len(pattern)
|
||||
re, err := robotsRegexp(pattern)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
return score, re.MatchString(path)
|
||||
}
|
||||
|
||||
// robotsRegexp turns a robots path pattern into an anchored-at-the-start
|
||||
// regexp. Everything but `*` and a trailing `$` is a literal, so the pattern is
|
||||
// quoted first and the two metacharacters are put back afterwards.
|
||||
func robotsRegexp(pattern string) (*regexp.Regexp, error) {
|
||||
end := ""
|
||||
if strings.HasSuffix(pattern, "$") {
|
||||
pattern = strings.TrimSuffix(pattern, "$")
|
||||
end = "$"
|
||||
}
|
||||
parts := strings.Split(pattern, "*")
|
||||
for i, p := range parts {
|
||||
parts[i] = regexp.QuoteMeta(p)
|
||||
}
|
||||
return regexp.Compile("^" + strings.Join(parts, ".*") + end)
|
||||
}
|
||||
|
||||
// robotsCache holds parsed rules per host so a crawl of ten pages on one site
|
||||
// reads robots.txt once. TTL because a site may change its mind, and a daemon
|
||||
// that runs for weeks would otherwise never notice.
|
||||
type robotsCache struct {
|
||||
ttl time.Duration
|
||||
mu sync.Mutex
|
||||
m map[string]robotsEntry
|
||||
}
|
||||
|
||||
type robotsEntry struct {
|
||||
rules Rules
|
||||
at time.Time
|
||||
}
|
||||
|
||||
func newRobotsCache(ttl time.Duration) *robotsCache {
|
||||
return &robotsCache{ttl: ttl, m: map[string]robotsEntry{}}
|
||||
}
|
||||
|
||||
func (c *robotsCache) get(host string, now time.Time) (Rules, bool) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
e, ok := c.m[host]
|
||||
if !ok || now.Sub(e.at) > c.ttl {
|
||||
return Rules{}, false
|
||||
}
|
||||
return e.rules, true
|
||||
}
|
||||
|
||||
func (c *robotsCache) put(host string, r Rules, now time.Time) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.m[host] = robotsEntry{rules: r, at: now}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package crawl
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const robotsBody = `# a comment
|
||||
User-agent: *
|
||||
Disallow: /private
|
||||
Disallow: /tmp/
|
||||
Crawl-delay: 5
|
||||
|
||||
User-agent: Maven
|
||||
Disallow: /
|
||||
Allow: /public
|
||||
`
|
||||
|
||||
func TestParseRobotsPicksTheMostSpecificGroup(t *testing.T) {
|
||||
// The Maven group applies to us even though we send a longer UA string.
|
||||
r := ParseRobots(robotsBody, "Maven/1.0 (self-hosted personal assistant)")
|
||||
if r.Allowed("/anything") {
|
||||
t.Error("Disallow: / in our own group was ignored")
|
||||
}
|
||||
if !r.Allowed("/public/page") {
|
||||
t.Error("Allow: /public must beat the shorter Disallow: /")
|
||||
}
|
||||
|
||||
// A different agent falls into the * group.
|
||||
star := ParseRobots(robotsBody, "SomeoneElse/2")
|
||||
if !star.Allowed("/anything") {
|
||||
t.Error("the * group disallows nothing but /private and /tmp/")
|
||||
}
|
||||
if star.Allowed("/private/x") || star.Allowed("/tmp/") {
|
||||
t.Error("the * group's disallows were not applied")
|
||||
}
|
||||
if star.Delay != 5*time.Second {
|
||||
t.Errorf("crawl-delay = %v, want 5s", star.Delay)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRobotsEmptyMeansAllowAll(t *testing.T) {
|
||||
for _, body := range []string{"", "# nothing here\n", "User-agent: *\nDisallow:\n"} {
|
||||
if !ParseRobots(body, "Maven").Allowed("/whatever") {
|
||||
t.Errorf("body %q must allow everything", body)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRobotsWildcards(t *testing.T) {
|
||||
r := ParseRobots("User-agent: *\nDisallow: /*.pdf$\nDisallow: /a/*/secret\n", "Maven")
|
||||
if r.Allowed("/docs/manual.pdf") {
|
||||
t.Error("*.pdf$ did not match")
|
||||
}
|
||||
if !r.Allowed("/docs/manual.pdf.html") {
|
||||
t.Error("$ must anchor at the end")
|
||||
}
|
||||
if r.Allowed("/a/b/secret") {
|
||||
t.Error("/a/*/secret did not match")
|
||||
}
|
||||
if !r.Allowed("/a/b/public") {
|
||||
t.Error("unrelated path was refused")
|
||||
}
|
||||
}
|
||||
|
||||
// Consecutive User-agent lines share one group, which is common in the wild.
|
||||
func TestRobotsSharedGroup(t *testing.T) {
|
||||
r := ParseRobots("User-agent: Googlebot\nUser-agent: Maven\nDisallow: /x\n", "Maven/1.0")
|
||||
if r.Allowed("/x/y") {
|
||||
t.Fatal("a shared group's rules were not applied to the second agent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRobotsCacheTTL(t *testing.T) {
|
||||
c := newRobotsCache(time.Minute)
|
||||
now := time.Now()
|
||||
c.put("example.com", ParseRobots("User-agent: *\nDisallow: /\n", "Maven"), now)
|
||||
if _, ok := c.get("example.com", now.Add(30*time.Second)); !ok {
|
||||
t.Error("a fresh entry must be served from cache")
|
||||
}
|
||||
if _, ok := c.get("example.com", now.Add(2*time.Minute)); ok {
|
||||
t.Error("an expired entry must be re-read")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
package crawl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Scheduled crawls: a page is re-read on an interval, and when its TEXT changed
|
||||
// the new text is written as a note. Nothing is dispatched — same rule as the
|
||||
// feed poller (Vikunja #258). A page that announced its own change would be a
|
||||
// nag, and "the docs page changed" is not worth interrupting anyone for.
|
||||
//
|
||||
// Dedup is by content hash, so a page that re-renders identically writes nothing
|
||||
// and a rotating ad slot does not count as news.
|
||||
|
||||
// WatchConfig — one page to keep an eye on.
|
||||
type WatchConfig struct {
|
||||
Name string // note source is "crawl:<Name>"
|
||||
URL string // http(s), guarded by the fetcher
|
||||
Interval time.Duration // 0 ⇒ Watcher's default
|
||||
}
|
||||
|
||||
// Notes is core's note-writing half (same shape as ipc.CoreAPI's method).
|
||||
type Notes interface {
|
||||
WriteNote(ctx context.Context, ts time.Time, text string, embedding []float32, source string) (int64, error)
|
||||
}
|
||||
|
||||
// Hashes remembers the last text hash per watch, durably, so a restart does not
|
||||
// re-note an unchanged page. The daemon backs this with config facts
|
||||
// ("crawl:hash:<name>").
|
||||
type Hashes interface {
|
||||
LastHash(ctx context.Context, name string) (string, error)
|
||||
SetHash(ctx context.Context, name, hash string) error
|
||||
}
|
||||
|
||||
// Embedder embeds a note on its way into the store. nil ⇒ no vector.
|
||||
type Embedder interface {
|
||||
Embed(ctx context.Context, text string) ([]float32, error)
|
||||
}
|
||||
|
||||
// DefaultWatchInterval — pages change slowly, and every check is a request in
|
||||
// someone's log.
|
||||
const DefaultWatchInterval = 6 * time.Hour
|
||||
|
||||
// Watcher re-reads watched pages on their interval.
|
||||
type Watcher struct {
|
||||
c *Crawler
|
||||
watches []WatchConfig
|
||||
notes Notes
|
||||
hashes Hashes
|
||||
embed Embedder
|
||||
interval time.Duration
|
||||
nextDue map[string]time.Time
|
||||
}
|
||||
|
||||
// NewWatcher wires the scheduled half, or returns nil when there is nothing to
|
||||
// watch. Callers check for nil: no watches, no goroutine, no request.
|
||||
func NewWatcher(c *Crawler, watches []WatchConfig, notes Notes, hashes Hashes, embed Embedder, defaultInterval time.Duration) *Watcher {
|
||||
if c == nil || notes == nil {
|
||||
return nil
|
||||
}
|
||||
var valid []WatchConfig
|
||||
for _, w := range watches {
|
||||
if strings.TrimSpace(w.Name) == "" || strings.TrimSpace(w.URL) == "" {
|
||||
log.Printf("crawl: skipping a watch with no name or no url")
|
||||
continue
|
||||
}
|
||||
valid = append(valid, w)
|
||||
}
|
||||
if len(valid) == 0 {
|
||||
return nil
|
||||
}
|
||||
if defaultInterval <= 0 {
|
||||
defaultInterval = DefaultWatchInterval
|
||||
}
|
||||
return &Watcher{
|
||||
c: c, watches: valid, notes: notes, hashes: hashes, embed: embed,
|
||||
interval: defaultInterval, nextDue: map[string]time.Time{},
|
||||
}
|
||||
}
|
||||
|
||||
// Watches returns the configured watches.
|
||||
func (w *Watcher) Watches() []WatchConfig { return w.watches }
|
||||
|
||||
// CheckDue re-reads every watch whose interval elapsed and returns how many
|
||||
// notes were written. Errors are logged per watch, never returned: one dead page
|
||||
// must not stop the others.
|
||||
func (w *Watcher) CheckDue(ctx context.Context, now time.Time) int {
|
||||
written := 0
|
||||
for _, watch := range w.watches {
|
||||
if due, ok := w.nextDue[watch.Name]; ok && now.Before(due) {
|
||||
continue
|
||||
}
|
||||
interval := watch.Interval
|
||||
if interval <= 0 {
|
||||
interval = w.interval
|
||||
}
|
||||
w.nextDue[watch.Name] = now.Add(interval)
|
||||
changed, err := w.Check(ctx, watch, now)
|
||||
if err != nil {
|
||||
log.Printf("crawl: watch %s: %v", watch.Name, err)
|
||||
continue
|
||||
}
|
||||
if changed {
|
||||
log.Printf("crawl: watch %s: page changed, noted", watch.Name)
|
||||
written++
|
||||
}
|
||||
}
|
||||
return written
|
||||
}
|
||||
|
||||
// Check re-reads one watch now and reports whether it wrote a note.
|
||||
func (w *Watcher) Check(ctx context.Context, watch WatchConfig, now time.Time) (bool, error) {
|
||||
page, err := w.c.Page(ctx, watch.URL)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
// Title included: a page whose headline changed has changed.
|
||||
h := Hash(page.Title + "\n" + page.Text)
|
||||
if w.hashes != nil {
|
||||
prev, err := w.hashes.LastHash(ctx, watch.Name)
|
||||
if err != nil {
|
||||
log.Printf("crawl: watch %s: read hash: %v", watch.Name, err)
|
||||
}
|
||||
if prev == h {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
text := NoteText(watch, page)
|
||||
var vec []float32
|
||||
if w.embed != nil {
|
||||
v, err := w.embed.Embed(ctx, text)
|
||||
if err != nil {
|
||||
log.Printf("crawl: watch %s: embed: %v", watch.Name, err)
|
||||
} else {
|
||||
vec = v
|
||||
}
|
||||
}
|
||||
if _, err := w.notes.WriteNote(ctx, now, text, vec, SourceFor(watch.Name)); err != nil {
|
||||
return false, fmt.Errorf("write note: %w", err)
|
||||
}
|
||||
if w.hashes != nil {
|
||||
if err := w.hashes.SetHash(ctx, watch.Name, h); err != nil {
|
||||
log.Printf("crawl: watch %s: save hash: %v", watch.Name, err)
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// SourceFor is the note source for a watch, and SourcePrefix is what the answer
|
||||
// path matches to recognise one.
|
||||
func SourceFor(name string) string { return SourcePrefix + name }
|
||||
|
||||
// SourcePrefix — provenance for anything read off the network on a schedule.
|
||||
const SourcePrefix = "crawl:"
|
||||
|
||||
// noteRunes — how much of a watched page goes into a note. Shorter than what the
|
||||
// on-demand path reads: a note is a record of a change, not an archive.
|
||||
const noteRunes = 800
|
||||
|
||||
// NoteText renders a watched page as a note body.
|
||||
func NoteText(watch WatchConfig, page Page) string {
|
||||
var b strings.Builder
|
||||
if page.Title != "" {
|
||||
b.WriteString(page.Title)
|
||||
} else {
|
||||
b.WriteString(watch.Name)
|
||||
}
|
||||
b.WriteString("\n")
|
||||
b.WriteString(TrimRunes(page.Text, noteRunes))
|
||||
b.WriteString("\n")
|
||||
b.WriteString(watch.URL)
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package crawl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type note struct {
|
||||
text string
|
||||
source string
|
||||
}
|
||||
|
||||
type fakeNotes struct{ notes []note }
|
||||
|
||||
func (n *fakeNotes) WriteNote(_ context.Context, _ time.Time, text string, _ []float32, source string) (int64, error) {
|
||||
n.notes = append(n.notes, note{text, source})
|
||||
return int64(len(n.notes)), nil
|
||||
}
|
||||
|
||||
type fakeHashes struct{ m map[string]string }
|
||||
|
||||
func newHashes() *fakeHashes { return &fakeHashes{m: map[string]string{}} }
|
||||
func (f *fakeHashes) LastHash(_ context.Context, name string) (string, error) {
|
||||
return f.m[name], nil
|
||||
}
|
||||
func (f *fakeHashes) SetHash(_ context.Context, name, h string) error { f.m[name] = h; return nil }
|
||||
|
||||
var t0 = time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC)
|
||||
|
||||
func TestWatchNotesAChangedPage(t *testing.T) {
|
||||
f := &fakeFetcher{pages: map[string]Response{
|
||||
"https://example.org/docs": {Body: []byte(htmlPage)},
|
||||
}}
|
||||
notes := &fakeNotes{}
|
||||
hashes := newHashes()
|
||||
w := NewWatcher(newTestCrawler(f), []WatchConfig{{Name: "docs", URL: "https://example.org/docs"}},
|
||||
notes, hashes, nil, time.Hour)
|
||||
if w == nil {
|
||||
t.Fatal("NewWatcher returned nil for a configured watch")
|
||||
}
|
||||
if n := w.CheckDue(context.Background(), t0); n != 1 {
|
||||
t.Fatalf("first check wrote %d notes, want 1", n)
|
||||
}
|
||||
if notes.notes[0].source != "crawl:docs" {
|
||||
t.Errorf("source = %q, want crawl:docs", notes.notes[0].source)
|
||||
}
|
||||
if !strings.Contains(notes.notes[0].text, "https://example.org/docs") {
|
||||
t.Errorf("note does not carry the url: %q", notes.notes[0].text)
|
||||
}
|
||||
|
||||
// Unchanged page, interval elapsed: nothing written.
|
||||
if n := w.CheckDue(context.Background(), t0.Add(2*time.Hour)); n != 0 {
|
||||
t.Fatalf("an unchanged page wrote %d notes", n)
|
||||
}
|
||||
|
||||
// Changed page: one note.
|
||||
f.pages["https://example.org/docs"] = Response{Body: []byte(strings.Replace(htmlPage, "синее", "серое", 1))}
|
||||
if n := w.CheckDue(context.Background(), t0.Add(4*time.Hour)); n != 1 {
|
||||
t.Fatalf("a changed page wrote %d notes, want 1", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchIntervalIsRespected(t *testing.T) {
|
||||
f := &fakeFetcher{pages: map[string]Response{"https://example.org/d": {Body: []byte(htmlPage)}}}
|
||||
w := NewWatcher(newTestCrawler(f), []WatchConfig{{Name: "d", URL: "https://example.org/d", Interval: time.Hour}},
|
||||
&fakeNotes{}, newHashes(), nil, 0)
|
||||
w.CheckDue(context.Background(), t0)
|
||||
before := len(f.calls)
|
||||
w.CheckDue(context.Background(), t0.Add(time.Minute))
|
||||
if len(f.calls) != before {
|
||||
t.Fatal("the page was re-read inside its interval")
|
||||
}
|
||||
}
|
||||
|
||||
// The hash is durable so a restart does not re-note an unchanged page.
|
||||
func TestWatchHashSurvivesRestart(t *testing.T) {
|
||||
f := &fakeFetcher{pages: map[string]Response{"https://example.org/d": {Body: []byte(htmlPage)}}}
|
||||
hashes := newHashes()
|
||||
watches := []WatchConfig{{Name: "d", URL: "https://example.org/d"}}
|
||||
NewWatcher(newTestCrawler(f), watches, &fakeNotes{}, hashes, nil, time.Hour).CheckDue(context.Background(), t0)
|
||||
|
||||
notes2 := &fakeNotes{}
|
||||
NewWatcher(newTestCrawler(f), watches, notes2, hashes, nil, time.Hour).CheckDue(context.Background(), t0.Add(time.Hour))
|
||||
if len(notes2.notes) != 0 {
|
||||
t.Fatalf("a fresh watcher re-noted an unchanged page: %q", notes2.notes[0].text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchDeadPageDoesNotStopTheOthers(t *testing.T) {
|
||||
f := &fakeFetcher{pages: map[string]Response{"https://example.org/live": {Body: []byte(htmlPage)}}}
|
||||
notes := &fakeNotes{}
|
||||
w := NewWatcher(newTestCrawler(f), []WatchConfig{
|
||||
{Name: "dead", URL: "https://example.org/gone"},
|
||||
{Name: "live", URL: "https://example.org/live"},
|
||||
}, notes, newHashes(), nil, time.Hour)
|
||||
if n := w.CheckDue(context.Background(), t0); n != 1 {
|
||||
t.Fatalf("wrote %d notes, want 1 (the live page)", n)
|
||||
}
|
||||
if notes.notes[0].source != "crawl:live" {
|
||||
t.Fatalf("source = %q", notes.notes[0].source)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoWatchesMeansNoWatcher(t *testing.T) {
|
||||
c := newTestCrawler(&fakeFetcher{})
|
||||
if NewWatcher(c, nil, &fakeNotes{}, nil, nil, 0) != nil {
|
||||
t.Fatal("no watches must mean no watcher")
|
||||
}
|
||||
if NewWatcher(nil, []WatchConfig{{Name: "a", URL: "u"}}, &fakeNotes{}, nil, nil, 0) != nil {
|
||||
t.Fatal("no crawler must mean no watcher")
|
||||
}
|
||||
if NewWatcher(c, []WatchConfig{{Name: "", URL: ""}}, &fakeNotes{}, nil, nil, 0) != nil {
|
||||
t.Fatal("a watch with no name or url is not a configuration")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
package email
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/llm"
|
||||
"github.com/kami/maven/internal/persona"
|
||||
)
|
||||
|
||||
// Extraction — turning one mail into task CANDIDATES, and nothing else.
|
||||
//
|
||||
// The output of this file can only ever become rows in `tasks` with status
|
||||
// "candidate" (store.TaskCandidate), written through the one intake seam
|
||||
// (ipc.CaptureTaskReq, Vikunja #130). That bound is the whole design:
|
||||
//
|
||||
// - No reminder. A reminder FIRES; it speaks to him unprompted. A 1.7B that
|
||||
// misreads "встреча была в четверг" as a future appointment would then wake
|
||||
// him up about it. A candidate that is wrong is a line on a review page he
|
||||
// dismisses in one click, which is the correct cost of a model being wrong
|
||||
// about someone's mail.
|
||||
// - No fact. A fact is a claim Maven will later recite as true. Nothing read
|
||||
// out of a marketing mail deserves that standing.
|
||||
// - No calendar event, no note, no action. Extraction writes candidates or
|
||||
// writes nothing.
|
||||
//
|
||||
// The due date the model may return is stored on the candidate (tasks.due_ts),
|
||||
// which no scheduler reads — it is there so the review page can sort by it.
|
||||
//
|
||||
// Privacy: the mail text goes to the resident model on this box and nowhere
|
||||
// else. It is never search input (CLAUDE.md: "his notes and facts are never
|
||||
// search input" — mail is the same class), and Evidence keeps only the subject
|
||||
// line, so the review page shows him where a candidate came from without the
|
||||
// store growing a copy of his mailbox.
|
||||
|
||||
// MaxCandidates — at most this many candidates per message, enforced by the
|
||||
// grammar. A mail with four tasks in it is a mail he has to read himself; a
|
||||
// model allowed ten will produce ten.
|
||||
const MaxCandidates = 3
|
||||
|
||||
// SourcePrefix — provenance for everything this package captures. The mailbox
|
||||
// name is appended: "email:INBOX". Same vocabulary as tap:voice / poll:netdata.
|
||||
const SourcePrefix = "email:"
|
||||
|
||||
// Candidate — one piece of work the model thinks the mail is asking for.
|
||||
type Candidate struct {
|
||||
Text string `json:"text"`
|
||||
// Due — "YYYY-MM-DD" or empty. A date the model read out of the text, not a
|
||||
// date it computed: relative wording ("до пятницы") is left in Text, because
|
||||
// a small model resolving "пятница" against today's date gets it wrong often
|
||||
// enough that a stored wrong date is worse than no date.
|
||||
Due string `json:"due"`
|
||||
}
|
||||
|
||||
// Completer — the llama-server seam, same shape memeval and the router use, so
|
||||
// the one resident model serves this caller too.
|
||||
type Completer interface {
|
||||
Complete(ctx context.Context, r llm.Req) (string, error)
|
||||
}
|
||||
|
||||
// Extractor reads a message and returns candidates. It holds no store and no
|
||||
// writer on purpose: this type cannot persist anything, so "extraction never
|
||||
// acts" is a property of the code, not of a review.
|
||||
type Extractor struct {
|
||||
llm Completer
|
||||
// MaxCandidates — 0 ⇒ MaxCandidates.
|
||||
max int
|
||||
// ContextBlock — the shared persona block, optional. Extraction output is
|
||||
// not spoken, so the persona matters less here than in the phraser; it is
|
||||
// wired anyway so a candidate reads in her voice on the review page.
|
||||
contextBlock func() string
|
||||
}
|
||||
|
||||
func NewExtractor(c Completer, max int, contextBlock func() string) *Extractor {
|
||||
if max <= 0 || max > MaxCandidates {
|
||||
max = MaxCandidates
|
||||
}
|
||||
return &Extractor{llm: c, max: max, contextBlock: contextBlock}
|
||||
}
|
||||
|
||||
// extractGrammar — GBNF pinning the answer to a bounded array of fixed-shape
|
||||
// candidates. Same reasoning as memeval's evalGrammar and the router's
|
||||
// routeGrammar: the shape and the length bound are what keep a small model from
|
||||
// drifting into prose or spending the token budget repeating one field.
|
||||
//
|
||||
// The empty array is reachable, deliberately: most mail contains no task, and a
|
||||
// model with no way to say "nothing" invents something.
|
||||
const extractGrammar = `
|
||||
root ::= "[" ws (item ("," ws item){0,2})? ws "]"
|
||||
item ::= "{" ws "\"text\"" ws ":" ws text "," ws "\"due\"" ws ":" ws due ws "}"
|
||||
text ::= "\"" ([^"\\] | "\\" .){1,120} "\""
|
||||
due ::= "\"\"" | "\"" [0-9]{4} "-" [0-9]{2} "-" [0-9]{2} "\""
|
||||
ws ::= [ \t\n]*
|
||||
`
|
||||
|
||||
// extractSystem — the extraction prompt.
|
||||
//
|
||||
// Written around the two failure modes a small model has on this task: it
|
||||
// summarises when asked to extract (turning a mail into "письмо от Антона"),
|
||||
// and it invents an obligation from any polite closing sentence. Hence the
|
||||
// insistence on a verb phrase, and the explicit permission to return [].
|
||||
const extractSystem = `Ты читаешь одно письмо из его почты и достаёшь из него дела, которые письмо от него требует.
|
||||
|
||||
Правила:
|
||||
- Отвечай ТОЛЬКО массивом JSON. Каждый элемент: {"text": "...", "due": "ГГГГ-ММ-ДД" или ""}.
|
||||
- text — короткая формулировка дела по-русски, с глаголом: "оплатить счёт за интернет", "отправить акт". Не пересказывай письмо и не описывай его.
|
||||
- Дело — это то, что должен сделать ОН. Рассылка, реклама, уведомление, отчёт, письмо «просто к сведению» — дел не содержат.
|
||||
- Если письмо ничего от него не требует, верни пустой массив []. Это нормальный ответ, так бывает чаще всего.
|
||||
- Ничего не придумывай. Если срока в письме нет — "".
|
||||
- due заполняй только когда в письме стоит конкретная дата. Слова вроде «до пятницы» оставь в text, дату не вычисляй.
|
||||
- Максимум три дела. Лучше одно точное, чем три общих.`
|
||||
|
||||
// Extract returns the candidates in one message.
|
||||
//
|
||||
// Junk is refused without an LLM call — cheapest possible defence, and the
|
||||
// reason the header filter exists. An empty message (no subject, no body) is
|
||||
// likewise not worth a round trip.
|
||||
//
|
||||
// A parse failure is an error the caller logs and moves past. It is never
|
||||
// silently turned into zero candidates, because "the model went off the rails"
|
||||
// and "the mail contains no task" want different reactions from a human reading
|
||||
// the log.
|
||||
func (e *Extractor) Extract(ctx context.Context, msg Message) ([]Candidate, error) {
|
||||
if msg.Junk {
|
||||
return nil, nil
|
||||
}
|
||||
user := renderForModel(msg)
|
||||
if user == "" {
|
||||
return nil, nil
|
||||
}
|
||||
raw, err := e.llm.Complete(ctx, llm.Req{
|
||||
System: persona.Prepend(e.contextBlock, extractSystem),
|
||||
User: user,
|
||||
Grammar: extractGrammar,
|
||||
MaxTokens: 512,
|
||||
RepeatPenalty: 1.1,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("email: extract: %w", err)
|
||||
}
|
||||
items, err := parseCandidates(raw)
|
||||
if err != nil {
|
||||
// The raw reply is NOT in the error: it is a transformation of his mail,
|
||||
// and this error reaches the daemon log.
|
||||
return nil, fmt.Errorf("email: extract: unparsable reply (%d bytes)", len(raw))
|
||||
}
|
||||
out := make([]Candidate, 0, len(items))
|
||||
seen := map[string]bool{}
|
||||
for _, it := range items {
|
||||
it.Text = strings.TrimSpace(it.Text)
|
||||
if it.Text == "" {
|
||||
continue
|
||||
}
|
||||
key := strings.ToLower(strings.Join(strings.Fields(it.Text), " "))
|
||||
if seen[key] {
|
||||
continue // the model repeating itself is not two tasks
|
||||
}
|
||||
seen[key] = true
|
||||
if _, ok := ParseDue(it.Due); !ok {
|
||||
it.Due = "" // a date the grammar allowed but the calendar does not
|
||||
}
|
||||
out = append(out, it)
|
||||
if len(out) >= e.max {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// renderForModel is the user turn: subject, sender and body, labelled. Only
|
||||
// these three fields — no headers, no recipient list, no message-id, nothing
|
||||
// that would let the model start reasoning about routing metadata.
|
||||
func renderForModel(msg Message) string {
|
||||
var b strings.Builder
|
||||
if msg.From != "" {
|
||||
fmt.Fprintf(&b, "От: %s\n", msg.From)
|
||||
}
|
||||
if msg.Subject != "" {
|
||||
fmt.Fprintf(&b, "Тема: %s\n", msg.Subject)
|
||||
}
|
||||
if msg.Body != "" {
|
||||
fmt.Fprintf(&b, "\n%s\n", msg.Body)
|
||||
}
|
||||
if msg.Subject == "" && msg.Body == "" {
|
||||
return ""
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// parseCandidates decodes the grammar-constrained reply, tolerating the
|
||||
// wrappers a Thinking model sometimes leaves around it (a fenced block, or
|
||||
// leading reasoning before the array).
|
||||
func parseCandidates(raw string) ([]Candidate, error) {
|
||||
s := strings.TrimSpace(raw)
|
||||
if i := strings.Index(s, "["); i > 0 {
|
||||
s = s[i:]
|
||||
}
|
||||
if j := strings.LastIndex(s, "]"); j >= 0 {
|
||||
s = s[:j+1]
|
||||
}
|
||||
var out []Candidate
|
||||
if err := json.Unmarshal([]byte(s), &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ParseDue turns the model's "YYYY-MM-DD" into a time in UTC. Exported because
|
||||
// the daemon-side intake stores it on the candidate.
|
||||
//
|
||||
// The zero-value/empty case returns ok=false rather than an error: no date is
|
||||
// the common answer, not a failure.
|
||||
func ParseDue(s string) (time.Time, bool) {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return time.Time{}, false
|
||||
}
|
||||
t, err := time.Parse("2006-01-02", s)
|
||||
if err != nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return t, true
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
package email
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/kami/maven/internal/llm"
|
||||
)
|
||||
|
||||
// fakeLLM returns a canned reply and records the request, so a test can assert
|
||||
// on the grammar and on what of the mail was sent.
|
||||
type fakeLLM struct {
|
||||
reply string
|
||||
err error
|
||||
got llm.Req
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *fakeLLM) Complete(_ context.Context, r llm.Req) (string, error) {
|
||||
f.calls++
|
||||
f.got = r
|
||||
return f.reply, f.err
|
||||
}
|
||||
|
||||
func msgFor(subject, body string) Message {
|
||||
return Message{UID: 1, From: "anton@example.org", Subject: subject, Body: body}
|
||||
}
|
||||
|
||||
func TestExtractCandidates(t *testing.T) {
|
||||
f := &fakeLLM{reply: `[{"text":"отправить акт","due":""},{"text":"оплатить счёт","due":"2026-08-05"}]`}
|
||||
e := NewExtractor(f, 0, nil)
|
||||
got, err := e.Extract(context.Background(), msgFor("Акт и счёт", "Надо отправить акт и оплатить счёт до 5 августа."))
|
||||
if err != nil {
|
||||
t.Fatalf("extract: %v", err)
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("got %d candidates, want 2: %+v", len(got), got)
|
||||
}
|
||||
if got[0].Text != "отправить акт" || got[1].Due != "2026-08-05" {
|
||||
t.Errorf("candidates = %+v", got)
|
||||
}
|
||||
if f.got.Grammar == "" {
|
||||
t.Error("extraction must be grammar-constrained")
|
||||
}
|
||||
// The subject and body go to the model; nothing else about the message does.
|
||||
if !strings.Contains(f.got.User, "Акт и счёт") || !strings.Contains(f.got.User, "оплатить счёт") {
|
||||
t.Errorf("user turn = %q", f.got.User)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractEmptyArrayIsNotAnError(t *testing.T) {
|
||||
f := &fakeLLM{reply: "[]"}
|
||||
got, err := NewExtractor(f, 0, nil).Extract(context.Background(), msgFor("FYI", "Просто к сведению."))
|
||||
if err != nil || len(got) != 0 {
|
||||
t.Fatalf("got (%v, %v), want (empty, nil) — no task is the normal answer", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Junk must never reach the model: the header filter exists so the resident
|
||||
// model is not spent on newsletters.
|
||||
func TestExtractSkipsJunkWithoutCallingModel(t *testing.T) {
|
||||
f := &fakeLLM{reply: `[{"text":"купить всё со скидкой","due":""}]`}
|
||||
msg := msgFor("Скидки", "Sale!")
|
||||
msg.Junk = true
|
||||
got, err := NewExtractor(f, 0, nil).Extract(context.Background(), msg)
|
||||
if err != nil || got != nil {
|
||||
t.Fatalf("got (%v, %v), want (nil, nil)", got, err)
|
||||
}
|
||||
if f.calls != 0 {
|
||||
t.Errorf("model called %d times for junk, want 0", f.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractEmptyMessageIsNotSent(t *testing.T) {
|
||||
f := &fakeLLM{reply: "[]"}
|
||||
if _, err := NewExtractor(f, 0, nil).Extract(context.Background(), Message{UID: 3}); err != nil {
|
||||
t.Fatalf("extract: %v", err)
|
||||
}
|
||||
if f.calls != 0 {
|
||||
t.Errorf("model called %d times for an empty message, want 0", f.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractCaps(t *testing.T) {
|
||||
f := &fakeLLM{reply: `[{"text":"a","due":""},{"text":"b","due":""},{"text":"c","due":""}]`}
|
||||
got, err := NewExtractor(f, 2, nil).Extract(context.Background(), msgFor("s", "b"))
|
||||
if err != nil {
|
||||
t.Fatalf("extract: %v", err)
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Errorf("got %d, want the configured cap of 2", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractDropsRepeatsAndBadDates(t *testing.T) {
|
||||
f := &fakeLLM{reply: `[{"text":"Отправить акт","due":"2026-02-31"},{"text":"отправить акт","due":""},{"text":" ","due":""}]`}
|
||||
got, err := NewExtractor(f, 0, nil).Extract(context.Background(), msgFor("s", "b"))
|
||||
if err != nil {
|
||||
t.Fatalf("extract: %v", err)
|
||||
}
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("got %d candidates, want 1 (repeat and blank dropped): %+v", len(got), got)
|
||||
}
|
||||
if got[0].Due != "" {
|
||||
t.Errorf("due = %q, want empty — 2026-02-31 is not a date", got[0].Due)
|
||||
}
|
||||
}
|
||||
|
||||
// A Thinking model sometimes wraps the array; and when it emits something
|
||||
// unparsable the caller must hear about it rather than see "no tasks".
|
||||
func TestParseCandidatesTolerance(t *testing.T) {
|
||||
got, err := parseCandidates("думаю... [{\"text\":\"x\",\"due\":\"\"}] всё")
|
||||
if err != nil || len(got) != 1 || got[0].Text != "x" {
|
||||
t.Fatalf("got (%+v, %v)", got, err)
|
||||
}
|
||||
if _, err := parseCandidates("нет никакого JSON"); err == nil {
|
||||
t.Error("unparsable output must be an error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractParseErrorHidesMailText(t *testing.T) {
|
||||
f := &fakeLLM{reply: "он просил отправить акт, вот такой ответ"}
|
||||
_, err := NewExtractor(f, 0, nil).Extract(context.Background(), msgFor("Акт", "секретный текст"))
|
||||
if err == nil {
|
||||
t.Fatal("want an error")
|
||||
}
|
||||
if strings.Contains(err.Error(), "акт") || strings.Contains(err.Error(), "секретный") {
|
||||
t.Errorf("error text leaks mail content: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDue(t *testing.T) {
|
||||
if _, ok := ParseDue(""); ok {
|
||||
t.Error("empty due must be (zero, false)")
|
||||
}
|
||||
if got, ok := ParseDue("2026-08-05"); !ok || got.Year() != 2026 || got.Month() != 8 || got.Day() != 5 {
|
||||
t.Errorf("ParseDue = (%v, %v)", got, ok)
|
||||
}
|
||||
if _, ok := ParseDue("05.08.2026"); ok {
|
||||
t.Error("a non-ISO date must not parse")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package email
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// FetchSince is the whole read path in one call: connect, log in, examine the
|
||||
// mailbox read-only, list what arrived since a date, fetch and parse the ones
|
||||
// the caller has not seen, log out.
|
||||
//
|
||||
// It is a function rather than a long-lived object because a mail poller should
|
||||
// not hold an authenticated session (and therefore his credential in a live TLS
|
||||
// state) between polls. Connect, read, drop.
|
||||
//
|
||||
// skip decides which UIDs are already known — the poller's seen-set. max bounds
|
||||
// one poll: a mailbox that received 400 messages overnight must not turn into
|
||||
// 400 LLM calls, and the newest max are the ones a task could still be hiding
|
||||
// in. Junk messages are returned too, flagged, so the caller can mark them seen
|
||||
// without a second protocol round.
|
||||
type FetchSince struct {
|
||||
Addr string // host or host:993
|
||||
User string
|
||||
Mailbox string // e.g. "INBOX"
|
||||
Timeout time.Duration
|
||||
Since time.Time
|
||||
Max int
|
||||
Skip func(uid uint32) bool
|
||||
}
|
||||
|
||||
// Run performs one read. password is passed here, not stored in the struct, so
|
||||
// the configuration of a mailbox and the secret for it are never the same value
|
||||
// sitting in the same place.
|
||||
func (f FetchSince) Run(password string) ([]Message, error) {
|
||||
return f.RunWith(password, nil)
|
||||
}
|
||||
|
||||
// RunWith is Run with an explicit connection function, which is how the reader
|
||||
// daemon and the tests substitute an in-process server. nil ⇒ Dial, i.e.
|
||||
// implicit TLS with certificate verification; there is no configuration path
|
||||
// that reaches this, so no deployment can end up talking cleartext IMAP.
|
||||
func (f FetchSince) RunWith(password string, dial func(addr string, timeout time.Duration) (*Conn, error)) ([]Message, error) {
|
||||
if f.Addr == "" || f.User == "" || f.Mailbox == "" {
|
||||
return nil, fmt.Errorf("email: mailbox not configured (addr/user/mailbox)")
|
||||
}
|
||||
if dial == nil {
|
||||
dial = Dial
|
||||
}
|
||||
c, err := dial(f.Addr, f.Timeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer c.Close()
|
||||
if err := c.Login(f.User, password); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer c.Logout()
|
||||
if err := c.Select(f.Mailbox); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
uids, err := c.SearchSince(f.Since)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Newest UIDs first — IMAP hands them back ascending, and when Max clips the
|
||||
// list the recent mail is what matters.
|
||||
wanted := make([]uint32, 0, len(uids))
|
||||
for i := len(uids) - 1; i >= 0; i-- {
|
||||
if f.Skip != nil && f.Skip(uids[i]) {
|
||||
continue
|
||||
}
|
||||
wanted = append(wanted, uids[i])
|
||||
if f.Max > 0 && len(wanted) >= f.Max {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
out := make([]Message, 0, len(wanted))
|
||||
for _, uid := range wanted {
|
||||
raw, err := c.Fetch(uid)
|
||||
if err != nil {
|
||||
// One unreadable message does not abandon the poll; the rest of the
|
||||
// mailbox is still worth reading. The error names the UID, not the
|
||||
// message.
|
||||
return out, fmt.Errorf("email: fetch uid %d: %w", uid, err)
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
continue // vanished between SEARCH and FETCH
|
||||
}
|
||||
msg, err := ParseMessage(uid, raw)
|
||||
if err != nil {
|
||||
continue // unparsable headers — nothing to review, skip silently
|
||||
}
|
||||
out = append(out, msg)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package email
|
||||
|
||||
import (
|
||||
"net"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestFetchSinceRun(t *testing.T) {
|
||||
mk := func(subject string) string {
|
||||
return "Subject: " + subject + "\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nbody\r\n"
|
||||
}
|
||||
f := &fakeIMAP{
|
||||
uids: []uint32{1, 2, 3},
|
||||
msgs: map[uint32]string{1: mk("one"), 2: mk("two"), 3: mk("three")},
|
||||
}
|
||||
fs := FetchSince{
|
||||
Addr: "mail.example:993", User: "kami", Mailbox: "INBOX",
|
||||
Timeout: 5 * time.Second,
|
||||
Since: time.Date(2026, 7, 30, 0, 0, 0, 0, time.UTC),
|
||||
Max: 2,
|
||||
Skip: func(uid uint32) bool { return uid == 3 },
|
||||
}
|
||||
msgs, err := fs.RunWith("secret", func(addr string, timeout time.Duration) (*Conn, error) {
|
||||
cli, srv := net.Pipe()
|
||||
go f.serve(t, srv)
|
||||
return NewConn(cli, timeout)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("run: %v", err)
|
||||
}
|
||||
// Newest first, the already-seen UID skipped, Max respected.
|
||||
if len(msgs) != 2 {
|
||||
t.Fatalf("got %d messages, want 2: %+v", len(msgs), msgs)
|
||||
}
|
||||
if msgs[0].Subject != "two" || msgs[1].Subject != "one" {
|
||||
t.Errorf("subjects = %q,%q, want two,one (newest first)", msgs[0].Subject, msgs[1].Subject)
|
||||
}
|
||||
if strings.Contains(strings.Join(f.cmds, " "), "UID FETCH 3") {
|
||||
t.Error("a skipped UID must not be fetched again")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchSinceRequiresConfig(t *testing.T) {
|
||||
if _, err := (FetchSince{}).Run("secret"); err == nil {
|
||||
t.Fatal("an unconfigured mailbox must not be read")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
package email
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A minimal IMAP4rev1 client — LOGIN, SELECT, UID SEARCH, UID FETCH with
|
||||
// BODY.PEEK, LOGOUT, and nothing else.
|
||||
//
|
||||
// Why hand-rolled instead of go-imap: the whole surface Maven needs is five
|
||||
// commands, and this is the one code path that holds his mailbox credential and
|
||||
// reads his private mail. A ~200-line client with no dependencies is auditable
|
||||
// in one sitting; a general-purpose IMAP library is a much larger amount of
|
||||
// code doing much more than we asked, in the most sensitive place in the tree.
|
||||
// If IDLE, CONDSTORE or server-side threading ever become worth having, that
|
||||
// trade should be re-made deliberately.
|
||||
//
|
||||
// BODY.PEEK[] rather than BODY[] is load-bearing: Maven reads his mail and must
|
||||
// leave no trace of having done so. Reading a message here does not mark it
|
||||
// \Seen, so the unread state in his own mail client stays his.
|
||||
|
||||
// DefaultIMAPPort — implicit-TLS IMAP. There is no cleartext and no STARTTLS
|
||||
// path in this client: an option to send his password over a plain socket is an
|
||||
// option to get it wrong once.
|
||||
const DefaultIMAPPort = "993"
|
||||
|
||||
// Conn — one authenticated IMAP connection. Not safe for concurrent use; the
|
||||
// poller drives one connection at a time.
|
||||
type Conn struct {
|
||||
rwc io.ReadWriteCloser
|
||||
r *bufio.Reader
|
||||
tag int
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
// Dial opens an implicit-TLS connection and reads the server greeting.
|
||||
func Dial(addr string, timeout time.Duration) (*Conn, error) {
|
||||
host, _, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
host, addr = addr, net.JoinHostPort(addr, DefaultIMAPPort)
|
||||
}
|
||||
d := &net.Dialer{Timeout: timeout}
|
||||
// ServerName is set from the host we asked for: certificate verification is
|
||||
// the only thing standing between his password and a MITM on the way out.
|
||||
c, err := tls.DialWithDialer(d, "tcp", addr, &tls.Config{ServerName: host, MinVersion: tls.VersionTLS12})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("email: dial %s: %w", addr, err)
|
||||
}
|
||||
return NewConn(c, timeout)
|
||||
}
|
||||
|
||||
// NewConn wraps an already-open stream (the tests speak IMAP over a pipe) and
|
||||
// consumes the greeting.
|
||||
func NewConn(rwc io.ReadWriteCloser, timeout time.Duration) (*Conn, error) {
|
||||
c := &Conn{rwc: rwc, r: bufio.NewReaderSize(rwc, 64<<10), timeout: timeout}
|
||||
line, err := c.readLine()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("email: greeting: %w", err)
|
||||
}
|
||||
if !strings.HasPrefix(line, "* OK") && !strings.HasPrefix(line, "* PREAUTH") {
|
||||
c.rwc.Close()
|
||||
return nil, fmt.Errorf("email: server refused connection: %s", line)
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (c *Conn) Close() error { return c.rwc.Close() }
|
||||
|
||||
// Login authenticates with LOGIN. The password is passed as an argument and
|
||||
// never stored on the Conn: nothing in this package keeps a credential alive
|
||||
// past the command that uses it, so no struct dump or panic trace can carry it.
|
||||
func (c *Conn) Login(user, pass string) error {
|
||||
// The command line itself is never logged (see exec) — a LOGIN line IS the
|
||||
// credential.
|
||||
if _, err := c.exec(fmt.Sprintf("LOGIN %s %s", quote(user), quote(pass))); err != nil {
|
||||
return fmt.Errorf("email: login: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Select opens a mailbox read-only. EXAMINE, not SELECT: read-only at the
|
||||
// protocol level means no command in this session can change a flag, expunge a
|
||||
// message, or move anything, even by mistake.
|
||||
func (c *Conn) Select(mailbox string) error {
|
||||
if _, err := c.exec(fmt.Sprintf("EXAMINE %s", quote(mailbox))); err != nil {
|
||||
return fmt.Errorf("email: examine %s: %w", mailbox, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SearchSince returns the UIDs of messages received on or after since. An
|
||||
// unlimited search is not offered: the first poll against a years-old mailbox
|
||||
// would otherwise fetch everything and hand a decade of mail to the model.
|
||||
//
|
||||
// The IMAP SINCE key has date granularity (and compares the server's internal
|
||||
// date), so the result can include messages slightly older than since. The
|
||||
// caller dedupes by UID anyway, so a wider window costs one extra fetch.
|
||||
func (c *Conn) SearchSince(since time.Time) ([]uint32, error) {
|
||||
cmd := fmt.Sprintf("UID SEARCH SINCE %s", since.Format("2-Jan-2006"))
|
||||
lines, err := c.exec(cmd)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("email: search: %w", err)
|
||||
}
|
||||
var uids []uint32
|
||||
for _, l := range lines {
|
||||
rest, ok := untagged(l, "SEARCH")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, f := range strings.Fields(rest) {
|
||||
n, err := strconv.ParseUint(f, 10, 32)
|
||||
if err == nil {
|
||||
uids = append(uids, uint32(n))
|
||||
}
|
||||
}
|
||||
}
|
||||
return uids, nil
|
||||
}
|
||||
|
||||
var literalSize = regexp.MustCompile(`\{(\d+)\}$`)
|
||||
|
||||
// Fetch returns the raw RFC 5322 bytes of one message, by UID.
|
||||
//
|
||||
// Returns (nil, nil) when the UID no longer exists — a message he deleted
|
||||
// between SEARCH and FETCH is normal, not an error.
|
||||
func (c *Conn) Fetch(uid uint32) ([]byte, error) {
|
||||
tag := c.nextTag()
|
||||
if err := c.send(fmt.Sprintf("%s UID FETCH %d (BODY.PEEK[])", tag, uid)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var raw []byte
|
||||
for {
|
||||
line, err := c.readLine()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("email: fetch %d: %w", uid, err)
|
||||
}
|
||||
if done, err := c.tagged(tag, line); done {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("email: fetch %d: %w", uid, err)
|
||||
}
|
||||
return raw, nil
|
||||
}
|
||||
m := literalSize.FindStringSubmatch(strings.TrimSpace(line))
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
n, err := strconv.Atoi(m[1])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
buf := make([]byte, n)
|
||||
if _, err := io.ReadFull(c.r, buf); err != nil {
|
||||
return nil, fmt.Errorf("email: fetch %d: literal: %w", uid, err)
|
||||
}
|
||||
if raw == nil {
|
||||
raw = buf
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Logout ends the session politely. A failure is not worth reporting — the
|
||||
// connection is being closed either way.
|
||||
func (c *Conn) Logout() {
|
||||
_, _ = c.exec("LOGOUT")
|
||||
}
|
||||
|
||||
// ---- protocol plumbing -----------------------------------------------------
|
||||
|
||||
func (c *Conn) nextTag() string {
|
||||
c.tag++
|
||||
return fmt.Sprintf("a%03d", c.tag)
|
||||
}
|
||||
|
||||
// exec sends one command and returns the untagged response lines.
|
||||
//
|
||||
// Neither the command nor the response is ever logged here. LOGIN goes through
|
||||
// this function, and a debug line "sent: a001 LOGIN ..." is how a credential
|
||||
// ends up in a log file forever.
|
||||
func (c *Conn) exec(cmd string) ([]string, error) {
|
||||
tag := c.nextTag()
|
||||
if err := c.send(tag + " " + cmd); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var lines []string
|
||||
for {
|
||||
line, err := c.readLine()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if done, err := c.tagged(tag, line); done {
|
||||
return lines, err
|
||||
}
|
||||
lines = append(lines, line)
|
||||
// A response line may carry a literal (e.g. a header FETCH). Nothing we
|
||||
// send asks for one outside Fetch, but skip it if it appears so the
|
||||
// stream stays aligned.
|
||||
if m := literalSize.FindStringSubmatch(strings.TrimSpace(line)); m != nil {
|
||||
if n, err := strconv.Atoi(m[1]); err == nil {
|
||||
if _, err := io.CopyN(io.Discard, c.r, int64(n)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// tagged reports whether line completes the command with this tag, and turns a
|
||||
// NO/BAD completion into an error. The error text is the server's, which never
|
||||
// echoes a password.
|
||||
func (c *Conn) tagged(tag, line string) (bool, error) {
|
||||
if !strings.HasPrefix(line, tag+" ") {
|
||||
return false, nil
|
||||
}
|
||||
rest := strings.TrimSpace(line[len(tag):])
|
||||
switch {
|
||||
case strings.HasPrefix(rest, "OK"):
|
||||
return true, nil
|
||||
case strings.HasPrefix(rest, "NO"), strings.HasPrefix(rest, "BAD"):
|
||||
return true, fmt.Errorf("server said: %s", rest)
|
||||
default:
|
||||
return true, fmt.Errorf("unexpected completion: %s", rest)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Conn) send(line string) error {
|
||||
c.setDeadline()
|
||||
if _, err := io.WriteString(c.rwc, line+"\r\n"); err != nil {
|
||||
return fmt.Errorf("email: write: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Conn) readLine() (string, error) {
|
||||
c.setDeadline()
|
||||
line, err := c.r.ReadString('\n')
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return strings.TrimRight(line, "\r\n"), nil
|
||||
}
|
||||
|
||||
// setDeadline applies the per-connection timeout when the transport supports
|
||||
// one. A hung IMAP server must not park the poller forever.
|
||||
func (c *Conn) setDeadline() {
|
||||
if c.timeout <= 0 {
|
||||
return
|
||||
}
|
||||
if d, ok := c.rwc.(interface{ SetDeadline(time.Time) error }); ok {
|
||||
_ = d.SetDeadline(time.Now().Add(c.timeout))
|
||||
}
|
||||
}
|
||||
|
||||
// untagged splits "* SEARCH 1 2 3" into its payload when the key matches.
|
||||
func untagged(line, key string) (string, bool) {
|
||||
if !strings.HasPrefix(line, "* ") {
|
||||
return "", false
|
||||
}
|
||||
rest := strings.TrimSpace(line[2:])
|
||||
if !strings.HasPrefix(rest, key) {
|
||||
return "", false
|
||||
}
|
||||
return strings.TrimSpace(rest[len(key):]), true
|
||||
}
|
||||
|
||||
// quote renders an IMAP quoted string. Passwords routinely contain characters
|
||||
// that would otherwise end the argument early, and CR/LF are stripped rather
|
||||
// than escaped because there is no legal way to send them — a credential file
|
||||
// with a stray newline must not become a second command.
|
||||
func quote(s string) string {
|
||||
s = strings.NewReplacer("\r", "", "\n", "").Replace(s)
|
||||
return `"` + strings.NewReplacer(`\`, `\\`, `"`, `\"`).Replace(s) + `"`
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
package email
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fakeIMAP is a scripted server: enough of IMAP to exercise the client, and
|
||||
// nothing more. It records the commands it received so a test can assert on the
|
||||
// protocol (BODY.PEEK rather than BODY, EXAMINE rather than SELECT).
|
||||
type fakeIMAP struct {
|
||||
msgs map[uint32]string
|
||||
uids []uint32
|
||||
cmds []string
|
||||
failOn string // substring of a command to answer NO
|
||||
}
|
||||
|
||||
func (f *fakeIMAP) serve(t *testing.T, c net.Conn) {
|
||||
t.Helper()
|
||||
defer c.Close()
|
||||
fmt.Fprint(c, "* OK fake IMAP ready\r\n")
|
||||
r := bufio.NewReader(c)
|
||||
for {
|
||||
line, err := r.ReadString('\n')
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
line = strings.TrimRight(line, "\r\n")
|
||||
parts := strings.SplitN(line, " ", 2)
|
||||
if len(parts) != 2 {
|
||||
return
|
||||
}
|
||||
tag, cmd := parts[0], parts[1]
|
||||
f.cmds = append(f.cmds, cmd)
|
||||
if f.failOn != "" && strings.Contains(cmd, f.failOn) {
|
||||
fmt.Fprintf(c, "%s NO computer says no\r\n", tag)
|
||||
continue
|
||||
}
|
||||
upper := strings.ToUpper(cmd)
|
||||
switch {
|
||||
case strings.HasPrefix(upper, "LOGIN"), strings.HasPrefix(upper, "EXAMINE"):
|
||||
fmt.Fprintf(c, "%s OK done\r\n", tag)
|
||||
case strings.HasPrefix(upper, "UID SEARCH"):
|
||||
var ids []string
|
||||
for _, u := range f.uids {
|
||||
ids = append(ids, strconv.FormatUint(uint64(u), 10))
|
||||
}
|
||||
fmt.Fprintf(c, "* SEARCH %s\r\n", strings.Join(ids, " "))
|
||||
fmt.Fprintf(c, "%s OK search done\r\n", tag)
|
||||
case strings.HasPrefix(upper, "UID FETCH"):
|
||||
uid64, _ := strconv.ParseUint(strings.Fields(cmd)[2], 10, 32)
|
||||
raw, ok := f.msgs[uint32(uid64)]
|
||||
if ok {
|
||||
fmt.Fprintf(c, "* 1 FETCH (UID %d BODY[] {%d}\r\n", uid64, len(raw))
|
||||
fmt.Fprint(c, raw)
|
||||
fmt.Fprint(c, ")\r\n")
|
||||
}
|
||||
fmt.Fprintf(c, "%s OK fetch done\r\n", tag)
|
||||
case strings.HasPrefix(upper, "LOGOUT"):
|
||||
fmt.Fprint(c, "* BYE\r\n")
|
||||
fmt.Fprintf(c, "%s OK bye\r\n", tag)
|
||||
return
|
||||
default:
|
||||
fmt.Fprintf(c, "%s BAD unknown\r\n", tag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// dialFake wires a client Conn to an in-process server over net.Pipe.
|
||||
func dialFake(t *testing.T, f *fakeIMAP) *Conn {
|
||||
t.Helper()
|
||||
cli, srv := net.Pipe()
|
||||
go f.serve(t, srv)
|
||||
c, err := NewConn(cli, 5*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("greeting: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { c.Close() })
|
||||
return c
|
||||
}
|
||||
|
||||
func TestIMAPRoundTrip(t *testing.T) {
|
||||
body := "Subject: hello\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nCall the bank.\r\n"
|
||||
f := &fakeIMAP{uids: []uint32{4, 9}, msgs: map[uint32]string{4: body, 9: body}}
|
||||
c := dialFake(t, f)
|
||||
|
||||
if err := c.Login("kami", `pa"ss\word`); err != nil {
|
||||
t.Fatalf("login: %v", err)
|
||||
}
|
||||
if err := c.Select("INBOX"); err != nil {
|
||||
t.Fatalf("select: %v", err)
|
||||
}
|
||||
uids, err := c.SearchSince(time.Date(2026, 8, 1, 0, 0, 0, 0, time.UTC))
|
||||
if err != nil {
|
||||
t.Fatalf("search: %v", err)
|
||||
}
|
||||
if len(uids) != 2 || uids[0] != 4 || uids[1] != 9 {
|
||||
t.Fatalf("uids = %v, want [4 9]", uids)
|
||||
}
|
||||
raw, err := c.Fetch(9)
|
||||
if err != nil {
|
||||
t.Fatalf("fetch: %v", err)
|
||||
}
|
||||
if string(raw) != body {
|
||||
t.Errorf("fetched %q, want the literal verbatim", raw)
|
||||
}
|
||||
c.Logout()
|
||||
|
||||
joined := strings.Join(f.cmds, "\n")
|
||||
// Read-only at the protocol level, and peeking — Maven must leave no trace
|
||||
// of having read his mail.
|
||||
if !strings.Contains(joined, "EXAMINE") || strings.Contains(joined, "SELECT ") {
|
||||
t.Errorf("want EXAMINE (read-only), got:\n%s", joined)
|
||||
}
|
||||
if !strings.Contains(joined, "BODY.PEEK[]") {
|
||||
t.Errorf("want BODY.PEEK, got:\n%s", joined)
|
||||
}
|
||||
// The password must have been quoted and escaped, not truncated at the quote.
|
||||
if !strings.Contains(joined, `"pa\"ss\\word"`) {
|
||||
t.Errorf("password not quoted correctly:\n%s", joined)
|
||||
}
|
||||
// SINCE must carry the IMAP date form.
|
||||
if !strings.Contains(joined, "SINCE 1-Aug-2026") {
|
||||
t.Errorf("want a SINCE date, got:\n%s", joined)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIMAPServerNoIsAnError(t *testing.T) {
|
||||
f := &fakeIMAP{failOn: "LOGIN"}
|
||||
c := dialFake(t, f)
|
||||
err := c.Login("kami", "wrong")
|
||||
if err == nil {
|
||||
t.Fatal("a NO completion must be an error")
|
||||
}
|
||||
// The error is the server's text; it must not echo the credential.
|
||||
if strings.Contains(err.Error(), "wrong") {
|
||||
t.Errorf("error leaks the password: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIMAPFetchMissingUID(t *testing.T) {
|
||||
f := &fakeIMAP{uids: []uint32{1}, msgs: map[uint32]string{}}
|
||||
c := dialFake(t, f)
|
||||
raw, err := c.Fetch(1)
|
||||
if err != nil {
|
||||
t.Fatalf("fetch: %v", err)
|
||||
}
|
||||
if raw != nil {
|
||||
t.Errorf("a vanished UID should give nil, got %q", raw)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuoteStripsNewlines(t *testing.T) {
|
||||
if got := quote("pass\r\nA1 LOGOUT"); strings.ContainsAny(got, "\r\n") {
|
||||
t.Errorf("quote kept a line break: %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package email
|
||||
|
||||
import (
|
||||
"net/mail"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The junk filter — the cheapest and most important half of reading mail.
|
||||
//
|
||||
// A mailbox is mostly machine-generated: newsletters, receipts nobody acts on,
|
||||
// social notifications, marketing. Sending all of it to a 1.7B and asking "is
|
||||
// there a task here" produces confident nonsense at a rate proportional to the
|
||||
// volume, so junk is decided by HEADERS, before any model sees the message.
|
||||
//
|
||||
// The rules are all bulk-mail markers that senders set on themselves, never
|
||||
// guesses about content:
|
||||
//
|
||||
// - List-Unsubscribe / List-Id — by definition a mailing list. If he can
|
||||
// unsubscribe from it, it is not asking him to do anything.
|
||||
// - Precedence: bulk|junk|list — the sender declaring itself bulk.
|
||||
// - Auto-Submitted other than "no" (RFC 3834) — generated by a machine.
|
||||
// - X-Spam-Flag: YES, X-Spam-Status: Yes — the spam filter upstream already
|
||||
// decided; we do not second-guess it in the other direction.
|
||||
// - X-GM-LABELS / X-Gmail-Labels containing a Gmail category — Gmail's own
|
||||
// Promotions/Social/Forums/Spam classification, when the server sends it.
|
||||
//
|
||||
// Deliberately NOT here: sender allow/deny lists and subject keyword matching.
|
||||
// Both are configuration that ages badly and both would be a place for his
|
||||
// contacts to end up in a config file. If a real correspondent's mail is being
|
||||
// dropped, the fix is a rule about a header, not a list of names.
|
||||
//
|
||||
// A junk verdict never deletes anything and never touches a flag on the server.
|
||||
// It means "do not spend the model on this", nothing more.
|
||||
|
||||
// junkHeaders — headers whose mere presence marks bulk mail.
|
||||
var junkPresence = []string{"List-Unsubscribe", "List-Id", "List-Post"}
|
||||
|
||||
// gmailCategories — Gmail's category labels, lowercased as they appear in
|
||||
// X-GM-LABELS. "important" and "inbox" are labels too, and are NOT categories.
|
||||
// Matching is by these exact tokens (substring is fine — they are namespaced
|
||||
// and cannot appear in a hand-made label by accident), so a user label named
|
||||
// "Social Club" is not mistaken for Gmail's Social category.
|
||||
var gmailCategories = []string{
|
||||
"category_promotions", "category_social", "category_forums", "category_updates",
|
||||
`\spam`, `\junk`,
|
||||
}
|
||||
|
||||
// classifyJunk returns whether the message is bulk/automated and why. The
|
||||
// reason is a short header name, safe to log — it names the marker, never the
|
||||
// sender or the subject.
|
||||
func classifyJunk(h mail.Header) (bool, string) {
|
||||
for _, name := range junkPresence {
|
||||
if strings.TrimSpace(h.Get(name)) != "" {
|
||||
return true, strings.ToLower(name)
|
||||
}
|
||||
}
|
||||
switch strings.ToLower(strings.TrimSpace(h.Get("Precedence"))) {
|
||||
case "bulk", "junk", "list":
|
||||
return true, "precedence"
|
||||
}
|
||||
if v := strings.ToLower(strings.TrimSpace(h.Get("Auto-Submitted"))); v != "" && v != "no" {
|
||||
return true, "auto-submitted"
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(h.Get("X-Spam-Flag")), "yes") {
|
||||
return true, "x-spam-flag"
|
||||
}
|
||||
if v := strings.ToLower(strings.TrimSpace(h.Get("X-Spam-Status"))); strings.HasPrefix(v, "yes") {
|
||||
return true, "x-spam-status"
|
||||
}
|
||||
labels := strings.ToLower(h.Get("X-GM-LABELS") + " " + h.Get("X-Gmail-Labels"))
|
||||
for _, c := range gmailCategories {
|
||||
if c == "" {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(labels, c) {
|
||||
return true, "gmail-category"
|
||||
}
|
||||
}
|
||||
return false, ""
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package email
|
||||
|
||||
import (
|
||||
"net/mail"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func headers(t *testing.T, raw string) mail.Header {
|
||||
t.Helper()
|
||||
m, err := mail.ReadMessage(strings.NewReader(strings.ReplaceAll(raw, "\n", "\r\n") + "\r\n\r\nbody\r\n"))
|
||||
if err != nil {
|
||||
t.Fatalf("read headers: %v", err)
|
||||
}
|
||||
return m.Header
|
||||
}
|
||||
|
||||
func TestClassifyJunk(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
raw string
|
||||
junk bool
|
||||
reason string
|
||||
}{
|
||||
{"personal", "From: a@b.c\nSubject: привет", false, ""},
|
||||
{"list-unsubscribe", "From: a@b.c\nList-Unsubscribe: <mailto:u@b.c>", true, "list-unsubscribe"},
|
||||
{"list-id", "From: a@b.c\nList-Id: <golang-nuts.example>", true, "list-id"},
|
||||
{"precedence bulk", "From: a@b.c\nPrecedence: bulk", true, "precedence"},
|
||||
{"auto-submitted", "From: a@b.c\nAuto-Submitted: auto-generated", true, "auto-submitted"},
|
||||
{"auto-submitted no", "From: a@b.c\nAuto-Submitted: no", false, ""},
|
||||
{"spam flag", "From: a@b.c\nX-Spam-Flag: YES", true, "x-spam-flag"},
|
||||
{"spam status", "From: a@b.c\nX-Spam-Status: Yes, score=9.1", true, "x-spam-status"},
|
||||
{"spam status no", "From: a@b.c\nX-Spam-Status: No, score=0.1", false, ""},
|
||||
{"gmail promo", "From: a@b.c\nX-Gmail-Labels: Inbox,CATEGORY_PROMOTIONS", true, "gmail-category"},
|
||||
{"user label", "From: a@b.c\nX-Gmail-Labels: Social Club,Important", false, ""},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
junk, reason := classifyJunk(headers(t, c.raw))
|
||||
if junk != c.junk || reason != c.reason {
|
||||
t.Errorf("classifyJunk = (%v, %q), want (%v, %q)", junk, reason, c.junk, c.reason)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewsletterFixtureIsJunk(t *testing.T) {
|
||||
msg, err := ParseMessage(9, fixture(t, "newsletter.eml"))
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if !msg.Junk {
|
||||
t.Fatal("a newsletter with List-Unsubscribe + Precedence: bulk must be junk")
|
||||
}
|
||||
// The reason is what gets logged, so it must never carry mail content.
|
||||
if strings.Contains(msg.JunkReason, "@") || strings.Contains(msg.JunkReason, "Скидки") {
|
||||
t.Errorf("junk reason leaks content: %q", msg.JunkReason)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
// Package email is the reading half of the email reader (Vikunja #246,
|
||||
// docs/plans/01-email-reader.md): a small IMAP client, a MIME-to-plaintext
|
||||
// converter, and the junk filter that decides a message is not worth reading at
|
||||
// all. Extraction lives in extract.go and writes nothing itself.
|
||||
//
|
||||
// Two constraints shape everything here, both from CLAUDE.md:
|
||||
//
|
||||
// - Mail is personal. Nothing in this package logs a body, a subject, or an
|
||||
// address; callers get the text and decide. Mail text is never search input
|
||||
// — no function here reaches the network except the IMAP connection itself.
|
||||
// - Off unless configured. There is no default host, no default account, and
|
||||
// no fallback that would make a mailbox get read because a field was empty.
|
||||
//
|
||||
// The IMAP subset is deliberately tiny (LOGIN, SELECT, UID SEARCH, UID FETCH
|
||||
// with BODY.PEEK, LOGOUT). No IDLE: a poll every few minutes is what a task
|
||||
// candidate needs, and IDLE would mean holding a connection and a credential
|
||||
// open forever for latency nobody is waiting on.
|
||||
package email
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"mime/multipart"
|
||||
"mime/quotedprintable"
|
||||
"net/mail"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// MaxBodyBytes — how much of one message body is kept. A task hides in the
|
||||
// first screenful; the rest is signature, quoted history and legal boilerplate,
|
||||
// and it would only spend the resident model's 4096-token context.
|
||||
const MaxBodyBytes = 4000
|
||||
|
||||
// Message — one mail, reduced to the fields extraction and review need.
|
||||
//
|
||||
// Raw is deliberately absent: once a message is parsed the original bytes are
|
||||
// dropped, so no caller can accidentally log or forward the whole mail.
|
||||
type Message struct {
|
||||
UID uint32
|
||||
From string
|
||||
Subject string
|
||||
Date string // as sent, unparsed — display only
|
||||
Body string // plaintext, decoded, HTML-stripped, truncated
|
||||
// Junk is set by the junk filter (see junk.go). A junk message is carried
|
||||
// rather than dropped so the poller can count it and still mark it seen.
|
||||
Junk bool
|
||||
JunkReason string
|
||||
}
|
||||
|
||||
// ParseMessage turns one RFC 5322 message into a Message.
|
||||
//
|
||||
// It never fails on a body it cannot understand: an unparsable or
|
||||
// unsupported-charset body yields an empty Body and the headers still come
|
||||
// through, because a subject line alone is often the whole task ("Счёт за
|
||||
// интернет"). Only a message whose headers cannot be read at all is an error.
|
||||
func ParseMessage(uid uint32, raw []byte) (Message, error) {
|
||||
m, err := mail.ReadMessage(strings.NewReader(string(raw)))
|
||||
if err != nil {
|
||||
return Message{}, fmt.Errorf("email: parse message: %w", err)
|
||||
}
|
||||
msg := Message{
|
||||
UID: uid,
|
||||
From: decodeHeader(m.Header.Get("From")),
|
||||
Subject: decodeHeader(m.Header.Get("Subject")),
|
||||
Date: m.Header.Get("Date"),
|
||||
}
|
||||
msg.Junk, msg.JunkReason = classifyJunk(m.Header)
|
||||
body, err := plaintextBody(m.Header.Get("Content-Type"), m.Header.Get("Content-Transfer-Encoding"), m.Body)
|
||||
if err == nil {
|
||||
msg.Body = truncate(collapse(body), MaxBodyBytes)
|
||||
}
|
||||
return msg, nil
|
||||
}
|
||||
|
||||
// plaintextBody walks the MIME tree and returns the best plaintext it can.
|
||||
//
|
||||
// Preference order inside a multipart: text/plain first, text/html stripped
|
||||
// only when there is no plain part. multipart/mixed attachments are skipped
|
||||
// wholesale — an attachment is a file, not a sentence, and reading one would
|
||||
// mean parsing arbitrary formats from the network.
|
||||
func plaintextBody(contentType, encoding string, body io.Reader) (string, error) {
|
||||
mediaType, params, err := mime.ParseMediaType(contentType)
|
||||
if contentType == "" || err != nil {
|
||||
// No Content-Type at all is legal and means text/plain; a broken one is
|
||||
// treated the same rather than dropping the message.
|
||||
mediaType, params = "text/plain", nil
|
||||
}
|
||||
switch {
|
||||
case strings.HasPrefix(mediaType, "multipart/"):
|
||||
boundary := params["boundary"]
|
||||
if boundary == "" {
|
||||
return "", fmt.Errorf("email: multipart without boundary")
|
||||
}
|
||||
return multipartText(multipart.NewReader(body, boundary))
|
||||
case mediaType == "text/html":
|
||||
raw, err := decodeBody(body, encoding, params["charset"])
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return stripHTML(raw), nil
|
||||
case mediaType == "text/plain":
|
||||
return decodeBody(body, encoding, params["charset"])
|
||||
default:
|
||||
// A single-part non-text message (a bare PDF, say). No body, headers only.
|
||||
return "", nil
|
||||
}
|
||||
}
|
||||
|
||||
// multipartText reads one multipart level, recursing into nested multiparts.
|
||||
// Returns the plain part if any part yielded one, else the stripped HTML.
|
||||
func multipartText(mr *multipart.Reader) (string, error) {
|
||||
var plain, html string
|
||||
for {
|
||||
part, err := mr.NextPart()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
// A truncated multipart still gives up whatever came before it.
|
||||
break
|
||||
}
|
||||
if part.FileName() != "" {
|
||||
part.Close()
|
||||
continue // attachment
|
||||
}
|
||||
ct := part.Header.Get("Content-Type")
|
||||
mediaType, _, _ := mime.ParseMediaType(ct)
|
||||
text, err := plaintextBody(ct, part.Header.Get("Content-Transfer-Encoding"), part)
|
||||
part.Close()
|
||||
if err != nil || strings.TrimSpace(text) == "" {
|
||||
continue
|
||||
}
|
||||
if mediaType == "text/html" && !strings.HasPrefix(mediaType, "multipart/") {
|
||||
if html == "" {
|
||||
html = text
|
||||
}
|
||||
continue
|
||||
}
|
||||
if plain == "" {
|
||||
plain = text
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(plain) != "" {
|
||||
return plain, nil
|
||||
}
|
||||
return html, nil
|
||||
}
|
||||
|
||||
// decodeBody applies the transfer encoding, then the charset.
|
||||
//
|
||||
// Charset support is UTF-8 (and ASCII, its subset) only, on purpose: x/text's
|
||||
// encoding tables are not vendored here, and guessing at windows-1251 bytes
|
||||
// would feed the model mojibake it would happily extract a task from. An
|
||||
// unsupported charset returns an error, which ParseMessage turns into an empty
|
||||
// body — subject-only, which is honest.
|
||||
func decodeBody(r io.Reader, encoding, charset string) (string, error) {
|
||||
switch strings.ToLower(strings.TrimSpace(encoding)) {
|
||||
case "quoted-printable":
|
||||
r = quotedprintable.NewReader(r)
|
||||
case "base64":
|
||||
r = newBase64Reader(r)
|
||||
}
|
||||
b, err := io.ReadAll(io.LimitReader(r, 1<<20))
|
||||
if err != nil && len(b) == 0 {
|
||||
return "", fmt.Errorf("email: read body: %w", err)
|
||||
}
|
||||
switch cs := strings.ToLower(strings.TrimSpace(charset)); cs {
|
||||
case "", "utf-8", "utf8", "us-ascii", "ascii":
|
||||
return string(b), nil
|
||||
default:
|
||||
return "", fmt.Errorf("email: unsupported charset %q", cs)
|
||||
}
|
||||
}
|
||||
|
||||
// decodeHeader decodes RFC 2047 encoded words ("=?utf-8?B?...?="), which is how
|
||||
// every Russian subject line arrives. Undecodable headers come back as-is
|
||||
// rather than empty: a mangled subject is still a hint, and it is only ever
|
||||
// shown to him as evidence.
|
||||
func decodeHeader(v string) string {
|
||||
dec := new(mime.WordDecoder)
|
||||
out, err := dec.DecodeHeader(v)
|
||||
if err != nil {
|
||||
return collapse(v)
|
||||
}
|
||||
return collapse(out)
|
||||
}
|
||||
|
||||
var (
|
||||
scriptStyle = regexp.MustCompile(`(?is)<(script|style)\b[^>]*>.*?</\s*(script|style)\s*>`)
|
||||
htmlBreak = regexp.MustCompile(`(?i)<\s*(br\s*/?|/p|/div|/tr|/li|/h[1-6])\s*>`)
|
||||
htmlTag = regexp.MustCompile(`(?s)<[^>]*>`)
|
||||
htmlComment = regexp.MustCompile(`(?s)<!--.*?-->`)
|
||||
)
|
||||
|
||||
// stripHTML reduces an HTML part to text. A regex stripper, not a parser:
|
||||
// x/net/html is not vendored, and the consumer is a model reading prose — a
|
||||
// stray angle bracket costs nothing, whereas a new dependency for the privacy-
|
||||
// sensitive path costs review.
|
||||
func stripHTML(s string) string {
|
||||
s = scriptStyle.ReplaceAllString(s, " ")
|
||||
s = htmlComment.ReplaceAllString(s, " ")
|
||||
s = htmlBreak.ReplaceAllString(s, "\n")
|
||||
s = htmlTag.ReplaceAllString(s, " ")
|
||||
return unescapeEntities(s)
|
||||
}
|
||||
|
||||
var entities = strings.NewReplacer(
|
||||
" ", " ", "&", "&", "<", "<", ">", ">",
|
||||
""", `"`, "'", "'", "'", "'", "—", "—", "–", "–",
|
||||
)
|
||||
|
||||
func unescapeEntities(s string) string { return entities.Replace(s) }
|
||||
|
||||
// collapse squeezes runs of whitespace, keeping single newlines. Mail bodies
|
||||
// arrive with hard-wrapped lines and blocks of blank space; the model does not
|
||||
// need them and they are pure context budget.
|
||||
func collapse(s string) string {
|
||||
lines := strings.Split(strings.ReplaceAll(s, "\r\n", "\n"), "\n")
|
||||
var out []string
|
||||
blank := 0
|
||||
for _, l := range lines {
|
||||
l = strings.TrimSpace(strings.Join(strings.Fields(l), " "))
|
||||
if l == "" {
|
||||
blank++
|
||||
if blank > 1 {
|
||||
continue
|
||||
}
|
||||
out = append(out, "")
|
||||
continue
|
||||
}
|
||||
blank = 0
|
||||
out = append(out, l)
|
||||
}
|
||||
return strings.TrimSpace(strings.Join(out, "\n"))
|
||||
}
|
||||
|
||||
// truncate cuts to n bytes on a rune boundary.
|
||||
func truncate(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
cut := s[:n]
|
||||
for len(cut) > 0 && !isRuneStart(cut[len(cut)-1]) {
|
||||
cut = cut[:len(cut)-1]
|
||||
}
|
||||
return strings.TrimSpace(cut) + "…"
|
||||
}
|
||||
|
||||
func isRuneStart(b byte) bool { return b&0xC0 != 0x80 }
|
||||
|
||||
// newBase64Reader — base64.NewDecoder already skips the CRLFs mail bodies wrap
|
||||
// with, so this is only a named seam for decodeBody to read cleanly.
|
||||
func newBase64Reader(r io.Reader) io.Reader {
|
||||
return base64.NewDecoder(base64.StdEncoding, r)
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package email
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func fixture(t *testing.T, name string) []byte {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(filepath.Join("testdata", name))
|
||||
if err != nil {
|
||||
t.Fatalf("read fixture %s: %v", name, err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func TestParsePlainRussian(t *testing.T) {
|
||||
msg, err := ParseMessage(7, fixture(t, "plain_ru.eml"))
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if msg.UID != 7 {
|
||||
t.Errorf("uid = %d, want 7", msg.UID)
|
||||
}
|
||||
if want := "Нужно закрыть задачу"; msg.Subject != want {
|
||||
t.Errorf("subject = %q, want %q", msg.Subject, want)
|
||||
}
|
||||
if !strings.Contains(msg.From, "Антон") {
|
||||
t.Errorf("from = %q, want the decoded display name", msg.From)
|
||||
}
|
||||
if !strings.Contains(msg.Body, "Надо отправить акт до пятницы.") {
|
||||
t.Errorf("body = %q, want the quoted-printable text decoded", msg.Body)
|
||||
}
|
||||
if msg.Junk {
|
||||
t.Errorf("a personal mail must not be junk (%s)", msg.JunkReason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseHTMLOnlyIsStripped(t *testing.T) {
|
||||
msg, err := ParseMessage(1, fixture(t, "html_only.eml"))
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if strings.Contains(msg.Body, "<") || strings.Contains(msg.Body, "color:red") || strings.Contains(msg.Body, "x()") {
|
||||
t.Errorf("body still has markup/script/style: %q", msg.Body)
|
||||
}
|
||||
for _, want := range []string{"Счёт за интернет: 700", "Оплатить до 5 августа."} {
|
||||
if !strings.Contains(msg.Body, want) {
|
||||
t.Errorf("body = %q, want it to contain %q", msg.Body, want)
|
||||
}
|
||||
}
|
||||
// must have become a real space, not vanished into the number.
|
||||
if strings.Contains(msg.Body, " ") {
|
||||
t.Errorf("entity left unescaped: %q", msg.Body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsePrefersPlainAndSkipsAttachments(t *testing.T) {
|
||||
msg, err := ParseMessage(2, fixture(t, "mixed_attachment.eml"))
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if got := strings.TrimSpace(msg.Body); got != "Sign the contract before Monday." {
|
||||
t.Errorf("body = %q, want the text/plain alternative only", got)
|
||||
}
|
||||
if strings.Contains(msg.Body, "PDF") {
|
||||
t.Errorf("attachment bytes leaked into the body: %q", msg.Body)
|
||||
}
|
||||
}
|
||||
|
||||
// An unsupported charset must degrade to headers-only rather than to mojibake
|
||||
// the model would then extract a task from.
|
||||
func TestParseUnsupportedCharsetKeepsHeaders(t *testing.T) {
|
||||
msg, err := ParseMessage(3, fixture(t, "cp1251.eml"))
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if msg.Subject != "Legacy" {
|
||||
t.Errorf("subject = %q, want Legacy", msg.Subject)
|
||||
}
|
||||
if msg.Body != "" {
|
||||
t.Errorf("body = %q, want empty for an undecodable charset", msg.Body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTruncatesLongBody(t *testing.T) {
|
||||
var b strings.Builder
|
||||
b.WriteString("Subject: long\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n")
|
||||
for i := 0; i < 2000; i++ {
|
||||
b.WriteString("длинная строка ")
|
||||
}
|
||||
msg, err := ParseMessage(4, []byte(b.String()))
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if len(msg.Body) > MaxBodyBytes+8 {
|
||||
t.Errorf("body kept %d bytes, want ≤ %d", len(msg.Body), MaxBodyBytes)
|
||||
}
|
||||
if !strings.HasSuffix(msg.Body, "…") {
|
||||
t.Errorf("truncated body should be marked: %q", msg.Body[len(msg.Body)-20:])
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollapseSqueezesBlankLines(t *testing.T) {
|
||||
got := collapse(" a b \r\n\r\n\r\n\r\n c \r\n")
|
||||
if got != "a b\n\nc" {
|
||||
t.Errorf("collapse = %q, want %q", got, "a b\n\nc")
|
||||
}
|
||||
}
|
||||
Vendored
+7
@@ -0,0 +1,7 @@
|
||||
From: legacy@example.org
|
||||
To: kami@example.org
|
||||
Subject: Legacy
|
||||
Date: Fri, 01 Aug 2026 05:00:00 +0400
|
||||
Content-Type: text/plain; charset="windows-1251"
|
||||
|
||||
Ï
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
From: billing@isp.example
|
||||
To: kami@example.org
|
||||
Subject: =?utf-8?B?0KHRh9GR0YIg0LfQsCDQuNC90YLQtdGA0L3QtdGC?=
|
||||
Date: Fri, 01 Aug 2026 08:00:00 +0400
|
||||
MIME-Version: 1.0
|
||||
Content-Type: multipart/alternative; boundary="B1"
|
||||
|
||||
--B1
|
||||
Content-Type: text/html; charset="utf-8"
|
||||
Content-Transfer-Encoding: base64
|
||||
|
||||
PGh0bWw+PGhlYWQ+PHN0eWxlPnB7Y29sb3I6cmVkfTwvc3R5bGU+PC9oZWFkPjxib2R5PjxwPtCh0YfRkdGCINC30LAg0LjQvdGC0LXRgNC90LXRgjogNzAwJm5ic3A74oK9PC9wPjxwPtCe0L/Qu9Cw0YLQuNGC0Ywg0LTQviA1INCw0LLQs9GD0YHRgtCwLjwvcD48c2NyaXB0PngoKTwvc2NyaXB0PjwvYm9keT48L2h0bWw+
|
||||
--B1--
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
From: hr@work.example
|
||||
To: kami@example.org
|
||||
Subject: Contract
|
||||
Date: Fri, 01 Aug 2026 07:00:00 +0400
|
||||
MIME-Version: 1.0
|
||||
Content-Type: multipart/mixed; boundary="M1"
|
||||
|
||||
--M1
|
||||
Content-Type: multipart/alternative; boundary="A1"
|
||||
|
||||
--A1
|
||||
Content-Type: text/plain; charset="utf-8"
|
||||
|
||||
Sign the contract before Monday.
|
||||
--A1
|
||||
Content-Type: text/html; charset="utf-8"
|
||||
|
||||
<p>Sign the contract before Monday.</p>
|
||||
--A1--
|
||||
--M1
|
||||
Content-Type: application/pdf; name="contract.pdf"
|
||||
Content-Disposition: attachment; filename="contract.pdf"
|
||||
Content-Transfer-Encoding: base64
|
||||
|
||||
JVBERi0xLjQgbm90IHJlYWxseSBhIHBkZg==
|
||||
--M1--
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
From: news@shop.example
|
||||
To: kami@example.org
|
||||
Subject: =?utf-8?B?0KHQutC40LTQutC4INGC0L7Qu9GM0LrQviDRgdC10LPQvtC00L3Rjw==?=
|
||||
Date: Fri, 01 Aug 2026 06:00:00 +0400
|
||||
List-Unsubscribe: <mailto:unsub@shop.example>
|
||||
Precedence: bulk
|
||||
Content-Type: text/plain; charset="utf-8"
|
||||
|
||||
Sale!
|
||||
Vendored
+14
@@ -0,0 +1,14 @@
|
||||
From: =?utf-8?B?0JDQvdGC0L7QvQ==?= <anton@example.org>
|
||||
To: kami@example.org
|
||||
Subject: =?utf-8?B?0J3Rg9C20L3QviDQt9Cw0LrRgNGL0YLRjCDQt9Cw0LTQsNGH0YM=?=
|
||||
Date: Fri, 01 Aug 2026 09:12:00 +0400
|
||||
Content-Type: text/plain; charset="utf-8"
|
||||
Content-Transfer-Encoding: quoted-printable
|
||||
Message-ID: <plain-ru@example.org>
|
||||
|
||||
=D0=9F=D1=80=D0=B8=D0=B2=D0=B5=D1=82! =D0=9D=D0=B0=D0=B4=D0=BE =D0=BE=D1=82=
|
||||
=D0=BF=D1=80=D0=B0=D0=B2=D0=B8=D1=82=D1=8C =D0=B0=D0=BA=D1=82 =D0=B4=D0=BE =
|
||||
=D0=BF=D1=8F=D1=82=D0=BD=D0=B8=D1=86=D1=8B.
|
||||
|
||||
--
|
||||
Anton
|
||||
@@ -138,6 +138,88 @@ type CaptureTaskResp struct {
|
||||
Created bool `json:"created"`
|
||||
}
|
||||
|
||||
// IngestMailReq — one message a mail reader has fetched, handed to core for
|
||||
// extraction (Vikunja #246).
|
||||
//
|
||||
// The mail reader (cmd/mavmaild) holds the IMAP credential and core never sees
|
||||
// it, the same split mavpoll uses for the zenmoney token. What crosses this
|
||||
// boundary is only the message text, because extraction runs on the resident
|
||||
// model and llama-server lives inside core's process.
|
||||
//
|
||||
// Body is already plaintext and truncated by internal/email; core does not
|
||||
// re-parse MIME and never stores the body. Junk means the reader's header
|
||||
// filter already classified the message as bulk — core is told rather than
|
||||
// asked, so a junk message can be counted without a model call.
|
||||
//
|
||||
// This method is available only when core has an email block configured AND a
|
||||
// llama-server phraser; otherwise it answers ErrUnknownMethod, which is what
|
||||
// "off unless configured" looks like at the wire.
|
||||
type IngestMailReq struct {
|
||||
Mailbox string `json:"mailbox"`
|
||||
UID uint32 `json:"uid"`
|
||||
From string `json:"from,omitempty"`
|
||||
Subject string `json:"subject,omitempty"`
|
||||
Date string `json:"date,omitempty"`
|
||||
Body string `json:"body,omitempty"`
|
||||
Junk bool `json:"junk,omitempty"`
|
||||
}
|
||||
|
||||
// IngestMailResp — what core did with the message. TaskIDs are the rows
|
||||
// CaptureTask returned; Created counts the ones that were new (a re-read
|
||||
// mailbox dedupes to Created=0). Skipped is set when nothing was asked of the
|
||||
// model at all — junk, or an empty message.
|
||||
//
|
||||
// Nothing here echoes the mail back. The reader logs counts.
|
||||
type IngestMailResp struct {
|
||||
TaskIDs []int64 `json:"task_ids,omitempty"`
|
||||
Created int `json:"created"`
|
||||
Skipped bool `json:"skipped,omitempty"`
|
||||
}
|
||||
|
||||
// SwapModelReq — load another resident model without restarting the daemon
|
||||
// (Vikunja #250). ModelPath must be one of the paths in phraser.swap_models;
|
||||
// anything else is ErrForbidden, and an unconfigured allowlist makes the whole
|
||||
// method ErrUnknownMethod.
|
||||
//
|
||||
// NGpuLayers and NCtx are zero for "keep what is loaded now", which is the
|
||||
// normal case — the same laptop iGPU, a different gguf.
|
||||
//
|
||||
// This is an owner action. It is AuthStepUp in the authority table, it is not on
|
||||
// CoreAPI, and no act, intent or timer can reach it: swapping the model is not
|
||||
// something Maven does to herself.
|
||||
type SwapModelReq struct {
|
||||
ModelPath string `json:"model_path"`
|
||||
NGpuLayers int `json:"n_gpu_layers,omitempty"`
|
||||
NCtx int `json:"n_ctx,omitempty"`
|
||||
}
|
||||
|
||||
// SwapModelResp — what the daemon ended up serving. Model is the identity the
|
||||
// new llama-server reported for itself, not an echo of the request: if the file
|
||||
// was not the model the operator thought it was, this is where it shows.
|
||||
//
|
||||
// RolledBack is true when the requested model failed to load or would not answer
|
||||
// and the previous one was put back. In that case the call also returns an error
|
||||
// — the swap did not happen — and Model names the model still serving.
|
||||
type SwapModelResp struct {
|
||||
Model string `json:"model"`
|
||||
ModelPath string `json:"model_path"`
|
||||
BaseURL string `json:"base_url"`
|
||||
RolledBack bool `json:"rolled_back,omitempty"`
|
||||
TookMs int64 `json:"took_ms"`
|
||||
}
|
||||
|
||||
// ModelStatusResp — which model is resident and which ones may be swapped in.
|
||||
// Read-only; the authed page renders it. Swappable is the configured allowlist,
|
||||
// so an empty list means the capability is off.
|
||||
type ModelStatusResp struct {
|
||||
Model string `json:"model"`
|
||||
ModelPath string `json:"model_path"`
|
||||
BaseURL string `json:"base_url"`
|
||||
NGpuLayers int `json:"n_gpu_layers"`
|
||||
NCtx int `json:"n_ctx"`
|
||||
Swappable []string `json:"swappable,omitempty"`
|
||||
}
|
||||
|
||||
type listTasksReq struct {
|
||||
Status string `json:"status"` // "" all | "live" | candidate|open|done|dropped
|
||||
}
|
||||
|
||||
@@ -448,6 +448,39 @@ func (c *Client) SetTaskStatus(ctx context.Context, id int64, status string, ts
|
||||
return c.call(ctx, MethodSetTaskStatus, setTaskStatusReq{ID: id, Status: status, Ts: ts}, nil)
|
||||
}
|
||||
|
||||
// IngestMail hands one fetched message to core for extraction. ErrUnknownMethod
|
||||
// means core has no email block configured — the caller should stop asking, not
|
||||
// retry.
|
||||
func (c *Client) IngestMail(ctx context.Context, req IngestMailReq) (IngestMailResp, error) {
|
||||
var r IngestMailResp
|
||||
if err := c.call(ctx, MethodIngestMail, req, &r); err != nil {
|
||||
return IngestMailResp{}, err
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// SwapModel asks core to load another resident model (Vikunja #250).
|
||||
// ErrUnknownMethod means core has no phraser.swap_models allowlist configured;
|
||||
// ErrForbidden means the path is not on it, or step-up was not asserted. A
|
||||
// non-nil error with RolledBack set means nothing changed — the old model is
|
||||
// still serving.
|
||||
func (c *Client) SwapModel(ctx context.Context, req SwapModelReq) (SwapModelResp, error) {
|
||||
var r SwapModelResp
|
||||
if err := c.call(ctx, MethodSwapModel, req, &r); err != nil {
|
||||
return SwapModelResp{}, err
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// ModelStatus reports the resident model and the swap allowlist. Read-only.
|
||||
func (c *Client) ModelStatus(ctx context.Context) (ModelStatusResp, error) {
|
||||
var r ModelStatusResp
|
||||
if err := c.call(ctx, MethodModelStatus, nil, &r); err != nil {
|
||||
return ModelStatusResp{}, err
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func (c *Client) DismissProposedRoutine(ctx context.Context, id int64) error {
|
||||
return c.call(ctx, MethodDismissProposedRoutine, dismissProposedRoutineReq{ID: id}, nil)
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
@@ -565,3 +566,77 @@ func mustJSON(v any) []byte {
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// TestIngestMail_OffUnlessConfigured — with no IngestMailFn set (the default,
|
||||
// and what an unconfigured core looks like) the method does not exist. A mail
|
||||
// reader gets a refusal it can act on rather than a silent success.
|
||||
func TestIngestMail_OffUnlessConfigured(t *testing.T) {
|
||||
_, _, cli, _ := newServerWithStore(t)
|
||||
if _, err := cli.IngestMail(context.Background(), IngestMailReq{Mailbox: "INBOX", UID: 1}); !errors.Is(err, ErrUnknownMethod) {
|
||||
t.Fatalf("IngestMail error = %v, want ErrUnknownMethod", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestIngestMail_Hook — when the daemon wires the hook, the message crosses the
|
||||
// boundary intact and the response comes back.
|
||||
func TestIngestMail_Hook(t *testing.T) {
|
||||
_, srv, cli, _ := newServerWithStore(t)
|
||||
var got IngestMailReq
|
||||
srv.IngestMailFn = func(_ context.Context, req IngestMailReq) (IngestMailResp, error) {
|
||||
got = req
|
||||
return IngestMailResp{TaskIDs: []int64{7}, Created: 1}, nil
|
||||
}
|
||||
resp, err := cli.IngestMail(context.Background(), IngestMailReq{
|
||||
Mailbox: "INBOX", UID: 12, Subject: "Счёт", Body: "Оплатить.", Junk: false,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("IngestMail: %v", err)
|
||||
}
|
||||
if resp.Created != 1 || len(resp.TaskIDs) != 1 || resp.TaskIDs[0] != 7 {
|
||||
t.Errorf("resp = %+v", resp)
|
||||
}
|
||||
if got.UID != 12 || got.Subject != "Счёт" || got.Body != "Оплатить." {
|
||||
t.Errorf("req across the wire = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSwapModel_OffUnlessConfigured — no allowlist in the config means the
|
||||
// daemon never sets the hook, and the method does not exist. That is what "off
|
||||
// unless configured" looks like at the wire for the model swap (Vikunja #250).
|
||||
func TestSwapModel_OffUnlessConfigured(t *testing.T) {
|
||||
_, _, cli, _ := newServerWithStore(t)
|
||||
if _, err := cli.SwapModel(context.Background(), SwapModelReq{ModelPath: "/m/x.gguf"}); !errors.Is(err, ErrUnknownMethod) {
|
||||
t.Fatalf("SwapModel error = %v, want ErrUnknownMethod", err)
|
||||
}
|
||||
if _, err := cli.ModelStatus(context.Background()); !errors.Is(err, ErrUnknownMethod) {
|
||||
t.Fatalf("ModelStatus error = %v, want ErrUnknownMethod", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSwapModel_Hook — the request crosses the boundary intact and the reported
|
||||
// identity comes back. A refusal from the daemon's allowlist arrives as
|
||||
// ErrForbidden, which is what a caller keys its error message off.
|
||||
func TestSwapModel_Hook(t *testing.T) {
|
||||
_, srv, cli, _ := newServerWithStore(t)
|
||||
var got SwapModelReq
|
||||
srv.SwapModelFn = func(_ context.Context, req SwapModelReq) (SwapModelResp, error) {
|
||||
got = req
|
||||
if req.ModelPath != "/m/allowed.gguf" {
|
||||
return SwapModelResp{}, fmt.Errorf("%w: not allowlisted", ErrForbidden)
|
||||
}
|
||||
return SwapModelResp{Model: "allowed", ModelPath: req.ModelPath, BaseURL: "http://127.0.0.1:9", TookMs: 12}, nil
|
||||
}
|
||||
resp, err := cli.SwapModel(context.Background(), SwapModelReq{ModelPath: "/m/allowed.gguf", NCtx: 4096})
|
||||
if err != nil {
|
||||
t.Fatalf("SwapModel: %v", err)
|
||||
}
|
||||
if resp.Model != "allowed" || resp.TookMs != 12 {
|
||||
t.Errorf("resp = %+v", resp)
|
||||
}
|
||||
if got.NCtx != 4096 {
|
||||
t.Errorf("req across the wire = %+v", got)
|
||||
}
|
||||
if _, err := cli.SwapModel(context.Background(), SwapModelReq{ModelPath: "/etc/shadow"}); !errors.Is(err, ErrForbidden) {
|
||||
t.Fatalf("swap to a non-allowlisted path = %v; want ErrForbidden", err)
|
||||
}
|
||||
}
|
||||
|
||||
+77
-4
@@ -421,6 +421,30 @@ type Server struct {
|
||||
// Set by the daemon; nil ⇒ MethodStoreEncryptionKey returns ErrUnknownMethod.
|
||||
WrapKeyFn WrapKeyFunc
|
||||
|
||||
// IngestMailFn — extracts task candidates from one fetched message. Set by
|
||||
// the daemon only when an email block is configured AND there is a
|
||||
// llama-server to extract with; nil ⇒ MethodIngestMail returns
|
||||
// ErrUnknownMethod, so a mail reader pointed at a core that is not
|
||||
// configured for mail is refused rather than silently ignored.
|
||||
//
|
||||
// Like StepUp/WrapKeyFn/UnlockFn this bypasses CoreAPI: it is not a store
|
||||
// operation, it needs the resident model, and it must not become a method
|
||||
// every CoreAPI implementation has to carry.
|
||||
IngestMailFn IngestMailFunc
|
||||
|
||||
// SwapModelFn / ModelStatusFn — the on-the-fly resident model swap (Vikunja
|
||||
// #250) and its read side. Set by the daemon only when phraser.swap_models
|
||||
// lists at least one model AND the phraser owns a llama-server; nil ⇒ both
|
||||
// methods answer ErrUnknownMethod, which is what "off unless configured"
|
||||
// looks like at the wire.
|
||||
//
|
||||
// They bypass CoreAPI for the same reason IngestMailFn does: this is not a
|
||||
// store operation, it needs the daemon's llama-server, and no other CoreAPI
|
||||
// implementation should have to carry it. MethodSwapModel is AuthStepUp in
|
||||
// internal/auth — owner-triggered, never an act and never a timer.
|
||||
SwapModelFn SwapModelFunc
|
||||
ModelStatusFn ModelStatusFunc
|
||||
|
||||
// UnlockFn — unwraps the store encryption key from the wrapped blob using
|
||||
// the passkey credential public key, opens the encrypted store, and wires
|
||||
// the rest of the daemon (voice, loop, delivery). Set by the daemon when
|
||||
@@ -439,6 +463,15 @@ type WrapKeyFunc func(ctx context.Context, publicKey []byte) error
|
||||
// public key and completes daemon initialization.
|
||||
type UnlockFunc func(ctx context.Context, publicKey []byte) error
|
||||
|
||||
// SwapModelFunc — loads another resident model in place of the live one.
|
||||
type SwapModelFunc func(ctx context.Context, req SwapModelReq) (SwapModelResp, error)
|
||||
|
||||
// ModelStatusFunc — reports the resident model and the swap allowlist.
|
||||
type ModelStatusFunc func(ctx context.Context) (ModelStatusResp, error)
|
||||
|
||||
// IngestMailFunc — core-side mail extraction. Returns what was captured.
|
||||
type IngestMailFunc func(ctx context.Context, req IngestMailReq) (IngestMailResp, error)
|
||||
|
||||
// CheckFunc — the auth hook signature. Wired by the daemon (auth.Gate.Check
|
||||
// satisfies this); dispatch calls it once per request after param-unmarshal
|
||||
// independence (it gets the raw params, may unmarshal what it needs — ipc
|
||||
@@ -606,9 +639,11 @@ func withoutParams[R any](fn func(ctx context.Context, api CoreAPI) (R, error))
|
||||
// existed) as an argument — so SetAPI's runtime swap (the unlock transition)
|
||||
// is still honored on the very next request with no extra plumbing here.
|
||||
//
|
||||
// MethodAssertStepUp, MethodStoreEncryptionKey and MethodUnlock are NOT in
|
||||
// this table: they bypass CoreAPI entirely (s.StepUp / s.WrapKeyFn /
|
||||
// s.UnlockFn), so dispatch special-cases them before consulting the table.
|
||||
// MethodAssertStepUp, MethodStoreEncryptionKey, MethodUnlock,
|
||||
// MethodIngestMail, MethodSwapModel and MethodModelStatus are NOT in this
|
||||
// table: they bypass CoreAPI entirely
|
||||
// (s.StepUp / s.WrapKeyFn / s.UnlockFn / s.IngestMailFn), so dispatch
|
||||
// special-cases them before consulting the table.
|
||||
var methodTable = map[Method]handlerFunc{
|
||||
MethodWriteFact: withParams(func(ctx context.Context, api CoreAPI, p WriteFactReq) (idResp, error) {
|
||||
id, err := api.WriteFact(ctx, p)
|
||||
@@ -816,7 +851,7 @@ func (s *Server) dispatch(ctx context.Context, req Request) (json.RawMessage, er
|
||||
}
|
||||
}
|
||||
|
||||
// These three bypass CoreAPI entirely — they drive Server fields set
|
||||
// These bypass CoreAPI entirely — they drive Server fields set
|
||||
// directly by the daemon (StepUp / WrapKeyFn / UnlockFn), not store
|
||||
// state, so they can never be table entries keyed on a CoreAPI method.
|
||||
switch req.Method {
|
||||
@@ -845,6 +880,44 @@ func (s *Server) dispatch(ctx context.Context, req Request) (json.RawMessage, er
|
||||
return marshalResult(nil), s.UnlockFn(ctx, p.PublicKey)
|
||||
}
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnknownMethod, req.Method)
|
||||
|
||||
case MethodIngestMail:
|
||||
if s.IngestMailFn != nil {
|
||||
var p IngestMailReq
|
||||
if err := unmarshalParams(req.Params, &p); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := s.IngestMailFn(ctx, p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return marshalResult(resp), nil
|
||||
}
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnknownMethod, req.Method)
|
||||
|
||||
case MethodSwapModel:
|
||||
if s.SwapModelFn != nil {
|
||||
var p SwapModelReq
|
||||
if err := unmarshalParams(req.Params, &p); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := s.SwapModelFn(ctx, p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return marshalResult(resp), nil
|
||||
}
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnknownMethod, req.Method)
|
||||
|
||||
case MethodModelStatus:
|
||||
if s.ModelStatusFn != nil {
|
||||
resp, err := s.ModelStatusFn(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return marshalResult(resp), nil
|
||||
}
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnknownMethod, req.Method)
|
||||
}
|
||||
|
||||
h, ok := methodTable[req.Method]
|
||||
|
||||
@@ -50,6 +50,9 @@ const (
|
||||
MethodCaptureTask Method = "capture_task"
|
||||
MethodListTasks Method = "list_tasks"
|
||||
MethodSetTaskStatus Method = "set_task_status"
|
||||
MethodIngestMail Method = "ingest_mail"
|
||||
MethodSwapModel Method = "swap_model"
|
||||
MethodModelStatus Method = "model_status"
|
||||
)
|
||||
|
||||
// Request — one frame from module to core. Params is the JSON-encoded argument
|
||||
|
||||
+27
-1
@@ -10,10 +10,17 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Client struct {
|
||||
// mu guards base only. The base URL changes when the daemon swaps the
|
||||
// resident model (Vikunja #250): llama-server is relaunched on a fresh
|
||||
// port, and every holder of this client — the LLM router, the replier, the
|
||||
// mail extractor — must follow without being rebuilt. One mutexed field is
|
||||
// the whole mechanism; a swap re-points the client, it does not replace it.
|
||||
mu sync.RWMutex
|
||||
base string
|
||||
http *http.Client
|
||||
}
|
||||
@@ -22,6 +29,25 @@ func New(baseURL string, timeout time.Duration) *Client {
|
||||
return &Client{base: baseURL, http: &http.Client{Timeout: timeout}}
|
||||
}
|
||||
|
||||
// SetBaseURL re-points the client at another llama-server. Safe to call while
|
||||
// requests are in flight: a request that already read the old base finishes
|
||||
// against the old base (or fails, and every caller of Complete has a fallback),
|
||||
// and the next one uses the new base. It is deliberately NOT a queue-and-retry —
|
||||
// the phraser quiesces around a swap, so the window is small and a lost turn
|
||||
// degrades to the classifier rather than hanging.
|
||||
func (c *Client) SetBaseURL(base string) {
|
||||
c.mu.Lock()
|
||||
c.base = base
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
// BaseURL is the server this client currently talks to.
|
||||
func (c *Client) BaseURL() string {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
return c.base
|
||||
}
|
||||
|
||||
type Req struct {
|
||||
System string
|
||||
User string
|
||||
@@ -63,7 +89,7 @@ func (c *Client) Complete(ctx context.Context, r Req) (string, error) {
|
||||
RepeatPenalty: r.RepeatPenalty,
|
||||
Stop: r.Stop,
|
||||
})
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", c.base+"/v1/chat/completions", bytes.NewReader(b))
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", c.BaseURL()+"/v1/chat/completions", bytes.NewReader(b))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -57,3 +57,37 @@ func TestComplete(t *testing.T) {
|
||||
t.Errorf("got %q, want %q", got, "ok")
|
||||
}
|
||||
}
|
||||
|
||||
// TestSetBaseURL — a model swap re-points every holder of the client rather than
|
||||
// rebuilding the router, the replier and the extractors (Vikunja #250).
|
||||
func TestSetBaseURL(t *testing.T) {
|
||||
var hit string
|
||||
srvA := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hit = "A"
|
||||
w.Write([]byte(`{"choices":[{"message":{"content":"a"}}]}`))
|
||||
}))
|
||||
defer srvA.Close()
|
||||
srvB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hit = "B"
|
||||
w.Write([]byte(`{"choices":[{"message":{"content":"b"}}]}`))
|
||||
}))
|
||||
defer srvB.Close()
|
||||
|
||||
c := New(srvA.URL, 5*time.Second)
|
||||
if _, err := c.Complete(context.Background(), Req{User: "x"}); err != nil {
|
||||
t.Fatalf("Complete against A: %v", err)
|
||||
}
|
||||
if hit != "A" {
|
||||
t.Fatalf("first request went to %q; want A", hit)
|
||||
}
|
||||
c.SetBaseURL(srvB.URL)
|
||||
if got := c.BaseURL(); got != srvB.URL {
|
||||
t.Errorf("BaseURL = %q; want %q", got, srvB.URL)
|
||||
}
|
||||
if _, err := c.Complete(context.Background(), Req{User: "x"}); err != nil {
|
||||
t.Fatalf("Complete against B: %v", err)
|
||||
}
|
||||
if hit != "B" {
|
||||
t.Errorf("request after the swap went to %q; want B", hit)
|
||||
}
|
||||
}
|
||||
|
||||
+150
-34
@@ -27,14 +27,46 @@ var listenRE = regexp.MustCompile(`listening on (https?://\S+)`)
|
||||
type LLMPhraser struct {
|
||||
cfg Config
|
||||
client *http.Client
|
||||
port string
|
||||
cmd *exec.Cmd
|
||||
cancel context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
|
||||
// tmpl — the hand-written Russian nudges. Default path for nudges; see
|
||||
// Config.LLMNudges. nil only if the template file failed to load.
|
||||
tmpl *NudgeTemplates
|
||||
|
||||
// spawnCtx — the parent of every llama-server this phraser starts, i.e. the
|
||||
// daemon's own context. Deliberately NOT the per-request context of the call
|
||||
// that asked for a model swap: that one is cancelled the moment the request
|
||||
// returns, which would kill the model it had just loaded.
|
||||
spawnCtx context.Context
|
||||
cancel context.CancelFunc
|
||||
|
||||
// launch / probe — the two side effects of a swap, injectable so the swap
|
||||
// logic is testable without a real llama-server and a real model file.
|
||||
// launch is nil when this phraser does not own its server (NewLLMPhraserAt),
|
||||
// which is also what makes Swap refuse there.
|
||||
launch func(ctx context.Context, cfg Config) (backend, error)
|
||||
probe func(ctx context.Context, base string) (string, error)
|
||||
|
||||
// swapMu — single-flight around Swap. Held for the whole swap, including the
|
||||
// model load, so two concurrent swap requests can never both be loading.
|
||||
swapMu sync.Mutex
|
||||
|
||||
// mu guards everything below: the live backend, the swap gate and the
|
||||
// in-flight request count. See acquire/quiesce in swap.go.
|
||||
mu sync.Mutex
|
||||
be backend
|
||||
live liveModel
|
||||
swapping bool
|
||||
inflight int
|
||||
observers []func(baseURL string)
|
||||
}
|
||||
|
||||
// liveModel — what is actually loaded right now. Distinct from Config, which
|
||||
// stays immutable after construction: a swap changes these three fields and
|
||||
// nothing else, so no reader of cfg (prompts, grammar, timeouts) races a swap.
|
||||
type liveModel struct {
|
||||
ModelPath string
|
||||
NGpuLayers int
|
||||
NCtx int
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
@@ -85,15 +117,21 @@ func DefaultConfig(modelPath string) Config {
|
||||
func NewLLMPhraser(ctx context.Context, cfg Config) (*LLMPhraser, error) {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
p := &LLMPhraser{
|
||||
cfg: cfg,
|
||||
client: &http.Client{Timeout: cfg.Timeout},
|
||||
cancel: cancel,
|
||||
tmpl: loadNudgeTemplates(),
|
||||
cfg: cfg,
|
||||
client: &http.Client{Timeout: cfg.Timeout},
|
||||
tmpl: loadNudgeTemplates(),
|
||||
spawnCtx: ctx,
|
||||
cancel: cancel,
|
||||
launch: spawnLlamaServer,
|
||||
probe: defaultProbe,
|
||||
live: liveModel{ModelPath: cfg.ModelPath, NGpuLayers: cfg.NGpuLayers, NCtx: cfg.NCtx},
|
||||
}
|
||||
if err := p.start(ctx); err != nil {
|
||||
be, err := p.launch(ctx, cfg)
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, err
|
||||
}
|
||||
p.be = be
|
||||
return p, nil
|
||||
}
|
||||
|
||||
@@ -106,11 +144,17 @@ func NewLLMPhraser(ctx context.Context, cfg Config) (*LLMPhraser, error) {
|
||||
// still uses NewLLMPhraser and still owns its own child process.
|
||||
func NewLLMPhraserAt(baseURL string, cfg Config) *LLMPhraser {
|
||||
return &LLMPhraser{
|
||||
cfg: cfg,
|
||||
client: &http.Client{Timeout: cfg.Timeout},
|
||||
port: strings.TrimSuffix(baseURL, "/"),
|
||||
cancel: func() {},
|
||||
tmpl: loadNudgeTemplates(),
|
||||
cfg: cfg,
|
||||
client: &http.Client{Timeout: cfg.Timeout},
|
||||
tmpl: loadNudgeTemplates(),
|
||||
spawnCtx: context.Background(),
|
||||
cancel: func() {},
|
||||
probe: defaultProbe,
|
||||
// launch stays nil: we did not start this server, so we must not stop it.
|
||||
// Swap therefore refuses here (ErrSwapNotOwned) instead of killing a
|
||||
// server another process depends on.
|
||||
be: borrowedBackend(strings.TrimSuffix(baseURL, "/")),
|
||||
live: liveModel{ModelPath: cfg.ModelPath, NGpuLayers: cfg.NGpuLayers, NCtx: cfg.NCtx},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -126,16 +170,66 @@ func loadNudgeTemplates() *NudgeTemplates {
|
||||
return nt
|
||||
}
|
||||
|
||||
func (p *LLMPhraser) start(ctx context.Context) error {
|
||||
// backend — one llama-server this phraser talks to. Two implementations: a
|
||||
// llamaProc we spawned and must reap, and a borrowedBackend someone else owns.
|
||||
type backend interface {
|
||||
BaseURL() string
|
||||
Close() error
|
||||
}
|
||||
|
||||
// borrowedBackend — a server started and owned by someone else (the phrasing
|
||||
// scorer's shared llama-server). Closing it is a no-op by construction.
|
||||
type borrowedBackend string
|
||||
|
||||
func (b borrowedBackend) BaseURL() string { return string(b) }
|
||||
func (b borrowedBackend) Close() error { return nil }
|
||||
|
||||
// llamaProc — a llama-server child process plus the goroutine reading its
|
||||
// stderr. Close kills and reaps it; see the Pdeathsig note in spawnLlamaServer.
|
||||
type llamaProc struct {
|
||||
base string
|
||||
cmd *exec.Cmd
|
||||
cancel context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
func (l *llamaProc) BaseURL() string { return l.base }
|
||||
|
||||
func (l *llamaProc) Close() error {
|
||||
l.cancel()
|
||||
if l.cmd != nil && l.cmd.Process != nil {
|
||||
_ = l.cmd.Process.Kill()
|
||||
_ = l.cmd.Wait() // reap the process — without Wait, the child becomes a zombie
|
||||
}
|
||||
l.wg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
// spawnLlamaServer starts one llama-server for cfg and waits until it says which
|
||||
// address it is listening on. ctx owns the process lifetime, so it must be the
|
||||
// daemon's context, not a request's.
|
||||
func spawnLlamaServer(ctx context.Context, cfg Config) (backend, error) {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
p, err := startLlamaProc(ctx, cfg)
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, err
|
||||
}
|
||||
p.cancel = cancel
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func startLlamaProc(ctx context.Context, cfg Config) (*llamaProc, error) {
|
||||
p := &llamaProc{}
|
||||
args := []string{
|
||||
"-m", p.cfg.ModelPath,
|
||||
"-m", cfg.ModelPath,
|
||||
"--host", "127.0.0.1",
|
||||
"--port", extractPort(p.cfg.Listen),
|
||||
"-c", fmt.Sprintf("%d", p.cfg.NCtx),
|
||||
"-ngl", fmt.Sprintf("%d", p.cfg.NGpuLayers),
|
||||
"--port", extractPort(cfg.Listen),
|
||||
"-c", fmt.Sprintf("%d", cfg.NCtx),
|
||||
"-ngl", fmt.Sprintf("%d", cfg.NGpuLayers),
|
||||
"--no-webui",
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, p.cfg.BinPath, args...)
|
||||
cmd := exec.CommandContext(ctx, cfg.BinPath, args...)
|
||||
// Pdeathsig: the kernel SIGKILLs llama-server the moment mavend dies — by
|
||||
// ANY means, including SIGKILL/OOM/panic where our Close() never runs. Without
|
||||
// it a hard-killed mavend orphans its llama-server (reparented to init, keeps
|
||||
@@ -148,12 +242,12 @@ func (p *LLMPhraser) start(ctx context.Context) error {
|
||||
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return fmt.Errorf("llm: stderr pipe: %w", err)
|
||||
return nil, fmt.Errorf("llm: stderr pipe: %w", err)
|
||||
}
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
stderr.Close()
|
||||
return fmt.Errorf("llm: start: %w", err)
|
||||
return nil, fmt.Errorf("llm: start: %w", err)
|
||||
}
|
||||
|
||||
portCh := make(chan string, 1)
|
||||
@@ -186,32 +280,44 @@ func (p *LLMPhraser) start(ctx context.Context) error {
|
||||
|
||||
select {
|
||||
case addr := <-portCh:
|
||||
p.port = addr
|
||||
return nil
|
||||
p.base = addr
|
||||
return p, nil
|
||||
case err := <-errCh:
|
||||
_ = cmd.Process.Kill()
|
||||
_ = cmd.Wait()
|
||||
return fmt.Errorf("llm: server output: %w", err)
|
||||
return nil, fmt.Errorf("llm: server output: %w", err)
|
||||
case <-ctx.Done():
|
||||
_ = cmd.Process.Kill()
|
||||
_ = cmd.Wait()
|
||||
return ctx.Err()
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(60 * time.Second):
|
||||
_ = cmd.Process.Kill()
|
||||
_ = cmd.Wait()
|
||||
return fmt.Errorf("llm: server did not start within 60s")
|
||||
return nil, fmt.Errorf("llm: server did not start within 60s")
|
||||
}
|
||||
}
|
||||
|
||||
func (p *LLMPhraser) BaseURL() string { return p.port }
|
||||
// BaseURL is the llama-server this phraser talks to right now. It changes when
|
||||
// the model is swapped, so callers that cache it must register an observer
|
||||
// (OnSwap) rather than keeping the string forever.
|
||||
func (p *LLMPhraser) BaseURL() string {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.be == nil {
|
||||
return ""
|
||||
}
|
||||
return p.be.BaseURL()
|
||||
}
|
||||
|
||||
func (p *LLMPhraser) Close() error {
|
||||
p.cancel()
|
||||
if p.cmd != nil && p.cmd.Process != nil {
|
||||
_ = p.cmd.Process.Kill()
|
||||
_ = p.cmd.Wait() // reap the process — without Wait, the child becomes a zombie
|
||||
p.mu.Lock()
|
||||
be := p.be
|
||||
p.be = nil
|
||||
p.mu.Unlock()
|
||||
if be != nil {
|
||||
return be.Close()
|
||||
}
|
||||
p.wg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -350,6 +456,11 @@ func chatSystemPrompt(block func() string) string {
|
||||
// the LLM completion endpoint. Like chatWithSystem but for an arbitrary message
|
||||
// slice — the caller owns the system prompt placement.
|
||||
func (p *LLMPhraser) chatWithMessages(ctx context.Context, msgs []chatMsg, maxTokens int) (string, error) {
|
||||
base, release, err := p.acquire()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer release()
|
||||
req := chatReq{
|
||||
Messages: msgs,
|
||||
Temperature: 0.7,
|
||||
@@ -360,7 +471,7 @@ func (p *LLMPhraser) chatWithMessages(ctx context.Context, msgs []chatMsg, maxTo
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("llm: marshal: %w", err)
|
||||
}
|
||||
httpReq, err := http.NewRequestWithContext(ctx, "POST", p.port+"/v1/chat/completions", bytes.NewReader(body))
|
||||
httpReq, err := http.NewRequestWithContext(ctx, "POST", base+"/v1/chat/completions", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("llm: request: %w", err)
|
||||
}
|
||||
@@ -493,6 +604,11 @@ func (p *LLMPhraser) chat(ctx context.Context, userPrompt string) (string, error
|
||||
}
|
||||
|
||||
func (p *LLMPhraser) chatWithSystem(ctx context.Context, system, user string, maxTokens int) (string, error) {
|
||||
base, release, err := p.acquire()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer release()
|
||||
req := chatReq{
|
||||
Messages: []chatMsg{
|
||||
{Role: "system", Content: system},
|
||||
@@ -507,7 +623,7 @@ func (p *LLMPhraser) chatWithSystem(ctx context.Context, system, user string, ma
|
||||
return "", fmt.Errorf("llm: marshal: %w", err)
|
||||
}
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(ctx, "POST", p.port+"/v1/chat/completions", bytes.NewReader(body))
|
||||
httpReq, err := http.NewRequestWithContext(ctx, "POST", base+"/v1/chat/completions", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("llm: request: %w", err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,330 @@
|
||||
package phraser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/llm"
|
||||
)
|
||||
|
||||
// Swapping the resident model without restarting the daemon (Vikunja #250).
|
||||
//
|
||||
// Three properties this file exists to hold, in order of importance:
|
||||
//
|
||||
// 1. NEVER two models resident at once. The deploy target is a laptop iGPU
|
||||
// with the whole 1.7B offloaded to it (`n_gpu_layers: 99`); loading a second
|
||||
// model beside the first is how you OOM the box, and a blue/green swap that
|
||||
// "keeps the old one warm until the new one answers" does exactly that. So
|
||||
// the old server is killed FIRST and the new one loaded after. The cost of
|
||||
// that ordering is a window with no model at all, which is why:
|
||||
//
|
||||
// 2. A swap is atomic from a turn's point of view. An in-flight turn finishes
|
||||
// on the old model — Swap waits for the last one to return before killing
|
||||
// anything. A turn that arrives during the swap is REFUSED immediately with
|
||||
// ErrSwapping rather than blocked: every phrasing path already has a
|
||||
// fallback (templates, "вот что я нашла", the classifier for routing), so a
|
||||
// fast refusal degrades one turn instead of hanging it for the length of a
|
||||
// model load. No turn ever gets half of one model and half of another.
|
||||
//
|
||||
// 3. A failed load rolls back to the model that was working. The new server is
|
||||
// probed (it must say which model it loaded) before it is published; if the
|
||||
// launch or the probe fails, the previous config is relaunched and the
|
||||
// phraser goes back to serving. Only if the rollback ALSO fails is the
|
||||
// phraser left without a backend, and then it says so loudly and every turn
|
||||
// degrades rather than breaks.
|
||||
//
|
||||
// Not here, deliberately: nothing calls Swap on a timer, and no act or intent can
|
||||
// reach it. It is an IPC method behind the step-up gate, i.e. owner-triggered.
|
||||
|
||||
var (
|
||||
// ErrSwapping — a turn arrived while the model was being swapped. Callers
|
||||
// treat it like any other LLM error and use their fallback.
|
||||
ErrSwapping = errors.New("phraser: model swap in progress")
|
||||
|
||||
// ErrSwapNotOwned — this phraser did not start its llama-server, so it must
|
||||
// not stop one (NewLLMPhraserAt: the eval harness shares a server).
|
||||
ErrSwapNotOwned = errors.New("phraser: llama-server is not ours to swap")
|
||||
|
||||
// ErrNoBackend — no model is loaded at all. Only reachable after a failed
|
||||
// swap whose rollback also failed.
|
||||
ErrNoBackend = errors.New("phraser: no llama-server loaded")
|
||||
|
||||
// ErrSwapBusy — a turn was still running when the drain deadline expired, so
|
||||
// the swap was abandoned. Nothing was killed; ask again.
|
||||
ErrSwapBusy = errors.New("phraser: turns still in flight, swap abandoned")
|
||||
)
|
||||
|
||||
// SwapSpec — what to load. Zero NGpuLayers/NCtx keep whatever is live, so the
|
||||
// common case ("same settings, different gguf") is one field.
|
||||
type SwapSpec struct {
|
||||
ModelPath string
|
||||
NGpuLayers int
|
||||
NCtx int
|
||||
}
|
||||
|
||||
// SwapResult — what happened. Model is the identity the NEW server reported, so
|
||||
// it is evidence rather than an echo of the request: if the file at ModelPath is
|
||||
// not what the operator thought it was, this is where that shows up.
|
||||
type SwapResult struct {
|
||||
Model string
|
||||
BaseURL string
|
||||
ModelPath string
|
||||
RolledBack bool
|
||||
Took time.Duration
|
||||
}
|
||||
|
||||
// drainTimeout — how long Swap waits for in-flight turns before giving up. A
|
||||
// turn is at most Config.Timeout (30s in deploy) plus the model's own latency;
|
||||
// 90s covers a slow Thinking generation without wedging the caller forever.
|
||||
const drainTimeout = 90 * time.Second
|
||||
|
||||
// probeTimeout — how long the new server gets to answer "which model do you
|
||||
// have". The load itself is bounded by spawnLlamaServer's own 60s wait.
|
||||
const probeTimeout = 30 * time.Second
|
||||
|
||||
// defaultProbe asks the server which model it has loaded. This is the health
|
||||
// check: a server that answers /v1/models has finished loading weights and is
|
||||
// serving, and its answer is the identity we report back.
|
||||
func defaultProbe(ctx context.Context, base string) (string, error) {
|
||||
return llm.ModelID(ctx, base)
|
||||
}
|
||||
|
||||
// OnSwap registers a callback fired with the new base URL every time the live
|
||||
// backend changes, including after a rollback. Holders of an *llm.Client (the
|
||||
// LLM router, the replier, the mail extractor) register SetBaseURL here so a
|
||||
// swap re-points them without rebuilding the router or the handler.
|
||||
//
|
||||
// Callbacks run with no lock held, in registration order.
|
||||
func (p *LLMPhraser) OnSwap(fn func(baseURL string)) {
|
||||
if fn == nil {
|
||||
return
|
||||
}
|
||||
p.mu.Lock()
|
||||
p.observers = append(p.observers, fn)
|
||||
p.mu.Unlock()
|
||||
}
|
||||
|
||||
// LiveModel is the model file currently loaded (and its load settings). Empty
|
||||
// ModelPath means no model is loaded.
|
||||
func (p *LLMPhraser) LiveModel() (path string, nGpuLayers, nCtx int) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return p.live.ModelPath, p.live.NGpuLayers, p.live.NCtx
|
||||
}
|
||||
|
||||
// acquire reserves a slot for one request and returns the base URL to use.
|
||||
// Every request path must call it and must call the returned release exactly
|
||||
// once — that count is what Swap drains.
|
||||
func (p *LLMPhraser) acquire() (string, func(), error) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.swapping {
|
||||
return "", nil, ErrSwapping
|
||||
}
|
||||
if p.be == nil {
|
||||
return "", nil, ErrNoBackend
|
||||
}
|
||||
p.inflight++
|
||||
base := p.be.BaseURL()
|
||||
var once bool
|
||||
return base, func() {
|
||||
p.mu.Lock()
|
||||
if !once {
|
||||
once = true
|
||||
p.inflight--
|
||||
}
|
||||
p.mu.Unlock()
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Swap loads another model in place of the live one. See the file comment for
|
||||
// the properties it guarantees. Returns the new model's reported identity, or
|
||||
// an error plus RolledBack=true when the old model was put back.
|
||||
//
|
||||
// ctx bounds the drain and the probe. It does NOT own the new server's lifetime
|
||||
// — that is the daemon's context, captured at construction — so a swap survives
|
||||
// the request that asked for it.
|
||||
func (p *LLMPhraser) Swap(ctx context.Context, spec SwapSpec) (SwapResult, error) {
|
||||
if spec.ModelPath == "" {
|
||||
return SwapResult{}, fmt.Errorf("phraser: swap needs a model path")
|
||||
}
|
||||
p.swapMu.Lock()
|
||||
defer p.swapMu.Unlock()
|
||||
|
||||
if p.launch == nil {
|
||||
return SwapResult{}, ErrSwapNotOwned
|
||||
}
|
||||
|
||||
started := time.Now()
|
||||
oldLive := p.liveSnapshot()
|
||||
newLive := liveModel{
|
||||
ModelPath: spec.ModelPath,
|
||||
NGpuLayers: pickInt(spec.NGpuLayers, oldLive.NGpuLayers),
|
||||
NCtx: pickInt(spec.NCtx, oldLive.NCtx),
|
||||
}
|
||||
if newLive == oldLive && p.BaseURL() != "" {
|
||||
// Already serving exactly this. Report the live identity rather than
|
||||
// pointlessly unloading and reloading the same weights.
|
||||
base := p.BaseURL()
|
||||
id, err := p.probeWith(ctx, base)
|
||||
if err != nil {
|
||||
return SwapResult{}, err
|
||||
}
|
||||
return SwapResult{Model: id, BaseURL: base, ModelPath: oldLive.ModelPath, Took: time.Since(started)}, nil
|
||||
}
|
||||
|
||||
if err := p.quiesce(ctx); err != nil {
|
||||
return SwapResult{}, err
|
||||
}
|
||||
defer p.resume()
|
||||
|
||||
// Property 1: the old model leaves the GPU before the new one arrives.
|
||||
p.mu.Lock()
|
||||
old := p.be
|
||||
p.be = nil
|
||||
p.mu.Unlock()
|
||||
if old != nil {
|
||||
_ = old.Close()
|
||||
}
|
||||
|
||||
be, err := p.loadAndProbe(ctx, newLive)
|
||||
if err != nil {
|
||||
log.Printf("phraser: swap to %s FAILED (%v) — rolling back to %s", newLive.ModelPath, err, oldLive.ModelPath)
|
||||
rb, rbErr := p.loadAndProbe(ctx, oldLive)
|
||||
if rbErr != nil {
|
||||
log.Printf("phraser: ROLLBACK to %s ALSO FAILED (%v) — no model is loaded, every phrasing path is on its fallback and routing is on the classifier until the daemon is restarted", oldLive.ModelPath, rbErr)
|
||||
return SwapResult{RolledBack: true, Took: time.Since(started)},
|
||||
fmt.Errorf("phraser: swap failed (%w) and rollback failed too: %v", err, rbErr)
|
||||
}
|
||||
p.publish(rb, oldLive)
|
||||
return SwapResult{
|
||||
Model: rb.id, BaseURL: rb.be.BaseURL(), ModelPath: oldLive.ModelPath,
|
||||
RolledBack: true, Took: time.Since(started),
|
||||
},
|
||||
fmt.Errorf("phraser: swap to %s failed, rolled back to %s: %w", newLive.ModelPath, oldLive.ModelPath, err)
|
||||
}
|
||||
p.publish(be, newLive)
|
||||
log.Printf("phraser: model swapped to %s (%s) at %s in %s", newLive.ModelPath, be.id, be.be.BaseURL(), time.Since(started).Round(time.Millisecond))
|
||||
return SwapResult{
|
||||
Model: be.id, BaseURL: be.be.BaseURL(), ModelPath: newLive.ModelPath,
|
||||
Took: time.Since(started),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// loaded — a started server plus the identity it reported.
|
||||
type loaded struct {
|
||||
be backend
|
||||
id string
|
||||
}
|
||||
|
||||
// loadAndProbe starts a server for lm and verifies it answers. A server that
|
||||
// starts but will not say what it loaded is treated as a failed load and is
|
||||
// killed here — publishing it would hand every turn to a backend we could not
|
||||
// confirm.
|
||||
func (p *LLMPhraser) loadAndProbe(ctx context.Context, lm liveModel) (loaded, error) {
|
||||
cfg := p.cfg
|
||||
cfg.ModelPath = lm.ModelPath
|
||||
cfg.NGpuLayers = lm.NGpuLayers
|
||||
cfg.NCtx = lm.NCtx
|
||||
// p.spawnCtx, not ctx: the process must outlive the request asking for it.
|
||||
be, err := p.launch(p.spawnCtx, cfg)
|
||||
if err != nil {
|
||||
return loaded{}, err
|
||||
}
|
||||
id, err := p.probeWith(ctx, be.BaseURL())
|
||||
if err != nil {
|
||||
_ = be.Close()
|
||||
return loaded{}, fmt.Errorf("phraser: %s started but would not answer: %w", lm.ModelPath, err)
|
||||
}
|
||||
return loaded{be: be, id: id}, nil
|
||||
}
|
||||
|
||||
func (p *LLMPhraser) probeWith(ctx context.Context, base string) (string, error) {
|
||||
probe := p.probe
|
||||
if probe == nil {
|
||||
probe = defaultProbe
|
||||
}
|
||||
pctx, cancel := context.WithTimeout(ctx, probeTimeout)
|
||||
defer cancel()
|
||||
return probe(pctx, base)
|
||||
}
|
||||
|
||||
// quiesce closes the door on new turns and waits for the ones already running.
|
||||
// Polling rather than a sync.Cond: the wait happens once per swap, a 25ms poll
|
||||
// is invisible next to a model load, and a poll cannot deadlock on a release
|
||||
// path that panicked.
|
||||
func (p *LLMPhraser) quiesce(ctx context.Context) error {
|
||||
p.mu.Lock()
|
||||
if p.swapping {
|
||||
p.mu.Unlock()
|
||||
return ErrSwapping
|
||||
}
|
||||
p.swapping = true
|
||||
inflight := p.inflight
|
||||
p.mu.Unlock()
|
||||
if inflight == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(drainTimeout)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
p.resume()
|
||||
return ctx.Err()
|
||||
case <-time.After(25 * time.Millisecond):
|
||||
}
|
||||
p.mu.Lock()
|
||||
inflight = p.inflight
|
||||
p.mu.Unlock()
|
||||
if inflight == 0 {
|
||||
return nil
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
// Nothing has been killed yet, so abandoning is free: reopen the door
|
||||
// and let the operator try again rather than cutting a live turn off
|
||||
// mid-generation.
|
||||
p.resume()
|
||||
return fmt.Errorf("%w (%d still running after %s)", ErrSwapBusy, inflight, drainTimeout)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *LLMPhraser) resume() {
|
||||
p.mu.Lock()
|
||||
p.swapping = false
|
||||
p.mu.Unlock()
|
||||
}
|
||||
|
||||
// publish makes l the live backend and tells everyone holding a base URL.
|
||||
func (p *LLMPhraser) publish(l loaded, lm liveModel) {
|
||||
p.mu.Lock()
|
||||
p.be = l.be
|
||||
p.live = lm
|
||||
obs := make([]func(string), len(p.observers))
|
||||
copy(obs, p.observers)
|
||||
p.mu.Unlock()
|
||||
base := l.be.BaseURL()
|
||||
for _, fn := range obs {
|
||||
fn(base)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *LLMPhraser) liveSnapshot() liveModel {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return p.live
|
||||
}
|
||||
|
||||
// pickInt returns v when the caller set it, and fallback otherwise. 0 is the
|
||||
// "unset" value: -1 already means "offload every layer" and deploy uses 99, so
|
||||
// nothing legitimate asks for exactly zero GPU layers through this path.
|
||||
func pickInt(v, fallback int) int {
|
||||
if v == 0 {
|
||||
return fallback
|
||||
}
|
||||
return v
|
||||
}
|
||||
@@ -0,0 +1,311 @@
|
||||
package phraser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fakeModel — a stand-in llama-server. It answers /v1/models with its own name
|
||||
// and /v1/chat/completions with a phrasing-contract reply that names itself, so
|
||||
// a test can tell WHICH model answered a turn — the property the swap is about.
|
||||
type fakeModel struct {
|
||||
srv *httptest.Server
|
||||
name string
|
||||
closed atomic.Bool
|
||||
}
|
||||
|
||||
func newFakeModel(t *testing.T, name string) *fakeModel {
|
||||
t.Helper()
|
||||
f := &fakeModel{name: name}
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/v1/models", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(`{"data":[{"id":"/models/` + name + `.gguf"}]}`))
|
||||
})
|
||||
mux.HandleFunc("/v1/chat/completions", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(`{"choices":[{"message":{"content":"{\"response\":\"` + name + `\",\"mood\":\"neutral\"}"}}]}`))
|
||||
})
|
||||
f.srv = httptest.NewServer(mux)
|
||||
t.Cleanup(f.srv.Close)
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *fakeModel) BaseURL() string { return f.srv.URL }
|
||||
func (f *fakeModel) Close() error { f.closed.Store(true); return nil }
|
||||
|
||||
// fakeFleet is the injected launcher: it hands out a prepared fakeModel per
|
||||
// model path, and refuses paths the test did not prepare (that is what a bad
|
||||
// gguf looks like from here). It also asserts the invariant that matters on a
|
||||
// laptop iGPU: never two servers alive at the same time.
|
||||
type fakeFleet struct {
|
||||
mu sync.Mutex
|
||||
models map[string]string // model path → fake name
|
||||
live int
|
||||
maxLive int
|
||||
launch int
|
||||
}
|
||||
|
||||
func (fl *fakeFleet) launcher(t *testing.T) func(context.Context, Config) (backend, error) {
|
||||
return func(ctx context.Context, cfg Config) (backend, error) {
|
||||
fl.mu.Lock()
|
||||
name, ok := fl.models[cfg.ModelPath]
|
||||
fl.launch++
|
||||
if !ok {
|
||||
fl.mu.Unlock()
|
||||
return nil, errors.New("no such model file: " + cfg.ModelPath)
|
||||
}
|
||||
fl.live++
|
||||
if fl.live > fl.maxLive {
|
||||
fl.maxLive = fl.live
|
||||
}
|
||||
fl.mu.Unlock()
|
||||
f := newFakeModel(t, name)
|
||||
return &fleetBackend{fleet: fl, model: f}, nil
|
||||
}
|
||||
}
|
||||
|
||||
type fleetBackend struct {
|
||||
fleet *fakeFleet
|
||||
model *fakeModel
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func (b *fleetBackend) BaseURL() string { return b.model.BaseURL() }
|
||||
func (b *fleetBackend) Close() error {
|
||||
b.once.Do(func() {
|
||||
b.fleet.mu.Lock()
|
||||
b.fleet.live--
|
||||
b.fleet.mu.Unlock()
|
||||
})
|
||||
return b.model.Close()
|
||||
}
|
||||
|
||||
// newSwapPhraser builds an LLMPhraser with an injected launcher, so the swap
|
||||
// path is exercised without a gguf or a GPU.
|
||||
func newSwapPhraser(t *testing.T, fl *fakeFleet, modelPath string) *LLMPhraser {
|
||||
t.Helper()
|
||||
cfg := DefaultConfig(modelPath)
|
||||
cfg.Timeout = 5 * time.Second
|
||||
p := &LLMPhraser{
|
||||
cfg: cfg,
|
||||
client: &http.Client{Timeout: cfg.Timeout},
|
||||
spawnCtx: context.Background(),
|
||||
cancel: func() {},
|
||||
launch: fl.launcher(t),
|
||||
probe: defaultProbe,
|
||||
live: liveModel{ModelPath: modelPath, NGpuLayers: cfg.NGpuLayers, NCtx: cfg.NCtx},
|
||||
}
|
||||
be, err := p.launch(p.spawnCtx, cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("initial launch: %v", err)
|
||||
}
|
||||
p.be = be
|
||||
t.Cleanup(func() { p.Close() })
|
||||
return p
|
||||
}
|
||||
|
||||
func TestSwap_LoadsNewModelAndRepointsHolders(t *testing.T) {
|
||||
fl := &fakeFleet{models: map[string]string{"/m/old.gguf": "old", "/m/new.gguf": "new"}}
|
||||
p := newSwapPhraser(t, fl, "/m/old.gguf")
|
||||
|
||||
// A holder of the base URL (the LLM router's client, in the daemon).
|
||||
var seen []string
|
||||
p.OnSwap(func(base string) { seen = append(seen, base) })
|
||||
|
||||
before, err := p.PhraseChat(context.Background(), "привет", nil)
|
||||
if err != nil || before != "old" {
|
||||
t.Fatalf("before swap: %q, %v; want the old model to answer", before, err)
|
||||
}
|
||||
|
||||
res, err := p.Swap(context.Background(), SwapSpec{ModelPath: "/m/new.gguf"})
|
||||
if err != nil {
|
||||
t.Fatalf("Swap: %v", err)
|
||||
}
|
||||
if res.Model != "new" {
|
||||
t.Errorf("res.Model = %q; want the identity the NEW server reported (%q)", res.Model, "new")
|
||||
}
|
||||
if res.RolledBack {
|
||||
t.Errorf("res.RolledBack = true on a successful swap")
|
||||
}
|
||||
after, err := p.PhraseChat(context.Background(), "привет", nil)
|
||||
if err != nil || after != "new" {
|
||||
t.Fatalf("after swap: %q, %v; want the new model to answer", after, err)
|
||||
}
|
||||
if path, _, _ := p.LiveModel(); path != "/m/new.gguf" {
|
||||
t.Errorf("LiveModel = %q; want /m/new.gguf", path)
|
||||
}
|
||||
if len(seen) != 1 || seen[0] != p.BaseURL() {
|
||||
t.Errorf("observers saw %v; want exactly one call with the new base %q", seen, p.BaseURL())
|
||||
}
|
||||
if fl.maxLive > 1 {
|
||||
t.Errorf("%d servers were alive at once; the iGPU only fits one model", fl.maxLive)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSwap_FailedLoadRollsBackToTheWorkingModel(t *testing.T) {
|
||||
fl := &fakeFleet{models: map[string]string{"/m/old.gguf": "old"}}
|
||||
p := newSwapPhraser(t, fl, "/m/old.gguf")
|
||||
|
||||
res, err := p.Swap(context.Background(), SwapSpec{ModelPath: "/m/broken.gguf"})
|
||||
if err == nil {
|
||||
t.Fatal("Swap to a model that will not load returned nil error")
|
||||
}
|
||||
if !res.RolledBack {
|
||||
t.Errorf("res.RolledBack = false; a failed swap must say it rolled back")
|
||||
}
|
||||
if res.Model != "old" {
|
||||
t.Errorf("res.Model = %q; want the old model back", res.Model)
|
||||
}
|
||||
// The point of the rollback: turns keep working.
|
||||
got, err := p.PhraseChat(context.Background(), "привет", nil)
|
||||
if err != nil || got != "old" {
|
||||
t.Fatalf("after rollback: %q, %v; want the old model serving again", got, err)
|
||||
}
|
||||
if path, _, _ := p.LiveModel(); path != "/m/old.gguf" {
|
||||
t.Errorf("LiveModel = %q; want the old model", path)
|
||||
}
|
||||
if fl.maxLive > 1 {
|
||||
t.Errorf("%d servers alive at once during a rollback", fl.maxLive)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSwap_ProbeFailureIsTreatedAsAFailedLoad(t *testing.T) {
|
||||
// A server that starts but will not say what it loaded must never be
|
||||
// published — we would be serving turns from a backend we cannot confirm.
|
||||
fl := &fakeFleet{models: map[string]string{"/m/old.gguf": "old", "/m/mute.gguf": "mute"}}
|
||||
p := newSwapPhraser(t, fl, "/m/old.gguf")
|
||||
// Fail the probe once — for the newly launched server — and let the
|
||||
// rollback's probe through.
|
||||
calls := 0
|
||||
p.probe = func(ctx context.Context, base string) (string, error) {
|
||||
calls++
|
||||
if calls == 1 {
|
||||
return "", errors.New("no answer from the new server")
|
||||
}
|
||||
return defaultProbe(ctx, base)
|
||||
}
|
||||
|
||||
_, err := p.Swap(context.Background(), SwapSpec{ModelPath: "/m/mute.gguf"})
|
||||
if err == nil {
|
||||
t.Fatal("Swap published a server that failed its probe")
|
||||
}
|
||||
if path, _, _ := p.LiveModel(); path != "/m/old.gguf" {
|
||||
t.Errorf("LiveModel = %q; want the old model after a failed probe", path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSwap_RollbackFailureLeavesNoBackendAndDegrades(t *testing.T) {
|
||||
fl := &fakeFleet{models: map[string]string{"/m/old.gguf": "old"}}
|
||||
p := newSwapPhraser(t, fl, "/m/old.gguf")
|
||||
// Make the rollback fail too: the old file "disappears" mid-swap.
|
||||
fl.mu.Lock()
|
||||
delete(fl.models, "/m/old.gguf")
|
||||
fl.mu.Unlock()
|
||||
|
||||
_, err := p.Swap(context.Background(), SwapSpec{ModelPath: "/m/broken.gguf"})
|
||||
if err == nil {
|
||||
t.Fatal("Swap returned nil when both the load and the rollback failed")
|
||||
}
|
||||
// Nothing is loaded, and the request path says so rather than panicking.
|
||||
if _, _, aerr := p.acquire(); !errors.Is(aerr, ErrNoBackend) {
|
||||
t.Errorf("acquire error = %v; want ErrNoBackend", aerr)
|
||||
}
|
||||
// Phrasing degrades to its fallback instead of failing the turn.
|
||||
got, err := p.PhraseChat(context.Background(), "привет", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("PhraseChat after a total failure returned an error: %v", err)
|
||||
}
|
||||
if got == "" {
|
||||
t.Error("PhraseChat returned empty; the fallback must still say something")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSwap_WaitsForInFlightTurnAndRefusesNewOnes(t *testing.T) {
|
||||
fl := &fakeFleet{models: map[string]string{"/m/old.gguf": "old", "/m/new.gguf": "new"}}
|
||||
p := newSwapPhraser(t, fl, "/m/old.gguf")
|
||||
|
||||
// Hold one turn open by taking a slot directly — the same slot every
|
||||
// request path takes.
|
||||
base, release, err := p.acquire()
|
||||
if err != nil {
|
||||
t.Fatalf("acquire: %v", err)
|
||||
}
|
||||
if base == "" {
|
||||
t.Fatal("acquire returned an empty base URL")
|
||||
}
|
||||
|
||||
swapped := make(chan error, 1)
|
||||
go func() { _, e := p.Swap(context.Background(), SwapSpec{ModelPath: "/m/new.gguf"}); swapped <- e }()
|
||||
|
||||
// While the swap waits to drain, a NEW turn is refused immediately rather
|
||||
// than blocked for the length of a model load.
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for {
|
||||
_, rel, aerr := p.acquire()
|
||||
if rel != nil {
|
||||
rel()
|
||||
}
|
||||
if errors.Is(aerr, ErrSwapping) {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("new turns were never refused during a swap (last error: %v)", aerr)
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
|
||||
// The swap cannot have completed while our turn was still in flight.
|
||||
select {
|
||||
case e := <-swapped:
|
||||
t.Fatalf("Swap finished before the in-flight turn released: %v", e)
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
|
||||
release()
|
||||
if e := <-swapped; e != nil {
|
||||
t.Fatalf("Swap after drain: %v", e)
|
||||
}
|
||||
got, err := p.PhraseChat(context.Background(), "привет", nil)
|
||||
if err != nil || got != "new" {
|
||||
t.Fatalf("after swap: %q, %v; want the new model", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSwap_RefusedWhenWeDoNotOwnTheServer(t *testing.T) {
|
||||
// NewLLMPhraserAt points at a shared server the eval harness owns. Swapping
|
||||
// there would kill a server another process depends on.
|
||||
p := NewLLMPhraserAt("http://127.0.0.1:1/", DefaultConfig("/m/old.gguf"))
|
||||
if _, err := p.Swap(context.Background(), SwapSpec{ModelPath: "/m/new.gguf"}); !errors.Is(err, ErrSwapNotOwned) {
|
||||
t.Fatalf("Swap on a borrowed server = %v; want ErrSwapNotOwned", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSwap_SameModelIsANoOp(t *testing.T) {
|
||||
fl := &fakeFleet{models: map[string]string{"/m/old.gguf": "old"}}
|
||||
p := newSwapPhraser(t, fl, "/m/old.gguf")
|
||||
launchesBefore := fl.launch
|
||||
|
||||
res, err := p.Swap(context.Background(), SwapSpec{ModelPath: "/m/old.gguf"})
|
||||
if err != nil {
|
||||
t.Fatalf("Swap to the live model: %v", err)
|
||||
}
|
||||
if res.Model != "old" {
|
||||
t.Errorf("res.Model = %q; want old", res.Model)
|
||||
}
|
||||
if fl.launch != launchesBefore {
|
||||
t.Errorf("%d extra launches; swapping to the live model must not reload weights", fl.launch-launchesBefore)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSwap_EmptyModelPathRefused(t *testing.T) {
|
||||
fl := &fakeFleet{models: map[string]string{"/m/old.gguf": "old"}}
|
||||
p := newSwapPhraser(t, fl, "/m/old.gguf")
|
||||
if _, err := p.Swap(context.Background(), SwapSpec{}); err == nil {
|
||||
t.Fatal("Swap with no model path returned nil error")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package router
|
||||
|
||||
import "strings"
|
||||
|
||||
// Feed queries — "что нового в лентах?", "что нового по технологиям?"
|
||||
// (Vikunja #258).
|
||||
//
|
||||
// Deterministic matching, like the calendar, plan and habit matchers above it:
|
||||
// the LLM router says this is a query, and this decides whether it is a question
|
||||
// about the feeds. A model deciding that would occasionally answer "что нового?"
|
||||
// out of world knowledge, which is the one thing a feed reader exists to avoid.
|
||||
|
||||
// FeedQuery — a parsed "what's new" question. Category is the topic he named
|
||||
// ("технологии"), empty when he asked about the feeds in general.
|
||||
type FeedQuery struct {
|
||||
Category string
|
||||
}
|
||||
|
||||
// feedNouns — the words that make a question about the feeds themselves.
|
||||
var feedNouns = []string{
|
||||
"лента", "ленте", "ленты", "лентах", "лентам",
|
||||
"новости", "новостей", "новостях", "новостям",
|
||||
"новое", "нового", "новенького",
|
||||
"feed", "feeds", "news", "headlines",
|
||||
}
|
||||
|
||||
// newnessMarkers — the "что нового" half. "нового" alone is in feedNouns
|
||||
// because it carries the question on its own ("что нового?"); a bare "лента"
|
||||
// needs the ask, which is what askMarkers below is for.
|
||||
var askMarkers = []string{
|
||||
"что", "какие", "какое", "расскажи", "почитай", "прочитай", "покажи",
|
||||
"what", "any", "tell", "show", "read",
|
||||
}
|
||||
|
||||
// ParseFeedQuery reports whether an utterance asks what is new in the feeds, and
|
||||
// which topic if it names one after "по"/"о"/"про"/"about".
|
||||
//
|
||||
// Both a feed noun and an ask are required. "у меня новая лента в инстаграме" is
|
||||
// a statement and must not be read as a request to recite headlines.
|
||||
func ParseFeedQuery(text string) (FeedQuery, bool) {
|
||||
toks := planTokens(text)
|
||||
noun, ask := false, false
|
||||
for _, t := range toks {
|
||||
for _, n := range feedNouns {
|
||||
if t == n {
|
||||
noun = true
|
||||
break
|
||||
}
|
||||
}
|
||||
for _, a := range askMarkers {
|
||||
if t == a {
|
||||
ask = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if !noun || !ask {
|
||||
return FeedQuery{}, false
|
||||
}
|
||||
return FeedQuery{Category: feedCategory(toks)}, true
|
||||
}
|
||||
|
||||
// categoryPreps — the prepositions a topic follows. Russian marks the topic with
|
||||
// a preposition ("по технологиям", "про политику"), so the word after one is the
|
||||
// category; there is no stemming here, and the match against the configured
|
||||
// category is a prefix comparison for exactly that reason.
|
||||
var categoryPreps = map[string]bool{"по": true, "о": true, "об": true, "про": true, "about": true, "on": true}
|
||||
|
||||
func feedCategory(toks []string) string {
|
||||
for i, t := range toks {
|
||||
if categoryPreps[t] && i+1 < len(toks) {
|
||||
next := toks[i+1]
|
||||
// "по новостям" names no topic, it repeats the noun.
|
||||
for _, n := range feedNouns {
|
||||
if next == n {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
return next
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// CategoryMatches reports whether a note's text plausibly belongs to the
|
||||
// category he named. Russian inflects the topic ("технологиям" vs the configured
|
||||
// "технологии"), and there is no stemmer in this repo, so the comparison is on a
|
||||
// common prefix — long enough that "полит" and "погод" stay apart, short enough
|
||||
// to survive a case ending.
|
||||
func CategoryMatches(text, category string) bool {
|
||||
if category == "" {
|
||||
return true
|
||||
}
|
||||
stem := categoryStem(category)
|
||||
if stem == "" {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(strings.ToLower(text), stem)
|
||||
}
|
||||
|
||||
// categoryStem cuts a word down to the part inflection leaves alone. 5 runes is
|
||||
// the compromise: shorter words are used whole.
|
||||
func categoryStem(word string) string {
|
||||
r := []rune(strings.ToLower(strings.TrimSpace(word)))
|
||||
if len(r) > 5 {
|
||||
r = r[:5]
|
||||
}
|
||||
return string(r)
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package router
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestParseFeedQuery(t *testing.T) {
|
||||
cases := []struct {
|
||||
text string
|
||||
ok bool
|
||||
category string
|
||||
}{
|
||||
{"что нового в лентах?", true, ""},
|
||||
{"что нового?", true, ""},
|
||||
{"какие новости?", true, ""},
|
||||
{"что нового по технологиям?", true, "технологиям"},
|
||||
{"расскажи новости про политику", true, "политику"},
|
||||
{"что нового по новостям", true, ""},
|
||||
{"what's new in the feeds?", true, ""},
|
||||
{"any news about kubernetes", true, "kubernetes"},
|
||||
// Statements, not requests.
|
||||
{"у меня новая лента в инстаграме", false, ""},
|
||||
{"новости меня утомили", false, ""},
|
||||
{"напомни полить цветы", false, ""},
|
||||
{"", false, ""},
|
||||
}
|
||||
for _, c := range cases {
|
||||
q, ok := ParseFeedQuery(c.text)
|
||||
if ok != c.ok {
|
||||
t.Errorf("ParseFeedQuery(%q) ok = %v, want %v", c.text, ok, c.ok)
|
||||
continue
|
||||
}
|
||||
if ok && q.Category != c.category {
|
||||
t.Errorf("ParseFeedQuery(%q) category = %q, want %q", c.text, q.Category, c.category)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCategoryMatches(t *testing.T) {
|
||||
// The inflected form he says must match the form the config spells.
|
||||
if !CategoryMatches("Новый релиз [технологии]", "технологиям") {
|
||||
t.Error("inflected category did not match")
|
||||
}
|
||||
if CategoryMatches("Новый релиз [технологии]", "политику") {
|
||||
t.Error("unrelated category matched")
|
||||
}
|
||||
if !CategoryMatches("anything", "") {
|
||||
t.Error("an empty category must match everything")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Finding a URL in an utterance (Vikunja #259).
|
||||
//
|
||||
// This is deliberately strict: a scheme is required. "посмотри на example.org"
|
||||
// is not treated as a fetch request, because a bare dotted word is also how
|
||||
// people say file names, versions and Russian abbreviations, and the cost of a
|
||||
// false positive here is an outbound request nobody asked for.
|
||||
//
|
||||
// Note where this runs: an utterance from STT. Whisper will mangle a spoken URL,
|
||||
// which is fine — the URL that survives is one he pasted into the web chat, and
|
||||
// a mangled one simply fails to match.
|
||||
var urlRE = regexp.MustCompile(`(?i)\bhttps?://[^\s<>"']+`)
|
||||
|
||||
// FirstURL returns the first http(s) URL in text.
|
||||
//
|
||||
// Trailing punctuation is trimmed: he ends sentences, and "…/page." is not a
|
||||
// path component. A closing bracket is only trimmed when it has no opener,
|
||||
// because a wikipedia URL legitimately ends in one.
|
||||
func FirstURL(text string) (string, bool) {
|
||||
m := urlRE.FindString(text)
|
||||
if m == "" {
|
||||
return "", false
|
||||
}
|
||||
m = strings.TrimRight(m, ".,;:!?…")
|
||||
if strings.HasSuffix(m, ")") && strings.Count(m, "(") == 0 {
|
||||
m = strings.TrimSuffix(m, ")")
|
||||
}
|
||||
// A scheme with nothing after it is not a URL.
|
||||
if rest := strings.SplitN(m, "//", 2); len(rest) < 2 || rest[1] == "" {
|
||||
return "", false
|
||||
}
|
||||
return m, true
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package router
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestFirstURL(t *testing.T) {
|
||||
cases := []struct {
|
||||
text string
|
||||
want string
|
||||
}{
|
||||
{"посмотри https://example.org/page — что там?", "https://example.org/page"},
|
||||
{"почитай http://example.org/a/b?x=1 и скажи", "http://example.org/a/b?x=1"},
|
||||
{"вот ссылка: https://example.org/page.", "https://example.org/page"},
|
||||
{"https://ru.wikipedia.org/wiki/Небо_(значения)", "https://ru.wikipedia.org/wiki/Небо_(значения)"},
|
||||
// No scheme ⇒ no fetch. A bare dotted word is not an instruction to
|
||||
// reach out to the network.
|
||||
{"посмотри на example.org", ""},
|
||||
{"открой файл config.json", ""},
|
||||
{"что нового?", ""},
|
||||
{"https://", ""},
|
||||
{"", ""},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got, ok := FirstURL(c.text)
|
||||
if c.want == "" {
|
||||
if ok {
|
||||
t.Errorf("FirstURL(%q) = %q, want no match", c.text, got)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !ok || got != c.want {
|
||||
t.Errorf("FirstURL(%q) = %q, %v; want %q", c.text, got, ok, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
// Package rss reads RSS 2.0 and Atom feeds, and does nothing else with them.
|
||||
//
|
||||
// Parsing and polling are split from delivery on purpose: a feed is a source
|
||||
// Maven can be ASKED about, not a thing that speaks. Nothing in this package
|
||||
// dispatches, nudges or notifies — the poller writes notes, and the answer path
|
||||
// reads them when he asks "что нового в лентах?". "Not a nag" is the oldest
|
||||
// constraint in the spec, and a news feed is the single most tempting way to
|
||||
// break it.
|
||||
//
|
||||
// Stdlib only (encoding/xml). Feeds are XML from strangers, so the parser takes
|
||||
// what it recognises and ignores the rest rather than failing a whole feed over
|
||||
// one malformed item.
|
||||
package rss
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"html"
|
||||
"io"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Item is one feed entry, normalised across RSS and Atom.
|
||||
type Item struct {
|
||||
Title string
|
||||
Link string
|
||||
Summary string // plain text, tags stripped, entities decoded
|
||||
Published time.Time // zero when the feed did not say
|
||||
ID string // guid / atom id, falling back to the link
|
||||
}
|
||||
|
||||
// Feed is a parsed document.
|
||||
type Feed struct {
|
||||
Title string
|
||||
Items []Item
|
||||
}
|
||||
|
||||
// feedDoc covers both dialects in one struct. RSS puts items under
|
||||
// channel>item, Atom puts entries at the top level, and the field names barely
|
||||
// overlap — so both sets are declared and whichever the document filled in wins.
|
||||
type feedDoc struct {
|
||||
ChannelTitle string `xml:"channel>title"`
|
||||
AtomTitle string `xml:"title"`
|
||||
|
||||
Items []struct {
|
||||
Title string `xml:"title"`
|
||||
Link string `xml:"link"`
|
||||
Description string `xml:"description"`
|
||||
Encoded string `xml:"encoded"` // content:encoded
|
||||
GUID string `xml:"guid"`
|
||||
PubDate string `xml:"pubDate"`
|
||||
Date string `xml:"date"` // dc:date
|
||||
} `xml:"channel>item"`
|
||||
|
||||
Entries []struct {
|
||||
Title string `xml:"title"`
|
||||
Links []struct {
|
||||
Href string `xml:"href,attr"`
|
||||
Rel string `xml:"rel,attr"`
|
||||
} `xml:"link"`
|
||||
Summary string `xml:"summary"`
|
||||
Content string `xml:"content"`
|
||||
ID string `xml:"id"`
|
||||
Updated string `xml:"updated"`
|
||||
Published string `xml:"published"`
|
||||
} `xml:"entry"`
|
||||
}
|
||||
|
||||
// Parse reads a feed document.
|
||||
func Parse(r io.Reader) (Feed, error) {
|
||||
var doc feedDoc
|
||||
dec := xml.NewDecoder(r)
|
||||
// Feeds in the wild declare windows-1251 and worse. We only ever read
|
||||
// UTF-8; a charset we cannot decode is a feed we do not read, which is
|
||||
// better than mojibake in his notes.
|
||||
dec.Strict = false
|
||||
if err := dec.Decode(&doc); err != nil {
|
||||
return Feed{}, fmt.Errorf("rss: bad xml: %w", err)
|
||||
}
|
||||
|
||||
f := Feed{Title: strings.TrimSpace(doc.ChannelTitle)}
|
||||
if f.Title == "" {
|
||||
f.Title = strings.TrimSpace(doc.AtomTitle)
|
||||
}
|
||||
for _, it := range doc.Items {
|
||||
item := Item{
|
||||
Title: PlainText(it.Title),
|
||||
Link: strings.TrimSpace(it.Link),
|
||||
Summary: PlainText(firstNonEmpty(it.Description, it.Encoded)),
|
||||
Published: parseTime(firstNonEmpty(it.PubDate, it.Date)),
|
||||
ID: strings.TrimSpace(firstNonEmpty(it.GUID, it.Link)),
|
||||
}
|
||||
if item.Title != "" || item.Link != "" {
|
||||
f.Items = append(f.Items, item)
|
||||
}
|
||||
}
|
||||
for _, e := range doc.Entries {
|
||||
link := ""
|
||||
for _, l := range e.Links {
|
||||
if l.Rel == "" || l.Rel == "alternate" {
|
||||
link = strings.TrimSpace(l.Href)
|
||||
break
|
||||
}
|
||||
}
|
||||
if link == "" && len(e.Links) > 0 {
|
||||
link = strings.TrimSpace(e.Links[0].Href)
|
||||
}
|
||||
item := Item{
|
||||
Title: PlainText(e.Title),
|
||||
Link: link,
|
||||
Summary: PlainText(firstNonEmpty(e.Summary, e.Content)),
|
||||
Published: parseTime(firstNonEmpty(e.Published, e.Updated)),
|
||||
ID: strings.TrimSpace(firstNonEmpty(e.ID, link)),
|
||||
}
|
||||
if item.Title != "" || item.Link != "" {
|
||||
f.Items = append(f.Items, item)
|
||||
}
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func firstNonEmpty(vals ...string) string {
|
||||
for _, v := range vals {
|
||||
if strings.TrimSpace(v) != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// timeLayouts — RFC1123/822 for RSS, RFC3339 for Atom, plus the near-misses
|
||||
// real feeds ship (no seconds, numeric zone where a name is expected).
|
||||
var timeLayouts = []string{
|
||||
time.RFC1123Z,
|
||||
time.RFC1123,
|
||||
time.RFC822Z,
|
||||
time.RFC822,
|
||||
time.RFC3339,
|
||||
"2006-01-02T15:04:05Z0700",
|
||||
"2006-01-02 15:04:05",
|
||||
"2006-01-02",
|
||||
"Mon, 02 Jan 2006 15:04:05 -0700",
|
||||
"Mon, 2 Jan 2006 15:04:05 -0700",
|
||||
"Mon, 2 Jan 2006 15:04:05 MST",
|
||||
}
|
||||
|
||||
// parseTime returns the zero time on anything it cannot read. An undated item
|
||||
// is still an item; the poller dedupes by ID, so a missing date costs nothing.
|
||||
func parseTime(s string) time.Time {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return time.Time{}
|
||||
}
|
||||
for _, l := range timeLayouts {
|
||||
if t, err := time.Parse(l, s); err == nil {
|
||||
return t.UTC()
|
||||
}
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
var (
|
||||
// RE2 has no backreferences, so the two tags are spelled out rather than
|
||||
// captured and matched against themselves.
|
||||
scriptRE = regexp.MustCompile(`(?is)<script\b[^>]*>.*?</script>|<style\b[^>]*>.*?</style>`)
|
||||
tagRE = regexp.MustCompile(`(?s)<[^>]*>`)
|
||||
)
|
||||
|
||||
// PlainText strips markup and decodes entities — feed summaries are HTML, and
|
||||
// what reaches a note (and possibly the TTS) must be text. Exported because the
|
||||
// crawler's extractor needs exactly this on a bigger input.
|
||||
func PlainText(s string) string {
|
||||
s = scriptRE.ReplaceAllString(s, " ")
|
||||
s = tagRE.ReplaceAllString(s, " ")
|
||||
s = html.UnescapeString(s)
|
||||
return strings.TrimSpace(strings.Join(strings.Fields(s), " "))
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package rss
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const rss2 = `<?xml version="1.0"?>
|
||||
<rss version="2.0">
|
||||
<channel>
|
||||
<title>Хабр</title>
|
||||
<item>
|
||||
<title>Новая уязвимость в ядре</title>
|
||||
<link>https://example.org/a</link>
|
||||
<description><p>Патч уже <b>вышел</b>.</p></description>
|
||||
<guid>tag:example.org,a</guid>
|
||||
<pubDate>Mon, 28 Jul 2026 10:00:00 +0000</pubDate>
|
||||
</item>
|
||||
<item>
|
||||
<title>Без даты</title>
|
||||
<link>https://example.org/b</link>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>`
|
||||
|
||||
const atom = `<?xml version="1.0" encoding="utf-8"?>
|
||||
<feed xmlns="http://www.w3.org/2005/Atom">
|
||||
<title>Example Atom</title>
|
||||
<entry>
|
||||
<title>Release 2.0</title>
|
||||
<link rel="alternate" href="https://example.com/rel"/>
|
||||
<link rel="edit" href="https://example.com/edit"/>
|
||||
<id>urn:uuid:1</id>
|
||||
<updated>2026-07-30T12:30:00Z</updated>
|
||||
<summary>Ships & works</summary>
|
||||
</entry>
|
||||
</feed>`
|
||||
|
||||
func TestParseRSS2(t *testing.T) {
|
||||
f, err := Parse(strings.NewReader(rss2))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.Title != "Хабр" {
|
||||
t.Fatalf("title = %q", f.Title)
|
||||
}
|
||||
if len(f.Items) != 2 {
|
||||
t.Fatalf("items = %d, want 2", len(f.Items))
|
||||
}
|
||||
it := f.Items[0]
|
||||
if it.Title != "Новая уязвимость в ядре" {
|
||||
t.Errorf("title = %q", it.Title)
|
||||
}
|
||||
if it.Summary != "Патч уже вышел ." && it.Summary != "Патч уже вышел." {
|
||||
t.Errorf("summary = %q — tags must be stripped and entities decoded", it.Summary)
|
||||
}
|
||||
if it.ID != "tag:example.org,a" {
|
||||
t.Errorf("id = %q", it.ID)
|
||||
}
|
||||
if want := time.Date(2026, 7, 28, 10, 0, 0, 0, time.UTC); !it.Published.Equal(want) {
|
||||
t.Errorf("published = %v, want %v", it.Published, want)
|
||||
}
|
||||
if !f.Items[1].Published.IsZero() {
|
||||
t.Errorf("undated item got a date: %v", f.Items[1].Published)
|
||||
}
|
||||
if f.Items[1].ID != "https://example.org/b" {
|
||||
t.Errorf("id falls back to the link, got %q", f.Items[1].ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseAtom(t *testing.T) {
|
||||
f, err := Parse(strings.NewReader(atom))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.Title != "Example Atom" || len(f.Items) != 1 {
|
||||
t.Fatalf("feed = %+v", f)
|
||||
}
|
||||
it := f.Items[0]
|
||||
if it.Link != "https://example.com/rel" {
|
||||
t.Errorf("link = %q, want the alternate link", it.Link)
|
||||
}
|
||||
if it.Summary != "Ships & works" {
|
||||
t.Errorf("summary = %q", it.Summary)
|
||||
}
|
||||
if want := time.Date(2026, 7, 30, 12, 30, 0, 0, time.UTC); !it.Published.Equal(want) {
|
||||
t.Errorf("published = %v, want %v", it.Published, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseGarbage(t *testing.T) {
|
||||
if _, err := Parse(strings.NewReader("<html><body>not a feed")); err == nil {
|
||||
t.Fatal("want an error on a non-feed document")
|
||||
}
|
||||
// A feed with an item that has neither title nor link contributes nothing
|
||||
// rather than an empty note.
|
||||
f, err := Parse(strings.NewReader(`<rss><channel><item><description>x</description></item></channel></rss>`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(f.Items) != 0 {
|
||||
t.Fatalf("items = %d, want 0", len(f.Items))
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlainTextDropsScript(t *testing.T) {
|
||||
got := PlainText(`<p>hi</p><script>alert("x")</script><style>b{}</style> there`)
|
||||
if got != "hi there" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,324 @@
|
||||
package rss
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// FeedConfig — one feed to read. A feed with no Name or no URL is ignored.
|
||||
type FeedConfig struct {
|
||||
Name string // short id; the note source is "rss:<Name>"
|
||||
URL string // http(s) only, enforced by the fetcher
|
||||
Category string // free text ("технологии"), used to answer "что по X?"
|
||||
Interval time.Duration // 0 ⇒ the poller's default
|
||||
Include []string // when non-empty, keep only items matching one of these
|
||||
Exclude []string // drop items matching any of these, even if included
|
||||
}
|
||||
|
||||
// Fetcher is the guarded HTTP door (internal/webfetch). An interface so the
|
||||
// poller is testable without a network and so it CANNOT fetch by any other
|
||||
// means: no http.Client is constructed in this package.
|
||||
type Fetcher interface {
|
||||
Get(ctx context.Context, url string) (*Body, error)
|
||||
}
|
||||
|
||||
// Body is the minimum the poller needs from a response.
|
||||
type Body struct{ Bytes []byte }
|
||||
|
||||
// Notes is core's note-writing half. Same shape as ipc.CoreAPI's method, so the
|
||||
// daemon passes its API straight in.
|
||||
type Notes interface {
|
||||
WriteNote(ctx context.Context, ts time.Time, text string, embedding []float32, source string) (int64, error)
|
||||
}
|
||||
|
||||
// Marks remembers how far a feed was read. Durable, because the alternative is
|
||||
// re-writing yesterday's headlines as fresh notes after every restart. The
|
||||
// daemon backs this with config facts (key "rss:latest:<feed>").
|
||||
type Marks interface {
|
||||
LastMark(ctx context.Context, feed string) (time.Time, error)
|
||||
SetMark(ctx context.Context, feed string, at time.Time) error
|
||||
}
|
||||
|
||||
// Embedder embeds a note on its way into the store so recall can find it. nil ⇒
|
||||
// notes are written without a vector (still readable by the recent-notes path).
|
||||
type Embedder interface {
|
||||
Embed(ctx context.Context, text string) ([]float32, error)
|
||||
}
|
||||
|
||||
// Ranker is the relevance seam. The plan called for scoring each item against
|
||||
// an interest profile built from his notes; that profile does not exist yet, and
|
||||
// a threshold over an embedder with no profile to compare to is a random filter
|
||||
// with a confident name. So the seam is here, nil in the daemon, and the filter
|
||||
// that actually runs is the per-feed keyword one — a rule he can read and
|
||||
// predict. When there IS a profile, implement this and pass it.
|
||||
//
|
||||
// Note what a Ranker must NOT be: anything that sends his notes outward. The
|
||||
// scoring happens locally against a local embedder; the feed item is the input,
|
||||
// his memory is never the payload.
|
||||
type Ranker interface {
|
||||
Relevant(ctx context.Context, text string) (bool, error)
|
||||
}
|
||||
|
||||
// Config — poller-wide settings.
|
||||
type Config struct {
|
||||
DefaultInterval time.Duration // 0 ⇒ DefaultPollInterval
|
||||
MaxItems int // most notes written per feed per poll; 0 ⇒ DefaultMaxItems
|
||||
MaxAge time.Duration // ignore items older than this on a cold start; 0 ⇒ DefaultMaxAge
|
||||
}
|
||||
|
||||
// Defaults chosen to be quiet: a feed read every half hour, at most a handful of
|
||||
// items kept, and a cold start that does not import a month of history.
|
||||
const (
|
||||
DefaultPollInterval = 30 * time.Minute
|
||||
DefaultMaxItems = 5
|
||||
DefaultMaxAge = 24 * time.Hour
|
||||
)
|
||||
|
||||
// Poller reads feeds on a schedule and writes what survives filtering as notes.
|
||||
type Poller struct {
|
||||
feeds []FeedConfig
|
||||
fetch Fetcher
|
||||
notes Notes
|
||||
marks Marks
|
||||
embed Embedder
|
||||
ranker Ranker
|
||||
cfg Config
|
||||
nextDue map[string]time.Time
|
||||
seen map[string]map[string]bool // feed → item ID, for items with no date
|
||||
}
|
||||
|
||||
// NewPoller wires a poller. Returns nil when there is nothing to poll — a
|
||||
// capability is off unless configured, and callers check for nil.
|
||||
func NewPoller(feeds []FeedConfig, fetch Fetcher, notes Notes, marks Marks, embed Embedder, ranker Ranker, cfg Config) *Poller {
|
||||
var valid []FeedConfig
|
||||
for _, f := range feeds {
|
||||
if strings.TrimSpace(f.Name) == "" || strings.TrimSpace(f.URL) == "" {
|
||||
log.Printf("rss: skipping a feed with no name or no url")
|
||||
continue
|
||||
}
|
||||
valid = append(valid, f)
|
||||
}
|
||||
if len(valid) == 0 || fetch == nil || notes == nil {
|
||||
return nil
|
||||
}
|
||||
if cfg.DefaultInterval <= 0 {
|
||||
cfg.DefaultInterval = DefaultPollInterval
|
||||
}
|
||||
if cfg.MaxItems <= 0 {
|
||||
cfg.MaxItems = DefaultMaxItems
|
||||
}
|
||||
if cfg.MaxAge <= 0 {
|
||||
cfg.MaxAge = DefaultMaxAge
|
||||
}
|
||||
return &Poller{
|
||||
feeds: valid, fetch: fetch, notes: notes, marks: marks,
|
||||
embed: embed, ranker: ranker, cfg: cfg,
|
||||
nextDue: map[string]time.Time{},
|
||||
seen: map[string]map[string]bool{},
|
||||
}
|
||||
}
|
||||
|
||||
// Feeds returns the configured feeds (the answer path lists categories).
|
||||
func (p *Poller) Feeds() []FeedConfig { return p.feeds }
|
||||
|
||||
// PollDue reads every feed whose interval has elapsed and returns how many
|
||||
// notes were written. Errors are logged per feed, never returned: one dead feed
|
||||
// must not stop the others, and there is nobody waiting on this.
|
||||
func (p *Poller) PollDue(ctx context.Context, now time.Time) int {
|
||||
written := 0
|
||||
for _, f := range p.feeds {
|
||||
if due, ok := p.nextDue[f.Name]; ok && now.Before(due) {
|
||||
continue
|
||||
}
|
||||
interval := f.Interval
|
||||
if interval <= 0 {
|
||||
interval = p.cfg.DefaultInterval
|
||||
}
|
||||
p.nextDue[f.Name] = now.Add(interval)
|
||||
n, err := p.PollFeed(ctx, f, now)
|
||||
if err != nil {
|
||||
// The URL is configured by him and not a secret, so it is loggable;
|
||||
// item titles are not logged, only counts.
|
||||
log.Printf("rss: feed %s: %v", f.Name, err)
|
||||
continue
|
||||
}
|
||||
if n > 0 {
|
||||
log.Printf("rss: feed %s: %d new item(s) noted", f.Name, n)
|
||||
}
|
||||
written += n
|
||||
}
|
||||
return written
|
||||
}
|
||||
|
||||
// PollFeed reads one feed now, regardless of its schedule.
|
||||
func (p *Poller) PollFeed(ctx context.Context, f FeedConfig, now time.Time) (int, error) {
|
||||
body, err := p.fetch.Get(ctx, f.URL)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
feed, err := Parse(strings.NewReader(string(body.Bytes)))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
mark := p.mark(ctx, f.Name, now)
|
||||
newest := mark
|
||||
written := 0
|
||||
for _, it := range feed.Items {
|
||||
if written >= p.cfg.MaxItems {
|
||||
break
|
||||
}
|
||||
if !p.fresh(f, it, mark, now) {
|
||||
continue
|
||||
}
|
||||
if !Matches(f, it) {
|
||||
continue
|
||||
}
|
||||
if p.ranker != nil {
|
||||
ok, err := p.ranker.Relevant(ctx, it.Title+" "+it.Summary)
|
||||
if err != nil {
|
||||
log.Printf("rss: feed %s: relevance: %v", f.Name, err)
|
||||
} else if !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if err := p.write(ctx, f, it, now); err != nil {
|
||||
return written, err
|
||||
}
|
||||
written++
|
||||
if it.Published.After(newest) {
|
||||
newest = it.Published
|
||||
}
|
||||
}
|
||||
if p.marks != nil && newest.After(mark) {
|
||||
if err := p.marks.SetMark(ctx, f.Name, newest); err != nil {
|
||||
log.Printf("rss: feed %s: save mark: %v", f.Name, err)
|
||||
}
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
|
||||
// mark — how far this feed was read. A feed with no mark starts MaxAge ago, so
|
||||
// a first poll takes today's headlines instead of the whole archive.
|
||||
func (p *Poller) mark(ctx context.Context, feed string, now time.Time) time.Time {
|
||||
cold := now.Add(-p.cfg.MaxAge)
|
||||
if p.marks == nil {
|
||||
return cold
|
||||
}
|
||||
at, err := p.marks.LastMark(ctx, feed)
|
||||
if err != nil || at.IsZero() {
|
||||
return cold
|
||||
}
|
||||
return at
|
||||
}
|
||||
|
||||
// fresh — two dedup rules, because feeds are inconsistent about dates. A dated
|
||||
// item must be newer than the mark; an undated one is kept once per process by
|
||||
// ID. Both are needed: dates alone re-import undated feeds forever, IDs alone
|
||||
// lose their memory on restart.
|
||||
func (p *Poller) fresh(f FeedConfig, it Item, mark, now time.Time) bool {
|
||||
if !it.Published.IsZero() {
|
||||
if !it.Published.After(mark) {
|
||||
return false
|
||||
}
|
||||
// A feed that dates its items in the future (or a clock skew) must not
|
||||
// win the mark and mute everything after it.
|
||||
return !it.Published.After(now.Add(time.Hour))
|
||||
}
|
||||
id := it.ID
|
||||
if id == "" {
|
||||
id = it.Title
|
||||
}
|
||||
if p.seen[f.Name] == nil {
|
||||
p.seen[f.Name] = map[string]bool{}
|
||||
}
|
||||
if p.seen[f.Name][id] {
|
||||
return false
|
||||
}
|
||||
p.seen[f.Name][id] = true
|
||||
return true
|
||||
}
|
||||
|
||||
// write stores one item as a note. Source "rss:<feed>" is what the answer path
|
||||
// filters on, and what makes a feed note distinguishable from something he said.
|
||||
func (p *Poller) write(ctx context.Context, f FeedConfig, it Item, now time.Time) error {
|
||||
text := NoteText(f, it)
|
||||
var vec []float32
|
||||
if p.embed != nil {
|
||||
v, err := p.embed.Embed(ctx, text)
|
||||
if err != nil {
|
||||
log.Printf("rss: feed %s: embed: %v", f.Name, err)
|
||||
} else {
|
||||
vec = v
|
||||
}
|
||||
}
|
||||
ts := it.Published
|
||||
if ts.IsZero() {
|
||||
ts = now
|
||||
}
|
||||
if _, err := p.notes.WriteNote(ctx, ts, text, vec, SourceFor(f.Name)); err != nil {
|
||||
return fmt.Errorf("write note: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SourceFor is the note source for a feed.
|
||||
func SourceFor(feed string) string { return "rss:" + feed }
|
||||
|
||||
// SourcePrefix — what the answer path matches to find feed notes.
|
||||
const SourcePrefix = "rss:"
|
||||
|
||||
// NoteText renders an item as the note body. The category is included because
|
||||
// "что нового по технологиям?" is answered by reading notes, and a note has to
|
||||
// carry enough to be recognised as belonging to that category.
|
||||
func NoteText(f FeedConfig, it Item) string {
|
||||
var b strings.Builder
|
||||
b.WriteString(it.Title)
|
||||
if f.Category != "" {
|
||||
fmt.Fprintf(&b, " [%s]", f.Category)
|
||||
}
|
||||
if it.Summary != "" {
|
||||
b.WriteString("\n")
|
||||
b.WriteString(trimRunes(it.Summary, 500))
|
||||
}
|
||||
if it.Link != "" {
|
||||
b.WriteString("\n")
|
||||
b.WriteString(it.Link)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// trimRunes cuts on a rune boundary — a note is Russian as often as English and
|
||||
// half a cyrillic letter is a broken note.
|
||||
func trimRunes(s string, max int) string {
|
||||
r := []rune(s)
|
||||
if len(r) <= max {
|
||||
return s
|
||||
}
|
||||
return strings.TrimSpace(string(r[:max])) + "…"
|
||||
}
|
||||
|
||||
// Matches applies the per-feed keyword filter: keep when Include is empty or one
|
||||
// include matches, drop when any exclude matches. Case-insensitive substring,
|
||||
// which for Russian is the honest choice — no stemmer here, so "выборы" does not
|
||||
// match "выборах", and a filter he writes is a filter he can predict.
|
||||
func Matches(f FeedConfig, it Item) bool {
|
||||
hay := strings.ToLower(it.Title + " " + it.Summary)
|
||||
for _, x := range f.Exclude {
|
||||
if x = strings.ToLower(strings.TrimSpace(x)); x != "" && strings.Contains(hay, x) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if len(f.Include) == 0 {
|
||||
return true
|
||||
}
|
||||
for _, in := range f.Include {
|
||||
if in = strings.ToLower(strings.TrimSpace(in)); in != "" && strings.Contains(hay, in) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
package rss
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type fakeFetch struct {
|
||||
body string
|
||||
err error
|
||||
calls int
|
||||
urls []string
|
||||
}
|
||||
|
||||
func (f *fakeFetch) Get(_ context.Context, url string) (*Body, error) {
|
||||
f.calls++
|
||||
f.urls = append(f.urls, url)
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
return &Body{Bytes: []byte(f.body)}, nil
|
||||
}
|
||||
|
||||
type writtenNote struct {
|
||||
ts time.Time
|
||||
text string
|
||||
source string
|
||||
vec []float32
|
||||
}
|
||||
|
||||
type fakeNotes struct{ notes []writtenNote }
|
||||
|
||||
func (n *fakeNotes) WriteNote(_ context.Context, ts time.Time, text string, vec []float32, source string) (int64, error) {
|
||||
n.notes = append(n.notes, writtenNote{ts, text, source, vec})
|
||||
return int64(len(n.notes)), nil
|
||||
}
|
||||
|
||||
type fakeMarks struct{ m map[string]time.Time }
|
||||
|
||||
func newMarks() *fakeMarks { return &fakeMarks{m: map[string]time.Time{}} }
|
||||
func (f *fakeMarks) LastMark(_ context.Context, feed string) (time.Time, error) {
|
||||
return f.m[feed], nil
|
||||
}
|
||||
func (f *fakeMarks) SetMark(_ context.Context, feed string, at time.Time) error {
|
||||
f.m[feed] = at
|
||||
return nil
|
||||
}
|
||||
|
||||
var now = time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
func TestPollWritesNotesWithSource(t *testing.T) {
|
||||
fetch := &fakeFetch{body: rss2}
|
||||
notes := &fakeNotes{}
|
||||
marks := newMarks()
|
||||
p := NewPoller([]FeedConfig{{Name: "habr", URL: "https://example.org/rss", Category: "технологии"}},
|
||||
fetch, notes, marks, nil, nil, Config{})
|
||||
if p == nil {
|
||||
t.Fatal("NewPoller returned nil for a configured feed")
|
||||
}
|
||||
n := p.PollDue(context.Background(), now)
|
||||
if n != 2 || len(notes.notes) != 2 {
|
||||
t.Fatalf("wrote %d notes (returned %d), want 2", len(notes.notes), n)
|
||||
}
|
||||
if notes.notes[0].source != "rss:habr" {
|
||||
t.Errorf("source = %q, want rss:habr", notes.notes[0].source)
|
||||
}
|
||||
if !strings.Contains(notes.notes[0].text, "технологии") {
|
||||
t.Errorf("note does not carry its category: %q", notes.notes[0].text)
|
||||
}
|
||||
if !strings.Contains(notes.notes[0].text, "https://example.org/a") {
|
||||
t.Errorf("note does not carry its link: %q", notes.notes[0].text)
|
||||
}
|
||||
// The undated item is stamped with now, the dated one with its own date.
|
||||
if !notes.notes[1].ts.Equal(now) {
|
||||
t.Errorf("undated item ts = %v, want now", notes.notes[1].ts)
|
||||
}
|
||||
}
|
||||
|
||||
// The whole point of a mark: polling twice must not re-note the same headlines.
|
||||
func TestSecondPollIsQuiet(t *testing.T) {
|
||||
fetch := &fakeFetch{body: rss2}
|
||||
notes := &fakeNotes{}
|
||||
p := NewPoller([]FeedConfig{{Name: "habr", URL: "u", Interval: time.Minute}}, fetch, notes, newMarks(), nil, nil, Config{})
|
||||
p.PollDue(context.Background(), now)
|
||||
before := len(notes.notes)
|
||||
p.PollDue(context.Background(), now.Add(2*time.Minute))
|
||||
if len(notes.notes) != before {
|
||||
t.Fatalf("second poll wrote %d extra notes", len(notes.notes)-before)
|
||||
}
|
||||
}
|
||||
|
||||
// A mark that survives a restart is the durable half; simulate one by building a
|
||||
// fresh poller over the same marks.
|
||||
func TestMarkSurvivesRestart(t *testing.T) {
|
||||
marks := newMarks()
|
||||
fetch := &fakeFetch{body: rss2}
|
||||
notes := &fakeNotes{}
|
||||
feeds := []FeedConfig{{Name: "habr", URL: "u"}}
|
||||
NewPoller(feeds, fetch, notes, marks, nil, nil, Config{}).PollDue(context.Background(), now)
|
||||
if len(notes.notes) != 2 {
|
||||
t.Fatalf("first run wrote %d", len(notes.notes))
|
||||
}
|
||||
notes2 := &fakeNotes{}
|
||||
NewPoller(feeds, fetch, notes2, marks, nil, nil, Config{}).PollDue(context.Background(), now.Add(time.Hour))
|
||||
// The dated item is behind the mark. The undated one has no date to compare,
|
||||
// so it comes back — accepted and documented in fresh(): an undated feed is
|
||||
// deduped per process, not forever.
|
||||
for _, n := range notes2.notes {
|
||||
if strings.Contains(n.text, "уязвимость") {
|
||||
t.Fatalf("dated item re-noted after restart: %q", n.text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntervalIsRespected(t *testing.T) {
|
||||
fetch := &fakeFetch{body: rss2}
|
||||
p := NewPoller([]FeedConfig{{Name: "habr", URL: "u", Interval: time.Hour}}, fetch, &fakeNotes{}, newMarks(), nil, nil, Config{})
|
||||
p.PollDue(context.Background(), now)
|
||||
p.PollDue(context.Background(), now.Add(time.Minute))
|
||||
if fetch.calls != 1 {
|
||||
t.Fatalf("fetched %d times inside one interval, want 1", fetch.calls)
|
||||
}
|
||||
p.PollDue(context.Background(), now.Add(2*time.Hour))
|
||||
if fetch.calls != 2 {
|
||||
t.Fatalf("fetched %d times, want 2 after the interval elapsed", fetch.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestColdStartIgnoresOldItems(t *testing.T) {
|
||||
old := `<rss><channel><item><title>Старое</title><link>l</link>` +
|
||||
`<pubDate>Mon, 01 Jun 2026 10:00:00 +0000</pubDate></item></channel></rss>`
|
||||
notes := &fakeNotes{}
|
||||
p := NewPoller([]FeedConfig{{Name: "f", URL: "u"}}, &fakeFetch{body: old}, notes, newMarks(), nil, nil, Config{MaxAge: 24 * time.Hour})
|
||||
if n := p.PollDue(context.Background(), now); n != 0 {
|
||||
t.Fatalf("cold start imported %d old items, want 0", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaxItemsCap(t *testing.T) {
|
||||
var b strings.Builder
|
||||
b.WriteString("<rss><channel>")
|
||||
for i := 0; i < 10; i++ {
|
||||
b.WriteString("<item><title>t")
|
||||
b.WriteByte(byte('0' + i))
|
||||
b.WriteString("</title><link>https://example.org/")
|
||||
b.WriteByte(byte('0' + i))
|
||||
b.WriteString("</link></item>")
|
||||
}
|
||||
b.WriteString("</channel></rss>")
|
||||
notes := &fakeNotes{}
|
||||
p := NewPoller([]FeedConfig{{Name: "f", URL: "u"}}, &fakeFetch{body: b.String()}, notes, newMarks(), nil, nil, Config{MaxItems: 3})
|
||||
if n := p.PollDue(context.Background(), now); n != 3 {
|
||||
t.Fatalf("wrote %d notes, want the cap of 3", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeywordFilter(t *testing.T) {
|
||||
f := FeedConfig{Include: []string{"ядр"}, Exclude: []string{"реклама"}}
|
||||
if !Matches(f, Item{Title: "Новое ядро"}) {
|
||||
t.Error("include did not match")
|
||||
}
|
||||
if Matches(f, Item{Title: "Новое ядро", Summary: "Реклама внутри"}) {
|
||||
t.Error("exclude must win over include")
|
||||
}
|
||||
if Matches(f, Item{Title: "Погода"}) {
|
||||
t.Error("non-matching item passed the include filter")
|
||||
}
|
||||
if !Matches(FeedConfig{}, Item{Title: "что угодно"}) {
|
||||
t.Error("an unfiltered feed must keep everything")
|
||||
}
|
||||
}
|
||||
|
||||
type fakeRanker struct{ keep bool }
|
||||
|
||||
func (r fakeRanker) Relevant(context.Context, string) (bool, error) { return r.keep, nil }
|
||||
|
||||
func TestRankerCanDropEverything(t *testing.T) {
|
||||
notes := &fakeNotes{}
|
||||
p := NewPoller([]FeedConfig{{Name: "f", URL: "u"}}, &fakeFetch{body: rss2}, notes, newMarks(), nil, fakeRanker{false}, Config{})
|
||||
if n := p.PollDue(context.Background(), now); n != 0 {
|
||||
t.Fatalf("ranker rejected everything but %d notes were written", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchErrorIsSurvivable(t *testing.T) {
|
||||
notes := &fakeNotes{}
|
||||
p := NewPoller([]FeedConfig{
|
||||
{Name: "dead", URL: "u1"},
|
||||
{Name: "live", URL: "u2"},
|
||||
}, &fakeFetch{err: errors.New("boom")}, notes, newMarks(), nil, nil, Config{})
|
||||
if n := p.PollDue(context.Background(), now); n != 0 {
|
||||
t.Fatalf("n = %d", n)
|
||||
}
|
||||
// Both feeds were attempted: one dead feed does not abort the round.
|
||||
if p.nextDue["live"].IsZero() {
|
||||
t.Fatal("the second feed was never attempted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoFeedsMeansNoPoller(t *testing.T) {
|
||||
if p := NewPoller(nil, &fakeFetch{}, &fakeNotes{}, nil, nil, nil, Config{}); p != nil {
|
||||
t.Fatal("NewPoller must return nil when nothing is configured")
|
||||
}
|
||||
if p := NewPoller([]FeedConfig{{Name: "", URL: ""}}, &fakeFetch{}, &fakeNotes{}, nil, nil, nil, Config{}); p != nil {
|
||||
t.Fatal("a feed with no name or url is not a configuration")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
package update
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Result — the full account of one Apply. Every field is filled in on the
|
||||
// failure paths too, because "what state is my box in" is the only question that
|
||||
// matters after a failed update.
|
||||
type Result struct {
|
||||
Verified bool
|
||||
SnapshotID string // the rollback target; named even when the rollback failed
|
||||
Installed []string
|
||||
Restarted bool
|
||||
Healthy bool
|
||||
RolledBack bool
|
||||
// RollbackHealthy — whether she answered again after the restore. False with
|
||||
// RolledBack true is the manual-recovery case.
|
||||
RollbackHealthy bool
|
||||
Steps []Step
|
||||
Took time.Duration
|
||||
}
|
||||
|
||||
// Apply is the whole update, in the only order that is safe.
|
||||
//
|
||||
// It is called by a human running cmd/mavupdate on the box. Nothing else calls
|
||||
// it: no timer, no IPC method, no web route, no act. See the package comment.
|
||||
func (u *Updater) Apply(ctx context.Context) (Result, error) {
|
||||
start := u.now()
|
||||
res := Result{}
|
||||
defer func() { res.Took = u.now().Sub(start) }()
|
||||
|
||||
// 0. She has to be answering before we start. Otherwise a failed update and
|
||||
// a box that was already broken look identical afterwards, and the rollback
|
||||
// has no baseline to prove itself against.
|
||||
u.log("preflight: checking the running daemon")
|
||||
if err := u.health(ctx, u.cfg.HealthSocket); err != nil {
|
||||
return res, fmt.Errorf("%w: %v", ErrUnhealthyBefore, err)
|
||||
}
|
||||
|
||||
// 1. Snapshot what is deployed now, BEFORE the build.
|
||||
//
|
||||
// The order matters and it is not the obvious one. `make build` writes its
|
||||
// binaries into the working tree, and on the docker deployment the working
|
||||
// tree IS the install dir — so snapshotting after the build would snapshot
|
||||
// the new artifacts and leave nothing to roll back to. The snapshot is the
|
||||
// only thing standing between a bad build and a box that needs a screwdriver,
|
||||
// so it is taken first, while the deployed bytes are still the old ones.
|
||||
names := append(append([]string{}, u.cfg.Binaries...), u.cfg.ConfigFiles...)
|
||||
snap, err := u.store.Save(u.cfg.InstallDir, names, u.gitHead(ctx), "pre-update")
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
res.SnapshotID = snap.ID
|
||||
u.log("snapshot: %s (%d files) in %s", snap.ID, len(snap.Files), snap.Dir())
|
||||
|
||||
// 2. Build and test before anything is deployed. A broken tree costs time
|
||||
// and nothing else — but `make build` has already overwritten the binaries in
|
||||
// the tree, so restore them: otherwise a later restart by hand would deploy
|
||||
// code that failed its own tests. Nothing has been restarted, so this is a
|
||||
// file restore with no restart and no health check.
|
||||
steps, err := u.Verify(ctx)
|
||||
res.Steps = append(res.Steps, steps...)
|
||||
if err != nil {
|
||||
if rerr := snap.Restore(u.cfg.InstallDir); rerr != nil {
|
||||
u.log("verify failed and the artifacts could not be put back: %v — the previous ones are in %s", rerr, snap.Dir())
|
||||
} else {
|
||||
res.RolledBack = true
|
||||
u.log("verify failed; the previously deployed artifacts are back in place, she was never restarted")
|
||||
}
|
||||
return res, err
|
||||
}
|
||||
res.Verified = true
|
||||
|
||||
// 3. Install. Per-file temp+rename, so an interruption leaves whole files.
|
||||
// Config is snapshotted but never overwritten — an update does not get to
|
||||
// replace the operator's config.
|
||||
installed, err := u.install()
|
||||
res.Installed = installed
|
||||
if err != nil {
|
||||
// Files may be half-swapped across the set, so restore before returning
|
||||
// even though nothing has been restarted yet.
|
||||
u.log("install failed: %v — restoring", err)
|
||||
return u.rollback(ctx, snap, res, err)
|
||||
}
|
||||
u.log("install: %d artifact(s) into %s", len(installed), u.cfg.InstallDir)
|
||||
|
||||
// 4. Restart, then 5. prove she answers.
|
||||
if err := u.restart(ctx, &res); err != nil {
|
||||
return u.rollback(ctx, snap, res, err)
|
||||
}
|
||||
u.log("restart: ok, waiting for her to answer (up to %s)", u.cfg.healthTimeout())
|
||||
if err := u.waitHealthy(ctx, u.cfg.healthTimeout()); err != nil {
|
||||
return u.rollback(ctx, snap, res, err)
|
||||
}
|
||||
res.Healthy = true
|
||||
u.log("health: she answers on %s — update committed", u.cfg.HealthSocket)
|
||||
|
||||
if err := u.store.Prune(u.cfg.KeepSnapshots); err != nil {
|
||||
u.log("prune: %v (harmless)", err)
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// Rollback restores a snapshot by id (empty = the newest) and restarts. Exposed
|
||||
// separately so the operator can undo an update that verified, restarted and
|
||||
// answered a Presence call but is wrong in a way no health check can see.
|
||||
func (u *Updater) Rollback(ctx context.Context, id string) (Result, error) {
|
||||
var snap Snapshot
|
||||
var err error
|
||||
if id == "" {
|
||||
snaps, lerr := u.store.List()
|
||||
if lerr != nil {
|
||||
return Result{}, lerr
|
||||
}
|
||||
if len(snaps) == 0 {
|
||||
return Result{}, errors.New("update: no snapshots to roll back to")
|
||||
}
|
||||
snap = snaps[0]
|
||||
} else if snap, err = u.store.Load(id); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
res := Result{SnapshotID: snap.ID}
|
||||
return u.rollback(ctx, snap, res, errors.New("operator asked for a rollback"))
|
||||
}
|
||||
|
||||
// rollback restores the snapshot and restarts, then reports whether that worked.
|
||||
// It depends on nothing that the update changed: file copies out of the snapshot
|
||||
// dir and the same restart command. No build, no migration, no cooperation from
|
||||
// the code being replaced.
|
||||
func (u *Updater) rollback(ctx context.Context, snap Snapshot, res Result, cause error) (Result, error) {
|
||||
// A rollback interrupted halfway is the one outcome worse than the failure
|
||||
// that triggered it, so it does not inherit the caller's cancellation: a
|
||||
// Ctrl-C during the health wait must not abandon the restore mid-restart.
|
||||
ctx = context.WithoutCancel(ctx)
|
||||
res.RolledBack = true
|
||||
u.log("rollback: restoring snapshot %s over %s", snap.ID, u.cfg.InstallDir)
|
||||
if err := snap.Restore(u.cfg.InstallDir); err != nil {
|
||||
u.log("rollback: RESTORE FAILED: %v", err)
|
||||
return res, fmt.Errorf("%w: %v (after %v); the previous artifacts are in %s — copy them back by hand", ErrRollbackFailed, err, cause, snap.Dir())
|
||||
}
|
||||
// A restore with no restart leaves the failed process running, so a failed
|
||||
// restart here is still the manual-recovery case.
|
||||
if err := u.restart(ctx, &res); err != nil {
|
||||
u.log("rollback: RESTART FAILED: %v", err)
|
||||
return res, fmt.Errorf("%w: restored %s but the restart failed: %v (after %v)", ErrRollbackFailed, snap.ID, err, cause)
|
||||
}
|
||||
if err := u.waitHealthy(ctx, u.cfg.healthTimeout()); err != nil {
|
||||
u.log("rollback: she still does not answer: %v", err)
|
||||
return res, fmt.Errorf("%w: restored %s and restarted but she does not answer: %v (after %v)", ErrRollbackFailed, snap.ID, err, cause)
|
||||
}
|
||||
res.RollbackHealthy = true
|
||||
u.log("rollback: she answers again on the previous build (%s)", snap.ID)
|
||||
return res, fmt.Errorf("%w to %s: %v", ErrRolledBack, snap.ID, cause)
|
||||
}
|
||||
|
||||
// install copies the freshly built binaries from SourceDir into InstallDir.
|
||||
//
|
||||
// When the two are the same directory — the docker deployment builds the image
|
||||
// from the working tree — this is a no-op by design rather than by accident: the
|
||||
// artifacts are already where they belong and the restart command rebuilds the
|
||||
// image from them.
|
||||
func (u *Updater) install() ([]string, error) {
|
||||
if filepath.Clean(u.cfg.SourceDir) == filepath.Clean(u.cfg.InstallDir) {
|
||||
return u.cfg.Binaries, nil
|
||||
}
|
||||
var done []string
|
||||
for _, name := range u.cfg.Binaries {
|
||||
src := filepath.Join(u.cfg.SourceDir, name)
|
||||
fi, err := os.Stat(src)
|
||||
if err != nil {
|
||||
return done, fmt.Errorf("update: install %s: %w (did `make build` produce it?)", name, err)
|
||||
}
|
||||
if _, err := copyFile(src, filepath.Join(u.cfg.InstallDir, name), fi.Mode().Perm()); err != nil {
|
||||
return done, fmt.Errorf("update: install %s: %w", name, err)
|
||||
}
|
||||
done = append(done, name)
|
||||
}
|
||||
return done, nil
|
||||
}
|
||||
|
||||
func (u *Updater) restart(ctx context.Context, res *Result) error {
|
||||
u.log("restart: %v", u.cfg.RestartCmd)
|
||||
out, err := u.run(ctx, u.cfg.SourceDir, u.cfg.RestartCmd)
|
||||
if err != nil {
|
||||
res.Steps = append(res.Steps, Step{Name: "restart", Argv: u.cfg.RestartCmd, Err: err, Output: tail(out, 4000)})
|
||||
return fmt.Errorf("update: restart %v: %w", u.cfg.RestartCmd, err)
|
||||
}
|
||||
res.Restarted = true
|
||||
res.Steps = append(res.Steps, Step{Name: "restart", Argv: u.cfg.RestartCmd})
|
||||
return nil
|
||||
}
|
||||
|
||||
// gitHead records which commit produced a snapshot, for the operator's benefit.
|
||||
// Best-effort: a tree without git is not a reason to refuse to snapshot.
|
||||
func (u *Updater) gitHead(ctx context.Context) string {
|
||||
out, err := u.run(ctx, u.cfg.SourceDir, []string{"git", "rev-parse", "HEAD"})
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(out)
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package update
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
)
|
||||
|
||||
// The health check is the whole basis for rolling back, so it has to mean
|
||||
// something. "The process is running" does not: mavend can be up with a dead
|
||||
// store, a socket it never bound, or a config it failed to parse. What is
|
||||
// checked instead is that she answers a real read over the real IPC socket —
|
||||
// which exercises the socket, the dispatch table and the store in one call.
|
||||
//
|
||||
// Presence is the method used because it is read-only (safe to retry), needs no
|
||||
// arguments, and touches the store. It cannot write anything, so a health check
|
||||
// never leaves a trace in her memory.
|
||||
|
||||
// DialHealth connects to the mavend socket and performs one read.
|
||||
func DialHealth(ctx context.Context, socket string) error {
|
||||
c, err := ipc.Dial(socket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("update: health dial: %w", err)
|
||||
}
|
||||
defer c.Close()
|
||||
if _, err := c.Presence(ctx); err != nil {
|
||||
return fmt.Errorf("update: health read: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// waitHealthy retries the health check until it passes or the timeout elapses.
|
||||
// A restart is not instantaneous — she loads a 1.7B on boot — so the first few
|
||||
// failures are expected and are not a reason to roll back.
|
||||
func (u *Updater) waitHealthy(ctx context.Context, timeout time.Duration) error {
|
||||
deadline := u.now().Add(timeout)
|
||||
delay := 500 * time.Millisecond
|
||||
var last error
|
||||
for {
|
||||
attemptCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
err := u.health(attemptCtx, u.cfg.HealthSocket)
|
||||
cancel()
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
last = err
|
||||
if u.now().After(deadline) {
|
||||
return fmt.Errorf("update: not healthy after %s: %w", timeout, last)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(delay):
|
||||
}
|
||||
if delay < 5*time.Second {
|
||||
delay *= 2
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,267 @@
|
||||
package update
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A snapshot is a byte-for-byte copy of the deployed artifacts plus a manifest
|
||||
// of their sha256 sums, taken before an install.
|
||||
//
|
||||
// It is copies, not hardlinks and not a git stash, for one reason: the restore
|
||||
// path must work when everything else is broken. A hardlink into the install dir
|
||||
// would be clobbered by the very install it exists to undo, and a git-based
|
||||
// undo needs a toolchain, a clean tree, and a rebuild — three things a failed
|
||||
// update is likely to have taken away. Copying two dozen megabytes of Go
|
||||
// binaries costs a second and needs nothing but the filesystem.
|
||||
//
|
||||
// The sums are what make a restore verifiable rather than hopeful: Restore
|
||||
// re-hashes every file it writes, so "the old bytes are back" is checked, not
|
||||
// assumed.
|
||||
|
||||
// FileRec — one file in a snapshot.
|
||||
type FileRec struct {
|
||||
Name string `json:"name"` // relative name inside the install dir
|
||||
SHA256 string `json:"sha256"` // of the snapshotted bytes
|
||||
Mode os.FileMode `json:"mode"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
// Snapshot — the manifest. Written last, so a directory without a readable
|
||||
// manifest.json is an aborted snapshot and is never offered as a rollback target.
|
||||
type Snapshot struct {
|
||||
ID string `json:"id"` // sortable timestamp, also the directory name
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Commit string `json:"commit,omitempty"` // git HEAD of the tree that produced it, when known
|
||||
Note string `json:"note,omitempty"`
|
||||
Files []FileRec `json:"files"`
|
||||
|
||||
dir string // absolute path, filled in by List/Load
|
||||
}
|
||||
|
||||
// Dir — where this snapshot's file copies live.
|
||||
func (s Snapshot) Dir() string { return s.dir }
|
||||
|
||||
const manifestName = "manifest.json"
|
||||
|
||||
// Store is a directory of snapshots.
|
||||
type Store struct {
|
||||
Dir string
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func (st *Store) clock() time.Time {
|
||||
if st.now != nil {
|
||||
return st.now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// Save copies names (relative to srcDir) into a new snapshot and writes the
|
||||
// manifest. A name that does not exist is skipped rather than fatal: the first
|
||||
// ever run happens on a box where some artifact may legitimately be missing, and
|
||||
// refusing to snapshot then would mean refusing to update.
|
||||
func (st *Store) Save(srcDir string, names []string, commit, note string) (Snapshot, error) {
|
||||
ts := st.clock().UTC()
|
||||
snap := Snapshot{
|
||||
ID: ts.Format("20060102-150405"),
|
||||
CreatedAt: ts,
|
||||
Commit: commit,
|
||||
Note: note,
|
||||
}
|
||||
snap.dir = filepath.Join(st.Dir, snap.ID)
|
||||
if err := os.MkdirAll(snap.dir, 0o700); err != nil {
|
||||
return Snapshot{}, fmt.Errorf("update: snapshot dir: %w", err)
|
||||
}
|
||||
for _, name := range names {
|
||||
src := filepath.Join(srcDir, name)
|
||||
fi, err := os.Stat(src)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
return Snapshot{}, fmt.Errorf("update: snapshot %s: %w", name, err)
|
||||
}
|
||||
if fi.IsDir() {
|
||||
return Snapshot{}, fmt.Errorf("update: snapshot %s: is a directory (only files are deployable artifacts)", name)
|
||||
}
|
||||
dst := filepath.Join(snap.dir, name)
|
||||
if err := os.MkdirAll(filepath.Dir(dst), 0o700); err != nil {
|
||||
return Snapshot{}, err
|
||||
}
|
||||
sum, err := copyFile(src, dst, fi.Mode().Perm())
|
||||
if err != nil {
|
||||
return Snapshot{}, fmt.Errorf("update: snapshot %s: %w", name, err)
|
||||
}
|
||||
snap.Files = append(snap.Files, FileRec{Name: name, SHA256: sum, Mode: fi.Mode().Perm(), Size: fi.Size()})
|
||||
}
|
||||
if len(snap.Files) == 0 {
|
||||
os.RemoveAll(snap.dir)
|
||||
return Snapshot{}, fmt.Errorf("update: snapshot of %s is empty — none of the listed artifacts exist", srcDir)
|
||||
}
|
||||
// Manifest last: its presence is what makes the snapshot usable.
|
||||
blob, err := json.MarshalIndent(snap, "", " ")
|
||||
if err != nil {
|
||||
return Snapshot{}, err
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(snap.dir, manifestName), blob, 0o600); err != nil {
|
||||
return Snapshot{}, fmt.Errorf("update: snapshot manifest: %w", err)
|
||||
}
|
||||
return snap, nil
|
||||
}
|
||||
|
||||
// List returns the complete snapshots, newest first.
|
||||
func (st *Store) List() ([]Snapshot, error) {
|
||||
ents, err := os.ReadDir(st.Dir)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
var out []Snapshot
|
||||
for _, e := range ents {
|
||||
if !e.IsDir() {
|
||||
continue
|
||||
}
|
||||
s, err := st.Load(e.Name())
|
||||
if err != nil {
|
||||
continue // aborted or hand-mangled: not a rollback target
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].ID > out[j].ID })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Load reads one snapshot's manifest.
|
||||
func (st *Store) Load(id string) (Snapshot, error) {
|
||||
dir := filepath.Join(st.Dir, id)
|
||||
blob, err := os.ReadFile(filepath.Join(dir, manifestName))
|
||||
if err != nil {
|
||||
return Snapshot{}, err
|
||||
}
|
||||
var s Snapshot
|
||||
if err := json.Unmarshal(blob, &s); err != nil {
|
||||
return Snapshot{}, fmt.Errorf("update: manifest %s: %w", id, err)
|
||||
}
|
||||
s.dir = dir
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Restore copies a snapshot's files back over dstDir and verifies every write
|
||||
// against the manifest sum. Only the named files are touched; anything else in
|
||||
// dstDir is left alone.
|
||||
//
|
||||
// This is the function the whole package exists to be able to run. It uses the
|
||||
// filesystem and nothing else — no toolchain, no build, no cooperation from the
|
||||
// code being replaced.
|
||||
func (s Snapshot) Restore(dstDir string) error {
|
||||
if s.dir == "" {
|
||||
return errors.New("update: snapshot has no directory (load it through the store)")
|
||||
}
|
||||
for _, f := range s.Files {
|
||||
src := filepath.Join(s.dir, f.Name)
|
||||
sum, err := hashFile(src)
|
||||
if err != nil {
|
||||
return fmt.Errorf("update: restore %s: %w", f.Name, err)
|
||||
}
|
||||
if sum != f.SHA256 {
|
||||
return fmt.Errorf("update: restore %s: snapshot is corrupt (sha256 %s, manifest says %s)", f.Name, sum, f.SHA256)
|
||||
}
|
||||
dst := filepath.Join(dstDir, f.Name)
|
||||
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
got, err := copyFile(src, dst, f.Mode)
|
||||
if err != nil {
|
||||
return fmt.Errorf("update: restore %s: %w", f.Name, err)
|
||||
}
|
||||
if got != f.SHA256 {
|
||||
return fmt.Errorf("update: restore %s: wrote the wrong bytes (sha256 %s)", f.Name, got)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Prune keeps the newest keep snapshots and removes the rest. The newest is
|
||||
// never pruned regardless of keep — it is the rollback target.
|
||||
func (st *Store) Prune(keep int) error {
|
||||
if keep < 1 {
|
||||
keep = 1
|
||||
}
|
||||
snaps, err := st.List()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, s := range snaps[min(keep, len(snaps)):] {
|
||||
if err := os.RemoveAll(s.dir); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// copyFile writes src to dst atomically (temp + rename, so a reader never sees a
|
||||
// half file and an interrupted copy leaves the old one intact) and returns the
|
||||
// sha256 of what was written.
|
||||
func copyFile(src, dst string, mode os.FileMode) (string, error) {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer in.Close()
|
||||
if mode == 0 {
|
||||
mode = 0o644
|
||||
}
|
||||
tmp, err := os.CreateTemp(filepath.Dir(dst), ".update-*")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
tmpName := tmp.Name()
|
||||
defer os.Remove(tmpName) // no-op once the rename succeeds
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(io.MultiWriter(tmp, h), in); err != nil {
|
||||
tmp.Close()
|
||||
return "", err
|
||||
}
|
||||
// fsync before the rename: a binary that is renamed into place but whose
|
||||
// bytes are still in the page cache is exactly the file a power cut turns
|
||||
// into an unbootable daemon.
|
||||
if err := tmp.Sync(); err != nil {
|
||||
tmp.Close()
|
||||
return "", err
|
||||
}
|
||||
if err := tmp.Chmod(mode); err != nil {
|
||||
tmp.Close()
|
||||
return "", err
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := os.Rename(tmpName, dst); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
|
||||
func hashFile(p string) (string, error) {
|
||||
f, err := os.Open(p)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer f.Close()
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(h, f); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
@@ -0,0 +1,270 @@
|
||||
// Package update applies a new build of Maven to the box she runs on, with a
|
||||
// verified-before-committed install and an automatic rollback (Vikunja #249).
|
||||
//
|
||||
// # What this package refuses to be
|
||||
//
|
||||
// This is the highest-risk capability in the backlog — code that changes the
|
||||
// running system — so the refusals are as much of the design as the features,
|
||||
// and they are enforced here rather than described in a doc:
|
||||
//
|
||||
// - It is never automatic and never on a timer. There is no checker, no
|
||||
// channel, no "check for updates" call and nothing that fires from the tick
|
||||
// loop. Apply runs exactly when a human runs cmd/mavupdate on the box.
|
||||
// - The daemon cannot update itself. mavend does not import this package and
|
||||
// there is no IPC method and no web route that reaches it, so no act, no
|
||||
// intent, no tool and no LLM output can start an update. The trigger needs
|
||||
// shell access to the host, which is a strictly higher bar than the step-up
|
||||
// passkey gate that guards /tools — an update is not a thing to expose to
|
||||
// anything reachable over the network.
|
||||
// - It does not fetch code. Nothing here talks to a release server, a
|
||||
// registry, or GitHub. The new version is whatever is in the working tree
|
||||
// the operator points it at, which he pulled himself. Downloading and
|
||||
// running code on the strength of a checksum in the same download is not a
|
||||
// property we can verify on one box.
|
||||
// - It does not supervise its own death. The plan asked for an in-process
|
||||
// crash-loop detector; a process cannot reliably notice that it keeps
|
||||
// dying, and one that thinks it can is worse than nothing. Restart-on-crash
|
||||
// belongs to whatever starts mavend (compose `restart: unless-stopped`,
|
||||
// systemd `Restart=`). What this package guarantees instead is narrower and
|
||||
// real: within one Apply, the new build is proven to answer before the old
|
||||
// one is considered replaced, and if it does not answer the old bytes go
|
||||
// back and are proven to answer again.
|
||||
//
|
||||
// # The order of operations, and why
|
||||
//
|
||||
// Apply is: health-check the CURRENT daemon → build → test → snapshot → install
|
||||
// → restart → health-check → rollback on any failure.
|
||||
//
|
||||
// The first health check is not ceremony. If she is already not answering, a
|
||||
// failed update and a broken box are indistinguishable afterwards, and the
|
||||
// rollback has nothing to prove itself against — so Apply refuses to start.
|
||||
//
|
||||
// Build and test run BEFORE anything is written to the install dir, so a broken
|
||||
// tree costs nothing but time. Install is per-file write-temp-then-rename, so a
|
||||
// crash mid-install leaves whole files, not half ones.
|
||||
//
|
||||
// The rollback path deliberately depends on nothing that just changed: it copies
|
||||
// byte-for-byte from a snapshot taken before the install and re-runs the same
|
||||
// restart command. It does not ask the new binary to do anything, does not run
|
||||
// a migration, and does not need the update to have gotten far enough to leave
|
||||
// a working anything behind.
|
||||
//
|
||||
// # What is out of scope on purpose
|
||||
//
|
||||
// The database is not snapshotted or rolled back. It is encrypted, live, and
|
||||
// often larger than the disk headroom; a store rolled back under a schema that
|
||||
// already migrated forward loses writes silently, which is worse than a failed
|
||||
// update. Schema compatibility is store.Migrate's job. A snapshot here is the
|
||||
// deployable artifacts only: binaries and config.
|
||||
package update
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrNotConfigured — no update block in the config. The capability does not
|
||||
// exist unless the operator described his own deployment.
|
||||
ErrNotConfigured = errors.New("update: not configured")
|
||||
|
||||
// ErrUnhealthyBefore — the daemon was already not answering when Apply
|
||||
// started. Refused: see the package comment.
|
||||
ErrUnhealthyBefore = errors.New("update: the running daemon is not healthy — refusing to update on top of a broken box")
|
||||
|
||||
// ErrVerifyFailed — build or test failed. Nothing was installed.
|
||||
ErrVerifyFailed = errors.New("update: verification failed")
|
||||
|
||||
// ErrRolledBack — the new build was installed and did not come up healthy,
|
||||
// so the previous snapshot was restored. Wraps the underlying failure.
|
||||
ErrRolledBack = errors.New("update: rolled back")
|
||||
|
||||
// ErrRollbackFailed — the worst case: the new build failed AND the restore
|
||||
// did not bring her back. The operator has to fix the box by hand; the
|
||||
// snapshot directory is named in the result so he knows what to copy.
|
||||
ErrRollbackFailed = errors.New("update: ROLLBACK FAILED — manual recovery required")
|
||||
)
|
||||
|
||||
// Config — the operator's description of his own deployment. Every path is
|
||||
// absolute and validated; nothing is guessed, because guessing wrong here means
|
||||
// overwriting the wrong file.
|
||||
type Config struct {
|
||||
// SourceDir — the git working tree to build. The operator pulls it himself;
|
||||
// this package never fetches.
|
||||
SourceDir string `json:"source_dir"`
|
||||
|
||||
// InstallDir — where the built binaries are copied to. On the docker
|
||||
// deployment this is the tree the image is built from, so it is usually the
|
||||
// same as SourceDir and Install is a no-op copy; on a bare-metal deployment
|
||||
// it is /opt/maven/bin.
|
||||
InstallDir string `json:"install_dir"`
|
||||
|
||||
// SnapshotDir — where the pre-install copies live. Must not be inside
|
||||
// InstallDir: a restore reading from a directory the install is writing to
|
||||
// is not a restore.
|
||||
SnapshotDir string `json:"snapshot_dir"`
|
||||
|
||||
// Binaries — the artifact names to snapshot and install, relative to
|
||||
// SourceDir (built) and InstallDir (deployed). Listed explicitly rather than
|
||||
// globbed so a stray file in the tree never gets deployed.
|
||||
Binaries []string `json:"binaries"`
|
||||
|
||||
// ConfigFiles — extra files to snapshot alongside the binaries, relative to
|
||||
// InstallDir. Snapshotted, never overwritten by an install: the operator's
|
||||
// config is not something an update gets to replace.
|
||||
ConfigFiles []string `json:"config_files,omitempty"`
|
||||
|
||||
// RestartCmd — how this deployment restarts mavend, e.g.
|
||||
// ["docker","compose","up","-d","--build","mavend"] or
|
||||
// ["systemctl","restart","mavend"]. Run in SourceDir. Required: there is no
|
||||
// portable default and picking one would mean restarting the wrong thing.
|
||||
RestartCmd []string `json:"restart_cmd"`
|
||||
|
||||
// HealthSocket — mavend's IPC socket, used to prove she answers after a
|
||||
// restart. Required: without a health check there is no signal to roll back
|
||||
// on, and an update that cannot detect its own failure is not what this
|
||||
// package is for.
|
||||
HealthSocket string `json:"health_socket"`
|
||||
|
||||
// HealthTimeoutSec — how long to wait for the restarted daemon to answer.
|
||||
// Default 90s; she loads a 1.7B on boot, so this is not a couple of seconds.
|
||||
HealthTimeoutSec int `json:"health_timeout_sec,omitempty"`
|
||||
|
||||
// VerifyTimeoutMin — cap on `make build` + `make test`. Default 20m.
|
||||
VerifyTimeoutMin int `json:"verify_timeout_min,omitempty"`
|
||||
|
||||
// KeepSnapshots — how many snapshots to retain. Default 5, minimum 1: the
|
||||
// most recent one is the rollback target and is never pruned.
|
||||
KeepSnapshots int `json:"keep_snapshots,omitempty"`
|
||||
}
|
||||
|
||||
// Validate — fail at startup, not halfway through an install.
|
||||
func (c Config) Validate() error {
|
||||
if c.SourceDir == "" || c.InstallDir == "" || c.SnapshotDir == "" {
|
||||
return errors.New("update: source_dir, install_dir and snapshot_dir are all required")
|
||||
}
|
||||
for _, p := range []string{c.SourceDir, c.InstallDir, c.SnapshotDir} {
|
||||
if !filepath.IsAbs(p) {
|
||||
return fmt.Errorf("update: %q must be an absolute path", p)
|
||||
}
|
||||
}
|
||||
if within(c.SnapshotDir, c.InstallDir) {
|
||||
return fmt.Errorf("update: snapshot_dir %q is inside install_dir %q — a restore must not read from what the install writes", c.SnapshotDir, c.InstallDir)
|
||||
}
|
||||
if len(c.Binaries) == 0 {
|
||||
return errors.New("update: binaries is empty — nothing to install")
|
||||
}
|
||||
for _, b := range append(append([]string{}, c.Binaries...), c.ConfigFiles...) {
|
||||
if filepath.IsAbs(b) || strings.Contains(b, "..") {
|
||||
return fmt.Errorf("update: %q must be a plain relative name", b)
|
||||
}
|
||||
}
|
||||
if len(c.RestartCmd) == 0 {
|
||||
return errors.New("update: restart_cmd is required — there is no safe default for restarting someone else's deployment")
|
||||
}
|
||||
if c.HealthSocket == "" {
|
||||
return errors.New("update: health_socket is required — an update that cannot check its own result cannot roll back on failure")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c Config) withDefaults() Config {
|
||||
if c.HealthTimeoutSec <= 0 {
|
||||
c.HealthTimeoutSec = 90
|
||||
}
|
||||
if c.VerifyTimeoutMin <= 0 {
|
||||
c.VerifyTimeoutMin = 20
|
||||
}
|
||||
if c.KeepSnapshots < 1 {
|
||||
c.KeepSnapshots = 5
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func (c Config) healthTimeout() time.Duration {
|
||||
return time.Duration(c.HealthTimeoutSec) * time.Second
|
||||
}
|
||||
|
||||
func (c Config) verifyTimeout() time.Duration {
|
||||
return time.Duration(c.VerifyTimeoutMin) * time.Minute
|
||||
}
|
||||
|
||||
// within reports whether p is dir or lives under it.
|
||||
func within(p, dir string) bool {
|
||||
p, dir = filepath.Clean(p), filepath.Clean(dir)
|
||||
if p == dir {
|
||||
return true
|
||||
}
|
||||
rel, err := filepath.Rel(dir, p)
|
||||
return err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator))
|
||||
}
|
||||
|
||||
// Runner runs one command and returns its combined output. Injected so the
|
||||
// tests can drive build/test/restart failures without a toolchain, a container
|
||||
// or a real daemon to break.
|
||||
type Runner func(ctx context.Context, dir string, argv []string) (string, error)
|
||||
|
||||
// ExecRunner is the real one.
|
||||
func ExecRunner(ctx context.Context, dir string, argv []string) (string, error) {
|
||||
cmd := exec.CommandContext(ctx, argv[0], argv[1:]...)
|
||||
cmd.Dir = dir
|
||||
out, err := cmd.CombinedOutput()
|
||||
return string(out), err
|
||||
}
|
||||
|
||||
// HealthCheck proves the daemon at socket answers. Injected for the same reason
|
||||
// as Runner.
|
||||
type HealthCheck func(ctx context.Context, socket string) error
|
||||
|
||||
// Logger receives one line per step. The CLI prints these as they happen: an
|
||||
// update that goes quiet for four minutes during `make test` reads as a hang.
|
||||
type Logger func(format string, args ...any)
|
||||
|
||||
// Updater is the whole capability. Construct with New and call Apply or
|
||||
// Rollback; there is no background goroutine and nothing starts on its own.
|
||||
type Updater struct {
|
||||
cfg Config
|
||||
store *Store
|
||||
run Runner
|
||||
health HealthCheck
|
||||
log Logger
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// New builds an Updater. Every seam has a real default; the tests replace them.
|
||||
func New(cfg Config, opts ...Option) (*Updater, error) {
|
||||
if err := cfg.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
u := &Updater{
|
||||
cfg: cfg.withDefaults(),
|
||||
store: &Store{Dir: cfg.SnapshotDir},
|
||||
run: ExecRunner,
|
||||
health: DialHealth,
|
||||
log: func(string, ...any) {},
|
||||
now: time.Now,
|
||||
}
|
||||
for _, o := range opts {
|
||||
o(u)
|
||||
}
|
||||
u.store.now = u.now
|
||||
return u, nil
|
||||
}
|
||||
|
||||
// Option — a constructor seam.
|
||||
type Option func(*Updater)
|
||||
|
||||
func WithRunner(r Runner) Option { return func(u *Updater) { u.run = r } }
|
||||
func WithHealth(h HealthCheck) Option { return func(u *Updater) { u.health = h } }
|
||||
func WithLogger(l Logger) Option { return func(u *Updater) { u.log = l } }
|
||||
func WithClock(f func() time.Time) Option {
|
||||
return func(u *Updater) { u.now = f }
|
||||
}
|
||||
|
||||
// Snapshots lists what is available to roll back to, newest first.
|
||||
func (u *Updater) Snapshots() ([]Snapshot, error) { return u.store.List() }
|
||||
@@ -0,0 +1,437 @@
|
||||
package update
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The tests drive the whole orchestration against a fake box: a directory tree
|
||||
// standing in for the install dir, an injected Runner standing in for
|
||||
// make/git/docker, and an injected HealthCheck standing in for mavend. That is
|
||||
// what makes the failure paths — the ones that matter — testable at all: you
|
||||
// cannot ask a real deployment to fail its health check on demand, and the
|
||||
// rollback path is exactly the path nobody exercises by hand.
|
||||
|
||||
type fakeBox struct {
|
||||
t *testing.T
|
||||
root string
|
||||
|
||||
// what the fake `make build` writes into the source tree
|
||||
newBytes string
|
||||
// scripted failures
|
||||
buildErr error
|
||||
testErr error
|
||||
restartErr error
|
||||
|
||||
// health: fails until the Nth call, then follows healthy
|
||||
healthErrs int // remaining failures to serve
|
||||
healthy bool
|
||||
healthChecks int
|
||||
// deployedAtRestart records the installed bytes each time restart runs, so a
|
||||
// test can prove the rollback put the old bytes back BEFORE restarting.
|
||||
deployedAtRestart []string
|
||||
|
||||
ran []string
|
||||
}
|
||||
|
||||
func newFakeBox(t *testing.T) *fakeBox {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
for _, d := range []string{"src", "install", "snapshots"} {
|
||||
if err := os.MkdirAll(filepath.Join(root, d), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// The currently deployed build, and a config file next to it.
|
||||
write(t, filepath.Join(root, "install", "mavend"), "OLD-BUILD")
|
||||
write(t, filepath.Join(root, "install", "mavend.json"), `{"tick_interval":"60s"}`)
|
||||
// The source tree already contains a stale binary; `make build` overwrites it.
|
||||
write(t, filepath.Join(root, "src", "mavend"), "STALE")
|
||||
return &fakeBox{t: t, root: root, newBytes: "NEW-BUILD", healthy: true}
|
||||
}
|
||||
|
||||
func (b *fakeBox) cfg() Config {
|
||||
return Config{
|
||||
SourceDir: filepath.Join(b.root, "src"),
|
||||
InstallDir: filepath.Join(b.root, "install"),
|
||||
SnapshotDir: filepath.Join(b.root, "snapshots"),
|
||||
Binaries: []string{"mavend"},
|
||||
ConfigFiles: []string{"mavend.json"},
|
||||
RestartCmd: []string{"restart-the-thing"},
|
||||
HealthSocket: filepath.Join(b.root, "mavend.sock"),
|
||||
HealthTimeoutSec: 1,
|
||||
KeepSnapshots: 3,
|
||||
}
|
||||
}
|
||||
|
||||
func (b *fakeBox) run(ctx context.Context, dir string, argv []string) (string, error) {
|
||||
b.ran = append(b.ran, strings.Join(argv, " "))
|
||||
switch strings.Join(argv, " ") {
|
||||
case "make build":
|
||||
if b.buildErr != nil {
|
||||
return "ld: undefined reference to everything", b.buildErr
|
||||
}
|
||||
// A real build writes its artifacts into the working tree — the behaviour
|
||||
// the snapshot-before-build ordering exists to survive.
|
||||
write(b.t, filepath.Join(b.root, "src", "mavend"), b.newBytes)
|
||||
return "built", nil
|
||||
case "make test":
|
||||
if b.testErr != nil {
|
||||
return "--- FAIL: TestSomething", b.testErr
|
||||
}
|
||||
return "ok", nil
|
||||
case "git rev-parse HEAD":
|
||||
return "cafebabecafebabecafebabecafebabecafebabe\n", nil
|
||||
case "restart-the-thing":
|
||||
b.deployedAtRestart = append(b.deployedAtRestart, read(b.t, filepath.Join(b.root, "install", "mavend")))
|
||||
if b.restartErr != nil {
|
||||
return "no such container", b.restartErr
|
||||
}
|
||||
return "restarted", nil
|
||||
}
|
||||
return "", errors.New("unexpected command: " + strings.Join(argv, " "))
|
||||
}
|
||||
|
||||
func (b *fakeBox) health(ctx context.Context, socket string) error {
|
||||
b.healthChecks++
|
||||
if b.healthErrs > 0 {
|
||||
b.healthErrs--
|
||||
return errors.New("connection refused")
|
||||
}
|
||||
if !b.healthy {
|
||||
return errors.New("she does not answer")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *fakeBox) updater(t *testing.T, extra ...Option) *Updater {
|
||||
t.Helper()
|
||||
opts := append([]Option{WithRunner(b.run), WithHealth(b.health)}, extra...)
|
||||
u, err := New(b.cfg(), opts...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return u
|
||||
}
|
||||
|
||||
func (b *fakeBox) deployed() string { return read(b.t, filepath.Join(b.root, "install", "mavend")) }
|
||||
|
||||
func write(t *testing.T, path, content string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path, []byte(content), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func read(t *testing.T, path string) string {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func TestApply_HappyPath(t *testing.T) {
|
||||
b := newFakeBox(t)
|
||||
res, err := b.updater(t).Apply(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Apply: %v", err)
|
||||
}
|
||||
if !res.Verified || !res.Restarted || !res.Healthy || res.RolledBack {
|
||||
t.Fatalf("result = %+v; want verified+restarted+healthy and no rollback", res)
|
||||
}
|
||||
if got := b.deployed(); got != "NEW-BUILD" {
|
||||
t.Errorf("deployed binary = %q; want the new build", got)
|
||||
}
|
||||
// The order is the property: health, snapshot, build, test, install, restart.
|
||||
want := []string{"git rev-parse HEAD", "make build", "make test", "restart-the-thing"}
|
||||
if strings.Join(b.ran, "|") != strings.Join(want, "|") {
|
||||
t.Errorf("commands ran = %v; want %v", b.ran, want)
|
||||
}
|
||||
if res.SnapshotID == "" {
|
||||
t.Error("no snapshot was taken")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApply_RefusesWhenSheIsAlreadyDown(t *testing.T) {
|
||||
// A box that is already broken has no baseline for the rollback to prove
|
||||
// itself against, so the update never starts.
|
||||
b := newFakeBox(t)
|
||||
b.healthy = false
|
||||
res, err := b.updater(t).Apply(context.Background())
|
||||
if !errors.Is(err, ErrUnhealthyBefore) {
|
||||
t.Fatalf("Apply on an unhealthy box = %v; want ErrUnhealthyBefore", err)
|
||||
}
|
||||
if len(b.ran) != 0 {
|
||||
t.Errorf("a refused update still ran %v", b.ran)
|
||||
}
|
||||
if res.SnapshotID != "" {
|
||||
t.Error("a refused update still took a snapshot")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApply_TestFailureDeploysNothingAndPutsTheTreeBack(t *testing.T) {
|
||||
b := newFakeBox(t)
|
||||
b.testErr = errors.New("exit status 1")
|
||||
res, err := b.updater(t).Apply(context.Background())
|
||||
if !errors.Is(err, ErrVerifyFailed) {
|
||||
t.Fatalf("Apply with failing tests = %v; want ErrVerifyFailed", err)
|
||||
}
|
||||
if res.Verified {
|
||||
t.Error("result claims verified after a failing test suite")
|
||||
}
|
||||
for _, c := range b.ran {
|
||||
if c == "restart-the-thing" {
|
||||
t.Fatal("a failed verification restarted the daemon")
|
||||
}
|
||||
}
|
||||
if got := b.deployed(); got != "OLD-BUILD" {
|
||||
t.Errorf("deployed binary = %q; want the old build untouched", got)
|
||||
}
|
||||
// The failing output is kept so the operator can see why.
|
||||
var found bool
|
||||
for _, s := range res.Steps {
|
||||
if s.Name == "test" && strings.Contains(s.Output, "FAIL") {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Error("the failing test output was not retained")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApply_BuildFailureIsCaughtBeforeTheTests(t *testing.T) {
|
||||
b := newFakeBox(t)
|
||||
b.buildErr = errors.New("exit status 2")
|
||||
if _, err := b.updater(t).Apply(context.Background()); !errors.Is(err, ErrVerifyFailed) {
|
||||
t.Fatalf("Apply with a failing build = %v; want ErrVerifyFailed", err)
|
||||
}
|
||||
for _, c := range b.ran {
|
||||
if c == "make test" {
|
||||
t.Error("ran the test suite after the build failed")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestApply_UnhealthyAfterRestartRollsBackToTheOldBytes(t *testing.T) {
|
||||
// The case the package exists for: everything verifies, the new build
|
||||
// installs, and then she does not come up.
|
||||
b := newFakeBox(t)
|
||||
b.healthErrs = 1 // the preflight check passes, then she stops answering
|
||||
b.healthy = false
|
||||
u := b.updater(t)
|
||||
// Once the rollback restores the old build, she answers again.
|
||||
restored := false
|
||||
u.health = func(ctx context.Context, socket string) error {
|
||||
b.healthChecks++
|
||||
if b.deployed() == "OLD-BUILD" && restored {
|
||||
return nil
|
||||
}
|
||||
if b.healthChecks == 1 {
|
||||
return nil // preflight: the old build is up
|
||||
}
|
||||
if b.deployed() == "OLD-BUILD" {
|
||||
restored = true
|
||||
return nil
|
||||
}
|
||||
return errors.New("she does not answer on the new build")
|
||||
}
|
||||
res, err := u.Apply(context.Background())
|
||||
if !errors.Is(err, ErrRolledBack) {
|
||||
t.Fatalf("Apply with a dead new build = %v; want ErrRolledBack", err)
|
||||
}
|
||||
if !res.RolledBack || !res.RollbackHealthy || res.Healthy {
|
||||
t.Fatalf("result = %+v; want rolled back and healthy again on the old build", res)
|
||||
}
|
||||
if got := b.deployed(); got != "OLD-BUILD" {
|
||||
t.Errorf("deployed binary after the rollback = %q; want OLD-BUILD", got)
|
||||
}
|
||||
// And the restore happened BEFORE the second restart, not after it.
|
||||
if len(b.deployedAtRestart) != 2 {
|
||||
t.Fatalf("restarts = %v; want two (the update and the rollback)", b.deployedAtRestart)
|
||||
}
|
||||
if b.deployedAtRestart[0] != "NEW-BUILD" || b.deployedAtRestart[1] != "OLD-BUILD" {
|
||||
t.Errorf("bytes in place at each restart = %v; want [NEW-BUILD OLD-BUILD]", b.deployedAtRestart)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApply_RestartFailureRollsBack(t *testing.T) {
|
||||
b := newFakeBox(t)
|
||||
b.restartErr = errors.New("exit status 1")
|
||||
res, err := b.updater(t).Apply(context.Background())
|
||||
// The rollback's own restart fails too, so this is the manual-recovery case —
|
||||
// and it says so instead of reporting a tidy rollback.
|
||||
if !errors.Is(err, ErrRollbackFailed) {
|
||||
t.Fatalf("Apply with a broken restart command = %v; want ErrRollbackFailed", err)
|
||||
}
|
||||
if !res.RolledBack || res.RollbackHealthy {
|
||||
t.Fatalf("result = %+v; want rolled back but not healthy", res)
|
||||
}
|
||||
if got := b.deployed(); got != "OLD-BUILD" {
|
||||
t.Errorf("deployed binary = %q; want the old bytes restored even so", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApply_RollbackNeedsNoBuildAndNoNewCode(t *testing.T) {
|
||||
// The rollback must not depend on the toolchain, the source tree, or the
|
||||
// code it is replacing. Prove it: delete the source tree's binary and make
|
||||
// every command except the restart fail, then roll back.
|
||||
b := newFakeBox(t)
|
||||
if _, err := b.updater(t).Apply(context.Background()); err != nil {
|
||||
t.Fatalf("setup Apply: %v", err)
|
||||
}
|
||||
if b.deployed() != "NEW-BUILD" {
|
||||
t.Fatal("setup did not deploy")
|
||||
}
|
||||
os.RemoveAll(filepath.Join(b.root, "src"))
|
||||
if err := os.MkdirAll(filepath.Join(b.root, "src"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.buildErr = errors.New("no toolchain here")
|
||||
b.testErr = errors.New("no toolchain here")
|
||||
b.ran = nil
|
||||
|
||||
res, err := b.updater(t).Rollback(context.Background(), "")
|
||||
if err != nil && !errors.Is(err, ErrRolledBack) {
|
||||
t.Fatalf("Rollback: %v", err)
|
||||
}
|
||||
if !res.RollbackHealthy {
|
||||
t.Fatalf("result = %+v; want a healthy rollback", res)
|
||||
}
|
||||
if got := b.deployed(); got != "OLD-BUILD" {
|
||||
t.Errorf("deployed binary = %q; want OLD-BUILD", got)
|
||||
}
|
||||
for _, c := range b.ran {
|
||||
if strings.HasPrefix(c, "make") {
|
||||
t.Errorf("the rollback ran %q — it must not need a build", c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestApply_ConfigIsSnapshottedButNeverOverwritten(t *testing.T) {
|
||||
b := newFakeBox(t)
|
||||
// A config in the source tree must not be deployed over the operator's.
|
||||
write(t, filepath.Join(b.root, "src", "mavend.json"), `{"tick_interval":"1s"}`)
|
||||
if _, err := b.updater(t).Apply(context.Background()); err != nil {
|
||||
t.Fatalf("Apply: %v", err)
|
||||
}
|
||||
if got := read(t, filepath.Join(b.root, "install", "mavend.json")); !strings.Contains(got, "60s") {
|
||||
t.Errorf("installed config = %q; an update must not replace his config", got)
|
||||
}
|
||||
snaps, err := b.updater(t).Snapshots()
|
||||
if err != nil || len(snaps) == 0 {
|
||||
t.Fatalf("Snapshots: %v %v", snaps, err)
|
||||
}
|
||||
var names []string
|
||||
for _, f := range snaps[0].Files {
|
||||
names = append(names, f.Name)
|
||||
}
|
||||
if len(names) != 2 {
|
||||
t.Errorf("snapshot files = %v; want the binary and the config", names)
|
||||
}
|
||||
if snaps[0].Commit == "" {
|
||||
t.Error("the snapshot did not record which commit produced it")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRollback_CorruptSnapshotIsRefusedNotRestored(t *testing.T) {
|
||||
b := newFakeBox(t)
|
||||
if _, err := b.updater(t).Apply(context.Background()); err != nil {
|
||||
t.Fatalf("setup Apply: %v", err)
|
||||
}
|
||||
snaps, _ := b.updater(t).Snapshots()
|
||||
// Something ate the snapshot. Restoring it would deploy garbage.
|
||||
write(t, filepath.Join(snaps[0].Dir(), "mavend"), "CORRUPT")
|
||||
_, err := b.updater(t).Rollback(context.Background(), snaps[0].ID)
|
||||
if !errors.Is(err, ErrRollbackFailed) || !strings.Contains(err.Error(), "corrupt") {
|
||||
t.Fatalf("Rollback of a corrupt snapshot = %v; want a refusal naming the corruption", err)
|
||||
}
|
||||
if got := b.deployed(); got != "NEW-BUILD" {
|
||||
t.Errorf("deployed binary = %q; a refused restore must change nothing", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRollback_NoSnapshots(t *testing.T) {
|
||||
b := newFakeBox(t)
|
||||
if _, err := b.updater(t).Rollback(context.Background(), ""); err == nil {
|
||||
t.Error("Rollback with no snapshots succeeded; want an error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrune_KeepsTheNewestAsTheRollbackTarget(t *testing.T) {
|
||||
b := newFakeBox(t)
|
||||
st := &Store{Dir: filepath.Join(b.root, "snapshots")}
|
||||
base := time.Date(2026, 8, 1, 3, 0, 0, 0, time.UTC)
|
||||
for i := 0; i < 4; i++ {
|
||||
i := i
|
||||
st.now = func() time.Time { return base.Add(time.Duration(i) * time.Minute) }
|
||||
if _, err := st.Save(filepath.Join(b.root, "install"), []string{"mavend"}, "", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := st.Prune(0); err != nil { // 0 is clamped to 1, never to zero
|
||||
t.Fatal(err)
|
||||
}
|
||||
snaps, err := st.List()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(snaps) != 1 {
|
||||
t.Fatalf("kept %d snapshots; want 1", len(snaps))
|
||||
}
|
||||
if snaps[0].ID != "20260801-030300" {
|
||||
t.Errorf("kept %s; want the newest", snaps[0].ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestList_IgnoresSnapshotsWithNoManifest(t *testing.T) {
|
||||
// An interrupted snapshot has files but no manifest. It must never be offered
|
||||
// as a rollback target — restoring a half-copied binary is the worst outcome
|
||||
// in the package.
|
||||
b := newFakeBox(t)
|
||||
dir := filepath.Join(b.root, "snapshots", "20260801-000000")
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
write(t, filepath.Join(dir, "mavend"), "HALF")
|
||||
snaps, err := (&Store{Dir: filepath.Join(b.root, "snapshots")}).List()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(snaps) != 0 {
|
||||
t.Errorf("List returned %d snapshots; want none (no manifest)", len(snaps))
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigValidate(t *testing.T) {
|
||||
ok := (&fakeBox{root: t.TempDir()}).cfg()
|
||||
if err := ok.Validate(); err != nil {
|
||||
t.Fatalf("valid config rejected: %v", err)
|
||||
}
|
||||
bad := map[string]func(c Config) Config{
|
||||
"relative source": func(c Config) Config { c.SourceDir = "src"; return c },
|
||||
"no restart command": func(c Config) Config { c.RestartCmd = nil; return c },
|
||||
"no health socket": func(c Config) Config { c.HealthSocket = ""; return c },
|
||||
"no binaries": func(c Config) Config { c.Binaries = nil; return c },
|
||||
"escaping artifact name": func(c Config) Config { c.Binaries = []string{"../../etc/passwd"}; return c },
|
||||
"absolute artifact name": func(c Config) Config { c.Binaries = []string{"/usr/bin/mavend"}; return c },
|
||||
"snapshots inside install": func(c Config) Config { c.SnapshotDir = filepath.Join(c.InstallDir, "snaps"); return c },
|
||||
}
|
||||
for name, mutate := range bad {
|
||||
if err := mutate(ok).Validate(); err == nil {
|
||||
t.Errorf("%s was accepted; want a startup failure", name)
|
||||
}
|
||||
}
|
||||
// And New refuses an invalid config outright rather than half-configuring.
|
||||
if _, err := New(mutate(ok, "no health socket", bad)); err == nil {
|
||||
t.Error("New accepted a config with no health socket")
|
||||
}
|
||||
}
|
||||
|
||||
func mutate(c Config, key string, m map[string]func(Config) Config) Config { return m[key](c) }
|
||||
@@ -0,0 +1,65 @@
|
||||
package update
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Verification is "does this tree build and does it pass its own tests", run
|
||||
// before a single byte is written to the install dir.
|
||||
//
|
||||
// It is `make build` and `make test`, not `go build`: the CGO daemons need the
|
||||
// vendored toolchain and the whisper/piper include and library paths wired
|
||||
// through the Makefile, and a bare `go build` on them fails in a way that has
|
||||
// nothing to do with the change being deployed. `make test` is the -race suite
|
||||
// with the CGO env set, and it is the only evidence available on a single box
|
||||
// that the new code does what the old code did.
|
||||
//
|
||||
// This is not a substitute for a second environment. A test suite that passes
|
||||
// says the code is self-consistent; it does not say the new build will start
|
||||
// against this machine's actual models, sockets and encrypted store. That is
|
||||
// what the post-restart health check is for, and it is why the install is
|
||||
// reversible rather than merely careful.
|
||||
|
||||
// Step — one verification or orchestration step and how it went. Kept so the CLI
|
||||
// can print a truthful account of what was done, including on the failure path.
|
||||
type Step struct {
|
||||
Name string
|
||||
Argv []string
|
||||
Took time.Duration
|
||||
Err error
|
||||
Output string // combined output, only retained for failures
|
||||
}
|
||||
|
||||
// Verify runs the build and the test suite in SourceDir.
|
||||
func (u *Updater) Verify(ctx context.Context) ([]Step, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, u.cfg.verifyTimeout())
|
||||
defer cancel()
|
||||
var steps []Step
|
||||
for _, argv := range [][]string{{"make", "build"}, {"make", "test"}} {
|
||||
u.log("verify: %v (this takes a while)", argv)
|
||||
start := u.now()
|
||||
out, err := u.run(ctx, u.cfg.SourceDir, argv)
|
||||
st := Step{Name: argv[len(argv)-1], Argv: argv, Took: u.now().Sub(start), Err: err}
|
||||
if err != nil {
|
||||
st.Output = tail(out, 4000)
|
||||
}
|
||||
steps = append(steps, st)
|
||||
if err != nil {
|
||||
u.log("verify: %v FAILED after %s", argv, st.Took.Round(time.Second))
|
||||
return steps, fmt.Errorf("%w: %v: %v", ErrVerifyFailed, argv, err)
|
||||
}
|
||||
u.log("verify: %v ok in %s", argv, st.Took.Round(time.Second))
|
||||
}
|
||||
return steps, nil
|
||||
}
|
||||
|
||||
// tail keeps the last n bytes — a failing `make test` prints far more than is
|
||||
// useful, and the failure is always at the end.
|
||||
func tail(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
return "…" + s[len(s)-n:]
|
||||
}
|
||||
@@ -0,0 +1,301 @@
|
||||
// Package webfetch is the one door Maven uses to read something off the
|
||||
// network, and it is a narrow one.
|
||||
//
|
||||
// "Never phones home" stopped being a hard constraint on 2026-07-31, but what
|
||||
// replaced it is not "she may fetch anything": local sources come first (Kiwix
|
||||
// on the box), external fetching is off unless configured, and only the
|
||||
// utterance ever leaves — never his notes, facts or history. That policy is
|
||||
// enforced by the callers. What THIS package enforces is the part that must be
|
||||
// code rather than a paragraph in a plan, because it protects the homelab from
|
||||
// its own assistant:
|
||||
//
|
||||
// - http/https only — no file://, no ftp://, no gopher;
|
||||
// - no private address, ever: loopback, RFC1918 (which is what makes the
|
||||
// 10.42.0.0/24 wireguard tunnel and the 192.168.1.0/24 LAN unreachable),
|
||||
// link-local incl. the 169.254.169.254 cloud metadata address, CGNAT,
|
||||
// unique-local v6. Checked in the dialer's Control hook, so it holds for
|
||||
// every address the resolver returns AND for every hop of a redirect
|
||||
// chain — a DNS name that resolves to 127.0.0.1 is refused at connect
|
||||
// time, which a pre-flight lookup could not promise (rebinding);
|
||||
// - an allowlist, when one is configured, and a denylist that always wins;
|
||||
// - a response size cap, a total timeout, a redirect cap;
|
||||
// - one request per host per interval, so a poll loop with a bug is slow
|
||||
// rather than an outbound flood.
|
||||
//
|
||||
// Everything above is on by default with sane numbers: a zero Config is a
|
||||
// usable, conservative fetcher. There is no cache and no retry — a feed poll
|
||||
// or a page read that fails is simply not answered this round.
|
||||
package webfetch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Defaults. Small on purpose: this reads feeds and article pages, not ISOs.
|
||||
const (
|
||||
DefaultTimeout = 20 * time.Second
|
||||
DefaultMaxBytes = 2 << 20 // 2 MiB
|
||||
DefaultMaxRedirects = 3
|
||||
DefaultHostInterval = time.Second
|
||||
DefaultUserAgent = "Maven/1.0 (self-hosted personal assistant)"
|
||||
)
|
||||
|
||||
// Errors callers distinguish. Everything else is wrapped transport error.
|
||||
var (
|
||||
ErrScheme = errors.New("webfetch: only http and https are allowed")
|
||||
ErrBlocked = errors.New("webfetch: host is not allowed")
|
||||
ErrPrivate = errors.New("webfetch: refusing to connect to a private address")
|
||||
ErrTooLarge = errors.New("webfetch: response exceeds the size cap")
|
||||
ErrRedirects = errors.New("webfetch: too many redirects")
|
||||
ErrStatus = errors.New("webfetch: non-2xx status")
|
||||
)
|
||||
|
||||
// Config are the limits. Every zero value means "the default above", so
|
||||
// Config{} is safe; the only field that changes behaviour by being empty is
|
||||
// AllowHosts (empty ⇒ any public host that is not denied).
|
||||
type Config struct {
|
||||
// AllowHosts — when non-empty, the ONLY hosts that may be fetched. An
|
||||
// entry matches the host itself and its subdomains ("example.com" allows
|
||||
// "news.example.com"). This is the knob to reach for when a capability
|
||||
// should read two feeds and nothing else.
|
||||
AllowHosts []string
|
||||
// DenyHosts — same matching, checked first and always winning.
|
||||
DenyHosts []string
|
||||
|
||||
Timeout time.Duration // whole request, including redirects and body read
|
||||
MaxBytes int64 // response body cap
|
||||
MaxRedirects int // 0 ⇒ default; negative ⇒ no redirects followed
|
||||
HostInterval time.Duration // minimum spacing between requests to one host
|
||||
UserAgent string
|
||||
|
||||
// AllowPrivate disables the private-address guard. It exists for tests
|
||||
// (httptest listens on 127.0.0.1) and for an explicitly configured
|
||||
// on-box mirror. Nothing in deploy/mavend.json sets it, and it should
|
||||
// stay that way: with it on, any URL Maven is handed becomes an SSRF
|
||||
// probe of the LAN and the wireguard range.
|
||||
AllowPrivate bool
|
||||
}
|
||||
|
||||
// Response is a fetched body, already bounded by MaxBytes.
|
||||
type Response struct {
|
||||
URL string // final URL after redirects
|
||||
Status int
|
||||
ContentType string
|
||||
Body []byte
|
||||
}
|
||||
|
||||
// Fetcher performs guarded GETs. Safe for concurrent use; the per-host rate
|
||||
// limiter is shared, which is the point of sharing one Fetcher.
|
||||
type Fetcher struct {
|
||||
cfg Config
|
||||
http *http.Client
|
||||
|
||||
mu sync.Mutex
|
||||
last map[string]time.Time // host → when we last dialed it
|
||||
}
|
||||
|
||||
// New builds a fetcher from cfg, filling in defaults.
|
||||
func New(cfg Config) *Fetcher {
|
||||
if cfg.Timeout <= 0 {
|
||||
cfg.Timeout = DefaultTimeout
|
||||
}
|
||||
if cfg.MaxBytes <= 0 {
|
||||
cfg.MaxBytes = DefaultMaxBytes
|
||||
}
|
||||
if cfg.MaxRedirects == 0 {
|
||||
cfg.MaxRedirects = DefaultMaxRedirects
|
||||
}
|
||||
if cfg.HostInterval <= 0 {
|
||||
cfg.HostInterval = DefaultHostInterval
|
||||
}
|
||||
if cfg.UserAgent == "" {
|
||||
cfg.UserAgent = DefaultUserAgent
|
||||
}
|
||||
f := &Fetcher{cfg: cfg, last: map[string]time.Time{}}
|
||||
|
||||
dialer := &net.Dialer{Timeout: 10 * time.Second}
|
||||
if !cfg.AllowPrivate {
|
||||
// The guard lives here rather than in a pre-flight net.LookupHost so
|
||||
// that it sees the address actually being connected to: every A/AAAA
|
||||
// the resolver handed back, on every redirect hop, with no window in
|
||||
// which the name could be re-pointed at the LAN.
|
||||
dialer.Control = func(_, address string, _ syscall.RawConn) error {
|
||||
host, _, err := net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ip := net.ParseIP(host)
|
||||
if ip == nil || IsPrivateIP(ip) {
|
||||
return fmt.Errorf("%w: %s", ErrPrivate, host)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
f.http = &http.Client{
|
||||
Timeout: cfg.Timeout,
|
||||
Transport: &http.Transport{DialContext: dialer.DialContext},
|
||||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||||
if len(via) > f.cfg.MaxRedirects {
|
||||
return ErrRedirects
|
||||
}
|
||||
// A redirect is a fresh URL and gets the full check: an allowed
|
||||
// host must not be able to bounce us onto a denied one.
|
||||
return f.checkURL(req.URL)
|
||||
},
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// Get fetches rawURL. The body is capped: a larger response is an error, not a
|
||||
// truncation, because half an XML document is worse than none.
|
||||
func (f *Fetcher) Get(ctx context.Context, rawURL string) (*Response, error) {
|
||||
u, err := url.Parse(strings.TrimSpace(rawURL))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("webfetch: bad url %q: %w", rawURL, err)
|
||||
}
|
||||
if err := f.checkURL(u); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := f.waitTurn(ctx, u.Hostname()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("User-Agent", f.cfg.UserAgent)
|
||||
req.Header.Set("Accept-Encoding", "identity")
|
||||
|
||||
resp, err := f.http.Do(req)
|
||||
if err != nil {
|
||||
// http.Client wraps our sentinels in *url.Error; unwrap so callers can
|
||||
// still tell "blocked" from "the network is down".
|
||||
for _, sentinel := range []error{ErrPrivate, ErrBlocked, ErrRedirects, ErrScheme} {
|
||||
if errors.Is(err, sentinel) {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, f.cfg.MaxBytes+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(body)) > f.cfg.MaxBytes {
|
||||
return nil, fmt.Errorf("%w (%d bytes)", ErrTooLarge, f.cfg.MaxBytes)
|
||||
}
|
||||
if resp.StatusCode < 200 || resp.StatusCode > 299 {
|
||||
return nil, fmt.Errorf("%w: %d", ErrStatus, resp.StatusCode)
|
||||
}
|
||||
return &Response{
|
||||
URL: resp.Request.URL.String(),
|
||||
Status: resp.StatusCode,
|
||||
ContentType: resp.Header.Get("Content-Type"),
|
||||
Body: body,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// checkURL applies the scheme rule and the host lists. The address rule is the
|
||||
// dialer's job (see New).
|
||||
func (f *Fetcher) checkURL(u *url.URL) error {
|
||||
switch u.Scheme {
|
||||
case "http", "https":
|
||||
default:
|
||||
return fmt.Errorf("%w: %q", ErrScheme, u.Scheme)
|
||||
}
|
||||
host := strings.ToLower(u.Hostname())
|
||||
if host == "" {
|
||||
return fmt.Errorf("%w: no host", ErrBlocked)
|
||||
}
|
||||
if HostMatches(host, f.cfg.DenyHosts) {
|
||||
return fmt.Errorf("%w: %s is denied", ErrBlocked, host)
|
||||
}
|
||||
if len(f.cfg.AllowHosts) > 0 && !HostMatches(host, f.cfg.AllowHosts) {
|
||||
return fmt.Errorf("%w: %s is not on the allowlist", ErrBlocked, host)
|
||||
}
|
||||
// A literal private address is refused here as well as in the dialer, so
|
||||
// the error is the specific one even when no connection is attempted.
|
||||
if !f.cfg.AllowPrivate {
|
||||
if ip := net.ParseIP(host); ip != nil && IsPrivateIP(ip) {
|
||||
return fmt.Errorf("%w: %s", ErrPrivate, host)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// waitTurn blocks until this host's rate-limit interval has elapsed. It holds
|
||||
// no lock while sleeping, so two hosts never wait on each other.
|
||||
func (f *Fetcher) waitTurn(ctx context.Context, host string) error {
|
||||
for {
|
||||
f.mu.Lock()
|
||||
now := time.Now()
|
||||
earliest := f.last[host].Add(f.cfg.HostInterval)
|
||||
if !now.Before(earliest) {
|
||||
f.last[host] = now
|
||||
f.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
f.mu.Unlock()
|
||||
wait := time.NewTimer(earliest.Sub(now))
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
wait.Stop()
|
||||
return ctx.Err()
|
||||
case <-wait.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// HostMatches reports whether host equals one of pats or is a subdomain of one.
|
||||
// Exported because the crawler applies the same rule to links it decides not to
|
||||
// follow, before it ever builds a request.
|
||||
func HostMatches(host string, pats []string) bool {
|
||||
host = strings.ToLower(strings.TrimSuffix(host, "."))
|
||||
for _, p := range pats {
|
||||
p = strings.ToLower(strings.TrimSpace(strings.TrimPrefix(p, "*.")))
|
||||
if p == "" {
|
||||
continue
|
||||
}
|
||||
if host == p || strings.HasSuffix(host, "."+p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// cgnat is 100.64.0.0/10 — carrier NAT, not covered by net.IP's helpers and not
|
||||
// somewhere a personal assistant has business connecting.
|
||||
var cgnat = &net.IPNet{IP: net.IPv4(100, 64, 0, 0).To4(), Mask: net.CIDRMask(10, 32)}
|
||||
|
||||
// IsPrivateIP reports whether ip is somewhere Maven must never reach out to:
|
||||
// the box itself, the LAN, the wireguard range (10.42.0.0/24 ⊂ 10/8), the cloud
|
||||
// metadata address (169.254.169.254 ⊂ link-local), or anything unroutable.
|
||||
func IsPrivateIP(ip net.IP) bool {
|
||||
if ip.IsLoopback() || ip.IsPrivate() || ip.IsUnspecified() ||
|
||||
ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
|
||||
ip.IsInterfaceLocalMulticast() || ip.IsMulticast() {
|
||||
return true
|
||||
}
|
||||
if v4 := ip.To4(); v4 != nil && cgnat.Contains(v4) {
|
||||
return true
|
||||
}
|
||||
// IPv4-mapped/compatible forms of the above are handled by To4() inside the
|
||||
// stdlib helpers; what is left is v6 unique-local (fc00::/7).
|
||||
if len(ip) == net.IPv6len && ip.To4() == nil && ip[0]&0xfe == 0xfc {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
package webfetch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The limits in this package are the reason a crawler is allowed to exist on
|
||||
// this box at all, so each one has a test that fails loudly if it is removed.
|
||||
|
||||
func TestPrivateAddressesAreRefused(t *testing.T) {
|
||||
// The wireguard range (10.42.0.0/24), the LAN (192.168.1.0/24) and the
|
||||
// cloud metadata address are the three that matter here; the rest come
|
||||
// along for free.
|
||||
for _, s := range []string{
|
||||
"127.0.0.1", "127.1.2.3", "10.42.0.7", "10.0.0.5", "192.168.1.104",
|
||||
"172.16.4.4", "169.254.169.254", "100.64.1.1", "0.0.0.0",
|
||||
"::1", "fc00::1", "fd12:3456::1", "fe80::1",
|
||||
} {
|
||||
if !IsPrivateIP(net.ParseIP(s)) {
|
||||
t.Errorf("IsPrivateIP(%s) = false, want true", s)
|
||||
}
|
||||
}
|
||||
for _, s := range []string{"8.8.8.8", "1.1.1.1", "93.184.216.34", "2606:2800:220:1::1"} {
|
||||
if IsPrivateIP(net.ParseIP(s)) {
|
||||
t.Errorf("IsPrivateIP(%s) = true, want false", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetRefusesPrivateLiteral(t *testing.T) {
|
||||
f := New(Config{})
|
||||
for _, u := range []string{
|
||||
"http://127.0.0.1:8034/search",
|
||||
"http://10.42.0.1/",
|
||||
"http://192.168.1.104/dash",
|
||||
"http://[::1]:9100/mcp",
|
||||
} {
|
||||
if _, err := f.Get(context.Background(), u); !errors.Is(err, ErrPrivate) {
|
||||
t.Errorf("Get(%s) error = %v, want ErrPrivate", u, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A hostname that resolves into private space must fail too — that is the
|
||||
// rebinding case, and it is why the check lives in the dialer.
|
||||
func TestGetRefusesPrivateResolution(t *testing.T) {
|
||||
f := New(Config{})
|
||||
if _, err := f.Get(context.Background(), "http://localhost:8034/"); !errors.Is(err, ErrPrivate) {
|
||||
t.Fatalf("Get(localhost) error = %v, want ErrPrivate", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetRefusesNonHTTPSchemes(t *testing.T) {
|
||||
f := New(Config{})
|
||||
for _, u := range []string{"file:///etc/passwd", "ftp://example.com/x", "gopher://example.com"} {
|
||||
if _, err := f.Get(context.Background(), u); !errors.Is(err, ErrScheme) {
|
||||
t.Errorf("Get(%s) error = %v, want ErrScheme", u, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// testFetcher — a fetcher pointed at an httptest server, which necessarily
|
||||
// listens on loopback. AllowPrivate is the test-only escape hatch.
|
||||
func testFetcher(t *testing.T, cfg Config) *Fetcher {
|
||||
t.Helper()
|
||||
cfg.AllowPrivate = true
|
||||
if cfg.HostInterval == 0 {
|
||||
cfg.HostInterval = time.Nanosecond
|
||||
}
|
||||
return New(cfg)
|
||||
}
|
||||
|
||||
func TestAllowAndDenyLists(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("ok"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
f := testFetcher(t, Config{AllowHosts: []string{"example.com"}})
|
||||
if _, err := f.Get(context.Background(), srv.URL); !errors.Is(err, ErrBlocked) {
|
||||
t.Fatalf("off-allowlist host: error = %v, want ErrBlocked", err)
|
||||
}
|
||||
f = testFetcher(t, Config{DenyHosts: []string{"127.0.0.1"}})
|
||||
if _, err := f.Get(context.Background(), srv.URL); !errors.Is(err, ErrBlocked) {
|
||||
t.Fatalf("denied host: error = %v, want ErrBlocked", err)
|
||||
}
|
||||
f = testFetcher(t, Config{AllowHosts: []string{"127.0.0.1"}})
|
||||
if _, err := f.Get(context.Background(), srv.URL); err != nil {
|
||||
t.Fatalf("allowlisted host: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostMatchesSubdomains(t *testing.T) {
|
||||
pats := []string{"example.com", "*.news.org"}
|
||||
for _, h := range []string{"example.com", "news.example.com", "a.b.example.com", "news.org", "feeds.news.org"} {
|
||||
if !HostMatches(h, pats) {
|
||||
t.Errorf("HostMatches(%q) = false, want true", h)
|
||||
}
|
||||
}
|
||||
for _, h := range []string{"notexample.com", "example.com.evil.net", "org"} {
|
||||
if HostMatches(h, pats) {
|
||||
t.Errorf("HostMatches(%q) = true, want false", h)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSizeCap(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(strings.Repeat("x", 5000)))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
f := testFetcher(t, Config{MaxBytes: 100})
|
||||
if _, err := f.Get(context.Background(), srv.URL); !errors.Is(err, ErrTooLarge) {
|
||||
t.Fatalf("error = %v, want ErrTooLarge", err)
|
||||
}
|
||||
f = testFetcher(t, Config{MaxBytes: 6000})
|
||||
resp, err := f.Get(context.Background(), srv.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("under the cap: %v", err)
|
||||
}
|
||||
if len(resp.Body) != 5000 {
|
||||
t.Fatalf("body = %d bytes, want 5000", len(resp.Body))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedirectCap(t *testing.T) {
|
||||
var srv *httptest.Server
|
||||
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, srv.URL+"/again", http.StatusFound)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
f := testFetcher(t, Config{MaxRedirects: 2})
|
||||
if _, err := f.Get(context.Background(), srv.URL); !errors.Is(err, ErrRedirects) {
|
||||
t.Fatalf("error = %v, want ErrRedirects", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A redirect off the allowlist is the interesting redirect: the first hop is
|
||||
// permitted, the second must not be.
|
||||
func TestRedirectRecheckedAgainstDenylist(t *testing.T) {
|
||||
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("secret"))
|
||||
}))
|
||||
defer target.Close()
|
||||
hop := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, target.URL, http.StatusFound)
|
||||
}))
|
||||
defer hop.Close()
|
||||
|
||||
// Reach the hop under the name "localhost" and allow only that name; the
|
||||
// redirect lands on the same box under its literal address, which the
|
||||
// allowlist does not cover. Without the CheckRedirect hook this fetch
|
||||
// succeeds and returns "secret".
|
||||
f := testFetcher(t, Config{AllowHosts: []string{"localhost"}})
|
||||
viaName := strings.Replace(hop.URL, "127.0.0.1", "localhost", 1)
|
||||
if _, err := f.Get(context.Background(), viaName); !errors.Is(err, ErrBlocked) {
|
||||
t.Fatalf("error = %v, want ErrBlocked", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPerHostRateLimit(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("ok"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
f := testFetcher(t, Config{HostInterval: 60 * time.Millisecond})
|
||||
start := time.Now()
|
||||
for i := 0; i < 3; i++ {
|
||||
if _, err := f.Get(context.Background(), srv.URL); err != nil {
|
||||
t.Fatalf("request %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed < 120*time.Millisecond {
|
||||
t.Fatalf("three requests took %s, want at least 120ms of spacing", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRateLimitHonoursContext(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
|
||||
defer srv.Close()
|
||||
|
||||
f := testFetcher(t, Config{HostInterval: 10 * time.Second})
|
||||
if _, err := f.Get(context.Background(), srv.URL); err != nil {
|
||||
t.Fatalf("first request: %v", err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
|
||||
defer cancel()
|
||||
if _, err := f.Get(ctx, srv.URL); !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("error = %v, want DeadlineExceeded", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNon2xxIsAnError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "nope", http.StatusInternalServerError)
|
||||
}))
|
||||
defer srv.Close()
|
||||
f := testFetcher(t, Config{})
|
||||
if _, err := f.Get(context.Background(), srv.URL); !errors.Is(err, ErrStatus) {
|
||||
t.Fatalf("error = %v, want ErrStatus", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUserAgentIsSent(t *testing.T) {
|
||||
got := make(chan string, 1)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
got <- r.Header.Get("User-Agent")
|
||||
}))
|
||||
defer srv.Close()
|
||||
f := testFetcher(t, Config{UserAgent: "Maven/test"})
|
||||
if _, err := f.Get(context.Background(), srv.URL); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ua := <-got; ua != "Maven/test" {
|
||||
t.Fatalf("user-agent = %q", ua)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user