Compare commits
15 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| aa1a26532c | |||
| d92349ca6e | |||
| 8d5e357b57 | |||
| 95ae900a58 | |||
| be066a4b04 | |||
| ad074cea31 | |||
| 2c1b0eede0 | |||
| cb3641e7bb | |||
| ee7bec11e3 | |||
| f42d1594ef | |||
| b4646155b4 | |||
| da647e87d0 | |||
| bf6ccf9aea | |||
| 7b2b96b957 | |||
| c8444813e2 |
@@ -7,6 +7,8 @@
|
||||
/mavpoll
|
||||
/mavcaldav
|
||||
/mavwaked
|
||||
/mavmaild
|
||||
/mavupdate
|
||||
|
||||
# Certs (private keys, don't commit)
|
||||
certs/
|
||||
@@ -34,6 +36,10 @@ deps
|
||||
deploy/db_key.env
|
||||
# Deploy secret (telegram bot token + chat id) — never commit
|
||||
deploy/telegram.env
|
||||
# zenmoney API token, read by mavpoll (never in argv, never committed)
|
||||
deploy/zenmoney.token
|
||||
# IMAP password, read by mavmaild (never in argv, never committed)
|
||||
deploy/imap.password
|
||||
|
||||
# Temp files
|
||||
/tmp/
|
||||
|
||||
@@ -34,7 +34,7 @@ CGO daemons (`mavend`, `mavsttd`, `mavttsd`, `mavenclient`) need the vendored to
|
||||
and libs wired through the Makefile — **do not** call `go build` on them bare, use `make`:
|
||||
|
||||
```sh
|
||||
make build # all 8 binaries
|
||||
make build # all 9 binaries
|
||||
make build-web # single daemon (pure-Go ones: web/waked/poll/caldav build without CGO)
|
||||
make test # go test -race across ./internal/... ./cmd/... with CGO env set
|
||||
```
|
||||
@@ -62,6 +62,7 @@ Pure-Go packages (`router`, `memory`, `mavweb`, …) run under a plain `go test
|
||||
| `mavenclient` | Voice loop client (mic → stt → core → tts). |
|
||||
| `mavpoll` | Telegram long-poll reach. |
|
||||
| `mavcaldav` | CalDAV calendar sync. |
|
||||
| `mavmaild` | Mail reader (IMAP, read-only). Holds the IMAP password; core never sees it. |
|
||||
|
||||
Daemons are wired socket-to-socket, not linked. `internal/ipc` is the client/server wire
|
||||
protocol; the config in `deploy/mavend.json` (with `${VAR}` env expansion from gitignored
|
||||
|
||||
+2
-1
@@ -51,7 +51,8 @@ RUN go build -o /out/mavend ./cmd/mavend && \
|
||||
go build -o /out/mavttsd ./cmd/mavttsd && \
|
||||
go build -o /out/mavweb ./cmd/mavweb && \
|
||||
go build -o /out/mavpoll ./cmd/mavpoll && \
|
||||
go build -o /out/mavcaldav ./cmd/mavcaldav
|
||||
go build -o /out/mavcaldav ./cmd/mavcaldav && \
|
||||
go build -o /out/mavmaild ./cmd/mavmaild
|
||||
|
||||
# llama.cpp Vulkan build — the phraser/router LFM engine (llama-server). Built
|
||||
# from source (not a prebuilt vendored blob) so the binary's glibc/GLIBCXX match
|
||||
|
||||
@@ -20,7 +20,7 @@ PIPER_ESPEAK := $(shell pwd)/deps/piper/espeak-ng-data
|
||||
|
||||
all: build
|
||||
|
||||
build: build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav
|
||||
build: build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav build-mail build-update
|
||||
|
||||
build-stt:
|
||||
CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
|
||||
@@ -50,6 +50,15 @@ build-poll:
|
||||
build-caldav:
|
||||
$(GO) build $(GOFLAGS) -o mavcaldav ./cmd/mavcaldav/
|
||||
|
||||
build-mail:
|
||||
$(GO) build $(GOFLAGS) -o mavmaild ./cmd/mavmaild/
|
||||
|
||||
# mavupdate is an operator CLI, not a daemon: nothing runs it but a human on the
|
||||
# box. It is built with the rest so a broken update path is caught by `make
|
||||
# build` rather than the first time it is needed.
|
||||
build-update:
|
||||
$(GO) build $(GOFLAGS) -o mavupdate ./cmd/mavupdate/
|
||||
|
||||
run-web: build-web
|
||||
./mavweb -addr :9200 -voice 127.0.0.1:9100
|
||||
|
||||
@@ -185,4 +194,4 @@ download-embedder:
|
||||
@echo ' sudo cp onnxruntime-linux-x64-1.15.1/lib/libonnxruntime.so* /usr/local/lib/'
|
||||
|
||||
clean:
|
||||
rm -f mavend mavenclient mavsttd mavttsd mavweb mavpoll mavcaldav mavwaked
|
||||
rm -f mavend mavenclient mavsttd mavttsd mavweb mavpoll mavcaldav mavwaked mavmaild
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"log"
|
||||
|
||||
"github.com/kami/maven/internal/mcp"
|
||||
"github.com/kami/maven/internal/router"
|
||||
"github.com/kami/maven/internal/tool"
|
||||
)
|
||||
@@ -52,6 +53,12 @@ func (h *reactiveHandler) actionAct(ctx context.Context, dec router.Decision) st
|
||||
return "выполнить «" + phrase + "»? скажи «да» или «нет»."
|
||||
case errors.Is(err, tool.ErrNotEnabled):
|
||||
return h.proposeGap(ctx, dec)
|
||||
case errors.Is(err, mcp.ErrNeedsArgs):
|
||||
// An MCP tool that wants named arguments a spoken verb cannot
|
||||
// supply. Guessing them would be a wrong act, so she says so
|
||||
// instead — the tool is still runnable from the authed surface,
|
||||
// where a human types them.
|
||||
return "этому инструменту нужны аргументы, которые я из голоса не соберу — я не буду угадывать."
|
||||
}
|
||||
log.Printf("voice: tool %s: %v", dec.Slots.Fn, err)
|
||||
if out != "" {
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/router"
|
||||
"github.com/kami/maven/internal/zenmoney"
|
||||
)
|
||||
|
||||
// Money questions (Vikunja #125).
|
||||
//
|
||||
// This is the whole read side: mavpoll holds the zenmoney token and writes
|
||||
// facts(kind=env, source=poll:zenmoney); core reads them back when he asks.
|
||||
// Core never sees the token, never calls zenmoney, and has no rule on these
|
||||
// keys — a total is never a reason for Maven to speak first. Maven is not a
|
||||
// nag, least of all about his money.
|
||||
//
|
||||
// Nothing here can reach the external search capability: the figures are read
|
||||
// from the store and rendered locally, and his financial data is never search
|
||||
// input.
|
||||
|
||||
// queryMoney — "сколько я потратил сегодня?", "покажи мои траты".
|
||||
//
|
||||
// Answers only from the latest fact the poller wrote. Three honest outcomes and
|
||||
// no fourth: the figure, "the fact is old and here is its date", or "money
|
||||
// tracking is not connected". It never computes, estimates or rounds a total of
|
||||
// its own — an invented number about his money is the worst thing this could do.
|
||||
func (h *reactiveHandler) queryMoney(ctx context.Context, t *queryTurn) (string, bool) {
|
||||
window, ok := router.ParseMoneyQuery(t.dec.Utterance)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
key, phrase := zenmoney.KeySpentMonth, "в этом месяце"
|
||||
if window == router.MoneyToday {
|
||||
key, phrase = zenmoney.KeySpentToday, "сегодня"
|
||||
}
|
||||
fact, err := h.api.LatestFactBySource(ctx, key, zenmoney.Source)
|
||||
if err != nil {
|
||||
// No fact at all is the normal state when the capability is off. Claim
|
||||
// the turn anyway: falling through to recall would answer a question
|
||||
// about money with whatever note happens to be nearest.
|
||||
if !isNoFactErr(err) {
|
||||
log.Printf("voice: money fact: %v", err)
|
||||
}
|
||||
return "я не отслеживаю траты — не подключено.", true
|
||||
}
|
||||
val, err := zenmoney.ParseFactValue(fact.Value)
|
||||
if err != nil {
|
||||
log.Printf("voice: money fact: decode: %v", err)
|
||||
return "не получилось прочитать траты.", true
|
||||
}
|
||||
reply := val.FormatRU(phrase)
|
||||
if reply == "" {
|
||||
return "по тратам пока нечего сказать.", true
|
||||
}
|
||||
// A stale fact is reported as stale rather than spoken as today's number.
|
||||
if h.now().Sub(fact.Ts) > zenmoney.StaleAfter {
|
||||
return "данные от " + fact.Ts.Local().Format("02.01") + ": " + reply, true
|
||||
}
|
||||
return reply, true
|
||||
}
|
||||
|
||||
// isNoFactErr — ErrNoFact survives the wire wrapped, so unwrap for it.
|
||||
func isNoFactErr(err error) bool {
|
||||
for e := err; e != nil; {
|
||||
if e == ipc.ErrNoFact {
|
||||
return true
|
||||
}
|
||||
u, ok := e.(interface{ Unwrap() error })
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
e = u.Unwrap()
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/router"
|
||||
"github.com/kami/maven/internal/zenmoney"
|
||||
)
|
||||
|
||||
// moneyAPI answers only LatestFactBySource; everything else is unimplemented,
|
||||
// which is the assertion that answering a money question costs no model call
|
||||
// and reaches no network.
|
||||
type moneyAPI struct {
|
||||
ipc.UnimplementedCoreAPI
|
||||
|
||||
fact ipc.Fact
|
||||
err error
|
||||
gotKey string
|
||||
gotSrc string
|
||||
callCnt int
|
||||
}
|
||||
|
||||
func (a *moneyAPI) LatestFactBySource(_ context.Context, key, source string) (ipc.Fact, error) {
|
||||
a.gotKey, a.gotSrc = key, source
|
||||
a.callCnt++
|
||||
return a.fact, a.err
|
||||
}
|
||||
|
||||
func moneyNow() time.Time { return time.Date(2026, 8, 15, 20, 0, 0, 0, time.UTC) }
|
||||
|
||||
func moneyFact(ts time.Time, val string) ipc.Fact {
|
||||
return ipc.Fact{Kind: "env", Key: zenmoney.KeySpentMonth, Value: val, Source: zenmoney.Source, Ts: ts}
|
||||
}
|
||||
|
||||
func TestQueryMoneyAnswersFromTheFact(t *testing.T) {
|
||||
api := &moneyAPI{fact: moneyFact(moneyNow(), `{"spent":[{"currency":"RUB","amount":1749.5}],"count":3}`)}
|
||||
h := &reactiveHandler{api: api, now: moneyNow}
|
||||
reply, ok := h.queryMoney(context.Background(), &queryTurn{
|
||||
dec: router.Decision{Utterance: "сколько я потратил в этом месяце?"},
|
||||
})
|
||||
if !ok {
|
||||
t.Fatal("the money source must claim a money question")
|
||||
}
|
||||
if api.gotKey != zenmoney.KeySpentMonth || api.gotSrc != zenmoney.Source {
|
||||
t.Errorf("read %q/%q, want the month key from the poller's source", api.gotKey, api.gotSrc)
|
||||
}
|
||||
if !strings.Contains(reply, "1749.5") {
|
||||
t.Errorf("reply = %q, want the exact figure", reply)
|
||||
}
|
||||
if !strings.Contains(reply, "в этом месяце") {
|
||||
t.Errorf("reply = %q, want the window named", reply)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryMoneyPicksTodaysKey(t *testing.T) {
|
||||
api := &moneyAPI{fact: moneyFact(moneyNow(), `{"spent":[{"currency":"RUB","amount":250}],"count":1}`)}
|
||||
h := &reactiveHandler{api: api, now: moneyNow}
|
||||
if _, ok := h.queryMoney(context.Background(), &queryTurn{
|
||||
dec: router.Decision{Utterance: "сколько я потратил сегодня?"},
|
||||
}); !ok {
|
||||
t.Fatal("expected the source to claim it")
|
||||
}
|
||||
if api.gotKey != zenmoney.KeySpentToday {
|
||||
t.Errorf("key = %q, want today's", api.gotKey)
|
||||
}
|
||||
}
|
||||
|
||||
// The capability is off unless configured, and then there is no fact. She says
|
||||
// so instead of letting the recall pass answer a money question from a note.
|
||||
func TestQueryMoneySaysNotConnected(t *testing.T) {
|
||||
h := &reactiveHandler{api: &moneyAPI{err: ipc.ErrNoFact}, now: moneyNow}
|
||||
reply, ok := h.queryMoney(context.Background(), &queryTurn{
|
||||
dec: router.Decision{Utterance: "сколько я потратил?"},
|
||||
})
|
||||
if !ok {
|
||||
t.Fatal("expected the source to claim it")
|
||||
}
|
||||
if !strings.Contains(reply, "не подключено") {
|
||||
t.Errorf("reply = %q, want an honest 'not connected'", reply)
|
||||
}
|
||||
// No number of any kind in that answer.
|
||||
for _, d := range []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9"} {
|
||||
if strings.Contains(reply, d) {
|
||||
t.Errorf("reply %q contains a digit — nothing was read, so there is no figure", reply)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A fact older than the staleness bound is dated rather than spoken as if it
|
||||
// were current: the poller can be down, and last week's total presented as
|
||||
// today's is a lie by omission.
|
||||
func TestQueryMoneyDatesAStaleFact(t *testing.T) {
|
||||
old := moneyNow().Add(-72 * time.Hour)
|
||||
api := &moneyAPI{fact: moneyFact(old, `{"spent":[{"currency":"RUB","amount":100}],"count":1}`)}
|
||||
h := &reactiveHandler{api: api, now: moneyNow}
|
||||
reply, _ := h.queryMoney(context.Background(), &queryTurn{
|
||||
dec: router.Decision{Utterance: "сколько я потратил?"},
|
||||
})
|
||||
if !strings.Contains(reply, "данные от") {
|
||||
t.Errorf("reply = %q, want the stale fact dated", reply)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryMoneyPassesOtherQuestions(t *testing.T) {
|
||||
api := &moneyAPI{}
|
||||
h := &reactiveHandler{api: api, now: moneyNow}
|
||||
for _, u := range []string{"какая погода?", "я потратил весь день на это", "какие у меня задачи?"} {
|
||||
if _, ok := h.queryMoney(context.Background(), &queryTurn{dec: router.Decision{Utterance: u}}); ok {
|
||||
t.Errorf("the money source claimed %q", u)
|
||||
}
|
||||
}
|
||||
if api.callCnt != 0 {
|
||||
t.Error("a non-money question must not read the money facts")
|
||||
}
|
||||
}
|
||||
|
||||
// Money must be answered before the recall sources, or a question about
|
||||
// spending gets answered by the nearest note.
|
||||
func TestQuerySourcesOrderMoneyBeforeRecall(t *testing.T) {
|
||||
moneyAt, notesAt := -1, -1
|
||||
for i, src := range querySources {
|
||||
switch src.name {
|
||||
case "money":
|
||||
moneyAt = i
|
||||
case "notes":
|
||||
notesAt = i
|
||||
}
|
||||
}
|
||||
if moneyAt < 0 || notesAt < 0 {
|
||||
t.Fatalf("sources missing: money=%d notes=%d", moneyAt, notesAt)
|
||||
}
|
||||
if moneyAt > notesAt {
|
||||
t.Errorf("money source at %d, after notes at %d", moneyAt, notesAt)
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,12 @@ import (
|
||||
// actionNote handles router.IntentNote: embed the note, persist it, and
|
||||
// index it for recall.
|
||||
func (h *reactiveHandler) actionNote(ctx context.Context, dec router.Decision) string {
|
||||
// An utterance that explicitly files a task is work, not recall, and
|
||||
// belongs in the task store (Vikunja #130). Checked before the embedding
|
||||
// is paid for. Everything else is a note, exactly as before.
|
||||
if reply, ok := h.captureTaskFromNote(ctx, dec); ok {
|
||||
return reply
|
||||
}
|
||||
// embed the note text with the same model the classifier uses, persist
|
||||
// via CoreAPI (source=tap:voice). Semantic recall lives in `notes`, not
|
||||
// facts — no predicate reads it (spec's two-memory split).
|
||||
|
||||
@@ -8,10 +8,12 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/crawl"
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/memory"
|
||||
"github.com/kami/maven/internal/morning"
|
||||
"github.com/kami/maven/internal/router"
|
||||
"github.com/kami/maven/internal/rss"
|
||||
"github.com/kami/maven/internal/weather"
|
||||
)
|
||||
|
||||
@@ -51,11 +53,39 @@ var querySources = []querySource{
|
||||
// the more specific ask (its matcher requires a plan word), so the calendar
|
||||
// listing would otherwise swallow it.
|
||||
{"day-plan", (*reactiveHandler).queryDayPlan},
|
||||
// Also before "calendar": "что я обычно делаю по средам?" names a weekday,
|
||||
// and the habit question is the more specific one. Its matcher requires a
|
||||
// habit marker ("обычно", "каждый", …), so a question about this coming
|
||||
// Wednesday still reaches the calendar.
|
||||
{"habits", (*reactiveHandler).queryHabits},
|
||||
// Before "calendar" and before the recall sources: "что мне нужно
|
||||
// сделать?" is a question about the task list, and the notes pass would
|
||||
// otherwise answer it with whatever note happens to be nearest. Its
|
||||
// matcher requires a task noun or an explicit "что … сделать", so a
|
||||
// date-bearing question still reaches the calendar.
|
||||
{"tasks", (*reactiveHandler).queryTasks},
|
||||
// Before the recall sources too: "сколько я потратил?" is a question about
|
||||
// the money facts the poller wrote, and the notes pass would otherwise
|
||||
// answer it from whatever he once said about spending. Its matcher needs a
|
||||
// money noun plus an actual ask, so "я потратил весь день" is untouched.
|
||||
{"money", (*reactiveHandler).queryMoney},
|
||||
// Before the recall sources and before general knowledge: "что нового?" is
|
||||
// a question about the feeds she reads, and general knowledge would answer
|
||||
// it by inventing news. Its matcher needs a feed noun plus an ask, so
|
||||
// "у меня новая лента в инстаграме" is untouched.
|
||||
{"feeds", (*reactiveHandler).queryFeeds},
|
||||
{"calendar", (*reactiveHandler).queryCalendar},
|
||||
{"weather", (*reactiveHandler).queryWeather},
|
||||
{"embed", (*reactiveHandler).queryEmbed},
|
||||
{"memory", (*reactiveHandler).queryMemory},
|
||||
{"notes", (*reactiveHandler).queryNotes},
|
||||
// LAST before the model answers from memory, and that position is the whole
|
||||
// design (Vikunja #259): local sources first. The model, his own notes and
|
||||
// facts, and — once internal/kiwix is wired into this chain — the offline
|
||||
// ZIMs all get their turn before anything touches the network. This source
|
||||
// only claims a turn where he named a URL out loud, so it never competes
|
||||
// with a local answer.
|
||||
{"web", (*reactiveHandler).queryWeb},
|
||||
{"general-knowledge", (*reactiveHandler).queryGeneral},
|
||||
}
|
||||
|
||||
@@ -135,6 +165,92 @@ func isRestOfDayQuery(text string) bool {
|
||||
return strings.Contains(s, "дальше") || strings.Contains(s, "next")
|
||||
}
|
||||
|
||||
// habitFactWindow — how many recent facts the behaviour profile is counted
|
||||
// over. Enough for a season of habits without scanning the whole store on every
|
||||
// question; the profile is recomputed on read, so the bound is the cost control.
|
||||
const habitFactWindow = 2000
|
||||
|
||||
// queryHabits — "что я обычно делаю по вторникам?" (Vikunja #254). Counts the
|
||||
// answer out of the fact log rather than asking the model to summarise a life:
|
||||
// see internal/memory/behavior.go for why nothing here is generated.
|
||||
func (h *reactiveHandler) queryHabits(ctx context.Context, t *queryTurn) (string, bool) {
|
||||
q, ok := router.ParseHabitQuery(t.dec.Utterance)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
facts, err := h.api.RecentFacts(ctx, habitFactWindow)
|
||||
if err != nil {
|
||||
log.Printf("voice: habits: recent facts: %v", err)
|
||||
return "не получилось посмотреть записи.", true
|
||||
}
|
||||
obs := make([]memory.Observation, 0, len(facts))
|
||||
for _, f := range facts {
|
||||
obs = append(obs, memory.Observation{At: f.Ts, Key: f.Key, Kind: f.Kind})
|
||||
}
|
||||
profile := memory.BuildProfile(obs, h.now())
|
||||
if q.HasWeekday {
|
||||
return profile.FormatWeekdayRU(q.Weekday), true
|
||||
}
|
||||
return profile.FormatOverallRU(), true
|
||||
}
|
||||
|
||||
// feedNoteWindow — how many recent notes are scanned for feed items, and
|
||||
// feedReadOut — how many headlines she actually reads back. She summarises the
|
||||
// top of the pile, she does not recite a river.
|
||||
const (
|
||||
feedNoteWindow = 200
|
||||
feedReadOut = 3
|
||||
)
|
||||
|
||||
// queryFeeds — "что нового в лентах?", "что нового по технологиям?"
|
||||
// (Vikunja #258).
|
||||
//
|
||||
// This is the ONLY way a feed item reaches him. The poller writes notes and
|
||||
// never speaks; asking is the trigger. If that ever changes, the thing that
|
||||
// changed is "Maven is not a nag", not a detail of this file.
|
||||
func (h *reactiveHandler) queryFeeds(ctx context.Context, t *queryTurn) (string, bool) {
|
||||
q, ok := router.ParseFeedQuery(t.dec.Utterance)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
if !h.feedsOn {
|
||||
// Claim the turn rather than fall through: "не читаю ленты" is true, and
|
||||
// letting general knowledge answer "что нового?" would be an invented
|
||||
// news bulletin.
|
||||
return "я пока не читаю ленты — они не настроены.", true
|
||||
}
|
||||
notes, err := h.api.RecentNotes(ctx, feedNoteWindow)
|
||||
if err != nil {
|
||||
log.Printf("voice: feeds: recent notes: %v", err)
|
||||
return "не получилось посмотреть ленты.", true
|
||||
}
|
||||
var picked []string
|
||||
for _, n := range notes {
|
||||
if !strings.HasPrefix(n.Source, rss.SourcePrefix) {
|
||||
continue
|
||||
}
|
||||
if !router.CategoryMatches(n.Text, q.Category) {
|
||||
continue
|
||||
}
|
||||
// The note carries title, summary and link; she reads the title.
|
||||
title := n.Text
|
||||
if i := strings.IndexByte(title, '\n'); i > 0 {
|
||||
title = title[:i]
|
||||
}
|
||||
picked = append(picked, strings.TrimSpace(title))
|
||||
if len(picked) == feedReadOut {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(picked) == 0 {
|
||||
if q.Category != "" {
|
||||
return "по этой теме в лентах пока ничего.", true
|
||||
}
|
||||
return "в лентах пока ничего нового.", true
|
||||
}
|
||||
return "вот что нового: " + strings.Join(picked, "; "), true
|
||||
}
|
||||
|
||||
// queryCalendar — "что у меня сегодня?", "планы на завтра?"
|
||||
// h.now(), not time.Now(): the handler's clock is the injected one, so this
|
||||
// source can be tested at a fixed time like the rest.
|
||||
@@ -263,6 +379,57 @@ func (h *reactiveHandler) queryNotes(ctx context.Context, t *queryTurn) (string,
|
||||
return reply, true
|
||||
}
|
||||
|
||||
// webPageContextRunes — how much of a fetched page is handed to the phraser.
|
||||
// Less than the crawler keeps: the rest of the 4096-token window belongs to the
|
||||
// prompt, the persona block and the reply.
|
||||
const webPageContextRunes = 1500
|
||||
|
||||
// queryWeb — "посмотри https://example.org/x — что там?" (Vikunja #259).
|
||||
//
|
||||
// It claims a turn ONLY when he named a URL, which is what keeps a fallback from
|
||||
// becoming a habit: no URL, no fetch, and the model answers from what is local.
|
||||
// What leaves the box is the URL and nothing else — no note, no fact, no history
|
||||
// travels with it.
|
||||
func (h *reactiveHandler) queryWeb(ctx context.Context, t *queryTurn) (string, bool) {
|
||||
link, ok := router.FirstURL(t.dec.Utterance)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
if h.crawler == nil {
|
||||
// Claim rather than fall through: he asked about a specific page, and
|
||||
// letting the model answer from the URL's spelling alone is how a small
|
||||
// model invents a page's contents.
|
||||
return "я не читаю страницы — это не настроено.", true
|
||||
}
|
||||
ctxFetch, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
page, err := h.crawler.Page(ctxFetch, link)
|
||||
if err != nil {
|
||||
if errors.Is(err, crawl.ErrRobots) {
|
||||
return "эта страница закрыта для чтения — robots.txt не разрешает.", true
|
||||
}
|
||||
log.Printf("voice: web: %v", err)
|
||||
return "не получилось прочитать страницу.", true
|
||||
}
|
||||
if page.Text == "" {
|
||||
return "страница открылась, но читать там нечего.", true
|
||||
}
|
||||
// The page is handed to the phraser the same way a note is: as context for
|
||||
// the question he actually asked. She answers the question, she does not
|
||||
// recite the page.
|
||||
snippet := page.Title + "\n" + crawl.TrimRunes(page.Text, webPageContextRunes)
|
||||
reply, perr := h.phraser.PhraseQuery(ctx, t.dec.Utterance, []string{snippet})
|
||||
if perr != nil {
|
||||
log.Printf("voice: web: phrase: %v", perr)
|
||||
}
|
||||
if reply == "" {
|
||||
// No phraser (or it failed): read back the top of the page rather than
|
||||
// pretend the fetch did not happen.
|
||||
return "вот что на странице: " + crawl.TrimRunes(page.Text, 300), true
|
||||
}
|
||||
return reply, true
|
||||
}
|
||||
|
||||
// queryGeneral — general knowledge from the phraser, the last source before
|
||||
// giving up. It always claims: either the model answers or Maven says she
|
||||
// doesn't know.
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/router"
|
||||
"github.com/kami/maven/internal/store"
|
||||
"github.com/kami/maven/internal/tasks"
|
||||
)
|
||||
|
||||
// Task capture on the voice/chat path (Vikunja #130).
|
||||
//
|
||||
// Two halves, both deliberately small:
|
||||
//
|
||||
// - captureTaskFromNote runs at the top of actionNote. An utterance that
|
||||
// explicitly files a task ("добавь в задачи купить молоко") goes to the task
|
||||
// store instead of the note store. Anything without an explicit marker is
|
||||
// still a note — see router.ParseTaskCapture for why "надо бы поспать" must
|
||||
// not become a task.
|
||||
// - queryTasks is a query source that reads the list back.
|
||||
//
|
||||
// Nothing here speaks unprompted. Tasks are answered when asked about; no tick
|
||||
// rule reads the table.
|
||||
|
||||
// captureTaskFromNote claims the turn when the utterance explicitly files a
|
||||
// task, returning the reply. ("", false) hands the turn back to the note path.
|
||||
func (h *reactiveHandler) captureTaskFromNote(ctx context.Context, dec router.Decision) (string, bool) {
|
||||
cap, ok := router.ParseTaskCapture(dec.Utterance)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
resp, err := h.api.CaptureTask(ctx, ipc.CaptureTaskReq{
|
||||
Text: cap.Text,
|
||||
Source: "tap:voice",
|
||||
Status: store.TaskOpen, // he stated it himself — not a candidate
|
||||
Weight: cap.Weight, // 0 unless he said "срочно" / "важно"
|
||||
Ts: h.now(),
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("voice: capture task: %v", err)
|
||||
return "не получилось записать задачу.", true
|
||||
}
|
||||
if !resp.Created {
|
||||
return "это уже в списке.", true
|
||||
}
|
||||
return "записала: " + cap.Text, true
|
||||
}
|
||||
|
||||
// queryTasks — "какие у меня задачи?", "что мне нужно сделать?".
|
||||
//
|
||||
// Reads the live set and recites it in priority order (Vikunja #129). The order
|
||||
// is computed by internal/tasks from what he told her — deadlines, the urgency
|
||||
// he stated, how long a task has been sitting — never asked of the model. The
|
||||
// rendering is the package's too, so the spoken list and the /tasks page can
|
||||
// never disagree about what comes first.
|
||||
func (h *reactiveHandler) queryTasks(ctx context.Context, t *queryTurn) (string, bool) {
|
||||
if !router.IsTaskListQuery(t.dec.Utterance) {
|
||||
return "", false
|
||||
}
|
||||
live, err := h.api.ListTasks(ctx, "live")
|
||||
if err != nil {
|
||||
log.Printf("voice: list tasks: %v", err)
|
||||
return "не получилось посмотреть задачи.", true
|
||||
}
|
||||
return tasks.FormatRU(tasks.Rank(taskItems(live), h.now())), true
|
||||
}
|
||||
|
||||
// taskItems maps wire rows onto the ranker's input. Written here rather than in
|
||||
// internal/tasks so the ranker stays a pure package with no ipc (and therefore
|
||||
// no store, and therefore no cgo) dependency — the same posture as
|
||||
// internal/morning and internal/memory.
|
||||
func taskItems(ts []ipc.Task) []tasks.Item {
|
||||
out := make([]tasks.Item, len(ts))
|
||||
for i, t := range ts {
|
||||
out[i] = tasks.Item{
|
||||
ID: t.ID, Text: t.Text, Status: t.Status,
|
||||
Created: t.CreatedTs, Due: t.Due, Weight: t.Weight,
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/router"
|
||||
)
|
||||
|
||||
// taskAPI answers only the three task methods; every other call is
|
||||
// unimplemented, which is the assertion that capture needs nothing else — in
|
||||
// particular no embedder, so a filed task costs no model call.
|
||||
type taskAPI struct {
|
||||
ipc.UnimplementedCoreAPI
|
||||
|
||||
captured []ipc.CaptureTaskReq
|
||||
created bool
|
||||
capErr error
|
||||
|
||||
tasks []ipc.Task
|
||||
listArg string
|
||||
listErr error
|
||||
}
|
||||
|
||||
func (a *taskAPI) CaptureTask(_ context.Context, req ipc.CaptureTaskReq) (ipc.CaptureTaskResp, error) {
|
||||
a.captured = append(a.captured, req)
|
||||
if a.capErr != nil {
|
||||
return ipc.CaptureTaskResp{}, a.capErr
|
||||
}
|
||||
return ipc.CaptureTaskResp{ID: 1, Created: a.created}, nil
|
||||
}
|
||||
|
||||
func (a *taskAPI) ListTasks(_ context.Context, status string) ([]ipc.Task, error) {
|
||||
a.listArg = status
|
||||
return a.tasks, a.listErr
|
||||
}
|
||||
|
||||
func taskNow() time.Time { return time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC) }
|
||||
|
||||
func taskHandler(api ipc.CoreAPI) *reactiveHandler {
|
||||
return &reactiveHandler{api: api, now: taskNow}
|
||||
}
|
||||
|
||||
func TestCaptureTaskFromNoteFilesTheTask(t *testing.T) {
|
||||
api := &taskAPI{created: true}
|
||||
h := taskHandler(api)
|
||||
reply, ok := h.captureTaskFromNote(context.Background(), router.Decision{
|
||||
Intent: router.IntentNote, Utterance: "добавь в задачи купить молоко",
|
||||
})
|
||||
if !ok {
|
||||
t.Fatal("an explicit capture must claim the turn")
|
||||
}
|
||||
if len(api.captured) != 1 {
|
||||
t.Fatalf("captured %d, want 1", len(api.captured))
|
||||
}
|
||||
got := api.captured[0]
|
||||
if got.Text != "купить молоко" {
|
||||
t.Errorf("text = %q, want the marker stripped", got.Text)
|
||||
}
|
||||
if got.Source != "tap:voice" {
|
||||
t.Errorf("source = %q, want tap:voice", got.Source)
|
||||
}
|
||||
if got.Status != "open" {
|
||||
t.Errorf("status = %q — work he stated is open, never a candidate", got.Status)
|
||||
}
|
||||
if !got.Ts.Equal(taskNow()) {
|
||||
t.Errorf("ts = %v, want the handler clock", got.Ts)
|
||||
}
|
||||
if !strings.Contains(reply, "купить молоко") {
|
||||
t.Errorf("reply = %q, want it to read the task back", reply)
|
||||
}
|
||||
}
|
||||
|
||||
// A note is still a note: capture only fires on an explicit marker, so
|
||||
// ordinary recall is untouched.
|
||||
func TestCaptureTaskFromNotePassesOrdinaryNotes(t *testing.T) {
|
||||
api := &taskAPI{}
|
||||
h := taskHandler(api)
|
||||
for _, u := range []string{"надо бы поспать", "мне понравился этот фильм", "запиши что я пил воду"} {
|
||||
if _, ok := h.captureTaskFromNote(context.Background(), router.Decision{Utterance: u}); ok {
|
||||
t.Errorf("%q was captured as a task", u)
|
||||
}
|
||||
}
|
||||
if len(api.captured) != 0 {
|
||||
t.Errorf("captured %d requests, want none", len(api.captured))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCaptureTaskFromNoteSaysAlreadyOnTheList(t *testing.T) {
|
||||
h := taskHandler(&taskAPI{created: false})
|
||||
reply, ok := h.captureTaskFromNote(context.Background(), router.Decision{Utterance: "добавь в задачи купить молоко"})
|
||||
if !ok {
|
||||
t.Fatal("expected the capture path to claim it")
|
||||
}
|
||||
if !strings.Contains(reply, "уже") {
|
||||
t.Errorf("reply = %q — a deduped capture must not claim it saved something new", reply)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCaptureTaskFromNoteReportsStoreFailure(t *testing.T) {
|
||||
h := taskHandler(&taskAPI{capErr: errors.New("db is on fire")})
|
||||
reply, ok := h.captureTaskFromNote(context.Background(), router.Decision{Utterance: "добавь задачу починить кран"})
|
||||
if !ok {
|
||||
t.Fatal("a failed capture still claims the turn — the note path must not double-write")
|
||||
}
|
||||
if !strings.Contains(reply, "не получилось") {
|
||||
t.Errorf("reply = %q, want an honest failure", reply)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryTasksRecitesTheLiveList(t *testing.T) {
|
||||
api := &taskAPI{tasks: []ipc.Task{
|
||||
{ID: 1, Text: "купить молоко", Status: "open"},
|
||||
{ID: 2, Text: "продлить страховку", Status: "candidate"},
|
||||
}}
|
||||
h := taskHandler(api)
|
||||
reply, ok := h.queryTasks(context.Background(), &queryTurn{
|
||||
dec: router.Decision{Intent: router.IntentQuery, Utterance: "какие у меня задачи?"},
|
||||
})
|
||||
if !ok {
|
||||
t.Fatal("the task source must claim a task-list question")
|
||||
}
|
||||
if api.listArg != "live" {
|
||||
t.Errorf("ListTasks(%q), want \"live\" — a resolved task is not outstanding work", api.listArg)
|
||||
}
|
||||
if !strings.Contains(reply, "купить молоко") || !strings.Contains(reply, "продлить страховку") {
|
||||
t.Errorf("reply = %q, want both tasks", reply)
|
||||
}
|
||||
// The candidate must be named as unconfirmed, not recited as his work.
|
||||
openIdx := strings.Index(reply, "купить молоко")
|
||||
candIdx := strings.Index(reply, "продлить страховку")
|
||||
if !(openIdx < candIdx) {
|
||||
t.Errorf("reply = %q, want confirmed work before candidates", reply)
|
||||
}
|
||||
if !strings.Contains(reply, "не подтвердил") {
|
||||
t.Errorf("reply = %q, want the candidate flagged as unconfirmed", reply)
|
||||
}
|
||||
}
|
||||
|
||||
// The stated urgency rides through capture as a weight, so the ranker can use
|
||||
// it later (Vikunja #129). "срочно" is not part of the task text.
|
||||
func TestCaptureTaskCarriesStatedUrgency(t *testing.T) {
|
||||
api := &taskAPI{created: true}
|
||||
h := taskHandler(api)
|
||||
if _, ok := h.captureTaskFromNote(context.Background(), router.Decision{
|
||||
Utterance: "добавь в задачи срочно оплатить интернет",
|
||||
}); !ok {
|
||||
t.Fatal("expected a capture")
|
||||
}
|
||||
got := api.captured[0]
|
||||
if got.Text != "оплатить интернет" {
|
||||
t.Errorf("text = %q, want the urgency word out of the task", got.Text)
|
||||
}
|
||||
if got.Weight == 0 {
|
||||
t.Error("weight = 0 — he said срочно and it was dropped")
|
||||
}
|
||||
}
|
||||
|
||||
// The recital is ordered by the ranker, not by insertion: a deadline he named
|
||||
// comes before undated work.
|
||||
func TestQueryTasksRecitesInPriorityOrder(t *testing.T) {
|
||||
due := taskNow()
|
||||
api := &taskAPI{tasks: []ipc.Task{
|
||||
{ID: 1, Text: "купить молоко", Status: "open", CreatedTs: taskNow()},
|
||||
{ID: 2, Text: "оплатить интернет", Status: "open", CreatedTs: taskNow(), Due: &due},
|
||||
}}
|
||||
h := taskHandler(api)
|
||||
reply, _ := h.queryTasks(context.Background(), &queryTurn{
|
||||
dec: router.Decision{Utterance: "какие у меня задачи?"},
|
||||
})
|
||||
if strings.Index(reply, "оплатить интернет") > strings.Index(reply, "купить молоко") {
|
||||
t.Errorf("reply = %q, want the dated task first", reply)
|
||||
}
|
||||
if !strings.Contains(reply, "сегодня") {
|
||||
t.Errorf("reply = %q, want the reason named", reply)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryTasksEmptyList(t *testing.T) {
|
||||
h := taskHandler(&taskAPI{})
|
||||
reply, ok := h.queryTasks(context.Background(), &queryTurn{
|
||||
dec: router.Decision{Utterance: "что мне нужно сделать?"},
|
||||
})
|
||||
if !ok {
|
||||
t.Fatal("expected the task source to claim it")
|
||||
}
|
||||
if reply != "задач нет." {
|
||||
t.Errorf("reply = %q", reply)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryTasksPassesOtherQuestions(t *testing.T) {
|
||||
api := &taskAPI{}
|
||||
h := taskHandler(api)
|
||||
for _, u := range []string{"как дела?", "какая погода в москве?", "что у меня сегодня?"} {
|
||||
if _, ok := h.queryTasks(context.Background(), &queryTurn{dec: router.Decision{Utterance: u}}); ok {
|
||||
t.Errorf("the task source claimed %q", u)
|
||||
}
|
||||
}
|
||||
if api.listArg != "" {
|
||||
t.Error("a non-task question must not read the task list")
|
||||
}
|
||||
}
|
||||
|
||||
// The chain must reach the task source before the recall sources, or "что мне
|
||||
// нужно сделать?" gets answered by whatever note is nearest.
|
||||
func TestQuerySourcesOrderTasksBeforeRecall(t *testing.T) {
|
||||
var tasksAt, notesAt = -1, -1
|
||||
for i, src := range querySources {
|
||||
switch src.name {
|
||||
case "tasks":
|
||||
tasksAt = i
|
||||
case "notes":
|
||||
notesAt = i
|
||||
}
|
||||
}
|
||||
if tasksAt < 0 || notesAt < 0 {
|
||||
t.Fatalf("sources missing: tasks=%d notes=%d", tasksAt, notesAt)
|
||||
}
|
||||
if tasksAt > notesAt {
|
||||
t.Errorf("tasks source at %d, after notes at %d", tasksAt, notesAt)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
// mavend/capture.go — core's half of the meeting recorder (Vikunja #253,
|
||||
// docs/plans/08-hearing.md).
|
||||
//
|
||||
// The split: a client that has a microphone (mavenclient, or a phone on the PWA)
|
||||
// is told to start, streams frames over ipc.MethodCaptureAppend, and is told to
|
||||
// stop. Core keeps the PCM, stores it as a WAV blob under the same media store
|
||||
// and the same retention as images, transcribes it through the ONE STT Maven has
|
||||
// (mavsttd's whisper.cpp, reused — not a second engine), and summarises the
|
||||
// transcript on the resident model in windows that fit n_ctx 4096.
|
||||
//
|
||||
// # Off unless configured, twice over
|
||||
//
|
||||
// No `media` block ⇒ nowhere to keep audio ⇒ the four capture methods do not
|
||||
// exist. No `capture` block with enabled ⇒ they still do not exist. On an
|
||||
// unconfigured box there is no wire path that starts a recording, which is the
|
||||
// only guarantee worth making about a capability like this one.
|
||||
//
|
||||
// # What this file refuses to do
|
||||
//
|
||||
// - Nothing listens. There is no VAD hook here, no wake-word branch, no
|
||||
// "start when you hear a meeting". The plan document's keyword-triggered
|
||||
// recorder is refused in internal/capture's package comment for the reason
|
||||
// that applies here too: noticing a keyword requires listening, which is
|
||||
// the behaviour this capability must not have.
|
||||
// - No transcript note by default. The summary is written where he will read
|
||||
// it; the verbatim record of what other people said takes a deliberate
|
||||
// capture.save_transcript.
|
||||
// - The transcript is never search input beyond this box, and the audio never
|
||||
// leaves it at all.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/capture"
|
||||
"github.com/kami/maven/internal/config"
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/llm"
|
||||
"github.com/kami/maven/internal/phraser"
|
||||
"github.com/kami/maven/internal/router"
|
||||
"github.com/kami/maven/internal/store"
|
||||
)
|
||||
|
||||
// captureSummaryTimeout — the budget for one Stop, which is a map-reduce over
|
||||
// the whole meeting: one model call per transcript window plus a reduce, each of
|
||||
// which is seconds on this box. Forty windows is the configured ceiling, so the
|
||||
// budget has to be minutes, not the 60s the reply path uses.
|
||||
const captureSummaryTimeout = 20 * time.Minute
|
||||
|
||||
// llmCompleter adapts *llm.Client to capture.Completer. The pure package names
|
||||
// the two strings it needs and stays free of the llm request struct; the client
|
||||
// itself is the swap-aware one from llmClientFor, so a model swap re-points it.
|
||||
type llmCompleter struct {
|
||||
c *llm.Client
|
||||
maxTokens int
|
||||
}
|
||||
|
||||
func (l llmCompleter) Complete(ctx context.Context, system, user string) (string, error) {
|
||||
return l.c.Complete(ctx, llm.Req{System: system, User: user, MaxTokens: l.maxTokens})
|
||||
}
|
||||
|
||||
// captureWiring — the recorder plus what it needs to write the result down.
|
||||
type captureWiring struct {
|
||||
rec *capture.Recorder
|
||||
st *store.Store
|
||||
emb router.Embedder
|
||||
cfg *config.CaptureConfig
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// newCaptureWiring returns nil when the recorder should not exist: no media
|
||||
// store, no capture block, capture disabled, or no STT to transcribe with.
|
||||
//
|
||||
// A missing llama-server is NOT a reason to return nil. Without one the
|
||||
// recording is still made, stored and transcribed, and the summary is simply
|
||||
// absent — the honest degradation, and much better than refusing to record a
|
||||
// meeting that is happening now.
|
||||
func newCaptureWiring(keeper *mediaKeeper, st *store.Store, voiceW *voiceWiring, phr phraser.Phraser, emb router.Embedder, cfg *config.Config) *captureWiring {
|
||||
if keeper == nil || !cfg.Capture.Records() {
|
||||
return nil
|
||||
}
|
||||
tr := transcriberOf(voiceW)
|
||||
if tr == nil {
|
||||
// Voice off ⇒ no STT client ⇒ nothing could turn the audio into words.
|
||||
// Storing hours of unreadable audio of other people is worse than not
|
||||
// recording, so this is a refusal, not a degradation.
|
||||
log.Printf("capture: enabled but voice/stt is not wired — meeting capture disabled")
|
||||
return nil
|
||||
}
|
||||
|
||||
cc := cfg.Capture
|
||||
var sum *capture.Summarizer
|
||||
if lp, ok := phr.(*phraser.LLMPhraser); ok {
|
||||
client := llmClientFor(lp, captureSummaryTimeout)
|
||||
sum = capture.NewSummarizer(
|
||||
llmCompleter{c: client, maxTokens: 512},
|
||||
cc.ChunkRunes, cc.MaxChunks, contextBlockFn(cfg, time.Now),
|
||||
)
|
||||
} else {
|
||||
log.Printf("capture: no llama-server phraser — meetings are transcribed, not summarised")
|
||||
}
|
||||
|
||||
rec, err := capture.New(keeper.store, tr, sum, capture.Config{
|
||||
MaxDuration: cc.MaxDuration(),
|
||||
STTWindow: time.Duration(cc.STTWindow),
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("capture: %v — meeting capture disabled", err)
|
||||
return nil
|
||||
}
|
||||
log.Printf("capture: enabled, sessions capped at %s", rec.MaxDuration())
|
||||
return &captureWiring{rec: rec, st: st, emb: emb, cfg: cc, now: time.Now}
|
||||
}
|
||||
|
||||
// start handles ipc.MethodCaptureStart.
|
||||
func (c *captureWiring) start(_ context.Context, req ipc.CaptureStartReq) (ipc.CaptureStartResp, error) {
|
||||
s, err := c.rec.Start(req.Label)
|
||||
if err != nil {
|
||||
return ipc.CaptureStartResp{}, err
|
||||
}
|
||||
// The label is logged; nothing that was said ever is.
|
||||
log.Printf("capture: started %q", s.Label)
|
||||
return ipc.CaptureStartResp{
|
||||
Label: s.Label,
|
||||
Started: s.Started,
|
||||
MaxSeconds: int(c.rec.MaxDuration().Seconds()),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// append handles ipc.MethodCaptureAppend. ErrExpired is reported as a successful
|
||||
// response with Expired set rather than an error: the cap firing is the designed
|
||||
// behaviour, and the client needs the flag to stop sending and call stop.
|
||||
func (c *captureWiring) append(_ context.Context, req ipc.CaptureAppendReq) (ipc.CaptureAppendResp, error) {
|
||||
err := c.rec.Append(req.Audio)
|
||||
st := c.rec.Status()
|
||||
if errors.Is(err, capture.ErrExpired) {
|
||||
log.Printf("capture: %q hit the %s cap — stopping", st.Label, c.rec.MaxDuration())
|
||||
return ipc.CaptureAppendResp{Seconds: st.Duration.Seconds(), Expired: true}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return ipc.CaptureAppendResp{}, err
|
||||
}
|
||||
return ipc.CaptureAppendResp{Seconds: st.Duration.Seconds()}, nil
|
||||
}
|
||||
|
||||
// stop handles ipc.MethodCaptureStop.
|
||||
//
|
||||
// The error handling here mirrors vision's, and for the same reason: the audio is
|
||||
// stored first, so a transcription or summary failure returns what exists rather
|
||||
// than nothing. A response can carry a blob id with no transcript (STT failed,
|
||||
// re-runnable), or a transcript with no summary (the model failed, the words are
|
||||
// kept) — both are degraded successes and neither is an error to the caller.
|
||||
func (c *captureWiring) stop(ctx context.Context, req ipc.CaptureStopReq) (ipc.CaptureStopResp, error) {
|
||||
if req.Discard {
|
||||
// "забудь, не записывай" — nothing is stored, transcribed or noted.
|
||||
if !c.rec.Abort() {
|
||||
return ipc.CaptureStopResp{}, capture.ErrNoSession
|
||||
}
|
||||
log.Printf("capture: session discarded on request")
|
||||
return ipc.CaptureStopResp{Discarded: true}, nil
|
||||
}
|
||||
|
||||
res, err := c.rec.Stop(ctx)
|
||||
resp := ipc.CaptureStopResp{
|
||||
BlobID: res.BlobID,
|
||||
Label: res.Label,
|
||||
Started: res.Started,
|
||||
Seconds: res.Duration.Seconds(),
|
||||
Transcript: res.Transcript,
|
||||
Summary: res.Summary,
|
||||
Chunks: res.Chunks,
|
||||
}
|
||||
if err != nil {
|
||||
if res.BlobID == "" && res.Transcript == "" {
|
||||
// Nothing survived: no session, or an empty recording. There is
|
||||
// nothing to hand back, so this is a real error.
|
||||
return ipc.CaptureStopResp{}, err
|
||||
}
|
||||
log.Printf("capture: %q partially finished: %v", res.Label, err)
|
||||
}
|
||||
|
||||
if id, werr := c.writeNotes(ctx, res); werr != nil {
|
||||
log.Printf("capture: note write for %q failed: %v", res.Label, werr)
|
||||
} else {
|
||||
resp.NoteID = id
|
||||
}
|
||||
log.Printf("capture: finished %q — %s of audio, %d summary chunk(s)",
|
||||
res.Label, res.Duration.Round(time.Second), res.Chunks)
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// writeNotes stores the summary as a note, and the transcript too when
|
||||
// capture.save_transcript is set. Returns the summary note's id, or 0 when there
|
||||
// was no summary to write.
|
||||
//
|
||||
// The note source carries the blob id, which is the only link back to the audio.
|
||||
// When retention prunes the blob the note remains — words about a meeting are a
|
||||
// far lighter thing to keep than a recording of it.
|
||||
func (c *captureWiring) writeNotes(ctx context.Context, res capture.Result) (int64, error) {
|
||||
source := "capture:meeting"
|
||||
if res.BlobID != "" {
|
||||
source = "capture:meeting:" + res.BlobID[:12]
|
||||
}
|
||||
var id int64
|
||||
if text := res.Summary; text != "" {
|
||||
var err error
|
||||
id, err = c.writeNote(ctx, text, source)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("summary note: %w", err)
|
||||
}
|
||||
}
|
||||
if c.cfg.SaveTranscript && res.Transcript != "" {
|
||||
if _, err := c.writeNote(ctx, res.Transcript, source+":transcript"); err != nil {
|
||||
return id, fmt.Errorf("transcript note: %w", err)
|
||||
}
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func (c *captureWiring) writeNote(ctx context.Context, text, source string) (int64, error) {
|
||||
var vec []float32
|
||||
if c.emb != nil {
|
||||
// EmbedPassage, not Embed: this is text being searched FOR, and the e5
|
||||
// embedder is asymmetric. Backwards here makes the meeting unfindable by
|
||||
// the question that should have matched it.
|
||||
var err error
|
||||
vec, err = router.EmbedPassage(ctx, c.emb, text)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("embed: %w", err)
|
||||
}
|
||||
}
|
||||
return c.st.WriteNote(ctx, c.now(), text, vec, source)
|
||||
}
|
||||
|
||||
// status handles ipc.MethodCaptureStatus.
|
||||
func (c *captureWiring) status(_ context.Context) (ipc.CaptureStatusResp, error) {
|
||||
st := c.rec.Status()
|
||||
return ipc.CaptureStatusResp{
|
||||
Running: st.Running,
|
||||
Label: st.Label,
|
||||
Started: st.Started,
|
||||
Seconds: st.Duration.Seconds(),
|
||||
Bytes: st.Bytes,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// wireCapture installs the four IPC hooks, or leaves them nil so every capture
|
||||
// method reports ErrUnknownMethod. Takes the media keeper wireVision already
|
||||
// opened: one blob store, one retention loop, images and audio side by side.
|
||||
func wireCapture(srv *ipc.Server, keeper *mediaKeeper, st *store.Store, voiceW *voiceWiring, phr phraser.Phraser, cfg *config.Config) {
|
||||
cw := newCaptureWiring(keeper, st, voiceW, phr, embedderOf(voiceW), cfg)
|
||||
if cw == nil {
|
||||
return
|
||||
}
|
||||
srv.CaptureStartFn = cw.start
|
||||
srv.CaptureAppendFn = cw.append
|
||||
srv.CaptureStopFn = cw.stop
|
||||
srv.CaptureStatusFn = cw.status
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
// mavend/crawls.go — the driver for reading web pages (Vikunja #259,
|
||||
// docs/plans/14-web-crawler.md). The crawler is pure and lives in
|
||||
// internal/crawl; this is the impure half: the guarded fetcher, a ticker for the
|
||||
// scheduled watches, and the fact-backed dedup hashes.
|
||||
//
|
||||
// Two paths, one config block, both off unless configured:
|
||||
//
|
||||
// - ON DEMAND — he names a URL out loud and she reads it. That is the
|
||||
// `queryWeb` source in actions_query.go, LAST in the chain: after his
|
||||
// memory, after the notes, and (once Kiwix is wired into the chain) after
|
||||
// the local ZIMs. A local read costs nothing and leaks nothing; a fetch puts
|
||||
// a URL in someone's log, so it goes last.
|
||||
// - SCHEDULED — a watched page is re-read on its interval, and a page whose
|
||||
// text changed is written as a note. It does NOT announce itself. Same rule
|
||||
// as the feed poller: notes, never nudges.
|
||||
//
|
||||
// Only the URL goes out. Nothing here reads a note, a fact, the persona block or
|
||||
// the history, and internal/crawl has no access to the store at all.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/config"
|
||||
"github.com/kami/maven/internal/crawl"
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/router"
|
||||
"github.com/kami/maven/internal/webfetch"
|
||||
)
|
||||
|
||||
// newCrawler builds the crawler from the `crawl` block, or returns nil when
|
||||
// there is none. Every caller checks for nil, and nil means no page is ever
|
||||
// fetched.
|
||||
func newCrawler(cfg *config.Config) *crawl.Crawler {
|
||||
if cfg.Crawl == nil {
|
||||
return nil
|
||||
}
|
||||
cc := cfg.Crawl
|
||||
|
||||
hosts := append([]string(nil), cc.AllowHosts...)
|
||||
// A watched page's own host is always reachable; otherwise an allowlist and
|
||||
// a watch list would have to be kept in sync by hand.
|
||||
for _, w := range cc.Watches {
|
||||
if u, err := url.Parse(w.URL); err == nil && u.Hostname() != "" {
|
||||
hosts = append(hosts, u.Hostname())
|
||||
}
|
||||
}
|
||||
// An allowlist plus on-demand is a contradiction worth logging rather than
|
||||
// silently resolving: he asked for arbitrary pages AND for a fixed list.
|
||||
// The allowlist wins, because it is the narrower instruction.
|
||||
if len(hosts) > 0 && cc.OnDemand && len(cc.AllowHosts) > 0 {
|
||||
log.Printf("crawl: allow_hosts is set, so on-demand reading is limited to those hosts")
|
||||
}
|
||||
ua := cc.UserAgent
|
||||
if ua == "" {
|
||||
ua = webfetch.DefaultUserAgent
|
||||
}
|
||||
fetcher := webfetch.New(webfetch.Config{
|
||||
AllowHosts: hosts,
|
||||
DenyHosts: cc.DenyHosts,
|
||||
Timeout: time.Duration(cc.Timeout),
|
||||
MaxBytes: cc.MaxBytes,
|
||||
UserAgent: ua,
|
||||
})
|
||||
// The user-agent handed to the crawler is the one the fetcher sends: obeying
|
||||
// robots rules written for a different name would be a lie.
|
||||
return crawl.New(&crawlFetcher{f: fetcher}, crawl.Config{
|
||||
UserAgent: ua,
|
||||
MaxRunes: cc.MaxRunes,
|
||||
})
|
||||
}
|
||||
|
||||
// onDemandCrawler returns a crawler for the answer path, or nil when on-demand
|
||||
// reading is off. The scheduled watches can be on while this is off: reading a
|
||||
// fixed list of pages on a timer and reading whatever URL is in an utterance are
|
||||
// different permissions, and the config keeps them separate.
|
||||
func onDemandCrawler(cfg *config.Config) *crawl.Crawler {
|
||||
if cfg.Crawl == nil || !cfg.Crawl.OnDemand {
|
||||
return nil
|
||||
}
|
||||
return newCrawler(cfg)
|
||||
}
|
||||
|
||||
// crawlWorker — ticker + watcher for the scheduled half.
|
||||
type crawlWorker struct {
|
||||
watcher *crawl.Watcher
|
||||
interval time.Duration
|
||||
}
|
||||
|
||||
// crawlTickInterval — how often the worker asks what is due. Per-watch cadence
|
||||
// is the watcher's business.
|
||||
const crawlTickInterval = 15 * time.Minute
|
||||
|
||||
// newCrawlWorker wires the scheduled crawls, or nil when nothing is watched.
|
||||
func newCrawlWorker(c *crawl.Crawler, api ipc.CoreAPI, emb router.Embedder, cfg *config.Config) *crawlWorker {
|
||||
if c == nil || cfg.Crawl == nil || len(cfg.Crawl.Watches) == 0 {
|
||||
return nil
|
||||
}
|
||||
watches := make([]crawl.WatchConfig, 0, len(cfg.Crawl.Watches))
|
||||
for _, w := range cfg.Crawl.Watches {
|
||||
watches = append(watches, crawl.WatchConfig{
|
||||
Name: w.Name,
|
||||
URL: w.URL,
|
||||
Interval: time.Duration(w.Interval),
|
||||
})
|
||||
}
|
||||
watcher := crawl.NewWatcher(c, watches, api, &factHashes{api: api},
|
||||
crawlEmbedder(emb), time.Duration(cfg.Crawl.Interval))
|
||||
if watcher == nil {
|
||||
log.Printf("crawl: configured but nothing watchable — scheduled crawls disabled")
|
||||
return nil
|
||||
}
|
||||
log.Printf("crawl: watching %d page(s), checking what is due every %s", len(watches), crawlTickInterval)
|
||||
return &crawlWorker{watcher: watcher, interval: crawlTickInterval}
|
||||
}
|
||||
|
||||
// run checks what is due until ctx is canceled. The first round runs
|
||||
// immediately; it writes notes only, so an early round startles nobody.
|
||||
func (w *crawlWorker) run(ctx context.Context) {
|
||||
w.watcher.CheckDue(ctx, time.Now())
|
||||
t := time.NewTicker(w.interval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case now := <-t.C:
|
||||
w.watcher.CheckDue(ctx, now)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// crawlFetcher adapts webfetch to crawl.Fetcher, which is the seam that keeps
|
||||
// net/http out of the crawler package.
|
||||
type crawlFetcher struct{ f *webfetch.Fetcher }
|
||||
|
||||
func (a *crawlFetcher) Get(ctx context.Context, u string) (*crawl.Response, error) {
|
||||
resp, err := a.f.Get(ctx, u)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &crawl.Response{URL: resp.URL, ContentType: resp.ContentType, Body: resp.Body}, nil
|
||||
}
|
||||
|
||||
// factHashes stores each watch's last content hash as a config fact, so a
|
||||
// restart does not re-note an unchanged page. Same mechanism the feed reader
|
||||
// uses for its marks, and inspectable on /dash.
|
||||
type factHashes struct{ api ipc.CoreAPI }
|
||||
|
||||
func hashKey(name string) string { return "crawl:hash:" + name }
|
||||
|
||||
func (h *factHashes) LastHash(ctx context.Context, name string) (string, error) {
|
||||
f, err := h.api.LatestFact(ctx, hashKey(name))
|
||||
if err != nil {
|
||||
// No hash yet is not an error: the watcher treats "" as "never read".
|
||||
return "", nil
|
||||
}
|
||||
return f.Value, nil
|
||||
}
|
||||
|
||||
func (h *factHashes) SetHash(ctx context.Context, name, hash string) error {
|
||||
_, err := h.api.WriteFact(ctx, ipc.WriteFactReq{
|
||||
Ts: time.Now(),
|
||||
Kind: "config",
|
||||
Key: hashKey(name),
|
||||
Value: hash,
|
||||
Source: "poll:crawl",
|
||||
Confidence: 1.0,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// crawlEmbedder adapts router.Embedder for the watcher, embedding with
|
||||
// EmbedPassage (a page is text being searched FOR, and the e5 embedder is
|
||||
// asymmetric).
|
||||
func crawlEmbedder(emb router.Embedder) crawl.Embedder {
|
||||
if emb == nil {
|
||||
return nil
|
||||
}
|
||||
return passageEmbedder{emb}
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/kami/maven/internal/config"
|
||||
"github.com/kami/maven/internal/crawl"
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/phraser"
|
||||
"github.com/kami/maven/internal/router"
|
||||
"github.com/kami/maven/internal/voice"
|
||||
)
|
||||
|
||||
// The default config reads nothing. This is the whole "off unless configured"
|
||||
// contract for the crawler, asserted at the wiring level rather than trusted.
|
||||
func TestCrawlOffByDefault(t *testing.T) {
|
||||
cfg := &config.Config{}
|
||||
if c := newCrawler(cfg); c != nil {
|
||||
t.Error("newCrawler with no crawl block returned a crawler")
|
||||
}
|
||||
if c := onDemandCrawler(cfg); c != nil {
|
||||
t.Error("onDemandCrawler with no crawl block returned a crawler")
|
||||
}
|
||||
if w := newCrawlWorker(nil, nil, nil, cfg); w != nil {
|
||||
t.Error("newCrawlWorker with no crawl block returned a worker")
|
||||
}
|
||||
// Watches configured but on_demand off ⇒ the answer path still reads
|
||||
// nothing: a timer over a fixed list is not permission for arbitrary URLs.
|
||||
withWatch := &config.Config{Crawl: &config.CrawlConfig{
|
||||
Watches: []config.CrawlWatchConfig{{Name: "p", URL: "https://example.org/p"}},
|
||||
}}
|
||||
if c := onDemandCrawler(withWatch); c != nil {
|
||||
t.Error("onDemandCrawler honoured a watch list as on-demand permission")
|
||||
}
|
||||
if c := newCrawler(withWatch); c == nil {
|
||||
t.Error("newCrawler returned nil for a configured watch")
|
||||
}
|
||||
}
|
||||
|
||||
// The wired fetcher must refuse a private address, because the crawler on this
|
||||
// box sits one hop from the whole homelab. Same guard the webfetch tests cover;
|
||||
// this asserts the daemon actually wires it.
|
||||
func TestCrawlerRefusesPrivateAddress(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
w.Write([]byte("<html><body>secret</body></html>"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := newCrawler(&config.Config{Crawl: &config.CrawlConfig{OnDemand: true}})
|
||||
if c == nil {
|
||||
t.Fatal("newCrawler returned nil for an on-demand config")
|
||||
}
|
||||
if _, err := c.Page(context.Background(), srv.URL); err == nil {
|
||||
t.Fatalf("reading %s succeeded; a loopback address must be refused", srv.URL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFactHashesRoundTrip(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
st := newTestStore(t)
|
||||
h := &factHashes{api: ipc.NewStoreAPI(st)}
|
||||
|
||||
got, err := h.LastHash(ctx, "page")
|
||||
if err != nil {
|
||||
t.Fatalf("LastHash on a fresh store: %v", err)
|
||||
}
|
||||
if got != "" {
|
||||
t.Errorf("LastHash = %q, want empty for a never-read page", got)
|
||||
}
|
||||
if err := h.SetHash(ctx, "page", "deadbeef"); err != nil {
|
||||
t.Fatalf("SetHash: %v", err)
|
||||
}
|
||||
got, err = h.LastHash(ctx, "page")
|
||||
if err != nil {
|
||||
t.Fatalf("LastHash: %v", err)
|
||||
}
|
||||
if got != "deadbeef" {
|
||||
t.Errorf("LastHash = %q, want deadbeef", got)
|
||||
}
|
||||
if key := hashKey("page"); key != "crawl:hash:page" {
|
||||
t.Errorf("hashKey = %q", key)
|
||||
}
|
||||
}
|
||||
|
||||
// stubCrawlFetcher serves one fixed page to every URL, so queryWeb can be
|
||||
// exercised without a network or an allowlist.
|
||||
type stubCrawlFetcher struct{ body, ctype string }
|
||||
|
||||
func (s *stubCrawlFetcher) Get(_ context.Context, u string) (*crawl.Response, error) {
|
||||
ct := s.ctype
|
||||
if ct == "" {
|
||||
ct = "text/html"
|
||||
}
|
||||
if strings.HasSuffix(u, "/robots.txt") {
|
||||
return &crawl.Response{URL: u, ContentType: "text/plain", Body: []byte("")}, nil
|
||||
}
|
||||
return &crawl.Response{URL: u, ContentType: ct, Body: []byte(s.body)}, nil
|
||||
}
|
||||
|
||||
func buildWebHandler(c *crawl.Crawler) *reactiveHandler {
|
||||
return &reactiveHandler{
|
||||
replier: voice.NewStubReplier(),
|
||||
phraser: phraser.NewStub(),
|
||||
crawler: c,
|
||||
}
|
||||
}
|
||||
|
||||
func askWeb(h *reactiveHandler, q string) (string, bool) {
|
||||
return h.queryWeb(context.Background(), &queryTurn{
|
||||
dec: router.Decision{Intent: router.IntentQuery, Utterance: q},
|
||||
})
|
||||
}
|
||||
|
||||
func TestQueryWebPassesWithoutAURL(t *testing.T) {
|
||||
h := buildWebHandler(crawl.New(&stubCrawlFetcher{body: "<html><body>x</body></html>"}, crawl.Config{}))
|
||||
if reply, ok := askWeb(h, "почему небо синее?"); ok {
|
||||
t.Errorf("the web source claimed a question with no URL: %q", reply)
|
||||
}
|
||||
}
|
||||
|
||||
// Not configured is said out loud rather than falling through, so a small model
|
||||
// never invents a page's contents from its URL.
|
||||
func TestQueryWebSaysWhenNotConfigured(t *testing.T) {
|
||||
h := buildWebHandler(nil)
|
||||
reply, ok := askWeb(h, "посмотри https://example.org/page")
|
||||
if !ok {
|
||||
t.Fatal("the web source did not claim a question with a URL")
|
||||
}
|
||||
if !strings.Contains(reply, "не настроено") {
|
||||
t.Errorf("reply = %q, want the not-configured answer", reply)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryWebReadsThePage(t *testing.T) {
|
||||
h := buildWebHandler(crawl.New(&stubCrawlFetcher{
|
||||
body: "<html><head><title>Заголовок</title></head><body><p>текст страницы</p></body></html>",
|
||||
}, crawl.Config{}))
|
||||
reply, ok := askWeb(h, "посмотри https://example.org/page — что там?")
|
||||
if !ok {
|
||||
t.Fatal("the web source did not claim a question with a URL")
|
||||
}
|
||||
if !strings.Contains(reply, "текст страницы") {
|
||||
t.Errorf("reply = %q, want the page text read back", reply)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryWebRefusesNonHTML(t *testing.T) {
|
||||
h := buildWebHandler(crawl.New(&stubCrawlFetcher{
|
||||
body: "\x00\x01binary", ctype: "application/octet-stream",
|
||||
}, crawl.Config{}))
|
||||
reply, ok := askWeb(h, "почитай https://example.org/blob.bin")
|
||||
if !ok {
|
||||
t.Fatal("the web source did not claim a question with a URL")
|
||||
}
|
||||
if !strings.Contains(reply, "не получилось") {
|
||||
t.Errorf("reply = %q, want the read-failed answer", reply)
|
||||
}
|
||||
}
|
||||
|
||||
// robots.txt is honoured on the answer path too, and she says so instead of
|
||||
// reporting a generic failure.
|
||||
func TestQueryWebObeysRobots(t *testing.T) {
|
||||
h := buildWebHandler(crawl.New(&robotsDenyFetcher{}, crawl.Config{}))
|
||||
reply, ok := askWeb(h, "посмотри https://example.org/private")
|
||||
if !ok {
|
||||
t.Fatal("the web source did not claim a question with a URL")
|
||||
}
|
||||
if !strings.Contains(reply, "robots.txt") {
|
||||
t.Errorf("reply = %q, want the robots answer", reply)
|
||||
}
|
||||
}
|
||||
|
||||
type robotsDenyFetcher struct{}
|
||||
|
||||
func (robotsDenyFetcher) Get(_ context.Context, u string) (*crawl.Response, error) {
|
||||
if strings.HasSuffix(u, "/robots.txt") {
|
||||
return &crawl.Response{URL: u, ContentType: "text/plain",
|
||||
Body: []byte("User-agent: *\nDisallow: /private\n")}, nil
|
||||
}
|
||||
return &crawl.Response{URL: u, ContentType: "text/html", Body: []byte("<html>nope</html>")}, nil
|
||||
}
|
||||
@@ -141,3 +141,87 @@ func TestDayPlanSourcePrecedesCalendar(t *testing.T) {
|
||||
t.Errorf("day-plan at %d must come before calendar at %d", plan, cal)
|
||||
}
|
||||
}
|
||||
|
||||
// habitAPI answers only RecentFacts — the whole input the behaviour profile
|
||||
// needs (Vikunja #254). Nothing is asked of the LLM, so nothing else is wired.
|
||||
type habitAPI struct {
|
||||
ipc.UnimplementedCoreAPI
|
||||
facts []ipc.Fact
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (a *habitAPI) RecentFacts(_ context.Context, _ int) ([]ipc.Fact, error) {
|
||||
a.calls++
|
||||
return a.facts, a.err
|
||||
}
|
||||
|
||||
// tuesdayFacts — n weekly Tuesday rows for key, ending before now.
|
||||
func tuesdayFacts(key string, hh, weeks int, now time.Time) []ipc.Fact {
|
||||
d := now
|
||||
for d.Weekday() != time.Tuesday {
|
||||
d = d.AddDate(0, 0, -1)
|
||||
}
|
||||
var out []ipc.Fact
|
||||
for i := 0; i < weeks; i++ {
|
||||
day := d.AddDate(0, 0, -7*i)
|
||||
out = append(out, ipc.Fact{
|
||||
Ts: time.Date(day.Year(), day.Month(), day.Day(), hh, 0, 0, 0, now.Location()),
|
||||
Kind: "self",
|
||||
Key: key,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestQueryHabitsAnswersFromCountedFacts(t *testing.T) {
|
||||
now := planDay() // a Monday
|
||||
api := &habitAPI{facts: tuesdayFacts("workout", 19, 4, now)}
|
||||
h := &reactiveHandler{api: api, now: func() time.Time { return now }}
|
||||
|
||||
reply, ok := h.queryHabits(context.Background(), &queryTurn{
|
||||
dec: router.Decision{Intent: router.IntentQuery, Utterance: "что я обычно делаю по вторникам?"},
|
||||
})
|
||||
if !ok {
|
||||
t.Fatal("the habit source must claim a habit question")
|
||||
}
|
||||
if want := "по вторникам ты обычно тренируешься около 19:00."; reply != want {
|
||||
t.Errorf("reply = %q, want %q", reply, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryHabitsPassesOnEverythingElse(t *testing.T) {
|
||||
now := planDay()
|
||||
for _, q := range []string{"что я делаю в среду?", "что у меня сегодня?", "какие планы на сегодня?", ""} {
|
||||
api := &habitAPI{}
|
||||
h := &reactiveHandler{api: api, now: func() time.Time { return now }}
|
||||
if reply, ok := h.queryHabits(context.Background(), &queryTurn{
|
||||
dec: router.Decision{Intent: router.IntentQuery, Utterance: q},
|
||||
}); ok {
|
||||
t.Errorf("%q was claimed by the habit source (reply %q)", q, reply)
|
||||
}
|
||||
if api.calls != 0 {
|
||||
t.Errorf("%q scanned the fact log for a profile it does not want", q)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Both specific sources must precede the calendar listing, which matches any
|
||||
// utterance naming a day.
|
||||
func TestHabitSourcePrecedesCalendar(t *testing.T) {
|
||||
habits, cal := -1, -1
|
||||
for i, s := range querySources {
|
||||
switch s.name {
|
||||
case "habits":
|
||||
habits = i
|
||||
case "calendar":
|
||||
cal = i
|
||||
}
|
||||
}
|
||||
if habits < 0 || cal < 0 {
|
||||
t.Fatalf("sources missing: habits=%d calendar=%d", habits, cal)
|
||||
}
|
||||
if habits > cal {
|
||||
t.Errorf("habits at %d must come before calendar at %d", habits, cal)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
// mavend/feeds.go — the driver for RSS/Atom reading (Vikunja #258,
|
||||
// docs/plans/13-rss-news-feeds.md). The reader itself is pure and lives in
|
||||
// internal/rss; this is the impure half: a ticker, the guarded fetcher, and the
|
||||
// two adapters that let a pure package talk to the store.
|
||||
//
|
||||
// Why in-core rather than its own daemon like mavmaild and mavpoll: those two
|
||||
// hold a CREDENTIAL (an IMAP password, a zenmoney token), and the reason they
|
||||
// are separate processes is that core must never see it. A feed URL is public,
|
||||
// there is no secret to isolate, and a whole extra binary and compose service
|
||||
// would buy nothing. The other half of the mavpoll precedent — off unless
|
||||
// configured — is kept: no `feeds` block, no poller, no outbound request.
|
||||
//
|
||||
// It is its own goroutine, not a step on the tick: the tick has a delivery
|
||||
// deadline behind it, and a feed read is a network round-trip that nobody is
|
||||
// waiting on.
|
||||
//
|
||||
// Nothing here dispatches. A feed that announced itself would be a nag, so the
|
||||
// only output is notes with source "rss:<feed>", which the answer path reads
|
||||
// when he asks ("что нового в лентах?" — see queryFeeds in actions_query.go).
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/config"
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/router"
|
||||
"github.com/kami/maven/internal/rss"
|
||||
"github.com/kami/maven/internal/stt"
|
||||
"github.com/kami/maven/internal/webfetch"
|
||||
)
|
||||
|
||||
// feedWorker — ticker + poller.
|
||||
type feedWorker struct {
|
||||
poller *rss.Poller
|
||||
interval time.Duration
|
||||
}
|
||||
|
||||
// feedTickInterval — how often the worker asks the poller what is due. Per-feed
|
||||
// cadence is the poller's business; this is just the granularity.
|
||||
const feedTickInterval = 5 * time.Minute
|
||||
|
||||
// newFeedWorker wires feed reading, or returns nil when it must not run:
|
||||
// no `feeds` block (the normal case), or nothing valid in it. Every caller
|
||||
// checks for nil.
|
||||
func newFeedWorker(api ipc.CoreAPI, emb router.Embedder, cfg *config.Config) *feedWorker {
|
||||
if cfg.Feeds == nil {
|
||||
return nil
|
||||
}
|
||||
fc := cfg.Feeds
|
||||
|
||||
feeds := make([]rss.FeedConfig, 0, len(fc.Sources))
|
||||
hosts := append([]string(nil), fc.AllowHosts...)
|
||||
for _, s := range fc.Sources {
|
||||
feeds = append(feeds, rss.FeedConfig{
|
||||
Name: s.Name,
|
||||
URL: s.URL,
|
||||
Category: s.Category,
|
||||
Interval: time.Duration(s.Interval),
|
||||
Include: s.Include,
|
||||
Exclude: s.Exclude,
|
||||
})
|
||||
// Each configured feed's own host is allowed. The allowlist is then
|
||||
// exactly "the feeds he asked for", so a redirect off to somewhere else
|
||||
// is refused by the fetcher rather than followed.
|
||||
if u, err := url.Parse(s.URL); err == nil && u.Hostname() != "" {
|
||||
hosts = append(hosts, u.Hostname())
|
||||
}
|
||||
}
|
||||
|
||||
fetcher := webfetch.New(webfetch.Config{
|
||||
AllowHosts: hosts,
|
||||
Timeout: time.Duration(fc.Timeout),
|
||||
MaxBytes: fc.MaxBytes,
|
||||
})
|
||||
poller := rss.NewPoller(feeds, &feedFetcher{f: fetcher}, api, &factMarks{api: api},
|
||||
embedderFor(emb), nil, rss.Config{
|
||||
DefaultInterval: time.Duration(fc.PollInterval),
|
||||
MaxItems: fc.MaxItems,
|
||||
MaxAge: time.Duration(fc.MaxAge),
|
||||
})
|
||||
if poller == nil {
|
||||
log.Printf("feeds: configured but nothing pollable — feed reading disabled")
|
||||
return nil
|
||||
}
|
||||
log.Printf("feeds: reading %d feed(s), checking what is due every %s", len(feeds), feedTickInterval)
|
||||
return &feedWorker{poller: poller, interval: feedTickInterval}
|
||||
}
|
||||
|
||||
// run polls what is due until ctx is canceled. The first round runs immediately
|
||||
// so a restart does not blind her for the first interval; it writes notes only,
|
||||
// so an early round cannot startle anyone.
|
||||
func (w *feedWorker) run(ctx context.Context) {
|
||||
w.poller.PollDue(ctx, time.Now())
|
||||
t := time.NewTicker(w.interval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case now := <-t.C:
|
||||
w.poller.PollDue(ctx, now)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// embedderOf — the voice wiring's embedder, or nil when voice is not wired.
|
||||
// Feed notes are embedded with the SAME model the rest of the store uses, or not
|
||||
// at all; a second embedder would write vectors nothing can search.
|
||||
func embedderOf(w *voiceWiring) router.Embedder {
|
||||
if w == nil {
|
||||
return nil
|
||||
}
|
||||
return w.embedder
|
||||
}
|
||||
|
||||
// transcriberOf — the STT the voice path is using, or nil when voice is off.
|
||||
// The meeting recorder reuses it rather than dialling mavsttd a second time:
|
||||
// Maven has one speech-to-text engine and adding a second would mean two
|
||||
// whisper contexts competing for the same iGPU.
|
||||
func transcriberOf(w *voiceWiring) stt.Transcriber {
|
||||
if w == nil {
|
||||
return nil
|
||||
}
|
||||
return w.transcriber
|
||||
}
|
||||
|
||||
// feedFetcher adapts webfetch to rss.Fetcher — the pure package names the two
|
||||
// fields it needs and stays free of net/http.
|
||||
type feedFetcher struct{ f *webfetch.Fetcher }
|
||||
|
||||
func (a *feedFetcher) Get(ctx context.Context, u string) (*rss.Body, error) {
|
||||
resp, err := a.f.Get(ctx, u)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &rss.Body{Bytes: resp.Body}, nil
|
||||
}
|
||||
|
||||
// factMarks stores "how far this feed was read" as a config fact, the same
|
||||
// mechanism the plan named and the same one the pattern tick uses for its own
|
||||
// bookkeeping. Durable, inspectable on /dash, and cheap.
|
||||
type factMarks struct{ api ipc.CoreAPI }
|
||||
|
||||
func markKey(feed string) string { return "rss:latest:" + feed }
|
||||
|
||||
func (m *factMarks) LastMark(ctx context.Context, feed string) (time.Time, error) {
|
||||
f, err := m.api.LatestFact(ctx, markKey(feed))
|
||||
if err != nil {
|
||||
// No mark yet is not an error worth propagating: the poller treats a
|
||||
// zero time as a cold start.
|
||||
return time.Time{}, nil
|
||||
}
|
||||
t, err := time.Parse(time.RFC3339, f.Value)
|
||||
if err != nil {
|
||||
return time.Time{}, nil
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
func (m *factMarks) SetMark(ctx context.Context, feed string, at time.Time) error {
|
||||
_, err := m.api.WriteFact(ctx, ipc.WriteFactReq{
|
||||
Ts: time.Now(),
|
||||
Kind: "config",
|
||||
Key: markKey(feed),
|
||||
Value: at.UTC().Format(time.RFC3339),
|
||||
Source: "poll:rss",
|
||||
Confidence: 1.0,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// embedderFor adapts router.Embedder to rss.Embedder, and returns nil when
|
||||
// there is none — a note without a vector is still a note the recent-notes path
|
||||
// can read.
|
||||
//
|
||||
// EmbedPassage, not Embed: a feed item is text being searched FOR, and the e5
|
||||
// embedder is asymmetric. Getting this backwards makes the item unfindable by
|
||||
// the question that should have matched it.
|
||||
func embedderFor(emb router.Embedder) rss.Embedder {
|
||||
if emb == nil {
|
||||
return nil
|
||||
}
|
||||
return passageEmbedder{emb}
|
||||
}
|
||||
|
||||
type passageEmbedder struct{ e router.Embedder }
|
||||
|
||||
func (p passageEmbedder) Embed(ctx context.Context, text string) ([]float32, error) {
|
||||
return router.EmbedPassage(ctx, p.e, text)
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/config"
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/phraser"
|
||||
"github.com/kami/maven/internal/router"
|
||||
"github.com/kami/maven/internal/rss"
|
||||
"github.com/kami/maven/internal/voice"
|
||||
)
|
||||
|
||||
// buildFeedHandler — a handler with the given feed notes already stored. No
|
||||
// embedder: the feed source answers from recent notes by source, which is what
|
||||
// makes it work for notes written before an embedder existed.
|
||||
func buildFeedHandler(t *testing.T, feedsOn bool, notes ...ipc.Note) *reactiveHandler {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
st := newTestStore(t)
|
||||
now := time.Now()
|
||||
for i, n := range notes {
|
||||
ts := now.Add(time.Duration(i) * time.Minute)
|
||||
if _, err := st.WriteNote(ctx, ts, n.Text, nil, n.Source); err != nil {
|
||||
t.Fatalf("WriteNote: %v", err)
|
||||
}
|
||||
}
|
||||
return &reactiveHandler{
|
||||
api: ipc.NewStoreAPI(st),
|
||||
replier: voice.NewStubReplier(),
|
||||
phraser: phraser.NewStub(),
|
||||
now: func() time.Time { return now },
|
||||
feedsOn: feedsOn,
|
||||
embedder: nil,
|
||||
}
|
||||
}
|
||||
|
||||
func askFeeds(t *testing.T, h *reactiveHandler, q string) (string, bool) {
|
||||
t.Helper()
|
||||
return h.queryFeeds(context.Background(), &queryTurn{
|
||||
dec: router.Decision{Intent: router.IntentQuery, Utterance: q},
|
||||
})
|
||||
}
|
||||
|
||||
func TestQueryFeedsReadsFeedNotes(t *testing.T) {
|
||||
h := buildFeedHandler(t, true,
|
||||
ipc.Note{Text: "Новая уязвимость в ядре [технологии]\nпатч вышел\nhttps://example.org/a", Source: "rss:habr"},
|
||||
ipc.Note{Text: "что-то он сам сказал", Source: "tap:voice"},
|
||||
)
|
||||
reply, ok := askFeeds(t, h, "что нового в лентах?")
|
||||
if !ok {
|
||||
t.Fatal("the feed source did not claim the question")
|
||||
}
|
||||
if !strings.Contains(reply, "уязвимость") {
|
||||
t.Errorf("reply = %q, want the headline", reply)
|
||||
}
|
||||
if strings.Contains(reply, "он сам сказал") {
|
||||
t.Errorf("a note he dictated leaked into the feed answer: %q", reply)
|
||||
}
|
||||
// She reads the headline, not the summary and not the URL.
|
||||
if strings.Contains(reply, "https://") || strings.Contains(reply, "патч вышел") {
|
||||
t.Errorf("reply = %q, want the title line only", reply)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryFeedsByCategory(t *testing.T) {
|
||||
h := buildFeedHandler(t, true,
|
||||
ipc.Note{Text: "Релиз ядра [технологии]", Source: "rss:habr"},
|
||||
ipc.Note{Text: "Выборы отложены [политика]", Source: "rss:news"},
|
||||
)
|
||||
reply, ok := askFeeds(t, h, "что нового по технологиям?")
|
||||
if !ok {
|
||||
t.Fatal("not claimed")
|
||||
}
|
||||
if !strings.Contains(reply, "ядра") || strings.Contains(reply, "Выборы") {
|
||||
t.Fatalf("reply = %q, want only the технологии item", reply)
|
||||
}
|
||||
reply, _ = askFeeds(t, h, "что нового по спорту?")
|
||||
if !strings.Contains(reply, "ничего") {
|
||||
t.Fatalf("reply = %q, want an honest empty answer for an unread category", reply)
|
||||
}
|
||||
}
|
||||
|
||||
// "не настроены" and "ничего нового" are different truths, and neither may be
|
||||
// answered by the model inventing a bulletin.
|
||||
func TestQueryFeedsOffAndEmptyDiffer(t *testing.T) {
|
||||
off := buildFeedHandler(t, false)
|
||||
reply, ok := askFeeds(t, off, "что нового?")
|
||||
if !ok || !strings.Contains(reply, "не настроены") {
|
||||
t.Fatalf("feeds off: reply = %q, ok = %v", reply, ok)
|
||||
}
|
||||
on := buildFeedHandler(t, true)
|
||||
reply, ok = askFeeds(t, on, "что нового?")
|
||||
if !ok || !strings.Contains(reply, "ничего нового") {
|
||||
t.Fatalf("feeds on but empty: reply = %q, ok = %v", reply, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryFeedsPassesOnANonFeedQuestion(t *testing.T) {
|
||||
h := buildFeedHandler(t, true)
|
||||
if reply, ok := askFeeds(t, h, "напомни полить цветы"); ok {
|
||||
t.Fatalf("claimed an unrelated question with %q", reply)
|
||||
}
|
||||
}
|
||||
|
||||
// The mark is what stops a restart from re-noting yesterday's headlines, so the
|
||||
// fact round-trip is worth a test of its own.
|
||||
func TestFactMarksRoundTrip(t *testing.T) {
|
||||
st := newTestStore(t)
|
||||
m := &factMarks{api: ipc.NewStoreAPI(st)}
|
||||
ctx := context.Background()
|
||||
|
||||
at, err := m.LastMark(ctx, "habr")
|
||||
if err != nil || !at.IsZero() {
|
||||
t.Fatalf("no mark yet: got %v, %v — want zero time and no error", at, err)
|
||||
}
|
||||
want := time.Date(2026, 7, 28, 10, 0, 0, 0, time.UTC)
|
||||
if err := m.SetMark(ctx, "habr", want); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := m.LastMark(ctx, "habr")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !got.Equal(want) {
|
||||
t.Fatalf("mark = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// Off unless configured, checked at the wiring seam: no `feeds` block ⇒ no
|
||||
// worker ⇒ no outbound request is possible.
|
||||
func TestNewFeedWorkerOffByDefault(t *testing.T) {
|
||||
st := newTestStore(t)
|
||||
api := ipc.NewStoreAPI(st)
|
||||
if w := newFeedWorker(api, nil, &config.Config{}); w != nil {
|
||||
t.Fatal("a config with no feeds block wired a feed worker")
|
||||
}
|
||||
// An empty sources list is normalised to "off" by config.Load; the worker
|
||||
// refuses it too, so a hand-built Config cannot switch it on by accident.
|
||||
if w := newFeedWorker(api, nil, &config.Config{Feeds: &config.FeedsConfig{}}); w != nil {
|
||||
t.Fatal("an empty sources list wired a feed worker")
|
||||
}
|
||||
cfg := &config.Config{Feeds: &config.FeedsConfig{Sources: []config.FeedSourceConfig{
|
||||
{Name: "habr", URL: "https://example.org/rss"},
|
||||
}}}
|
||||
w := newFeedWorker(api, nil, cfg)
|
||||
if w == nil {
|
||||
t.Fatal("a configured feed did not wire a worker")
|
||||
}
|
||||
if got := w.poller.Feeds(); len(got) != 1 || got[0].Name != "habr" {
|
||||
t.Fatalf("feeds = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The fetcher the worker builds must be allowlisted to the configured feeds and
|
||||
// nothing else — the crawler's SSRF guards are only worth as much as the
|
||||
// allowlist handed to them.
|
||||
func TestFeedWorkerFetcherIsAllowlisted(t *testing.T) {
|
||||
cfg := &config.Config{Feeds: &config.FeedsConfig{Sources: []config.FeedSourceConfig{
|
||||
{Name: "habr", URL: "https://feeds.example.org/rss"},
|
||||
}}}
|
||||
w := newFeedWorker(ipc.NewStoreAPI(newTestStore(t)), nil, cfg)
|
||||
if w == nil {
|
||||
t.Fatal("no worker")
|
||||
}
|
||||
// PollFeed goes through the guarded fetcher; a feed URL pointing at the box
|
||||
// itself must fail rather than be read.
|
||||
_, err := w.poller.PollFeed(context.Background(), rss.FeedConfig{
|
||||
Name: "evil", URL: "http://127.0.0.1:9100/mcp",
|
||||
}, time.Now())
|
||||
if err == nil {
|
||||
t.Fatal("the poller fetched a private address")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
// mavend/mail.go — core's half of the email reader (Vikunja #246,
|
||||
// docs/plans/01-email-reader.md).
|
||||
//
|
||||
// The split: cmd/mavmaild holds the IMAP credential, connects to the mailbox
|
||||
// and converts messages to plaintext; it hands each message to core over
|
||||
// ipc.MethodIngestMail. Core runs the extraction on the resident model —
|
||||
// llama-server lives in this process, spawned by the phraser — and writes what
|
||||
// comes back through the one task intake seam.
|
||||
//
|
||||
// What this file may produce is exactly one thing: rows in `tasks` with status
|
||||
// "candidate". No fact, no reminder, no note, no nudge, no calendar event. A
|
||||
// 1.7B misreading a mail can therefore put a wrong line on a review page and
|
||||
// nothing else; it can never make Maven speak, and it can never make her
|
||||
// recite something out of an advert as true.
|
||||
//
|
||||
// Off unless configured twice over: no `email` block in mavend.json ⇒ the IPC
|
||||
// method does not exist; no llama-server phraser ⇒ same. A reader pointed at a
|
||||
// core that is not set up for mail gets ErrUnknownMethod rather than silence.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/config"
|
||||
"github.com/kami/maven/internal/email"
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/phraser"
|
||||
"github.com/kami/maven/internal/store"
|
||||
)
|
||||
|
||||
// evidenceMaxChars — how much of the subject line is kept as a candidate's
|
||||
// evidence. Enough to recognise the mail on /tasks, not enough to turn the task
|
||||
// list into a copy of his mailbox.
|
||||
const evidenceMaxChars = 160
|
||||
|
||||
// mailIntake — extraction + capture for one message at a time.
|
||||
type mailIntake struct {
|
||||
st *store.Store
|
||||
ex *email.Extractor
|
||||
timeout time.Duration
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// newMailIntake returns nil when mail ingestion must not be available, which is
|
||||
// the default. Both preconditions are real:
|
||||
//
|
||||
// - no cfg.Email ⇒ not configured, and a capability is off unless configured;
|
||||
// - no llama-server phraser ⇒ nothing to extract with. There is deliberately
|
||||
// no keyword fallback: "the subject line became a task" is not extraction,
|
||||
// it is a mailbox rendered as a to-do list, and it would fill the review
|
||||
// page faster than he could clear it.
|
||||
func newMailIntake(st *store.Store, phr phraser.Phraser, cfg *config.Config) *mailIntake {
|
||||
if cfg.Email == nil {
|
||||
return nil
|
||||
}
|
||||
lp, ok := phr.(*phraser.LLMPhraser)
|
||||
if !ok {
|
||||
log.Printf("mail intake: configured but no llama-server phraser — mail ingestion disabled")
|
||||
return nil
|
||||
}
|
||||
timeout := time.Duration(cfg.Email.Timeout)
|
||||
if timeout <= 0 {
|
||||
timeout = config.DefaultEmailTimeout
|
||||
}
|
||||
ex := email.NewExtractor(llmClientFor(lp, timeout), cfg.Email.MaxTasks, contextBlockFn(cfg, time.Now))
|
||||
log.Printf("mail intake: enabled (max %d candidates per message, timeout %s)", cfg.Email.MaxTasks, timeout)
|
||||
return &mailIntake{st: st, ex: ex, timeout: timeout, now: time.Now}
|
||||
}
|
||||
|
||||
// ingest handles one ipc.MethodIngestMail call.
|
||||
//
|
||||
// Junk and empty messages are answered Skipped without touching the model — the
|
||||
// reader's header filter is what keeps the resident model off newsletters.
|
||||
//
|
||||
// Every candidate is captured with Status "candidate", Source "email:<mailbox>"
|
||||
// and the subject as Evidence. CaptureTask dedupes on normalised text among
|
||||
// live rows, so a mailbox re-read after a restart produces Created=0 rather
|
||||
// than a second copy of every task.
|
||||
func (m *mailIntake) ingest(ctx context.Context, req ipc.IngestMailReq) (ipc.IngestMailResp, error) {
|
||||
msg := email.Message{
|
||||
UID: req.UID,
|
||||
From: req.From,
|
||||
Subject: req.Subject,
|
||||
Date: req.Date,
|
||||
Body: req.Body,
|
||||
Junk: req.Junk,
|
||||
}
|
||||
if msg.Junk || (msg.Subject == "" && msg.Body == "") {
|
||||
return ipc.IngestMailResp{Skipped: true}, nil
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, m.timeout)
|
||||
defer cancel()
|
||||
cands, err := m.ex.Extract(ctx, msg)
|
||||
if err != nil {
|
||||
// The error from internal/email never carries mail text; keep it that way
|
||||
// by not adding the subject here.
|
||||
return ipc.IngestMailResp{}, fmt.Errorf("mail intake: uid %d: %w", req.UID, err)
|
||||
}
|
||||
if len(cands) == 0 {
|
||||
return ipc.IngestMailResp{}, nil
|
||||
}
|
||||
|
||||
source := email.SourcePrefix + req.Mailbox
|
||||
evidence := truncateRunes(req.Subject, evidenceMaxChars)
|
||||
now := m.now()
|
||||
var resp ipc.IngestMailResp
|
||||
for _, c := range cands {
|
||||
t := store.Task{
|
||||
CreatedTs: now,
|
||||
Text: c.Text,
|
||||
Source: source,
|
||||
Evidence: evidence,
|
||||
// The one status this path may ever write. Anything Maven derived from
|
||||
// something she read is a suggestion until he confirms it on /tasks.
|
||||
Status: store.TaskCandidate,
|
||||
}
|
||||
if due, ok := email.ParseDue(c.Due); ok {
|
||||
t.Due = &due
|
||||
}
|
||||
id, created, err := m.st.CaptureTask(ctx, t)
|
||||
if err != nil {
|
||||
return resp, fmt.Errorf("mail intake: capture: %w", err)
|
||||
}
|
||||
resp.TaskIDs = append(resp.TaskIDs, id)
|
||||
if created {
|
||||
resp.Created++
|
||||
}
|
||||
}
|
||||
// Counts only: the log line names the mailbox and the UID, never the subject,
|
||||
// the sender or the task text. Reviewing a candidate is what /tasks is for.
|
||||
log.Printf("mail intake: %s uid %d → %d candidate(s), %d new", source, req.UID, len(cands), resp.Created)
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// wireMailIntake installs the IPC hook, or leaves it nil so the method reports
|
||||
// ErrUnknownMethod. Called on both startup paths (unlocked boot and passkey
|
||||
// unlock) so mail behaves the same either way.
|
||||
func wireMailIntake(srv *ipc.Server, st *store.Store, phr phraser.Phraser, cfg *config.Config) {
|
||||
mi := newMailIntake(st, phr, cfg)
|
||||
if mi == nil {
|
||||
return
|
||||
}
|
||||
srv.IngestMailFn = mi.ingest
|
||||
}
|
||||
|
||||
// truncateRunes cuts a string to n runes, marking the cut.
|
||||
func truncateRunes(s string, n int) string {
|
||||
r := []rune(s)
|
||||
if len(r) <= n {
|
||||
return s
|
||||
}
|
||||
return string(r[:n]) + "…"
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/config"
|
||||
"github.com/kami/maven/internal/email"
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/llm"
|
||||
"github.com/kami/maven/internal/store"
|
||||
)
|
||||
|
||||
// mailLLM — a canned extraction reply.
|
||||
type mailLLM struct {
|
||||
reply string
|
||||
calls int
|
||||
}
|
||||
|
||||
func (m *mailLLM) Complete(_ context.Context, _ llm.Req) (string, error) {
|
||||
m.calls++
|
||||
return m.reply, nil
|
||||
}
|
||||
|
||||
func newTestIntake(t *testing.T, reply string) (*mailIntake, *store.Store, *mailLLM) {
|
||||
t.Helper()
|
||||
st := newTestStore(t)
|
||||
fake := &mailLLM{reply: reply}
|
||||
return &mailIntake{
|
||||
st: st,
|
||||
ex: email.NewExtractor(fake, 0, nil),
|
||||
timeout: 5 * time.Second,
|
||||
now: func() time.Time { return time.Date(2026, 8, 1, 10, 0, 0, 0, time.UTC) },
|
||||
}, st, fake
|
||||
}
|
||||
|
||||
func ingestReq() ipc.IngestMailReq {
|
||||
return ipc.IngestMailReq{
|
||||
Mailbox: "INBOX", UID: 42,
|
||||
From: "billing@isp.example",
|
||||
Subject: "Счёт за интернет",
|
||||
Body: "Оплатите счёт до 5 августа.",
|
||||
}
|
||||
}
|
||||
|
||||
// The one property that matters: a mail-derived task is a candidate, attributed
|
||||
// to the mailbox, with the subject as reviewable evidence — and nothing else is
|
||||
// written.
|
||||
func TestIngestCapturesCandidates(t *testing.T) {
|
||||
mi, st, _ := newTestIntake(t, `[{"text":"оплатить счёт за интернет","due":"2026-08-05"}]`)
|
||||
resp, err := mi.ingest(context.Background(), ingestReq())
|
||||
if err != nil {
|
||||
t.Fatalf("ingest: %v", err)
|
||||
}
|
||||
if resp.Created != 1 || len(resp.TaskIDs) != 1 {
|
||||
t.Fatalf("resp = %+v, want one created task", resp)
|
||||
}
|
||||
tasks, err := st.ListTasks(context.Background(), "")
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if len(tasks) != 1 {
|
||||
t.Fatalf("got %d tasks, want 1", len(tasks))
|
||||
}
|
||||
got := tasks[0]
|
||||
if got.Status != store.TaskCandidate {
|
||||
t.Errorf("status = %q, want %q — mail may only produce candidates", got.Status, store.TaskCandidate)
|
||||
}
|
||||
if got.Source != "email:INBOX" {
|
||||
t.Errorf("source = %q, want email:INBOX", got.Source)
|
||||
}
|
||||
if got.Evidence != "Счёт за интернет" {
|
||||
t.Errorf("evidence = %q, want the subject line", got.Evidence)
|
||||
}
|
||||
if got.Due == nil || got.Due.Format("2006-01-02") != "2026-08-05" {
|
||||
t.Errorf("due = %v, want 2026-08-05", got.Due)
|
||||
}
|
||||
// Nothing else may have been written: no reminder, no fact.
|
||||
rem, err := st.ListReminders(context.Background(), 10)
|
||||
if err != nil {
|
||||
t.Fatalf("list reminders: %v", err)
|
||||
}
|
||||
if len(rem) != 0 {
|
||||
t.Errorf("mail created %d reminders; a misread mail must never be able to fire", len(rem))
|
||||
}
|
||||
}
|
||||
|
||||
// Re-reading a mailbox must not grow the list — CaptureTask dedupes among live
|
||||
// rows, and the intake relies on exactly that.
|
||||
func TestIngestSameMailTwiceIsIdempotent(t *testing.T) {
|
||||
mi, st, _ := newTestIntake(t, `[{"text":"оплатить счёт","due":""}]`)
|
||||
if _, err := mi.ingest(context.Background(), ingestReq()); err != nil {
|
||||
t.Fatalf("first ingest: %v", err)
|
||||
}
|
||||
resp, err := mi.ingest(context.Background(), ingestReq())
|
||||
if err != nil {
|
||||
t.Fatalf("second ingest: %v", err)
|
||||
}
|
||||
if resp.Created != 0 || len(resp.TaskIDs) != 1 {
|
||||
t.Errorf("resp = %+v, want the existing row and Created=0", resp)
|
||||
}
|
||||
tasks, _ := st.ListTasks(context.Background(), "")
|
||||
if len(tasks) != 1 {
|
||||
t.Errorf("got %d tasks after two reads, want 1", len(tasks))
|
||||
}
|
||||
}
|
||||
|
||||
func TestIngestJunkSkipsTheModel(t *testing.T) {
|
||||
mi, st, fake := newTestIntake(t, `[{"text":"купить со скидкой","due":""}]`)
|
||||
req := ingestReq()
|
||||
req.Junk = true
|
||||
resp, err := mi.ingest(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("ingest: %v", err)
|
||||
}
|
||||
if !resp.Skipped || resp.Created != 0 {
|
||||
t.Errorf("resp = %+v, want skipped", resp)
|
||||
}
|
||||
if fake.calls != 0 {
|
||||
t.Errorf("model called %d times for junk, want 0", fake.calls)
|
||||
}
|
||||
if tasks, _ := st.ListTasks(context.Background(), ""); len(tasks) != 0 {
|
||||
t.Errorf("junk produced %d tasks, want 0", len(tasks))
|
||||
}
|
||||
}
|
||||
|
||||
func TestIngestEmptyMessageSkipped(t *testing.T) {
|
||||
mi, _, fake := newTestIntake(t, "[]")
|
||||
resp, err := mi.ingest(context.Background(), ipc.IngestMailReq{Mailbox: "INBOX", UID: 1})
|
||||
if err != nil || !resp.Skipped {
|
||||
t.Fatalf("resp = %+v, err = %v; want skipped", resp, err)
|
||||
}
|
||||
if fake.calls != 0 {
|
||||
t.Errorf("model called %d times for an empty message, want 0", fake.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIngestNoTasksWritesNothing(t *testing.T) {
|
||||
mi, st, _ := newTestIntake(t, "[]")
|
||||
resp, err := mi.ingest(context.Background(), ingestReq())
|
||||
if err != nil {
|
||||
t.Fatalf("ingest: %v", err)
|
||||
}
|
||||
if resp.Created != 0 || len(resp.TaskIDs) != 0 || resp.Skipped {
|
||||
t.Errorf("resp = %+v, want nothing captured and not skipped", resp)
|
||||
}
|
||||
if tasks, _ := st.ListTasks(context.Background(), ""); len(tasks) != 0 {
|
||||
t.Errorf("got %d tasks, want 0", len(tasks))
|
||||
}
|
||||
}
|
||||
|
||||
func TestIngestTruncatesEvidence(t *testing.T) {
|
||||
mi, st, _ := newTestIntake(t, `[{"text":"дело","due":""}]`)
|
||||
req := ingestReq()
|
||||
req.Subject = strings.Repeat("щ", 400)
|
||||
if _, err := mi.ingest(context.Background(), req); err != nil {
|
||||
t.Fatalf("ingest: %v", err)
|
||||
}
|
||||
tasks, _ := st.ListTasks(context.Background(), "")
|
||||
if len(tasks) != 1 {
|
||||
t.Fatalf("got %d tasks, want 1", len(tasks))
|
||||
}
|
||||
if n := len([]rune(tasks[0].Evidence)); n > evidenceMaxChars+1 {
|
||||
t.Errorf("evidence kept %d runes, want ≤ %d", n, evidenceMaxChars)
|
||||
}
|
||||
}
|
||||
|
||||
// Off unless configured: no email block ⇒ no intake, so the IPC method does not
|
||||
// exist at all.
|
||||
func TestNewMailIntakeOffWithoutConfig(t *testing.T) {
|
||||
st := newTestStore(t)
|
||||
if mi := newMailIntake(st, nil, &config.Config{}); mi != nil {
|
||||
t.Error("no email block must mean no mail intake")
|
||||
}
|
||||
// Configured but with a non-LLM phraser: still off — there is no fallback
|
||||
// extraction, by design.
|
||||
if mi := newMailIntake(st, nil, &config.Config{Email: &config.EmailConfig{}}); mi != nil {
|
||||
t.Error("without a llama-server phraser there is nothing to extract with")
|
||||
}
|
||||
}
|
||||
@@ -170,6 +170,8 @@ func run(args []string) error {
|
||||
eco *ecosystemWiring
|
||||
factWorker *factEnrichmentWorker
|
||||
evalWorker *memoryEvalWorker // nil ⇒ memory evaluation off (the default)
|
||||
feedWkr *feedWorker // nil ⇒ no feed is read (the default)
|
||||
crawlWkr *crawlWorker // nil ⇒ no page is watched (the default)
|
||||
)
|
||||
|
||||
if !locked {
|
||||
@@ -265,6 +267,8 @@ func run(args []string) error {
|
||||
tl = newTickLoop(st, gatherer, dispatcher, phr, rules, tickInterval, repeatInterval, autotuneInterval, cfg.Digest, routinesFromConfig(cfg.Routines), config.MorningRoutinesFromConfig(cfg.MorningRoutines), cfg.PatternProposals)
|
||||
factWorker = newFactEnrichmentWorker(st, eco, time.Duration(cfg.FactEnrichmentInterval))
|
||||
evalWorker = newMemoryEvalWorker(st, phr, cfg)
|
||||
feedWkr = newFeedWorker(ipc.NewStoreAPI(st), embedderOf(voiceW), cfg)
|
||||
crawlWkr = newCrawlWorker(newCrawler(cfg), ipc.NewStoreAPI(st), embedderOf(voiceW), cfg)
|
||||
|
||||
coreAPI = &daemonAPI{
|
||||
CoreAPI: ipc.NewStoreAPI(st),
|
||||
@@ -276,6 +280,9 @@ func run(args []string) error {
|
||||
api := coreAPI.(*daemonAPI)
|
||||
api.chatFn = voiceW.handler.handleText
|
||||
}
|
||||
if voiceW != nil && voiceW.mcp != nil {
|
||||
coreAPI.(*daemonAPI).getMCPServers = voiceW.mcp.status
|
||||
}
|
||||
} else {
|
||||
// locked mode: no real store yet, so there's no meaningful CoreAPI to
|
||||
// serve. srv.Check below is the actual guard — every CoreAPI call is
|
||||
@@ -320,6 +327,21 @@ func run(args []string) error {
|
||||
|
||||
srv.StepUp = func(ctx context.Context) error { return passkeySess.Assert(ctx, auth.Scope{}) }
|
||||
|
||||
// Mail ingestion (Vikunja #246): the hook stays nil unless an email block is
|
||||
// configured and there is a llama-server to extract with, in which case
|
||||
// ipc.MethodIngestMail reports ErrUnknownMethod.
|
||||
if !locked {
|
||||
wireMailIntake(srv, st, phr, cfg)
|
||||
wireModelSwap(srv, phr, cfg)
|
||||
// Vision + the media blob store (Vikunja #252). Both stay dark without a
|
||||
// media block; MethodDescribeImage answers ErrUnknownMethod then.
|
||||
keeper := wireVision(ctx, srv, st, embedderOf(voiceW), cfg)
|
||||
// The meeting recorder (Vikunja #253) shares that blob store and its
|
||||
// retention loop. Off unless a capture block enables it, in which case
|
||||
// all four capture methods answer ErrUnknownMethod.
|
||||
wireCapture(srv, keeper, st, voiceW, phr, cfg)
|
||||
}
|
||||
|
||||
// WrapKeyFn — wraps the env key with a passkey credential public key and
|
||||
// persists the wrapped blob. Only wired when the daemon has the key in
|
||||
// memory (env key mode). Called by mavweb after passkey enrollment.
|
||||
@@ -444,6 +466,8 @@ func run(args []string) error {
|
||||
tl = newTickLoop(st, gatherer, dispatcher, phr, rules, tickInterval, repeatInterval, autotuneInterval, cfg.Digest, routinesFromConfig(cfg.Routines), config.MorningRoutinesFromConfig(cfg.MorningRoutines), cfg.PatternProposals)
|
||||
factWorker = newFactEnrichmentWorker(st, eco, time.Duration(cfg.FactEnrichmentInterval))
|
||||
evalWorker = newMemoryEvalWorker(st, phr, cfg)
|
||||
feedWkr = newFeedWorker(ipc.NewStoreAPI(st), embedderOf(voiceW), cfg)
|
||||
crawlWkr = newCrawlWorker(newCrawler(cfg), ipc.NewStoreAPI(st), embedderOf(voiceW), cfg)
|
||||
|
||||
// Swap the CoreAPI from the locked placeholder to the real store adapter.
|
||||
newAPI := &daemonAPI{
|
||||
@@ -457,6 +481,10 @@ func run(args []string) error {
|
||||
}
|
||||
srv.SetAPI(newAPI)
|
||||
srv.Check = (&auth.Gate{Enrollment: auth.NewFloorEnrollment(), Session: passkeySess}).Check
|
||||
wireMailIntake(srv, st, phr, cfg)
|
||||
wireModelSwap(srv, phr, cfg)
|
||||
keeper := wireVision(ctx, srv, st, embedderOf(voiceW), cfg)
|
||||
wireCapture(srv, keeper, st, voiceW, phr, cfg)
|
||||
|
||||
// Start voice server.
|
||||
if voiceW != nil {
|
||||
@@ -488,6 +516,25 @@ func run(args []string) error {
|
||||
}()
|
||||
}
|
||||
|
||||
// Start feed reading (nil unless configured).
|
||||
if feedWkr != nil {
|
||||
go func() {
|
||||
feedWkr.run(ctx)
|
||||
}()
|
||||
}
|
||||
|
||||
// Start the watched-page crawls (nil unless configured).
|
||||
if crawlWkr != nil {
|
||||
go func() {
|
||||
crawlWkr.run(ctx)
|
||||
}()
|
||||
}
|
||||
|
||||
// Keep MCP connections alive (nil unless configured).
|
||||
if voiceW != nil && voiceW.mcp != nil {
|
||||
go voiceW.mcp.run(ctx)
|
||||
}
|
||||
|
||||
dl.unlock()
|
||||
log.Printf("mavend: unlocked via passkey assertion")
|
||||
return nil
|
||||
@@ -533,6 +580,27 @@ func run(args []string) error {
|
||||
evalWorker.run(ctx)
|
||||
}()
|
||||
}
|
||||
if feedWkr != nil {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
feedWkr.run(ctx)
|
||||
}()
|
||||
}
|
||||
if crawlWkr != nil {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
crawlWkr.run(ctx)
|
||||
}()
|
||||
}
|
||||
if voiceW != nil && voiceW.mcp != nil {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
voiceW.mcp.run(ctx)
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
<-ctx.Done()
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/config"
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/mcp"
|
||||
"github.com/kami/maven/internal/store"
|
||||
"github.com/kami/maven/internal/webfetch"
|
||||
)
|
||||
|
||||
// mcpRefreshInterval — how often the manager re-dials a server that is down.
|
||||
// The manager applies its own backoff on top, so this being short is cheap.
|
||||
const mcpRefreshInterval = time.Minute
|
||||
|
||||
// mcpWiring — the MCP client, when the `mcp` block configures at least one
|
||||
// enabled server. nil ⇒ nothing was configured, nothing is connected, and an
|
||||
// allowlist row that happens to look like an MCP row refuses to run.
|
||||
//
|
||||
// It lives on the voice wiring because MCP tools ARE acts: they run through
|
||||
// tool.Executor, the enabled allowlist and the confirm turn, which only exist
|
||||
// on the voice/chat path. No voice surface ⇒ nothing that could call a tool.
|
||||
type mcpWiring struct {
|
||||
mgr *mcp.Manager
|
||||
st *store.Store
|
||||
}
|
||||
|
||||
// wireMCP builds the manager, connects, and proposes what it found. It never
|
||||
// fails the daemon: a server that is unreachable at boot is logged and retried,
|
||||
// because Maven starting is not contingent on someone else's process.
|
||||
func wireMCP(cfg *config.Config, st *store.Store) *mcpWiring {
|
||||
servers := cfg.MCPServers()
|
||||
if len(servers) == 0 {
|
||||
return nil
|
||||
}
|
||||
limits := webfetch.Config{}
|
||||
if cfg.MCP != nil {
|
||||
limits.AllowHosts = cfg.MCP.AllowHosts
|
||||
limits.DenyHosts = cfg.MCP.DenyHosts
|
||||
limits.MaxBytes = cfg.MCP.MaxBytes
|
||||
limits.Timeout = time.Duration(cfg.MCP.Timeout)
|
||||
}
|
||||
mgr, err := mcp.NewManager(mcp.WebfetchDoor(limits), servers)
|
||||
if err != nil {
|
||||
// Validation already ran in config.validate, so this is a programming
|
||||
// error rather than a config one. Still not fatal: MCP off is a working
|
||||
// Maven.
|
||||
log.Printf("mcp: not wired: %v", err)
|
||||
return nil
|
||||
}
|
||||
w := &mcpWiring{mgr: mgr, st: st}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
mgr.Connect(ctx)
|
||||
w.propose(ctx)
|
||||
return w
|
||||
}
|
||||
|
||||
// propose writes a 'proposed' allowlist row for every discovered tool. It does
|
||||
// NOT enable anything: a configured server is a place Maven may look, not a
|
||||
// capability she has. Kami enables what he wants on /tools, behind step-up,
|
||||
// which is the same gate a shell tool goes through.
|
||||
//
|
||||
// Re-running on every boot is idempotent — ProposeMCPTool never touches an
|
||||
// existing row, so a tool he disabled stays disabled and one he enabled keeps
|
||||
// the cmd he enabled it with.
|
||||
func (w *mcpWiring) propose(ctx context.Context) {
|
||||
if w == nil {
|
||||
return
|
||||
}
|
||||
now := time.Now()
|
||||
fresh := 0
|
||||
for _, t := range w.mgr.Tools() {
|
||||
name := mcp.LocalName(t.Server, t.Name)
|
||||
// No readOnlyHint ⇒ assume it mutates ⇒ the confirm turn. Being wrong
|
||||
// in this direction only costs a question.
|
||||
destructive := !t.ReadOnly
|
||||
provenance := fmt.Sprintf("mcp %s/%s", t.Server, t.Name)
|
||||
if t.Description != "" {
|
||||
provenance += ": " + t.Description
|
||||
}
|
||||
ok, err := w.st.ProposeMCPTool(ctx, name, mcp.Scope(t.Server),
|
||||
mcp.Cmd(t.Server, t.Name), destructive, provenance, now)
|
||||
if err != nil {
|
||||
log.Printf("mcp: propose %s: %v", name, err)
|
||||
continue
|
||||
}
|
||||
if ok {
|
||||
fresh++
|
||||
}
|
||||
}
|
||||
if fresh > 0 {
|
||||
log.Printf("mcp: %d new tool proposal(s) waiting on /tools", fresh)
|
||||
}
|
||||
}
|
||||
|
||||
// run re-dials downed servers and picks up tools that appeared, until ctx is
|
||||
// canceled.
|
||||
func (w *mcpWiring) run(ctx context.Context) {
|
||||
if w == nil {
|
||||
return
|
||||
}
|
||||
t := time.NewTicker(mcpRefreshInterval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
w.mgr.Refresh(ctx)
|
||||
w.propose(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// status maps the manager's view onto the wire type the web surface reads.
|
||||
func (w *mcpWiring) status() []ipc.MCPServerStatus {
|
||||
if w == nil {
|
||||
return nil
|
||||
}
|
||||
in := w.mgr.Status()
|
||||
out := make([]ipc.MCPServerStatus, 0, len(in))
|
||||
for _, s := range in {
|
||||
out = append(out, ipc.MCPServerStatus{
|
||||
Name: s.Name,
|
||||
Transport: s.Transport,
|
||||
Target: s.Target,
|
||||
Connected: s.Connected,
|
||||
Server: s.Server,
|
||||
Tools: s.Tools,
|
||||
Err: s.Err,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (w *mcpWiring) close() {
|
||||
if w == nil {
|
||||
return
|
||||
}
|
||||
_ = w.mgr.Close()
|
||||
}
|
||||
|
||||
// caller is the tool.MCPCaller the executor gets, or nil when MCP is off.
|
||||
func (w *mcpWiring) caller() *mcp.Manager {
|
||||
if w == nil {
|
||||
return nil
|
||||
}
|
||||
return w.mgr
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/kami/maven/internal/config"
|
||||
)
|
||||
|
||||
func TestWireMCPOffWhenUnconfigured(t *testing.T) {
|
||||
st := newTestStore(t)
|
||||
for name, cfg := range map[string]*config.Config{
|
||||
"no block": {},
|
||||
"nothing enabled": {MCP: &config.MCPConfig{Servers: []config.MCPServerConfig{
|
||||
{Name: "vikunja", URL: "http://192.168.1.104:9100/mcp"},
|
||||
}}},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if w := wireMCP(cfg, st); w != nil {
|
||||
t.Fatal("MCP must be off unless a server is configured AND enabled")
|
||||
}
|
||||
})
|
||||
}
|
||||
// nil wiring must be safe to use everywhere it is reachable.
|
||||
var w *mcpWiring
|
||||
w.close()
|
||||
w.propose(context.Background())
|
||||
if w.status() != nil || w.caller() != nil {
|
||||
t.Fatal("a nil wiring must report nothing")
|
||||
}
|
||||
}
|
||||
|
||||
// An unreachable server must not stop the daemon, must be reported as down, and
|
||||
// must propose nothing.
|
||||
func TestWireMCPUnreachableServerIsNotFatal(t *testing.T) {
|
||||
st := newTestStore(t)
|
||||
w := wireMCP(&config.Config{MCP: &config.MCPConfig{Servers: []config.MCPServerConfig{{
|
||||
Name: "dead", Command: "/nonexistent/mcp-server", Enabled: true,
|
||||
}}}}, st)
|
||||
if w == nil {
|
||||
t.Fatal("a configured server should still wire")
|
||||
}
|
||||
defer w.close()
|
||||
st2 := w.status()
|
||||
if len(st2) != 1 || st2[0].Connected || st2[0].Err == "" {
|
||||
t.Fatalf("status = %+v", st2)
|
||||
}
|
||||
tools, err := st.ListTools(context.Background(), "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(tools) != 0 {
|
||||
t.Fatalf("a server that never answered must propose nothing, got %+v", tools)
|
||||
}
|
||||
}
|
||||
|
||||
// A url server whose address is private is refused by webfetch unless that
|
||||
// server sets allow_private. This is the guard the whole MCP path rides on, so
|
||||
// it is asserted here too, at the wiring level.
|
||||
func TestWireMCPPrivateURLRefusedWithoutAllowPrivate(t *testing.T) {
|
||||
st := newTestStore(t)
|
||||
w := wireMCP(&config.Config{MCP: &config.MCPConfig{Servers: []config.MCPServerConfig{{
|
||||
Name: "lan", URL: "http://127.0.0.1:9100/mcp", Enabled: true,
|
||||
}}}}, st)
|
||||
if w == nil {
|
||||
t.Fatal("should wire")
|
||||
}
|
||||
defer w.close()
|
||||
s := w.status()[0]
|
||||
if s.Connected {
|
||||
t.Fatal("a loopback server must not connect without allow_private")
|
||||
}
|
||||
if !strings.Contains(s.Err, "private address") {
|
||||
t.Fatalf("err = %q, want the private-address refusal", s.Err)
|
||||
}
|
||||
}
|
||||
@@ -16,7 +16,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/config"
|
||||
"github.com/kami/maven/internal/llm"
|
||||
"github.com/kami/maven/internal/memeval"
|
||||
"github.com/kami/maven/internal/phraser"
|
||||
"github.com/kami/maven/internal/store"
|
||||
@@ -50,7 +49,7 @@ func newMemoryEvalWorker(st *store.Store, phr phraser.Phraser, cfg *config.Confi
|
||||
}
|
||||
// A generous per-request timeout: this is a long prompt to a Thinking model
|
||||
// and nobody is waiting on the answer.
|
||||
client := llm.New(lp.BaseURL(), 5*time.Minute)
|
||||
client := llmClientFor(lp, 5*time.Minute)
|
||||
ev := memeval.NewEvaluator(st, st, client, memeval.Config{
|
||||
MaxItems: cfg.MemoryEval.MaxItems,
|
||||
MinConfidence: cfg.MemoryEval.MinConfidence,
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/config"
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/llm"
|
||||
"github.com/kami/maven/internal/phraser"
|
||||
)
|
||||
|
||||
// Swapping the resident model while the daemon runs (Vikunja #250).
|
||||
//
|
||||
// Off unless configured: with no phraser.swap_models allowlist the two IPC
|
||||
// methods are never wired, so they answer ErrUnknownMethod. When it is wired the
|
||||
// swap method is AuthStepUp (internal/auth), which means an authed human surface
|
||||
// only — there is no act, no intent and no timer that reaches it. The daemon
|
||||
// never decides to change its own brain.
|
||||
//
|
||||
// The allowlist is exact-match against paths a human wrote in mavend.json. The
|
||||
// request carries a path and llama-server is started with it as `-m`, so
|
||||
// anything looser would turn "swap the model" into "load any file on my disk".
|
||||
func wireModelSwap(srv *ipc.Server, phr phraser.Phraser, cfg *config.Config) {
|
||||
if cfg.Phraser == nil || len(cfg.Phraser.SwapModels) == 0 {
|
||||
return
|
||||
}
|
||||
lp, ok := phr.(*phraser.LLMPhraser)
|
||||
if !ok {
|
||||
log.Printf("model swap: phraser.swap_models is set but there is no llama-server phraser — swap disabled")
|
||||
return
|
||||
}
|
||||
allowed := map[string]bool{}
|
||||
for _, m := range cfg.Phraser.SwapModels {
|
||||
allowed[filepath.Clean(m)] = true
|
||||
}
|
||||
// The configured model is always swappable back to, listed or not: the way
|
||||
// out of a bad swap must not depend on remembering to allowlist the model
|
||||
// you are already running.
|
||||
allowed[filepath.Clean(cfg.Phraser.ModelPath)] = true
|
||||
|
||||
srv.SwapModelFn = func(ctx context.Context, req ipc.SwapModelReq) (ipc.SwapModelResp, error) {
|
||||
path := filepath.Clean(req.ModelPath)
|
||||
if !allowed[path] {
|
||||
log.Printf("model swap: REFUSED %q — not in phraser.swap_models", req.ModelPath)
|
||||
return ipc.SwapModelResp{}, fmt.Errorf("%w: %q is not in phraser.swap_models", ipc.ErrForbidden, req.ModelPath)
|
||||
}
|
||||
res, err := lp.Swap(ctx, phraser.SwapSpec{
|
||||
ModelPath: path,
|
||||
NGpuLayers: req.NGpuLayers,
|
||||
NCtx: req.NCtx,
|
||||
})
|
||||
resp := ipc.SwapModelResp{
|
||||
Model: res.Model,
|
||||
ModelPath: res.ModelPath,
|
||||
BaseURL: res.BaseURL,
|
||||
RolledBack: res.RolledBack,
|
||||
TookMs: res.Took.Milliseconds(),
|
||||
}
|
||||
if err != nil {
|
||||
// A rolled-back swap is a failure that left a working daemon behind.
|
||||
// Both halves matter to the caller, so the response is filled in even
|
||||
// though the error is returned.
|
||||
log.Printf("model swap: %v", err)
|
||||
return resp, err
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
srv.ModelStatusFn = func(ctx context.Context) (ipc.ModelStatusResp, error) {
|
||||
path, ngl, nctx := lp.LiveModel()
|
||||
base := lp.BaseURL()
|
||||
resp := ipc.ModelStatusResp{
|
||||
ModelPath: path,
|
||||
BaseURL: base,
|
||||
NGpuLayers: ngl,
|
||||
NCtx: nctx,
|
||||
Swappable: cfg.Phraser.SwapModels,
|
||||
}
|
||||
if base == "" {
|
||||
resp.Model = llm.UnknownModel
|
||||
return resp, nil
|
||||
}
|
||||
id, err := llm.ModelID(ctx, base)
|
||||
if err != nil {
|
||||
// Report the honest "I could not confirm it" rather than echoing the
|
||||
// configured filename as if the server had said it.
|
||||
resp.Model = llm.UnknownModel
|
||||
return resp, nil
|
||||
}
|
||||
resp.Model = id
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
log.Printf("model swap: enabled, %d allowlisted model(s) — step-up required", len(cfg.Phraser.SwapModels))
|
||||
}
|
||||
|
||||
// llmClientFor builds a completion client on the phraser's llama-server and
|
||||
// keeps it pointed at the right one across a model swap.
|
||||
//
|
||||
// Without the OnSwap registration every holder of a base URL — the LLM router,
|
||||
// the replier, the mail extractor, the memory evaluator — would keep talking to
|
||||
// the port of a server that no longer exists, and the daemon would degrade to
|
||||
// the classifier permanently after the first swap. The client is re-pointed, not
|
||||
// rebuilt, so nothing that holds it has to know a swap happened.
|
||||
func llmClientFor(lp *phraser.LLMPhraser, timeout time.Duration) *llm.Client {
|
||||
c := llm.New(lp.BaseURL(), timeout)
|
||||
lp.OnSwap(func(base string) { c.SetBaseURL(base) })
|
||||
return c
|
||||
}
|
||||
@@ -941,6 +941,7 @@ type daemonAPI struct {
|
||||
getMorningStatus func(ctx context.Context) []ipc.MorningRoutineStatus
|
||||
getDayPlan func(ctx context.Context) ipc.DayPlan
|
||||
chatFn func(ctx context.Context, text string) string
|
||||
getMCPServers func() []ipc.MCPServerStatus
|
||||
}
|
||||
|
||||
func (d *daemonAPI) Chat(ctx context.Context, text string) (string, error) {
|
||||
@@ -950,6 +951,16 @@ func (d *daemonAPI) Chat(ctx context.Context, text string) (string, error) {
|
||||
return d.chatFn(ctx, text), nil
|
||||
}
|
||||
|
||||
// MCPServers — the configured MCP servers and their health (Vikunja #251).
|
||||
// Empty, not an error, when the mcp block is absent: "not configured" is the
|
||||
// default state and the web surface renders it as such.
|
||||
func (d *daemonAPI) MCPServers(ctx context.Context) ([]ipc.MCPServerStatus, error) {
|
||||
if d.getMCPServers == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return d.getMCPServers(), nil
|
||||
}
|
||||
|
||||
func (d *daemonAPI) TickTrace(ctx context.Context) (ipc.TickTrace, error) {
|
||||
trace := d.getTrace()
|
||||
if trace == nil {
|
||||
|
||||
@@ -0,0 +1,257 @@
|
||||
// mavend/vision.go — core's half of image understanding (Vikunja #252,
|
||||
// docs/plans/07-vision.md).
|
||||
//
|
||||
// The split: any surface that can receive a picture (mavweb upload, a Telegram
|
||||
// photo through mavpoll, a path he names) hands the bytes to core over
|
||||
// ipc.MethodDescribeImage. Core stores them content-addressed under
|
||||
// media.dir, prepares a downscaled JPEG, and asks a local vision server what it
|
||||
// is. The description comes back as words; nothing about the image is echoed.
|
||||
//
|
||||
// Off unless configured twice over: no `media` block ⇒ nowhere to keep the
|
||||
// bytes, so the method does not exist; no `vision` block with enabled + a local
|
||||
// endpoint ⇒ the store is wired but the describing half refuses, and the method
|
||||
// still does not exist. A surface cannot make Maven look at pictures by merely
|
||||
// sending one.
|
||||
//
|
||||
// Two things this file deliberately does not do:
|
||||
//
|
||||
// - No cloud vision call, ever. internal/vision refuses a non-private
|
||||
// endpoint at construction; there is no config shape here that could reach
|
||||
// an upstream API even if someone wanted one.
|
||||
// - No automatic memory. SaveNote is opt-in per call. Glancing at a screenshot
|
||||
// is not the same act as remembering it, and a 1.7B-class VLM's guess about
|
||||
// a photo is not a fact worth carrying around.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/config"
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/media"
|
||||
"github.com/kami/maven/internal/router"
|
||||
"github.com/kami/maven/internal/store"
|
||||
"github.com/kami/maven/internal/vision"
|
||||
)
|
||||
|
||||
// prunePeriod — how often stored blobs are checked against media.retention.
|
||||
// Hourly is far more often than needed for a 7-day retention and costs a
|
||||
// directory walk over a handful of sidecars; the point is that the promise is
|
||||
// kept by a loop that runs, not by an operator remembering a cron.
|
||||
const prunePeriod = time.Hour
|
||||
|
||||
// mediaKeeper — the blob store plus the loop that enforces its retention. The
|
||||
// two are one object because a store without the loop is a directory that grows
|
||||
// forever, and shipping that would break the only interesting promise this
|
||||
// capability makes.
|
||||
type mediaKeeper struct {
|
||||
store *media.Store
|
||||
}
|
||||
|
||||
// openMediaStore builds the blob store from config, or returns nil when media is
|
||||
// not configured. A relative dir resolves against StateDir, the same rule the db
|
||||
// and socket paths follow.
|
||||
func openMediaStore(cfg *config.Config) *mediaKeeper {
|
||||
dir := cfg.Media.StoreDir()
|
||||
if dir == "" {
|
||||
return nil
|
||||
}
|
||||
if !filepath.IsAbs(dir) && cfg.StateDir != "" {
|
||||
dir = filepath.Join(cfg.StateDir, dir)
|
||||
}
|
||||
st, err := media.Open(dir, cfg.Media.MaxBytes, time.Duration(cfg.Media.Retention))
|
||||
if err != nil {
|
||||
log.Printf("media: %v — image and audio intake disabled", err)
|
||||
return nil
|
||||
}
|
||||
log.Printf("media: blob store at %s, retention %s", st.Dir(), st.Retention())
|
||||
return &mediaKeeper{store: st}
|
||||
}
|
||||
|
||||
// runPrune deletes over-retention blobs on a loop until ctx ends. It prunes once
|
||||
// immediately, so a daemon restarted after a long downtime does not sit on a
|
||||
// month of stale recordings until the first tick.
|
||||
func (k *mediaKeeper) runPrune(ctx context.Context) {
|
||||
prune := func() {
|
||||
n, err := k.store.Prune()
|
||||
if err != nil {
|
||||
log.Printf("media: prune: %v", err)
|
||||
return
|
||||
}
|
||||
if n > 0 {
|
||||
log.Printf("media: pruned %d blob(s) older than %s", n, k.store.Retention())
|
||||
}
|
||||
}
|
||||
prune()
|
||||
t := time.NewTicker(prunePeriod)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
prune()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// visionIntake — one image at a time: store, prepare, describe, optionally note.
|
||||
type visionIntake struct {
|
||||
in *vision.Intake
|
||||
st *store.Store
|
||||
emb router.Embedder
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// newVisionIntake returns nil when there is nothing to wire. keeper == nil means
|
||||
// no media block, which disables the method outright; a missing or disabled
|
||||
// vision block still wires the method, because storing an image and answering
|
||||
// "I can't look at it yet" is more useful than pretending the surface does not
|
||||
// exist — and it is exactly the state this box is in until a vision model is on
|
||||
// disk.
|
||||
func newVisionIntake(keeper *mediaKeeper, st *store.Store, emb router.Embedder, cfg *config.Config) *visionIntake {
|
||||
if keeper == nil {
|
||||
return nil
|
||||
}
|
||||
vc := cfg.Vision
|
||||
maxDim := 0
|
||||
var provider vision.Provider = vision.Disabled{}
|
||||
if vc.LooksAtImages() {
|
||||
p, err := vision.NewLocal(vision.Config{
|
||||
Endpoint: vc.Endpoint,
|
||||
Model: vc.Model,
|
||||
Timeout: time.Duration(vc.Timeout),
|
||||
MaxTokens: vc.MaxTokens,
|
||||
Prompt: vc.Prompt,
|
||||
})
|
||||
if err != nil {
|
||||
// A public endpoint, a hostname, a bad URL. Logged once here rather
|
||||
// than failing every turn, and the store still works.
|
||||
log.Printf("vision: %v — she can store images but not describe them", err)
|
||||
} else {
|
||||
provider = p
|
||||
maxDim = vc.MaxDim
|
||||
log.Printf("vision: enabled against %s", p.Endpoint())
|
||||
}
|
||||
} else {
|
||||
log.Printf("vision: not configured — images are stored, not described")
|
||||
}
|
||||
return &visionIntake{
|
||||
in: vision.NewIntake(keeper.store, provider, maxDim),
|
||||
st: st,
|
||||
emb: emb,
|
||||
now: time.Now,
|
||||
}
|
||||
}
|
||||
|
||||
// describe handles one ipc.MethodDescribeImage call.
|
||||
//
|
||||
// A description failure is NOT an error out of this method when the bytes were
|
||||
// stored: the caller gets the id and an empty description, which is honest ("it
|
||||
// is kept, I cannot read it yet") and re-runnable. A failure to store, or bytes
|
||||
// that are not an image at all, is an error — there is nothing to come back to.
|
||||
func (v *visionIntake) describe(ctx context.Context, req ipc.DescribeImageReq) (ipc.DescribeImageResp, error) {
|
||||
if len(req.Data) == 0 && req.ID == "" {
|
||||
return ipc.DescribeImageResp{}, fmt.Errorf("describe image: neither data nor id")
|
||||
}
|
||||
|
||||
var (
|
||||
res vision.Result
|
||||
err error
|
||||
)
|
||||
if req.ID != "" {
|
||||
res, err = v.in.Rerun(ctx, req.ID, req.Question)
|
||||
} else {
|
||||
res, err = v.in.Accept(ctx, req.Data, sourceOrDefault(req.Source), req.Question)
|
||||
}
|
||||
if res.Blob.ID == "" {
|
||||
// Nothing was stored: bad format, over the size cap, unwritable dir.
|
||||
return ipc.DescribeImageResp{}, fmt.Errorf("describe image: %w", err)
|
||||
}
|
||||
|
||||
resp := ipc.DescribeImageResp{
|
||||
ID: res.Blob.ID,
|
||||
Description: res.Description,
|
||||
Width: res.Image.Width,
|
||||
Height: res.Image.Height,
|
||||
}
|
||||
if err != nil {
|
||||
// Bytes are safe, words are not available. The log names the blob and the
|
||||
// reason; it never names what was in the picture.
|
||||
if errors.Is(err, vision.ErrDisabled) {
|
||||
log.Printf("vision: stored %s, no vision model configured", res.Blob)
|
||||
} else {
|
||||
log.Printf("vision: stored %s, describe failed: %v", res.Blob, err)
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
if req.SaveNote {
|
||||
id, werr := v.writeNote(ctx, res)
|
||||
if werr != nil {
|
||||
// The description is still returned: losing the note is worse as a
|
||||
// silent failure than as a log line next to a successful answer.
|
||||
log.Printf("vision: note write for %s failed: %v", res.Blob, werr)
|
||||
} else {
|
||||
resp.NoteID = id
|
||||
}
|
||||
}
|
||||
log.Printf("vision: described %s (%dx%d)", res.Blob, res.Image.Width, res.Image.Height)
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// writeNote stores the description as an ordinary note so it is recallable. The
|
||||
// note carries the blob id in its source, which is the only link back to the
|
||||
// bytes — the note text is words about the picture, never the picture.
|
||||
func (v *visionIntake) writeNote(ctx context.Context, res vision.Result) (int64, error) {
|
||||
var vec []float32
|
||||
if v.emb != nil {
|
||||
// EmbedPassage, not Embed: a description is text being searched FOR, and
|
||||
// the e5 embedder is asymmetric. Backwards here makes it unfindable by
|
||||
// the question that should have matched it.
|
||||
var err error
|
||||
vec, err = router.EmbedPassage(ctx, v.emb, res.Description)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("embed: %w", err)
|
||||
}
|
||||
}
|
||||
source := "media:image:" + res.Blob.ID[:12]
|
||||
return v.st.WriteNote(ctx, v.now(), res.Description, vec, source)
|
||||
}
|
||||
|
||||
// sourceOrDefault labels a blob whose sender did not say where it came from.
|
||||
func sourceOrDefault(s string) string {
|
||||
if s == "" {
|
||||
return "unknown"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// wireVision installs the IPC hook and starts the retention loop, or leaves the
|
||||
// hook nil so ipc.MethodDescribeImage reports ErrUnknownMethod. Called on both
|
||||
// startup paths (unlocked boot and passkey unlock) so vision behaves the same
|
||||
// either way.
|
||||
//
|
||||
// Returns the media keeper so the meeting recorder can share it: one blob store
|
||||
// with one retention loop holds both the images and the audio, which is the
|
||||
// whole point of internal/media being a shared package. nil ⇒ no media block,
|
||||
// and neither capability exists.
|
||||
func wireVision(ctx context.Context, srv *ipc.Server, st *store.Store, emb router.Embedder, cfg *config.Config) *mediaKeeper {
|
||||
keeper := openMediaStore(cfg)
|
||||
if keeper == nil {
|
||||
return nil
|
||||
}
|
||||
go keeper.runPrune(ctx)
|
||||
|
||||
vi := newVisionIntake(keeper, st, emb, cfg)
|
||||
if vi == nil {
|
||||
return keeper
|
||||
}
|
||||
srv.DescribeImageFn = vi.describe
|
||||
return keeper
|
||||
}
|
||||
@@ -52,6 +52,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/audio"
|
||||
"github.com/kami/maven/internal/crawl"
|
||||
"github.com/kami/maven/internal/dialogue"
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/memory"
|
||||
@@ -82,6 +83,15 @@ type reactiveHandler struct {
|
||||
replier voice.Replier
|
||||
now func() time.Time
|
||||
|
||||
// crawler reads a web page he names out loud (queryWeb). nil ⇒ on-demand
|
||||
// page reading is off, which is the default: no `crawl` block, no fetch.
|
||||
crawler *crawl.Crawler
|
||||
|
||||
// feedsOn — whether any RSS feed is configured (config.Feeds). It changes
|
||||
// only what she SAYS when asked and nothing is there: "ленты не настроены"
|
||||
// instead of "ничего нового", which are different truths.
|
||||
feedsOn bool
|
||||
|
||||
weatherProvider weather.Provider
|
||||
weatherLocation string // default location for weather queries
|
||||
|
||||
|
||||
+36
-11
@@ -40,6 +40,15 @@ type voiceWiring struct {
|
||||
// mavsttd / mavttsd don't keep a stale conn into a restarting daemon.
|
||||
sttClient *worker.Client
|
||||
ttsClient *worker.Client
|
||||
// transcriber — the STT in use, exposed so the meeting recorder
|
||||
// (cmd/mavend/capture.go) can reuse it. Maven has exactly one STT and does
|
||||
// not grow a second one for capture: this is the same whisper.cpp worker the
|
||||
// voice path talks to.
|
||||
transcriber stt.Transcriber
|
||||
// mcp — the MCP client, nil unless the `mcp` block configures an enabled
|
||||
// server (Vikunja #251). Its tools land in the same allowlist as every
|
||||
// other act, so nothing else here has to know about it.
|
||||
mcp *mcpWiring
|
||||
}
|
||||
|
||||
// close releases the listener + worker conns. Safe to call on nil (when
|
||||
@@ -60,6 +69,7 @@ func (w *voiceWiring) close() {
|
||||
if w.ttsClient != nil {
|
||||
_ = w.ttsClient.Close()
|
||||
}
|
||||
w.mcp.close()
|
||||
}
|
||||
|
||||
// wireVoice builds the audio path from cfg + a CoreAPI + a router. Returns
|
||||
@@ -87,6 +97,7 @@ func wireVoice(cfg *config.Config, coreAPI ipc.CoreAPI, phr phraser.Phraser, mem
|
||||
} else {
|
||||
transcriber = stt.NewStub()
|
||||
}
|
||||
w.transcriber = transcriber
|
||||
|
||||
// ----- tts (Stub in-process OR Remote) -----
|
||||
var synthesizer tts.Synthesizer
|
||||
@@ -131,6 +142,14 @@ func wireVoice(cfg *config.Config, coreAPI ipc.CoreAPI, phr phraser.Phraser, mem
|
||||
// daemon restart.
|
||||
seedTools(coreAPI, cfg.Voice.Tools)
|
||||
exec := tool.NewExecutor(coreAPI, time.Duration(cfg.Voice.ToolTimeout))
|
||||
// MCP servers (Vikunja #251): discovery PROPOSES tools into the same
|
||||
// allowlist, so an MCP tool is enabled by hand on /tools like any other and
|
||||
// runs through the same confirm turn. Off unless the `mcp` block configures
|
||||
// an enabled server.
|
||||
w.mcp = wireMCP(cfg, dataStore)
|
||||
if w.mcp != nil {
|
||||
exec = exec.WithMCP(w.mcp.caller())
|
||||
}
|
||||
matcher := tool.NewMatcher(coreAPI)
|
||||
|
||||
// ----- weather provider (Open-Meteo when configured, Stub otherwise) -----
|
||||
@@ -148,7 +167,9 @@ func wireVoice(cfg *config.Config, coreAPI ipc.CoreAPI, phr phraser.Phraser, mem
|
||||
// The replier uses the same llama-server as the phraser.
|
||||
var llmClient *llm.Client
|
||||
if lp, ok := phr.(*phraser.LLMPhraser); ok {
|
||||
llmClient = llm.New(lp.BaseURL(), 60*time.Second)
|
||||
// llmClientFor, not llm.New: this client must follow the phraser onto
|
||||
// the new llama-server when the resident model is swapped (Vikunja #250).
|
||||
llmClient = llmClientFor(lp, 60*time.Second)
|
||||
}
|
||||
// ----- router (the cascade; floor examples seed the classifier) -----
|
||||
// The act matcher's allowlist is exactly the enabled tool names — the
|
||||
@@ -201,16 +222,20 @@ func wireVoice(cfg *config.Config, coreAPI ipc.CoreAPI, phr phraser.Phraser, mem
|
||||
|
||||
// ----- the handler (the reactive path; closes over stt / tts / router / coreAPI / memory) -----
|
||||
h := &reactiveHandler{
|
||||
stt: transcriber,
|
||||
tts: synthesizer,
|
||||
router: rtr,
|
||||
embedder: emb,
|
||||
api: coreAPI,
|
||||
tools: exec,
|
||||
matcher: matcher,
|
||||
replier: replier,
|
||||
phraser: phr,
|
||||
now: time.Now,
|
||||
stt: transcriber,
|
||||
tts: synthesizer,
|
||||
router: rtr,
|
||||
embedder: emb,
|
||||
api: coreAPI,
|
||||
tools: exec,
|
||||
matcher: matcher,
|
||||
replier: replier,
|
||||
phraser: phr,
|
||||
now: time.Now,
|
||||
feedsOn: cfg.Feeds != nil,
|
||||
// nil unless `crawl.on_demand` is on: reading a page he names is a
|
||||
// capability, and capabilities are off unless configured.
|
||||
crawler: onDemandCrawler(cfg),
|
||||
weatherProvider: weatherProvider,
|
||||
weatherLocation: weatherLocation,
|
||||
memStore: memStore,
|
||||
|
||||
@@ -0,0 +1,332 @@
|
||||
// mavmaild — the mail reader module (Vikunja #246,
|
||||
// docs/plans/01-email-reader.md).
|
||||
//
|
||||
// Every so often it opens one IMAP mailbox read-only, fetches the messages it
|
||||
// has not read yet, and hands each one to core over ipc.MethodIngestMail. Core
|
||||
// runs the extraction on the resident model and writes what comes back as task
|
||||
// CANDIDATES he reviews on /tasks. Nothing here writes to the store, nothing
|
||||
// here can create a reminder, and nothing here speaks.
|
||||
//
|
||||
// Why a separate daemon rather than a loop inside mavend, when extraction has
|
||||
// to happen in mavend anyway: the credential. mavpoll set the precedent with the
|
||||
// zenmoney token (#125) — the module that talks to a third party holds the
|
||||
// secret, reads it from a FILE so it never appears in `ps`, in
|
||||
// docker-compose.yml or in shell history, and core never sees it. Core learns
|
||||
// that mail exists only as message text on one IPC method; it cannot connect to
|
||||
// the mailbox even if it wanted to, and a compromised core yields no mail
|
||||
// password.
|
||||
//
|
||||
// Off unless configured: without -password-file there is nothing to run, and
|
||||
// the daemon says so and exits. If core has no `email` block the very first
|
||||
// ingest comes back ErrUnknownMethod and this daemon stops polling instead of
|
||||
// hammering a socket that will keep refusing.
|
||||
//
|
||||
// Mail is personal, so the log is counts and UIDs: how many messages were
|
||||
// fetched, how many were bulk, how many candidates came back. No subject, no
|
||||
// sender, no body, ever — reviewing a candidate is what /tasks is for.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/email"
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := run(os.Args[1:]); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "mavmaild:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(args []string) error {
|
||||
fs := flag.NewFlagSet("mavmaild", flag.ContinueOnError)
|
||||
socket := fs.String("socket", "", "core IPC socket path (required)")
|
||||
server := fs.String("imap", "", "IMAP server, host or host:993 (required)")
|
||||
user := fs.String("user", "", "IMAP username (required)")
|
||||
passFile := fs.String("password-file", "", "file holding the IMAP password (required — never passed as a flag value)")
|
||||
mailbox := fs.String("mailbox", "INBOX", "mailbox to read, read-only")
|
||||
interval := fs.Duration("interval", 15*time.Minute, "how often to read the mailbox")
|
||||
lookback := fs.Duration("lookback", 72*time.Hour, "how far back to search on each poll")
|
||||
max := fs.Int("max", 25, "most messages to fetch in one poll")
|
||||
timeout := fs.Duration("timeout", 30*time.Second, "IMAP network timeout")
|
||||
statePath := fs.String("state", "", "file remembering which UIDs were read (default: none — every poll re-reads the window)")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
if *socket == "" {
|
||||
return fmt.Errorf("-socket is required")
|
||||
}
|
||||
if *server == "" || *user == "" || *passFile == "" {
|
||||
return fmt.Errorf("mail reading is off unless configured: set -imap, -user and -password-file")
|
||||
}
|
||||
|
||||
// The password is read from a file, never taken as a flag value: an argv
|
||||
// secret is visible in `ps` to every user on the box and lands in the compose
|
||||
// file and the shell history. Read once at start — a rotated password means a
|
||||
// restart, which is cheaper than re-reading his credential every quarter hour.
|
||||
raw, err := os.ReadFile(*passFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read password file: %w", err)
|
||||
}
|
||||
password := strings.TrimSpace(string(raw))
|
||||
if password == "" {
|
||||
return fmt.Errorf("password file %s is empty", *passFile)
|
||||
}
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
core, err := ipc.DialWait(*socket, 60*time.Second)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer core.Close()
|
||||
|
||||
r := &reader{
|
||||
core: core,
|
||||
addr: *server,
|
||||
user: *user,
|
||||
mailbox: *mailbox,
|
||||
lookback: *lookback,
|
||||
max: *max,
|
||||
timeout: *timeout,
|
||||
state: newSeenState(*statePath),
|
||||
}
|
||||
if err := r.state.load(); err != nil {
|
||||
// A missing or corrupt state file must not stop mail from being read: the
|
||||
// worst case is re-reading the window, and capture dedupes on text.
|
||||
log.Printf("mavmaild: state: %v (starting from an empty seen-set)", err)
|
||||
}
|
||||
|
||||
// The password is never logged, not even its length.
|
||||
log.Printf("mavmaild: reading %s on %s every %s (lookback %s, max %d/poll)",
|
||||
*mailbox, *server, *interval, *lookback, *max)
|
||||
|
||||
r.pollOnce(ctx, password) // don't idle a full interval on start
|
||||
t := time.NewTicker(*interval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Printf("mavmaild: bye")
|
||||
return nil
|
||||
case <-t.C:
|
||||
if r.disabled {
|
||||
// Core told us mail ingestion is not configured. Nothing will change
|
||||
// without a core restart, and a restart restarts us too.
|
||||
log.Printf("mavmaild: core does not accept mail — idling")
|
||||
return nil
|
||||
}
|
||||
r.pollOnce(ctx, password)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// mailIngester — the slice of core this daemon uses. One method: hand over a
|
||||
// message. It cannot write a fact, create a reminder or read the store, and the
|
||||
// interface says so.
|
||||
type mailIngester interface {
|
||||
IngestMail(ctx context.Context, req ipc.IngestMailReq) (ipc.IngestMailResp, error)
|
||||
}
|
||||
|
||||
type reader struct {
|
||||
core mailIngester
|
||||
addr string
|
||||
user string
|
||||
mailbox string
|
||||
lookback time.Duration
|
||||
max int
|
||||
timeout time.Duration
|
||||
state *seenState
|
||||
|
||||
// dial — connection seam for the tests; nil ⇒ implicit TLS.
|
||||
dial func(addr string, timeout time.Duration) (*email.Conn, error)
|
||||
|
||||
// disabled — core answered ErrUnknownMethod, i.e. it has no email block.
|
||||
disabled bool
|
||||
}
|
||||
|
||||
// pollOnce — one read of the mailbox, then one ingest per message.
|
||||
//
|
||||
// A fetch error aborts this poll and nothing else; the next tick tries again.
|
||||
// An ingest error for one message does not skip the rest — one mail the model
|
||||
// choked on should not hide the four behind it.
|
||||
func (r *reader) pollOnce(ctx context.Context, password string) {
|
||||
msgs, err := r.fetch(password)
|
||||
if err != nil {
|
||||
// The error may name a UID; it never names a subject or a sender.
|
||||
log.Printf("mavmaild: fetch: %v", err)
|
||||
if len(msgs) == 0 {
|
||||
return
|
||||
}
|
||||
}
|
||||
var junk, candidates, created int
|
||||
for _, m := range msgs {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if m.Junk {
|
||||
junk++
|
||||
// Marked seen without a model call: the header filter already decided,
|
||||
// and re-classifying it every quarter hour would be pure waste.
|
||||
r.state.mark(m.UID)
|
||||
continue
|
||||
}
|
||||
resp, err := r.core.IngestMail(ctx, ipc.IngestMailReq{
|
||||
Mailbox: r.mailbox,
|
||||
UID: m.UID,
|
||||
From: m.From,
|
||||
Subject: m.Subject,
|
||||
Date: m.Date,
|
||||
Body: m.Body,
|
||||
})
|
||||
if errors.Is(err, ipc.ErrUnknownMethod) {
|
||||
log.Printf("mavmaild: core has no email block configured — mail ingestion is off; stopping")
|
||||
r.disabled = true
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
// Not marked seen: an ingest that failed should be retried next poll.
|
||||
log.Printf("mavmaild: ingest uid %d: %v", m.UID, err)
|
||||
continue
|
||||
}
|
||||
r.state.mark(m.UID)
|
||||
candidates += len(resp.TaskIDs)
|
||||
created += resp.Created
|
||||
}
|
||||
if err := r.state.save(); err != nil {
|
||||
log.Printf("mavmaild: state: %v", err)
|
||||
}
|
||||
log.Printf("mavmaild: %s: %d read, %d bulk, %d candidate(s), %d new", r.mailbox, len(msgs), junk, candidates, created)
|
||||
}
|
||||
|
||||
// fetch reads the mailbox. Messages already in the seen-set are not fetched at
|
||||
// all, so a steady mailbox costs one SEARCH per poll and nothing else.
|
||||
func (r *reader) fetch(password string) ([]email.Message, error) {
|
||||
f := email.FetchSince{
|
||||
Addr: r.addr,
|
||||
User: r.user,
|
||||
Mailbox: r.mailbox,
|
||||
Timeout: r.timeout,
|
||||
Since: time.Now().Add(-r.lookback),
|
||||
Max: r.max,
|
||||
Skip: r.state.seen,
|
||||
}
|
||||
return f.RunWith(password, r.dial)
|
||||
}
|
||||
|
||||
// ---- seen state ------------------------------------------------------------
|
||||
|
||||
// seenState — the UIDs already handed to core, persisted so a restart does not
|
||||
// re-read (and re-extract, at multi-second LLM cost) the whole lookback window.
|
||||
//
|
||||
// Correctness does not depend on it: ipc.CaptureTask dedupes on normalised text
|
||||
// among live tasks, so a re-read produces no duplicate rows. This exists to save
|
||||
// the model's time, which is why a broken state file is a log line rather than a
|
||||
// failure.
|
||||
//
|
||||
// UIDs are per-mailbox and monotonic, so the set is kept as a high-water mark
|
||||
// plus the stragglers above it. If the server ever changes UIDVALIDITY, UIDs
|
||||
// reset and the window is simply re-read once — dedupe absorbs it.
|
||||
type seenState struct {
|
||||
path string
|
||||
high uint32
|
||||
set map[uint32]bool
|
||||
dirty bool
|
||||
}
|
||||
|
||||
func newSeenState(path string) *seenState {
|
||||
return &seenState{path: path, set: map[uint32]bool{}}
|
||||
}
|
||||
|
||||
type seenFile struct {
|
||||
High uint32 `json:"high"`
|
||||
UIDs []uint32 `json:"uids,omitempty"`
|
||||
}
|
||||
|
||||
func (s *seenState) seen(uid uint32) bool {
|
||||
return uid <= s.high || s.set[uid]
|
||||
}
|
||||
|
||||
func (s *seenState) mark(uid uint32) {
|
||||
if s.seen(uid) {
|
||||
return
|
||||
}
|
||||
s.set[uid] = true
|
||||
s.dirty = true
|
||||
// Advance the high-water mark through any contiguous run, so the explicit set
|
||||
// stays small on a mailbox read in order.
|
||||
for {
|
||||
next := s.high + 1
|
||||
if !s.set[next] {
|
||||
break
|
||||
}
|
||||
delete(s.set, next)
|
||||
s.high = next
|
||||
}
|
||||
}
|
||||
|
||||
func (s *seenState) load() error {
|
||||
if s.path == "" {
|
||||
return nil
|
||||
}
|
||||
b, err := os.ReadFile(s.path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil // first run
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var f seenFile
|
||||
if err := json.Unmarshal(b, &f); err != nil {
|
||||
return fmt.Errorf("parse %s: %w", s.path, err)
|
||||
}
|
||||
s.high = f.High
|
||||
for _, u := range f.UIDs {
|
||||
s.set[u] = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// save writes the state atomically (temp file + rename), 0600: it is a list of
|
||||
// message ids from his mailbox, which is metadata about his mail.
|
||||
func (s *seenState) save() error {
|
||||
if s.path == "" || !s.dirty {
|
||||
return nil
|
||||
}
|
||||
uids := make([]uint32, 0, len(s.set))
|
||||
for u := range s.set {
|
||||
uids = append(uids, u)
|
||||
}
|
||||
sort.Slice(uids, func(i, j int) bool { return uids[i] < uids[j] })
|
||||
b, err := json.Marshal(seenFile{High: s.high, UIDs: uids})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := s.path + ".tmp"
|
||||
if err := os.MkdirAll(filepath.Dir(s.path), 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(tmp, b, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(tmp, s.path); err != nil {
|
||||
return err
|
||||
}
|
||||
s.dirty = false
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/email"
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
)
|
||||
|
||||
// ---- a scripted IMAP server, same shape internal/email's tests use ---------
|
||||
|
||||
type fakeIMAP struct {
|
||||
msgs map[uint32]string
|
||||
uids []uint32
|
||||
cmds []string
|
||||
}
|
||||
|
||||
func (f *fakeIMAP) serve(c net.Conn) {
|
||||
defer c.Close()
|
||||
fmt.Fprint(c, "* OK fake ready\r\n")
|
||||
r := bufio.NewReader(c)
|
||||
for {
|
||||
line, err := r.ReadString('\n')
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
parts := strings.SplitN(strings.TrimRight(line, "\r\n"), " ", 2)
|
||||
if len(parts) != 2 {
|
||||
return
|
||||
}
|
||||
tag, cmd := parts[0], parts[1]
|
||||
f.cmds = append(f.cmds, cmd)
|
||||
upper := strings.ToUpper(cmd)
|
||||
switch {
|
||||
case strings.HasPrefix(upper, "LOGIN"), strings.HasPrefix(upper, "EXAMINE"):
|
||||
fmt.Fprintf(c, "%s OK\r\n", tag)
|
||||
case strings.HasPrefix(upper, "UID SEARCH"):
|
||||
var ids []string
|
||||
for _, u := range f.uids {
|
||||
ids = append(ids, strconv.FormatUint(uint64(u), 10))
|
||||
}
|
||||
fmt.Fprintf(c, "* SEARCH %s\r\n%s OK\r\n", strings.Join(ids, " "), tag)
|
||||
case strings.HasPrefix(upper, "UID FETCH"):
|
||||
uid, _ := strconv.ParseUint(strings.Fields(cmd)[2], 10, 32)
|
||||
if raw, ok := f.msgs[uint32(uid)]; ok {
|
||||
fmt.Fprintf(c, "* 1 FETCH (UID %d BODY[] {%d}\r\n%s)\r\n", uid, len(raw), raw)
|
||||
}
|
||||
fmt.Fprintf(c, "%s OK\r\n", tag)
|
||||
case strings.HasPrefix(upper, "LOGOUT"):
|
||||
fmt.Fprintf(c, "* BYE\r\n%s OK\r\n", tag)
|
||||
return
|
||||
default:
|
||||
fmt.Fprintf(c, "%s BAD\r\n", tag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeIMAP) dial(_ string, timeout time.Duration) (*email.Conn, error) {
|
||||
cli, srv := net.Pipe()
|
||||
go f.serve(srv)
|
||||
return email.NewConn(cli, timeout)
|
||||
}
|
||||
|
||||
// ---- a fake core -----------------------------------------------------------
|
||||
|
||||
type fakeCore struct {
|
||||
got []ipc.IngestMailReq
|
||||
resp ipc.IngestMailResp
|
||||
err error
|
||||
}
|
||||
|
||||
func (c *fakeCore) IngestMail(_ context.Context, req ipc.IngestMailReq) (ipc.IngestMailResp, error) {
|
||||
c.got = append(c.got, req)
|
||||
if c.err != nil {
|
||||
return ipc.IngestMailResp{}, c.err
|
||||
}
|
||||
return c.resp, nil
|
||||
}
|
||||
|
||||
func mail(subject, body string, extraHeaders ...string) string {
|
||||
h := "Subject: " + subject + "\r\nFrom: a@b.c\r\nContent-Type: text/plain; charset=utf-8\r\n"
|
||||
for _, e := range extraHeaders {
|
||||
h += e + "\r\n"
|
||||
}
|
||||
return h + "\r\n" + body + "\r\n"
|
||||
}
|
||||
|
||||
func newTestReader(t *testing.T, f *fakeIMAP, core *fakeCore, statePath string) *reader {
|
||||
t.Helper()
|
||||
return &reader{
|
||||
core: core, addr: "mail.example:993", user: "kami", mailbox: "INBOX",
|
||||
lookback: 72 * time.Hour, max: 25, timeout: 5 * time.Second,
|
||||
state: newSeenState(statePath),
|
||||
dial: f.dial,
|
||||
}
|
||||
}
|
||||
|
||||
func TestPollHandsMessagesToCore(t *testing.T) {
|
||||
f := &fakeIMAP{
|
||||
uids: []uint32{1, 2},
|
||||
msgs: map[uint32]string{
|
||||
1: mail("Счёт", "Оплатить до 5 августа."),
|
||||
2: mail("Скидки", "Sale!", "List-Unsubscribe: <mailto:u@x>"),
|
||||
},
|
||||
}
|
||||
core := &fakeCore{resp: ipc.IngestMailResp{TaskIDs: []int64{1}, Created: 1}}
|
||||
r := newTestReader(t, f, core, "")
|
||||
r.pollOnce(context.Background(), "secret")
|
||||
|
||||
// The newsletter is filtered before core is asked: only the real mail crosses.
|
||||
if len(core.got) != 1 {
|
||||
t.Fatalf("core saw %d messages, want 1 (the bulk one must not cross): %+v", len(core.got), core.got)
|
||||
}
|
||||
got := core.got[0]
|
||||
if got.UID != 1 || got.Mailbox != "INBOX" || got.Subject != "Счёт" {
|
||||
t.Errorf("ingest req = %+v", got)
|
||||
}
|
||||
if !strings.Contains(got.Body, "Оплатить") {
|
||||
t.Errorf("body = %q", got.Body)
|
||||
}
|
||||
}
|
||||
|
||||
// A second poll must not re-send what core already saw — extraction is a
|
||||
// multi-second LLM call per message.
|
||||
func TestPollSkipsSeenUIDs(t *testing.T) {
|
||||
f := &fakeIMAP{uids: []uint32{5}, msgs: map[uint32]string{5: mail("Счёт", "текст")}}
|
||||
core := &fakeCore{}
|
||||
r := newTestReader(t, f, core, "")
|
||||
r.pollOnce(context.Background(), "secret")
|
||||
r.pollOnce(context.Background(), "secret")
|
||||
if len(core.got) != 1 {
|
||||
t.Errorf("core saw %d messages over two polls, want 1", len(core.got))
|
||||
}
|
||||
}
|
||||
|
||||
// An ingest that failed is NOT marked seen: the next poll retries it.
|
||||
func TestPollRetriesFailedIngest(t *testing.T) {
|
||||
f := &fakeIMAP{uids: []uint32{5}, msgs: map[uint32]string{5: mail("Счёт", "текст")}}
|
||||
core := &fakeCore{err: fmt.Errorf("llama-server is warming up")}
|
||||
r := newTestReader(t, f, core, "")
|
||||
r.pollOnce(context.Background(), "secret")
|
||||
core.err = nil
|
||||
r.pollOnce(context.Background(), "secret")
|
||||
if len(core.got) != 2 {
|
||||
t.Errorf("core saw %d attempts, want 2 (a failed ingest is retried)", len(core.got))
|
||||
}
|
||||
}
|
||||
|
||||
// Core without an email block ⇒ stop, don't hammer the socket.
|
||||
func TestPollStopsWhenCoreRefusesMail(t *testing.T) {
|
||||
f := &fakeIMAP{uids: []uint32{1, 2}, msgs: map[uint32]string{1: mail("a", "b"), 2: mail("c", "d")}}
|
||||
core := &fakeCore{err: fmt.Errorf("call: %w", ipc.ErrUnknownMethod)}
|
||||
r := newTestReader(t, f, core, "")
|
||||
r.pollOnce(context.Background(), "secret")
|
||||
if !r.disabled {
|
||||
t.Error("ErrUnknownMethod must disable the reader")
|
||||
}
|
||||
if len(core.got) != 1 {
|
||||
t.Errorf("core saw %d messages, want 1 — stop at the first refusal", len(core.got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeenStatePersists(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "state", "seen.json")
|
||||
f := &fakeIMAP{uids: []uint32{9}, msgs: map[uint32]string{9: mail("Счёт", "текст")}}
|
||||
core := &fakeCore{}
|
||||
r := newTestReader(t, f, core, path)
|
||||
r.pollOnce(context.Background(), "secret")
|
||||
|
||||
fi, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatalf("state file: %v", err)
|
||||
}
|
||||
// A list of message ids from his mailbox is metadata about his mail.
|
||||
if perm := fi.Mode().Perm(); perm != 0o600 {
|
||||
t.Errorf("state file mode = %v, want 0600", perm)
|
||||
}
|
||||
|
||||
// A fresh reader with the same state file must not re-read the message.
|
||||
core2 := &fakeCore{}
|
||||
r2 := newTestReader(t, f, core2, path)
|
||||
if err := r2.state.load(); err != nil {
|
||||
t.Fatalf("load: %v", err)
|
||||
}
|
||||
r2.pollOnce(context.Background(), "secret")
|
||||
if len(core2.got) != 0 {
|
||||
t.Errorf("after a restart core saw %d messages, want 0", len(core2.got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeenStateHighWaterMark(t *testing.T) {
|
||||
s := newSeenState("")
|
||||
s.mark(1)
|
||||
s.mark(3)
|
||||
s.mark(2)
|
||||
if s.high != 3 {
|
||||
t.Errorf("high = %d, want 3 (contiguous run collapses)", s.high)
|
||||
}
|
||||
if len(s.set) != 0 {
|
||||
t.Errorf("explicit set = %v, want empty", s.set)
|
||||
}
|
||||
if !s.seen(2) || s.seen(4) {
|
||||
t.Errorf("seen(2)=%v seen(4)=%v", s.seen(2), s.seen(4))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeenStateCorruptFileIsNotFatal(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "seen.json")
|
||||
if err := os.WriteFile(path, []byte("{not json"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := newSeenState(path)
|
||||
if err := s.load(); err == nil {
|
||||
t.Error("a corrupt state file should report an error the caller logs")
|
||||
}
|
||||
if s.seen(1) {
|
||||
t.Error("a corrupt state file must leave an empty seen-set, not a poisoned one")
|
||||
}
|
||||
}
|
||||
|
||||
// Off unless configured, and the credential is never a flag value.
|
||||
func TestRunRequiresConfig(t *testing.T) {
|
||||
if err := run([]string{}); err == nil {
|
||||
t.Error("no -socket must be an error")
|
||||
}
|
||||
if err := run([]string{"-socket", "/tmp/nope.sock"}); err == nil {
|
||||
t.Error("no mailbox configuration must be an error, not a default mailbox")
|
||||
}
|
||||
// There is no -password flag at all: only -password-file.
|
||||
if err := run([]string{"-socket", "/x", "-imap", "h", "-user", "u", "-password", "p"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "flag provided but not defined") {
|
||||
t.Errorf("a -password flag must not exist; err = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRejectsEmptyPasswordFile(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "pass")
|
||||
if err := os.WriteFile(path, []byte(" \n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err := run([]string{"-socket", "/x/y.sock", "-imap", "h", "-user", "u", "-password-file", path})
|
||||
if err == nil || !strings.Contains(err.Error(), "empty") {
|
||||
t.Errorf("an empty password file must be refused before dialling; err = %v", err)
|
||||
}
|
||||
}
|
||||
+124
-3
@@ -9,6 +9,12 @@
|
||||
// Two sources, each its own provenance (the loop's rules trust source):
|
||||
// - netdata → poll:netdata resource alarms (disk/mem/cert/temp)
|
||||
// - kuma → poll:uptimekuma service up/down (the source of truth for it)
|
||||
// - zenmoney → poll:zenmoney spending/income totals (Vikunja #125)
|
||||
//
|
||||
// The zenmoney source is why the token lives HERE and not in core: the poller
|
||||
// already owns every other third-party credential, it holds no store key, and
|
||||
// core never needs to know an account exists to answer a question about a fact
|
||||
// the poller wrote. It is off unless -zenmoney-token-file is given.
|
||||
//
|
||||
// Netdata needs no auth over the wg-fronted net. Kuma's /metrics needs an API
|
||||
// key (basic-auth); without -kuma the whole kuma path is skipped (netdata-only
|
||||
@@ -37,6 +43,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/zenmoney"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@@ -52,6 +59,9 @@ func run(args []string) error {
|
||||
netdataURL := fs.String("netdata", "http://127.0.0.1:19999", "netdata base URL ('' to disable)")
|
||||
kumaURL := fs.String("kuma", "", "uptime-kuma metrics URL, e.g. http://127.0.0.1:3001/metrics ('' to disable)")
|
||||
kumaKey := fs.String("kuma-key", "", "uptime-kuma API key (basic-auth username)")
|
||||
zenTokenFile := fs.String("zenmoney-token-file", "", "file holding the zenmoney API token ('' disables money tracking)")
|
||||
zenURL := fs.String("zenmoney-url", zenmoney.DefaultBaseURL, "zenmoney API base URL (tests/self-hosted proxies)")
|
||||
zenInterval := fs.Duration("zenmoney-interval", time.Hour, "how often to read zenmoney (money does not move every minute)")
|
||||
wgIface := fs.String("wg", "", "wireguard interface for the presence signal, e.g. wg0 or 'all' ('' to disable)")
|
||||
wgCmd := fs.String("wg-cmd", "wg", "wg binary (use e.g. 'sudo wg' if the poller lacks CAP_NET_ADMIN)")
|
||||
interval := fs.Duration("interval", 60*time.Second, "poll cadence")
|
||||
@@ -62,8 +72,24 @@ func run(args []string) error {
|
||||
if *socket == "" {
|
||||
return fmt.Errorf("-socket is required")
|
||||
}
|
||||
if *netdataURL == "" && *kumaURL == "" && *wgIface == "" {
|
||||
return fmt.Errorf("nothing to poll: set -netdata, -kuma and/or -wg")
|
||||
if *netdataURL == "" && *kumaURL == "" && *wgIface == "" && *zenTokenFile == "" {
|
||||
return fmt.Errorf("nothing to poll: set -netdata, -kuma, -wg and/or -zenmoney-token-file")
|
||||
}
|
||||
|
||||
// The token is read from a file, never taken as a flag value: an argv token
|
||||
// is visible in `ps` to every user on the box and lands in the compose file
|
||||
// and the shell history. Read once at start — a rotated token means a
|
||||
// restart, which is cheaper than re-reading his credential every hour.
|
||||
var zen *zenmoney.Client
|
||||
if *zenTokenFile != "" {
|
||||
raw, err := os.ReadFile(*zenTokenFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read zenmoney token: %w", err)
|
||||
}
|
||||
zen, err = zenmoney.New(strings.TrimSpace(string(raw)), *zenURL, *timeout*3)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
@@ -83,9 +109,13 @@ func run(args []string) error {
|
||||
kumaKey: *kumaKey,
|
||||
wgIface: *wgIface,
|
||||
wgCmd: *wgCmd,
|
||||
zen: zen,
|
||||
zenEvery: *zenInterval,
|
||||
}
|
||||
|
||||
log.Printf("mavpoll: polling every %s (netdata=%q kuma=%q wg=%q)", *interval, *netdataURL, *kumaURL, *wgIface)
|
||||
// The token is never logged, not even its length.
|
||||
log.Printf("mavpoll: polling every %s (netdata=%q kuma=%q wg=%q zenmoney=%v every %s)",
|
||||
*interval, *netdataURL, *kumaURL, *wgIface, zen != nil, *zenInterval)
|
||||
p.pollOnce(ctx) // fire immediately; don't idle a full interval on start
|
||||
t := time.NewTicker(*interval)
|
||||
defer t.Stop()
|
||||
@@ -108,6 +138,12 @@ type poller struct {
|
||||
kumaKey string
|
||||
wgIface string
|
||||
wgCmd string
|
||||
|
||||
// zen is nil unless a token file was configured — money tracking is a
|
||||
// capability, off by default like weather and telegram.
|
||||
zen *zenmoney.Client
|
||||
zenEvery time.Duration
|
||||
zenLast time.Time
|
||||
}
|
||||
|
||||
// pollOnce — one sweep of both sources. A failure in one source logs and does
|
||||
@@ -129,6 +165,66 @@ func (p *poller) pollOnce(ctx context.Context) {
|
||||
log.Printf("mavpoll: wg: %v", err)
|
||||
}
|
||||
}
|
||||
// Money on its own, much slower cadence: a bank feed that updates hourly
|
||||
// polled every minute is 60 pointless reads of his financial history.
|
||||
if p.zen != nil && now.Sub(p.zenLast) >= p.zenEvery {
|
||||
p.zenLast = now
|
||||
if err := p.pollZenmoney(ctx, now); err != nil {
|
||||
log.Printf("mavpoll: zenmoney: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- zenmoney: spending/income totals → money facts ------------------------
|
||||
|
||||
// pollZenmoney reads today's and this month's totals and writes them as
|
||||
// facts(kind=env, source=poll:zenmoney) (Vikunja #125).
|
||||
//
|
||||
// Two properties this function exists to hold:
|
||||
//
|
||||
// - An empty or failed read writes NOTHING. zenmoney.Summary.Value() refuses
|
||||
// to encode a summary built from zero transactions, so a poller that cannot
|
||||
// reach the API leaves the last good fact in place rather than overwriting
|
||||
// it with a zero Maven would then recite as fact.
|
||||
// - Nothing about the money leaves the box except the diff request itself, to
|
||||
// the service that already holds his bank sessions. The totals are written
|
||||
// to the store and read back only when he asks; they are never search input
|
||||
// and no tick rule fires on them.
|
||||
//
|
||||
// Both windows are read from one diff call each. Two calls an hour against an
|
||||
// API whose whole job is this is not worth caching.
|
||||
// moneyWindow — one fact key and the period it covers.
|
||||
type moneyWindow struct {
|
||||
key string
|
||||
from, to time.Time
|
||||
}
|
||||
|
||||
func (p *poller) pollZenmoney(ctx context.Context, now time.Time) error {
|
||||
dFrom, dTo := zenmoney.DayWindow(now)
|
||||
mFrom, mTo := zenmoney.MonthWindow(now)
|
||||
windows := []moneyWindow{
|
||||
{zenmoney.KeySpentToday, dFrom, dTo},
|
||||
{zenmoney.KeySpentMonth, mFrom, mTo},
|
||||
}
|
||||
var firstErr error
|
||||
for _, w := range windows {
|
||||
sum, err := p.zen.Since(ctx, w.from, w.to)
|
||||
if err != nil {
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
continue
|
||||
}
|
||||
val, ok := sum.Value()
|
||||
if !ok {
|
||||
// Nothing read. Silence, not a zero.
|
||||
continue
|
||||
}
|
||||
if err := p.writeIfChangedRaw(ctx, w.key, zenmoney.Source, val, now); err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
}
|
||||
return firstErr
|
||||
}
|
||||
|
||||
// ---- wireguard: latest handshake → presence signal -------------------------
|
||||
@@ -301,6 +397,31 @@ func (p *poller) writeIfChanged(ctx context.Context, key, source, val string, no
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeIfChangedRaw is writeIfChanged for values that are already JSON (the
|
||||
// money facts store an object, not a string). Kept separate rather than
|
||||
// generalising writeIfChanged, because the string-valued env facts encoding
|
||||
// their own value is the convention the rules rely on.
|
||||
//
|
||||
// The log line names the key and the source, never the figures: mavpoll's log
|
||||
// is not the place his spending ends up.
|
||||
func (p *poller) writeIfChangedRaw(ctx context.Context, key, source, jsonVal string, now time.Time) error {
|
||||
prev, err := p.core.LatestFactBySource(ctx, key, source)
|
||||
switch {
|
||||
case err == nil && prev.Value == jsonVal:
|
||||
return nil
|
||||
case err != nil && err != ipc.ErrNoFact && !isNoFact(err):
|
||||
return fmt.Errorf("read %s: %w", key, err)
|
||||
}
|
||||
if _, err := p.core.WriteFact(ctx, ipc.WriteFactReq{
|
||||
Ts: now, Kind: "env", Key: key, Value: jsonVal,
|
||||
Source: source, Confidence: 1.0,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("write %s: %w", key, err)
|
||||
}
|
||||
log.Printf("mavpoll: %s updated (%s)", key, source)
|
||||
return nil
|
||||
}
|
||||
|
||||
// isNoFact — ErrNoFact rehydrated over the wire is wrapped (fmt.Errorf %w), so
|
||||
// errors.Is is the right check; keep a helper so the switch above reads clean.
|
||||
func isNoFact(err error) bool {
|
||||
|
||||
@@ -1,8 +1,17 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/zenmoney"
|
||||
)
|
||||
|
||||
func TestMaxSeverity(t *testing.T) {
|
||||
@@ -62,3 +71,120 @@ func TestParseMaxHandshake(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- zenmoney (Vikunja #125) ----------------------------------------------
|
||||
|
||||
// factCore records the facts the poller wrote and answers "no fact yet".
|
||||
type factCore struct {
|
||||
ipc.UnimplementedCoreAPI
|
||||
|
||||
written []ipc.WriteFactReq
|
||||
prev map[string]string
|
||||
}
|
||||
|
||||
func (c *factCore) LatestFactBySource(_ context.Context, key, source string) (ipc.Fact, error) {
|
||||
if v, ok := c.prev[key+"|"+source]; ok {
|
||||
return ipc.Fact{Key: key, Source: source, Value: v}, nil
|
||||
}
|
||||
return ipc.Fact{}, ipc.ErrNoFact
|
||||
}
|
||||
|
||||
func (c *factCore) WriteFact(_ context.Context, req ipc.WriteFactReq) (int64, error) {
|
||||
c.written = append(c.written, req)
|
||||
return int64(len(c.written)), nil
|
||||
}
|
||||
|
||||
func zenFixtureServer(t *testing.T, body []byte, status int) *httptest.Server {
|
||||
t.Helper()
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if status != http.StatusOK {
|
||||
w.WriteHeader(status)
|
||||
return
|
||||
}
|
||||
w.Write(body)
|
||||
}))
|
||||
}
|
||||
|
||||
func TestPollZenmoneyWritesMoneyFacts(t *testing.T) {
|
||||
body, err := os.ReadFile("../../internal/zenmoney/testdata/diff.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv := zenFixtureServer(t, body, http.StatusOK)
|
||||
defer srv.Close()
|
||||
zen, err := zenmoney.New("tok", srv.URL, time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
core := &factCore{}
|
||||
p := &poller{core: core, zen: zen}
|
||||
now := time.Date(2026, 8, 1, 21, 0, 0, 0, time.UTC)
|
||||
if err := p.pollZenmoney(context.Background(), now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(core.written) != 2 {
|
||||
t.Fatalf("wrote %d facts, want today + month", len(core.written))
|
||||
}
|
||||
for _, f := range core.written {
|
||||
if f.Kind != "env" || f.Source != zenmoney.Source {
|
||||
t.Errorf("fact = %+v, want kind=env source=%s", f, zenmoney.Source)
|
||||
}
|
||||
if _, err := zenmoney.ParseFactValue(f.Value); err != nil {
|
||||
t.Errorf("fact value %q does not decode: %v", f.Value, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A read that returns nothing for the window writes NOTHING. Silence, not a
|
||||
// zero: an invented 0 would be recited back to him as fact.
|
||||
func TestPollZenmoneyWritesNothingWhenEmpty(t *testing.T) {
|
||||
srv := zenFixtureServer(t, []byte(`{"serverTimestamp":1,"instrument":[],"transaction":[]}`), http.StatusOK)
|
||||
defer srv.Close()
|
||||
zen, _ := zenmoney.New("tok", srv.URL, time.Second)
|
||||
core := &factCore{}
|
||||
p := &poller{core: core, zen: zen}
|
||||
if err := p.pollZenmoney(context.Background(), time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(core.written) != 0 {
|
||||
t.Errorf("wrote %+v, want no fact at all", core.written)
|
||||
}
|
||||
}
|
||||
|
||||
// An API failure must not overwrite the last good total either.
|
||||
func TestPollZenmoneyFailureWritesNothing(t *testing.T) {
|
||||
srv := zenFixtureServer(t, nil, http.StatusUnauthorized)
|
||||
defer srv.Close()
|
||||
zen, _ := zenmoney.New("bad", srv.URL, time.Second)
|
||||
core := &factCore{}
|
||||
p := &poller{core: core, zen: zen}
|
||||
if err := p.pollZenmoney(context.Background(), time.Now()); err == nil {
|
||||
t.Error("want the 401 reported")
|
||||
}
|
||||
if len(core.written) != 0 {
|
||||
t.Errorf("wrote %+v on a failed read", core.written)
|
||||
}
|
||||
}
|
||||
|
||||
// Unchanged totals do not churn the facts table.
|
||||
func TestWriteIfChangedRawSkipsUnchanged(t *testing.T) {
|
||||
core := &factCore{prev: map[string]string{
|
||||
zenmoney.KeySpentToday + "|" + zenmoney.Source: `{"count":1}`,
|
||||
}}
|
||||
p := &poller{core: core}
|
||||
if err := p.writeIfChangedRaw(context.Background(), zenmoney.KeySpentToday, zenmoney.Source, `{"count":1}`, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(core.written) != 0 {
|
||||
t.Errorf("wrote %+v for an unchanged value", core.written)
|
||||
}
|
||||
}
|
||||
|
||||
// Money tracking is off unless configured: no token file, no zenmoney client,
|
||||
// and the poller still refuses to start with nothing at all to poll.
|
||||
func TestRunRequiresSomethingToPoll(t *testing.T) {
|
||||
err := run([]string{"-socket", "/tmp/nope.sock", "-netdata", "", "-kuma", "", "-wg", ""})
|
||||
if err == nil || !strings.Contains(err.Error(), "nothing to poll") {
|
||||
t.Errorf("err = %v, want a 'nothing to poll' refusal", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
// Command mavupdate deploys a new build of Maven to the box she runs on, with
|
||||
// an automatic rollback when the new build does not come up (Vikunja #249).
|
||||
//
|
||||
// It is a CLI on purpose, and it is the ONLY trigger for the update path.
|
||||
//
|
||||
// The obvious design — an IPC method plus a button on the web UI behind the
|
||||
// step-up passkey gate, the way /tools works — was considered and refused. A
|
||||
// step-up gate protects against the wrong person clicking; it does not change
|
||||
// the fact that anything reachable over the network becomes, in the event of a
|
||||
// mavweb bug, a remote arbitrary-code path with a build system attached. An
|
||||
// update needs shell access on the host, which is a strictly higher bar than
|
||||
// the gate that guards the tool allowlist. That is deliberate and it is the
|
||||
// reason there is no MethodApplyUpdate anywhere in internal/ipc.
|
||||
//
|
||||
// Consequently: mavend does not import internal/update, nothing runs on a timer,
|
||||
// nothing checks a release server, and no act, intent, tool or LLM output can
|
||||
// reach any of this. She cannot update herself. She can be updated, by him.
|
||||
//
|
||||
// mavupdate -config deploy/mavend.json list # snapshots available to roll back to
|
||||
// mavupdate -config deploy/mavend.json verify # make build + make test, deploys nothing
|
||||
// mavupdate -config deploy/mavend.json apply -yes # the whole thing
|
||||
// mavupdate -config deploy/mavend.json rollback [id] # restore + restart (default: newest)
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/config"
|
||||
"github.com/kami/maven/internal/update"
|
||||
)
|
||||
|
||||
func main() {
|
||||
cfgPath := flag.String("config", "deploy/mavend.json", "path to mavend.json (the update block is read from it)")
|
||||
yes := flag.Bool("yes", false, "required by `apply` and `rollback`: yes, restart the daemon")
|
||||
flag.Usage = usage
|
||||
flag.Parse()
|
||||
|
||||
// The stdlib flag package stops parsing at the first non-flag argument, so a
|
||||
// `-yes` written after the subcommand (which is how anyone would type it, and
|
||||
// how the usage text shows it) lands in Args instead of the flag. Pick it out
|
||||
// by hand rather than silently treating "apply -yes" as an unconfirmed apply.
|
||||
var args []string
|
||||
for _, a := range flag.Args() {
|
||||
if a == "-yes" || a == "--yes" {
|
||||
*yes = true
|
||||
continue
|
||||
}
|
||||
args = append(args, a)
|
||||
}
|
||||
if len(args) == 0 {
|
||||
usage()
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
cfg, err := config.Load(*cfgPath)
|
||||
if err != nil {
|
||||
die("config: %v", err)
|
||||
}
|
||||
if cfg.Update == nil {
|
||||
die("no `update` block in %s — the update capability is off unless configured.\nSee the package comment in internal/update for what it does and does not do.", *cfgPath)
|
||||
}
|
||||
|
||||
logf := func(format string, a ...any) {
|
||||
fmt.Fprintf(os.Stderr, "%s %s\n", time.Now().Format("15:04:05"), fmt.Sprintf(format, a...))
|
||||
}
|
||||
u, err := update.New(*cfg.Update, update.WithLogger(logf))
|
||||
if err != nil {
|
||||
die("%v", err)
|
||||
}
|
||||
|
||||
// Ctrl-C cancels the build or the health wait. It cannot cancel a rollback
|
||||
// midway into leaving the box in an unknown state, because the rollback runs
|
||||
// on its own context — see cmdApply.
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
switch args[0] {
|
||||
case "list":
|
||||
cmdList(u)
|
||||
case "verify":
|
||||
cmdVerify(ctx, u)
|
||||
case "apply":
|
||||
if !*yes {
|
||||
die("apply restarts mavend and can roll her back. Re-run with -yes if that is what you want.")
|
||||
}
|
||||
cmdApply(ctx, u)
|
||||
case "rollback":
|
||||
if !*yes {
|
||||
die("rollback restores the previous artifacts and restarts mavend. Re-run with -yes.")
|
||||
}
|
||||
id := ""
|
||||
if len(args) > 1 {
|
||||
id = args[1]
|
||||
}
|
||||
cmdRollback(ctx, u, id)
|
||||
default:
|
||||
usage()
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
|
||||
func cmdList(u *update.Updater) {
|
||||
snaps, err := u.Snapshots()
|
||||
if err != nil {
|
||||
die("snapshots: %v", err)
|
||||
}
|
||||
if len(snaps) == 0 {
|
||||
fmt.Println("no snapshots yet — the first `apply` takes one before it builds anything")
|
||||
return
|
||||
}
|
||||
fmt.Printf("%-18s %-12s %s\n", "SNAPSHOT", "COMMIT", "FILES")
|
||||
for _, s := range snaps {
|
||||
commit := s.Commit
|
||||
if len(commit) > 12 {
|
||||
commit = commit[:12]
|
||||
}
|
||||
if commit == "" {
|
||||
commit = "-"
|
||||
}
|
||||
fmt.Printf("%-18s %-12s %d\n", s.ID, commit, len(s.Files))
|
||||
}
|
||||
fmt.Printf("\nrollback to the newest with: mavupdate rollback -yes\n")
|
||||
}
|
||||
|
||||
func cmdVerify(ctx context.Context, u *update.Updater) {
|
||||
steps, err := u.Verify(ctx)
|
||||
report(steps)
|
||||
if err != nil {
|
||||
die("%v", err)
|
||||
}
|
||||
fmt.Println("verified: the tree builds and passes its own tests. Nothing was deployed — run `apply -yes` for that.")
|
||||
}
|
||||
|
||||
func cmdApply(ctx context.Context, u *update.Updater) {
|
||||
res, err := u.Apply(ctx)
|
||||
report(res.Steps)
|
||||
summarize(res)
|
||||
switch {
|
||||
case err == nil:
|
||||
fmt.Println("\nupdate committed: she answers on the new build.")
|
||||
case errors.Is(err, update.ErrRollbackFailed):
|
||||
die("\n%v\n\nSHE IS PROBABLY DOWN. The previous artifacts are in the snapshot dir; copy them\nover the install dir and restart by hand.", err)
|
||||
case errors.Is(err, update.ErrRolledBack):
|
||||
die("\n%v\n\nShe is answering again on the previous build. Nothing was lost; fix the change and retry.", err)
|
||||
default:
|
||||
die("\n%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func cmdRollback(ctx context.Context, u *update.Updater, id string) {
|
||||
res, err := u.Rollback(ctx, id)
|
||||
report(res.Steps)
|
||||
summarize(res)
|
||||
if err != nil && !errors.Is(err, update.ErrRolledBack) {
|
||||
die("\n%v", err)
|
||||
}
|
||||
fmt.Printf("\nrolled back to %s; she answers on it.\n", res.SnapshotID)
|
||||
}
|
||||
|
||||
func report(steps []update.Step) {
|
||||
for _, s := range steps {
|
||||
status := "ok"
|
||||
if s.Err != nil {
|
||||
status = "FAILED: " + s.Err.Error()
|
||||
}
|
||||
fmt.Printf(" %-8s %-8s %s\n", s.Name, s.Took.Round(time.Second), status)
|
||||
if s.Output != "" {
|
||||
fmt.Printf("---- %s output ----\n%s\n-------------------\n", s.Name, s.Output)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func summarize(res update.Result) {
|
||||
fmt.Printf("\nverified=%v snapshot=%s installed=%d restarted=%v healthy=%v rolled_back=%v rollback_healthy=%v took=%s\n",
|
||||
res.Verified, res.SnapshotID, len(res.Installed), res.Restarted, res.Healthy, res.RolledBack, res.RollbackHealthy, res.Took.Round(time.Second))
|
||||
}
|
||||
|
||||
func usage() {
|
||||
fmt.Fprint(os.Stderr, `mavupdate — deploy a new build of Maven, with rollback.
|
||||
|
||||
mavupdate [-config path] list
|
||||
mavupdate [-config path] verify
|
||||
mavupdate [-config path] apply -yes
|
||||
mavupdate [-config path] rollback [snapshot-id] -yes
|
||||
|
||||
apply is: health-check the running daemon, snapshot the deployed artifacts,
|
||||
make build, make test, install, restart, health-check — and restore the
|
||||
snapshot if any of that fails. It never fetches code and never runs by itself.
|
||||
|
||||
`)
|
||||
flag.PrintDefaults()
|
||||
}
|
||||
|
||||
func die(format string, a ...any) {
|
||||
fmt.Fprintf(os.Stderr, format+"\n", a...)
|
||||
os.Exit(1)
|
||||
}
|
||||
@@ -67,6 +67,14 @@ type fakeCore struct {
|
||||
// for handleChatAPI tests
|
||||
chatText string
|
||||
chatErr error
|
||||
|
||||
// for the MCP section of /tools
|
||||
mcpServers []ipc.MCPServerStatus
|
||||
mcpErr error
|
||||
}
|
||||
|
||||
func (f *fakeCore) MCPServers(context.Context) ([]ipc.MCPServerStatus, error) {
|
||||
return f.mcpServers, f.mcpErr
|
||||
}
|
||||
|
||||
func (f *fakeCore) Chat(_ context.Context, text string) (string, error) {
|
||||
@@ -1118,3 +1126,49 @@ func TestHandleChatAPI_FailOpenByDefault(t *testing.T) {
|
||||
t.Errorf("core.Chat text = %q, want %q", core.chatText, "привет")
|
||||
}
|
||||
}
|
||||
|
||||
// The MCP section renders the configured servers, and a proposal that already
|
||||
// knows its cmd prefills the enable form so the argv is not retyped by hand.
|
||||
func TestHandleTools_GET_MCPSection(t *testing.T) {
|
||||
core := &fakeCore{
|
||||
proposed: []ipc.Tool{{
|
||||
Name: "vikunja_list_tasks", Scope: "mcp:vikunja",
|
||||
Cmd: []string{"mcp", "vikunja", "list_tasks"}, Destructive: true,
|
||||
Utterance: "mcp vikunja/list_tasks: List tasks in a project.",
|
||||
}},
|
||||
mcpServers: []ipc.MCPServerStatus{
|
||||
{Name: "vikunja", Transport: "http", Target: "http://192.168.1.104:9100/mcp", Connected: true, Server: "vikunja 0.1.0", Tools: 4},
|
||||
{Name: "files", Transport: "stdio", Target: "mcp-server-fs /srv", Err: "start: no such file"},
|
||||
},
|
||||
}
|
||||
rr := httptest.NewRecorder()
|
||||
handleTools(rr, httptest.NewRequest(http.MethodGet, "/tools", nil), core, nil, false)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d", rr.Code)
|
||||
}
|
||||
body := rr.Body.String()
|
||||
for _, want := range []string{
|
||||
"MCP servers", "vikunja", "192.168.1.104:9100/mcp", "vikunja 0.1.0",
|
||||
"files", "no such file",
|
||||
`value="mcp vikunja list_tasks"`, // the enable form is prefilled
|
||||
"checked", // and pre-marked destructive (no readOnlyHint)
|
||||
} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("missing %q in /tools output", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MCP off (or an older core that does not know the method) renders the section
|
||||
// empty instead of breaking the page.
|
||||
func TestHandleTools_GET_MCPUnavailable(t *testing.T) {
|
||||
core := &fakeCore{mcpErr: ipc.ErrNotImplemented}
|
||||
rr := httptest.NewRecorder()
|
||||
handleTools(rr, httptest.NewRequest(http.MethodGet, "/tools", nil), core, nil, false)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rr.Code)
|
||||
}
|
||||
if !strings.Contains(rr.Body.String(), "no MCP servers configured") {
|
||||
t.Error("expected the empty-state copy")
|
||||
}
|
||||
}
|
||||
|
||||
+233
-4
@@ -25,6 +25,7 @@ import (
|
||||
"github.com/kami/maven/internal/audio"
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/pattern"
|
||||
"github.com/kami/maven/internal/tasks"
|
||||
"github.com/kami/maven/internal/voice"
|
||||
"github.com/kami/maven/internal/webauthn"
|
||||
)
|
||||
@@ -58,6 +59,9 @@ var notificationsHTML string
|
||||
//go:embed reminders.html
|
||||
var remindersHTML string
|
||||
|
||||
//go:embed tasks.html
|
||||
var tasksHTML string
|
||||
|
||||
//go:embed voice.html
|
||||
var voiceHTML string
|
||||
|
||||
@@ -97,6 +101,7 @@ var sidebarSections = []struct {
|
||||
Pages: []struct{ Label, URL, Key string }{
|
||||
{Label: "Rule Trace", URL: "/trace", Key: "trace"},
|
||||
{Label: "Notifications", URL: "/notifications", Key: "notifications"},
|
||||
{Label: "Tasks", URL: "/tasks", Key: "tasks"},
|
||||
{Label: "Reminders", URL: "/reminders", Key: "reminders"},
|
||||
{Label: "Routines", URL: "/routines", Key: "routines"},
|
||||
{Label: "Morning", URL: "/morning", Key: "morning"},
|
||||
@@ -119,6 +124,7 @@ var sidebarSections = []struct {
|
||||
Label: "Settings",
|
||||
Pages: []struct{ Label, URL, Key string }{
|
||||
{Label: "Tools", URL: "/tools", Key: "tools"},
|
||||
{Label: "Model", URL: "/models", Key: "models"},
|
||||
{Label: "Passkey", URL: "/auth/passkey", Key: "passkey"},
|
||||
},
|
||||
},
|
||||
@@ -170,6 +176,8 @@ func pageIcon(key string) string {
|
||||
return `<svg class=icon width="14" height="14"><use href="/ethos-icons.svg#i-wave"/></svg>`
|
||||
case "notifications":
|
||||
return `<svg class=icon width="14" height="14"><use href="/ethos-icons.svg#i-bell"/></svg>`
|
||||
case "tasks":
|
||||
return `<svg class=icon width="14" height="14"><use href="/ethos-icons.svg#i-grid"/></svg>`
|
||||
case "reminders":
|
||||
return `<svg class=icon width="14" height="14"><use href="/ethos-icons.svg#i-calendar"/></svg>`
|
||||
case "routines":
|
||||
@@ -184,6 +192,8 @@ func pageIcon(key string) string {
|
||||
return `<svg class=icon width="14" height="14"><use href="/ethos-icons.svg#i-grid"/></svg>`
|
||||
case "tools":
|
||||
return `<svg class=icon width="14" height="14"><use href="/ethos-icons.svg#i-settings"/></svg>`
|
||||
case "models":
|
||||
return `<svg class=icon width="14" height="14"><use href="/ethos-icons.svg#i-wave"/></svg>`
|
||||
case "passkey":
|
||||
return `<svg class=icon width="14" height="14"><use href="/ethos-icons.svg#i-lock"/></svg>`
|
||||
default:
|
||||
@@ -202,6 +212,8 @@ func pageTitle(key string) string {
|
||||
return "Rule Trace"
|
||||
case "notifications":
|
||||
return "Notifications"
|
||||
case "tasks":
|
||||
return "Tasks"
|
||||
case "reminders":
|
||||
return "Reminders"
|
||||
case "routines":
|
||||
@@ -216,6 +228,8 @@ func pageTitle(key string) string {
|
||||
return "Ecosystem"
|
||||
case "tools":
|
||||
return "Tools"
|
||||
case "models":
|
||||
return "Resident Model"
|
||||
case "passkey":
|
||||
return "Passkey"
|
||||
default:
|
||||
@@ -408,6 +422,11 @@ func main() {
|
||||
mux.HandleFunc("/reminders", func(w http.ResponseWriter, r *http.Request) {
|
||||
handleReminders(w, r, core)
|
||||
})
|
||||
// /tasks — capture + review. POST is not step-up gated; see handleTasks for
|
||||
// why a task write is not in the same class as /tools or /routines.
|
||||
mux.HandleFunc("/tasks", func(w http.ResponseWriter, r *http.Request) {
|
||||
handleTasks(w, r, core)
|
||||
})
|
||||
mux.HandleFunc("/morning", func(w http.ResponseWriter, r *http.Request) {
|
||||
handleMorning(w, r, core)
|
||||
})
|
||||
@@ -465,6 +484,12 @@ func main() {
|
||||
mux.HandleFunc("/routines", func(w http.ResponseWriter, r *http.Request) {
|
||||
handleRoutines(w, r, core, stepUpSession, *requireStepUp)
|
||||
})
|
||||
// /models — the resident-model surface (Vikunja #250). Same step-up gate as
|
||||
// /tools, and for a comparable reason: which model is loaded decides how every
|
||||
// utterance is routed and how every reply is worded. GET is read-only.
|
||||
mux.HandleFunc("/models", func(w http.ResponseWriter, r *http.Request) {
|
||||
handleModels(w, r, core, stepUpSession, *requireStepUp)
|
||||
})
|
||||
|
||||
// State-changing routes on this server, and their gate (Vikunja #317):
|
||||
//
|
||||
@@ -667,7 +692,7 @@ const toolsHTML = `{{template "shellTop" "tools"}}
|
||||
{{if .Msg}}<div class="msg msg-ok">{{.Msg}}</div>{{end}}
|
||||
<section class=card>
|
||||
<h2 class=card-title>proposed <span class=badge>{{len .Proposed}}</span></h2>
|
||||
{{if .Proposed}}<p class=hint>maven drafted these from acts she couldn't run. Fill the command (argv, space-separated) and enable.</p>
|
||||
{{if .Proposed}}<p class=hint>maven drafted these from acts she couldn't run. Fill the command (argv, space-separated) and enable. A row in an <code>mcp:</code> scope came from an MCP server and already knows what it calls — check the command, then enable.</p>
|
||||
<div class=scroll><table><tr><th>name</th><th>scope</th><th>from utterance</th><th>enable as</th></tr>
|
||||
{{range .Proposed}}<tr>
|
||||
<td><code>{{.Name}}</code></td><td><span class=badge>{{.Scope}}</span></td><td>{{.Utterance}}</td>
|
||||
@@ -675,8 +700,8 @@ const toolsHTML = `{{template "shellTop" "tools"}}
|
||||
<input type=hidden name=name value="{{.Name}}">
|
||||
<input type=hidden name=scope value="{{.Scope}}">
|
||||
<input type=hidden name=action value=enable>
|
||||
<input type=text name=cmd class=input-wide placeholder="systemctl restart" required>
|
||||
<label><input type=checkbox name=destructive> destructive</label>
|
||||
<input type=text name=cmd class=input-wide placeholder="systemctl restart" value="{{join .Cmd " "}}" required>
|
||||
<label><input type=checkbox name=destructive {{if .Destructive}}checked{{end}}> destructive</label>
|
||||
<button class=btn>enable</button></form>
|
||||
<form method=post action=/tools class=inline-form>
|
||||
<input type=hidden name=name value="{{.Name}}">
|
||||
@@ -705,6 +730,19 @@ const toolsHTML = `{{template "shellTop" "tools"}}
|
||||
<div class=hint>enable proposed tools above, or ask maven to configure one</div>
|
||||
</div>{{end}}
|
||||
</section>
|
||||
<section class=card>
|
||||
<h2 class=card-title>MCP servers <span class=badge>{{len .MCP}}</span></h2>
|
||||
{{if .MCP}}<p class=hint>servers she connects OUT to. Their tools appear above as proposals — a configured server is a place she may look, not a capability she has. A <code>stdio</code> target is a process on this box; an <code>http</code> one on a loopback or LAN address is inside the network, so treat its tools accordingly.</p>
|
||||
<div class=scroll><table><tr><th>name</th><th>transport</th><th>target</th><th>state</th><th>tools</th></tr>
|
||||
{{range .MCP}}<tr><td><code>{{.Name}}</code></td><td><span class=badge>{{.Transport}}</span></td><td><code>{{.Target}}</code></td>
|
||||
<td>{{if .Connected}}connected{{if .Server}} — {{.Server}}{{end}}{{else}}<span class=red>down</span>{{if .Err}} — {{.Err}}{{end}}{{end}}</td>
|
||||
<td>{{.Tools}}</td></tr>{{end}}</table></div>
|
||||
{{else}}<div class=empty>
|
||||
<svg class=icon width="20" height="20"><use href="/ethos-icons.svg#i-settings"/></svg>
|
||||
<div>no MCP servers configured</div>
|
||||
<div class=hint>add an <code>mcp.servers</code> block to mavend.json to let her use an external tool server</div>
|
||||
</div>{{end}}
|
||||
</section>
|
||||
{{template "shellBottom"}}`
|
||||
|
||||
// routinesHTML — proposed routine review surface. One row per thing maven
|
||||
@@ -744,6 +782,8 @@ var passkeyTmpl = template.Must(template.New("passkey").Funcs(shellFuncs()).Pars
|
||||
|
||||
var voiceTmpl = template.Must(template.New("voice").Funcs(shellFuncs()).Parse(shellTopHTML + voiceHTML + shellBottomHTML))
|
||||
|
||||
var tasksTmpl = template.Must(template.New("tasks").Funcs(shellFuncs()).Parse(shellTopHTML + tasksHTML + shellBottomHTML))
|
||||
|
||||
var routinesTmpl = template.Must(template.New("routines").Funcs(shellFuncs()).Parse(shellTopHTML + routinesHTML + shellBottomHTML))
|
||||
|
||||
var traceTmpl = template.Must(template.New("trace").Funcs(func() template.FuncMap {
|
||||
@@ -814,6 +854,187 @@ func handleReminders(w http.ResponseWriter, r *http.Request, core ipc.CoreAPI) {
|
||||
}
|
||||
}
|
||||
|
||||
// taskRow is one line on /tasks, with every timestamp already formatted so the
|
||||
// template holds no date logic.
|
||||
type taskRow struct {
|
||||
ID int64
|
||||
Text string
|
||||
Source string
|
||||
Evidence string
|
||||
Status string
|
||||
Due string
|
||||
Created string
|
||||
Resolved string
|
||||
// Why — the ranker's reason for this row's position (Vikunja #129), in
|
||||
// Russian, empty when nothing distinguished the task. Blank is the honest
|
||||
// rendering: he never said this one mattered more.
|
||||
Why string
|
||||
}
|
||||
|
||||
// handleTasks serves the task review surface (GET) and the four writes it
|
||||
// offers (POST): add, confirm, done, drop.
|
||||
//
|
||||
// Not step-up gated, unlike /tools and /routines, and the difference is the
|
||||
// point: enabling a tool defines argv Maven will execute, and accepting a
|
||||
// routine hands the tick loop a new standing reason to interrupt him. A task is
|
||||
// neither — nothing in the tick loop reads the tasks table, so the worst a
|
||||
// weaker caller can do here is write a line onto a list he reads himself. It
|
||||
// still sits behind whatever transport auth fronts mavweb, like every other
|
||||
// page.
|
||||
//
|
||||
// "confirm" is the only interesting move: it promotes a candidate Maven derived
|
||||
// from something she read into work he owns. That review step is why derived
|
||||
// tasks are captured as candidates in the first place.
|
||||
func handleTasks(w http.ResponseWriter, r *http.Request, core ipc.CoreAPI) {
|
||||
if core == nil {
|
||||
http.Error(w, "tasks disabled (no -core)", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
ctx := r.Context()
|
||||
var msg, errMsg string
|
||||
if r.Method == http.MethodPost {
|
||||
var err error
|
||||
msg, err = applyTaskPost(ctx, core, r)
|
||||
if err != nil {
|
||||
log.Printf("tasks: %v", err)
|
||||
errMsg = err.Error()
|
||||
}
|
||||
}
|
||||
|
||||
all, err := core.ListTasks(ctx, "")
|
||||
if err != nil {
|
||||
log.Printf("tasks: list: %v", err)
|
||||
http.Error(w, "tasks error: "+err.Error(), http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
// Live rows are ordered by the same ranker the spoken list uses, so the page
|
||||
// and the voice reply can never disagree about what comes first. Resolved
|
||||
// rows keep store order (newest first) — ranking finished work is pointless.
|
||||
var live []tasks.Item
|
||||
var resolved []taskRow
|
||||
for _, t := range all {
|
||||
switch t.Status {
|
||||
case "candidate", "open":
|
||||
live = append(live, tasks.Item{
|
||||
ID: t.ID, Text: t.Text, Status: t.Status,
|
||||
Created: t.CreatedTs, Due: t.Due, Weight: t.Weight,
|
||||
})
|
||||
default:
|
||||
resolved = append(resolved, taskRow{
|
||||
ID: t.ID, Text: t.Text, Source: t.Source, Evidence: t.Evidence,
|
||||
Status: t.Status, Created: fmtTaskTime(&t.CreatedTs),
|
||||
Due: fmtTaskDate(t.Due), Resolved: fmtTaskTime(t.Resolved),
|
||||
})
|
||||
}
|
||||
}
|
||||
byID := make(map[int64]ipc.Task, len(all))
|
||||
for _, t := range all {
|
||||
byID[t.ID] = t
|
||||
}
|
||||
var cands, open []taskRow
|
||||
for _, r := range tasks.Rank(live, time.Now()) {
|
||||
t := byID[r.ID]
|
||||
row := taskRow{
|
||||
ID: t.ID, Text: t.Text, Source: t.Source, Evidence: t.Evidence,
|
||||
Status: t.Status, Created: fmtTaskTime(&t.CreatedTs),
|
||||
Due: fmtTaskDate(t.Due), Resolved: fmtTaskTime(t.Resolved),
|
||||
Why: r.Reason,
|
||||
}
|
||||
if t.Status == "candidate" {
|
||||
// A candidate's due date is Maven's reading of a mail, so its
|
||||
// ranking reason is not shown as if he had set a priority.
|
||||
row.Why = ""
|
||||
cands = append(cands, row)
|
||||
} else {
|
||||
open = append(open, row)
|
||||
}
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
if err := tasksTmpl.Execute(w, struct {
|
||||
Msg, Err string
|
||||
Candidates []taskRow
|
||||
Open []taskRow
|
||||
Resolved []taskRow
|
||||
}{msg, errMsg, cands, open, resolved}); err != nil {
|
||||
log.Printf("tasks render: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// applyTaskPost performs one write and returns the message to show. A bad
|
||||
// request returns an error, which the page renders inline rather than as a
|
||||
// bare 400 — this is a form surface, not an API.
|
||||
func applyTaskPost(ctx context.Context, core ipc.CoreAPI, r *http.Request) (string, error) {
|
||||
action := r.FormValue("action")
|
||||
if action == "add" {
|
||||
text := strings.TrimSpace(r.FormValue("text"))
|
||||
if text == "" {
|
||||
return "", errors.New("empty task text")
|
||||
}
|
||||
req := ipc.CaptureTaskReq{Text: text, Source: "tap:web", Status: "open", Ts: time.Now()}
|
||||
// Importance is his, stated on the form. Out-of-range values are
|
||||
// clamped rather than rejected — a bad select is not worth a 400.
|
||||
if v := r.FormValue("weight"); v != "" {
|
||||
var wgt int
|
||||
if n, _ := fmt.Sscanf(v, "%d", &wgt); n != 1 || wgt < 0 {
|
||||
return "", fmt.Errorf("bad weight %q", v)
|
||||
}
|
||||
if wgt > tasks.MaxWeight {
|
||||
wgt = tasks.MaxWeight
|
||||
}
|
||||
req.Weight = wgt
|
||||
}
|
||||
if d := r.FormValue("due"); d != "" {
|
||||
due, err := time.ParseInLocation("2006-01-02", d, time.Local)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("bad due date %q", d)
|
||||
}
|
||||
req.Due = &due
|
||||
}
|
||||
resp, err := core.CaptureTask(ctx, req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !resp.Created {
|
||||
return "already on the list", nil
|
||||
}
|
||||
return "added task", nil
|
||||
}
|
||||
|
||||
var id int64
|
||||
if n, _ := fmt.Sscanf(r.FormValue("id"), "%d", &id); n != 1 {
|
||||
return "", errors.New("invalid id")
|
||||
}
|
||||
var status, msg string
|
||||
switch action {
|
||||
case "confirm":
|
||||
status, msg = "open", "confirmed task"
|
||||
case "done":
|
||||
status, msg = "done", "task done"
|
||||
case "drop":
|
||||
status, msg = "dropped", "dropped task"
|
||||
default:
|
||||
return "", fmt.Errorf("unknown action %q", action)
|
||||
}
|
||||
if err := core.SetTaskStatus(ctx, id, status, time.Now()); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return msg, nil
|
||||
}
|
||||
|
||||
func fmtTaskTime(t *time.Time) string {
|
||||
if t == nil || t.IsZero() {
|
||||
return "—"
|
||||
}
|
||||
return t.Local().Format("02 Jan 15:04")
|
||||
}
|
||||
|
||||
func fmtTaskDate(t *time.Time) string {
|
||||
if t == nil || t.IsZero() {
|
||||
return "—"
|
||||
}
|
||||
return t.Local().Format("02 Jan")
|
||||
}
|
||||
|
||||
// routineRow is one line on the page: what maven noticed, in her words, and
|
||||
// how long ago she noticed it.
|
||||
type routineRow struct {
|
||||
@@ -1112,12 +1333,20 @@ func handleTools(w http.ResponseWriter, r *http.Request, core ipc.CoreAPI, sessi
|
||||
http.Error(w, "core read failed", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
// MCP is off by default and an older core may not know the method at all,
|
||||
// so a failure here renders an empty section rather than breaking the page.
|
||||
servers, err := core.MCPServers(ctx)
|
||||
if err != nil {
|
||||
log.Printf("tools: mcp servers: %v", err)
|
||||
servers = nil
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
if err := toolsTmpl.Execute(w, struct {
|
||||
Msg string
|
||||
Proposed []ipc.Tool
|
||||
Enabled []ipc.Tool
|
||||
}{msg, proposed, enabled}); err != nil {
|
||||
MCP []ipc.MCPServerStatus
|
||||
}{msg, proposed, enabled, servers}); err != nil {
|
||||
log.Printf("tools render: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"html/template"
|
||||
"log"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/webauthn"
|
||||
)
|
||||
|
||||
// The resident-model surface (Vikunja #250).
|
||||
//
|
||||
// GET shows which model llama-server actually has loaded and which files the
|
||||
// daemon is configured to allow. POST swaps to one of them, behind the same
|
||||
// step-up gate as POST /tools: the loaded model decides how every utterance is
|
||||
// routed and how every reply is worded, so it is an owner action.
|
||||
//
|
||||
// There is nothing on this page Maven can press. The swap is an IPC method rated
|
||||
// AuthStepUp in internal/auth, unreachable from an act, an intent or a timer.
|
||||
|
||||
// modelController — the two non-CoreAPI methods this page needs. *ipc.Client
|
||||
// satisfies it; a core without a swap allowlist answers ErrUnknownMethod, which
|
||||
// the page renders as "not configured" rather than an error.
|
||||
type modelController interface {
|
||||
ModelStatus(ctx context.Context) (ipc.ModelStatusResp, error)
|
||||
SwapModel(ctx context.Context, req ipc.SwapModelReq) (ipc.SwapModelResp, error)
|
||||
}
|
||||
|
||||
var modelsTmpl = template.Must(template.New("models").Funcs(shellFuncs()).Parse(shellTopHTML + modelsHTML + shellBottomHTML))
|
||||
|
||||
const modelsHTML = `{{template "shellTop" "models"}}
|
||||
<h1>Resident model</h1>
|
||||
<p class=hint>swapping requires step-up — <a href=/auth/passkey>assert a passkey</a> first. The old model is unloaded before the new one is loaded (one model fits the iGPU at a time), so turns during the load are refused and fall back to the classifier.</p>
|
||||
{{if .Msg}}<div class="msg msg-ok">{{.Msg}}</div>{{end}}
|
||||
{{if .Err}}<div class="msg msg-err">{{.Err}}</div>{{end}}
|
||||
{{if .Off}}
|
||||
<section class=card>
|
||||
<h2 class=card-title>swap not configured</h2>
|
||||
<p class=hint>this core has no <code>phraser.swap_models</code> allowlist, so there is nothing to swap to. Add the gguf paths you allow to <code>deploy/mavend.json</code> and restart once.</p>
|
||||
</section>
|
||||
{{else}}
|
||||
<section class=card>
|
||||
<h2 class=card-title>loaded now</h2>
|
||||
<div class=scroll><table>
|
||||
<tr><th>model</th><td><code>{{.Status.Model}}</code></td></tr>
|
||||
<tr><th>file</th><td><code>{{.Status.ModelPath}}</code></td></tr>
|
||||
<tr><th>server</th><td><code>{{.Status.BaseURL}}</code></td></tr>
|
||||
<tr><th>n_ctx</th><td>{{.Status.NCtx}}</td></tr>
|
||||
<tr><th>n_gpu_layers</th><td>{{.Status.NGpuLayers}}</td></tr>
|
||||
</table></div>
|
||||
<p class=hint>the model name is what llama-server reports for itself, not what the config says it should be.</p>
|
||||
</section>
|
||||
<section class=card>
|
||||
<h2 class=card-title>allowed models <span class=badge>{{len .Status.Swappable}}</span></h2>
|
||||
{{if .Status.Swappable}}<div class=scroll><table><tr><th>file</th><th></th></tr>
|
||||
{{range .Status.Swappable}}<tr><td><code>{{.}}</code></td>
|
||||
<td><form method=post action=/models class=inline-form>
|
||||
<input type=hidden name=model_path value="{{.}}">
|
||||
<button class=btn>load this one</button></form></td></tr>{{end}}
|
||||
</table></div>
|
||||
{{else}}<div class=empty><div>no models allowlisted</div></div>{{end}}
|
||||
</section>
|
||||
{{end}}
|
||||
{{template "shellBottom"}}`
|
||||
|
||||
type modelsPage struct {
|
||||
Msg string
|
||||
Err string
|
||||
Off bool
|
||||
Status ipc.ModelStatusResp
|
||||
}
|
||||
|
||||
// handleModels renders the model surface (GET) and applies a swap (POST).
|
||||
//
|
||||
// A failed swap is reported as a failure with the model that is still serving
|
||||
// named, because that is the state the operator needs: the daemon rolled back
|
||||
// and is answering turns, it just is not answering them with what he asked for.
|
||||
func handleModels(w http.ResponseWriter, r *http.Request, core ipc.CoreAPI, session *webauthn.PasskeySession, requireStepUp bool) {
|
||||
if core == nil {
|
||||
http.Error(w, "models disabled (no -core)", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
mc, ok := core.(modelController)
|
||||
if !ok {
|
||||
http.Error(w, "models unavailable: core connection does not support model swap", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
ctx := r.Context()
|
||||
page := modelsPage{}
|
||||
|
||||
if r.Method == http.MethodPost {
|
||||
if !stepUpOK(session, requireStepUp) {
|
||||
http.Error(w, "step-up required: assert a passkey first", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
path := strings.TrimSpace(r.FormValue("model_path"))
|
||||
if path == "" {
|
||||
http.Error(w, "model_path required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
req := ipc.SwapModelReq{ModelPath: path}
|
||||
if v, err := strconv.Atoi(r.FormValue("n_ctx")); err == nil {
|
||||
req.NCtx = v
|
||||
}
|
||||
res, err := mc.SwapModel(ctx, req)
|
||||
switch {
|
||||
case err == nil:
|
||||
page.Msg = "loaded " + res.Model + " (" + strconv.FormatInt(res.TookMs, 10) + "ms)"
|
||||
log.Printf("models: swapped to %s (%s) in %dms", res.ModelPath, res.Model, res.TookMs)
|
||||
case errors.Is(err, ipc.ErrForbidden):
|
||||
http.Error(w, "refused: that model is not in phraser.swap_models, or step-up was not asserted", http.StatusForbidden)
|
||||
return
|
||||
case errors.Is(err, ipc.ErrUnknownMethod):
|
||||
http.Error(w, "swap not configured on this core", http.StatusServiceUnavailable)
|
||||
return
|
||||
case res.RolledBack:
|
||||
page.Err = "swap failed, rolled back to " + res.Model + " — she is still answering, with the old model"
|
||||
log.Printf("models: swap to %s failed, rolled back: %v", path, err)
|
||||
default:
|
||||
page.Err = "swap failed: " + err.Error()
|
||||
log.Printf("models: swap to %s failed: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
st, err := mc.ModelStatus(ctx)
|
||||
if err != nil {
|
||||
if errors.Is(err, ipc.ErrUnknownMethod) {
|
||||
page.Off = true
|
||||
} else {
|
||||
log.Printf("models: status: %v", err)
|
||||
http.Error(w, "core read failed", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
}
|
||||
page.Status = st
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
if err := modelsTmpl.Execute(w, page); err != nil {
|
||||
log.Printf("models render: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/webauthn"
|
||||
)
|
||||
|
||||
// fakeModelCore is a core that supports the two model methods. It records what
|
||||
// the page asked for, so the tests can assert the gate rather than the HTML.
|
||||
type fakeModelCore struct {
|
||||
ipc.UnimplementedCoreAPI
|
||||
|
||||
status ipc.ModelStatusResp
|
||||
statusErr error
|
||||
|
||||
swapResp ipc.SwapModelResp
|
||||
swapErr error
|
||||
swapped []ipc.SwapModelReq
|
||||
}
|
||||
|
||||
func (f *fakeModelCore) ModelStatus(ctx context.Context) (ipc.ModelStatusResp, error) {
|
||||
return f.status, f.statusErr
|
||||
}
|
||||
|
||||
func (f *fakeModelCore) SwapModel(ctx context.Context, req ipc.SwapModelReq) (ipc.SwapModelResp, error) {
|
||||
f.swapped = append(f.swapped, req)
|
||||
return f.swapResp, f.swapErr
|
||||
}
|
||||
|
||||
func modelsGET(t *testing.T, core ipc.CoreAPI) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
w := httptest.NewRecorder()
|
||||
handleModels(w, httptest.NewRequest(http.MethodGet, "/models", nil), core, nil, false)
|
||||
return w
|
||||
}
|
||||
|
||||
func modelsPOST(t *testing.T, core ipc.CoreAPI, session *webauthn.PasskeySession, requireStepUp bool, path string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
r := httptest.NewRequest(http.MethodPost, "/models", strings.NewReader("model_path="+path))
|
||||
r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
w := httptest.NewRecorder()
|
||||
handleModels(w, r, core, session, requireStepUp)
|
||||
return w
|
||||
}
|
||||
|
||||
func TestModels_GETShowsTheLoadedModelAndTheAllowlist(t *testing.T) {
|
||||
core := &fakeModelCore{status: ipc.ModelStatusResp{
|
||||
Model: "Qwen3-1.7B-UD-Q4_K_XL",
|
||||
ModelPath: "/opt/maven/models/llm/qwen3.gguf",
|
||||
BaseURL: "http://127.0.0.1:18099",
|
||||
NCtx: 4096,
|
||||
Swappable: []string{"/opt/maven/models/llm/qwen3.gguf", "/opt/maven/models/llm/qwen3-cpt.gguf"},
|
||||
}}
|
||||
w := modelsGET(t, core)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("GET /models = %d; want 200", w.Code)
|
||||
}
|
||||
body := w.Body.String()
|
||||
for _, want := range []string{"Qwen3-1.7B-UD-Q4_K_XL", "qwen3-cpt.gguf", "4096"} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("page does not mention %q", want)
|
||||
}
|
||||
}
|
||||
if len(core.swapped) != 0 {
|
||||
t.Errorf("a GET swapped the model: %v", core.swapped)
|
||||
}
|
||||
}
|
||||
|
||||
func TestModels_POSTRequiresStepUpWhenFailingClosed(t *testing.T) {
|
||||
// No WebAuthn configured (nil session) + -require-stepup ⇒ deny, exactly
|
||||
// like POST /tools. Nothing reaches core.
|
||||
core := &fakeModelCore{}
|
||||
w := modelsPOST(t, core, nil, true, "/opt/maven/models/llm/qwen3.gguf")
|
||||
if w.Code != http.StatusForbidden {
|
||||
t.Fatalf("POST /models without assertable step-up = %d; want 403", w.Code)
|
||||
}
|
||||
if len(core.swapped) != 0 {
|
||||
t.Fatalf("a denied POST still called SwapModel: %v", core.swapped)
|
||||
}
|
||||
}
|
||||
|
||||
func TestModels_POSTSwapsAndReportsTheModelThatAnswered(t *testing.T) {
|
||||
core := &fakeModelCore{
|
||||
swapResp: ipc.SwapModelResp{Model: "qwen3-cpt", ModelPath: "/m/cpt.gguf", TookMs: 4200},
|
||||
status: ipc.ModelStatusResp{Model: "qwen3-cpt", ModelPath: "/m/cpt.gguf"},
|
||||
}
|
||||
w := modelsPOST(t, core, nil, false, "/m/cpt.gguf")
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("POST /models = %d; want 200", w.Code)
|
||||
}
|
||||
if len(core.swapped) != 1 || core.swapped[0].ModelPath != "/m/cpt.gguf" {
|
||||
t.Fatalf("SwapModel calls = %v; want one for /m/cpt.gguf", core.swapped)
|
||||
}
|
||||
if !strings.Contains(w.Body.String(), "loaded qwen3-cpt") {
|
||||
t.Errorf("page does not report which model was loaded:\n%s", w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestModels_RolledBackSwapSaysSheIsStillAnswering(t *testing.T) {
|
||||
core := &fakeModelCore{
|
||||
swapResp: ipc.SwapModelResp{Model: "qwen3", ModelPath: "/m/old.gguf", RolledBack: true},
|
||||
swapErr: errBrokenModel{},
|
||||
status: ipc.ModelStatusResp{Model: "qwen3", ModelPath: "/m/old.gguf"},
|
||||
}
|
||||
w := modelsPOST(t, core, nil, false, "/m/cpt.gguf")
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("POST /models after a rollback = %d; want 200 with the failure rendered", w.Code)
|
||||
}
|
||||
body := w.Body.String()
|
||||
if !strings.Contains(body, "rolled back to qwen3") {
|
||||
t.Errorf("page does not say it rolled back:\n%s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestModels_RefusedPathIs403(t *testing.T) {
|
||||
core := &fakeModelCore{swapErr: ipc.ErrForbidden}
|
||||
w := modelsPOST(t, core, nil, false, "/etc/passwd")
|
||||
if w.Code != http.StatusForbidden {
|
||||
t.Fatalf("POST /models with a non-allowlisted path = %d; want 403", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestModels_UnconfiguredCoreRendersOff(t *testing.T) {
|
||||
core := &fakeModelCore{statusErr: ipc.ErrUnknownMethod}
|
||||
w := modelsGET(t, core)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("GET /models against a core without the swap = %d; want 200", w.Code)
|
||||
}
|
||||
if !strings.Contains(w.Body.String(), "swap not configured") {
|
||||
t.Errorf("page does not say the capability is off:\n%s", w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestModels_CoreWithoutTheMethodsIs503(t *testing.T) {
|
||||
// An in-process CoreAPI (no swap methods) must not 500 the page.
|
||||
w := modelsGET(t, ipc.UnimplementedCoreAPI{})
|
||||
if w.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("GET /models on a core without the methods = %d; want 503", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
type errBrokenModel struct{}
|
||||
|
||||
func (errBrokenModel) Error() string { return "llm: server did not start" }
|
||||
@@ -0,0 +1,83 @@
|
||||
{{template "shellTop" "tasks"}}
|
||||
<h1>Tasks</h1>
|
||||
{{if .Msg}}<div class="msg msg-ok">{{.Msg}}</div>{{end}}
|
||||
{{if .Err}}<div class="msg msg-err">{{.Err}}</div>{{end}}
|
||||
|
||||
<section class=card>
|
||||
<h2 class=card-title>add</h2>
|
||||
<form method=post action=/tasks class=inline-form>
|
||||
<input type=hidden name=action value=add>
|
||||
<input type=text name=text placeholder="что нужно сделать" size=44 required>
|
||||
<input type=date name=due title="due date (optional)">
|
||||
<select name=weight title="importance (optional)">
|
||||
<option value=0>normal</option>
|
||||
<option value=2>важно</option>
|
||||
<option value=3>срочно</option>
|
||||
</select>
|
||||
<button class=btn>add</button>
|
||||
</form>
|
||||
</section>
|
||||
|
||||
{{if .Candidates}}
|
||||
<section class=card>
|
||||
<h2 class=card-title>found, not confirmed <span class=badge>{{len .Candidates}}</span></h2>
|
||||
<div class=hint>maven derived these from something she read. nothing counts as your work until you confirm it.</div>
|
||||
<div class=scroll><table>
|
||||
<tr><th>task</th><th>where from</th><th>due</th><th>captured</th><th></th><th></th></tr>
|
||||
{{range .Candidates}}<tr>
|
||||
<td class=text-max>{{.Text}}</td>
|
||||
<td class=hint>{{.Source}}{{if .Evidence}} — {{.Evidence}}{{end}}</td>
|
||||
<td>{{.Due}}</td>
|
||||
<td class=muted>{{.Created}}</td>
|
||||
<td><form method=post action=/tasks class=inline-form>
|
||||
<input type=hidden name=id value="{{.ID}}">
|
||||
<input type=hidden name=action value=confirm>
|
||||
<button class=btn>confirm</button></form></td>
|
||||
<td><form method=post action=/tasks class=inline-form>
|
||||
<input type=hidden name=id value="{{.ID}}">
|
||||
<input type=hidden name=action value=drop>
|
||||
<button class="btn btn-muted">drop</button></form></td>
|
||||
</tr>{{end}}</table></div>
|
||||
</section>
|
||||
{{end}}
|
||||
|
||||
<section class=card>
|
||||
<h2 class=card-title>open <span class=badge>{{len .Open}}</span></h2>
|
||||
<div class=hint>most pressing first — by the deadlines and the urgency you gave, nothing guessed.</div>
|
||||
{{if .Open}}<div class=scroll><table>
|
||||
<tr><th>task</th><th>why</th><th>from</th><th>due</th><th>captured</th><th></th><th></th></tr>
|
||||
{{range .Open}}<tr>
|
||||
<td class=text-max>{{.Text}}</td>
|
||||
<td class=hint>{{.Why}}</td>
|
||||
<td class=hint>{{.Source}}</td>
|
||||
<td>{{.Due}}</td>
|
||||
<td class=muted>{{.Created}}</td>
|
||||
<td><form method=post action=/tasks class=inline-form>
|
||||
<input type=hidden name=id value="{{.ID}}">
|
||||
<input type=hidden name=action value=done>
|
||||
<button class=btn>done</button></form></td>
|
||||
<td><form method=post action=/tasks class=inline-form>
|
||||
<input type=hidden name=id value="{{.ID}}">
|
||||
<input type=hidden name=action value=drop>
|
||||
<button class="btn btn-muted">drop</button></form></td>
|
||||
</tr>{{end}}</table></div>
|
||||
{{else}}<div class=empty>
|
||||
<svg class=icon width="20" height="20"><use href="/ethos-icons.svg#i-grid"/></svg>
|
||||
<div>no open tasks</div>
|
||||
<div class=hint>add one above, or tell maven "добавь в задачи …"</div>
|
||||
</div>{{end}}
|
||||
</section>
|
||||
|
||||
{{if .Resolved}}
|
||||
<section class=card>
|
||||
<h2 class=card-title>resolved <span class=badge>{{len .Resolved}}</span></h2>
|
||||
<div class=scroll><table>
|
||||
<tr><th>task</th><th>status</th><th>when</th></tr>
|
||||
{{range .Resolved}}<tr>
|
||||
<td class=text-max>{{.Text}}</td>
|
||||
<td><span class="badge {{.Status}}">{{.Status}}</span></td>
|
||||
<td class=muted>{{.Resolved}}</td>
|
||||
</tr>{{end}}</table></div>
|
||||
</section>
|
||||
{{end}}
|
||||
{{template "shellBottom"}}
|
||||
@@ -0,0 +1,225 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/tasks"
|
||||
)
|
||||
|
||||
// fakeTaskCore serves the /tasks handler: a canned list plus a log of the
|
||||
// writes the page made.
|
||||
type fakeTaskCore struct {
|
||||
ipc.UnimplementedCoreAPI
|
||||
|
||||
tasks []ipc.Task
|
||||
listErr error
|
||||
|
||||
captured []ipc.CaptureTaskReq
|
||||
created bool
|
||||
captureErr error
|
||||
|
||||
statusID int64
|
||||
statusVal string
|
||||
statusErr error
|
||||
}
|
||||
|
||||
func (f *fakeTaskCore) ListTasks(_ context.Context, status string) ([]ipc.Task, error) {
|
||||
if f.listErr != nil {
|
||||
return nil, f.listErr
|
||||
}
|
||||
return f.tasks, nil
|
||||
}
|
||||
|
||||
func (f *fakeTaskCore) CaptureTask(_ context.Context, req ipc.CaptureTaskReq) (ipc.CaptureTaskResp, error) {
|
||||
f.captured = append(f.captured, req)
|
||||
if f.captureErr != nil {
|
||||
return ipc.CaptureTaskResp{}, f.captureErr
|
||||
}
|
||||
return ipc.CaptureTaskResp{ID: 7, Created: f.created}, nil
|
||||
}
|
||||
|
||||
func (f *fakeTaskCore) SetTaskStatus(_ context.Context, id int64, status string, _ time.Time) error {
|
||||
f.statusID, f.statusVal = id, status
|
||||
return f.statusErr
|
||||
}
|
||||
|
||||
func TestHandleTasksSplitsCandidatesFromOpen(t *testing.T) {
|
||||
now := time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC)
|
||||
resolved := now.Add(time.Hour)
|
||||
core := &fakeTaskCore{tasks: []ipc.Task{
|
||||
{ID: 1, Text: "купить молоко", Source: "tap:voice", Status: "open", CreatedTs: now},
|
||||
{ID: 2, Text: "продлить страховку", Source: "email:kami", Evidence: "полис истекает", Status: "candidate", CreatedTs: now},
|
||||
{ID: 3, Text: "полить цветы", Source: "tap:web", Status: "done", CreatedTs: now, Resolved: &resolved},
|
||||
}}
|
||||
rec := httptest.NewRecorder()
|
||||
handleTasks(rec, httptest.NewRequest(http.MethodGet, "/tasks", nil), core)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d", rec.Code)
|
||||
}
|
||||
body := rec.Body.String()
|
||||
for _, want := range []string{
|
||||
"купить молоко", "продлить страховку", "полить цветы",
|
||||
"полис истекает", // the evidence trail is visible for review
|
||||
"found, not confirmed", // candidates get their own section
|
||||
} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("body missing %q", want)
|
||||
}
|
||||
}
|
||||
// The candidate must offer confirm, and the open task must not.
|
||||
if !strings.Contains(body, "value=confirm") {
|
||||
t.Error("candidate row has no confirm action")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleTasksAddCaptures(t *testing.T) {
|
||||
core := &fakeTaskCore{created: true}
|
||||
form := url.Values{"action": {"add"}, "text": {" позвонить в банк "}, "due": {"2026-08-05"}}
|
||||
req := httptest.NewRequest(http.MethodPost, "/tasks", strings.NewReader(form.Encode()))
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
rec := httptest.NewRecorder()
|
||||
handleTasks(rec, req, core)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d", rec.Code)
|
||||
}
|
||||
if len(core.captured) != 1 {
|
||||
t.Fatalf("captured %d requests, want 1", len(core.captured))
|
||||
}
|
||||
got := core.captured[0]
|
||||
if got.Text != "позвонить в банк" {
|
||||
t.Errorf("text = %q, want trimmed", got.Text)
|
||||
}
|
||||
if got.Source != "tap:web" {
|
||||
t.Errorf("source = %q, want tap:web", got.Source)
|
||||
}
|
||||
if got.Status != "open" {
|
||||
t.Errorf("status = %q — a task he typed himself is open, not a candidate", got.Status)
|
||||
}
|
||||
if got.Due == nil || got.Due.Format("2006-01-02") != "2026-08-05" {
|
||||
t.Errorf("due = %v", got.Due)
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), "added task") {
|
||||
t.Error("no confirmation message")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleTasksAddSaysAlreadyOnTheList(t *testing.T) {
|
||||
core := &fakeTaskCore{created: false}
|
||||
form := url.Values{"action": {"add"}, "text": {"купить молоко"}}
|
||||
req := httptest.NewRequest(http.MethodPost, "/tasks", strings.NewReader(form.Encode()))
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
rec := httptest.NewRecorder()
|
||||
handleTasks(rec, req, core)
|
||||
if !strings.Contains(rec.Body.String(), "already on the list") {
|
||||
t.Error("a deduped capture must not claim it saved something new")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleTasksStatusActions(t *testing.T) {
|
||||
for _, tc := range []struct{ action, want string }{
|
||||
{"confirm", "open"},
|
||||
{"done", "done"},
|
||||
{"drop", "dropped"},
|
||||
} {
|
||||
core := &fakeTaskCore{}
|
||||
form := url.Values{"action": {tc.action}, "id": {"42"}}
|
||||
req := httptest.NewRequest(http.MethodPost, "/tasks", strings.NewReader(form.Encode()))
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
handleTasks(httptest.NewRecorder(), req, core)
|
||||
if core.statusID != 42 || core.statusVal != tc.want {
|
||||
t.Errorf("%s → SetTaskStatus(%d, %q), want (42, %q)", tc.action, core.statusID, core.statusVal, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleTasksRejectsBadPost(t *testing.T) {
|
||||
core := &fakeTaskCore{}
|
||||
form := url.Values{"action": {"explode"}, "id": {"1"}}
|
||||
req := httptest.NewRequest(http.MethodPost, "/tasks", strings.NewReader(form.Encode()))
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
rec := httptest.NewRecorder()
|
||||
handleTasks(rec, req, core)
|
||||
// The page still renders, with the error inline — and nothing was written.
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d", rec.Code)
|
||||
}
|
||||
if core.statusVal != "" || len(core.captured) != 0 {
|
||||
t.Error("an unknown action must write nothing")
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), "unknown action") {
|
||||
t.Error("error not surfaced on the page")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleTasksNoCore(t *testing.T) {
|
||||
rec := httptest.NewRecorder()
|
||||
handleTasks(rec, httptest.NewRequest(http.MethodGet, "/tasks", nil), nil)
|
||||
if rec.Code != http.StatusServiceUnavailable {
|
||||
t.Errorf("status = %d, want 503", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// The open list is ordered by the ranker, and the reason is shown so the page
|
||||
// says why a task is first instead of asking him to trust the order.
|
||||
func TestHandleTasksOrdersOpenByRank(t *testing.T) {
|
||||
now := time.Now()
|
||||
due := now
|
||||
core := &fakeTaskCore{tasks: []ipc.Task{
|
||||
{ID: 1, Text: "купить молоко", Status: "open", CreatedTs: now},
|
||||
{ID: 2, Text: "оплатить интернет", Status: "open", CreatedTs: now, Due: &due},
|
||||
}}
|
||||
rec := httptest.NewRecorder()
|
||||
handleTasks(rec, httptest.NewRequest(http.MethodGet, "/tasks", nil), core)
|
||||
body := rec.Body.String()
|
||||
if strings.Index(body, "оплатить интернет") > strings.Index(body, "купить молоко") {
|
||||
t.Error("want the dated task rendered first")
|
||||
}
|
||||
if !strings.Contains(body, "сегодня") {
|
||||
t.Error("want the ranker's reason shown in the why column")
|
||||
}
|
||||
}
|
||||
|
||||
// A candidate is ranked into place but never carries a priority reason: its due
|
||||
// date is Maven's reading of a mail, not something he stated.
|
||||
func TestHandleTasksHidesCandidateReason(t *testing.T) {
|
||||
now := time.Now()
|
||||
due := now
|
||||
core := &fakeTaskCore{tasks: []ipc.Task{
|
||||
{ID: 1, Text: "продлить страховку", Status: "candidate", CreatedTs: now, Due: &due},
|
||||
}}
|
||||
rec := httptest.NewRecorder()
|
||||
handleTasks(rec, httptest.NewRequest(http.MethodGet, "/tasks", nil), core)
|
||||
if strings.Contains(rec.Body.String(), "сегодня") {
|
||||
t.Error("a candidate must not be shown with a priority reason")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyTaskPostCarriesWeight(t *testing.T) {
|
||||
core := &fakeTaskCore{created: true}
|
||||
form := url.Values{"action": {"add"}, "text": {"оплатить интернет"}, "weight": {"3"}}
|
||||
req := httptest.NewRequest(http.MethodPost, "/tasks", strings.NewReader(form.Encode()))
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
handleTasks(httptest.NewRecorder(), req, core)
|
||||
if len(core.captured) != 1 || core.captured[0].Weight != 3 {
|
||||
t.Fatalf("captured = %+v, want weight 3", core.captured)
|
||||
}
|
||||
}
|
||||
|
||||
// Out of range clamps rather than 400s; a non-number is a real client error.
|
||||
func TestApplyTaskPostClampsWeight(t *testing.T) {
|
||||
core := &fakeTaskCore{created: true}
|
||||
form := url.Values{"action": {"add"}, "text": {"что-то"}, "weight": {"99"}}
|
||||
req := httptest.NewRequest(http.MethodPost, "/tasks", strings.NewReader(form.Encode()))
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
handleTasks(httptest.NewRecorder(), req, core)
|
||||
if core.captured[0].Weight != tasks.MaxWeight {
|
||||
t.Errorf("weight = %d, want the cap", core.captured[0].Weight)
|
||||
}
|
||||
}
|
||||
@@ -36,6 +36,67 @@ present (see `.dockerignore`).
|
||||
| `/var/lib/maven` (volume) | encrypted db at rest |
|
||||
| `/dev/shm` (tmpfs) | decrypted db working copy (RAM only) |
|
||||
|
||||
## Reading the outside world (off by default)
|
||||
|
||||
`mavend.json` ships without a `feeds` block, which means no RSS/Atom feed is
|
||||
fetched and no outbound request is made. Switching it on is adding the block:
|
||||
|
||||
```json
|
||||
"feeds": {
|
||||
"poll_interval": "30m",
|
||||
"max_items": 5,
|
||||
"max_age": "24h",
|
||||
"sources": [
|
||||
{ "name": "habr", "url": "https://habr.com/ru/rss/best/daily/",
|
||||
"category": "технологии", "exclude": ["реклама"] }
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
What it does and does not do:
|
||||
|
||||
- items are written as notes with source `rss:<name>`, visible on `/dash`;
|
||||
- **nothing is announced.** She reads them back when asked — "что нового в
|
||||
лентах?", "что нового по технологиям?" — and never on arrival. There is no
|
||||
severity or channel knob here on purpose;
|
||||
- the fetcher is allowlisted to the hosts of the configured feeds, plus any
|
||||
`allow_hosts`. It refuses non-http(s) schemes and every private address
|
||||
(loopback, the LAN, the `10.42.0.0/24` wg range, cloud metadata). It caps the
|
||||
response at 2 MiB and redirects at 3, and makes at most one request per host
|
||||
per second. See `internal/webfetch`;
|
||||
- how far each feed was read is stored as a config fact `rss:latest:<name>`, so
|
||||
a restart does not re-note yesterday's headlines.
|
||||
|
||||
### Reading a page (`crawl`, also off by default)
|
||||
|
||||
There is no `crawl` block either, so no page is fetched. Two halves, separately
|
||||
switched:
|
||||
|
||||
```json
|
||||
"crawl": {
|
||||
"on_demand": true,
|
||||
"interval": "6h",
|
||||
"max_runes": 4000,
|
||||
"watches": [
|
||||
{ "name": "changelog", "url": "https://example.org/changelog", "interval": "12h" }
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
- `on_demand` lets her read a page he names in the utterance: "посмотри
|
||||
https://example.org/x — что там?". The page becomes context for his question,
|
||||
and only the URL leaves the box. Without a URL nothing is fetched, so this is
|
||||
a fallback and not a habit;
|
||||
- `watches` re-reads a fixed list on its interval and writes a note when the
|
||||
text changed. Like the feeds, it announces nothing;
|
||||
- the answer path sits **last** in the query chain, behind his memory, his notes
|
||||
and (once wired) the local Kiwix ZIMs. A local read costs nothing;
|
||||
- `robots.txt` is fetched first and obeyed with no override; a `Disallow` is a
|
||||
refusal she says out loud. `Crawl-delay` is honoured;
|
||||
- same guarded fetcher as the feeds: allowlist/denylist, no private addresses,
|
||||
size cap, redirect cap, timeout, one request per host per second;
|
||||
- dedup state is the config fact `crawl:hash:<name>`.
|
||||
|
||||
## Not yet verified / host-dependent
|
||||
|
||||
This stack is correct-by-construction but has **not been build-tested here**
|
||||
@@ -53,3 +114,49 @@ build on the target host, most likely in one of these:
|
||||
work fine over the core socket.
|
||||
- **netdata** — `mavpoll` reaches it via `host.docker.internal`; adjust if
|
||||
netdata runs elsewhere.
|
||||
|
||||
## Updating her (`mavupdate`, Vikunja #249)
|
||||
|
||||
Off unless configured, and there is deliberately no button for it. There is no
|
||||
IPC method, no web route, no timer and no act that starts an update — the trigger
|
||||
is a human running `mavupdate` on the host, which needs shell access, a strictly
|
||||
higher bar than the step-up passkey gate that guards `/tools`. She cannot update
|
||||
herself; she can be updated. Nothing here ever fetches code: the new version is
|
||||
whatever you pulled into the working tree yourself.
|
||||
|
||||
Add an `update` block to `mavend.json` (mavend ignores it — only the CLI reads
|
||||
it), with paths as they exist **on the host**, not inside a container:
|
||||
|
||||
```json
|
||||
"update": {
|
||||
"source_dir": "/home/kami/apps/Maven",
|
||||
"install_dir": "/home/kami/apps/Maven",
|
||||
"snapshot_dir": "/var/lib/maven-snapshots",
|
||||
"binaries": ["mavend", "mavweb", "mavsttd", "mavttsd", "mavwaked",
|
||||
"mavenclient", "mavpoll", "mavcaldav", "mavmaild"],
|
||||
"config_files": ["deploy/mavend.json"],
|
||||
"restart_cmd": ["docker", "compose", "up", "-d", "--build"],
|
||||
"health_socket": "/var/lib/docker/volumes/maven_sockets/_data/mavend.sock",
|
||||
"health_timeout_sec": 120
|
||||
}
|
||||
```
|
||||
|
||||
`snapshot_dir` must be outside `install_dir` (a restore must not read from what
|
||||
the install writes) and `health_socket` is required: an update that cannot check
|
||||
its own result cannot roll itself back, so the config is refused without one.
|
||||
|
||||
Then:
|
||||
|
||||
```sh
|
||||
mavupdate -config deploy/mavend.json verify # make build + make test, deploys nothing
|
||||
mavupdate -config deploy/mavend.json apply -yes # snapshot, verify, install, restart, health-check
|
||||
mavupdate -config deploy/mavend.json list # what you can roll back to
|
||||
mavupdate -config deploy/mavend.json rollback -yes # restore the previous artifacts and restart
|
||||
```
|
||||
|
||||
`apply` refuses to start if she is not already answering — otherwise a failed
|
||||
update and a box that was already broken are indistinguishable afterwards. On any
|
||||
failure after the install it restores the snapshot, restarts, and checks again;
|
||||
if that also fails it says so loudly and names the directory to copy back by hand.
|
||||
The database is never snapshotted or rolled back (see the package comment in
|
||||
`internal/update`); schema compatibility is `store.Migrate`'s job.
|
||||
|
||||
@@ -31,6 +31,20 @@
|
||||
"cooldown": "24h"
|
||||
},
|
||||
|
||||
"mcp": {
|
||||
"timeout": "15s",
|
||||
"servers": [
|
||||
{
|
||||
"name": "vikunja",
|
||||
"url": "http://192.168.1.104:9100/mcp",
|
||||
"allow_private": true,
|
||||
"allow_tools": ["list_projects", "list_tasks", "get_task_details", "create_task"],
|
||||
"max_tools": 6,
|
||||
"enabled": false
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
"nexus": { "url": "http://nexus:9740" },
|
||||
"praxis": { "url": "http://praxis:8989" },
|
||||
"hexis": { "url": "http://hexis:9741" },
|
||||
|
||||
@@ -101,9 +101,42 @@ services:
|
||||
"-netdata", "http://127.0.0.1:19999",
|
||||
"-kuma", "http://127.0.0.1:3001/metrics",
|
||||
"-kuma-key", "uk5_mavpoll-key"]
|
||||
# Money tracking (Vikunja #125) is OFF: it needs a zenmoney token,
|
||||
# which mavpoll reads from a FILE so it never appears in `ps`, in
|
||||
# this file, or in shell history. To enable, mount the token and
|
||||
# append: "-zenmoney-token-file", "/run/secrets/zenmoney.token"
|
||||
# (optionally "-zenmoney-interval", "1h"). Core never sees the
|
||||
# token — the poller writes facts(kind=env, source=poll:zenmoney)
|
||||
# and mavend only reads those back when he asks.
|
||||
depends_on: [mavend]
|
||||
volumes:
|
||||
- sockets:/run/maven
|
||||
# - ./deploy/zenmoney.token:/run/secrets/zenmoney.token:ro
|
||||
|
||||
# The mail reader (Vikunja #246) is OFF and commented out: it needs an IMAP
|
||||
# account, and there is none on this box. mavmaild reads the password from a
|
||||
# FILE so it never appears in `ps`, in this file, or in shell history — the
|
||||
# same rule mavpoll follows for the zenmoney token. Core never sees the
|
||||
# password: the reader hands core message text on one IPC method, and core
|
||||
# writes what the model extracts as task CANDIDATES he reviews on /tasks.
|
||||
# Nothing here can create a reminder, so a misread mail cannot fire.
|
||||
#
|
||||
# To enable: write the password to deploy/imap.password (0600, gitignored),
|
||||
# add an "email": {} block to deploy/mavend.json, and uncomment this service.
|
||||
# mavmaild:
|
||||
# <<: *image
|
||||
# command: ["mavmaild", "-socket", "/run/maven/mavend.sock",
|
||||
# "-imap", "imap.example.org:993",
|
||||
# "-user", "kami@example.org",
|
||||
# "-password-file", "/run/secrets/imap.password",
|
||||
# "-mailbox", "INBOX",
|
||||
# "-interval", "15m",
|
||||
# "-state", "/var/lib/maven/mail-seen.json"]
|
||||
# depends_on: [mavend]
|
||||
# volumes:
|
||||
# - sockets:/run/maven
|
||||
# - dbdata:/var/lib/maven
|
||||
# - ./deploy/imap.password:/run/secrets/imap.password:ro
|
||||
|
||||
volumes:
|
||||
dbdata:
|
||||
|
||||
+101
-22
@@ -1,27 +1,106 @@
|
||||
# Plan: Vision — Image Understanding Capability
|
||||
|
||||
**Goal:** Maven can "see" — accept images (from mavweb upload, Telegram, or filesystem paths), run vision inference via a local or remote multimodal model, and answer questions about the image content or extract structured information.
|
||||
**Goal:** Maven can "see" — accept images (from mavweb upload, Telegram, or filesystem paths), store them, run inference via a **local** multimodal model, and answer questions about the image content or extract text from it.
|
||||
|
||||
**Done when:**
|
||||
- Vision model backend is configurable: local multimodal LLM (e.g., LLaVA, Qwen-VL via `llama-server` mmproj) or remote API
|
||||
- `internal/vision/` package handles image preprocessing, model inference, result parsing
|
||||
- Voice/text commands like "что на картинке?" or "прочитай текст с экрана" route to the vision handler
|
||||
- Extracted information can be written as facts/notes through `ipc.CoreAPI`
|
||||
- Telegram image messages are processed through the same pipeline
|
||||
**Status (2026-08-01):** intake, storage, config seam and the provider are shipped. The
|
||||
describing half is **BLOCKED on a model download** — see "What is blocked" below.
|
||||
|
||||
**Scope:**
|
||||
- New `internal/vision/` package — image loader (Go stdlib `image` + `golang.org/x/image`), inference client
|
||||
- New config block: `voice.vision` in `config.Config` — `{enabled, provider, model_path, mmproj_path, remote_url}`
|
||||
- Router intent extension: new `IntentVision` or reuse `IntentQuery` with a vision flag
|
||||
- Reuses `internal/llm.Client` for API-compatible backends (OpenAI-compatible vision API)
|
||||
- Reuses `internal/ipc.CoreAPI` for writing extracted data
|
||||
## What shipped
|
||||
|
||||
**Steps:**
|
||||
1. Create `internal/vision/provider.go` — `Provider` interface with `Describe(image []byte, prompt string) (string, error)` and `ExtractText(image []byte) (string, error)`
|
||||
2. Implement `LocalProvider` — spawns `llama-server` with mmproj, sends multimodal chat completion requests
|
||||
3. Implement `RemoteProvider` — calls an OpenAI-compatible vision API endpoint, reuses `internal/llm.Client`
|
||||
4. Create `internal/vision/processor.go` — image preprocessing (resize, format conversion to JPEG/PNG, base64 encoding)
|
||||
5. Wire vision into `cmd/mavend/voice.go:reactiveHandler` — detect vision intent from router (new `IntentVision` or a `Slots.HasImage` flag)
|
||||
6. Add IPC method `MethodDescribeImage` for programmatic access (mavweb upload, telegram bot)
|
||||
7. Add vision config block to `config.Config` and wire in `cmd/mavend/main.go`
|
||||
8. Test with a local multimodal model: send an image via mavweb, verify description and text extraction
|
||||
| Piece | Where |
|
||||
|---|---|
|
||||
| Blob store (content-addressed, retention-pruned) | `internal/media/store.go` |
|
||||
| Image decode / flatten / downscale / JPEG | `internal/media/image.go` |
|
||||
| `Provider` seam + `Disabled` floor + `LocalProvider` | `internal/vision/vision.go` |
|
||||
| Store-then-describe orchestration, re-runnable | `internal/vision/intake.go` |
|
||||
| Config blocks `media` and `vision` | `internal/config/config.go` |
|
||||
| IPC method `describe_image` (`AuthRead`) | `internal/ipc/{wire,api,client,server}.go`, `internal/auth/policy.go` |
|
||||
| Daemon wiring + hourly retention prune | `cmd/mavend/vision.go` |
|
||||
|
||||
`internal/media` is deliberately shared: hearing (#253) and speaker recognition (#255) have
|
||||
the same intake problem — a blob arrives, gets stored, gets described — and they store their
|
||||
audio in the same place under the same retention.
|
||||
|
||||
## Design decisions worth knowing
|
||||
|
||||
**Store before describe.** `Intake.Accept` writes the blob to disk *first*, then asks the
|
||||
model. If the model is missing or broken — which is this box's actual state — the answer is
|
||||
"it's kept, I can't read it yet" with a content-addressed id, and `Intake.Rerun(id, question)`
|
||||
describes it later. Nothing is lost to a missing model.
|
||||
|
||||
**No `RemoteProvider`.** The original step 3 called for "an OpenAI-compatible vision API
|
||||
endpoint". Refused. The surviving hard constraint in CLAUDE.md after "never phones home" was
|
||||
deprecated is *no cloud model, inference stays on the box*, and a photo of his flat is the
|
||||
worst possible exception. `vision.NewLocal` therefore validates the endpoint at construction:
|
||||
loopback, a private IP, or `localhost`. A hostname is refused too — it could resolve anywhere,
|
||||
and resolving it would mean trusting DNS with his pictures.
|
||||
|
||||
**Blobs are not in the database.** The sqlite store is small, encrypted and read every tick;
|
||||
a 40 MB blob has no business there. What lands in the database is the *text* the blob produced,
|
||||
as an ordinary note (`source: media:image:<id-prefix>`), and only when the caller asks for it
|
||||
(`save_note`). Glancing at a screenshot is not the same act as remembering it.
|
||||
|
||||
**Images are never search input and never embedded.** Only the derived description
|
||||
participates in recall, and only after he can see it as a note.
|
||||
|
||||
**Retention is enforced by a loop, not by a promise.** `media.retention` defaults to 7 days
|
||||
and `cmd/mavend` prunes hourly, starting at boot. A store that grows forever would be the real
|
||||
failure mode of this capability.
|
||||
|
||||
**No webp.** The stdlib has no webp decoder and this repo takes no new dependencies (the box
|
||||
is offline). `media.SniffImage` recognises webp well enough to refuse it *by name*, so the log
|
||||
says "webp is not supported" instead of "not an image". Telegram sends webp for stickers; that
|
||||
is a known gap, not a mystery.
|
||||
|
||||
**Text extraction is not a second method.** "прочитай текст с картинки" is a prompt. A VLM has
|
||||
no separate OCR mode to select, and a second interface method would only duplicate the first.
|
||||
|
||||
## What is blocked, and on what
|
||||
|
||||
There is **no vision-capable gguf and no mmproj file on this box**. Checked 2026-08-01:
|
||||
|
||||
```
|
||||
/mnt/hdd1/llms/{Bonsai,LFM2.5,llama3.2,ministral,nemotron3-nano,qwen3,qwen3.5}
|
||||
```
|
||||
|
||||
— sixteen ggufs, all text-only, no `*mmproj*` anywhere. The resident Qwen3-1.7B is text-only
|
||||
by construction, so vision needs a *second* model. The ≤1.7B ceiling in CLAUDE.md is about the
|
||||
resident router/phraser, not about a second model loaded on demand — but iGPU VRAM still is,
|
||||
so keep it small.
|
||||
|
||||
To unblock, download one pair to `/mnt/hdd1/llms/vision/` (bind-mounted to
|
||||
`/opt/maven/models/llm`), a gguf **and** its mmproj:
|
||||
|
||||
- `Qwen2.5-VL-3B-Instruct` (Q4_K_M + `mmproj-F16.gguf`) — the safe default; reads Russian, and
|
||||
its OCR is the best of this size class.
|
||||
- `SmolVLM2-2.2B-Instruct` — smaller and faster, weaker at Cyrillic text in images.
|
||||
- `moondream2` — smallest, English-only in practice. Do not bother, per the sub-500M lesson.
|
||||
|
||||
Then run a second llama-server on 8081 with `--mmproj`, point `vision.endpoint` at it, and
|
||||
walk the QA steps on Vikunja #252.
|
||||
|
||||
## Config
|
||||
|
||||
```json
|
||||
"media": { "dir": "media", "retention": "168h", "max_bytes": 67108864 },
|
||||
"vision": {
|
||||
"enabled": true,
|
||||
"endpoint": "http://127.0.0.1:8081",
|
||||
"model": "qwen2.5-vl-3b",
|
||||
"max_dim": 896,
|
||||
"max_tokens": 300,
|
||||
"timeout": "90s"
|
||||
}
|
||||
```
|
||||
|
||||
Both absent by default. No `media` block ⇒ `describe_image` does not exist at all; a `media`
|
||||
block with no `vision` block ⇒ images are stored and honestly not described.
|
||||
|
||||
## Still open
|
||||
|
||||
- **Router intent.** "что на картинке?" does not route anywhere yet. Adding an intent is
|
||||
premature while nothing can answer it; the IPC method is the surface a Telegram photo or a
|
||||
mavweb upload calls today.
|
||||
- **Telegram photo path** in `mavpoll` (download the file, call `DescribeImage`).
|
||||
- **mavweb upload page** and a `/media` listing so stored blobs are visible and deletable from
|
||||
the authed surface.
|
||||
|
||||
+117
-24
@@ -1,28 +1,121 @@
|
||||
# Plan: Hearing — Audio Stream Monitoring & Meeting Summarization
|
||||
# Plan: Hearing — Meeting Capture & Summarisation
|
||||
|
||||
**Goal:** Maven can "hear" ambient audio from workpc — microphone input during meetings, system audio — and on demand (or on trigger) produce transcripts, summaries, or extract action items. A typical use case: "Maven, запиши встречу" starts capture, "хватит" stops it, and Maven writes a summary note.
|
||||
**Goal:** "Maven, запиши встречу" starts a recording, "хватит" stops it, and she writes a
|
||||
summary note. The audio stays on the box, is pruned by retention, and nothing is recorded that
|
||||
nobody asked for.
|
||||
|
||||
**Done when:**
|
||||
- `internal/audio/capture.go` — remote microphone capture client (receives PCM stream from workpc over WebSocket or the existing voice TCP protocol)
|
||||
- `internal/stt/` — streaming transcription (uses existing `stt.Transcriber` interface, extended with streaming support)
|
||||
- Meeting capture triggered by voice command (IntentCapture) or configurable keyword ("maven record")
|
||||
- Raw audio is either streamed to STT in real-time or saved to a WAV file and transcribed after capture ends
|
||||
- Transcription + LLM summary is written as a note (`source:capture:meeting`) through `ipc.CoreAPI`
|
||||
- New `mavheary` module (`cmd/mavheard/`) — the workpc-side agent that captures mic/speaker audio and streams it to mavend
|
||||
**Status (2026-08-01):** the recorder, the storage, the chunked transcription, the map-reduce
|
||||
summariser, the config seam and the four IPC methods are shipped and tested. What is not
|
||||
shipped is the workpc-side microphone agent and the router intent — see "Still open".
|
||||
|
||||
**Scope:**
|
||||
- New `cmd/mavheard/` — workpc-side agent: captures microphone (PortAudio or ALSA `arecord`), streams over WebSocket to mavend
|
||||
- `internal/audio/` extended with capture types: `MicCapture`, `SystemCapture`, `FileCapture`
|
||||
- `internal/stt/stt.go` extended with `StreamingTranscriber` interface (or reuse existing with chunked input)
|
||||
- Router: new `IntentCapture` intent for start/stop commands
|
||||
- Reuses `internal/llm.Client` for summarization
|
||||
- Reuses `internal/voice/server.go` TCP protocol for streaming audio
|
||||
## What shipped
|
||||
|
||||
**Steps:**
|
||||
1. Create `cmd/mavheard/main.go` — workpc-side daemon: captures microphone via `arecord` pipe or PortAudio, opens WebSocket or TCP connection to mavend, streams PCM frames
|
||||
2. Create `internal/audio/capture.go` — `Capture` interface: `Start()`, `Stop()`, `AudioCh <-chan Audio`; implement `MicCapture` (reads from `mavheard` stream) and `FileCapture` (reads WAV)
|
||||
3. Extend `internal/stt/stt.go` — add `TranscribeStream(ctx, audio <-chan Audio) (string, error)` to `Transcriber` interface; `Stub` returns empty; `Remote` forwards chunks to worker socket
|
||||
4. Add `IntentCapture` to `internal/router/intent.go` — slots: `Action` ("start"/"stop"/"status"), `Duration`
|
||||
5. Wire capture handler in `cmd/mavend/voice.go:reactiveHandler` — start = spawn goroutine receiving audio, stream to STT; stop = finalize, send to LLM for summarization, write note via `WriteNote`
|
||||
6. Add capture config to `voice` block in `config.Config` — `{capture_enabled, capture_timeout}`
|
||||
7. Test with a recorded WAV file — simulate a meeting, verify transcription + summary note is created
|
||||
| Piece | Where |
|
||||
|---|---|
|
||||
| Session state machine: start / append / stop / abort / status | `internal/capture/capture.go` |
|
||||
| Map-reduce summarisation against `n_ctx` 4096 | `internal/capture/summarize.go` |
|
||||
| Audio blobs in the shared store, pruned by `media.retention` | `internal/media` (from #252) |
|
||||
| Config block `capture`, off by default | `internal/config/config.go` |
|
||||
| IPC `capture_start` / `capture_append` / `capture_stop` / `capture_status` | `internal/ipc/{wire,api,client,server}.go` |
|
||||
| Authority: the three write methods `AuthWrite`, status `AuthRead` | `internal/auth/policy.go` |
|
||||
| Daemon wiring, note write, STT reuse | `cmd/mavend/capture.go` |
|
||||
|
||||
The audio lands in the same content-addressed blob store as images, under the same retention
|
||||
loop, because #252 and #253 have the same intake problem and solving it twice would mean two
|
||||
directories to remember to prune.
|
||||
|
||||
## The refusals, and why
|
||||
|
||||
**Nothing listens.** The original step 8 called for capture "triggered by voice command
|
||||
(IntentCapture) **or configurable keyword ('maven record')**". The keyword half is refused.
|
||||
Noticing a keyword requires listening to the room continuously, which is precisely the
|
||||
behaviour this capability must not have, and the refusal is in the code rather than in a
|
||||
comment: `Recorder.Append` is the only way audio enters, and it returns `ErrNoSession` unless
|
||||
someone explicitly started a session. Audio arriving at an idle core is dropped, not buffered
|
||||
"just in case".
|
||||
|
||||
**Off unless configured, twice over.** No `media` block ⇒ nowhere to keep audio ⇒ the four
|
||||
methods do not exist. No `capture` block with `enabled: true` ⇒ they still do not exist. On an
|
||||
unconfigured box there is no wire path at all that begins a recording. That is the only
|
||||
guarantee worth making here, and it is the reason the hooks use the nil-hook ⇒
|
||||
`ErrUnknownMethod` pattern rather than an in-handler check.
|
||||
|
||||
**A forgotten session ends itself.** `max_minutes` defaults to 120 and is checked on every
|
||||
append, not on a timer that could be missed. Past the cap `Append` returns `ErrExpired`
|
||||
permanently, so a client that ignores the error cannot grow the recording; the audio collected
|
||||
before the cap is kept and `Stop` still works.
|
||||
|
||||
**"Забудь, не записывай" leaves nothing behind.** `capture_stop` with `discard: true` throws
|
||||
the session away without storing, transcribing or summarising anything — not a blob with a note
|
||||
saying it was abandoned. Nothing.
|
||||
|
||||
**The transcript is not saved by default.** The summary is written where he will read it; the
|
||||
verbatim record of what other people said in a room is a heavier thing to keep and takes a
|
||||
deliberate `save_transcript: true`. The audio blob is pruned by `media.retention` either way.
|
||||
|
||||
**No second STT.** Step 3 of the original plan extended the `Transcriber` interface with
|
||||
streaming. Not needed and not done: whisper.cpp already runs as `mavsttd`, and `internal/capture`
|
||||
takes the ordinary `stt.Transcriber` the voice path already holds (exposed as
|
||||
`voiceWiring.transcriber`). Long recordings are handed over in five-minute windows —
|
||||
`chunkAudio`, cut on sample boundaries — for the same reason whisper itself works in 30-second
|
||||
windows: an hour of PCM in one call either times out or blocks the voice path for minutes.
|
||||
Capture with voice off is refused rather than degraded, because storing hours of unreadable
|
||||
audio of other people is worse than not recording.
|
||||
|
||||
**Not `AuthStepUp`.** Recording people is invasive enough to argue for the top rung, and it is
|
||||
still wrong: step-up needs a passkey gesture, which the voice path cannot make, so
|
||||
"запиши встречу" could never work by voice — the only way he will actually use this. `AuthWrite`
|
||||
plus the off-unless-configured gate is the honest combination.
|
||||
|
||||
## Long audio against a 4096-token context
|
||||
|
||||
The resident model is a Thinking variant at `n_ctx` 4096, so an hour of transcript does not fit
|
||||
in one prompt and never will. `summarize.go` does map-reduce and nothing cleverer: split the
|
||||
transcript on sentence boundaries into 3000-rune windows (about 1100 Qwen tokens of Russian,
|
||||
leaving room for the persona block, the reasoning and the answer), summarise each, then
|
||||
summarise the summaries. A transcript that fits in one window skips the reduce step.
|
||||
|
||||
Truncation was the alternative and is rejected: a truncated meeting summary reads as complete
|
||||
and is not, and he would act on it. Past `max_chunks` (40, roughly the two-hour cap) the
|
||||
transcript *is* cut, and the summary says so in the note.
|
||||
|
||||
Two degradations are deliberate and both are reported rather than hidden:
|
||||
|
||||
- No llama-server ⇒ transcript, no summary. The words exist.
|
||||
- The reduce call fails ⇒ the per-chunk summaries are returned joined. Real work, not thrown
|
||||
away over the last call.
|
||||
|
||||
The map and reduce prompts contain no first person at all, so the persona's feminine-form rules
|
||||
have nothing to get wrong in them; the reply she actually gives him is phrased by the ordinary
|
||||
replier, which does carry the persona.
|
||||
|
||||
## Config
|
||||
|
||||
```json
|
||||
"media": { "dir": "media", "retention": "168h" },
|
||||
"capture": {
|
||||
"enabled": true,
|
||||
"max_minutes": 120,
|
||||
"stt_window": "5m",
|
||||
"chunk_runes": 3000,
|
||||
"max_chunks": 40,
|
||||
"save_transcript": false
|
||||
}
|
||||
```
|
||||
|
||||
Both absent by default. `capture` alone does nothing without `media`.
|
||||
|
||||
## Still open
|
||||
|
||||
- **`cmd/mavheard`** — the workpc-side microphone agent. Deferred, not refused: the core half
|
||||
is the part with the invariants in it, and a mic client is straightforward once there is a
|
||||
stable wire to stream at. It should be an explicit-start process, not a resident one, for the
|
||||
same reason the recorder has no keyword trigger. The four IPC methods are the wire it will
|
||||
use; `mavenclient` already has the mic plumbing to borrow.
|
||||
- **Router intent.** "запиши встречу" / "хватит" does not route anywhere yet. It needs the
|
||||
`system` intent plus slots, and it needs care: "хватит" is also how someone tells her to stop
|
||||
talking, so the recorder's stop and the speech barge-in must not collide.
|
||||
- **A `/dash` panel** showing a running session, so a recording is visible on a surface and not
|
||||
only in a log line.
|
||||
- **Speaker attribution** — who said what — is #255 and is blocked on a model; see
|
||||
`docs/plans/10-speaker-recognition.md`.
|
||||
|
||||
@@ -27,3 +27,36 @@
|
||||
6. Wire voice query — `"что я обычно делаю?"` routes to `IntentQuery` → behavior profile lookup → LLM-phrased answer
|
||||
7. Add IPC read method `MethodGetBehaviorProfile` so mavweb can display it on `/dash`
|
||||
8. Test with synthetic fact history — verify weekly schedule is correctly inferred
|
||||
|
||||
---
|
||||
|
||||
## Status (2026-08-01) — partially shipped, deliberately narrowed
|
||||
|
||||
Shipped on `overnight/behavior-profile`:
|
||||
|
||||
- `internal/memory/behavior.go` — `BuildProfile` counts habits per weekday out of
|
||||
self-facts: distinct-day counts (`MinHabitDays = 2`), a median time-of-day, and
|
||||
`FormatWeekdayRU` / `FormatOverallRU` for the spoken answer.
|
||||
- `internal/router/habit.go` — `ParseHabitQuery`, which requires a habit marker
|
||||
("обычно", "каждую", "привычки", …) and parses the weekday deterministically.
|
||||
- `cmd/mavend/actions_query.go` — a `habits` query source, so "что я обычно делаю
|
||||
по вторникам?" is answered.
|
||||
|
||||
**Not shipped, and not to be shipped as written:**
|
||||
|
||||
- *Step 3, LLM-generated profile stored as a fact.* The profile is COUNTED, not
|
||||
generated. A 1.7B asked to summarise a year of habits produces fluent claims
|
||||
about the owner's life that no row supports, and a wrong claim about him is the
|
||||
most expensive kind of wrong maven can be. Counting is verifiable and cheap.
|
||||
- *Step 5, incremental updates on fact write.* There is no cache to keep fresh —
|
||||
the profile is recomputed on the question, so a new fact is already in the next
|
||||
answer. A cached profile that can disagree with its own rows is two truths.
|
||||
- *Step 4, proactive daily plan proposals via the dispatcher.* Maven is not a nag,
|
||||
and a nudge at 08:00 every day proposing the day is the definition of one. The
|
||||
sanctioned path from "she noticed a pattern" to "she acts on it" already exists:
|
||||
`internal/pattern/detector.go` proposes a routine, and the owner accepts it on
|
||||
`/routines`. It goes through him.
|
||||
|
||||
Still open, if wanted later: `MethodGetBehaviorProfile` + a `/dash` panel (step 7).
|
||||
The counted profile needs no new IPC method to be *asked* about — the query source
|
||||
reads `RecentFacts` over the existing surface — so this is a display concern only.
|
||||
|
||||
@@ -27,3 +27,28 @@
|
||||
7. Add voice query handler — `"что нового?"` queries `RecentNotes` filtered by source prefix `rss:` and phrases via `phraser.PhraseQuery`
|
||||
8. Add `feeds` block to `config.Config` and `deploy/mavend.json`
|
||||
9. Test with a live RSS feed (e.g., `https://news.ycombinator.com/rss`) — verify items appear in notes table
|
||||
|
||||
---
|
||||
|
||||
## Shipped 2026-08-01 (#258)
|
||||
|
||||
`internal/webfetch` (the guarded HTTP door: scheme, allow/deny hosts, private-address
|
||||
refusal in the dialer, size cap, redirect cap, per-host rate limit), `internal/rss`
|
||||
(RSS 2.0 + Atom parser, poller with durable marks and a keyword filter),
|
||||
`cmd/mavend/feeds.go` (ticker, fetcher adapter, `rss:latest:<feed>` fact marks),
|
||||
config block `feeds`, and the `feeds` query source with `router.ParseFeedQuery`.
|
||||
|
||||
Deviations from the plan above, both deliberate:
|
||||
|
||||
- **Step 5 (breaking-news nudges) was not built.** A feed that dispatches is a nag,
|
||||
and the one thing Maven is not is a nag. Items are read when asked and nowhere else.
|
||||
If breaking news is ever wanted, it belongs behind the existing delivery policy
|
||||
(severity, quiet hours, digest), not in the poller.
|
||||
- **Step 3 (embedder relevance) is a seam, not an implementation.** `rss.Ranker`
|
||||
exists and is wired nil. Scoring items against an "interest profile" needs a
|
||||
profile, and there is none yet; a threshold with nothing to compare against is a
|
||||
random filter with a confident name. The filter that runs is the per-feed
|
||||
include/exclude keyword list, which he can read and predict.
|
||||
|
||||
No new dependency: stdlib `encoding/xml`, no gofeed. Stock deploy config has no
|
||||
`feeds` block, so the capability is off.
|
||||
|
||||
@@ -28,3 +28,41 @@
|
||||
7. Add IPC methods `MethodTriggerCrawl(name)`, `MethodListCrawls`, `MethodGetCrawlResult(name)`
|
||||
8. Add `crawls` block to `config.Config` and `deploy/mavend.json`
|
||||
9. Test with a static HTML page — verify extraction matches expected values, verify scheduling fires correctly
|
||||
|
||||
## Shipped 2026-08-01 (#259)
|
||||
|
||||
Built as `internal/crawl` (pure: robots, extraction, watcher) plus
|
||||
`cmd/mavend/crawls.go` (fetcher, ticker, dedup facts), on top of the guarded
|
||||
`internal/webfetch` door added with the feed reader (#258). Off unless
|
||||
configured, in two separately-switched halves: `crawl.on_demand` for a URL he
|
||||
names, `crawl.watches` for a scheduled re-read.
|
||||
|
||||
**Limits are code, not documentation** (`internal/webfetch`, tested one test per
|
||||
limit): host allowlist/denylist, no private addresses (loopback, RFC1918 —
|
||||
hence the LAN and the `10.42.0.0/24` wg range —, link-local incl. cloud
|
||||
metadata, CGNAT, v6 ULA) enforced in the dialer's `Control` hook so DNS
|
||||
rebinding and every redirect hop are covered, response size cap, redirect cap,
|
||||
timeout, one request per host per second. `robots.txt` is fetched first, cached
|
||||
per host, and a `Disallow` is refused with no override.
|
||||
|
||||
Deliberate deviations from the plan above:
|
||||
|
||||
- **No CSS selectors and no LLM structured extraction** (steps 2). The output is
|
||||
plaintext handed to the phraser as context for the question he asked. A 1.7B
|
||||
extracting a JSON price table from 4000 runes is a worse bet than reading, and
|
||||
`goquery` is not vendored.
|
||||
- **No `crawl` act verb and no new IPC methods** (steps 5, 7). Reading a page is
|
||||
a query source (`queryWeb` in `actions_query.go`, last in the chain, behind
|
||||
Kiwix once that is wired), not an action he commands. Nothing needs a new wire
|
||||
method to work.
|
||||
- **Notes, not facts.** A page's text is not a fact about him. Only the dedup
|
||||
hash is a fact (`crawl:hash:<name>`, kind `config`, source `poll:crawl`).
|
||||
- **Nothing is dispatched.** A changed page writes a note; it does not nudge.
|
||||
Not a nag.
|
||||
- **No `/tools` crawl history page.** The notes and the hash facts are already
|
||||
visible on `/dash`.
|
||||
|
||||
**No new dependency.** The vendored tree has no `x/net/html`, no `goquery` and
|
||||
no `temoto/robotstxt`, so robots parsing and HTML-to-text are stdlib
|
||||
(`regexp`, `html`) — RE2 has no backreferences, hence the `pairsRE` builder in
|
||||
`extract.go`.
|
||||
|
||||
@@ -393,3 +393,52 @@ func mustWriteFactParams(source string) []byte {
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// TestRequirement_SwapModel — loading a different resident model is an owner
|
||||
// action at the same rung as mutating the tool allowlist: it decides how every
|
||||
// utterance is routed and how every reply is worded. The read side is not.
|
||||
func TestRequirement_SwapModel(t *testing.T) {
|
||||
if got := Requirement(ipc.MethodSwapModel); got != AuthStepUp {
|
||||
t.Errorf("SwapModel authority = %v; want AuthStepUp", got)
|
||||
}
|
||||
if got := Requirement(ipc.MethodModelStatus); got != AuthRead {
|
||||
t.Errorf("ModelStatus authority = %v; want AuthRead", got)
|
||||
}
|
||||
// A surface that cannot carry a passkey gesture cannot swap the model, no
|
||||
// matter what it is enrolled as — this is the "never through voice" property.
|
||||
voice := Scope{Surface: SurfaceVoice, Module: "voice", SourceScope: []string{"*"}}
|
||||
if err := Can(ipc.MethodSwapModel, voice, nil); !errors.Is(err, ErrForbidden) {
|
||||
t.Errorf("voice swapping the model = %v; want ErrForbidden", err)
|
||||
}
|
||||
// And with no step-up session asserted, the gate refuses even a capable surface.
|
||||
noSession := &Gate{Enrollment: NewFloorEnrollment()}
|
||||
if err := noSession.Check(context.Background(), ipc.MethodSwapModel, nil); !errors.Is(err, ipc.ErrForbidden) {
|
||||
t.Errorf("SwapModel with no asserted step-up = %v; want ErrForbidden", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRequirement_Capture — recording other people is a write, not a read: it
|
||||
// puts audio of them on disk. The read side, "что ты записываешь?", is not.
|
||||
//
|
||||
// It is deliberately NOT AuthStepUp. Step-up needs a passkey gesture, which the
|
||||
// voice path cannot make, so putting it there would mean "запиши встречу" could
|
||||
// never work by voice. The real gate on this capability is that the methods do
|
||||
// not exist at all unless the operator enabled a capture block.
|
||||
func TestRequirement_Capture(t *testing.T) {
|
||||
for _, m := range []ipc.Method{
|
||||
ipc.MethodCaptureStart, ipc.MethodCaptureAppend, ipc.MethodCaptureStop,
|
||||
} {
|
||||
if got := Requirement(m); got != AuthWrite {
|
||||
t.Errorf("%s authority = %v; want AuthWrite", m, got)
|
||||
}
|
||||
}
|
||||
if got := Requirement(ipc.MethodCaptureStatus); got != AuthRead {
|
||||
t.Errorf("CaptureStatus authority = %v; want AuthRead", got)
|
||||
}
|
||||
// Voice can start one: it is the surface he will actually use to say
|
||||
// "запиши встречу", and it carries AuthWrite.
|
||||
voice := Scope{Surface: SurfaceVoice, Module: "voice", SourceScope: []string{"*"}}
|
||||
if err := Can(ipc.MethodCaptureStart, voice, nil); err != nil {
|
||||
t.Errorf("voice starting a capture = %v; want allowed", err)
|
||||
}
|
||||
}
|
||||
|
||||
+51
-1
@@ -53,6 +53,28 @@ func Requirement(m ipc.Method) Authority {
|
||||
// asserted — never a module or the voice/chat path. maven can propose
|
||||
// (MethodProposeTool, no step-up: she has no passkey) but never en/disable.
|
||||
return AuthStepUp
|
||||
case ipc.MethodSwapModel:
|
||||
// Swapping the resident model changes what routes every utterance and
|
||||
// what words every reply. It is the owner's call, from a surface that can
|
||||
// carry a passkey gesture — the same rung as mutating the tool allowlist,
|
||||
// and for the same reason: nothing Maven says or does may reach it.
|
||||
// MethodModelStatus is only the read side, so it stays at AuthRead.
|
||||
return AuthStepUp
|
||||
case ipc.MethodCaptureStart, ipc.MethodCaptureAppend, ipc.MethodCaptureStop:
|
||||
// Recording a meeting (Vikunja #253). AuthWrite, not AuthRead: it puts
|
||||
// audio of other people on disk, which is a heavier thing than reading a
|
||||
// fact, and it is not something a read-only surface should be able to
|
||||
// begin. Append and Stop sit on the same rung as Start deliberately —
|
||||
// a surface that may not start a recording has no business feeding or
|
||||
// harvesting one either.
|
||||
//
|
||||
// Not AuthStepUp, and this is the interesting line: step-up needs a
|
||||
// passkey gesture, which the voice path cannot make. Putting it here
|
||||
// would mean "запиши встречу" could never work by voice, and the real
|
||||
// gate on this capability is elsewhere and stronger — the methods do not
|
||||
// exist at all unless the operator enabled a capture block, and no
|
||||
// recording can begin without someone saying so.
|
||||
return AuthWrite
|
||||
case ipc.MethodWriteFact:
|
||||
return AuthWrite
|
||||
case ipc.MethodAssertStepUp:
|
||||
@@ -65,7 +87,35 @@ func Requirement(m ipc.Method) Authority {
|
||||
ipc.MethodCreateReminder,
|
||||
ipc.MethodMarkReminder,
|
||||
ipc.MethodRecordNudge,
|
||||
ipc.MethodResolveNudge:
|
||||
ipc.MethodResolveNudge,
|
||||
// Task capture (Vikunja #130). Listed explicitly rather than left to
|
||||
// the default so the intent is on the record: capturing a task is a
|
||||
// module write, not an allowlist mutation and not a new standing reason
|
||||
// for Maven to speak — nothing in the tick loop reads tasks. It stays
|
||||
// at AuthRead, the same rung as CreateReminder, which is the closest
|
||||
// existing analogue.
|
||||
ipc.MethodCaptureTask,
|
||||
ipc.MethodListTasks,
|
||||
ipc.MethodSetTaskStatus,
|
||||
// Mail ingestion (Vikunja #246). AuthRead because of what the method can
|
||||
// produce: candidate tasks and nothing else. It cannot write a fact, set a
|
||||
// reminder, or touch the tool allowlist, so a compromised mail reader can
|
||||
// at worst put junk on a review page he clears in one click.
|
||||
ipc.MethodIngestMail,
|
||||
// Looking at one image (Vikunja #252). AuthRead because of what it can
|
||||
// produce: words about a picture, and optionally a note. It cannot write
|
||||
// a fact, set a reminder, or touch the tool allowlist. The invasive part
|
||||
// of this capability is not the authority rung — it is that the bytes are
|
||||
// kept on disk, which media.retention bounds, and that they never leave
|
||||
// the box, which internal/vision enforces by refusing a non-private
|
||||
// endpoint.
|
||||
ipc.MethodDescribeImage,
|
||||
// "что ты записываешь?" — the read side of the recorder. It reports a
|
||||
// label, a start time and a byte count, begins nothing and keeps nothing.
|
||||
ipc.MethodCaptureStatus,
|
||||
// The read side of the model swap: which model is resident, which ones are
|
||||
// allowlisted. It loads nothing and changes nothing.
|
||||
ipc.MethodModelStatus:
|
||||
return AuthRead
|
||||
}
|
||||
// Unknown method ⇒ AuthRead, but ipc.dispatch returns ErrUnknownMethod
|
||||
|
||||
@@ -0,0 +1,404 @@
|
||||
// Package capture is Maven's meeting recorder (Vikunja #253,
|
||||
// docs/plans/08-hearing.md).
|
||||
//
|
||||
// One session at a time, with an explicit start and an explicit stop:
|
||||
//
|
||||
// Start("встреча") → audio frames appended → Stop() → transcript → summary
|
||||
//
|
||||
// # Nothing here listens
|
||||
//
|
||||
// This is the most invasive capability in the backlog and the design is
|
||||
// constrained accordingly. The constraints are the code, not a preamble:
|
||||
//
|
||||
// - There is no ambient path. `Session.Append` is the only way audio enters,
|
||||
// and it only accepts frames while a session someone started is running.
|
||||
// A keyword-triggered recorder ("maven record" heard in the room) was in the
|
||||
// plan document and is refused: it requires listening in order to notice the
|
||||
// keyword, which is the exact behaviour this capability must not have.
|
||||
// - A session that is not stopped stops itself. MaxDuration is a hard cap
|
||||
// checked on every Append, not a suggestion; a forgotten recording is a
|
||||
// recording that ends, not one that runs until the disk is full.
|
||||
// - Audio is stored under internal/media, which means retention prunes it and
|
||||
// it never leaves the box. Both the audio blob and the transcript stay
|
||||
// local; only the summary is written where he will read it.
|
||||
// - The transcript is never search input for anything outside this box. It is
|
||||
// text about a conversation with other people in it.
|
||||
//
|
||||
// # Long audio against a 4096-token context
|
||||
//
|
||||
// The resident model is a Thinking variant at n_ctx 4096, so an hour of meeting
|
||||
// transcript does not fit in one prompt and never will. summarize.go does the
|
||||
// obvious map-reduce: split the transcript on sentence boundaries into windows
|
||||
// that fit, summarise each, then summarise the summaries. That is handled
|
||||
// explicitly rather than by truncation, because a truncated meeting summary is
|
||||
// worse than none — it looks complete and is not.
|
||||
//
|
||||
// # Transcription
|
||||
//
|
||||
// There is exactly one STT in Maven and this package does not add a second: it
|
||||
// takes an stt.Transcriber, which in deploy is the whisper.cpp worker behind
|
||||
// cmd/mavsttd. Long audio is transcribed in windows too (see chunkAudio), for
|
||||
// the same reason whisper itself works in 30s windows — handing a worker an hour
|
||||
// of PCM in one call is a request that either times out or blocks everything
|
||||
// else for minutes.
|
||||
package capture
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/audio"
|
||||
"github.com/kami/maven/internal/media"
|
||||
"github.com/kami/maven/internal/stt"
|
||||
)
|
||||
|
||||
// DefaultMaxDuration — how long one capture may run before it stops itself.
|
||||
// Two hours covers a long meeting and bounds the damage of a forgotten session:
|
||||
// at 16 kHz mono that is about 230 MB of PCM, which is over media's default
|
||||
// per-blob cap, so a session at the limit is stored truncated rather than
|
||||
// refused. That trade is deliberate — a partial recording of a meeting he asked
|
||||
// for beats an error after two hours.
|
||||
const DefaultMaxDuration = 2 * time.Hour
|
||||
|
||||
// DefaultSTTWindow — how much audio goes to the transcriber in one call. Five
|
||||
// minutes of 16 kHz mono is under 10 MB, transcribes in well under whisper's
|
||||
// own timeout on this box, and keeps the worker responsive to the voice path
|
||||
// between windows.
|
||||
const DefaultSTTWindow = 5 * time.Minute
|
||||
|
||||
// Errors callers distinguish.
|
||||
var (
|
||||
// ErrDisabled — capture is not configured. A capability is off unless
|
||||
// configured, and a recorder most of all.
|
||||
ErrDisabled = errors.New("capture: not configured")
|
||||
// ErrBusy — a session is already running. One at a time: two concurrent
|
||||
// recordings would make "хватит" ambiguous.
|
||||
ErrBusy = errors.New("capture: a session is already running")
|
||||
// ErrNoSession — stop or append with nothing running.
|
||||
ErrNoSession = errors.New("capture: nothing is being recorded")
|
||||
// ErrBadFormat — a frame is not the canonical 16 kHz mono PCM shape.
|
||||
ErrBadFormat = errors.New("capture: audio format not supported")
|
||||
// ErrEmptyCapture — the session ended with no audio in it.
|
||||
ErrEmptyCapture = errors.New("capture: nothing was recorded")
|
||||
// ErrExpired — the session hit MaxDuration and was closed. Returned from
|
||||
// Append so the caller stops sending; the audio collected so far is kept.
|
||||
ErrExpired = errors.New("capture: session reached its time limit")
|
||||
)
|
||||
|
||||
// Session — one recording in progress. Not created directly; Recorder.Start
|
||||
// makes it. Guarded by a mutex because frames arrive from a network goroutine
|
||||
// while a status call may read from another.
|
||||
type Session struct {
|
||||
Label string
|
||||
Started time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
pcm []byte
|
||||
format audio.Format
|
||||
expired bool
|
||||
}
|
||||
|
||||
// Duration is how much audio has been collected, from the bytes rather than the
|
||||
// wall clock: a stream that dropped frames should report the audio that exists,
|
||||
// not the time that passed.
|
||||
func (s *Session) Duration() time.Duration {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.duration()
|
||||
}
|
||||
|
||||
func (s *Session) duration() time.Duration {
|
||||
a := audio.Audio{Format: s.format, Bytes: s.pcm}
|
||||
return time.Duration(a.Duration() * float64(time.Second))
|
||||
}
|
||||
|
||||
// Bytes is how much PCM has been collected. For a status line.
|
||||
func (s *Session) Bytes() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return len(s.pcm)
|
||||
}
|
||||
|
||||
// Status — what a "что записываешь?" answer needs, and what /dash shows. It is
|
||||
// the read side of a running session and is safe to ask for at any time.
|
||||
type Status struct {
|
||||
Running bool `json:"running"`
|
||||
Label string `json:"label,omitempty"`
|
||||
Started time.Time `json:"started,omitempty"`
|
||||
Duration time.Duration `json:"duration,omitempty"`
|
||||
Bytes int `json:"bytes,omitempty"`
|
||||
}
|
||||
|
||||
// Recorder owns the single session slot, the blob store and the two models a
|
||||
// finished capture needs. Build it with New; a zero Recorder is not usable.
|
||||
type Recorder struct {
|
||||
blobs *media.Store
|
||||
tr stt.Transcriber
|
||||
sum *Summarizer
|
||||
maxDuration time.Duration
|
||||
sttWindow time.Duration
|
||||
now func() time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
current *Session
|
||||
}
|
||||
|
||||
// Config — the recorder's knobs, built from config.CaptureConfig by the daemon.
|
||||
type Config struct {
|
||||
// MaxDuration — hard cap on one session. 0 ⇒ DefaultMaxDuration.
|
||||
MaxDuration time.Duration
|
||||
// STTWindow — audio per transcription call. 0 ⇒ DefaultSTTWindow.
|
||||
STTWindow time.Duration
|
||||
}
|
||||
|
||||
// New builds a Recorder. blobs and tr are required — a recorder with nowhere to
|
||||
// put the audio, or nothing to transcribe it with, is not a recorder. sum may be
|
||||
// nil: the transcript is still produced and stored, and the summary is simply
|
||||
// absent, which is the honest degradation when there is no llama-server.
|
||||
func New(blobs *media.Store, tr stt.Transcriber, sum *Summarizer, cfg Config) (*Recorder, error) {
|
||||
if blobs == nil {
|
||||
return nil, errors.New("capture: no blob store")
|
||||
}
|
||||
if tr == nil {
|
||||
return nil, errors.New("capture: no transcriber")
|
||||
}
|
||||
maxDur := cfg.MaxDuration
|
||||
if maxDur <= 0 {
|
||||
maxDur = DefaultMaxDuration
|
||||
}
|
||||
window := cfg.STTWindow
|
||||
if window <= 0 {
|
||||
window = DefaultSTTWindow
|
||||
}
|
||||
return &Recorder{
|
||||
blobs: blobs,
|
||||
tr: tr,
|
||||
sum: sum,
|
||||
maxDuration: maxDur,
|
||||
sttWindow: window,
|
||||
now: time.Now,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// MaxDuration is the configured hard cap. For the reply that tells him how long
|
||||
// she will keep going if he forgets to say "хватит".
|
||||
func (r *Recorder) MaxDuration() time.Duration { return r.maxDuration }
|
||||
|
||||
// Start opens a session. label is what the meeting is called ("встреча с
|
||||
// подрядчиком"); it ends up in the summary note so the note is findable.
|
||||
// ErrBusy if one is already running — the caller says so rather than silently
|
||||
// discarding the first recording.
|
||||
func (r *Recorder) Start(label string) (*Session, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.current != nil {
|
||||
return nil, fmt.Errorf("%w: %q since %s", ErrBusy, r.current.Label,
|
||||
r.current.Started.Format(time.Kitchen))
|
||||
}
|
||||
s := &Session{
|
||||
Label: strings.TrimSpace(label),
|
||||
Started: r.now().UTC(),
|
||||
format: audio.PCM16kMono,
|
||||
}
|
||||
r.current = s
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Append adds one frame to the running session. ErrNoSession when nothing is
|
||||
// running, which is the guard that makes an ambient path impossible: a stream
|
||||
// arriving at a Recorder nobody started is refused frame by frame.
|
||||
//
|
||||
// ErrExpired once the session is at MaxDuration. The audio collected so far is
|
||||
// kept and Stop still works — the cap ends the recording, it does not throw it
|
||||
// away.
|
||||
func (r *Recorder) Append(a audio.Audio) error {
|
||||
if !a.Format.IsValid() {
|
||||
return fmt.Errorf("%w: %+v", ErrBadFormat, a.Format)
|
||||
}
|
||||
r.mu.Lock()
|
||||
s := r.current
|
||||
r.mu.Unlock()
|
||||
if s == nil {
|
||||
return ErrNoSession
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.expired {
|
||||
return ErrExpired
|
||||
}
|
||||
s.pcm = append(s.pcm, a.Bytes...)
|
||||
if s.duration() >= r.maxDuration {
|
||||
s.expired = true
|
||||
return ErrExpired
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Status reports the running session, or Running=false.
|
||||
func (r *Recorder) Status() Status {
|
||||
r.mu.Lock()
|
||||
s := r.current
|
||||
r.mu.Unlock()
|
||||
if s == nil {
|
||||
return Status{}
|
||||
}
|
||||
return Status{
|
||||
Running: true,
|
||||
Label: s.Label,
|
||||
Started: s.Started,
|
||||
Duration: s.Duration(),
|
||||
Bytes: s.Bytes(),
|
||||
}
|
||||
}
|
||||
|
||||
// Result — a finished capture.
|
||||
type Result struct {
|
||||
// BlobID — the stored audio, content-addressed. Empty only if storing failed.
|
||||
BlobID string
|
||||
// Label / Started / Duration — what was recorded and when.
|
||||
Label string
|
||||
Started time.Time
|
||||
Duration time.Duration
|
||||
// Transcript — the full text, joined across STT windows.
|
||||
Transcript string
|
||||
// Summary — the map-reduced summary, or empty when no summarizer was wired
|
||||
// or the model failed. Empty summary with a non-empty transcript is a
|
||||
// degraded success, not a failure: the words are there.
|
||||
Summary string
|
||||
// Chunks — how many windows the transcript was summarised in. 1 means it fit
|
||||
// in one prompt. Reported so a suspiciously vague summary can be explained.
|
||||
Chunks int
|
||||
}
|
||||
|
||||
// Stop ends the session and produces the result: store the audio, transcribe it
|
||||
// in windows, summarise it in windows. The session slot is freed before any of
|
||||
// the slow work starts, so a stuck model cannot block the next recording.
|
||||
//
|
||||
// The order matters and is the same as vision's: the audio is stored FIRST. If
|
||||
// transcription or summarisation fails, the recording is still on disk and can
|
||||
// be run again; a meeting that happened once must not be lost to a model error.
|
||||
func (r *Recorder) Stop(ctx context.Context) (Result, error) {
|
||||
r.mu.Lock()
|
||||
s := r.current
|
||||
r.current = nil
|
||||
r.mu.Unlock()
|
||||
if s == nil {
|
||||
return Result{}, ErrNoSession
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
pcm := s.pcm
|
||||
format := s.format
|
||||
s.mu.Unlock()
|
||||
|
||||
res := Result{Label: s.Label, Started: s.Started}
|
||||
if len(pcm) == 0 {
|
||||
return res, ErrEmptyCapture
|
||||
}
|
||||
full := audio.Audio{Format: format, Bytes: pcm}
|
||||
res.Duration = time.Duration(full.Duration() * float64(time.Second))
|
||||
|
||||
// Stored as WAV, not headerless PCM: a blob on disk that `aplay` and whisper
|
||||
// can both open without being told the format is worth 44 bytes.
|
||||
wav, err := audio.WAVFromPCM(format, pcm)
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("capture: wav: %w", err)
|
||||
}
|
||||
blob, err := r.blobs.Put(media.KindAudio, "audio/wav", "capture:meeting", wav)
|
||||
if err != nil {
|
||||
// Over the per-blob cap is the expected case for a very long meeting.
|
||||
// Report it and keep going: a transcript without the audio still beats
|
||||
// nothing, and the words are what he will read.
|
||||
return res, fmt.Errorf("capture: store audio: %w", err)
|
||||
}
|
||||
res.BlobID = blob.ID
|
||||
|
||||
text, err := r.transcribe(ctx, full)
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("capture: transcribe: %w", err)
|
||||
}
|
||||
res.Transcript = text
|
||||
if strings.TrimSpace(text) == "" {
|
||||
return res, ErrEmptyCapture
|
||||
}
|
||||
|
||||
if r.sum == nil {
|
||||
return res, nil
|
||||
}
|
||||
summary, chunks, err := r.sum.Summarize(ctx, s.Label, text)
|
||||
res.Chunks = chunks
|
||||
if err != nil {
|
||||
// Degraded success: the transcript is real and stored, only the summary
|
||||
// is missing. The caller writes the transcript note and says so.
|
||||
return res, fmt.Errorf("capture: summarize: %w", err)
|
||||
}
|
||||
res.Summary = summary
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// Abort throws the running session away without transcribing or storing it.
|
||||
// This is what "забудь, не записывай" must map to: a recording someone changed
|
||||
// their mind about leaves nothing behind, not a blob with a note saying it was
|
||||
// abandoned. Returns whether anything was running.
|
||||
func (r *Recorder) Abort() bool {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.current == nil {
|
||||
return false
|
||||
}
|
||||
r.current = nil
|
||||
return true
|
||||
}
|
||||
|
||||
// transcribe runs the transcriber over the audio in windows and joins the text.
|
||||
// A window that fails is fatal: a summary of a meeting with a silent hole in the
|
||||
// middle is a summary that misleads.
|
||||
func (r *Recorder) transcribe(ctx context.Context, a audio.Audio) (string, error) {
|
||||
windows := chunkAudio(a, r.sttWindow)
|
||||
parts := make([]string, 0, len(windows))
|
||||
for i, w := range windows {
|
||||
text, _, err := r.tr.Transcribe(ctx, w)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("window %d/%d: %w", i+1, len(windows), err)
|
||||
}
|
||||
if t := strings.TrimSpace(text); t != "" {
|
||||
parts = append(parts, t)
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, " "), nil
|
||||
}
|
||||
|
||||
// chunkAudio splits audio into windows of at most window duration, cut on
|
||||
// sample boundaries. A window shorter than one sample is impossible; audio
|
||||
// shorter than one window comes back as a single element, so the caller never
|
||||
// special-cases the short case.
|
||||
func chunkAudio(a audio.Audio, window time.Duration) []audio.Audio {
|
||||
bytesPerSample := a.Format.SampleBits / 8 * a.Format.Channels
|
||||
if bytesPerSample <= 0 || a.Format.SampleRate <= 0 || window <= 0 {
|
||||
return []audio.Audio{a}
|
||||
}
|
||||
per := int(window.Seconds()) * a.Format.SampleRate * bytesPerSample
|
||||
if per <= 0 || len(a.Bytes) <= per {
|
||||
return []audio.Audio{a}
|
||||
}
|
||||
var out []audio.Audio
|
||||
for off := 0; off < len(a.Bytes); off += per {
|
||||
end := off + per
|
||||
if end > len(a.Bytes) {
|
||||
end = len(a.Bytes)
|
||||
}
|
||||
// Never cut mid-sample: a split inside an int16 shifts every following
|
||||
// sample by a byte and turns the tail of the window into noise.
|
||||
end -= (end - off) % bytesPerSample
|
||||
if end <= off {
|
||||
break
|
||||
}
|
||||
out = append(out, audio.Audio{Format: a.Format, Bytes: a.Bytes[off:end]})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,363 @@
|
||||
package capture
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/audio"
|
||||
"github.com/kami/maven/internal/media"
|
||||
)
|
||||
|
||||
// fakeTranscriber returns a fixed phrase per call so a windowed transcription is
|
||||
// visible in the joined output.
|
||||
type fakeTranscriber struct {
|
||||
calls int
|
||||
err error
|
||||
phrase string
|
||||
}
|
||||
|
||||
func (f *fakeTranscriber) Transcribe(_ context.Context, a audio.Audio) (string, float64, error) {
|
||||
f.calls++
|
||||
if f.err != nil {
|
||||
return "", 0, f.err
|
||||
}
|
||||
p := f.phrase
|
||||
if p == "" {
|
||||
p = "окно"
|
||||
}
|
||||
return fmt.Sprintf("%s%d", p, f.calls), 1.0, nil
|
||||
}
|
||||
|
||||
// fakeCompleter records prompts and replies from a script.
|
||||
type fakeCompleter struct {
|
||||
replies []string
|
||||
systems []string
|
||||
users []string
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeCompleter) Complete(_ context.Context, system, user string) (string, error) {
|
||||
f.systems = append(f.systems, system)
|
||||
f.users = append(f.users, user)
|
||||
if f.err != nil {
|
||||
return "", f.err
|
||||
}
|
||||
if len(f.replies) == 0 {
|
||||
return "итог", nil
|
||||
}
|
||||
r := f.replies[0]
|
||||
f.replies = f.replies[1:]
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// frame builds n seconds of silence in the canonical format.
|
||||
func frame(seconds float64) audio.Audio {
|
||||
n := int(seconds*16000) * 2
|
||||
return audio.Audio{Format: audio.PCM16kMono, Bytes: make([]byte, n)}
|
||||
}
|
||||
|
||||
func testRecorder(t *testing.T, tr *fakeTranscriber, sum *Summarizer, cfg Config) (*Recorder, *media.Store) {
|
||||
t.Helper()
|
||||
blobs, err := media.Open(t.TempDir(), 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r, err := New(blobs, tr, sum, cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return r, blobs
|
||||
}
|
||||
|
||||
func TestNewRequiresStoreAndTranscriber(t *testing.T) {
|
||||
blobs, err := media.Open(t.TempDir(), 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := New(nil, &fakeTranscriber{}, nil, Config{}); err == nil {
|
||||
t.Error("recorder built with no blob store")
|
||||
}
|
||||
if _, err := New(blobs, nil, nil, Config{}); err == nil {
|
||||
t.Error("recorder built with no transcriber")
|
||||
}
|
||||
}
|
||||
|
||||
// The invariant that matters most: audio arriving at a recorder nobody started
|
||||
// is refused. There is no ambient path in.
|
||||
func TestAppendWithoutStartIsRefused(t *testing.T) {
|
||||
r, _ := testRecorder(t, &fakeTranscriber{}, nil, Config{})
|
||||
if err := r.Append(frame(1)); !errors.Is(err, ErrNoSession) {
|
||||
t.Fatalf("got %v, want ErrNoSession", err)
|
||||
}
|
||||
if r.Status().Running {
|
||||
t.Error("a refused frame started a session")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStopWithoutStartIsRefused(t *testing.T) {
|
||||
r, _ := testRecorder(t, &fakeTranscriber{}, nil, Config{})
|
||||
if _, err := r.Stop(context.Background()); !errors.Is(err, ErrNoSession) {
|
||||
t.Fatalf("got %v, want ErrNoSession", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOneSessionAtATime(t *testing.T) {
|
||||
r, _ := testRecorder(t, &fakeTranscriber{}, nil, Config{})
|
||||
if _, err := r.Start("встреча"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := r.Start("вторая"); !errors.Is(err, ErrBusy) {
|
||||
t.Fatalf("got %v, want ErrBusy", err)
|
||||
}
|
||||
if _, err := r.Stop(context.Background()); !errors.Is(err, ErrEmptyCapture) {
|
||||
t.Fatalf("empty stop: %v", err)
|
||||
}
|
||||
// The slot is free again after a stop, even a failed one.
|
||||
if _, err := r.Start("третья"); err != nil {
|
||||
t.Errorf("slot not released: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoundTripStoresAudioTranscriptAndSummary(t *testing.T) {
|
||||
tr := &fakeTranscriber{phrase: "совещание"}
|
||||
sum := NewSummarizer(&fakeCompleter{replies: []string{"— решили купить насос"}}, 0, 0, nil)
|
||||
r, blobs := testRecorder(t, tr, sum, Config{})
|
||||
|
||||
if _, err := r.Start("встреча с подрядчиком"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 3; i++ {
|
||||
if err := r.Append(frame(2)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
res, err := r.Stop(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("stop: %v", err)
|
||||
}
|
||||
if res.BlobID == "" {
|
||||
t.Error("no audio blob stored")
|
||||
}
|
||||
blob, data, err := blobs.Read(res.BlobID)
|
||||
if err != nil {
|
||||
t.Fatalf("blob unreadable: %v", err)
|
||||
}
|
||||
if blob.Kind != media.KindAudio || blob.Source != "capture:meeting" {
|
||||
t.Errorf("blob metadata = %+v", blob)
|
||||
}
|
||||
if string(data[:4]) != "RIFF" {
|
||||
t.Error("audio was not stored as a playable WAV")
|
||||
}
|
||||
if res.Transcript == "" {
|
||||
t.Error("no transcript")
|
||||
}
|
||||
if !strings.Contains(res.Summary, "насос") {
|
||||
t.Errorf("summary = %q", res.Summary)
|
||||
}
|
||||
if !strings.Contains(res.Summary, "встреча с подрядчиком") {
|
||||
t.Errorf("label missing from summary: %q", res.Summary)
|
||||
}
|
||||
if res.Duration != 6*time.Second {
|
||||
t.Errorf("duration = %v, want 6s", res.Duration)
|
||||
}
|
||||
}
|
||||
|
||||
// A forgotten session stops itself, and the audio collected before the cap is
|
||||
// kept rather than thrown away.
|
||||
func TestMaxDurationEndsTheSessionAndKeepsAudio(t *testing.T) {
|
||||
tr := &fakeTranscriber{}
|
||||
r, _ := testRecorder(t, tr, nil, Config{MaxDuration: 4 * time.Second})
|
||||
if _, err := r.Start("длинная"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.Append(frame(3)); err != nil {
|
||||
t.Fatalf("first frame: %v", err)
|
||||
}
|
||||
if err := r.Append(frame(3)); !errors.Is(err, ErrExpired) {
|
||||
t.Fatalf("got %v, want ErrExpired", err)
|
||||
}
|
||||
// Further frames keep being refused, so a client that ignores the error
|
||||
// cannot grow the recording past the cap.
|
||||
if err := r.Append(frame(3)); !errors.Is(err, ErrExpired) {
|
||||
t.Fatalf("post-expiry frame: %v", err)
|
||||
}
|
||||
res, err := r.Stop(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("stop after expiry: %v", err)
|
||||
}
|
||||
if res.Duration != 6*time.Second {
|
||||
t.Errorf("duration = %v, want the 6s collected before the cap", res.Duration)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppendRejectsWrongFormat(t *testing.T) {
|
||||
r, _ := testRecorder(t, &fakeTranscriber{}, nil, Config{})
|
||||
if _, err := r.Start("x"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bad := audio.Audio{Format: audio.Format{SampleRate: 44100, Channels: 2, SampleBits: 16, Encoding: "pcm_s16le"}, Bytes: make([]byte, 100)}
|
||||
if err := r.Append(bad); !errors.Is(err, ErrBadFormat) {
|
||||
t.Fatalf("got %v, want ErrBadFormat", err)
|
||||
}
|
||||
}
|
||||
|
||||
// "забудь, не записывай" must leave nothing behind — no blob, no transcript.
|
||||
func TestAbortLeavesNothing(t *testing.T) {
|
||||
tr := &fakeTranscriber{}
|
||||
r, blobs := testRecorder(t, tr, nil, Config{})
|
||||
if _, err := r.Start("зря начали"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.Append(frame(5)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !r.Abort() {
|
||||
t.Fatal("Abort reported nothing running")
|
||||
}
|
||||
if r.Status().Running {
|
||||
t.Error("session survived Abort")
|
||||
}
|
||||
list, err := blobs.List(media.KindAudio)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(list) != 0 {
|
||||
t.Errorf("Abort stored %d blob(s)", len(list))
|
||||
}
|
||||
if tr.calls != 0 {
|
||||
t.Errorf("Abort transcribed anyway (%d calls)", tr.calls)
|
||||
}
|
||||
if r.Abort() {
|
||||
t.Error("second Abort reported a session")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatusReportsTheRunningSession(t *testing.T) {
|
||||
r, _ := testRecorder(t, &fakeTranscriber{}, nil, Config{})
|
||||
if got := r.Status(); got.Running {
|
||||
t.Error("idle recorder reports running")
|
||||
}
|
||||
if _, err := r.Start("планёрка"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.Append(frame(10)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st := r.Status()
|
||||
if !st.Running || st.Label != "планёрка" {
|
||||
t.Fatalf("status = %+v", st)
|
||||
}
|
||||
if st.Duration != 10*time.Second {
|
||||
t.Errorf("duration = %v", st.Duration)
|
||||
}
|
||||
if st.Bytes != 10*16000*2 {
|
||||
t.Errorf("bytes = %d", st.Bytes)
|
||||
}
|
||||
}
|
||||
|
||||
// Long audio goes to the transcriber in windows: handing a whisper worker an
|
||||
// hour of PCM in one call blocks the voice path for minutes.
|
||||
func TestLongAudioIsTranscribedInWindows(t *testing.T) {
|
||||
tr := &fakeTranscriber{}
|
||||
r, _ := testRecorder(t, tr, nil, Config{STTWindow: 2 * time.Second})
|
||||
if _, err := r.Start("длинная"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.Append(frame(9)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
res, err := r.Stop(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("stop: %v", err)
|
||||
}
|
||||
if tr.calls != 5 { // 2+2+2+2+1
|
||||
t.Errorf("transcriber called %d times, want 5", tr.calls)
|
||||
}
|
||||
if !strings.Contains(res.Transcript, "окно5") {
|
||||
t.Errorf("last window missing from transcript: %q", res.Transcript)
|
||||
}
|
||||
}
|
||||
|
||||
// A hole in the middle of a meeting summary would mislead, so a failed window is
|
||||
// fatal — but the audio is already stored and re-runnable.
|
||||
func TestTranscriptionFailureKeepsTheAudio(t *testing.T) {
|
||||
tr := &fakeTranscriber{err: errors.New("whisper is down")}
|
||||
r, blobs := testRecorder(t, tr, nil, Config{})
|
||||
if _, err := r.Start("встреча"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.Append(frame(2)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
res, err := r.Stop(context.Background())
|
||||
if err == nil {
|
||||
t.Fatal("transcription failure was not reported")
|
||||
}
|
||||
if res.BlobID == "" {
|
||||
t.Fatal("no blob id to retry with")
|
||||
}
|
||||
if _, _, err := blobs.Read(res.BlobID); err != nil {
|
||||
t.Errorf("audio was not kept: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// No llama-server ⇒ transcript only. That is the honest degradation, not an
|
||||
// error.
|
||||
func TestNoSummarizerStillProducesATranscript(t *testing.T) {
|
||||
r, _ := testRecorder(t, &fakeTranscriber{}, nil, Config{})
|
||||
if _, err := r.Start("встреча"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.Append(frame(1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
res, err := r.Stop(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("stop: %v", err)
|
||||
}
|
||||
if res.Transcript == "" {
|
||||
t.Error("no transcript")
|
||||
}
|
||||
if res.Summary != "" {
|
||||
t.Errorf("summary appeared from nowhere: %q", res.Summary)
|
||||
}
|
||||
}
|
||||
|
||||
// A summariser failure is a degraded success: the words exist and are returned.
|
||||
func TestSummaryFailureStillReturnsTheTranscript(t *testing.T) {
|
||||
sum := NewSummarizer(&fakeCompleter{err: errors.New("llama is down")}, 0, 0, nil)
|
||||
r, _ := testRecorder(t, &fakeTranscriber{}, sum, Config{})
|
||||
if _, err := r.Start("встреча"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.Append(frame(1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
res, err := r.Stop(context.Background())
|
||||
if err == nil {
|
||||
t.Fatal("summary failure was not reported")
|
||||
}
|
||||
if res.Transcript == "" {
|
||||
t.Error("transcript lost to a summary failure")
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkAudioNeverCutsMidSample(t *testing.T) {
|
||||
a := audio.Audio{Format: audio.PCM16kMono, Bytes: make([]byte, 16000*2*5+1)}
|
||||
for _, w := range chunkAudio(a, 2*time.Second) {
|
||||
if len(w.Bytes)%2 != 0 {
|
||||
t.Fatalf("window of %d bytes cuts an int16 in half", len(w.Bytes))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkAudioShortInputIsOneWindow(t *testing.T) {
|
||||
a := frame(1)
|
||||
if got := chunkAudio(a, time.Minute); len(got) != 1 {
|
||||
t.Errorf("got %d windows, want 1", len(got))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,261 @@
|
||||
package capture
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// DefaultChunkRunes — how much transcript goes into one summarisation prompt.
|
||||
//
|
||||
// The resident model runs at n_ctx 4096 and is a Thinking variant, so reasoning
|
||||
// tokens need room too. Russian runs roughly 2.5–3 characters per token on a
|
||||
// Qwen tokenizer, so 3000 runes is about 1100 tokens of transcript, leaving the
|
||||
// prompt, the persona block, the reasoning and the answer comfortable space.
|
||||
// This is the same reasoning internal/crawl used to land on 4000 runes, tightened
|
||||
// because a meeting transcript is denser in named entities than a web page and
|
||||
// the reduce step has to fit several summaries at once.
|
||||
const DefaultChunkRunes = 3000
|
||||
|
||||
// DefaultMaxChunks — how many windows one meeting may be summarised in. Forty
|
||||
// chunks at 3000 runes is roughly a two-hour meeting, which is MaxDuration; past
|
||||
// that the transcript is truncated and the summary says so, because forty-one
|
||||
// sequential model calls on this box is half an hour of work nobody is waiting
|
||||
// through.
|
||||
const DefaultMaxChunks = 40
|
||||
|
||||
// ErrNoSummary — the model returned nothing usable for every chunk.
|
||||
var ErrNoSummary = errors.New("capture: model produced no summary")
|
||||
|
||||
// Completer is the one thing the summarizer needs from a model: text in, text
|
||||
// out. It is an interface rather than an *llm.Client so this package stays pure
|
||||
// and testable, and so the daemon can pass whatever it already has.
|
||||
type Completer interface {
|
||||
Complete(ctx context.Context, system, user string) (string, error)
|
||||
}
|
||||
|
||||
// Summarizer turns a transcript into something worth reading. It is map-reduce
|
||||
// and nothing cleverer: summarise each window, then summarise the summaries.
|
||||
//
|
||||
// Truncation was the alternative and is rejected. A truncated meeting summary
|
||||
// reads as complete and is not, which is worse than no summary at all — he would
|
||||
// act on it.
|
||||
type Summarizer struct {
|
||||
llm Completer
|
||||
chunkRunes int
|
||||
maxChunks int
|
||||
// context is the persona/context block the daemon prepends to every prompt,
|
||||
// or empty. Passed in rather than built here so this package does not import
|
||||
// internal/persona and the feminine self-reference rules stay in one place.
|
||||
context func() string
|
||||
}
|
||||
|
||||
// NewSummarizer wires a summarizer. llm nil ⇒ nil Summarizer, which Recorder
|
||||
// treats as "transcript only", the honest degradation with no llama-server.
|
||||
// chunkRunes ≤ 0 ⇒ DefaultChunkRunes; maxChunks ≤ 0 ⇒ DefaultMaxChunks.
|
||||
func NewSummarizer(llm Completer, chunkRunes, maxChunks int, contextBlock func() string) *Summarizer {
|
||||
if llm == nil {
|
||||
return nil
|
||||
}
|
||||
if chunkRunes <= 0 {
|
||||
chunkRunes = DefaultChunkRunes
|
||||
}
|
||||
if maxChunks <= 0 {
|
||||
maxChunks = DefaultMaxChunks
|
||||
}
|
||||
if contextBlock == nil {
|
||||
contextBlock = func() string { return "" }
|
||||
}
|
||||
return &Summarizer{llm: llm, chunkRunes: chunkRunes, maxChunks: maxChunks, context: contextBlock}
|
||||
}
|
||||
|
||||
// chunkPrompt — the map step. Deliberately plain: this is not Maven speaking to
|
||||
// him, it is a model condensing text, so there is no first person in it at all
|
||||
// and therefore nothing for the persona's gender rules to get wrong. The reply
|
||||
// she gives him afterwards is phrased by the ordinary replier, which does carry
|
||||
// the persona.
|
||||
const chunkPrompt = `Ты обрабатываешь фрагмент расшифровки разговора.
|
||||
Сожми его до 2-4 пунктов: о чём говорили, какие решения приняли, какие задачи назвали.
|
||||
Без вступлений и выводов. Только по тексту — не придумывай того, чего в нём нет.
|
||||
Если во фрагменте нет ничего содержательного, ответь одним словом: пусто.`
|
||||
|
||||
// reducePrompt — the reduce step. Same rules, over the chunk summaries.
|
||||
const reducePrompt = `Ниже — конспекты фрагментов одной встречи, по порядку.
|
||||
Собери из них один короткий итог: о чём была встреча, какие решения приняли, что кому делать.
|
||||
Не повторяйся, не придумывай, не добавляй вступлений.`
|
||||
|
||||
// emptyMarker — what the map step answers for a chunk with nothing in it. Such
|
||||
// chunks are dropped before the reduce step rather than padding it with noise.
|
||||
const emptyMarker = "пусто"
|
||||
|
||||
// Summarize returns the summary and the number of chunks the transcript was
|
||||
// split into. One chunk means it fit in a single prompt and the reduce step was
|
||||
// skipped, which is the common case for a short meeting and saves a model call.
|
||||
func (s *Summarizer) Summarize(ctx context.Context, label, transcript string) (string, int, error) {
|
||||
if s == nil {
|
||||
return "", 0, ErrDisabled
|
||||
}
|
||||
chunks := ChunkText(transcript, s.chunkRunes)
|
||||
if len(chunks) == 0 {
|
||||
return "", 0, ErrEmptyCapture
|
||||
}
|
||||
truncated := false
|
||||
if len(chunks) > s.maxChunks {
|
||||
chunks = chunks[:s.maxChunks]
|
||||
truncated = true
|
||||
}
|
||||
|
||||
system := s.context() + chunkPrompt
|
||||
parts := make([]string, 0, len(chunks))
|
||||
for i, c := range chunks {
|
||||
out, err := s.llm.Complete(ctx, system, c)
|
||||
if err != nil {
|
||||
return "", len(chunks), fmt.Errorf("chunk %d/%d: %w", i+1, len(chunks), err)
|
||||
}
|
||||
out = strings.TrimSpace(out)
|
||||
if out == "" || strings.EqualFold(out, emptyMarker) {
|
||||
continue
|
||||
}
|
||||
parts = append(parts, out)
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "", len(chunks), ErrNoSummary
|
||||
}
|
||||
|
||||
summary := parts[0]
|
||||
if len(parts) > 1 {
|
||||
joined := strings.Join(parts, "\n\n")
|
||||
reduced, err := s.llm.Complete(ctx, s.context()+reducePrompt, joined)
|
||||
if err != nil {
|
||||
// The per-chunk summaries are real work; hand them over rather than
|
||||
// losing them to a failure in the last step.
|
||||
return joined, len(chunks), fmt.Errorf("reduce: %w", err)
|
||||
}
|
||||
if r := strings.TrimSpace(reduced); r != "" {
|
||||
summary = r
|
||||
} else {
|
||||
summary = joined
|
||||
}
|
||||
}
|
||||
if label != "" {
|
||||
summary = label + "\n\n" + summary
|
||||
}
|
||||
if truncated {
|
||||
// Said in the note, not swallowed: a summary that silently covers the
|
||||
// first hour of a three-hour meeting is the failure mode this guards.
|
||||
summary += fmt.Sprintf("\n\n(расшифровка обрезана: обработано %d фрагментов из большего числа)", s.maxChunks)
|
||||
}
|
||||
return summary, len(chunks), nil
|
||||
}
|
||||
|
||||
// ChunkText splits text into windows of at most maxRunes runes, cutting on
|
||||
// sentence boundaries where it can and on a word boundary otherwise. Exported
|
||||
// because it is the part worth testing on its own and the part a future
|
||||
// transcript viewer will want.
|
||||
//
|
||||
// A sentence longer than maxRunes (a transcript with no punctuation at all,
|
||||
// which whisper does produce) is cut on whitespace rather than dropped or run
|
||||
// past the limit.
|
||||
func ChunkText(text string, maxRunes int) []string {
|
||||
text = strings.TrimSpace(text)
|
||||
if text == "" {
|
||||
return nil
|
||||
}
|
||||
if maxRunes <= 0 {
|
||||
maxRunes = DefaultChunkRunes
|
||||
}
|
||||
if len([]rune(text)) <= maxRunes {
|
||||
return []string{text}
|
||||
}
|
||||
|
||||
var out []string
|
||||
var cur []rune
|
||||
flush := func() {
|
||||
if s := strings.TrimSpace(string(cur)); s != "" {
|
||||
out = append(out, s)
|
||||
}
|
||||
cur = cur[:0]
|
||||
}
|
||||
for _, sent := range splitSentences(text) {
|
||||
sr := []rune(sent)
|
||||
if len(sr) > maxRunes {
|
||||
// Oversized sentence: emit what is buffered, then cut this one on
|
||||
// word boundaries.
|
||||
flush()
|
||||
for _, piece := range splitWords(sr, maxRunes) {
|
||||
out = append(out, piece)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if len(cur)+len(sr) > maxRunes {
|
||||
flush()
|
||||
}
|
||||
cur = append(cur, sr...)
|
||||
}
|
||||
flush()
|
||||
return out
|
||||
}
|
||||
|
||||
// splitSentences cuts on sentence-ending punctuation followed by a space,
|
||||
// keeping the punctuation with the sentence it ends. Good enough for a
|
||||
// transcript: whisper emits periods and question marks, and being wrong about an
|
||||
// abbreviation costs a slightly uneven chunk, nothing more.
|
||||
func splitSentences(text string) []string {
|
||||
runes := []rune(text)
|
||||
var out []string
|
||||
start := 0
|
||||
for i := 0; i < len(runes); i++ {
|
||||
if runes[i] != '.' && runes[i] != '!' && runes[i] != '?' && runes[i] != '\n' {
|
||||
continue
|
||||
}
|
||||
// Consume a run of punctuation ("?!", "...") so it stays together.
|
||||
j := i
|
||||
for j+1 < len(runes) && isSentenceEnd(runes[j+1]) {
|
||||
j++
|
||||
}
|
||||
if j+1 < len(runes) && !unicode.IsSpace(runes[j+1]) {
|
||||
i = j
|
||||
continue
|
||||
}
|
||||
end := j + 1
|
||||
for end < len(runes) && unicode.IsSpace(runes[end]) {
|
||||
end++
|
||||
}
|
||||
out = append(out, string(runes[start:end]))
|
||||
start = end
|
||||
i = end - 1
|
||||
}
|
||||
if start < len(runes) {
|
||||
out = append(out, string(runes[start:]))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func isSentenceEnd(r rune) bool {
|
||||
return r == '.' || r == '!' || r == '?'
|
||||
}
|
||||
|
||||
// splitWords cuts an oversized run on whitespace, falling back to a hard cut
|
||||
// when a single "word" is itself longer than the limit.
|
||||
func splitWords(runes []rune, maxRunes int) []string {
|
||||
var out []string
|
||||
for len(runes) > maxRunes {
|
||||
cut := maxRunes
|
||||
for cut > 0 && !unicode.IsSpace(runes[cut]) {
|
||||
cut--
|
||||
}
|
||||
if cut == 0 {
|
||||
cut = maxRunes
|
||||
}
|
||||
if s := strings.TrimSpace(string(runes[:cut])); s != "" {
|
||||
out = append(out, s)
|
||||
}
|
||||
runes = runes[cut:]
|
||||
}
|
||||
if s := strings.TrimSpace(string(runes)); s != "" {
|
||||
out = append(out, s)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
package capture
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNilSummarizerWithoutAModel(t *testing.T) {
|
||||
if s := NewSummarizer(nil, 0, 0, nil); s != nil {
|
||||
t.Fatal("a summarizer with no model is not nil")
|
||||
}
|
||||
var s *Summarizer
|
||||
if _, _, err := s.Summarize(context.Background(), "x", "текст"); !errors.Is(err, ErrDisabled) {
|
||||
t.Fatalf("got %v, want ErrDisabled", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The common case: a short meeting fits in one prompt, so there is exactly one
|
||||
// model call and no reduce step.
|
||||
func TestShortTranscriptSkipsTheReduceStep(t *testing.T) {
|
||||
f := &fakeCompleter{replies: []string{"— договорились о смете"}}
|
||||
s := NewSummarizer(f, 0, 0, nil)
|
||||
out, chunks, err := s.Summarize(context.Background(), "смета", "Обсудили смету. Решили подписать.")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if chunks != 1 {
|
||||
t.Errorf("chunks = %d, want 1", chunks)
|
||||
}
|
||||
if len(f.users) != 1 {
|
||||
t.Fatalf("%d model calls, want 1", len(f.users))
|
||||
}
|
||||
if !strings.Contains(out, "смете") || !strings.HasPrefix(out, "смета") {
|
||||
t.Errorf("summary = %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLongTranscriptIsMappedThenReduced(t *testing.T) {
|
||||
f := &roleCompleter{mapReply: "часть", reduceReply: "общий итог"}
|
||||
s := NewSummarizer(f, 40, 0, nil)
|
||||
long := strings.Repeat("Говорили про насос и трубы. ", 12)
|
||||
out, chunks, err := s.Summarize(context.Background(), "", long)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if chunks < 2 {
|
||||
t.Fatalf("chunks = %d, want the transcript split", chunks)
|
||||
}
|
||||
// One map call per chunk, then exactly one reduce.
|
||||
if f.maps != chunks {
|
||||
t.Errorf("%d map calls for %d chunks", f.maps, chunks)
|
||||
}
|
||||
if f.reduces != 1 {
|
||||
t.Errorf("%d reduce calls, want 1", f.reduces)
|
||||
}
|
||||
if out != "общий итог" {
|
||||
t.Errorf("summary = %q, want the reduced text", out)
|
||||
}
|
||||
}
|
||||
|
||||
// Losing every per-chunk summary because the last call failed would throw away
|
||||
// most of the work.
|
||||
func TestReduceFailureReturnsTheJoinedParts(t *testing.T) {
|
||||
f := &roleCompleter{mapReply: "часть", reduceFails: true}
|
||||
s := NewSummarizer(f, 40, 0, nil)
|
||||
long := strings.Repeat("Говорили про насос и трубы. ", 12)
|
||||
out, _, err := s.Summarize(context.Background(), "", long)
|
||||
if err == nil {
|
||||
t.Fatal("reduce failure was not reported")
|
||||
}
|
||||
if !strings.Contains(out, "часть1") || !strings.Contains(out, "часть2") {
|
||||
t.Errorf("per-chunk work was lost: %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkFailureIsReported(t *testing.T) {
|
||||
f := &fakeCompleter{err: errors.New("llama is down")}
|
||||
s := NewSummarizer(f, 0, 0, nil)
|
||||
if _, _, err := s.Summarize(context.Background(), "", "текст"); err == nil {
|
||||
t.Fatal("chunk failure was not reported")
|
||||
}
|
||||
}
|
||||
|
||||
// "пусто" chunks are noise; they must not pad the reduce prompt, and a
|
||||
// transcript that is entirely empty chunks is an honest ErrNoSummary rather than
|
||||
// an invented summary.
|
||||
func TestEmptyChunksAreDropped(t *testing.T) {
|
||||
f := &roleCompleter{mapReply: "пусто", literalMap: true, reduceReply: "не должно вызываться"}
|
||||
s := NewSummarizer(f, 40, 0, nil)
|
||||
long := strings.Repeat("Тишина в комнате. ", 12)
|
||||
if _, _, err := s.Summarize(context.Background(), "", long); !errors.Is(err, ErrNoSummary) {
|
||||
t.Fatalf("got %v, want ErrNoSummary", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmptyTranscriptIsRefused(t *testing.T) {
|
||||
s := NewSummarizer(&fakeCompleter{}, 0, 0, nil)
|
||||
if _, _, err := s.Summarize(context.Background(), "", " \n "); !errors.Is(err, ErrEmptyCapture) {
|
||||
t.Fatalf("got %v, want ErrEmptyCapture", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A summary that silently covers the first fraction of a long meeting is the
|
||||
// failure mode; it has to say so.
|
||||
func TestTruncationIsStatedInTheSummary(t *testing.T) {
|
||||
f := &fakeCompleter{replies: []string{"a", "b", "итог"}}
|
||||
s := NewSummarizer(f, 30, 2, nil)
|
||||
long := strings.Repeat("Говорили про насос и про трубы. ", 20)
|
||||
out, chunks, err := s.Summarize(context.Background(), "", long)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if chunks != 2 {
|
||||
t.Errorf("chunks = %d, want the cap of 2", chunks)
|
||||
}
|
||||
if !strings.Contains(out, "обрезана") {
|
||||
t.Errorf("truncation not stated: %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
// The persona block belongs to the daemon, not this package, and must reach the
|
||||
// model when it is supplied.
|
||||
func TestContextBlockIsPrependedToEveryPrompt(t *testing.T) {
|
||||
f := &fakeCompleter{replies: []string{"итог"}}
|
||||
s := NewSummarizer(f, 0, 0, func() string { return "ПЕРСОНА\n\n" })
|
||||
if _, _, err := s.Summarize(context.Background(), "", "Обсудили смету."); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i, sys := range f.systems {
|
||||
if !strings.HasPrefix(sys, "ПЕРСОНА") {
|
||||
t.Errorf("call %d lost the context block: %q", i, sys)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The map/reduce prompts must contain no first person at all: the persona's
|
||||
// feminine forms live in the replier, and a first-person instruction here is a
|
||||
// place for the model to write "я рад".
|
||||
func TestPromptsHaveNoFirstPerson(t *testing.T) {
|
||||
for name, p := range map[string]string{"chunk": chunkPrompt, "reduce": reducePrompt} {
|
||||
for _, bad := range []string{" я ", "рад", "поняла", "мне ", "вы ", "ваш"} {
|
||||
if strings.Contains(strings.ToLower(" "+p+" "), bad) {
|
||||
t.Errorf("%s prompt contains %q", name, bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkTextSplitsOnSentenceBoundaries(t *testing.T) {
|
||||
text := "Раз два три. Четыре пять шесть. Семь восемь девять."
|
||||
got := ChunkText(text, 20)
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("got %d chunks: %q", len(got), got)
|
||||
}
|
||||
for _, c := range got {
|
||||
if !strings.HasSuffix(c, ".") {
|
||||
t.Errorf("chunk does not end on a sentence: %q", c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkTextPacksSentencesUpToTheLimit(t *testing.T) {
|
||||
text := "Раз. Два. Три. Четыре."
|
||||
got := ChunkText(text, 12)
|
||||
if len(got) < 2 {
|
||||
t.Fatalf("nothing was split: %q", got)
|
||||
}
|
||||
for _, c := range got {
|
||||
if n := len([]rune(c)); n > 12 {
|
||||
t.Errorf("chunk of %d runes exceeds the limit: %q", n, c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// whisper does emit long unpunctuated runs; those must be cut on whitespace, not
|
||||
// dropped and not run past the context limit.
|
||||
func TestChunkTextCutsUnpunctuatedRuns(t *testing.T) {
|
||||
text := strings.TrimSpace(strings.Repeat("слово ", 50))
|
||||
got := ChunkText(text, 30)
|
||||
if len(got) < 2 {
|
||||
t.Fatalf("unpunctuated run was not split: %d chunks", len(got))
|
||||
}
|
||||
total := 0
|
||||
for _, c := range got {
|
||||
if n := len([]rune(c)); n > 30 {
|
||||
t.Errorf("chunk of %d runes exceeds the limit", n)
|
||||
}
|
||||
total += strings.Count(c, "слово")
|
||||
}
|
||||
if total != 50 {
|
||||
t.Errorf("%d of 50 words survived chunking", total)
|
||||
}
|
||||
}
|
||||
|
||||
// A single token longer than the window must still come out, hard-cut.
|
||||
func TestChunkTextHandlesOneOversizedWord(t *testing.T) {
|
||||
text := strings.Repeat("я", 70)
|
||||
got := ChunkText(text, 20)
|
||||
if len(got) != 4 {
|
||||
t.Fatalf("got %d chunks, want 4", len(got))
|
||||
}
|
||||
if joined := strings.Join(got, ""); len([]rune(joined)) != 70 {
|
||||
t.Errorf("%d runes survived, want 70", len([]rune(joined)))
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkTextShortInputAndEmpty(t *testing.T) {
|
||||
if got := ChunkText("коротко", 100); len(got) != 1 || got[0] != "коротко" {
|
||||
t.Errorf("got %q", got)
|
||||
}
|
||||
if got := ChunkText(" ", 100); got != nil {
|
||||
t.Errorf("blank text produced %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// roleCompleter answers by which prompt it was handed, so a test does not have
|
||||
// to predict how many chunks the text splits into. Map replies are numbered
|
||||
// ("часть1", "часть2", …) unless literalMap is set.
|
||||
type roleCompleter struct {
|
||||
mapReply string
|
||||
literalMap bool
|
||||
reduceReply string
|
||||
reduceFails bool
|
||||
maps int
|
||||
reduces int
|
||||
}
|
||||
|
||||
func (f *roleCompleter) Complete(_ context.Context, system, _ string) (string, error) {
|
||||
if strings.Contains(system, "конспекты фрагментов") {
|
||||
f.reduces++
|
||||
if f.reduceFails {
|
||||
return "", errors.New("llama fell over")
|
||||
}
|
||||
return f.reduceReply, nil
|
||||
}
|
||||
f.maps++
|
||||
if f.literalMap {
|
||||
return f.mapReply, nil
|
||||
}
|
||||
return fmt.Sprintf("%s%d", f.mapReply, f.maps), nil
|
||||
}
|
||||
@@ -18,11 +18,14 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/delivery/ntfysink"
|
||||
"github.com/kami/maven/internal/delivery/telegramsink"
|
||||
"github.com/kami/maven/internal/mcp"
|
||||
"github.com/kami/maven/internal/morning"
|
||||
"github.com/kami/maven/internal/update"
|
||||
"github.com/robfig/cron/v3"
|
||||
)
|
||||
|
||||
@@ -108,6 +111,16 @@ type Config struct {
|
||||
// calls its /v1/chat/completions endpoint to phrase nudges and reminders.
|
||||
Phraser *PhraserConfig `json:"phraser,omitempty"`
|
||||
|
||||
// Update — how THIS box deploys a new build of Maven (Vikunja #249). nil ⇒
|
||||
// the update capability does not exist, which is the state to leave it in
|
||||
// unless the operator has read internal/update's package comment.
|
||||
//
|
||||
// mavend never reads this block: the daemon does not import internal/update
|
||||
// and cannot update itself. It lives here because cmd/mavupdate — a CLI the
|
||||
// owner runs on the host, the only trigger there is — reads the same config
|
||||
// file to find the socket it health-checks.
|
||||
Update *update.Config `json:"update,omitempty"`
|
||||
|
||||
// Voice — the client↔core surface + the stt/tts modules the daemon
|
||||
// wires. nil ⇒ the daemon doesn't wire voice: the TCP listener stays
|
||||
// down, the dispatcher's Voice slot stays nil (the routing table's
|
||||
@@ -150,6 +163,21 @@ type Config struct {
|
||||
// absent ⇒ no evaluation loop at all. See MemoryEvalConfig.
|
||||
MemoryEval *MemoryEvalConfig `json:"memory_eval,omitempty"`
|
||||
|
||||
// Email — mail ingestion (Vikunja #246). nil / absent ⇒ core refuses
|
||||
// ipc.MethodIngestMail outright, so a mail reader cannot make Maven read a
|
||||
// mailbox by merely existing. See EmailConfig; the IMAP host and credential
|
||||
// live in the reader (cmd/mavmaild), never here.
|
||||
Email *EmailConfig `json:"email,omitempty"`
|
||||
|
||||
// Feeds — RSS/Atom feed reading (Vikunja #258). nil / absent ⇒ no feed is
|
||||
// ever fetched: reading the outside world is off unless configured, like
|
||||
// the weather and telegram. See FeedsConfig.
|
||||
Feeds *FeedsConfig `json:"feeds,omitempty"`
|
||||
|
||||
// Crawl — reading a web page (Vikunja #259). nil / absent ⇒ Maven never
|
||||
// fetches a page: not on request, not on a schedule. See CrawlConfig.
|
||||
Crawl *CrawlConfig `json:"crawl,omitempty"`
|
||||
|
||||
// Praxis — the ecosystem attention-state service. When configured, maven
|
||||
// calls the Praxis HTTP tools API for attention listing and item lifecycle.
|
||||
// Maven never touches Praxis's database directly (ecosystem invariant: no
|
||||
@@ -165,6 +193,137 @@ type Config struct {
|
||||
// discovers and executes capabilities through Hexis for ecosystem actions.
|
||||
// nil ⇒ no capability-aware routing.
|
||||
Hexis *HexisConfig `json:"hexis,omitempty"`
|
||||
|
||||
// Vision — image understanding (Vikunja #252). nil / absent ⇒ she cannot
|
||||
// look at pictures at all: the intake refuses, and no vision server is
|
||||
// contacted. See VisionConfig.
|
||||
Vision *VisionConfig `json:"vision,omitempty"`
|
||||
|
||||
// Media — where images and captured audio are kept on disk, and for how
|
||||
// long. nil / absent ⇒ no blob store is wired, which is what disables both
|
||||
// vision intake and meeting capture regardless of their own blocks: nothing
|
||||
// in this repo holds a recording only in memory. See MediaConfig.
|
||||
Media *MediaConfig `json:"media,omitempty"`
|
||||
|
||||
// Capture — meeting recording and summarisation (Vikunja #253). nil /
|
||||
// absent ⇒ the recorder does not exist: the start/stop methods are not
|
||||
// served at all, so nothing on this box can begin a recording. This is the
|
||||
// most invasive capability Maven has and it is the one most firmly off by
|
||||
// default. See CaptureConfig.
|
||||
Capture *CaptureConfig `json:"capture,omitempty"`
|
||||
|
||||
// MCP — Model Context Protocol servers Maven connects OUT to (Vikunja
|
||||
// #251). nil / absent / no enabled server ⇒ no connection is made and no
|
||||
// tool is discovered, like every other capability that reaches outside the
|
||||
// box. She is a client here, never a server: nothing exposes her own
|
||||
// capabilities to an outside caller. See MCPConfig.
|
||||
MCP *MCPConfig `json:"mcp,omitempty"`
|
||||
}
|
||||
|
||||
// MCPConfig — the MCP client block. Servers are dark until one has
|
||||
// `"enabled": true`, and a discovered tool is only ever PROPOSED: Kami enables
|
||||
// it on /tools, on the authed surface, exactly as he would a shell tool. The
|
||||
// voice path can never grant a capability to itself.
|
||||
type MCPConfig struct {
|
||||
// Servers — the configured servers. Each needs exactly one of command
|
||||
// (a subprocess on this box) or url (a streamable-HTTP endpoint).
|
||||
Servers []MCPServerConfig `json:"servers,omitempty"`
|
||||
|
||||
// Timeout — per-call budget for every server that does not set its own.
|
||||
// 0 ⇒ mcp.DefaultTimeout (15s). A tool slower than this is not usable in a
|
||||
// spoken turn.
|
||||
Timeout Duration `json:"timeout,omitempty"`
|
||||
|
||||
// AllowHosts / DenyHosts — the host lists for the shared webfetch door that
|
||||
// url servers go through. Deny wins. Private addresses are refused
|
||||
// unconditionally unless the individual server sets allow_private.
|
||||
AllowHosts []string `json:"allow_hosts,omitempty"`
|
||||
DenyHosts []string `json:"deny_hosts,omitempty"`
|
||||
|
||||
// MaxBytes — cap on one JSON-RPC response. 0 ⇒ webfetch.DefaultMaxBytes.
|
||||
MaxBytes int64 `json:"max_bytes,omitempty"`
|
||||
}
|
||||
|
||||
// MCPServerConfig — one MCP server.
|
||||
type MCPServerConfig struct {
|
||||
// Name — the local handle. It prefixes every tool this server contributes
|
||||
// ("vikunja" + "list_tasks" ⇒ the allowlist row "vikunja_list_tasks") and
|
||||
// becomes the store scope "mcp:<name>", so its provenance is readable on
|
||||
// /tools without opening the config.
|
||||
Name string `json:"name"`
|
||||
|
||||
// Command / Args / Env / Dir — a stdio server: a child process of mavend,
|
||||
// on this box, under this user. argv, never a shell string.
|
||||
Command string `json:"command,omitempty"`
|
||||
Args []string `json:"args,omitempty"`
|
||||
Env []string `json:"env,omitempty"`
|
||||
Dir string `json:"dir,omitempty"`
|
||||
|
||||
// URL — a streamable-HTTP endpoint. It is fetched through
|
||||
// internal/webfetch, so the SSRF guard, the redirect cap, the size cap and
|
||||
// the one-request-per-host-per-second limit all apply.
|
||||
URL string `json:"url,omitempty"`
|
||||
|
||||
// AllowPrivate — let THIS server be a loopback or LAN address. The Vikunja
|
||||
// server on homesrv is "http://localhost:9100/mcp", which is refused
|
||||
// without this flag. Understand what it means before setting it: a local
|
||||
// server is a DIFFERENT trust level from a public one. It is inside the
|
||||
// network, it usually needs no credential, and it can change things that
|
||||
// matter — so an argument the router got wrong lands somewhere real. Set it
|
||||
// only for a server you run yourself, and prefer allow_tools with it.
|
||||
AllowPrivate bool `json:"allow_private,omitempty"`
|
||||
|
||||
// AllowTools — when set, the ONLY remote tool names taken from this server.
|
||||
// This is the knob that keeps the catalogue deliberate: the resident model
|
||||
// is a 1.7B with a 4096-token context, and a tool name it half-remembers is
|
||||
// a wrong act, so fewer and better-chosen beats complete.
|
||||
AllowTools []string `json:"allow_tools,omitempty"`
|
||||
|
||||
// MaxTools — cap on this server's contribution. 0 ⇒ mcp.DefaultMaxTools (12).
|
||||
MaxTools int `json:"max_tools,omitempty"`
|
||||
|
||||
// Timeout — per-call budget for this server. 0 ⇒ MCPConfig.Timeout.
|
||||
Timeout Duration `json:"timeout,omitempty"`
|
||||
|
||||
// Enabled — false (the default) keeps a configured server described but
|
||||
// dark, so a block can be written and reviewed before it is switched on.
|
||||
Enabled bool `json:"enabled,omitempty"`
|
||||
}
|
||||
|
||||
// MCPServers maps the config blocks onto the mcp package's own type. It lives
|
||||
// here so config validation and daemon wiring cannot drift on the mapping.
|
||||
// Returns nil when nothing is configured or nothing is enabled.
|
||||
func (c *Config) MCPServers() []mcp.ServerConfig {
|
||||
if c.MCP == nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]mcp.ServerConfig, 0, len(c.MCP.Servers))
|
||||
for _, s := range c.MCP.Servers {
|
||||
if !s.Enabled {
|
||||
continue
|
||||
}
|
||||
timeout := time.Duration(s.Timeout)
|
||||
if timeout <= 0 {
|
||||
timeout = time.Duration(c.MCP.Timeout)
|
||||
}
|
||||
out = append(out, mcp.ServerConfig{
|
||||
Name: s.Name,
|
||||
Command: s.Command,
|
||||
Args: s.Args,
|
||||
Env: s.Env,
|
||||
Dir: s.Dir,
|
||||
URL: s.URL,
|
||||
AllowPrivate: s.AllowPrivate,
|
||||
AllowTools: s.AllowTools,
|
||||
MaxTools: s.MaxTools,
|
||||
Timeout: timeout,
|
||||
Enabled: true,
|
||||
})
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// PraxisConfig — maven's connection to the Praxis attention service.
|
||||
@@ -334,6 +493,132 @@ type VoiceConfig struct {
|
||||
ToolTimeout Duration `json:"tool_timeout,omitempty"`
|
||||
}
|
||||
|
||||
// MediaConfig — the on-disk blob store for images and captured audio
|
||||
// (internal/media). It is shared by all three senses: vision intake, meeting
|
||||
// capture, and speaker enrolment samples all write here.
|
||||
//
|
||||
// Absent ⇒ off, and off means Maven cannot accept an image or start a recording
|
||||
// at all. That default is deliberate: a capability that keeps photos and audio of
|
||||
// people on disk should require someone to have typed a path.
|
||||
type MediaConfig struct {
|
||||
// Dir — the blob store root, created 0700. Relative paths resolve against
|
||||
// StateDir. Required; an empty dir means the store is not wired.
|
||||
Dir string `json:"dir,omitempty"`
|
||||
|
||||
// Retention — how long a blob is kept before the tick prunes it. 0 ⇒
|
||||
// media.DefaultRetention (7 days). This is the knob that stops recordings
|
||||
// of people accumulating; raising it past a few weeks should need a reason.
|
||||
Retention Duration `json:"retention,omitempty"`
|
||||
|
||||
// MaxBytes — per-blob cap. 0 ⇒ media.DefaultMaxBytes (64 MiB).
|
||||
MaxBytes int64 `json:"max_bytes,omitempty"`
|
||||
}
|
||||
|
||||
// StoreDir reports the configured blob directory, or "" when media is not
|
||||
// wired. Safe on a nil receiver.
|
||||
func (m *MediaConfig) StoreDir() string {
|
||||
if m == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(m.Dir)
|
||||
}
|
||||
|
||||
// VisionConfig — the vision provider (internal/vision, docs/plans/07-vision.md).
|
||||
//
|
||||
// Absent, or enabled=false, ⇒ the daemon wires vision.Disabled and every attempt
|
||||
// to look at an image answers that vision is not set up. There is no cloud
|
||||
// option in this block on purpose: Endpoint must be a loopback or private
|
||||
// address and internal/vision refuses anything else at startup, because
|
||||
// inference stays on the box and a photo of his flat is the last thing to make
|
||||
// an exception for.
|
||||
type VisionConfig struct {
|
||||
// Enabled — may she look at images. Default false.
|
||||
Enabled bool `json:"enabled,omitempty"`
|
||||
|
||||
// Endpoint — base URL of a llama-server running a vision model with its
|
||||
// mmproj, e.g. "http://127.0.0.1:8081". Loopback / private only.
|
||||
Endpoint string `json:"endpoint,omitempty"`
|
||||
|
||||
// Model — model name sent in the request. llama-server ignores it.
|
||||
Model string `json:"model,omitempty"`
|
||||
|
||||
// MaxDim — longest edge the image is scaled to before inference. 0 ⇒
|
||||
// media.DefaultMaxDim (896).
|
||||
MaxDim int `json:"max_dim,omitempty"`
|
||||
|
||||
// MaxTokens — cap on the description. 0 ⇒ vision.DefaultMaxTokens (300).
|
||||
MaxTokens int `json:"max_tokens,omitempty"`
|
||||
|
||||
// Timeout — per-description budget. 0 ⇒ vision.DefaultTimeout (90s). A small
|
||||
// VLM on an iGPU is slow; a tight timeout here just means no answer ever.
|
||||
Timeout Duration `json:"timeout,omitempty"`
|
||||
|
||||
// Prompt — the default question when he only sent a picture. Empty ⇒
|
||||
// vision.DefaultPrompt (Russian, "опиши что на изображении").
|
||||
Prompt string `json:"prompt,omitempty"`
|
||||
}
|
||||
|
||||
// LooksAtImages reports whether vision is configured well enough to try. Safe on
|
||||
// a nil receiver, and false without an endpoint — enabled with nothing to talk
|
||||
// to is a misconfiguration, not a capability.
|
||||
func (v *VisionConfig) LooksAtImages() bool {
|
||||
return v != nil && v.Enabled && strings.TrimSpace(v.Endpoint) != ""
|
||||
}
|
||||
|
||||
// CaptureConfig — the meeting recorder (internal/capture,
|
||||
// docs/plans/08-hearing.md).
|
||||
//
|
||||
// Absent, or enabled=false, ⇒ the recorder is not wired and the capture methods
|
||||
// return "unknown method", so no client can start a recording however it asks.
|
||||
// A media block is required too: audio is never held only in memory.
|
||||
//
|
||||
// There is deliberately no "auto", no keyword trigger and no duration default
|
||||
// long enough to be forgotten about. Recording other people is an explicit act
|
||||
// with a start, a stop, and a cap.
|
||||
type CaptureConfig struct {
|
||||
// Enabled — may she record a meeting when asked. Default false.
|
||||
Enabled bool `json:"enabled,omitempty"`
|
||||
|
||||
// MaxMinutes — hard cap on one session; it stops itself there. 0 ⇒
|
||||
// capture.DefaultMaxDuration (120 minutes).
|
||||
MaxMinutes int `json:"max_minutes,omitempty"`
|
||||
|
||||
// STTWindow — audio handed to whisper per call. 0 ⇒
|
||||
// capture.DefaultSTTWindow (5m). Larger windows transcribe slightly better
|
||||
// and block the STT worker for longer.
|
||||
STTWindow Duration `json:"stt_window,omitempty"`
|
||||
|
||||
// ChunkRunes — transcript runes per summarisation prompt. 0 ⇒
|
||||
// capture.DefaultChunkRunes (3000), sized for the resident model's n_ctx of
|
||||
// 4096. Raise this only if the resident model's context grows.
|
||||
ChunkRunes int `json:"chunk_runes,omitempty"`
|
||||
|
||||
// MaxChunks — how many windows one meeting may be summarised in before the
|
||||
// transcript is truncated and the summary says so. 0 ⇒
|
||||
// capture.DefaultMaxChunks (40).
|
||||
MaxChunks int `json:"max_chunks,omitempty"`
|
||||
|
||||
// SaveTranscript — write the full transcript as a note alongside the
|
||||
// summary. Default false: a verbatim record of what other people said in a
|
||||
// room is a heavier thing to keep than a four-line summary, so it takes a
|
||||
// deliberate yes. The audio blob is pruned by media.retention either way.
|
||||
SaveTranscript bool `json:"save_transcript,omitempty"`
|
||||
}
|
||||
|
||||
// Records reports whether the recorder should be wired. Safe on a nil receiver.
|
||||
func (c *CaptureConfig) Records() bool {
|
||||
return c != nil && c.Enabled
|
||||
}
|
||||
|
||||
// MaxDuration is the configured session cap as a duration, or 0 for the
|
||||
// package default. Safe on a nil receiver.
|
||||
func (c *CaptureConfig) MaxDuration() time.Duration {
|
||||
if c == nil || c.MaxMinutes <= 0 {
|
||||
return 0
|
||||
}
|
||||
return time.Duration(c.MaxMinutes) * time.Minute
|
||||
}
|
||||
|
||||
// WeatherConfig configures the weather provider for voice queries.
|
||||
type WeatherConfig struct {
|
||||
Provider string `json:"provider,omitempty"` // "open-meteo" or "" → stub
|
||||
@@ -396,6 +681,109 @@ func (p *PatternProposalConfig) AnnounceProposals() bool {
|
||||
return p != nil && p.Notify
|
||||
}
|
||||
|
||||
// FeedsConfig — the RSS/Atom reader (Vikunja #258, docs/plans/13-rss-news-feeds.md).
|
||||
//
|
||||
// Absent ⇒ off, and off means no outbound request at all. Present with an empty
|
||||
// `sources` list is also off — a poller with nothing to poll is not wired.
|
||||
//
|
||||
// What a feed may NOT do here: speak. Items are written as notes with source
|
||||
// "rss:<name>" and read back when he asks; nothing is dispatched, nudged or
|
||||
// announced on arrival. That is the "not a nag" constraint, and it is why there
|
||||
// is no severity or channel field in this block to reach for.
|
||||
type FeedsConfig struct {
|
||||
// Sources — the feeds to read. Empty ⇒ the reader stays down.
|
||||
Sources []FeedSourceConfig `json:"sources,omitempty"`
|
||||
|
||||
// PollInterval — default per-feed cadence. 0 ⇒ rss.DefaultPollInterval (30m).
|
||||
PollInterval Duration `json:"poll_interval,omitempty"`
|
||||
|
||||
// MaxItems — most items kept from one feed in one poll. 0 ⇒
|
||||
// rss.DefaultMaxItems (5). This is the "не завали мне /dash" knob.
|
||||
MaxItems int `json:"max_items,omitempty"`
|
||||
|
||||
// MaxAge — on a first poll (no saved mark), how far back to take items.
|
||||
// 0 ⇒ rss.DefaultMaxAge (24h), so switching a feed on imports today, not
|
||||
// the archive.
|
||||
MaxAge Duration `json:"max_age,omitempty"`
|
||||
|
||||
// AllowHosts — when set, the reader may only connect to these hosts (and
|
||||
// their subdomains). The feed URLs' own hosts are added automatically, so
|
||||
// this is only needed to be stricter than that.
|
||||
AllowHosts []string `json:"allow_hosts,omitempty"`
|
||||
|
||||
// Timeout — per-request budget. 0 ⇒ webfetch.DefaultTimeout.
|
||||
Timeout Duration `json:"timeout,omitempty"`
|
||||
|
||||
// MaxBytes — response size cap. 0 ⇒ webfetch.DefaultMaxBytes (2 MiB).
|
||||
MaxBytes int64 `json:"max_bytes,omitempty"`
|
||||
}
|
||||
|
||||
// FeedSourceConfig — one feed.
|
||||
type FeedSourceConfig struct {
|
||||
Name string `json:"name"` // note source is "rss:<name>"
|
||||
URL string `json:"url"` // http(s) only
|
||||
Category string `json:"category,omitempty"` // "технологии" — what "что нового по X?" matches
|
||||
Interval Duration `json:"interval,omitempty"` // 0 ⇒ FeedsConfig.PollInterval
|
||||
Include []string `json:"include,omitempty"` // keep only items containing one of these
|
||||
Exclude []string `json:"exclude,omitempty"` // drop items containing any of these
|
||||
}
|
||||
|
||||
// CrawlConfig — the web crawler (Vikunja #259, docs/plans/14-web-crawler.md).
|
||||
//
|
||||
// Absent ⇒ off, and off means no page is ever fetched. Present with neither
|
||||
// `on_demand` nor a `watches` entry is also off: there would be nothing to do.
|
||||
//
|
||||
// The crawler is the LAST place an answer is looked for, behind the model, his
|
||||
// own memory and the local Kiwix ZIMs. That ordering lives in the query-source
|
||||
// chain (cmd/mavend/actions_query.go), not here, but it is the reason this block
|
||||
// is small: it is a fallback, not a search engine.
|
||||
//
|
||||
// Only the URL leaves the box. His notes, facts, persona block and history are
|
||||
// never part of a request — the crawler package cannot even read the store.
|
||||
type CrawlConfig struct {
|
||||
// OnDemand — may he ask her to read a page he names out loud
|
||||
// ("посмотри https://… — что там пишут?"). false ⇒ the on-demand answer
|
||||
// source stays off and only the watches below run.
|
||||
OnDemand bool `json:"on_demand,omitempty"`
|
||||
|
||||
// Watches — pages re-read on a schedule. A page whose text changed is
|
||||
// written as a note (source "crawl:<name>"); nothing is announced.
|
||||
Watches []CrawlWatchConfig `json:"watches,omitempty"`
|
||||
|
||||
// Interval — default watch cadence. 0 ⇒ crawl.DefaultWatchInterval (6h).
|
||||
Interval Duration `json:"interval,omitempty"`
|
||||
|
||||
// AllowHosts — when set, the ONLY hosts the crawler may reach (subdomains
|
||||
// included). Watched pages' own hosts are added automatically. Setting this
|
||||
// is how "she may read the arch wiki and nothing else" is expressed.
|
||||
AllowHosts []string `json:"allow_hosts,omitempty"`
|
||||
|
||||
// DenyHosts — never reachable, checked first. Private addresses do not need
|
||||
// to be listed: they are refused unconditionally (see internal/webfetch).
|
||||
DenyHosts []string `json:"deny_hosts,omitempty"`
|
||||
|
||||
// UserAgent — sent on every request AND matched against robots.txt groups.
|
||||
// Empty ⇒ webfetch.DefaultUserAgent.
|
||||
UserAgent string `json:"user_agent,omitempty"`
|
||||
|
||||
// Timeout — per-request budget. 0 ⇒ webfetch.DefaultTimeout.
|
||||
Timeout Duration `json:"timeout,omitempty"`
|
||||
|
||||
// MaxBytes — response size cap. 0 ⇒ webfetch.DefaultMaxBytes (2 MiB).
|
||||
MaxBytes int64 `json:"max_bytes,omitempty"`
|
||||
|
||||
// MaxRunes — how much extracted text is kept. 0 ⇒ crawl.DefaultMaxRunes
|
||||
// (4000), which is what fits a 4096-token context alongside a prompt.
|
||||
MaxRunes int `json:"max_runes,omitempty"`
|
||||
}
|
||||
|
||||
// CrawlWatchConfig — one page kept an eye on.
|
||||
type CrawlWatchConfig struct {
|
||||
Name string `json:"name"` // note source is "crawl:<name>"
|
||||
URL string `json:"url"`
|
||||
Interval Duration `json:"interval,omitempty"` // 0 ⇒ CrawlConfig.Interval
|
||||
}
|
||||
|
||||
// MemoryEvalConfig — the background memory-evaluation loop (Vikunja #248).
|
||||
// Absent ⇒ off, like every other capability that costs something the owner did
|
||||
// not ask for. Each evaluation is a full LLM round-trip on the one resident
|
||||
@@ -418,6 +806,26 @@ type MemoryEvalConfig struct {
|
||||
MinConfidence float64 `json:"min_confidence,omitempty"`
|
||||
}
|
||||
|
||||
// EmailConfig — core's half of the email reader: how many task candidates one
|
||||
// message may produce, and how long the extraction call may take.
|
||||
//
|
||||
// There is deliberately nothing about a mailbox here. Core does not connect to
|
||||
// IMAP, does not know an account exists, and holds no mail credential — the
|
||||
// reader daemon does, the same split mavpoll uses for the zenmoney token. This
|
||||
// block only says "extraction is allowed, with these bounds".
|
||||
type EmailConfig struct {
|
||||
// MaxTasks — candidates per message. 0 ⇒ email.MaxCandidates (3).
|
||||
MaxTasks int `json:"max_tasks,omitempty"`
|
||||
|
||||
// Timeout — per-message extraction budget. 0 ⇒ DefaultEmailTimeout. This is
|
||||
// a Thinking model reading a mail; nobody is waiting on the answer, but a
|
||||
// hung llama-server must not pin the reader's connection forever.
|
||||
Timeout Duration `json:"timeout,omitempty"`
|
||||
}
|
||||
|
||||
// DefaultEmailTimeout — extraction budget per message.
|
||||
const DefaultEmailTimeout = 2 * time.Minute
|
||||
|
||||
// PhraserConfig — the LLM-backed phraser seam. The daemon spawns llama-server
|
||||
// as a managed subprocess and sends chat-completion requests to phrase nudge
|
||||
// and reminder messages. nil ⇒ the template-based Stub is used instead.
|
||||
@@ -441,6 +849,21 @@ type PhraserConfig struct {
|
||||
// persona and invented units). Chat, query and reminder phrasing always go
|
||||
// through the model regardless. See phraser.Config.LLMNudges.
|
||||
LLMNudges bool `json:"llm_nudges,omitempty"`
|
||||
|
||||
// SwapModels — the gguf files the running daemon is allowed to swap to
|
||||
// without a restart (Vikunja #250). Empty (the default) means the swap
|
||||
// capability does not exist: ipc.MethodSwapModel answers ErrUnknownMethod,
|
||||
// exactly like an unconfigured weather or telegram block.
|
||||
//
|
||||
// It is an allowlist and not a directory on purpose. The request carries a
|
||||
// path, and llama-server is started with it as `-m`; anything short of an
|
||||
// exact match against a list a human wrote in this file would make "swap the
|
||||
// model" mean "load a file of your choosing off my disk". ModelPath is
|
||||
// always swappable back to whether or not it is listed.
|
||||
//
|
||||
// Paths must be absolute — the daemon's working directory is not the
|
||||
// operator's, and a relative path here would resolve somewhere surprising.
|
||||
SwapModels []string `json:"swap_models,omitempty"`
|
||||
}
|
||||
|
||||
// EmbedderConfig — paths for the ONNX multilingual embedder. The daemon
|
||||
@@ -607,6 +1030,30 @@ func (c *Config) applyDefaults() {
|
||||
c.MemoryEval.Interval = Duration(DefaultMemoryEvalInterval)
|
||||
}
|
||||
|
||||
// Same rule again: absent stays nil (⇒ mail ingestion refused), present gets
|
||||
// the timeout default so `{}` is a valid "on with the defaults".
|
||||
if c.Email != nil && c.Email.Timeout <= 0 {
|
||||
c.Email.Timeout = Duration(DefaultEmailTimeout)
|
||||
}
|
||||
|
||||
// A feeds block with no sources is the same as no block: nothing to poll,
|
||||
// nothing wired. Normalising it to nil keeps that "off" in one place.
|
||||
if c.Feeds != nil && len(c.Feeds.Sources) == 0 {
|
||||
c.Feeds = nil
|
||||
}
|
||||
|
||||
// Same rule for MCP: a block with no server, or none enabled, is the same
|
||||
// as no block at all. Normalising it to nil keeps "off" in one place.
|
||||
if c.MCP != nil && len(c.MCPServers()) == 0 {
|
||||
c.MCP = nil
|
||||
}
|
||||
|
||||
// Same rule for the crawler: a block that neither answers on demand nor
|
||||
// watches anything has nothing to do, so it is normalised to "off".
|
||||
if c.Crawl != nil && !c.Crawl.OnDemand && len(c.Crawl.Watches) == 0 {
|
||||
c.Crawl = nil
|
||||
}
|
||||
|
||||
if c.Voice != nil {
|
||||
if c.Voice.RouterThreshold <= 0 {
|
||||
c.Voice.RouterThreshold = DefaultRouterThreshold
|
||||
@@ -664,6 +1111,22 @@ func (c *Config) validate() error {
|
||||
if c.Phraser.ModelPath == "" {
|
||||
return errors.New("phraser.model_path is required")
|
||||
}
|
||||
// A relative entry in the swap allowlist would resolve against the
|
||||
// daemon's working directory, so the path a human reads in this file
|
||||
// would not be the path llama-server is handed. Fail at startup.
|
||||
for _, m := range c.Phraser.SwapModels {
|
||||
if !filepath.IsAbs(m) {
|
||||
return fmt.Errorf("phraser.swap_models: %q must be an absolute path", m)
|
||||
}
|
||||
}
|
||||
}
|
||||
// The update block is validated here even though mavend never acts on it: a
|
||||
// half-written update config that is only noticed by cmd/mavupdate is noticed
|
||||
// at the worst possible moment, halfway through deploying a new build.
|
||||
if c.Update != nil {
|
||||
if err := c.Update.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if c.Voice != nil && c.Voice.Enabled {
|
||||
if c.Voice.Bind == "" {
|
||||
@@ -689,6 +1152,12 @@ func (c *Config) validate() error {
|
||||
return fmt.Errorf("routine %q: bad cron %q: %w", r.Name, r.Cron, err)
|
||||
}
|
||||
}
|
||||
// An MCP block with a typo (no name, both command and url, a bare hostname
|
||||
// as the url) fails here, at startup, rather than at the first turn that
|
||||
// needed the tool.
|
||||
if err := mcp.Validate(c.MCPServers()); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(c.MorningRoutines) > 0 {
|
||||
if err := morning.Validate(morningRoutinesFromConfig(c.MorningRoutines)); err != nil {
|
||||
return err
|
||||
|
||||
@@ -293,3 +293,76 @@ func TestPatternProposalNotifyDefaultsOff(t *testing.T) {
|
||||
t.Errorf("cooldown = %v, want 6h", c.PatternProposals.Cooldown)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSwapModelsAbsentMeansOff — the swap capability does not exist unless the
|
||||
// operator lists the models he allows (Vikunja #250).
|
||||
func TestSwapModelsAbsentMeansOff(t *testing.T) {
|
||||
c, err := Load(writeConfig(t, `{"phraser": {"model_path": "/m/qwen.gguf"}}`))
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
if len(c.Phraser.SwapModels) != 0 {
|
||||
t.Errorf("swap_models = %v; want empty when unconfigured", c.Phraser.SwapModels)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSwapModelsParsedAndMustBeAbsolute(t *testing.T) {
|
||||
c, err := Load(writeConfig(t, `{"phraser": {
|
||||
"model_path": "/m/qwen.gguf",
|
||||
"swap_models": ["/m/qwen.gguf", "/m/qwen-cpt.gguf"]
|
||||
}}`))
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
if len(c.Phraser.SwapModels) != 2 {
|
||||
t.Fatalf("swap_models = %v; want 2 entries", c.Phraser.SwapModels)
|
||||
}
|
||||
// A relative entry would resolve against the daemon's cwd, not the operator's.
|
||||
if _, err := Load(writeConfig(t, `{"phraser": {
|
||||
"model_path": "/m/qwen.gguf",
|
||||
"swap_models": ["models/llm/qwen.gguf"]
|
||||
}}`)); err == nil {
|
||||
t.Error("Load accepted a relative swap_models entry; want a startup failure")
|
||||
}
|
||||
}
|
||||
|
||||
// TestUpdateBlockAbsentMeansOff — mavend never updates itself; the block only
|
||||
// exists so cmd/mavupdate can find the deployment it is asked to update
|
||||
// (Vikunja #249). Absent is the normal state.
|
||||
func TestUpdateBlockAbsentMeansOff(t *testing.T) {
|
||||
c, err := Load(writeConfig(t, `{}`))
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
if c.Update != nil {
|
||||
t.Errorf("update = %+v; want nil when unconfigured", c.Update)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateBlockValidatedAtStartup(t *testing.T) {
|
||||
good := `{"update": {
|
||||
"source_dir": "/srv/maven",
|
||||
"install_dir": "/srv/maven",
|
||||
"snapshot_dir": "/var/lib/maven/snapshots",
|
||||
"binaries": ["mavend", "mavweb"],
|
||||
"restart_cmd": ["docker", "compose", "up", "-d", "--build", "mavend"],
|
||||
"health_socket": "/run/maven/mavend.sock"
|
||||
}}`
|
||||
c, err := Load(writeConfig(t, good))
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
if c.Update == nil || len(c.Update.Binaries) != 2 {
|
||||
t.Fatalf("update block = %+v; want it parsed", c.Update)
|
||||
}
|
||||
// A block with no health check cannot detect its own failure, so it cannot
|
||||
// roll back — refused at load, not halfway through a deploy.
|
||||
noHealth := `{"update": {
|
||||
"source_dir": "/srv/maven", "install_dir": "/srv/maven",
|
||||
"snapshot_dir": "/var/lib/maven/snapshots",
|
||||
"binaries": ["mavend"], "restart_cmd": ["true"]
|
||||
}}`
|
||||
if _, err := Load(writeConfig(t, noHealth)); err == nil {
|
||||
t.Error("Load accepted an update block with no health_socket")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestMCPAbsentIsOff(t *testing.T) {
|
||||
c, err := Load(writeConfig(t, `{}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.MCP != nil {
|
||||
t.Error("no mcp block ⇒ nil")
|
||||
}
|
||||
if got := c.MCPServers(); got != nil {
|
||||
t.Errorf("MCPServers() = %+v, want nil", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A described-but-not-enabled server must not be wired. This is how a block can
|
||||
// sit in the config file, reviewed, before it is switched on.
|
||||
func TestMCPDisabledServerIsOff(t *testing.T) {
|
||||
c, err := Load(writeConfig(t, `{"mcp":{"servers":[
|
||||
{"name":"vikunja","url":"http://localhost:9100/mcp","allow_private":true}]}}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.MCP != nil {
|
||||
t.Errorf("a block with nothing enabled must normalise to nil, got %+v", c.MCP)
|
||||
}
|
||||
if got := c.MCPServers(); len(got) != 0 {
|
||||
t.Errorf("MCPServers() = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMCPEnabledServerMapping(t *testing.T) {
|
||||
c, err := Load(writeConfig(t, `{"mcp":{
|
||||
"timeout":"5s",
|
||||
"servers":[
|
||||
{"name":"vikunja","url":"http://localhost:9100/mcp","allow_private":true,
|
||||
"allow_tools":["list_tasks"],"max_tools":3,"enabled":true},
|
||||
{"name":"files","command":"mcp-server-fs","args":["/srv"],"timeout":"1s","enabled":true},
|
||||
{"name":"off","command":"nope"}
|
||||
]}}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := c.MCPServers()
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("servers = %+v", got)
|
||||
}
|
||||
if got[0].Name != "vikunja" || !got[0].AllowPrivate || got[0].MaxTools != 3 ||
|
||||
len(got[0].AllowTools) != 1 || got[0].Timeout != 5*time.Second {
|
||||
t.Errorf("vikunja mapped wrong: %+v", got[0])
|
||||
}
|
||||
if got[1].Command != "mcp-server-fs" || len(got[1].Args) != 1 || got[1].Timeout != time.Second {
|
||||
t.Errorf("files mapped wrong: %+v", got[1])
|
||||
}
|
||||
// allow_private is per server and must not leak to the other one.
|
||||
if got[1].AllowPrivate {
|
||||
t.Error("allow_private leaked between servers")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMCPBadServerFailsAtStartup(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"no name": `{"mcp":{"servers":[{"command":"x","enabled":true}]}}`,
|
||||
"both": `{"mcp":{"servers":[{"name":"a","command":"x","url":"http://a.test","enabled":true}]}}`,
|
||||
"neither": `{"mcp":{"servers":[{"name":"a","enabled":true}]}}`,
|
||||
"bad scheme": `{"mcp":{"servers":[{"name":"a","url":"unix:///run/x.sock","enabled":true}]}}`,
|
||||
"duplicate": `{"mcp":{"servers":[{"name":"a","command":"x","enabled":true},{"name":"a","command":"y","enabled":true}]}}`,
|
||||
"spacey name": `{"mcp":{"servers":[{"name":"a b","command":"x","enabled":true}]}}`,
|
||||
}
|
||||
for name, body := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if _, err := Load(writeConfig(t, body)); err == nil {
|
||||
t.Fatal("want a startup error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Absent blocks must read as off on a nil receiver: the daemon calls these
|
||||
// helpers before it knows whether the operator configured anything.
|
||||
func TestSensesOffByDefault(t *testing.T) {
|
||||
var cfg Config
|
||||
if cfg.Media.StoreDir() != "" {
|
||||
t.Error("media store dir is set with no media block")
|
||||
}
|
||||
if cfg.Vision.LooksAtImages() {
|
||||
t.Error("vision is on with no vision block")
|
||||
}
|
||||
if cfg.Capture.Records() {
|
||||
t.Error("the recorder is on with no capture block")
|
||||
}
|
||||
if cfg.Capture.MaxDuration() != 0 {
|
||||
t.Error("a nil capture block invented a duration")
|
||||
}
|
||||
}
|
||||
|
||||
// The recorder is the capability that most needs its default to be off, so it
|
||||
// gets its own test rather than a line in the one above.
|
||||
func TestCaptureIsOffUntilExplicitlyEnabled(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
c *CaptureConfig
|
||||
want bool
|
||||
}{
|
||||
{"absent", nil, false},
|
||||
{"present but not enabled", &CaptureConfig{MaxMinutes: 60}, false},
|
||||
{"enabled", &CaptureConfig{Enabled: true}, true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := c.c.Records(); got != c.want {
|
||||
t.Errorf("%s: Records() = %v, want %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCaptureBlockParsesFromJSON(t *testing.T) {
|
||||
raw := `{"capture":{"enabled":true,"max_minutes":45,"stt_window":"2m",
|
||||
"chunk_runes":2000,"max_chunks":10,"save_transcript":true}}`
|
||||
var cfg Config
|
||||
if err := json.Unmarshal([]byte(raw), &cfg); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
if !cfg.Capture.Records() {
|
||||
t.Fatal("capture did not parse as enabled")
|
||||
}
|
||||
if cfg.Capture.MaxDuration() != 45*time.Minute {
|
||||
t.Errorf("max duration = %v", cfg.Capture.MaxDuration())
|
||||
}
|
||||
if time.Duration(cfg.Capture.STTWindow) != 2*time.Minute {
|
||||
t.Errorf("stt window = %v", time.Duration(cfg.Capture.STTWindow))
|
||||
}
|
||||
if cfg.Capture.ChunkRunes != 2000 || cfg.Capture.MaxChunks != 10 {
|
||||
t.Errorf("summariser limits = %+v", cfg.Capture)
|
||||
}
|
||||
if !cfg.Capture.SaveTranscript {
|
||||
t.Error("save_transcript did not parse")
|
||||
}
|
||||
}
|
||||
|
||||
// Keeping the verbatim record of what other people said is the heavier act, so
|
||||
// it is separately opt-in from recording at all.
|
||||
func TestTranscriptIsNotSavedByDefault(t *testing.T) {
|
||||
var cfg Config
|
||||
if err := json.Unmarshal([]byte(`{"capture":{"enabled":true}}`), &cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.Capture.SaveTranscript {
|
||||
t.Error("transcripts are saved without anyone asking")
|
||||
}
|
||||
if cfg.Capture.MaxDuration() != 0 {
|
||||
t.Error("max_minutes defaulted in config instead of in the package")
|
||||
}
|
||||
}
|
||||
|
||||
// enabled with nothing to talk to is a misconfiguration, not a capability.
|
||||
func TestVisionNeedsBothEnabledAndEndpoint(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
v *VisionConfig
|
||||
want bool
|
||||
}{
|
||||
{"absent", nil, false},
|
||||
{"endpoint but not enabled", &VisionConfig{Endpoint: "http://127.0.0.1:8081"}, false},
|
||||
{"enabled but no endpoint", &VisionConfig{Enabled: true}, false},
|
||||
{"enabled, blank endpoint", &VisionConfig{Enabled: true, Endpoint: " "}, false},
|
||||
{"both", &VisionConfig{Enabled: true, Endpoint: "http://127.0.0.1:8081"}, true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := c.v.LooksAtImages(); got != c.want {
|
||||
t.Errorf("%s: LooksAtImages() = %v, want %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSensesBlocksParseFromJSON(t *testing.T) {
|
||||
raw := `{
|
||||
"db_path": "/tmp/x.db",
|
||||
"socket_path": "/tmp/x.sock",
|
||||
"media": {"dir": "media", "retention": "48h", "max_bytes": 1048576},
|
||||
"vision": {
|
||||
"enabled": true,
|
||||
"endpoint": "http://127.0.0.1:8081",
|
||||
"model": "qwen2.5-vl",
|
||||
"max_dim": 640,
|
||||
"max_tokens": 200,
|
||||
"timeout": "45s",
|
||||
"prompt": "Что тут?"
|
||||
}
|
||||
}`
|
||||
var cfg Config
|
||||
if err := json.Unmarshal([]byte(raw), &cfg); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
if cfg.Media.StoreDir() != "media" {
|
||||
t.Errorf("media dir = %q", cfg.Media.StoreDir())
|
||||
}
|
||||
if time.Duration(cfg.Media.Retention) != 48*time.Hour {
|
||||
t.Errorf("retention = %v", time.Duration(cfg.Media.Retention))
|
||||
}
|
||||
if cfg.Media.MaxBytes != 1<<20 {
|
||||
t.Errorf("max_bytes = %d", cfg.Media.MaxBytes)
|
||||
}
|
||||
if !cfg.Vision.LooksAtImages() {
|
||||
t.Fatal("vision did not parse as enabled")
|
||||
}
|
||||
if cfg.Vision.MaxDim != 640 || cfg.Vision.MaxTokens != 200 {
|
||||
t.Errorf("vision limits = %+v", cfg.Vision)
|
||||
}
|
||||
if time.Duration(cfg.Vision.Timeout) != 45*time.Second {
|
||||
t.Errorf("vision timeout = %v", time.Duration(cfg.Vision.Timeout))
|
||||
}
|
||||
if cfg.Vision.Prompt != "Что тут?" {
|
||||
t.Errorf("prompt = %q", cfg.Vision.Prompt)
|
||||
}
|
||||
}
|
||||
|
||||
// A media dir set with no vision block is a valid state, and the useful one on a
|
||||
// box with no vision model: images can be kept, they just cannot be described.
|
||||
func TestMediaWithoutVisionIsValid(t *testing.T) {
|
||||
var cfg Config
|
||||
if err := json.Unmarshal([]byte(`{"media":{"dir":"/srv/media"}}`), &cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.Media.StoreDir() != "/srv/media" {
|
||||
t.Errorf("dir = %q", cfg.Media.StoreDir())
|
||||
}
|
||||
if cfg.Vision.LooksAtImages() {
|
||||
t.Error("vision came on by itself")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
// Package crawl reads a web page: fetch, robots check, HTML to text.
|
||||
//
|
||||
// It is the LAST place Maven looks for an answer, and that ordering is the whole
|
||||
// design. "Never phones home" is deprecated, but what replaced it puts local
|
||||
// sources first: the resident model, then his own memory, then the Kiwix ZIMs on
|
||||
// the box (internal/kiwix), and only then the network. A local read costs
|
||||
// nothing and leaks nothing; a fetch costs a round-trip and puts a URL in
|
||||
// someone's access log. So this package exists to be the fallback, not the
|
||||
// front door — see the querySources chain in cmd/mavend/actions_query.go for
|
||||
// where it actually sits.
|
||||
//
|
||||
// What never leaves the box: his notes, his facts, the persona block, the
|
||||
// conversation history. Only the URL is requested and, for the on-demand path,
|
||||
// only because he said it out loud. Nothing here reads the store.
|
||||
//
|
||||
// The limits are not in this package — they are in internal/webfetch, which is
|
||||
// the only way anything here touches a socket: http(s) only, host allow/deny,
|
||||
// private-address refusal, size cap, redirect cap, per-host rate limit. What
|
||||
// this package adds is politeness (robots.txt) and dedup.
|
||||
package crawl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Errors callers distinguish.
|
||||
var (
|
||||
ErrRobots = errors.New("crawl: robots.txt disallows this path")
|
||||
ErrNotHTML = errors.New("crawl: response is not html or text")
|
||||
)
|
||||
|
||||
// Fetcher is the guarded HTTP door (internal/webfetch adapted by the daemon). An
|
||||
// interface so this package constructs no http.Client of its own and can be
|
||||
// tested without a network.
|
||||
type Fetcher interface {
|
||||
Get(ctx context.Context, url string) (*Response, error)
|
||||
}
|
||||
|
||||
// Response is the minimum a crawl needs from a fetch.
|
||||
type Response struct {
|
||||
URL string
|
||||
ContentType string
|
||||
Body []byte
|
||||
}
|
||||
|
||||
// Config — crawler knobs.
|
||||
type Config struct {
|
||||
// UserAgent is the name matched against robots.txt groups. It must be the
|
||||
// same string the fetcher sends, or Maven would be claiming one identity
|
||||
// and obeying the rules for another.
|
||||
UserAgent string
|
||||
// MaxRunes caps extracted text. 0 ⇒ DefaultMaxRunes.
|
||||
MaxRunes int
|
||||
// RobotsTTL — how long a parsed robots.txt is trusted. 0 ⇒ 1h.
|
||||
RobotsTTL time.Duration
|
||||
// Now is injectable for tests. nil ⇒ time.Now.
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// Crawler fetches and extracts pages. Safe for concurrent use.
|
||||
type Crawler struct {
|
||||
fetch Fetcher
|
||||
cfg Config
|
||||
robots *robotsCache
|
||||
}
|
||||
|
||||
// New builds a crawler. Returns nil when there is no fetcher, which is how the
|
||||
// daemon expresses "crawling is off unless configured".
|
||||
func New(fetch Fetcher, cfg Config) *Crawler {
|
||||
if fetch == nil {
|
||||
return nil
|
||||
}
|
||||
if cfg.UserAgent == "" {
|
||||
cfg.UserAgent = "Maven"
|
||||
}
|
||||
if cfg.MaxRunes <= 0 {
|
||||
cfg.MaxRunes = DefaultMaxRunes
|
||||
}
|
||||
if cfg.RobotsTTL <= 0 {
|
||||
cfg.RobotsTTL = time.Hour
|
||||
}
|
||||
if cfg.Now == nil {
|
||||
cfg.Now = time.Now
|
||||
}
|
||||
return &Crawler{fetch: fetch, cfg: cfg, robots: newRobotsCache(cfg.RobotsTTL)}
|
||||
}
|
||||
|
||||
// Page fetches rawURL and returns its text. It checks robots.txt first and
|
||||
// refuses a disallowed path with ErrRobots — there is no override.
|
||||
func (c *Crawler) Page(ctx context.Context, rawURL string) (Page, error) {
|
||||
u, err := url.Parse(strings.TrimSpace(rawURL))
|
||||
if err != nil {
|
||||
return Page{}, fmt.Errorf("crawl: bad url %q: %w", rawURL, err)
|
||||
}
|
||||
ok, err := c.allowed(ctx, u)
|
||||
if err != nil {
|
||||
return Page{}, err
|
||||
}
|
||||
if !ok {
|
||||
return Page{}, fmt.Errorf("%w: %s", ErrRobots, u.Path)
|
||||
}
|
||||
resp, err := c.fetch.Get(ctx, u.String())
|
||||
if err != nil {
|
||||
return Page{}, err
|
||||
}
|
||||
// A PDF or an image is bytes Maven cannot read; saying so beats storing
|
||||
// binary garbage as a "note".
|
||||
ct := strings.ToLower(resp.ContentType)
|
||||
if ct != "" && !strings.Contains(ct, "html") && !strings.Contains(ct, "text/") &&
|
||||
!strings.Contains(ct, "xml") && !strings.Contains(ct, "json") {
|
||||
return Page{}, fmt.Errorf("%w: %s", ErrNotHTML, resp.ContentType)
|
||||
}
|
||||
return Extract(resp.URL, resp.Body, c.cfg.MaxRunes), nil
|
||||
}
|
||||
|
||||
// allowed consults robots.txt for u's host, reading it at most once per TTL.
|
||||
//
|
||||
// A robots.txt that cannot be fetched (404, a timeout, a blocked host) means
|
||||
// allow, per the standard. The one thing that is NOT fail-open is an explicit
|
||||
// Disallow.
|
||||
func (c *Crawler) allowed(ctx context.Context, u *url.URL) (bool, error) {
|
||||
host := u.Host
|
||||
now := c.cfg.Now()
|
||||
rules, ok := c.robots.get(host, now)
|
||||
if !ok {
|
||||
robotsURL := u.Scheme + "://" + host + "/robots.txt"
|
||||
resp, err := c.fetch.Get(ctx, robotsURL)
|
||||
switch {
|
||||
case err != nil:
|
||||
// Note what is NOT swallowed: a refusal from the guarded fetcher.
|
||||
// If webfetch says this host is denied or private, the page fetch
|
||||
// would fail the same way, and reporting the real reason beats
|
||||
// reporting a robots verdict we never got.
|
||||
if isFatalFetchError(err) {
|
||||
return false, err
|
||||
}
|
||||
rules = Rules{}
|
||||
default:
|
||||
rules = ParseRobots(string(resp.Body), c.cfg.UserAgent)
|
||||
}
|
||||
c.robots.put(host, rules, now)
|
||||
}
|
||||
path := u.EscapedPath()
|
||||
if u.RawQuery != "" {
|
||||
path += "?" + u.RawQuery
|
||||
}
|
||||
return rules.Allowed(path), nil
|
||||
}
|
||||
|
||||
// isFatalFetchError — a fetch failure that means "this host is off limits"
|
||||
// rather than "there is no robots.txt here". The sentinel set is webfetch's, but
|
||||
// this package must not import it (the interface exists precisely so it does
|
||||
// not), so the check is on the message. Ugly and honest: the alternative is a
|
||||
// dependency inversion for two strings.
|
||||
func isFatalFetchError(err error) bool {
|
||||
s := err.Error()
|
||||
return strings.Contains(s, "not allowed") || strings.Contains(s, "private address") ||
|
||||
strings.Contains(s, "only http and https")
|
||||
}
|
||||
|
||||
// Hash is the dedup key for a crawl result: the sha256 of the extracted text,
|
||||
// hex, first 16 chars. Text and not raw HTML, because a page whose only change
|
||||
// is a rotating ad slot or a CSRF token has not changed.
|
||||
func Hash(text string) string {
|
||||
sum := sha256.Sum256([]byte(strings.TrimSpace(text)))
|
||||
return hex.EncodeToString(sum[:])[:16]
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package crawl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fakeFetcher serves canned pages by URL and counts requests, so a test can
|
||||
// assert that robots.txt was read once and that a refusal never reached the page.
|
||||
type fakeFetcher struct {
|
||||
pages map[string]Response
|
||||
err error
|
||||
calls []string
|
||||
}
|
||||
|
||||
func (f *fakeFetcher) Get(_ context.Context, u string) (*Response, error) {
|
||||
f.calls = append(f.calls, u)
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
r, ok := f.pages[u]
|
||||
if !ok {
|
||||
return nil, errors.New("http 404")
|
||||
}
|
||||
if r.URL == "" {
|
||||
r.URL = u
|
||||
}
|
||||
if r.ContentType == "" {
|
||||
r.ContentType = "text/html; charset=utf-8"
|
||||
}
|
||||
return &r, nil
|
||||
}
|
||||
|
||||
const htmlPage = `<html><head><title>Почему небо синее</title>
|
||||
<style>body{color:red}</style><script>track()</script></head>
|
||||
<body><nav>меню</nav><h1>Небо</h1>
|
||||
<p>Свет рассеивается на молекулах воздуха.</p>
|
||||
<p>Короткие волны рассеиваются сильнее.</p>
|
||||
<footer>© 2026</footer></body></html>`
|
||||
|
||||
func newTestCrawler(f *fakeFetcher) *Crawler {
|
||||
return New(f, Config{UserAgent: "Maven/1.0", Now: func() time.Time { return time.Unix(0, 0) }})
|
||||
}
|
||||
|
||||
func TestPageExtractsText(t *testing.T) {
|
||||
f := &fakeFetcher{pages: map[string]Response{
|
||||
"https://example.org/sky": {Body: []byte(htmlPage)},
|
||||
}}
|
||||
page, err := newTestCrawler(f).Page(context.Background(), "https://example.org/sky")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if page.Title != "Почему небо синее" {
|
||||
t.Errorf("title = %q", page.Title)
|
||||
}
|
||||
if !strings.Contains(page.Text, "Свет рассеивается") {
|
||||
t.Errorf("body text missing: %q", page.Text)
|
||||
}
|
||||
for _, junk := range []string{"track()", "color:red", "меню", "© 2026"} {
|
||||
if strings.Contains(page.Text, junk) {
|
||||
t.Errorf("%q survived extraction: %q", junk, page.Text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRobotsIsCheckedAndObeyed(t *testing.T) {
|
||||
f := &fakeFetcher{pages: map[string]Response{
|
||||
"https://example.org/robots.txt": {Body: []byte("User-agent: *\nDisallow: /secret\n"), ContentType: "text/plain"},
|
||||
"https://example.org/secret/x": {Body: []byte(htmlPage)},
|
||||
"https://example.org/open": {Body: []byte(htmlPage)},
|
||||
}}
|
||||
c := newTestCrawler(f)
|
||||
if _, err := c.Page(context.Background(), "https://example.org/secret/x"); !errors.Is(err, ErrRobots) {
|
||||
t.Fatalf("error = %v, want ErrRobots", err)
|
||||
}
|
||||
for _, u := range f.calls {
|
||||
if strings.Contains(u, "/secret") {
|
||||
t.Fatal("the disallowed page was fetched anyway")
|
||||
}
|
||||
}
|
||||
if _, err := c.Page(context.Background(), "https://example.org/open"); err != nil {
|
||||
t.Fatalf("allowed page: %v", err)
|
||||
}
|
||||
// robots.txt was read once for the host, not once per page.
|
||||
robotsReads := 0
|
||||
for _, u := range f.calls {
|
||||
if strings.HasSuffix(u, "/robots.txt") {
|
||||
robotsReads++
|
||||
}
|
||||
}
|
||||
if robotsReads != 1 {
|
||||
t.Fatalf("robots.txt read %d times, want 1", robotsReads)
|
||||
}
|
||||
}
|
||||
|
||||
// No robots.txt means allow — that is the standard, and the alternative makes
|
||||
// most of the web unreadable.
|
||||
func TestMissingRobotsAllows(t *testing.T) {
|
||||
f := &fakeFetcher{pages: map[string]Response{
|
||||
"https://example.org/page": {Body: []byte(htmlPage)},
|
||||
}}
|
||||
if _, err := newTestCrawler(f).Page(context.Background(), "https://example.org/page"); err != nil {
|
||||
t.Fatalf("err = %v, want the page", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A refusal from the guarded fetcher must surface as itself, not be laundered
|
||||
// into "no robots.txt, go ahead".
|
||||
func TestFetcherRefusalIsNotSwallowed(t *testing.T) {
|
||||
f := &fakeFetcher{err: errors.New("webfetch: refusing to connect to a private address: 127.0.0.1")}
|
||||
_, err := newTestCrawler(f).Page(context.Background(), "http://127.0.0.1:9100/mcp")
|
||||
if err == nil || !strings.Contains(err.Error(), "private address") {
|
||||
t.Fatalf("error = %v, want the fetcher's refusal", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNonTextIsRefused(t *testing.T) {
|
||||
f := &fakeFetcher{pages: map[string]Response{
|
||||
"https://example.org/f.pdf": {Body: []byte("%PDF-1.7"), ContentType: "application/pdf"},
|
||||
}}
|
||||
if _, err := newTestCrawler(f).Page(context.Background(), "https://example.org/f.pdf"); !errors.Is(err, ErrNotHTML) {
|
||||
t.Fatalf("error = %v, want ErrNotHTML", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaxRunesCapsText(t *testing.T) {
|
||||
long := "<html><body><p>" + strings.Repeat("привет ", 2000) + "</p></body></html>"
|
||||
f := &fakeFetcher{pages: map[string]Response{"https://example.org/l": {Body: []byte(long)}}}
|
||||
c := New(f, Config{MaxRunes: 50})
|
||||
page, err := c.Page(context.Background(), "https://example.org/l")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := len([]rune(page.Text)); n > 51 {
|
||||
t.Fatalf("text = %d runes, want the 50-rune cap", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewWithoutFetcherIsNil(t *testing.T) {
|
||||
if New(nil, Config{}) != nil {
|
||||
t.Fatal("a crawler with no fetcher must be nil — crawling is off unless configured")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHashIgnoresNothingButText(t *testing.T) {
|
||||
if Hash("a") == Hash("b") {
|
||||
t.Fatal("different text hashed the same")
|
||||
}
|
||||
if Hash(" same \n") != Hash("same") {
|
||||
t.Fatal("surrounding whitespace changed the hash")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package crawl
|
||||
|
||||
import (
|
||||
"html"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// HTML → text, with a regexp and no tokenizer.
|
||||
//
|
||||
// golang.org/x/net/html is not vendored and the network is not assumed, so this
|
||||
// is stdlib. That is less of a compromise than it sounds: the unit of context
|
||||
// here is a few hundred words for a 4096-token model to read, exactly like the
|
||||
// Kiwix snippet, so what matters is dropping script/style/nav noise and keeping
|
||||
// paragraph boundaries. A DOM would buy correctness on malformed markup that is
|
||||
// then thrown away by truncation anyway.
|
||||
//
|
||||
// What this deliberately does NOT do: run JavaScript, follow links, or extract
|
||||
// structured fields with CSS selectors or an LLM prompt. The plan's step 2 asked
|
||||
// for the last of those; see docs/plans/14-web-crawler.md for why it was left
|
||||
// out for now.
|
||||
|
||||
var (
|
||||
// RE2 has no backreferences, so each tag pair is spelled out rather than
|
||||
// captured and matched against itself.
|
||||
dropRE = regexp.MustCompile(pairsRE("script", "style", "noscript", "svg", "head", "nav", "footer", "form"))
|
||||
titleRE = regexp.MustCompile(`(?is)<title\b[^>]*>(.*?)</title>`)
|
||||
h1RE = regexp.MustCompile(`(?is)<h1\b[^>]*>(.*?)</h1>`)
|
||||
// Block-level tags become newlines so paragraphs survive as paragraphs.
|
||||
blockRE = regexp.MustCompile(`(?is)</?(p|div|br|li|tr|h[1-6]|section|article|blockquote|pre)\b[^>]*>`)
|
||||
tagRE = regexp.MustCompile(`(?s)<[^>]*>`)
|
||||
commentRE = regexp.MustCompile(`(?s)<!--.*?-->`)
|
||||
spaceRE = regexp.MustCompile(`[ \t\f\v]+`)
|
||||
blankRE = regexp.MustCompile(`\n{2,}`)
|
||||
)
|
||||
|
||||
// pairsRE builds `(?is)<tag …>…</tag>|…` for the given tags.
|
||||
func pairsRE(tags ...string) string {
|
||||
parts := make([]string, 0, len(tags))
|
||||
for _, t := range tags {
|
||||
parts = append(parts, `<`+t+`\b[^>]*>.*?</`+t+`>`)
|
||||
}
|
||||
return `(?is)` + strings.Join(parts, "|")
|
||||
}
|
||||
|
||||
// Page is an extracted page.
|
||||
type Page struct {
|
||||
URL string
|
||||
Title string
|
||||
Text string // plain text, paragraphs separated by single newlines
|
||||
}
|
||||
|
||||
// Extract turns a fetched HTML document into a Page. maxRunes caps the text (0 ⇒
|
||||
// DefaultMaxRunes); the cap is on runes, not bytes, because a Russian page cut
|
||||
// at a byte boundary ends in half a letter.
|
||||
func Extract(url string, body []byte, maxRunes int) Page {
|
||||
if maxRunes <= 0 {
|
||||
maxRunes = DefaultMaxRunes
|
||||
}
|
||||
s := string(body)
|
||||
s = commentRE.ReplaceAllString(s, " ")
|
||||
|
||||
title := firstGroup(titleRE, s)
|
||||
if title == "" {
|
||||
title = firstGroup(h1RE, s)
|
||||
}
|
||||
|
||||
s = dropRE.ReplaceAllString(s, "\n")
|
||||
s = blockRE.ReplaceAllString(s, "\n")
|
||||
s = tagRE.ReplaceAllString(s, " ")
|
||||
s = html.UnescapeString(s)
|
||||
s = spaceRE.ReplaceAllString(s, " ")
|
||||
|
||||
var lines []string
|
||||
for _, l := range strings.Split(s, "\n") {
|
||||
if l = strings.TrimSpace(l); l != "" {
|
||||
lines = append(lines, l)
|
||||
}
|
||||
}
|
||||
text := blankRE.ReplaceAllString(strings.Join(lines, "\n"), "\n")
|
||||
|
||||
return Page{URL: url, Title: title, Text: TrimRunes(text, maxRunes)}
|
||||
}
|
||||
|
||||
// DefaultMaxRunes — how much of a page is kept. ~4000 runes is a long answer's
|
||||
// worth of context and still leaves room in a 4096-token window for the prompt
|
||||
// and the reply.
|
||||
const DefaultMaxRunes = 4000
|
||||
|
||||
func firstGroup(re *regexp.Regexp, s string) string {
|
||||
m := re.FindStringSubmatch(s)
|
||||
if len(m) < 2 {
|
||||
return ""
|
||||
}
|
||||
t := tagRE.ReplaceAllString(m[1], " ")
|
||||
return strings.TrimSpace(strings.Join(strings.Fields(html.UnescapeString(t)), " "))
|
||||
}
|
||||
|
||||
// TrimRunes cuts s to at most max runes, on a rune boundary.
|
||||
func TrimRunes(s string, max int) string {
|
||||
r := []rune(s)
|
||||
if len(r) <= max {
|
||||
return s
|
||||
}
|
||||
return strings.TrimSpace(string(r[:max])) + "…"
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
package crawl
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// robots.txt, parsed the small way: no wildcards beyond the two the standard
|
||||
// actually defines (`*` inside a path and `$` at the end), no sitemaps, no
|
||||
// crawl-delay-per-agent gymnastics. A personal assistant reading a handful of
|
||||
// pages does not need a spec-complete implementation; it needs to not be rude,
|
||||
// and to be auditable in one sitting.
|
||||
//
|
||||
// Two rules worth stating because they are choices, not accidents:
|
||||
//
|
||||
// - a missing or unreadable robots.txt means ALLOW. That is what the standard
|
||||
// says (404 ⇒ unrestricted), and the alternative would make a site that
|
||||
// simply has no robots.txt unreadable;
|
||||
// - an explicit Disallow means REFUSE, and Maven does not offer an override.
|
||||
// There is no "but he asked me to" flag: the page is not read.
|
||||
|
||||
// Rules is a parsed robots.txt for one user-agent.
|
||||
type Rules struct {
|
||||
allow []string
|
||||
disallow []string
|
||||
// Delay is Crawl-delay in seconds when the group named one, 0 otherwise.
|
||||
// The fetcher's own per-host rate limit is the floor; this can only make
|
||||
// Maven slower, never faster.
|
||||
Delay time.Duration
|
||||
}
|
||||
|
||||
// ParseRobots reads robots.txt and returns the rules that apply to agent.
|
||||
//
|
||||
// Group selection follows the standard: the most specific matching group wins,
|
||||
// which here means an exact user-agent match beats `*`. Lines that are neither
|
||||
// are ignored rather than guessed at.
|
||||
func ParseRobots(body string, agent string) Rules {
|
||||
agent = strings.ToLower(agent)
|
||||
|
||||
type group struct {
|
||||
agents []string
|
||||
allow []string
|
||||
disallow []string
|
||||
delay time.Duration
|
||||
}
|
||||
var groups []group
|
||||
var cur *group
|
||||
// startNew tracks whether the next User-agent line opens a new group or
|
||||
// joins the current one: consecutive User-agent lines share their rules.
|
||||
startNew := true
|
||||
|
||||
for _, raw := range strings.Split(body, "\n") {
|
||||
line := raw
|
||||
if i := strings.IndexByte(line, '#'); i >= 0 {
|
||||
line = line[:i]
|
||||
}
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
key, val, ok := strings.Cut(line, ":")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
key = strings.ToLower(strings.TrimSpace(key))
|
||||
val = strings.TrimSpace(val)
|
||||
|
||||
switch key {
|
||||
case "user-agent":
|
||||
if startNew || cur == nil {
|
||||
groups = append(groups, group{})
|
||||
cur = &groups[len(groups)-1]
|
||||
startNew = false
|
||||
}
|
||||
cur.agents = append(cur.agents, strings.ToLower(val))
|
||||
case "disallow":
|
||||
if cur == nil {
|
||||
continue
|
||||
}
|
||||
startNew = true
|
||||
// "Disallow:" with an empty value allows everything, and is not a
|
||||
// path rule at all.
|
||||
if val != "" {
|
||||
cur.disallow = append(cur.disallow, val)
|
||||
}
|
||||
case "allow":
|
||||
if cur == nil {
|
||||
continue
|
||||
}
|
||||
startNew = true
|
||||
if val != "" {
|
||||
cur.allow = append(cur.allow, val)
|
||||
}
|
||||
case "crawl-delay":
|
||||
if cur == nil {
|
||||
continue
|
||||
}
|
||||
startNew = true
|
||||
if d, err := time.ParseDuration(val + "s"); err == nil && d > 0 {
|
||||
cur.delay = d
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var star, exact *group
|
||||
for i := range groups {
|
||||
for _, a := range groups[i].agents {
|
||||
if a == "*" && star == nil {
|
||||
star = &groups[i]
|
||||
}
|
||||
// A robots.txt names "maven", we send "Maven/1.0 (…)": match on
|
||||
// prefix, which is how every crawler reads this field.
|
||||
if a != "*" && a != "" && strings.HasPrefix(agent, a) {
|
||||
exact = &groups[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
g := exact
|
||||
if g == nil {
|
||||
g = star
|
||||
}
|
||||
if g == nil {
|
||||
return Rules{}
|
||||
}
|
||||
return Rules{allow: g.allow, disallow: g.disallow, Delay: g.delay}
|
||||
}
|
||||
|
||||
// Allowed reports whether path may be fetched. Longest matching rule wins, and
|
||||
// Allow beats Disallow at equal length — the standard's tie-break, and the one
|
||||
// that makes "Disallow: /" plus "Allow: /public" mean what it looks like.
|
||||
func (r Rules) Allowed(path string) bool {
|
||||
if path == "" {
|
||||
path = "/"
|
||||
}
|
||||
best, allowed := -1, true
|
||||
for _, p := range r.disallow {
|
||||
if n, ok := matchPath(p, path); ok && n > best {
|
||||
best, allowed = n, false
|
||||
}
|
||||
}
|
||||
for _, p := range r.allow {
|
||||
if n, ok := matchPath(p, path); ok && n >= best {
|
||||
best, allowed = n, true
|
||||
}
|
||||
}
|
||||
return allowed
|
||||
}
|
||||
|
||||
// matchPath applies a robots path pattern and returns the pattern's length as
|
||||
// the specificity score. `*` matches any run of characters, `$` anchors the end.
|
||||
// A pattern is a PREFIX match otherwise, which is what "Disallow: /admin" means.
|
||||
func matchPath(pattern, path string) (int, bool) {
|
||||
score := len(pattern)
|
||||
re, err := robotsRegexp(pattern)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
return score, re.MatchString(path)
|
||||
}
|
||||
|
||||
// robotsRegexp turns a robots path pattern into an anchored-at-the-start
|
||||
// regexp. Everything but `*` and a trailing `$` is a literal, so the pattern is
|
||||
// quoted first and the two metacharacters are put back afterwards.
|
||||
func robotsRegexp(pattern string) (*regexp.Regexp, error) {
|
||||
end := ""
|
||||
if strings.HasSuffix(pattern, "$") {
|
||||
pattern = strings.TrimSuffix(pattern, "$")
|
||||
end = "$"
|
||||
}
|
||||
parts := strings.Split(pattern, "*")
|
||||
for i, p := range parts {
|
||||
parts[i] = regexp.QuoteMeta(p)
|
||||
}
|
||||
return regexp.Compile("^" + strings.Join(parts, ".*") + end)
|
||||
}
|
||||
|
||||
// robotsCache holds parsed rules per host so a crawl of ten pages on one site
|
||||
// reads robots.txt once. TTL because a site may change its mind, and a daemon
|
||||
// that runs for weeks would otherwise never notice.
|
||||
type robotsCache struct {
|
||||
ttl time.Duration
|
||||
mu sync.Mutex
|
||||
m map[string]robotsEntry
|
||||
}
|
||||
|
||||
type robotsEntry struct {
|
||||
rules Rules
|
||||
at time.Time
|
||||
}
|
||||
|
||||
func newRobotsCache(ttl time.Duration) *robotsCache {
|
||||
return &robotsCache{ttl: ttl, m: map[string]robotsEntry{}}
|
||||
}
|
||||
|
||||
func (c *robotsCache) get(host string, now time.Time) (Rules, bool) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
e, ok := c.m[host]
|
||||
if !ok || now.Sub(e.at) > c.ttl {
|
||||
return Rules{}, false
|
||||
}
|
||||
return e.rules, true
|
||||
}
|
||||
|
||||
func (c *robotsCache) put(host string, r Rules, now time.Time) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.m[host] = robotsEntry{rules: r, at: now}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package crawl
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const robotsBody = `# a comment
|
||||
User-agent: *
|
||||
Disallow: /private
|
||||
Disallow: /tmp/
|
||||
Crawl-delay: 5
|
||||
|
||||
User-agent: Maven
|
||||
Disallow: /
|
||||
Allow: /public
|
||||
`
|
||||
|
||||
func TestParseRobotsPicksTheMostSpecificGroup(t *testing.T) {
|
||||
// The Maven group applies to us even though we send a longer UA string.
|
||||
r := ParseRobots(robotsBody, "Maven/1.0 (self-hosted personal assistant)")
|
||||
if r.Allowed("/anything") {
|
||||
t.Error("Disallow: / in our own group was ignored")
|
||||
}
|
||||
if !r.Allowed("/public/page") {
|
||||
t.Error("Allow: /public must beat the shorter Disallow: /")
|
||||
}
|
||||
|
||||
// A different agent falls into the * group.
|
||||
star := ParseRobots(robotsBody, "SomeoneElse/2")
|
||||
if !star.Allowed("/anything") {
|
||||
t.Error("the * group disallows nothing but /private and /tmp/")
|
||||
}
|
||||
if star.Allowed("/private/x") || star.Allowed("/tmp/") {
|
||||
t.Error("the * group's disallows were not applied")
|
||||
}
|
||||
if star.Delay != 5*time.Second {
|
||||
t.Errorf("crawl-delay = %v, want 5s", star.Delay)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRobotsEmptyMeansAllowAll(t *testing.T) {
|
||||
for _, body := range []string{"", "# nothing here\n", "User-agent: *\nDisallow:\n"} {
|
||||
if !ParseRobots(body, "Maven").Allowed("/whatever") {
|
||||
t.Errorf("body %q must allow everything", body)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRobotsWildcards(t *testing.T) {
|
||||
r := ParseRobots("User-agent: *\nDisallow: /*.pdf$\nDisallow: /a/*/secret\n", "Maven")
|
||||
if r.Allowed("/docs/manual.pdf") {
|
||||
t.Error("*.pdf$ did not match")
|
||||
}
|
||||
if !r.Allowed("/docs/manual.pdf.html") {
|
||||
t.Error("$ must anchor at the end")
|
||||
}
|
||||
if r.Allowed("/a/b/secret") {
|
||||
t.Error("/a/*/secret did not match")
|
||||
}
|
||||
if !r.Allowed("/a/b/public") {
|
||||
t.Error("unrelated path was refused")
|
||||
}
|
||||
}
|
||||
|
||||
// Consecutive User-agent lines share one group, which is common in the wild.
|
||||
func TestRobotsSharedGroup(t *testing.T) {
|
||||
r := ParseRobots("User-agent: Googlebot\nUser-agent: Maven\nDisallow: /x\n", "Maven/1.0")
|
||||
if r.Allowed("/x/y") {
|
||||
t.Fatal("a shared group's rules were not applied to the second agent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRobotsCacheTTL(t *testing.T) {
|
||||
c := newRobotsCache(time.Minute)
|
||||
now := time.Now()
|
||||
c.put("example.com", ParseRobots("User-agent: *\nDisallow: /\n", "Maven"), now)
|
||||
if _, ok := c.get("example.com", now.Add(30*time.Second)); !ok {
|
||||
t.Error("a fresh entry must be served from cache")
|
||||
}
|
||||
if _, ok := c.get("example.com", now.Add(2*time.Minute)); ok {
|
||||
t.Error("an expired entry must be re-read")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
package crawl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Scheduled crawls: a page is re-read on an interval, and when its TEXT changed
|
||||
// the new text is written as a note. Nothing is dispatched — same rule as the
|
||||
// feed poller (Vikunja #258). A page that announced its own change would be a
|
||||
// nag, and "the docs page changed" is not worth interrupting anyone for.
|
||||
//
|
||||
// Dedup is by content hash, so a page that re-renders identically writes nothing
|
||||
// and a rotating ad slot does not count as news.
|
||||
|
||||
// WatchConfig — one page to keep an eye on.
|
||||
type WatchConfig struct {
|
||||
Name string // note source is "crawl:<Name>"
|
||||
URL string // http(s), guarded by the fetcher
|
||||
Interval time.Duration // 0 ⇒ Watcher's default
|
||||
}
|
||||
|
||||
// Notes is core's note-writing half (same shape as ipc.CoreAPI's method).
|
||||
type Notes interface {
|
||||
WriteNote(ctx context.Context, ts time.Time, text string, embedding []float32, source string) (int64, error)
|
||||
}
|
||||
|
||||
// Hashes remembers the last text hash per watch, durably, so a restart does not
|
||||
// re-note an unchanged page. The daemon backs this with config facts
|
||||
// ("crawl:hash:<name>").
|
||||
type Hashes interface {
|
||||
LastHash(ctx context.Context, name string) (string, error)
|
||||
SetHash(ctx context.Context, name, hash string) error
|
||||
}
|
||||
|
||||
// Embedder embeds a note on its way into the store. nil ⇒ no vector.
|
||||
type Embedder interface {
|
||||
Embed(ctx context.Context, text string) ([]float32, error)
|
||||
}
|
||||
|
||||
// DefaultWatchInterval — pages change slowly, and every check is a request in
|
||||
// someone's log.
|
||||
const DefaultWatchInterval = 6 * time.Hour
|
||||
|
||||
// Watcher re-reads watched pages on their interval.
|
||||
type Watcher struct {
|
||||
c *Crawler
|
||||
watches []WatchConfig
|
||||
notes Notes
|
||||
hashes Hashes
|
||||
embed Embedder
|
||||
interval time.Duration
|
||||
nextDue map[string]time.Time
|
||||
}
|
||||
|
||||
// NewWatcher wires the scheduled half, or returns nil when there is nothing to
|
||||
// watch. Callers check for nil: no watches, no goroutine, no request.
|
||||
func NewWatcher(c *Crawler, watches []WatchConfig, notes Notes, hashes Hashes, embed Embedder, defaultInterval time.Duration) *Watcher {
|
||||
if c == nil || notes == nil {
|
||||
return nil
|
||||
}
|
||||
var valid []WatchConfig
|
||||
for _, w := range watches {
|
||||
if strings.TrimSpace(w.Name) == "" || strings.TrimSpace(w.URL) == "" {
|
||||
log.Printf("crawl: skipping a watch with no name or no url")
|
||||
continue
|
||||
}
|
||||
valid = append(valid, w)
|
||||
}
|
||||
if len(valid) == 0 {
|
||||
return nil
|
||||
}
|
||||
if defaultInterval <= 0 {
|
||||
defaultInterval = DefaultWatchInterval
|
||||
}
|
||||
return &Watcher{
|
||||
c: c, watches: valid, notes: notes, hashes: hashes, embed: embed,
|
||||
interval: defaultInterval, nextDue: map[string]time.Time{},
|
||||
}
|
||||
}
|
||||
|
||||
// Watches returns the configured watches.
|
||||
func (w *Watcher) Watches() []WatchConfig { return w.watches }
|
||||
|
||||
// CheckDue re-reads every watch whose interval elapsed and returns how many
|
||||
// notes were written. Errors are logged per watch, never returned: one dead page
|
||||
// must not stop the others.
|
||||
func (w *Watcher) CheckDue(ctx context.Context, now time.Time) int {
|
||||
written := 0
|
||||
for _, watch := range w.watches {
|
||||
if due, ok := w.nextDue[watch.Name]; ok && now.Before(due) {
|
||||
continue
|
||||
}
|
||||
interval := watch.Interval
|
||||
if interval <= 0 {
|
||||
interval = w.interval
|
||||
}
|
||||
w.nextDue[watch.Name] = now.Add(interval)
|
||||
changed, err := w.Check(ctx, watch, now)
|
||||
if err != nil {
|
||||
log.Printf("crawl: watch %s: %v", watch.Name, err)
|
||||
continue
|
||||
}
|
||||
if changed {
|
||||
log.Printf("crawl: watch %s: page changed, noted", watch.Name)
|
||||
written++
|
||||
}
|
||||
}
|
||||
return written
|
||||
}
|
||||
|
||||
// Check re-reads one watch now and reports whether it wrote a note.
|
||||
func (w *Watcher) Check(ctx context.Context, watch WatchConfig, now time.Time) (bool, error) {
|
||||
page, err := w.c.Page(ctx, watch.URL)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
// Title included: a page whose headline changed has changed.
|
||||
h := Hash(page.Title + "\n" + page.Text)
|
||||
if w.hashes != nil {
|
||||
prev, err := w.hashes.LastHash(ctx, watch.Name)
|
||||
if err != nil {
|
||||
log.Printf("crawl: watch %s: read hash: %v", watch.Name, err)
|
||||
}
|
||||
if prev == h {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
text := NoteText(watch, page)
|
||||
var vec []float32
|
||||
if w.embed != nil {
|
||||
v, err := w.embed.Embed(ctx, text)
|
||||
if err != nil {
|
||||
log.Printf("crawl: watch %s: embed: %v", watch.Name, err)
|
||||
} else {
|
||||
vec = v
|
||||
}
|
||||
}
|
||||
if _, err := w.notes.WriteNote(ctx, now, text, vec, SourceFor(watch.Name)); err != nil {
|
||||
return false, fmt.Errorf("write note: %w", err)
|
||||
}
|
||||
if w.hashes != nil {
|
||||
if err := w.hashes.SetHash(ctx, watch.Name, h); err != nil {
|
||||
log.Printf("crawl: watch %s: save hash: %v", watch.Name, err)
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// SourceFor is the note source for a watch, and SourcePrefix is what the answer
|
||||
// path matches to recognise one.
|
||||
func SourceFor(name string) string { return SourcePrefix + name }
|
||||
|
||||
// SourcePrefix — provenance for anything read off the network on a schedule.
|
||||
const SourcePrefix = "crawl:"
|
||||
|
||||
// noteRunes — how much of a watched page goes into a note. Shorter than what the
|
||||
// on-demand path reads: a note is a record of a change, not an archive.
|
||||
const noteRunes = 800
|
||||
|
||||
// NoteText renders a watched page as a note body.
|
||||
func NoteText(watch WatchConfig, page Page) string {
|
||||
var b strings.Builder
|
||||
if page.Title != "" {
|
||||
b.WriteString(page.Title)
|
||||
} else {
|
||||
b.WriteString(watch.Name)
|
||||
}
|
||||
b.WriteString("\n")
|
||||
b.WriteString(TrimRunes(page.Text, noteRunes))
|
||||
b.WriteString("\n")
|
||||
b.WriteString(watch.URL)
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package crawl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type note struct {
|
||||
text string
|
||||
source string
|
||||
}
|
||||
|
||||
type fakeNotes struct{ notes []note }
|
||||
|
||||
func (n *fakeNotes) WriteNote(_ context.Context, _ time.Time, text string, _ []float32, source string) (int64, error) {
|
||||
n.notes = append(n.notes, note{text, source})
|
||||
return int64(len(n.notes)), nil
|
||||
}
|
||||
|
||||
type fakeHashes struct{ m map[string]string }
|
||||
|
||||
func newHashes() *fakeHashes { return &fakeHashes{m: map[string]string{}} }
|
||||
func (f *fakeHashes) LastHash(_ context.Context, name string) (string, error) {
|
||||
return f.m[name], nil
|
||||
}
|
||||
func (f *fakeHashes) SetHash(_ context.Context, name, h string) error { f.m[name] = h; return nil }
|
||||
|
||||
var t0 = time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC)
|
||||
|
||||
func TestWatchNotesAChangedPage(t *testing.T) {
|
||||
f := &fakeFetcher{pages: map[string]Response{
|
||||
"https://example.org/docs": {Body: []byte(htmlPage)},
|
||||
}}
|
||||
notes := &fakeNotes{}
|
||||
hashes := newHashes()
|
||||
w := NewWatcher(newTestCrawler(f), []WatchConfig{{Name: "docs", URL: "https://example.org/docs"}},
|
||||
notes, hashes, nil, time.Hour)
|
||||
if w == nil {
|
||||
t.Fatal("NewWatcher returned nil for a configured watch")
|
||||
}
|
||||
if n := w.CheckDue(context.Background(), t0); n != 1 {
|
||||
t.Fatalf("first check wrote %d notes, want 1", n)
|
||||
}
|
||||
if notes.notes[0].source != "crawl:docs" {
|
||||
t.Errorf("source = %q, want crawl:docs", notes.notes[0].source)
|
||||
}
|
||||
if !strings.Contains(notes.notes[0].text, "https://example.org/docs") {
|
||||
t.Errorf("note does not carry the url: %q", notes.notes[0].text)
|
||||
}
|
||||
|
||||
// Unchanged page, interval elapsed: nothing written.
|
||||
if n := w.CheckDue(context.Background(), t0.Add(2*time.Hour)); n != 0 {
|
||||
t.Fatalf("an unchanged page wrote %d notes", n)
|
||||
}
|
||||
|
||||
// Changed page: one note.
|
||||
f.pages["https://example.org/docs"] = Response{Body: []byte(strings.Replace(htmlPage, "синее", "серое", 1))}
|
||||
if n := w.CheckDue(context.Background(), t0.Add(4*time.Hour)); n != 1 {
|
||||
t.Fatalf("a changed page wrote %d notes, want 1", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchIntervalIsRespected(t *testing.T) {
|
||||
f := &fakeFetcher{pages: map[string]Response{"https://example.org/d": {Body: []byte(htmlPage)}}}
|
||||
w := NewWatcher(newTestCrawler(f), []WatchConfig{{Name: "d", URL: "https://example.org/d", Interval: time.Hour}},
|
||||
&fakeNotes{}, newHashes(), nil, 0)
|
||||
w.CheckDue(context.Background(), t0)
|
||||
before := len(f.calls)
|
||||
w.CheckDue(context.Background(), t0.Add(time.Minute))
|
||||
if len(f.calls) != before {
|
||||
t.Fatal("the page was re-read inside its interval")
|
||||
}
|
||||
}
|
||||
|
||||
// The hash is durable so a restart does not re-note an unchanged page.
|
||||
func TestWatchHashSurvivesRestart(t *testing.T) {
|
||||
f := &fakeFetcher{pages: map[string]Response{"https://example.org/d": {Body: []byte(htmlPage)}}}
|
||||
hashes := newHashes()
|
||||
watches := []WatchConfig{{Name: "d", URL: "https://example.org/d"}}
|
||||
NewWatcher(newTestCrawler(f), watches, &fakeNotes{}, hashes, nil, time.Hour).CheckDue(context.Background(), t0)
|
||||
|
||||
notes2 := &fakeNotes{}
|
||||
NewWatcher(newTestCrawler(f), watches, notes2, hashes, nil, time.Hour).CheckDue(context.Background(), t0.Add(time.Hour))
|
||||
if len(notes2.notes) != 0 {
|
||||
t.Fatalf("a fresh watcher re-noted an unchanged page: %q", notes2.notes[0].text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchDeadPageDoesNotStopTheOthers(t *testing.T) {
|
||||
f := &fakeFetcher{pages: map[string]Response{"https://example.org/live": {Body: []byte(htmlPage)}}}
|
||||
notes := &fakeNotes{}
|
||||
w := NewWatcher(newTestCrawler(f), []WatchConfig{
|
||||
{Name: "dead", URL: "https://example.org/gone"},
|
||||
{Name: "live", URL: "https://example.org/live"},
|
||||
}, notes, newHashes(), nil, time.Hour)
|
||||
if n := w.CheckDue(context.Background(), t0); n != 1 {
|
||||
t.Fatalf("wrote %d notes, want 1 (the live page)", n)
|
||||
}
|
||||
if notes.notes[0].source != "crawl:live" {
|
||||
t.Fatalf("source = %q", notes.notes[0].source)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoWatchesMeansNoWatcher(t *testing.T) {
|
||||
c := newTestCrawler(&fakeFetcher{})
|
||||
if NewWatcher(c, nil, &fakeNotes{}, nil, nil, 0) != nil {
|
||||
t.Fatal("no watches must mean no watcher")
|
||||
}
|
||||
if NewWatcher(nil, []WatchConfig{{Name: "a", URL: "u"}}, &fakeNotes{}, nil, nil, 0) != nil {
|
||||
t.Fatal("no crawler must mean no watcher")
|
||||
}
|
||||
if NewWatcher(c, []WatchConfig{{Name: "", URL: ""}}, &fakeNotes{}, nil, nil, 0) != nil {
|
||||
t.Fatal("a watch with no name or url is not a configuration")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
package email
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/llm"
|
||||
"github.com/kami/maven/internal/persona"
|
||||
)
|
||||
|
||||
// Extraction — turning one mail into task CANDIDATES, and nothing else.
|
||||
//
|
||||
// The output of this file can only ever become rows in `tasks` with status
|
||||
// "candidate" (store.TaskCandidate), written through the one intake seam
|
||||
// (ipc.CaptureTaskReq, Vikunja #130). That bound is the whole design:
|
||||
//
|
||||
// - No reminder. A reminder FIRES; it speaks to him unprompted. A 1.7B that
|
||||
// misreads "встреча была в четверг" as a future appointment would then wake
|
||||
// him up about it. A candidate that is wrong is a line on a review page he
|
||||
// dismisses in one click, which is the correct cost of a model being wrong
|
||||
// about someone's mail.
|
||||
// - No fact. A fact is a claim Maven will later recite as true. Nothing read
|
||||
// out of a marketing mail deserves that standing.
|
||||
// - No calendar event, no note, no action. Extraction writes candidates or
|
||||
// writes nothing.
|
||||
//
|
||||
// The due date the model may return is stored on the candidate (tasks.due_ts),
|
||||
// which no scheduler reads — it is there so the review page can sort by it.
|
||||
//
|
||||
// Privacy: the mail text goes to the resident model on this box and nowhere
|
||||
// else. It is never search input (CLAUDE.md: "his notes and facts are never
|
||||
// search input" — mail is the same class), and Evidence keeps only the subject
|
||||
// line, so the review page shows him where a candidate came from without the
|
||||
// store growing a copy of his mailbox.
|
||||
|
||||
// MaxCandidates — at most this many candidates per message, enforced by the
|
||||
// grammar. A mail with four tasks in it is a mail he has to read himself; a
|
||||
// model allowed ten will produce ten.
|
||||
const MaxCandidates = 3
|
||||
|
||||
// SourcePrefix — provenance for everything this package captures. The mailbox
|
||||
// name is appended: "email:INBOX". Same vocabulary as tap:voice / poll:netdata.
|
||||
const SourcePrefix = "email:"
|
||||
|
||||
// Candidate — one piece of work the model thinks the mail is asking for.
|
||||
type Candidate struct {
|
||||
Text string `json:"text"`
|
||||
// Due — "YYYY-MM-DD" or empty. A date the model read out of the text, not a
|
||||
// date it computed: relative wording ("до пятницы") is left in Text, because
|
||||
// a small model resolving "пятница" against today's date gets it wrong often
|
||||
// enough that a stored wrong date is worse than no date.
|
||||
Due string `json:"due"`
|
||||
}
|
||||
|
||||
// Completer — the llama-server seam, same shape memeval and the router use, so
|
||||
// the one resident model serves this caller too.
|
||||
type Completer interface {
|
||||
Complete(ctx context.Context, r llm.Req) (string, error)
|
||||
}
|
||||
|
||||
// Extractor reads a message and returns candidates. It holds no store and no
|
||||
// writer on purpose: this type cannot persist anything, so "extraction never
|
||||
// acts" is a property of the code, not of a review.
|
||||
type Extractor struct {
|
||||
llm Completer
|
||||
// MaxCandidates — 0 ⇒ MaxCandidates.
|
||||
max int
|
||||
// ContextBlock — the shared persona block, optional. Extraction output is
|
||||
// not spoken, so the persona matters less here than in the phraser; it is
|
||||
// wired anyway so a candidate reads in her voice on the review page.
|
||||
contextBlock func() string
|
||||
}
|
||||
|
||||
func NewExtractor(c Completer, max int, contextBlock func() string) *Extractor {
|
||||
if max <= 0 || max > MaxCandidates {
|
||||
max = MaxCandidates
|
||||
}
|
||||
return &Extractor{llm: c, max: max, contextBlock: contextBlock}
|
||||
}
|
||||
|
||||
// extractGrammar — GBNF pinning the answer to a bounded array of fixed-shape
|
||||
// candidates. Same reasoning as memeval's evalGrammar and the router's
|
||||
// routeGrammar: the shape and the length bound are what keep a small model from
|
||||
// drifting into prose or spending the token budget repeating one field.
|
||||
//
|
||||
// The empty array is reachable, deliberately: most mail contains no task, and a
|
||||
// model with no way to say "nothing" invents something.
|
||||
const extractGrammar = `
|
||||
root ::= "[" ws (item ("," ws item){0,2})? ws "]"
|
||||
item ::= "{" ws "\"text\"" ws ":" ws text "," ws "\"due\"" ws ":" ws due ws "}"
|
||||
text ::= "\"" ([^"\\] | "\\" .){1,120} "\""
|
||||
due ::= "\"\"" | "\"" [0-9]{4} "-" [0-9]{2} "-" [0-9]{2} "\""
|
||||
ws ::= [ \t\n]*
|
||||
`
|
||||
|
||||
// extractSystem — the extraction prompt.
|
||||
//
|
||||
// Written around the two failure modes a small model has on this task: it
|
||||
// summarises when asked to extract (turning a mail into "письмо от Антона"),
|
||||
// and it invents an obligation from any polite closing sentence. Hence the
|
||||
// insistence on a verb phrase, and the explicit permission to return [].
|
||||
const extractSystem = `Ты читаешь одно письмо из его почты и достаёшь из него дела, которые письмо от него требует.
|
||||
|
||||
Правила:
|
||||
- Отвечай ТОЛЬКО массивом JSON. Каждый элемент: {"text": "...", "due": "ГГГГ-ММ-ДД" или ""}.
|
||||
- text — короткая формулировка дела по-русски, с глаголом: "оплатить счёт за интернет", "отправить акт". Не пересказывай письмо и не описывай его.
|
||||
- Дело — это то, что должен сделать ОН. Рассылка, реклама, уведомление, отчёт, письмо «просто к сведению» — дел не содержат.
|
||||
- Если письмо ничего от него не требует, верни пустой массив []. Это нормальный ответ, так бывает чаще всего.
|
||||
- Ничего не придумывай. Если срока в письме нет — "".
|
||||
- due заполняй только когда в письме стоит конкретная дата. Слова вроде «до пятницы» оставь в text, дату не вычисляй.
|
||||
- Максимум три дела. Лучше одно точное, чем три общих.`
|
||||
|
||||
// Extract returns the candidates in one message.
|
||||
//
|
||||
// Junk is refused without an LLM call — cheapest possible defence, and the
|
||||
// reason the header filter exists. An empty message (no subject, no body) is
|
||||
// likewise not worth a round trip.
|
||||
//
|
||||
// A parse failure is an error the caller logs and moves past. It is never
|
||||
// silently turned into zero candidates, because "the model went off the rails"
|
||||
// and "the mail contains no task" want different reactions from a human reading
|
||||
// the log.
|
||||
func (e *Extractor) Extract(ctx context.Context, msg Message) ([]Candidate, error) {
|
||||
if msg.Junk {
|
||||
return nil, nil
|
||||
}
|
||||
user := renderForModel(msg)
|
||||
if user == "" {
|
||||
return nil, nil
|
||||
}
|
||||
raw, err := e.llm.Complete(ctx, llm.Req{
|
||||
System: persona.Prepend(e.contextBlock, extractSystem),
|
||||
User: user,
|
||||
Grammar: extractGrammar,
|
||||
MaxTokens: 512,
|
||||
RepeatPenalty: 1.1,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("email: extract: %w", err)
|
||||
}
|
||||
items, err := parseCandidates(raw)
|
||||
if err != nil {
|
||||
// The raw reply is NOT in the error: it is a transformation of his mail,
|
||||
// and this error reaches the daemon log.
|
||||
return nil, fmt.Errorf("email: extract: unparsable reply (%d bytes)", len(raw))
|
||||
}
|
||||
out := make([]Candidate, 0, len(items))
|
||||
seen := map[string]bool{}
|
||||
for _, it := range items {
|
||||
it.Text = strings.TrimSpace(it.Text)
|
||||
if it.Text == "" {
|
||||
continue
|
||||
}
|
||||
key := strings.ToLower(strings.Join(strings.Fields(it.Text), " "))
|
||||
if seen[key] {
|
||||
continue // the model repeating itself is not two tasks
|
||||
}
|
||||
seen[key] = true
|
||||
if _, ok := ParseDue(it.Due); !ok {
|
||||
it.Due = "" // a date the grammar allowed but the calendar does not
|
||||
}
|
||||
out = append(out, it)
|
||||
if len(out) >= e.max {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// renderForModel is the user turn: subject, sender and body, labelled. Only
|
||||
// these three fields — no headers, no recipient list, no message-id, nothing
|
||||
// that would let the model start reasoning about routing metadata.
|
||||
func renderForModel(msg Message) string {
|
||||
var b strings.Builder
|
||||
if msg.From != "" {
|
||||
fmt.Fprintf(&b, "От: %s\n", msg.From)
|
||||
}
|
||||
if msg.Subject != "" {
|
||||
fmt.Fprintf(&b, "Тема: %s\n", msg.Subject)
|
||||
}
|
||||
if msg.Body != "" {
|
||||
fmt.Fprintf(&b, "\n%s\n", msg.Body)
|
||||
}
|
||||
if msg.Subject == "" && msg.Body == "" {
|
||||
return ""
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// parseCandidates decodes the grammar-constrained reply, tolerating the
|
||||
// wrappers a Thinking model sometimes leaves around it (a fenced block, or
|
||||
// leading reasoning before the array).
|
||||
func parseCandidates(raw string) ([]Candidate, error) {
|
||||
s := strings.TrimSpace(raw)
|
||||
if i := strings.Index(s, "["); i > 0 {
|
||||
s = s[i:]
|
||||
}
|
||||
if j := strings.LastIndex(s, "]"); j >= 0 {
|
||||
s = s[:j+1]
|
||||
}
|
||||
var out []Candidate
|
||||
if err := json.Unmarshal([]byte(s), &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ParseDue turns the model's "YYYY-MM-DD" into a time in UTC. Exported because
|
||||
// the daemon-side intake stores it on the candidate.
|
||||
//
|
||||
// The zero-value/empty case returns ok=false rather than an error: no date is
|
||||
// the common answer, not a failure.
|
||||
func ParseDue(s string) (time.Time, bool) {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return time.Time{}, false
|
||||
}
|
||||
t, err := time.Parse("2006-01-02", s)
|
||||
if err != nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return t, true
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
package email
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/kami/maven/internal/llm"
|
||||
)
|
||||
|
||||
// fakeLLM returns a canned reply and records the request, so a test can assert
|
||||
// on the grammar and on what of the mail was sent.
|
||||
type fakeLLM struct {
|
||||
reply string
|
||||
err error
|
||||
got llm.Req
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *fakeLLM) Complete(_ context.Context, r llm.Req) (string, error) {
|
||||
f.calls++
|
||||
f.got = r
|
||||
return f.reply, f.err
|
||||
}
|
||||
|
||||
func msgFor(subject, body string) Message {
|
||||
return Message{UID: 1, From: "anton@example.org", Subject: subject, Body: body}
|
||||
}
|
||||
|
||||
func TestExtractCandidates(t *testing.T) {
|
||||
f := &fakeLLM{reply: `[{"text":"отправить акт","due":""},{"text":"оплатить счёт","due":"2026-08-05"}]`}
|
||||
e := NewExtractor(f, 0, nil)
|
||||
got, err := e.Extract(context.Background(), msgFor("Акт и счёт", "Надо отправить акт и оплатить счёт до 5 августа."))
|
||||
if err != nil {
|
||||
t.Fatalf("extract: %v", err)
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("got %d candidates, want 2: %+v", len(got), got)
|
||||
}
|
||||
if got[0].Text != "отправить акт" || got[1].Due != "2026-08-05" {
|
||||
t.Errorf("candidates = %+v", got)
|
||||
}
|
||||
if f.got.Grammar == "" {
|
||||
t.Error("extraction must be grammar-constrained")
|
||||
}
|
||||
// The subject and body go to the model; nothing else about the message does.
|
||||
if !strings.Contains(f.got.User, "Акт и счёт") || !strings.Contains(f.got.User, "оплатить счёт") {
|
||||
t.Errorf("user turn = %q", f.got.User)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractEmptyArrayIsNotAnError(t *testing.T) {
|
||||
f := &fakeLLM{reply: "[]"}
|
||||
got, err := NewExtractor(f, 0, nil).Extract(context.Background(), msgFor("FYI", "Просто к сведению."))
|
||||
if err != nil || len(got) != 0 {
|
||||
t.Fatalf("got (%v, %v), want (empty, nil) — no task is the normal answer", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Junk must never reach the model: the header filter exists so the resident
|
||||
// model is not spent on newsletters.
|
||||
func TestExtractSkipsJunkWithoutCallingModel(t *testing.T) {
|
||||
f := &fakeLLM{reply: `[{"text":"купить всё со скидкой","due":""}]`}
|
||||
msg := msgFor("Скидки", "Sale!")
|
||||
msg.Junk = true
|
||||
got, err := NewExtractor(f, 0, nil).Extract(context.Background(), msg)
|
||||
if err != nil || got != nil {
|
||||
t.Fatalf("got (%v, %v), want (nil, nil)", got, err)
|
||||
}
|
||||
if f.calls != 0 {
|
||||
t.Errorf("model called %d times for junk, want 0", f.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractEmptyMessageIsNotSent(t *testing.T) {
|
||||
f := &fakeLLM{reply: "[]"}
|
||||
if _, err := NewExtractor(f, 0, nil).Extract(context.Background(), Message{UID: 3}); err != nil {
|
||||
t.Fatalf("extract: %v", err)
|
||||
}
|
||||
if f.calls != 0 {
|
||||
t.Errorf("model called %d times for an empty message, want 0", f.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractCaps(t *testing.T) {
|
||||
f := &fakeLLM{reply: `[{"text":"a","due":""},{"text":"b","due":""},{"text":"c","due":""}]`}
|
||||
got, err := NewExtractor(f, 2, nil).Extract(context.Background(), msgFor("s", "b"))
|
||||
if err != nil {
|
||||
t.Fatalf("extract: %v", err)
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Errorf("got %d, want the configured cap of 2", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractDropsRepeatsAndBadDates(t *testing.T) {
|
||||
f := &fakeLLM{reply: `[{"text":"Отправить акт","due":"2026-02-31"},{"text":"отправить акт","due":""},{"text":" ","due":""}]`}
|
||||
got, err := NewExtractor(f, 0, nil).Extract(context.Background(), msgFor("s", "b"))
|
||||
if err != nil {
|
||||
t.Fatalf("extract: %v", err)
|
||||
}
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("got %d candidates, want 1 (repeat and blank dropped): %+v", len(got), got)
|
||||
}
|
||||
if got[0].Due != "" {
|
||||
t.Errorf("due = %q, want empty — 2026-02-31 is not a date", got[0].Due)
|
||||
}
|
||||
}
|
||||
|
||||
// A Thinking model sometimes wraps the array; and when it emits something
|
||||
// unparsable the caller must hear about it rather than see "no tasks".
|
||||
func TestParseCandidatesTolerance(t *testing.T) {
|
||||
got, err := parseCandidates("думаю... [{\"text\":\"x\",\"due\":\"\"}] всё")
|
||||
if err != nil || len(got) != 1 || got[0].Text != "x" {
|
||||
t.Fatalf("got (%+v, %v)", got, err)
|
||||
}
|
||||
if _, err := parseCandidates("нет никакого JSON"); err == nil {
|
||||
t.Error("unparsable output must be an error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractParseErrorHidesMailText(t *testing.T) {
|
||||
f := &fakeLLM{reply: "он просил отправить акт, вот такой ответ"}
|
||||
_, err := NewExtractor(f, 0, nil).Extract(context.Background(), msgFor("Акт", "секретный текст"))
|
||||
if err == nil {
|
||||
t.Fatal("want an error")
|
||||
}
|
||||
if strings.Contains(err.Error(), "акт") || strings.Contains(err.Error(), "секретный") {
|
||||
t.Errorf("error text leaks mail content: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDue(t *testing.T) {
|
||||
if _, ok := ParseDue(""); ok {
|
||||
t.Error("empty due must be (zero, false)")
|
||||
}
|
||||
if got, ok := ParseDue("2026-08-05"); !ok || got.Year() != 2026 || got.Month() != 8 || got.Day() != 5 {
|
||||
t.Errorf("ParseDue = (%v, %v)", got, ok)
|
||||
}
|
||||
if _, ok := ParseDue("05.08.2026"); ok {
|
||||
t.Error("a non-ISO date must not parse")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package email
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// FetchSince is the whole read path in one call: connect, log in, examine the
|
||||
// mailbox read-only, list what arrived since a date, fetch and parse the ones
|
||||
// the caller has not seen, log out.
|
||||
//
|
||||
// It is a function rather than a long-lived object because a mail poller should
|
||||
// not hold an authenticated session (and therefore his credential in a live TLS
|
||||
// state) between polls. Connect, read, drop.
|
||||
//
|
||||
// skip decides which UIDs are already known — the poller's seen-set. max bounds
|
||||
// one poll: a mailbox that received 400 messages overnight must not turn into
|
||||
// 400 LLM calls, and the newest max are the ones a task could still be hiding
|
||||
// in. Junk messages are returned too, flagged, so the caller can mark them seen
|
||||
// without a second protocol round.
|
||||
type FetchSince struct {
|
||||
Addr string // host or host:993
|
||||
User string
|
||||
Mailbox string // e.g. "INBOX"
|
||||
Timeout time.Duration
|
||||
Since time.Time
|
||||
Max int
|
||||
Skip func(uid uint32) bool
|
||||
}
|
||||
|
||||
// Run performs one read. password is passed here, not stored in the struct, so
|
||||
// the configuration of a mailbox and the secret for it are never the same value
|
||||
// sitting in the same place.
|
||||
func (f FetchSince) Run(password string) ([]Message, error) {
|
||||
return f.RunWith(password, nil)
|
||||
}
|
||||
|
||||
// RunWith is Run with an explicit connection function, which is how the reader
|
||||
// daemon and the tests substitute an in-process server. nil ⇒ Dial, i.e.
|
||||
// implicit TLS with certificate verification; there is no configuration path
|
||||
// that reaches this, so no deployment can end up talking cleartext IMAP.
|
||||
func (f FetchSince) RunWith(password string, dial func(addr string, timeout time.Duration) (*Conn, error)) ([]Message, error) {
|
||||
if f.Addr == "" || f.User == "" || f.Mailbox == "" {
|
||||
return nil, fmt.Errorf("email: mailbox not configured (addr/user/mailbox)")
|
||||
}
|
||||
if dial == nil {
|
||||
dial = Dial
|
||||
}
|
||||
c, err := dial(f.Addr, f.Timeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer c.Close()
|
||||
if err := c.Login(f.User, password); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer c.Logout()
|
||||
if err := c.Select(f.Mailbox); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
uids, err := c.SearchSince(f.Since)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Newest UIDs first — IMAP hands them back ascending, and when Max clips the
|
||||
// list the recent mail is what matters.
|
||||
wanted := make([]uint32, 0, len(uids))
|
||||
for i := len(uids) - 1; i >= 0; i-- {
|
||||
if f.Skip != nil && f.Skip(uids[i]) {
|
||||
continue
|
||||
}
|
||||
wanted = append(wanted, uids[i])
|
||||
if f.Max > 0 && len(wanted) >= f.Max {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
out := make([]Message, 0, len(wanted))
|
||||
for _, uid := range wanted {
|
||||
raw, err := c.Fetch(uid)
|
||||
if err != nil {
|
||||
// One unreadable message does not abandon the poll; the rest of the
|
||||
// mailbox is still worth reading. The error names the UID, not the
|
||||
// message.
|
||||
return out, fmt.Errorf("email: fetch uid %d: %w", uid, err)
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
continue // vanished between SEARCH and FETCH
|
||||
}
|
||||
msg, err := ParseMessage(uid, raw)
|
||||
if err != nil {
|
||||
continue // unparsable headers — nothing to review, skip silently
|
||||
}
|
||||
out = append(out, msg)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package email
|
||||
|
||||
import (
|
||||
"net"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestFetchSinceRun(t *testing.T) {
|
||||
mk := func(subject string) string {
|
||||
return "Subject: " + subject + "\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nbody\r\n"
|
||||
}
|
||||
f := &fakeIMAP{
|
||||
uids: []uint32{1, 2, 3},
|
||||
msgs: map[uint32]string{1: mk("one"), 2: mk("two"), 3: mk("three")},
|
||||
}
|
||||
fs := FetchSince{
|
||||
Addr: "mail.example:993", User: "kami", Mailbox: "INBOX",
|
||||
Timeout: 5 * time.Second,
|
||||
Since: time.Date(2026, 7, 30, 0, 0, 0, 0, time.UTC),
|
||||
Max: 2,
|
||||
Skip: func(uid uint32) bool { return uid == 3 },
|
||||
}
|
||||
msgs, err := fs.RunWith("secret", func(addr string, timeout time.Duration) (*Conn, error) {
|
||||
cli, srv := net.Pipe()
|
||||
go f.serve(t, srv)
|
||||
return NewConn(cli, timeout)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("run: %v", err)
|
||||
}
|
||||
// Newest first, the already-seen UID skipped, Max respected.
|
||||
if len(msgs) != 2 {
|
||||
t.Fatalf("got %d messages, want 2: %+v", len(msgs), msgs)
|
||||
}
|
||||
if msgs[0].Subject != "two" || msgs[1].Subject != "one" {
|
||||
t.Errorf("subjects = %q,%q, want two,one (newest first)", msgs[0].Subject, msgs[1].Subject)
|
||||
}
|
||||
if strings.Contains(strings.Join(f.cmds, " "), "UID FETCH 3") {
|
||||
t.Error("a skipped UID must not be fetched again")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchSinceRequiresConfig(t *testing.T) {
|
||||
if _, err := (FetchSince{}).Run("secret"); err == nil {
|
||||
t.Fatal("an unconfigured mailbox must not be read")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
package email
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A minimal IMAP4rev1 client — LOGIN, SELECT, UID SEARCH, UID FETCH with
|
||||
// BODY.PEEK, LOGOUT, and nothing else.
|
||||
//
|
||||
// Why hand-rolled instead of go-imap: the whole surface Maven needs is five
|
||||
// commands, and this is the one code path that holds his mailbox credential and
|
||||
// reads his private mail. A ~200-line client with no dependencies is auditable
|
||||
// in one sitting; a general-purpose IMAP library is a much larger amount of
|
||||
// code doing much more than we asked, in the most sensitive place in the tree.
|
||||
// If IDLE, CONDSTORE or server-side threading ever become worth having, that
|
||||
// trade should be re-made deliberately.
|
||||
//
|
||||
// BODY.PEEK[] rather than BODY[] is load-bearing: Maven reads his mail and must
|
||||
// leave no trace of having done so. Reading a message here does not mark it
|
||||
// \Seen, so the unread state in his own mail client stays his.
|
||||
|
||||
// DefaultIMAPPort — implicit-TLS IMAP. There is no cleartext and no STARTTLS
|
||||
// path in this client: an option to send his password over a plain socket is an
|
||||
// option to get it wrong once.
|
||||
const DefaultIMAPPort = "993"
|
||||
|
||||
// Conn — one authenticated IMAP connection. Not safe for concurrent use; the
|
||||
// poller drives one connection at a time.
|
||||
type Conn struct {
|
||||
rwc io.ReadWriteCloser
|
||||
r *bufio.Reader
|
||||
tag int
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
// Dial opens an implicit-TLS connection and reads the server greeting.
|
||||
func Dial(addr string, timeout time.Duration) (*Conn, error) {
|
||||
host, _, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
host, addr = addr, net.JoinHostPort(addr, DefaultIMAPPort)
|
||||
}
|
||||
d := &net.Dialer{Timeout: timeout}
|
||||
// ServerName is set from the host we asked for: certificate verification is
|
||||
// the only thing standing between his password and a MITM on the way out.
|
||||
c, err := tls.DialWithDialer(d, "tcp", addr, &tls.Config{ServerName: host, MinVersion: tls.VersionTLS12})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("email: dial %s: %w", addr, err)
|
||||
}
|
||||
return NewConn(c, timeout)
|
||||
}
|
||||
|
||||
// NewConn wraps an already-open stream (the tests speak IMAP over a pipe) and
|
||||
// consumes the greeting.
|
||||
func NewConn(rwc io.ReadWriteCloser, timeout time.Duration) (*Conn, error) {
|
||||
c := &Conn{rwc: rwc, r: bufio.NewReaderSize(rwc, 64<<10), timeout: timeout}
|
||||
line, err := c.readLine()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("email: greeting: %w", err)
|
||||
}
|
||||
if !strings.HasPrefix(line, "* OK") && !strings.HasPrefix(line, "* PREAUTH") {
|
||||
c.rwc.Close()
|
||||
return nil, fmt.Errorf("email: server refused connection: %s", line)
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (c *Conn) Close() error { return c.rwc.Close() }
|
||||
|
||||
// Login authenticates with LOGIN. The password is passed as an argument and
|
||||
// never stored on the Conn: nothing in this package keeps a credential alive
|
||||
// past the command that uses it, so no struct dump or panic trace can carry it.
|
||||
func (c *Conn) Login(user, pass string) error {
|
||||
// The command line itself is never logged (see exec) — a LOGIN line IS the
|
||||
// credential.
|
||||
if _, err := c.exec(fmt.Sprintf("LOGIN %s %s", quote(user), quote(pass))); err != nil {
|
||||
return fmt.Errorf("email: login: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Select opens a mailbox read-only. EXAMINE, not SELECT: read-only at the
|
||||
// protocol level means no command in this session can change a flag, expunge a
|
||||
// message, or move anything, even by mistake.
|
||||
func (c *Conn) Select(mailbox string) error {
|
||||
if _, err := c.exec(fmt.Sprintf("EXAMINE %s", quote(mailbox))); err != nil {
|
||||
return fmt.Errorf("email: examine %s: %w", mailbox, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SearchSince returns the UIDs of messages received on or after since. An
|
||||
// unlimited search is not offered: the first poll against a years-old mailbox
|
||||
// would otherwise fetch everything and hand a decade of mail to the model.
|
||||
//
|
||||
// The IMAP SINCE key has date granularity (and compares the server's internal
|
||||
// date), so the result can include messages slightly older than since. The
|
||||
// caller dedupes by UID anyway, so a wider window costs one extra fetch.
|
||||
func (c *Conn) SearchSince(since time.Time) ([]uint32, error) {
|
||||
cmd := fmt.Sprintf("UID SEARCH SINCE %s", since.Format("2-Jan-2006"))
|
||||
lines, err := c.exec(cmd)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("email: search: %w", err)
|
||||
}
|
||||
var uids []uint32
|
||||
for _, l := range lines {
|
||||
rest, ok := untagged(l, "SEARCH")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, f := range strings.Fields(rest) {
|
||||
n, err := strconv.ParseUint(f, 10, 32)
|
||||
if err == nil {
|
||||
uids = append(uids, uint32(n))
|
||||
}
|
||||
}
|
||||
}
|
||||
return uids, nil
|
||||
}
|
||||
|
||||
var literalSize = regexp.MustCompile(`\{(\d+)\}$`)
|
||||
|
||||
// Fetch returns the raw RFC 5322 bytes of one message, by UID.
|
||||
//
|
||||
// Returns (nil, nil) when the UID no longer exists — a message he deleted
|
||||
// between SEARCH and FETCH is normal, not an error.
|
||||
func (c *Conn) Fetch(uid uint32) ([]byte, error) {
|
||||
tag := c.nextTag()
|
||||
if err := c.send(fmt.Sprintf("%s UID FETCH %d (BODY.PEEK[])", tag, uid)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var raw []byte
|
||||
for {
|
||||
line, err := c.readLine()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("email: fetch %d: %w", uid, err)
|
||||
}
|
||||
if done, err := c.tagged(tag, line); done {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("email: fetch %d: %w", uid, err)
|
||||
}
|
||||
return raw, nil
|
||||
}
|
||||
m := literalSize.FindStringSubmatch(strings.TrimSpace(line))
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
n, err := strconv.Atoi(m[1])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
buf := make([]byte, n)
|
||||
if _, err := io.ReadFull(c.r, buf); err != nil {
|
||||
return nil, fmt.Errorf("email: fetch %d: literal: %w", uid, err)
|
||||
}
|
||||
if raw == nil {
|
||||
raw = buf
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Logout ends the session politely. A failure is not worth reporting — the
|
||||
// connection is being closed either way.
|
||||
func (c *Conn) Logout() {
|
||||
_, _ = c.exec("LOGOUT")
|
||||
}
|
||||
|
||||
// ---- protocol plumbing -----------------------------------------------------
|
||||
|
||||
func (c *Conn) nextTag() string {
|
||||
c.tag++
|
||||
return fmt.Sprintf("a%03d", c.tag)
|
||||
}
|
||||
|
||||
// exec sends one command and returns the untagged response lines.
|
||||
//
|
||||
// Neither the command nor the response is ever logged here. LOGIN goes through
|
||||
// this function, and a debug line "sent: a001 LOGIN ..." is how a credential
|
||||
// ends up in a log file forever.
|
||||
func (c *Conn) exec(cmd string) ([]string, error) {
|
||||
tag := c.nextTag()
|
||||
if err := c.send(tag + " " + cmd); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var lines []string
|
||||
for {
|
||||
line, err := c.readLine()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if done, err := c.tagged(tag, line); done {
|
||||
return lines, err
|
||||
}
|
||||
lines = append(lines, line)
|
||||
// A response line may carry a literal (e.g. a header FETCH). Nothing we
|
||||
// send asks for one outside Fetch, but skip it if it appears so the
|
||||
// stream stays aligned.
|
||||
if m := literalSize.FindStringSubmatch(strings.TrimSpace(line)); m != nil {
|
||||
if n, err := strconv.Atoi(m[1]); err == nil {
|
||||
if _, err := io.CopyN(io.Discard, c.r, int64(n)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// tagged reports whether line completes the command with this tag, and turns a
|
||||
// NO/BAD completion into an error. The error text is the server's, which never
|
||||
// echoes a password.
|
||||
func (c *Conn) tagged(tag, line string) (bool, error) {
|
||||
if !strings.HasPrefix(line, tag+" ") {
|
||||
return false, nil
|
||||
}
|
||||
rest := strings.TrimSpace(line[len(tag):])
|
||||
switch {
|
||||
case strings.HasPrefix(rest, "OK"):
|
||||
return true, nil
|
||||
case strings.HasPrefix(rest, "NO"), strings.HasPrefix(rest, "BAD"):
|
||||
return true, fmt.Errorf("server said: %s", rest)
|
||||
default:
|
||||
return true, fmt.Errorf("unexpected completion: %s", rest)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Conn) send(line string) error {
|
||||
c.setDeadline()
|
||||
if _, err := io.WriteString(c.rwc, line+"\r\n"); err != nil {
|
||||
return fmt.Errorf("email: write: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Conn) readLine() (string, error) {
|
||||
c.setDeadline()
|
||||
line, err := c.r.ReadString('\n')
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return strings.TrimRight(line, "\r\n"), nil
|
||||
}
|
||||
|
||||
// setDeadline applies the per-connection timeout when the transport supports
|
||||
// one. A hung IMAP server must not park the poller forever.
|
||||
func (c *Conn) setDeadline() {
|
||||
if c.timeout <= 0 {
|
||||
return
|
||||
}
|
||||
if d, ok := c.rwc.(interface{ SetDeadline(time.Time) error }); ok {
|
||||
_ = d.SetDeadline(time.Now().Add(c.timeout))
|
||||
}
|
||||
}
|
||||
|
||||
// untagged splits "* SEARCH 1 2 3" into its payload when the key matches.
|
||||
func untagged(line, key string) (string, bool) {
|
||||
if !strings.HasPrefix(line, "* ") {
|
||||
return "", false
|
||||
}
|
||||
rest := strings.TrimSpace(line[2:])
|
||||
if !strings.HasPrefix(rest, key) {
|
||||
return "", false
|
||||
}
|
||||
return strings.TrimSpace(rest[len(key):]), true
|
||||
}
|
||||
|
||||
// quote renders an IMAP quoted string. Passwords routinely contain characters
|
||||
// that would otherwise end the argument early, and CR/LF are stripped rather
|
||||
// than escaped because there is no legal way to send them — a credential file
|
||||
// with a stray newline must not become a second command.
|
||||
func quote(s string) string {
|
||||
s = strings.NewReplacer("\r", "", "\n", "").Replace(s)
|
||||
return `"` + strings.NewReplacer(`\`, `\\`, `"`, `\"`).Replace(s) + `"`
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
package email
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fakeIMAP is a scripted server: enough of IMAP to exercise the client, and
|
||||
// nothing more. It records the commands it received so a test can assert on the
|
||||
// protocol (BODY.PEEK rather than BODY, EXAMINE rather than SELECT).
|
||||
type fakeIMAP struct {
|
||||
msgs map[uint32]string
|
||||
uids []uint32
|
||||
cmds []string
|
||||
failOn string // substring of a command to answer NO
|
||||
}
|
||||
|
||||
func (f *fakeIMAP) serve(t *testing.T, c net.Conn) {
|
||||
t.Helper()
|
||||
defer c.Close()
|
||||
fmt.Fprint(c, "* OK fake IMAP ready\r\n")
|
||||
r := bufio.NewReader(c)
|
||||
for {
|
||||
line, err := r.ReadString('\n')
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
line = strings.TrimRight(line, "\r\n")
|
||||
parts := strings.SplitN(line, " ", 2)
|
||||
if len(parts) != 2 {
|
||||
return
|
||||
}
|
||||
tag, cmd := parts[0], parts[1]
|
||||
f.cmds = append(f.cmds, cmd)
|
||||
if f.failOn != "" && strings.Contains(cmd, f.failOn) {
|
||||
fmt.Fprintf(c, "%s NO computer says no\r\n", tag)
|
||||
continue
|
||||
}
|
||||
upper := strings.ToUpper(cmd)
|
||||
switch {
|
||||
case strings.HasPrefix(upper, "LOGIN"), strings.HasPrefix(upper, "EXAMINE"):
|
||||
fmt.Fprintf(c, "%s OK done\r\n", tag)
|
||||
case strings.HasPrefix(upper, "UID SEARCH"):
|
||||
var ids []string
|
||||
for _, u := range f.uids {
|
||||
ids = append(ids, strconv.FormatUint(uint64(u), 10))
|
||||
}
|
||||
fmt.Fprintf(c, "* SEARCH %s\r\n", strings.Join(ids, " "))
|
||||
fmt.Fprintf(c, "%s OK search done\r\n", tag)
|
||||
case strings.HasPrefix(upper, "UID FETCH"):
|
||||
uid64, _ := strconv.ParseUint(strings.Fields(cmd)[2], 10, 32)
|
||||
raw, ok := f.msgs[uint32(uid64)]
|
||||
if ok {
|
||||
fmt.Fprintf(c, "* 1 FETCH (UID %d BODY[] {%d}\r\n", uid64, len(raw))
|
||||
fmt.Fprint(c, raw)
|
||||
fmt.Fprint(c, ")\r\n")
|
||||
}
|
||||
fmt.Fprintf(c, "%s OK fetch done\r\n", tag)
|
||||
case strings.HasPrefix(upper, "LOGOUT"):
|
||||
fmt.Fprint(c, "* BYE\r\n")
|
||||
fmt.Fprintf(c, "%s OK bye\r\n", tag)
|
||||
return
|
||||
default:
|
||||
fmt.Fprintf(c, "%s BAD unknown\r\n", tag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// dialFake wires a client Conn to an in-process server over net.Pipe.
|
||||
func dialFake(t *testing.T, f *fakeIMAP) *Conn {
|
||||
t.Helper()
|
||||
cli, srv := net.Pipe()
|
||||
go f.serve(t, srv)
|
||||
c, err := NewConn(cli, 5*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("greeting: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { c.Close() })
|
||||
return c
|
||||
}
|
||||
|
||||
func TestIMAPRoundTrip(t *testing.T) {
|
||||
body := "Subject: hello\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nCall the bank.\r\n"
|
||||
f := &fakeIMAP{uids: []uint32{4, 9}, msgs: map[uint32]string{4: body, 9: body}}
|
||||
c := dialFake(t, f)
|
||||
|
||||
if err := c.Login("kami", `pa"ss\word`); err != nil {
|
||||
t.Fatalf("login: %v", err)
|
||||
}
|
||||
if err := c.Select("INBOX"); err != nil {
|
||||
t.Fatalf("select: %v", err)
|
||||
}
|
||||
uids, err := c.SearchSince(time.Date(2026, 8, 1, 0, 0, 0, 0, time.UTC))
|
||||
if err != nil {
|
||||
t.Fatalf("search: %v", err)
|
||||
}
|
||||
if len(uids) != 2 || uids[0] != 4 || uids[1] != 9 {
|
||||
t.Fatalf("uids = %v, want [4 9]", uids)
|
||||
}
|
||||
raw, err := c.Fetch(9)
|
||||
if err != nil {
|
||||
t.Fatalf("fetch: %v", err)
|
||||
}
|
||||
if string(raw) != body {
|
||||
t.Errorf("fetched %q, want the literal verbatim", raw)
|
||||
}
|
||||
c.Logout()
|
||||
|
||||
joined := strings.Join(f.cmds, "\n")
|
||||
// Read-only at the protocol level, and peeking — Maven must leave no trace
|
||||
// of having read his mail.
|
||||
if !strings.Contains(joined, "EXAMINE") || strings.Contains(joined, "SELECT ") {
|
||||
t.Errorf("want EXAMINE (read-only), got:\n%s", joined)
|
||||
}
|
||||
if !strings.Contains(joined, "BODY.PEEK[]") {
|
||||
t.Errorf("want BODY.PEEK, got:\n%s", joined)
|
||||
}
|
||||
// The password must have been quoted and escaped, not truncated at the quote.
|
||||
if !strings.Contains(joined, `"pa\"ss\\word"`) {
|
||||
t.Errorf("password not quoted correctly:\n%s", joined)
|
||||
}
|
||||
// SINCE must carry the IMAP date form.
|
||||
if !strings.Contains(joined, "SINCE 1-Aug-2026") {
|
||||
t.Errorf("want a SINCE date, got:\n%s", joined)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIMAPServerNoIsAnError(t *testing.T) {
|
||||
f := &fakeIMAP{failOn: "LOGIN"}
|
||||
c := dialFake(t, f)
|
||||
err := c.Login("kami", "wrong")
|
||||
if err == nil {
|
||||
t.Fatal("a NO completion must be an error")
|
||||
}
|
||||
// The error is the server's text; it must not echo the credential.
|
||||
if strings.Contains(err.Error(), "wrong") {
|
||||
t.Errorf("error leaks the password: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIMAPFetchMissingUID(t *testing.T) {
|
||||
f := &fakeIMAP{uids: []uint32{1}, msgs: map[uint32]string{}}
|
||||
c := dialFake(t, f)
|
||||
raw, err := c.Fetch(1)
|
||||
if err != nil {
|
||||
t.Fatalf("fetch: %v", err)
|
||||
}
|
||||
if raw != nil {
|
||||
t.Errorf("a vanished UID should give nil, got %q", raw)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuoteStripsNewlines(t *testing.T) {
|
||||
if got := quote("pass\r\nA1 LOGOUT"); strings.ContainsAny(got, "\r\n") {
|
||||
t.Errorf("quote kept a line break: %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package email
|
||||
|
||||
import (
|
||||
"net/mail"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The junk filter — the cheapest and most important half of reading mail.
|
||||
//
|
||||
// A mailbox is mostly machine-generated: newsletters, receipts nobody acts on,
|
||||
// social notifications, marketing. Sending all of it to a 1.7B and asking "is
|
||||
// there a task here" produces confident nonsense at a rate proportional to the
|
||||
// volume, so junk is decided by HEADERS, before any model sees the message.
|
||||
//
|
||||
// The rules are all bulk-mail markers that senders set on themselves, never
|
||||
// guesses about content:
|
||||
//
|
||||
// - List-Unsubscribe / List-Id — by definition a mailing list. If he can
|
||||
// unsubscribe from it, it is not asking him to do anything.
|
||||
// - Precedence: bulk|junk|list — the sender declaring itself bulk.
|
||||
// - Auto-Submitted other than "no" (RFC 3834) — generated by a machine.
|
||||
// - X-Spam-Flag: YES, X-Spam-Status: Yes — the spam filter upstream already
|
||||
// decided; we do not second-guess it in the other direction.
|
||||
// - X-GM-LABELS / X-Gmail-Labels containing a Gmail category — Gmail's own
|
||||
// Promotions/Social/Forums/Spam classification, when the server sends it.
|
||||
//
|
||||
// Deliberately NOT here: sender allow/deny lists and subject keyword matching.
|
||||
// Both are configuration that ages badly and both would be a place for his
|
||||
// contacts to end up in a config file. If a real correspondent's mail is being
|
||||
// dropped, the fix is a rule about a header, not a list of names.
|
||||
//
|
||||
// A junk verdict never deletes anything and never touches a flag on the server.
|
||||
// It means "do not spend the model on this", nothing more.
|
||||
|
||||
// junkHeaders — headers whose mere presence marks bulk mail.
|
||||
var junkPresence = []string{"List-Unsubscribe", "List-Id", "List-Post"}
|
||||
|
||||
// gmailCategories — Gmail's category labels, lowercased as they appear in
|
||||
// X-GM-LABELS. "important" and "inbox" are labels too, and are NOT categories.
|
||||
// Matching is by these exact tokens (substring is fine — they are namespaced
|
||||
// and cannot appear in a hand-made label by accident), so a user label named
|
||||
// "Social Club" is not mistaken for Gmail's Social category.
|
||||
var gmailCategories = []string{
|
||||
"category_promotions", "category_social", "category_forums", "category_updates",
|
||||
`\spam`, `\junk`,
|
||||
}
|
||||
|
||||
// classifyJunk returns whether the message is bulk/automated and why. The
|
||||
// reason is a short header name, safe to log — it names the marker, never the
|
||||
// sender or the subject.
|
||||
func classifyJunk(h mail.Header) (bool, string) {
|
||||
for _, name := range junkPresence {
|
||||
if strings.TrimSpace(h.Get(name)) != "" {
|
||||
return true, strings.ToLower(name)
|
||||
}
|
||||
}
|
||||
switch strings.ToLower(strings.TrimSpace(h.Get("Precedence"))) {
|
||||
case "bulk", "junk", "list":
|
||||
return true, "precedence"
|
||||
}
|
||||
if v := strings.ToLower(strings.TrimSpace(h.Get("Auto-Submitted"))); v != "" && v != "no" {
|
||||
return true, "auto-submitted"
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(h.Get("X-Spam-Flag")), "yes") {
|
||||
return true, "x-spam-flag"
|
||||
}
|
||||
if v := strings.ToLower(strings.TrimSpace(h.Get("X-Spam-Status"))); strings.HasPrefix(v, "yes") {
|
||||
return true, "x-spam-status"
|
||||
}
|
||||
labels := strings.ToLower(h.Get("X-GM-LABELS") + " " + h.Get("X-Gmail-Labels"))
|
||||
for _, c := range gmailCategories {
|
||||
if c == "" {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(labels, c) {
|
||||
return true, "gmail-category"
|
||||
}
|
||||
}
|
||||
return false, ""
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package email
|
||||
|
||||
import (
|
||||
"net/mail"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func headers(t *testing.T, raw string) mail.Header {
|
||||
t.Helper()
|
||||
m, err := mail.ReadMessage(strings.NewReader(strings.ReplaceAll(raw, "\n", "\r\n") + "\r\n\r\nbody\r\n"))
|
||||
if err != nil {
|
||||
t.Fatalf("read headers: %v", err)
|
||||
}
|
||||
return m.Header
|
||||
}
|
||||
|
||||
func TestClassifyJunk(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
raw string
|
||||
junk bool
|
||||
reason string
|
||||
}{
|
||||
{"personal", "From: a@b.c\nSubject: привет", false, ""},
|
||||
{"list-unsubscribe", "From: a@b.c\nList-Unsubscribe: <mailto:u@b.c>", true, "list-unsubscribe"},
|
||||
{"list-id", "From: a@b.c\nList-Id: <golang-nuts.example>", true, "list-id"},
|
||||
{"precedence bulk", "From: a@b.c\nPrecedence: bulk", true, "precedence"},
|
||||
{"auto-submitted", "From: a@b.c\nAuto-Submitted: auto-generated", true, "auto-submitted"},
|
||||
{"auto-submitted no", "From: a@b.c\nAuto-Submitted: no", false, ""},
|
||||
{"spam flag", "From: a@b.c\nX-Spam-Flag: YES", true, "x-spam-flag"},
|
||||
{"spam status", "From: a@b.c\nX-Spam-Status: Yes, score=9.1", true, "x-spam-status"},
|
||||
{"spam status no", "From: a@b.c\nX-Spam-Status: No, score=0.1", false, ""},
|
||||
{"gmail promo", "From: a@b.c\nX-Gmail-Labels: Inbox,CATEGORY_PROMOTIONS", true, "gmail-category"},
|
||||
{"user label", "From: a@b.c\nX-Gmail-Labels: Social Club,Important", false, ""},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
junk, reason := classifyJunk(headers(t, c.raw))
|
||||
if junk != c.junk || reason != c.reason {
|
||||
t.Errorf("classifyJunk = (%v, %q), want (%v, %q)", junk, reason, c.junk, c.reason)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewsletterFixtureIsJunk(t *testing.T) {
|
||||
msg, err := ParseMessage(9, fixture(t, "newsletter.eml"))
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if !msg.Junk {
|
||||
t.Fatal("a newsletter with List-Unsubscribe + Precedence: bulk must be junk")
|
||||
}
|
||||
// The reason is what gets logged, so it must never carry mail content.
|
||||
if strings.Contains(msg.JunkReason, "@") || strings.Contains(msg.JunkReason, "Скидки") {
|
||||
t.Errorf("junk reason leaks content: %q", msg.JunkReason)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
// Package email is the reading half of the email reader (Vikunja #246,
|
||||
// docs/plans/01-email-reader.md): a small IMAP client, a MIME-to-plaintext
|
||||
// converter, and the junk filter that decides a message is not worth reading at
|
||||
// all. Extraction lives in extract.go and writes nothing itself.
|
||||
//
|
||||
// Two constraints shape everything here, both from CLAUDE.md:
|
||||
//
|
||||
// - Mail is personal. Nothing in this package logs a body, a subject, or an
|
||||
// address; callers get the text and decide. Mail text is never search input
|
||||
// — no function here reaches the network except the IMAP connection itself.
|
||||
// - Off unless configured. There is no default host, no default account, and
|
||||
// no fallback that would make a mailbox get read because a field was empty.
|
||||
//
|
||||
// The IMAP subset is deliberately tiny (LOGIN, SELECT, UID SEARCH, UID FETCH
|
||||
// with BODY.PEEK, LOGOUT). No IDLE: a poll every few minutes is what a task
|
||||
// candidate needs, and IDLE would mean holding a connection and a credential
|
||||
// open forever for latency nobody is waiting on.
|
||||
package email
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"mime/multipart"
|
||||
"mime/quotedprintable"
|
||||
"net/mail"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// MaxBodyBytes — how much of one message body is kept. A task hides in the
|
||||
// first screenful; the rest is signature, quoted history and legal boilerplate,
|
||||
// and it would only spend the resident model's 4096-token context.
|
||||
const MaxBodyBytes = 4000
|
||||
|
||||
// Message — one mail, reduced to the fields extraction and review need.
|
||||
//
|
||||
// Raw is deliberately absent: once a message is parsed the original bytes are
|
||||
// dropped, so no caller can accidentally log or forward the whole mail.
|
||||
type Message struct {
|
||||
UID uint32
|
||||
From string
|
||||
Subject string
|
||||
Date string // as sent, unparsed — display only
|
||||
Body string // plaintext, decoded, HTML-stripped, truncated
|
||||
// Junk is set by the junk filter (see junk.go). A junk message is carried
|
||||
// rather than dropped so the poller can count it and still mark it seen.
|
||||
Junk bool
|
||||
JunkReason string
|
||||
}
|
||||
|
||||
// ParseMessage turns one RFC 5322 message into a Message.
|
||||
//
|
||||
// It never fails on a body it cannot understand: an unparsable or
|
||||
// unsupported-charset body yields an empty Body and the headers still come
|
||||
// through, because a subject line alone is often the whole task ("Счёт за
|
||||
// интернет"). Only a message whose headers cannot be read at all is an error.
|
||||
func ParseMessage(uid uint32, raw []byte) (Message, error) {
|
||||
m, err := mail.ReadMessage(strings.NewReader(string(raw)))
|
||||
if err != nil {
|
||||
return Message{}, fmt.Errorf("email: parse message: %w", err)
|
||||
}
|
||||
msg := Message{
|
||||
UID: uid,
|
||||
From: decodeHeader(m.Header.Get("From")),
|
||||
Subject: decodeHeader(m.Header.Get("Subject")),
|
||||
Date: m.Header.Get("Date"),
|
||||
}
|
||||
msg.Junk, msg.JunkReason = classifyJunk(m.Header)
|
||||
body, err := plaintextBody(m.Header.Get("Content-Type"), m.Header.Get("Content-Transfer-Encoding"), m.Body)
|
||||
if err == nil {
|
||||
msg.Body = truncate(collapse(body), MaxBodyBytes)
|
||||
}
|
||||
return msg, nil
|
||||
}
|
||||
|
||||
// plaintextBody walks the MIME tree and returns the best plaintext it can.
|
||||
//
|
||||
// Preference order inside a multipart: text/plain first, text/html stripped
|
||||
// only when there is no plain part. multipart/mixed attachments are skipped
|
||||
// wholesale — an attachment is a file, not a sentence, and reading one would
|
||||
// mean parsing arbitrary formats from the network.
|
||||
func plaintextBody(contentType, encoding string, body io.Reader) (string, error) {
|
||||
mediaType, params, err := mime.ParseMediaType(contentType)
|
||||
if contentType == "" || err != nil {
|
||||
// No Content-Type at all is legal and means text/plain; a broken one is
|
||||
// treated the same rather than dropping the message.
|
||||
mediaType, params = "text/plain", nil
|
||||
}
|
||||
switch {
|
||||
case strings.HasPrefix(mediaType, "multipart/"):
|
||||
boundary := params["boundary"]
|
||||
if boundary == "" {
|
||||
return "", fmt.Errorf("email: multipart without boundary")
|
||||
}
|
||||
return multipartText(multipart.NewReader(body, boundary))
|
||||
case mediaType == "text/html":
|
||||
raw, err := decodeBody(body, encoding, params["charset"])
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return stripHTML(raw), nil
|
||||
case mediaType == "text/plain":
|
||||
return decodeBody(body, encoding, params["charset"])
|
||||
default:
|
||||
// A single-part non-text message (a bare PDF, say). No body, headers only.
|
||||
return "", nil
|
||||
}
|
||||
}
|
||||
|
||||
// multipartText reads one multipart level, recursing into nested multiparts.
|
||||
// Returns the plain part if any part yielded one, else the stripped HTML.
|
||||
func multipartText(mr *multipart.Reader) (string, error) {
|
||||
var plain, html string
|
||||
for {
|
||||
part, err := mr.NextPart()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
// A truncated multipart still gives up whatever came before it.
|
||||
break
|
||||
}
|
||||
if part.FileName() != "" {
|
||||
part.Close()
|
||||
continue // attachment
|
||||
}
|
||||
ct := part.Header.Get("Content-Type")
|
||||
mediaType, _, _ := mime.ParseMediaType(ct)
|
||||
text, err := plaintextBody(ct, part.Header.Get("Content-Transfer-Encoding"), part)
|
||||
part.Close()
|
||||
if err != nil || strings.TrimSpace(text) == "" {
|
||||
continue
|
||||
}
|
||||
if mediaType == "text/html" && !strings.HasPrefix(mediaType, "multipart/") {
|
||||
if html == "" {
|
||||
html = text
|
||||
}
|
||||
continue
|
||||
}
|
||||
if plain == "" {
|
||||
plain = text
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(plain) != "" {
|
||||
return plain, nil
|
||||
}
|
||||
return html, nil
|
||||
}
|
||||
|
||||
// decodeBody applies the transfer encoding, then the charset.
|
||||
//
|
||||
// Charset support is UTF-8 (and ASCII, its subset) only, on purpose: x/text's
|
||||
// encoding tables are not vendored here, and guessing at windows-1251 bytes
|
||||
// would feed the model mojibake it would happily extract a task from. An
|
||||
// unsupported charset returns an error, which ParseMessage turns into an empty
|
||||
// body — subject-only, which is honest.
|
||||
func decodeBody(r io.Reader, encoding, charset string) (string, error) {
|
||||
switch strings.ToLower(strings.TrimSpace(encoding)) {
|
||||
case "quoted-printable":
|
||||
r = quotedprintable.NewReader(r)
|
||||
case "base64":
|
||||
r = newBase64Reader(r)
|
||||
}
|
||||
b, err := io.ReadAll(io.LimitReader(r, 1<<20))
|
||||
if err != nil && len(b) == 0 {
|
||||
return "", fmt.Errorf("email: read body: %w", err)
|
||||
}
|
||||
switch cs := strings.ToLower(strings.TrimSpace(charset)); cs {
|
||||
case "", "utf-8", "utf8", "us-ascii", "ascii":
|
||||
return string(b), nil
|
||||
default:
|
||||
return "", fmt.Errorf("email: unsupported charset %q", cs)
|
||||
}
|
||||
}
|
||||
|
||||
// decodeHeader decodes RFC 2047 encoded words ("=?utf-8?B?...?="), which is how
|
||||
// every Russian subject line arrives. Undecodable headers come back as-is
|
||||
// rather than empty: a mangled subject is still a hint, and it is only ever
|
||||
// shown to him as evidence.
|
||||
func decodeHeader(v string) string {
|
||||
dec := new(mime.WordDecoder)
|
||||
out, err := dec.DecodeHeader(v)
|
||||
if err != nil {
|
||||
return collapse(v)
|
||||
}
|
||||
return collapse(out)
|
||||
}
|
||||
|
||||
var (
|
||||
scriptStyle = regexp.MustCompile(`(?is)<(script|style)\b[^>]*>.*?</\s*(script|style)\s*>`)
|
||||
htmlBreak = regexp.MustCompile(`(?i)<\s*(br\s*/?|/p|/div|/tr|/li|/h[1-6])\s*>`)
|
||||
htmlTag = regexp.MustCompile(`(?s)<[^>]*>`)
|
||||
htmlComment = regexp.MustCompile(`(?s)<!--.*?-->`)
|
||||
)
|
||||
|
||||
// stripHTML reduces an HTML part to text. A regex stripper, not a parser:
|
||||
// x/net/html is not vendored, and the consumer is a model reading prose — a
|
||||
// stray angle bracket costs nothing, whereas a new dependency for the privacy-
|
||||
// sensitive path costs review.
|
||||
func stripHTML(s string) string {
|
||||
s = scriptStyle.ReplaceAllString(s, " ")
|
||||
s = htmlComment.ReplaceAllString(s, " ")
|
||||
s = htmlBreak.ReplaceAllString(s, "\n")
|
||||
s = htmlTag.ReplaceAllString(s, " ")
|
||||
return unescapeEntities(s)
|
||||
}
|
||||
|
||||
var entities = strings.NewReplacer(
|
||||
" ", " ", "&", "&", "<", "<", ">", ">",
|
||||
""", `"`, "'", "'", "'", "'", "—", "—", "–", "–",
|
||||
)
|
||||
|
||||
func unescapeEntities(s string) string { return entities.Replace(s) }
|
||||
|
||||
// collapse squeezes runs of whitespace, keeping single newlines. Mail bodies
|
||||
// arrive with hard-wrapped lines and blocks of blank space; the model does not
|
||||
// need them and they are pure context budget.
|
||||
func collapse(s string) string {
|
||||
lines := strings.Split(strings.ReplaceAll(s, "\r\n", "\n"), "\n")
|
||||
var out []string
|
||||
blank := 0
|
||||
for _, l := range lines {
|
||||
l = strings.TrimSpace(strings.Join(strings.Fields(l), " "))
|
||||
if l == "" {
|
||||
blank++
|
||||
if blank > 1 {
|
||||
continue
|
||||
}
|
||||
out = append(out, "")
|
||||
continue
|
||||
}
|
||||
blank = 0
|
||||
out = append(out, l)
|
||||
}
|
||||
return strings.TrimSpace(strings.Join(out, "\n"))
|
||||
}
|
||||
|
||||
// truncate cuts to n bytes on a rune boundary.
|
||||
func truncate(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
cut := s[:n]
|
||||
for len(cut) > 0 && !isRuneStart(cut[len(cut)-1]) {
|
||||
cut = cut[:len(cut)-1]
|
||||
}
|
||||
return strings.TrimSpace(cut) + "…"
|
||||
}
|
||||
|
||||
func isRuneStart(b byte) bool { return b&0xC0 != 0x80 }
|
||||
|
||||
// newBase64Reader — base64.NewDecoder already skips the CRLFs mail bodies wrap
|
||||
// with, so this is only a named seam for decodeBody to read cleanly.
|
||||
func newBase64Reader(r io.Reader) io.Reader {
|
||||
return base64.NewDecoder(base64.StdEncoding, r)
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package email
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func fixture(t *testing.T, name string) []byte {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(filepath.Join("testdata", name))
|
||||
if err != nil {
|
||||
t.Fatalf("read fixture %s: %v", name, err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func TestParsePlainRussian(t *testing.T) {
|
||||
msg, err := ParseMessage(7, fixture(t, "plain_ru.eml"))
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if msg.UID != 7 {
|
||||
t.Errorf("uid = %d, want 7", msg.UID)
|
||||
}
|
||||
if want := "Нужно закрыть задачу"; msg.Subject != want {
|
||||
t.Errorf("subject = %q, want %q", msg.Subject, want)
|
||||
}
|
||||
if !strings.Contains(msg.From, "Антон") {
|
||||
t.Errorf("from = %q, want the decoded display name", msg.From)
|
||||
}
|
||||
if !strings.Contains(msg.Body, "Надо отправить акт до пятницы.") {
|
||||
t.Errorf("body = %q, want the quoted-printable text decoded", msg.Body)
|
||||
}
|
||||
if msg.Junk {
|
||||
t.Errorf("a personal mail must not be junk (%s)", msg.JunkReason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseHTMLOnlyIsStripped(t *testing.T) {
|
||||
msg, err := ParseMessage(1, fixture(t, "html_only.eml"))
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if strings.Contains(msg.Body, "<") || strings.Contains(msg.Body, "color:red") || strings.Contains(msg.Body, "x()") {
|
||||
t.Errorf("body still has markup/script/style: %q", msg.Body)
|
||||
}
|
||||
for _, want := range []string{"Счёт за интернет: 700", "Оплатить до 5 августа."} {
|
||||
if !strings.Contains(msg.Body, want) {
|
||||
t.Errorf("body = %q, want it to contain %q", msg.Body, want)
|
||||
}
|
||||
}
|
||||
// must have become a real space, not vanished into the number.
|
||||
if strings.Contains(msg.Body, " ") {
|
||||
t.Errorf("entity left unescaped: %q", msg.Body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsePrefersPlainAndSkipsAttachments(t *testing.T) {
|
||||
msg, err := ParseMessage(2, fixture(t, "mixed_attachment.eml"))
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if got := strings.TrimSpace(msg.Body); got != "Sign the contract before Monday." {
|
||||
t.Errorf("body = %q, want the text/plain alternative only", got)
|
||||
}
|
||||
if strings.Contains(msg.Body, "PDF") {
|
||||
t.Errorf("attachment bytes leaked into the body: %q", msg.Body)
|
||||
}
|
||||
}
|
||||
|
||||
// An unsupported charset must degrade to headers-only rather than to mojibake
|
||||
// the model would then extract a task from.
|
||||
func TestParseUnsupportedCharsetKeepsHeaders(t *testing.T) {
|
||||
msg, err := ParseMessage(3, fixture(t, "cp1251.eml"))
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if msg.Subject != "Legacy" {
|
||||
t.Errorf("subject = %q, want Legacy", msg.Subject)
|
||||
}
|
||||
if msg.Body != "" {
|
||||
t.Errorf("body = %q, want empty for an undecodable charset", msg.Body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTruncatesLongBody(t *testing.T) {
|
||||
var b strings.Builder
|
||||
b.WriteString("Subject: long\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n")
|
||||
for i := 0; i < 2000; i++ {
|
||||
b.WriteString("длинная строка ")
|
||||
}
|
||||
msg, err := ParseMessage(4, []byte(b.String()))
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if len(msg.Body) > MaxBodyBytes+8 {
|
||||
t.Errorf("body kept %d bytes, want ≤ %d", len(msg.Body), MaxBodyBytes)
|
||||
}
|
||||
if !strings.HasSuffix(msg.Body, "…") {
|
||||
t.Errorf("truncated body should be marked: %q", msg.Body[len(msg.Body)-20:])
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollapseSqueezesBlankLines(t *testing.T) {
|
||||
got := collapse(" a b \r\n\r\n\r\n\r\n c \r\n")
|
||||
if got != "a b\n\nc" {
|
||||
t.Errorf("collapse = %q, want %q", got, "a b\n\nc")
|
||||
}
|
||||
}
|
||||
Vendored
+7
@@ -0,0 +1,7 @@
|
||||
From: legacy@example.org
|
||||
To: kami@example.org
|
||||
Subject: Legacy
|
||||
Date: Fri, 01 Aug 2026 05:00:00 +0400
|
||||
Content-Type: text/plain; charset="windows-1251"
|
||||
|
||||
Ï
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
From: billing@isp.example
|
||||
To: kami@example.org
|
||||
Subject: =?utf-8?B?0KHRh9GR0YIg0LfQsCDQuNC90YLQtdGA0L3QtdGC?=
|
||||
Date: Fri, 01 Aug 2026 08:00:00 +0400
|
||||
MIME-Version: 1.0
|
||||
Content-Type: multipart/alternative; boundary="B1"
|
||||
|
||||
--B1
|
||||
Content-Type: text/html; charset="utf-8"
|
||||
Content-Transfer-Encoding: base64
|
||||
|
||||
PGh0bWw+PGhlYWQ+PHN0eWxlPnB7Y29sb3I6cmVkfTwvc3R5bGU+PC9oZWFkPjxib2R5PjxwPtCh0YfRkdGCINC30LAg0LjQvdGC0LXRgNC90LXRgjogNzAwJm5ic3A74oK9PC9wPjxwPtCe0L/Qu9Cw0YLQuNGC0Ywg0LTQviA1INCw0LLQs9GD0YHRgtCwLjwvcD48c2NyaXB0PngoKTwvc2NyaXB0PjwvYm9keT48L2h0bWw+
|
||||
--B1--
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
From: hr@work.example
|
||||
To: kami@example.org
|
||||
Subject: Contract
|
||||
Date: Fri, 01 Aug 2026 07:00:00 +0400
|
||||
MIME-Version: 1.0
|
||||
Content-Type: multipart/mixed; boundary="M1"
|
||||
|
||||
--M1
|
||||
Content-Type: multipart/alternative; boundary="A1"
|
||||
|
||||
--A1
|
||||
Content-Type: text/plain; charset="utf-8"
|
||||
|
||||
Sign the contract before Monday.
|
||||
--A1
|
||||
Content-Type: text/html; charset="utf-8"
|
||||
|
||||
<p>Sign the contract before Monday.</p>
|
||||
--A1--
|
||||
--M1
|
||||
Content-Type: application/pdf; name="contract.pdf"
|
||||
Content-Disposition: attachment; filename="contract.pdf"
|
||||
Content-Transfer-Encoding: base64
|
||||
|
||||
JVBERi0xLjQgbm90IHJlYWxseSBhIHBkZg==
|
||||
--M1--
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
From: news@shop.example
|
||||
To: kami@example.org
|
||||
Subject: =?utf-8?B?0KHQutC40LTQutC4INGC0L7Qu9GM0LrQviDRgdC10LPQvtC00L3Rjw==?=
|
||||
Date: Fri, 01 Aug 2026 06:00:00 +0400
|
||||
List-Unsubscribe: <mailto:unsub@shop.example>
|
||||
Precedence: bulk
|
||||
Content-Type: text/plain; charset="utf-8"
|
||||
|
||||
Sale!
|
||||
Vendored
+14
@@ -0,0 +1,14 @@
|
||||
From: =?utf-8?B?0JDQvdGC0L7QvQ==?= <anton@example.org>
|
||||
To: kami@example.org
|
||||
Subject: =?utf-8?B?0J3Rg9C20L3QviDQt9Cw0LrRgNGL0YLRjCDQt9Cw0LTQsNGH0YM=?=
|
||||
Date: Fri, 01 Aug 2026 09:12:00 +0400
|
||||
Content-Type: text/plain; charset="utf-8"
|
||||
Content-Transfer-Encoding: quoted-printable
|
||||
Message-ID: <plain-ru@example.org>
|
||||
|
||||
=D0=9F=D1=80=D0=B8=D0=B2=D0=B5=D1=82! =D0=9D=D0=B0=D0=B4=D0=BE =D0=BE=D1=82=
|
||||
=D0=BF=D1=80=D0=B0=D0=B2=D0=B8=D1=82=D1=8C =D0=B0=D0=BA=D1=82 =D0=B4=D0=BE =
|
||||
=D0=BF=D1=8F=D1=82=D0=BD=D0=B8=D1=86=D1=8B.
|
||||
|
||||
--
|
||||
Anton
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/audio"
|
||||
)
|
||||
|
||||
// DTOs — wire-level data. Decoupled from internal/store so the protocol is
|
||||
@@ -93,6 +95,268 @@ type WriteFactReq struct {
|
||||
Subject string `json:"subject,omitempty"`
|
||||
}
|
||||
|
||||
// Task — one captured piece of work (Vikunja #130). Status is
|
||||
// "candidate" (Maven derived it and it is unconfirmed), "open" (his work),
|
||||
// "done" or "dropped". Source is provenance in the facts vocabulary:
|
||||
// "tap:voice", "tap:web", "email:<account>". Evidence is the trail a derived
|
||||
// task came from, empty for anything he stated himself.
|
||||
type Task struct {
|
||||
ID int64 `json:"id"`
|
||||
CreatedTs time.Time `json:"created_ts"`
|
||||
Text string `json:"text"`
|
||||
Source string `json:"source"`
|
||||
Evidence string `json:"evidence,omitempty"`
|
||||
Status string `json:"status"`
|
||||
Due *time.Time `json:"due,omitempty"`
|
||||
Weight int `json:"weight,omitempty"`
|
||||
Resolved *time.Time `json:"resolved,omitempty"`
|
||||
}
|
||||
|
||||
// CaptureTaskReq — THE INTAKE SEAM. Everything that captures a task goes
|
||||
// through this one shape: the voice path, the web form, and (Vikunja #246) the
|
||||
// email reader, which has not been built yet.
|
||||
//
|
||||
// An extractor that reads mail sets Source "email:<account>", Status
|
||||
// "candidate", and Evidence to whatever makes the task reviewable (the subject
|
||||
// line). It must NOT set Status "open" — work Maven inferred from something she
|
||||
// read is a suggestion until the owner confirms it on the /tasks page. Capture
|
||||
// is idempotent on normalised text among live tasks, so re-reading the same
|
||||
// mailbox is free.
|
||||
type CaptureTaskReq struct {
|
||||
Text string `json:"text"`
|
||||
Source string `json:"source"`
|
||||
Evidence string `json:"evidence,omitempty"`
|
||||
Status string `json:"status,omitempty"` // "" ⇒ open
|
||||
Due *time.Time `json:"due,omitempty"`
|
||||
Weight int `json:"weight,omitempty"`
|
||||
Ts time.Time `json:"ts"`
|
||||
}
|
||||
|
||||
// CaptureTaskResp — Created is false when the same live task already existed,
|
||||
// in which case ID is the existing row. A caller tells the owner "уже в
|
||||
// списке" rather than claiming it saved something new.
|
||||
type CaptureTaskResp struct {
|
||||
ID int64 `json:"id"`
|
||||
Created bool `json:"created"`
|
||||
}
|
||||
|
||||
// IngestMailReq — one message a mail reader has fetched, handed to core for
|
||||
// extraction (Vikunja #246).
|
||||
//
|
||||
// The mail reader (cmd/mavmaild) holds the IMAP credential and core never sees
|
||||
// it, the same split mavpoll uses for the zenmoney token. What crosses this
|
||||
// boundary is only the message text, because extraction runs on the resident
|
||||
// model and llama-server lives inside core's process.
|
||||
//
|
||||
// Body is already plaintext and truncated by internal/email; core does not
|
||||
// re-parse MIME and never stores the body. Junk means the reader's header
|
||||
// filter already classified the message as bulk — core is told rather than
|
||||
// asked, so a junk message can be counted without a model call.
|
||||
//
|
||||
// This method is available only when core has an email block configured AND a
|
||||
// llama-server phraser; otherwise it answers ErrUnknownMethod, which is what
|
||||
// "off unless configured" looks like at the wire.
|
||||
type IngestMailReq struct {
|
||||
Mailbox string `json:"mailbox"`
|
||||
UID uint32 `json:"uid"`
|
||||
From string `json:"from,omitempty"`
|
||||
Subject string `json:"subject,omitempty"`
|
||||
Date string `json:"date,omitempty"`
|
||||
Body string `json:"body,omitempty"`
|
||||
Junk bool `json:"junk,omitempty"`
|
||||
}
|
||||
|
||||
// IngestMailResp — what core did with the message. TaskIDs are the rows
|
||||
// CaptureTask returned; Created counts the ones that were new (a re-read
|
||||
// mailbox dedupes to Created=0). Skipped is set when nothing was asked of the
|
||||
// model at all — junk, or an empty message.
|
||||
//
|
||||
// Nothing here echoes the mail back. The reader logs counts.
|
||||
type IngestMailResp struct {
|
||||
TaskIDs []int64 `json:"task_ids,omitempty"`
|
||||
Created int `json:"created"`
|
||||
Skipped bool `json:"skipped,omitempty"`
|
||||
}
|
||||
|
||||
// DescribeImageReq — one image handed to core to look at (Vikunja #252).
|
||||
//
|
||||
// Data is the raw image file as received (png / jpeg / gif). Core sniffs it and
|
||||
// refuses anything else; a declared content type is not part of this request
|
||||
// because the sender's claim about its own bytes is not evidence. Base64 on the
|
||||
// wire via the usual JSON marshal of []byte.
|
||||
//
|
||||
// Question is what he asked about the picture ("что тут написано?"). Empty ⇒
|
||||
// core uses its configured default prompt.
|
||||
//
|
||||
// Source is provenance recorded on the stored blob: "telegram", "web:upload".
|
||||
//
|
||||
// Exactly one of Data or ID is set. ID re-describes an image core already has —
|
||||
// a different question, or the first attempt that succeeds after a vision model
|
||||
// finally lands on disk.
|
||||
//
|
||||
// The method exists only when core has both a media store and an enabled vision
|
||||
// block; otherwise it answers ErrUnknownMethod, which is what "off unless
|
||||
// configured" looks like at the wire. A surface cannot make Maven look at
|
||||
// pictures by merely sending one.
|
||||
type DescribeImageReq struct {
|
||||
Data []byte `json:"data,omitempty"`
|
||||
ID string `json:"id,omitempty"`
|
||||
Source string `json:"source,omitempty"`
|
||||
Question string `json:"question,omitempty"`
|
||||
// SaveNote — also write the description as a note (source
|
||||
// "media:image:<id-prefix>") so it is recallable later. Default false: a
|
||||
// glance at a screenshot is not automatically a memory.
|
||||
SaveNote bool `json:"save_note,omitempty"`
|
||||
}
|
||||
|
||||
// DescribeImageResp — what she saw. ID is the stored blob's content address, and
|
||||
// it is set even when Description is empty because the description failed: the
|
||||
// bytes are on disk and the same id can be retried. NoteID is non-zero only when
|
||||
// SaveNote was set and the write succeeded.
|
||||
//
|
||||
// The image itself is never echoed back.
|
||||
type DescribeImageResp struct {
|
||||
ID string `json:"id"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Width int `json:"width,omitempty"`
|
||||
Height int `json:"height,omitempty"`
|
||||
NoteID int64 `json:"note_id,omitempty"`
|
||||
}
|
||||
|
||||
// CaptureStartReq — begin recording a meeting (Vikunja #253).
|
||||
//
|
||||
// Label is what the meeting is called ("встреча с подрядчиком"); it goes into
|
||||
// the summary note so the note is findable later. Empty is allowed.
|
||||
//
|
||||
// There is no "auto", no keyword and no schedule in this request, and there will
|
||||
// not be: the only way audio enters the recorder is a client that was told to
|
||||
// start, appending frames it was told to append. All four capture methods answer
|
||||
// ErrUnknownMethod unless the operator enabled a capture block, so a surface
|
||||
// cannot start a recording by asking nicely.
|
||||
type CaptureStartReq struct {
|
||||
Label string `json:"label,omitempty"`
|
||||
}
|
||||
|
||||
// CaptureStartResp — the session that opened. MaxSeconds is the hard cap after
|
||||
// which it stops itself; the caller tells him, so a forgotten recording is his
|
||||
// own informed choice rather than a surprise.
|
||||
type CaptureStartResp struct {
|
||||
Label string `json:"label,omitempty"`
|
||||
Started time.Time `json:"started"`
|
||||
MaxSeconds int `json:"max_seconds"`
|
||||
}
|
||||
|
||||
// CaptureAppendReq — one chunk of audio for the running session. Refused with
|
||||
// "nothing is being recorded" when no session is open, which is the guard that
|
||||
// makes an ambient path impossible: audio arriving at an idle core is dropped on
|
||||
// the floor, not buffered "just in case".
|
||||
type CaptureAppendReq struct {
|
||||
Audio audio.Audio `json:"audio"`
|
||||
}
|
||||
|
||||
// CaptureAppendResp — how much has been collected, so a client can show a timer
|
||||
// and notice the cap coming. Expired means the session hit its limit and closed;
|
||||
// stop sending and call capture_stop, the audio so far is kept.
|
||||
type CaptureAppendResp struct {
|
||||
Seconds float64 `json:"seconds"`
|
||||
Expired bool `json:"expired,omitempty"`
|
||||
}
|
||||
|
||||
// CaptureStopReq — end the running session.
|
||||
//
|
||||
// Discard throws the recording away without transcribing, storing or
|
||||
// summarising anything. This is what "забудь, не записывай" maps to, and it is a
|
||||
// flag rather than a separate method so the client that says "stop" and the
|
||||
// client that says "stop and forget" take the same path to the same session.
|
||||
type CaptureStopReq struct {
|
||||
Discard bool `json:"discard,omitempty"`
|
||||
}
|
||||
|
||||
// CaptureStopResp — the finished capture. BlobID is the stored WAV, kept under
|
||||
// media.retention like any other blob and pruned with it.
|
||||
//
|
||||
// A response with a Transcript and an empty Summary is a degraded success: the
|
||||
// words exist, only the model failed. A response with a BlobID and neither is
|
||||
// the audio surviving a transcription failure — the same id can be run again.
|
||||
// Discarded is true when nothing was kept.
|
||||
type CaptureStopResp struct {
|
||||
BlobID string `json:"blob_id,omitempty"`
|
||||
Label string `json:"label,omitempty"`
|
||||
Started time.Time `json:"started,omitempty"`
|
||||
Seconds float64 `json:"seconds,omitempty"`
|
||||
Transcript string `json:"transcript,omitempty"`
|
||||
Summary string `json:"summary,omitempty"`
|
||||
Chunks int `json:"chunks,omitempty"`
|
||||
NoteID int64 `json:"note_id,omitempty"`
|
||||
Discarded bool `json:"discarded,omitempty"`
|
||||
}
|
||||
|
||||
// CaptureStatusResp — what "что ты записываешь?" needs, and what /dash shows.
|
||||
// Running=false with everything else empty is the normal state.
|
||||
type CaptureStatusResp struct {
|
||||
Running bool `json:"running"`
|
||||
Label string `json:"label,omitempty"`
|
||||
Started time.Time `json:"started,omitempty"`
|
||||
Seconds float64 `json:"seconds,omitempty"`
|
||||
Bytes int `json:"bytes,omitempty"`
|
||||
}
|
||||
|
||||
// SwapModelReq — load another resident model without restarting the daemon
|
||||
// (Vikunja #250). ModelPath must be one of the paths in phraser.swap_models;
|
||||
// anything else is ErrForbidden, and an unconfigured allowlist makes the whole
|
||||
// method ErrUnknownMethod.
|
||||
//
|
||||
// NGpuLayers and NCtx are zero for "keep what is loaded now", which is the
|
||||
// normal case — the same laptop iGPU, a different gguf.
|
||||
//
|
||||
// This is an owner action. It is AuthStepUp in the authority table, it is not on
|
||||
// CoreAPI, and no act, intent or timer can reach it: swapping the model is not
|
||||
// something Maven does to herself.
|
||||
type SwapModelReq struct {
|
||||
ModelPath string `json:"model_path"`
|
||||
NGpuLayers int `json:"n_gpu_layers,omitempty"`
|
||||
NCtx int `json:"n_ctx,omitempty"`
|
||||
}
|
||||
|
||||
// SwapModelResp — what the daemon ended up serving. Model is the identity the
|
||||
// new llama-server reported for itself, not an echo of the request: if the file
|
||||
// was not the model the operator thought it was, this is where it shows.
|
||||
//
|
||||
// RolledBack is true when the requested model failed to load or would not answer
|
||||
// and the previous one was put back. In that case the call also returns an error
|
||||
// — the swap did not happen — and Model names the model still serving.
|
||||
type SwapModelResp struct {
|
||||
Model string `json:"model"`
|
||||
ModelPath string `json:"model_path"`
|
||||
BaseURL string `json:"base_url"`
|
||||
RolledBack bool `json:"rolled_back,omitempty"`
|
||||
TookMs int64 `json:"took_ms"`
|
||||
}
|
||||
|
||||
// ModelStatusResp — which model is resident and which ones may be swapped in.
|
||||
// Read-only; the authed page renders it. Swappable is the configured allowlist,
|
||||
// so an empty list means the capability is off.
|
||||
type ModelStatusResp struct {
|
||||
Model string `json:"model"`
|
||||
ModelPath string `json:"model_path"`
|
||||
BaseURL string `json:"base_url"`
|
||||
NGpuLayers int `json:"n_gpu_layers"`
|
||||
NCtx int `json:"n_ctx"`
|
||||
Swappable []string `json:"swappable,omitempty"`
|
||||
}
|
||||
|
||||
type listTasksReq struct {
|
||||
Status string `json:"status"` // "" all | "live" | candidate|open|done|dropped
|
||||
}
|
||||
type listTasksResp struct {
|
||||
Tasks []Task `json:"tasks"`
|
||||
}
|
||||
type setTaskStatusReq struct {
|
||||
ID int64 `json:"id"`
|
||||
Status string `json:"status"`
|
||||
Ts time.Time `json:"ts"`
|
||||
}
|
||||
|
||||
// idReq — methods keyed by a single id.
|
||||
type idReq struct {
|
||||
ID int64 `json:"id"`
|
||||
@@ -180,6 +444,19 @@ type Tool struct {
|
||||
Updated time.Time `json:"updated"`
|
||||
}
|
||||
|
||||
// MCPServerStatus — one configured MCP server, as the web surface sees it.
|
||||
// Target is the command or url; Tools is how many tools discovery kept after
|
||||
// allow_tools / max_tools, not how many the server offers.
|
||||
type MCPServerStatus struct {
|
||||
Name string `json:"name"`
|
||||
Transport string `json:"transport"` // "stdio" (a local subprocess) or "http"
|
||||
Target string `json:"target"`
|
||||
Connected bool `json:"connected"`
|
||||
Server string `json:"server,omitempty"` // the server's own name + version
|
||||
Tools int `json:"tools"`
|
||||
Err string `json:"err,omitempty"`
|
||||
}
|
||||
|
||||
// chatReq / chatResp — text chat round-trip for the IPC Chat method.
|
||||
type chatReq struct {
|
||||
Text string `json:"text"`
|
||||
@@ -294,6 +571,18 @@ type CoreAPI interface {
|
||||
// loop takes the schedule from there — no reminder is created (Vikunja #366).
|
||||
AcceptProposedRoutine(ctx context.Context, id int64) error
|
||||
|
||||
// CaptureTask records a task. See CaptureTaskReq — this is the single
|
||||
// intake seam for the voice path, the web form and the future email
|
||||
// extractor. Idempotent per live normalised text; the response says
|
||||
// whether a row was actually created.
|
||||
CaptureTask(ctx context.Context, req CaptureTaskReq) (CaptureTaskResp, error)
|
||||
// ListTasks returns tasks in one status, newest first. "" is every row,
|
||||
// "live" is candidate + open (outstanding work).
|
||||
ListTasks(ctx context.Context, status string) ([]Task, error)
|
||||
// SetTaskStatus moves a task forward once: candidate→open|dropped,
|
||||
// open→done|dropped. Any other move is refused.
|
||||
SetTaskStatus(ctx context.Context, id int64, status string, ts time.Time) error
|
||||
|
||||
// TickTrace returns the most recent tick's rule trace. The daemon caches
|
||||
// this after every tick; the store adapter returns an error (trace is not
|
||||
// persisted — it's a daemon-level cache).
|
||||
@@ -306,6 +595,14 @@ type CoreAPI interface {
|
||||
// TickTrace.
|
||||
MorningStatus(ctx context.Context) ([]MorningRoutineStatus, error)
|
||||
|
||||
// MCPServers reports the configured MCP servers and their health
|
||||
// (Vikunja #251). Read-only introspection for /tools — there is no
|
||||
// "call this tool" method on purpose: an MCP tool runs through the same
|
||||
// allowlist, confirm turn and act path as any other tool, and a second
|
||||
// mutation path would be a second thing to get wrong. Empty when the
|
||||
// mcp config block is absent, which is the default.
|
||||
MCPServers(ctx context.Context) ([]MCPServerStatus, error)
|
||||
|
||||
// DayPlan returns today's ordered plan — calendar events, pending
|
||||
// reminders and any morning checklist still outstanding (see
|
||||
// internal/morning.BuildPlan) — plus the spoken RU rendering of it.
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
package ipc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/audio"
|
||||
)
|
||||
|
||||
// The load-bearing default for the most invasive capability Maven has: on a core
|
||||
// that was never configured to record, there is no wire path that starts a
|
||||
// recording, feeds one, or harvests one. Every one of the four methods refuses.
|
||||
func TestCapture_OffUnlessConfigured(t *testing.T) {
|
||||
_, _, cli, _ := newServerWithStore(t)
|
||||
ctx := context.Background()
|
||||
|
||||
if _, err := cli.CaptureStart(ctx, CaptureStartReq{Label: "встреча"}); !errors.Is(err, ErrUnknownMethod) {
|
||||
t.Errorf("CaptureStart error = %v, want ErrUnknownMethod", err)
|
||||
}
|
||||
if _, err := cli.CaptureAppend(ctx, CaptureAppendReq{}); !errors.Is(err, ErrUnknownMethod) {
|
||||
t.Errorf("CaptureAppend error = %v, want ErrUnknownMethod", err)
|
||||
}
|
||||
if _, err := cli.CaptureStop(ctx, CaptureStopReq{}); !errors.Is(err, ErrUnknownMethod) {
|
||||
t.Errorf("CaptureStop error = %v, want ErrUnknownMethod", err)
|
||||
}
|
||||
if _, err := cli.CaptureStatus(ctx); !errors.Is(err, ErrUnknownMethod) {
|
||||
t.Errorf("CaptureStatus error = %v, want ErrUnknownMethod", err)
|
||||
}
|
||||
}
|
||||
|
||||
// With the hooks wired, a whole session crosses the boundary intact: the label
|
||||
// out, the audio in, the summary back.
|
||||
func TestCapture_RoundTrip(t *testing.T) {
|
||||
_, srv, cli, _ := newServerWithStore(t)
|
||||
ctx := context.Background()
|
||||
|
||||
started := time.Now().UTC().Truncate(time.Second)
|
||||
var gotLabel string
|
||||
var gotBytes int
|
||||
var gotDiscard bool
|
||||
|
||||
srv.CaptureStartFn = func(_ context.Context, req CaptureStartReq) (CaptureStartResp, error) {
|
||||
gotLabel = req.Label
|
||||
return CaptureStartResp{Label: req.Label, Started: started, MaxSeconds: 7200}, nil
|
||||
}
|
||||
srv.CaptureAppendFn = func(_ context.Context, req CaptureAppendReq) (CaptureAppendResp, error) {
|
||||
gotBytes = len(req.Audio.Bytes)
|
||||
return CaptureAppendResp{Seconds: 1.5}, nil
|
||||
}
|
||||
srv.CaptureStopFn = func(_ context.Context, req CaptureStopReq) (CaptureStopResp, error) {
|
||||
gotDiscard = req.Discard
|
||||
return CaptureStopResp{BlobID: "abc", Summary: "— решили купить насос", Chunks: 1}, nil
|
||||
}
|
||||
srv.CaptureStatusFn = func(context.Context) (CaptureStatusResp, error) {
|
||||
return CaptureStatusResp{Running: true, Label: "встреча", Seconds: 1.5}, nil
|
||||
}
|
||||
|
||||
start, err := cli.CaptureStart(ctx, CaptureStartReq{Label: "встреча с подрядчиком"})
|
||||
if err != nil {
|
||||
t.Fatalf("CaptureStart: %v", err)
|
||||
}
|
||||
if gotLabel != "встреча с подрядчиком" || start.MaxSeconds != 7200 {
|
||||
t.Errorf("start = %+v (label seen: %q)", start, gotLabel)
|
||||
}
|
||||
if !start.Started.Equal(started) {
|
||||
t.Errorf("started = %v, want %v", start.Started, started)
|
||||
}
|
||||
|
||||
// Audio must survive the JSON round trip byte for byte — a base64 mistake
|
||||
// here would be silence in the transcript, not a visible error.
|
||||
pcm := []byte{1, 2, 3, 4, 5, 6, 7, 8}
|
||||
ap, err := cli.CaptureAppend(ctx, CaptureAppendReq{
|
||||
Audio: audio.Audio{Format: audio.PCM16kMono, Bytes: pcm},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CaptureAppend: %v", err)
|
||||
}
|
||||
if gotBytes != len(pcm) {
|
||||
t.Errorf("%d bytes arrived, sent %d", gotBytes, len(pcm))
|
||||
}
|
||||
if ap.Seconds != 1.5 || ap.Expired {
|
||||
t.Errorf("append resp = %+v", ap)
|
||||
}
|
||||
|
||||
st, err := cli.CaptureStatus(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("CaptureStatus: %v", err)
|
||||
}
|
||||
if !st.Running || st.Label != "встреча" {
|
||||
t.Errorf("status = %+v", st)
|
||||
}
|
||||
|
||||
stop, err := cli.CaptureStop(ctx, CaptureStopReq{})
|
||||
if err != nil {
|
||||
t.Fatalf("CaptureStop: %v", err)
|
||||
}
|
||||
if gotDiscard {
|
||||
t.Error("a plain stop arrived as a discard")
|
||||
}
|
||||
if stop.BlobID != "abc" || stop.Summary == "" {
|
||||
t.Errorf("stop = %+v", stop)
|
||||
}
|
||||
}
|
||||
|
||||
// "забудь, не записывай" has to reach core as a discard, not as an ordinary
|
||||
// stop that quietly keeps everything.
|
||||
func TestCapture_DiscardCrossesTheWire(t *testing.T) {
|
||||
_, srv, cli, _ := newServerWithStore(t)
|
||||
var gotDiscard bool
|
||||
srv.CaptureStopFn = func(_ context.Context, req CaptureStopReq) (CaptureStopResp, error) {
|
||||
gotDiscard = req.Discard
|
||||
return CaptureStopResp{Discarded: req.Discard}, nil
|
||||
}
|
||||
resp, err := cli.CaptureStop(context.Background(), CaptureStopReq{Discard: true})
|
||||
if err != nil {
|
||||
t.Fatalf("CaptureStop: %v", err)
|
||||
}
|
||||
if !gotDiscard || !resp.Discarded {
|
||||
t.Errorf("discard lost: sent true, core saw %v, resp %+v", gotDiscard, resp)
|
||||
}
|
||||
}
|
||||
@@ -69,8 +69,10 @@ var readOnlyMethods = map[Method]bool{
|
||||
MethodLookupTool: true,
|
||||
MethodListTools: true,
|
||||
MethodListProposedRoutines: true,
|
||||
MethodListTasks: true,
|
||||
MethodTickTrace: true,
|
||||
MethodMorningStatus: true,
|
||||
MethodMCPServers: true,
|
||||
MethodDayPlan: true,
|
||||
}
|
||||
|
||||
@@ -427,6 +429,114 @@ func (c *Client) ListProposedRoutines(ctx context.Context) ([]ProposedRoutine, e
|
||||
return r.Routines, nil
|
||||
}
|
||||
|
||||
func (c *Client) CaptureTask(ctx context.Context, req CaptureTaskReq) (CaptureTaskResp, error) {
|
||||
var r CaptureTaskResp
|
||||
if err := c.call(ctx, MethodCaptureTask, req, &r); err != nil {
|
||||
return CaptureTaskResp{}, err
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func (c *Client) ListTasks(ctx context.Context, status string) ([]Task, error) {
|
||||
var r listTasksResp
|
||||
if err := c.call(ctx, MethodListTasks, listTasksReq{Status: status}, &r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return r.Tasks, nil
|
||||
}
|
||||
|
||||
func (c *Client) SetTaskStatus(ctx context.Context, id int64, status string, ts time.Time) error {
|
||||
return c.call(ctx, MethodSetTaskStatus, setTaskStatusReq{ID: id, Status: status, Ts: ts}, nil)
|
||||
}
|
||||
|
||||
// IngestMail hands one fetched message to core for extraction. ErrUnknownMethod
|
||||
// means core has no email block configured — the caller should stop asking, not
|
||||
// retry.
|
||||
func (c *Client) IngestMail(ctx context.Context, req IngestMailReq) (IngestMailResp, error) {
|
||||
var r IngestMailResp
|
||||
if err := c.call(ctx, MethodIngestMail, req, &r); err != nil {
|
||||
return IngestMailResp{}, err
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// DescribeImage hands one image to core to look at (Vikunja #252).
|
||||
// ErrUnknownMethod means core has no media store or vision is off — the caller
|
||||
// should stop asking, not retry. A response with an ID and an empty Description
|
||||
// means the bytes were stored but nothing could describe them yet, which is the
|
||||
// expected state on a box with no vision model on disk.
|
||||
func (c *Client) DescribeImage(ctx context.Context, req DescribeImageReq) (DescribeImageResp, error) {
|
||||
var r DescribeImageResp
|
||||
if err := c.call(ctx, MethodDescribeImage, req, &r); err != nil {
|
||||
return DescribeImageResp{}, err
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// CaptureStart begins recording a meeting (Vikunja #253). ErrUnknownMethod
|
||||
// means the operator has not enabled capture — the caller should say so and stop
|
||||
// asking, not retry.
|
||||
func (c *Client) CaptureStart(ctx context.Context, req CaptureStartReq) (CaptureStartResp, error) {
|
||||
var r CaptureStartResp
|
||||
if err := c.call(ctx, MethodCaptureStart, req, &r); err != nil {
|
||||
return CaptureStartResp{}, err
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// CaptureAppend hands one chunk of audio to the running session. An error means
|
||||
// the frame was not kept: either nothing is being recorded, or the session hit
|
||||
// its time limit. Either way the client stops sending.
|
||||
func (c *Client) CaptureAppend(ctx context.Context, req CaptureAppendReq) (CaptureAppendResp, error) {
|
||||
var r CaptureAppendResp
|
||||
if err := c.call(ctx, MethodCaptureAppend, req, &r); err != nil {
|
||||
return CaptureAppendResp{}, err
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// CaptureStop ends the session. Slow — it transcribes and summarises the whole
|
||||
// recording — so pass a context with room. Set Discard to throw the recording
|
||||
// away instead.
|
||||
func (c *Client) CaptureStop(ctx context.Context, req CaptureStopReq) (CaptureStopResp, error) {
|
||||
var r CaptureStopResp
|
||||
if err := c.call(ctx, MethodCaptureStop, req, &r); err != nil {
|
||||
return CaptureStopResp{}, err
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// CaptureStatus reports the running session, if any.
|
||||
func (c *Client) CaptureStatus(ctx context.Context) (CaptureStatusResp, error) {
|
||||
var r CaptureStatusResp
|
||||
if err := c.call(ctx, MethodCaptureStatus, nil, &r); err != nil {
|
||||
return CaptureStatusResp{}, err
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// SwapModel asks core to load another resident model (Vikunja #250).
|
||||
// ErrUnknownMethod means core has no phraser.swap_models allowlist configured;
|
||||
// ErrForbidden means the path is not on it, or step-up was not asserted. A
|
||||
// non-nil error with RolledBack set means nothing changed — the old model is
|
||||
// still serving.
|
||||
func (c *Client) SwapModel(ctx context.Context, req SwapModelReq) (SwapModelResp, error) {
|
||||
var r SwapModelResp
|
||||
if err := c.call(ctx, MethodSwapModel, req, &r); err != nil {
|
||||
return SwapModelResp{}, err
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// ModelStatus reports the resident model and the swap allowlist. Read-only.
|
||||
func (c *Client) ModelStatus(ctx context.Context) (ModelStatusResp, error) {
|
||||
var r ModelStatusResp
|
||||
if err := c.call(ctx, MethodModelStatus, nil, &r); err != nil {
|
||||
return ModelStatusResp{}, err
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func (c *Client) DismissProposedRoutine(ctx context.Context, id int64) error {
|
||||
return c.call(ctx, MethodDismissProposedRoutine, dismissProposedRoutineReq{ID: id}, nil)
|
||||
}
|
||||
@@ -451,6 +561,14 @@ func (c *Client) TickTrace(ctx context.Context) (TickTrace, error) {
|
||||
return t, nil
|
||||
}
|
||||
|
||||
func (c *Client) MCPServers(ctx context.Context) ([]MCPServerStatus, error) {
|
||||
var s []MCPServerStatus
|
||||
if err := c.call(ctx, MethodMCPServers, nil, &s); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (c *Client) MorningStatus(ctx context.Context) ([]MorningRoutineStatus, error) {
|
||||
var s []MorningRoutineStatus
|
||||
if err := c.call(ctx, MethodMorningStatus, nil, &s); err != nil {
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
@@ -565,3 +566,77 @@ func mustJSON(v any) []byte {
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// TestIngestMail_OffUnlessConfigured — with no IngestMailFn set (the default,
|
||||
// and what an unconfigured core looks like) the method does not exist. A mail
|
||||
// reader gets a refusal it can act on rather than a silent success.
|
||||
func TestIngestMail_OffUnlessConfigured(t *testing.T) {
|
||||
_, _, cli, _ := newServerWithStore(t)
|
||||
if _, err := cli.IngestMail(context.Background(), IngestMailReq{Mailbox: "INBOX", UID: 1}); !errors.Is(err, ErrUnknownMethod) {
|
||||
t.Fatalf("IngestMail error = %v, want ErrUnknownMethod", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestIngestMail_Hook — when the daemon wires the hook, the message crosses the
|
||||
// boundary intact and the response comes back.
|
||||
func TestIngestMail_Hook(t *testing.T) {
|
||||
_, srv, cli, _ := newServerWithStore(t)
|
||||
var got IngestMailReq
|
||||
srv.IngestMailFn = func(_ context.Context, req IngestMailReq) (IngestMailResp, error) {
|
||||
got = req
|
||||
return IngestMailResp{TaskIDs: []int64{7}, Created: 1}, nil
|
||||
}
|
||||
resp, err := cli.IngestMail(context.Background(), IngestMailReq{
|
||||
Mailbox: "INBOX", UID: 12, Subject: "Счёт", Body: "Оплатить.", Junk: false,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("IngestMail: %v", err)
|
||||
}
|
||||
if resp.Created != 1 || len(resp.TaskIDs) != 1 || resp.TaskIDs[0] != 7 {
|
||||
t.Errorf("resp = %+v", resp)
|
||||
}
|
||||
if got.UID != 12 || got.Subject != "Счёт" || got.Body != "Оплатить." {
|
||||
t.Errorf("req across the wire = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSwapModel_OffUnlessConfigured — no allowlist in the config means the
|
||||
// daemon never sets the hook, and the method does not exist. That is what "off
|
||||
// unless configured" looks like at the wire for the model swap (Vikunja #250).
|
||||
func TestSwapModel_OffUnlessConfigured(t *testing.T) {
|
||||
_, _, cli, _ := newServerWithStore(t)
|
||||
if _, err := cli.SwapModel(context.Background(), SwapModelReq{ModelPath: "/m/x.gguf"}); !errors.Is(err, ErrUnknownMethod) {
|
||||
t.Fatalf("SwapModel error = %v, want ErrUnknownMethod", err)
|
||||
}
|
||||
if _, err := cli.ModelStatus(context.Background()); !errors.Is(err, ErrUnknownMethod) {
|
||||
t.Fatalf("ModelStatus error = %v, want ErrUnknownMethod", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSwapModel_Hook — the request crosses the boundary intact and the reported
|
||||
// identity comes back. A refusal from the daemon's allowlist arrives as
|
||||
// ErrForbidden, which is what a caller keys its error message off.
|
||||
func TestSwapModel_Hook(t *testing.T) {
|
||||
_, srv, cli, _ := newServerWithStore(t)
|
||||
var got SwapModelReq
|
||||
srv.SwapModelFn = func(_ context.Context, req SwapModelReq) (SwapModelResp, error) {
|
||||
got = req
|
||||
if req.ModelPath != "/m/allowed.gguf" {
|
||||
return SwapModelResp{}, fmt.Errorf("%w: not allowlisted", ErrForbidden)
|
||||
}
|
||||
return SwapModelResp{Model: "allowed", ModelPath: req.ModelPath, BaseURL: "http://127.0.0.1:9", TookMs: 12}, nil
|
||||
}
|
||||
resp, err := cli.SwapModel(context.Background(), SwapModelReq{ModelPath: "/m/allowed.gguf", NCtx: 4096})
|
||||
if err != nil {
|
||||
t.Fatalf("SwapModel: %v", err)
|
||||
}
|
||||
if resp.Model != "allowed" || resp.TookMs != 12 {
|
||||
t.Errorf("resp = %+v", resp)
|
||||
}
|
||||
if got.NCtx != 4096 {
|
||||
t.Errorf("req across the wire = %+v", got)
|
||||
}
|
||||
if _, err := cli.SwapModel(context.Background(), SwapModelReq{ModelPath: "/etc/shadow"}); !errors.Is(err, ErrForbidden) {
|
||||
t.Fatalf("swap to a non-allowlisted path = %v; want ErrForbidden", err)
|
||||
}
|
||||
}
|
||||
|
||||
+250
-4
@@ -211,6 +211,10 @@ func (a *storeAPI) MorningStatus(ctx context.Context) ([]MorningRoutineStatus, e
|
||||
return nil, errors.New("store: morning status not available via direct store API")
|
||||
}
|
||||
|
||||
func (a *storeAPI) MCPServers(ctx context.Context) ([]MCPServerStatus, error) {
|
||||
return nil, nil // no manager behind a bare store: nothing configured
|
||||
}
|
||||
|
||||
func (a *storeAPI) DayPlan(ctx context.Context) (DayPlan, error) {
|
||||
return DayPlan{}, errors.New("store: day plan not available via direct store API")
|
||||
}
|
||||
@@ -231,6 +235,48 @@ func (a *storeAPI) DeleteTool(ctx context.Context, name string) error {
|
||||
return mapErr(a.s.DeleteTool(ctx, name))
|
||||
}
|
||||
|
||||
func (a *storeAPI) CaptureTask(ctx context.Context, req CaptureTaskReq) (CaptureTaskResp, error) {
|
||||
id, created, err := a.s.CaptureTask(ctx, store.Task{
|
||||
CreatedTs: req.Ts,
|
||||
Text: req.Text,
|
||||
Source: req.Source,
|
||||
Evidence: req.Evidence,
|
||||
Status: req.Status,
|
||||
Due: req.Due,
|
||||
Weight: req.Weight,
|
||||
})
|
||||
if err != nil {
|
||||
return CaptureTaskResp{}, mapErr(err)
|
||||
}
|
||||
return CaptureTaskResp{ID: id, Created: created}, nil
|
||||
}
|
||||
|
||||
func (a *storeAPI) ListTasks(ctx context.Context, status string) ([]Task, error) {
|
||||
ts, err := a.s.ListTasks(ctx, status)
|
||||
if err != nil {
|
||||
return nil, mapErr(err)
|
||||
}
|
||||
out := make([]Task, len(ts))
|
||||
for i, t := range ts {
|
||||
out[i] = Task{
|
||||
ID: t.ID,
|
||||
CreatedTs: t.CreatedTs,
|
||||
Text: t.Text,
|
||||
Source: t.Source,
|
||||
Evidence: t.Evidence,
|
||||
Status: t.Status,
|
||||
Due: t.Due,
|
||||
Weight: t.Weight,
|
||||
Resolved: t.ResolvedTs,
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (a *storeAPI) SetTaskStatus(ctx context.Context, id int64, status string, ts time.Time) error {
|
||||
return mapErr(a.s.SetTaskStatus(ctx, id, status, ts))
|
||||
}
|
||||
|
||||
func (a *storeAPI) ListProposedRoutines(ctx context.Context) ([]ProposedRoutine, error) {
|
||||
rs, err := a.s.ListProposedRoutines(ctx)
|
||||
if err != nil {
|
||||
@@ -379,6 +425,54 @@ type Server struct {
|
||||
// Set by the daemon; nil ⇒ MethodStoreEncryptionKey returns ErrUnknownMethod.
|
||||
WrapKeyFn WrapKeyFunc
|
||||
|
||||
// IngestMailFn — extracts task candidates from one fetched message. Set by
|
||||
// the daemon only when an email block is configured AND there is a
|
||||
// llama-server to extract with; nil ⇒ MethodIngestMail returns
|
||||
// ErrUnknownMethod, so a mail reader pointed at a core that is not
|
||||
// configured for mail is refused rather than silently ignored.
|
||||
//
|
||||
// Like StepUp/WrapKeyFn/UnlockFn this bypasses CoreAPI: it is not a store
|
||||
// operation, it needs the resident model, and it must not become a method
|
||||
// every CoreAPI implementation has to carry.
|
||||
IngestMailFn IngestMailFunc
|
||||
|
||||
// SwapModelFn / ModelStatusFn — the on-the-fly resident model swap (Vikunja
|
||||
// #250) and its read side. Set by the daemon only when phraser.swap_models
|
||||
// lists at least one model AND the phraser owns a llama-server; nil ⇒ both
|
||||
// methods answer ErrUnknownMethod, which is what "off unless configured"
|
||||
// looks like at the wire.
|
||||
//
|
||||
// They bypass CoreAPI for the same reason IngestMailFn does: this is not a
|
||||
// store operation, it needs the daemon's llama-server, and no other CoreAPI
|
||||
// implementation should have to carry it. MethodSwapModel is AuthStepUp in
|
||||
// internal/auth — owner-triggered, never an act and never a timer.
|
||||
SwapModelFn SwapModelFunc
|
||||
ModelStatusFn ModelStatusFunc
|
||||
|
||||
// DescribeImageFn — looks at one image (Vikunja #252). Set by the daemon only
|
||||
// when a media store is configured AND vision is enabled with a local
|
||||
// endpoint; nil ⇒ MethodDescribeImage answers ErrUnknownMethod, so a surface
|
||||
// cannot make Maven accept a photo by merely sending one.
|
||||
//
|
||||
// It bypasses CoreAPI for the same reason IngestMailFn does: it needs a blob
|
||||
// store and a vision server, neither of which is a store operation, and no
|
||||
// other CoreAPI implementation should have to carry it.
|
||||
DescribeImageFn DescribeImageFunc
|
||||
|
||||
// Capture* — the meeting recorder (Vikunja #253). Set by the daemon only
|
||||
// when a media store is configured AND capture.enabled is true; nil ⇒ all
|
||||
// four methods answer ErrUnknownMethod. That is the load-bearing default for
|
||||
// this capability: on an unconfigured box there is no wire path that begins a
|
||||
// recording, so nothing can be recorded by accident, by a bug in a surface,
|
||||
// or by a model deciding it would be helpful.
|
||||
//
|
||||
// They bypass CoreAPI because a recorder needs a blob store, an STT worker
|
||||
// and a llama-server, none of which is a store operation.
|
||||
CaptureStartFn CaptureStartFunc
|
||||
CaptureAppendFn CaptureAppendFunc
|
||||
CaptureStopFn CaptureStopFunc
|
||||
CaptureStatusFn CaptureStatusFunc
|
||||
|
||||
// UnlockFn — unwraps the store encryption key from the wrapped blob using
|
||||
// the passkey credential public key, opens the encrypted store, and wires
|
||||
// the rest of the daemon (voice, loop, delivery). Set by the daemon when
|
||||
@@ -397,6 +491,25 @@ type WrapKeyFunc func(ctx context.Context, publicKey []byte) error
|
||||
// public key and completes daemon initialization.
|
||||
type UnlockFunc func(ctx context.Context, publicKey []byte) error
|
||||
|
||||
// SwapModelFunc — loads another resident model in place of the live one.
|
||||
type SwapModelFunc func(ctx context.Context, req SwapModelReq) (SwapModelResp, error)
|
||||
|
||||
// ModelStatusFunc — reports the resident model and the swap allowlist.
|
||||
type ModelStatusFunc func(ctx context.Context) (ModelStatusResp, error)
|
||||
|
||||
// IngestMailFunc — core-side mail extraction. Returns what was captured.
|
||||
type IngestMailFunc func(ctx context.Context, req IngestMailReq) (IngestMailResp, error)
|
||||
|
||||
// DescribeImageFunc — core-side image intake + description.
|
||||
type DescribeImageFunc func(ctx context.Context, req DescribeImageReq) (DescribeImageResp, error)
|
||||
|
||||
// CaptureStartFunc / CaptureAppendFunc / CaptureStopFunc / CaptureStatusFunc —
|
||||
// the four core-side halves of the meeting recorder.
|
||||
type CaptureStartFunc func(ctx context.Context, req CaptureStartReq) (CaptureStartResp, error)
|
||||
type CaptureAppendFunc func(ctx context.Context, req CaptureAppendReq) (CaptureAppendResp, error)
|
||||
type CaptureStopFunc func(ctx context.Context, req CaptureStopReq) (CaptureStopResp, error)
|
||||
type CaptureStatusFunc func(ctx context.Context) (CaptureStatusResp, error)
|
||||
|
||||
// CheckFunc — the auth hook signature. Wired by the daemon (auth.Gate.Check
|
||||
// satisfies this); dispatch calls it once per request after param-unmarshal
|
||||
// independence (it gets the raw params, may unmarshal what it needs — ipc
|
||||
@@ -564,9 +677,12 @@ func withoutParams[R any](fn func(ctx context.Context, api CoreAPI) (R, error))
|
||||
// existed) as an argument — so SetAPI's runtime swap (the unlock transition)
|
||||
// is still honored on the very next request with no extra plumbing here.
|
||||
//
|
||||
// MethodAssertStepUp, MethodStoreEncryptionKey and MethodUnlock are NOT in
|
||||
// this table: they bypass CoreAPI entirely (s.StepUp / s.WrapKeyFn /
|
||||
// s.UnlockFn), so dispatch special-cases them before consulting the table.
|
||||
// MethodAssertStepUp, MethodStoreEncryptionKey, MethodUnlock,
|
||||
// MethodIngestMail, MethodSwapModel, MethodModelStatus,
|
||||
// MethodDescribeImage and the four MethodCapture* methods are NOT in this
|
||||
// table: they bypass CoreAPI entirely (s.StepUp / s.WrapKeyFn / s.UnlockFn /
|
||||
// s.IngestMailFn / s.DescribeImageFn / s.Capture*Fn), so dispatch
|
||||
// special-cases them before consulting the table.
|
||||
var methodTable = map[Method]handlerFunc{
|
||||
MethodWriteFact: withParams(func(ctx context.Context, api CoreAPI, p WriteFactReq) (idResp, error) {
|
||||
id, err := api.WriteFact(ctx, p)
|
||||
@@ -700,6 +816,22 @@ var methodTable = map[Method]handlerFunc{
|
||||
MethodDeleteTool: withParamsVoid(func(ctx context.Context, api CoreAPI, p disableToolReq) error {
|
||||
return api.DeleteTool(ctx, p.Name)
|
||||
}),
|
||||
MethodCaptureTask: withParams(func(ctx context.Context, api CoreAPI, p CaptureTaskReq) (CaptureTaskResp, error) {
|
||||
return api.CaptureTask(ctx, p)
|
||||
}),
|
||||
MethodListTasks: withParams(func(ctx context.Context, api CoreAPI, p listTasksReq) (listTasksResp, error) {
|
||||
out, err := api.ListTasks(ctx, p.Status)
|
||||
if err != nil {
|
||||
return listTasksResp{}, err
|
||||
}
|
||||
if out == nil {
|
||||
out = []Task{}
|
||||
}
|
||||
return listTasksResp{Tasks: out}, nil
|
||||
}),
|
||||
MethodSetTaskStatus: withParamsVoid(func(ctx context.Context, api CoreAPI, p setTaskStatusReq) error {
|
||||
return api.SetTaskStatus(ctx, p.ID, p.Status, p.Ts)
|
||||
}),
|
||||
MethodListProposedRoutines: withoutParams(func(ctx context.Context, api CoreAPI) (listProposedRoutinesResp, error) {
|
||||
out, err := api.ListProposedRoutines(ctx)
|
||||
if err != nil {
|
||||
@@ -739,6 +871,16 @@ var methodTable = map[Method]handlerFunc{
|
||||
MethodMorningStatus: withoutParams(func(ctx context.Context, api CoreAPI) ([]MorningRoutineStatus, error) {
|
||||
return api.MorningStatus(ctx)
|
||||
}),
|
||||
MethodMCPServers: withoutParams(func(ctx context.Context, api CoreAPI) ([]MCPServerStatus, error) {
|
||||
out, err := api.MCPServers(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if out == nil {
|
||||
out = []MCPServerStatus{}
|
||||
}
|
||||
return out, nil
|
||||
}),
|
||||
}
|
||||
|
||||
// dispatch unmarshals params for req.Method and calls the matching CoreAPI
|
||||
@@ -758,7 +900,7 @@ func (s *Server) dispatch(ctx context.Context, req Request) (json.RawMessage, er
|
||||
}
|
||||
}
|
||||
|
||||
// These three bypass CoreAPI entirely — they drive Server fields set
|
||||
// These bypass CoreAPI entirely — they drive Server fields set
|
||||
// directly by the daemon (StepUp / WrapKeyFn / UnlockFn), not store
|
||||
// state, so they can never be table entries keyed on a CoreAPI method.
|
||||
switch req.Method {
|
||||
@@ -787,6 +929,110 @@ func (s *Server) dispatch(ctx context.Context, req Request) (json.RawMessage, er
|
||||
return marshalResult(nil), s.UnlockFn(ctx, p.PublicKey)
|
||||
}
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnknownMethod, req.Method)
|
||||
|
||||
case MethodIngestMail:
|
||||
if s.IngestMailFn != nil {
|
||||
var p IngestMailReq
|
||||
if err := unmarshalParams(req.Params, &p); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := s.IngestMailFn(ctx, p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return marshalResult(resp), nil
|
||||
}
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnknownMethod, req.Method)
|
||||
|
||||
case MethodSwapModel:
|
||||
if s.SwapModelFn != nil {
|
||||
var p SwapModelReq
|
||||
if err := unmarshalParams(req.Params, &p); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := s.SwapModelFn(ctx, p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return marshalResult(resp), nil
|
||||
}
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnknownMethod, req.Method)
|
||||
|
||||
case MethodDescribeImage:
|
||||
if s.DescribeImageFn != nil {
|
||||
var p DescribeImageReq
|
||||
if err := unmarshalParams(req.Params, &p); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := s.DescribeImageFn(ctx, p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return marshalResult(resp), nil
|
||||
}
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnknownMethod, req.Method)
|
||||
|
||||
case MethodCaptureStart:
|
||||
if s.CaptureStartFn != nil {
|
||||
var p CaptureStartReq
|
||||
if err := unmarshalParams(req.Params, &p); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := s.CaptureStartFn(ctx, p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return marshalResult(resp), nil
|
||||
}
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnknownMethod, req.Method)
|
||||
|
||||
case MethodCaptureAppend:
|
||||
if s.CaptureAppendFn != nil {
|
||||
var p CaptureAppendReq
|
||||
if err := unmarshalParams(req.Params, &p); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := s.CaptureAppendFn(ctx, p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return marshalResult(resp), nil
|
||||
}
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnknownMethod, req.Method)
|
||||
|
||||
case MethodCaptureStop:
|
||||
if s.CaptureStopFn != nil {
|
||||
var p CaptureStopReq
|
||||
if err := unmarshalParams(req.Params, &p); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := s.CaptureStopFn(ctx, p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return marshalResult(resp), nil
|
||||
}
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnknownMethod, req.Method)
|
||||
|
||||
case MethodCaptureStatus:
|
||||
if s.CaptureStatusFn != nil {
|
||||
resp, err := s.CaptureStatusFn(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return marshalResult(resp), nil
|
||||
}
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnknownMethod, req.Method)
|
||||
|
||||
case MethodModelStatus:
|
||||
if s.ModelStatusFn != nil {
|
||||
resp, err := s.ModelStatusFn(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return marshalResult(resp), nil
|
||||
}
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnknownMethod, req.Method)
|
||||
}
|
||||
|
||||
h, ok := methodTable[req.Method]
|
||||
|
||||
@@ -89,6 +89,15 @@ func (UnimplementedCoreAPI) DisableTool(ctx context.Context, name string) error
|
||||
func (UnimplementedCoreAPI) DeleteTool(ctx context.Context, name string) error {
|
||||
return ErrNotImplemented
|
||||
}
|
||||
func (UnimplementedCoreAPI) CaptureTask(ctx context.Context, req CaptureTaskReq) (CaptureTaskResp, error) {
|
||||
return CaptureTaskResp{}, ErrNotImplemented
|
||||
}
|
||||
func (UnimplementedCoreAPI) ListTasks(ctx context.Context, status string) ([]Task, error) {
|
||||
return nil, ErrNotImplemented
|
||||
}
|
||||
func (UnimplementedCoreAPI) SetTaskStatus(ctx context.Context, id int64, status string, ts time.Time) error {
|
||||
return ErrNotImplemented
|
||||
}
|
||||
func (UnimplementedCoreAPI) ListProposedRoutines(ctx context.Context) ([]ProposedRoutine, error) {
|
||||
return nil, ErrNotImplemented
|
||||
}
|
||||
@@ -113,6 +122,9 @@ func (UnimplementedCoreAPI) TickTrace(ctx context.Context) (TickTrace, error) {
|
||||
func (UnimplementedCoreAPI) MorningStatus(ctx context.Context) ([]MorningRoutineStatus, error) {
|
||||
return nil, ErrNotImplemented
|
||||
}
|
||||
func (UnimplementedCoreAPI) MCPServers(ctx context.Context) ([]MCPServerStatus, error) {
|
||||
return nil, ErrNotImplemented
|
||||
}
|
||||
func (UnimplementedCoreAPI) DayPlan(ctx context.Context) (DayPlan, error) {
|
||||
return DayPlan{}, ErrNotImplemented
|
||||
}
|
||||
|
||||
@@ -45,8 +45,20 @@ const (
|
||||
MethodRevertFact Method = "revert_fact"
|
||||
MethodTickTrace Method = "tick_trace"
|
||||
MethodMorningStatus Method = "morning_status"
|
||||
MethodMCPServers Method = "mcp_servers"
|
||||
MethodDayPlan Method = "day_plan"
|
||||
MethodChat Method = "chat"
|
||||
MethodCaptureTask Method = "capture_task"
|
||||
MethodListTasks Method = "list_tasks"
|
||||
MethodSetTaskStatus Method = "set_task_status"
|
||||
MethodIngestMail Method = "ingest_mail"
|
||||
MethodSwapModel Method = "swap_model"
|
||||
MethodModelStatus Method = "model_status"
|
||||
MethodDescribeImage Method = "describe_image"
|
||||
MethodCaptureStart Method = "capture_start"
|
||||
MethodCaptureAppend Method = "capture_append"
|
||||
MethodCaptureStop Method = "capture_stop"
|
||||
MethodCaptureStatus Method = "capture_status"
|
||||
)
|
||||
|
||||
// Request — one frame from module to core. Params is the JSON-encoded argument
|
||||
|
||||
+27
-1
@@ -10,10 +10,17 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Client struct {
|
||||
// mu guards base only. The base URL changes when the daemon swaps the
|
||||
// resident model (Vikunja #250): llama-server is relaunched on a fresh
|
||||
// port, and every holder of this client — the LLM router, the replier, the
|
||||
// mail extractor — must follow without being rebuilt. One mutexed field is
|
||||
// the whole mechanism; a swap re-points the client, it does not replace it.
|
||||
mu sync.RWMutex
|
||||
base string
|
||||
http *http.Client
|
||||
}
|
||||
@@ -22,6 +29,25 @@ func New(baseURL string, timeout time.Duration) *Client {
|
||||
return &Client{base: baseURL, http: &http.Client{Timeout: timeout}}
|
||||
}
|
||||
|
||||
// SetBaseURL re-points the client at another llama-server. Safe to call while
|
||||
// requests are in flight: a request that already read the old base finishes
|
||||
// against the old base (or fails, and every caller of Complete has a fallback),
|
||||
// and the next one uses the new base. It is deliberately NOT a queue-and-retry —
|
||||
// the phraser quiesces around a swap, so the window is small and a lost turn
|
||||
// degrades to the classifier rather than hanging.
|
||||
func (c *Client) SetBaseURL(base string) {
|
||||
c.mu.Lock()
|
||||
c.base = base
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
// BaseURL is the server this client currently talks to.
|
||||
func (c *Client) BaseURL() string {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
return c.base
|
||||
}
|
||||
|
||||
type Req struct {
|
||||
System string
|
||||
User string
|
||||
@@ -63,7 +89,7 @@ func (c *Client) Complete(ctx context.Context, r Req) (string, error) {
|
||||
RepeatPenalty: r.RepeatPenalty,
|
||||
Stop: r.Stop,
|
||||
})
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", c.base+"/v1/chat/completions", bytes.NewReader(b))
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", c.BaseURL()+"/v1/chat/completions", bytes.NewReader(b))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -57,3 +57,37 @@ func TestComplete(t *testing.T) {
|
||||
t.Errorf("got %q, want %q", got, "ok")
|
||||
}
|
||||
}
|
||||
|
||||
// TestSetBaseURL — a model swap re-points every holder of the client rather than
|
||||
// rebuilding the router, the replier and the extractors (Vikunja #250).
|
||||
func TestSetBaseURL(t *testing.T) {
|
||||
var hit string
|
||||
srvA := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hit = "A"
|
||||
w.Write([]byte(`{"choices":[{"message":{"content":"a"}}]}`))
|
||||
}))
|
||||
defer srvA.Close()
|
||||
srvB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hit = "B"
|
||||
w.Write([]byte(`{"choices":[{"message":{"content":"b"}}]}`))
|
||||
}))
|
||||
defer srvB.Close()
|
||||
|
||||
c := New(srvA.URL, 5*time.Second)
|
||||
if _, err := c.Complete(context.Background(), Req{User: "x"}); err != nil {
|
||||
t.Fatalf("Complete against A: %v", err)
|
||||
}
|
||||
if hit != "A" {
|
||||
t.Fatalf("first request went to %q; want A", hit)
|
||||
}
|
||||
c.SetBaseURL(srvB.URL)
|
||||
if got := c.BaseURL(); got != srvB.URL {
|
||||
t.Errorf("BaseURL = %q; want %q", got, srvB.URL)
|
||||
}
|
||||
if _, err := c.Complete(context.Background(), Req{User: "x"}); err != nil {
|
||||
t.Fatalf("Complete against B: %v", err)
|
||||
}
|
||||
if hit != "B" {
|
||||
t.Errorf("request after the swap went to %q; want B", hit)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
package mcp
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// CmdPrefix is the reserved first argv element that marks an allowlist row as
|
||||
// an MCP call rather than a process. An MCP tool row looks like
|
||||
//
|
||||
// name: "vikunja_list_tasks" cmd: ["mcp", "vikunja", "list_tasks"]
|
||||
//
|
||||
// which is why there is no new column and no migration: the store, the /tools
|
||||
// page, ProposeTool, EnableTool, DisableTool, the act matcher and the confirm
|
||||
// turn all keep working unchanged. The executor is the only place that has to
|
||||
// know the difference, and it is one branch on Cmd[0].
|
||||
//
|
||||
// The rest of the allowlist discipline is inherited whole: a row that is not
|
||||
// status='enabled' does not run, and a row marked destructive does not run on
|
||||
// first hearing. Nothing here can enable itself — discovery only proposes.
|
||||
const CmdPrefix = "mcp"
|
||||
|
||||
// Cmd builds the argv encoding for a discovered tool.
|
||||
func Cmd(server, tool string) []string { return []string{CmdPrefix, server, tool} }
|
||||
|
||||
// ParseCmd recognises an MCP allowlist row. ok=false for an ordinary process
|
||||
// tool, which is what almost every row is.
|
||||
func ParseCmd(cmd []string) (server, tool string, ok bool) {
|
||||
if len(cmd) != 3 || cmd[0] != CmdPrefix {
|
||||
return "", "", false
|
||||
}
|
||||
if cmd[1] == "" || cmd[2] == "" {
|
||||
return "", "", false
|
||||
}
|
||||
return cmd[1], cmd[2], true
|
||||
}
|
||||
|
||||
var notName = regexp.MustCompile(`[^a-z0-9_]+`)
|
||||
|
||||
// LocalName is the allowlist name for a discovered tool: the server handle, an
|
||||
// underscore, the remote name, lowercased and stripped of anything that is not
|
||||
// a word character. Namespacing by server is what keeps two servers that both
|
||||
// offer "search" from colliding, and what makes the provenance of a row on the
|
||||
// /tools page obvious without opening the diff.
|
||||
func LocalName(server, tool string) string {
|
||||
clean := func(s string) string {
|
||||
return strings.Trim(notName.ReplaceAllString(strings.ToLower(strings.TrimSpace(s)), "_"), "_")
|
||||
}
|
||||
s, t := clean(server), clean(tool)
|
||||
switch {
|
||||
case s == "":
|
||||
return t
|
||||
case t == "":
|
||||
return s
|
||||
}
|
||||
return s + "_" + t
|
||||
}
|
||||
|
||||
// Scope is the store scope for a server's rows, so the /tools page can group
|
||||
// them and a human can tell at a glance where a capability came from.
|
||||
func Scope(server string) string { return "mcp:" + server }
|
||||
@@ -0,0 +1,267 @@
|
||||
package mcp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
// Errors callers distinguish.
|
||||
var (
|
||||
// ErrClosed — the transport is gone (subprocess died, client closed).
|
||||
ErrClosed = errors.New("mcp: connection is closed")
|
||||
// ErrNotInitialized — a call was made before the initialize handshake.
|
||||
ErrNotInitialized = errors.New("mcp: not initialized")
|
||||
// ErrToolFailed — the server ran the tool and reported an error result.
|
||||
ErrToolFailed = errors.New("mcp: tool reported an error")
|
||||
)
|
||||
|
||||
// Tool is one tool a server offers, in the form Maven cares about.
|
||||
//
|
||||
// ReadOnly comes from the server's own readOnlyHint annotation and decides
|
||||
// whether the allowlist row is marked destructive: no hint, or a false one,
|
||||
// means "assume it mutates", which routes the call through the confirm turn.
|
||||
// Guessing wrong in that direction only costs a question.
|
||||
type Tool struct {
|
||||
Server string
|
||||
Name string
|
||||
Description string
|
||||
InputSchema json.RawMessage
|
||||
ReadOnly bool
|
||||
}
|
||||
|
||||
// Resource is one resource a server offers. Contents are fetched separately —
|
||||
// listing is cheap, reading is not.
|
||||
type Resource struct {
|
||||
Server string
|
||||
URI string
|
||||
Name string
|
||||
MIMEType string
|
||||
}
|
||||
|
||||
// ServerInfo is what came back from the handshake.
|
||||
type ServerInfo struct {
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
ProtocolVersion string `json:"-"`
|
||||
}
|
||||
|
||||
// Client is one connected MCP server. Safe for concurrent use.
|
||||
type Client struct {
|
||||
name string
|
||||
tr transport
|
||||
next atomic.Int64
|
||||
|
||||
mu sync.Mutex
|
||||
info ServerInfo
|
||||
ready bool
|
||||
}
|
||||
|
||||
// newClient wraps a transport. Callers use Dial* in manager.go.
|
||||
func newClient(name string, tr transport) *Client {
|
||||
return &Client{name: name, tr: tr}
|
||||
}
|
||||
|
||||
// Name — the local name of this server (the config key, not the server's own).
|
||||
func (c *Client) Name() string { return c.name }
|
||||
|
||||
// Info — what the server said about itself during the handshake.
|
||||
func (c *Client) Info() ServerInfo {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.info
|
||||
}
|
||||
|
||||
// Initialize performs the MCP handshake and sends notifications/initialized.
|
||||
// Capabilities we declare are empty on purpose: Maven consumes, she does not
|
||||
// offer sampling or roots back to the server.
|
||||
func (c *Client) Initialize(ctx context.Context) error {
|
||||
var out struct {
|
||||
ProtocolVersion string `json:"protocolVersion"`
|
||||
ServerInfo ServerInfo `json:"serverInfo"`
|
||||
}
|
||||
err := c.call(ctx, "initialize", map[string]any{
|
||||
"protocolVersion": ProtocolVersion,
|
||||
"capabilities": map[string]any{},
|
||||
"clientInfo": map[string]any{"name": "maven", "version": "1.0"},
|
||||
}, &out)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(out.ProtocolVersion) == "" {
|
||||
return fmt.Errorf("mcp: %s: handshake returned no protocol version", c.name)
|
||||
}
|
||||
out.ServerInfo.ProtocolVersion = out.ProtocolVersion
|
||||
c.mu.Lock()
|
||||
c.info, c.ready = out.ServerInfo, true
|
||||
c.mu.Unlock()
|
||||
// Best effort: a stateless HTTP server may not care, and a failure here is
|
||||
// not worth dropping a working connection over.
|
||||
_ = c.tr.Notify(ctx, "notifications/initialized", map[string]any{})
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListTools discovers the server's tools.
|
||||
func (c *Client) ListTools(ctx context.Context) ([]Tool, error) {
|
||||
if !c.initialized() {
|
||||
return nil, ErrNotInitialized
|
||||
}
|
||||
var out struct {
|
||||
Tools []struct {
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
InputSchema json.RawMessage `json:"inputSchema"`
|
||||
Annotations *struct {
|
||||
ReadOnlyHint bool `json:"readOnlyHint"`
|
||||
} `json:"annotations"`
|
||||
} `json:"tools"`
|
||||
}
|
||||
if err := c.call(ctx, "tools/list", map[string]any{}, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tools := make([]Tool, 0, len(out.Tools))
|
||||
for _, t := range out.Tools {
|
||||
if strings.TrimSpace(t.Name) == "" {
|
||||
continue
|
||||
}
|
||||
tools = append(tools, Tool{
|
||||
Server: c.name,
|
||||
Name: t.Name,
|
||||
Description: strings.TrimSpace(t.Description),
|
||||
InputSchema: t.InputSchema,
|
||||
ReadOnly: t.Annotations != nil && t.Annotations.ReadOnlyHint,
|
||||
})
|
||||
}
|
||||
return tools, nil
|
||||
}
|
||||
|
||||
// CallTool runs one tool and returns its text content, joined by newlines.
|
||||
// Non-text content (images, blobs) is dropped: everything downstream of here
|
||||
// is a spoken or written sentence.
|
||||
//
|
||||
// args is exactly what the router produced. Nothing else — no history, no
|
||||
// notes, no persona — is in scope here, by construction.
|
||||
func (c *Client) CallTool(ctx context.Context, name string, args map[string]any) (string, error) {
|
||||
if !c.initialized() {
|
||||
return "", ErrNotInitialized
|
||||
}
|
||||
if args == nil {
|
||||
args = map[string]any{}
|
||||
}
|
||||
var out struct {
|
||||
IsError bool `json:"isError"`
|
||||
Content []struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text"`
|
||||
} `json:"content"`
|
||||
}
|
||||
if err := c.call(ctx, "tools/call", map[string]any{"name": name, "arguments": args}, &out); err != nil {
|
||||
return "", err
|
||||
}
|
||||
var parts []string
|
||||
for _, ct := range out.Content {
|
||||
if ct.Type == "text" && strings.TrimSpace(ct.Text) != "" {
|
||||
parts = append(parts, strings.TrimSpace(ct.Text))
|
||||
}
|
||||
}
|
||||
text := strings.Join(parts, "\n")
|
||||
if out.IsError {
|
||||
return text, fmt.Errorf("%w: %s/%s: %s", ErrToolFailed, c.name, name, text)
|
||||
}
|
||||
return text, nil
|
||||
}
|
||||
|
||||
// ListResources discovers the server's resources. A server without the
|
||||
// resources capability answers with an error; that is not fatal, the caller
|
||||
// gets an empty list.
|
||||
func (c *Client) ListResources(ctx context.Context) ([]Resource, error) {
|
||||
if !c.initialized() {
|
||||
return nil, ErrNotInitialized
|
||||
}
|
||||
var out struct {
|
||||
Resources []struct {
|
||||
URI string `json:"uri"`
|
||||
Name string `json:"name"`
|
||||
MIMEType string `json:"mimeType"`
|
||||
} `json:"resources"`
|
||||
}
|
||||
if err := c.call(ctx, "resources/list", map[string]any{}, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := make([]Resource, 0, len(out.Resources))
|
||||
for _, r := range out.Resources {
|
||||
if strings.TrimSpace(r.URI) == "" {
|
||||
continue
|
||||
}
|
||||
res = append(res, Resource{Server: c.name, URI: r.URI, Name: r.Name, MIMEType: r.MIMEType})
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// ReadResource returns a resource's text contents, joined by newlines. This is
|
||||
// the RAG-hint path: the text can be pasted into a router or phraser prompt.
|
||||
func (c *Client) ReadResource(ctx context.Context, uri string) (string, error) {
|
||||
if !c.initialized() {
|
||||
return "", ErrNotInitialized
|
||||
}
|
||||
var out struct {
|
||||
Contents []struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"contents"`
|
||||
}
|
||||
if err := c.call(ctx, "resources/read", map[string]any{"uri": uri}, &out); err != nil {
|
||||
return "", err
|
||||
}
|
||||
var parts []string
|
||||
for _, ct := range out.Contents {
|
||||
if strings.TrimSpace(ct.Text) != "" {
|
||||
parts = append(parts, strings.TrimSpace(ct.Text))
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, "\n"), nil
|
||||
}
|
||||
|
||||
// Close drops the connection.
|
||||
func (c *Client) Close() error {
|
||||
c.mu.Lock()
|
||||
c.ready = false
|
||||
c.mu.Unlock()
|
||||
return c.tr.Close()
|
||||
}
|
||||
|
||||
func (c *Client) initialized() bool {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.ready
|
||||
}
|
||||
|
||||
// alive reports whether the underlying transport can still carry a call. HTTP
|
||||
// is stateless, so it is always alive; a dead subprocess is not.
|
||||
func (c *Client) alive() bool {
|
||||
if s, ok := c.tr.(*stdioTransport); ok {
|
||||
return s.alive()
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *Client) call(ctx context.Context, method string, params any, out any) error {
|
||||
req := &rpcRequest{JSONRPC: "2.0", ID: c.next.Add(1), Method: method, Params: params}
|
||||
resp, err := c.tr.Call(ctx, req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("mcp: %s: %s: %w", c.name, method, err)
|
||||
}
|
||||
if resp.Error != nil {
|
||||
return fmt.Errorf("mcp: %s: %s: %w", c.name, method, resp.Error)
|
||||
}
|
||||
if out == nil || len(resp.Result) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := json.Unmarshal(resp.Result, out); err != nil {
|
||||
return fmt.Errorf("mcp: %s: %s: decode result: %w", c.name, method, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
package mcp
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Poster is the HTTP seam: internal/webfetch.Fetcher satisfies it. The
|
||||
// transport takes it as an interface so a test can serve a fake without a
|
||||
// listener, and so that the ONLY implementation wired in production is the
|
||||
// guarded fetcher — an MCP endpoint cannot get a bare http.Client this way.
|
||||
type Poster interface {
|
||||
Post(ctx context.Context, rawURL, contentType string, body []byte, hdr map[string]string) (*PostResponse, error)
|
||||
}
|
||||
|
||||
// PostResponse is the shape webfetch returns, restated here so this package
|
||||
// does not depend on it structurally.
|
||||
type PostResponse struct {
|
||||
Status int
|
||||
ContentType string
|
||||
Body []byte
|
||||
Header map[string]string
|
||||
}
|
||||
|
||||
// httpTransport speaks streamable HTTP: every request is a POST to one
|
||||
// endpoint, and the reply is either a JSON object or a text/event-stream frame
|
||||
// carrying one. Both are accepted — servers pick per response, and the two the
|
||||
// LAN runs disagree about which.
|
||||
type httpTransport struct {
|
||||
poster Poster
|
||||
url string
|
||||
|
||||
mu sync.Mutex
|
||||
session string // Mcp-Session-Id, echoed back when the server issues one
|
||||
}
|
||||
|
||||
func newHTTPTransport(post Poster, endpoint string) *httpTransport {
|
||||
return &httpTransport{poster: post, url: endpoint}
|
||||
}
|
||||
|
||||
func (t *httpTransport) Call(ctx context.Context, req *rpcRequest) (*rpcResponse, error) {
|
||||
body, err := t.send(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
frame, err := decodeFrame(body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var resp rpcResponse
|
||||
if err := json.Unmarshal(frame, &resp); err != nil {
|
||||
return nil, fmt.Errorf("mcp: decode response: %w", err)
|
||||
}
|
||||
return &resp, nil
|
||||
}
|
||||
|
||||
func (t *httpTransport) Notify(ctx context.Context, method string, params any) error {
|
||||
_, err := t.send(ctx, &rpcRequest{JSONRPC: "2.0", Method: method, Params: params})
|
||||
return err
|
||||
}
|
||||
|
||||
func (t *httpTransport) send(ctx context.Context, req *rpcRequest) ([]byte, error) {
|
||||
req.JSONRPC = "2.0"
|
||||
raw, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hdr := map[string]string{"Accept": "application/json, text/event-stream"}
|
||||
t.mu.Lock()
|
||||
if t.session != "" {
|
||||
hdr["Mcp-Session-Id"] = t.session
|
||||
}
|
||||
t.mu.Unlock()
|
||||
|
||||
resp, err := t.poster.Post(ctx, t.url, "application/json", raw, hdr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if sid := headerGet(resp.Header, "Mcp-Session-Id"); sid != "" {
|
||||
t.mu.Lock()
|
||||
t.session = sid
|
||||
t.mu.Unlock()
|
||||
}
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
func (t *httpTransport) Close() error {
|
||||
t.mu.Lock()
|
||||
t.session = ""
|
||||
t.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func headerGet(h map[string]string, key string) string {
|
||||
if h == nil {
|
||||
return ""
|
||||
}
|
||||
if v, ok := h[key]; ok {
|
||||
return v
|
||||
}
|
||||
lower := strings.ToLower(key)
|
||||
for k, v := range h {
|
||||
if strings.ToLower(k) == lower {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// decodeFrame pulls the JSON object out of a body that is either raw JSON or
|
||||
// SSE. For SSE we take the LAST data: payload that parses, which is the
|
||||
// response — earlier frames on the stream are progress notifications.
|
||||
func decodeFrame(body []byte) ([]byte, error) {
|
||||
trimmed := bytes.TrimSpace(body)
|
||||
if len(trimmed) == 0 {
|
||||
return nil, errors.New("mcp: empty response body")
|
||||
}
|
||||
if trimmed[0] == '{' || trimmed[0] == '[' {
|
||||
return trimmed, nil
|
||||
}
|
||||
var last []byte
|
||||
sc := bufio.NewScanner(bytes.NewReader(trimmed))
|
||||
sc.Buffer(make([]byte, 0, 64<<10), maxLine)
|
||||
for sc.Scan() {
|
||||
line := strings.TrimSpace(sc.Text())
|
||||
if !strings.HasPrefix(line, "data:") {
|
||||
continue
|
||||
}
|
||||
payload := strings.TrimSpace(strings.TrimPrefix(line, "data:"))
|
||||
if payload == "" {
|
||||
continue
|
||||
}
|
||||
var probe map[string]json.RawMessage
|
||||
if json.Unmarshal([]byte(payload), &probe) != nil {
|
||||
continue
|
||||
}
|
||||
if _, isResp := probe["id"]; isResp {
|
||||
last = []byte(payload)
|
||||
}
|
||||
}
|
||||
if err := sc.Err(); err != nil {
|
||||
return nil, fmt.Errorf("mcp: read event stream: %w", err)
|
||||
}
|
||||
if last == nil {
|
||||
return nil, errors.New("mcp: no JSON-RPC response in event stream")
|
||||
}
|
||||
return last, nil
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
// Package mcp is Maven's Model Context Protocol CLIENT. She is a host: she
|
||||
// connects OUT to MCP servers, discovers the tools and resources they offer,
|
||||
// and hands them to the parts of her that already exist for this — the tool
|
||||
// allowlist in the store, the confirm turn for anything that mutates, the
|
||||
// stage-3 gate that makes an uncertain act ask instead of run.
|
||||
//
|
||||
// She is not an MCP server. Nothing here exposes her own capabilities to an
|
||||
// outside caller; docs/plans/06-mcp-support.md asks for the host direction only.
|
||||
//
|
||||
// Boundaries, in code rather than in prose:
|
||||
//
|
||||
// - OFF unless configured. No mcp_servers block ⇒ no manager, no goroutine,
|
||||
// no socket.
|
||||
// - A remote server is reached through internal/webfetch, so the SSRF guard,
|
||||
// the size cap, the redirect cap and the per-host rate limit all apply to
|
||||
// an MCP endpoint exactly as they do to a news feed. Reaching a loopback
|
||||
// or LAN server means explicitly setting allow_private on THAT server —
|
||||
// a different trust level, spelled out per server rather than globally.
|
||||
// - Only the tool name and the arguments the router produced are sent. This
|
||||
// package never sees his notes, facts, history or the persona block, and
|
||||
// has no API through which a caller could pass them.
|
||||
// - Discovery proposes, it does not enable. A discovered tool lands as a
|
||||
// 'proposed' row; a human enables it on the authed surface.
|
||||
package mcp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// ProtocolVersion — the spec revision we ask for in the initialize handshake.
|
||||
// A server that answers with a different one is accepted (the spec says the
|
||||
// client may proceed if it can support what came back); we only refuse when it
|
||||
// answers with no version at all, which means it is not an MCP server.
|
||||
const ProtocolVersion = "2025-06-18"
|
||||
|
||||
// rpcRequest / rpcResponse — JSON-RPC 2.0. Deliberately hand-rolled: the wire
|
||||
// format is four fields, and the repo vendors its dependencies, so pulling a
|
||||
// library in for this would cost more than it saves.
|
||||
type rpcRequest struct {
|
||||
JSONRPC string `json:"jsonrpc"`
|
||||
ID int64 `json:"id,omitempty"`
|
||||
Method string `json:"method"`
|
||||
Params any `json:"params,omitempty"`
|
||||
}
|
||||
|
||||
type rpcResponse struct {
|
||||
JSONRPC string `json:"jsonrpc"`
|
||||
ID *int64 `json:"id"`
|
||||
Result json.RawMessage `json:"result,omitempty"`
|
||||
Error *rpcError `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type rpcError struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
func (e *rpcError) Error() string { return fmt.Sprintf("mcp: rpc error %d: %s", e.Code, e.Message) }
|
||||
|
||||
// transport carries one JSON-RPC conversation. Implementations: stdioTransport
|
||||
// (a subprocess on this box) and httpTransport (streamable HTTP, guarded by
|
||||
// webfetch). Both must be safe for concurrent use by the Client.
|
||||
type transport interface {
|
||||
// Call sends a request and returns the matching response.
|
||||
Call(ctx context.Context, req *rpcRequest) (*rpcResponse, error)
|
||||
// Notify sends a notification (no id, no reply expected).
|
||||
Notify(ctx context.Context, method string, params any) error
|
||||
// Close releases the transport (kills the subprocess, drops the session).
|
||||
Close() error
|
||||
}
|
||||
@@ -0,0 +1,523 @@
|
||||
package mcp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Defaults for a server block. Small numbers on purpose — see MaxTools.
|
||||
const (
|
||||
// DefaultTimeout bounds one JSON-RPC call. A tool that takes longer than
|
||||
// this is not usable in a spoken turn anyway.
|
||||
DefaultTimeout = 15 * time.Second
|
||||
// DefaultMaxTools caps how many tools ONE server may contribute. The
|
||||
// resident model is a 1.7B with a 4096-token context: a catalogue of forty
|
||||
// tool names does not fit in its head, and a name it half-remembers is a
|
||||
// wrong act. Twelve per server is already generous.
|
||||
DefaultMaxTools = 12
|
||||
// DefaultReconnectEvery is how long the manager waits before re-dialing a
|
||||
// server whose connection died.
|
||||
DefaultReconnectEvery = 30 * time.Second
|
||||
)
|
||||
|
||||
// ErrNoServer — the named server is not configured or not connected.
|
||||
var ErrNoServer = errors.New("mcp: no such server")
|
||||
|
||||
// ServerConfig is one configured MCP server. Off unless present.
|
||||
//
|
||||
// Exactly one of Command (a subprocess on this box) or URL (a remote or
|
||||
// loopback HTTP endpoint) must be set.
|
||||
type ServerConfig struct {
|
||||
// Name is the local handle. It prefixes every tool this server
|
||||
// contributes, so it must be short and a valid identifier-ish word.
|
||||
Name string `json:"name"`
|
||||
// Command + Args + Env + Dir describe a stdio server: a child process of
|
||||
// mavend, on this box, under this user. argv, never a shell string.
|
||||
Command string `json:"command,omitempty"`
|
||||
Args []string `json:"args,omitempty"`
|
||||
Env []string `json:"env,omitempty"`
|
||||
Dir string `json:"dir,omitempty"`
|
||||
// URL is a streamable-HTTP endpoint. It goes through internal/webfetch, so
|
||||
// it inherits the SSRF guard, the size cap and the per-host rate limit.
|
||||
URL string `json:"url,omitempty"`
|
||||
// AllowPrivate lets THIS server be a loopback or LAN address
|
||||
// (http://localhost:9100/mcp is the Vikunja server on homesrv). It is a
|
||||
// per-server hole in the private-address guard and it is not the same trust
|
||||
// level as a public endpoint: whatever is behind it is inside the network,
|
||||
// so an argument the router got wrong reaches something that matters. Set
|
||||
// it only for a server you run.
|
||||
AllowPrivate bool `json:"allow_private,omitempty"`
|
||||
// AllowTools, when non-empty, is the ONLY set of remote tool names taken
|
||||
// from this server. This is the knob for keeping the catalogue small and
|
||||
// deliberate rather than "whatever the server grew this week".
|
||||
AllowTools []string `json:"allow_tools,omitempty"`
|
||||
// MaxTools caps the contribution (0 ⇒ DefaultMaxTools).
|
||||
MaxTools int `json:"max_tools,omitempty"`
|
||||
// Timeout bounds one call (0 ⇒ DefaultTimeout).
|
||||
Timeout time.Duration `json:"-"`
|
||||
// Enabled=false keeps a configured server described but dark.
|
||||
Enabled bool `json:"enabled"`
|
||||
}
|
||||
|
||||
// PosterFactory builds the HTTP door for one server. It is a factory rather
|
||||
// than a single shared Poster because allow_private is per server: the fetcher
|
||||
// that may reach http://localhost:9100/mcp must NOT be the same fetcher another
|
||||
// server's public URL goes through, or one loopback exemption would quietly
|
||||
// unlock the LAN for all of them.
|
||||
type PosterFactory func(cfg ServerConfig) (Poster, error)
|
||||
|
||||
// Manager owns the connections. Nothing here starts unless at least one server
|
||||
// is configured and enabled.
|
||||
type Manager struct {
|
||||
newPoster PosterFactory
|
||||
mu sync.Mutex
|
||||
conns map[string]*conn
|
||||
order []string
|
||||
}
|
||||
|
||||
type conn struct {
|
||||
cfg ServerConfig
|
||||
client *Client
|
||||
tools []Tool
|
||||
lastErr error
|
||||
lastTry time.Time
|
||||
dialedAt time.Time
|
||||
}
|
||||
|
||||
// NewManager builds a manager for the enabled servers in cfgs. newPoster is
|
||||
// the guarded HTTP door factory for url servers; pass nil only when no url
|
||||
// server is configured (a nil factory with a url server is reported per server
|
||||
// at dial time rather than fatally, so one bad block never stops the daemon).
|
||||
//
|
||||
// Dialing is lazy: NewManager validates and records, Connect dials.
|
||||
func NewManager(newPoster PosterFactory, cfgs []ServerConfig) (*Manager, error) {
|
||||
m := &Manager{newPoster: newPoster, conns: map[string]*conn{}}
|
||||
for _, c := range cfgs {
|
||||
if !c.Enabled {
|
||||
continue
|
||||
}
|
||||
if err := validate(c); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, dup := m.conns[c.Name]; dup {
|
||||
return nil, fmt.Errorf("mcp: duplicate server name %q", c.Name)
|
||||
}
|
||||
if c.Timeout <= 0 {
|
||||
c.Timeout = DefaultTimeout
|
||||
}
|
||||
if c.MaxTools <= 0 {
|
||||
c.MaxTools = DefaultMaxTools
|
||||
}
|
||||
m.conns[c.Name] = &conn{cfg: c}
|
||||
m.order = append(m.order, c.Name)
|
||||
}
|
||||
sort.Strings(m.order)
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// Validate checks a set of server blocks without dialling anything, so a typo
|
||||
// fails at startup rather than at the first turn that needed the tool.
|
||||
func Validate(cfgs []ServerConfig) error {
|
||||
seen := map[string]bool{}
|
||||
for _, c := range cfgs {
|
||||
if err := validate(c); err != nil {
|
||||
return err
|
||||
}
|
||||
if seen[c.Name] {
|
||||
return fmt.Errorf("mcp: duplicate server name %q", c.Name)
|
||||
}
|
||||
seen[c.Name] = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validate(c ServerConfig) error {
|
||||
if strings.TrimSpace(c.Name) == "" {
|
||||
return errors.New("mcp: server needs a name")
|
||||
}
|
||||
if strings.ContainsAny(c.Name, " \t/:") {
|
||||
return fmt.Errorf("mcp: server name %q must be one word without spaces, slashes or colons", c.Name)
|
||||
}
|
||||
hasCmd, hasURL := c.Command != "", c.URL != ""
|
||||
if hasCmd == hasURL {
|
||||
return fmt.Errorf("mcp: server %q needs exactly one of command or url", c.Name)
|
||||
}
|
||||
if hasURL && !strings.HasPrefix(c.URL, "http://") && !strings.HasPrefix(c.URL, "https://") {
|
||||
return fmt.Errorf("mcp: server %q url must be http or https", c.Name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Servers — the configured, enabled server names, sorted.
|
||||
func (m *Manager) Servers() []string {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return append([]string(nil), m.order...)
|
||||
}
|
||||
|
||||
// Empty reports whether nothing is configured. The daemon uses it to skip
|
||||
// wiring entirely.
|
||||
func (m *Manager) Empty() bool {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return len(m.conns) == 0
|
||||
}
|
||||
|
||||
// Connect dials every configured server, handshakes, and discovers tools.
|
||||
// A server that fails is recorded and retried later by Refresh — one bad
|
||||
// server never blocks the others, and never blocks boot.
|
||||
func (m *Manager) Connect(ctx context.Context) {
|
||||
for _, name := range m.Servers() {
|
||||
if err := m.dial(ctx, name); err != nil {
|
||||
log.Printf("mcp: %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) dial(ctx context.Context, name string) error {
|
||||
m.mu.Lock()
|
||||
c, ok := m.conns[name]
|
||||
if !ok {
|
||||
m.mu.Unlock()
|
||||
return ErrNoServer
|
||||
}
|
||||
cfg := c.cfg
|
||||
c.lastTry = time.Now()
|
||||
m.mu.Unlock()
|
||||
|
||||
var tr transport
|
||||
var err error
|
||||
if cfg.Command != "" {
|
||||
tr, err = newStdioTransport(ctx, append([]string{cfg.Command}, cfg.Args...), cfg.Env, cfg.Dir)
|
||||
} else if m.newPoster == nil {
|
||||
err = fmt.Errorf("server %q has a url but no http door was wired", name)
|
||||
} else {
|
||||
var poster Poster
|
||||
if poster, err = m.newPoster(cfg); err == nil {
|
||||
tr = newHTTPTransport(poster, cfg.URL)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
m.fail(name, err)
|
||||
return err
|
||||
}
|
||||
|
||||
cl := newClient(name, tr)
|
||||
ictx, cancel := context.WithTimeout(ctx, cfg.Timeout)
|
||||
defer cancel()
|
||||
if err := cl.Initialize(ictx); err != nil {
|
||||
_ = cl.Close()
|
||||
m.fail(name, err)
|
||||
return err
|
||||
}
|
||||
tools, err := cl.ListTools(ictx)
|
||||
if err != nil {
|
||||
// A server with no tools capability is still a usable resource server.
|
||||
log.Printf("mcp: %s: list tools: %v", name, err)
|
||||
tools = nil
|
||||
}
|
||||
tools = filterTools(cfg, tools)
|
||||
|
||||
m.mu.Lock()
|
||||
if old := m.conns[name].client; old != nil {
|
||||
_ = old.Close()
|
||||
}
|
||||
m.conns[name].client = cl
|
||||
m.conns[name].tools = tools
|
||||
m.conns[name].lastErr = nil
|
||||
m.conns[name].dialedAt = time.Now()
|
||||
m.mu.Unlock()
|
||||
log.Printf("mcp: %s connected (%s %s), %d tool(s)", name, cl.Info().Name, cl.Info().Version, len(tools))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Manager) fail(name string, err error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if c := m.conns[name]; c != nil {
|
||||
c.lastErr = err
|
||||
c.client = nil
|
||||
c.tools = nil
|
||||
}
|
||||
}
|
||||
|
||||
// filterTools applies AllowTools and MaxTools, and drops nameless entries.
|
||||
// Sorted first, so the cap is deterministic rather than "whatever order the
|
||||
// server felt like".
|
||||
func filterTools(cfg ServerConfig, in []Tool) []Tool {
|
||||
sort.Slice(in, func(i, j int) bool { return in[i].Name < in[j].Name })
|
||||
out := make([]Tool, 0, len(in))
|
||||
for _, t := range in {
|
||||
if len(cfg.AllowTools) > 0 && !contains(cfg.AllowTools, t.Name) {
|
||||
continue
|
||||
}
|
||||
out = append(out, t)
|
||||
}
|
||||
if cfg.MaxTools > 0 && len(out) > cfg.MaxTools {
|
||||
log.Printf("mcp: %s offers %d tools, taking the first %d (raise max_tools or set allow_tools)",
|
||||
cfg.Name, len(out), cfg.MaxTools)
|
||||
out = out[:cfg.MaxTools]
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func contains(hay []string, needle string) bool {
|
||||
for _, h := range hay {
|
||||
if h == needle {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Refresh re-dials any server that is down, if enough time has passed since the
|
||||
// last attempt. Call it from the daemon's periodic tick — it is cheap when
|
||||
// everything is up.
|
||||
func (m *Manager) Refresh(ctx context.Context) {
|
||||
now := time.Now()
|
||||
var stale []string
|
||||
m.mu.Lock()
|
||||
for _, name := range m.order {
|
||||
c := m.conns[name]
|
||||
down := c.client == nil || !c.client.alive()
|
||||
if down && now.Sub(c.lastTry) >= DefaultReconnectEvery {
|
||||
stale = append(stale, name)
|
||||
}
|
||||
}
|
||||
m.mu.Unlock()
|
||||
for _, name := range stale {
|
||||
if err := m.dial(ctx, name); err != nil {
|
||||
log.Printf("mcp: %s: reconnect: %v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Tools — every discovered tool across connected servers, sorted by
|
||||
// server then name.
|
||||
func (m *Manager) Tools() []Tool {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
var out []Tool
|
||||
for _, name := range m.order {
|
||||
out = append(out, m.conns[name].tools...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Status is one server's health, for the web surface.
|
||||
type Status struct {
|
||||
Name string
|
||||
Transport string // "stdio" or "http"
|
||||
Target string // command or url
|
||||
Connected bool
|
||||
Server string // the server's own name+version
|
||||
Tools int
|
||||
Err string
|
||||
}
|
||||
|
||||
// Status reports every configured server.
|
||||
func (m *Manager) Status() []Status {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
out := make([]Status, 0, len(m.order))
|
||||
for _, name := range m.order {
|
||||
c := m.conns[name]
|
||||
s := Status{Name: name, Tools: len(c.tools)}
|
||||
if c.cfg.Command != "" {
|
||||
s.Transport, s.Target = "stdio", strings.Join(append([]string{c.cfg.Command}, c.cfg.Args...), " ")
|
||||
} else {
|
||||
s.Transport, s.Target = "http", c.cfg.URL
|
||||
}
|
||||
if c.client != nil {
|
||||
s.Connected = true
|
||||
s.Server = strings.TrimSpace(c.client.Info().Name + " " + c.client.Info().Version)
|
||||
}
|
||||
if c.lastErr != nil {
|
||||
s.Err = c.lastErr.Error()
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Call runs server's tool with args. Args come from the router and nothing
|
||||
// else; there is no path here through which a note or a fact could travel.
|
||||
func (m *Manager) Call(ctx context.Context, server, tool string, args map[string]any) (string, error) {
|
||||
m.mu.Lock()
|
||||
c := m.conns[server]
|
||||
m.mu.Unlock()
|
||||
if c == nil {
|
||||
return "", fmt.Errorf("%w: %s", ErrNoServer, server)
|
||||
}
|
||||
m.mu.Lock()
|
||||
cl, timeout, known := c.client, c.cfg.Timeout, false
|
||||
for _, t := range c.tools {
|
||||
if t.Name == tool {
|
||||
known = true
|
||||
break
|
||||
}
|
||||
}
|
||||
m.mu.Unlock()
|
||||
if cl == nil {
|
||||
return "", fmt.Errorf("mcp: %s is not connected", server)
|
||||
}
|
||||
// The discovered-and-filtered set is the second allowlist: even an enabled
|
||||
// store row cannot reach a tool the server stopped offering, or one
|
||||
// allow_tools excludes.
|
||||
if !known {
|
||||
return "", fmt.Errorf("mcp: %s offers no tool %q", server, tool)
|
||||
}
|
||||
cctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
return cl.CallTool(cctx, tool, args)
|
||||
}
|
||||
|
||||
// Resources lists resources across connected servers.
|
||||
func (m *Manager) Resources(ctx context.Context) []Resource {
|
||||
m.mu.Lock()
|
||||
clients := make([]*Client, 0, len(m.order))
|
||||
for _, name := range m.order {
|
||||
if cl := m.conns[name].client; cl != nil {
|
||||
clients = append(clients, cl)
|
||||
}
|
||||
}
|
||||
m.mu.Unlock()
|
||||
var out []Resource
|
||||
for _, cl := range clients {
|
||||
rs, err := cl.ListResources(ctx)
|
||||
if err != nil {
|
||||
continue // no resources capability; not an error worth logging per tick
|
||||
}
|
||||
out = append(out, rs...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ReadResource reads one resource from one server.
|
||||
func (m *Manager) ReadResource(ctx context.Context, server, uri string) (string, error) {
|
||||
m.mu.Lock()
|
||||
c := m.conns[server]
|
||||
var cl *Client
|
||||
var timeout time.Duration
|
||||
if c != nil {
|
||||
cl, timeout = c.client, c.cfg.Timeout
|
||||
}
|
||||
m.mu.Unlock()
|
||||
if cl == nil {
|
||||
return "", fmt.Errorf("%w: %s", ErrNoServer, server)
|
||||
}
|
||||
cctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
return cl.ReadResource(cctx, uri)
|
||||
}
|
||||
|
||||
// Close shuts every connection down.
|
||||
func (m *Manager) Close() error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
for _, name := range m.order {
|
||||
if cl := m.conns[name].client; cl != nil {
|
||||
_ = cl.Close()
|
||||
m.conns[name].client = nil
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ErrNeedsArgs — the tool requires arguments that a voice verb cannot supply.
|
||||
var ErrNeedsArgs = errors.New("mcp: tool needs named arguments")
|
||||
|
||||
// CallPositional is the voice path's way in. The router gives an act a verb and
|
||||
// a tail of positional words; an MCP tool wants a named-argument object. There
|
||||
// is no general mapping between those two, and inventing one is exactly the
|
||||
// improvisation this codebase refuses, so the rule is deliberately narrow:
|
||||
//
|
||||
// - a tool with no required properties runs with no arguments (a spare tail
|
||||
// is ignored — "покажи проекты пожалуйста" should still list projects);
|
||||
// - a READ-ONLY tool with exactly one required property, of type string or
|
||||
// integer/number, gets the tail bound to it;
|
||||
// - anything else is refused with ErrNeedsArgs. Such a tool is still callable
|
||||
// with explicit arguments from the authed surface, where a human types
|
||||
// them.
|
||||
//
|
||||
// The refusal is the point, and the read-only condition on it was learned the
|
||||
// hard way while testing against the Vikunja server: `update_task` requires
|
||||
// only `task_id` and takes every other field as optional, so calling it with
|
||||
// one guessed argument and no others BLANKED the fields it did not receive. A
|
||||
// mutating tool therefore never gets a guessed argument — the one thing a
|
||||
// partially-filled write can do is destroy what it did not mention. A mutating
|
||||
// tool with nothing required is still fine: nothing was guessed, and it still
|
||||
// goes through the confirm turn.
|
||||
func (m *Manager) CallPositional(ctx context.Context, server, tool string, args []string) (string, error) {
|
||||
m.mu.Lock()
|
||||
c := m.conns[server]
|
||||
var schema json.RawMessage
|
||||
found, readOnly := false, false
|
||||
if c != nil {
|
||||
for _, t := range c.tools {
|
||||
if t.Name == tool {
|
||||
schema, readOnly, found = t.InputSchema, t.ReadOnly, true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
m.mu.Unlock()
|
||||
if !found {
|
||||
return "", fmt.Errorf("mcp: %s offers no tool %q", server, tool)
|
||||
}
|
||||
named, err := bindPositional(schema, args, readOnly)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return m.Call(ctx, server, tool, named)
|
||||
}
|
||||
|
||||
// bindPositional implements the rule documented on CallPositional.
|
||||
func bindPositional(schema json.RawMessage, args []string, readOnly bool) (map[string]any, error) {
|
||||
var s struct {
|
||||
Required []string `json:"required"`
|
||||
Properties map[string]struct {
|
||||
Type string `json:"type"`
|
||||
} `json:"properties"`
|
||||
}
|
||||
if len(schema) > 0 {
|
||||
if err := json.Unmarshal(schema, &s); err != nil {
|
||||
return nil, fmt.Errorf("mcp: unreadable input schema: %w", err)
|
||||
}
|
||||
}
|
||||
switch len(s.Required) {
|
||||
case 0:
|
||||
return map[string]any{}, nil
|
||||
case 1:
|
||||
name := s.Required[0]
|
||||
if !readOnly {
|
||||
return nil, fmt.Errorf("%w: %q, and a tool that writes never gets a guessed one", ErrNeedsArgs, name)
|
||||
}
|
||||
tail := strings.TrimSpace(strings.Join(args, " "))
|
||||
if tail == "" {
|
||||
return nil, fmt.Errorf("%w: %q", ErrNeedsArgs, name)
|
||||
}
|
||||
switch s.Properties[name].Type {
|
||||
case "string", "":
|
||||
return map[string]any{name: tail}, nil
|
||||
case "integer", "number":
|
||||
n, err := strconv.ParseFloat(tail, 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %q wants a number, got %q", ErrNeedsArgs, name, tail)
|
||||
}
|
||||
return map[string]any{name: n}, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("%w: %q is a %s", ErrNeedsArgs, name, s.Properties[name].Type)
|
||||
}
|
||||
default:
|
||||
return nil, fmt.Errorf("%w: %s", ErrNeedsArgs, strings.Join(s.Required, ", "))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,565 @@
|
||||
package mcp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fakePoster answers POSTs from a canned handler, in either JSON or SSE form.
|
||||
type fakePoster struct {
|
||||
mu sync.Mutex
|
||||
handler func(method string, params json.RawMessage) (any, *rpcError)
|
||||
sse bool
|
||||
session string
|
||||
seen []map[string]string // headers of each request, for the session test
|
||||
calls []string
|
||||
}
|
||||
|
||||
func (f *fakePoster) Post(_ context.Context, _, _ string, body []byte, hdr map[string]string) (*PostResponse, error) {
|
||||
var req struct {
|
||||
ID *int64 `json:"id"`
|
||||
Method string `json:"method"`
|
||||
Params json.RawMessage `json:"params"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &req); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f.mu.Lock()
|
||||
f.seen = append(f.seen, hdr)
|
||||
f.calls = append(f.calls, req.Method)
|
||||
f.mu.Unlock()
|
||||
|
||||
if req.ID == nil { // notification
|
||||
return &PostResponse{Status: 202, Body: []byte(`{}`)}, nil
|
||||
}
|
||||
result, rerr := f.handler(req.Method, req.Params)
|
||||
resp := map[string]any{"jsonrpc": "2.0", "id": *req.ID}
|
||||
if rerr != nil {
|
||||
resp["error"] = map[string]any{"code": rerr.Code, "message": rerr.Message}
|
||||
} else {
|
||||
resp["result"] = result
|
||||
}
|
||||
raw, _ := json.Marshal(resp)
|
||||
out := &PostResponse{Status: 200, Body: raw, ContentType: "application/json", Header: map[string]string{}}
|
||||
if f.sse {
|
||||
out.ContentType = "text/event-stream"
|
||||
out.Body = []byte("event: message\ndata: {\"jsonrpc\":\"2.0\",\"method\":\"notifications/progress\"}\n\nevent: message\ndata: " + string(raw) + "\n\n")
|
||||
}
|
||||
if f.session != "" {
|
||||
out.Header["Mcp-Session-Id"] = f.session
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// echoServer is a handler with two tools, one read-only and one not.
|
||||
func echoServer() func(string, json.RawMessage) (any, *rpcError) {
|
||||
return func(method string, params json.RawMessage) (any, *rpcError) {
|
||||
switch method {
|
||||
case "initialize":
|
||||
return map[string]any{
|
||||
"protocolVersion": ProtocolVersion,
|
||||
"serverInfo": map[string]any{"name": "fake", "version": "0.1"},
|
||||
}, nil
|
||||
case "tools/list":
|
||||
return map[string]any{"tools": []any{
|
||||
map[string]any{
|
||||
"name": "read_thing", "description": "reads",
|
||||
"inputSchema": map[string]any{"type": "object"},
|
||||
"annotations": map[string]any{"readOnlyHint": true},
|
||||
},
|
||||
map[string]any{"name": "break_thing", "description": "mutates"},
|
||||
}}, nil
|
||||
case "tools/call":
|
||||
var p struct {
|
||||
Name string `json:"name"`
|
||||
Args map[string]any `json:"arguments"`
|
||||
}
|
||||
_ = json.Unmarshal(params, &p)
|
||||
if p.Name == "break_thing" {
|
||||
return map[string]any{"isError": true, "content": []any{
|
||||
map[string]any{"type": "text", "text": "не вышло"}}}, nil
|
||||
}
|
||||
return map[string]any{"content": []any{
|
||||
map[string]any{"type": "text", "text": fmt.Sprintf("%s:%v", p.Name, p.Args["q"])},
|
||||
map[string]any{"type": "image", "text": "ignored"},
|
||||
}}, nil
|
||||
case "resources/list":
|
||||
return map[string]any{"resources": []any{
|
||||
map[string]any{"uri": "note://one", "name": "one", "mimeType": "text/plain"},
|
||||
map[string]any{"uri": "", "name": "nameless"},
|
||||
}}, nil
|
||||
case "resources/read":
|
||||
return map[string]any{"contents": []any{map[string]any{"text": "тело ресурса"}}}, nil
|
||||
}
|
||||
return nil, &rpcError{Code: -32601, Message: "method not found"}
|
||||
}
|
||||
}
|
||||
|
||||
func dialFake(t *testing.T, p *fakePoster) *Client {
|
||||
t.Helper()
|
||||
c := newClient("fake", newHTTPTransport(p, "http://example.test/mcp"))
|
||||
if err := c.Initialize(context.Background()); err != nil {
|
||||
t.Fatalf("initialize: %v", err)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func TestHandshakeAndDiscovery(t *testing.T) {
|
||||
for _, sse := range []bool{false, true} {
|
||||
name := "json"
|
||||
if sse {
|
||||
name = "sse"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
p := &fakePoster{handler: echoServer(), sse: sse}
|
||||
c := dialFake(t, p)
|
||||
if got := c.Info().Name; got != "fake" {
|
||||
t.Fatalf("server name = %q", got)
|
||||
}
|
||||
if got := c.Info().ProtocolVersion; got != ProtocolVersion {
|
||||
t.Fatalf("protocol = %q", got)
|
||||
}
|
||||
tools, err := c.ListTools(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("list tools: %v", err)
|
||||
}
|
||||
if len(tools) != 2 {
|
||||
t.Fatalf("tools = %+v", tools)
|
||||
}
|
||||
byName := map[string]Tool{}
|
||||
for _, tl := range tools {
|
||||
byName[tl.Name] = tl
|
||||
}
|
||||
if !byName["read_thing"].ReadOnly {
|
||||
t.Error("read_thing should be read-only (readOnlyHint true)")
|
||||
}
|
||||
// The important direction: no annotation ⇒ assume it mutates.
|
||||
if byName["break_thing"].ReadOnly {
|
||||
t.Error("break_thing has no readOnlyHint, must NOT be treated as read-only")
|
||||
}
|
||||
if byName["read_thing"].Server != "fake" {
|
||||
t.Error("tool should carry its server handle")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCallToolTextOnly(t *testing.T) {
|
||||
c := dialFake(t, &fakePoster{handler: echoServer()})
|
||||
out, err := c.CallTool(context.Background(), "read_thing", map[string]any{"q": "привет"})
|
||||
if err != nil {
|
||||
t.Fatalf("call: %v", err)
|
||||
}
|
||||
if out != "read_thing:привет" {
|
||||
t.Fatalf("out = %q (non-text content must be dropped)", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCallToolErrorResult(t *testing.T) {
|
||||
c := dialFake(t, &fakePoster{handler: echoServer()})
|
||||
out, err := c.CallTool(context.Background(), "break_thing", nil)
|
||||
if err == nil {
|
||||
t.Fatal("isError result must surface as an error")
|
||||
}
|
||||
if out != "не вышло" {
|
||||
t.Fatalf("text should still come back, got %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResources(t *testing.T) {
|
||||
c := dialFake(t, &fakePoster{handler: echoServer()})
|
||||
rs, err := c.ListResources(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("list resources: %v", err)
|
||||
}
|
||||
if len(rs) != 1 || rs[0].URI != "note://one" {
|
||||
t.Fatalf("resources = %+v (a uri-less entry must be dropped)", rs)
|
||||
}
|
||||
body, err := c.ReadResource(context.Background(), "note://one")
|
||||
if err != nil {
|
||||
t.Fatalf("read: %v", err)
|
||||
}
|
||||
if body != "тело ресурса" {
|
||||
t.Fatalf("body = %q", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCallBeforeInitializeRefused(t *testing.T) {
|
||||
c := newClient("fake", newHTTPTransport(&fakePoster{handler: echoServer()}, "http://example.test/mcp"))
|
||||
if _, err := c.CallTool(context.Background(), "read_thing", nil); err != ErrNotInitialized {
|
||||
t.Fatalf("err = %v, want ErrNotInitialized", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionIDEchoed(t *testing.T) {
|
||||
p := &fakePoster{handler: echoServer(), session: "sess-1"}
|
||||
c := dialFake(t, p)
|
||||
if _, err := c.ListTools(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
last := p.seen[len(p.seen)-1]
|
||||
if last["Mcp-Session-Id"] != "sess-1" {
|
||||
t.Fatalf("session header not echoed: %+v", last)
|
||||
}
|
||||
if !strings.Contains(last["Accept"], "text/event-stream") {
|
||||
t.Fatalf("Accept must offer both forms: %q", last["Accept"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandshakeWithoutProtocolVersionRefused(t *testing.T) {
|
||||
p := &fakePoster{handler: func(m string, _ json.RawMessage) (any, *rpcError) {
|
||||
return map[string]any{"serverInfo": map[string]any{"name": "not-mcp"}}, nil
|
||||
}}
|
||||
c := newClient("x", newHTTPTransport(p, "http://example.test/mcp"))
|
||||
if err := c.Initialize(context.Background()); err == nil {
|
||||
t.Fatal("a reply with no protocolVersion is not an MCP server")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCErrorSurfaces(t *testing.T) {
|
||||
c := dialFake(t, &fakePoster{handler: echoServer()})
|
||||
if _, err := c.callRaw(context.Background(), "nope/nope"); err == nil {
|
||||
t.Fatal("want an rpc error")
|
||||
} else if !strings.Contains(err.Error(), "method not found") {
|
||||
t.Fatalf("err = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// callRaw is a test-only shim so the rpc-error path can be exercised without a
|
||||
// typed wrapper for a method the server does not implement.
|
||||
func (c *Client) callRaw(ctx context.Context, method string) (any, error) {
|
||||
var out any
|
||||
err := c.call(ctx, method, map[string]any{}, &out)
|
||||
return out, err
|
||||
}
|
||||
|
||||
func TestDecodeFrame(t *testing.T) {
|
||||
cases := []struct {
|
||||
name, in, want string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "plain json", in: `{"id":1,"result":{}}`, want: `{"id":1,"result":{}}`},
|
||||
{name: "sse single", in: "event: message\ndata: {\"id\":1,\"result\":1}\n\n", want: `{"id":1,"result":1}`},
|
||||
{
|
||||
name: "sse picks the response not the notification",
|
||||
in: "data: {\"method\":\"notifications/progress\"}\n\ndata: {\"id\":2,\"result\":2}\n\n",
|
||||
want: `{"id":2,"result":2}`,
|
||||
},
|
||||
{name: "empty", in: " ", wantErr: true},
|
||||
{name: "sse with no response", in: "data: {\"method\":\"x\"}\n\n", wantErr: true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, err := decodeFrame([]byte(tc.in))
|
||||
if tc.wantErr {
|
||||
if err == nil {
|
||||
t.Fatalf("want error, got %q", got)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(got) != tc.want {
|
||||
t.Fatalf("got %q want %q", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidate(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
cfg ServerConfig
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "stdio ok", cfg: ServerConfig{Name: "a", Command: "echo"}},
|
||||
{name: "http ok", cfg: ServerConfig{Name: "a", URL: "http://x.test/mcp"}},
|
||||
{name: "no name", cfg: ServerConfig{Command: "echo"}, wantErr: true},
|
||||
{name: "spacey name", cfg: ServerConfig{Name: "a b", Command: "echo"}, wantErr: true},
|
||||
{name: "neither", cfg: ServerConfig{Name: "a"}, wantErr: true},
|
||||
{name: "both", cfg: ServerConfig{Name: "a", Command: "echo", URL: "http://x.test"}, wantErr: true},
|
||||
{name: "bad scheme", cfg: ServerConfig{Name: "a", URL: "file:///etc/passwd"}, wantErr: true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
err := Validate([]ServerConfig{tc.cfg})
|
||||
if (err != nil) != tc.wantErr {
|
||||
t.Fatalf("err = %v, wantErr = %v", err, tc.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
if err := Validate([]ServerConfig{{Name: "a", Command: "x"}, {Name: "a", Command: "y"}}); err == nil {
|
||||
t.Error("duplicate names must be refused")
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagerOffWhenNothingEnabled(t *testing.T) {
|
||||
m, err := NewManager(nil, []ServerConfig{{Name: "a", Command: "echo"}}) // Enabled=false
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !m.Empty() {
|
||||
t.Fatal("a server that is not enabled must not be wired")
|
||||
}
|
||||
m.Connect(context.Background())
|
||||
if got := m.Tools(); len(got) != 0 {
|
||||
t.Fatalf("tools = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagerDiscoversAndCalls(t *testing.T) {
|
||||
p := &fakePoster{handler: echoServer()}
|
||||
m, err := NewManager(func(ServerConfig) (Poster, error) { return p, nil },
|
||||
[]ServerConfig{{Name: "fake", URL: "http://example.test/mcp", Enabled: true}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Connect(context.Background())
|
||||
defer m.Close()
|
||||
|
||||
tools := m.Tools()
|
||||
if len(tools) != 2 {
|
||||
t.Fatalf("tools = %+v", tools)
|
||||
}
|
||||
out, err := m.Call(context.Background(), "fake", "read_thing", map[string]any{"q": "да"})
|
||||
if err != nil {
|
||||
t.Fatalf("call: %v", err)
|
||||
}
|
||||
if out != "read_thing:да" {
|
||||
t.Fatalf("out = %q", out)
|
||||
}
|
||||
// The discovered set is a second allowlist.
|
||||
if _, err := m.Call(context.Background(), "fake", "not_offered", nil); err == nil {
|
||||
t.Error("a tool the server does not offer must be refused")
|
||||
}
|
||||
if _, err := m.Call(context.Background(), "other", "read_thing", nil); err == nil {
|
||||
t.Error("an unconfigured server must be refused")
|
||||
}
|
||||
st := m.Status()
|
||||
if len(st) != 1 || !st[0].Connected || st[0].Transport != "http" || st[0].Tools != 2 {
|
||||
t.Fatalf("status = %+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagerAllowToolsAndMaxTools(t *testing.T) {
|
||||
p := &fakePoster{handler: echoServer()}
|
||||
mk := func(cfg ServerConfig) *Manager {
|
||||
cfg.Name, cfg.URL, cfg.Enabled = "fake", "http://example.test/mcp", true
|
||||
m, err := NewManager(func(ServerConfig) (Poster, error) { return p, nil }, []ServerConfig{cfg})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Connect(context.Background())
|
||||
return m
|
||||
}
|
||||
m := mk(ServerConfig{AllowTools: []string{"read_thing"}})
|
||||
defer m.Close()
|
||||
if got := m.Tools(); len(got) != 1 || got[0].Name != "read_thing" {
|
||||
t.Fatalf("allow_tools ignored: %+v", got)
|
||||
}
|
||||
if _, err := m.Call(context.Background(), "fake", "break_thing", nil); err == nil {
|
||||
t.Error("a tool excluded by allow_tools must be unreachable")
|
||||
}
|
||||
m2 := mk(ServerConfig{MaxTools: 1})
|
||||
defer m2.Close()
|
||||
if got := m2.Tools(); len(got) != 1 || got[0].Name != "break_thing" {
|
||||
t.Fatalf("max_tools should keep the first name-sorted tool: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagerURLServerWithoutHTTPDoor(t *testing.T) {
|
||||
m, err := NewManager(nil, []ServerConfig{{Name: "fake", URL: "http://example.test/mcp", Enabled: true}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Connect(context.Background())
|
||||
st := m.Status()
|
||||
if len(st) != 1 || st[0].Connected || st[0].Err == "" {
|
||||
t.Fatalf("a url server with no poster must be recorded as failed: %+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagerReconnectAfterFailure(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
fail := true
|
||||
m, err := NewManager(func(ServerConfig) (Poster, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if fail {
|
||||
return nil, fmt.Errorf("down")
|
||||
}
|
||||
return &fakePoster{handler: echoServer()}, nil
|
||||
}, []ServerConfig{{Name: "fake", URL: "http://example.test/mcp", Enabled: true}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer m.Close()
|
||||
m.Connect(context.Background())
|
||||
if m.Status()[0].Connected {
|
||||
t.Fatal("should be down")
|
||||
}
|
||||
mu.Lock()
|
||||
fail = false
|
||||
mu.Unlock()
|
||||
// Refresh honours the backoff, so pretend the last attempt was long ago.
|
||||
m.mu.Lock()
|
||||
m.conns["fake"].lastTry = time.Now().Add(-2 * DefaultReconnectEvery)
|
||||
m.mu.Unlock()
|
||||
m.Refresh(context.Background())
|
||||
if !m.Status()[0].Connected {
|
||||
t.Fatalf("should have reconnected: %+v", m.Status())
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalNameAndCmd(t *testing.T) {
|
||||
cases := [][3]string{
|
||||
{"vikunja", "list_tasks", "vikunja_list_tasks"},
|
||||
{"Vikunja", "Get Task Details", "vikunja_get_task_details"},
|
||||
{"fs", "read-file", "fs_read_file"},
|
||||
{"", "search", "search"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := LocalName(c[0], c[1]); got != c[2] {
|
||||
t.Errorf("LocalName(%q,%q) = %q want %q", c[0], c[1], got, c[2])
|
||||
}
|
||||
}
|
||||
server, tool, ok := ParseCmd(Cmd("vikunja", "list_tasks"))
|
||||
if !ok || server != "vikunja" || tool != "list_tasks" {
|
||||
t.Fatalf("ParseCmd round-trip: %q %q %v", server, tool, ok)
|
||||
}
|
||||
for _, bad := range [][]string{nil, {"systemctl", "restart", "nginx"}, {"mcp", "vikunja"}, {"mcp", "", "x"}} {
|
||||
if _, _, ok := ParseCmd(bad); ok {
|
||||
t.Errorf("ParseCmd(%v) must not claim an ordinary tool row", bad)
|
||||
}
|
||||
}
|
||||
if Scope("vikunja") != "mcp:vikunja" {
|
||||
t.Error("scope")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBindPositional(t *testing.T) {
|
||||
cases := []struct {
|
||||
name, schema string
|
||||
args []string
|
||||
mutating bool
|
||||
want map[string]any
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "no required runs with nothing",
|
||||
// A spare tail is fine: "покажи проекты пожалуйста" still lists them.
|
||||
schema: `{"type":"object","properties":{},"required":[]}`,
|
||||
args: []string{"пожалуйста"},
|
||||
want: map[string]any{},
|
||||
},
|
||||
{
|
||||
name: "empty schema",
|
||||
schema: ``,
|
||||
want: map[string]any{},
|
||||
},
|
||||
{
|
||||
name: "one required string gets the tail",
|
||||
schema: `{"properties":{"q":{"type":"string"}},"required":["q"]}`,
|
||||
args: []string{"почему", "небо", "синее"},
|
||||
want: map[string]any{"q": "почему небо синее"},
|
||||
},
|
||||
{
|
||||
name: "one required string with no tail",
|
||||
schema: `{"properties":{"q":{"type":"string"}},"required":["q"]}`,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "one required integer parses",
|
||||
schema: `{"properties":{"task_id":{"type":"integer"}},"required":["task_id"]}`,
|
||||
args: []string{"251"},
|
||||
want: map[string]any{"task_id": float64(251)},
|
||||
},
|
||||
{
|
||||
name: "one required integer with words",
|
||||
schema: `{"properties":{"task_id":{"type":"integer"}},"required":["task_id"]}`,
|
||||
args: []string{"двести", "пятьдесят", "один"},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "two required is refused rather than guessed",
|
||||
schema: `{"properties":{"a":{"type":"string"},"b":{"type":"string"}},"required":["a","b"]}`,
|
||||
args: []string{"что-то"},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "one required object is refused",
|
||||
schema: `{"properties":{"payload":{"type":"object"}},"required":["payload"]}`,
|
||||
args: []string{"что-то"},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
// Learned from Vikunja's update_task: required ["task_id"], every
|
||||
// other field optional, so one guessed argument blanks the rest.
|
||||
name: "one required on a mutating tool is refused",
|
||||
schema: `{"properties":{"task_id":{"type":"integer"}},"required":["task_id"]}`,
|
||||
args: []string{"251"},
|
||||
mutating: true,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
// Nothing was guessed, so there is nothing to get wrong. It still
|
||||
// goes through the confirm turn upstream.
|
||||
name: "no required on a mutating tool still runs",
|
||||
schema: `{"properties":{},"required":[]}`,
|
||||
mutating: true,
|
||||
want: map[string]any{},
|
||||
},
|
||||
{
|
||||
name: "unreadable schema",
|
||||
schema: `not json`,
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, err := bindPositional(json.RawMessage(tc.schema), tc.args, !tc.mutating)
|
||||
if tc.wantErr {
|
||||
if err == nil {
|
||||
t.Fatalf("want an error, got %v", got)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fmt.Sprint(got) != fmt.Sprint(tc.want) {
|
||||
t.Fatalf("got %v want %v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCallPositionalThroughManager(t *testing.T) {
|
||||
p := &fakePoster{handler: echoServer()}
|
||||
m, err := NewManager(func(ServerConfig) (Poster, error) { return p, nil },
|
||||
[]ServerConfig{{Name: "fake", URL: "http://example.test/mcp", Enabled: true}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Connect(context.Background())
|
||||
defer m.Close()
|
||||
// echoServer's tools declare no required properties.
|
||||
out, err := m.CallPositional(context.Background(), "fake", "read_thing", []string{"хвост"})
|
||||
if err != nil {
|
||||
t.Fatalf("call: %v", err)
|
||||
}
|
||||
if out != "read_thing:<nil>" {
|
||||
t.Fatalf("out = %q", out)
|
||||
}
|
||||
if _, err := m.CallPositional(context.Background(), "fake", "absent", nil); err == nil {
|
||||
t.Error("an unknown tool must be refused")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
package mcp
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// maxLine bounds one JSON-RPC frame from a subprocess. A tool result bigger
|
||||
// than this is a misbehaving server, not something to buffer.
|
||||
const maxLine = 1 << 20 // 1 MiB
|
||||
|
||||
// stdioTransport speaks newline-delimited JSON-RPC to a child process. This is
|
||||
// the local transport: the server runs on this box, under this user, and gets
|
||||
// no network guard because it never touches the network on our behalf.
|
||||
//
|
||||
// Args are argv, never a shell string — the same discipline internal/tool
|
||||
// keeps, for the same reason.
|
||||
type stdioTransport struct {
|
||||
mu sync.Mutex
|
||||
cmd *exec.Cmd
|
||||
in io.WriteCloser
|
||||
out *bufio.Reader
|
||||
dead bool
|
||||
}
|
||||
|
||||
func newStdioTransport(ctx context.Context, argv []string, env []string, dir string) (*stdioTransport, error) {
|
||||
if len(argv) == 0 {
|
||||
return nil, errors.New("mcp: stdio server needs a command")
|
||||
}
|
||||
cmd := exec.Command(argv[0], argv[1:]...)
|
||||
cmd.Dir = dir
|
||||
if len(env) > 0 {
|
||||
cmd.Env = append(os.Environ(), env...)
|
||||
}
|
||||
cmd.Stderr = os.Stderr
|
||||
in, err := cmd.StdinPipe()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("mcp: stdin pipe: %w", err)
|
||||
}
|
||||
out, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("mcp: stdout pipe: %w", err)
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
return nil, fmt.Errorf("mcp: start %q: %w", argv[0], err)
|
||||
}
|
||||
return &stdioTransport{cmd: cmd, in: in, out: bufio.NewReaderSize(out, 64<<10)}, nil
|
||||
}
|
||||
|
||||
func (t *stdioTransport) Call(ctx context.Context, req *rpcRequest) (*rpcResponse, error) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if t.dead {
|
||||
return nil, ErrClosed
|
||||
}
|
||||
if err := t.write(req); err != nil {
|
||||
t.dead = true
|
||||
return nil, err
|
||||
}
|
||||
// Read until the frame with our id turns up; anything else on the pipe is
|
||||
// a notification or a server-initiated request we do not answer.
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
line, err := t.readLine()
|
||||
if err != nil {
|
||||
t.dead = true
|
||||
return nil, err
|
||||
}
|
||||
var resp rpcResponse
|
||||
if err := json.Unmarshal(line, &resp); err != nil {
|
||||
continue // not a response frame; ignore rather than break the turn
|
||||
}
|
||||
if resp.ID == nil || *resp.ID != req.ID {
|
||||
continue
|
||||
}
|
||||
return &resp, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (t *stdioTransport) Notify(ctx context.Context, method string, params any) error {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if t.dead {
|
||||
return ErrClosed
|
||||
}
|
||||
return t.write(&rpcRequest{JSONRPC: "2.0", Method: method, Params: params})
|
||||
}
|
||||
|
||||
func (t *stdioTransport) write(req *rpcRequest) error {
|
||||
req.JSONRPC = "2.0"
|
||||
raw, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := t.in.Write(append(raw, '\n')); err != nil {
|
||||
return fmt.Errorf("mcp: write %s: %w", req.Method, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *stdioTransport) readLine() ([]byte, error) {
|
||||
for {
|
||||
line, err := t.out.ReadString('\n')
|
||||
if err != nil {
|
||||
if len(strings.TrimSpace(line)) == 0 {
|
||||
return nil, fmt.Errorf("mcp: read: %w", err)
|
||||
}
|
||||
return []byte(line), nil
|
||||
}
|
||||
if len(line) > maxLine {
|
||||
return nil, fmt.Errorf("mcp: frame exceeds %d bytes", maxLine)
|
||||
}
|
||||
if s := strings.TrimSpace(line); s != "" {
|
||||
return []byte(s), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (t *stdioTransport) Close() error {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
t.dead = true
|
||||
if t.in != nil {
|
||||
_ = t.in.Close()
|
||||
}
|
||||
if t.cmd.Process != nil {
|
||||
_ = t.cmd.Process.Kill()
|
||||
_ = t.cmd.Wait()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// alive reports whether the transport can still carry a call. The manager uses
|
||||
// it to decide on a reconnect instead of retrying into a dead pipe.
|
||||
func (t *stdioTransport) alive() bool {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
return !t.dead
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
package mcp
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The stdio transport is tested against a real subprocess — this test binary,
|
||||
// re-executed with MAVEN_MCP_FAKE set, acting as a minimal MCP server. No
|
||||
// python, no fixture file, no network.
|
||||
func TestMain(m *testing.M) {
|
||||
if os.Getenv("MAVEN_MCP_FAKE") != "" {
|
||||
fakeStdioServer()
|
||||
return
|
||||
}
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
func fakeStdioServer() {
|
||||
h := echoServer()
|
||||
sc := bufio.NewScanner(os.Stdin)
|
||||
out := bufio.NewWriter(os.Stdout)
|
||||
defer out.Flush()
|
||||
for sc.Scan() {
|
||||
line := strings.TrimSpace(sc.Text())
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
var req struct {
|
||||
ID *int64 `json:"id"`
|
||||
Method string `json:"method"`
|
||||
Params json.RawMessage `json:"params"`
|
||||
}
|
||||
if json.Unmarshal([]byte(line), &req) != nil {
|
||||
continue
|
||||
}
|
||||
if req.ID == nil {
|
||||
// A notification gets no reply, but we emit an unrelated
|
||||
// notification so the client's frame-skipping is exercised.
|
||||
_, _ = out.WriteString("{\"jsonrpc\":\"2.0\",\"method\":\"notifications/message\"}\n")
|
||||
_ = out.Flush()
|
||||
continue
|
||||
}
|
||||
result, rerr := h(req.Method, req.Params)
|
||||
resp := map[string]any{"jsonrpc": "2.0", "id": *req.ID}
|
||||
if rerr != nil {
|
||||
resp["error"] = map[string]any{"code": rerr.Code, "message": rerr.Message}
|
||||
} else {
|
||||
resp["result"] = result
|
||||
}
|
||||
raw, _ := json.Marshal(resp)
|
||||
_, _ = out.Write(append(raw, '\n'))
|
||||
_ = out.Flush()
|
||||
if os.Getenv("MAVEN_MCP_FAKE") == "die" && req.Method == "tools/list" {
|
||||
return // hang up, so the reconnect path has something to see
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func stdioManager(t *testing.T, mode string) *Manager {
|
||||
t.Helper()
|
||||
self, err := os.Executable()
|
||||
if err != nil {
|
||||
t.Skipf("no executable path: %v", err)
|
||||
}
|
||||
if _, err := exec.LookPath(self); err != nil && !strings.Contains(self, "/") {
|
||||
t.Skip("test binary not executable")
|
||||
}
|
||||
m, err := NewManager(nil, []ServerConfig{{
|
||||
Name: "fake",
|
||||
Command: self,
|
||||
Env: []string{"MAVEN_MCP_FAKE=" + mode},
|
||||
Enabled: true,
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Connect(context.Background())
|
||||
return m
|
||||
}
|
||||
|
||||
func TestStdioTransportEndToEnd(t *testing.T) {
|
||||
m := stdioManager(t, "1")
|
||||
defer m.Close()
|
||||
st := m.Status()
|
||||
if len(st) != 1 || !st[0].Connected {
|
||||
t.Fatalf("status = %+v", st)
|
||||
}
|
||||
if st[0].Transport != "stdio" {
|
||||
t.Fatalf("transport = %q", st[0].Transport)
|
||||
}
|
||||
if got := len(m.Tools()); got != 2 {
|
||||
t.Fatalf("tools = %d", got)
|
||||
}
|
||||
out, err := m.Call(context.Background(), "fake", "read_thing", map[string]any{"q": "стдио"})
|
||||
if err != nil {
|
||||
t.Fatalf("call: %v", err)
|
||||
}
|
||||
if out != "read_thing:стдио" {
|
||||
t.Fatalf("out = %q", out)
|
||||
}
|
||||
res := m.Resources(context.Background())
|
||||
if len(res) != 1 || res[0].URI != "note://one" {
|
||||
t.Fatalf("resources = %+v", res)
|
||||
}
|
||||
body, err := m.ReadResource(context.Background(), "fake", "note://one")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if body != "тело ресурса" {
|
||||
t.Fatalf("body = %q", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStdioServerThatDiesIsNotUsable(t *testing.T) {
|
||||
m := stdioManager(t, "die")
|
||||
defer m.Close()
|
||||
// The server hung up after tools/list; the next call must fail cleanly
|
||||
// rather than hang or panic.
|
||||
if _, err := m.Call(context.Background(), "fake", "read_thing", nil); err == nil {
|
||||
t.Fatal("a call into a dead server must error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStdioMissingCommand(t *testing.T) {
|
||||
m, err := NewManager(nil, []ServerConfig{{
|
||||
Name: "nope", Command: "/nonexistent/mcp-server-that-is-not-there", Enabled: true,
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Connect(context.Background())
|
||||
st := m.Status()
|
||||
if st[0].Connected || st[0].Err == "" {
|
||||
t.Fatalf("a missing binary must be recorded, not fatal: %+v", st)
|
||||
}
|
||||
if got := len(m.Tools()); got != 0 {
|
||||
t.Fatalf("tools = %d", got)
|
||||
}
|
||||
if !strings.Contains(fmt.Sprint(st[0].Err), "start") {
|
||||
t.Logf("err = %q", st[0].Err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package mcp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/kami/maven/internal/webfetch"
|
||||
)
|
||||
|
||||
// WebfetchDoor builds the PosterFactory used in production: one guarded
|
||||
// webfetch.Fetcher per url server, with that server's allow_private and the
|
||||
// shared host lists and limits.
|
||||
//
|
||||
// One fetcher PER server is the point. allow_private is a hole in the
|
||||
// private-address guard, and a hole punched for the Vikunja server on loopback
|
||||
// must not become a hole for some public endpoint that happens to redirect at
|
||||
// the LAN. Rate limiting is per fetcher too, which is the right shape here:
|
||||
// separate servers are separate hosts.
|
||||
func WebfetchDoor(limits webfetch.Config) PosterFactory {
|
||||
return func(cfg ServerConfig) (Poster, error) {
|
||||
c := limits
|
||||
c.AllowPrivate = cfg.AllowPrivate
|
||||
if c.Timeout <= 0 && cfg.Timeout > 0 {
|
||||
c.Timeout = cfg.Timeout
|
||||
}
|
||||
return fetcherPoster{webfetch.New(c)}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// fetcherPoster adapts webfetch.Fetcher to Poster. It exists so this package
|
||||
// does not have to know webfetch's Response type, and so a test can substitute
|
||||
// a fake without a listener.
|
||||
type fetcherPoster struct{ f *webfetch.Fetcher }
|
||||
|
||||
func (p fetcherPoster) Post(ctx context.Context, rawURL, contentType string, body []byte, hdr map[string]string) (*PostResponse, error) {
|
||||
resp, err := p.f.Post(ctx, rawURL, contentType, body, hdr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("mcp: post %s: %w", rawURL, err)
|
||||
}
|
||||
return &PostResponse{
|
||||
Status: resp.Status,
|
||||
ContentType: resp.ContentType,
|
||||
Body: resp.Body,
|
||||
Header: resp.Header,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
package media
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/draw"
|
||||
"image/gif"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// DefaultMaxDim — the longest edge an image is scaled down to before it goes to
|
||||
// a vision model. 896 is the tile size the current crop of small
|
||||
// vision-language models (Qwen2.5-VL, SmolVLM, moondream) work in; sending a
|
||||
// 12-megapixel phone photo instead just costs the box minutes of prefill for
|
||||
// tiles that get pooled away anyway.
|
||||
const DefaultMaxDim = 896
|
||||
|
||||
// JPEGQuality for the re-encode. 85 is the usual "no visible artefacts" point,
|
||||
// and the re-encode exists to shrink the payload, not to archive it — the
|
||||
// original bytes stay in the blob store untouched.
|
||||
const JPEGQuality = 85
|
||||
|
||||
// ErrUnsupportedImage — the bytes are not an image format this build can
|
||||
// decode. Notably webp: the stdlib has no webp decoder and this repo takes no
|
||||
// new dependencies, so a webp arriving from Telegram is refused here with a
|
||||
// clear error rather than handed to a model as garbage.
|
||||
var ErrUnsupportedImage = errors.New("media: unsupported image format")
|
||||
|
||||
// SniffImage identifies image bytes by magic number and returns the mime. It
|
||||
// exists because a caller-declared content type is a claim, and the store's file
|
||||
// extension (and the vision provider's data URI) should follow the bytes.
|
||||
//
|
||||
// Returns ErrUnsupportedImage for anything unrecognised, including webp — which
|
||||
// is recognised well enough to name in the error, so the log says "webp is not
|
||||
// supported" instead of "not an image".
|
||||
func SniffImage(data []byte) (string, error) {
|
||||
switch {
|
||||
case len(data) >= 3 && data[0] == 0xFF && data[1] == 0xD8 && data[2] == 0xFF:
|
||||
return "image/jpeg", nil
|
||||
case len(data) >= 8 && string(data[:8]) == "\x89PNG\r\n\x1a\n":
|
||||
return "image/png", nil
|
||||
case len(data) >= 6 && (string(data[:6]) == "GIF87a" || string(data[:6]) == "GIF89a"):
|
||||
return "image/gif", nil
|
||||
case len(data) >= 12 && string(data[:4]) == "RIFF" && string(data[8:12]) == "WEBP":
|
||||
return "", fmt.Errorf("%w: webp (no decoder in this build)", ErrUnsupportedImage)
|
||||
}
|
||||
return "", ErrUnsupportedImage
|
||||
}
|
||||
|
||||
// Image — an image prepared for a vision model: JPEG bytes, downscaled, with
|
||||
// the dimensions it ended up at. It is deliberately a separate type from Blob:
|
||||
// a Blob is what he sent, an Image is what the model sees, and the two are not
|
||||
// the same bytes.
|
||||
type Image struct {
|
||||
JPEG []byte
|
||||
Width int
|
||||
Height int
|
||||
// Source names where the original came from ("telegram", "web:upload"),
|
||||
// carried through only so a log line can say what was looked at.
|
||||
Source string
|
||||
}
|
||||
|
||||
// DataURI renders the image as a `data:image/jpeg;base64,...` URI, which is how
|
||||
// every OpenAI-compatible multimodal endpoint takes an image. The string is
|
||||
// large (roughly 4/3 of the JPEG); nothing caches it.
|
||||
func (im Image) DataURI() string {
|
||||
return "data:image/jpeg;base64," + base64.StdEncoding.EncodeToString(im.JPEG)
|
||||
}
|
||||
|
||||
// PrepareImage decodes data, scales it so its longest edge is at most maxDim
|
||||
// (never up — a small image is left alone), and re-encodes it as JPEG.
|
||||
// maxDim ≤ 0 ⇒ DefaultMaxDim.
|
||||
//
|
||||
// An image with an alpha channel is composited onto white rather than having
|
||||
// alpha dropped to black, because the common case is a screenshot or a
|
||||
// transparent-background diagram, and text on black-on-black is unreadable to
|
||||
// the model for no reason.
|
||||
func PrepareImage(data []byte, source string, maxDim int) (Image, error) {
|
||||
if len(data) == 0 {
|
||||
return Image{}, ErrEmpty
|
||||
}
|
||||
if maxDim <= 0 {
|
||||
maxDim = DefaultMaxDim
|
||||
}
|
||||
mime, err := SniffImage(data)
|
||||
if err != nil {
|
||||
return Image{}, err
|
||||
}
|
||||
src, err := decode(data, mime)
|
||||
if err != nil {
|
||||
return Image{}, fmt.Errorf("media: decode %s: %w", mime, err)
|
||||
}
|
||||
|
||||
dst := flattenAndScale(src, maxDim)
|
||||
var buf bytes.Buffer
|
||||
if err := jpeg.Encode(&buf, dst, &jpeg.Options{Quality: JPEGQuality}); err != nil {
|
||||
return Image{}, fmt.Errorf("media: encode jpeg: %w", err)
|
||||
}
|
||||
b := dst.Bounds()
|
||||
return Image{JPEG: buf.Bytes(), Width: b.Dx(), Height: b.Dy(), Source: source}, nil
|
||||
}
|
||||
|
||||
func decode(data []byte, mime string) (image.Image, error) {
|
||||
r := bytes.NewReader(data)
|
||||
switch strings.ToLower(mime) {
|
||||
case "image/jpeg":
|
||||
return jpeg.Decode(r)
|
||||
case "image/png":
|
||||
return png.Decode(r)
|
||||
case "image/gif":
|
||||
return gif.Decode(r)
|
||||
}
|
||||
return nil, ErrUnsupportedImage
|
||||
}
|
||||
|
||||
// flattenAndScale composites onto white and box-scales down to maxDim. The
|
||||
// scaler is a plain area average over the source pixels mapping to each
|
||||
// destination pixel — nearest-neighbour would alias small text into noise,
|
||||
// which defeats the point of reading a screenshot, and an area average is a
|
||||
// dozen lines against pulling in golang.org/x/image on an offline box.
|
||||
func flattenAndScale(src image.Image, maxDim int) *image.RGBA {
|
||||
sb := src.Bounds()
|
||||
sw, sh := sb.Dx(), sb.Dy()
|
||||
dw, dh := fit(sw, sh, maxDim)
|
||||
|
||||
flat := image.NewRGBA(image.Rect(0, 0, sw, sh))
|
||||
draw.Draw(flat, flat.Bounds(), image.NewUniform(image.White), image.Point{}, draw.Src)
|
||||
draw.Draw(flat, flat.Bounds(), src, sb.Min, draw.Over)
|
||||
if dw == sw && dh == sh {
|
||||
return flat
|
||||
}
|
||||
|
||||
dst := image.NewRGBA(image.Rect(0, 0, dw, dh))
|
||||
for y := 0; y < dh; y++ {
|
||||
y0, y1 := y*sh/dh, (y+1)*sh/dh
|
||||
if y1 <= y0 {
|
||||
y1 = y0 + 1
|
||||
}
|
||||
for x := 0; x < dw; x++ {
|
||||
x0, x1 := x*sw/dw, (x+1)*sw/dw
|
||||
if x1 <= x0 {
|
||||
x1 = x0 + 1
|
||||
}
|
||||
var r, g, b, n uint32
|
||||
for sy := y0; sy < y1; sy++ {
|
||||
for sx := x0; sx < x1; sx++ {
|
||||
i := flat.PixOffset(sx, sy)
|
||||
r += uint32(flat.Pix[i])
|
||||
g += uint32(flat.Pix[i+1])
|
||||
b += uint32(flat.Pix[i+2])
|
||||
n++
|
||||
}
|
||||
}
|
||||
o := dst.PixOffset(x, y)
|
||||
dst.Pix[o] = uint8(r / n)
|
||||
dst.Pix[o+1] = uint8(g / n)
|
||||
dst.Pix[o+2] = uint8(b / n)
|
||||
dst.Pix[o+3] = 0xFF
|
||||
}
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
// fit returns the largest w×h with the same aspect ratio whose longest edge is
|
||||
// at most maxDim, never enlarging. Both edges are clamped to at least 1 so a
|
||||
// 2000×1 strip does not scale to zero height.
|
||||
func fit(w, h, maxDim int) (int, int) {
|
||||
if w <= maxDim && h <= maxDim {
|
||||
return w, h
|
||||
}
|
||||
if w >= h {
|
||||
nh := h * maxDim / w
|
||||
if nh < 1 {
|
||||
nh = 1
|
||||
}
|
||||
return maxDim, nh
|
||||
}
|
||||
nw := w * maxDim / h
|
||||
if nw < 1 {
|
||||
nw = 1
|
||||
}
|
||||
return nw, maxDim
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
package media
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/gif"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// pngBytes builds a w×h test image: left half red, right half a light grey, so
|
||||
// a downscale that averages produces a predictable mid value and a scaler that
|
||||
// silently returns the wrong region is visible.
|
||||
func pngBytes(t *testing.T, w, h int) []byte {
|
||||
t.Helper()
|
||||
img := image.NewRGBA(image.Rect(0, 0, w, h))
|
||||
for y := 0; y < h; y++ {
|
||||
for x := 0; x < w; x++ {
|
||||
if x < w/2 {
|
||||
img.Set(x, y, color.RGBA{255, 0, 0, 255})
|
||||
} else {
|
||||
img.Set(x, y, color.RGBA{200, 200, 200, 255})
|
||||
}
|
||||
}
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := png.Encode(&buf, img); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func TestSniffImage(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
data []byte
|
||||
want string
|
||||
}{
|
||||
{"png", pngBytes(t, 4, 4), "image/png"},
|
||||
{"jpeg", jpegBytes(t, 4, 4), "image/jpeg"},
|
||||
{"gif", gifBytes(t, 4, 4), "image/gif"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got, err := SniffImage(c.data)
|
||||
if err != nil {
|
||||
t.Errorf("%s: %v", c.name, err)
|
||||
continue
|
||||
}
|
||||
if got != c.want {
|
||||
t.Errorf("%s: got %q want %q", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// webp is common from Telegram and there is no stdlib decoder, so it must be
|
||||
// refused by name rather than mis-sniffed or fed to a model as noise.
|
||||
func TestSniffRefusesWebpByName(t *testing.T) {
|
||||
webp := append([]byte("RIFF\x00\x00\x00\x00WEBP"), make([]byte, 8)...)
|
||||
_, err := SniffImage(webp)
|
||||
if !errors.Is(err, ErrUnsupportedImage) {
|
||||
t.Fatalf("got %v, want ErrUnsupportedImage", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "webp") {
|
||||
t.Errorf("error does not name the format: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSniffRefusesGarbage(t *testing.T) {
|
||||
for _, data := range [][]byte{nil, []byte("hello"), []byte("\x00\x01\x02\x03")} {
|
||||
if _, err := SniffImage(data); !errors.Is(err, ErrUnsupportedImage) {
|
||||
t.Errorf("SniffImage(%q) = %v", data, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareImageDownscalesLongestEdge(t *testing.T) {
|
||||
im, err := PrepareImage(pngBytes(t, 2000, 1000), "web:upload", 500)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare: %v", err)
|
||||
}
|
||||
if im.Width != 500 || im.Height != 250 {
|
||||
t.Errorf("got %dx%d, want 500x250", im.Width, im.Height)
|
||||
}
|
||||
if _, err := jpeg.Decode(bytes.NewReader(im.JPEG)); err != nil {
|
||||
t.Errorf("output is not decodable jpeg: %v", err)
|
||||
}
|
||||
if im.Source != "web:upload" {
|
||||
t.Errorf("source lost: %q", im.Source)
|
||||
}
|
||||
}
|
||||
|
||||
// Tall images scale on the other axis; a scaler that only handles landscape is
|
||||
// the classic version of this bug.
|
||||
func TestPrepareImageHandlesPortrait(t *testing.T) {
|
||||
im, err := PrepareImage(pngBytes(t, 400, 1600), "telegram", 800)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare: %v", err)
|
||||
}
|
||||
if im.Height != 800 || im.Width != 200 {
|
||||
t.Errorf("got %dx%d, want 200x800", im.Width, im.Height)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareImageNeverEnlarges(t *testing.T) {
|
||||
im, err := PrepareImage(pngBytes(t, 64, 32), "telegram", 896)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare: %v", err)
|
||||
}
|
||||
if im.Width != 64 || im.Height != 32 {
|
||||
t.Errorf("got %dx%d, want the original 64x32", im.Width, im.Height)
|
||||
}
|
||||
}
|
||||
|
||||
// A degenerate strip must not scale to zero on the short axis — jpeg.Encode
|
||||
// fails on a zero-height image, which would turn a weird screenshot into a
|
||||
// hard error.
|
||||
func TestPrepareImageClampsDegenerateAspect(t *testing.T) {
|
||||
im, err := PrepareImage(pngBytes(t, 2000, 2), "web:upload", 100)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare: %v", err)
|
||||
}
|
||||
if im.Height < 1 || im.Width != 100 {
|
||||
t.Errorf("got %dx%d", im.Width, im.Height)
|
||||
}
|
||||
}
|
||||
|
||||
// Transparent pixels composite onto white, not black: the common case is a
|
||||
// screenshot or a diagram, and dark-on-black is unreadable to the model.
|
||||
func TestPrepareImageFlattensAlphaOntoWhite(t *testing.T) {
|
||||
img := image.NewRGBA(image.Rect(0, 0, 8, 8)) // fully transparent
|
||||
var buf bytes.Buffer
|
||||
if err := png.Encode(&buf, img); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
im, err := PrepareImage(buf.Bytes(), "web:upload", 8)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare: %v", err)
|
||||
}
|
||||
decoded, err := jpeg.Decode(bytes.NewReader(im.JPEG))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r, g, b, _ := decoded.At(4, 4).RGBA()
|
||||
if r>>8 < 240 || g>>8 < 240 || b>>8 < 240 {
|
||||
t.Errorf("transparent pixel became rgb(%d,%d,%d), want near-white", r>>8, g>>8, b>>8)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareImageRejectsEmpty(t *testing.T) {
|
||||
if _, err := PrepareImage(nil, "x", 0); !errors.Is(err, ErrEmpty) {
|
||||
t.Errorf("got %v, want ErrEmpty", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDataURIIsAJPEGDataURI(t *testing.T) {
|
||||
im, err := PrepareImage(pngBytes(t, 16, 16), "x", 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
uri := im.DataURI()
|
||||
if !strings.HasPrefix(uri, "data:image/jpeg;base64,") {
|
||||
t.Fatalf("bad prefix: %.40s", uri)
|
||||
}
|
||||
if len(uri) <= len("data:image/jpeg;base64,") {
|
||||
t.Error("data uri carries no payload")
|
||||
}
|
||||
}
|
||||
|
||||
func jpegBytes(t *testing.T, w, h int) []byte {
|
||||
t.Helper()
|
||||
img := image.NewRGBA(image.Rect(0, 0, w, h))
|
||||
var buf bytes.Buffer
|
||||
if err := jpeg.Encode(&buf, img, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func gifBytes(t *testing.T, w, h int) []byte {
|
||||
t.Helper()
|
||||
img := image.NewPaletted(image.Rect(0, 0, w, h), []color.Color{color.Black, color.White})
|
||||
var buf bytes.Buffer
|
||||
if err := gif.Encode(&buf, img, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
// Package media is the intake for everything Maven sees or hears that is not
|
||||
// text: a photo he sends her, a meeting she was asked to record, a voice sample
|
||||
// used to enrol a speaker. All three senses (vision, hearing, speaker
|
||||
// recognition) share one problem — a blob arrives, it has to be stored, and
|
||||
// something has to describe it — so the storing half lives here once instead of
|
||||
// three times.
|
||||
//
|
||||
// # What this package is
|
||||
//
|
||||
// A content-addressed blob store on the local filesystem. Put returns a Blob
|
||||
// keyed by the sha256 of its bytes, so the same photo sent twice is one file.
|
||||
// Each blob gets a sidecar `.json` with its kind, mime, size, source and
|
||||
// creation time; the sidecar is the whole index, because at personal scale a
|
||||
// directory walk is cheaper than another sqlite table and the store has to be
|
||||
// readable with `ls` when something goes wrong.
|
||||
//
|
||||
// Blobs are NOT in the sqlite database. The database is small, encrypted, and
|
||||
// read on every tick; a 40 MB meeting recording has no business in it. What
|
||||
// goes in the database is the *text* a blob produced — a transcript, a
|
||||
// description — written as an ordinary note, which is the durable artefact and
|
||||
// the only part worth recalling later.
|
||||
//
|
||||
// # Invariants (these are the point of the package, not decoration)
|
||||
//
|
||||
// - Nothing is captured that was not asked for. This package never records;
|
||||
// it stores what a caller hands it, and every caller is an explicit act
|
||||
// with a start and a stop. There is no ambient path in, and none may be
|
||||
// added: see the refusal recorded in docs/plans/08-hearing.md.
|
||||
// - A blob never leaves the box. No provider in this repo may upload one, and
|
||||
// the vision provider refuses a non-private endpoint for exactly that
|
||||
// reason (internal/vision).
|
||||
// - A blob is never search input and never embedded. His photos and the audio
|
||||
// of his meetings are not corpus. Only text derived from them, once he can
|
||||
// see it as a note, participates in recall.
|
||||
// - Storage is bounded. Retention is a config knob with a default, Prune
|
||||
// enforces it, and an unpruned store is a bug: audio of people accumulating
|
||||
// forever on disk is the failure mode this capability has to avoid.
|
||||
//
|
||||
// # Layout
|
||||
//
|
||||
// <dir>/<kind>/<aa>/<sha256>.<ext> the bytes
|
||||
// <dir>/<kind>/<aa>/<sha256>.json the sidecar metadata
|
||||
//
|
||||
// `aa` is the first two hex chars of the digest — one fan-out level, enough to
|
||||
// keep a directory listing usable after a few thousand blobs.
|
||||
package media
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Kind — what a blob is. Two values today; the kind is a directory name and a
|
||||
// retention bucket, so adding a third is additive.
|
||||
type Kind string
|
||||
|
||||
const (
|
||||
// KindImage — a still image (png / jpeg / gif / webp bytes as received).
|
||||
KindImage Kind = "image"
|
||||
// KindAudio — raw PCM in the canonical internal/audio format, or a WAV
|
||||
// container. Meeting captures and enrolment samples both land here.
|
||||
KindAudio Kind = "audio"
|
||||
)
|
||||
|
||||
// Valid reports whether k is a kind this package will store. An unknown kind is
|
||||
// refused at Put rather than creating a stray directory.
|
||||
func (k Kind) Valid() bool { return k == KindImage || k == KindAudio }
|
||||
|
||||
// Errors callers distinguish. ErrNotFound is the only one a caller usually
|
||||
// handles; the rest mean the call was wrong.
|
||||
var (
|
||||
// ErrNotFound — no blob with that id in this store.
|
||||
ErrNotFound = errors.New("media: not found")
|
||||
// ErrEmpty — Put was handed zero bytes. Storing an empty capture would
|
||||
// leave a sidecar claiming a recording exists when it does not.
|
||||
ErrEmpty = errors.New("media: empty payload")
|
||||
// ErrTooLarge — the payload is over the store's cap. The cap exists so a
|
||||
// runaway capture cannot fill the disk that mavend's database lives on.
|
||||
ErrTooLarge = errors.New("media: payload too large")
|
||||
// ErrBadKind — unknown Kind.
|
||||
ErrBadKind = errors.New("media: unknown kind")
|
||||
// ErrBadID — the id is not a 64-char lowercase hex digest, so it cannot
|
||||
// have come from this store and must not be turned into a path.
|
||||
ErrBadID = errors.New("media: malformed id")
|
||||
)
|
||||
|
||||
// Blob — one stored item. ID is the sha256 of the bytes in lowercase hex, which
|
||||
// makes it both the primary key and the dedupe mechanism. Path is absolute and
|
||||
// local; it is a debugging affordance and the argument a subprocess (whisper,
|
||||
// llama-server) is pointed at, never something handed to a network client.
|
||||
type Blob struct {
|
||||
ID string `json:"id"`
|
||||
Kind Kind `json:"kind"`
|
||||
MIME string `json:"mime"`
|
||||
Size int64 `json:"size"`
|
||||
Source string `json:"source"` // provenance: "telegram", "web:upload", "capture:meeting", "enroll"
|
||||
Created time.Time `json:"created"` // UTC
|
||||
Path string `json:"-"` // filled by the store; not part of the sidecar
|
||||
}
|
||||
|
||||
// Age is how long ago the blob was stored, measured against now. Prune uses it;
|
||||
// it is exported because the /media surface will want to show it.
|
||||
func (b Blob) Age(now time.Time) time.Duration { return now.Sub(b.Created) }
|
||||
|
||||
// String is a one-line summary for logs. Deliberately does not include Path:
|
||||
// a log line is not the place to spell out where his meeting audio lives.
|
||||
func (b Blob) String() string {
|
||||
return fmt.Sprintf("%s %s %dB from %s", b.Kind, shortID(b.ID), b.Size, b.Source)
|
||||
}
|
||||
|
||||
// shortID trims a digest to something readable in a log line. Twelve hex chars
|
||||
// is unambiguous at personal scale and short enough to fit next to the rest.
|
||||
func shortID(id string) string {
|
||||
if len(id) <= 12 {
|
||||
return id
|
||||
}
|
||||
return id[:12]
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user