diff --git a/.gitignore b/.gitignore index 4d7a279..8b29eb9 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,9 @@ /mavweb /mavpoll /mavcaldav +/mavwaked +/mavmaild +/mavupdate # Certs (private keys, don't commit) certs/ @@ -33,6 +36,10 @@ deps deploy/db_key.env # Deploy secret (telegram bot token + chat id) — never commit deploy/telegram.env +# zenmoney API token, read by mavpoll (never in argv, never committed) +deploy/zenmoney.token +# IMAP password, read by mavmaild (never in argv, never committed) +deploy/imap.password # Temp files /tmp/ @@ -45,3 +52,5 @@ coverage.out # Agent worktrees and local agent state .claude/ +/models/stt +/models/tts diff --git a/CLAUDE.md b/CLAUDE.md index 26b9bf5..8f7d79d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -34,7 +34,7 @@ CGO daemons (`mavend`, `mavsttd`, `mavttsd`, `mavenclient`) need the vendored to and libs wired through the Makefile — **do not** call `go build` on them bare, use `make`: ```sh -make build # all 8 binaries +make build # all 9 binaries make build-web # single daemon (pure-Go ones: web/waked/poll/caldav build without CGO) make test # go test -race across ./internal/... ./cmd/... with CGO env set ``` @@ -62,6 +62,7 @@ Pure-Go packages (`router`, `memory`, `mavweb`, …) run under a plain `go test | `mavenclient` | Voice loop client (mic → stt → core → tts). | | `mavpoll` | Telegram long-poll reach. | | `mavcaldav` | CalDAV calendar sync. | +| `mavmaild` | Mail reader (IMAP, read-only). Holds the IMAP password; core never sees it. | Daemons are wired socket-to-socket, not linked. `internal/ipc` is the client/server wire protocol; the config in `deploy/mavend.json` (with `${VAR}` env expansion from gitignored @@ -90,8 +91,19 @@ fallback. Any LLM error falls through to the classifier so a turn never breaks o Measured on the 77-case RU fixture (`MODEL-BAKEOFF-31-07-2026.md`): the classifier scores 36.8% full accuracy at p50 31ms; Qwen3-1.7B scores 67.5% intent-only / 72.7% through the cascade at p50 ≈2.7s. Accuracy roughly doubled, latency is ~90× worse, and that trade was -accepted deliberately. Still open: `Confidence: 1.0` is hardcoded in `llmrouter.go`, so the -LLM path never asks for clarification (6/6 refusal cases missed) — Vikunja #359. +accepted deliberately. `Confidence: 1.0` used to be hardcoded in `llmrouter.go`, so the LLM +path could never ask for clarification (6/6 refusal cases missed on the fixture) — Vikunja +#359. Fixed 31-07-2026 with structural signal (single-token utterance, keyless fact, act with +no allowlisted fn) feeding the same stage-3 gate the classifier path already had — see +`gateLLMDecision` in `router.go`. Note the second half of that bug: the LLM branch never +consulted `r.threshold` at all, so a correct low confidence would have been discarded anyway. + +Re-measured on the fixture after the fix: **missed clarify 6/6 → 1**, at the cost of 3 false +clarifies and 2.6pt of full accuracy (72.7% → 70.1%, intent-only 67.5% → 74.0%). Two of the +three false clarifies are acts the model mis-routed and the gate caught — asking beats wrongly +executing, so the fixture and the daemon disagree about what is correct there. The third, +`"поужинал"`, is a real defect: **the single-token rule is an English intuition and does not +transfer to Russian**, where one word is routinely a whole sentence. Narrow or drop it. ## LLM output contract diff --git a/Dockerfile b/Dockerfile index ccc65bc..dd3851c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,7 +51,8 @@ RUN go build -o /out/mavend ./cmd/mavend && \ go build -o /out/mavttsd ./cmd/mavttsd && \ go build -o /out/mavweb ./cmd/mavweb && \ go build -o /out/mavpoll ./cmd/mavpoll && \ - go build -o /out/mavcaldav ./cmd/mavcaldav + go build -o /out/mavcaldav ./cmd/mavcaldav && \ + go build -o /out/mavmaild ./cmd/mavmaild # llama.cpp Vulkan build — the phraser/router LFM engine (llama-server). Built # from source (not a prebuilt vendored blob) so the binary's glibc/GLIBCXX match diff --git a/Makefile b/Makefile index 5e34b96..c3e8d7b 100644 --- a/Makefile +++ b/Makefile @@ -16,11 +16,11 @@ PIPER_BIN := $(shell pwd)/deps/piper/piper PIPER_MODEL := $(shell pwd)/models/tts/ru_RU-irina-medium.onnx PIPER_ESPEAK := $(shell pwd)/deps/piper/espeak-ng-data -.PHONY: all build build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav clean test fmt-check vet run-stt run-tts run-web download-embedder deps-go eval-router eval-recall eval-phrasing eval-models +.PHONY: simulate stt-fixtures test-stt-golden all build build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav clean test fmt-check vet run-stt run-tts run-web download-embedder deps-go eval-router eval-recall eval-phrasing eval-models all: build -build: build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav +build: build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav build-mail build-update build-stt: CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \ @@ -50,6 +50,15 @@ build-poll: build-caldav: $(GO) build $(GOFLAGS) -o mavcaldav ./cmd/mavcaldav/ +build-mail: + $(GO) build $(GOFLAGS) -o mavmaild ./cmd/mavmaild/ + +# mavupdate is an operator CLI, not a daemon: nothing runs it but a human on the +# box. It is built with the rest so a broken update path is caught by `make +# build` rather than the first time it is needed. +build-update: + $(GO) build $(GOFLAGS) -o mavupdate ./cmd/mavupdate/ + run-web: build-web ./mavweb -addr :9200 -voice 127.0.0.1:9100 @@ -82,6 +91,14 @@ vet: CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \ $(GO) vet ./internal/... ./cmd/... +# simulate — replay every scripted day under cmd/mavend/testdata/scenarios +# through the real router, store, tick loop and intake journal, on a fake clock +# (Vikunja #284). Verbose so the transcript of each scenario lands in the +# terminal. Also runs as part of `make test`; this target is for reading it. +simulate: + CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \ + $(GO) test -v -count=1 -run TestSimulator ./cmd/mavend/ + test: fmt-check vet CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \ $(GO) test -race -coverprofile=coverage.out ./internal/... ./cmd/... @@ -130,6 +147,22 @@ eval-models: MAVEN_LLM_URL="$(MAVEN_LLM_URL)" $(GO) test -v -count=1 -timeout 60m \ -run TestLLMRouterBaseline ./internal/router/eval/ +# stt-fixtures — regenerate the golden STT audio in cmd/mavsttd/testdata from +# the piper voices (#288). The committed WAVs are synthesised, never recorded, +# so this is the only way they should ever change. The spoken text is read out +# of testdata/golden_v1.json, so edit the transcript there and rerun this. +# +# test-stt-golden runs both golden tests: TestGoldenAudioTranscription, which +# scores the fixtures against ggml-small and self-skips when the model is +# absent, and TestGoldenFixturesAreCanonical, which checks the committed audio +# and the manifest with no model at all. +stt-fixtures: + ./scripts/gen-stt-fixtures.sh + +test-stt-golden: + CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \ + $(GO) test -v -count=1 -run TestGolden ./cmd/mavsttd/ + run-stt: build-stt LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \ ./mavsttd -socket /tmp/maven/stt.sock -model $(WHISPER_MODEL) @@ -185,4 +218,4 @@ download-embedder: @echo ' sudo cp onnxruntime-linux-x64-1.15.1/lib/libonnxruntime.so* /usr/local/lib/' clean: - rm -f mavend mavenclient mavsttd mavttsd mavweb mavpoll mavcaldav mavwaked + rm -f mavend mavenclient mavsttd mavttsd mavweb mavpoll mavcaldav mavwaked mavmaild diff --git a/cmd/mavcaldav/main.go b/cmd/mavcaldav/main.go index f0fb09f..4c120bd 100644 --- a/cmd/mavcaldav/main.go +++ b/cmd/mavcaldav/main.go @@ -1,17 +1,24 @@ -// mavcaldav — the CalDAV poller module. +// mavcaldav — the CalDAV module: reads calendars into facts, and renders +// maven's own reminders back out to a calendar she owns. // -// Polls a Radicale (or any CalDAV) server for today's events and writes -// `facts (kind=env, source=poll:caldav)` through core's IPC socket. -// Key-free, restart-free, fail-independent — crashes can't touch the -// store key, worst case a stale calendar_busy fact until the next poll. +// READ side (unchanged behaviour): polls a Radicale (or any CalDAV) server for +// today's events and writes `facts (kind=env, source=poll:caldav)` through +// core's IPC socket. Key-free, restart-free, fail-independent — crashes can't +// touch the store key, worst case a stale calendar_busy fact until the next +// poll. Two facts: // -// Two facts written: // - calendar_busy ("true"/"false") — read by the loop gate to suppress // nudges during meetings // - calendar_event (" @ -") — per-event for query // -// Append-only discipline: a fact is written only when its value CHANGED -// vs the latest for that key+source. +// Append-only discipline: a fact is written only when its value CHANGED vs the +// latest for that key+source. +// +// RENDER side (Vikunja #127, off unless -render-url is given): publishes each +// pending reminder as a single-event iCal resource in a collection maven owns. +// The calendar is a view, sqlite is the store — see render.go. The render URL +// must differ from the read URL, checked at startup, so the render target can +// never be a calendar maven is only supposed to read. package main import ( @@ -27,6 +34,7 @@ import ( "syscall" "time" + "github.com/kami/maven/internal/calendar" "github.com/kami/maven/internal/ipc" ) @@ -43,6 +51,10 @@ func run(args []string) error { url := fs.String("url", "", "CalDAV calendar URL, e.g. http://localhost:5232/kami/personal (required)") user := fs.String("user", "", "CalDAV basic-auth username (required)") pass := fs.String("pass", "", "CalDAV basic-auth password (required)") + renderURL := fs.String("render-url", "", "CalDAV collection maven publishes her own reminders to; empty disables rendering") + renderUser := fs.String("render-user", "", "basic-auth username for -render-url (defaults to -user)") + renderPass := fs.String("render-pass", "", "basic-auth password for -render-url (defaults to -pass)") + renderDur := fs.Duration("render-duration", calendar.DefaultReminderDuration, "how long a rendered reminder occupies") interval := fs.Duration("interval", 5*time.Minute, "poll cadence") timeout := fs.Duration("timeout", 10*time.Second, "per-request HTTP timeout") if err := fs.Parse(args); err != nil { @@ -54,6 +66,9 @@ func run(args []string) error { if *url == "" || *user == "" || *pass == "" { return fmt.Errorf("-url, -user, -pass are required") } + if err := checkRenderTarget([]string{*url}, *renderURL); err != nil { + return err + } ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() @@ -64,16 +79,36 @@ func run(args []string) error { } defer core.Close() + hc := &http.Client{Timeout: *timeout} p := &poller{ core: core, - http: &http.Client{Timeout: *timeout}, + http: hc, url: strings.TrimRight(*url, "/"), user: *user, pass: *pass, } + var rend *renderer + if *renderURL != "" { + ru, rp := *renderUser, *renderPass + if ru == "" { + ru = *user + } + if rp == "" { + rp = *pass + } + rend = newRenderer(core, hc, *renderURL, ru, rp, *renderDur) + log.Printf("mavcaldav: rendering reminders to %s", *renderURL) + } + log.Printf("mavcaldav: polling %s every %s", *url, *interval) - p.pollOnce(ctx) // fire immediately + tick := func() { + p.pollOnce(ctx) + if rend != nil { + rend.renderOnce(ctx) + } + } + tick() // fire immediately t := time.NewTicker(*interval) defer t.Stop() for { @@ -82,11 +117,39 @@ func run(args []string) error { log.Printf("mavcaldav: bye") return nil case <-t.C: - p.pollOnce(ctx) + tick() } } } +// checkRenderTarget refuses a render URL that is also one of the read URLs. +// This is the structural half of #127's "cannot write to your work calendar": +// the write credential and the write URL are separate flags, and a calendar +// maven is known to only read is rejected as a target at startup rather than +// trusted at runtime. +// +// It takes the whole read set, not one URL. The guarantee in the package +// comment is about every calendar maven reads, and a second read target added +// later must not quietly fall outside the check. +func checkRenderTarget(readURLs []string, renderURL string) error { + if renderURL == "" { + return nil + } + for _, read := range readURLs { + if read == "" { + continue + } + if sameCollection(read, renderURL) { + return fmt.Errorf("-render-url must differ from the read URL %s: maven renders into a calendar she owns, never into one she reads", read) + } + } + return nil +} + +func sameCollection(a, b string) bool { + return strings.EqualFold(strings.TrimRight(a, "/"), strings.TrimRight(b, "/")) +} + type poller struct { core ipc.CoreAPI http *http.Client @@ -95,12 +158,6 @@ type poller struct { pass string } -type icalEvent struct { - start time.Time - end time.Time - summary string -} - func (p *poller) pollOnce(ctx context.Context) { now := time.Now() events, err := p.fetchEvents(ctx, now) @@ -109,38 +166,30 @@ func (p *poller) pollOnce(ctx context.Context) { return } - busy := false - for _, e := range events { - if !now.Before(e.start) && now.Before(e.end) { - busy = true - break - } - } busyVal := "false" - if busy { + if calendar.Busy(events, now) { busyVal = "true" } // Write calendar_busy on change. - if err := p.writeIfChanged(ctx, "calendar_busy", "poll:caldav", busyVal, now); err != nil { + if err := p.writeIfChanged(ctx, "calendar_busy", calendar.SourcePersonal, busyVal, now); err != nil { log.Printf("mavcaldav: write calendar_busy: %v", err) return } - // Write per-event facts (one per event, keyed by event summary + start). + // Write per-event facts (one per event, keyed by day + event summary). // This lets the note RAG path answer "what's on my calendar" without // reaching back to Radicale. for _, e := range events { - val := fmt.Sprintf("%s @ %s-%s", e.summary, e.start.Format("15:04"), e.end.Format("15:04")) - eventKey := fmt.Sprintf("calendar_event_%s_%s", e.start.Format("20060102"), safeKey(e.summary)) - if err := p.writeIfChanged(ctx, eventKey, "poll:caldav", val, e.start); err != nil { - log.Printf("mavcaldav: write %s: %v", eventKey, err) + key := calendar.FactKey(e) + if err := p.writeIfChanged(ctx, key, calendar.SourcePersonal, calendar.FactValue(e), e.Start); err != nil { + log.Printf("mavcaldav: write %s: %v", key, err) } } } // fetchEvents GETs the calendar URL and parses VEVENTs from the iCal response. -func (p *poller) fetchEvents(ctx context.Context, now time.Time) ([]icalEvent, error) { +func (p *poller) fetchEvents(ctx context.Context, now time.Time) ([]calendar.Event, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.url, nil) if err != nil { return nil, err @@ -162,119 +211,13 @@ func (p *poller) fetchEvents(ctx context.Context, now time.Time) ([]icalEvent, e return nil, fmt.Errorf("GET %s: %s", p.url, resp.Status) } - return parseICal(body, now), nil -} - -// parseICal scans iCal text for VEVENT components. Returns events that overlap -// with today (UTC day boundaries) to keep the response manageable. -func parseICal(body []byte, now time.Time) []icalEvent { - todayStart := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC) - todayEnd := todayStart.AddDate(0, 0, 1) - - var events []icalEvent - text := string(body) - for { - veventStart := strings.Index(text, "BEGIN:VEVENT") - if veventStart < 0 { - break - } - text = text[veventStart+len("BEGIN:VEVENT"):] - veventEnd := strings.Index(text, "END:VEVENT") - if veventEnd < 0 { - break - } - block := text[:veventEnd] - text = text[veventEnd+len("END:VEVENT"):] - - e := parseVEVENT(block) - if e == nil { - continue - } - // Only keep events overlapping today. - if e.end.After(todayStart) && e.start.Before(todayEnd) { - events = append(events, *e) - } - } - return events -} - -// parseVEVENT extracts start, end, summary from a VEVENT block. -// Supports both UTC (DTEND:20260703T100000Z) and local (DTSTART;TZID=...:...) -// formats. Returns nil for all-day events (no DTSTART/DTEND time component) or -// parse failures. -func parseVEVENT(block string) *icalEvent { - var e icalEvent - lines := strings.Split(block, "\n") - for _, line := range lines { - line = strings.TrimSpace(line) - switch { - case strings.HasPrefix(line, "DTSTART"): - if t, ok := parseDT(line); ok { - e.start = t - } - case strings.HasPrefix(line, "DTEND"): - if t, ok := parseDT(line); ok { - e.end = t - } - case strings.HasPrefix(line, "SUMMARY"): - if idx := strings.Index(line, ":"); idx >= 0 { - e.summary = strings.TrimSpace(line[idx+1:]) - } - } - } - if e.start.IsZero() || e.end.IsZero() { - return nil - } - return &e -} - -// parseDT parses a DTSTART/DTEND value. Supports: -// - UTC: DTEND:20260703T100000Z -// - Local: DTSTART;TZID=Europe/Moscow:20260703T130000 -// - Value-date (all-day): DTSTART;VALUE=DATE:20260703 (returns zero time) -func parseDT(line string) (time.Time, bool) { - if strings.Contains(line, "VALUE=DATE:") { - return time.Time{}, false // all-day, skip - } - idx := strings.LastIndex(line, ":") - if idx < 0 { - return time.Time{}, false - } - val := line[idx+1:] - val = strings.TrimSuffix(val, "Z") - - // Try UTC first (has Z suffix, or ended in Z before TrimSuffix). - if strings.HasSuffix(line, "Z") { - t, err := time.Parse("20060102T150405", val) - if err != nil { - return time.Time{}, false - } - return t.UTC(), true - } - - // Local time — treat as UTC for simplicity (CalDAV server and poller - // run in the same timezone; the gate only needs busy/not-busy accuracy). - t, err := time.Parse("20060102T150405", val) - if err != nil { - return time.Time{}, false - } - return t.UTC(), true -} - -// safeKey makes an event summary safe to use as a fact key (alphanumeric + dash). -func safeKey(s string) string { - var b strings.Builder - for _, r := range s { - if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '-' { - b.WriteRune(r) - } else if r == ' ' || r == '_' { - b.WriteRune('-') - } - } - return b.String() + return calendar.ParseICalDay(body, now), nil } // writeIfChanged writes a fact only when the value differs from the latest. +// Everything this poller writes is a calendar read, which is full confidence by +// definition; a source that is not, such as the notification relay, does not +// come through here. func (p *poller) writeIfChanged(ctx context.Context, key, source, val string, ts time.Time) error { prev, err := p.core.LatestFactBySource(ctx, key, source) switch { diff --git a/cmd/mavcaldav/main_test.go b/cmd/mavcaldav/main_test.go index 6972eda..c3d82d3 100644 --- a/cmd/mavcaldav/main_test.go +++ b/cmd/mavcaldav/main_test.go @@ -12,7 +12,7 @@ import ( ) type fakeCore struct { - ipc.CoreAPI + ipc.UnimplementedCoreAPI facts map[string]ipc.Fact // composite key "key|source" → Fact writeLog []ipc.WriteFactReq writeErr error @@ -51,165 +51,6 @@ func (f *fakeCore) WriteFact(_ context.Context, req ipc.WriteFactReq) (int64, er return int64(len(f.writeLog)), nil } -// --------------------------------------------------------------------------- -// Parsing tests -// --------------------------------------------------------------------------- - -func TestParseICal(t *testing.T) { - now := time.Date(2026, 7, 3, 12, 0, 0, 0, time.UTC) - - body := []byte(`BEGIN:VCALENDAR -BEGIN:VEVENT -DTSTART:20260703T090000Z -DTEND:20260703T100000Z -SUMMARY:Morning standup -END:VEVENT -BEGIN:VEVENT -DTSTART:20260703T140000Z -DTEND:20260703T150000Z -SUMMARY:Team sync -END:VEVENT -BEGIN:VEVENT -DTSTART:20260702T140000Z -DTEND:20260702T150000Z -SUMMARY:Yesterday retro -END:VEVENT -BEGIN:VEVENT -DTSTART:20260704T090000Z -DTEND:20260704T100000Z -SUMMARY:Tomorrow standup -END:VEVENT -BEGIN:VEVENT -DTSTART;VALUE=DATE:20260704 -DTEND;VALUE=DATE:20260705 -SUMMARY:All-day event -END:VEVENT -END:VCALENDAR`) - - events := parseICal(body, now) - - if len(events) != 2 { - t.Fatalf("got %d events, want 2 (today events, no all-day/past/future)", len(events)) - } - - // Morning standup — overlaps today. - if events[0].summary != "Morning standup" { - t.Errorf("events[0].summary = %q, want %q", events[0].summary, "Morning standup") - } - wantStart0 := time.Date(2026, 7, 3, 9, 0, 0, 0, time.UTC) - if !events[0].start.Equal(wantStart0) { - t.Errorf("events[0].start = %v, want %v", events[0].start, wantStart0) - } - wantEnd0 := time.Date(2026, 7, 3, 10, 0, 0, 0, time.UTC) - if !events[0].end.Equal(wantEnd0) { - t.Errorf("events[0].end = %v, want %v", events[0].end, wantEnd0) - } - - // Team sync — overlaps today. - if events[1].summary != "Team sync" { - t.Errorf("events[1].summary = %q, want %q", events[1].summary, "Team sync") - } - wantStart1 := time.Date(2026, 7, 3, 14, 0, 0, 0, time.UTC) - if !events[1].start.Equal(wantStart1) { - t.Errorf("events[1].start = %v, want %v", events[1].start, wantStart1) - } - wantEnd1 := time.Date(2026, 7, 3, 15, 0, 0, 0, time.UTC) - if !events[1].end.Equal(wantEnd1) { - t.Errorf("events[1].end = %v, want %v", events[1].end, wantEnd1) - } -} - -func TestParseVEVENT(t *testing.T) { - // Normal event with TZID in DTSTART and UTC DTEND. - block := "DTSTART;TZID=Europe/Moscow:20260703T130000\nDTEND:20260703T140000Z\nSUMMARY:Stand up meeting" - e := parseVEVENT(block) - if e == nil { - t.Fatal("expected non-nil icalEvent") - } - wantStart := time.Date(2026, 7, 3, 13, 0, 0, 0, time.UTC) - if !e.start.Equal(wantStart) { - t.Errorf("start = %v, want %v", e.start, wantStart) - } - wantEnd := time.Date(2026, 7, 3, 14, 0, 0, 0, time.UTC) - if !e.end.Equal(wantEnd) { - t.Errorf("end = %v, want %v", e.end, wantEnd) - } - if e.summary != "Stand up meeting" { - t.Errorf("summary = %q, want %q", e.summary, "Stand up meeting") - } - - // All-day event (VALUE=DATE) → nil. - allDay := "DTSTART;VALUE=DATE:20260703\nDTEND;VALUE=DATE:20260704\nSUMMARY:All-day" - if e2 := parseVEVENT(allDay); e2 != nil { - t.Error("expected nil for all-day event") - } -} - -func TestParseDT(t *testing.T) { - tests := []struct { - name string - line string - want time.Time - wantOK bool - }{ - { - name: "UTC", - line: "DTEND:20260703T100000Z", - want: time.Date(2026, 7, 3, 10, 0, 0, 0, time.UTC), - wantOK: true, - }, - { - name: "local time", - line: "DTSTART;TZID=Europe/Moscow:20260703T130000", - want: time.Date(2026, 7, 3, 13, 0, 0, 0, time.UTC), - wantOK: true, - }, - { - name: "all-day", - line: "DTSTART;VALUE=DATE:20260703", - want: time.Time{}, - wantOK: false, - }, - { - name: "invalid", - line: "DTSTART:garbage", - want: time.Time{}, - wantOK: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, ok := parseDT(tt.line) - if ok != tt.wantOK { - t.Errorf("ok = %v, want %v", ok, tt.wantOK) - } - if !got.Equal(tt.want) { - t.Errorf("got = %v, want %v", got, tt.want) - } - }) - } -} - -func TestSafeKey(t *testing.T) { - tests := []struct { - input string - want string - }{ - {"Stand up meeting", "Stand-up-meeting"}, - {"Hello_World", "Hello-World"}, - {"special@#$chars!!", "specialchars"}, - {"ALL_CAPS_123", "ALL-CAPS-123"}, - } - - for _, tt := range tests { - got := safeKey(tt.input) - if got != tt.want { - t.Errorf("safeKey(%q) = %q, want %q", tt.input, got, tt.want) - } - } -} - // --------------------------------------------------------------------------- // Core logic tests // --------------------------------------------------------------------------- @@ -349,13 +190,15 @@ func TestPollOnce(t *testing.T) { t.Errorf("calendar_busy ts is zero") } - // Second write: calendar_event__ = " @ HH:MM-HH:MM" + // Second write: calendar_event__ = " @ HH:MM-HH:MM". + // The iCal states the event in UTC and the fact is stamped on the owner's + // clock, so the expected key date and times are the local reading of it. eventReq := fc.writeLog[1] - expectedKey := "calendar_event_" + start.Format("20060102") + "_Current-meeting" + expectedKey := "calendar_event_" + start.Local().Format("20060102") + "_Current-meeting" if eventReq.Key != expectedKey { t.Errorf("event key = %q, want %q", eventReq.Key, expectedKey) } - expectedVal := "Current meeting @ " + start.Format("15:04") + "-" + end.Format("15:04") + expectedVal := "Current meeting @ " + start.Local().Format("15:04") + "-" + end.Local().Format("15:04") if eventReq.Value != expectedVal { t.Errorf("event value = %q, want %q", eventReq.Value, expectedVal) } diff --git a/cmd/mavcaldav/render.go b/cmd/mavcaldav/render.go new file mode 100644 index 0000000..4568317 --- /dev/null +++ b/cmd/mavcaldav/render.go @@ -0,0 +1,222 @@ +package main + +import ( + "context" + "encoding/xml" + "fmt" + "io" + "log" + "net/http" + "net/url" + "strings" + "time" + + "github.com/kami/maven/internal/calendar" + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/store" +) + +// renderer is the write half of maven's own local calendar (Vikunja #127). +// +// It is a RENDER TARGET, not a store. sqlite stays canonical: every tick the +// renderer reads the pending reminders out of core and publishes each one as a +// single-event iCal resource in a CalDAV collection maven owns. Nothing is ever +// read back from that collection, and losing it costs nothing — the next tick +// rebuilds it. +// +// It structurally cannot write to a calendar maven only reads. The URL comes +// from its own flag, checked at startup against every read URL (see +// run in main.go), and the only paths it ever addresses carry +// calendar.ReminderUIDPrefix — so even pointed at the wrong collection it can +// only touch resources it created. +type renderer struct { + core ipc.CoreAPI + http *http.Client + url string + user string + pass string + dur time.Duration + + // published maps reminder id → the body last successfully PUT, so an + // unchanged reminder costs nothing. Purely an optimisation: a restart + // re-publishes every reminder once, which is idempotent. + published map[int64]string + + // reconciled — whether the collection has been read once since start. It + // has to be, because published is in-memory: withdrawal used to cover only + // the reminders THIS process published, so a reminder that fired while the + // daemon was down kept its event in the calendar forever, and nothing ever + // revisited it. + reconciled bool +} + +func newRenderer(core ipc.CoreAPI, hc *http.Client, url, user, pass string, dur time.Duration) *renderer { + return &renderer{ + core: core, + http: hc, + url: strings.TrimRight(url, "/"), + user: user, + pass: pass, + dur: dur, + published: make(map[int64]string), + } +} + +// renderOnce publishes every pending reminder and withdraws the ones that are +// no longer pending. Errors are logged and skipped: a calendar maven cannot +// reach must never break the reminder itself, which lives in sqlite. +func (r *renderer) renderOnce(ctx context.Context) { + reminders, err := r.core.ListReminders(ctx, renderMaxReminders) + if err != nil { + log.Printf("mavcaldav: list reminders: %v", err) + return + } + + live := make(map[int64]bool, len(reminders)) + for _, rem := range reminders { + if rem.Status != store.ReminderPending { + continue + } + live[rem.ID] = true + e := calendar.ReminderEvent(rem.ID, fireTime(rem), rem.Payload, r.dur) + body := calendar.RenderICal([]calendar.Event{e}) + if r.published[rem.ID] == body { + continue + } + if err := r.put(ctx, calendar.ReminderPath(rem.ID), body); err != nil { + log.Printf("mavcaldav: render reminder %d: %v", rem.ID, err) + continue + } + r.published[rem.ID] = body + log.Printf("mavcaldav: rendered reminder %d (%s)", rem.ID, e.Summary) + } + + stale := make(map[int64]bool) + for id := range r.published { + if !live[id] { + stale[id] = true + } + } + if !r.reconciled { + remote, err := r.listPublished(ctx) + if err != nil { + // Try again next tick. A collection maven cannot read is not a + // reason to stop publishing to it. + log.Printf("mavcaldav: reconcile: %v", err) + } else { + r.reconciled = true + for _, id := range remote { + if !live[id] { + stale[id] = true + } + } + } + } + for id := range stale { + if err := r.delete(ctx, calendar.ReminderPath(id)); err != nil { + log.Printf("mavcaldav: withdraw reminder %d: %v", id, err) + continue + } + delete(r.published, id) + log.Printf("mavcaldav: withdrew reminder %d", id) + } +} + +// listPublished PROPFINDs the collection and returns the reminder ids maven has +// events for in it. Only resources carrying calendar.ReminderUIDPrefix are +// reported, so a reconciliation pass can never propose deleting a file maven +// did not create — the same bound every other path in this file has. +func (r *renderer) listPublished(ctx context.Context) ([]int64, error) { + const body = `` + + `` + req, err := http.NewRequestWithContext(ctx, "PROPFIND", r.url+"/", strings.NewReader(body)) + if err != nil { + return nil, err + } + req.SetBasicAuth(r.user, r.pass) + req.Header.Set("Content-Type", "application/xml; charset=utf-8") + req.Header.Set("Depth", "1") + + resp, err := r.http.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + raw, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20)) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusMultiStatus && (resp.StatusCode < 200 || resp.StatusCode >= 300) { + return nil, fmt.Errorf("PROPFIND %s: %s", r.url, resp.Status) + } + + var ms struct { + Responses []struct { + Href string `xml:"href"` + } `xml:"response"` + } + if err := xml.Unmarshal(raw, &ms); err != nil { + return nil, fmt.Errorf("PROPFIND %s: %w", r.url, err) + } + var ids []int64 + for _, resp := range ms.Responses { + href, err := url.PathUnescape(strings.TrimSpace(resp.Href)) + if err != nil { + continue + } + if id, ok := calendar.ReminderIDFromPath(href); ok { + ids = append(ids, id) + } + } + return ids, nil +} + +// renderMaxReminders bounds the read. Reminders past this count are older than +// anything a calendar view is useful for. +const renderMaxReminders = 200 + +// fireTime prefers NextFireTs — for a recurring reminder that is the occurrence +// worth showing; FireTs is the original statement. +func fireTime(rem ipc.Reminder) time.Time { + if !rem.NextFireTs.IsZero() { + return rem.NextFireTs + } + return rem.FireTs +} + +func (r *renderer) put(ctx context.Context, name, body string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodPut, r.url+"/"+name, strings.NewReader(body)) + if err != nil { + return err + } + req.SetBasicAuth(r.user, r.pass) + req.Header.Set("Content-Type", "text/calendar; charset=utf-8") + return r.do(req, name) +} + +func (r *renderer) delete(ctx context.Context, name string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodDelete, r.url+"/"+name, nil) + if err != nil { + return err + } + req.SetBasicAuth(r.user, r.pass) + return r.do(req, name) +} + +// do runs the request and treats any 2xx, plus 404 on a DELETE, as success — +// a resource that is already gone is the state the caller wanted. +func (r *renderer) do(req *http.Request, name string) error { + resp, err := r.http.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<16)) + switch { + case resp.StatusCode >= 200 && resp.StatusCode < 300: + return nil + case req.Method == http.MethodDelete && resp.StatusCode == http.StatusNotFound: + return nil + } + return fmt.Errorf("%s %s: %s", req.Method, name, resp.Status) +} diff --git a/cmd/mavcaldav/render_test.go b/cmd/mavcaldav/render_test.go new file mode 100644 index 0000000..53c759d --- /dev/null +++ b/cmd/mavcaldav/render_test.go @@ -0,0 +1,301 @@ +package main + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/kami/maven/internal/ipc" +) + +// reminderCore is a fakeCore that also answers ListReminders. +type reminderCore struct { + fakeCore + reminders []ipc.Reminder + listErr error +} + +func (c *reminderCore) ListReminders(context.Context, int) ([]ipc.Reminder, error) { + if c.listErr != nil { + return nil, c.listErr + } + return c.reminders, nil +} + +// calSrv records what a CalDAV collection received. existing seeds resources +// that were already in the collection before this process started, which is +// what a restart looks like from the renderer's side. +type calSrv struct { + mu sync.Mutex + puts map[string]string + dels []string + existing []string + propfind int + status int + *httptest.Server +} + +func newCalSrv() *calSrv { + s := &calSrv{puts: map[string]string{}, status: http.StatusCreated} + s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + s.mu.Lock() + defer s.mu.Unlock() + switch r.Method { + case http.MethodPut: + s.puts[strings.TrimPrefix(r.URL.Path, "/cal/")] = string(body) + case http.MethodDelete: + s.dels = append(s.dels, strings.TrimPrefix(r.URL.Path, "/cal/")) + case "PROPFIND": + s.propfind++ + w.Header().Set("Content-Type", "application/xml; charset=utf-8") + w.WriteHeader(http.StatusMultiStatus) + io.WriteString(w, s.multistatusLocked(r.URL.Path)) + return + } + w.WriteHeader(s.status) + })) + return s +} + +// multistatusLocked renders the collection listing. Caller holds the lock. +func (s *calSrv) multistatusLocked(base string) string { + var b strings.Builder + b.WriteString(``) + b.WriteString("" + base + "") + names := append([]string{}, s.existing...) + for name := range s.puts { + names = append(names, name) + } + for _, name := range names { + if slices.Contains(s.dels, name) { + continue + } + b.WriteString("/cal/" + name + "") + } + b.WriteString("") + return b.String() +} + +func (s *calSrv) deleted() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string{}, s.dels...) +} + +func (s *calSrv) putCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.puts) +} + +func TestRenderOncePublishesPendingReminders(t *testing.T) { + fire := time.Date(2026, 8, 1, 18, 30, 0, 0, time.UTC) + srv := newCalSrv() + defer srv.Close() + + core := &reminderCore{reminders: []ipc.Reminder{ + {ID: 7, FireTs: fire, Payload: "позвонить маме", Status: "pending"}, + {ID: 8, FireTs: fire, Payload: "уже сделано", Status: "fired"}, + {ID: 9, FireTs: fire, Payload: "отменено", Status: "cancelled"}, + }} + r := newRenderer(core, srv.Client(), srv.URL+"/cal/", "u", "p", 0) + r.renderOnce(context.Background()) + + srv.mu.Lock() + body, ok := srv.puts["maven-reminder-7.ics"] + n := len(srv.puts) + srv.mu.Unlock() + + if n != 1 { + t.Fatalf("expected exactly the pending reminder to be published, got %d PUTs", n) + } + if !ok { + t.Fatal("pending reminder 7 was not published") + } + if !strings.Contains(body, "SUMMARY:позвонить маме") { + t.Errorf("payload missing from rendered body:\n%s", body) + } + if !strings.Contains(body, "UID:maven-reminder-7") { + t.Errorf("UID missing from rendered body:\n%s", body) + } +} + +func TestRenderOnceSkipsUnchanged(t *testing.T) { + srv := newCalSrv() + defer srv.Close() + core := &reminderCore{reminders: []ipc.Reminder{ + {ID: 1, FireTs: time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC), Payload: "выпить воды", Status: "pending"}, + }} + r := newRenderer(core, srv.Client(), srv.URL+"/cal", "u", "p", 0) + r.renderOnce(context.Background()) + r.renderOnce(context.Background()) + if got := srv.putCount(); got != 1 { + t.Fatalf("an unchanged reminder was re-published: %d distinct PUTs", got) + } +} + +func TestRenderOnceWithdrawsResolvedReminders(t *testing.T) { + srv := newCalSrv() + defer srv.Close() + core := &reminderCore{reminders: []ipc.Reminder{ + {ID: 5, FireTs: time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC), Payload: "встреча", Status: "pending"}, + }} + r := newRenderer(core, srv.Client(), srv.URL+"/cal", "u", "p", 0) + r.renderOnce(context.Background()) + + core.reminders[0].Status = "fired" + r.renderOnce(context.Background()) + + srv.mu.Lock() + dels := append([]string(nil), srv.dels...) + srv.mu.Unlock() + if len(dels) != 1 || dels[0] != "maven-reminder-5.ics" { + t.Fatalf("resolved reminder was not withdrawn: %v", dels) + } + if len(r.published) != 0 { + t.Errorf("published map still holds %v", r.published) + } +} + +// A calendar maven cannot reach must never break anything: sqlite is canonical. +func TestRenderOnceSurvivesServerErrors(t *testing.T) { + srv := newCalSrv() + srv.status = http.StatusInternalServerError + defer srv.Close() + core := &reminderCore{reminders: []ipc.Reminder{ + {ID: 1, FireTs: time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC), Payload: "x", Status: "pending"}, + }} + r := newRenderer(core, srv.Client(), srv.URL+"/cal", "u", "p", 0) + r.renderOnce(context.Background()) + if len(r.published) != 0 { + t.Error("a failed PUT must not be recorded as published, or it never retries") + } +} + +func TestRenderOnceUsesNextFireForRecurring(t *testing.T) { + srv := newCalSrv() + defer srv.Close() + next := time.Date(2026, 8, 2, 7, 0, 0, 0, time.UTC) + core := &reminderCore{reminders: []ipc.Reminder{{ + ID: 3, + FireTs: time.Date(2026, 8, 1, 7, 0, 0, 0, time.UTC), + NextFireTs: next, + Payload: "зарядка", + Status: "pending", + Cron: "0 7 * * *", + }}} + r := newRenderer(core, srv.Client(), srv.URL+"/cal", "u", "p", 0) + r.renderOnce(context.Background()) + + srv.mu.Lock() + body := srv.puts["maven-reminder-3.ics"] + srv.mu.Unlock() + if !strings.Contains(body, "DTSTART:20260802T070000Z") { + t.Errorf("recurring reminder should render its next occurrence:\n%s", body) + } +} + +func TestCheckRenderTargetRefusesTheCalendarItReads(t *testing.T) { + read := "http://localhost:5232/kami/personal" + if err := checkRenderTarget([]string{read}, ""); err != nil { + t.Fatalf("rendering off must be fine: %v", err) + } + if err := checkRenderTarget([]string{read}, "http://localhost:5232/kami/maven"); err != nil { + t.Fatalf("a distinct collection must be accepted: %v", err) + } + if err := checkRenderTarget([]string{read}, read); err == nil { + t.Error("rendering into the read calendar must be refused") + } + if err := checkRenderTarget([]string{read}, read+"/"); err == nil { + t.Error("a trailing slash must not defeat the check") + } + if err := checkRenderTarget([]string{read}, strings.ToUpper(read)); err == nil { + t.Error("case must not defeat the check") + } + // Every read target is checked, not the first one. A second calendar to + // read must not fall outside the guarantee just by being added later. + work := "http://localhost:5232/kami/work" + if err := checkRenderTarget([]string{read, work}, work); err == nil { + t.Error("rendering into the second read calendar must be refused") + } + if err := checkRenderTarget([]string{read, work}, "http://localhost:5232/kami/maven"); err != nil { + t.Fatalf("a collection maven owns must still be accepted: %v", err) + } +} + +// Withdrawal has to survive a restart. published is in-memory, so a fresh +// process knows nothing about the events an earlier one wrote: fire a reminder, +// restart mavcaldav, and its event used to sit in the collection forever +// because nothing ever revisited it. The first tick reads the collection and +// reconciles what it finds against what is pending. +func TestRenderOnceWithdrawsAfterRestart(t *testing.T) { + srv := newCalSrv() + defer srv.Close() + // Left behind by a previous process: 4 is still pending, 5 has fired. + // The third file is not maven's and must not be touched. + srv.existing = []string{"maven-reminder-4.ics", "maven-reminder-5.ics", "dentist.ics"} + + core := &reminderCore{reminders: []ipc.Reminder{ + {ID: 4, FireTs: time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC), Payload: "выпить воды", Status: "pending"}, + {ID: 5, FireTs: time.Date(2026, 8, 1, 8, 0, 0, 0, time.UTC), Payload: "уже прозвенело", Status: "fired"}, + }} + r := newRenderer(core, srv.Client(), srv.URL+"/cal", "u", "p", 0) + r.renderOnce(context.Background()) + + dels := srv.deleted() + if len(dels) != 1 || dels[0] != "maven-reminder-5.ics" { + t.Fatalf("deleted %v, want only the fired reminder's event", dels) + } + + // The collection is read once, not on every tick. + r.renderOnce(context.Background()) + srv.mu.Lock() + n := srv.propfind + srv.mu.Unlock() + if n != 1 { + t.Errorf("PROPFIND ran %d times, want once per process", n) + } +} + +// A collection maven cannot read is not a reason to stop publishing to it, and +// the reconciliation must be retried rather than skipped for the process. +func TestRenderOnceRetriesReconcile(t *testing.T) { + srv := newCalSrv() + defer srv.Close() + srv.existing = []string{"maven-reminder-6.ics"} + failing := &http.Client{Transport: &propfindFailure{base: srv.Client().Transport}} + + core := &reminderCore{} + r := newRenderer(core, failing, srv.URL+"/cal", "u", "p", 0) + r.renderOnce(context.Background()) + if got := srv.deleted(); len(got) != 0 { + t.Fatalf("nothing can be withdrawn on a failed read: %v", got) + } + if r.reconciled { + t.Fatal("a failed read must not count as reconciled") + } + + r.http = srv.Client() + r.renderOnce(context.Background()) + if got := srv.deleted(); len(got) != 1 || got[0] != "maven-reminder-6.ics" { + t.Fatalf("deleted %v, want the orphaned event on the retry", got) + } +} + +// propfindFailure fails PROPFIND and passes everything else through. +type propfindFailure struct{ base http.RoundTripper } + +func (f *propfindFailure) RoundTrip(req *http.Request) (*http.Response, error) { + if req.Method == "PROPFIND" { + return nil, errors.New("collection unreachable") + } + return f.base.RoundTrip(req) +} diff --git a/cmd/mavend/actions.go b/cmd/mavend/actions.go new file mode 100644 index 0000000..f6a9956 --- /dev/null +++ b/cmd/mavend/actions.go @@ -0,0 +1,71 @@ +// actionTable dispatches applyAction's per-intent bodies. Each of the 7 +// intents (fact, reminder, note, query, act, chat, system) has one handler +// here with the signature: +// +// func(h *reactiveHandler, ctx context.Context, dec router.Decision) string +// +// same contract as applyAction itself: "" means "let the Replier phrase the +// reply", a non-empty string OVERRIDES it. This is a straight extraction of +// applyAction's old switch cases (formerly ~300 lines in voice.go) — no +// reordering of side effects, no new abstractions inside a handler. +// +// What does NOT belong in this table, because it is not per-intent: +// +// - the dec.Clarify short-circuit ("" when the router's stage-3 fired) — +// stays in applyAction, before dispatch, since it applies to every +// intent identically. +// - the destructive-act confirm gate (park / resolveConfirm / confirmTTL) +// and the enabled-tool allowlist. Both live entirely inside +// actionAct/handleAct in actions_act.go, exactly where they lived in the old +// switch's IntentAct case — they are act-specific (a fact or a note +// can't be destructive), not shared across intents, so they do not need +// to move to a separate layer. The important invariant, preserved +// as-is: applyAction runs identically whether dec came from a fresh +// route or from a completed clarify answer (see finishClarified in +// clarify.go and its comment "filling in an argument never grants +// authority") — a handler must never special-case a clarify-completed +// decision to skip the confirm gate or the allowlist. +// - detectPattern and dialogue-session bookkeeping (rememberTurn, +// followUpMerge) run in the callers (runTurn, +// finishClarified), not per-intent, and are untouched by this slice. +// +// Each handler lives in actions_.go; the small ones (chat, system) +// and the table itself stay here. +// +// Adding an intent: write its handler in its own file, add one line to +// actionHandlers. Do not grow applyAction's switch back. +package main + +import ( + "context" + "log" + + "github.com/kami/maven/internal/router" +) + +// actionHandlers is the per-intent dispatch table used by applyAction. +var actionHandlers = map[router.Intent]func(*reactiveHandler, context.Context, router.Decision) string{ + router.IntentFact: (*reactiveHandler).actionFact, + router.IntentReminder: (*reactiveHandler).actionReminder, + router.IntentAct: (*reactiveHandler).actionAct, + router.IntentChat: (*reactiveHandler).actionChat, + router.IntentSystem: (*reactiveHandler).actionSystem, + router.IntentNote: (*reactiveHandler).actionNote, + router.IntentQuery: (*reactiveHandler).actionQuery, +} + +func (h *reactiveHandler) actionChat(ctx context.Context, dec router.Decision) string { + // Conversational: build history from dialogue session (prior user turns) + // and let the LLM respond from general knowledge + context. + history := h.chatHistory() + reply, err := h.phraser.PhraseChat(ctx, dec.Utterance, history) + if err != nil { + log.Printf("voice: chat: %v", err) + return "поговорили." + } + return reply +} + +func (h *reactiveHandler) actionSystem(ctx context.Context, dec router.Decision) string { + return h.replySystem(ctx, dec) +} diff --git a/cmd/mavend/actions_act.go b/cmd/mavend/actions_act.go new file mode 100644 index 0000000..2b96855 --- /dev/null +++ b/cmd/mavend/actions_act.go @@ -0,0 +1,80 @@ +package main + +import ( + "context" + "errors" + "log" + + "github.com/kami/maven/internal/mcp" + "github.com/kami/maven/internal/router" + "github.com/kami/maven/internal/tool" +) + +// actionAct handles router.IntentAct: match a verb to an enabled tool, offer +// it to the ecosystems first, and run it behind the confirm gate and the +// allowlist. proposeGap and the confirm gate itself live in confirm.go. +func (h *reactiveHandler) actionAct(ctx context.Context, dec router.Decision) string { + // tool executor: run the matched fn against the enabled allowlist. + // HasFn=false ⇒ try the matcher (for LLM-routed acts where the verb + // didn't go through the stage-0 act grammar). + if !dec.Slots.HasFn && dec.Slots.Text != "" && h.matcher != nil { + if fn, args, ok := h.matcher.Match(dec.Slots.Text); ok { + dec.Slots.Fn, dec.Slots.Args, dec.Slots.HasFn = fn, args, true + } + } + + // Praxis ecosystem tools: intercept before the system command executor. + if h.ecosystem != nil && h.ecosystem.praxis != nil && dec.Slots.HasFn { + if reply := h.handlePraxisAct(ctx, dec); reply != "" { + return reply + } + } + + // Hexis ecosystem action: if ecosystem is configured and we have a verb + // + entity text, try to resolve the entity and execute via Hexis. + if h.ecosystem != nil && h.ecosystem.hexis != nil && dec.Slots.Text != "" { + if reply := h.handleHexisAct(ctx, dec); reply != "" { + return reply + } + } + + // HasFn still false ⇒ no allowlist match: scaffold a 'proposed' tool + // the user can enable on the authed surface ("earn the right to ask"). + if !dec.Slots.HasFn { + return h.proposeGap(ctx, dec) + } + out, err := h.tools.Exec(ctx, dec.Slots.Fn, dec.Slots.Args, false) + if err != nil { + switch { + case errors.Is(err, tool.ErrNeedsConfirm): + // destructive: park it and ask. The next utterance answers. + phrase := actPhrase(dec.Slots.Fn, dec.Slots.Args) + h.park(dec.Slots.Fn, dec.Slots.Args, phrase) + return "выполнить «" + phrase + "»? скажи «да» или «нет»." + case errors.Is(err, tool.ErrNotEnabled): + return h.proposeGap(ctx, dec) + case errors.Is(err, tool.ErrNotConnected), errors.Is(err, mcp.ErrNotConnected), errors.Is(err, mcp.ErrNoServer): + // The row is enabled and the backend is gone. Drafting a proposal + // for it (the ErrNotEnabled path) would be answering the wrong + // question. + return "этот инструмент включён, но сервер, который его выполняет, сейчас не подключён." + case errors.Is(err, mcp.ErrToolGone): + return "сервер больше не предлагает этот инструмент — я сняла его с разрешённых, посмотри на /tools." + case errors.Is(err, mcp.ErrNeedsArgs): + // An MCP tool that wants named arguments a spoken verb cannot + // supply. Guessing them would be a wrong act, so she says so + // instead — the tool is still runnable from the authed surface, + // where a human types them. + return "этому инструменту нужны аргументы, которые я из голоса не соберу — я не буду угадывать." + } + log.Printf("voice: tool %s: %v", dec.Slots.Fn, err) + if out != "" { + return "не получилось выполнить команду: " + firstLine(out) + } + return "не получилось выполнить команду." + } + if out != "" { + return "готово: " + firstLine(out) + } + return "готово." +} diff --git a/cmd/mavend/actions_fact.go b/cmd/mavend/actions_fact.go new file mode 100644 index 0000000..855bf4b --- /dev/null +++ b/cmd/mavend/actions_fact.go @@ -0,0 +1,64 @@ +package main + +import ( + "context" + "log" + "strconv" + + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/router" +) + +// actionFact handles router.IntentFact: persist a tapped self-fact, index +// it for recall, and let pattern detection propose a routine. +func (h *reactiveHandler) actionFact(ctx context.Context, dec router.Decision) string { + if !dec.Slots.HasKey { + return "не разобрала, что записать — попробуй иначе." + } + now := h.now() + req := ipc.WriteFactReq{ + Ts: now, + Kind: "self", + Key: dec.Slots.Key, + Value: dec.Slots.Value, + Source: "tap:voice", + Confidence: 1.0, + // Subject: the key doubles as the entity-resolution candidate — + // a voice-tapped fact's key is usually the thing/person it's + // about ("espresso_machine", "kate"), so queueing it for Nexus + // resolution costs one async lookup and is a no-op (not_found) + // for the abstract self-state keys (mood, water) that aren't + // entities at all. + Subject: dec.Slots.Key, + } + factID, err := h.api.WriteFact(ctx, req) + if err != nil { + log.Printf("voice: write fact: %v", err) + return "не получилось сохранить факт." + } + // Index the fact utterance in long-term memory (best-effort, must not + // fail the fact write). Facts aren't in the notes table, so this is the + // only recall path for them — "когда я пил воду?" reads back from here. + if h.memStore != nil { + if vec, err := router.EmbedPassage(ctx, h.embedder, dec.Utterance); err != nil { + log.Printf("voice: embed fact for memory: %v", err) + } else if err := h.memStore.Insert(ctx, "fact:"+dec.Slots.Key+":"+strconv.FormatInt(now.Unix(), 10), vec, map[string]string{ + "source": "voice", + "type": "fact", + "text": dec.Utterance, + "ts": strconv.FormatInt(now.Unix(), 10), + }); err != nil { + log.Printf("voice: memory insert fact: %v", err) + } + } + // Event extraction + pattern detection (best-effort, must not fail the + // fact write). If the fact describes a recognizable action, it becomes a + // normalized event; if ≥3 events for the same action+object show stable + // intervals, a proposed routine is created and parked for confirmation. + if h.dataStore != nil { + if phrase := h.detectPattern(ctx, factID, dec.Slots.Key, dec.Slots.Value, now); phrase != "" { + return phrase // "ты заправляешь ... напоминать?" + } + } + return "" // replier phrases the success reply +} diff --git a/cmd/mavend/actions_money.go b/cmd/mavend/actions_money.go new file mode 100644 index 0000000..ec65ba8 --- /dev/null +++ b/cmd/mavend/actions_money.go @@ -0,0 +1,93 @@ +package main + +import ( + "context" + "errors" + "log" + + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/router" + "github.com/kami/maven/internal/zenmoney" +) + +// Money questions (Vikunja #125). +// +// This is the whole read side: mavpoll holds the zenmoney token and writes +// facts(kind=env, source=poll:zenmoney); core reads them back when he asks. +// Core never sees the token, never calls zenmoney, and has no rule on these +// keys — a total is never a reason for Maven to speak first. Maven is not a +// nag, least of all about his money. +// +// Nothing here can reach the external search capability: the figures are read +// from the store and rendered locally, and his financial data is never search +// input. + +// queryMoney — "сколько я потратил сегодня?", "покажи мои траты". +// +// Answers only from the latest fact the poller wrote. Three honest outcomes and +// no fourth: the figure, "the fact is old and here is its date", or "money +// tracking is not connected". It never computes, estimates or rounds a total of +// its own — an invented number about his money is the worst thing this could do. +func (h *reactiveHandler) queryMoney(ctx context.Context, t *queryTurn) (string, bool) { + q, ok := router.ParseMoneyQuery(t.dec.Utterance) + if !ok { + return "", false + } + if q.Window == router.MoneyUnsupported { + // Two windows are stored and no others. Answering "сколько я потратил + // вчера?" with the month-to-date total answers a different question + // with a real number, which is the shape of a lie he cannot spot. + return "я храню только сегодняшние траты и за этот месяц.", true + } + key, phrase := zenmoney.KeySpentMonth, "в этом месяце" + if q.Window == router.MoneyToday { + key, phrase = zenmoney.KeySpentToday, "сегодня" + } + fact, err := h.api.LatestFactBySource(ctx, key, zenmoney.Source) + if err != nil { + // No fact at all is the normal state when the capability is off. Claim + // the turn anyway: falling through to recall would answer a question + // about money with whatever note happens to be nearest. + if !isNoFactErr(err) { + log.Printf("voice: money fact: %v", err) + } + return "я не отслеживаю траты — не подключено.", true + } + val, err := zenmoney.ParseFactValue(fact.Value) + if err != nil { + log.Printf("voice: money fact: decode: %v", err) + return "не получилось прочитать траты.", true + } + now := h.now() + if q.Window == router.MoneyToday && !val.CoversDay(now) { + // The day window rolled over and the poller had nothing to write, + // because he has not spent anything yet today. The fact is fresh by ts + // and covers yesterday, so no staleness check can catch it — only the + // window stamp inside the value can. + return "сегодня пока ничего не вижу.", true + } + reply := val.FormatRU(phrase) + if q.Income { + reply = val.FormatIncomeRU(phrase) + } + if reply == "" { + return "по тратам пока нечего сказать.", true + } + // A stale fact is reported as stale rather than spoken as today's number. + // The age is measured from when the figure was last READ, not from when it + // last changed: a month with no spending in it does not go stale. + asOf := val.AsOf + if asOf.IsZero() { + asOf = fact.Ts + } + if now.Sub(asOf) > zenmoney.StaleAfter { + return "данные от " + asOf.Local().Format("02.01") + ": " + reply, true + } + return reply, true +} + +// isNoFactErr — ErrNoFact survives the wire wrapped, so unwrap for it. The +// hand-rolled loop this replaces missed any error implementing Is(error) bool. +func isNoFactErr(err error) bool { + return errors.Is(err, ipc.ErrNoFact) +} diff --git a/cmd/mavend/actions_money_test.go b/cmd/mavend/actions_money_test.go new file mode 100644 index 0000000..5f5657f --- /dev/null +++ b/cmd/mavend/actions_money_test.go @@ -0,0 +1,228 @@ +package main + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/router" + "github.com/kami/maven/internal/zenmoney" +) + +// moneyAPI answers only LatestFactBySource; everything else is unimplemented, +// which is the assertion that answering a money question costs no model call +// and reaches no network. +type moneyAPI struct { + ipc.UnimplementedCoreAPI + + fact ipc.Fact + err error + gotKey string + gotSrc string + callCnt int +} + +func (a *moneyAPI) LatestFactBySource(_ context.Context, key, source string) (ipc.Fact, error) { + a.gotKey, a.gotSrc = key, source + a.callCnt++ + return a.fact, a.err +} + +func moneyNow() time.Time { return time.Date(2026, 8, 15, 20, 0, 0, 0, time.UTC) } + +func moneyFact(ts time.Time, val string) ipc.Fact { + return ipc.Fact{Kind: "env", Key: zenmoney.KeySpentMonth, Value: val, Source: zenmoney.Source, Ts: ts} +} + +func TestQueryMoneyAnswersFromTheFact(t *testing.T) { + api := &moneyAPI{fact: moneyFact(moneyNow(), `{"spent":[{"currency":"RUB","amount":1749.5}],"count":3}`)} + h := &reactiveHandler{api: api, now: moneyNow} + reply, ok := h.queryMoney(context.Background(), &queryTurn{ + dec: router.Decision{Utterance: "сколько я потратил в этом месяце?"}, + }) + if !ok { + t.Fatal("the money source must claim a money question") + } + if api.gotKey != zenmoney.KeySpentMonth || api.gotSrc != zenmoney.Source { + t.Errorf("read %q/%q, want the month key from the poller's source", api.gotKey, api.gotSrc) + } + if !strings.Contains(reply, "1749.5") { + t.Errorf("reply = %q, want the exact figure", reply) + } + if !strings.Contains(reply, "в этом месяце") { + t.Errorf("reply = %q, want the window named", reply) + } +} + +func TestQueryMoneyPicksTodaysKey(t *testing.T) { + api := &moneyAPI{fact: moneyFact(moneyNow(), `{"spent":[{"currency":"RUB","amount":250}],"count":1}`)} + h := &reactiveHandler{api: api, now: moneyNow} + if _, ok := h.queryMoney(context.Background(), &queryTurn{ + dec: router.Decision{Utterance: "сколько я потратил сегодня?"}, + }); !ok { + t.Fatal("expected the source to claim it") + } + if api.gotKey != zenmoney.KeySpentToday { + t.Errorf("key = %q, want today's", api.gotKey) + } +} + +// The capability is off unless configured, and then there is no fact. She says +// so instead of letting the recall pass answer a money question from a note. +func TestQueryMoneySaysNotConnected(t *testing.T) { + h := &reactiveHandler{api: &moneyAPI{err: ipc.ErrNoFact}, now: moneyNow} + reply, ok := h.queryMoney(context.Background(), &queryTurn{ + dec: router.Decision{Utterance: "сколько я потратил?"}, + }) + if !ok { + t.Fatal("expected the source to claim it") + } + if !strings.Contains(reply, "не подключено") { + t.Errorf("reply = %q, want an honest 'not connected'", reply) + } + // No number of any kind in that answer. + for _, d := range []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9"} { + if strings.Contains(reply, d) { + t.Errorf("reply %q contains a digit — nothing was read, so there is no figure", reply) + } + } +} + +// A fact older than the staleness bound is dated rather than spoken as if it +// were current: the poller can be down, and last week's total presented as +// today's is a lie by omission. +func TestQueryMoneyDatesAStaleFact(t *testing.T) { + old := moneyNow().Add(-72 * time.Hour) + api := &moneyAPI{fact: moneyFact(old, `{"spent":[{"currency":"RUB","amount":100}],"count":1}`)} + h := &reactiveHandler{api: api, now: moneyNow} + reply, _ := h.queryMoney(context.Background(), &queryTurn{ + dec: router.Decision{Utterance: "сколько я потратил?"}, + }) + if !strings.Contains(reply, "данные от") { + t.Errorf("reply = %q, want the stale fact dated", reply) + } +} + +func TestQueryMoneyPassesOtherQuestions(t *testing.T) { + api := &moneyAPI{} + h := &reactiveHandler{api: api, now: moneyNow} + for _, u := range []string{"какая погода?", "я потратил весь день на это", "какие у меня задачи?"} { + if _, ok := h.queryMoney(context.Background(), &queryTurn{dec: router.Decision{Utterance: u}}); ok { + t.Errorf("the money source claimed %q", u) + } + } + if api.callCnt != 0 { + t.Error("a non-money question must not read the money facts") + } +} + +// Money must be answered before the recall sources, or a question about +// spending gets answered by the nearest note. +func TestQuerySourcesOrderMoneyBeforeRecall(t *testing.T) { + moneyAt, notesAt := -1, -1 + for i, src := range querySources { + switch src.name { + case "money": + moneyAt = i + case "notes": + notesAt = i + } + } + if moneyAt < 0 || notesAt < 0 { + t.Fatalf("sources missing: money=%d notes=%d", moneyAt, notesAt) + } + if moneyAt > notesAt { + t.Errorf("money source at %d, after notes at %d", moneyAt, notesAt) + } +} + +// The day window rolls over at midnight and the poller writes nothing until the +// first spend of the new day, so the last money_today fact is fresh by ts and +// covers yesterday. No staleness check can catch that. +func TestQueryMoneyRefusesYesterdaysDayTotal(t *testing.T) { + yesterday, _ := zenmoney.DayWindow(moneyNow().AddDate(0, 0, -1)) + sum := zenmoney.Summary{From: yesterday, Spent: []zenmoney.Money{{Currency: "RUB", Amount: 1749.5}}, Count: 3} + val, ok := sum.Value(moneyNow().AddDate(0, 0, -1).Add(2 * time.Hour)) + if !ok { + t.Fatal("want a fact value") + } + api := &moneyAPI{fact: ipc.Fact{ + Kind: "env", Key: zenmoney.KeySpentToday, Value: val, + Source: zenmoney.Source, Ts: moneyNow().Add(-11 * time.Hour), + }} + h := &reactiveHandler{api: api, now: moneyNow} + reply, claimed := h.queryMoney(context.Background(), &queryTurn{ + dec: router.Decision{Utterance: "сколько я потратил сегодня?"}, + }) + if !claimed { + t.Fatal("expected the source to claim it") + } + if strings.Contains(reply, "1749.5") { + t.Errorf("reply = %q — that is yesterday's spending spoken as today's", reply) + } +} + +// Ts advances only when the number moves, so a quiet month used to be reported +// as stale while being current. The read stamp inside the value is what the +// staleness check means. +func TestQueryMoneyMeasuresStalenessFromTheRead(t *testing.T) { + from, _ := zenmoney.MonthWindow(moneyNow()) + sum := zenmoney.Summary{From: from, Spent: []zenmoney.Money{{Currency: "RUB", Amount: 100}}, Count: 1} + val, _ := sum.Value(moneyNow().Add(-time.Hour)) + // The fact itself last CHANGED three days ago: nothing was spent since. + api := &moneyAPI{fact: ipc.Fact{ + Kind: "env", Key: zenmoney.KeySpentMonth, Value: val, + Source: zenmoney.Source, Ts: moneyNow().Add(-72 * time.Hour), + }} + h := &reactiveHandler{api: api, now: moneyNow} + reply, _ := h.queryMoney(context.Background(), &queryTurn{ + dec: router.Decision{Utterance: "сколько я потратил в этом месяце?"}, + }) + if strings.Contains(reply, "данные от") { + t.Errorf("reply = %q — the figure was read an hour ago and is current", reply) + } +} + +// Two windows are stored and no others. Answering "вчера" with the +// month-to-date total answers a different question with a real number. +func TestQueryMoneyRefusesWindowsItDoesNotKeep(t *testing.T) { + api := &moneyAPI{} + h := &reactiveHandler{api: api, now: moneyNow} + reply, ok := h.queryMoney(context.Background(), &queryTurn{ + dec: router.Decision{Utterance: "сколько я потратил вчера?"}, + }) + if !ok { + t.Fatal("a money question must be claimed, not passed to recall") + } + if !strings.Contains(reply, "только") { + t.Errorf("reply = %q, want her to say which windows she keeps", reply) + } + if api.callCnt != 0 { + t.Error("a window she does not keep must not read a fact") + } +} + +// "сколько я заработал" reads the same fact and must lead with the income. +func TestQueryMoneyLeadsWithIncomeWhenAsked(t *testing.T) { + from, _ := zenmoney.MonthWindow(moneyNow()) + sum := zenmoney.Summary{ + From: from, + Spent: []zenmoney.Money{{Currency: "RUB", Amount: 100}}, + Earned: []zenmoney.Money{{Currency: "RUB", Amount: 3000}}, + Count: 2, + } + val, _ := sum.Value(moneyNow()) + api := &moneyAPI{fact: ipc.Fact{ + Kind: "env", Key: zenmoney.KeySpentMonth, Value: val, + Source: zenmoney.Source, Ts: moneyNow(), + }} + h := &reactiveHandler{api: api, now: moneyNow} + reply, _ := h.queryMoney(context.Background(), &queryTurn{ + dec: router.Decision{Utterance: "сколько я заработал в этом месяце?"}, + }) + if strings.Index(reply, "3000") > strings.Index(reply, "100") { + t.Errorf("reply = %q, want the income he asked about first", reply) + } +} diff --git a/cmd/mavend/actions_note.go b/cmd/mavend/actions_note.go new file mode 100644 index 0000000..c7c20a5 --- /dev/null +++ b/cmd/mavend/actions_note.go @@ -0,0 +1,47 @@ +package main + +import ( + "context" + "log" + "strconv" + + "github.com/kami/maven/internal/router" +) + +// actionNote handles router.IntentNote: embed the note, persist it, and +// index it for recall. +func (h *reactiveHandler) actionNote(ctx context.Context, dec router.Decision) string { + // An utterance that explicitly files a task is work, not recall, and + // belongs in the task store (Vikunja #130). Checked before the embedding + // is paid for. Everything else is a note, exactly as before. + if reply, ok := h.captureTaskFromNote(ctx, dec); ok { + return reply + } + // embed the note text with the same model the classifier uses, persist + // via CoreAPI (source=tap:voice). Semantic recall lives in `notes`, not + // facts — no predicate reads it (spec's two-memory split). + vec, err := router.EmbedPassage(ctx, h.embedder, dec.Utterance) + if err != nil { + log.Printf("voice: embed note: %v", err) + return "не получилось сохранить заметку." + } + noteTs := h.now() + noteID, err := h.api.WriteNote(ctx, noteTs, dec.Utterance, vec, "tap:voice") + if err != nil { + log.Printf("voice: write note: %v", err) + return "не получилось сохранить заметку." + } + // Insert into long-term memory (best-effort, must not fail the note write). + // text/ts in the meta make a Search hit self-describing (see bestRecall). + if h.memStore != nil { + if err := h.memStore.Insert(ctx, "note:"+strconv.FormatInt(noteID, 10), vec, map[string]string{ + "source": "voice", + "type": "note", + "text": dec.Utterance, + "ts": strconv.FormatInt(noteTs.Unix(), 10), + }); err != nil { + log.Printf("voice: memory insert: %v", err) + } + } + return "" // replier phrases the "saved" reply +} diff --git a/cmd/mavend/actions_query.go b/cmd/mavend/actions_query.go new file mode 100644 index 0000000..5c3ef61 --- /dev/null +++ b/cmd/mavend/actions_query.go @@ -0,0 +1,510 @@ +package main + +import ( + "context" + "errors" + "fmt" + "log" + "strings" + "time" + + "github.com/kami/maven/internal/crawl" + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/memory" + "github.com/kami/maven/internal/morning" + "github.com/kami/maven/internal/router" + "github.com/kami/maven/internal/rss" + "github.com/kami/maven/internal/store" + "github.com/kami/maven/internal/weather" +) + +// queryTurn is the per-turn scratch a chain of query sources shares: the +// decision being answered plus the work an earlier source already paid for +// (the query embedding, the notes it pulled). Sources read and fill it in +// order, so a later source never re-embeds. +type queryTurn struct { + dec router.Decision + vec []float32 + notes []ipc.Note +} + +// querySource — one answer source in the chain actionQuery walks. answer +// returns (reply, true) when this source claims the question, ("", false) +// when it passes to the next one. name is for reading the table, not logged. +// +// A struct of one func rather than an interface: every source is a plain +// method on *reactiveHandler with no state of its own (what state a turn has +// lives in queryTurn), so an interface would mean one empty type per source +// to satisfy it — ceremony for nothing. Same reasoning as confirmResolver in +// confirm.go, and the table then reads like actionHandlers: a flat list of +// method expressions you extend with one line. +type querySource struct { + name string + answer func(*reactiveHandler, context.Context, *queryTurn) (string, bool) +} + +// querySources is the ordered chain actionQuery walks; first source to claim +// answers the turn. THE ORDER IS LOAD-BEARING — see the memory-before-notes +// comment on queryMemory: running the notes-only pass first was #373, and the +// gate was never the bug. Adding a source (Kiwix, RSS, crawler, email) is one +// line here plus its method; where you put the line is the whole decision. +var querySources = []querySource{ + {"fact-by-key", (*reactiveHandler).queryFactByKey}, + // Before "calendar" on purpose: both match "…на сегодня", and the plan is + // the more specific ask (its matcher requires a plan word), so the calendar + // listing would otherwise swallow it. + {"day-plan", (*reactiveHandler).queryDayPlan}, + // Also before "calendar": "что я обычно делаю по средам?" names a weekday, + // and the habit question is the more specific one. Its matcher requires a + // habit marker ("обычно", "каждый", …), so a question about this coming + // Wednesday still reaches the calendar. + {"habits", (*reactiveHandler).queryHabits}, + // Before "calendar" and before the recall sources: "что мне нужно + // сделать?" is a question about the task list, and the notes pass would + // otherwise answer it with whatever note happens to be nearest. Its + // matcher requires a task noun or an explicit "что … сделать", so a + // date-bearing question still reaches the calendar. + {"tasks", (*reactiveHandler).queryTasks}, + // Before the recall sources too: "сколько я потратил?" is a question about + // the money facts the poller wrote, and the notes pass would otherwise + // answer it from whatever he once said about spending. Its matcher needs a + // money noun plus an actual ask, so "я потратил весь день" is untouched. + {"money", (*reactiveHandler).queryMoney}, + // Before the recall sources and before general knowledge: "что нового?" is + // a question about the feeds she reads, and general knowledge would answer + // it by inventing news. Its matcher needs a feed noun plus an ask, so + // "у меня новая лента в инстаграме" is untouched. + {"feeds", (*reactiveHandler).queryFeeds}, + // Before "calendar" and before the recall sources: "что включено дома?" is + // a question about the house, and the notes pass would otherwise answer it + // from whatever he once said about the lights. Its matcher needs a house + // marker plus an ask plus a device word, and it bails out on weather + // wording, so "какая температура на улице?" still reaches the weather + // source. + {"home", (*reactiveHandler).queryHome}, + // Next to "home" and for the same reason: "какие устройства в сети?" is a + // question about the LAN, and the recall pass would otherwise answer it + // from an old note about the router. Its matcher needs a network word plus + // an ask plus a device noun, so "интернет не работает" is untouched. + {"network", (*reactiveHandler).queryNetwork}, + {"calendar", (*reactiveHandler).queryCalendar}, + {"weather", (*reactiveHandler).queryWeather}, + {"embed", (*reactiveHandler).queryEmbed}, + {"memory", (*reactiveHandler).queryMemory}, + {"notes", (*reactiveHandler).queryNotes}, + // LAST before the model answers from memory, and that position is the whole + // design (Vikunja #259): local sources first. His memory, his notes and — + // once internal/kiwix is wired into this chain — the offline ZIMs all get + // their turn before anything touches the network. The model does NOT: it + // answers after this, because a URL he said out loud is an instruction and + // a 1.7B guessing at a page it cannot read is how contents get invented. + // This source only claims a turn where he named a URL, so it never competes + // with a local answer. + {"web", (*reactiveHandler).queryWeb}, + {"general-knowledge", (*reactiveHandler).queryGeneral}, +} + +func (h *reactiveHandler) actionQuery(ctx context.Context, dec router.Decision) string { + t := &queryTurn{dec: dec} + for _, src := range querySources { + if reply, ok := src.answer(h, ctx, t); ok { + return reply + } + } + return "не знаю." +} + +// queryFactByKey — when the dialogue layer resolved an anaphoric reference to +// a prior fact's key (e.g. "когда я это сделал?" after "запиши что я пил +// воду"), look up the fact's value directly. +func (h *reactiveHandler) queryFactByKey(ctx context.Context, t *queryTurn) (string, bool) { + dec := t.dec + if !dec.Slots.HasKey || dec.Slots.Key == "" { + return "", false + } + f, err := h.api.LatestFact(ctx, dec.Slots.Key) + if err != nil { + return "", false + } + if dec.Slots.HasTime { + // The query asks about timing — the fact's own timestamp is the + // answer it's looking for. Format as a natural reply. + return fmt.Sprintf("я записала это %s", formatTime(f.Ts)), true + } + // General fact reference: describe what we know. + if dec.Utterance == "" { + return fmt.Sprintf("вот что я знаю: %s — %s", dec.Slots.Key, f.Value), true + } + // The utterance still carries the question; fall through to normal RAG + // with the resolved key in context. + return "", false +} + +// queryDayPlan — "какие планы на сегодня?", "что у меня по плану?", "что +// дальше?" (Vikunja #128). Recites the day: calendar events, pending +// reminders, and every morning checklist item today still has no evidence for, +// including the ones whose window has closed. +// +// Read-only by construction — the plan is assembled and rendered core-side and +// nothing here schedules or announces. "что дальше?" asks for the rest of the +// day, so that phrasing trims what has already passed. +// +// What surface this belongs on is still open, tracked as Vikunja #431 ("Board +// surface: Maven holds the work board, runs the intake form, never argues"). +// The spoken recital here is the current answer, not the decided one. +func (h *reactiveHandler) queryDayPlan(ctx context.Context, t *queryTurn) (string, bool) { + if !router.IsDayPlanQuery(t.dec.Utterance) { + return "", false + } + plan, err := h.api.DayPlan(ctx) + if err != nil { + log.Printf("voice: day plan: %v", err) + return "не получилось собрать план.", true + } + if !router.IsRestOfDayQuery(t.dec.Utterance) { + return plan.Spoken, true + } + // Rebuild the pure plan so the rest-of-day rendering is the same code that + // rendered the whole day — one formatter, one persona. + p := morning.Plan{Date: plan.Date} + for _, it := range plan.Items { + p.Items = append(p.Items, morning.PlanEntry{ + At: it.At, + Text: it.Text, + Kind: morning.PlanKind(it.Kind), + Uncertain: it.Uncertain, + }) + } + return p.After(h.now()).FormatRU(), true +} + +// habitFactWindow — how many recent SELF facts the behaviour profile is counted +// over. Enough for a season of habits without scanning the whole store on every +// question; the profile is recomputed on read, so the bound is the cost control. +// +// The read is kind-filtered in SQL, and that is the load-bearing part. When this +// was a plain recent-facts read the window was a row budget over every writer, +// and the machine writers dwarf the taps: mavpoll writes a wg_handshake row +// whenever a peer rehandshakes, which is roughly every two minutes per peer, so +// 2000 rows was under three days of history. A weekday habit needs +// memory.MinHabitDays distinct Tuesdays, which such a window can never hold, so +// she answered "по вторникам у меня пока нет ничего постоянного" forever on a +// store with a year of taps in it. Self facts come from voice taps, and he does +// not tap seven hundred times a day. +const habitFactWindow = 2000 + +// queryHabits — "что я обычно делаю по вторникам?" (Vikunja #254). Counts the +// answer out of the fact log rather than asking the model to summarise a life: +// see internal/memory/behavior.go for why nothing here is generated. +func (h *reactiveHandler) queryHabits(ctx context.Context, t *queryTurn) (string, bool) { + q, ok := router.ParseHabitQuery(t.dec.Utterance) + if !ok { + return "", false + } + facts, err := h.api.RecentActiveFactsByKind(ctx, string(store.KindSelf), habitFactWindow) + if err != nil { + log.Printf("voice: habits: recent facts: %v", err) + return "не получилось посмотреть записи.", true + } + obs := make([]memory.Observation, 0, len(facts)) + for _, f := range facts { + obs = append(obs, memory.Observation{At: f.Ts, Key: f.Key, Kind: f.Kind}) + } + profile := memory.BuildProfile(obs, h.now()) + if q.HasWeekday { + return profile.FormatWeekdayRU(q.Weekday), true + } + if q.Weekend { + return profile.FormatWeekendRU(), true + } + return profile.FormatOverallRU(), true +} + +// feedNoteWindow — how many recent FEED notes are scanned, and +// feedReadOut — how many headlines she actually reads back. She summarises the +// top of the pile, she does not recite a river. +const ( + feedNoteWindow = 200 + feedReadOut = 3 +) + +// queryFeeds — "что нового в лентах?", "что нового по технологиям?" +// (Vikunja #258). +// +// This is the ONLY way a feed item reaches him. The poller writes notes and +// never speaks; asking is the trigger. If that ever changes, the thing that +// changed is "Maven is not a nag", not a detail of this file. +func (h *reactiveHandler) queryFeeds(ctx context.Context, t *queryTurn) (string, bool) { + q, ok := router.ParseFeedQuery(t.dec.Utterance) + if !ok { + return "", false + } + if !h.feedsOn { + // Claim the turn rather than fall through: "не читаю ленты" is true, and + // letting general knowledge answer "что нового?" would be an invented + // news bulletin. + return "я пока не читаю ленты — они не настроены.", true + } + // By source, not the last 200 notes of any kind: a busy day of voice notes + // used to push the newest headline out of the window, and she answered "в + // лентах пока ничего нового" while the poller was working fine. + notes, err := h.api.RecentNotesFromSource(ctx, rss.SourcePrefix, feedNoteWindow) + if err != nil { + log.Printf("voice: feeds: recent notes: %v", err) + return "не получилось посмотреть ленты.", true + } + var picked []string + for _, n := range notes { + if !router.CategoryMatches(rss.NoteCategory(n.Text), q.Category) { + continue + } + // The note carries title, summary, category tag and link; she reads the + // title alone. The tag is for the match above, and piper reads brackets + // out loud. + picked = append(picked, rss.NoteHeadline(n.Text)) + if len(picked) == feedReadOut { + break + } + } + if len(picked) == 0 { + if q.Category != "" { + return "по этой теме в лентах пока ничего.", true + } + return "в лентах пока ничего нового.", true + } + return "вот что нового: " + strings.Join(picked, "; "), true +} + +// queryCalendar — "что у меня сегодня?", "планы на завтра?" +// h.now(), not time.Now(): the handler's clock is the injected one, so this +// source can be tested at a fixed time like the rest. +func (h *reactiveHandler) queryCalendar(ctx context.Context, t *queryTurn) (string, bool) { + date, ok := router.ParseCalendarDate(t.dec.Utterance, h.now()) + if !ok { + return "", false + } + events, err := h.api.CalendarEvents(ctx, date, date.Add(24*time.Hour)) + if err != nil { + log.Printf("voice: calendar events: %v", err) + return "не получилось проверить календарь.", true + } + // Provenance travels with each event. A work meeting relayed off a phone + // notification (source ambient:notif, #126) is stored below full confidence + // and gets hedged; a CalDAV read is recited plainly. + entries := make([]router.CalendarEntry, len(events)) + for i, e := range events { + entries[i] = router.CalendarEntry{Text: e.Value, Uncertain: e.Confidence < 1.0} + } + var f router.CalendarEventFormatter + return f.FormatEntries(entries, date), true +} + +// queryHome answers a question about the house. Read-only by construction: it +// calls States and nothing else, so there is no confirm turn here — the only +// way to CHANGE something is an enabled allowlist row through tool.Executor. +func (h *reactiveHandler) queryHome(ctx context.Context, t *queryTurn) (string, bool) { + if !isHomeQuery(t.dec.Utterance) { + return "", false + } + if h.home == nil { + // Fall through rather than claim the turn. A capability that is off + // must not change what an unconfigured box answers: "какая температура + // в доме?" on a Maven with no smarthome block reached recall before + // this source existed, and a stored fact is a better answer than + // "дом не подключён" from a house that was never configured. The + // unreachable case is different and homeSummary covers it. + return "", false + } + ctxH, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + return h.home.homeSummary(ctxH) +} + +// queryNetwork answers a question about the LAN with a bounded scan. There is +// no confirm turn because nothing is changed, and no way to widen the range +// because Scan takes no target — the utterance selects the question, never the +// subnet. +func (h *reactiveHandler) queryNetwork(ctx context.Context, t *queryTurn) (string, bool) { + if !isNetworkQuery(t.dec.Utterance) { + return "", false + } + if h.netscan == nil { + // Fall through, same as queryHome: an unconfigured scanner must not + // swallow "сколько устройств в сети?" before recall has looked. + return "", false + } + return h.netscan.scanSummary(ctx) +} + +func (h *reactiveHandler) queryWeather(ctx context.Context, t *queryTurn) (string, bool) { + if !isWeatherQuery(t.dec.Utterance) { + return "", false + } + loc := extractWeatherLocation(t.dec.Utterance, h.weatherLocation) + if loc == "" { + // He named no city and voice.weather.default_location is unset. Saying + // so is the only honest answer; picking a city would be inventing one. + return "не знаю, для какого города — задай voice.weather.default_location или назови город.", true + } + ctxWT, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + w, err := h.weatherProvider.CurrentWeather(ctxWT, loc) + if errors.Is(err, weather.ErrNotConfigured) { + return "погода не настроена.", true + } + if err != nil { + log.Printf("voice: weather: %v", err) + return "не получилось узнать погоду.", true + } + return fmt.Sprintf("в %s сейчас %.0f градусов, %s.", w.Location, w.Temperature, w.Condition), true +} + +// queryEmbed isn't an answer source — it's the shared cost the two recall +// sources below both need, run once, in the position it always ran in. It +// only claims the turn when the embedder fails. +func (h *reactiveHandler) queryEmbed(ctx context.Context, t *queryTurn) (string, bool) { + vec, err := router.EmbedQuery(ctx, h.embedder, t.dec.Utterance) + if err != nil { + log.Printf("voice: embed query: %v", err) + return "не получилось найти ответ.", true + } + t.vec = vec + return "", false +} + +// queryMemory — long-term memory first: ONE search over everything Maven +// remembers (notes and facts share this index) and ONE confidence gate, so +// the memory that is clearly the best match answers — a note just as much as +// a fact. +// +// This used to run only after the notes-only source below had already +// rejected the same note at the same score, which no note could ever survive +// a second time: the branch could only return a fact (#373). Order, not the +// gate, was the bug — the set of questions Maven answers is unchanged, only +// which memory gets to answer them. +func (h *reactiveHandler) queryMemory(ctx context.Context, t *queryTurn) (string, bool) { + if h.memStore == nil { + return "", false + } + hits, herr := h.memStore.Search(ctx, t.vec, 3) + if herr != nil { + log.Printf("voice: memory search: %v", herr) + return "", false + } + hit, ok := bestRecall(hits, h.queryMinScore, h.queryMinMargin) + if !ok { + return "", false + } + text := hit.Meta["text"] + // A note is phrased in Maven's voice; a fact is read back as it was + // stored. + if hit.Meta["type"] == "note" { + if reply, perr := h.phraser.PhraseQuery(ctx, t.dec.Utterance, []string{text}); perr == nil && reply != "" { + return reply, true + } + } + return text, true +} + +// queryNotes — notes-only pass, for notes the vector index above does not +// hold (an older note written before it existed). Same gate, notes-only +// candidates. +// +// Confidence gate: below it, say "I don't know" rather than read back the +// least-unrelated note — a confident wrong recall is worse than a gap (spec's +// "not a guesser-of-truth"). Same instinct as the loop's since(key)==null → +// don't fire. Two parts: an absolute cosine floor, and a margin over the +// runner-up, which is the part that works with the e5 embedder's narrow score +// band. See memory.Confident. Failing the gate passes the turn on to general +// knowledge, which is what "don't read back the runner-up" means here. +func (h *reactiveHandler) queryNotes(ctx context.Context, t *queryTurn) (string, bool) { + notes, err := h.api.QueryNotes(ctx, t.vec, 5) + if err != nil { + log.Printf("voice: query notes: %v", err) + return "не получилось найти ответ.", true + } + t.notes = notes + noteScores := make([]float64, len(notes)) + for i, n := range notes { + noteScores[i] = n.Score + } + if !memory.ConfidentScores(noteScores, h.queryMinScore, h.queryMinMargin) { + return "", false + } + texts := make([]string, len(notes)) + for i, n := range notes { + texts[i] = n.Text + } + reply, err := h.phraser.PhraseQuery(ctx, t.dec.Utterance, texts) + if err != nil { + log.Printf("voice: phrase query: %v", err) + } + if reply == "" { + reply = "вот что я нашла: " + texts[0] + } + return reply, true +} + +// webPageContextRunes — how much of a fetched page is handed to the phraser. +// Less than the crawler keeps: the rest of the 4096-token window belongs to the +// prompt, the persona block and the reply. +const webPageContextRunes = 1500 + +// queryWeb — "посмотри https://example.org/x — что там?" (Vikunja #259). +// +// It claims a turn ONLY when he named a URL, which is what keeps a fallback from +// becoming a habit: no URL, no fetch, and the model answers from what is local. +// What leaves the box is the URL and nothing else — no note, no fact, no history +// travels with it. +func (h *reactiveHandler) queryWeb(ctx context.Context, t *queryTurn) (string, bool) { + link, ok := router.FirstURL(t.dec.Utterance) + if !ok { + return "", false + } + if h.crawler == nil { + // Fall through. Reading pages is off unless configured, and on a daemon + // where it was never turned on the older behaviour is right: the model + // answers the question as if the URL had not been said. Announcing a + // configuration status is for a capability that exists and failed, not + // for one he never asked for. + return "", false + } + ctxFetch, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + page, err := h.crawler.Page(ctxFetch, link) + if err != nil { + if errors.Is(err, crawl.ErrRobots) { + return "эта страница закрыта для чтения — robots.txt не разрешает.", true + } + log.Printf("voice: web: %v", err) + return "не получилось прочитать страницу.", true + } + if page.Text == "" { + return "страница открылась, но читать там нечего.", true + } + // The page is handed to the phraser the same way a note is: as context for + // the question he actually asked. She answers the question, she does not + // recite the page. + snippet := page.Title + "\n" + crawl.TrimRunes(page.Text, webPageContextRunes) + reply, perr := h.phraser.PhraseQuery(ctx, t.dec.Utterance, []string{snippet}) + if perr != nil { + log.Printf("voice: web: phrase: %v", perr) + } + if reply == "" { + // No phraser (or it failed): read back the top of the page rather than + // pretend the fetch did not happen. + return "вот что на странице: " + crawl.TrimRunes(page.Text, 300), true + } + return reply, true +} + +// queryGeneral — general knowledge from the phraser, the last source before +// giving up. It always claims: either the model answers or Maven says she +// doesn't know. +func (h *reactiveHandler) queryGeneral(ctx context.Context, t *queryTurn) (string, bool) { + reply, err := h.phraser.PhraseQuery(ctx, t.dec.Utterance, nil) + if err != nil || reply == "" { + return "не знаю.", true + } + return reply, true +} diff --git a/cmd/mavend/actions_reminder.go b/cmd/mavend/actions_reminder.go new file mode 100644 index 0000000..ce2a632 --- /dev/null +++ b/cmd/mavend/actions_reminder.go @@ -0,0 +1,33 @@ +package main + +import ( + "context" + "log" + + "github.com/kami/maven/internal/router" +) + +// actionReminder handles router.IntentReminder: parse the time when stage-0 +// skipped the extractor, then create the reminder. +func (h *reactiveHandler) actionReminder(ctx context.Context, dec router.Decision) string { + if !dec.Slots.HasTime { + // Stage-0 (reminder-wakeword grammar) skips the extractor, so the + // time wasn't parsed. Run the parser as a fallback. + if dec.Stage == 0 && h.timeParser != nil { + t, ok, err := h.timeParser.Parse(ctx, dec.Utterance, h.now()) + if err == nil && ok { + dec.Slots.Time = t + dec.Slots.HasTime = true + } + } + if !dec.Slots.HasTime { + return "не получилось разобрать время напоминания." + } + } + payload := `{"text":` + jsonString(dec.Utterance) + `}` + if _, err := h.api.CreateReminder(ctx, dec.Slots.Time, payload, ""); err != nil { + log.Printf("voice: create reminder: %v", err) + return "не получилось поставить напоминание." + } + return "" +} diff --git a/cmd/mavend/actions_task.go b/cmd/mavend/actions_task.go new file mode 100644 index 0000000..99187f0 --- /dev/null +++ b/cmd/mavend/actions_task.go @@ -0,0 +1,89 @@ +package main + +import ( + "context" + "log" + + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/router" + "github.com/kami/maven/internal/store" + "github.com/kami/maven/internal/tasks" +) + +// Task capture on the voice/chat path (Vikunja #130). +// +// Two halves, both deliberately small: +// +// - captureTaskFromNote runs at the top of actionNote. An utterance that +// explicitly files a task ("добавь в задачи купить молоко") goes to the task +// store instead of the note store. Anything without an explicit marker is +// still a note — see router.ParseTaskCapture for why "надо бы поспать" must +// not become a task. +// - queryTasks is a query source that reads the list back. +// +// Nothing here speaks unprompted. Tasks are answered when asked about; no tick +// rule reads the table. + +// captureTaskFromNote claims the turn when the utterance explicitly files a +// task, returning the reply. ("", false) hands the turn back to the note path. +func (h *reactiveHandler) captureTaskFromNote(ctx context.Context, dec router.Decision) (string, bool) { + cap, ok := router.ParseTaskCapture(dec.Utterance) + if !ok { + return "", false + } + resp, err := h.api.CaptureTask(ctx, ipc.CaptureTaskReq{ + Text: cap.Text, + Source: "tap:voice", + Status: store.TaskOpen, // he stated it himself — not a candidate + Weight: cap.Weight, // 0 unless he said "срочно" / "важно" + Ts: h.now(), + }) + if err != nil { + log.Printf("voice: capture task: %v", err) + return "не получилось записать задачу.", true + } + if resp.Promoted { + // It was a candidate Maven derived from something she read, and he has + // now said it himself. Saying "уже в списке" here would be answering a + // confirmation with a shrug. + return "поняла, беру в работу: " + cap.Text, true + } + if !resp.Created { + return "это уже в списке.", true + } + return "записала: " + cap.Text, true +} + +// queryTasks — "какие у меня задачи?", "что мне нужно сделать?". +// +// Reads the live set and recites it in priority order (Vikunja #129). The order +// is computed by internal/tasks from what he told her — deadlines, the urgency +// he stated, how long a task has been sitting — never asked of the model. The +// rendering is the package's too, so the spoken list and the /tasks page can +// never disagree about what comes first. +func (h *reactiveHandler) queryTasks(ctx context.Context, t *queryTurn) (string, bool) { + if !router.IsTaskListQuery(t.dec.Utterance) { + return "", false + } + live, err := h.api.ListTasks(ctx, "live") + if err != nil { + log.Printf("voice: list tasks: %v", err) + return "не получилось посмотреть задачи.", true + } + return tasks.FormatRU(tasks.Rank(taskItems(live), h.now())), true +} + +// taskItems maps wire rows onto the ranker's input. Written here rather than in +// internal/tasks so the ranker stays a pure package with no ipc (and therefore +// no store, and therefore no cgo) dependency — the same posture as +// internal/morning and internal/memory. +func taskItems(ts []ipc.Task) []tasks.Item { + out := make([]tasks.Item, len(ts)) + for i, t := range ts { + out[i] = tasks.Item{ + ID: t.ID, Text: t.Text, Status: t.Status, + Created: t.CreatedTs, Due: t.Due, Weight: t.Weight, + } + } + return out +} diff --git a/cmd/mavend/actions_task_test.go b/cmd/mavend/actions_task_test.go new file mode 100644 index 0000000..628b30d --- /dev/null +++ b/cmd/mavend/actions_task_test.go @@ -0,0 +1,254 @@ +package main + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/router" +) + +// taskAPI answers only the three task methods; every other call is +// unimplemented, which is the assertion that capture needs nothing else — in +// particular no embedder, so a filed task costs no model call. +type taskAPI struct { + ipc.UnimplementedCoreAPI + + captured []ipc.CaptureTaskReq + created bool + promoted bool + capErr error + + tasks []ipc.Task + listArg string + listErr error +} + +func (a *taskAPI) CaptureTask(_ context.Context, req ipc.CaptureTaskReq) (ipc.CaptureTaskResp, error) { + a.captured = append(a.captured, req) + if a.capErr != nil { + return ipc.CaptureTaskResp{}, a.capErr + } + return ipc.CaptureTaskResp{ID: 1, Created: a.created, Promoted: a.promoted}, nil +} + +func (a *taskAPI) ListTasks(_ context.Context, status string) ([]ipc.Task, error) { + a.listArg = status + return a.tasks, a.listErr +} + +func taskNow() time.Time { return time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC) } + +func taskHandler(api ipc.CoreAPI) *reactiveHandler { + return &reactiveHandler{api: api, now: taskNow} +} + +func TestCaptureTaskFromNoteFilesTheTask(t *testing.T) { + api := &taskAPI{created: true} + h := taskHandler(api) + reply, ok := h.captureTaskFromNote(context.Background(), router.Decision{ + Intent: router.IntentNote, Utterance: "добавь в задачи купить молоко", + }) + if !ok { + t.Fatal("an explicit capture must claim the turn") + } + if len(api.captured) != 1 { + t.Fatalf("captured %d, want 1", len(api.captured)) + } + got := api.captured[0] + if got.Text != "купить молоко" { + t.Errorf("text = %q, want the marker stripped", got.Text) + } + if got.Source != "tap:voice" { + t.Errorf("source = %q, want tap:voice", got.Source) + } + if got.Status != "open" { + t.Errorf("status = %q — work he stated is open, never a candidate", got.Status) + } + if !got.Ts.Equal(taskNow()) { + t.Errorf("ts = %v, want the handler clock", got.Ts) + } + if !strings.Contains(reply, "купить молоко") { + t.Errorf("reply = %q, want it to read the task back", reply) + } +} + +// A note is still a note: capture only fires on an explicit marker, so +// ordinary recall is untouched. +func TestCaptureTaskFromNotePassesOrdinaryNotes(t *testing.T) { + api := &taskAPI{} + h := taskHandler(api) + for _, u := range []string{"надо бы поспать", "мне понравился этот фильм", "запиши что я пил воду"} { + if _, ok := h.captureTaskFromNote(context.Background(), router.Decision{Utterance: u}); ok { + t.Errorf("%q was captured as a task", u) + } + } + if len(api.captured) != 0 { + t.Errorf("captured %d requests, want none", len(api.captured)) + } +} + +func TestCaptureTaskFromNoteSaysAlreadyOnTheList(t *testing.T) { + h := taskHandler(&taskAPI{created: false}) + reply, ok := h.captureTaskFromNote(context.Background(), router.Decision{Utterance: "добавь в задачи купить молоко"}) + if !ok { + t.Fatal("expected the capture path to claim it") + } + if !strings.Contains(reply, "уже") { + t.Errorf("reply = %q — a deduped capture must not claim it saved something new", reply) + } +} + +func TestCaptureTaskFromNoteReportsStoreFailure(t *testing.T) { + h := taskHandler(&taskAPI{capErr: errors.New("db is on fire")}) + reply, ok := h.captureTaskFromNote(context.Background(), router.Decision{Utterance: "добавь задачу починить кран"}) + if !ok { + t.Fatal("a failed capture still claims the turn — the note path must not double-write") + } + if !strings.Contains(reply, "не получилось") { + t.Errorf("reply = %q, want an honest failure", reply) + } +} + +func TestQueryTasksRecitesTheLiveList(t *testing.T) { + api := &taskAPI{tasks: []ipc.Task{ + {ID: 1, Text: "купить молоко", Status: "open"}, + {ID: 2, Text: "продлить страховку", Status: "candidate"}, + }} + h := taskHandler(api) + reply, ok := h.queryTasks(context.Background(), &queryTurn{ + dec: router.Decision{Intent: router.IntentQuery, Utterance: "какие у меня задачи?"}, + }) + if !ok { + t.Fatal("the task source must claim a task-list question") + } + if api.listArg != "live" { + t.Errorf("ListTasks(%q), want \"live\" — a resolved task is not outstanding work", api.listArg) + } + if !strings.Contains(reply, "купить молоко") || !strings.Contains(reply, "продлить страховку") { + t.Errorf("reply = %q, want both tasks", reply) + } + // The candidate must be named as unconfirmed, not recited as his work. + openIdx := strings.Index(reply, "купить молоко") + candIdx := strings.Index(reply, "продлить страховку") + if !(openIdx < candIdx) { + t.Errorf("reply = %q, want confirmed work before candidates", reply) + } + if !strings.Contains(reply, "не подтвердил") { + t.Errorf("reply = %q, want the candidate flagged as unconfirmed", reply) + } +} + +// The stated urgency rides through capture as a weight, so the ranker can use +// it later (Vikunja #129). "срочно" is not part of the task text. +func TestCaptureTaskCarriesStatedUrgency(t *testing.T) { + api := &taskAPI{created: true} + h := taskHandler(api) + if _, ok := h.captureTaskFromNote(context.Background(), router.Decision{ + Utterance: "добавь в задачи срочно оплатить интернет", + }); !ok { + t.Fatal("expected a capture") + } + got := api.captured[0] + if got.Text != "оплатить интернет" { + t.Errorf("text = %q, want the urgency word out of the task", got.Text) + } + if got.Weight == 0 { + t.Error("weight = 0 — he said срочно and it was dropped") + } +} + +// The recital is ordered by the ranker, not by insertion: a deadline he named +// comes before undated work. +func TestQueryTasksRecitesInPriorityOrder(t *testing.T) { + due := taskNow() + api := &taskAPI{tasks: []ipc.Task{ + {ID: 1, Text: "купить молоко", Status: "open", CreatedTs: taskNow()}, + {ID: 2, Text: "оплатить интернет", Status: "open", CreatedTs: taskNow(), Due: &due}, + }} + h := taskHandler(api) + reply, _ := h.queryTasks(context.Background(), &queryTurn{ + dec: router.Decision{Utterance: "какие у меня задачи?"}, + }) + if strings.Index(reply, "оплатить интернет") > strings.Index(reply, "купить молоко") { + t.Errorf("reply = %q, want the dated task first", reply) + } + if !strings.Contains(reply, "сегодня") { + t.Errorf("reply = %q, want the reason named", reply) + } +} + +func TestQueryTasksEmptyList(t *testing.T) { + h := taskHandler(&taskAPI{}) + reply, ok := h.queryTasks(context.Background(), &queryTurn{ + dec: router.Decision{Utterance: "что мне нужно сделать?"}, + }) + if !ok { + t.Fatal("expected the task source to claim it") + } + if reply != "задач нет." { + t.Errorf("reply = %q", reply) + } +} + +func TestQueryTasksPassesOtherQuestions(t *testing.T) { + api := &taskAPI{} + h := taskHandler(api) + for _, u := range []string{"как дела?", "какая погода в москве?", "что у меня сегодня?"} { + if _, ok := h.queryTasks(context.Background(), &queryTurn{dec: router.Decision{Utterance: u}}); ok { + t.Errorf("the task source claimed %q", u) + } + } + if api.listArg != "" { + t.Error("a non-task question must not read the task list") + } +} + +// The chain must reach the task source before the recall sources, or "что мне +// нужно сделать?" gets answered by whatever note is nearest. +func TestQuerySourcesOrderTasksBeforeRecall(t *testing.T) { + var tasksAt, notesAt = -1, -1 + for i, src := range querySources { + switch src.name { + case "tasks": + tasksAt = i + case "notes": + notesAt = i + } + } + if tasksAt < 0 || notesAt < 0 { + t.Fatalf("sources missing: tasks=%d notes=%d", tasksAt, notesAt) + } + if tasksAt > notesAt { + t.Errorf("tasks source at %d, after notes at %d", tasksAt, notesAt) + } +} + +// Saying a task out loud that Maven had only proposed is a confirmation. She +// used to answer "это уже в списке" and then read it back, in the same +// conversation, as something he had not confirmed. +func TestCaptureTaskFromNoteAcknowledgesAPromotion(t *testing.T) { + api := &taskAPI{promoted: true} + h := taskHandler(api) + reply, ok := h.captureTaskFromNote(context.Background(), router.Decision{ + Intent: router.IntentNote, Utterance: "добавь в задачи продлить страховку", + }) + if !ok { + t.Fatal("an explicit capture must claim the turn") + } + if strings.Contains(reply, "уже в списке") { + t.Errorf("reply = %q — he just confirmed it, that is not a duplicate", reply) + } + if !strings.Contains(reply, "продлить страховку") { + t.Errorf("reply = %q, want the task named back", reply) + } + // Persona: feminine, informal. + for _, bad := range []string{"рад ", "вы ", "ваш"} { + if strings.Contains(reply, bad) { + t.Errorf("reply %q contains %q", reply, bad) + } + } +} diff --git a/cmd/mavend/capture.go b/cmd/mavend/capture.go new file mode 100644 index 0000000..2393976 --- /dev/null +++ b/cmd/mavend/capture.go @@ -0,0 +1,313 @@ +// mavend/capture.go — core's half of the meeting recorder (Vikunja #253, +// docs/plans/08-hearing.md). +// +// The split: a client that has a microphone (mavenclient, or a phone on the PWA) +// is told to start, streams frames over ipc.MethodCaptureAppend, and is told to +// stop. Core keeps the PCM, stores it as a WAV blob under the same media store +// and the same retention as images, transcribes it through the ONE STT Maven has +// (mavsttd's whisper.cpp, reused — not a second engine), and summarises the +// transcript on the resident model in windows that fit n_ctx 4096. +// +// # Off unless configured, twice over +// +// No `media` block ⇒ nowhere to keep audio ⇒ the four capture methods do not +// exist. No `capture` block with enabled ⇒ they still do not exist. On an +// unconfigured box there is no wire path that starts a recording, which is the +// only guarantee worth making about a capability like this one. +// +// # What this file refuses to do +// +// - Nothing listens. There is no VAD hook here, no wake-word branch, no +// "start when you hear a meeting". The plan document's keyword-triggered +// recorder is refused in internal/capture's package comment for the reason +// that applies here too: noticing a keyword requires listening, which is +// the behaviour this capability must not have. +// - No transcript note by default. The summary is written where he will read +// it; the verbatim record of what other people said takes a deliberate +// capture.save_transcript. +// - The transcript is never search input beyond this box, and the audio never +// leaves it at all. +package main + +import ( + "context" + "errors" + "fmt" + "log" + "sync" + "time" + + "github.com/kami/maven/internal/capture" + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/llm" + "github.com/kami/maven/internal/phraser" + "github.com/kami/maven/internal/router" + "github.com/kami/maven/internal/store" +) + +// captureSummaryTimeout — the budget for one summary, which is a map-reduce over +// the whole meeting: one model call per transcript window plus a reduce, each of +// which is seconds on this box. Forty windows is the configured ceiling, so the +// budget has to be minutes, not the 60s the reply path uses. It is spent on a +// background goroutine, never inside the capture_stop request: a client that +// asks Maven to stop recording gets the transcript back in seconds. +const captureSummaryTimeout = 20 * time.Minute + +// llmCompleter adapts *llm.Client to capture.Completer. The pure package names +// the two strings it needs and stays free of the llm request struct; the client +// itself is the swap-aware one from llmClientFor, so a model swap re-points it. +type llmCompleter struct { + c *llm.Client + maxTokens int +} + +func (l llmCompleter) Complete(ctx context.Context, system, user string) (string, error) { + return l.c.Complete(ctx, llm.Req{System: system, User: user, MaxTokens: l.maxTokens}) +} + +// captureWiring — the recorder plus what it needs to write the result down. +type captureWiring struct { + rec *capture.Recorder + st *store.Store + emb router.Embedder + cfg *config.CaptureConfig + now func() time.Time + + // ctx and wg belong to the daemon, not to the request. Summarising happens + // after the reply has gone out, so it needs a lifetime that outlives the + // call and a shutdown that waits for it. + ctx context.Context + wg *sync.WaitGroup +} + +// newCaptureWiring returns nil when the recorder should not exist: no media +// store, no capture block, capture disabled, or no STT to transcribe with. +// +// A missing llama-server is NOT a reason to return nil. Without one the +// recording is still made, stored and transcribed, and the summary is simply +// absent — the honest degradation, and much better than refusing to record a +// meeting that is happening now. +func newCaptureWiring(ctx context.Context, wg *sync.WaitGroup, keeper *mediaKeeper, st *store.Store, voiceW *voiceWiring, phr phraser.Phraser, emb router.Embedder, cfg *config.Config) *captureWiring { + if keeper == nil || !cfg.Capture.Records() { + return nil + } + tr := transcriberOf(voiceW) + if tr == nil { + // Voice off ⇒ no STT client ⇒ nothing could turn the audio into words. + // Storing hours of unreadable audio of other people is worse than not + // recording, so this is a refusal, not a degradation. + log.Printf("capture: enabled but voice/stt is not wired — meeting capture disabled") + return nil + } + + cc := cfg.Capture + var sum *capture.Summarizer + if lp, ok := phr.(*phraser.LLMPhraser); ok { + client := llmClientFor(lp, captureSummaryTimeout) + sum = capture.NewSummarizer( + llmCompleter{c: client, maxTokens: 512}, + cc.ChunkRunes, cc.MaxChunks, contextBlockFn(cfg, time.Now), + ) + } else { + log.Printf("capture: no llama-server phraser — meetings are transcribed, not summarised") + } + + rec, err := capture.New(keeper.store, tr, sum, capture.Config{ + MaxDuration: cc.MaxDuration(), + STTWindow: time.Duration(cc.STTWindow), + }) + if err != nil { + log.Printf("capture: %v — meeting capture disabled", err) + return nil + } + log.Printf("capture: enabled, sessions capped at %s", rec.MaxDuration()) + return &captureWiring{rec: rec, st: st, emb: emb, cfg: cc, now: time.Now, ctx: ctx, wg: wg} +} + +// start handles ipc.MethodCaptureStart. +func (c *captureWiring) start(_ context.Context, req ipc.CaptureStartReq) (ipc.CaptureStartResp, error) { + s, err := c.rec.Start(req.Label) + if err != nil { + return ipc.CaptureStartResp{}, err + } + // The label is logged; nothing that was said ever is. + log.Printf("capture: started %q", s.Label) + return ipc.CaptureStartResp{ + Label: s.Label, + Started: s.Started, + Token: s.Token, + MaxSeconds: int(c.rec.MaxDuration().Seconds()), + }, nil +} + +// append handles ipc.MethodCaptureAppend. ErrExpired is reported as a successful +// response with Expired set rather than an error: the cap firing is the designed +// behaviour, and the client needs the flag to stop sending and call stop. +func (c *captureWiring) append(_ context.Context, req ipc.CaptureAppendReq) (ipc.CaptureAppendResp, error) { + err := c.rec.Append(req.Token, req.Audio) + st := c.rec.Status() + if errors.Is(err, capture.ErrExpired) { + log.Printf("capture: %q hit the %s cap — stopping", st.Label, c.rec.MaxDuration()) + return ipc.CaptureAppendResp{Seconds: st.Duration.Seconds(), Expired: true}, nil + } + if err != nil { + return ipc.CaptureAppendResp{}, err + } + return ipc.CaptureAppendResp{Seconds: st.Duration.Seconds()}, nil +} + +// stop handles ipc.MethodCaptureStop. +// +// The error handling here mirrors vision's, and for the same reason: the audio is +// stored first, so a transcription failure returns what exists rather than +// nothing. A response can carry a blob id with no transcript (STT failed, +// re-runnable) — a degraded success, not an error to the caller. +// +// Summarising is NOT done here. A two-hour meeting is forty model calls, which +// on this box is minutes, and holding the IPC request open for them means the +// client that said "стоп" sits there with no answer while its own deadline runs +// out. Stop returns the transcript, and the summary note is written by a +// goroutine in the daemon's WaitGroup afterwards. +func (c *captureWiring) stop(ctx context.Context, req ipc.CaptureStopReq) (ipc.CaptureStopResp, error) { + if req.Discard { + // "забудь, не записывай" — nothing is stored, transcribed or noted. + if !c.rec.Abort(req.Token) { + return ipc.CaptureStopResp{}, capture.ErrNoSession + } + log.Printf("capture: session discarded on request") + return ipc.CaptureStopResp{Discarded: true}, nil + } + + res, err := c.rec.Stop(ctx, req.Token) + resp := ipc.CaptureStopResp{ + BlobID: res.BlobID, + Label: res.Label, + Started: res.Started, + Seconds: res.Duration.Seconds(), + Transcript: res.Transcript, + Summary: res.Summary, + Chunks: res.Chunks, + } + if err != nil { + if res.BlobID == "" && res.Transcript == "" { + // Nothing survived: no session, or an empty recording. There is + // nothing to hand back, so this is a real error. + return ipc.CaptureStopResp{}, err + } + log.Printf("capture: %q partially finished: %v", res.Label, err) + } + + c.summarizeLater(res) + log.Printf("capture: finished %q — %s of audio, %d bytes of transcript", + res.Label, res.Duration.Round(time.Second), len(res.Transcript)) + return resp, nil +} + +// summarizeLater runs the map-reduce and writes the notes after stop replied. +// The context is the daemon's, not the request's: the request is already +// answered, and cancelling the summary because the client hung up would throw +// away the only readable record of the meeting. +func (c *captureWiring) summarizeLater(res capture.Result) { + if res.Transcript == "" { + return + } + c.wg.Add(1) + go func() { + defer c.wg.Done() + ctx, cancel := context.WithTimeout(c.ctx, captureSummaryTimeout) + defer cancel() + if err := c.rec.Summarize(ctx, &res); err != nil { + // Not fatal: writeNotes falls back to the transcript, so a dead + // llama-server costs the summary and not the meeting. + log.Printf("capture: summary for %q failed: %v", res.Label, err) + } + if _, err := c.writeNotes(ctx, res); err != nil { + log.Printf("capture: note write for %q failed: %v", res.Label, err) + return + } + log.Printf("capture: summarised %q in %d chunk(s)", res.Label, res.Chunks) + }() +} + +// writeNotes stores the summary as a note, and the transcript too when +// capture.save_transcript is set. Returns the id of the note that carries the +// meeting. +// +// With no summary the transcript is written instead, whatever save_transcript +// says. That flag is about keeping the verbatim record IN ADDITION to a summary, +// not about whether the meeting is remembered at all. Without this fallback a +// llama-server that was down at stop time meant an hour of recorded meeting left +// no note behind and nothing recalled it later. +// +// The note source carries the blob id, which is the only link back to the audio. +// When retention prunes the blob the note remains — words about a meeting are a +// far lighter thing to keep than a recording of it. +func (c *captureWiring) writeNotes(ctx context.Context, res capture.Result) (int64, error) { + source := "capture:meeting" + if res.BlobID != "" { + source = "capture:meeting:" + res.BlobID[:12] + } + var id int64 + if text := res.Summary; text != "" { + var err error + id, err = c.writeNote(ctx, text, source) + if err != nil { + return 0, fmt.Errorf("summary note: %w", err) + } + } else if res.Transcript != "" { + var err error + id, err = c.writeNote(ctx, res.Transcript, source+":transcript") + if err != nil { + return 0, fmt.Errorf("transcript note: %w", err) + } + return id, nil + } + if c.cfg.SaveTranscript && res.Transcript != "" { + if _, err := c.writeNote(ctx, res.Transcript, source+":transcript"); err != nil { + return id, fmt.Errorf("transcript note: %w", err) + } + } + return id, nil +} + +func (c *captureWiring) writeNote(ctx context.Context, text, source string) (int64, error) { + var vec []float32 + if c.emb != nil { + // EmbedPassage, not Embed: this is text being searched FOR, and the e5 + // embedder is asymmetric. Backwards here makes the meeting unfindable by + // the question that should have matched it. + var err error + vec, err = router.EmbedPassage(ctx, c.emb, text) + if err != nil { + return 0, fmt.Errorf("embed: %w", err) + } + } + return c.st.WriteNote(ctx, c.now(), text, vec, source) +} + +// status handles ipc.MethodCaptureStatus. +func (c *captureWiring) status(_ context.Context) (ipc.CaptureStatusResp, error) { + st := c.rec.Status() + return ipc.CaptureStatusResp{ + Running: st.Running, + Label: st.Label, + Started: st.Started, + Seconds: st.Duration.Seconds(), + Bytes: st.Bytes, + }, nil +} + +// wireCapture installs the four IPC hooks, or leaves them nil so every capture +// method reports ErrUnknownMethod. Takes the media keeper wireVision already +// opened: one blob store, one retention loop, images and audio side by side. +func wireCapture(ctx context.Context, wg *sync.WaitGroup, srv *ipc.Server, keeper *mediaKeeper, st *store.Store, voiceW *voiceWiring, phr phraser.Phraser, cfg *config.Config) { + cw := newCaptureWiring(ctx, wg, keeper, st, voiceW, phr, embedderOf(voiceW), cfg) + if cw == nil { + return + } + srv.CaptureStartFn = cw.start + srv.CaptureAppendFn = cw.append + srv.CaptureStopFn = cw.stop + srv.CaptureStatusFn = cw.status +} diff --git a/cmd/mavend/capture_test.go b/cmd/mavend/capture_test.go new file mode 100644 index 0000000..e09bdad --- /dev/null +++ b/cmd/mavend/capture_test.go @@ -0,0 +1,118 @@ +package main + +import ( + "context" + "strings" + "sync" + "testing" + "time" + + "github.com/kami/maven/internal/audio" + "github.com/kami/maven/internal/capture" + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/media" +) + +// silentTranscriber stands in for mavsttd: one fixed phrase per window, so the +// wiring can be tested without whisper. +type silentTranscriber struct{} + +func (silentTranscriber) Transcribe(_ context.Context, _ audio.Audio) (string, float64, error) { + return "решили купить насос", 1.0, nil +} + +func testCaptureWiring(t *testing.T) (*captureWiring, *sync.WaitGroup) { + t.Helper() + blobs, err := media.Open(t.TempDir(), 0, 0) + if err != nil { + t.Fatal(err) + } + rec, err := capture.New(blobs, silentTranscriber{}, nil, capture.Config{}) + if err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + return &captureWiring{ + rec: rec, + st: newTestStore(t), + cfg: &config.CaptureConfig{}, + now: time.Now, + ctx: context.Background(), + wg: &wg, + }, &wg +} + +// A frame carrying the wrong token must not land in the running session. Append +// and stop used to address "whatever is running now", so a client whose session +// had already ended went on recording into somebody else's meeting, and any +// client could end a recording it never started. +func TestCaptureRefusesAnotherClientsToken(t *testing.T) { + c, _ := testCaptureWiring(t) + start, err := c.start(context.Background(), ipc.CaptureStartReq{Label: "встреча"}) + if err != nil { + t.Fatal(err) + } + if start.Token == "" { + t.Fatal("start handed back no session token") + } + if _, err := c.append(context.Background(), ipc.CaptureAppendReq{ + Token: "not-mine", + Audio: audio.Audio{Format: audio.PCM16kMono, Bytes: make([]byte, 3200)}, + }); err == nil { + t.Error("a frame with the wrong token was accepted") + } + if _, err := c.stop(context.Background(), ipc.CaptureStopReq{Token: "not-mine"}); err == nil { + t.Error("a stop with the wrong token ended the session") + } + if st, _ := c.status(context.Background()); !st.Running { + t.Error("the session was ended by a client that does not own it") + } +} + +// Stop answers with the transcript and does not wait for the summary. The +// summary is up to forty model calls, and holding the IPC request for them meant +// the client that said "стоп" sat with no answer for minutes. +// +// With no summariser wired the note still has to be written, from the transcript. +// save_transcript is about keeping the verbatim record IN ADDITION to a summary, +// not about whether the meeting is remembered at all — without this fallback a +// dead llama-server meant an hour of meeting left no note behind. +func TestStopReturnsTranscriptAndNotesItWithoutASummary(t *testing.T) { + c, wg := testCaptureWiring(t) + start, err := c.start(context.Background(), ipc.CaptureStartReq{Label: "планёрка"}) + if err != nil { + t.Fatal(err) + } + if _, err := c.append(context.Background(), ipc.CaptureAppendReq{ + Token: start.Token, + Audio: audio.Audio{Format: audio.PCM16kMono, Bytes: make([]byte, 32000)}, + }); err != nil { + t.Fatal(err) + } + resp, err := c.stop(context.Background(), ipc.CaptureStopReq{Token: start.Token}) + if err != nil { + t.Fatalf("stop: %v", err) + } + if resp.Transcript == "" { + t.Fatal("stop returned no transcript") + } + if resp.Summary != "" { + t.Errorf("summary = %q, want none inside the request", resp.Summary) + } + wg.Wait() + + notes, err := c.st.RecentNotes(context.Background(), 10) + if err != nil { + t.Fatal(err) + } + var found bool + for _, n := range notes { + if strings.Contains(n.Text, "насос") { + found = true + } + } + if !found { + t.Fatalf("the meeting left no note behind: %+v", notes) + } +} diff --git a/cmd/mavend/clarify.go b/cmd/mavend/clarify.go index 5e78d6c..75d6691 100644 --- a/cmd/mavend/clarify.go +++ b/cmd/mavend/clarify.go @@ -17,7 +17,8 @@ import ( const clarifyTTL = 90 * time.Second // wantedSlots — what each intent needs before she can act on it. First entry is -// the one she asks about; the rest are only used to decide act-vs-drop. +// the one she asks about this turn; the rest are asked about on later turns, one +// per turn, as each answer lands (see askRemainingGap). // // Intents not listed here are never worth a question: note and query act on the // raw utterance, chat and system have nothing to fill in. For those a clarify @@ -25,8 +26,7 @@ const clarifyTTL = 90 * time.Second // is worse than admitting she missed it. // A reminder wants BOTH what to remind about and when. Subject first: "напомни // в 11" has a time and nothing to say at 11, and a reminder with no subject is -// not worth setting. Order here is the order she asks in — she still only asks -// about the first one missing. +// not worth setting. Order here is the order she asks in. var wantedSlots = map[router.Intent][]dialogue.Slot{ router.IntentReminder: {dialogue.SlotText, dialogue.SlotTime}, router.IntentFact: {dialogue.SlotKey}, @@ -140,7 +140,8 @@ func missingFor(dec router.Decision) []dialogue.Slot { // ("", false) when she has no idea what is missing. // // One question about one thing: if two slots are missing she asks about the -// first and lets the rest go. Two questions in a row is an interrogation. +// first only. Two questions in one breath is an interrogation. The second gap +// is picked up on the turn after the first one is answered (askRemainingGap). func clarifyQuestion(dec router.Decision) (dialogue.Slot, string, bool) { missing := missingFor(dec) if len(missing) == 0 { @@ -199,11 +200,27 @@ func (h *reactiveHandler) resolveClarifyAnswer(ctx context.Context, text string) intent := router.Intent(q.Intent) answer := h.extractor.Extract(ctx, intent, text, h.now()) merged := q.Answer(text, toDialogueSlots(answer)) + // Fold a newly answered subject into the raw utterance. Downstream actions + // phrase from Utterance, not from the text slot — actionReminder stores it + // as the reminder payload — so a reminder clarified out of a bare "напомни" + // would fire at 11:00 saying "напомни" and nothing else. + q.Utterance = foldAnswerIntoUtterance(q.Utterance, merged.Text) if len(dialogue.StillMissing(q.Missing, merged)) > 0 { return h.reaskOrGiveUp(q, merged, text), true } h.clarifyStore.Delete(voiceDialogueID) + // One gap filled is not the same as a complete request. askClarify parks + // only the first gap, because one question per turn is the rule, but a + // reminder wants both a subject and a time. "напомни" with neither used to + // ask "О чём напомнить?", accept "позвонить маме", and then hand applyAction + // a reminder with no time, which answered "не получилось разобрать время + // напоминания." — an error for a request she never finished asking about. + // Re-enter the loop instead, one question at a time as before. + if reply, asked := h.askRemainingGap(q, intent, merged); asked { + return reply, true + } + // Rebuild the decision as if it had routed cleanly, then run it down the // normal path. Clarify is deliberately false and the intent is unchanged: // filling in an argument never grants authority, so the completed decision @@ -218,6 +235,55 @@ func (h *reactiveHandler) resolveClarifyAnswer(ctx context.Context, text string) return h.finishClarified(ctx, dec), true } +// foldAnswerIntoUtterance appends an answered subject to the original words, +// unless they already carry it. "напомни" + "позвонить маме" reads as the +// request he would have made in one breath. Nothing is appended when the +// subject is empty or already present, so re-asking the same question twice +// cannot grow the utterance. +func foldAnswerIntoUtterance(utterance, subject string) string { + subject = strings.TrimSpace(subject) + if subject == "" || strings.Contains(utterance, subject) { + return utterance + } + if strings.TrimSpace(utterance) == "" { + return subject + } + return strings.TrimSpace(utterance) + " " + subject +} + +// askRemainingGap re-parks the request when the answer closed one gap and +// wantedSlots still names another. Returns ("", false) when the request is +// complete, when there is no question for what is left, or when she is out of +// attempts — in all three the caller runs the decision as it stands, which for +// the out-of-attempts case is the old behaviour and is the right one: she has +// already asked enough. +// +// The attempt budget is shared with the re-ask path on purpose. A second gap +// costs a question exactly like a second try at the first one does, so the cap +// still bounds how many times she can speak before acting or letting go. +func (h *reactiveHandler) askRemainingGap(q *dialogue.PendingQuestion, intent router.Intent, merged dialogue.Slots) (string, bool) { + remaining := dialogue.StillMissing(wantedSlots[intent], merged) + if len(remaining) == 0 { + return "", false + } + question, ok := clarifyQuestions[remaining[0]] + if !ok || !q.CanAsk() { + return "", false + } + h.clarifyStore.Put(voiceDialogueID, &dialogue.PendingQuestion{ + Intent: q.Intent, + Slots: merged, + Missing: []dialogue.Slot{remaining[0]}, + Utterance: q.Utterance, + Asked: h.now(), + TTL: clarifyTTL, + Attempts: q.Attempts + 1, + MaxAttempts: q.MaxAttempts, + }) + log.Printf("voice: clarify — one gap filled, still missing %s for intent=%s, asking again (attempt %d)", remaining[0], intent, q.Attempts+1) + return question, true +} + // reaskOrGiveUp handles an answer that left the gap open: ask the same question // again while she has attempts left, otherwise say she did not understand and // let the request go. Never returns "" — a mute give-up reads as "done". diff --git a/cmd/mavend/clarify_test.go b/cmd/mavend/clarify_test.go index e280e32..e07298a 100644 --- a/cmd/mavend/clarify_test.go +++ b/cmd/mavend/clarify_test.go @@ -10,6 +10,7 @@ import ( "github.com/kami/maven/internal/dialogue" "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/phraser/eval" "github.com/kami/maven/internal/router" "github.com/kami/maven/internal/store" "github.com/kami/maven/internal/tool" @@ -330,3 +331,137 @@ func TestNoPendingQuestionFallsThrough(t *testing.T) { t.Fatalf("no open question ⇒ must not be treated as an answer, got %q", reply) } } + +// TestClarifyAsksAboutTheSecondGapToo — "напомни" with neither a subject nor a +// time. She asks about the subject, he gives it, and the request is still not +// complete. The old code handed applyAction a reminder with no time, which +// answered with a parse error for a question she never asked. +func TestClarifyAsksAboutTheSecondGapToo(t *testing.T) { + ctx := context.Background() + h, st, _ := newClarifyHandler(t) + + question, asked := h.askClarify(clarifyDec(router.IntentReminder, router.Slots{}, "напомни")) + if !asked || question != "О чём напомнить?" { + t.Fatalf("expected the subject question, got %q asked=%v", question, asked) + } + + reply, handled := h.resolveClarifyAnswer(ctx, "позвонить маме") + if !handled { + t.Fatal("the answer must be consumed as an answer") + } + if reply != "Когда?" { + t.Fatalf("a filled subject with no time must ask about the time, got %q", reply) + } + q := h.clarifyStore.Get(voiceDialogueID, h.now()) + if q == nil { + t.Fatal("the second gap must leave a question armed") + } + if q.Slots.Text == "" { + t.Fatalf("the re-parked question lost the answered subject: %+v", q.Slots) + } + + if reply, handled := h.resolveClarifyAnswer(ctx, "в 11:00"); !handled || reply == clarifyGaveUp { + t.Fatalf("the time answer must complete the reminder, handled=%v reply=%q", handled, reply) + } + reminders, err := st.DueReminders(ctx, h.now().Add(48*time.Hour)) + if err != nil || len(reminders) != 1 { + t.Fatalf("expected one reminder: %v err=%v", reminders, err) + } + if !strings.Contains(reminders[0].Payload, "маме") { + t.Fatalf("the reminder lost the subject: %q", reminders[0].Payload) + } +} + +// TestClarifySecondGapRespectsTheAttemptCap — the second gap spends a question +// out of the same budget, so it cannot turn a capped exchange into an endless +// one. With one attempt allowed she acts on what she has instead of asking. +func TestClarifySecondGapRespectsTheAttemptCap(t *testing.T) { + ctx := context.Background() + h, _, _ := newClarifyHandler(t) + h.clarifyMaxAttempts = 1 + + if _, asked := h.askClarify(clarifyDec(router.IntentReminder, router.Slots{}, "напомни")); !asked { + t.Fatal("expected the subject question") + } + reply, handled := h.resolveClarifyAnswer(ctx, "позвонить маме") + if !handled { + t.Fatal("the answer must be consumed") + } + if reply == "Когда?" { + t.Fatal("out of attempts she must not ask a second question") + } + if h.clarifyStore.Get(voiceDialogueID, h.now()) != nil { + t.Fatal("no question may stay armed past the cap") + } +} + +// TestClarifyProseHoldsThePersona — these lines are hand-written Russian that +// the phrasing eval never sees, because they never go through the phraser. They +// carry feminine self-reference ("ждала", "отпустила") and address him with a +// plain imperative, and they are exactly the kind of string someone later edits +// reaching for a synonym. Run the eval's own persona checks over them here. +func TestClarifyProseHoldsThePersona(t *testing.T) { + // Only the persona checks. Length and on-topic do not apply: these are not + // nudges, they have no rule to be on topic about, and the expiry lines are + // deliberately longer than a nudge ceiling. + want := map[string]bool{ + eval.CheckFeminine: true, + eval.CheckHisGender: true, + eval.CheckAddress: true, + eval.CheckCringe: true, + } + lines := append([]string{clarifyGaveUp}, clarifyExpiredVariants...) + for _, q := range clarifyQuestions { + lines = append(lines, q) + } + for _, line := range lines { + for _, r := range eval.RunChecks(eval.Case{}, line, "neutral") { + // The apology clause of the cringe check is scoped to nudges: it + // exists because apologising for a greenlit nudge undermines it. + // These lines are the opposite case. She did not understand him, or + // she let his request go, and "прости" there is ordinary speech + // rather than grovelling. Every other cringe rule still applies: + // pet names, emoji, exclamations, fake concern, praise. + // checkCringe returns the first break it finds, so this skip also + // hides a later one in the same line. Kept narrow on purpose: it + // only fires on a leading "apology (…)" detail. + if r.Name == eval.CheckCringe && strings.HasPrefix(r.Detail, "apology") { + continue + } + if want[r.Name] && !r.Pass { + t.Errorf("%q fails %s: %s", line, r.Name, r.Detail) + } + } + } +} + +// TestExpiryNoticeSurvivesAConfirmTurn — she asks a question, he walks off, the +// question expires, he comes back and answers a confirm that is still parked. +// The confirm turn used to return before the notice was even computed, so he +// answered the confirm and never heard that the older request was let go. +func TestExpiryNoticeSurvivesAConfirmTurn(t *testing.T) { + ctx := context.Background() + h, _, now := newClarifyHandler(t) + + if _, asked := h.askClarify(clarifyDec(router.IntentReminder, router.Slots{Text: "напомни"}, "напомни")); !asked { + t.Fatal("expected a question") + } + // A confirm parked with a longer life than the question, so only the + // question is stale when he speaks. + h.pending = &pendingAct{fn: "delete_backups", phrase: "удалить бэкапы", expiry: now.Add(time.Hour)} + *now = now.Add(clarifyTTL + time.Second) + + reply := h.handleText(ctx, "нет") + if !isClarifyExpired(reply) { + t.Fatalf("the expired question must be announced on a confirm turn too, got %q", reply) + } + if trimClarifyExpired(reply) == "" { + t.Fatalf("the confirm answer must survive the notice, got only the notice: %q", reply) + } + if h.pending != nil { + t.Fatal("the confirm must still have been consumed") + } + if h.clarifyStore.Get(voiceDialogueID, h.now()) != nil { + t.Fatal("the expired question must be gone") + } +} diff --git a/cmd/mavend/coldstart_test.go b/cmd/mavend/coldstart_test.go new file mode 100644 index 0000000..67f5c9f --- /dev/null +++ b/cmd/mavend/coldstart_test.go @@ -0,0 +1,190 @@ +package main + +import ( + "bytes" + "context" + "crypto/rand" + "io" + "os" + "path/filepath" + "testing" + "time" + + "github.com/kami/maven/internal/store" + "github.com/kami/maven/internal/webauthn" +) + +func randBytes(t *testing.T, n int) []byte { + t.Helper() + b := make([]byte, n) + if _, err := io.ReadFull(rand.Reader, b); err != nil { + t.Fatalf("rand: %v", err) + } + b[0] |= 1 + return b +} + +func TestDaemonLockStartsLockedAndFlips(t *testing.T) { + dl := newDaemonLock(true) + if !dl.isLocked() { + t.Fatal("newDaemonLock(true) is not locked") + } + dl.unlock(nil) + if dl.isLocked() { + t.Fatal("still locked after unlock") + } + if newDaemonLock(false).isLocked() { + t.Fatal("newDaemonLock(false) reports locked") + } +} + +// closeStore must be safe on a daemon that never unlocked and safe twice — +// shutdown runs it unconditionally. +func TestDaemonLockCloseStoreIsSafeWhenNeverUnlocked(t *testing.T) { + dl := newDaemonLock(true) + if err := dl.closeStore(); err != nil { + t.Fatalf("closeStore with no store: %v", err) + } + if err := dl.closeStore(); err != nil { + t.Fatalf("second closeStore: %v", err) + } +} + +// The data-loss bug: in locked mode the store is opened on an IPC goroutine +// inside UnlockFn, and shutdown runs on main. Without the handoff nothing +// calls Close, and Close is what re-encrypts the tmpfs working copy back over +// the ciphertext file — so every write of a cold-started session vanished. +func TestDaemonLockSealsTheStoreOpenedAfterUnlock(t *testing.T) { + dir := t.TempDir() + dbPath := filepath.Join(dir, "maven.db") + tmpfs := filepath.Join(dir, "work") + key := randBytes(t, 32) + // Store.Close zeroes the key slice it was handed (encState.key is the + // caller's backing array), so the next boot needs its own copy — exactly + // as mavend keeps envKeyBytes separate from the config's key. + nextBoot := bytes.Clone(key) + ctx := context.Background() + + // Cold start: locked, no store. + dl := newDaemonLock(true) + + // ... unlock arrives, opens the store and hands it over. + st, err := store.OpenEncrypted(ctx, dbPath, tmpfs, key) + if err != nil { + t.Fatalf("OpenEncrypted: %v", err) + } + dl.unlock(st) + if _, err := st.WriteNote(ctx, time.Now(), "заметка после холодного старта", nil, "test"); err != nil { + t.Fatalf("WriteNote: %v", err) + } + + // Shutdown. + if err := dl.closeStore(); err != nil { + t.Fatalf("closeStore: %v", err) + } + if err := dl.closeStore(); err != nil { + t.Fatalf("second closeStore after a real store: %v", err) + } + + // Next boot with the same key must see the write. + st2, err := store.OpenEncrypted(ctx, dbPath, tmpfs, nextBoot) + if err != nil { + t.Fatalf("reopen: %v", err) + } + defer st2.Close() + notes, err := st2.RecentNotes(ctx, 10) + if err != nil { + t.Fatalf("RecentNotes: %v", err) + } + if len(notes) != 1 { + t.Fatalf("got %d notes after a cold-started session, want 1 — the session was lost", len(notes)) + } +} + +// The whole point of the wrapped blob: what sits in the state dir must not let +// anyone open the database. Nothing written there may contain the key, and the +// ciphertext must not be readable with a wrong one. +func TestColdStartLeavesNoPlaintextKeyOnDisk(t *testing.T) { + dir := t.TempDir() + dbPath := filepath.Join(dir, "maven.db") + tmpfs := filepath.Join(dir, "work") + wrappedPath := filepath.Join(dir, "db_key.wrapped") + key := randBytes(t, 32) + secret := randBytes(t, 32) + ctx := context.Background() + + blob, err := webauthn.WrapKey(key, secret) + if err != nil { + t.Fatalf("WrapKey: %v", err) + } + if err := os.WriteFile(wrappedPath, blob, 0o600); err != nil { + t.Fatalf("write wrapped key: %v", err) + } + + st, err := store.OpenEncrypted(ctx, dbPath, tmpfs, key) + if err != nil { + t.Fatalf("OpenEncrypted: %v", err) + } + if _, err := st.WriteNote(ctx, time.Now(), "секрет", nil, "test"); err != nil { + t.Fatalf("WriteNote: %v", err) + } + if err := st.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + // Walk everything in the state dir; none of it may contain the key. + err = filepath.Walk(dir, func(p string, info os.FileInfo, err error) error { + if err != nil || info.IsDir() { + return err + } + b, rerr := os.ReadFile(p) + if rerr != nil { + return nil // unreadable is not a leak + } + if bytes.Contains(b, key) { + t.Errorf("%s contains the plaintext encryption key", p) + } + return nil + }) + if err != nil { + t.Fatalf("walk: %v", err) + } + + // The wrapped file must have owner-only permissions. + fi, err := os.Stat(wrappedPath) + if err != nil { + t.Fatalf("stat: %v", err) + } + if perm := fi.Mode().Perm(); perm != 0o600 { + t.Errorf("wrapped key file mode = %o, want 600", perm) + } + + // A wrong passkey must not open the store. + if _, _, err := webauthn.UnwrapKey(blob, randBytes(t, 32)); err == nil { + t.Fatal("a wrong PRF secret unwrapped the key") + } + if _, err := store.OpenEncrypted(ctx, dbPath, filepath.Join(dir, "work2"), randBytes(t, 32)); err == nil { + t.Fatal("the encrypted store opened under a wrong key") + } + + // And the right one round-trips back to a readable database. + got, version, err := webauthn.UnwrapKey(blob, secret) + if err != nil { + t.Fatalf("UnwrapKey: %v", err) + } + if version != webauthn.BlobV2 { + t.Errorf("blob version = %v, want v2", version) + } + st2, err := store.OpenEncrypted(ctx, dbPath, tmpfs, got) + if err != nil { + t.Fatalf("reopen with the unwrapped key: %v", err) + } + defer st2.Close() + notes, err := st2.RecentNotes(ctx, 10) + if err != nil { + t.Fatalf("RecentNotes: %v", err) + } + if len(notes) != 1 { + t.Fatalf("got %d notes, want 1", len(notes)) + } +} diff --git a/cmd/mavend/confirm.go b/cmd/mavend/confirm.go new file mode 100644 index 0000000..83171fb --- /dev/null +++ b/cmd/mavend/confirm.go @@ -0,0 +1,216 @@ +package main + +import ( + "context" + "log" + "strings" + "time" + + "github.com/kami/maven/internal/router" +) + +// pendingHexisExec — a mutating Hexis capability parked awaiting a spoken +// confirm. The confirmation is bound to the resolved capability + canonical +// target entity so a later "да" can only execute exactly what was proposed +// (ecosystem invariant: protected actions require bound confirmation). +type pendingHexisExec struct { + capabilityID string + capName string + entityID string + displayName string + expiry time.Time +} + +// pendingRoutineConfirm — a proposed routine awaiting a spoken y/n to become +// a recurring reminder. Set by detectPattern after creating a proposal. +type pendingRoutineConfirm struct { + routineID int64 + action string + object string + interval float64 + phrase string + expiry time.Time +} + +// pendingAct — a destructive act awaiting a spoken confirm. +type pendingAct struct { + fn string + args []string + phrase string + expiry time.Time +} + +// confirmTTL — how long a parked destructive confirm stays answerable. Short: +// a confirm is a same-breath gesture; a stale prompt shouldn't fire on an +// unrelated later "да". +const confirmTTL = 90 * time.Second + +// park stores a destructive act awaiting confirmation. Overwrites any prior +// pending (last-asked wins — single-user box). +func (h *reactiveHandler) park(fn string, args []string, phrase string) { + h.mu.Lock() + h.pending = &pendingAct{fn: fn, args: args, phrase: phrase, expiry: h.now().Add(confirmTTL)} + h.mu.Unlock() +} + +// resolveConfirm interprets an utterance as the answer to a parked destructive +// act OR a parked routine proposal. Returns (reply, true) when it consumed the +// utterance as a y/n answer; ("", false) when there's nothing pending (or the +// parked act expired), so the caller routes the utterance normally. An +// unrecognised answer cancels the pending and routes normally — a confirm that +// can't be answered clearly is safer abandoned than left armed. +func (h *reactiveHandler) resolveConfirm(ctx context.Context, text string) (string, bool) { + h.mu.Lock() + defer h.mu.Unlock() + + for _, r := range h.confirmResolvers(ctx) { + if !r.claim() { + continue + } + // The slot is already cleared by claim(): every branch below drops the + // pending, including the unclear one — a confirm that can't be + // answered clearly is safer abandoned than left armed. + switch classifyConfirm(text) { + case confirmYes: + return r.yes(), true + case confirmNo: + return r.no(), true + default: + return "", false + } + } + return "", false +} + +// confirmResolver — one parked-confirm slot in the chain. claim() reports +// whether this slot holds a live pending, taking it (and dropping an expired +// one) as it goes; yes/no then run the answer. Only ever called with h.mu held. +type confirmResolver struct { + claim func() bool + yes func() string + no func() string +} + +// confirmResolvers builds the ordered chain resolveConfirm walks. Order is +// deliberate: the routine proposal is checked before the tool confirm so a +// routine confirm doesn't get eaten by a stale tool pending. +func (h *reactiveHandler) confirmResolvers(ctx context.Context) []confirmResolver { + var pr *pendingRoutineConfirm + var hx *pendingHexisExec + var p *pendingAct + + return []confirmResolver{ + // Routine proposal. + { + claim: func() bool { + pr, h.pendingRoutine = h.pendingRoutine, nil + return pr != nil && !h.now().After(pr.expiry) + }, + yes: func() string { + // Only record the acceptance. The tick loop reads accepted + // routines and nudges on their own interval. Building a + // reminder here made a routine fire exactly once (Vikunja #366). + if err := h.dataStore.AcceptProposedRoutine(ctx, pr.routineID, h.now()); err != nil { + log.Printf("voice: accept proposed routine: %v", err) + return "не получилось запомнить рутину." + } + return "буду напоминать." + }, + no: func() string { + if err := h.dataStore.DismissProposedRoutine(ctx, pr.routineID); err != nil { + log.Printf("voice: dismiss proposed routine: %v", err) + } + return "хорошо, не буду." + }, + }, + // Hexis execution confirm. Bound to the exact capability + target that + // was proposed; a stray "да" can only run that, nothing else. + { + claim: func() bool { + hx, h.pendingHexis = h.pendingHexis, nil + return hx != nil && !h.now().After(hx.expiry) + }, + yes: func() string { + return h.execHexis(ctx, hx.capabilityID, hx.capName, hx.entityID, hx.displayName) + }, + no: func() string { return "отменила." }, + }, + // Tool confirm. + { + claim: func() bool { + p, h.pending = h.pending, nil + return p != nil && !h.now().After(p.expiry) + }, + yes: func() string { + out, err := h.tools.Exec(ctx, p.fn, p.args, true) // confirmed + if err != nil { + log.Printf("voice: tool %s (confirmed): %v", p.fn, err) + if out != "" { + return "не получилось выполнить команду: " + firstLine(out) + } + return "не получилось выполнить команду." + } + if out != "" { + return "готово: " + firstLine(out) + } + return "готово." + }, + no: func() string { return "отменила." }, + }, + } +} + +// proposeGap scaffolds a 'proposed' tool for an act whose verb isn't enabled. +// maven drafts the registration (name = the verb, provenance = the utterance); +// a human enables it on the authed surface. She suggests, never enables. +func (h *reactiveHandler) proposeGap(ctx context.Context, dec router.Decision) string { + name := firstWord(stripWake(dec.Utterance)) + if name == "" { + return "не разобрала команду — попробуй иначе." + } + newly, err := h.api.ProposeTool(ctx, name, dec.Utterance, "", h.now()) + if err != nil { + log.Printf("voice: propose tool %q: %v", name, err) + return "команды «" + name + "» нет в списке разрешённых." + } + if newly { + return "команды «" + name + "» нет в списке. Предложила её добавить — включи через клиент." + } + return "команды «" + name + "» пока нет в списке — она уже предложена, включи через клиент." +} + +// confirmVerdict — the parse of a y/n confirm answer. +type confirmVerdict int + +const ( + confirmUnknown confirmVerdict = iota + confirmYes + confirmNo +) + +// classifyConfirm reads a short ru/en yes-or-no answer. Substring match on the +// stems so inflections/fillers ("да, давай", "нет, отмени") still land. +func classifyConfirm(text string) confirmVerdict { + t := strings.ToLower(strings.TrimSpace(text)) + // negatives first — "не надо" contains no "да", but check no-stems before + // yes so a leading "нет" isn't shadowed. + for _, no := range []string{"нет", "не надо", "отмен", "стоп", "no", "cancel", "stop", "don't"} { + if strings.Contains(t, no) { + return confirmNo + } + } + for _, yes := range []string{"да", "ага", "давай", "подтвер", "конечно", "yes", "yeah", "yep", "confirm", "ок", "okay", "ok"} { + if strings.Contains(t, yes) { + return confirmYes + } + } + return confirmUnknown +} + +// actPhrase renders "fn arg1 arg2" for the confirm prompt. +func actPhrase(fn string, args []string) string { + if len(args) == 0 { + return fn + } + return fn + " " + strings.Join(args, " ") +} diff --git a/cmd/mavend/crawls.go b/cmd/mavend/crawls.go new file mode 100644 index 0000000..a2d8a7c --- /dev/null +++ b/cmd/mavend/crawls.go @@ -0,0 +1,219 @@ +// mavend/crawls.go — the driver for reading web pages (Vikunja #259, +// docs/plans/14-web-crawler.md). The crawler is pure and lives in +// internal/crawl; this is the impure half: the guarded fetcher, a ticker for the +// scheduled watches, and the fact-backed dedup hashes. +// +// Two paths, one config block, both off unless configured: +// +// - ON DEMAND — he names a URL out loud and she reads it. That is the +// `queryWeb` source in actions_query.go, LAST in the chain: after his +// memory, after the notes, and (once Kiwix is wired into the chain) after +// the local ZIMs. A local read costs nothing and leaks nothing; a fetch puts +// a URL in someone's log, so it goes last. +// - SCHEDULED — a watched page is re-read on its interval, and a page whose +// text changed is written as a note. It does NOT announce itself. Same rule +// as the feed poller: notes, never nudges. +// +// Only the URL goes out. Nothing here reads a note, a fact, the persona block or +// the history, and internal/crawl has no access to the store at all. +package main + +import ( + "context" + "errors" + "fmt" + "log" + "net/url" + "time" + + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/crawl" + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/router" + "github.com/kami/maven/internal/webfetch" +) + +// newCrawler builds the crawler from the `crawl` block, or returns nil when +// there is none. Every caller checks for nil, and nil means no page is ever +// fetched. +func newCrawler(cfg *config.Config) *crawl.Crawler { + if cfg.Crawl == nil { + return nil + } + cc := cfg.Crawl + // The WATCH crawler, and only it, reaches the watched hosts. webfetch reads + // a non-empty allow list as "these and nothing else", so folding the watch + // hosts in turned a single watch into an allowlist for everything: a config + // with one watch and on_demand true silently refused every other page he + // pasted, with "не получилось прочитать страницу." and no clue why. + return crawlerWithHosts(cc, crawlHosts(cc, true)) +} + +// crawlHosts — the allowlist for one of the two crawlers. forWatches adds the +// watched pages' own hosts, so a watch does not have to be allowlisted by hand. +// +// The on-demand crawler gets his allow_hosts and nothing else. webfetch reads a +// non-empty list as "these and nothing else", so adding the watch hosts there +// would silently narrow on-demand reading to the watched sites. +func crawlHosts(cc *config.CrawlConfig, forWatches bool) []string { + hosts := append([]string(nil), cc.AllowHosts...) + if !forWatches { + return hosts + } + for _, w := range cc.Watches { + if u, err := url.Parse(w.URL); err == nil && u.Hostname() != "" { + hosts = append(hosts, u.Hostname()) + } + } + return hosts +} + +// crawlerWithHosts builds a crawler over one allowlist. Two callers, two lists: +// see newCrawler and onDemandCrawler. +func crawlerWithHosts(cc *config.CrawlConfig, hosts []string) *crawl.Crawler { + ua := cc.UserAgent + if ua == "" { + ua = webfetch.DefaultUserAgent + } + fetcher := webfetch.New(webfetch.Config{ + AllowHosts: hosts, + DenyHosts: cc.DenyHosts, + Timeout: time.Duration(cc.Timeout), + MaxBytes: cc.MaxBytes, + UserAgent: ua, + }) + // The user-agent handed to the crawler is the one the fetcher sends: obeying + // robots rules written for a different name would be a lie. + return crawl.New(&crawlFetcher{f: fetcher}, crawl.Config{ + UserAgent: ua, + MaxRunes: cc.MaxRunes, + }) +} + +// onDemandCrawler returns a crawler for the answer path, or nil when on-demand +// reading is off. The scheduled watches can be on while this is off: reading a +// fixed list of pages on a timer and reading whatever URL is in an utterance are +// different permissions, and the config keeps them separate. +func onDemandCrawler(cfg *config.Config) *crawl.Crawler { + if cfg.Crawl == nil || !cfg.Crawl.OnDemand { + return nil + } + cc := cfg.Crawl + // His own allow_hosts, and nothing added behind his back. Empty means "any + // host that is not denied and not private", which is what on-demand reading + // of a URL he just said out loud has to mean. + if len(cc.AllowHosts) > 0 { + log.Printf("crawl: allow_hosts is set, so on-demand reading is limited to those %d host(s)", len(cc.AllowHosts)) + } + return crawlerWithHosts(cc, crawlHosts(cc, false)) +} + +// crawlWorker — ticker + watcher for the scheduled half. +type crawlWorker struct { + watcher *crawl.Watcher + interval time.Duration +} + +// crawlTickInterval — how often the worker asks what is due. Per-watch cadence +// is the watcher's business. +const crawlTickInterval = 15 * time.Minute + +// newCrawlWorker wires the scheduled crawls, or nil when nothing is watched. +func newCrawlWorker(c *crawl.Crawler, api ipc.CoreAPI, emb router.Embedder, cfg *config.Config) *crawlWorker { + if c == nil || cfg.Crawl == nil || len(cfg.Crawl.Watches) == 0 { + return nil + } + watches := make([]crawl.WatchConfig, 0, len(cfg.Crawl.Watches)) + for _, w := range cfg.Crawl.Watches { + watches = append(watches, crawl.WatchConfig{ + Name: w.Name, + URL: w.URL, + Interval: time.Duration(w.Interval), + }) + } + watcher := crawl.NewWatcher(c, watches, api, &factHashes{api: api}, + crawlEmbedder(emb), time.Duration(cfg.Crawl.Interval)) + if watcher == nil { + log.Printf("crawl: configured but nothing watchable — scheduled crawls disabled") + return nil + } + log.Printf("crawl: watching %d page(s), checking what is due every %s", len(watches), crawlTickInterval) + return &crawlWorker{watcher: watcher, interval: crawlTickInterval} +} + +// run checks what is due until ctx is canceled. The first round runs +// immediately; it writes notes only, so an early round startles nobody. +func (w *crawlWorker) run(ctx context.Context) { + w.watcher.CheckDue(ctx, time.Now()) + t := time.NewTicker(w.interval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case now := <-t.C: + w.watcher.CheckDue(ctx, now) + } + } +} + +// crawlFetcher adapts webfetch to crawl.Fetcher, which is the seam that keeps +// net/http out of the crawler package. +type crawlFetcher struct{ f *webfetch.Fetcher } + +// Get maps webfetch's sentinels onto crawl's. This adapter is the one place +// that imports both packages, so the mapping belongs here; the crawler used to +// match on three substrings of a message it could not see the definition of, +// and a reworded error would have quietly turned a blocked host into "there is +// no robots.txt here". +func (a *crawlFetcher) Get(ctx context.Context, u string) (*crawl.Response, error) { + resp, err := a.f.Get(ctx, u) + if err != nil { + switch { + case errors.Is(err, webfetch.ErrBlocked), errors.Is(err, webfetch.ErrPrivate), errors.Is(err, webfetch.ErrScheme): + return nil, fmt.Errorf("%w: %v", crawl.ErrFetchRefused, err) + case errors.Is(err, webfetch.ErrStatus): + return nil, fmt.Errorf("%w: %v", crawl.ErrFetchStatus, err) + } + return nil, err + } + return &crawl.Response{URL: resp.URL, ContentType: resp.ContentType, Body: resp.Body}, nil +} + +// factHashes stores each watch's last content hash as a config fact, so a +// restart does not re-note an unchanged page. Same mechanism the feed reader +// uses for its marks, and inspectable on /dash. +type factHashes struct{ api ipc.CoreAPI } + +func hashKey(name string) string { return "crawl:hash:" + name } + +func (h *factHashes) LastHash(ctx context.Context, name string) (string, error) { + f, err := h.api.LatestFact(ctx, hashKey(name)) + if err != nil { + // No hash yet is not an error: the watcher treats "" as "never read". + return "", nil + } + return f.Value, nil +} + +func (h *factHashes) SetHash(ctx context.Context, name, hash string) error { + _, err := h.api.WriteFact(ctx, ipc.WriteFactReq{ + Ts: time.Now(), + Kind: "config", + Key: hashKey(name), + Value: hash, + Source: "poll:crawl", + Confidence: 1.0, + }) + return err +} + +// crawlEmbedder adapts router.Embedder for the watcher, embedding with +// EmbedPassage (a page is text being searched FOR, and the e5 embedder is +// asymmetric). +func crawlEmbedder(emb router.Embedder) crawl.Embedder { + if emb == nil { + return nil + } + return passageEmbedder{emb} +} diff --git a/cmd/mavend/crawls_test.go b/cmd/mavend/crawls_test.go new file mode 100644 index 0000000..5055880 --- /dev/null +++ b/cmd/mavend/crawls_test.go @@ -0,0 +1,240 @@ +package main + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/crawl" + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/phraser" + "github.com/kami/maven/internal/router" + "github.com/kami/maven/internal/voice" + "github.com/kami/maven/internal/webfetch" +) + +// The default config reads nothing. This is the whole "off unless configured" +// contract for the crawler, asserted at the wiring level rather than trusted. +func TestCrawlOffByDefault(t *testing.T) { + cfg := &config.Config{} + if c := newCrawler(cfg); c != nil { + t.Error("newCrawler with no crawl block returned a crawler") + } + if c := onDemandCrawler(cfg); c != nil { + t.Error("onDemandCrawler with no crawl block returned a crawler") + } + if w := newCrawlWorker(nil, nil, nil, cfg); w != nil { + t.Error("newCrawlWorker with no crawl block returned a worker") + } + // Watches configured but on_demand off ⇒ the answer path still reads + // nothing: a timer over a fixed list is not permission for arbitrary URLs. + withWatch := &config.Config{Crawl: &config.CrawlConfig{ + Watches: []config.CrawlWatchConfig{{Name: "p", URL: "https://example.org/p"}}, + }} + if c := onDemandCrawler(withWatch); c != nil { + t.Error("onDemandCrawler honoured a watch list as on-demand permission") + } + if c := newCrawler(withWatch); c == nil { + t.Error("newCrawler returned nil for a configured watch") + } +} + +// The wired fetcher must refuse a private address, because the crawler on this +// box sits one hop from the whole homelab. Same guard the webfetch tests cover; +// this asserts the daemon actually wires it. +func TestCrawlerRefusesPrivateAddress(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + w.Write([]byte("secret")) + })) + defer srv.Close() + + c := newCrawler(&config.Config{Crawl: &config.CrawlConfig{OnDemand: true}}) + if c == nil { + t.Fatal("newCrawler returned nil for an on-demand config") + } + if _, err := c.Page(context.Background(), srv.URL); err == nil { + t.Fatalf("reading %s succeeded; a loopback address must be refused", srv.URL) + } +} + +func TestFactHashesRoundTrip(t *testing.T) { + ctx := context.Background() + st := newTestStore(t) + h := &factHashes{api: ipc.NewStoreAPI(st)} + + got, err := h.LastHash(ctx, "page") + if err != nil { + t.Fatalf("LastHash on a fresh store: %v", err) + } + if got != "" { + t.Errorf("LastHash = %q, want empty for a never-read page", got) + } + if err := h.SetHash(ctx, "page", "deadbeef"); err != nil { + t.Fatalf("SetHash: %v", err) + } + got, err = h.LastHash(ctx, "page") + if err != nil { + t.Fatalf("LastHash: %v", err) + } + if got != "deadbeef" { + t.Errorf("LastHash = %q, want deadbeef", got) + } + if key := hashKey("page"); key != "crawl:hash:page" { + t.Errorf("hashKey = %q", key) + } +} + +// stubCrawlFetcher serves one fixed page to every URL, so queryWeb can be +// exercised without a network or an allowlist. +type stubCrawlFetcher struct{ body, ctype string } + +func (s *stubCrawlFetcher) Get(_ context.Context, u string) (*crawl.Response, error) { + ct := s.ctype + if ct == "" { + ct = "text/html" + } + if strings.HasSuffix(u, "/robots.txt") { + return &crawl.Response{URL: u, ContentType: "text/plain", Body: []byte("")}, nil + } + return &crawl.Response{URL: u, ContentType: ct, Body: []byte(s.body)}, nil +} + +func buildWebHandler(c *crawl.Crawler) *reactiveHandler { + return &reactiveHandler{ + replier: voice.NewStubReplier(), + phraser: phraser.NewStub(), + crawler: c, + } +} + +func askWeb(h *reactiveHandler, q string) (string, bool) { + return h.queryWeb(context.Background(), &queryTurn{ + dec: router.Decision{Intent: router.IntentQuery, Utterance: q}, + }) +} + +func TestQueryWebPassesWithoutAURL(t *testing.T) { + h := buildWebHandler(crawl.New(&stubCrawlFetcher{body: "x"}, crawl.Config{})) + if reply, ok := askWeb(h, "почему небо синее?"); ok { + t.Errorf("the web source claimed a question with no URL: %q", reply) + } +} + +// A daemon where page reading was never turned on — the default — answers the +// question the way it did before the capability existed. Claiming the turn to +// report a configuration status is for something that exists and failed. +func TestQueryWebPassesWhenNotConfigured(t *testing.T) { + h := buildWebHandler(nil) + if reply, ok := askWeb(h, "посмотри https://example.org/page"); ok { + t.Fatalf("an unconfigured crawler claimed the turn with %q", reply) + } +} + +func TestQueryWebReadsThePage(t *testing.T) { + h := buildWebHandler(crawl.New(&stubCrawlFetcher{ + body: "Заголовок

текст страницы

", + }, crawl.Config{})) + reply, ok := askWeb(h, "посмотри https://example.org/page — что там?") + if !ok { + t.Fatal("the web source did not claim a question with a URL") + } + if !strings.Contains(reply, "текст страницы") { + t.Errorf("reply = %q, want the page text read back", reply) + } +} + +func TestQueryWebRefusesNonHTML(t *testing.T) { + h := buildWebHandler(crawl.New(&stubCrawlFetcher{ + body: "\x00\x01binary", ctype: "application/octet-stream", + }, crawl.Config{})) + reply, ok := askWeb(h, "почитай https://example.org/blob.bin") + if !ok { + t.Fatal("the web source did not claim a question with a URL") + } + if !strings.Contains(reply, "не получилось") { + t.Errorf("reply = %q, want the read-failed answer", reply) + } +} + +// robots.txt is honoured on the answer path too, and she says so instead of +// reporting a generic failure. +func TestQueryWebObeysRobots(t *testing.T) { + h := buildWebHandler(crawl.New(&robotsDenyFetcher{}, crawl.Config{})) + reply, ok := askWeb(h, "посмотри https://example.org/private") + if !ok { + t.Fatal("the web source did not claim a question with a URL") + } + if !strings.Contains(reply, "robots.txt") { + t.Errorf("reply = %q, want the robots answer", reply) + } +} + +type robotsDenyFetcher struct{} + +func (robotsDenyFetcher) Get(_ context.Context, u string) (*crawl.Response, error) { + if strings.HasSuffix(u, "/robots.txt") { + return &crawl.Response{URL: u, ContentType: "text/plain", + Body: []byte("User-agent: *\nDisallow: /private\n")}, nil + } + return &crawl.Response{URL: u, ContentType: "text/html", Body: []byte("nope")}, nil +} + +// TestCrawlHostsKeepsAWatchOutOfTheOnDemandAllowlist — the on-demand crawler +// used to be built over allow_hosts PLUS every watched host. webfetch reads a +// non-empty allow list as "these and nothing else", so one watch on a config +// with no allow_hosts at all turned unrestricted on-demand reading into +// "the watched site only", and every other URL he pasted came back as +// "не получилось прочитать страницу." with nothing in the log to explain it. +func TestCrawlHostsKeepsAWatchOutOfTheOnDemandAllowlist(t *testing.T) { + cc := &config.CrawlConfig{ + OnDemand: true, + Watches: []config.CrawlWatchConfig{{Name: "p", URL: "https://watched.example/p"}}, + } + if got := crawlHosts(cc, false); len(got) != 0 { + t.Errorf("on-demand allowlist = %v; a watch is not an allowlist entry, and an empty list is what means \"anything public\"", got) + } + if got := crawlHosts(cc, true); len(got) != 1 || got[0] != "watched.example" { + t.Errorf("watch allowlist = %v; want the watched host so a watch needs no hand-written entry", got) + } + + // With allow_hosts set, his list is what on-demand gets, unchanged. + cc.AllowHosts = []string{"wiki.example"} + on := crawlHosts(cc, false) + if len(on) != 1 || on[0] != "wiki.example" { + t.Errorf("on-demand allowlist = %v; want exactly his allow_hosts", on) + } + if got := crawlHosts(cc, true); len(got) != 2 { + t.Errorf("watch allowlist = %v; want his hosts plus the watched one", got) + } +} + +// TestCrawlFetcherReportsARefusalAsARefusal — internal/crawl cannot import +// webfetch, so it used to recognise a guard refusal by matching substrings of +// webfetch's message text. This adapter owns both packages and is where the +// translation belongs. +func TestCrawlFetcherReportsARefusalAsARefusal(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "boom", http.StatusBadGateway) + })) + defer srv.Close() + + blocked := &crawlFetcher{f: webfetch.New(webfetch.Config{AllowHosts: []string{"wiki.example"}})} + if _, err := blocked.Get(context.Background(), "https://other.example/a"); !errors.Is(err, crawl.ErrFetchRefused) { + t.Errorf("a host outside allow_hosts = %v; want crawl.ErrFetchRefused", err) + } + if _, err := blocked.Get(context.Background(), "file:///etc/passwd"); !errors.Is(err, crawl.ErrFetchRefused) { + t.Errorf("a non-http scheme = %v; want crawl.ErrFetchRefused", err) + } + + // A 5xx is a different thing: the server answered, badly. robots.txt over + // this must refuse the crawl rather than read it as "no rules". + open := &crawlFetcher{f: webfetch.New(webfetch.Config{AllowHosts: []string{"127.0.0.1"}, AllowPrivate: true})} + if _, err := open.Get(context.Background(), srv.URL+"/robots.txt"); !errors.Is(err, crawl.ErrFetchStatus) { + t.Errorf("a 502 = %v; want crawl.ErrFetchStatus", err) + } +} diff --git a/cmd/mavend/dayplan_test.go b/cmd/mavend/dayplan_test.go new file mode 100644 index 0000000..2ae4e8a --- /dev/null +++ b/cmd/mavend/dayplan_test.go @@ -0,0 +1,353 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "strings" + "testing" + "time" + + "github.com/kami/maven/internal/calendar" + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/router" + "github.com/kami/maven/internal/store" +) + +// planAPI answers only DayPlan; every other call is unimplemented, which is +// exactly the assertion that the plan source needs nothing else. +type planAPI struct { + ipc.UnimplementedCoreAPI + plan ipc.DayPlan + err error + calls int +} + +func (a *planAPI) DayPlan(context.Context) (ipc.DayPlan, error) { + a.calls++ + if a.err != nil { + return ipc.DayPlan{}, a.err + } + return a.plan, nil +} + +func planDay() time.Time { return time.Date(2026, 8, 3, 12, 0, 0, 0, time.UTC) } + +func samplePlan() ipc.DayPlan { + day := planDay() + mid := time.Date(2026, 8, 3, 0, 0, 0, 0, time.UTC) + return ipc.DayPlan{ + Date: mid, + Items: []ipc.DayPlanItem{ + {At: day.Add(-2 * time.Hour), Text: "Standup @ 10:00-10:30", Kind: "event"}, + {At: day.Add(2 * time.Hour), Text: "Планёрка @ 14:00-14:30", Kind: "event", Uncertain: true}, + {At: day.Add(6 * time.Hour), Text: "позвонить маме", Kind: "reminder"}, + }, + Spoken: "план на 03.08.2026: 10:00 — Standup @ 10:00-10:30; " + + "похоже, 14:00 — Планёрка @ 14:00-14:30; 18:00 — позвонить маме.", + } +} + +func planHandler(api ipc.CoreAPI) *reactiveHandler { + return &reactiveHandler{api: api, now: planDay} +} + +func TestQueryDayPlanRecitesTheDay(t *testing.T) { + api := &planAPI{plan: samplePlan()} + h := planHandler(api) + reply, ok := h.queryDayPlan(context.Background(), &queryTurn{ + dec: router.Decision{Intent: router.IntentQuery, Utterance: "какие планы на сегодня?"}, + }) + if !ok { + t.Fatal("the plan source must claim a plan question") + } + if reply != api.plan.Spoken { + t.Errorf("reply = %q, want the core's spoken plan %q", reply, api.plan.Spoken) + } +} + +// "что дальше?" is the rest of the day, not the whole day: what has already +// happened is not a plan. +func TestQueryDayPlanTrimsToRestOfDay(t *testing.T) { + h := planHandler(&planAPI{plan: samplePlan()}) + reply, ok := h.queryDayPlan(context.Background(), &queryTurn{ + dec: router.Decision{Intent: router.IntentQuery, Utterance: "что дальше?"}, + }) + if !ok { + t.Fatal("expected the plan source to claim it") + } + if strings.Contains(reply, "Standup") { + t.Errorf("a passed item must not be read back: %q", reply) + } + if !strings.Contains(reply, "Планёрка") || !strings.Contains(reply, "позвонить маме") { + t.Errorf("the rest of the day is missing: %q", reply) + } + // Provenance survives the trim. + if !strings.Contains(reply, "похоже,") { + t.Errorf("a relayed event must stay hedged: %q", reply) + } +} + +// "что дальше?" after the last item of the day. The day was not empty, it is +// over, and the whole-day empty line says something false about a day he just +// lived through. +func TestQueryDayPlanRestOfDayWhenNothingIsLeft(t *testing.T) { + plan := samplePlan() + h := &reactiveHandler{api: &planAPI{plan: plan}, now: func() time.Time { + return time.Date(2026, 8, 3, 23, 0, 0, 0, time.UTC) + }} + reply, ok := h.queryDayPlan(context.Background(), &queryTurn{ + dec: router.Decision{Intent: router.IntentQuery, Utterance: "что дальше?"}, + }) + if !ok { + t.Fatal("expected the plan source to claim it") + } + if strings.Contains(reply, plan.Date.Format("02.01.2006")) { + t.Errorf("the day had things on it and they are done, not empty: %q", reply) + } + if reply != "на сегодня больше ничего не запланировано." { + t.Errorf("reply = %q", reply) + } +} + +// A question that is not about the plan must fall through, or the plan buries +// the calendar listing and the weather behind it. +func TestQueryDayPlanPassesOnEverythingElse(t *testing.T) { + for _, q := range []string{ + "что у меня сегодня?", + "какие планы на завтра?", + // The plan can only be built for the clock's own day. Naming another + // one has to fall through, not get answered with today. + "какие планы на понедельник?", + "какие планы на неделю?", + "какие планы на выходные?", + "what are my plans for friday?", + "когда планёрка?", + "какая погода?", + "", + } { + api := &planAPI{plan: samplePlan()} + reply, ok := planHandler(api).queryDayPlan(context.Background(), &queryTurn{ + dec: router.Decision{Intent: router.IntentQuery, Utterance: q}, + }) + if ok { + t.Errorf("%q was claimed by the plan source (reply %q)", q, reply) + } + if api.calls != 0 { + t.Errorf("%q hit the core for a plan it does not want", q) + } + } +} + +func TestQueryDayPlanCoreFailure(t *testing.T) { + h := planHandler(&planAPI{err: errors.New("socket closed")}) + reply, ok := h.queryDayPlan(context.Background(), &queryTurn{ + dec: router.Decision{Intent: router.IntentQuery, Utterance: "план на сегодня"}, + }) + if !ok { + t.Fatal("a failed plan read must still answer, not fall through to RAG") + } + if reply != "не получилось собрать план." { + t.Errorf("reply = %q", reply) + } +} + +// The day plan must sit before the calendar listing: both match "…на сегодня", +// and the more specific matcher has to get first refusal (see #373 for what +// happens when the order is wrong). +func TestDayPlanSourcePrecedesCalendar(t *testing.T) { + plan, cal := -1, -1 + for i, s := range querySources { + switch s.name { + case "day-plan": + plan = i + case "calendar": + cal = i + } + } + if plan < 0 || cal < 0 { + t.Fatalf("sources missing: day-plan=%d calendar=%d", plan, cal) + } + if plan > cal { + t.Errorf("day-plan at %d must come before calendar at %d", plan, cal) + } +} + +// habitAPI answers only the kind-filtered fact read — the whole input the +// behaviour profile needs (Vikunja #254). Nothing is asked of the LLM, so +// nothing else is wired. RecentFacts is left unimplemented on purpose: the +// profile must not read the mixed window, and a caller that does fails here. +type habitAPI struct { + ipc.UnimplementedCoreAPI + facts []ipc.Fact + err error + calls int + kind string +} + +func (a *habitAPI) RecentActiveFactsByKind(_ context.Context, kind string, _ int) ([]ipc.Fact, error) { + a.calls++ + a.kind = kind + return a.facts, a.err +} + +// tuesdayFacts — n weekly Tuesday rows for key, ending before now. +func tuesdayFacts(key string, hh, weeks int, now time.Time) []ipc.Fact { + d := now + for d.Weekday() != time.Tuesday { + d = d.AddDate(0, 0, -1) + } + var out []ipc.Fact + for i := 0; i < weeks; i++ { + day := d.AddDate(0, 0, -7*i) + out = append(out, ipc.Fact{ + Ts: time.Date(day.Year(), day.Month(), day.Day(), hh, 0, 0, 0, now.Location()), + Kind: "self", + Key: key, + }) + } + return out +} + +func TestQueryHabitsAnswersFromCountedFacts(t *testing.T) { + now := planDay() // a Monday + api := &habitAPI{facts: tuesdayFacts("workout", 19, 4, now)} + h := &reactiveHandler{api: api, now: func() time.Time { return now }} + + reply, ok := h.queryHabits(context.Background(), &queryTurn{ + dec: router.Decision{Intent: router.IntentQuery, Utterance: "что я обычно делаю по вторникам?"}, + }) + if !ok { + t.Fatal("the habit source must claim a habit question") + } + if want := "по вторникам ты обычно тренируешься около 19:00."; reply != want { + t.Errorf("reply = %q, want %q", reply, want) + } +} + +func TestQueryHabitsPassesOnEverythingElse(t *testing.T) { + now := planDay() + for _, q := range []string{"что я делаю в среду?", "что у меня сегодня?", "какие планы на сегодня?", ""} { + api := &habitAPI{} + h := &reactiveHandler{api: api, now: func() time.Time { return now }} + if reply, ok := h.queryHabits(context.Background(), &queryTurn{ + dec: router.Decision{Intent: router.IntentQuery, Utterance: q}, + }); ok { + t.Errorf("%q was claimed by the habit source (reply %q)", q, reply) + } + if api.calls != 0 { + t.Errorf("%q scanned the fact log for a profile it does not want", q) + } + } +} + +// Both specific sources must precede the calendar listing, which matches any +// utterance naming a day. +func TestHabitSourcePrecedesCalendar(t *testing.T) { + habits, cal := -1, -1 + for i, s := range querySources { + switch s.name { + case "habits": + habits = i + case "calendar": + cal = i + } + } + if habits < 0 || cal < 0 { + t.Fatalf("sources missing: habits=%d calendar=%d", habits, cal) + } + if habits > cal { + t.Errorf("habits at %d must come before calendar at %d", habits, cal) + } +} + +// TestQueryHabitsReadsSelfFactsOnly — the profile window is a budget over rows, +// so it must be spent on the rows the profile can use. Reading the mixed table +// let one chatty poller (wg_handshake, roughly every two minutes per peer) push +// every tap out of the window, and she then reported no habits on a store that +// held them. +func TestQueryHabitsReadsSelfFactsOnly(t *testing.T) { + now := planDay() + api := &habitAPI{facts: tuesdayFacts("workout", 19, 4, now)} + h := &reactiveHandler{api: api, now: func() time.Time { return now }} + + if _, ok := h.queryHabits(context.Background(), &queryTurn{ + dec: router.Decision{Intent: router.IntentQuery, Utterance: "что я обычно делаю по вторникам?"}, + }); !ok { + t.Fatal("the habit source must claim a habit question") + } + if api.kind != string(store.KindSelf) { + t.Errorf("profile read kind %q, want %q", api.kind, store.KindSelf) + } +} + +// TestHabitQueryWithPlanWordReachesHabits — the whole chain, not just the +// matchers: a habit question carrying "планы" used to be answered by the day +// plan with today's calendar, because day-plan sits above habits. +func TestHabitQueryWithPlanWordReachesHabits(t *testing.T) { + now := planDay() + api := &habitAPI{facts: tuesdayFacts("workout", 19, 4, now)} + h := &reactiveHandler{api: api, now: func() time.Time { return now }} + + reply := h.actionQuery(context.Background(), router.Decision{ + Intent: router.IntentQuery, + Utterance: "какие у меня обычно планы по вторникам?", + }) + if want := "по вторникам ты обычно тренируешься около 19:00."; reply != want { + t.Errorf("reply = %q, want %q", reply, want) + } +} + +// The plan reads the store on the owner's clock: one line per event, the hour +// printed once, and reminders selected by fire time rather than by how +// recently they were stated. +func TestTickDayPlanReadsTheStore(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + tl := newTestTickLoop(t, st, &fakeSink{}, nil) + + now := time.Date(2026, 8, 3, 12, 0, 0, 0, time.Local) + day := time.Date(2026, 8, 3, 0, 0, 0, 0, time.Local) + ev := calendar.Event{ + Summary: "Standup", + Start: day.Add(14 * time.Hour), + End: day.Add(14*time.Hour + 30*time.Minute), + } + // Rescheduled: same key, a second row. + if _, err := st.WriteFact(ctx, ev.Start, store.KindEnv, calendar.FactKey(ev), + calendar.FactValue(ev), calendar.SourcePersonal, 1.0, sql.NullInt64{}); err != nil { + t.Fatalf("WriteFact: %v", err) + } + moved := ev + moved.Start, moved.End = day.Add(16*time.Hour), day.Add(16*time.Hour+30*time.Minute) + if _, err := st.WriteFact(ctx, moved.Start, store.KindEnv, calendar.FactKey(moved), + calendar.FactValue(moved), calendar.SourcePersonal, 1.0, sql.NullInt64{}); err != nil { + t.Fatalf("WriteFact: %v", err) + } + // One reminder today, one next year. Both are pending; only today's is a + // plan for today. + if _, err := st.CreateReminder(ctx, day.Add(18*time.Hour), "позвонить маме", ""); err != nil { + t.Fatalf("CreateReminder: %v", err) + } + if _, err := st.CreateReminder(ctx, day.AddDate(1, 0, 0), "продлить страховку", ""); err != nil { + t.Fatalf("CreateReminder: %v", err) + } + + plan := tl.dayPlan(ctx, now) + if len(plan.Items) != 2 { + t.Fatalf("got %d items, want the moved standup and today's reminder: %+v", len(plan.Items), plan.Items) + } + ev0 := plan.Items[0] + if ev0.Kind != "event" || ev0.At.In(time.Local).Format("15:04") != "16:00" { + t.Errorf("event = %+v, want the 16:00 one", ev0) + } + if ev0.Text != "Standup" { + t.Errorf("text = %q — the plan prints the hour itself", ev0.Text) + } + if plan.Items[1].Text != "позвонить маме" { + t.Errorf("second item = %+v", plan.Items[1]) + } + if strings.Contains(plan.Spoken, "страховку") { + t.Errorf("a reminder for next year is not today's plan: %q", plan.Spoken) + } +} diff --git a/cmd/mavend/digest_test.go b/cmd/mavend/digest_test.go new file mode 100644 index 0000000..3e808a9 --- /dev/null +++ b/cmd/mavend/digest_test.go @@ -0,0 +1,158 @@ +package main + +import ( + "context" + "testing" + "time" + + "github.com/kami/maven/internal/loop" + "github.com/kami/maven/internal/store" +) + +// Vikunja #281 — the fourth delivery outcome: a care candidate the restraint +// gate suppresses (quiet hours / away / calendar-busy) is not necessarily +// lost. If it's worth resurfacing (loop.DigestEligible), it's durably held +// (internal/store's digest_entries) and spoken as one bundle once speaking +// is appropriate again — never while the suppression reason still holds. + +func breakTrace(blockedBy string) *loop.TickTrace { + return &loop.TickTrace{ + RuleTraces: []loop.RuleTrace{{ + RuleName: "break", + Severity: loop.Sev2, + PredicateResult: true, + GateResult: false, + GateBlockedBy: blockedBy, + }}, + } +} + +// TestSuppressedCareDigestsAcrossQuietHours — a Sev2 care candidate blocked +// by quiet hours is enqueued into the durable digest, and is spoken as a +// "digest" nudge only once quiet hours actually end — never while still +// suppressed (that would just be a second way to nag through quiet hours). +func TestSuppressedCareDigestsAcrossQuietHours(t *testing.T) { + st := newTestStore(t) + sink := &fakeSink{} + tl := newTestTickLoop(t, st, sink, nil) + ctx := context.Background() + now := refNow() + + quiet := loop.State{Now: now, QuietHours: true, Presence: store.Present} + tl.enqueueSuppressedDigest(ctx, breakTrace("quiet_hours"), quiet, now) + + entries, err := st.PendingDigestEntries(ctx, now) + if err != nil { + t.Fatalf("pending: %v", err) + } + if len(entries) != 1 || entries[0].Rule != "break" { + t.Fatalf("want 1 pending digest entry for break, got %+v", entries) + } + + // still quiet hours: draining now must not speak — the same restraint + // that suppressed the live nudge must suppress the bundle too. + tl.maybeDrainDigest(ctx, quiet, now) + if len(sink.sends) != 0 { + t.Fatalf("digest must not drain while quiet hours holds, got %+v", sink.sends) + } + + // quiet hours end: this is the moment speaking is appropriate again. + after := now.Add(time.Hour) + clear := loop.State{Now: after, QuietHours: false, Presence: store.Present} + tl.maybeDrainDigest(ctx, clear, after) + + if len(sink.sends) != 1 { + t.Fatalf("want exactly 1 dispatched digest bundle, got %d: %+v", len(sink.sends), sink.sends) + } + if sink.sends[0].RuleName != "digest" { + t.Fatalf("want RuleName digest, got %q", sink.sends[0].RuleName) + } + + remaining, err := st.PendingDigestEntries(ctx, after) + if err != nil { + t.Fatalf("pending after drain: %v", err) + } + if len(remaining) != 0 { + t.Fatalf("drained entry must no longer be pending, got %+v", remaining) + } +} + +// TestSuppressedCareDigestDedupesAcrossTicks — quiet hours holding for +// several ticks must not enqueue several copies of the same suppressed +// nudge; he hears it once when the bundle finally drains. +func TestSuppressedCareDigestDedupesAcrossTicks(t *testing.T) { + st := newTestStore(t) + sink := &fakeSink{} + tl := newTestTickLoop(t, st, sink, nil) + ctx := context.Background() + now := refNow() + + quiet := loop.State{Now: now, QuietHours: true, Presence: store.Present} + for i := 0; i < 3; i++ { + tl.enqueueSuppressedDigest(ctx, breakTrace("quiet_hours"), quiet, now.Add(time.Duration(i)*time.Minute)) + } + + entries, err := st.PendingDigestEntries(ctx, now) + if err != nil { + t.Fatalf("pending: %v", err) + } + if len(entries) != 1 { + t.Fatalf("3 suppressions of the same nudge must collapse to 1 pending entry, got %d", len(entries)) + } +} + +// TestSuppressedCareDigestExpiresRatherThanDeliveringLate — an entry that +// aged out before the suppression cleared is dropped, not spoken late: a +// two-day-old "you skipped a break" is noise, not news. +func TestSuppressedCareDigestExpiresRatherThanDeliveringLate(t *testing.T) { + st := newTestStore(t) + sink := &fakeSink{} + tl := newTestTickLoop(t, st, sink, nil) + ctx := context.Background() + now := refNow() + + quiet := loop.State{Now: now, QuietHours: true, Presence: store.Present} + tl.enqueueSuppressedDigest(ctx, breakTrace("quiet_hours"), quiet, now) + + // well past digestExpiry (24h) before the suppression ever clears. + stale := now.Add(48 * time.Hour) + tl.expireStaleDigest(ctx, stale) + + clear := loop.State{Now: stale, QuietHours: false, Presence: store.Present} + tl.maybeDrainDigest(ctx, clear, stale) + + if len(sink.sends) != 0 { + t.Fatalf("a stale digest entry must be dropped, not delivered late; got %+v", sink.sends) + } +} + +// TestSuppressedCareDigestIgnoresHighSeverity — defense in depth at the +// wiring layer: even if a RuleTrace somehow showed a high-severity rule +// blocked by a care-only gate reason, the tick driver must not durably +// digest it. Alarms bypass the gate and deliver now, unchanged; they must +// never be silently delayed into a bundle. +func TestSuppressedCareDigestIgnoresHighSeverity(t *testing.T) { + st := newTestStore(t) + sink := &fakeSink{} + tl := newTestTickLoop(t, st, sink, nil) + ctx := context.Background() + now := refNow() + + trace := &loop.TickTrace{RuleTraces: []loop.RuleTrace{{ + RuleName: "service_down", + Severity: loop.Sev4, + PredicateResult: true, + GateResult: false, + GateBlockedBy: "quiet_hours", + }}} + quiet := loop.State{Now: now, QuietHours: true, Presence: store.Present} + tl.enqueueSuppressedDigest(ctx, trace, quiet, now) + + entries, err := st.PendingDigestEntries(ctx, now) + if err != nil { + t.Fatalf("pending: %v", err) + } + if len(entries) != 0 { + t.Fatalf("high severity must never be digested, got %+v", entries) + } +} diff --git a/cmd/mavend/ecosystem.go b/cmd/mavend/ecosystem.go index 6a7e3a1..bca3a5a 100644 --- a/cmd/mavend/ecosystem.go +++ b/cmd/mavend/ecosystem.go @@ -6,6 +6,7 @@ import ( "crypto/rand" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "log" @@ -31,18 +32,84 @@ func correlationIDFromCtx(ctx context.Context) string { return id } -// setEcosystemHeaders stamps the version and correlation headers common to -// every outgoing ecosystem request. -func setEcosystemHeaders(req *http.Request, ctx context.Context, versionHeader string) { +// ecosystemAPIVersion is the contract version Maven speaks to Nexus and +// Praxis. It is sent on every request so a service that has moved on can +// refuse or adapt explicitly instead of misreading an older payload. +const ecosystemAPIVersion = "v1" + +// mavenRequester identifies the calling system on every ecosystem request, so +// a trace on the far side can attribute a call to Maven rather than to an +// anonymous HTTP client. +const mavenRequester = "maven" + +// setEcosystemHeaders stamps the version, requester, auth and correlation +// headers common to every outgoing ecosystem request. token may be empty, +// which means the transport itself is trusted (loopback or unix socket). +// +// The correlation ID is read from the context and never minted here. Minting +// one per request sent the far side an ID that existed nowhere on this side, +// and gave a single multi-hop action as many unrelated IDs as it made calls. +// Callers that start an action assign the ID once (handleHexisAct, +// handlePraxisAct, resolveEntityReference) and every hop inherits it. +func setEcosystemHeaders(req *http.Request, ctx context.Context, versionHeader, token string) { req.Header.Set("Content-Type", "application/json") - req.Header.Set(versionHeader, "v1") + req.Header.Set(versionHeader, ecosystemAPIVersion) + req.Header.Set("Accept", "application/json") + req.Header.Set("X-Requested-By", mavenRequester) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } if id := correlationIDFromCtx(ctx); id != "" { req.Header.Set("X-Correlation-ID", id) } } +// ecosystemError is the typed failure every ecosystem client returns, so +// callers can tell a transport failure from a refusal from a contract +// mismatch without matching on message text. The distinction matters: +// "the service is down" and "the service rejected my version" degrade the +// same way to the user but not to whoever reads the trace. +type ecosystemError struct { + Service string // "nexus", "praxis", "hexis" + Op string // logical operation, e.g. "resolve" + Status int // HTTP status, 0 when the call never got an answer + Err error +} + +func (e *ecosystemError) Error() string { + if e.Status != 0 { + return fmt.Sprintf("%s %s: http %d: %v", e.Service, e.Op, e.Status, e.Err) + } + return fmt.Sprintf("%s %s: %v", e.Service, e.Op, e.Err) +} + +func (e *ecosystemError) Unwrap() error { return e.Err } + +// Unauthorized reports a rejected or missing credential. +func (e *ecosystemError) Unauthorized() bool { + return e.Status == http.StatusUnauthorized || e.Status == http.StatusForbidden +} + +// ContractMismatch reports that the far side refused the version Maven speaks. +func (e *ecosystemError) ContractMismatch() bool { + return e.Status == http.StatusNotAcceptable || e.Status == http.StatusUpgradeRequired +} + +// Unreachable reports a call that never produced an HTTP answer at all +// (connection refused, timeout, cancelled). +func (e *ecosystemError) Unreachable() bool { return e.Status == 0 } + +// httpError builds an ecosystemError from a response status. +func httpError(service, op string, status int) *ecosystemError { + return &ecosystemError{ + Service: service, Op: op, Status: status, + Err: errors.New(http.StatusText(status)), + } +} + type nexusClient struct { baseURL string + token string httpClient *http.Client } @@ -53,6 +120,13 @@ func newNexusClient(url string) *nexusClient { } } +// withToken sets the bearer token sent on every request. Returns the client so +// wiring reads as one expression. +func (c *nexusClient) withToken(token string) *nexusClient { + c.token = token + return c +} + type nexusEntity struct { ID string `json:"id"` Type string `json:"type"` @@ -107,22 +181,22 @@ func (c *nexusClient) Resolve(ctx context.Context, query string, types []string) if err != nil { return nil, fmt.Errorf("create request: %w", err) } - setEcosystemHeaders(req, ctx, "X-Nexus-Version") + setEcosystemHeaders(req, ctx, "X-Nexus-Version", c.token) resp, err := c.httpClient.Do(req) if err != nil { - return nil, fmt.Errorf("do request: %w", err) + return nil, &ecosystemError{Service: "nexus", Op: "resolve", Err: err} } defer resp.Body.Close() bodyBytes, _ := io.ReadAll(resp.Body) if resp.StatusCode != 200 { - return nil, fmt.Errorf("nexus: %s", http.StatusText(resp.StatusCode)) + return nil, httpError("nexus", "resolve", resp.StatusCode) } var result nexusResolveResult if err := json.Unmarshal(bodyBytes, &result); err != nil { - return nil, fmt.Errorf("decode: %w", err) + return nil, &ecosystemError{Service: "nexus", Op: "resolve", Status: resp.StatusCode, Err: err} } return &result, nil } @@ -130,16 +204,16 @@ func (c *nexusClient) Resolve(ctx context.Context, query string, types []string) func (c *nexusClient) Health(ctx context.Context) error { req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+"/health", nil) if err != nil { - return err + return &ecosystemError{Service: "nexus", Op: "health", Err: err} } - setEcosystemHeaders(req, ctx, "X-Nexus-Version") + setEcosystemHeaders(req, ctx, "X-Nexus-Version", c.token) resp, err := c.httpClient.Do(req) if err != nil { - return err + return &ecosystemError{Service: "nexus", Op: "health", Err: err} } resp.Body.Close() if resp.StatusCode != 200 { - return fmt.Errorf("nexus health: %s", http.StatusText(resp.StatusCode)) + return httpError("nexus", "health", resp.StatusCode) } return nil } @@ -149,6 +223,7 @@ func (c *nexusClient) Health(ctx context.Context) error { // so attention/changes/lifecycle all go over this HTTP contract against praxisd. type praxisClient struct { baseURL string + token string httpClient *http.Client } @@ -159,27 +234,38 @@ func newPraxisClient(url string) *praxisClient { } } -// getJSON performs a GET and decodes the JSON body into out. -func (c *praxisClient) getJSON(ctx context.Context, path string, out any) error { +func (c *praxisClient) withToken(token string) *praxisClient { + c.token = token + return c +} + +// getJSON performs a GET and decodes the JSON body into out. op is the logical +// operation name for errors and traces: the path carries the query string, and +// after entity scoping that means an entity id in every log line built from the +// error, next to a trace that redacts far less than that. +func (c *praxisClient) getJSON(ctx context.Context, op, path string, out any) error { req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+path, nil) if err != nil { return err } - setEcosystemHeaders(req, ctx, "X-Praxis-Version") + setEcosystemHeaders(req, ctx, "X-Praxis-Version", c.token) resp, err := c.httpClient.Do(req) if err != nil { - return err + return &ecosystemError{Service: "praxis", Op: op, Err: err} } defer resp.Body.Close() if resp.StatusCode != 200 { - return fmt.Errorf("praxis: %s", http.StatusText(resp.StatusCode)) + return httpError("praxis", op, resp.StatusCode) } - return json.NewDecoder(resp.Body).Decode(out) + if err := json.NewDecoder(resp.Body).Decode(out); err != nil { + return &ecosystemError{Service: "praxis", Op: op, Status: resp.StatusCode, Err: err} + } + return nil } func (c *praxisClient) ListAttention(ctx context.Context, limit int) ([]map[string]any, error) { var out []map[string]any - err := c.getJSON(ctx, fmt.Sprintf("/api/v1/tools/attention?limit=%d", limit), &out) + err := c.getJSON(ctx, "attention", fmt.Sprintf("/api/v1/tools/attention?limit=%d", limit), &out) return out, err } @@ -189,13 +275,14 @@ func (c *praxisClient) ListAttention(ctx context.Context, limit int) ([]map[stri // instead of filtering the unscoped list client-side. func (c *praxisClient) ListAttentionForEntity(ctx context.Context, entityID string, limit int) ([]map[string]any, error) { var out []map[string]any - err := c.getJSON(ctx, fmt.Sprintf("/api/v1/tools/attention?limit=%d&entity_id=%s", limit, url.QueryEscape(entityID)), &out) + err := c.getJSON(ctx, "attention_for_entity", + fmt.Sprintf("/api/v1/tools/attention?limit=%d&entity_id=%s", limit, url.QueryEscape(entityID)), &out) return out, err } func (c *praxisClient) ListChanges(ctx context.Context, limit int) ([]map[string]any, error) { var out []map[string]any - err := c.getJSON(ctx, fmt.Sprintf("/api/v1/tools/changes?limit=%d", limit), &out) + err := c.getJSON(ctx, "changes", fmt.Sprintf("/api/v1/tools/changes?limit=%d", limit), &out) return out, err } @@ -221,24 +308,32 @@ type praxisItem struct { // postItemAction posts {"item_id": id} to a Praxis tools lifecycle endpoint // and decodes the resulting item. Shared by Surface/Acknowledge/Resolve/Ignore. -func (c *praxisClient) postItemAction(ctx context.Context, path, itemID string) (*praxisItem, error) { - body, _ := json.Marshal(map[string]any{"item_id": itemID}) +func (c *praxisClient) postItemAction(ctx context.Context, op, path, itemID string) (*praxisItem, error) { + return c.postJSON(ctx, op, path, map[string]any{"item_id": itemID}) +} + +// postJSON posts a body to a Praxis lifecycle endpoint and decodes the item. +// Every failure is a *ecosystemError, including the transport and decode ones: +// these are the paths that mutate remote state, and the question worth +// answering afterwards is whether the call never left or was refused. +func (c *praxisClient) postJSON(ctx context.Context, op, path string, payload map[string]any) (*praxisItem, error) { + body, _ := json.Marshal(payload) req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+path, bytes.NewReader(body)) if err != nil { - return nil, err + return nil, &ecosystemError{Service: "praxis", Op: op, Err: err} } - setEcosystemHeaders(req, ctx, "X-Praxis-Version") + setEcosystemHeaders(req, ctx, "X-Praxis-Version", c.token) resp, err := c.httpClient.Do(req) if err != nil { - return nil, err + return nil, &ecosystemError{Service: "praxis", Op: op, Err: err} } defer resp.Body.Close() if resp.StatusCode != 200 { - return nil, fmt.Errorf("praxis %s: %s", path, http.StatusText(resp.StatusCode)) + return nil, httpError("praxis", op, resp.StatusCode) } var out praxisItem if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { - return nil, fmt.Errorf("decode: %w", err) + return nil, &ecosystemError{Service: "praxis", Op: op, Status: resp.StatusCode, Err: err} } return &out, nil } @@ -247,46 +342,28 @@ func (c *praxisClient) postItemAction(ctx context.Context, path, itemID string) // ECOSYSTEM-SPEC.md §2.3). Callers that read attention aloud must call this, never // Acknowledge, so "I mentioned it" stays distinguishable from "you told me you saw it". func (c *praxisClient) Surface(ctx context.Context, itemID string) (*praxisItem, error) { - return c.postItemAction(ctx, "/api/v1/tools/surface", itemID) + return c.postItemAction(ctx, "surface", "/api/v1/tools/surface", itemID) } func (c *praxisClient) Acknowledge(ctx context.Context, itemID string) (*praxisItem, error) { - return c.postItemAction(ctx, "/api/v1/tools/acknowledge", itemID) + return c.postItemAction(ctx, "acknowledge", "/api/v1/tools/acknowledge", itemID) } func (c *praxisClient) Resolve(ctx context.Context, itemID string) (*praxisItem, error) { - return c.postItemAction(ctx, "/api/v1/tools/resolve", itemID) + return c.postItemAction(ctx, "resolve", "/api/v1/tools/resolve", itemID) } func (c *praxisClient) Ignore(ctx context.Context, itemID string) (*praxisItem, error) { - return c.postItemAction(ctx, "/api/v1/tools/ignore", itemID) + return c.postItemAction(ctx, "ignore", "/api/v1/tools/ignore", itemID) } func (c *praxisClient) Pin(ctx context.Context, itemID string, pinned bool) (*praxisItem, error) { - body, _ := json.Marshal(map[string]any{"item_id": itemID, "pinned": pinned}) - req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/api/v1/tools/pin", bytes.NewReader(body)) - if err != nil { - return nil, err - } - setEcosystemHeaders(req, ctx, "X-Praxis-Version") - resp, err := c.httpClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - if resp.StatusCode != 200 { - return nil, fmt.Errorf("praxis pin: %s", http.StatusText(resp.StatusCode)) - } - var out praxisItem - if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { - return nil, fmt.Errorf("decode: %w", err) - } - return &out, nil + return c.postJSON(ctx, "pin", "/api/v1/tools/pin", map[string]any{"item_id": itemID, "pinned": pinned}) } func (c *praxisClient) GetItem(ctx context.Context, itemID string) (*praxisItem, error) { var out praxisItem - err := c.getJSON(ctx, "/api/v1/tools/items/"+itemID, &out) + err := c.getJSON(ctx, "get_item", "/api/v1/tools/items/"+itemID, &out) if err != nil { return nil, err } @@ -295,7 +372,7 @@ func (c *praxisClient) GetItem(ctx context.Context, itemID string) (*praxisItem, func (c *praxisClient) Search(ctx context.Context, query string, limit int) ([]praxisItem, error) { var out []praxisItem - err := c.getJSON(ctx, fmt.Sprintf("/api/v1/tools/search?q=%s&limit=%d", url.QueryEscape(query), limit), &out) + err := c.getJSON(ctx, "search", fmt.Sprintf("/api/v1/tools/search?q=%s&limit=%d", url.QueryEscape(query), limit), &out) return out, err } @@ -311,7 +388,7 @@ func wireEcosystem(cfg *config.Config) *ecosystemWiring { // Nexus identity service if cfg.Nexus != nil && cfg.Nexus.URL != "" { - w.nexus = newNexusClient(cfg.Nexus.URL) + w.nexus = newNexusClient(cfg.Nexus.URL).withToken(cfg.Nexus.Token) log.Printf("ecosystem: nexus at %s", cfg.Nexus.URL) } else { log.Printf("ecosystem: nexus not configured") @@ -319,7 +396,7 @@ func wireEcosystem(cfg *config.Config) *ecosystemWiring { // Hexis capability service if cfg.Hexis != nil && cfg.Hexis.URL != "" { - w.hexis = hexisclient.New(cfg.Hexis.URL) + w.hexis = hexisclient.New(cfg.Hexis.URL).WithToken(cfg.Hexis.Token) log.Printf("ecosystem: hexis at %s", cfg.Hexis.URL) } else { log.Printf("ecosystem: hexis not configured") @@ -327,7 +404,7 @@ func wireEcosystem(cfg *config.Config) *ecosystemWiring { // Praxis attention service (HTTP tools API — never the DB directly) if cfg.Praxis != nil && cfg.Praxis.URL != "" { - w.praxis = newPraxisClient(cfg.Praxis.URL) + w.praxis = newPraxisClient(cfg.Praxis.URL).withToken(cfg.Praxis.Token) log.Printf("ecosystem: praxis at %s", cfg.Praxis.URL) } else { log.Printf("ecosystem: praxis not configured") @@ -352,7 +429,19 @@ func (w *ecosystemWiring) resolveEntityReference(ctx context.Context, text strin log.Printf("ecosystem: nexus resolve error: %v", err) return "", "", nil, err } - if result.Status == "resolved" && result.Entity != nil { + if result.Status == "resolved" { + // "resolved" with nothing to resolve to is a contract violation, not a + // miss. Treating it as "no such entity" let the caller fall straight + // through to the local executor with his verb intact, which is a + // dependency failure reaching execution. + if result.Entity == nil || result.Entity.ID == "" { + err := &ecosystemError{ + Service: "nexus", Op: "resolve", Status: 200, + Err: errors.New("resolved status with no entity"), + } + log.Printf("ecosystem: %v", err) + return "", "", nil, err + } return result.Entity.ID, result.Entity.DisplayName, nil, nil } if result.Status == "ambiguous" { @@ -372,6 +461,12 @@ func (w *ecosystemWiring) resolveEntityReference(ctx context.Context, text strin // healthy and genuinely has nothing registered for this entity. Callers must // not conflate the two: a dependency failure must not silently read as "no // capabilities" and fall through to unrelated local execution. +// +// The correlation header is stamped in the client's do(), so discovery and +// execution can be joined on the Hexis side as long as both hops carry the +// same ID through ctx. (This used to say the header went out on Execute only; +// that was never true of the vendored code and is not true after the 2026-08-01 +// re-vendor.) func (w *ecosystemWiring) discoverCapabilities(ctx context.Context, entityID string) ([]hexisclient.Capability, error) { if w == nil || w.hexis == nil || entityID == "" { return nil, nil diff --git a/cmd/mavend/ecosystem_acts.go b/cmd/mavend/ecosystem_acts.go new file mode 100644 index 0000000..867685d --- /dev/null +++ b/cmd/mavend/ecosystem_acts.go @@ -0,0 +1,634 @@ +package main + +import ( + "context" + "errors" + "fmt" + "log" + "strings" + "time" + + hexisclient "github.com/kami/hexis/pkg/client" + "github.com/kami/maven/internal/router" + "github.com/kami/maven/internal/store" +) + +// praxisCapability is one arm of the Praxis act dispatch. This is an interface +// rather than a map[string]func because each arm carries its own state: the +// verb aliases it answers to, the trace name it records, and its own reply +// formatting. The dispatch grows an arm per Praxis capability, so a new one is +// added to praxisCapabilities below and nothing else changes. +type praxisCapability interface { + // aliases are the verbs (router fn slots, EN and RU) this capability answers to. + aliases() []string + // handle runs the capability and returns the user-facing reply. + handle(ctx context.Context, h *reactiveHandler, px *praxisClient, dec router.Decision) string +} + +// praxisCapabilities is the registry handlePraxisAct consults, in order. +var praxisCapabilities = []praxisCapability{ + listAttentionCapability{}, + praxisItemAction{ + verbs: []string{"acknowledge_item", "принято", "понял", "поняла"}, + ask: "какой пункт отметить принятым?", + op: "acknowledge", + failure: "не получилось отметить принятым.", + success: "принято.", + call: func(ctx context.Context, px *praxisClient, id string) error { + _, err := px.Acknowledge(ctx, id) + return err + }, + }, + praxisItemAction{ + verbs: []string{"resolve_item", "сделано", "готово", "решено"}, + ask: "какой пункт отметить сделанным?", + op: "resolve", + failure: "не получилось отметить сделанным.", + success: "отмечено как сделано.", + call: func(ctx context.Context, px *praxisClient, id string) error { + _, err := px.Resolve(ctx, id) + return err + }, + }, + praxisItemAction{ + verbs: []string{"ignore_item", "игнорировать", "неважно"}, + ask: "какой пункт игнорировать?", + op: "ignore", + failure: "не получилось проигнорировать.", + success: "проигнорировано.", + call: func(ctx context.Context, px *praxisClient, id string) error { + _, err := px.Ignore(ctx, id) + return err + }, + }, + praxisItemAction{ + verbs: []string{"pin_item", "закрепить"}, + ask: "какой пункт закрепить?", + op: "pin", + failure: "не получилось закрепить.", + success: "закреплено.", + call: func(ctx context.Context, px *praxisClient, id string) error { + _, err := px.Pin(ctx, id, true) + return err + }, + }, + listChangesCapability{}, + entityAttentionCapability{}, +} + +// handlePraxisAct — dispatches ecosystem tool acts through the Praxis tools API. +// Returns "" when the act is not a Praxis verb (the caller falls through to the +// system command executor). Returns a reply string otherwise. +func (h *reactiveHandler) handlePraxisAct(ctx context.Context, dec router.Decision) string { + if h.ecosystem == nil || h.ecosystem.praxis == nil { + return "" + } + // Every hop of this action shares one correlation ID, assigned here, so a + // digest that calls attention once and surface N times reads as one turn + // on the Praxis side instead of N+1 unrelated request ids. + if correlationIDFromCtx(ctx) == "" { + ctx = withCorrelationID(ctx, newCorrelationID()) + } + px := h.ecosystem.praxis + for _, capability := range praxisCapabilities { + for _, alias := range capability.aliases() { + if alias == dec.Slots.Fn { + return capability.handle(ctx, h, px, dec) + } + } + } + // Not a Praxis verb — let the caller fall through. + return "" +} + +// praxisItemAction is the shared shape of the item-lifecycle capabilities: take +// an item id from the value slot, call one Praxis endpoint, trace the result. +type praxisItemAction struct { + verbs []string + ask string // reply when no item id was given + op string // trace + log name of the operation + failure string // reply when the Praxis call errors + success string + call func(ctx context.Context, px *praxisClient, id string) error +} + +func (a praxisItemAction) aliases() []string { return a.verbs } + +func (a praxisItemAction) handle(ctx context.Context, h *reactiveHandler, px *praxisClient, dec router.Decision) string { + id := dec.Slots.Value + if id == "" { + return a.ask + } + started := h.now() + if err := a.call(ctx, px, id); err != nil { + log.Printf("ecosystem: praxis %s %s: %v", a.op, id, err) + h.recordEcosystemTrace(ctx, "praxis", a.op, traceStatusForError(err), started, + mergeFields(traceErrorFields(err), map[string]any{"item_id": id})) + return a.failure + } + h.recordPraxisTrace(ctx, a.op, started, map[string]any{"item_id": id}) + return a.success +} + +// listAttentionCapability reads the attention digest and surfaces every item it speaks. +type listAttentionCapability struct{} + +func (listAttentionCapability) aliases() []string { + return []string{"list_attention", "attention", "внимание", "что требует внимания", "что нового"} +} + +func (listAttentionCapability) handle(ctx context.Context, h *reactiveHandler, px *praxisClient, _ router.Decision) string { + started := h.now() + items, err := px.ListAttention(ctx, 20) + if err != nil { + log.Printf("ecosystem: praxis attention: %v", err) + h.recordEcosystemTrace(ctx, "praxis", "list_attention", traceStatusForError(err), + started, traceErrorFields(err)) + return "не могу сейчас узнать, что требует внимания." + } + if len(items) == 0 { + return "ничего не требует внимания." + } + h.recordPraxisTrace(ctx, "list_attention", started, map[string]any{"count": len(items)}) + var parts []string + for _, item := range items { + title, _ := item["title"].(string) + // importance arrives as JSON number ⇒ float64 over the HTTP contract. + importance, _ := item["importance"].(float64) + rule, _ := item["rule"].(string) + s := title + if importance > 0 { + s += fmt.Sprintf(" (важность %d", int(importance)) + if rule != "" { + s += ": " + rule + } + s += ")" + } + parts = append(parts, s) + + // Speaking an item surfaces it, it does not acknowledge it + // (ECOSYSTEM-SPEC.md §2.3: surfaced != acknowledged). Best-effort: + // a failed surface call must not block delivering the digest. + if id, ok := item["id"].(string); ok && id != "" { + if _, err := px.Surface(ctx, id); err != nil { + log.Printf("ecosystem: praxis surface %s: %v", id, err) + } + } + } + return "требует внимания: " + strings.Join(parts, "; ") +} + +// listChangesCapability reads the recent-changes feed. +type listChangesCapability struct{} + +func (listChangesCapability) aliases() []string { + return []string{"list_changes", "changes", "изменения", "что изменилось"} +} + +func (listChangesCapability) handle(ctx context.Context, h *reactiveHandler, px *praxisClient, _ router.Decision) string { + started := h.now() + changes, err := px.ListChanges(ctx, 20) + if err != nil { + log.Printf("ecosystem: praxis changes: %v", err) + h.recordEcosystemTrace(ctx, "praxis", "list_changes", traceStatusForError(err), + started, traceErrorFields(err)) + return "не могу сейчас узнать об изменениях." + } + if len(changes) == 0 { + return "нет изменений." + } + h.recordPraxisTrace(ctx, "list_changes", started, map[string]any{"count": len(changes)}) + var parts []string + for _, c := range changes { + title, _ := c["title"].(string) + typ, _ := c["change_type"].(string) + parts = append(parts, fmt.Sprintf("%s (%s)", title, typ)) + } + return "изменения: " + strings.Join(parts, "; ") +} + +// entityAttentionCapability answers "what's going on with X" by resolving X to +// a canonical Nexus entity and asking Praxis for that entity's attention items +// (Vikunja #272). Unlike listAttentionCapability it is scoped: the entity_id +// travels to Praxis as a query parameter instead of Maven filtering an unscoped +// list client-side, which is what makes the ref canonical end to end. +// +// It also folds in what Maven herself knows about the same entity — facts the +// enrichment worker has already resolved to that entity_id — so one question +// gets one answer across both stores. +type entityAttentionCapability struct{} + +// aliases are matched against Slots.Fn, which carries a function slot from the +// act grammar and never free Russian, so only grammar names belong here. +func (entityAttentionCapability) aliases() []string { + return []string{"entity_attention", "entity_status"} +} + +func (entityAttentionCapability) handle(ctx context.Context, h *reactiveHandler, px *praxisClient, dec router.Decision) string { + subject := dec.Slots.Value + if subject == "" { + subject = dec.Slots.Text + } + if subject == "" { + return "про что именно спросить?" + } + if h.ecosystem == nil || h.ecosystem.nexus == nil { + // Without Nexus there is no canonical ref to scope by. Say so rather + // than quietly answering about something else. + return "не могу связать это с сущностью — Nexus не настроен." + } + + started := h.now() + entityID, displayName, ambiguous, err := h.ecosystem.resolveEntityReference(ctx, subject, nil) + if err != nil { + // The subject is his words, so the log gets the same redaction the + // trace gets. A trace that stores a rune count next to a log line + // storing the runes is not redacted at all. + log.Printf("ecosystem: entity attention resolve %s: %v", redactSubject(subject), err) + h.recordEcosystemTrace(ctx, "nexus", "resolve", traceStatusForError(err), started, + mergeFields(traceErrorFields(err), map[string]any{"subject": redactSubject(subject)})) + if unauthorizedEcosystemError(err) { + return "экосистема отклоняет доступ, проверь токен." + } + return "экосистема недоступна, попробуй ещё раз." + } + if len(ambiguous) > 0 { + return "уточни, что именно: " + strings.Join(ambiguous, ", ") + "?" + } + if entityID == "" { + return "не знаю такой сущности." + } + if displayName == "" { + displayName = subject + } + + queried := h.now() + items, err := px.ListAttentionForEntity(ctx, entityID, 20) + if err != nil { + log.Printf("ecosystem: praxis attention for %s: %v", entityID, err) + h.recordEcosystemTrace(ctx, "praxis", "entity_attention", traceStatusForError(err), + queried, mergeFields(traceErrorFields(err), map[string]any{"entity_id": entityID})) + return "не могу сейчас узнать, что требует внимания по «" + displayName + "»." + } + items, scoped := scopedToEntity(items, entityID) + if !scoped { + // A Praxis old enough to ignore an unknown query parameter answers the + // scoped question with the unscoped list. Reading that back as "по + // «X»: ..." is the exact fabrication the entity ref exists to prevent, + // so refuse the answer instead of relabelling someone else's items. + log.Printf("ecosystem: praxis returned unscoped items for %s, refusing to answer", entityID) + h.recordEcosystemTrace(ctx, "praxis", "entity_attention", traceFailed, queried, + map[string]any{"entity_id": entityID, "class": "unscoped_response"}) + return "не могу сейчас узнать, что требует внимания по «" + displayName + "»." + } + h.recordPraxisTrace(ctx, "entity_attention", queried, map[string]any{ + "entity_id": entityID, "count": len(items), + }) + + var parts []string + for _, item := range items { + title, _ := item["title"].(string) + if title == "" { + continue + } + parts = append(parts, title) + // Same surfaced != acknowledged rule as the unscoped digest. + if id, ok := item["id"].(string); ok && id != "" { + if _, err := px.Surface(ctx, id); err != nil { + log.Printf("ecosystem: praxis surface %s: %v", id, err) + } + } + } + if known := h.localFactsForEntity(ctx, entityID); known != "" { + parts = append(parts, known) + } + if len(parts) == 0 { + return "по «" + displayName + "» ничего нет." + } + return "по «" + displayName + "»: " + strings.Join(parts, "; ") +} + +// scopedToEntity drops items that carry an entity_id other than the one asked +// about, and reports whether the response can be trusted as scoped at all. An +// item without an entity_id is kept only when at least one sibling carries the +// matching id: a whole page with no entity_id is a Praxis that ignored the +// scope, not a page of untagged items. +func scopedToEntity(items []map[string]any, entityID string) ([]map[string]any, bool) { + if len(items) == 0 { + return items, true + } + var kept []map[string]any + var sawMatch, sawMismatch bool + for _, item := range items { + id, _ := item["entity_id"].(string) + switch { + case id == entityID: + sawMatch = true + kept = append(kept, item) + case id != "": + sawMismatch = true + default: + kept = append(kept, item) + } + } + if sawMatch { + return kept, true + } + if sawMismatch { + // Some items were tagged and none matched: the far side answered about + // other entities, so nothing here belongs to this one. + return nil, true + } + return nil, false +} + +// localFactsForEntity summarises Maven's own facts already resolved to this +// canonical entity. Empty when the store is unavailable or nothing matched — +// entity-scoped memory is an enrichment of the answer, never a precondition. +func (h *reactiveHandler) localFactsForEntity(ctx context.Context, entityID string) string { + if h.dataStore == nil || entityID == "" { + return "" + } + const spoken = 3 + // One over the spoken limit, so a truncation can be named rather than + // passed off as everything she knows. + facts, err := h.dataStore.FactsByEntity(ctx, entityID, spoken+1) + if err != nil { + log.Printf("ecosystem: facts by entity %s: %v", entityID, err) + return "" + } + more := false + if len(facts) > spoken { + facts, more = facts[:spoken], true + } + var parts []string + for _, f := range facts { + if f.Value != "" { + parts = append(parts, f.Value) + } + } + if len(parts) == 0 { + return "" + } + out := "я помню: " + strings.Join(parts, ", ") + if more { + out += ", и это не всё" + } + return out +} + +// mergeFields overlays b onto a and returns a. +func mergeFields(a, b map[string]any) map[string]any { + for k, v := range b { + a[k] = v + } + return a +} + +// recordPraxisTrace — records a completed Praxis call. Thin wrapper over +// recordEcosystemTrace so every ecosystem hop lands in one table with one +// shape. +func (h *reactiveHandler) recordPraxisTrace(ctx context.Context, operation string, started time.Time, details map[string]any) { + h.recordEcosystemTrace(ctx, "praxis", operation, traceOK, started, details) +} + +// traceStatus classifies an ecosystem call for the trace record. Kept coarse +// on purpose: a trace is read to answer "did this hop work, and how long did +// it take", not to re-derive the error. +const ( + traceOK = "ok" + traceFailed = "failed" // the call never got an answer + traceRefused = "refused" // the far side answered, and said no + traceAmbig = "ambiguous" + traceNotFound = "not_found" + tracePending = "pending" // deliberately not done yet, awaiting a confirm +) + +// traceStatusForError distinguishes "I could not reach it" from "it answered +// and refused". Both degrade the same way for him and not at all the same way +// for whoever reads the trace: one is a network or a dead service, the other +// is a token, a version or a rejected argument. +func traceStatusForError(err error) string { + var ee *ecosystemError + if errors.As(err, &ee) && !ee.Unreachable() { + return traceRefused + } + return traceFailed +} + +// redactSubject reduces a user utterance to something safe to persist in a +// trace: its length only. Traces are diagnostics, and his words are not +// diagnostics — the correlation ID is what ties a trace to the turn. +func redactSubject(s string) string { + return fmt.Sprintf("<%d chars>", len([]rune(s))) +} + +// recordEcosystemTrace writes one hop of a cross-service call: which service, +// which operation, the outcome, how long it took, and the correlation ID that +// stitches the hops together. It is written for every outcome, not only +// success — an unrecorded failure is exactly the hop you need when something +// went wrong at 3am. +// +// Traces go to their own store table, never to facts. One act turn produces +// three or four of them, at machine rate, while facts arrive at human rate: +// sharing the table meant the habit profile's 2000-row window, memeval's +// prompt snapshot and the /dash and /history pages all filled with traces and +// stopped seeing his actual facts. +func (h *reactiveHandler) recordEcosystemTrace(ctx context.Context, service, op, status string, started time.Time, fields map[string]any) { + if h.dataStore == nil { + return + } + tr := store.EcosystemTrace{ + Ts: h.now(), + Service: service, + Operation: op, + Status: status, + DurationMs: h.now().Sub(started).Milliseconds(), + CorrelationID: correlationIDFromCtx(ctx), + Fields: map[string]any{}, + } + for k, v := range fields { + switch k { + case "causation_id": + tr.CausationID, _ = v.(string) + case "http_status": + if n, ok := v.(int); ok { + tr.HTTPStatus = n + continue + } + tr.Fields[k] = v + default: + tr.Fields[k] = v + } + } + if _, err := h.dataStore.WriteEcosystemTrace(ctx, tr); err != nil { + log.Printf("ecosystem: record trace %s:%s: %v", service, op, err) + } +} + +// unauthorizedEcosystemError reports a credential the far side rejected. It +// gets its own reply: a missing or wrong token looks exactly like an outage to +// him, and "try again" is advice that will never work. +func unauthorizedEcosystemError(err error) bool { + var ee *ecosystemError + return errors.As(err, &ee) && ee.Unauthorized() +} + +// traceErrorFields describes an ecosystemError for a trace without leaking the +// payload: the HTTP status and the failure class, nothing else. +func traceErrorFields(err error) map[string]any { + fields := map[string]any{} + var ee *ecosystemError + if errors.As(err, &ee) { + fields["http_status"] = ee.Status + switch { + case ee.Unauthorized(): + fields["class"] = "unauthorized" + case ee.ContractMismatch(): + fields["class"] = "contract_mismatch" + case ee.Unreachable(): + fields["class"] = "unreachable" + default: + fields["class"] = "error" + } + return fields + } + fields["class"] = "error" + return fields +} + +// handleHexisAct — resolves entity references through Nexus and executes +// matching capabilities through Hexis. Returns a reply string when handled, +// or "" to fall through to the system command executor. +func (h *reactiveHandler) handleHexisAct(ctx context.Context, dec router.Decision) string { + if h.ecosystem == nil { + return "" + } + + // Every hop of this action shares one correlation ID, assigned here so + // resolution and discovery are traceable even when execution never + // happens. + if correlationIDFromCtx(ctx) == "" { + ctx = withCorrelationID(ctx, newCorrelationID()) + } + + // Resolve the utterance text as an entity reference through Nexus. An + // ambiguous match must stop and clarify — never guess a mutation target. + started := h.now() + entityID, displayName, ambiguous, err := h.ecosystem.resolveEntityReference(ctx, dec.Slots.Text, nil) + if err != nil { + h.recordEcosystemTrace(ctx, "nexus", "resolve", traceStatusForError(err), started, + mergeFields(traceErrorFields(err), map[string]any{"subject": redactSubject(dec.Slots.Text)})) + if unauthorizedEcosystemError(err) { + return "экосистема отклоняет доступ, проверь токен." + } + // A genuine Nexus dependency failure, not "no such entity" — stop here + // and report degradation rather than silently falling through to the + // local command executor (ECOSYSTEM-SPEC.md: services degrade + // independently, never a silent all-clear). + return "экосистема недоступна, попробуй ещё раз." + } + if len(ambiguous) > 0 { + h.recordEcosystemTrace(ctx, "nexus", "resolve", traceAmbig, started, + map[string]any{"candidates": len(ambiguous)}) + return "уточни, что именно: " + strings.Join(ambiguous, ", ") + "?" + } + if entityID == "" { + h.recordEcosystemTrace(ctx, "nexus", "resolve", traceNotFound, started, + map[string]any{"subject": redactSubject(dec.Slots.Text)}) + return "" + } + h.recordEcosystemTrace(ctx, "nexus", "resolve", traceOK, started, + map[string]any{"entity_id": entityID}) + + // Discover Hexis capabilities for this entity. A resolved entity with a + // genuine Hexis failure must not be treated as "no capabilities" and + // fall through to unrelated local execution. + discovered := h.now() + caps, err := h.ecosystem.discoverCapabilities(ctx, entityID) + if err != nil { + h.recordEcosystemTrace(ctx, "hexis", "capabilities", traceStatusForError(err), discovered, + mergeFields(traceErrorFields(err), map[string]any{"entity_id": entityID})) + if unauthorizedEcosystemError(err) { + return "экосистема отклоняет доступ, проверь токен." + } + return "экосистема недоступна, попробуй ещё раз." + } + h.recordEcosystemTrace(ctx, "hexis", "capabilities", traceOK, discovered, + map[string]any{"entity_id": entityID, "count": len(caps)}) + if len(caps) == 0 { + return "" + } + + // Match the user's verb to a capability by name/description. Collect all + // matches: more than one is itself ambiguous, so we ask rather than pick + // the first (ecosystem invariant: no arbitrary target for mutation). + verb := dec.Slots.Fn + if verb == "" { + verb = dec.Slots.Text + } + verbLower := strings.ToLower(verb) + + var matches []*hexisclient.Capability + for i, c := range caps { + if strings.Contains(strings.ToLower(c.Name), verbLower) || + (c.Description != "" && strings.Contains(strings.ToLower(c.Description), verbLower)) { + matches = append(matches, &caps[i]) + } + } + if len(matches) == 0 { + return "" + } + if len(matches) > 1 { + var names []string + for _, m := range matches { + names = append(names, m.Name) + } + return "какую команду для " + displayName + ": " + strings.Join(names, ", ") + "?" + } + matched := matches[0] + + // Read-only capabilities run immediately; mutating ones are parked for an + // explicit spoken confirm bound to this capability + target. + if !matched.ReadOnly { + h.mu.Lock() + h.pendingHexis = &pendingHexisExec{ + capabilityID: matched.ID, + capName: matched.Name, + entityID: entityID, + displayName: displayName, + expiry: h.now().Add(confirmTTL), + } + h.mu.Unlock() + h.recordEcosystemTrace(ctx, "hexis", "confirmation", tracePending, started, + map[string]any{"entity_id": entityID, "capability": matched.Name}) + return "выполнить «" + matched.Name + "» для " + displayName + "? скажи «да» или «нет»." + } + + return h.execHexis(ctx, matched.ID, matched.Name, entityID, displayName) +} + +// execHexis runs a resolved capability and records a cross-service trace with +// the correlation ID. It reports command success, never operational recovery +// (Praxis observes recovery independently). +func (h *reactiveHandler) execHexis(ctx context.Context, capID, capName, entityID, displayName string) string { + started := h.now() + causationID := correlationIDFromCtx(ctx) + correlationID, err := h.ecosystem.executeCapability(ctx, capID, entityID, nil) + traced := withCorrelationID(ctx, correlationID) + if err != nil { + log.Printf("ecosystem: hexis execute error (cor=%s): %v", correlationID, err) + h.recordEcosystemTrace(traced, "hexis", "execute", traceStatusForError(err), started, + mergeFields(traceErrorFields(err), map[string]any{ + "entity_id": entityID, "capability": capName, "causation_id": causationID, + })) + return "не получилось выполнить команду для " + displayName + "." + } + // One record per hop: the second write this used to make said the same + // thing under a different key, in a different shape. + h.recordEcosystemTrace(traced, "hexis", "execute", traceOK, started, map[string]any{ + "entity_id": entityID, "entity_name": displayName, + "capability": capName, "causation_id": causationID, + }) + return "команда выполнена для " + displayName + "." +} diff --git a/cmd/mavend/ecosystem_auth_test.go b/cmd/mavend/ecosystem_auth_test.go new file mode 100644 index 0000000..41c748f --- /dev/null +++ b/cmd/mavend/ecosystem_auth_test.go @@ -0,0 +1,65 @@ +package main + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/kami/maven/internal/config" +) + +// TestWireEcosystem_HexisToken — a configured Hexis token reaches the wire. +// +// This is the regression that closes the 2026-08-01 re-vendor. The copy of +// github.com/kami/hexis checked into vendor/ used to predate Client.WithToken, +// so a configured token could not be sent at all; wireEcosystem refused to wire +// Hexis rather than execute unauthenticated. Both halves of that are gone. The +// test asserts the outcome the refusal was standing in for: the header goes +// out, so nobody has to trust a boot log to know auth is on. +func TestWireEcosystem_HexisToken(t *testing.T) { + var gotAuth string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuth = r.Header.Get("Authorization") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`[]`)) + })) + defer srv.Close() + + cfg := &config.Config{Hexis: &config.HexisConfig{URL: srv.URL, Token: "s3cret"}} + w := wireEcosystem(cfg) + if w.hexis == nil { + t.Fatal("hexis not wired with a token configured") + } + if _, err := w.discoverCapabilities(context.Background(), "entity-1"); err != nil { + t.Fatalf("discoverCapabilities: %v", err) + } + if want := "Bearer s3cret"; gotAuth != want { + t.Errorf("Authorization = %q; want %q", gotAuth, want) + } +} + +// TestWireEcosystem_HexisNoToken — no token configured still wires, unauthed. +// Hexis without auth is a valid deployment on a trusted box, and the re-vendor +// must not have turned the token into a requirement. +func TestWireEcosystem_HexisNoToken(t *testing.T) { + var sawAuth bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + sawAuth = r.Header.Get("Authorization") != "" + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`[]`)) + })) + defer srv.Close() + + cfg := &config.Config{Hexis: &config.HexisConfig{URL: srv.URL}} + w := wireEcosystem(cfg) + if w.hexis == nil { + t.Fatal("hexis not wired without a token") + } + if _, err := w.discoverCapabilities(context.Background(), "entity-1"); err != nil { + t.Fatalf("discoverCapabilities: %v", err) + } + if sawAuth { + t.Error("Authorization header sent with no token configured") + } +} diff --git a/cmd/mavend/ecosystem_degraded_test.go b/cmd/mavend/ecosystem_degraded_test.go new file mode 100644 index 0000000..383959e --- /dev/null +++ b/cmd/mavend/ecosystem_degraded_test.go @@ -0,0 +1,468 @@ +package main + +import ( + "context" + "strings" + "testing" + "time" + + hexisclient "github.com/kami/hexis/pkg/client" + + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/store" +) + +// Phase-5 hardening suite (Vikunja #276). Everything here drives the shared +// fake ecosystem (fakeecosystem_test.go) rather than one-off inline handlers, +// so the same fault levers — SetFault, SetBody, SetDelay — cover every +// service. What is asserted is the degraded-mode contract: +// +// - services degrade independently: one outage never mutes the others, +// - a degraded reply is never silent, never fabricated, never "success", +// - contract drift (old shape, unknown fields, garbage) is survivable, +// - Maven never acts on an ambiguous target and never chains +// Praxis observation into Hexis execution on its own. + +// ecoHandler wires a handler against whichever of the three fakes is given +// (pass nil to leave a service unconfigured, which is a different state from +// "configured but down"). +func ecoHandler(t *testing.T, nexus, praxis, hexis *fakeServer) *reactiveHandler { + t.Helper() + st := newTestStore(t) + clock := newTickingClock(time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC), time.Millisecond) + w := &ecosystemWiring{} + if nexus != nil { + w.nexus = newNexusClient(nexus.URL) + } + if praxis != nil { + w.praxis = newPraxisClient(praxis.URL) + } + if hexis != nil { + w.hexis = hexisclient.New(hexis.URL) + } + return &reactiveHandler{ + api: ipc.NewStoreAPI(st), + dataStore: st, + now: clock.Now, + ecosystem: w, + } +} + +// traces reads the ecosystem trace table. Traces live there and not in facts, +// so a bounded reader of facts never fills up with machine-rate rows. +func traces(t *testing.T, h *reactiveHandler) []store.EcosystemTrace { + t.Helper() + out, err := h.dataStore.RecentEcosystemTraces(context.Background(), 100) + if err != nil { + t.Fatalf("read traces: %v", err) + } + return out +} + +// tracesFor returns the traces recorded for one service+operation. +func tracesFor(t *testing.T, h *reactiveHandler, service, op string) []store.EcosystemTrace { + t.Helper() + var out []store.EcosystemTrace + for _, tr := range traces(t, h) { + if tr.Service == service && tr.Operation == op { + out = append(out, tr) + } + } + return out +} + +// restartCaps is a read-only capability. Restarting a service is a mutation, +// so the read-only one this suite runs through the happy paths is named for +// what it is; the mutating restart lives in the confirmation tests. +func restartCaps() string { + return fixtureHexisCapabilities(map[string]any{ + "id": "cap_status", "name": "restart status", "read_only": true, + }) +} + +// TestEcosystem_OutagesLeaveNoSharedFailureState: the two act paths share a +// handler, a store and a clock, so what is worth asserting is that a failure +// on one leaves nothing behind that degrades the other. Faulting one disjoint +// call graph and exercising the other only tests the call graph. +func TestEcosystem_OutagesLeaveNoSharedFailureState(t *testing.T) { + ctx := context.Background() + nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) + praxis := newFakePraxis(t, fixturePraxisAttentionItems(map[string]any{ + "id": "item_1", "title": "disk almost full", "importance": 3.0, + })) + hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded")) + h := ecoHandler(t, nexus, praxis, hexis) + + // A Nexus outage during a Hexis act writes a failure trace, and a shared + // store is the one thing the Praxis path could inherit it through. + nexus.SetFault(503) + if reply := h.handleHexisAct(ctx, actDec("muzick indexer")); strings.Contains(reply, "выполнена") { + t.Fatalf("nexus outage must not report success, got %q", reply) + } + if len(tracesFor(t, h, "nexus", "resolve")) == 0 { + t.Fatal("the failed resolve must be recorded") + } + + nexus.SetFault(0) + reply := h.handlePraxisAct(ctx, praxisActDec("list_attention")) + if !strings.Contains(reply, "disk almost full") { + t.Fatalf("a recorded nexus failure must not degrade the praxis digest, got %q", reply) + } + if got := tracesFor(t, h, "praxis", "list_attention"); len(got) != 1 || got[0].Status != traceOK { + t.Fatalf("the praxis digest must trace its own success, got %+v", got) + } + + // And the reverse: a Praxis outage mid-session leaves the Hexis path whole. + praxis.SetFault(503) + if reply := h.handlePraxisAct(ctx, praxisActDec("list_attention")); strings.Contains(reply, "disk") { + t.Fatalf("praxis outage must not serve content, got %q", reply) + } + if reply := h.handleHexisAct(ctx, actDec("muzick indexer")); !strings.Contains(reply, "выполнена") { + t.Fatalf("a praxis outage must not block the hexis path, got %q", reply) + } +} + +// TestEcosystem_OneEndpointDownDoesNotMuteTheService: real outages are usually +// partial. Attention answering while surface is down must still deliver. +func TestEcosystem_OneEndpointDownDoesNotMuteTheService(t *testing.T) { + ctx := context.Background() + praxis := newFakePraxis(t, fixturePraxisAttentionItems(map[string]any{ + "id": "item_1", "title": "disk almost full", "importance": 3.0, + })) + h := ecoHandler(t, nil, praxis, nil) + + praxis.SetRouteFault("/api/v1/tools/surface", 503) + reply := h.handlePraxisAct(ctx, praxisActDec("list_attention")) + if !strings.Contains(reply, "disk almost full") { + t.Fatalf("a downed surface endpoint must not mute the digest, got %q", reply) + } + if praxis.Count("POST", "/api/v1/tools/surface") == 0 { + t.Fatal("expected the surface attempt") + } +} + +// TestEcosystem_ResolvedWithoutEntityFailsClosed: the contract violation that +// decodes cleanly. Nexus says "resolved" and delivers no entity; treating that +// as "no such entity" put the user's verb through to the local executor. +func TestEcosystem_ResolvedWithoutEntityFailsClosed(t *testing.T) { + ctx := context.Background() + nexus := newFakeNexus(t, fixtureNexusResolvedEmpty()) + hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded")) + h := ecoHandler(t, nexus, nil, hexis) + + reply := h.handleHexisAct(ctx, actDec("muzick indexer")) + if reply == "" { + t.Fatal("a resolve with no entity must degrade, not fall through to local execution") + } + if strings.Contains(reply, "выполнена") { + t.Fatalf("a resolve with no entity must not report success, got %q", reply) + } + if hexis.Count("", "/api/v1") != 0 { + t.Fatal("hexis must not be contacted after a contract-violating resolve") + } +} + +// TestEcosystem_RejectedCredentialSaysSo: 401 and 403 must not read as an +// outage. "Try again" is advice that never works for a misconfigured token. +func TestEcosystem_RejectedCredentialSaysSo(t *testing.T) { + ctx := context.Background() + for _, status := range []int{401, 403} { + nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) + hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded")) + h := ecoHandler(t, nexus, nil, hexis) + nexus.SetFault(status) + + reply := h.handleHexisAct(ctx, actDec("muzick indexer")) + if !strings.Contains(reply, "токен") { + t.Fatalf("http %d must read as a credential problem, got %q", status, reply) + } + tr := tracesFor(t, h, "nexus", "resolve") + if len(tr) != 1 || tr[0].Status != traceRefused || tr[0].HTTPStatus != status { + t.Fatalf("http %d must trace as refused with its status, got %+v", status, tr) + } + } +} + +// TestEcosystem_MalformedPraxisBodyDegrades: Praxis has the same decode path +// Nexus does, and a 200 carrying garbage there is a dependency failure too. +func TestEcosystem_MalformedPraxisBodyDegrades(t *testing.T) { + ctx := context.Background() + praxis := newFakePraxis(t, fixturePraxisAttentionItems(map[string]any{ + "id": "item_1", "title": "disk almost full", "importance": 3.0, + })) + h := ecoHandler(t, nil, praxis, nil) + + praxis.SetBody(`[{"title":`) + reply := h.handlePraxisAct(ctx, praxisActDec("list_attention")) + if reply == "" { + t.Fatal("a malformed praxis body must not answer with silence") + } + if strings.Contains(reply, "disk almost full") { + t.Fatalf("a malformed body must not produce content, got %q", reply) + } +} + +// TestEcosystem_MalformedNexusResponseFailsClosed: a 200 carrying garbage is a +// dependency failure, not "no such entity". It must stop before Hexis. +func TestEcosystem_MalformedNexusResponseFailsClosed(t *testing.T) { + ctx := context.Background() + nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) + hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded")) + h := ecoHandler(t, nexus, nil, hexis) + + nexus.SetBody(`{"status":"resolved","entity":`) + reply := h.handleHexisAct(ctx, actDec("muzick indexer")) + if reply == "" || strings.Contains(reply, "выполнена") { + t.Fatalf("malformed nexus body must degrade, got %q", reply) + } + if hexis.Count("", "/api/v1") != 0 { + t.Fatal("hexis must not be contacted after a malformed nexus response") + } +} + +// TestEcosystem_UnknownContractFieldsTolerated: a newer Nexus adding fields +// must not break an older Maven. Same for the older flat resolve shape. +func TestEcosystem_UnknownContractFieldsTolerated(t *testing.T) { + ctx := context.Background() + for name, body := range map[string]string{ + "future": fixtureNexusResolvedFuture("ent_muzick", "Muzick indexer", "service"), + "flat": fixtureNexusResolvedFlat("ent_muzick", "Muzick indexer", "service"), + } { + t.Run(name, func(t *testing.T) { + nexus := newFakeNexus(t, body) + hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded")) + h := ecoHandler(t, nexus, nil, hexis) + if reply := h.handleHexisAct(ctx, actDec("muzick indexer")); !strings.Contains(reply, "выполнена") { + t.Fatalf("%s contract shape must still resolve and execute, got %q", name, reply) + } + }) + } +} + +// TestEcosystem_CancelledContextDegrades: a caller hanging up (turn abandoned, +// deadline hit) must surface as degradation, never as a fabricated result. +func TestEcosystem_CancelledContextDegrades(t *testing.T) { + nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) + hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded")) + h := ecoHandler(t, nexus, nil, hexis) + nexus.SetDelay(2 * time.Second) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) + defer cancel() + reply := h.handleHexisAct(ctx, actDec("muzick indexer")) + if reply == "" || strings.Contains(reply, "выполнена") { + t.Fatalf("cancelled resolve must degrade, got %q", reply) + } + if hexis.Count("", "/api/v1") != 0 { + t.Fatal("hexis must not be contacted after a cancelled resolve") + } +} + +// TestEcosystem_ExecutionFailureIsNotSuccess: Hexis answering 200 with +// status=failed is a partial failure — the call worked, the command did not. +// Maven must report it as a failure and must not write a success trace. +func TestEcosystem_ExecutionFailureIsNotSuccess(t *testing.T) { + ctx := context.Background() + nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) + hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecutionFailed("exec_1", "unit not found")) + h := ecoHandler(t, nexus, nil, hexis) + + reply := h.handleHexisAct(ctx, actDec("muzick indexer")) + if strings.Contains(reply, "выполнена") { + t.Fatalf("failed execution must not read as success, got %q", reply) + } + if reply == "" { + t.Fatal("failed execution must say something") + } + for _, tr := range tracesFor(t, h, "hexis", "execute") { + if tr.Status == traceOK { + t.Fatalf("failed execution must not write a success trace: %+v", tr) + } + } +} + +// TestEcosystem_SuccessfulActionWritesATrace is the positive half the failure +// assertions above depend on: without it, "no success trace" passes with the +// trace writer deleted. It was, for a while — both writers used a fact kind the +// store's CHECK constraint rejects and the error was discarded. +func TestEcosystem_SuccessfulActionWritesATrace(t *testing.T) { + ctx := context.Background() + nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) + hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded")) + h := ecoHandler(t, nexus, nil, hexis) + + if reply := h.handleHexisAct(ctx, actDec("muzick indexer")); !strings.Contains(reply, "выполнена") { + t.Fatalf("setup: expected success, got %q", reply) + } + exec := tracesFor(t, h, "hexis", "execute") + if len(exec) != 1 || exec[0].Status != traceOK { + t.Fatalf("a successful execution must leave exactly one ok trace, got %+v", exec) + } + if exec[0].CorrelationID == "" { + t.Error("a trace with no correlation id cannot be stitched to anything") + } +} + +// TestEcosystem_TracesStayOutOfFacts: traces are written at machine rate and +// facts at human rate. One act turn used to write four fact rows, which pushed +// his facts out of every bounded reader (the habit profile's window, memeval's +// prompt, /dash, /history). +func TestEcosystem_TracesStayOutOfFacts(t *testing.T) { + ctx := context.Background() + nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) + hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded")) + h := ecoHandler(t, nexus, nil, hexis) + + if reply := h.handleHexisAct(ctx, actDec("muzick indexer")); !strings.Contains(reply, "выполнена") { + t.Fatalf("setup: expected success, got %q", reply) + } + if len(traces(t, h)) == 0 { + t.Fatal("setup: expected traces") + } + facts, err := h.dataStore.RecentFacts(ctx, 100) + if err != nil { + t.Fatalf("read facts: %v", err) + } + if len(facts) != 0 { + t.Fatalf("an ecosystem act must write no facts at all, got %+v", facts) + } +} + +// TestEcosystem_AmbiguousTargetBlocksExecution: ambiguity blocks mutation, and +// the clarification must name the candidates rather than pick one. +func TestEcosystem_AmbiguousTargetBlocksExecution(t *testing.T) { + ctx := context.Background() + nexus := newFakeNexus(t, fixtureNexusAmbiguous( + map[string]string{"entity_id": "ent_a", "display_name": "Muzick indexer"}, + map[string]string{"entity_id": "ent_b", "display_name": "Muzick web"}, + )) + hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded")) + h := ecoHandler(t, nexus, nil, hexis) + + reply := h.handleHexisAct(ctx, actDec("muzick")) + if !strings.Contains(reply, "Muzick indexer") || !strings.Contains(reply, "Muzick web") { + t.Fatalf("ambiguous resolve must list candidates, got %q", reply) + } + if hexis.Count("POST", "/api/v1/execute") != 0 { + t.Fatal("ambiguous target must never execute") + } +} + +// TestEcosystem_NoAutonomousPraxisToHexis: reading the attention digest is an +// observation. Maven must never turn an observed problem into a Hexis command +// by herself — she is not autonomous. +func TestEcosystem_NoAutonomousPraxisToHexis(t *testing.T) { + ctx := context.Background() + praxis := newFakePraxis(t, fixturePraxisAttentionItems( + map[string]any{"id": "item_1", "title": "muzick indexer is down", "importance": 4.0, "rule": "service_down"}, + )) + nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) + hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded")) + h := ecoHandler(t, nexus, praxis, hexis) + + _ = h.handlePraxisAct(ctx, praxisActDec("list_attention")) + if hexis.Count("", "/api/v1") != 0 { + t.Fatal("attention digest must not contact hexis on its own") + } + if nexus.Count("", "/api/v1/resolve") != 0 { + t.Fatal("attention digest must not resolve targets for autonomous action") + } +} + +// TestEcosystem_MutatingCapabilityWaitsForConfirmation: a non-read-only +// capability parks for an explicit spoken confirm bound to capability+target. +func TestEcosystem_MutatingCapabilityWaitsForConfirmation(t *testing.T) { + ctx := context.Background() + nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) + caps := fixtureHexisCapabilities(map[string]any{"id": "cap_restart", "name": "restart", "read_only": false}) + hexis := newFakeHexis(t, caps, fixtureHexisExecuted("exec_1", "succeeded")) + h := ecoHandler(t, nexus, nil, hexis) + + reply := h.handleHexisAct(ctx, actDec("restart")) + if !strings.Contains(reply, "restart") || !strings.Contains(reply, "да") { + t.Fatalf("mutating capability must ask for confirmation, got %q", reply) + } + if hexis.Count("POST", "/api/v1/execute") != 0 { + t.Fatal("mutating capability must not execute before confirmation") + } + h.mu.Lock() + pending := h.pendingHexis + h.mu.Unlock() + if pending == nil || pending.capabilityID != "cap_restart" || pending.entityID != "ent_muzick" { + t.Fatalf("confirmation must be bound to capability+target, got %+v", pending) + } +} + +// TestEcosystem_SurfaceFailureStillDelivers: surfacing is bookkeeping. If the +// surface call fails the digest must still be spoken — a partial failure +// downgrades bookkeeping, not the answer. +func TestEcosystem_SurfaceFailureStillDelivers(t *testing.T) { + ctx := context.Background() + praxis := newFakePraxis(t, fixturePraxisAttentionItems( + map[string]any{"id": "item_1", "title": "disk almost full", "importance": 3.0}, + )) + praxis.SetRouteFault("/api/v1/tools/surface", 500) + h := ecoHandler(t, nil, praxis, nil) + + reply := h.handlePraxisAct(ctx, praxisActDec("list_attention")) + if !strings.Contains(reply, "disk almost full") { + t.Fatalf("failed surface must not swallow the digest, got %q", reply) + } + if praxis.Count("POST", "/api/v1/tools/surface") == 0 { + t.Fatal("expected the surface attempt") + } +} + +// TestEcosystem_TotalOutageSaysSoForEveryPath: with all three down, every +// entry point degrades explicitly instead of returning empty or inventing. +func TestEcosystem_TotalOutageSaysSoForEveryPath(t *testing.T) { + ctx := context.Background() + nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) + praxis := newFakePraxis(t, fixturePraxisAttentionItems()) + hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded")) + for _, fs := range []*fakeServer{nexus, praxis, hexis} { + fs.SetFault(503) + } + h := ecoHandler(t, nexus, praxis, hexis) + + for name, reply := range map[string]string{ + "hexis act": h.handleHexisAct(ctx, actDec("muzick indexer")), + "attention": h.handlePraxisAct(ctx, praxisActDec("list_attention")), + "changes": h.handlePraxisAct(ctx, praxisActDec("list_changes")), + "acknowledge": h.handlePraxisAct(ctx, praxisItemDec("acknowledge_item", "item_1")), + } { + if reply == "" { + t.Errorf("%s: total outage must not answer with silence", name) + } + if strings.Contains(reply, "выполнена") { + t.Errorf("%s: total outage must not claim success: %q", name, reply) + } + } + for _, tr := range traces(t, h) { + if tr.Status == traceOK { + t.Fatalf("a total outage must not leave success traces behind: %+v", tr) + } + } + if len(tracesFor(t, h, "praxis", "acknowledge")) == 0 { + t.Fatal("the acknowledge arm must reach praxis and record the refusal") + } +} + +// TestEcosystem_RecoveryAfterOutageNeedsNoRestart: once the dependency comes +// back the very next turn works — no cached failure state, no restart. +func TestEcosystem_RecoveryAfterOutageNeedsNoRestart(t *testing.T) { + ctx := context.Background() + praxis := newFakePraxis(t, fixturePraxisAttentionItems( + map[string]any{"id": "item_1", "title": "disk almost full", "importance": 3.0}, + )) + h := ecoHandler(t, nil, praxis, nil) + + praxis.SetFault(503) + if reply := h.handlePraxisAct(ctx, praxisActDec("list_attention")); strings.Contains(reply, "disk") { + t.Fatalf("outage must not serve content, got %q", reply) + } + praxis.SetFault(0) + if reply := h.handlePraxisAct(ctx, praxisActDec("list_attention")); !strings.Contains(reply, "disk almost full") { + t.Fatalf("recovery must work on the next turn, got %q", reply) + } +} diff --git a/cmd/mavend/ecosystem_harness_test.go b/cmd/mavend/ecosystem_harness_test.go index 1c6e375..521d220 100644 --- a/cmd/mavend/ecosystem_harness_test.go +++ b/cmd/mavend/ecosystem_harness_test.go @@ -19,6 +19,13 @@ func praxisActDec(fn string) router.Decision { return router.Decision{Intent: router.IntentAct, Slots: router.Slots{Fn: fn, HasFn: true}} } +// praxisItemDec is praxisActDec for the lifecycle verbs, which need an item id +// in the value slot. Without one they answer "which item?" and never reach +// Praxis at all, which makes them useless for testing a Praxis outage. +func praxisItemDec(fn, itemID string) router.Decision { + return router.Decision{Intent: router.IntentAct, Slots: router.Slots{Fn: fn, HasFn: true, Value: itemID}} +} + func newPraxisTestHandler(t *testing.T, praxis *fakeServer) *reactiveHandler { t.Helper() st := newTestStore(t) diff --git a/cmd/mavend/ecosystem_test.go b/cmd/mavend/ecosystem_test.go index b603560..4fc557f 100644 --- a/cmd/mavend/ecosystem_test.go +++ b/cmd/mavend/ecosystem_test.go @@ -59,8 +59,11 @@ func newHexisTestHandler(t *testing.T, resolveBody string, caps string) (*reacti }, executed } -func actDec(text string) router.Decision { - return router.Decision{Intent: router.IntentAct, Slots: router.Slots{Text: text, Fn: "restart", HasFn: true}} +// actDec builds an act decision about subject. The verb is always "restart": +// the argument is the utterance the entity is resolved from, never the verb, +// so actDec("restart") reads as a verb and is not one. +func actDec(subject string) router.Decision { + return router.Decision{Intent: router.IntentAct, Slots: router.Slots{Text: subject, Fn: "restart", HasFn: true}} } func TestHexisMutatingRequiresConfirm(t *testing.T) { diff --git a/cmd/mavend/ecosystem_trace_test.go b/cmd/mavend/ecosystem_trace_test.go new file mode 100644 index 0000000..86d3b1c --- /dev/null +++ b/cmd/mavend/ecosystem_trace_test.go @@ -0,0 +1,316 @@ +package main + +import ( + "context" + "strings" + "testing" + + "github.com/kami/maven/internal/store" +) + +// Versioning, authentication and tracing of ecosystem calls (Vikunja #273). + +func findTrace(t *testing.T, h *reactiveHandler, service, op string) *store.EcosystemTrace { + t.Helper() + for _, tr := range traces(t, h) { + if tr.Service == service && tr.Operation == op { + found := tr + return &found + } + } + return nil +} + +// TestEcosystemHeaders_VersionRequesterAndAuth: every outgoing request carries +// the contract version, the requester, and the bearer token when configured. +func TestEcosystemHeaders_VersionRequesterAndAuth(t *testing.T) { + ctx := context.Background() + nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) + praxis := newFakePraxis(t, fixturePraxisAttentionItems()) + h := ecoHandler(t, nexus, praxis, nil) + h.ecosystem.nexus = newNexusClient(nexus.URL).withToken("nexus-secret") + h.ecosystem.praxis = newPraxisClient(praxis.URL).withToken("praxis-secret") + + _, _, _, err := h.ecosystem.resolveEntityReference(ctx, "muzick indexer", nil) + if err != nil { + t.Fatalf("resolve: %v", err) + } + // A bare client call carries whatever the caller assigned. Entry points + // assign the ID, the header layer only reads it, so mirror an action here. + if _, err := h.ecosystem.praxis.ListAttention(withCorrelationID(ctx, newCorrelationID()), 5); err != nil { + t.Fatalf("attention: %v", err) + } + + for _, tc := range []struct { + fs *fakeServer + versionHeader string + token string + }{ + {nexus, "X-Nexus-Version", "nexus-secret"}, + {praxis, "X-Praxis-Version", "praxis-secret"}, + } { + reqs := tc.fs.Requests() + if len(reqs) == 0 { + t.Fatalf("%s: no request captured", tc.versionHeader) + } + r := reqs[0] + if got := r.Header.Get(tc.versionHeader); got != ecosystemAPIVersion { + t.Errorf("%s = %q, want %q", tc.versionHeader, got, ecosystemAPIVersion) + } + if got := r.Header.Get("X-Requested-By"); got != mavenRequester { + t.Errorf("X-Requested-By = %q, want %q", got, mavenRequester) + } + if got := r.Header.Get("Authorization"); got != "Bearer "+tc.token { + t.Errorf("Authorization = %q, want bearer %q", got, tc.token) + } + if r.Header.Get("X-Correlation-ID") == "" { + t.Errorf("%s: missing correlation ID", tc.versionHeader) + } + } +} + +// TestEcosystemHeaders_NoTokenSendsNoAuth: an unconfigured token means the +// transport is trusted, not that a bogus header is sent. +func TestEcosystemHeaders_NoTokenSendsNoAuth(t *testing.T) { + ctx := context.Background() + nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) + h := ecoHandler(t, nexus, nil, nil) + + if _, _, _, err := h.ecosystem.resolveEntityReference(ctx, "muzick indexer", nil); err != nil { + t.Fatalf("resolve: %v", err) + } + if got := nexus.Requests()[0].Header.Get("Authorization"); got != "" { + t.Fatalf("unauthenticated client must send no Authorization header, got %q", got) + } +} + +// TestEcosystemError_ClassifiesRefusals: callers must be able to tell a +// rejected credential from a version refusal from an unreachable service +// without matching on message text. +func TestEcosystemError_ClassifiesRefusals(t *testing.T) { + ctx := context.Background() + for _, tc := range []struct { + name string + status int + check func(*ecosystemError) bool + wantCls string + }{ + {"unauthorized", 401, (*ecosystemError).Unauthorized, "unauthorized"}, + {"forbidden", 403, (*ecosystemError).Unauthorized, "unauthorized"}, + {"contract", 426, (*ecosystemError).ContractMismatch, "contract_mismatch"}, + } { + t.Run(tc.name, func(t *testing.T) { + nexus := newFakeNexus(t, fixtureNexusResolved("ent_x", "X", "service")) + nexus.SetFault(tc.status) + c := newNexusClient(nexus.URL) + _, err := c.Resolve(ctx, "x", nil) + ee, ok := err.(*ecosystemError) + if !ok { + t.Fatalf("expected *ecosystemError, got %T (%v)", err, err) + } + if ee.Service != "nexus" || ee.Status != tc.status { + t.Fatalf("unexpected typed error %+v", ee) + } + if !tc.check(ee) { + t.Fatalf("%s not classified: %+v", tc.name, ee) + } + if got := traceErrorFields(err)["class"]; got != tc.wantCls { + t.Fatalf("trace class = %v, want %s", got, tc.wantCls) + } + }) + } +} + +func TestEcosystemError_UnreachableHasNoStatus(t *testing.T) { + c := newNexusClient("http://127.0.0.1:1") + _, err := c.Resolve(context.Background(), "x", nil) + ee, ok := err.(*ecosystemError) + if !ok { + t.Fatalf("expected *ecosystemError, got %T", err) + } + if !ee.Unreachable() || ee.Unauthorized() || ee.ContractMismatch() { + t.Fatalf("a refused connection must classify as unreachable only: %+v", ee) + } +} + +// TestEcosystemTrace_SuccessfulActionTracesEveryHop: resolution, discovery and +// execution each leave a record sharing one correlation chain, with timing and +// status, and execution carries the causation link back to the resolve. +func TestEcosystemTrace_SuccessfulActionTracesEveryHop(t *testing.T) { + ctx := context.Background() + nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) + hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded")) + h := ecoHandler(t, nexus, nil, hexis) + + if reply := h.handleHexisAct(ctx, actDec("muzick indexer")); !strings.Contains(reply, "выполнена") { + t.Fatalf("setup: expected success, got %q", reply) + } + + var chain string + for _, want := range [][2]string{{"nexus", "resolve"}, {"hexis", "capabilities"}, {"hexis", "execute"}} { + d := findTrace(t, h, want[0], want[1]) + if d == nil { + t.Fatalf("missing trace for %s %s, got %+v", want[0], want[1], traces(t, h)) + } + if d.Status != traceOK { + t.Errorf("%s %s status = %v, want ok", want[0], want[1], d.Status) + } + if d.CorrelationID == "" { + t.Errorf("%s %s trace has no correlation id", want[0], want[1]) + } + if want[1] != "execute" { + if chain == "" { + chain = d.CorrelationID + } else if d.CorrelationID != chain { + t.Errorf("%s %s left the correlation chain: %s != %s", want[0], want[1], d.CorrelationID, chain) + } + } + } + exec := findTrace(t, h, "hexis", "execute") + if exec.CausationID == "" { + t.Error("execute trace must carry the causation id of the turn that caused it") + } + if exec.CorrelationID == exec.CausationID { + t.Error("execute correlation and causation must be distinguishable") + } +} + +// TestEcosystemTrace_OneCorrelationIDPerPraxisAction: a digest calls attention +// once and surface once per item. All of it is one turn, so the far side must +// see one ID and not N+1 unrelated ones. +func TestEcosystemTrace_OneCorrelationIDPerPraxisAction(t *testing.T) { + ctx := context.Background() + praxis := newFakePraxis(t, fixturePraxisAttentionItems( + map[string]any{"id": "item_1", "title": "disk almost full", "importance": 3.0}, + map[string]any{"id": "item_2", "title": "backup is stale", "importance": 2.0}, + )) + h := ecoHandler(t, nil, praxis, nil) + + if reply := h.handlePraxisAct(ctx, praxisActDec("list_attention")); !strings.Contains(reply, "disk almost full") { + t.Fatalf("setup: expected the digest, got %q", reply) + } + + reqs := praxis.Requests() + if len(reqs) < 3 { + t.Fatalf("expected attention plus one surface per item, got %d requests", len(reqs)) + } + first := reqs[0].Header.Get("X-Correlation-ID") + if first == "" { + t.Fatal("every ecosystem request must carry a correlation id") + } + for _, r := range reqs { + if got := r.Header.Get("X-Correlation-ID"); got != first { + t.Fatalf("%s %s carried %q, want the action's id %q", r.Method, r.Path, got, first) + } + } + tr := findTrace(t, h, "praxis", "list_attention") + if tr == nil || tr.CorrelationID != first { + t.Fatalf("the trace must carry the id that was actually sent, got %+v", tr) + } +} + +// TestEcosystemTrace_FailuresAreTracedToo: the whole point of the change — +// a failed hop is exactly the one worth having recorded. +func TestEcosystemTrace_FailuresAreTracedToo(t *testing.T) { + ctx := context.Background() + nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) + hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded")) + h := ecoHandler(t, nexus, nil, hexis) + nexus.SetFault(401) + + _ = h.handleHexisAct(ctx, actDec("muzick indexer")) + + d := findTrace(t, h, "nexus", "resolve") + if d == nil { + t.Fatal("a failed resolve must still be traced") + } + if d.Status != traceRefused { + t.Errorf("status = %v, want refused: the far side answered", d.Status) + } + if d.Fields["class"] != "unauthorized" { + t.Errorf("class = %v, want unauthorized", d.Fields["class"]) + } + if d.HTTPStatus != 401 { + t.Errorf("http_status = %v, want 401", d.HTTPStatus) + } +} + +// TestEcosystemTrace_UnreachableIsNotRefused: never got an answer and answered +// with a refusal are different failures, and the trace must say which. +func TestEcosystemTrace_UnreachableIsNotRefused(t *testing.T) { + ctx := context.Background() + h := ecoHandler(t, nil, nil, nil) + h.ecosystem.nexus = newNexusClient("http://127.0.0.1:1") + + _ = h.handleHexisAct(ctx, actDec("muzick indexer")) + + d := findTrace(t, h, "nexus", "resolve") + if d == nil { + t.Fatal("an unreachable resolve must still be traced") + } + if d.Status != traceFailed { + t.Errorf("status = %v, want failed", d.Status) + } + if d.Fields["class"] != "unreachable" { + t.Errorf("class = %v, want unreachable", d.Fields["class"]) + } +} + +// TestEcosystemTrace_RedactsTheUtterance: traces are diagnostics, his words +// are not. The subject must never be persisted verbatim. +func TestEcosystemTrace_RedactsTheUtterance(t *testing.T) { + ctx := context.Background() + nexus := newFakeNexus(t, fixtureNexusNotFound()) + h := ecoHandler(t, nexus, nil, nil) + + _ = h.handleHexisAct(ctx, actDec("перезапусти кофемашину")) + + recorded := traces(t, h) + if len(recorded) == 0 { + t.Fatal("expected a not_found resolve trace") + } + for _, tr := range recorded { + for k, v := range tr.Fields { + if s, ok := v.(string); ok && strings.Contains(s, "кофемашину") { + t.Fatalf("trace leaked the utterance in %s: %q", k, s) + } + } + } + d := findTrace(t, h, "nexus", "resolve") + if d.Status != traceNotFound { + t.Errorf("status = %v, want not_found", d.Status) + } + if d.Fields["subject"] != redactSubject("перезапусти кофемашину") { + t.Errorf("subject = %v, want a redacted length", d.Fields["subject"]) + } +} + +// TestEcosystemTrace_AmbiguityAndConfirmationAreRecorded: the two moments +// where Maven deliberately does not act still leave a trail. +func TestEcosystemTrace_AmbiguityAndConfirmationAreRecorded(t *testing.T) { + ctx := context.Background() + ambig := newFakeNexus(t, fixtureNexusAmbiguous( + map[string]string{"entity_id": "ent_a", "display_name": "Muzick indexer"}, + map[string]string{"entity_id": "ent_b", "display_name": "Muzick web"}, + )) + hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded")) + h := ecoHandler(t, ambig, nil, hexis) + _ = h.handleHexisAct(ctx, actDec("muzick")) + if d := findTrace(t, h, "nexus", "resolve"); d == nil || d.Status != traceAmbig { + t.Fatalf("ambiguous resolve must be traced as such, got %+v", d) + } + + nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) + mutating := fixtureHexisCapabilities(map[string]any{"id": "cap_restart", "name": "restart", "read_only": false}) + h2 := ecoHandler(t, nexus, nil, newFakeHexis(t, mutating, fixtureHexisExecuted("exec_1", "succeeded"))) + _ = h2.handleHexisAct(ctx, actDec("restart")) + d := findTrace(t, h2, "hexis", "confirmation") + if d == nil || d.Status != tracePending { + t.Fatalf("a parked confirmation must be traced, got %+v", d) + } + // The confirmation hop is measured from the top of the action, not from + // the instant it is recorded, which was always zero. + if d.DurationMs == 0 { + t.Error("the confirmation trace must report the time the action took to get there") + } +} diff --git a/cmd/mavend/entityrefs_test.go b/cmd/mavend/entityrefs_test.go new file mode 100644 index 0000000..4a4877f --- /dev/null +++ b/cmd/mavend/entityrefs_test.go @@ -0,0 +1,358 @@ +package main + +import ( + "context" + "database/sql" + "strings" + "testing" + "time" + + "github.com/kami/maven/internal/router" + "github.com/kami/maven/internal/store" +) + +// Entity-ref propagation, Maven side (Vikunja #272): the canonical Nexus +// entity_id must reach Praxis as a query scope rather than being resolved and +// then thrown away, and the enrichment that produces those ids must degrade +// visibly instead of silently. + +func entityAttentionDec(subject string) router.Decision { + return router.Decision{ + Intent: router.IntentAct, + Slots: router.Slots{Fn: "entity_attention", HasFn: true, Value: subject}, + } +} + +// TestEntityAttention_ScopesPraxisByCanonicalID: the resolved id must travel +// to Praxis in the request, not be used for client-side filtering. +func TestEntityAttention_ScopesPraxisByCanonicalID(t *testing.T) { + ctx := context.Background() + nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) + praxis := newFakePraxis(t, fixturePraxisAttentionScoped("ent_muzick", + map[string]any{"id": "item_1", "title": "indexer queue is backing up", "importance": 3.0}, + )) + h := ecoHandler(t, nexus, praxis, nil) + + reply := h.handlePraxisAct(ctx, entityAttentionDec("muzick indexer")) + if !strings.Contains(reply, "indexer queue is backing up") { + t.Fatalf("expected the scoped item in the reply, got %q", reply) + } + + var scoped bool + for _, r := range praxis.Requests() { + if r.Method == "GET" && strings.HasPrefix(r.Path, "/api/v1/tools/attention") && + strings.Contains(r.Query, "entity_id=ent_muzick") { + scoped = true + } + } + if !scoped { + t.Fatalf("expected attention scoped by entity_id, got requests %+v", praxis.Requests()) + } + if praxis.Count("POST", "/api/v1/tools/surface") == 0 { + t.Error("a spoken scoped item must be surfaced, like the unscoped digest") + } +} + +// TestEntityAttention_FoldsInLocalFactsForSameEntity: facts the enrichment +// worker already tagged with the same canonical id join the same answer. +func TestEntityAttention_FoldsInLocalFactsForSameEntity(t *testing.T) { + ctx := context.Background() + nexus := newFakeNexus(t, fixtureNexusResolved("ent_espresso", "the espresso machine", "device")) + praxis := newFakePraxis(t, fixturePraxisAttentionItems()) + h := ecoHandler(t, nexus, praxis, nil) + + id, err := h.dataStore.WriteFactAboutSubject(ctx, time.Now(), store.KindEnv, + "descaled", "the espresso machine", "descaled in june", "infer:pref", 0.8, sql.NullInt64{}) + if err != nil { + t.Fatalf("WriteFactAboutSubject: %v", err) + } + if err := h.dataStore.ResolveFactEntity(ctx, id, "ent_espresso", store.ResolutionResolved); err != nil { + t.Fatalf("ResolveFactEntity: %v", err) + } + + reply := h.handlePraxisAct(ctx, entityAttentionDec("the espresso machine")) + if !strings.Contains(reply, "descaled in june") { + t.Fatalf("expected entity-scoped local facts in the reply, got %q", reply) + } +} + +// TestEntityAttention_UnscopedPraxisResponseIsRefused: a Praxis old enough to +// ignore the entity_id parameter answers the scoped question with the whole +// unscoped list. Relabelling those items "по «X»" is the same fabrication the +// canonical ref exists to prevent, arriving through a different door. +func TestEntityAttention_UnscopedPraxisResponseIsRefused(t *testing.T) { + ctx := context.Background() + nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) + praxis := newFakePraxis(t, fixturePraxisAttentionItems( + map[string]any{"id": "item_1", "title": "disk almost full", "importance": 3.0}, + )) + h := ecoHandler(t, nexus, praxis, nil) + + reply := h.handlePraxisAct(ctx, entityAttentionDec("muzick indexer")) + if strings.Contains(reply, "disk almost full") { + t.Fatalf("an unscoped response must not be read back as entity-scoped, got %q", reply) + } + if reply == "" { + t.Fatal("refusing the answer must still say something") + } + if praxis.Count("POST", "/api/v1/tools/surface") != 0 { + t.Error("items that were never spoken must not be surfaced") + } +} + +// TestEntityAttention_ForeignItemsAreDropped: items tagged with another entity +// are dropped rather than spoken under this entity's name. +func TestEntityAttention_ForeignItemsAreDropped(t *testing.T) { + ctx := context.Background() + nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) + mixed := []map[string]any{ + {"id": "item_1", "title": "indexer queue is backing up", "importance": 3.0, "entity_id": "ent_muzick"}, + {"id": "item_2", "title": "the kettle is descaling", "importance": 1.0, "entity_id": "ent_kettle"}, + } + praxis := newFakePraxis(t, mustJSON(mixed)) + h := ecoHandler(t, nexus, praxis, nil) + + reply := h.handlePraxisAct(ctx, entityAttentionDec("muzick indexer")) + if !strings.Contains(reply, "indexer queue is backing up") { + t.Fatalf("the matching item must be spoken, got %q", reply) + } + if strings.Contains(reply, "kettle") { + t.Fatalf("another entity's item must not be spoken here, got %q", reply) + } +} + +// TestEntityAttention_TruncationIsNamed: reading three of many remembered +// facts must not be presented as everything she knows. +func TestEntityAttention_TruncationIsNamed(t *testing.T) { + ctx := context.Background() + nexus := newFakeNexus(t, fixtureNexusResolved("ent_espresso", "the espresso machine", "device")) + praxis := newFakePraxis(t, fixturePraxisAttentionItems()) + h := ecoHandler(t, nexus, praxis, nil) + + for i := 0; i < 5; i++ { + id, err := h.dataStore.WriteFactAboutSubject(ctx, time.Now(), store.KindEnv, + "note", "the espresso machine", "факт "+string(rune('а'+i)), "infer:pref", 0.8, sql.NullInt64{}) + if err != nil { + t.Fatalf("WriteFactAboutSubject: %v", err) + } + if err := h.dataStore.ResolveFactEntity(ctx, id, "ent_espresso", store.ResolutionResolved); err != nil { + t.Fatalf("ResolveFactEntity: %v", err) + } + } + + reply := h.handlePraxisAct(ctx, entityAttentionDec("the espresso machine")) + if !strings.Contains(reply, "и это не всё") { + t.Fatalf("a truncated recall must say it is truncated, got %q", reply) + } +} + +// TestEntityAttention_AmbiguousAsksInsteadOfGuessing. +func TestEntityAttention_AmbiguousAsksInsteadOfGuessing(t *testing.T) { + ctx := context.Background() + nexus := newFakeNexus(t, fixtureNexusAmbiguous( + map[string]string{"entity_id": "ent_a", "display_name": "Muzick indexer"}, + map[string]string{"entity_id": "ent_b", "display_name": "Muzick web"}, + )) + praxis := newFakePraxis(t, fixturePraxisAttentionItems()) + h := ecoHandler(t, nexus, praxis, nil) + + reply := h.handlePraxisAct(ctx, entityAttentionDec("muzick")) + if !strings.Contains(reply, "Muzick indexer") || !strings.Contains(reply, "Muzick web") { + t.Fatalf("ambiguous subject must ask, got %q", reply) + } + if praxis.Count("GET", "/api/v1/tools/attention") != 0 { + t.Fatal("an ambiguous subject must not be queried against praxis") + } +} + +// TestEntityAttention_MissingAndDegradedAreDistinct: "no such entity" and +// "Nexus is down" must not produce the same answer. +func TestEntityAttention_MissingAndDegradedAreDistinct(t *testing.T) { + ctx := context.Background() + nexus := newFakeNexus(t, fixtureNexusNotFound()) + praxis := newFakePraxis(t, fixturePraxisAttentionItems()) + h := ecoHandler(t, nexus, praxis, nil) + + missing := h.handlePraxisAct(ctx, entityAttentionDec("нечто")) + if missing == "" { + t.Fatal("an unknown entity must still get an answer") + } + + nexus.SetFault(503) + degraded := h.handlePraxisAct(ctx, entityAttentionDec("нечто")) + if degraded == missing { + t.Fatalf("outage and unknown-entity must not read the same: %q", degraded) + } +} + +// TestEntityAttention_DelayedNexusDegradesNotHangs: a slow Nexus past the +// caller's deadline degrades and never queries Praxis with an empty scope. +func TestEntityAttention_DelayedNexusDegradesNotHangs(t *testing.T) { + nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) + praxis := newFakePraxis(t, fixturePraxisAttentionItems()) + h := ecoHandler(t, nexus, praxis, nil) + nexus.SetDelay(2 * time.Second) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) + defer cancel() + reply := h.handlePraxisAct(ctx, entityAttentionDec("muzick indexer")) + if reply == "" { + t.Fatal("a delayed resolve must still answer") + } + if praxis.Count("GET", "/api/v1/tools/attention") != 0 { + t.Fatal("praxis must not be queried without a resolved scope") + } +} + +// TestEntityAttention_WithoutNexusSaysSo: no Nexus means no canonical ref, so +// the scoped query is refused rather than answered about something else. +func TestEntityAttention_WithoutNexusSaysSo(t *testing.T) { + ctx := context.Background() + praxis := newFakePraxis(t, fixturePraxisAttentionItems( + map[string]any{"id": "item_1", "title": "disk almost full", "importance": 3.0}, + )) + h := ecoHandler(t, nil, praxis, nil) + + reply := h.handlePraxisAct(ctx, entityAttentionDec("muzick indexer")) + if strings.Contains(reply, "disk almost full") { + t.Fatalf("without nexus, items must not be passed off as entity-scoped, got %q", reply) + } + if praxis.Count("GET", "/api/v1/tools/attention") != 0 { + t.Fatal("no canonical ref means no scoped query at all") + } +} + +// TestEnrichmentBackoff_HoldsAndReleases: repeated Nexus failures back the +// fact off instead of hammering, and the fact is retried once the window +// elapses. Nothing is ever given up on. +func TestEnrichmentBackoff_HoldsAndReleases(t *testing.T) { + ctx := context.Background() + nexus := newFakeNexus(t, fixtureNexusResolved("ent_espresso", "the espresso machine", "device")) + st := newTestStore(t) + if _, err := st.WriteFactAboutSubject(ctx, time.Now(), store.KindEnv, "likes", + "the espresso machine", `"true"`, "infer:pref", 0.8, sql.NullInt64{}); err != nil { + t.Fatalf("WriteFactAboutSubject: %v", err) + } + + clock := newFakeClock(time.Date(2026, 8, 1, 3, 0, 0, 0, time.UTC)) + w := newFactEnrichmentWorker(st, stubEcosystem(nexus.URL, ""), time.Hour) + w.now = clock.Now + + nexus.SetFault(503) + w.tick(ctx) + failedCalls := nexus.Count("POST", "/api/v1/resolve") + if failedCalls != 1 { + t.Fatalf("expected one resolve attempt, got %d", failedCalls) + } + + // Immediately after a failure the fact is in backoff: no second call. + w.tick(ctx) + if nexus.Count("POST", "/api/v1/resolve") != failedCalls { + t.Fatal("a fact in backoff must not be retried on the very next tick") + } + if s := w.status(ctx); s.Pending != 1 || s.InBackoff != 1 || s.MaxAttempts != 1 { + t.Fatalf("degradation must be reported, got %+v", s) + } + + // Once the window elapses and Nexus recovers, the fact resolves. + clock.Advance(2 * time.Minute) + nexus.SetFault(0) + w.tick(ctx) + facts, err := st.FactsByEntity(ctx, "ent_espresso", 10) + if err != nil { + t.Fatalf("FactsByEntity: %v", err) + } + if len(facts) != 1 { + t.Fatalf("expected the fact resolved after recovery, got %+v", facts) + } + if s := w.status(ctx); s.Pending != 0 || s.MaxAttempts != 0 { + t.Fatalf("recovery must clear the degradation report, got %+v", s) + } +} + +func TestEnrichmentBackoff_GrowsAndIsCapped(t *testing.T) { + if enrichmentBackoff(1) != time.Minute { + t.Fatalf("first retry should be a minute, got %v", enrichmentBackoff(1)) + } + if enrichmentBackoff(3) != 4*time.Minute { + t.Fatalf("third retry should be four minutes, got %v", enrichmentBackoff(3)) + } + if enrichmentBackoff(50) != time.Hour { + t.Fatalf("backoff must cap at an hour, got %v", enrichmentBackoff(50)) + } +} + +// TestEnrichment_BackedOffFactsDoNotStallTheQueue: the pending queue is ordered +// by id, so the oldest facts are pulled first whether or not they are eligible. +// A batch of facts in backoff at the head must not hold every slot and stop +// enrichment for everything younger. +func TestEnrichment_BackedOffFactsDoNotStallTheQueue(t *testing.T) { + ctx := context.Background() + st := newTestStore(t) + total := 5 + for i := 0; i < total; i++ { + if _, err := st.WriteFactAboutSubject(ctx, time.Now(), store.KindEnv, "likes", + "subject-"+string(rune('a'+i)), `"true"`, "infer:pref", 0.8, sql.NullInt64{}); err != nil { + t.Fatalf("WriteFactAboutSubject: %v", err) + } + } + + nexus := newFakeNexus(t, fixtureNexusResolved("ent_x", "X", "service")) + clock := newFakeClock(time.Date(2026, 8, 1, 3, 0, 0, 0, time.UTC)) + w := newFactEnrichmentWorker(st, stubEcosystem(nexus.URL, ""), time.Hour) + w.now = clock.Now + // A batch smaller than the queue, so with no scan the last fact never + // reaches the head while the first ones are backed off. + w.batch = total - 1 + + nexus.SetFault(503) + w.tick(ctx) + if got := nexus.Count("POST", "/api/v1/resolve"); got != total-1 { + t.Fatalf("expected the first batch attempted, got %d calls", got) + } + + // Second tick with Nexus healthy: the backed-off head must be skipped and + // the fact behind it resolved, not the same batch pulled and dropped. + nexus.SetFault(0) + w.tick(ctx) + facts, err := st.FactsByEntity(ctx, "ent_x", 10) + if err != nil { + t.Fatalf("FactsByEntity: %v", err) + } + if len(facts) == 0 { + t.Fatal("a due fact behind a backed-off batch must still be resolved") + } +} + +// TestEnrichment_StoreWriteFailureBacksOffToo: the one failure mode where the +// resolve worked and the write did not must be paced like any other, not +// retried at full rate forever. +func TestEnrichment_StoreWriteFailureBacksOffToo(t *testing.T) { + ctx := context.Background() + nexus := newFakeNexus(t, fixtureNexusResolved("ent_espresso", "the espresso machine", "device")) + st := newTestStore(t) + if _, err := st.WriteFactAboutSubject(ctx, time.Now(), store.KindEnv, "likes", + "the espresso machine", `"true"`, "infer:pref", 0.8, sql.NullInt64{}); err != nil { + t.Fatalf("WriteFactAboutSubject: %v", err) + } + pending, err := st.PendingFactResolutions(ctx, 10) + if err != nil || len(pending) != 1 { + t.Fatalf("setup: pending = %+v, %v", pending, err) + } + + clock := newFakeClock(time.Date(2026, 8, 1, 3, 0, 0, 0, time.UTC)) + w := newFactEnrichmentWorker(st, stubEcosystem(nexus.URL, ""), time.Hour) + w.now = clock.Now + + // Closing the store makes the resolution write fail while the Nexus call + // still succeeds — the split this path gets wrong. + if err := st.Close(); err != nil { + t.Fatalf("close store: %v", err) + } + if w.resolveOne(ctx, pending[0]) { + t.Fatal("a failed store write must not report success") + } + if w.due(pending[0].ID) { + t.Fatal("a failed store write must back the fact off like a failed resolve") + } +} diff --git a/cmd/mavend/factenrichment.go b/cmd/mavend/factenrichment.go index e228215..835e630 100644 --- a/cmd/mavend/factenrichment.go +++ b/cmd/mavend/factenrichment.go @@ -9,6 +9,7 @@ package main import ( "context" "log" + "sync" "time" "github.com/kami/maven/internal/store" @@ -24,10 +25,88 @@ type factEnrichmentWorker struct { eco *ecosystemWiring interval time.Duration batch int // facts resolved per tick; keeps a single slow tick bounded + now func() time.Time + + // Retry state for facts whose resolution failed transiently. Kept in + // memory rather than in the DB: a restart legitimately retries + // everything, and the backoff exists to spare a struggling Nexus, not + // to be durable. A fact is never given up on — degraded means slower, + // not dropped. + mu sync.Mutex + attempt map[int64]int // fact id → consecutive failures + nextTry map[int64]time.Time // fact id → earliest retry +} + +// enrichmentScanLimit bounds how deep a single tick (or status report) walks +// the pending queue looking for facts whose backoff has elapsed. The queue is +// ordered by id, so without a scan the oldest facts hold every batch slot +// whether or not they are eligible, and one permanently failing fact stalls +// every younger one behind it. +const enrichmentScanLimit = 1000 + +// enrichmentBackoff is the wait before retrying a fact after n consecutive +// failures, capped so a long Nexus outage still retries about hourly. +func enrichmentBackoff(n int) time.Duration { + d := time.Minute + for i := 1; i < n && d < time.Hour; i++ { + d *= 2 + } + if d > time.Hour { + d = time.Hour + } + return d } func newFactEnrichmentWorker(st *store.Store, eco *ecosystemWiring, interval time.Duration) *factEnrichmentWorker { - return &factEnrichmentWorker{store: st, eco: eco, interval: interval, batch: 20} + return &factEnrichmentWorker{ + store: st, + eco: eco, + interval: interval, + batch: 20, + now: time.Now, + attempt: map[int64]int{}, + nextTry: map[int64]time.Time{}, + } +} + +// enrichmentStatus is what the worker reports about its own health: how many +// facts are waiting, how many of those are currently in backoff, and the worst +// retry count among them. Degradation is reported, never hidden — a Nexus that +// has been down all day must be visible as a backlog, not as facts that +// silently never got tagged. +// +// All three numbers describe the same set of rows, the first +// enrichmentScanLimit pending facts. Counting Pending over a thousand rows +// while counting InBackoff over the twenty that reached the head of a batch +// described two different populations under one struct. +type enrichmentStatus struct { + Pending int + InBackoff int + MaxAttempts int + Scanned int // rows the other three counts were taken over +} + +func (w *factEnrichmentWorker) status(ctx context.Context) enrichmentStatus { + var st enrichmentStatus + pending, err := w.store.PendingFactResolutions(ctx, enrichmentScanLimit) + if err != nil { + log.Printf("factenrichment: status: %v", err) + return st + } + st.Pending = len(pending) + st.Scanned = len(pending) + w.mu.Lock() + defer w.mu.Unlock() + now := w.now() + for _, f := range pending { + if next, ok := w.nextTry[f.ID]; ok && now.Before(next) { + st.InBackoff++ + } + if n := w.attempt[f.ID]; n > st.MaxAttempts { + st.MaxAttempts = n + } + } + return st } func (w *factEnrichmentWorker) run(ctx context.Context) { @@ -52,22 +131,86 @@ func (w *factEnrichmentWorker) run(ctx context.Context) { } func (w *factEnrichmentWorker) tick(ctx context.Context) { - pending, err := w.store.PendingFactResolutions(ctx, w.batch) + // Scan past the facts that are still in backoff instead of letting them + // occupy the batch. The queue is ordered by id, so the oldest facts are + // pulled first whether or not they are eligible: twenty facts Nexus keeps + // rejecting would otherwise hold every slot forever and enrichment would + // stop with no error and no log line, because a tick that skips everything + // fails nothing. + pending, err := w.store.PendingFactResolutions(ctx, enrichmentScanLimit) if err != nil { log.Printf("factenrichment: list pending: %v", err) return } + w.forgetDeparted(pending) + skipped, failed, attempted := 0, 0, 0 for _, f := range pending { - w.resolveOne(ctx, f) + if attempted >= w.batch { + break + } + if !w.due(f.ID) { + skipped++ + continue + } + attempted++ + if !w.resolveOne(ctx, f) { + failed++ + } + } + if failed > 0 { + log.Printf("factenrichment: %d/%d resolutions failed this tick, %d held in backoff", + failed, attempted, skipped) + } + // Report the backlog every tick, not only when something failed: the + // stalled state worth seeing is the one where nothing failed because + // nothing was attempted. + if st := w.status(ctx); st.Pending > 0 { + log.Printf("factenrichment: %d facts pending entity resolution, %d in backoff, worst attempt %d (scanned %d)", + st.Pending, st.InBackoff, st.MaxAttempts, st.Scanned) } } -func (w *factEnrichmentWorker) resolveOne(ctx context.Context, f store.Fact) { +// forgetDeparted drops retry state for facts that are no longer pending. A +// fact can leave the queue without ever resolving here — voided, or resolved +// by a later write — and its entries would otherwise live as long as the +// process does. +func (w *factEnrichmentWorker) forgetDeparted(pending []store.Fact) { + live := make(map[int64]struct{}, len(pending)) + for _, f := range pending { + live[f.ID] = struct{}{} + } + w.mu.Lock() + defer w.mu.Unlock() + for id := range w.attempt { + if _, ok := live[id]; !ok { + delete(w.attempt, id) + } + } + for id := range w.nextTry { + if _, ok := live[id]; !ok { + delete(w.nextTry, id) + } + } +} + +// due reports whether a fact's backoff window has elapsed. +func (w *factEnrichmentWorker) due(id int64) bool { + w.mu.Lock() + defer w.mu.Unlock() + next, ok := w.nextTry[id] + return !ok || !w.now().Before(next) +} + +// resolveOne resolves one pending fact. It returns false when the attempt +// failed transiently: the fact stays pending and is retried on a backoff. +func (w *factEnrichmentWorker) resolveOne(ctx context.Context, f store.Fact) bool { entityID, _, ambiguous, err := w.eco.resolveEntityReference(ctx, f.Subject, nil) if err != nil { - // Transient (Nexus unreachable) — leave pending, retry next tick. - log.Printf("factenrichment: resolve fact %d subject %q: %v", f.ID, f.Subject, err) - return + // Transient (Nexus unreachable) — leave pending, back off, retry later. + // The subject is his words: log its length, the way the trace does. + log.Printf("factenrichment: resolve fact %d subject %s: %v", f.ID, redactSubject(f.Subject), err) + w.backOff(f.ID) + return false } state := store.ResolutionNotFound switch { @@ -77,6 +220,25 @@ func (w *factEnrichmentWorker) resolveOne(ctx context.Context, f store.Fact) { state = store.ResolutionAmbiguous } if err := w.store.ResolveFactEntity(ctx, f.ID, entityID, state); err != nil { + // A failed write leaves the fact pending exactly like a failed resolve + // does, so it gets the same pacing. Clearing the counters first meant + // this one path retried every tick, at full rate, with no ceiling. log.Printf("factenrichment: record resolution for fact %d: %v", f.ID, err) + w.backOff(f.ID) + return false } + w.mu.Lock() + delete(w.attempt, f.ID) + delete(w.nextTry, f.ID) + w.mu.Unlock() + return true +} + +// backOff records one more consecutive failure for a fact and pushes its next +// attempt out accordingly. +func (w *factEnrichmentWorker) backOff(id int64) { + w.mu.Lock() + defer w.mu.Unlock() + w.attempt[id]++ + w.nextTry[id] = w.now().Add(enrichmentBackoff(w.attempt[id])) } diff --git a/cmd/mavend/fakeecosystem_test.go b/cmd/mavend/fakeecosystem_test.go index ff08eb6..707f6ab 100644 --- a/cmd/mavend/fakeecosystem_test.go +++ b/cmd/mavend/fakeecosystem_test.go @@ -14,7 +14,9 @@ import ( type capturedRequest struct { Method string Path string + Query string Body []byte + Header http.Header } // fakeServer is the common shell behind fakeNexus/fakePraxis/fakeHexis: an @@ -25,9 +27,12 @@ type capturedRequest struct { type fakeServer struct { *httptest.Server - mu sync.Mutex - requests []capturedRequest - fault int // non-zero: every request gets this HTTP status instead of routing + mu sync.Mutex + requests []capturedRequest + fault int // non-zero: every request gets this HTTP status instead of routing + routeFaults map[string]int // path prefix → status, for one endpoint failing alone + garbage string // non-empty: returned 200 verbatim instead of routing (malformed-contract lever) + delay time.Duration } // newFakeServer starts a server dispatching to routes keyed by "METHOD @@ -47,14 +52,42 @@ func newFakeServer(t *testing.T, routes map[string]http.HandlerFunc) *fakeServer } } fs.mu.Lock() - fs.requests = append(fs.requests, capturedRequest{Method: r.Method, Path: r.URL.Path, Body: body}) + fs.requests = append(fs.requests, capturedRequest{ + Method: r.Method, + Path: r.URL.Path, + Query: r.URL.RawQuery, + Body: body, + Header: r.Header.Clone(), + }) fault := fs.fault + if fault == 0 { + for prefix, status := range fs.routeFaults { + if hasPrefix(r.URL.Path, prefix) { + fault = status + break + } + } + } + garbage := fs.garbage + delay := fs.delay fs.mu.Unlock() + if delay > 0 { + select { + case <-time.After(delay): + case <-r.Context().Done(): + return + } + } if fault != 0 { http.Error(w, "injected fault", fault) return } + if garbage != "" { + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(garbage)) + return + } for key, handler := range routes { method, prefix := splitRouteKey(key) @@ -90,6 +123,52 @@ func (fs *fakeServer) SetFault(status int) { fs.fault = status } +// SetRouteFault fails one endpoint while the rest of the server stays healthy, +// which is the shape most real outages take: attention answers and pin is +// down. Pass 0 to clear that route. A server-wide SetFault still wins. +func (fs *fakeServer) SetRouteFault(pathPrefix string, status int) { + fs.mu.Lock() + defer fs.mu.Unlock() + if fs.routeFaults == nil { + fs.routeFaults = map[string]int{} + } + if status == 0 { + delete(fs.routeFaults, pathPrefix) + return + } + fs.routeFaults[pathPrefix] = status +} + +// SetBody makes every subsequent request answer 200 with the given body, +// bypassing the route table. Used to serve a malformed or contract-violating +// payload where the transport itself is healthy. Pass "" to clear it. +func (fs *fakeServer) SetBody(body string) { + fs.mu.Lock() + defer fs.mu.Unlock() + fs.garbage = body +} + +// SetDelay stalls every subsequent request for d before answering, so callers +// can drive client timeouts and context cancellation deterministically. The +// delay is abandoned as soon as the client hangs up. +func (fs *fakeServer) SetDelay(d time.Duration) { + fs.mu.Lock() + defer fs.mu.Unlock() + fs.delay = d +} + +// Count returns how many captured requests used the given method and path +// prefix. "" matches any method. +func (fs *fakeServer) Count(method, prefix string) int { + n := 0 + for _, r := range fs.Requests() { + if (method == "" || r.Method == method) && hasPrefix(r.Path, prefix) { + n++ + } + } + return n +} + // Requests returns a snapshot of captured requests, in arrival order. func (fs *fakeServer) Requests() []capturedRequest { fs.mu.Lock() @@ -118,6 +197,40 @@ func fixtureNexusResolved(entityID, displayName, entityType string) string { }) } +// fixtureNexusResolvedFlat is the flat resolve shape documented in +// ECOSYSTEM-SPEC.md §1.5 (entity_id/entity_type/display_name at the top +// level) rather than the nested "entity" object — the older of the two +// wire shapes Maven must keep accepting. +func fixtureNexusResolvedFlat(entityID, displayName, entityType string) string { + return mustJSON(map[string]any{ + "status": "resolved", + "entity_id": entityID, + "entity_type": entityType, + "display_name": displayName, + }) +} + +// fixtureNexusResolvedFuture is a resolved response from a hypothetical newer +// Nexus: same required fields plus unknown ones. Decoding must ignore the +// extras, not fail — forward compatibility is what lets the ecosystem be +// upgraded one service at a time. +func fixtureNexusResolvedFuture(entityID, displayName, entityType string) string { + return mustJSON(map[string]any{ + "status": "resolved", + "entity": map[string]any{"id": entityID, "display_name": displayName, "type": entityType, "tenant": "home"}, + "provenance": map[string]any{"resolver": "v3", "graph_epoch": 42}, + "score_breakdown": []any{map[string]any{"signal": "alias", "weight": 0.9}}, + }) +} + +// fixtureNexusResolvedEmpty is the contract violation that decodes cleanly: +// Nexus claims a resolve and delivers no entity. It must not read as "no such +// entity", which would let the caller fall through to local execution with the +// user's verb intact. +func fixtureNexusResolvedEmpty() string { + return `{"status":"resolved"}` +} + func fixtureNexusNotFound() string { return `{"status":"not_found"}` } @@ -138,6 +251,23 @@ func fixtureHexisExecuted(id, status string) string { return mustJSON(map[string]any{"id": id, "status": status}) } +// fixtureHexisExecutionFailed is a well-formed Hexis response reporting that +// the command itself failed: the call succeeded, the execution did not. Maven +// must distinguish this from a transport failure and from success. +func fixtureHexisExecutionFailed(id, message string) string { + return mustJSON(map[string]any{"id": id, "status": "failed", "error": message}) +} + +// fixturePraxisAttentionScoped tags each item with an entity_id, which is what +// a Praxis that understands the entity_id query parameter returns. A Praxis +// that ignores it answers with untagged items from every entity. +func fixturePraxisAttentionScoped(entityID string, items ...map[string]any) string { + for _, item := range items { + item["entity_id"] = entityID + } + return mustJSON(items) +} + func fixturePraxisAttentionItems(items ...map[string]any) string { return mustJSON(items) } @@ -156,18 +286,28 @@ func mustJSON(v any) string { // (e.g. asserting age-based digest ordering without sleeping). type fakeClock struct { - mu sync.Mutex - t time.Time + mu sync.Mutex + t time.Time + step time.Duration // advanced on every read, so elapsed time is measurable } func newFakeClock(start time.Time) *fakeClock { return &fakeClock{t: start} } +// newTickingClock advances by step on every read. Durations measured across +// hops are then non-zero without sleeping, which is what lets a test tell a +// trace that measured something from one that measured nothing. +func newTickingClock(start time.Time, step time.Duration) *fakeClock { + return &fakeClock{t: start, step: step} +} + func (c *fakeClock) Now() time.Time { c.mu.Lock() defer c.mu.Unlock() - return c.t + now := c.t + c.t = c.t.Add(c.step) + return now } func (c *fakeClock) Advance(d time.Duration) { @@ -191,8 +331,13 @@ func newFakeNexus(t *testing.T, resolveBody string) *fakeServer { // fault is injected via SetFault. func newFakePraxis(t *testing.T, attentionBody string) *fakeServer { return newFakeServer(t, map[string]http.HandlerFunc{ - "GET /api/v1/tools/attention": jsonHandler(http.StatusOK, attentionBody), - "POST /api/v1/tools/surface": jsonHandler(http.StatusOK, `{}`), + "GET /api/v1/tools/attention": jsonHandler(http.StatusOK, attentionBody), + "GET /api/v1/tools/changes": jsonHandler(http.StatusOK, `[]`), + "POST /api/v1/tools/surface": jsonHandler(http.StatusOK, `{}`), + "POST /api/v1/tools/acknowledge": jsonHandler(http.StatusOK, `{}`), + "POST /api/v1/tools/resolve": jsonHandler(http.StatusOK, `{}`), + "POST /api/v1/tools/ignore": jsonHandler(http.StatusOK, `{}`), + "POST /api/v1/tools/pin": jsonHandler(http.StatusOK, `{}`), }) } diff --git a/cmd/mavend/feeds.go b/cmd/mavend/feeds.go new file mode 100644 index 0000000..99d9d8f --- /dev/null +++ b/cmd/mavend/feeds.go @@ -0,0 +1,194 @@ +// mavend/feeds.go — the driver for RSS/Atom reading (Vikunja #258, +// docs/plans/13-rss-news-feeds.md). The reader itself is pure and lives in +// internal/rss; this is the impure half: a ticker, the guarded fetcher, and the +// two adapters that let a pure package talk to the store. +// +// Why in-core rather than its own daemon like mavmaild and mavpoll: those two +// hold a CREDENTIAL (an IMAP password, a zenmoney token), and the reason they +// are separate processes is that core must never see it. A feed URL is public, +// there is no secret to isolate, and a whole extra binary and compose service +// would buy nothing. The other half of the mavpoll precedent — off unless +// configured — is kept: no `feeds` block, no poller, no outbound request. +// +// It is its own goroutine, not a step on the tick: the tick has a delivery +// deadline behind it, and a feed read is a network round-trip that nobody is +// waiting on. +// +// Nothing here dispatches. A feed that announced itself would be a nag, so the +// only output is notes with source "rss:", which the answer path reads +// when he asks ("что нового в лентах?" — see queryFeeds in actions_query.go). +package main + +import ( + "context" + "log" + "net/url" + "time" + + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/router" + "github.com/kami/maven/internal/rss" + "github.com/kami/maven/internal/stt" + "github.com/kami/maven/internal/webfetch" +) + +// feedWorker — ticker + poller. +type feedWorker struct { + poller *rss.Poller + interval time.Duration +} + +// feedTickInterval — how often the worker asks the poller what is due. Per-feed +// cadence is the poller's business; this is just the granularity. +const feedTickInterval = 5 * time.Minute + +// newFeedWorker wires feed reading, or returns nil when it must not run: +// no `feeds` block (the normal case), or nothing valid in it. Every caller +// checks for nil. +func newFeedWorker(api ipc.CoreAPI, emb router.Embedder, cfg *config.Config) *feedWorker { + if cfg.Feeds == nil { + return nil + } + fc := cfg.Feeds + + feeds := make([]rss.FeedConfig, 0, len(fc.Sources)) + hosts := append([]string(nil), fc.AllowHosts...) + for _, s := range fc.Sources { + feeds = append(feeds, rss.FeedConfig{ + Name: s.Name, + URL: s.URL, + Category: s.Category, + Interval: time.Duration(s.Interval), + Include: s.Include, + Exclude: s.Exclude, + }) + // Each configured feed's own host is allowed. The allowlist is then + // exactly "the feeds he asked for", so a redirect off to somewhere else + // is refused by the fetcher rather than followed. + if u, err := url.Parse(s.URL); err == nil && u.Hostname() != "" { + hosts = append(hosts, u.Hostname()) + } + } + + fetcher := webfetch.New(webfetch.Config{ + AllowHosts: hosts, + Timeout: time.Duration(fc.Timeout), + MaxBytes: fc.MaxBytes, + }) + poller := rss.NewPoller(feeds, &feedFetcher{f: fetcher}, api, &factMarks{api: api}, + embedderFor(emb), nil, rss.Config{ + DefaultInterval: time.Duration(fc.PollInterval), + MaxItems: fc.MaxItems, + MaxAge: time.Duration(fc.MaxAge), + }) + if poller == nil { + log.Printf("feeds: configured but nothing pollable — feed reading disabled") + return nil + } + log.Printf("feeds: reading %d feed(s), checking what is due every %s", len(feeds), feedTickInterval) + return &feedWorker{poller: poller, interval: feedTickInterval} +} + +// run polls what is due until ctx is canceled. The first round runs immediately +// so a restart does not blind her for the first interval; it writes notes only, +// so an early round cannot startle anyone. +func (w *feedWorker) run(ctx context.Context) { + w.poller.PollDue(ctx, time.Now()) + t := time.NewTicker(w.interval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case now := <-t.C: + w.poller.PollDue(ctx, now) + } + } +} + +// embedderOf — the voice wiring's embedder, or nil when voice is not wired. +// Feed notes are embedded with the SAME model the rest of the store uses, or not +// at all; a second embedder would write vectors nothing can search. +func embedderOf(w *voiceWiring) router.Embedder { + if w == nil { + return nil + } + return w.embedder +} + +// transcriberOf — the STT the voice path is using, or nil when voice is off. +// The meeting recorder reuses it rather than dialling mavsttd a second time: +// Maven has one speech-to-text engine and adding a second would mean two +// whisper contexts competing for the same iGPU. +func transcriberOf(w *voiceWiring) stt.Transcriber { + if w == nil { + return nil + } + return w.transcriber +} + +// feedFetcher adapts webfetch to rss.Fetcher — the pure package names the two +// fields it needs and stays free of net/http. +type feedFetcher struct{ f *webfetch.Fetcher } + +func (a *feedFetcher) Get(ctx context.Context, u string) (*rss.Body, error) { + resp, err := a.f.Get(ctx, u) + if err != nil { + return nil, err + } + return &rss.Body{Bytes: resp.Body}, nil +} + +// factMarks stores "how far this feed was read" as a config fact, the same +// mechanism the plan named and the same one the pattern tick uses for its own +// bookkeeping. Durable, inspectable on /dash, and cheap. +type factMarks struct{ api ipc.CoreAPI } + +func markKey(feed string) string { return "rss:latest:" + feed } + +func (m *factMarks) LastMark(ctx context.Context, feed string) (time.Time, error) { + f, err := m.api.LatestFact(ctx, markKey(feed)) + if err != nil { + // No mark yet is not an error worth propagating: the poller treats a + // zero time as a cold start. + return time.Time{}, nil + } + t, err := time.Parse(time.RFC3339, f.Value) + if err != nil { + return time.Time{}, nil + } + return t, nil +} + +func (m *factMarks) SetMark(ctx context.Context, feed string, at time.Time) error { + _, err := m.api.WriteFact(ctx, ipc.WriteFactReq{ + Ts: time.Now(), + Kind: "config", + Key: markKey(feed), + Value: at.UTC().Format(time.RFC3339), + Source: "poll:rss", + Confidence: 1.0, + }) + return err +} + +// embedderFor adapts router.Embedder to rss.Embedder, and returns nil when +// there is none — a note without a vector is still a note the recent-notes path +// can read. +// +// EmbedPassage, not Embed: a feed item is text being searched FOR, and the e5 +// embedder is asymmetric. Getting this backwards makes the item unfindable by +// the question that should have matched it. +func embedderFor(emb router.Embedder) rss.Embedder { + if emb == nil { + return nil + } + return passageEmbedder{emb} +} + +type passageEmbedder struct{ e router.Embedder } + +func (p passageEmbedder) Embed(ctx context.Context, text string) ([]float32, error) { + return router.EmbedPassage(ctx, p.e, text) +} diff --git a/cmd/mavend/feeds_test.go b/cmd/mavend/feeds_test.go new file mode 100644 index 0000000..0759c50 --- /dev/null +++ b/cmd/mavend/feeds_test.go @@ -0,0 +1,196 @@ +package main + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/phraser" + "github.com/kami/maven/internal/router" + "github.com/kami/maven/internal/rss" + "github.com/kami/maven/internal/voice" +) + +// buildFeedHandler — a handler with the given feed notes already stored. No +// embedder: the feed source answers from recent notes by source, which is what +// makes it work for notes written before an embedder existed. +func buildFeedHandler(t *testing.T, feedsOn bool, notes ...ipc.Note) *reactiveHandler { + t.Helper() + ctx := context.Background() + st := newTestStore(t) + now := time.Now() + for i, n := range notes { + ts := now.Add(time.Duration(i) * time.Minute) + if _, err := st.WriteNote(ctx, ts, n.Text, nil, n.Source); err != nil { + t.Fatalf("WriteNote: %v", err) + } + } + return &reactiveHandler{ + api: ipc.NewStoreAPI(st), + replier: voice.NewStubReplier(), + phraser: phraser.NewStub(), + now: func() time.Time { return now }, + feedsOn: feedsOn, + embedder: nil, + } +} + +func askFeeds(t *testing.T, h *reactiveHandler, q string) (string, bool) { + t.Helper() + return h.queryFeeds(context.Background(), &queryTurn{ + dec: router.Decision{Intent: router.IntentQuery, Utterance: q}, + }) +} + +func TestQueryFeedsReadsFeedNotes(t *testing.T) { + h := buildFeedHandler(t, true, + ipc.Note{Text: "Новая уязвимость в ядре [технологии]\nпатч вышел\nhttps://example.org/a", Source: "rss:habr"}, + ipc.Note{Text: "что-то он сам сказал", Source: "tap:voice"}, + ) + reply, ok := askFeeds(t, h, "что нового в лентах?") + if !ok { + t.Fatal("the feed source did not claim the question") + } + if !strings.Contains(reply, "уязвимость") { + t.Errorf("reply = %q, want the headline", reply) + } + if strings.Contains(reply, "он сам сказал") { + t.Errorf("a note he dictated leaked into the feed answer: %q", reply) + } + // She reads the headline, not the summary and not the URL. + if strings.Contains(reply, "https://") || strings.Contains(reply, "патч вышел") { + t.Errorf("reply = %q, want the title line only", reply) + } +} + +func TestQueryFeedsByCategory(t *testing.T) { + h := buildFeedHandler(t, true, + ipc.Note{Text: "Релиз ядра [технологии]", Source: "rss:habr"}, + ipc.Note{Text: "Выборы отложены [политика]", Source: "rss:news"}, + ) + reply, ok := askFeeds(t, h, "что нового по технологиям?") + if !ok { + t.Fatal("not claimed") + } + if !strings.Contains(reply, "ядра") || strings.Contains(reply, "Выборы") { + t.Fatalf("reply = %q, want only the технологии item", reply) + } + reply, _ = askFeeds(t, h, "что нового по спорту?") + if !strings.Contains(reply, "ничего") { + t.Fatalf("reply = %q, want an honest empty answer for an unread category", reply) + } +} + +// "не настроены" and "ничего нового" are different truths, and neither may be +// answered by the model inventing a bulletin. +func TestQueryFeedsOffAndEmptyDiffer(t *testing.T) { + off := buildFeedHandler(t, false) + reply, ok := askFeeds(t, off, "что нового в лентах?") + if !ok || !strings.Contains(reply, "не настроены") { + t.Fatalf("feeds off: reply = %q, ok = %v", reply, ok) + } + on := buildFeedHandler(t, true) + reply, ok = askFeeds(t, on, "что нового в лентах?") + if !ok || !strings.Contains(reply, "ничего нового") { + t.Fatalf("feeds on but empty: reply = %q, ok = %v", reply, ok) + } +} + +func TestQueryFeedsPassesOnANonFeedQuestion(t *testing.T) { + h := buildFeedHandler(t, true) + if reply, ok := askFeeds(t, h, "напомни полить цветы"); ok { + t.Fatalf("claimed an unrelated question with %q", reply) + } + // The bare greeting is not a request for headlines. It used to be answered + // with a configuration status. + if reply, ok := askFeeds(t, h, "что нового?"); ok { + t.Fatalf("claimed a greeting with %q", reply) + } +} + +// A busy day of his own notes must not push the newest headline out of the +// window the feed answer scans. +func TestQueryFeedsIsNotCrowdedOutByHisOwnNotes(t *testing.T) { + notes := []ipc.Note{{Text: "Релиз ядра [технологии]", Source: "rss:habr"}} + for i := 0; i < feedNoteWindow+10; i++ { + notes = append(notes, ipc.Note{Text: "мысль вслух", Source: "tap:voice"}) + } + h := buildFeedHandler(t, true, notes...) + reply, ok := askFeeds(t, h, "что нового в лентах?") + if !ok || !strings.Contains(reply, "ядра") { + t.Fatalf("reply = %q, ok = %v; the headline fell out of the window", reply, ok) + } +} + +// The mark is what stops a restart from re-noting yesterday's headlines, so the +// fact round-trip is worth a test of its own. +func TestFactMarksRoundTrip(t *testing.T) { + st := newTestStore(t) + m := &factMarks{api: ipc.NewStoreAPI(st)} + ctx := context.Background() + + at, err := m.LastMark(ctx, "habr") + if err != nil || !at.IsZero() { + t.Fatalf("no mark yet: got %v, %v — want zero time and no error", at, err) + } + want := time.Date(2026, 7, 28, 10, 0, 0, 0, time.UTC) + if err := m.SetMark(ctx, "habr", want); err != nil { + t.Fatal(err) + } + got, err := m.LastMark(ctx, "habr") + if err != nil { + t.Fatal(err) + } + if !got.Equal(want) { + t.Fatalf("mark = %v, want %v", got, want) + } +} + +// Off unless configured, checked at the wiring seam: no `feeds` block ⇒ no +// worker ⇒ no outbound request is possible. +func TestNewFeedWorkerOffByDefault(t *testing.T) { + st := newTestStore(t) + api := ipc.NewStoreAPI(st) + if w := newFeedWorker(api, nil, &config.Config{}); w != nil { + t.Fatal("a config with no feeds block wired a feed worker") + } + // An empty sources list is normalised to "off" by config.Load; the worker + // refuses it too, so a hand-built Config cannot switch it on by accident. + if w := newFeedWorker(api, nil, &config.Config{Feeds: &config.FeedsConfig{}}); w != nil { + t.Fatal("an empty sources list wired a feed worker") + } + cfg := &config.Config{Feeds: &config.FeedsConfig{Sources: []config.FeedSourceConfig{ + {Name: "habr", URL: "https://example.org/rss"}, + }}} + w := newFeedWorker(api, nil, cfg) + if w == nil { + t.Fatal("a configured feed did not wire a worker") + } + if got := w.poller.Feeds(); len(got) != 1 || got[0].Name != "habr" { + t.Fatalf("feeds = %+v", got) + } +} + +// The fetcher the worker builds must be allowlisted to the configured feeds and +// nothing else — the crawler's SSRF guards are only worth as much as the +// allowlist handed to them. +func TestFeedWorkerFetcherIsAllowlisted(t *testing.T) { + cfg := &config.Config{Feeds: &config.FeedsConfig{Sources: []config.FeedSourceConfig{ + {Name: "habr", URL: "https://feeds.example.org/rss"}, + }}} + w := newFeedWorker(ipc.NewStoreAPI(newTestStore(t)), nil, cfg) + if w == nil { + t.Fatal("no worker") + } + // PollFeed goes through the guarded fetcher; a feed URL pointing at the box + // itself must fail rather than be read. + _, err := w.poller.PollFeed(context.Background(), rss.FeedConfig{ + Name: "evil", URL: "http://127.0.0.1:9100/mcp", + }, time.Now()) + if err == nil { + t.Fatal("the poller fetched a private address") + } +} diff --git a/cmd/mavend/intake.go b/cmd/mavend/intake.go new file mode 100644 index 0000000..ba4954a --- /dev/null +++ b/cmd/mavend/intake.go @@ -0,0 +1,310 @@ +// mavend/intake.go — the unified event intake envelope, wired (Vikunja #283). +// +// internal/event defines the envelope and the bounded in-memory journal. This +// file is the one place that FILLS it, and the reason it is one place is worth +// stating, because the alternative was eight patches: +// +// Every intake path in Maven already converges on three writes, and all three +// are ipc.CoreAPI methods — +// +// WriteFact ← POST /api/ambient, mavcaldav, mavpoll's zenmoney + wg reads, +// /api/signal presence probes, the RSS/crawl watermarks +// WriteNote ← the RSS poller, the page crawler, meeting transcripts, +// image descriptions +// CaptureTask ← the voice path, the web form, and the mail reader +// +// — so decorating that ONE interface with a publish covers the lot without a +// caller knowing about events at all. cmd/mavmaild, cmd/mavcaldav, cmd/mavpoll, +// cmd/mavweb and the in-core feed/crawl/capture/vision workers are unchanged: +// they call the same interface they always called, and it now also narrates. +// +// The exception is cmd/mavend/mail.go, which reaches past the interface to +// st.CaptureTask directly. It publishes explicitly; see mailIntake.ingest. +// +// # Production behaviour when nobody is watching +// +// A nil *event.Bus makes Publish a no-op, and newIntakeAPI with a nil bus +// returns the wrapped API unchanged, so there is not even a decorator on the +// call path. The journal is memory-only and is never consulted by the tick +// loop, the router, or delivery — nothing Maven says depends on it. It is a +// read surface (`/events`, `recent_events`) and an observation seam for the +// simulator. +// +// # What is deliberately NOT here +// +// No dispatch. An event is a report that something arrived, never an +// instruction to speak: "a feed item appeared" becoming a notification is the +// nag this repo refuses. Digestion may one day read the journal; it will still +// go through internal/loop's rules and the severity/presence routing table. +package main + +import ( + "context" + "encoding/json" + "log" + "strings" + "time" + + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/event" + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/store" +) + +// newEventBus builds the journal, or returns nil when the operator turned it +// off (a negative config.intake_journal). nil is the "behave exactly as before" +// value all the way down: no decorator, no ring, no /events rows. +func newEventBus(cfg *config.Config) *event.Bus { + if cfg == nil { + // No config at all is a test, not an operator decision. Saying "off" + // here was noise in every suite that passes nil. + return nil + } + if cfg.IntakeJournal < 0 { + log.Printf("intake journal: off (intake_journal < 0)") + return nil + } + n := cfg.IntakeJournal + if n == 0 { + n = config.DefaultIntakeJournal + } + log.Printf("intake journal: keeping the last %d intake events in memory", n) + return event.NewBus(n) +} + +// intakeEventsFn is the daemonAPI.getEvents closure: the bus's ring rendered as +// the wire type. Returns nil for a nil bus, which the daemonAPI reports as an +// empty journal rather than an error. +func intakeEventsFn(bus *event.Bus) func(n int) []ipc.IntakeEvent { + if bus == nil { + return nil + } + return func(n int) []ipc.IntakeEvent { + evs := bus.Recent(n) + out := make([]ipc.IntakeEvent, 0, len(evs)) + for _, e := range evs { + out = append(out, ipc.IntakeEvent{ + Source: e.Source, + Kind: e.Kind, + EntityIDs: e.EntityIDs, + Title: e.Title, + Body: e.Body, + Priority: e.Priority, + OccurredAt: e.OccurredAt, + NoticedAt: e.NoticedAt, + }) + } + return out + } +} + +// intakeAPI decorates a CoreAPI, publishing one envelope per successful +// intake write. Embedding the interface means every other method passes +// through untouched, and a new CoreAPI method is inherited rather than +// silently dropped. +type intakeAPI struct { + ipc.CoreAPI + bus *event.Bus + now func() time.Time +} + +// newIntakeAPI wraps api so its intake writes are journalled. A nil bus +// returns api itself — no decorator, no allocation, no behaviour change. +func newIntakeAPI(api ipc.CoreAPI, bus *event.Bus, now func() time.Time) ipc.CoreAPI { + if bus == nil || api == nil { + return api + } + if now == nil { + now = time.Now + } + return &intakeAPI{CoreAPI: api, bus: bus, now: now} +} + +// WriteFact journals the fact after it lands. Order matters: an event is a +// report of something that HAPPENED, so a failed write publishes nothing. +func (a *intakeAPI) WriteFact(ctx context.Context, req ipc.WriteFactReq) (int64, error) { + id, err := a.CoreAPI.WriteFact(ctx, req) + if err != nil { + return id, err + } + if selfWrite(req) { + // Maven's own bookkeeping is not something that arrived. The feed + // watermark, the crawl hash, the praxis trace of an act she performed + // and a quiet-hours toggle he pressed all used to sit on a page headed + // "everything that arrived", and on a cold start a handful of feeds + // could evict real intake behind their marks. + return id, nil + } + // OccurredAt is req.Ts, not now: mavpoll's wg read carries the handshake + // instant and the ambient path carries the meeting's start. Flattening + // those to notice-time would make the journal lie about when things + // happened, which is the one thing it is for. + title := req.Key + if req.VoidsID != nil { + // A retraction is not a reading. Without this it published an envelope + // indistinguishable from a fresh value for the same key, on a page + // whose whole job is "what came in". + title = "отмена: " + req.Key + } + a.bus.Publish(event.Event{ + Source: req.Source, + Kind: event.SourceKind(req.Source, event.KindFact), + Title: title, + Body: req.Value, + Priority: factPriority(req), + OccurredAt: req.Ts, + EntityIDs: entityIDs(req.Subject), + Payload: factPayload(req), + }, a.now()) + return id, nil +} + +// WriteNote journals a note. This is the RSS and crawler path, and also the +// meeting transcript and image description paths, which write their derived +// text as ordinary notes. +func (a *intakeAPI) WriteNote(ctx context.Context, ts time.Time, text string, embedding []float32, source string) (int64, error) { + id, err := a.CoreAPI.WriteNote(ctx, ts, text, embedding, source) + if err != nil { + return id, err + } + title, body := splitFirstLine(text) + a.bus.Publish(event.Event{ + Source: source, + Kind: event.SourceKind(source, event.KindNote), + Title: title, + Body: body, + Priority: event.PriorityLow, + OccurredAt: ts, + }, a.now()) + return id, nil +} + +// CaptureTask journals a captured task, but only when a row was actually +// created. CaptureTask dedupes on normalised text among live rows, so a +// mailbox re-read after a restart must not refill the journal with tasks that +// were already there. +func (a *intakeAPI) CaptureTask(ctx context.Context, req ipc.CaptureTaskReq) (ipc.CaptureTaskResp, error) { + resp, err := a.CoreAPI.CaptureTask(ctx, req) + if err != nil || !resp.Created { + return resp, err + } + a.bus.Publish(publishableTask(store.Task{ + CreatedTs: req.Ts, + Text: req.Text, + Source: req.Source, + Evidence: req.Evidence, + Status: req.Status, + Due: req.Due, + }, a.now()), a.now()) + return resp, nil +} + +// publishableTask is the task→envelope shape, shared with mail.go, which +// captures through the store directly rather than through the interface. +// +// Priority is high for a candidate with a due date and normal otherwise. That +// is the only place this file makes a judgement, and it is a display hint on a +// review page — nothing routes on it. +func publishableTask(t store.Task, now time.Time) event.Event { + occurred := t.CreatedTs + if occurred.IsZero() { + occurred = now + } + prio := event.PriorityNormal + if t.Due != nil { + prio = event.PriorityHigh + } + return event.Event{ + Source: t.Source, + Kind: event.KindTask, + Title: t.Text, + Body: t.Evidence, + Priority: prio, + OccurredAt: occurred, + } +} + +// selfWrite reports whether a fact write is Maven describing her own state +// rather than something arriving from outside. The store's fact kinds are +// 'self', 'env' and 'config'; 'config' is where every watermark and toggle +// lands, and the praxis trace is an audit record of an act she performed, which +// is the same class of thing under an 'env' kind. +func selfWrite(req ipc.WriteFactReq) bool { + switch req.Kind { + case "config", "system": + return true + } + return strings.HasPrefix(req.Source, "praxis:trace") +} + +// factPriority is the attention hint for a fact write. Deliberately crude: +// a low-confidence inference (the ambient notification path writes below 1.0) +// is worth less attention than a read he or a credentialled poller made, and a +// retraction is a correction rather than news. +// +// Confidence is NOT recoverable from this, which is why the number itself goes +// into Payload: three display buckets must not be the only surviving trace of +// the distinction internal/calendar went out of its way to keep. +func factPriority(req ipc.WriteFactReq) string { + if req.VoidsID != nil { + return event.PriorityLow + } + if req.Confidence > 0 && req.Confidence < 1.0 { + return event.PriorityLow + } + return event.PriorityNormal +} + +// factDetail is the fact-shaped Payload: the fields the envelope's own flat +// shape cannot carry, kept so a reader can tell an inference from a +// credentialled read, and "nobody said" from "certain". +type factDetail struct { + // FactKind — the fact's own kind ('self', 'env', 'config'), a different + // taxonomy from Event.Kind. + FactKind string `json:"fact_kind,omitempty"` + // Confidence — the number itself, so an inference stays distinguishable + // from a credentialled read. nil when the writer set none, which the ipc + // layer rejects today; the pointer keeps "nobody said" and "certain" from + // collapsing into each other the way the priority bucket does. + Confidence *float64 `json:"confidence,omitempty"` + // VoidsID — the fact this one retracts. + VoidsID *int64 `json:"voids_id,omitempty"` +} + +func factPayload(req ipc.WriteFactReq) json.RawMessage { + d := factDetail{FactKind: req.Kind, VoidsID: req.VoidsID} + if req.Confidence != 0 { + c := req.Confidence + d.Confidence = &c + } + if d.FactKind == "" && d.Confidence == nil && d.VoidsID == nil { + return nil + } + b, err := json.Marshal(d) + if err != nil { + return nil + } + return b +} + +// entityIDs turns a fact's free-text Subject into the EntityIDs slot when it +// already looks resolved. Intake runs BEFORE the fact enrichment worker +// resolves a subject against Nexus, so this is almost always empty — the slot +// exists for the paths that do know (the ecosystem acts), not for guessing. +func entityIDs(subject string) []string { + subject = strings.TrimSpace(subject) + if subject == "" || !strings.HasPrefix(subject, "entity:") { + return nil + } + return []string{strings.TrimPrefix(subject, "entity:")} +} + +// splitFirstLine renders a note as title + body. Feed and crawl notes are +// written "headline\nsummary\nlink", so the first line is already the title. +func splitFirstLine(text string) (title, body string) { + text = strings.TrimSpace(text) + if i := strings.IndexByte(text, '\n'); i >= 0 { + return strings.TrimSpace(text[:i]), strings.TrimSpace(text[i+1:]) + } + return text, "" +} diff --git a/cmd/mavend/intake_test.go b/cmd/mavend/intake_test.go new file mode 100644 index 0000000..a73103d --- /dev/null +++ b/cmd/mavend/intake_test.go @@ -0,0 +1,287 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/event" + "github.com/kami/maven/internal/ipc" +) + +var intakeNow = time.Date(2026, 8, 1, 10, 0, 0, 0, time.UTC) + +func intakeClock() time.Time { return intakeNow } + +// failingAPI wraps the store adapter, failing the three intake writes on +// demand, so the "a failed write publishes nothing" invariant is testable. +type failingAPI struct { + ipc.CoreAPI + fail bool +} + +func (f *failingAPI) WriteFact(ctx context.Context, req ipc.WriteFactReq) (int64, error) { + if f.fail { + return 0, errors.New("injected") + } + return f.CoreAPI.WriteFact(ctx, req) +} + +func newIntakeTestAPI(t *testing.T) (ipc.CoreAPI, *event.Bus) { + t.Helper() + st := newTestStore(t) + bus := event.NewBus(32) + return newIntakeAPI(ipc.NewStoreAPI(st), bus, intakeClock), bus +} + +func TestIntakeAPIWithoutBusIsTheBareAPI(t *testing.T) { + // The adoption invariant: with the journal off there is not even a + // decorator on the intake path, so production behaves exactly as before. + st := newTestStore(t) + bare := ipc.NewStoreAPI(st) + if got := newIntakeAPI(bare, nil, intakeClock); got != ipc.CoreAPI(bare) { + t.Errorf("newIntakeAPI with a nil bus returned a wrapper, want the bare API") + } +} + +func TestNewEventBusOffWhenNegative(t *testing.T) { + if b := newEventBus(&config.Config{IntakeJournal: -1}); b != nil { + t.Error("intake_journal = -1 still built a bus") + } + if b := newEventBus(&config.Config{IntakeJournal: 4}); b == nil { + t.Error("intake_journal = 4 built no bus") + } +} + +func TestIntakeJournalsAFactWrite(t *testing.T) { + api, bus := newIntakeTestAPI(t) + ctx := context.Background() + // The ambient path's shape: an env fact below full confidence, timestamped + // at the meeting's start rather than at notice time. + start := intakeNow.Add(2 * time.Hour) + if _, err := api.WriteFact(ctx, ipc.WriteFactReq{ + Ts: start, Kind: "env", Key: "calendar_event_20260801_планёрка", + Value: "10:00-11:00 планёрка", Source: "ambient:notif", Confidence: 0.6, + }); err != nil { + t.Fatalf("WriteFact: %v", err) + } + got := bus.Recent(0) + if len(got) != 1 { + t.Fatalf("journal has %d entries, want 1", len(got)) + } + e := got[0] + if e.Source != "ambient:notif" || e.Kind != event.KindFact { + t.Errorf("source/kind = %q/%q", e.Source, e.Kind) + } + if e.Title != "calendar_event_20260801_планёрка" { + t.Errorf("title = %q, want the fact key", e.Title) + } + if !e.OccurredAt.Equal(start) { + t.Errorf("occurred_at = %v, want the fact's Ts %v — the journal must not flatten intake to notice time", e.OccurredAt, start) + } + if e.Priority != event.PriorityLow { + t.Errorf("priority = %q, want %q for a sub-1.0 confidence read", e.Priority, event.PriorityLow) + } +} + +func TestIntakeDoesNotJournalAFailedWrite(t *testing.T) { + st := newTestStore(t) + bus := event.NewBus(8) + api := newIntakeAPI(&failingAPI{CoreAPI: ipc.NewStoreAPI(st), fail: true}, bus, intakeClock) + if _, err := api.WriteFact(context.Background(), ipc.WriteFactReq{ + Ts: intakeNow, Kind: "env", Key: "k", Value: "v", Source: "poll:zenmoney", Confidence: 1, + }); err == nil { + t.Fatal("expected the injected error") + } + if bus.Len() != 0 { + t.Errorf("journal has %d entries after a failed write, want 0 — an event reports something that happened", bus.Len()) + } +} + +func TestIntakeJournalsANoteAsTitlePlusBody(t *testing.T) { + api, bus := newIntakeTestAPI(t) + // The RSS shape: "headline\nsummary\nlink". + if _, err := api.WriteNote(context.Background(), intakeNow, + "Вышло ядро 6.19\nкраткое содержание\nhttps://example.org/a", nil, "rss:tech"); err != nil { + t.Fatalf("WriteNote: %v", err) + } + got := bus.Recent(1) + if len(got) != 1 { + t.Fatalf("journal has %d entries, want 1", len(got)) + } + if got[0].Title != "Вышло ядро 6.19" { + t.Errorf("title = %q, want the headline", got[0].Title) + } + if got[0].Kind != event.KindNote { + t.Errorf("kind = %q, want %q", got[0].Kind, event.KindNote) + } + if got[0].Body == "" { + t.Error("body is empty, want the rest of the note") + } +} + +func TestIntakeJournalsOnlyCreatedTasks(t *testing.T) { + api, bus := newIntakeTestAPI(t) + ctx := context.Background() + req := ipc.CaptureTaskReq{Text: "оплатить интернет", Source: "email:inbox", Status: "candidate", Ts: intakeNow} + if _, err := api.CaptureTask(ctx, req); err != nil { + t.Fatalf("CaptureTask: %v", err) + } + // Same text again: CaptureTask dedupes among live rows, and a re-read of a + // mailbox must not refill the journal. + resp, err := api.CaptureTask(ctx, req) + if err != nil { + t.Fatalf("CaptureTask (repeat): %v", err) + } + if resp.Created { + t.Fatal("store did not dedupe; the test cannot check what it means to") + } + if bus.Len() != 1 { + t.Errorf("journal has %d entries, want 1 — a deduped capture must not publish", bus.Len()) + } + if got := bus.Recent(1)[0]; got.Kind != event.KindTask || got.Title != "оплатить интернет" { + t.Errorf("entry = %+v, want the captured task", got) + } +} + +func TestIntakeEventsFnRendersNewestFirst(t *testing.T) { + api, bus := newIntakeTestAPI(t) + ctx := context.Background() + for _, key := range []string{"a", "b", "c"} { + if _, err := api.WriteFact(ctx, ipc.WriteFactReq{ + Ts: intakeNow, Kind: "env", Key: key, Value: "1", Source: "poll:zenmoney", Confidence: 1, + }); err != nil { + t.Fatalf("WriteFact %s: %v", key, err) + } + } + fn := intakeEventsFn(bus) + got := fn(2) + if len(got) != 2 || got[0].Title != "c" || got[1].Title != "b" { + t.Errorf("intakeEventsFn(2) = %+v, want the two newest, newest first", got) + } + if intakeEventsFn(nil) != nil { + t.Error("intakeEventsFn(nil) returned a closure, want nil so daemonAPI reports an empty journal") + } +} + +func TestDaemonAPIRecentEventsEmptyWithoutABus(t *testing.T) { + d := &daemonAPI{CoreAPI: ipc.UnimplementedCoreAPI{}} + got, err := d.RecentEvents(context.Background(), 10) + if err != nil { + t.Fatalf("RecentEvents with no journal errored: %v", err) + } + if len(got) != 0 { + t.Errorf("got %d events, want none", len(got)) + } +} + +// Maven's own bookkeeping is not intake. The feed watermark, the crawl hash, +// the praxis trace of an act she performed and the quiet-hours toggle he +// pressed all landed on a page headed "everything that arrived", and on a cold +// start a handful of feeds could evict real intake behind their marks. +func TestIntakeSkipsHerOwnBookkeeping(t *testing.T) { + api, bus := newIntakeTestAPI(t) + ctx := context.Background() + for _, req := range []ipc.WriteFactReq{ + {Ts: intakeNow, Kind: "config", Key: "rss:latest:tech", Value: "2026-08-01T09:00:00Z", Source: "poll:rss", Confidence: 1.0}, + {Ts: intakeNow, Kind: "config", Key: "crawl:hash:kernel", Value: "deadbeef", Source: "poll:crawl", Confidence: 1.0}, + {Ts: intakeNow, Kind: "config", Key: "quiet_hours", Value: "true", Source: "tap:voice", Confidence: 1.0}, + {Ts: intakeNow, Kind: "env", Key: "praxis:list_attention", Value: "ok", Source: "praxis:trace", Confidence: 1.0}, + } { + if _, err := api.WriteFact(ctx, req); err != nil { + t.Fatalf("WriteFact(%s): %v", req.Key, err) + } + } + if n := bus.Len(); n != 0 { + t.Fatalf("journalled %d bookkeeping writes, want 0: %+v", n, bus.Recent(0)) + } + // A real arrival under the same decorator still lands. + if _, err := api.WriteFact(ctx, ipc.WriteFactReq{ + Ts: intakeNow, Kind: "env", Key: "spend_today", Value: "1200", + Source: "poll:zenmoney", Confidence: 1.0, + }); err != nil { + t.Fatal(err) + } + if bus.Len() != 1 { + t.Fatalf("a real intake write was dropped: %+v", bus.Recent(0)) + } +} + +// Confidence is the distinction between an inference and a credentialled read, +// and the three-value priority bucket cannot carry it: unset and 1.0 land in +// the same bucket, and 0.6 is gone entirely once mapped. Payload keeps it. +func TestIntakeCarriesConfidenceAndFactKind(t *testing.T) { + api, bus := newIntakeTestAPI(t) + ctx := context.Background() + if _, err := api.WriteFact(ctx, ipc.WriteFactReq{ + Ts: intakeNow, Kind: "env", Key: "calendar_event_x", Value: "18:00 планёрка", + Source: "ambient:notif", Confidence: 0.6, + }); err != nil { + t.Fatal(err) + } + if _, err := api.WriteFact(ctx, ipc.WriteFactReq{ + Ts: intakeNow, Kind: "self", Key: "mood", Value: "ok", Source: "tap:web", Confidence: 1.0, + }); err != nil { + t.Fatal(err) + } + got := bus.Recent(0) + if len(got) != 2 { + t.Fatalf("journal has %d entries, want 2", len(got)) + } + var relayed, stated factDetail + if err := json.Unmarshal(got[1].Payload, &relayed); err != nil { + t.Fatalf("payload: %v", err) + } + if relayed.Confidence == nil || *relayed.Confidence != 0.6 { + t.Errorf("confidence = %v, want 0.6 recoverable from the payload", relayed.Confidence) + } + if relayed.FactKind != "env" { + t.Errorf("fact_kind = %q, want env", relayed.FactKind) + } + // Both writes land in PriorityNormal or PriorityLow buckets that cannot be + // told apart from the outside; the payload is where the two numbers stay + // distinguishable. + if err := json.Unmarshal(got[0].Payload, &stated); err != nil { + t.Fatalf("payload: %v", err) + } + if stated.Confidence == nil || *stated.Confidence != 1.0 || stated.FactKind != "self" { + t.Errorf("payload = %+v, want confidence 1.0 and fact_kind self", stated) + } +} + +// A retraction is not an observation. It used to publish an envelope +// indistinguishable from a fresh reading of the same key. +func TestIntakeMarksARetraction(t *testing.T) { + api, bus := newIntakeTestAPI(t) + ctx := context.Background() + id, err := api.WriteFact(ctx, ipc.WriteFactReq{ + Ts: intakeNow, Kind: "env", Key: "weight", Value: "82", Source: "tap:web", Confidence: 1.0, + }) + if err != nil { + t.Fatal(err) + } + if _, err := api.WriteFact(ctx, ipc.WriteFactReq{ + Ts: intakeNow, Kind: "env", Key: "weight", Value: "81", Source: "tap:web", + Confidence: 1.0, VoidsID: &id, + }); err != nil { + t.Fatal(err) + } + e := bus.Recent(1)[0] + if e.Priority != event.PriorityLow { + t.Errorf("priority = %q, want low for a correction", e.Priority) + } + if !strings.HasPrefix(e.Title, "отмена:") { + t.Errorf("title = %q, want it marked as a retraction", e.Title) + } + var d factDetail + if err := json.Unmarshal(e.Payload, &d); err != nil { + t.Fatal(err) + } + if d.VoidsID == nil || *d.VoidsID != id { + t.Errorf("voids_id = %v, want %d", d.VoidsID, id) + } +} diff --git a/cmd/mavend/keyfile.go b/cmd/mavend/keyfile.go new file mode 100644 index 0000000..15f9ecb --- /dev/null +++ b/cmd/mavend/keyfile.go @@ -0,0 +1,111 @@ +package main + +// Writing the wrapped-key blob (Vikunja #14). +// +// The blob is the only thing that opens the database on a cold-started box, so +// the two rules here are about not losing it. +// +// # It is rewritten on every assertion, so the write must be atomic +// +// mavweb calls StoreEncryptionKey after every successful assertion, not only +// after enrolment. os.WriteFile truncates in place: a power cut or an OOM kill +// between the truncate and the write left a zero-length blob and no previous +// contents, on the path of every routine step-up. Write to a temp file in the +// same directory, fsync it, rename over the target, then fsync the directory. +// +// # Only one authenticator can hold the cold-start key +// +// A blob is wrapped under one credential's PRF output and nothing else opens +// it. mavweb sends an empty allowCredentials list and the credential store +// keeps more than one passkey, so an unconditional rewrite meant the last +// authenticator to assert silently locked out every other one — including the +// backup hardware key enrolled for exactly the cold-start case. So: a blob +// that already opens under this secret and already wraps this key is left +// alone, a v1 blob is upgraded in place, and a v2 blob belonging to a +// different credential is refused rather than overwritten. + +import ( + "bytes" + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/kami/maven/internal/webauthn" +) + +// errForeignBlob — the wrapped key on disk belongs to another credential. +// Refusing is the point: overwriting would lock that authenticator out. +var errForeignBlob = errors.New("wrapped key belongs to a different credential") + +// wrapKeyToFile wraps key under secret and persists it at path, unless the +// blob already there says not to. Reports whether it wrote anything. +func wrapKeyToFile(path string, key, secret []byte) (wrote bool, err error) { + existing, err := os.ReadFile(path) + switch { + case err == nil: + plain, version, uerr := webauthn.UnwrapKey(existing, secret) + switch { + case uerr == nil && version == webauthn.BlobV2 && bytes.Equal(plain, key): + // Already wrapped under this secret, around this key. The + // common case on every assertion after the first. + return false, nil + case uerr != nil && version == webauthn.BlobV2: + return false, fmt.Errorf("%w: %s does not open under this assertion's PRF output, so another passkey holds the cold-start key; delete it deliberately to re-wrap", errForeignBlob, path) + } + // A v1 blob (upgrade it), or a v2 blob wrapping a stale key under + // this same secret (the key was rotated). Both are rewrites. + case errors.Is(err, os.ErrNotExist): + // First wrap. + default: + return false, fmt.Errorf("read wrapped key: %w", err) + } + + blob, err := webauthn.WrapKey(key, secret) + if err != nil { + return false, fmt.Errorf("wrap encryption key: %w", err) + } + if err := writeFileAtomic(path, blob, 0o600); err != nil { + return false, fmt.Errorf("write wrapped key: %w", err) + } + return true, nil +} + +// writeFileAtomic writes data to path so that a reader sees either the whole +// new file or the whole old one, never a truncated blob. +func writeFileAtomic(path string, data []byte, perm os.FileMode) error { + dir := filepath.Dir(path) + f, err := os.CreateTemp(dir, filepath.Base(path)+".tmp*") + if err != nil { + return err + } + tmp := f.Name() + defer os.Remove(tmp) // no-op once the rename succeeded + + if err := f.Chmod(perm); err != nil { + f.Close() + return err + } + if _, err := f.Write(data); err != nil { + f.Close() + return err + } + if err := f.Sync(); err != nil { + f.Close() + return err + } + if err := f.Close(); err != nil { + return err + } + if err := os.Rename(tmp, path); err != nil { + return err + } + // The rename itself needs to reach the disk, or a crash can resurrect the + // old directory entry pointing at a file that is gone. + d, err := os.Open(dir) + if err != nil { + return err + } + defer d.Close() + return d.Sync() +} diff --git a/cmd/mavend/keyfile_test.go b/cmd/mavend/keyfile_test.go new file mode 100644 index 0000000..bf3cb1e --- /dev/null +++ b/cmd/mavend/keyfile_test.go @@ -0,0 +1,187 @@ +package main + +import ( + "bytes" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/kami/maven/internal/webauthn" +) + +func wrapPath(t *testing.T) string { + t.Helper() + return filepath.Join(t.TempDir(), "db_key.wrapped") +} + +// The first wrap writes a v2 blob that opens under the same secret. +func TestWrapKeyToFileWritesAnOpenableBlob(t *testing.T) { + path := wrapPath(t) + key := bytes.Repeat([]byte{1}, 32) + secret := bytes.Repeat([]byte{2}, 32) + + wrote, err := wrapKeyToFile(path, key, secret) + if err != nil || !wrote { + t.Fatalf("wrapKeyToFile = %v, %v; want a write", wrote, err) + } + blob, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read blob: %v", err) + } + plain, version, err := webauthn.UnwrapKey(blob, secret) + if err != nil || version != webauthn.BlobV2 || !bytes.Equal(plain, key) { + t.Fatalf("UnwrapKey = %x, %v, %v", plain, version, err) + } + if fi, err := os.Stat(path); err != nil || fi.Mode().Perm() != 0o600 { + t.Fatalf("mode = %v (%v), want 0600", fi.Mode().Perm(), err) + } +} + +// A blob that already wraps this key under this secret is left alone. Without +// this every assertion rewrote the one file that opens the database. +func TestWrapKeyToFileSkipsAnIdenticalBlob(t *testing.T) { + path := wrapPath(t) + key := bytes.Repeat([]byte{3}, 32) + secret := bytes.Repeat([]byte{4}, 32) + + if _, err := wrapKeyToFile(path, key, secret); err != nil { + t.Fatalf("first wrap: %v", err) + } + before, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read: %v", err) + } + wrote, err := wrapKeyToFile(path, key, secret) + if err != nil { + t.Fatalf("second wrap: %v", err) + } + if wrote { + t.Error("rewrote a blob that already opens under this secret") + } + after, _ := os.ReadFile(path) + if !bytes.Equal(before, after) { + t.Error("the blob changed on a no-op wrap") + } +} + +// Two enrolled authenticators, two PRF secrets, one blob. The second must not +// silently lock the first one out — the backup passkey enrolled for exactly +// the cold-start case is the one thing that used to stop working. +func TestWrapKeyToFileRefusesAnotherCredentialsBlob(t *testing.T) { + path := wrapPath(t) + key := bytes.Repeat([]byte{5}, 32) + phone := bytes.Repeat([]byte{6}, 32) + yubikey := bytes.Repeat([]byte{7}, 32) + + if _, err := wrapKeyToFile(path, key, phone); err != nil { + t.Fatalf("first wrap: %v", err) + } + before, _ := os.ReadFile(path) + + wrote, err := wrapKeyToFile(path, key, yubikey) + if !errors.Is(err, errForeignBlob) { + t.Fatalf("wrapKeyToFile = %v, %v; want errForeignBlob", wrote, err) + } + after, _ := os.ReadFile(path) + if !bytes.Equal(before, after) { + t.Fatal("the second authenticator overwrote the first one's blob") + } + if _, _, err := webauthn.UnwrapKey(after, phone); err != nil { + t.Fatalf("the first authenticator can no longer open the blob: %v", err) + } +} + +// A v1 blob is the pre-#14 format. It is upgraded in place rather than +// refused, because that is the only way off a format that protects nothing. +func TestWrapKeyToFileUpgradesALegacyBlob(t *testing.T) { + path := wrapPath(t) + key := bytes.Repeat([]byte{8}, 32) + secret := bytes.Repeat([]byte{9}, 32) + + // A v1 blob is a v2 blob with the magic stripped and the v1 info string; + // the package writes no v1, so build one the only way a test can: wrap + // v2 under a public key, then hand the file a body with no magic. What + // matters here is only that UnwrapKey classifies it as v1. + v2, err := webauthn.WrapKey(key, secret) + if err != nil { + t.Fatalf("WrapKey: %v", err) + } + legacy := v2[7:] // drop the magic + if err := os.WriteFile(path, legacy, 0o600); err != nil { + t.Fatalf("write legacy blob: %v", err) + } + if _, version, _ := webauthn.UnwrapKey(legacy, secret); version != webauthn.BlobV1 { + t.Fatalf("fixture is not read as v1 (got %v)", version) + } + + wrote, err := wrapKeyToFile(path, key, secret) + if err != nil || !wrote { + t.Fatalf("wrapKeyToFile = %v, %v; want the legacy blob upgraded", wrote, err) + } + blob, _ := os.ReadFile(path) + if _, version, err := webauthn.UnwrapKey(blob, secret); err != nil || version != webauthn.BlobV2 { + t.Fatalf("after upgrade: version %v, err %v", version, err) + } +} + +// A rotated at-rest key under the same credential is a rewrite, not a no-op. +func TestWrapKeyToFileRewritesARotatedKey(t *testing.T) { + path := wrapPath(t) + secret := bytes.Repeat([]byte{10}, 32) + old := bytes.Repeat([]byte{11}, 32) + fresh := bytes.Repeat([]byte{12}, 32) + + if _, err := wrapKeyToFile(path, old, secret); err != nil { + t.Fatalf("first wrap: %v", err) + } + wrote, err := wrapKeyToFile(path, fresh, secret) + if err != nil || !wrote { + t.Fatalf("wrapKeyToFile = %v, %v; want the rotated key written", wrote, err) + } + blob, _ := os.ReadFile(path) + plain, _, err := webauthn.UnwrapKey(blob, secret) + if err != nil || !bytes.Equal(plain, fresh) { + t.Fatalf("blob still wraps the old key (%v)", err) + } +} + +// The write never truncates the target in place, so a crash mid-write cannot +// leave a zero-length blob where the only copy of the wrapped key was. +func TestWriteFileAtomicLeavesNoTempFilesAndReplacesWhole(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "db_key.wrapped") + + if err := os.WriteFile(path, bytes.Repeat([]byte{0xaa}, 67), 0o600); err != nil { + t.Fatalf("seed: %v", err) + } + // Hold the old inode. A rename gives it a new one; a truncating write + // would keep it. + oldInfo, err := os.Stat(path) + if err != nil { + t.Fatalf("stat: %v", err) + } + + want := bytes.Repeat([]byte{0xbb}, 67) + if err := writeFileAtomic(path, want, 0o600); err != nil { + t.Fatalf("writeFileAtomic: %v", err) + } + got, err := os.ReadFile(path) + if err != nil || !bytes.Equal(got, want) { + t.Fatalf("content = %x (%v)", got, err) + } + newInfo, err := os.Stat(path) + if err != nil { + t.Fatalf("stat: %v", err) + } + if os.SameFile(oldInfo, newInfo) { + t.Error("the target was written in place, not renamed over") + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("readdir: %v", err) + } + if len(entries) != 1 { + t.Errorf("directory holds %d entries, want just the blob (a temp file leaked)", len(entries)) + } +} diff --git a/cmd/mavend/mail.go b/cmd/mavend/mail.go new file mode 100644 index 0000000..904dd1d --- /dev/null +++ b/cmd/mavend/mail.go @@ -0,0 +1,241 @@ +// mavend/mail.go — core's half of the email reader (Vikunja #246, +// docs/plans/01-email-reader.md). +// +// The split: cmd/mavmaild holds the IMAP credential, connects to the mailbox +// and converts messages to plaintext; it hands each message to core over +// ipc.MethodIngestMail. Core runs the extraction on the resident model — +// llama-server lives in this process, spawned by the phraser — and writes what +// comes back through the one task intake seam. +// +// What this file may produce is exactly one thing: rows in `tasks` with status +// "candidate". No fact, no reminder, no note, no nudge, no calendar event. A +// 1.7B misreading a mail can therefore put a wrong line on a review page and +// nothing else; it can never make Maven speak, and it can never make her +// recite something out of an advert as true. +// +// Off unless configured twice over: no `email` block in mavend.json ⇒ the IPC +// method does not exist; no llama-server phraser ⇒ same. A reader pointed at a +// core that is not set up for mail gets ErrUnknownMethod rather than silence. +package main + +import ( + "context" + "fmt" + "log" + "strings" + "time" + "unicode" + + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/email" + "github.com/kami/maven/internal/event" + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/phraser" + "github.com/kami/maven/internal/store" +) + +// evidenceMaxChars — how much of the subject line is kept as a candidate's +// evidence. Enough to recognise the mail on /tasks, not enough to turn the task +// list into a copy of his mailbox. +const evidenceMaxChars = 160 + +// captureTimeout — how long the capture writes get, separately from the +// extraction budget. A candidate the model already produced must not be lost +// because the model was slow. +const captureTimeout = 30 * time.Second + +// maxMailboxChars — a mailbox name is an IMAP folder, not free text. It ends up +// in the provenance string, which is a small controlled vocabulary. +const maxMailboxChars = 64 + +// validMailbox checks the name this method is willing to write provenance for. +// Empty is refused: "email:" is not a source. So is anything with a control +// character or a space-only value, so the source string stays greppable and +// stays one token. +func validMailbox(s string) (string, error) { + s = strings.TrimSpace(s) + if s == "" { + return "", fmt.Errorf("mail intake: mailbox is required") + } + if len([]rune(s)) > maxMailboxChars { + return "", fmt.Errorf("mail intake: mailbox name too long") + } + for _, r := range s { + if r < 0x20 || r == 0x7f || unicode.IsSpace(r) { + return "", fmt.Errorf("mail intake: mailbox name has whitespace or a control character") + } + } + return s, nil +} + +// mailIntake — extraction + capture for one message at a time. +type mailIntake struct { + st *store.Store + ex *email.Extractor + timeout time.Duration + now func() time.Time + // bus — the unified intake journal (Vikunja #283). This path captures + // through the store directly rather than through ipc.CoreAPI, so the + // decorator in intake.go does not see it and the publish is explicit here. + // nil is a working no-op. + bus *event.Bus +} + +// newMailIntake returns nil when mail ingestion must not be available, which is +// the default. Both preconditions are real: +// +// - no cfg.Email ⇒ not configured, and a capability is off unless configured; +// - no llama-server phraser ⇒ nothing to extract with. There is deliberately +// no keyword fallback: "the subject line became a task" is not extraction, +// it is a mailbox rendered as a to-do list, and it would fill the review +// page faster than he could clear it. +func newMailIntake(st *store.Store, phr phraser.Phraser, cfg *config.Config, bus *event.Bus) *mailIntake { + if cfg.Email == nil { + return nil + } + lp, ok := phr.(*phraser.LLMPhraser) + if !ok { + // The phraser is not an *LLMPhraser. Today that means there is no + // llama-server; if anything ever WRAPS the phraser it will mean that + // instead, so the line names the assertion rather than guessing why. + log.Printf("mail intake: configured but the phraser is not an *phraser.LLMPhraser (%T) — mail ingestion disabled", phr) + return nil + } + timeout := time.Duration(cfg.Email.Timeout) + if timeout <= 0 { + timeout = config.DefaultEmailTimeout + } + // Background client: extraction is a job nobody is waiting on, and it shares + // one llama-server slot with the voice turn. Through the gate it yields to + // anything he is waiting for and only one extraction runs at a time, so a + // first poll of 25 unseen messages cannot queue 25 model calls in front of + // him. See llm.Gate. + ex := email.NewExtractor(llmBackgroundClientFor(lp, timeout), cfg.Email.MaxTasks, contextBlockFn(cfg, time.Now)) + // The NORMALISED bound, not the configured one: with "email": {} in + // mavend.json the configured value is 0 and the daemon allows three. + log.Printf("mail intake: enabled (max %d candidates per message, timeout %s)", ex.Max(), timeout) + return &mailIntake{st: st, ex: ex, timeout: timeout, now: time.Now, bus: bus} +} + +// ingest handles one ipc.MethodIngestMail call. +// +// Junk and empty messages are answered Skipped without touching the model — the +// reader's header filter is what keeps the resident model off newsletters. +// +// Every candidate is captured with Status "candidate", Source "email:" +// and the subject as Evidence, under an ExternalID naming the message and the +// span it was extracted from. That key is unique over every row whatever its +// status, so a mailbox re-read after a restart produces Created=0 — and, more +// to the point, a task he already marked done is not re-proposed the next time +// the same unread message is read again. +func (m *mailIntake) ingest(ctx context.Context, req ipc.IngestMailReq) (ipc.IngestMailResp, error) { + // The mailbox name becomes provenance ("email:INBOX"), and the source + // vocabulary is what the loop's rules trust. An empty name gave "email:" and + // an arbitrary string gave an arbitrary source under that namespace. + mailbox, err := validMailbox(req.Mailbox) + if err != nil { + return ipc.IngestMailResp{}, err + } + msg := email.Message{ + UID: req.UID, + From: req.From, + Subject: req.Subject, + Date: req.Date, + Body: req.Body, + Junk: req.Junk, + } + if msg.Junk || (msg.Subject == "" && msg.Body == "") { + return ipc.IngestMailResp{Skipped: true}, nil + } + + // The timeout scopes the EXTRACTION and nothing else. It used to wrap the + // capture writes too, so a model that answered at 119 seconds of a 120 + // second budget left the first CaptureTask one second and the third none: + // the work was done, the answer was good, and it was dropped with a + // deadline error. Config calls this a per-message extraction budget, and now + // it is one. + exCtx, cancel := context.WithTimeout(ctx, m.timeout) + cands, err := m.ex.Extract(exCtx, msg) + cancel() + if err != nil { + // The error from internal/email never carries mail text; keep it that way + // by not adding the subject here. + return ipc.IngestMailResp{}, fmt.Errorf("mail intake: uid %d: %w", req.UID, err) + } + if len(cands) == 0 { + return ipc.IngestMailResp{}, nil + } + + // A fresh budget for the writes, derived from the caller's context rather + // than from the extraction's. Encrypted-store writes are fast; what this + // bounds is a stuck store, not the model. + ctx, cancel = context.WithTimeout(ctx, captureTimeout) + defer cancel() + + source := email.SourcePrefix + mailbox + evidence := truncateRunes(req.Subject, evidenceMaxChars) + now := m.now() + var resp ipc.IngestMailResp + for _, c := range cands { + t := store.Task{ + CreatedTs: now, + Text: c.Text, + Source: source, + Evidence: evidence, + // The one status this path may ever write. Anything Maven derived from + // something she read is a suggestion until he confirms it on /tasks. + Status: store.TaskCandidate, + } + t.ExternalID = mailExternalID(source, req.UID, c.Text) + if due, ok := email.ParseDue(c.Due); ok { + t.Due = &due + } + res, err := m.st.CaptureTask(ctx, t) + if err != nil { + return resp, fmt.Errorf("mail intake: capture: %w", err) + } + resp.TaskIDs = append(resp.TaskIDs, res.ID) + if res.Created { + resp.Created++ + // Only a row that was actually created. CaptureTask dedupes on + // normalised text among live rows, so a mailbox re-read after a + // restart must not refill the journal with tasks already in it. + m.bus.Publish(publishableTask(t, now), now) + } + } + // Counts only: the log line names the mailbox and the UID, never the subject, + // the sender or the task text. Reviewing a candidate is what /tasks is for. + log.Printf("mail intake: %s uid %d → %d candidate(s), %d new", source, req.UID, len(cands), resp.Created) + return resp, nil +} + +// wireMailIntake installs the IPC hook, or leaves it nil so the method reports +// ErrUnknownMethod. Called on both startup paths (unlocked boot and passkey +// unlock) so mail behaves the same either way. +func wireMailIntake(srv *ipc.Server, st *store.Store, phr phraser.Phraser, cfg *config.Config, bus *event.Bus) { + mi := newMailIntake(st, phr, cfg, bus) + if mi == nil { + return + } + srv.IngestMailFn = mi.ingest +} + +// truncateRunes cuts a string to n runes, marking the cut. +func truncateRunes(s string, n int) string { + r := []rune(s) + if len(r) <= n { + return s + } + return string(r[:n]) + "…" +} + +// mailExternalID names the message and the span a candidate was extracted +// from. The mailbox and UID identify the message; the normalised text +// identifies which of the candidates in it this is, so a message yielding two +// tasks gets two keys and a re-read of it gets neither twice. +// +// UIDs are stable per mailbox, and a mailbox that renumbers (UIDVALIDITY +// changing) re-proposes its tasks once, which is the safe direction. +func mailExternalID(source string, uid uint32, text string) string { + return fmt.Sprintf("%s#%d:%s", source, uid, store.NormalizeTaskText(text)) +} diff --git a/cmd/mavend/mail_test.go b/cmd/mavend/mail_test.go new file mode 100644 index 0000000..383d64b --- /dev/null +++ b/cmd/mavend/mail_test.go @@ -0,0 +1,241 @@ +package main + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/email" + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/llm" + "github.com/kami/maven/internal/store" +) + +// mailLLM — a canned extraction reply. +type mailLLM struct { + reply string + calls int +} + +func (m *mailLLM) Complete(_ context.Context, _ llm.Req) (string, error) { + m.calls++ + return m.reply, nil +} + +func newTestIntake(t *testing.T, reply string) (*mailIntake, *store.Store, *mailLLM) { + t.Helper() + st := newTestStore(t) + fake := &mailLLM{reply: reply} + return &mailIntake{ + st: st, + ex: email.NewExtractor(fake, 0, nil), + timeout: 5 * time.Second, + now: func() time.Time { return time.Date(2026, 8, 1, 10, 0, 0, 0, time.UTC) }, + }, st, fake +} + +func ingestReq() ipc.IngestMailReq { + return ipc.IngestMailReq{ + Mailbox: "INBOX", UID: 42, + From: "billing@isp.example", + Subject: "Счёт за интернет", + Body: "Оплатите счёт до 5 августа.", + } +} + +// The one property that matters: a mail-derived task is a candidate, attributed +// to the mailbox, with the subject as reviewable evidence — and nothing else is +// written. +func TestIngestCapturesCandidates(t *testing.T) { + mi, st, _ := newTestIntake(t, `[{"text":"оплатить счёт за интернет","due":"2026-08-05"}]`) + resp, err := mi.ingest(context.Background(), ingestReq()) + if err != nil { + t.Fatalf("ingest: %v", err) + } + if resp.Created != 1 || len(resp.TaskIDs) != 1 { + t.Fatalf("resp = %+v, want one created task", resp) + } + tasks, err := st.ListTasks(context.Background(), "") + if err != nil { + t.Fatalf("list: %v", err) + } + if len(tasks) != 1 { + t.Fatalf("got %d tasks, want 1", len(tasks)) + } + got := tasks[0] + if got.Status != store.TaskCandidate { + t.Errorf("status = %q, want %q — mail may only produce candidates", got.Status, store.TaskCandidate) + } + if got.Source != "email:INBOX" { + t.Errorf("source = %q, want email:INBOX", got.Source) + } + if got.Evidence != "Счёт за интернет" { + t.Errorf("evidence = %q, want the subject line", got.Evidence) + } + if got.Due == nil || got.Due.Format("2006-01-02") != "2026-08-05" { + t.Errorf("due = %v, want 2026-08-05", got.Due) + } + // Nothing else may have been written: no reminder, no fact. + rem, err := st.ListReminders(context.Background(), 10) + if err != nil { + t.Fatalf("list reminders: %v", err) + } + if len(rem) != 0 { + t.Errorf("mail created %d reminders; a misread mail must never be able to fire", len(rem)) + } +} + +// Re-reading a mailbox must not grow the list — CaptureTask dedupes among live +// rows, and the intake relies on exactly that. +func TestIngestSameMailTwiceIsIdempotent(t *testing.T) { + mi, st, _ := newTestIntake(t, `[{"text":"оплатить счёт","due":""}]`) + if _, err := mi.ingest(context.Background(), ingestReq()); err != nil { + t.Fatalf("first ingest: %v", err) + } + resp, err := mi.ingest(context.Background(), ingestReq()) + if err != nil { + t.Fatalf("second ingest: %v", err) + } + if resp.Created != 0 || len(resp.TaskIDs) != 1 { + t.Errorf("resp = %+v, want the existing row and Created=0", resp) + } + tasks, _ := st.ListTasks(context.Background(), "") + if len(tasks) != 1 { + t.Errorf("got %d tasks after two reads, want 1", len(tasks)) + } +} + +func TestIngestJunkSkipsTheModel(t *testing.T) { + mi, st, fake := newTestIntake(t, `[{"text":"купить со скидкой","due":""}]`) + req := ingestReq() + req.Junk = true + resp, err := mi.ingest(context.Background(), req) + if err != nil { + t.Fatalf("ingest: %v", err) + } + if !resp.Skipped || resp.Created != 0 { + t.Errorf("resp = %+v, want skipped", resp) + } + if fake.calls != 0 { + t.Errorf("model called %d times for junk, want 0", fake.calls) + } + if tasks, _ := st.ListTasks(context.Background(), ""); len(tasks) != 0 { + t.Errorf("junk produced %d tasks, want 0", len(tasks)) + } +} + +func TestIngestEmptyMessageSkipped(t *testing.T) { + mi, _, fake := newTestIntake(t, "[]") + resp, err := mi.ingest(context.Background(), ipc.IngestMailReq{Mailbox: "INBOX", UID: 1}) + if err != nil || !resp.Skipped { + t.Fatalf("resp = %+v, err = %v; want skipped", resp, err) + } + if fake.calls != 0 { + t.Errorf("model called %d times for an empty message, want 0", fake.calls) + } +} + +func TestIngestNoTasksWritesNothing(t *testing.T) { + mi, st, _ := newTestIntake(t, "[]") + resp, err := mi.ingest(context.Background(), ingestReq()) + if err != nil { + t.Fatalf("ingest: %v", err) + } + if resp.Created != 0 || len(resp.TaskIDs) != 0 || resp.Skipped { + t.Errorf("resp = %+v, want nothing captured and not skipped", resp) + } + if tasks, _ := st.ListTasks(context.Background(), ""); len(tasks) != 0 { + t.Errorf("got %d tasks, want 0", len(tasks)) + } +} + +func TestIngestTruncatesEvidence(t *testing.T) { + mi, st, _ := newTestIntake(t, `[{"text":"дело","due":""}]`) + req := ingestReq() + req.Subject = strings.Repeat("щ", 400) + if _, err := mi.ingest(context.Background(), req); err != nil { + t.Fatalf("ingest: %v", err) + } + tasks, _ := st.ListTasks(context.Background(), "") + if len(tasks) != 1 { + t.Fatalf("got %d tasks, want 1", len(tasks)) + } + if n := len([]rune(tasks[0].Evidence)); n > evidenceMaxChars+1 { + t.Errorf("evidence kept %d runes, want ≤ %d", n, evidenceMaxChars) + } +} + +// Off unless configured: no email block ⇒ no intake, so the IPC method does not +// exist at all. +func TestNewMailIntakeOffWithoutConfig(t *testing.T) { + st := newTestStore(t) + if mi := newMailIntake(st, nil, &config.Config{}, nil); mi != nil { + t.Error("no email block must mean no mail intake") + } + // Configured but with a non-LLM phraser: still off — there is no fallback + // extraction, by design. + if mi := newMailIntake(st, nil, &config.Config{Email: &config.EmailConfig{}}, nil); mi != nil { + t.Error("without a llama-server phraser there is nothing to extract with") + } +} + +// The mailbox name becomes the provenance string, which is the vocabulary the +// loop's rules trust. "email:" is not a source and neither is "email:anything +// he could post at the socket". +func TestIngestRejectsBadMailbox(t *testing.T) { + for _, name := range []string{"", " ", "IN BOX", "IN\nBOX", "IN\x00BOX", strings.Repeat("щ", maxMailboxChars+1)} { + mi, st, fake := newTestIntake(t, `[{"text":"дело","due":""}]`) + req := ingestReq() + req.Mailbox = name + if _, err := mi.ingest(context.Background(), req); err == nil { + t.Errorf("mailbox %q was accepted", name) + } + if fake.calls != 0 { + t.Errorf("mailbox %q reached the model", name) + } + if tasks, _ := st.ListTasks(context.Background(), ""); len(tasks) != 0 { + t.Errorf("mailbox %q wrote %d tasks", name, len(tasks)) + } + } +} + +// slowLLM burns most of the extraction budget before answering, the way a +// Thinking 1.7B does on a long mail. +type slowLLM struct { + reply string + delay time.Duration +} + +func (s *slowLLM) Complete(ctx context.Context, _ llm.Req) (string, error) { + select { + case <-time.After(s.delay): + return s.reply, nil + case <-ctx.Done(): + return "", ctx.Err() + } +} + +// The extraction budget must not also bound the writes. It used to be one +// context, so a model answering near the deadline lost the candidates it had +// just produced. +func TestIngestCapturesAfterASlowExtraction(t *testing.T) { + st := newTestStore(t) + mi := &mailIntake{ + st: st, + ex: email.NewExtractor(&slowLLM{reply: `[{"text":"оплатить счёт","due":""}]`, delay: 90 * time.Millisecond}, 0, nil), + timeout: 100 * time.Millisecond, + now: func() time.Time { return time.Date(2026, 8, 1, 10, 0, 0, 0, time.UTC) }, + } + resp, err := mi.ingest(context.Background(), ingestReq()) + if err != nil { + t.Fatalf("ingest: %v", err) + } + if resp.Created != 1 { + t.Fatalf("resp = %+v, want the candidate captured", resp) + } + if tasks, _ := st.ListTasks(context.Background(), ""); len(tasks) != 1 { + t.Errorf("got %d tasks, want 1", len(tasks)) + } +} diff --git a/cmd/mavend/main.go b/cmd/mavend/main.go index 2966968..b74b080 100644 --- a/cmd/mavend/main.go +++ b/cmd/mavend/main.go @@ -25,7 +25,7 @@ // When a passkey credential is enrolled AND no env key is set, the daemon // starts in LOCKED mode: the IPC server runs but rejects all store methods // except MethodAssertStepUp and MethodUnlock. A passkey assertion followed -// by MethodUnlock (with the same credential's public key) unwraps the at-rest +// by MethodUnlock (with that credential's WebAuthn PRF output) unwraps the at-rest // AES-256 key from a wrapped blob on disk (HKDF-SHA256 + AES-GCM) and opens // the encrypted store. After unlock, the daemon wires voice, loop, and // delivery and runs normally. @@ -34,10 +34,13 @@ // starts unlocked from the env key (pre-unlock behavior). Enrolling a passkey // while unlocked calls MethodStoreEncryptionKey to wrap the env key and // persist the wrapped blob — enabling cold-start unlock on the next boot -// after the env key is removed. +// after the env key is removed. That write happens once, when no blob +// exists; replacing an existing one takes an explicit request, see +// cmd/mavend/keyfile.go. package main import ( + "bytes" "context" "encoding/json" "errors" @@ -48,6 +51,7 @@ import ( "os" "os/signal" "sync" + "sync/atomic" "syscall" "time" @@ -66,12 +70,19 @@ import ( var errLocked = errors.New("mavend: daemon locked — complete passkey assertion first") -// daemonLock tracks whether the daemon is in locked (pre-unlock) mode. -// In locked mode, all CoreAPI methods return errLocked. The unlock path -// replaces the CoreAPI with the real store adapter and flips the flag. +// daemonLock tracks whether the daemon is in locked (pre-unlock) mode, and +// owns the store handle the unlock path creates. +// +// The store matters here because of who runs when. In locked mode there is no +// store at boot; one is opened inside UnlockFn, on an IPC goroutine, minutes +// or days later. Shutdown runs on the main goroutine. Without a handoff the +// main goroutine has nothing to close, and store.Close is what re-encrypts +// the tmpfs working copy back over the ciphertext file — so a daemon that +// cold-started lost every write of that session, silently, on the next boot. type daemonLock struct { mu sync.Mutex locked bool + st *store.Store } func newDaemonLock(locked bool) *daemonLock { @@ -84,10 +95,25 @@ func (l *daemonLock) isLocked() bool { return l.locked } -func (l *daemonLock) unlock() { +// unlock flips the flag and takes ownership of the store opened by UnlockFn. +func (l *daemonLock) unlock(st *store.Store) { l.mu.Lock() defer l.mu.Unlock() l.locked = false + l.st = st +} + +// closeStore seals the store the unlock path opened, if any. Safe to call +// when the daemon never unlocked, and safe to call twice. +func (l *daemonLock) closeStore() error { + l.mu.Lock() + st := l.st + l.st = nil + l.mu.Unlock() + if st == nil { + return nil + } + return st.Close() } func main() { @@ -97,100 +123,10 @@ func main() { } } -// lockedAPI is a dummy CoreAPI used while the daemon is locked. Every method -// returns errLocked. The wire protocol's StoreAPI methods all go through the -// Server dispatch on CoreAPI, so returning errLocked from each is correct. -type lockedAPI struct{} - -var _ ipc.CoreAPI = (*lockedAPI)(nil) - -func (l *lockedAPI) WriteFact(ctx context.Context, req ipc.WriteFactReq) (int64, error) { - return 0, errLocked -} -func (l *lockedAPI) LatestFact(ctx context.Context, key string) (ipc.Fact, error) { - return ipc.Fact{}, errLocked -} -func (l *lockedAPI) LatestFactBySource(ctx context.Context, key, source string) (ipc.Fact, error) { - return ipc.Fact{}, errLocked -} -func (l *lockedAPI) Since(ctx context.Context, key string, now time.Time) (time.Duration, error) { - return 0, errLocked -} -func (l *lockedAPI) Presence(ctx context.Context) (ipc.Presence, error) { - return ipc.Presence{}, errLocked -} -func (l *lockedAPI) CreateReminder(ctx context.Context, fire time.Time, payload, cron string) (int64, error) { - return 0, errLocked -} -func (l *lockedAPI) MarkReminder(ctx context.Context, id int64, status string) error { - return errLocked -} -func (l *lockedAPI) ListReminders(ctx context.Context, n int) ([]ipc.Reminder, error) { - return nil, errLocked -} -func (l *lockedAPI) RecordNudge(ctx context.Context, rule, channel, message string, ts time.Time) (int64, error) { - return 0, errLocked -} -func (l *lockedAPI) ResolveNudge(ctx context.Context, id int64, outcome string, ts time.Time) error { - return errLocked -} -func (l *lockedAPI) RecentOutcomes(ctx context.Context, rule string, n int) ([]string, error) { - return nil, errLocked -} -func (l *lockedAPI) RecentFacts(ctx context.Context, n int) ([]ipc.Fact, error) { - return nil, errLocked -} -func (l *lockedAPI) CalendarEvents(ctx context.Context, from, to time.Time) ([]ipc.Fact, error) { - return nil, errLocked -} -func (l *lockedAPI) RecentNudges(ctx context.Context, n int) ([]ipc.Nudge, error) { - return nil, errLocked -} -func (l *lockedAPI) WriteNote(ctx context.Context, ts time.Time, text string, embedding []float32, source string) (int64, error) { - return 0, errLocked -} -func (l *lockedAPI) QueryNotes(ctx context.Context, embedding []float32, k int) ([]ipc.Note, error) { - return nil, errLocked -} -func (l *lockedAPI) RecentNotes(ctx context.Context, n int) ([]ipc.Note, error) { - return nil, errLocked -} -func (l *lockedAPI) ProposeTool(ctx context.Context, name, utterance, scope string, ts time.Time) (bool, error) { - return false, errLocked -} -func (l *lockedAPI) EnableTool(ctx context.Context, name string, cmd []string, destructive bool, scope string, ts time.Time) error { - return errLocked -} -func (l *lockedAPI) DisableTool(ctx context.Context, name string) error { return errLocked } -func (l *lockedAPI) DeleteTool(ctx context.Context, name string) error { return errLocked } -func (l *lockedAPI) ListProposedRoutines(ctx context.Context) ([]ipc.ProposedRoutine, error) { - return nil, errLocked -} -func (l *lockedAPI) DismissProposedRoutine(ctx context.Context, id int64) error { return errLocked } -func (l *lockedAPI) AcceptProposedRoutine(ctx context.Context, id int64) error { - return errLocked -} -func (l *lockedAPI) LookupTool(ctx context.Context, name string) (ipc.Tool, error) { - return ipc.Tool{}, errLocked -} -func (l *lockedAPI) ListTools(ctx context.Context, status string) ([]ipc.Tool, error) { - return nil, errLocked -} -func (l *lockedAPI) RevertFact(ctx context.Context, key string) (int64, error) { return 0, errLocked } -func (l *lockedAPI) Chat(ctx context.Context, text string) (string, error) { - return "", errLocked -} -func (l *lockedAPI) TickTrace(ctx context.Context) (ipc.TickTrace, error) { - return ipc.TickTrace{}, errLocked -} -func (l *lockedAPI) MorningStatus(ctx context.Context) ([]ipc.MorningRoutineStatus, error) { - return nil, errLocked -} - func run(args []string) error { cfgPath := flag.String("config", defaultConfigPath(), "path to mavend JSON config") wrappedKeyPath := flag.String("wrapped-key-file", "", "path to wrapped encryption key blob (enables cold-start unlock)") - reembed := flag.Bool("reembed", false, "re-embed every stored note and fact with the configured embedder, then serve normally (run once after an embedder swap)") + reembed := flag.Bool("reembed", false, "re-embed every stored note and fact with the configured embedder, then serve normally (run once after an embedder swap; the daemon does not answer until it finishes)") flag.CommandLine.Parse(args) reembedOnStart = *reembed cfg, err := config.Load(*cfgPath) @@ -232,11 +168,28 @@ func run(args []string) error { var st *store.Store var envKeyBytes []byte // kept for WrapKeyFn (enrollment wraps this key) + // dbKey — the plaintext at-rest key, once the daemon has one. Set at boot + // in env-key mode and inside UnlockFn after a cold start. WrapKeyFn reads + // it from an IPC goroutine, hence the atomic: srv's function fields are + // installed before Serve and must not be reassigned afterwards. + var dbKey atomic.Pointer[[]byte] + + // wrappedPath resolves the blob location the same way for both the read + // at boot and every write, so a default-path deployment cannot wrap to + // one file and unwrap from another. + wrappedPath := func() string { + if *wrappedKeyPath != "" { + return *wrappedKeyPath + } + return cfg.DefaultWrappedKeyPath() + } + if !locked { // Normal boot: env key or plaintext (dev/CI) if envKey != nil { envKeyBytes = make([]byte, len(envKey)) copy(envKeyBytes, envKey) + dbKey.Store(&envKeyBytes) st, err = store.OpenEncrypted(ctx, cfg.DBPath, cfg.DBTmpfs, envKey) } else { st, err = store.Open(ctx, cfg.DBPath) @@ -245,6 +198,14 @@ func run(args []string) error { return fmt.Errorf("open store: %w", err) } defer st.Close() + } else { + // Locked boot: the store does not exist yet. Seal whatever UnlockFn + // opened, at shutdown, on this goroutine. + defer func() { + if err := dl.closeStore(); err != nil { + log.Printf("mavend: seal store on shutdown: %v", err) + } + }() } // ----- daemon components (only wired when unlocked) ----- @@ -259,8 +220,21 @@ func run(args []string) error { coreAPI ipc.CoreAPI eco *ecosystemWiring factWorker *factEnrichmentWorker + evalWorker *memoryEvalWorker // nil ⇒ memory evaluation off (the default) + feedWkr *feedWorker // nil ⇒ no feed is read (the default) + crawlWkr *crawlWorker // nil ⇒ no page is watched (the default) ) + // The unified intake journal (Vikunja #283). Built before anything else + // that holds a CoreAPI, because intakeAPI wraps that one interface and + // every intake path in the daemon reaches its sink through it. nil (the + // operator set intake_journal negative) means no decorator at all. + evBus := newEventBus(cfg) + // coreFor is what every in-process holder of a CoreAPI now takes, instead + // of a bare ipc.NewStoreAPI(st). Identical behaviour plus one published + // envelope per successful intake write. + coreFor := func() ipc.CoreAPI { return newIntakeAPI(ipc.NewStoreAPI(st), evBus, time.Now) } + if !locked { rules = loop.DefaultRules() gatherer = loop.NewGatherer(st, rules) @@ -304,7 +278,7 @@ func run(args []string) error { eco = wireEcosystem(cfg) // voice - voiceW, err = wireVoice(cfg, ipc.NewStoreAPI(st), phr, st.VectorMemory(), st, eco) + voiceW, err = wireVoice(cfg, coreFor(), phr, st.VectorMemory(), st, eco) if err != nil { return fmt.Errorf("wire voice: %w", err) } @@ -351,21 +325,34 @@ func run(args []string) error { tickInterval := time.Duration(cfg.TickInterval) repeatInterval := time.Duration(cfg.RepeatInterval) autotuneInterval := time.Duration(cfg.AutotuneInterval) - tl = newTickLoop(st, gatherer, dispatcher, phr, rules, tickInterval, repeatInterval, autotuneInterval, cfg.Digest, routinesFromConfig(cfg.Routines), config.MorningRoutinesFromConfig(cfg.MorningRoutines)) + tl = newTickLoop(st, gatherer, dispatcher, phr, rules, tickInterval, repeatInterval, autotuneInterval, cfg.Digest, routinesFromConfig(cfg.Routines), config.MorningRoutinesFromConfig(cfg.MorningRoutines), cfg.PatternProposals) factWorker = newFactEnrichmentWorker(st, eco, time.Duration(cfg.FactEnrichmentInterval)) + evalWorker = newMemoryEvalWorker(st, phr, cfg) + feedWkr = newFeedWorker(coreFor(), embedderOf(voiceW), cfg) + crawlWkr = newCrawlWorker(newCrawler(cfg), coreFor(), embedderOf(voiceW), cfg) coreAPI = &daemonAPI{ - CoreAPI: ipc.NewStoreAPI(st), + CoreAPI: coreFor(), getTrace: tl.trace, getMorningStatus: func(ctx context.Context) []ipc.MorningRoutineStatus { return tl.morningStatus(ctx, time.Now()) }, + getDayPlan: func(ctx context.Context) ipc.DayPlan { return tl.dayPlan(ctx, time.Now()) }, + getEvents: intakeEventsFn(evBus), } if voiceW != nil && voiceW.handler != nil { api := coreAPI.(*daemonAPI) api.chatFn = voiceW.handler.handleText } + if voiceW != nil && voiceW.mcp != nil { + coreAPI.(*daemonAPI).getMCPServers = voiceW.mcp.status + } } else { - // locked mode: dummy CoreAPI that returns errLocked for everything - coreAPI = &lockedAPI{} + // locked mode: no real store yet, so there's no meaningful CoreAPI to + // serve. srv.Check below is the actual guard — every CoreAPI call is + // refused before it reaches this value. This is just a safe non-nil + // placeholder: if the guard is ever bypassed by a bug, calls land + // here and fail loudly with ipc.ErrNotImplemented instead of a nil + // dereference or, worse, silently succeeding. + coreAPI = ipc.UnimplementedCoreAPI{} } // ----- IPC boundary (core ↔ modules) ----- @@ -376,11 +363,25 @@ func run(args []string) error { passkeySess := webauthn.NewPasskeySession(5 * time.Minute) - // Set Server.Check — in locked mode, block everything except unlock-path methods. + // Set Server.Check — the single authorization guard, run once by + // Server.dispatch before any CoreAPI method is called (see + // internal/ipc/server.go). In locked mode this is the ONLY thing + // standing between an unauthenticated caller and the store: it must + // default-deny, with an explicit allowlist for the two methods the + // unlock flow itself needs (MethodAssertStepUp, MethodUnlock — neither + // of which touches CoreAPI; dispatch handles them directly via + // srv.StepUp/srv.UnlockFn). Forgetting to allowlist a new unlock-path + // method fails safe (denied); forgetting to guard a new CoreAPI method + // is impossible because there is nothing left to forget — every method + // not in the allowlist is refused by construction. if locked { srv.Check = func(ctx context.Context, m ipc.Method, _ json.RawMessage) error { switch m { - case ipc.MethodAssertStepUp, ipc.MethodUnlock: + case ipc.MethodAssertStepUp, ipc.MethodUnlock, ipc.MethodPing: + // Ping is allowed for the same reason the two unlock methods + // are: it never reaches CoreAPI. It answers "she is up and + // locked", which is what mavupdate needs to tell a daemon + // waiting for a passkey apart from one that failed to start. return nil // allowed in locked mode default: return errLocked @@ -391,42 +392,115 @@ func run(args []string) error { } srv.StepUp = func(ctx context.Context) error { return passkeySess.Assert(ctx, auth.Scope{}) } + srv.LockedFn = dl.isLocked - // WrapKeyFn — wraps the env key with a passkey credential public key and - // persists the wrapped blob. Only wired when the daemon has the key in - // memory (env key mode). Called by mavweb after passkey enrollment. - if envKeyBytes != nil { - srv.WrapKeyFn = func(ctx context.Context, publicKey []byte) error { - blob, err := webauthn.WrapKey(envKeyBytes, publicKey) + // wg is declared here rather than next to srv.Serve because the media + // retention loop starts on this path too, and shutdown has to wait for a + // prune in flight: it deletes files. + var wg sync.WaitGroup + + // Mail ingestion (Vikunja #246): the hook stays nil unless an email block is + // configured and there is a llama-server to extract with, in which case + // ipc.MethodIngestMail reports ErrUnknownMethod. + if !locked { + wireMailIntake(srv, st, phr, cfg, evBus) + wireModelSwap(srv, phr, cfg) + // Vision + the media blob store (Vikunja #252). Both stay dark without a + // media block; MethodDescribeImage answers ErrUnknownMethod then. + keeper := wireVision(ctx, &wg, srv, st, embedderOf(voiceW), cfg) + // The meeting recorder (Vikunja #253) shares that blob store and its + // retention loop. Off unless a capture block enables it, in which case + // all four capture methods answer ErrUnknownMethod. + wireCapture(ctx, &wg, srv, keeper, st, voiceW, phr, cfg) + // Voice identification (Vikunja #255). Enrolment plumbing only until a + // speaker-embedding model exists on disk; off entirely without a speaker + // block, so no wire path takes a voiceprint on a default box. + wireSpeaker(srv, st, cfg) + } + + // WrapKeyFn — wraps the at-rest key under the passkey PRF secret and + // persists the wrapped blob. Called by mavweb after every assertion. + // + // It is wired in locked mode too, not only in env-key mode, and that is + // what makes a v1 blob recoverable. A box enrolled before Vikunja #14 + // cold-starts through the legacy public-key retry in mavweb, and the + // StoreEncryptionKey that follows rewrites the blob as v2. Without this + // the only escape from a v1 blob was putting MAVEN_DB_KEY back in the + // environment, which is the thing cold-start unlock exists to avoid. + // + // webauthn.WrapKey refuses anything that is not a 32-byte PRF output, so + // an authenticator without PRF support produces no wrapped file at all + // rather than a file that looks protected and is not. + if envKeyBytes != nil || locked { + srv.WrapKeyFn = func(ctx context.Context, secret []byte, explicit bool) error { + kp := dbKey.Load() + if kp == nil { + return errors.New("wrap encryption key: the daemon is locked and has no key yet (unlock first)") + } + wp := wrappedPath() + // Asserting a passkey is not a request to rewrite the cold-start + // key. Without this an assertion carrying a substituted PRF value + // re-wrapped the real database key under it, and a second + // authenticator silently replaced the first one's blob. + if !explicit { + if _, err := os.Stat(wp); err == nil { + return nil + } else if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("check wrapped key: %w", err) + } + } + wrote, err := wrapKeyToFile(wp, *kp, secret) if err != nil { - return fmt.Errorf("wrap encryption key: %w", err) + return err } - wp := *wrappedKeyPath - if wp == "" { - wp = cfg.DefaultWrappedKeyPath() + if wrote { + log.Printf("mavend: wrapped encryption key under this passkey's PRF output → %s", wp) } - if err := os.WriteFile(wp, blob, 0o600); err != nil { - return fmt.Errorf("write wrapped key: %w", err) - } - log.Printf("mavend: wrapped encryption key with passkey credential (%d bytes)", len(blob)) return nil } } - // UnlockFn — cold-start unlock: unwraps the encryption key from the wrapped - // blob using the passkey credential public key, opens the store, wires all + // UnlockFn — cold-start unlock: unwraps the encryption key from the + // wrapped blob using the passkey PRF secret, opens the store, wires all // daemon components, and replaces the locked API. if locked { - srv.UnlockFn = func(ctx context.Context, publicKey []byte) error { - wp := *wrappedKeyPath + var unlockMu sync.Mutex + srv.UnlockFn = func(ctx context.Context, secret []byte) error { + // One unlock at a time, and never a second one. Without this a + // concurrent pair of Unlock calls would each open a store and + // wire a full daemon, and the loser's goroutines would run + // against a store nobody closes. + unlockMu.Lock() + defer unlockMu.Unlock() + if !dl.isLocked() { + return nil // already unlocked; the caller does not need to know + } + + // Depth, not a boundary. MethodAssertStepUp is AuthRead, so + // anything that can open the same-uid socket can flip the + // session and reach MethodUnlock. What actually stops a local + // attacker is the 32-byte PRF output they do not have, and that + // was true before this check. What this check stops is an + // accidental unlock attempt from an unrelated local caller. + if !passkeySess.IsStepUp() { + return errors.New("unlock: no verified passkey assertion (assert first)") + } + + wp := wrappedPath() blob, err := os.ReadFile(wp) if err != nil { return fmt.Errorf("read wrapped key: %w", err) } - key, err := webauthn.UnwrapKey(blob, publicKey) + key, version, err := webauthn.UnwrapKey(blob, secret) if err != nil { return fmt.Errorf("unwrap key: %w", err) } + if version == webauthn.BlobV1 { + log.Printf("SECURITY: %s was unwrapped from a %s blob. The wrapping key is derived from the credential PUBLIC key, which mavweb also writes to its passkeys.json — anyone holding both files can recover the database key with no authenticator. Use the \"rewrite cold-start key\" button on /auth/webauthn with a PRF-capable authenticator to replace it with a v2 blob.", wp, version) + } + // WrapKeyFn needs the key to be able to rewrite the blob later. + keyCopy := bytes.Clone(key) + dbKey.Store(&keyCopy) // Open the store with the unwrapped key. st, err = store.OpenEncrypted(ctx, cfg.DBPath, cfg.DBTmpfs, key) if err != nil { @@ -472,7 +546,7 @@ func run(args []string) error { eco = wireEcosystem(cfg) - voiceW, err = wireVoice(cfg, ipc.NewStoreAPI(st), phr, st.VectorMemory(), st, eco) + voiceW, err = wireVoice(cfg, coreFor(), phr, st.VectorMemory(), st, eco) if err != nil { return fmt.Errorf("wire voice: %w", err) } @@ -513,20 +587,33 @@ func run(args []string) error { tickInterval := time.Duration(cfg.TickInterval) repeatInterval := time.Duration(cfg.RepeatInterval) autotuneInterval := time.Duration(cfg.AutotuneInterval) - tl = newTickLoop(st, gatherer, dispatcher, phr, rules, tickInterval, repeatInterval, autotuneInterval, cfg.Digest, routinesFromConfig(cfg.Routines), config.MorningRoutinesFromConfig(cfg.MorningRoutines)) + tl = newTickLoop(st, gatherer, dispatcher, phr, rules, tickInterval, repeatInterval, autotuneInterval, cfg.Digest, routinesFromConfig(cfg.Routines), config.MorningRoutinesFromConfig(cfg.MorningRoutines), cfg.PatternProposals) factWorker = newFactEnrichmentWorker(st, eco, time.Duration(cfg.FactEnrichmentInterval)) + evalWorker = newMemoryEvalWorker(st, phr, cfg) + feedWkr = newFeedWorker(coreFor(), embedderOf(voiceW), cfg) + crawlWkr = newCrawlWorker(newCrawler(cfg), coreFor(), embedderOf(voiceW), cfg) - // Swap the CoreAPI from lockedAPI to the real store adapter. + // Swap the CoreAPI from the locked placeholder to the real store adapter. newAPI := &daemonAPI{ - CoreAPI: ipc.NewStoreAPI(st), + CoreAPI: coreFor(), getTrace: tl.trace, getMorningStatus: func(ctx context.Context) []ipc.MorningRoutineStatus { return tl.morningStatus(ctx, time.Now()) }, + getDayPlan: func(ctx context.Context) ipc.DayPlan { return tl.dayPlan(ctx, time.Now()) }, + getEvents: intakeEventsFn(evBus), } if voiceW != nil && voiceW.handler != nil { newAPI.chatFn = voiceW.handler.handleText } srv.SetAPI(newAPI) srv.Check = (&auth.Gate{Enrollment: auth.NewFloorEnrollment(), Session: passkeySess}).Check + wireMailIntake(srv, st, phr, cfg, evBus) + wireModelSwap(srv, phr, cfg) + keeper := wireVision(ctx, &wg, srv, st, embedderOf(voiceW), cfg) + wireCapture(ctx, &wg, srv, keeper, st, voiceW, phr, cfg) + // Voice identification (Vikunja #255). Enrolment plumbing only until a + // speaker-embedding model exists on disk; off entirely without a speaker + // block, so no wire path takes a voiceprint on a default box. + wireSpeaker(srv, st, cfg) // Start voice server. if voiceW != nil { @@ -551,13 +638,43 @@ func run(args []string) error { factWorker.run(ctx) }() - dl.unlock() + // Start background memory evaluation (nil unless configured). + if evalWorker != nil { + go func() { + evalWorker.run(ctx) + }() + } + + // Start feed reading (nil unless configured). + if feedWkr != nil { + go func() { + feedWkr.run(ctx) + }() + } + + // Start the watched-page crawls (nil unless configured). + if crawlWkr != nil { + go func() { + crawlWkr.run(ctx) + }() + } + + // Keep MCP connections alive (nil unless configured). + if voiceW != nil && voiceW.mcp != nil { + go voiceW.mcp.run(ctx) + } + + // Re-enumerate the house for new devices (nil unless configured). + if voiceW != nil && voiceW.home != nil { + go voiceW.home.run(ctx) + } + + dl.unlock(st) log.Printf("mavend: unlocked via passkey assertion") return nil } } - var wg sync.WaitGroup wg.Add(1) go func() { defer wg.Done() @@ -589,6 +706,41 @@ func run(args []string) error { defer wg.Done() factWorker.run(ctx) }() + if evalWorker != nil { + wg.Add(1) + go func() { + defer wg.Done() + evalWorker.run(ctx) + }() + } + if feedWkr != nil { + wg.Add(1) + go func() { + defer wg.Done() + feedWkr.run(ctx) + }() + } + if crawlWkr != nil { + wg.Add(1) + go func() { + defer wg.Done() + crawlWkr.run(ctx) + }() + } + if voiceW != nil && voiceW.mcp != nil { + wg.Add(1) + go func() { + defer wg.Done() + voiceW.mcp.run(ctx) + }() + } + if voiceW != nil && voiceW.home != nil { + wg.Add(1) + go func() { + defer wg.Done() + voiceW.home.run(ctx) + }() + } } <-ctx.Done() diff --git a/cmd/mavend/mcp.go b/cmd/mavend/mcp.go new file mode 100644 index 0000000..c350b80 --- /dev/null +++ b/cmd/mavend/mcp.go @@ -0,0 +1,259 @@ +package main + +import ( + "context" + "fmt" + "log" + "time" + + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/mcp" + "github.com/kami/maven/internal/store" + "github.com/kami/maven/internal/webfetch" +) + +// mcpRefreshInterval — how often the manager is asked to re-dial servers that +// are down. It is a tick, not a retry rate: mcp.Manager holds a per-server +// backoff that starts at DefaultReconnectEvery and doubles to +// MaxReconnectEvery, so a permanently misconfigured stdio server is not +// re-exec'd once a minute forever. +const mcpRefreshInterval = time.Minute + +// mcpWiring — the MCP client, when the `mcp` block configures at least one +// enabled server. nil ⇒ nothing was configured, nothing is connected, and an +// allowlist row that happens to look like an MCP row refuses to run. +// +// It lives on the voice wiring because MCP tools ARE acts: they run through +// tool.Executor, the enabled allowlist and the confirm turn, which only exist +// on the voice/chat path. No voice surface ⇒ nothing that could call a tool. +type mcpWiring struct { + mgr *mcp.Manager + st *store.Store +} + +// wireMCP builds the manager. It does NOT dial: run does that, on its own +// goroutine, which is what makes "Maven starting is not contingent on someone +// else's process" true rather than merely intended. +// +// Dialing here used to be synchronous with a 30s budget, from wireVoice, from +// run. Connect dials serially and each HTTP dial is three requests against +// that server's timeout, so one black-holed endpoint cost 15s of boot and two +// cost the whole budget. On the passkey path wireVoice runs inside the unlock +// handler, so it delayed the answer to an unlock as well. Not failing and not +// blocking are different properties and only the first one held. +func wireMCP(cfg *config.Config, st *store.Store) *mcpWiring { + servers := cfg.MCPServers() + if len(servers) == 0 { + return nil + } + limits := webfetch.Config{} + if cfg.MCP != nil { + limits.AllowHosts = cfg.MCP.AllowHosts + limits.DenyHosts = cfg.MCP.DenyHosts + limits.MaxBytes = cfg.MCP.MaxBytes + limits.Timeout = time.Duration(cfg.MCP.Timeout) + limits.HostInterval = time.Duration(cfg.MCP.HostInterval) + } + mgr, err := mcp.NewManager(mcp.WebfetchDoor(limits), servers) + if err != nil { + // Validation already ran in config.validate, so this is a programming + // error rather than a config one. Still not fatal: MCP off is a working + // Maven. + log.Printf("mcp: not wired: %v", err) + return nil + } + return &mcpWiring{mgr: mgr, st: st} +} + +// connect dials every server and reconciles what came back. Called from run, +// under the daemon's context, so a shutdown during a slow dial is observed. +func (w *mcpWiring) connect(ctx context.Context) { + if w == nil { + return + } + w.mgr.Connect(ctx) + w.propose(ctx) +} + +// propose writes a 'proposed' allowlist row for every discovered tool, and +// reconciles the rows that already exist against what the server offers today. +// It does NOT enable anything: a configured server is a place Maven may look, +// not a capability she has. Kami enables what he wants on /tools, behind +// step-up, which is the same gate a shell tool goes through. +// +// Three things happen per discovered tool. +// +// A name not in the store becomes a proposal, carrying the tool's fingerprint. +// +// A name already in the store is reconciled against that fingerprint. A tool +// whose description, schema or readOnlyHint changed since it was approved drops +// back to 'proposed' and, if it stopped claiming read-only, to destructive=1. +// Insert-or-skip was not enough on its own: the cmd is a late-bound reference +// to a name the far end owns, so the server can redefine list_tasks into +// something that writes without the row changing at all. +// +// A row whose server is connected and no longer offers the tool is withdrawn. +func (w *mcpWiring) propose(ctx context.Context) { + if w == nil { + return + } + now := time.Now() + fresh, changed := 0, 0 + seen := map[string]string{} // local name → "server/tool", for collisions + for _, t := range w.mgr.Tools() { + name := mcp.LocalName(t.Server, t.Name) + remote := t.Server + "/" + t.Name + // Two different tools can flatten to one local name: server "vik" with + // tool "list_tasks" and server "vik_list" with tool "tasks" both give + // "vik_list_tasks". The store keys rows by name, so the second would + // land on the first one's row. Config-controlled and therefore rare, + // but silently reusing a row is the wrong way to lose that race. + if prev, dup := seen[name]; dup { + log.Printf("mcp: %s and %s both map to the allowlist name %q — skipping the second, rename a server", + prev, remote, name) + continue + } + seen[name] = remote + // No readOnlyHint ⇒ assume it mutates ⇒ the confirm turn. Being wrong + // in this direction only costs a question. + destructive := !t.ReadOnly + provenance := fmt.Sprintf("mcp %s/%s", t.Server, t.Name) + if t.Description != "" { + provenance += ": " + t.Description + } + fp := mcp.Fingerprint(t) + ok, err := w.st.ProposeMCPTool(ctx, name, mcp.Scope(t.Server), + mcp.Cmd(t.Server, t.Name), destructive, provenance, fp, now) + if err != nil { + log.Printf("mcp: propose %s: %v", name, err) + continue + } + if ok { + fresh++ + continue + } + // The row already existed. Its provenance is whatever the server said + // the first time; reconciling rewrites it, so what /tools shows is what + // the server says now. + ch, err := w.st.ReconcileMCPTool(ctx, name, fp, destructive, provenance, now) + if err != nil { + log.Printf("mcp: reconcile %s: %v", name, err) + continue + } + if !ch.Changed { + continue + } + changed++ + switch { + case ch.Demoted && ch.Escalated: + log.Printf("mcp: %s changed on the server and no longer claims read-only — disabled and marked destructive, re-approve it on /tools", name) + case ch.Demoted: + log.Printf("mcp: %s changed on the server since it was enabled — disabled, re-approve it on /tools", name) + default: + log.Printf("mcp: %s changed on the server; the proposal now shows the new description", name) + } + } + w.withdrawGone(ctx, seen, now) + if fresh > 0 { + log.Printf("mcp: %d new tool proposal(s) waiting on /tools", fresh) + } + if changed > 0 { + log.Printf("mcp: %d tool(s) changed since approval and need another look", changed) + } +} + +// withdrawGone disarms rows whose tool the server stopped offering. Only +// servers that are CONNECTED are considered: a tool missing because its server +// is down is not a tool that was withdrawn, and disabling a capability every +// time a process restarts would be worse than the problem. +func (w *mcpWiring) withdrawGone(ctx context.Context, seen map[string]string, now time.Time) { + live := map[string]bool{} + for _, name := range w.mgr.Connected() { + live[name] = true + } + if len(live) == 0 { + return + } + rows, err := w.st.ListTools(ctx, "") + if err != nil { + log.Printf("mcp: list tools: %v", err) + return + } + for _, row := range rows { + server, remote, ok := mcp.ParseCmd(row.Cmd) + if !ok || !live[server] { + continue + } + if _, still := seen[row.Name]; still { + continue + } + note := fmt.Sprintf("mcp %s/%s: no longer offered by the server", server, remote) + wasEnabled, err := w.st.WithdrawTool(ctx, row.Name, note, now) + if err != nil { + log.Printf("mcp: withdraw %s: %v", row.Name, err) + continue + } + if wasEnabled { + log.Printf("mcp: %s was enabled but %s no longer offers it — disabled", row.Name, server) + } + } +} + +// run re-dials downed servers and picks up tools that appeared, until ctx is +// canceled. +func (w *mcpWiring) run(ctx context.Context) { + if w == nil { + return + } + // The first dial happens here rather than at wiring time, so boot never + // waits on someone else's process. + w.connect(ctx) + t := time.NewTicker(mcpRefreshInterval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + w.mgr.Refresh(ctx) + w.propose(ctx) + } + } +} + +// status maps the manager's view onto the wire type the web surface reads. +func (w *mcpWiring) status() []ipc.MCPServerStatus { + if w == nil { + return nil + } + in := w.mgr.Status() + out := make([]ipc.MCPServerStatus, 0, len(in)) + for _, s := range in { + out = append(out, ipc.MCPServerStatus{ + Name: s.Name, + Transport: s.Transport, + Target: s.Target, + Connected: s.Connected, + Server: s.Server, + Tools: s.Tools, + Err: s.Err, + }) + } + return out +} + +func (w *mcpWiring) close() { + if w == nil { + return + } + _ = w.mgr.Close() +} + +// caller is the tool.MCPCaller the executor gets, or nil when MCP is off. +func (w *mcpWiring) caller() *mcp.Manager { + if w == nil { + return nil + } + return w.mgr +} diff --git a/cmd/mavend/mcp_test.go b/cmd/mavend/mcp_test.go new file mode 100644 index 0000000..b63bdbd --- /dev/null +++ b/cmd/mavend/mcp_test.go @@ -0,0 +1,96 @@ +package main + +import ( + "context" + "strings" + "testing" + + "github.com/kami/maven/internal/config" +) + +func TestWireMCPOffWhenUnconfigured(t *testing.T) { + st := newTestStore(t) + for name, cfg := range map[string]*config.Config{ + "no block": {}, + "nothing enabled": {MCP: &config.MCPConfig{Servers: []config.MCPServerConfig{ + {Name: "vikunja", URL: "http://192.168.1.104:9100/mcp"}, + }}}, + } { + t.Run(name, func(t *testing.T) { + if w := wireMCP(cfg, st); w != nil { + t.Fatal("MCP must be off unless a server is configured AND enabled") + } + }) + } + // nil wiring must be safe to use everywhere it is reachable. + var w *mcpWiring + w.close() + w.propose(context.Background()) + if w.status() != nil || w.caller() != nil { + t.Fatal("a nil wiring must report nothing") + } +} + +// Wiring must not dial. Boot used to block for the whole per-server timeout +// budget on a black-holed endpoint, and on the passkey path that delay landed +// inside the unlock handler. +func TestWireMCPDoesNotDial(t *testing.T) { + st := newTestStore(t) + w := wireMCP(&config.Config{MCP: &config.MCPConfig{Servers: []config.MCPServerConfig{{ + Name: "dead", Command: "/nonexistent/mcp-server", Enabled: true, + }}}}, st) + if w == nil { + t.Fatal("a configured server should wire") + } + defer w.close() + if s := w.status(); len(s) != 1 || s[0].Err != "" { + t.Fatalf("wireMCP dialled: %+v", s) + } +} + +// An unreachable server must not stop the daemon, must be reported as down, and +// must propose nothing. +func TestWireMCPUnreachableServerIsNotFatal(t *testing.T) { + st := newTestStore(t) + w := wireMCP(&config.Config{MCP: &config.MCPConfig{Servers: []config.MCPServerConfig{{ + Name: "dead", Command: "/nonexistent/mcp-server", Enabled: true, + }}}}, st) + if w == nil { + t.Fatal("a configured server should still wire") + } + defer w.close() + w.connect(context.Background()) + st2 := w.status() + if len(st2) != 1 || st2[0].Connected || st2[0].Err == "" { + t.Fatalf("status = %+v", st2) + } + tools, err := st.ListTools(context.Background(), "") + if err != nil { + t.Fatal(err) + } + if len(tools) != 0 { + t.Fatalf("a server that never answered must propose nothing, got %+v", tools) + } +} + +// A url server whose address is private is refused by webfetch unless that +// server sets allow_private. This is the guard the whole MCP path rides on, so +// it is asserted here too, at the wiring level. +func TestWireMCPPrivateURLRefusedWithoutAllowPrivate(t *testing.T) { + st := newTestStore(t) + w := wireMCP(&config.Config{MCP: &config.MCPConfig{Servers: []config.MCPServerConfig{{ + Name: "lan", URL: "http://127.0.0.1:9100/mcp", Enabled: true, + }}}}, st) + if w == nil { + t.Fatal("should wire") + } + defer w.close() + w.connect(context.Background()) + s := w.status()[0] + if s.Connected { + t.Fatal("a loopback server must not connect without allow_private") + } + if !strings.Contains(s.Err, "private address") { + t.Fatalf("err = %q, want the private-address refusal", s.Err) + } +} diff --git a/cmd/mavend/memoryeval.go b/cmd/mavend/memoryeval.go new file mode 100644 index 0000000..45016bc --- /dev/null +++ b/cmd/mavend/memoryeval.go @@ -0,0 +1,101 @@ +// mavend/memoryeval.go — the driver for background memory evaluation +// (Vikunja #248). The evaluator itself is pure-ish and lives in +// internal/memeval; this is the one impure part: a ticker, the store, and the +// resident model's base URL. +// +// It is its own goroutine and NOT a step on the main tick, deliberately. The +// tick runs every 60s and has a delivery deadline behind it; an evaluation is +// a multi-second LLM round-trip on the same llama-server that answers voice +// turns, and it happens hourly at most. Bolting it onto the tick would make +// every hour's tick the slow one for no benefit. +package main + +import ( + "context" + "log" + "time" + + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/memeval" + "github.com/kami/maven/internal/phraser" + "github.com/kami/maven/internal/store" +) + +// memoryEvalTimeout — the per-request deadline on one evaluation. +// +// It used to be five minutes, on the grounds that nobody waits for the answer. +// Nobody waits for the evaluation, but there is ONE resident model behind one +// llama-server, so a voice turn arriving mid-evaluation waited behind it: five +// minutes of evaluation was five minutes of a mute assistant. +// +// The background client now yields the slot while a turn is in flight, so the +// collision is handled where it belongs and this is a prompt budget again. +// Sixty seconds is long enough for a Thinking model here, and an evaluation cut +// off costs nothing, because it is retried at the next interval. Raise it if +// observations start truncating. +const memoryEvalTimeout = 60 * time.Second + +// memoryEvalWorker — ticker + evaluator. +type memoryEvalWorker struct { + eval *memeval.Evaluator + interval time.Duration +} + +// newMemoryEvalWorker wires the evaluation loop, or returns nil when it should +// not run at all. nil is the normal case and every caller must handle it: +// +// - no memory_eval config block ⇒ off (a capability is off unless configured); +// - no LLM phraser ⇒ nothing to evaluate with. There is no template fallback +// here on purpose: a "memory evaluation" assembled from string templates +// would be a fixed sentence pretending to be an observation. +func newMemoryEvalWorker(st *store.Store, phr phraser.Phraser, cfg *config.Config) *memoryEvalWorker { + if cfg.MemoryEval == nil { + return nil + } + lp, ok := phr.(*phraser.LLMPhraser) + if !ok { + log.Printf("memory eval: configured but no llama-server phraser — evaluation disabled") + return nil + } + interval := time.Duration(cfg.MemoryEval.Interval) + if interval <= 0 { + interval = config.DefaultMemoryEvalInterval + } + // Background: nobody is waiting on an observation, and it must not sit in + // front of a voice turn on the single llama-server slot. + client := llmBackgroundClientFor(lp, memoryEvalTimeout) + ev := memeval.NewEvaluator(st, st, client, memeval.Config{ + MaxItems: cfg.MemoryEval.MaxItems, + MinConfidence: cfg.MemoryEval.MinConfidence, + ContextBlock: contextBlockFn(cfg, time.Now), + }) + log.Printf("memory eval: enabled, every %s", interval) + return &memoryEvalWorker{eval: ev, interval: interval} +} + +// run evaluates every interval until ctx is canceled. +// +// The first evaluation waits a full interval rather than firing at startup, the +// opposite of the tick loop's cold-start behaviour. A tick that fires late is a +// nudge that arrives late; an evaluation that fires late is nothing at all, and +// the alternative is a heavy LLM call competing with startup — including with +// the first voice turn after a restart. +func (w *memoryEvalWorker) run(ctx context.Context) { + ticker := time.NewTicker(w.interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case now := <-ticker.C: + obs, err := w.eval.Evaluate(ctx, now) + if err != nil { + log.Printf("memory eval: %v", err) + continue + } + for _, o := range obs { + log.Printf("memory eval: noted (%.2f, %s): %s", o.Conf, o.Action, o.Text) + } + } + } +} diff --git a/cmd/mavend/modelswap.go b/cmd/mavend/modelswap.go new file mode 100644 index 0000000..6fb490d --- /dev/null +++ b/cmd/mavend/modelswap.go @@ -0,0 +1,149 @@ +package main + +import ( + "context" + "fmt" + "log" + "path/filepath" + "time" + + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/llm" + "github.com/kami/maven/internal/phraser" +) + +// Swapping the resident model while the daemon runs (Vikunja #250). +// +// Off unless configured: with no phraser.swap_models allowlist the two IPC +// methods are never wired, so they answer ErrUnknownMethod. When it is wired the +// swap method is AuthStepUp (internal/auth), which means an authed human surface +// only — there is no act, no intent and no timer that reaches it. The daemon +// never decides to change its own brain. +// +// The allowlist is exact-match against paths a human wrote in mavend.json. The +// request carries a path and llama-server is started with it as `-m`, so +// anything looser would turn "swap the model" into "load any file on my disk". +func wireModelSwap(srv *ipc.Server, phr phraser.Phraser, cfg *config.Config) { + if cfg.Phraser == nil || len(cfg.Phraser.SwapModels) == 0 { + return + } + lp, ok := phr.(*phraser.LLMPhraser) + if !ok { + log.Printf("model swap: phraser.swap_models is set but there is no llama-server phraser — swap disabled") + return + } + allowed := map[string]bool{} + for _, m := range cfg.Phraser.SwapModels { + allowed[filepath.Clean(m)] = true + } + // The configured model is always swappable back to, listed or not: the way + // out of a bad swap must not depend on remembering to allowlist the model + // you are already running. + allowed[filepath.Clean(cfg.Phraser.ModelPath)] = true + + srv.SwapModelFn = func(ctx context.Context, req ipc.SwapModelReq) (ipc.SwapModelResp, error) { + path := filepath.Clean(req.ModelPath) + if !allowed[path] { + log.Printf("model swap: REFUSED %q — not in phraser.swap_models", req.ModelPath) + return ipc.SwapModelResp{}, fmt.Errorf("%w: %q is not in phraser.swap_models", ipc.ErrForbidden, req.ModelPath) + } + res, err := lp.Swap(ctx, phraser.SwapSpec{ + ModelPath: path, + NGpuLayers: req.NGpuLayers, + NCtx: req.NCtx, + }) + resp := ipc.SwapModelResp{ + Model: res.Model, + ModelPath: res.ModelPath, + BaseURL: res.BaseURL, + RolledBack: res.RolledBack, + NoBackend: res.NoBackend, + TookMs: res.Took.Milliseconds(), + } + if err != nil { + // A rolled-back swap is a failure that left a working daemon behind. + // Both halves matter to the caller, so the response is filled in even + // though the error is returned. + log.Printf("model swap: %v", err) + return resp, err + } + return resp, nil + } + + srv.ModelStatusFn = func(ctx context.Context) (ipc.ModelStatusResp, error) { + path, ngl, nctx := lp.LiveModel() + base := lp.BaseURL() + resp := ipc.ModelStatusResp{ + ModelPath: path, + BaseURL: base, + NGpuLayers: ngl, + NCtx: nctx, + Swappable: cfg.Phraser.SwapModels, + } + if base == "" { + resp.Model = llm.UnknownModel + return resp, nil + } + id, err := llm.ModelID(ctx, base) + if err != nil { + // Report the honest "I could not confirm it" rather than echoing the + // configured filename as if the server had said it. + resp.Model = llm.UnknownModel + return resp, nil + } + resp.Model = id + return resp, nil + } + + log.Printf("model swap: enabled, %d allowlisted model(s) — step-up required", len(cfg.Phraser.SwapModels)) +} + +// llmClientFor builds a completion client on the phraser's llama-server and +// keeps it pointed at the right one across a model swap. +// +// Without the OnSwap registration every holder of a base URL — the LLM router, +// the replier, the mail extractor, the memory evaluator — would keep talking to +// the port of a server that no longer exists, and the daemon would degrade to +// the classifier permanently after the first swap. The client is re-pointed, not +// rebuilt, so nothing that holds it has to know a swap happened. +// SetSwapGate is the other half, and on the deploy shape it is the load-bearing +// one: +// llama-server is relaunched on the same fixed port, so SetBaseURL is usually a +// no-op, while the gate is what makes the swap's drain count these callers at +// all. Without it a swap can kill the server mid-routing-decision. +func llmClientFor(lp *phraser.LLMPhraser, timeout time.Duration) *llm.Client { + c := llm.New(lp.BaseURL(), timeout) + c.SetGate(residentGate, false) + c.SetSwapGate(lp) + lp.OnSwap(func(base string) { c.SetBaseURL(base) }) + return c +} + +// backgroundQuiet — how long background work stays off the resident model after +// a foreground request. Long enough to cover the gap between the router call and +// the phraser call of one turn (router p50 is ~2.7s on this box), short enough +// that a quiet mailbox is still read promptly. +const backgroundQuiet = 10 * time.Second + +// residentGate — the priority gate on the one llama-server slot, shared by every +// client llmClientFor builds. Package level because the daemon owns exactly one +// llama-server: two gates would be two opinions about one queue. +// +// The problem it solves: llama-server runs a single slot, so requests queue. Mail +// extraction is allowed two minutes, and a first poll can hand core 25 messages +// back to back. Without a gate a voice turn arriving mid-extraction waits for +// whatever is left of that budget, the router times out into the classifier +// cascade at its 36.8% floor, and the phraser just waits. +var residentGate = llm.NewGate(backgroundQuiet) + +// llmBackgroundClientFor is llmClientFor for work nobody is waiting on: mail +// extraction and memory evaluation. Same swap-following client, but it yields +// to voice turns and only one such request runs at a time. +func llmBackgroundClientFor(lp *phraser.LLMPhraser, timeout time.Duration) *llm.Client { + c := llm.New(lp.BaseURL(), timeout) + c.SetGate(residentGate, true) + c.SetSwapGate(lp) + lp.OnSwap(func(base string) { c.SetBaseURL(base) }) + return c +} diff --git a/cmd/mavend/netscan.go b/cmd/mavend/netscan.go new file mode 100644 index 0000000..83c1c35 --- /dev/null +++ b/cmd/mavend/netscan.go @@ -0,0 +1,278 @@ +package main + +import ( + "context" + "fmt" + "log" + "strings" + "sync" + "time" + + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/netscan" +) + +// scanBudget — the whole spoken scan, end to end. A voice turn that takes +// longer than this has already failed as a turn, so the scan returns whatever +// it found rather than keeping him waiting. +// +// It has to be consistent with the shipped defaults or every scan is truncated: +// a /24 at four ports is 1016 probes, which at netscan.DefaultRate of 100 a +// second is a little over ten seconds plus the tail dials. 30s leaves room for +// that without pretending a slower rate would fit. +const scanBudget = 30 * time.Second + +// scanCacheTTL — how long a scan answer is reused. Two questions in a row used +// to be two full sweeps of the LAN, up to a thousand connections each. The +// network does not change on the scale of a follow-up question, and the cheapest +// packet is the one not sent. +const scanCacheTTL = 2 * time.Minute + +// scanReadOut — how many hosts go into the written record's first lines before +// it says "и ещё N". Nothing reads addresses out loud; see scanSummary. +const scanReadOut = 20 + +// netWiring — the LAN scanner, when the `netscan` block is enabled. nil ⇒ Maven +// never puts a discovery packet on the network. +// +// Unlike the house, a scan is a READ, so it is a query source rather than an +// act: there is no allowlist row and no confirm turn, because nothing changes. +// What makes that safe is that the range is not an argument — see +// internal/netscan's package comment. +type netWiring struct { + scanner *netscan.Scanner + subnets []string + // api — where the address list is WRITTEN. The spoken answer is a count + // and a shape, so the detail has to land somewhere readable; a note under + // source "scan:lan" puts it on /history and, through the intake decorator, + // on /events. It is also the only record that Maven put packets on the LAN + // at all. nil ⇒ nothing is written, which is what the tests use. + api ipc.CoreAPI + now func() time.Time + + mu sync.Mutex + cached netscan.Result + cachedAt time.Time +} + +// wireNetScan builds the scanner. nil unless the block is enabled and valid. +func wireNetScan(cfg *config.Config, api ipc.CoreAPI) *netWiring { + nc, ok := cfg.NetScanner() + if !ok { + return nil + } + if err := netscan.Validate(nc); err != nil { + // config.validate already ran this, so reaching here is a programming + // error rather than a config one. Not fatal: the scanner off is a + // working Maven. + log.Printf("netscan: not wired: %v", err) + return nil + } + return &netWiring{scanner: netscan.New(nc), subnets: nc.Subnets, api: api, now: time.Now} +} + +// scan runs a scan, or reuses one younger than scanCacheTTL. +func (w *netWiring) scan(ctx context.Context) (netscan.Result, error) { + w.mu.Lock() + defer w.mu.Unlock() + now := w.now() + if !w.cachedAt.IsZero() && now.Sub(w.cachedAt) < scanCacheTTL { + return w.cached, nil + } + scanCtx, cancel := context.WithTimeout(ctx, scanBudget) + defer cancel() + res, err := w.scanner.Scan(scanCtx) + if err != nil { + return res, err + } + w.cached, w.cachedAt = res, now + // Written on a fresh scan only: the record is a trace of packets going out, + // so a cached answer must not forge a second one. + w.writeScanRecord(ctx, res) + return res, nil +} + +// scanSummary answers "какие устройства в сети?" in one spoken line. +// +// It does NOT read addresses out. This is the query path, so the reply goes to +// piper as well as to /chat, and "192.168.1.1 (80, 443); 192.168.1.14 (22)" is +// a digit stream nobody can follow through a speaker. She says how many and +// what shape they are; the addresses go into a note (see writeScanRecord). +func (w *netWiring) scanSummary(ctx context.Context) (string, bool) { + if w == nil { + return "", false + } + res, err := w.scan(ctx) + if err != nil { + log.Printf("netscan: scan: %v", err) + return "не получилось просканировать сеть.", true + } + // A truncated run is not a statement about the LAN. Saying "нашла 6 + // устройств" after stopping two thirds of the way through the range is a + // false claim, and the addresses at the end are the ones that go missing. + tail := "" + if res.Truncated { + tail = ", но успела посмотреть не всю сеть" + } + if len(res.Hosts) == 0 { + return "в сети никого не нашла" + tail + ".", true + } + out := fmt.Sprintf("нашла %d %s", len(res.Hosts), hostWord(len(res.Hosts))) + if shape := scanShape(res.Hosts); shape != "" { + out += ", " + shape + } + out += tail + if w.api != nil { + out += ". список записала" + } + return out + ".", true +} + +// scanShape describes the hosts by what they answer on, which is the part of +// the answer that carries meaning out loud: "два с вебом" says more about the +// flat than four octets do. +func scanShape(hosts []netscan.Host) string { + var web, ssh, quiet int + for _, h := range hosts { + hasWeb, hasSSH := false, false + for _, p := range h.Ports { + switch p { + case 80, 443, 8080: + hasWeb = true + case 22: + hasSSH = true + } + } + if hasWeb { + web++ + } + if hasSSH { + ssh++ + } + // No open port at all: seen only through the ARP cache. + if len(h.Ports) == 0 { + quiet++ + } + } + var parts []string + if web > 0 { + parts = append(parts, fmt.Sprintf("%d с вебом", web)) + } + if ssh > 0 { + parts = append(parts, fmt.Sprintf("%d с ssh", ssh)) + } + if quiet > 0 { + parts = append(parts, fmt.Sprintf("%d молча", quiet)) + } + if len(parts) == 0 { + return "" + } + return "из них " + strings.Join(parts, ", ") +} + +// writeScanRecord stores the address list as a note. This is both where the +// detail becomes readable and the only trace that a scan happened at all: a +// scan is a read, but "when did she last put packets on the LAN" deserves an +// answer. +func (w *netWiring) writeScanRecord(ctx context.Context, res netscan.Result) { + if w.api == nil { + return + } + head := fmt.Sprintf("сканирование сети: %d %s", len(res.Hosts), hostWord(len(res.Hosts))) + if res.Truncated { + head += " (не вся сеть)" + } + lines := []string{head, "подсети: " + strings.Join(w.subnets, ", ")} + shown := res.Hosts + if len(shown) > scanReadOut { + shown = shown[:scanReadOut] + } + for _, h := range shown { + s := h.Addr + if len(h.Ports) > 0 { + ps := make([]string, 0, len(h.Ports)) + for _, p := range h.Ports { + ps = append(ps, fmt.Sprintf("%d", p)) + } + s += " (" + strings.Join(ps, ", ") + ")" + } + if h.MAC != "" { + s += " " + h.MAC + } + lines = append(lines, s) + } + if len(res.Hosts) > len(shown) { + lines = append(lines, fmt.Sprintf("и ещё %d", len(res.Hosts)-len(shown))) + } + if _, err := w.api.WriteNote(ctx, w.now(), strings.Join(lines, "\n"), nil, "scan:lan"); err != nil { + log.Printf("netscan: write scan note: %v", err) + } +} + +// hostWord — Russian counts inflect the noun: 1 устройство, 2-4 устройства, +// 5+ устройств, and the teens are all the last form. +func hostWord(n int) string { + if n%100 >= 11 && n%100 <= 14 { + return "устройств" + } + switch n % 10 { + case 1: + return "устройство" + case 2, 3, 4: + return "устройства" + default: + return "устройств" + } +} + +// isNetworkQuery recognises a question about the LAN, narrowly. It needs a +// network word AND an ask: "интернет не работает" is a complaint, not a request +// to scan, and a scan she runs unasked is exactly the noisy behaviour the +// bounds exist to prevent. +func isNetworkQuery(u string) bool { + s := strings.ToLower(strings.TrimSpace(u)) + if s == "" { + return false + } + // Whole tokens for the network nouns: the bare substring "сети" is inside + // "посетил", so "сколько машин я посетил?" used to read as a request to + // scan the LAN. The prefix forms below are stems that have no such + // collisions. + network := false + for _, w := range []string{"сеть", "сети", "сетке", "сетку"} { + if homeWord(s, w) { + network = true + break + } + } + if !network { + for _, w := range []string{"локальн", "wifi", "wi-fi", "вайфай"} { + if strings.Contains(s, w) { + network = true + break + } + } + } + if !network { + return false + } + // An explicit ask to scan, or a phrase that can only be about the LAN. + // "кто в сети" carries no device noun but means nothing else. + for _, w := range []string{"просканируй", "сканируй", "скан", "просканир", "кто в сети", "кто в сетке"} { + if strings.Contains(s, w) { + return true + } + } + ask := strings.Contains(s, "?") || homeWord(s, "какие") || homeWord(s, "кто") || + homeWord(s, "что") || homeWord(s, "сколько") || strings.Contains(s, "покажи") + if !ask { + return false + } + for _, w := range []string{"устройств", "хост", "компьютер", "машин", "адрес"} { + if strings.Contains(s, w) { + return true + } + } + return false +} diff --git a/cmd/mavend/netscan_test.go b/cmd/mavend/netscan_test.go new file mode 100644 index 0000000..df87124 --- /dev/null +++ b/cmd/mavend/netscan_test.go @@ -0,0 +1,171 @@ +package main + +import ( + "context" + "net" + "strconv" + "strings" + "testing" + "time" + + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/ipc" +) + +func TestWireNetScanOffUnlessEnabled(t *testing.T) { + for name, cfg := range map[string]*config.Config{ + "no block": {}, + "written but dark": {NetScan: &config.NetScanConfig{ + Subnets: []string{"192.168.1.0/24"}, + }}, + "enabled but nothing to scan": {NetScan: &config.NetScanConfig{Enabled: true}}, + "enabled but public": {NetScan: &config.NetScanConfig{ + Subnets: []string{"8.8.8.0/24"}, Enabled: true, + }}, + "enabled but far too wide": {NetScan: &config.NetScanConfig{ + Subnets: []string{"10.0.0.0/8"}, Enabled: true, + }}, + } { + t.Run(name, func(t *testing.T) { + if w := wireNetScan(cfg, nil); w != nil { + t.Fatal("the scanner must not wire for this config") + } + }) + } + + var w *netWiring + if _, ok := w.scanSummary(context.Background()); ok { + t.Fatal("a nil wiring must not claim a query") + } + + ok := wireNetScan(&config.Config{NetScan: &config.NetScanConfig{ + Subnets: []string{"192.168.1.0/24"}, Enabled: true, + }}, nil) + if ok == nil { + t.Fatal("a valid enabled block should wire") + } +} + +// A loopback /32 with nothing listening on the scanned port: the summary must +// come back honest rather than inventing a host. This also exercises the real +// dialer end to end without touching anything outside this box. +func TestScanSummaryOnAnEmptyRange(t *testing.T) { + w := wireNetScan(&config.Config{NetScan: &config.NetScanConfig{ + // Port 1 on loopback: nothing listens and the connection is refused + // immediately, so the scan is fast and touches only this machine. + Subnets: []string{"127.0.0.1/32"}, Ports: []int{1}, Rate: 1000, Enabled: true, + }}, nil) + if w == nil { + t.Fatal("wireNetScan returned nil") + } + out, claimed := w.scanSummary(context.Background()) + if !claimed { + t.Fatal("the summary did not claim the turn") + } + if out == "" { + t.Fatal("empty summary") + } + // Persona: feminine self-reference, informal address, no pet names. + low := strings.ToLower(out) + for _, bad := range []string{"нашёл", "не смог ", "вы ", "ваш", "милый", "дорогой"} { + if strings.Contains(low, bad) { + t.Errorf("persona violation %q in %q", bad, out) + } + } +} + +func TestHostWordAgreesWithTheCount(t *testing.T) { + for n, want := range map[int]string{ + 1: "устройство", 2: "устройства", 4: "устройства", 5: "устройств", + 11: "устройств", 12: "устройств", 21: "устройство", 22: "устройства", + 25: "устройств", 111: "устройств", 101: "устройство", 0: "устройств", + } { + if got := hostWord(n); got != want { + t.Errorf("hostWord(%d) = %q, want %q", n, got, want) + } + } +} + +func TestIsNetworkQuery(t *testing.T) { + yes := []string{ + "какие устройства в сети?", + "кто в сети?", + "просканируй сеть", + "покажи устройства в локальной сети", + "сколько машин в сети", + } + no := []string{ + "", + "интернет не работает", + "сеть какая-то медленная", + "я в сети инстаграма", + "что включено дома?", + "напомни оплатить интернет", + } + for _, u := range yes { + if !isNetworkQuery(u) { + t.Errorf("isNetworkQuery(%q) = false, want true", u) + } + } + for _, u := range no { + if isNetworkQuery(u) { + t.Errorf("isNetworkQuery(%q) = true, want false", u) + } + } +} + +// notingAPI counts the notes a scan writes, and remembers the last one. +type notingAPI struct { + ipc.CoreAPI + n int + last string +} + +func (a *notingAPI) WriteNote(_ context.Context, _ time.Time, text string, _ []float32, _ string) (int64, error) { + a.n++ + a.last = text + return int64(a.n), nil +} + +// The spoken answer must not be a list of IP addresses. It goes to piper as +// well as to /chat, and six dotted quads read out as a digit stream is not an +// answer anybody can use. The addresses belong in the written record. +func TestScanSummarySpeaksACountAndWritesTheAddresses(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + _, portStr, _ := net.SplitHostPort(ln.Addr().String()) + port, _ := strconv.Atoi(portStr) + + api := ¬ingAPI{} + w := wireNetScan(&config.Config{NetScan: &config.NetScanConfig{ + Subnets: []string{"127.0.0.1/32"}, Ports: []int{port}, Rate: 1000, Enabled: true, + }}, api) + if w == nil { + t.Fatal("wireNetScan returned nil") + } + out, claimed := w.scanSummary(context.Background()) + if !claimed { + t.Fatal("the summary did not claim the turn") + } + if strings.Contains(out, "127.0.0.1") || strings.Contains(out, portStr) { + t.Errorf("the spoken reply reads addresses out loud: %q", out) + } + if !strings.Contains(out, "нашла 1 устройство") { + t.Errorf("reply = %q, want a count", out) + } + if api.n != 1 { + t.Fatalf("wrote %d notes, want 1", api.n) + } + if !strings.Contains(api.last, "127.0.0.1") { + t.Errorf("the written record has no addresses: %q", api.last) + } + + // A follow-up question inside the TTL reuses the answer: two questions in + // a row must not be two sweeps of the LAN. + if _, _ = w.scanSummary(context.Background()); api.n != 1 { + t.Errorf("a repeat question rescanned and rewrote the record (%d notes)", api.n) + } +} diff --git a/cmd/mavend/patterns.go b/cmd/mavend/patterns.go new file mode 100644 index 0000000..7fd5381 --- /dev/null +++ b/cmd/mavend/patterns.go @@ -0,0 +1,120 @@ +// mavend/patterns.go — the shared detect+propose step of pattern inference +// (Vikunja #43). Event *extraction* (fact -> action/object) happens at fact- +// write time in detectPattern below, tied to whichever channel wrote the +// fact. Detection — turning a run of events into a proposed routine — is +// channel-agnostic: it only needs what's already in the events table, so it +// runs both right after a voice fact-write (for the immediate "напоминать?" +// confirmation) and, proactively, from the digestion tick (tick.go's +// detectPatterns) over every action+object pair on record, not just the one +// that was just talked about. +package main + +import ( + "context" + "errors" + "fmt" + "log" + "time" + + "github.com/kami/maven/internal/pattern" + "github.com/kami/maven/internal/store" +) + +// detectAndPropose runs the pattern detector over every recorded event for +// action+object and, if a stable pattern is found and nothing has been +// proposed/accepted/dismissed for this pair yet, creates a proposed_routines +// row. Returns (nil, 0, nil) — not an error — whenever there is nothing new +// to report: too few events, irregular intervals, or a pair that already has +// a row in any status. That last case is the one that matters most: it is +// how a routine the owner already DISMISSED stays dismissed forever, because +// the row survives dismissal (status flips in place, see +// store.DismissProposedRoutine) and both the Lookup check here and the +// table's UNIQUE(action, object) constraint refuse to create a second one. +func detectAndPropose(ctx context.Context, ds *store.Store, action, object string, ts time.Time) (*pattern.ProposedRoutine, int64, error) { + events, err := ds.EventsFor(ctx, action, object) + if err != nil { + return nil, 0, fmt.Errorf("events for %s/%s: %w", action, object, err) + } + patEvents := make([]pattern.Event, len(events)) + for i, e := range events { + patEvents[i] = pattern.Event{ + FactID: e.FactID, + Action: e.Action, + Object: e.Object, + Ts: e.Ts, + } + } + r, err := pattern.Detect(patEvents) + if err != nil { + return nil, 0, fmt.Errorf("detect %s/%s: %w", action, object, err) + } + if r == nil { + return nil, 0, nil // not enough data or intervals too irregular + } + + // Belt: check first so the common "nothing new" case never even attempts + // an insert. Suspenders: CreateProposedRoutine's ON CONFLICT DO NOTHING + // (backed by the UNIQUE(action,object) constraint) is the actual + // guarantee — this Lookup is an optimization, not the source of truth. + existing, err := ds.LookupProposedRoutine(ctx, r.Action, r.Object) + if err != nil { + return nil, 0, fmt.Errorf("lookup proposed routine %s/%s: %w", action, object, err) + } + if existing != nil { + return nil, 0, nil // already proposed, accepted, or dismissed — say nothing + } + + id, err := ds.CreateProposedRoutine(ctx, r.Action, r.Object, r.IntervalDays, ts) + if err != nil { + if errors.Is(err, store.ErrProposedRoutineExists) { + return nil, 0, nil // lost a race with another caller — not an error + } + return nil, 0, fmt.Errorf("create proposed routine %s/%s: %w", action, object, err) + } + return r, id, nil +} + +// detectPattern extracts an event from the written fact and runs the pattern +// detector. If a stable recurring pattern is found and no proposed routine +// exists for this action+object yet, one is created and the user is prompted +// to confirm via the park() mechanism. Returns the suggestion phrase when a +// new proposal was created and parked; "" otherwise. +func (h *reactiveHandler) detectPattern(ctx context.Context, factID int64, key, value string, ts time.Time) string { + ev := pattern.Extract(factID, key, value, ts) + if ev == nil { + return "" // not an actionable event + } + if _, err := h.dataStore.CreateEvent(ctx, factID, ev.Action, ev.Object, ts); err != nil { + log.Printf("voice: create event: %v", err) + return "" + } + // Detect+propose (Vikunja #43) is shared with the digestion tick's + // proactive scan — see detectAndPropose above. Event *extraction* stays + // here, tied to this fact write; detection over the accumulated history does + // not need to happen right now for the voice path to have already done + // its job — it's dedupe-safe to also let the next tick find the same + // pattern independently. + r, id, err := detectAndPropose(ctx, h.dataStore, ev.Action, ev.Object, ts) + if err != nil { + log.Printf("voice: detect pattern %s/%s: %v", ev.Action, ev.Object, err) + return "" + } + if r == nil { + return "" // not enough data, too irregular, or already proposed/decided + } + log.Printf("voice: proposed routine: %s/%s every %.1f days", r.Action, r.Object, r.IntervalDays) + + // Park the proposal for voice confirmation. + phrase := pattern.PhraseRoutine(r) + h.mu.Lock() + h.pendingRoutine = &pendingRoutineConfirm{ + routineID: id, + action: r.Action, + object: r.Object, + interval: r.IntervalDays, + phrase: phrase, + expiry: ts.Add(confirmTTL), + } + h.mu.Unlock() + return phrase +} diff --git a/cmd/mavend/patterns_test.go b/cmd/mavend/patterns_test.go new file mode 100644 index 0000000..1b14524 --- /dev/null +++ b/cmd/mavend/patterns_test.go @@ -0,0 +1,285 @@ +package main + +import ( + "context" + "database/sql" + "strings" + "testing" + "time" + + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/delivery" + "github.com/kami/maven/internal/loop" + "github.com/kami/maven/internal/pattern" + "github.com/kami/maven/internal/store" +) + +// seedRefillEvents writes N weekly "refill/cat_water" events straight to the +// events table — this is what the tick reads, independent of any utterance. +func seedRefillEvents(t *testing.T, st *store.Store, ctx context.Context, base time.Time, n int) { + t.Helper() + for i := 0; i < n; i++ { + factID, err := st.WriteFact(ctx, base.Add(time.Duration(i)*7*24*time.Hour), store.KindSelf, + "cat_water", "refill", "test", 1.0, sql.NullInt64{}) + if err != nil { + t.Fatalf("write fact %d: %v", i, err) + } + if _, err := st.CreateEvent(ctx, factID, "refill", "cat_water", base.Add(time.Duration(i)*7*24*time.Hour)); err != nil { + t.Fatalf("create event %d: %v", i, err) + } + } +} + +// TestTickDetectsPatternFromStoredEvents proves the tick notices a pattern on +// its own, reading straight from the store — not as a side effect of a live +// utterance (Vikunja #43). MinEvents weekly events with no voice turn in +// sight must produce exactly one proposed routine. +func TestTickDetectsPatternFromStoredEvents(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := refNow() + seedRefillEvents(t, st, ctx, now, pattern.MinEvents) + + tl := newTestTickLoop(t, st, &fakeSink{}, nil) + tl.detectPatterns(ctx, now, loop.State{}) + + rows, err := st.ListProposedRoutines(ctx) + if err != nil { + t.Fatalf("list proposed routines: %v", err) + } + if len(rows) != 1 { + t.Fatalf("proposed routines = %d, want 1: %+v", len(rows), rows) + } + if rows[0].Action != "refill" || rows[0].Object != "cat_water" { + t.Errorf("proposed routine = %s/%s, want refill/cat_water", rows[0].Action, rows[0].Object) + } +} + +// TestTickPatternDetectionIsIdempotent proves running the tick's pattern scan +// twice does not spam a second proposal for the same pair, and that the store +// itself is what stops the duplicate (not tick-local state) — the whole point +// of the guard, since the tick has no memory of what it proposed last time. +func TestTickPatternDetectionIsIdempotent(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := refNow() + seedRefillEvents(t, st, ctx, now, pattern.MinEvents) + + tl := newTestTickLoop(t, st, &fakeSink{}, nil) + tl.detectPatterns(ctx, now, loop.State{}) + tl.detectPatterns(ctx, now.Add(time.Hour), loop.State{}) + + rows, err := st.ListProposedRoutines(ctx) + if err != nil { + t.Fatalf("list proposed routines: %v", err) + } + if len(rows) != 1 { + t.Fatalf("proposed routines after two ticks = %d, want 1 (no duplicate): %+v", len(rows), rows) + } +} + +// TestTickPatternDetectionRespectsDismissal proves the single worst failure +// mode here — a proposal the owner already said no to coming back on the next +// tick — cannot happen. Dismissal flips the row's status in place; it must +// still be there to block re-proposal. +func TestTickPatternDetectionRespectsDismissal(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := refNow() + seedRefillEvents(t, st, ctx, now, pattern.MinEvents) + + tl := newTestTickLoop(t, st, &fakeSink{}, nil) + tl.detectPatterns(ctx, now, loop.State{}) + + rows, err := st.ListProposedRoutines(ctx) + if err != nil { + t.Fatalf("list proposed routines: %v", err) + } + if len(rows) != 1 { + t.Fatalf("setup: proposed routines = %d, want 1", len(rows)) + } + if err := st.DismissProposedRoutine(ctx, rows[0].ID); err != nil { + t.Fatalf("dismiss: %v", err) + } + + // More events for the same pair arrive, and the tick runs again — a + // dismissed pattern must not resurface. + seedRefillEvents(t, st, ctx, now.Add(30*24*time.Hour), pattern.MinEvents) + tl.detectPatterns(ctx, now.Add(60*24*time.Hour), loop.State{}) + + proposed, err := st.ListProposedRoutinesByStatus(ctx, store.RoutineProposed) + if err != nil { + t.Fatalf("list proposed: %v", err) + } + if len(proposed) != 0 { + t.Fatalf("a dismissed pattern came back: %+v", proposed) + } + all, err := st.ListProposedRoutinesByStatus(ctx, "") + if err != nil { + t.Fatalf("list all: %v", err) + } + if len(all) != 1 { + t.Fatalf("total rows for the pair = %d, want 1 (still dismissed, not duplicated): %+v", len(all), all) + } + if all[0].Status != store.RoutineDismissed { + t.Errorf("status = %s, want dismissed", all[0].Status) + } +} + +// proposalRule — the rule name announceProposal uses for the seeded pair. +const proposalRule = "proposal:refill cat_water" + +// TestTickProposalSilentByDefault — detection is always on, announcing is not. +// With no pattern_proposals block the tick still records the proposal, and says +// nothing about it: Maven is not autonomous, so a behaviour that speaks without +// being asked stays off until it is configured. +func TestTickProposalSilentByDefault(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := refNow() + seedRefillEvents(t, st, ctx, now, pattern.MinEvents) + markPresent(t, st, ctx, now) + + sink := &fakeSink{} + tl := newTestTickLoop(t, st, sink, nil) + tl.tick(ctx, now) + + if n := countSends(sink, proposalRule); n != 0 { + t.Fatalf("announced %d proposals with no config, want 0", n) + } + rows, err := st.ListProposedRoutinesByStatus(ctx, store.RoutineProposed) + if err != nil { + t.Fatalf("list proposed: %v", err) + } + if len(rows) != 1 { + t.Fatalf("proposed routines = %d, want 1 (silent, but recorded)", len(rows)) + } +} + +// TestTickAnnouncesProposalWhenConfigured — with notify on, the proposal goes +// out once through the ordinary delivery path, worded by the detector itself. +// Later ticks stay quiet because the pair is already proposed: one pattern is +// one announcement, ever. +func TestTickAnnouncesProposalWhenConfigured(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := refNow() + seedRefillEvents(t, st, ctx, now, pattern.MinEvents) + markPresent(t, st, ctx, now) + + sink := &fakeSink{} + tl := newTestTickLoop(t, st, sink, nil) + tl.proposalCfg = &config.PatternProposalConfig{Notify: true} + tl.tick(ctx, now) + + var got *delivery.Sendable + for i := range sink.sends { + if sink.sends[i].RuleName == proposalRule { + got = &sink.sends[i] + } + } + if got == nil { + t.Fatalf("proposal was not announced; sends=%+v", sink.sends) + } + if !strings.Contains(got.Body, "напоминать?") { + t.Errorf("body = %q, want the detector's own question", got.Body) + } + if got.Channel != delivery.ChannelVoice { + t.Errorf("channel = %v, want voice (sev1, present)", got.Channel) + } + + // A month of further ticks: the pair already has a row, so there is + // nothing new to detect and nothing more to say. + sink.sends = nil + later := now.Add(40 * 24 * time.Hour) + markPresent(t, st, ctx, later) + tl.tick(ctx, later) + if n := countSends(sink, proposalRule); n != 0 { + t.Fatalf("re-announced an existing proposal %d times, want 0", n) + } +} + +// TestTickProposalRespectsGate — a proposal is the least urgent thing Maven can +// say, so it is sev1 and the restraint gate suppresses it. Away presence means +// it is not announced at all: it is not held, not retried, it just lives on +// /routines. The proposal row is still written — noticing is never gated. +func TestTickProposalRespectsGate(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := refNow() + seedRefillEvents(t, st, ctx, now, pattern.MinEvents) + // no presence probes ⇒ away ⇒ care-class gate blocks. + + sink := &fakeSink{} + tl := newTestTickLoop(t, st, sink, nil) + tl.proposalCfg = &config.PatternProposalConfig{Notify: true} + tl.tick(ctx, now) + + if n := countSends(sink, proposalRule); n != 0 { + t.Fatalf("away: announced %d proposals, want 0", n) + } + if !tl.lastProposalAt.IsZero() { + t.Error("cooldown clock advanced on a suppressed announcement") + } + rows, err := st.ListProposedRoutinesByStatus(ctx, store.RoutineProposed) + if err != nil { + t.Fatalf("list proposed: %v", err) + } + if len(rows) != 1 { + t.Fatalf("proposed routines = %d, want 1 (detection is never gated)", len(rows)) + } +} + +// TestTickProposalCooldownSpacesAnnouncements — two patterns detected on the +// same tick must not become two interruptions. The second one waits for the +// cooldown, and is on /routines meanwhile. +func TestTickProposalCooldownSpacesAnnouncements(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := refNow() + seedRefillEvents(t, st, ctx, now, pattern.MinEvents) + for i := 0; i < pattern.MinEvents; i++ { + ts := now.Add(time.Duration(i) * 3 * 24 * time.Hour) + factID, err := st.WriteFact(ctx, ts, store.KindSelf, "litter_box", "clean", "test", 1.0, sql.NullInt64{}) + if err != nil { + t.Fatalf("write fact: %v", err) + } + if _, err := st.CreateEvent(ctx, factID, "clean", "litter_box", ts); err != nil { + t.Fatalf("create event: %v", err) + } + } + markPresent(t, st, ctx, now) + + sink := &fakeSink{} + tl := newTestTickLoop(t, st, sink, nil) + tl.proposalCfg = &config.PatternProposalConfig{Notify: true, Cooldown: config.Duration(24 * time.Hour)} + tl.tick(ctx, now) + + announced := 0 + for _, s := range sink.sends { + if strings.HasPrefix(s.RuleName, "proposal:") { + announced++ + } + } + if announced != 1 { + t.Fatalf("announced %d proposals on one tick, want exactly 1", announced) + } + rows, err := st.ListProposedRoutinesByStatus(ctx, store.RoutineProposed) + if err != nil { + t.Fatalf("list proposed: %v", err) + } + if len(rows) != 2 { + t.Fatalf("proposed routines = %d, want 2 (both recorded, one announced)", len(rows)) + } + + // Still inside the cooldown: silence, even though a proposal is pending. + sink.sends = nil + soon := now.Add(time.Hour) + markPresent(t, st, ctx, soon) + tl.tick(ctx, soon) + for _, s := range sink.sends { + if strings.HasPrefix(s.RuleName, "proposal:") { + t.Fatalf("announced %q inside the cooldown", s.RuleName) + } + } +} diff --git a/cmd/mavend/quiet_toggle.go b/cmd/mavend/quiet_toggle.go new file mode 100644 index 0000000..13979cf --- /dev/null +++ b/cmd/mavend/quiet_toggle.go @@ -0,0 +1,206 @@ +// Quiet-mode toggle recognition — the pre-route keyword check that lets +// "тихий режим" flip the daemon-wide quiet_hours config without going through +// the router. Moved out of voice.go unchanged (Vikunja #321); the tests live in +// quiet_toggle_test.go. +package main + +import ( + "context" + "log" + "strings" + "unicode" + + "github.com/kami/maven/internal/ipc" +) + +// resolveQuietToggle — pre-route keyword check. Returns (reply, true) when +// the utterance is a quiet-on/off command; ("", false) otherwise. Called from +// runTurn BEFORE the router so a classifier miscue can't drop it — which means +// both the voice path and the text path (mavweb /api/chat, telegram) reach it, +// so a false positive here is a network-reachable way to flip a daemon-wide +// setting. See classifyQuietToggle for the matching rule. +// +// src is the channel the utterance arrived on, and it is written straight into +// the fact. Every toggle used to be stored as "tap:voice", including the ones +// typed into the web UI, which left the facts table claiming a microphone flipped +// a setting nobody spoke to. This is the one function where that matters most: +// when he goes looking at why quiet mode is on, provenance is the first column +// he reads. +func (h *reactiveHandler) resolveQuietToggle(ctx context.Context, text string, src turnSource) (string, bool) { + on, off := classifyQuietToggle(text) + if !on && !off { + return "", false + } + val := "false" + reply := "тихий режим выключен." + if on { + val = "true" + reply = "тихий режим включён. буду реже напоминать." + } + if _, err := h.api.WriteFact(ctx, ipc.WriteFactReq{ + Ts: h.now(), + Kind: "config", + Key: "quiet_hours", + Value: val, + Source: string(src), + Confidence: 1.0, + }); err != nil { + log.Printf("voice: write quiet_hours: %v", err) + return "не получилось переключить тихий режим.", true + } + return reply, true +} + +// quietInflections — the inflectional endings a stem may carry and still be +// the same word. Adjective/adverb/noun/verb endings, all ≤3 letters. This is +// what separates "тихий"/"тихом"/"тихо" (stem "тих" + a real ending) from +// "тихонько"/"потихоньку", which are different words: "онько" is not an +// ending, and "потихоньку" doesn't start with the stem at all. +var quietInflections = []string{ + "", "а", "е", "и", "й", "о", "у", "ы", "ю", "я", + "ая", "ее", "ей", "ем", "ие", "ий", "им", "их", "ия", "ию", "ое", "ой", "ом", "ую", "ые", "ый", "ым", "ых", "ья", + "ами", "ого", "ому", "ыми", "ать", "ить", "ять", +} + +// quietStem reports whether tok is the given stem carrying at most one +// inflectional ending. Word boundaries come from tokenisation (see +// quietTokens), not from a regexp — Go's \b is ASCII-oriented and treats every +// Cyrillic letter as a non-word character, so `\bтих\b` would happily match +// inside "тихонько". Comparing whole tokens sidesteps that entirely. +func quietStem(tok, stem string) bool { + if !strings.HasPrefix(tok, stem) { + return false + } + suffix := tok[len(stem):] + for _, e := range quietInflections { + if suffix == e { + return true + } + } + return false +} + +// quietTokens splits an utterance into lowercase word tokens, dropping +// punctuation and spacing. Unicode-aware, so Cyrillic words tokenise the same +// way ASCII ones do. +func quietTokens(text string) []string { + return strings.FieldsFunc(strings.ToLower(strings.TrimSpace(text)), func(r rune) bool { + return !unicode.IsLetter(r) && !unicode.IsDigit(r) + }) +} + +// quietPhrase matches a pattern (a sequence of stems) against the token list. +// Multi-word patterns match any contiguous run of tokens — "включи тихий +// режим" carries "тихий режим". Single-word patterns match ONLY when they are +// the whole utterance: bare "тихо" is a command, but "в комнате тихо" is a +// remark about the room and must not flip a daemon-wide setting. +func quietPhrase(tokens, pattern []string) bool { + if len(pattern) == 0 || len(tokens) < len(pattern) { + return false + } + if len(pattern) == 1 { + return len(tokens) == 1 && quietStem(tokens[0], pattern[0]) + } + for i := 0; i+len(pattern) <= len(tokens); i++ { + hit := true + for j, stem := range pattern { + if !quietStem(tokens[i+j], stem) { + hit = false + break + } + } + if hit { + return true + } + } + return false +} + +// quietOffPhrases / quietOnPhrases — the toggle vocabulary, as stem sequences. +// +// Note what is NOT here any more: the OFF list used to carry {"не", "тих"} and +// the ON list {"не", "шум"} / {"не", "беспоко"}. Both were adjacency patterns, +// and negation is not an adjacency phenomenon. "не надо тихий режим" put two +// tokens between "не" and "тих", so the OFF pattern missed, the ON pattern +// {"тих","режим"} matched, and asking for quiet mode to stop turned it on. +// Negation is handled by quietNegators below, over the whole utterance. +var ( + quietOffPhrases = [][]string{ + {"quiet", "off"}, {"quiet", "end"}, + {"громк", "режим"}, {"шумн", "режим"}, + {"отмен", "тих"}, {"выключ", "тих"}, + } + quietOnPhrases = [][]string{ + {"quiet", "on"}, {"quiet", "mode"}, + {"тих", "режим"}, {"не", "шум"}, {"не", "беспоко"}, + {"тих"}, + } +) + +// quietNegatorWords — negators that are whole words with no useful stem. +var quietNegatorWords = map[string]bool{ + "не": true, "нет": true, "хватит": true, "no": true, "not": true, "off": true, +} + +// quietNegatorStems — negators that inflect. Matched through quietStem, the +// same one-ending rule the toggle vocabulary uses, so "выключи", "выключить" +// and "выключай" all count and "выключатель" does not. +var quietNegatorStems = []string{"выключ", "отмен", "прекрат", "убер", "stop", "cancel", "disable"} + +// quietNegated reports whether the utterance carries a negator. Two ON phrases +// are themselves built on "не" — "не шуми", "не беспокой" — and those are +// requests FOR quiet, so they are excluded before the scan: a negator only +// counts when it is not part of the phrase that matched. +func quietNegated(tokens []string, matched []string) bool { + if len(matched) > 0 && matched[0] == "не" { + return false + } + for _, t := range tokens { + if quietNegatorWords[t] { + return true + } + for _, stem := range quietNegatorStems { + if quietStem(t, stem) { + return true + } + } + } + return false +} + +// classifyQuietToggle reads an utterance as a quiet-mode command. +// +// Explicit OFF phrases resolve first, for the same reason classifyConfirm +// checks negatives first: they are built out of the ON words ("выключи тихий" +// contains "тихий"), so scanning ON first would shadow them. An ON phrase that +// matches is then checked for negation across the whole utterance, so any way +// of saying "not quiet mode" turns it off rather than on. +func classifyQuietToggle(text string) (on, off bool) { + tokens := quietTokens(text) + for _, p := range quietOffPhrases { + if quietPhrase(tokens, p) { + return false, true + } + } + for _, p := range quietOnPhrases { + if quietPhrase(tokens, p) { + if quietNegated(tokens, p) { + return false, true + } + return true, false + } + } + // No ON phrase matched, but he negated a quiet word: "не тихо", "хватит + // тихого режима". The ON vocabulary cannot see these — bare "тих" only + // matches a one-token utterance, by design, so the negator pushes the token + // count past it — and reading them as "no command" would leave quiet mode + // on after he asked for it to stop. + if quietNegated(tokens, nil) { + for _, t := range tokens { + if quietStem(t, "тих") { + return false, true + } + } + } + return false, false +} diff --git a/cmd/mavend/quiet_toggle_test.go b/cmd/mavend/quiet_toggle_test.go new file mode 100644 index 0000000..c450b95 --- /dev/null +++ b/cmd/mavend/quiet_toggle_test.go @@ -0,0 +1,168 @@ +package main + +import ( + "context" + "testing" + "time" + + "github.com/kami/maven/internal/ipc" +) + +// quietFakeAPI records the WriteFact the toggle performs. +type quietFakeAPI struct { + ipc.UnimplementedCoreAPI + got ipc.WriteFactReq + call int +} + +func (a *quietFakeAPI) WriteFact(_ context.Context, req ipc.WriteFactReq) (int64, error) { + a.got, a.call = req, a.call+1 + return 1, nil +} + +// quietVerdict — what a phrase should do to the setting. +type quietVerdict int + +const ( + quietNone quietVerdict = iota + quietOn + quietOff +) + +func TestResolveQuietToggle(t *testing.T) { + cases := []struct { + text string + want quietVerdict + }{ + // ON vocabulary. + {"quiet on", quietOn}, + {"quiet mode", quietOn}, + {"тихий режим", quietOn}, + {"тихий", quietOn}, + {"не шуми", quietOn}, + {"не беспокоить", quietOn}, + {"тихо", quietOn}, + // ON, inflected / embedded in a sentence. + {"включи тихий режим", quietOn}, + {"побудь в тихом режиме", quietOn}, + {"Тихий Режим!", quietOn}, + {"тихая", quietOn}, + + // OFF vocabulary — all seven, incl. the three that used to say ON. + {"quiet off", quietOff}, + {"quiet end", quietOff}, + {"громкий режим", quietOff}, + {"шумный режим", quietOff}, + {"отмени тихий", quietOff}, + {"выключи тихий", quietOff}, + {"не тихо", quietOff}, + // OFF wins over the ON words it contains. + {"выключи тихий режим", quietOff}, + {"отмени тихий режим пожалуйста", quietOff}, + {"верни громкий режим", quietOff}, + + // False positives: "тихо"/"тихий" as ordinary Russian. + {"очень тихий сегодня день", quietNone}, + {"в комнате тихо", quietNone}, + {"тихонько напомни", quietNone}, + {"потихоньку", quietNone}, + {"тихонько", quietNone}, + {"он говорил тихим голосом весь вечер", quietNone}, + + // Unrelated. + {"напомни завтра позвонить маме", quietNone}, + {"какая погода", quietNone}, + {"", quietNone}, + } + + for _, tc := range cases { + t.Run(tc.text, func(t *testing.T) { + api := &quietFakeAPI{} + h := &reactiveHandler{api: api, now: func() time.Time { return time.Unix(0, 0).UTC() }} + reply, handled := h.resolveQuietToggle(context.Background(), tc.text, sourceVoice) + + if tc.want == quietNone { + if handled || reply != "" { + t.Fatalf("%q: got (%q, %v), want no match", tc.text, reply, handled) + } + if api.call != 0 { + t.Fatalf("%q: wrote a fact on a non-match", tc.text) + } + return + } + if !handled { + t.Fatalf("%q: not handled, want %v", tc.text, tc.want) + } + wantReply, wantVal := "тихий режим выключен.", "false" + if tc.want == quietOn { + wantReply, wantVal = "тихий режим включён. буду реже напоминать.", "true" + } + if reply != wantReply { + t.Errorf("%q: reply = %q, want %q", tc.text, reply, wantReply) + } + if api.call != 1 { + t.Fatalf("%q: WriteFact called %d times, want 1", tc.text, api.call) + } + if api.got.Kind != "config" || api.got.Key != "quiet_hours" || api.got.Source != "tap:voice" || api.got.Confidence != 1.0 { + t.Errorf("%q: request shape = %+v", tc.text, api.got) + } + if api.got.Value != wantVal { + t.Errorf("%q: value = %q, want %q", tc.text, api.got.Value, wantVal) + } + }) + } +} + +// TestQuietToggleNegationIsNotAdjacency — negation used to be an adjacency +// pattern ({"не","тих"} in the OFF list), so any word between the negator and +// the quiet word made the ON pattern win and asking for quiet mode to STOP +// turned it on. Negation is scanned over the whole utterance now. +func TestQuietToggleNegationIsNotAdjacency(t *testing.T) { + off := []string{ + "не надо тихий режим", + "не хочу тихий режим", + "тихий режим выключи", + "убери тихий режим", + "хватит тихого режима", + "прекрати тихий режим", + "тихий режим отмени пожалуйста", + } + for _, text := range off { + t.Run(text, func(t *testing.T) { + on, isOff := classifyQuietToggle(text) + if on || !isOff { + t.Fatalf("%q: want OFF, got on=%v off=%v", text, on, isOff) + } + }) + } + + // The two ON phrases that are themselves built on "не" must stay ON: they + // are requests FOR quiet, not negations of one. + for _, text := range []string{"не шуми", "не беспокоить"} { + t.Run(text, func(t *testing.T) { + on, isOff := classifyQuietToggle(text) + if !on || isOff { + t.Fatalf("%q: want ON, got on=%v off=%v", text, on, isOff) + } + }) + } +} + +// TestQuietToggleRecordsTheChannelItArrivedOn — the toggle is reachable from +// mavweb /api/chat and telegram, not only the microphone. Every write used to +// be stamped "tap:voice", so a toggle typed into the web UI claimed a mic wrote +// it and the provenance column lied about a daemon-wide setting. +func TestQuietToggleRecordsTheChannelItArrivedOn(t *testing.T) { + for _, src := range []turnSource{sourceVoice, sourceText} { + t.Run(string(src), func(t *testing.T) { + api := &quietFakeAPI{} + h := &reactiveHandler{api: api, now: func() time.Time { return time.Unix(0, 0).UTC() }} + if _, handled := h.resolveQuietToggle(context.Background(), "тихий режим", src); !handled { + t.Fatal("expected the toggle to match") + } + if api.got.Source != string(src) { + t.Errorf("source = %q, want %q", api.got.Source, src) + } + }) + } +} diff --git a/cmd/mavend/ruwords.go b/cmd/mavend/ruwords.go new file mode 100644 index 0000000..f14513f --- /dev/null +++ b/cmd/mavend/ruwords.go @@ -0,0 +1,180 @@ +// Package main — ruwords.go holds Russian language + calendar/time formatting +// helpers used by the voice reply paths (replySystem, the reminder/routine +// phrasing, etc). Pure functions, no receivers: weekday/month name tables, +// plural agreement, clock/date rendering, and the "do I actually know this +// place/day" guards that pick an honest reply over a confidently wrong one. +// Extend this file rather than voice.go for anything in that shape. +package main + +import ( + "fmt" + "strconv" + "strings" + "time" +) + +var ruWeekdays = []string{ + "воскресенье", "понедельник", "вторник", "среда", + "четверг", "пятница", "суббота", +} + +var ruMonths = []string{ + "января", "февраля", "марта", "апреля", "мая", "июня", + "июля", "августа", "сентября", "октября", "ноября", "декабря", +} + +// onlyLocalTimeReply — the honest answer when the user asks the time somewhere +// other than here. She only keeps one clock, and saying so is better than +// naming the wrong city's time. +// +// There used to be a city→time-zone table here. It was removed on purpose: the +// user only ever asks for local time, so the table was a second list of cities +// to keep in step with the weather one for no gain. +const onlyLocalTimeReply = "я знаю только местное время, про другие города пока не скажу." + +// notPlaceAfterV — words that follow "в" without naming a place, so +// mentionsUnknownPlace does not mistake them for a city. +var notPlaceAfterV = map[string]bool{ + "данный": true, "данную": true, "этот": true, "эту": true, + "котором": true, "какое": true, "какой": true, "который": true, + "общем": true, "точности": true, "курсе": true, "сутках": true, + "часах": true, "минутах": true, "секундах": true, "неделе": true, +} + +// mentionsUnknownPlace reports whether the question has a "в <слово>" phrase +// that looks like a place we do not know ("который час в киеве"). Used only to +// pick the honest "local time only" reply instead of answering local time as +// if it were the city's. +func mentionsUnknownPlace(u string) bool { + toks := strings.Fields(u) + for i := 0; i+1 < len(toks); i++ { + if toks[i] != "в" && toks[i] != "во" { + continue + } + next := strings.Trim(toks[i+1], ".,?!") + if next == "" || notPlaceAfterV[next] { + continue + } + // A number after "в" is a clock ("в 5 часов"), not a place. + if _, err := strconv.Atoi(strings.SplitN(next, ":", 2)[0]); err == nil { + continue + } + return true + } + return false +} + +// onlyNearDaysReply — she can work out today, tomorrow, the day after and +// yesterday, and nothing further. Said out loud instead of answering today's +// date for a day she did not understand. +const onlyNearDaysReply = "я считаю только сегодня, завтра, послезавтра и вчера — про другие дни пока не скажу." + +// dayWords — day references the calendar parser cannot resolve. A weekday name +// or a "через …" phrase means he asked about a specific other day. +var dayWords = []string{ + "понедельник", "вторник", "сред", "четверг", "пятниц", "суббот", "воскресен", + "через", "monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday", +} + +// mentionsUnknownDay reports whether the question names a day the calendar +// parser could not resolve. Mirror of mentionsUnknownPlace: it exists only to +// pick an honest reply over a confidently wrong one. +// +// Only called after ParseCalendarDate has already failed, so "завтра" and the +// other words it does know never reach here. +func mentionsUnknownDay(u string) bool { + for _, w := range dayWords { + if strings.Contains(u, w) { + return true + } + } + return false +} + +// ruClock renders the clock part of the time reply: "15 часов 4 минуты". +func ruClock(t time.Time) string { + h, m := t.Hour(), t.Minute() + hourWord := ruPlural(h, "час", "часа", "часов") + if m == 0 { + return fmt.Sprintf("%d %s ровно", h, hourWord) + } + return fmt.Sprintf("%d %s %d %s", h, hourWord, m, ruPlural(m, "минута", "минуты", "минут")) +} + +// dayPrefix names the day relative to now ("завтра", "вчера", …) so the date +// reply opens the way a person would say it. +func dayPrefix(now, day time.Time) string { + base := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) + switch int(day.Sub(base).Hours() / 24) { + case -1: + return "вчера" + case 0: + return "сегодня" + case 1: + return "завтра" + case 2: + return "послезавтра" + } + return "это" +} + +func ruPlural(n int, one, two, many string) string { + n = n % 100 + if n > 10 && n < 20 { + return many + } + n = n % 10 + switch n { + case 1: + return one + case 2, 3, 4: + return two + default: + return many + } +} + +// hasDurationWords checks whether u is asking about elapsed/remaining time +// rather than the current clock — guards replySystem from replying "сейчас +// X часов" to "сколько времени прошло". Mirrors the stage0.go build filter. +func hasDurationWords(u string) bool { + s := strings.ToLower(strings.TrimSpace(u)) + // First-word duration markers (same keywords as timeQueryBuild in stage0). + first := strings.Fields(s) + if len(first) > 0 { + switch first[0] { + case "прошло", "осталось", "пройдет", "минуло", "проходит": + return true + } + } + // Broader duration keywords appearing anywhere in the utterance. + if strings.Contains(s, "прошло") || strings.Contains(s, "осталось") { + return true + } + if strings.Contains(s, " до ") { + return true + } + return false +} + +// formatTime returns a human-readable Russian time string for a fact timestamp. +// Used by the query handler when answering "когда я это сделал?"-style questions. +func formatTime(t time.Time) string { + now := time.Now() + if t.After(now.Add(-2*time.Minute)) && t.Before(now.Add(2*time.Minute)) { + return "только что" + } + diff := now.Sub(t) + switch { + case diff < 10*time.Minute: + return "несколько минут назад" + case diff < 60*time.Minute: + return fmt.Sprintf("%d минут назад", int(diff.Minutes())) + case diff < 2*time.Hour: + return "час назад" + case diff < 24*time.Hour: + return fmt.Sprintf("%d часа назад", int(diff.Hours())) + default: + return t.Format("2 января 15:04") + } +} diff --git a/cmd/mavend/simulator_test.go b/cmd/mavend/simulator_test.go new file mode 100644 index 0000000..04a314c --- /dev/null +++ b/cmd/mavend/simulator_test.go @@ -0,0 +1,1042 @@ +// mavend/simulator_test.go — the replayable full-system simulator +// (Vikunja #284, 20-07-2026-BACKLOG.md item 7). +// +// # What it is +// +// A scripted day, replayed through the real mavend code paths, with every +// boundary faked and the clock under the scenario's control. A scenario is a +// JSON file in testdata/scenarios; the harness reads it, builds a world, walks +// the steps in order, and asserts on what actually happened: +// +// what Maven SAID — the reply text of every utterance +// what was SENT — every delivery.Sendable the dispatcher emitted +// what ARRIVED — the unified intake journal from #283 +// what TOOLS were called — the recorded requests against fake Praxis/Nexis/Hexis +// what did NOT happen — expect_no_send / expect_no_call, first-class +// +// The last one is the point. Maven's hard constraints are mostly negative — +// not a nag, not autonomous, nothing executed without confirmation — and a +// harness that can only assert on things that happened cannot test any of +// them. "Nothing was sent" is an assertion here, not an absence of one. +// +// # Determinism +// +// No time.Now() runs inside a replay. The scenario names a start instant, each +// step names a wall-clock offset from it, and the harness advances a fakeClock +// to that offset before running the step. Every clock reader in the world — +// the handler's `now`, the tick loop's `tick(ctx, now)`, the intake journal's +// publish stamp — is wired to that clock. Two runs of the same file produce +// the same transcript, and a scenario about 08:35 does not behave differently +// at 03:00 in CI. +// +// The tick is driven by the scenario, not by a ticker: tick() already takes +// `now` as an argument, so the only thing the daemon's ticker contributed was +// wall-clock timing, which is exactly what a replay must not have. +// +// # Why this shape and not a binary +// +// Vikunja #288 (golden-audio STT) deferred its tier-2 "audio → STT → router → +// phraser" scenarios to this task, and asked that they reuse a fixture format +// rather than inventing a third. A scenario here can name a WAV from +// cmd/mavsttd/testdata and the harness will feed it through the STT seam. As a +// test it runs under `make test` on every change, which a separate binary +// would not. +// +// # Production is untouched +// +// Every file this task adds is a _test.go file or testdata. There is no +// simulator in the daemon, no flag, no config key, and no code path that +// checks whether a simulation is running. The seams it uses — stt.Transcriber, +// tts.Synthesizer, router.Completer, delivery.Sink, ipc.CoreAPI, the +// event.Bus from #283 — all already existed for the production wiring. +package main + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "path/filepath" + "regexp" + "strings" + "sync" + "testing" + "time" + + "github.com/kami/maven/internal/audio" + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/delivery" + "github.com/kami/maven/internal/dialogue" + "github.com/kami/maven/internal/event" + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/llm" + "github.com/kami/maven/internal/loop" + "github.com/kami/maven/internal/phraser" + "github.com/kami/maven/internal/router" + "github.com/kami/maven/internal/store" + "github.com/kami/maven/internal/tool" + "github.com/kami/maven/internal/voice" +) + +// --------------------------------------------------------------------------- +// Scenario format +// --------------------------------------------------------------------------- + +// scenario — one scripted day. schema_version matches the convention already +// set by testdata/system_safety_scenarios.json. +type scenario struct { + SchemaVersion int `json:"schema_version"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + + // Start — the instant the day begins, RFC3339. Every step offset is + // relative to it, and nothing in the run reads a real clock. + Start string `json:"start"` + + // Script — what the resident model answers. The world has no llama-server; + // see scriptedLLM for how an entry is chosen. + Script []scriptEntry `json:"script,omitempty"` + + // Praxis / Nexus / Hexis — canned bodies for the ecosystem fakes. Absent ⇒ + // that service is not wired at all, which is the default box. + Praxis string `json:"praxis_attention,omitempty"` + Nexus string `json:"nexus_resolve,omitempty"` + Hexis string `json:"hexis_capabilities,omitempty"` + + // Tools — allowlist rows to enable before the first step. An act is only + // dispatched when its verb is on the enabled allowlist, so a scenario that + // wants to exercise one has to say which rows Kami had enabled. + Tools []toolRow `json:"tools,omitempty"` + + Steps []step `json:"steps"` +} + +// toolRow — one enabled allowlist row. Cmd is empty for an ecosystem verb, +// which is intercepted before the process executor is ever reached. +type toolRow struct { + Name string `json:"name"` + Cmd []string `json:"cmd,omitempty"` + Destructive bool `json:"destructive,omitempty"` +} + +// scriptEntry — one canned model answer. Match is a substring of the user +// message; the first entry whose Match is contained in it wins, and an entry +// with an empty Match is the catch-all. +// +// Route and Reply are separate because the same model serves both contracts +// (CLAUDE.md, "LLM output contract"): a grammar-constrained call is a routing +// call and gets Route, an unconstrained one is a phrasing call and gets Reply. +type scriptEntry struct { + Match string `json:"match"` + Route string `json:"route,omitempty"` + Reply string `json:"reply,omitempty"` +} + +// step — one scripted moment. At is "HH:MM" or "HH:MM:SS", interpreted in the +// start instant's location; the clock is advanced to it before the step runs. +// +// A step does exactly one thing (say / audio / signal / fact / tick / arrive) +// and then asserts. Assertions are evaluated against everything recorded since +// the run began, except expect_no_send and expect_no_call, which are scoped to +// this step — "nothing was sent because of THIS" is the useful question. +// +// The asymmetry is worth stating plainly, because it changes what a scenario +// author is writing. expect_sent_contains, expect_called and expect_events are +// RUN-scoped: they pass if the thing ever happened, at any earlier step. So +// repeating expect_events: ["rss:tech"] on a later step asserts nothing new, +// it just re-checks the earlier arrival. The negatives — expect_no_send, +// expect_not_called, expect_no_events — are STEP-scoped, and are the ones that +// say something about this moment. expect_reply_contains and +// expect_reply_lacks read the most recent reply only, so a step with no +// utterance re-checks the previous one. +type step struct { + At string `json:"at"` + Note string `json:"note,omitempty"` + + // --- stimuli (at most one per step) --- + + // Say — an utterance, as text, through the same runTurn the IPC chat path + // uses. + Say string `json:"say,omitempty"` + + // Audio — a WAV under cmd/mavsttd/testdata, fed through the STT seam. This + // is #288's deferred tier 2. The harness uses the deterministic stt stub + // unless a real transcriber is available, so the assertion a scenario can + // make about an audio step is about the PIPELINE, not about whisper's + // accuracy — that is what cmd/mavsttd/golden_test.go is for. + Audio string `json:"audio,omitempty"` + + // Signal — a presence/world fact arriving from a poller or /api/signal. + Signal *signalStep `json:"signal,omitempty"` + + // Arrive — an intake write from a module: an ambient notification, a feed + // item, a mail candidate. Goes through the same decorated ipc.CoreAPI the + // daemon gives those callers, so it lands in the journal exactly as it + // would in production. + Arrive *arriveStep `json:"arrive,omitempty"` + + // Tick — run one iteration of the proactive loop at this instant. + Tick bool `json:"tick,omitempty"` + + // Fault — make every ecosystem fake answer with this HTTP status from now + // on. The degraded-mode lever; ClearFault puts them back. + Fault int `json:"fault,omitempty"` + ClearFault bool `json:"clear_fault,omitempty"` + + // --- assertions --- + + ExpectReply []string `json:"expect_reply_contains,omitempty"` + ExpectNotReply []string `json:"expect_reply_lacks,omitempty"` + ExpectSent []string `json:"expect_sent_contains,omitempty"` + ExpectNoSend bool `json:"expect_no_send,omitempty"` + ExpectCalled []string `json:"expect_called,omitempty"` + ExpectNotCalled []string `json:"expect_not_called,omitempty"` + ExpectEvents []string `json:"expect_events,omitempty"` + ExpectNoEvents bool `json:"expect_no_events,omitempty"` +} + +type signalStep struct { + Key string `json:"key"` + Value string `json:"value"` + Source string `json:"source"` + Kind string `json:"kind,omitempty"` + + // Confidence — 0 ⇒ 1.0, an observation Maven made herself. A relayed + // notification is not that: the ambient path writes + // calendar.AmbientConfidence, 0.6, and factPriority in intake.go branches + // on exactly that difference. The field exists so a replay can reach the + // low branch, which it could not while write() hardcoded 1.0. + Confidence float64 `json:"confidence,omitempty"` +} + +type arriveStep struct { + // Note / Fact / Task — exactly one. Each mirrors the intake seam its real + // caller uses. + Note *arriveNote `json:"note,omitempty"` + Fact *signalStep `json:"fact,omitempty"` + Task *arriveTask `json:"task,omitempty"` + AsOf string `json:"as_of,omitempty"` // "HH:MM" — OccurredAt, when it differs from the step time + Source string `json:"source"` +} + +type arriveNote struct { + Text string `json:"text"` +} + +type arriveTask struct { + Text string `json:"text"` + Evidence string `json:"evidence,omitempty"` + Status string `json:"status,omitempty"` +} + +// --------------------------------------------------------------------------- +// The world +// --------------------------------------------------------------------------- + +// simWorld — every faked boundary plus the real components between them. +type simWorld struct { + t *testing.T + clock *fakeClock + loc *time.Location + start time.Time + + store *store.Store + api ipc.CoreAPI // the intake-decorated adapter, same as the daemon builds + bus *event.Bus + handler *reactiveHandler + tick *tickLoop + sink *recordingSink + llm *scriptedLLM + + praxis *fakeServer + nexus *fakeServer + hexis *fakeServer + + // transcript — everything that happened, in order. Printed on failure so a + // broken scenario is diagnosable without a debugger. + transcript []string + replies []string + + // fatalf — the abort seam. Defaults to t.Fatalf. It exists so a test can + // reach the harness's own refusals (a backwards step, a missing WAV) and + // assert on them instead of dying with the scenario. + fatalf func(format string, args ...any) + + // published — how many events the bus accepted, counted through a + // subscriber. bus.Len() saturates at the ring capacity and cannot answer + // "did anything arrive during this step" once a long scenario has filled + // it. + mu sync.Mutex + published int + + // audio — golden_v1.json, parsed once. A scenario with twenty audio steps + // used to read and parse the manifest twenty times. + audio map[string]string +} + +func (w *simWorld) publishCount() int { + w.mu.Lock() + defer w.mu.Unlock() + return w.published +} + +// recordingSink captures every send, mutex-guarded (the tick loop dispatches +// from its own goroutine in production and the race detector is on here). +type recordingSink struct { + mu sync.Mutex + sends []delivery.Sendable +} + +func (s *recordingSink) Send(_ context.Context, d delivery.Sendable) error { + s.mu.Lock() + defer s.mu.Unlock() + s.sends = append(s.sends, d) + return nil +} + +func (s *recordingSink) all() []delivery.Sendable { + s.mu.Lock() + defer s.mu.Unlock() + out := make([]delivery.Sendable, len(s.sends)) + copy(out, s.sends) + return out +} + +func (s *recordingSink) count() int { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.sends) +} + +// scriptedLLM stands in for llama-server on BOTH contracts the resident model +// serves: grammar-constrained routing and unconstrained phrasing. +// +// It is not a stub that ignores its input — a scenario that scripts an answer +// for "что я пропустил" and gets asked something else must fail, not silently +// return the wrong intent. An unmatched call returns an error, and the router +// then falls through to the classifier cascade exactly as it does in +// production when llama-server is unreachable. That fall-through is itself +// worth exercising: it is the failure floor CLAUDE.md refuses to let rot. +type scriptedLLM struct { + mu sync.Mutex + entries []scriptEntry + calls []llm.Req +} + +func (s *scriptedLLM) Complete(_ context.Context, r llm.Req) (string, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.calls = append(s.calls, r) + routing := r.Grammar != "" + for _, e := range s.entries { + if e.Match != "" && !strings.Contains(strings.ToLower(r.User), strings.ToLower(e.Match)) { + continue + } + if routing && e.Route != "" { + return e.Route, nil + } + if !routing && e.Reply != "" { + return e.Reply, nil + } + } + return "", fmt.Errorf("simulator: no scripted %s answer for %q", + map[bool]string{true: "route", false: "reply"}[routing], truncateRunes(r.User, 60)) +} + +// --------------------------------------------------------------------------- +// Building the world +// --------------------------------------------------------------------------- + +func newSimWorld(t *testing.T, sc scenario) *simWorld { + t.Helper() + + start, err := time.Parse(time.RFC3339, sc.Start) + if err != nil { + t.Fatalf("scenario %q: bad start %q: %v", sc.Name, sc.Start, err) + } + clock := newFakeClock(start) + + st := newTestStore(t) + bus := event.NewBus(512) + // The same decorator the daemon wires, on the same clock: intake in a + // replay is journalled exactly as it is in production. + api := newIntakeAPI(ipc.NewStoreAPI(st), bus, clock.Now) + + sink := &recordingSink{} + rules := loop.DefaultRules() + gatherer := loop.NewGatherer(st, rules) + dispatcher := delivery.NewDispatcher(delivery.Config{ + Voice: sink, Ntfy: sink, Telegram: sink, Nudges: st, Reminders: st, + }) + tl := newTickLoop(st, gatherer, dispatcher, phraser.NewStub(), rules, + time.Minute, 5*time.Minute, 0, nil, nil, nil, nil) + + scripted := &scriptedLLM{entries: sc.Script} + + w := &simWorld{ + t: t, clock: clock, loc: start.Location(), start: start, + store: st, api: api, bus: bus, tick: tl, sink: sink, llm: scripted, + } + w.fatalf = t.Fatalf + bus.Subscribe(func(event.Event) { + w.mu.Lock() + w.published++ + w.mu.Unlock() + }) + + // Allowlist rows the scenario asked for, enabled before the first step. An + // act only reaches a dispatch if a verb is on the enabled list, so without + // this a scenario cannot script one at all. + for _, tr := range sc.Tools { + cmd := tr.Cmd + if len(cmd) == 0 { + cmd = []string{"true"} + } + if err := st.EnableTool(context.Background(), tr.Name, cmd, tr.Destructive, "sim", start); err != nil { + t.Fatalf("scenario %q: enabling tool %q: %v", sc.Name, tr.Name, err) + } + } + + // Ecosystem fakes, wired only when the scenario supplies a body — a box + // with no praxis block has no praxis client, and a scenario must be able to + // reproduce that. + eco := &ecosystemWiring{} + if sc.Praxis != "" { + w.praxis = newFakePraxis(t, sc.Praxis) + eco.praxis = newPraxisClient(w.praxis.URL) + } + if sc.Nexus != "" { + w.nexus = newFakeNexus(t, sc.Nexus) + } + if sc.Hexis != "" { + w.hexis = newFakeHexis(t, sc.Hexis, fixtureHexisExecuted("exec_1", "completed")) + } + + // The router: the same cascade the daemon builds — stage-0 grammars, the + // LLM router on the scripted model, the classifier underneath. Keeping the + // classifier in is deliberate; it is the failure floor, and a scenario that + // scripts no route for an utterance exercises it. + emb := router.NewHashEmbedder(1024) + // The matcher reads the live enabled allowlist, same as the daemon's. It + // used to be built on a nil API, which meant any scenario that produced an + // act panicked the moment the matcher was consulted. + matcher := tool.NewMatcher(api) + rtr := buildRouter(emb, matcher, config.DefaultRouterThreshold, router.NewLLMRouter(scripted)) + + w.handler = &reactiveHandler{ + stt: simTranscriber{}, + tts: simSynthesizer{}, + router: rtr, + embedder: emb, + api: api, + matcher: matcher, + tools: tool.NewExecutor(api, 5*time.Second), + phraser: phraser.NewStub(), + replier: newLLMReplier(scripted, nil), + now: clock.Now, + memStore: st.VectorMemory(), + dataStore: st, + queryMinScore: config.DefaultQueryMinScore, + queryMinMargin: config.DefaultQueryMinMargin, + timeParser: router.StubDateTimeParser{}, + dialogueSessions: dialogue.NewSessionStore(time.Hour), + clarifyStore: dialogue.NewClarifyStore(time.Hour), + clarifyMaxAttempts: dialogue.DefaultMaxAttempts, + ecosystem: eco, + } + return w +} + +// simTranscriber — the STT seam. Deterministic by construction: it returns the +// text the harness parked for this step, so the pipeline under test is +// "audio arrives → a turn runs", not "whisper heard correctly". Transcription +// accuracy is cmd/mavsttd/golden_test.go's job (#288 tier 1), and duplicating +// it here would make every scenario depend on a 500 MB model. +type simTranscriber struct{ text string } + +func (s simTranscriber) Transcribe(_ context.Context, _ audio.Audio) (string, float64, error) { + return s.text, 1.0, nil +} + +// simSynthesizer — the TTS seam. A scenario asserts on what Maven SAID, which +// is the reply text; the waveform is not the artefact under test. +type simSynthesizer struct{} + +func (simSynthesizer) Synthesize(_ context.Context, _ string) (audio.Audio, error) { + return audio.Audio{Format: audio.PCM16kMono}, nil +} + +// --------------------------------------------------------------------------- +// Running +// --------------------------------------------------------------------------- + +func (w *simWorld) logf(format string, args ...any) { + w.transcript = append(w.transcript, + fmt.Sprintf("%s %s", w.clock.Now().In(w.loc).Format("15:04:05"), fmt.Sprintf(format, args...))) +} + +// dump prints the whole transcript. Called on any failure — a scenario that +// broke on step 7 is unreadable without the six steps before it. +func (w *simWorld) dump() { + w.t.Logf("--- replay transcript ---\n%s", strings.Join(w.transcript, "\n")) +} + +// advanceTo moves the clock to the step's offset. Time only ever moves +// FORWARD: a scenario with steps out of order is a bug in the scenario, and +// silently reordering it would hide the bug. +func (w *simWorld) advanceTo(at string) { + w.t.Helper() + if at == "" { + return + } + target := w.timeOf(at) + now := w.clock.Now() + if target.Before(now) { + w.fatalf("step at %s goes backwards from %s — scenario steps must be in order", + at, now.In(w.loc).Format("15:04:05")) + return + } + w.clock.Advance(target.Sub(now)) +} + +// timeOf resolves an "HH:MM" or "HH:MM:SS" step offset against the scenario's +// start day and location. +func (w *simWorld) timeOf(at string) time.Time { + w.t.Helper() + layout := "15:04" + if strings.Count(at, ":") == 2 { + layout = "15:04:05" + } + hm, err := time.Parse(layout, at) + if err != nil { + w.t.Fatalf("bad step time %q: %v", at, err) + } + return time.Date(w.start.Year(), w.start.Month(), w.start.Day(), + hm.Hour(), hm.Minute(), hm.Second(), 0, w.loc) +} + +func (w *simWorld) run(sc scenario) { + ctx := context.Background() + for i, s := range sc.Steps { + w.advanceTo(s.At) + if s.Note != "" { + w.logf("# %s", s.Note) + } + sendsBefore := w.sink.count() + callsBefore := w.callMark() + eventsBefore := w.publishCount() + + w.stimulate(ctx, s) + w.assert(i, s, sendsBefore, callsBefore, eventsBefore) + } +} + +func (w *simWorld) stimulate(ctx context.Context, s step) { + if s.Fault != 0 || s.ClearFault { + for _, fs := range []*fakeServer{w.praxis, w.nexus, w.hexis} { + if fs != nil { + fs.SetFault(s.Fault) + } + } + w.logf("fault=%d on every ecosystem fake", s.Fault) + } + + switch { + case s.Say != "": + reply := w.handler.runTurn(ctx, s.Say, sourceText) + w.replies = append(w.replies, reply) + w.logf("он: %s", s.Say) + w.logf("она: %s", reply) + + case s.Audio != "": + text := w.audioText(s.Audio) + // Swap in a transcriber parked with this step's text, then run the same + // push-to-talk entry point the voice client calls. + w.handler.stt = simTranscriber{text: text} + resp, err := w.handler.HandlePushToTalk(ctx, voicePTT(), 0) + if err != nil { + w.t.Fatalf("push-to-talk on %s: %v", s.Audio, err) + } + w.replies = append(w.replies, resp.ReplyText) + w.logf("[wav %s → %q]", filepath.Base(s.Audio), text) + w.logf("она: %s", resp.ReplyText) + + case s.Signal != nil: + w.write(ctx, *s.Signal, w.clock.Now()) + w.logf("сигнал: %s=%s (%s)", s.Signal.Key, s.Signal.Value, s.Signal.Source) + + case s.Arrive != nil: + w.arrive(ctx, *s.Arrive) + + case s.Tick: + w.tick.tick(ctx, w.clock.Now()) + w.logf("tick") + } +} + +func (w *simWorld) write(ctx context.Context, sig signalStep, ts time.Time) { + w.t.Helper() + kind := sig.Kind + if kind == "" { + kind = "env" + } + conf := sig.Confidence + if conf == 0 { + conf = 1.0 + } + if _, err := w.api.WriteFact(ctx, ipc.WriteFactReq{ + Ts: ts, Kind: kind, Key: sig.Key, Value: sig.Value, Source: sig.Source, Confidence: conf, + }); err != nil { + w.fatalf("write fact %s: %v", sig.Key, err) + } +} + +func (w *simWorld) arrive(ctx context.Context, a arriveStep) { + w.t.Helper() + // AsOf is when the thing HAPPENED, which for a feed item or a relayed + // notification is usually earlier than when Maven heard about it. It does + // not move the clock — only the timestamp on the row and the envelope. + ts := w.clock.Now() + if a.AsOf != "" { + ts = w.timeOf(a.AsOf) + } + switch { + case a.Fact != nil: + f := *a.Fact + if f.Source == "" { + f.Source = a.Source + } + w.write(ctx, f, ts) + w.logf("пришло: факт %s=%s (%s)", f.Key, f.Value, f.Source) + case a.Note != nil: + if _, err := w.api.WriteNote(ctx, ts, a.Note.Text, nil, a.Source); err != nil { + w.t.Fatalf("write note from %s: %v", a.Source, err) + } + w.logf("пришло: заметка от %s — %s", a.Source, truncateRunes(a.Note.Text, 60)) + case a.Task != nil: + status := a.Task.Status + if status == "" { + status = store.TaskCandidate + } + if _, err := w.api.CaptureTask(ctx, ipc.CaptureTaskReq{ + Text: a.Task.Text, Source: a.Source, Evidence: a.Task.Evidence, Status: status, Ts: ts, + }); err != nil { + w.t.Fatalf("capture task from %s: %v", a.Source, err) + } + w.logf("пришло: задача от %s — %s", a.Source, a.Task.Text) + default: + w.t.Fatalf("arrive step from %s carries nothing", a.Source) + } +} + +// audioText resolves a scenario's WAV reference to the text the fixture is +// known to contain, by reading cmd/mavsttd's golden manifest (#288's format, +// reused rather than duplicated). An unknown reference fails the scenario +// rather than quietly transcribing to "". +// +// The manifest is read and parsed once per world, not once per step: a +// scenario with twenty audio steps should cost one file read. +func (w *simWorld) audioText(ref string) string { + w.t.Helper() + manifest := filepath.Join("..", "mavsttd", "testdata", "golden_v1.json") + if w.audio == nil { + raw, err := os.ReadFile(manifest) + if err != nil { + w.fatalf("audio step %q: reading %s: %v", ref, manifest, err) + return "" + } + var m struct { + Cases []struct { + Name string `json:"name"` + WAV string `json:"wav"` + Text string `json:"text"` + } `json:"cases"` + } + if err := json.Unmarshal(raw, &m); err != nil { + w.fatalf("audio step %q: parsing %s: %v", ref, manifest, err) + return "" + } + w.audio = make(map[string]string, len(m.Cases)*2) + for _, c := range m.Cases { + w.audio[c.Name] = c.Text + w.audio[c.WAV] = c.Text + } + } + if text, ok := w.audio[ref]; ok && ref != "" { + return text + } + w.fatalf("audio step %q: no such case in %s", ref, manifest) + return "" +} + +func voicePTT() voice.PushToTalkReq { + return voice.PushToTalkReq{Audio: audio.Audio{Format: audio.PCM16kMono}} +} + +// fakes — every ecosystem fake, in a fixed order. Both the mark and the paths +// walk this same order, which is the whole point: they have to agree. +func (w *simWorld) fakes() []*fakeServer { return []*fakeServer{w.praxis, w.nexus, w.hexis} } + +// callMark takes a PER-SERVER snapshot of how many requests each fake has +// seen. It is not a total. +// +// A total cannot be used to slice the concatenated path list, and the harness +// used to do exactly that. callPaths concatenates praxis, then nexus, then +// hexis; a total counts arrivals across all three. With praxis on 3 requests +// and nexus on 1, the total is 4 and the list is [p1 p2 p3 n1]. A step that +// calls praxis once makes the list [p1 p2 p3 p4 n1], and paths[4:] is [n1]. +// The new praxis call sits at index 3 and is never looked at, so +// expect_not_called on praxis passed on a step that called praxis. The same +// slice reported the stale nexus call as new, so expect_not_called on +// "/resolve" failed on a step that resolved nothing. +func (w *simWorld) callMark() []int { + mark := make([]int, len(w.fakes())) + for i, fs := range w.fakes() { + if fs != nil { + mark[i] = len(fs.Requests()) + } + } + return mark +} + +// callPathsSince returns the calls each fake took after its own mark. A nil +// mark means "everything, from the beginning of the run". +func (w *simWorld) callPathsSince(mark []int) []string { + var out []string + for i, fs := range w.fakes() { + if fs == nil { + continue + } + reqs := fs.Requests() + from := 0 + if mark != nil && i < len(mark) { + from = mark[i] + } + if from > len(reqs) { + from = len(reqs) + } + for _, r := range reqs[from:] { + out = append(out, r.Method+" "+r.Path) + } + } + return out +} + +func (w *simWorld) callPaths() []string { return w.callPathsSince(nil) } + +// --------------------------------------------------------------------------- +// Assertions +// --------------------------------------------------------------------------- + +func (w *simWorld) assert(i int, s step, sendsBefore int, callsBefore []int, eventsBefore int) { + w.t.Helper() + where := fmt.Sprintf("step %d (%s)", i+1, s.At) + if s.Note != "" { + where += " " + s.Note + } + fail := func(format string, args ...any) { + w.dump() + w.t.Errorf("%s: %s", where, fmt.Sprintf(format, args...)) + } + + lastReply := "" + if len(w.replies) > 0 { + lastReply = w.replies[len(w.replies)-1] + } + for _, want := range s.ExpectReply { + if !containsFold(lastReply, want) { + fail("reply %q does not contain %q", lastReply, want) + } + } + for _, unwanted := range s.ExpectNotReply { + if containsFold(lastReply, unwanted) { + fail("reply %q contains %q and must not", lastReply, unwanted) + } + } + + sent := w.sink.all() + for _, want := range s.ExpectSent { + if !anyContains(sendableTexts(sent), want) { + fail("nothing sent mentions %q; sent so far: %v", want, sendableTexts(sent)) + } + } + // Scoped to this step on purpose: "nothing was sent BECAUSE OF THIS" is the + // question a not-a-nag constraint asks. + if s.ExpectNoSend && len(sent) > sendsBefore { + fail("expected nothing to be sent, got %v", sendableTexts(sent[sendsBefore:])) + } + + paths := w.callPaths() + for _, want := range s.ExpectCalled { + if !anyContains(paths, want) { + fail("no ecosystem call matches %q; calls so far: %v", want, paths) + } + } + since := w.callPathsSince(callsBefore) + for _, unwanted := range s.ExpectNotCalled { + if anyContains(since, unwanted) { + fail("an ecosystem call matched %q and must not have: %v", unwanted, since) + } + } + + evs := w.bus.Recent(0) + for _, want := range s.ExpectEvents { + if !anyContains(eventLines(evs), want) { + fail("no intake event matches %q; journal: %v", want, eventLines(evs)) + } + } + // Counted publishes, not bus.Len(): the ring saturates at its capacity, so + // a long scenario that filled it made every later expect_no_events pass + // unconditionally. + if s.ExpectNoEvents && w.publishCount() > eventsBefore { + fail("expected nothing to arrive, %d event(s) were published", + w.publishCount()-eventsBefore) + } +} + +func sendableTexts(sends []delivery.Sendable) []string { + out := make([]string, 0, len(sends)) + for _, s := range sends { + out = append(out, fmt.Sprintf("[%s] %s", s.RuleName, s.Body)) + } + return out +} + +func eventLines(evs []event.Event) []string { + out := make([]string, 0, len(evs)) + for _, e := range evs { + // Priority is in the line so a scenario can assert on it. It is the one + // field factPriority derives from confidence, and without it a replay + // could set a confidence but never see what the journal did with it. + out = append(out, fmt.Sprintf("%s/%s pri=%s %s %s", e.Source, e.Kind, e.Priority, e.Title, e.Body)) + } + return out +} + +func containsFold(hay, needle string) bool { + return strings.Contains(strings.ToLower(hay), strings.ToLower(needle)) +} + +func anyContains(hay []string, needle string) bool { + for _, h := range hay { + if containsFold(h, needle) { + return true + } + } + return false +} + +// --------------------------------------------------------------------------- +// The test +// --------------------------------------------------------------------------- + +const scenarioDir = "testdata/scenarios" + +// TestSimulatorScenarios replays every scenario file. Adding a scenario is +// adding a JSON file — no Go change, which is the property that makes this +// cheap enough to actually use. +func TestSimulatorScenarios(t *testing.T) { + entries, err := os.ReadDir(scenarioDir) + if err != nil { + t.Fatalf("reading %s: %v", scenarioDir, err) + } + var ran int + for _, ent := range entries { + if ent.IsDir() || !strings.HasSuffix(ent.Name(), ".json") { + continue + } + ran++ + name := strings.TrimSuffix(ent.Name(), ".json") + t.Run(name, func(t *testing.T) { + sc := loadScenario(t, filepath.Join(scenarioDir, ent.Name())) + w := newSimWorld(t, sc) + w.run(sc) + if testing.Verbose() { + w.dump() + } + }) + } + if ran == 0 { + t.Fatalf("no scenarios in %s — the harness would pass vacuously", scenarioDir) + } +} + +func loadScenario(t *testing.T, path string) scenario { + t.Helper() + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("reading %s: %v", path, err) + } + var sc scenario + dec := json.NewDecoder(strings.NewReader(string(raw))) + dec.DisallowUnknownFields() // a typo'd assertion key must fail, not be ignored + if err := dec.Decode(&sc); err != nil { + t.Fatalf("parsing %s: %v", path, err) + } + if sc.SchemaVersion != 1 { + t.Fatalf("%s: schema_version = %d, want 1", path, sc.SchemaVersion) + } + if sc.Name == "" || sc.Start == "" || len(sc.Steps) == 0 { + t.Fatalf("%s: a scenario needs a name, a start and at least one step", path) + } + return sc +} + +// TestSimulatorIsDeterministic replays one scenario twice and requires an +// identical transcript. This is the property the whole task rests on: if a +// time.Now() creeps into a replayed path, two runs diverge and this fails. +func TestSimulatorIsDeterministic(t *testing.T) { + path := filepath.Join(scenarioDir, "morning_missed.json") + sc := loadScenario(t, path) + + transcriptOf := func() string { + w := newSimWorld(t, sc) + w.run(sc) + return strings.Join(w.transcript, "\n") + } + first := transcriptOf() + second := transcriptOf() + if first != second { + t.Errorf("two replays of the same scenario diverged:\n--- first ---\n%s\n--- second ---\n%s", first, second) + } + // And every transcript timestamp must lie inside the scenario's own span. + // + // This used to compare the transcript against time.Now().Format("15:04"), + // which failed whenever the suite happened to run during the half hour the + // scenario covers: morning_missed logs 08:30 through 09:00, sc.Start + // contains only 08:30, so a run at 08:35 reported a wall-clock read that + // had not happened. A determinism test that depends on the time of day is + // the bug it is looking for. + start, err := time.Parse(time.RFC3339, sc.Start) + if err != nil { + t.Fatalf("bad start: %v", err) + } + last := start + for _, s := range sc.Steps { + if at := stepInstant(t, start, s.At); at.After(last) { + last = at + } + } + // Only the lines logf stamped. A note or a feed item can carry its own + // newlines, and those continuation lines have no timestamp. + stamp := regexp.MustCompile(`^(\d\d:\d\d:\d\d) `) + for _, line := range strings.Split(first, "\n") { + m := stamp.FindStringSubmatch(line) + if m == nil { + continue + } + at := stepInstant(t, start, m[1]) + if at.Before(start) || at.After(last) { + t.Errorf("transcript line %q is stamped outside the scenario span %s..%s — "+ + "something in the replay path read time.Now()", + line, start.Format("15:04:05"), last.Format("15:04:05")) + } + } +} + +// TestCallsSinceAreScopedPerServer pins the ordering bug that made +// expect_not_called unsound. The mark is per server; a total cannot slice a +// list that is concatenated per server. +func TestCallsSinceAreScopedPerServer(t *testing.T) { + sc := scenario{SchemaVersion: 1, Name: "x", Start: "2026-08-01T08:30:00+03:00", + Praxis: fixturePraxisAttentionItems(), Nexus: fixtureNexusResolved("ent_1", "thing", "device"), + Steps: []step{{At: "08:30"}}} + w := newSimWorld(t, sc) + + hit := func(fs *fakeServer, path string) { + t.Helper() + resp, err := http.Get(fs.URL + path) + if err != nil { + t.Fatalf("hitting %s: %v", path, err) + } + resp.Body.Close() + } + // Praxis runs ahead of nexus, so the concatenated list already has a nexus + // call sitting after three praxis ones. + hit(w.praxis, "/api/v1/tools/attention") + hit(w.praxis, "/api/v1/tools/attention") + hit(w.praxis, "/api/v1/tools/attention") + hit(w.nexus, "/api/v1/resolve") + + mark := w.callMark() + hit(w.praxis, "/api/v1/tools/attention") + + since := w.callPathsSince(mark) + if !anyContains(since, "attention") { + t.Errorf("the praxis call made after the mark is missing from %v", since) + } + if anyContains(since, "resolve") { + t.Errorf("a nexus call from before the mark was reported as new: %v", since) + } +} + +// TestPublishCountDoesNotSaturateWithTheRing pins expect_no_events on a bus +// that has already wrapped. bus.Len() stops at the capacity, so it can no +// longer answer "did anything arrive". +func TestPublishCountDoesNotSaturateWithTheRing(t *testing.T) { + sc := scenario{SchemaVersion: 1, Name: "x", Start: "2026-08-01T08:30:00+03:00", + Steps: []step{{At: "08:30"}}} + w := newSimWorld(t, sc) + + for i := 0; i < event.DefaultCapacity+5; i++ { + w.bus.Publish(event.Event{ + Source: "sim:test", Kind: event.KindFact, Title: fmt.Sprintf("f%d", i), + }, w.clock.Now()) + } + if got := w.bus.Len(); got != event.DefaultCapacity { + t.Fatalf("ring holds %d, expected it to be saturated at %d", got, event.DefaultCapacity) + } + before := w.publishCount() + w.bus.Publish(event.Event{Source: "sim:test", Kind: event.KindFact, Title: "one more"}, w.clock.Now()) + if w.publishCount() != before+1 { + t.Errorf("publish count went %d → %d on a full ring, expected it to keep counting", + before, w.publishCount()) + } +} + +// stepInstant resolves "HH:MM" or "HH:MM:SS" against the scenario's start day. +func stepInstant(t *testing.T, start time.Time, at string) time.Time { + t.Helper() + layout := "15:04" + if strings.Count(at, ":") == 2 { + layout = "15:04:05" + } + hm, err := time.Parse(layout, at) + if err != nil { + t.Fatalf("bad step time %q: %v", at, err) + } + return time.Date(start.Year(), start.Month(), start.Day(), + hm.Hour(), hm.Minute(), hm.Second(), 0, start.Location()) +} + +// TestSimulatorRefusesBackwardsSteps guards the one scenario-authoring mistake +// that would silently produce a meaningless run. +// +// It used to advance to 09:00 and then to 09:30 and assert the clock had +// moved, which is the forwards case: the backwards branch it is named after +// was never reached, because reaching it ended the test. The fatalf seam is +// what makes it testable. +func TestSimulatorRefusesBackwardsSteps(t *testing.T) { + sc := scenario{SchemaVersion: 1, Name: "x", Start: "2026-08-01T08:30:00+03:00", + Steps: []step{{At: "09:00"}}} + w := newSimWorld(t, sc) + var refusal string + w.fatalf = func(format string, args ...any) { refusal = fmt.Sprintf(format, args...) } + + w.advanceTo("09:00") + if got := w.clock.Now().In(w.loc).Format("15:04"); got != "09:00" { + t.Fatalf("clock at %s after advancing to 09:00", got) + } + if refusal != "" { + t.Fatalf("a forwards step was refused: %s", refusal) + } + + w.advanceTo("08:45") + if refusal == "" { + t.Fatal("a step going backwards to 08:45 was accepted") + } + if got := w.clock.Now().In(w.loc).Format("15:04"); got != "09:00" { + t.Errorf("the clock moved to %s on a refused step, it must stay at 09:00", got) + } +} diff --git a/cmd/mavend/smarthome.go b/cmd/mavend/smarthome.go new file mode 100644 index 0000000..03946b2 --- /dev/null +++ b/cmd/mavend/smarthome.go @@ -0,0 +1,248 @@ +package main + +import ( + "context" + "fmt" + "log" + "strings" + "time" + + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/smarthome" + "github.com/kami/maven/internal/store" +) + +// homeWiring — the Home Assistant client, when the `smarthome` block is present +// AND enabled. nil ⇒ the house is not wired, nothing was proposed, and an +// allowlist row that happens to look like a house row refuses to run. +// +// It lives on the voice wiring for the same reason MCP does: a house control IS +// an act. It goes through tool.Executor, the enabled allowlist and the confirm +// turn, all of which only exist on the voice/chat path. +type homeWiring struct { + client *smarthome.Client + st *store.Store + refresh time.Duration +} + +// wireSmartHome builds the client and proposes what it found. It never fails +// the daemon: an instance that is down at boot is logged and retried, because +// Maven starting is not contingent on someone else's process. +func wireSmartHome(cfg *config.Config, st *store.Store) *homeWiring { + hc, ok := cfg.SmartHomeClient() + if !ok || st == nil { + return nil + } + if err := smarthome.Validate(hc); err != nil { + // config.validate already ran this, so reaching here is a programming + // error rather than a config one. Still not fatal: the house off is a + // working Maven. + log.Printf("smarthome: not wired: %v", err) + return nil + } + w := &homeWiring{ + client: smarthome.NewClient(hc), + st: st, + refresh: time.Duration(cfg.SmartHome.Refresh), + } + // No first propose here. This runs inside wireVoice, inside run, before the + // IPC socket is serving, and on the locked path inside the passkey unlock + // handler. A Home Assistant box that is powered off but still on a routed + // subnet black-holes the connection rather than refusing it, so a + // synchronous enumeration held the daemon's start for the per-call timeout. + // run does the first propose off the ticker instead. + return w +} + +// caller is the tool.HomeCaller seam. +func (w *homeWiring) caller() *smarthome.Client { + if w == nil { + return nil + } + return w.client +} + +// propose writes a 'proposed' allowlist row for every controllable device. It +// does NOT enable anything: a reachable house is a place Maven may look, not a +// set of switches she may flip. Kami enables what he wants on /tools, behind +// step-up, which is the same gate a shell tool goes through. +// +// Sensors are read but never proposed — there is nothing to call on them. +func (w *homeWiring) propose(ctx context.Context) { + if w == nil { + return + } + ents, err := w.client.States(ctx) + if err != nil { + log.Printf("smarthome: read states: %v", err) + return + } + now := time.Now() + fresh, devices := 0, 0 + for _, e := range ents { + svcs := smarthome.Services(e.Domain) + if len(svcs) == 0 { + continue + } + devices++ + for _, s := range svcs { + name := smarthome.LocalName(e.ID, s.Verb) + provenance := "дом: " + s.Name + " → " + e.Name + " (" + e.ID + ")" + ok, err := w.st.ProposeSmartHomeTool(ctx, name, smarthome.Scope(e.Domain), + smarthome.Cmd(e.ID, s.Name), provenance, now) + if err != nil { + log.Printf("smarthome: propose %s: %v", name, err) + continue + } + if ok { + fresh++ + } + } + } + log.Printf("smarthome: %d entities, %d controllable", len(ents), devices) + if fresh > 0 { + log.Printf("smarthome: %d new device proposal(s) waiting on /tools", fresh) + } +} + +// run re-enumerates the house and picks up devices that appeared, until ctx is +// canceled. +func (w *homeWiring) run(ctx context.Context) { + if w == nil { + return + } + iv := w.refresh + if iv <= 0 { + iv = config.DefaultSmartHomeRefresh + } + t := time.NewTicker(iv) + defer t.Stop() + // The first enumeration, off the daemon's start path. wireSmartHome used to + // do it synchronously and a dead house delayed the socket coming up. + w.propose(ctx) + for { + select { + case <-ctx.Done(): + return + case <-t.C: + w.propose(ctx) + } + } +} + +// homeSummary answers "что дома?" — a read of the current entity states, one +// short line. Read-only: it can never call a service, so it needs no confirm +// and no allowlist row. +func (w *homeWiring) homeSummary(ctx context.Context) (string, bool) { + if w == nil { + return "", false + } + ents, err := w.client.States(ctx) + if err != nil { + log.Printf("smarthome: summary: %v", err) + return "не смогла достучаться до дома.", true + } + if len(ents) == 0 { + return "дом ничего не отдаёт.", true + } + var on []string + var sensors []string + dark := 0 + for _, e := range ents { + switch { + case e.Domain == "sensor" || e.Domain == "binary_sensor": + if e.State == "" || e.State == "unavailable" { + dark++ + continue + } + if len(sensors) < 3 { + sensors = append(sensors, e.Name+" "+e.State+e.Unit) + } + case e.State == "unavailable" || e.State == "unknown" || e.State == "": + // A lamp that is not reachable is not a lamp that is off. Counting + // it as neither used to make "всё выключено" and "one device is + // unreachable" read identically. + dark++ + case e.State == "on" || e.State == "open" || e.State == "unlocked": + on = append(on, e.Name) + } + } + var parts []string + switch { + case len(on) > 0: + shown, rest := on, 0 + if len(shown) > 5 { + rest = len(shown) - 5 + shown = shown[:5] + } + // Silent truncation on a status read is the same failure as the cap + // one layer up: she has to say the list is not the whole list. + line := "включено: " + strings.Join(shown, ", ") + if rest > 0 { + line += fmt.Sprintf(" и ещё %d", rest) + } + parts = append(parts, line) + case dark > 0 && len(sensors) == 0: + // Nothing is on and everything she can see is unreachable. "всё + // выключено" would be a claim about the house she cannot make. + return fmt.Sprintf("дом молчит: %d %s не отвечают.", dark, hostWord(dark)), true + default: + parts = append(parts, "всё выключено") + } + if len(sensors) > 0 { + parts = append(parts, strings.Join(sensors, ", ")) + } + if dark > 0 { + parts = append(parts, fmt.Sprintf("%d %s не отвечают", dark, hostWord(dark))) + } + return strings.Join(parts, "; ") + ".", true +} + +// isHomeQuery recognises a question about the house, narrowly. "дома" on its +// own is not enough — "я дома" is a fact, not a question — so it takes a house +// marker AND an ask AND either a device word or the word "включ…". Weather +// wording bails out first: "какая температура на улице?" belongs to the weather +// source, and both questions contain "температура". +func isHomeQuery(u string) bool { + s := strings.ToLower(strings.TrimSpace(u)) + if s == "" { + return false + } + for _, w := range []string{"погод", "на улице", "прогноз"} { + if strings.Contains(s, w) { + return false + } + } + for _, phrase := range []string{"что включено", "что выключено", "умный дом", "что в доме включено"} { + if strings.Contains(s, phrase) { + return true + } + } + house := homeWord(s, "дома") || strings.Contains(s, "в доме") || strings.Contains(s, "в квартире") + if !house { + return false + } + ask := strings.Contains(s, "?") || homeWord(s, "что") || homeWord(s, "какая") || + homeWord(s, "какой") || homeWord(s, "сколько") + if !ask { + return false + } + for _, w := range []string{"свет", "лампа", "лампы", "розетк", "датчик", "температур", "включ", "выключ"} { + if strings.Contains(s, w) { + return true + } + } + return false +} + +// homeWord — whole-token membership, so "дома" does not fire on "домашний". +// Punctuation is trimmed off each token because a spoken question arrives with +// a question mark glued to the last word. +func homeWord(s, w string) bool { + for _, tok := range strings.Fields(s) { + if strings.Trim(tok, ".,!?;:") == w { + return true + } + } + return false +} diff --git a/cmd/mavend/smarthome_test.go b/cmd/mavend/smarthome_test.go new file mode 100644 index 0000000..d7904ae --- /dev/null +++ b/cmd/mavend/smarthome_test.go @@ -0,0 +1,276 @@ +package main + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/kami/maven/internal/config" +) + +const haStatesFixture = `[ + {"entity_id":"light.living_room","state":"on","attributes":{"friendly_name":"Гостиная"}}, + {"entity_id":"switch.kettle","state":"off","attributes":{"friendly_name":"Чайник"}}, + {"entity_id":"sensor.bedroom_temp","state":"22.5","attributes":{"friendly_name":"Спальня","unit_of_measurement":"°C"}} +]` + +func TestWireSmartHomeOffUnlessEnabled(t *testing.T) { + st := newTestStore(t) + for name, cfg := range map[string]*config.Config{ + "no block": {}, + "written but dark": {SmartHome: &config.SmartHomeConfig{ + URL: "http://ha.lan:8123", Token: "t", + }}, + } { + t.Run(name, func(t *testing.T) { + if w := wireSmartHome(cfg, st); w != nil { + t.Fatal("the house must be off unless the block is enabled") + } + }) + } + // nil wiring must be safe everywhere it is reachable. + var w *homeWiring + w.propose(context.Background()) + w.run(context.Background()) + if w.caller() != nil { + t.Fatal("a nil wiring must have no caller") + } + if _, ok := w.homeSummary(context.Background()); ok { + t.Fatal("a nil wiring must not claim a query") + } +} + +// An unreachable instance must not stop the daemon and must propose nothing. +func TestWireSmartHomeUnreachableIsNotFatal(t *testing.T) { + st := newTestStore(t) + w := wireSmartHome(&config.Config{SmartHome: &config.SmartHomeConfig{ + // Port 1 on loopback: nothing listens, and it fails fast. + URL: "http://127.0.0.1:1", Token: "t", Enabled: true, + }}, st) + if w == nil { + t.Fatal("a configured house should still wire") + } + tools, err := st.ListTools(context.Background(), "") + if err != nil { + t.Fatal(err) + } + if len(tools) != 0 { + t.Fatalf("an instance that never answered must propose nothing, got %+v", tools) + } +} + +// Discovery proposes one row per controllable service, always destructive, +// always 'proposed'. A sensor gets no row: there is nothing to call on it. +func TestProposeOnlyProposesControllableDevices(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(haStatesFixture)) + })) + defer srv.Close() + + st := newTestStore(t) + w := wireSmartHome(&config.Config{SmartHome: &config.SmartHomeConfig{ + URL: srv.URL, Token: "t", Enabled: true, + }}, st) + if w == nil { + t.Fatal("wireSmartHome returned nil for an enabled, reachable house") + } + // Wiring alone must not have touched the house: enumeration happens off + // the ticker, not on the daemon's start path. + if pre, err := st.ListTools(context.Background(), ""); err != nil || len(pre) != 0 { + t.Fatalf("wireSmartHome enumerated the house synchronously: %+v (%v)", pre, err) + } + w.propose(context.Background()) + + tools, err := st.ListTools(context.Background(), "") + if err != nil { + t.Fatal(err) + } + got := map[string]bool{} + for _, tl := range tools { + got[tl.Name] = true + if tl.Status != "proposed" { + t.Errorf("%s status = %q: discovery must never enable", tl.Name, tl.Status) + } + if !tl.Destructive { + t.Errorf("%s is not destructive: every house control needs the confirm turn", tl.Name) + } + if len(tl.Cmd) == 0 || tl.Cmd[0] != "smarthome" { + t.Errorf("%s cmd = %v", tl.Name, tl.Cmd) + } + } + for _, want := range []string{ + "home_light_living_room_on", "home_light_living_room_off", + "home_switch_kettle_on", "home_switch_kettle_off", + } { + if !got[want] { + t.Errorf("missing proposal %q (have %v)", want, got) + } + } + if len(tools) != 4 { + t.Fatalf("got %d rows, want 4 — the sensor must not be proposed: %+v", len(tools), tools) + } + + // A second pass must be idempotent: re-discovery duplicates nothing and + // never rewrites a row Kami already enabled. + if err := st.EnableTool(context.Background(), "home_switch_kettle_on", + []string{"smarthome", "switch.kettle", "turn_on"}, true, "smarthome:switch", time.Now()); err != nil { + t.Fatal(err) + } + w.propose(context.Background()) + again, err := st.ListTools(context.Background(), "") + if err != nil { + t.Fatal(err) + } + if len(again) != 4 { + t.Fatalf("re-discovery duplicated rows: %d", len(again)) + } + for _, tl := range again { + if tl.Name == "home_switch_kettle_on" && tl.Status != "enabled" { + t.Errorf("re-discovery un-enabled a device he had enabled: %q", tl.Status) + } + } +} + +func TestHomeSummaryReadsState(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(haStatesFixture)) + })) + defer srv.Close() + + w := wireSmartHome(&config.Config{SmartHome: &config.SmartHomeConfig{ + URL: srv.URL, Token: "t", Enabled: true, + }}, newTestStore(t)) + out, ok := w.homeSummary(context.Background()) + if !ok { + t.Fatal("summary did not claim the turn") + } + if !strings.Contains(out, "Гостиная") { + t.Errorf("the lamp that is on should be named: %q", out) + } + if strings.Contains(out, "Чайник") { + t.Errorf("a device that is off should not be listed as on: %q", out) + } + if !strings.Contains(out, "22.5") { + t.Errorf("the sensor reading should be there: %q", out) + } + // Persona: no masculine self-reference, no "вы", no pet names. + for _, bad := range []string{"рад ", "готов ", "вы ", "ваш", "милый", "дорогой"} { + if strings.Contains(strings.ToLower(out), bad) { + t.Errorf("persona violation %q in %q", bad, out) + } + } +} + +func TestIsHomeQuery(t *testing.T) { + yes := []string{ + "что включено дома?", + "что выключено", + "какой свет горит дома", + "свет в доме включен?", + "какая температура в квартире?", + "покажи умный дом", + } + no := []string{ + "", + "я дома", + "буду дома в семь", + "какая погода дома", // weather wording wins + "какая температура на улице?", + "домашние дела", // "дома" must not fire on "домашние" + "что мне нужно сделать?", + "напомни выключить чайник в семь", // a reminder, not a house read + } + for _, u := range yes { + if !isHomeQuery(u) { + t.Errorf("isHomeQuery(%q) = false, want true", u) + } + } + for _, u := range no { + if isHomeQuery(u) { + t.Errorf("isHomeQuery(%q) = true, want false", u) + } + } +} + +// A house that black-holes the connection must not hold the daemon's start. +// wireSmartHome used to enumerate synchronously with a 30s context, inside +// wireVoice, inside run, before the IPC socket was serving — and on the locked +// path, inside the passkey unlock handler. +func TestWireSmartHomeDoesNotBlockOnTheHouse(t *testing.T) { + // A handler that never answers: the client's own timeout is the only way + // out, and it is ten seconds. + block := make(chan struct{}) + defer close(block) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-block + })) + defer srv.Close() + + done := make(chan *homeWiring, 1) + go func() { + done <- wireSmartHome(&config.Config{SmartHome: &config.SmartHomeConfig{ + URL: srv.URL, Token: "t", Enabled: true, + }}, newTestStore(t)) + }() + select { + case w := <-done: + if w == nil { + t.Fatal("a configured house should still wire") + } + case <-time.After(2 * time.Second): + t.Fatal("wireSmartHome waited on the house") + } +} + +// A lamp that is unreachable is not a lamp that is off, and a list she cut +// short has to say so. Both used to read as plain statements about the house. +func TestHomeSummaryDoesNotCallUnreachableDevicesOff(t *testing.T) { + const fixture = `[ + {"entity_id":"light.a","state":"unavailable","attributes":{"friendly_name":"Прихожая"}}, + {"entity_id":"light.b","state":"unavailable","attributes":{"friendly_name":"Кухня"}} +]` + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(fixture)) + })) + defer srv.Close() + w := wireSmartHome(&config.Config{SmartHome: &config.SmartHomeConfig{ + URL: srv.URL, Token: "t", Enabled: true, + }}, newTestStore(t)) + out, ok := w.homeSummary(context.Background()) + if !ok { + t.Fatal("summary did not claim the turn") + } + if strings.Contains(out, "всё выключено") { + t.Errorf("two unreachable lamps were reported as off: %q", out) + } + if !strings.Contains(out, "не отвечают") { + t.Errorf("the unreachable devices are not mentioned: %q", out) + } +} + +func TestHomeSummarySaysWhenTheListIsCutShort(t *testing.T) { + var b strings.Builder + b.WriteString("[") + for i := 0; i < 8; i++ { + if i > 0 { + b.WriteString(",") + } + fmt.Fprintf(&b, `{"entity_id":"light.l%d","state":"on","attributes":{"friendly_name":"лампа%d"}}`, i, i) + } + b.WriteString("]") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(b.String())) + })) + defer srv.Close() + w := wireSmartHome(&config.Config{SmartHome: &config.SmartHomeConfig{ + URL: srv.URL, Token: "t", Enabled: true, + }}, newTestStore(t)) + out, _ := w.homeSummary(context.Background()) + if !strings.Contains(out, "и ещё 3") { + t.Errorf("eight lamps on, five named, and nothing said about the rest: %q", out) + } +} diff --git a/cmd/mavend/speaker.go b/cmd/mavend/speaker.go new file mode 100644 index 0000000..f717400 --- /dev/null +++ b/cmd/mavend/speaker.go @@ -0,0 +1,162 @@ +// mavend/speaker.go — core's half of voice identification (Vikunja #255, +// docs/plans/10-speaker-recognition.md). +// +// # What is actually wired here, and what is not +// +// Nothing is, on this box. There is no speaker-embedding model on disk — no +// ECAPA, no x-vector, no titanet, no wespeaker, nothing in /mnt/hdd1/llms but +// text ggufs. Until one is downloaded, newSpeakerEmbedder returns nil. +// +// Without an embedder the capability has no runnable half. This comment used to +// say enrolment was real and only recognition was blocked, and the startup log +// said the same. Both were wrong: Recognizer.Enroll embeds every sample before +// it stores anything, so with no model it fails on the first sample and nothing +// is ever stored, which leaves List empty forever and Forget with nothing to +// delete. So the gate is cfg.Speaker.Recognizes() — enabled AND a model path — +// and a box without one gets no speaker methods, not three no-ops. +// +// This is deliberately not papered over with a hand-rolled MFCC floor. A +// biometric that is confidently wrong writes false claims about named people +// into his memory, and that is worse than a capability that is honestly absent. +// +// # Off unless configured +// +// No speaker block, or one without enabled, ⇒ the three methods do not exist and +// answer ErrUnknownMethod. On an unconfigured box there is no wire path that +// takes a voiceprint at all. +// +// # The refused design step +// +// The plan asks for unknown speakers to be enrolled on first interaction. That +// is refused in internal/speaker/enroll.go and there is no handler for it here: +// no request shape in the protocol enrols whoever just spoke. Taking a biometric +// of a guest who walked past the microphone is not something this daemon does. +package main + +import ( + "context" + "errors" + "log" + + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/speaker" + "github.com/kami/maven/internal/store" +) + +// speakerWiring holds the recognizer behind the three IPC handlers. +type speakerWiring struct { + rec *speaker.Recognizer +} + +// newSpeakerEmbedder loads the speaker-embedding model named by the config. +// +// It always returns nil today. The seam exists so that wiring a real model is a +// change to this one function and nothing else: give it a loader, and Identify +// starts working with no change to the store, the protocol, the auth table or +// the handlers. See the plan document for what to download. +func newSpeakerEmbedder(cfg *config.SpeakerConfig) speaker.Embedder { + _ = cfg + return nil +} + +// newSpeakerWiring builds the recognizer, or nil when the capability is off. +func newSpeakerWiring(st *store.Store, cfg *config.Config) *speakerWiring { + if cfg == nil || cfg.Speaker == nil { + return nil + } + if !cfg.Speaker.Recognizes() { + // Recognizes() was written as the gate and documented as one, and then + // never called. "enabled": true with no model_path used to wire all + // three methods and log "enrolment on", which is the one config shape + // where the operator most needs to be told otherwise. + if cfg.Speaker.Enabled { + log.Print("speaker: enabled but no model_path, so there is nothing to embed with; " + + "enrol, list and forget would all be no-ops, staying off " + + "(see docs/plans/10-speaker-recognition.md)") + } + return nil + } + if st == nil { + log.Print("speaker: enabled but there is no store to keep profiles in; staying off") + return nil + } + rec, err := speaker.New(newSpeakerEmbedder(cfg.Speaker), st.VectorMemory(), speaker.Config{ + Threshold: cfg.Speaker.Threshold, + MinSeconds: cfg.Speaker.MinSeconds, + }) + if err != nil { + log.Printf("speaker: %v; staying off", err) + return nil + } + if rec.Enabled() { + log.Printf("speaker: recognition on, threshold %.2f", rec.Threshold()) + } else { + log.Printf("speaker: model_path %q is configured but no embedding backend is built yet, "+ + "so enrol, list and forget are all no-ops (Vikunja #255)", cfg.Speaker.ModelPath) + } + return &speakerWiring{rec: rec} +} + +func (w *speakerWiring) enroll(ctx context.Context, req ipc.EnrollSpeakerReq) (ipc.EnrollSpeakerResp, error) { + p, err := w.rec.Enroll(ctx, req.ID, req.Name, req.Samples) + if err != nil { + return ipc.EnrollSpeakerResp{}, speakerErr(err) + } + return ipc.EnrollSpeakerResp{Speaker: toWireSpeaker(p)}, nil +} + +func (w *speakerWiring) list(ctx context.Context) (ipc.ListSpeakersResp, error) { + ps, err := w.rec.List(ctx) + if err != nil { + return ipc.ListSpeakersResp{}, speakerErr(err) + } + out := make([]ipc.Speaker, 0, len(ps)) + for _, p := range ps { + out = append(out, toWireSpeaker(p)) + } + return ipc.ListSpeakersResp{Speakers: out, Enabled: w.rec.Enabled()}, nil +} + +func (w *speakerWiring) forget(ctx context.Context, req ipc.ForgetSpeakerReq) error { + return speakerErr(w.rec.Forget(ctx, req.ID)) +} + +// toWireSpeaker drops the voiceprint. A listing says who is enrolled; it does +// not hand the biometric back out over the socket. +func toWireSpeaker(p speaker.Profile) ipc.Speaker { + return ipc.Speaker{ID: p.ID, Name: p.Name, Enrolled: p.Enrolled, Samples: p.Samples, Damaged: p.Damaged} +} + +// speakerErr maps the package sentinels onto the wire vocabulary so a surface +// can tell "you asked wrong" from "core broke". +func speakerErr(err error) error { + switch { + case err == nil: + return nil + case errors.Is(err, speaker.ErrDisabled): + // Not a core failure. The capability is present on the wire but has no + // embedding model behind it, which is the same thing an unconfigured + // method says, so say it the same way. + return ipc.ErrUnknownMethod + case errors.Is(err, speaker.ErrNotFound): + return ipc.ErrNoFact + case errors.Is(err, speaker.ErrBadID), + errors.Is(err, speaker.ErrBadFormat), + errors.Is(err, speaker.ErrTooShort): + return errors.Join(ipc.ErrBadParams, err) + default: + return err + } +} + +// wireSpeaker attaches the three handlers when the capability is configured. +func wireSpeaker(srv *ipc.Server, st *store.Store, cfg *config.Config) { + w := newSpeakerWiring(st, cfg) + if w == nil { + return + } + srv.EnrollSpeakerFn = w.enroll + srv.ListSpeakersFn = w.list + srv.ForgetSpeakerFn = w.forget +} diff --git a/cmd/mavend/speaker_wiring_test.go b/cmd/mavend/speaker_wiring_test.go new file mode 100644 index 0000000..96fd8bd --- /dev/null +++ b/cmd/mavend/speaker_wiring_test.go @@ -0,0 +1,50 @@ +package main + +import ( + "errors" + "testing" + + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/speaker" +) + +// "enabled": true with no model_path used to wire all three methods and log +// "enrolment on". Nothing behind them works without an embedder, so the +// capability stays off and the socket answers "no such method". +func TestSpeakerStaysOffWithoutAModelPath(t *testing.T) { + srv := &ipc.Server{} + cfg := &config.Config{Speaker: &config.SpeakerConfig{Enabled: true}} + + wireSpeaker(srv, nil, cfg) + + if srv.EnrollSpeakerFn != nil || srv.ListSpeakersFn != nil || srv.ForgetSpeakerFn != nil { + t.Error("speaker methods were wired with nothing to embed with") + } +} + +// The gate is Recognizes(), so a disabled block with a model path is off too. +func TestSpeakerStaysOffWhenDisabled(t *testing.T) { + srv := &ipc.Server{} + cfg := &config.Config{Speaker: &config.SpeakerConfig{ModelPath: "/nope/ecapa.onnx"}} + + wireSpeaker(srv, nil, cfg) + + if srv.EnrollSpeakerFn != nil { + t.Error("speaker methods were wired for a disabled block") + } +} + +// ErrDisabled is "this capability is off", not "core broke". It used to fall +// through speakerErr's default and reach the surface as an opaque failure. +func TestSpeakerErrMapsDisabledToUnknownMethod(t *testing.T) { + if got := speakerErr(speaker.ErrDisabled); !errors.Is(got, ipc.ErrUnknownMethod) { + t.Errorf("speakerErr(ErrDisabled) = %v, want ErrUnknownMethod", got) + } + if got := speakerErr(speaker.ErrNotFound); !errors.Is(got, ipc.ErrNoFact) { + t.Errorf("speakerErr(ErrNotFound) = %v, want ErrNoFact", got) + } + if got := speakerErr(nil); got != nil { + t.Errorf("speakerErr(nil) = %v", got) + } +} diff --git a/cmd/mavend/strutil.go b/cmd/mavend/strutil.go new file mode 100644 index 0000000..714a152 --- /dev/null +++ b/cmd/mavend/strutil.go @@ -0,0 +1,85 @@ +// Package main — strutil.go holds small, receiver-free string utilities used +// across the voice reply paths: trimming a wake token, pulling out the first +// word or first line, and a minimal JSON string encoder for the one payload +// shape that needs it. Extend this file rather than voice.go for anything in +// that shape. +package main + +import ( + "fmt" + "strings" + + "github.com/kami/maven/internal/router" +) + +// stripWake removes a leading wake token (any script the STT phonetically +// transcribes "Maven" as) so the verb is the first word. +func stripWake(u string) string { + stripped, had := router.StripWakeToken(u) + if !had { + return strings.TrimSpace(u) + } + return stripped +} + +// firstWord returns the first whitespace-delimited token (lowercased) — the +// proposed tool's name. +func firstWord(s string) string { + f := strings.Fields(s) + if len(f) == 0 { + return "" + } + return strings.ToLower(f[0]) +} + +// firstLine — the first non-empty line of a tool's output, for a short spoken +// reply (the full output goes to the log, not the TTS). Trimmed to keep the +// utterance sane if a command dumps a wall of text. +func firstLine(s string) string { + for _, line := range strings.Split(s, "\n") { + line = strings.TrimSpace(line) + if line != "" { + if len(line) > 200 { + line = line[:200] + } + return line + } + } + return "" +} + +// jsonString — a one-line JSON string encoder without dragging encoding/json +// into the top of this file. Used to wrap a reminder payload's text field; +// the router's reminder Slots are already absolute (DateTimeParser resolved +// relative→absolute), the payload shape is conventional {"text":...}. +func jsonString(s string) string { + // minimal JSON string escape — quotes + backslash + control chars. + // adequate for the reminder payload's text field; not a general JSON + // encoder. The chroma / RAG modules (when they land) use a real json + // encoder for richer payloads. Keep it inline here so the import + // direction stays narrow. + var b []byte + b = append(b, '"') + for _, r := range s { + switch r { + case '"': + b = append(b, '\\', '"') + case '\\': + b = append(b, '\\', '\\') + case '\n': + b = append(b, '\\', 'n') + case '\r': + b = append(b, '\\', 'r') + case '\t': + b = append(b, '\\', 't') + default: + if r < 0x20 { + b = append(b, []byte(fmt.Sprintf("\\u%04x", r))...) + } else { + b = append(b, []byte(string(r))...) + } + } + } + b = append(b, '"') + return string(b) +} diff --git a/cmd/mavend/testdata/scenarios/act_degraded.json b/cmd/mavend/testdata/scenarios/act_degraded.json new file mode 100644 index 0000000..e500b45 --- /dev/null +++ b/cmd/mavend/testdata/scenarios/act_degraded.json @@ -0,0 +1,58 @@ +{ + "schema_version": 1, + "name": "act_degraded", + "description": "The act path against a Praxis that goes down and comes back. This is the case the harness promised and did not have: the other two scenarios never produce an act, so the ecosystem fakes saw zero requests and the fault lever was inert. Here a scripted act reaches an enabled allowlist row, the row is a Praxis verb, and the same utterance runs healthy, then at 503, then healthy again. The degraded turn must say she cannot reach it and must not send anything at him off the back of it.", + "start": "2026-08-01T09:00:00+03:00", + "praxis_attention": "[{\"id\":\"item_1\",\"title\":\"medicine not taken\",\"importance\":3.0,\"rule\":\"morning_medicine\"}]", + "tools": [{ "name": "list_attention" }], + "script": [ + { + "match": "требует внимания", + "route": "[{\"intent\":\"act\",\"verb\":\"list_attention\"}]" + }, + { + "match": "", + "route": "[{\"intent\":\"chat\",\"text\":\"привет\"}]", + "reply": "{\"response\":\"Я рада тебя слышать.\",\"mood\":\"happy\"}" + } + ], + "steps": [ + { + "at": "09:00", + "note": "a healthy act reaches Praxis and speaks what it found", + "say": "что требует внимания?", + "expect_reply_contains": ["medicine not taken"], + "expect_called": ["/api/v1/tools/attention"], + "expect_no_send": true + }, + { + "at": "09:05", + "note": "the ecosystem goes down", + "fault": 503 + }, + { + "at": "09:10", + "note": "the same act against a 503. She says she cannot reach it. She does not invent an answer and she does not push anything at him.", + "say": "что требует внимания?", + "expect_reply_contains": ["не могу сейчас узнать"], + "expect_reply_lacks": ["medicine not taken"], + "expect_no_send": true + }, + { + "at": "09:15", + "note": "a tick while the ecosystem is down touches nothing out there — the proactive loop has no business calling Praxis", + "tick": true, + "expect_not_called": ["/api/v1"], + "expect_no_send": true, + "expect_no_events": true + }, + { + "at": "09:20", + "note": "recovery: the same act works again, so the degraded turn left no sticky state", + "clear_fault": true, + "say": "что требует внимания?", + "expect_reply_contains": ["medicine not taken"], + "expect_no_send": true + } + ] +} diff --git a/cmd/mavend/testdata/scenarios/evening_degraded.json b/cmd/mavend/testdata/scenarios/evening_degraded.json new file mode 100644 index 0000000..5d55974 --- /dev/null +++ b/cmd/mavend/testdata/scenarios/evening_degraded.json @@ -0,0 +1,67 @@ +{ + "schema_version": 1, + "name": "evening_degraded", + "description": "The tier-2 pipeline case #288 deferred here, plus degraded mode. A golden WAV goes in at the microphone end and comes out as a written fact, and then the ecosystem starts answering 503 and the proactive loop has to stay quiet instead of falling over. The audio step asserts the PIPELINE — mic to STT seam to router to store to TTS — not whisper's accuracy; cmd/mavsttd/golden_test.go owns accuracy.", + "start": "2026-08-01T21:00:00+03:00", + "praxis_attention": "[{\"id\":\"item_1\",\"title\":\"medicine not taken\",\"importance\":3.0,\"rule\":\"evening_medicine\"}]", + "script": [ + { + "match": "выпил воды", + "route": "[{\"intent\":\"fact\",\"key\":\"water\",\"value\":\"выпил\"}]" + }, + { + "match": "записала факт: water", + "reply": "{\"response\":\"Записала, что ты выпил воды.\",\"mood\":\"neutral\"}" + }, + { + "match": "", + "route": "[{\"intent\":\"chat\",\"text\":\"привет\"}]", + "reply": "{\"response\":\"Я рада тебя слышать.\",\"mood\":\"happy\"}" + } + ], + "steps": [ + { + "at": "21:00", + "note": "he speaks. The whole voice path runs: push-to-talk, the STT seam parked with the golden transcript, the real router, the real store write, the phrasing contract.", + "audio": "ru_fact", + "expect_reply_contains": ["записала"], + "expect_reply_lacks": ["записал ", "записал,", "записал.", "милый", "ваш"], + "expect_events": ["water"] + }, + { + "at": "21:05", + "note": "a healthy tick with him just having spoken stays silent", + "tick": true, + "expect_no_send": true + }, + { + "at": "21:10", + "note": "the ecosystem goes down", + "fault": 503 + }, + { + "at": "21:15", + "note": "a tick against a dead ecosystem must degrade, not send half a thought", + "tick": true, + "expect_no_send": true, + "expect_no_events": true + }, + { + "at": "21:20", + "note": "intake keeps working while the ecosystem is down — a write does not depend on it", + "arrive": { + "source": "rss:tech", + "note": { "text": "Патч 6.19.1 [tech]\nисправления\nhttps://example.org/b" } + }, + "expect_events": ["rss:tech"], + "expect_no_send": true + }, + { + "at": "21:25", + "note": "recovery", + "clear_fault": true, + "tick": true, + "expect_no_send": true + } + ] +} diff --git a/cmd/mavend/testdata/scenarios/morning_missed.json b/cmd/mavend/testdata/scenarios/morning_missed.json new file mode 100644 index 0000000..f398039 --- /dev/null +++ b/cmd/mavend/testdata/scenarios/morning_missed.json @@ -0,0 +1,97 @@ +{ + "schema_version": 1, + "name": "morning_missed", + "description": "The scenario from Vikunja #284's description, replayed. He appears at 08:30, things arrive through the morning while he is at the desk, and at 08:50 he asks what he missed. The assertions are as much about what did NOT happen — nothing was sent at him unprompted — as about what she said.", + "start": "2026-08-01T08:30:00+03:00", + "praxis_attention": "[{\"id\":\"item_1\",\"title\":\"medicine not taken\",\"importance\":3.0,\"rule\":\"morning_medicine\"}]", + "script": [ + { + "match": "выпил воды", + "route": "[{\"intent\":\"fact\",\"key\":\"water\",\"value\":\"выпил\"}]" + }, + { + "match": "записала факт: water", + "reply": "{\"response\":\"Записала, что ты выпил воды.\",\"mood\":\"neutral\"}" + }, + { + "match": "что я пропустил", + "route": "[{\"intent\":\"query\",\"text\":\"что я пропустил\"}]" + }, + { + "match": "", + "route": "[{\"intent\":\"chat\",\"text\":\"привет\"}]", + "reply": "{\"response\":\"Я рада тебя слышать.\",\"mood\":\"happy\"}" + } + ], + "steps": [ + { + "at": "08:30", + "note": "he appears at the desk", + "signal": { "key": "desk_active", "value": "true", "source": "infer:hyprland" }, + "expect_events": ["infer:hyprland"], + "expect_no_send": true + }, + { + "at": "08:32", + "note": "a feed item arrives, published half an hour ago", + "arrive": { + "source": "rss:tech", + "as_of": "08:02", + "note": { "text": "Вышло ядро 6.19 [tech]\nкраткое содержание\nhttps://example.org/a" } + }, + "expect_events": ["rss:tech"], + "expect_no_send": true + }, + { + "at": "08:35", + "note": "the mail reader extracts a candidate — a candidate is never spoken", + "arrive": { + "source": "email:inbox", + "task": { "text": "продлить домен", "evidence": "Домен истекает через 7 дней" } + }, + "expect_events": ["email:inbox", "продлить домен"], + "expect_no_send": true + }, + { + "at": "08:40", + "note": "the work calendar signal — a relayed notification, at the ambient path's own 0.6 rather than an observation she made herself. That is the branch factPriority takes, so the journal must file it low.", + "arrive": { + "source": "ambient:notif", + "fact": { + "key": "calendar_event_20260801_планёрка", + "value": "10:00-11:00 планёрка", + "confidence": 0.6 + } + }, + "expect_events": ["планёрка", "ambient:notif/fact pri=low"], + "expect_no_send": true + }, + { + "at": "08:45", + "note": "a tick with him present and nothing wrong must stay silent", + "tick": true, + "expect_no_send": true + }, + { + "at": "08:50", + "note": "he asks. The query path answers from local recall only: nothing stored clears the score gate, so she refuses rather than inventing a morning summary, and the replier is never reached. That refusal is the no-hallucination floor and this step pins it. Note what the persona check here is and is not: the reply is a constant in the Go source, so expect_reply_lacks pins that constant, not anything the model wrote. The step below is the one that reads model output.", + "say": "что я пропустил?", + "expect_reply_contains": ["не знаю"], + "expect_reply_lacks": ["рад ", "милый", "ваш"] + }, + { + "at": "08:55", + "note": "stating a fact writes it and says so, in the feminine. This reply comes back through the replier from the scripted model, so the persona check is against generated text rather than a constant. The masculine forms are listed with their following character — \"записал \" and \"записал,\" — because \"записала\" contains \"записал\", and the earlier check on the comma alone passed on \"записал что ты выпил воды\".", + "say": "я выпил воды", + "expect_reply_contains": ["записала"], + "expect_reply_lacks": ["записал ", "записал,", "записал.", "милый"], + "expect_events": ["water"] + }, + { + "at": "09:00", + "note": "a second tick, still nothing unprompted", + "tick": true, + "expect_no_send": true + } + ] +} diff --git a/cmd/mavend/tick.go b/cmd/mavend/tick.go index 39d1c83..922e8a0 100644 --- a/cmd/mavend/tick.go +++ b/cmd/mavend/tick.go @@ -19,11 +19,13 @@ import ( "sync" "time" + "github.com/kami/maven/internal/calendar" "github.com/kami/maven/internal/config" "github.com/kami/maven/internal/delivery" "github.com/kami/maven/internal/ipc" "github.com/kami/maven/internal/loop" "github.com/kami/maven/internal/morning" + "github.com/kami/maven/internal/pattern" "github.com/kami/maven/internal/phraser" "github.com/kami/maven/internal/routine" "github.com/kami/maven/internal/store" @@ -67,6 +69,14 @@ type tickLoop struct { morningRoutines []morning.Routine morningLast map[string]time.Time + // proposalCfg — announcement policy for routines the tick inferred itself. + // nil ⇒ detect silently, never announce (the default). lastProposalAt is + // the cooldown clock, in-memory on purpose: a restart is allowed to permit + // one more announcement, and a restart-per-day loop is a bigger problem + // than a duplicate proposal notice. + proposalCfg *config.PatternProposalConfig + lastProposalAt time.Time + // digestQ — in-memory queue of eligible nudges waiting for batch flush. // populated when digestCfg != nil && digestCfg.Enabled. digestQ []QueuedNudge @@ -92,6 +102,7 @@ func newTickLoop( digestCfg *config.DigestConfig, routines []routine.Routine, morningRoutines []morning.Routine, + proposalCfg *config.PatternProposalConfig, ) *tickLoop { return &tickLoop{ store: st, @@ -108,6 +119,7 @@ func newTickLoop( routineLast: make(map[string]time.Time), morningRoutines: morningRoutines, morningLast: make(map[string]time.Time), + proposalCfg: proposalCfg, lastPhrase: make(map[string]delivery.PhrasedNudge), } } @@ -180,6 +192,17 @@ func (t *tickLoop) tick(ctx context.Context, now time.Time) { // (with dedup) avoids re-queueing the same rule after a flush. t.maybeFlush(ctx, now, state) + // gate-suppressed digest (Vikunja #281): rules the restraint gate held + // back this tick (quiet hours / away / calendar-busy), not because they + // weren't due, but because it wasn't the moment. Some of those are worth + // resurfacing later instead of just being lost — loop.DigestEligible + // draws that line. This is a SEPARATE mechanism from the in-memory + // digestQ above: that one batches candidates the gate already ALLOWED to + // fire; this one durably holds candidates the gate BLOCKED. + t.enqueueSuppressedDigest(ctx, trace, state, now) + t.expireStaleDigest(ctx, now) + t.maybeDrainDigest(ctx, state, now) + // routines: operator-declared scheduled behaviors. fire the ones whose cron // crossed since last fire, delivered through the normal routing (voice when // present, away channels otherwise). bodies are literal operator text — not @@ -195,6 +218,14 @@ func (t *tickLoop) tick(ctx context.Context, now time.Time) { // nudge time. See internal/morning for the "why not four timers" rationale. t.fireMorningRoutines(ctx, now, state) + // pattern detection: scan every action+object pair with recorded events + // and propose a routine for any stable one not already decided (Vikunja + // #43). This used to only run as a side effect of the voice fact-write + // path, so a pattern already sitting in history went unnoticed until he + // happened to mention it again by voice. See patterns.go and + // detectPatterns below for how idempotence and dismissal are respected. + t.detectPatterns(ctx, now, state) + // reminders: gate-bypassing class. fired once, marked after a successful // delivery. a failed send leaves the reminder pending — the next tick // re-gathers and re-attempts. @@ -342,6 +373,252 @@ func (t *tickLoop) flushDigest(ctx context.Context, now time.Time, state loop.St t.digestQ = nil } +// detectPatterns runs the pattern detector proactively over every +// action+object pair that has ever produced an event, independent of +// whichever fact write (or channel) last touched it (Vikunja #43). This is +// what makes pattern inference actually proactive: it fires on the daemon's +// own schedule reading accumulated history, not only as a side effect of a +// live voice turn. +// +// Idempotence and noise are handled by the store, not here — this function +// is safe to call every tick: +// - Same pattern, tick after tick: detectAndPropose's LookupProposedRoutine +// check plus proposed_routines' UNIQUE(action, object) constraint (with +// CreateProposedRoutine's ON CONFLICT DO NOTHING) mean a pair that +// already has a row — in ANY status — produces no second row and no log +// spam beyond the one line at genuine creation. +// - A DISMISSED proposal must never come back. DismissProposedRoutine flips +// status in place; the row is never deleted. So the same Lookup check +// that stops a duplicate "proposed" also stops a "dismissed" one from +// resurrecting — there is nothing tick-specific to get right here beyond +// calling the same shared path the voice route already used. +// +// By default this only creates a row for the /routines page to show: it does +// not notify, ring, or speak. Detection is not the same act as disturbing him +// about it, and Maven is "not a nag, not autonomous" (CLAUDE.md). Announcing +// is opt-in through the pattern_proposals config block — see announceProposal +// for the restraints that apply even then. A proposal only starts producing +// recurring nudges once he accepts it (fireAcceptedRoutines). +func (t *tickLoop) detectPatterns(ctx context.Context, now time.Time, state loop.State) { + pairs, err := t.store.DistinctEventPairs(ctx) + if err != nil { + log.Printf("tick: distinct event pairs: %v", err) + return + } + announced := false + for _, p := range pairs { + r, _, err := detectAndPropose(ctx, t.store, p.Action, p.Object, now) + if err != nil { + log.Printf("tick: detect pattern %s/%s: %v", p.Action, p.Object, err) + continue + } + if r == nil { + continue // no stable pattern, or already proposed/accepted/dismissed + } + log.Printf("tick: proposed routine: %s/%s every %.1f days", r.Action, r.Object, r.IntervalDays) + // One announcement per tick at most, whatever the scan turned up. The + // rest are on /routines; they are not lost, they are just not shouted. + // Nor are they queued: the row now exists, so no later tick re-detects + // them and they are never announced. See announceProposal. + if announced { + continue + } + announced = t.announceProposal(ctx, r, now, state) + } +} + +// announceProposal offers a freshly inferred routine through the ordinary +// care-delivery path, if announcing is switched on at all. Returns true when +// something was actually sent. +// +// Everything here is restraint. The feature is off unless configured; when on +// it is sev1 (the lowest severity, so quiet hours, away presence and snooze +// all suppress it via loop.Gate exactly like a care nudge); it is spaced by +// proposalCfg.Cooldown across every pair, not per pair; and a suppressed or +// dropped announcement is NOT retried — the cooldown clock advances only on a +// real send, but the proposal row already exists, so the next tick will not +// re-detect it and nothing queues up behind it. A missed announcement means +// he reads it on /routines instead, which is the whole point of the page. +// +// What the cooldown is and is not. detectAndPropose returns non-nil only for a +// newly created row, so a pair gets exactly one chance to be spoken: the tick +// that first proposes it. Combined with one announcement per tick, the first +// tick over a populated history announces one pattern and permanently silences +// every other pattern found in the same pass. That is the intent, not an +// oversight — an inferred routine is not worth a second attempt at his +// attention, and /routines lists all of them. So the cooldown does not drain a +// backlog. It only spaces announcements of genuinely new pairs discovered on +// later ticks. If it should ever become "one per day until each is mentioned", +// that needs a queue rather than this counter. +// +// Cooldown gets its default here as well as in applyDefaults. That is +// deliberate: a tickLoop assembled directly in a test never goes through Load, +// and an unspaced announcer is not what those tests mean to exercise. +// +// The body is the detector's own literal Russian phrasing (pattern.PhraseRoutine +// — "ты заправляешь поилку раз в 7 дней — напоминать?"), not LLM-generated, so +// an inferred routine cannot arrive worded as something Maven never observed. +func (t *tickLoop) announceProposal(ctx context.Context, r *pattern.ProposedRoutine, now time.Time, state loop.State) bool { + if !t.proposalCfg.AnnounceProposals() { + return false + } + cooldown := time.Duration(t.proposalCfg.Cooldown) + if cooldown <= 0 { + cooldown = config.DefaultProposalCooldown + } + if !t.lastProposalAt.IsZero() && now.Sub(t.lastProposalAt) < cooldown { + return false + } + + rule := loop.Rule{Name: "proposal:" + r.Action + " " + r.Object, Severity: loop.Sev1} + if !loop.Gate(state, rule) { + return false + } + body := pattern.PhraseRoutine(r) + pn := delivery.PhrasedNudge{ + Candidate: loop.Candidate{Rule: rule, Severity: rule.Severity, State: state}, + Body: body, + Summary: body, + } + sent, err := t.dispatcher.DispatchNudge(ctx, pn, now) + if err != nil { + log.Printf("tick: announce proposal %s/%s: %v", r.Action, r.Object, err) + return false + } + if len(sent) == 0 { + return false // routing dropped it — /routines still has it. + } + t.lastProposalAt = now + return true +} + +// digestExpiry — how long a gate-suppressed care nudge stays worth +// resurfacing. 24h: these are daily-cadence rules (water/meal/break run on +// hour-scale cooldowns and re-derive from facts that reset every day), so a +// digest entry that outlives one full day is describing a day that's already +// over — "you skipped a break yesterday" said tomorrow evening is noise, not +// news. Bounding at one day also means a digest can never silently span a +// weekend of quiet hours into an unbounded backlog. +const digestExpiry = 24 * time.Hour + +// maxDigestSpokenItems — the bundle read-out is capped so "batched, not +// dropped" cannot regress into "she dumps twelve things on me the moment I +// walk in" — a digest that nags in bulk is worse than the drops it replaced. +// Anything beyond the cap is still marked drained (it did get its moment; +// the cap limits WORDS, not whether it counted) and folded into a trailing +// count instead of being spoken in full. +const maxDigestSpokenItems = 3 + +// enqueueSuppressedDigest scans this tick's trace for care candidates the +// gate blocked for a genuine restraint reason and durably records the +// digest-eligible ones (loop.DigestEligible). Phrasing happens once, here, +// at enqueue time — not re-derived at drain time — the same way queueNudge +// phrases once and caches, so a rule suppressed for hours isn't re-prompting +// the LLM every tick it stays blocked (EnqueueDigestEntry's rule+body dedupe +// makes repeat calls here harmless, but skipping the phrase call entirely +// when a pending entry already exists avoids the LLM round-trip too). +func (t *tickLoop) enqueueSuppressedDigest(ctx context.Context, trace *loop.TickTrace, state loop.State, now time.Time) { + if trace == nil { + return + } + for _, tr := range trace.RuleTraces { + if !tr.PredicateResult || tr.GateResult { + continue // didn't want to fire, or wasn't suppressed + } + if !loop.DigestEligible(tr.Severity, tr.GateBlockedBy) { + continue + } + rule := loop.Rule{Name: tr.RuleName, Severity: tr.Severity} + cand := loop.Candidate{Rule: rule, Severity: tr.Severity, State: state} + pn, err := t.phraser.PhraseNudge(ctx, cand) + if err != nil { + log.Printf("tick: phrase digest candidate %s: %v", tr.RuleName, err) + continue + } + expires := now.Add(digestExpiry) + if _, deduped, err := t.store.EnqueueDigestEntry(ctx, tr.RuleName, int(tr.Severity), pn.Body, now, expires); err != nil { + log.Printf("tick: enqueue digest entry %s: %v", tr.RuleName, err) + } else if deduped { + // same suppressed nudge already pending — nothing new to say. + continue + } + } +} + +// expireStaleDigest sweeps entries past their expiry once per tick — cheap +// bookkeeping, mirrors ReconcileStaleDeliveryAttempts's shape. +func (t *tickLoop) expireStaleDigest(ctx context.Context, now time.Time) { + n, err := t.store.ExpireStaleDigestEntries(ctx, now) + if err != nil { + log.Printf("tick: expire stale digest entries: %v", err) + return + } + if n > 0 { + log.Printf("tick: expired %d stale digest entr(y/ies) unspoken", n) + } +} + +// maybeDrainDigest speaks the pending digest bundle once the gate's +// suppression reasons have actually cleared — quiet hours over, back from +// away, out of the meeting. Draining while still suppressed would just be a +// second way to nag through quiet hours; the bundle waits for the same "is +// it allowed right now" condition a live nudge already waits for. +func (t *tickLoop) maybeDrainDigest(ctx context.Context, state loop.State, now time.Time) { + if state.QuietHours || state.CalendarBusy || state.Presence == store.Away { + return + } + entries, err := t.store.PendingDigestEntries(ctx, now) + if err != nil { + log.Printf("tick: pending digest entries: %v", err) + return + } + if len(entries) == 0 { + return + } + + spoken := entries + extra := 0 + if len(spoken) > maxDigestSpokenItems { + spoken = entries[:maxDigestSpokenItems] + extra = len(entries) - maxDigestSpokenItems + } + var b strings.Builder + maxSev := 0 + for i, e := range spoken { + if i > 0 { + b.WriteString(" · ") + } + b.WriteString(e.Body) + if e.Severity > maxSev { + maxSev = e.Severity + } + } + if extra > 0 { + fmt.Fprintf(&b, " · и ещё %d", extra) + } + body := b.String() + summary := fmt.Sprintf("%d отложенных уведомлений", len(entries)) + + cand := loop.Candidate{ + Rule: loop.Rule{Name: "digest", Severity: loop.Severity(maxSev)}, + Severity: loop.Severity(maxSev), + State: state, + } + pn := delivery.PhrasedNudge{Candidate: cand, Body: body, Summary: summary} + t.cachePhrase(pn) + if _, err := t.dispatcher.DispatchNudge(ctx, pn, now); err != nil { + log.Printf("tick: dispatch digest bundle: %v", err) + return // leave entries pending; retried next tick + } + ids := make([]int64, len(entries)) + for i, e := range entries { + ids[i] = e.ID + } + if err := t.store.DrainDigestEntries(ctx, ids, now); err != nil { + log.Printf("tick: drain digest entries: %v", err) + } +} + // routinesFromConfig maps the config's routine blocks to the engine type. // Validation (cron parses, name/body present, severity defaulted) already ran // in config.Load, so this is a pure field copy. @@ -530,6 +807,75 @@ func (t *tickLoop) morningStatus(ctx context.Context, now time.Time) []ipc.Morni return out } +// dayPlan is the read-only "what does today hold" query (Vikunja #128). It is +// the impure half of morning.BuildPlan: it reads the calendar events, the +// pending reminders and the checklist facts, and the pure builder orders them. +// +// It never dispatches. Asking for the plan is a query like any other; the only +// unprompted delivery in maven stays with the morning nudge and the +// dispatcher's policy. +func (t *tickLoop) dayPlan(ctx context.Context, now time.Time) ipc.DayPlan { + y, m, d := now.Date() + dayStart := time.Date(y, m, d, 0, 0, 0, 0, now.Location()) + dayEnd := dayStart.AddDate(0, 0, 1) + + var events []morning.PlanEntry + facts, err := t.store.CalendarEvents(ctx, dayStart, dayEnd) + if err != nil { + log.Printf("tick: day plan: calendar events: %v", err) + } + for _, f := range facts { + events = append(events, morning.PlanEntry{ + At: f.Ts, + // The plan prints the hour itself, so the "@ 14:00-14:30" tail the + // fact value carries would say it twice. + Text: calendar.FactSummary(f.Value), + Kind: morning.PlanEvent, + // Provenance below a calendar read (an ambient relay, #126) is + // hedged rather than recited as fact. + Uncertain: f.Confidence < 1.0, + }) + } + + var reminders []morning.PlanEntry + rems, err := t.store.PendingReminders(ctx, dayStart, dayEnd) + if err != nil { + log.Printf("tick: day plan: pending reminders: %v", err) + } + for _, r := range rems { + if r.Status != store.ReminderPending { + continue + } + fire := r.NextFireTs + if fire.IsZero() { + fire = r.FireTs + } + reminders = append(reminders, morning.PlanEntry{ + At: fire, + Text: strings.TrimSpace(r.Payload), + Kind: morning.PlanReminder, + }) + } + + var checklistFacts map[string]store.Fact + if len(t.morningRoutines) > 0 { + checklistFacts = t.gatherMorningFacts(ctx) + } + plan := morning.BuildPlan(t.morningRoutines, checklistFacts, events, reminders, now) + + out := ipc.DayPlan{Date: plan.Date, Spoken: plan.FormatRU()} + out.Items = make([]ipc.DayPlanItem, len(plan.Items)) + for i, it := range plan.Items { + out.Items[i] = ipc.DayPlanItem{ + At: it.At, + Text: it.Text, + Kind: string(it.Kind), + Uncertain: it.Uncertain, + } + } + return out +} + // tune — the feedback auto-tuner's impure step. runs on a slow cadence // (autotuneInterval, see run) so it doesn't write a fact every tick. for each // rule: @@ -609,7 +955,21 @@ type daemonAPI struct { ipc.CoreAPI getTrace func() *loop.TickTrace getMorningStatus func(ctx context.Context) []ipc.MorningRoutineStatus + getDayPlan func(ctx context.Context) ipc.DayPlan chatFn func(ctx context.Context, text string) string + getMCPServers func() []ipc.MCPServerStatus + getEvents func(n int) []ipc.IntakeEvent +} + +// RecentEvents — the unified intake journal (Vikunja #283). Empty, not an +// error, when no bus was wired: "nothing has arrived" and "the journal is off" +// look the same to a reader on purpose, because neither is a fault and the +// page renders both as an empty table. +func (d *daemonAPI) RecentEvents(ctx context.Context, n int) ([]ipc.IntakeEvent, error) { + if d.getEvents == nil { + return nil, nil + } + return d.getEvents(n), nil } func (d *daemonAPI) Chat(ctx context.Context, text string) (string, error) { @@ -619,6 +979,16 @@ func (d *daemonAPI) Chat(ctx context.Context, text string) (string, error) { return d.chatFn(ctx, text), nil } +// MCPServers — the configured MCP servers and their health (Vikunja #251). +// Empty, not an error, when the mcp block is absent: "not configured" is the +// default state and the web surface renders it as such. +func (d *daemonAPI) MCPServers(ctx context.Context) ([]ipc.MCPServerStatus, error) { + if d.getMCPServers == nil { + return nil, nil + } + return d.getMCPServers(), nil +} + func (d *daemonAPI) TickTrace(ctx context.Context) (ipc.TickTrace, error) { trace := d.getTrace() if trace == nil { @@ -634,6 +1004,13 @@ func (d *daemonAPI) MorningStatus(ctx context.Context) ([]ipc.MorningRoutineStat return d.getMorningStatus(ctx), nil } +func (d *daemonAPI) DayPlan(ctx context.Context) (ipc.DayPlan, error) { + if d.getDayPlan == nil { + return ipc.DayPlan{}, errors.New("mavend: day plan not available") + } + return d.getDayPlan(ctx), nil +} + func toIPCTickTrace(t loop.TickTrace) ipc.TickTrace { rules := make([]ipc.RuleTrace, len(t.RuleTraces)) for i, r := range t.RuleTraces { diff --git a/cmd/mavend/tick_test.go b/cmd/mavend/tick_test.go index 3333ce8..69dfcbd 100644 --- a/cmd/mavend/tick_test.go +++ b/cmd/mavend/tick_test.go @@ -46,7 +46,7 @@ func newTestTickLoop(t *testing.T, st *store.Store, sink delivery.Sink, digestCf Nudges: st, Reminders: st, }) - return newTickLoop(st, g, d, phraser.NewStub(), rules, time.Second, 5*time.Minute, 0, digestCfg, nil, nil) + return newTickLoop(st, g, d, phraser.NewStub(), rules, time.Second, 5*time.Minute, 0, digestCfg, nil, nil, nil) } func TestTickFiresRoutineWhenScheduleCrosses(t *testing.T) { @@ -63,7 +63,7 @@ func TestTickFiresRoutineWhenScheduleCrosses(t *testing.T) { sink := &fakeSink{} d := delivery.NewDispatcher(delivery.Config{Voice: sink, Ntfy: sink, Telegram: sink, Nudges: st, Reminders: st}) rs := []routine.Routine{{Name: "morning", Cron: "0 12 * * *", Body: "полдень, время воды", Severity: 1}} - tl := newTickLoop(st, g, d, phraser.NewStub(), rules, time.Second, 5*time.Minute, 0, nil, rs, nil) + tl := newTickLoop(st, g, d, phraser.NewStub(), rules, time.Second, 5*time.Minute, 0, nil, rs, nil, nil) // first tick: seeds, does not fire the routine. tl.tick(ctx, now) diff --git a/cmd/mavend/vision.go b/cmd/mavend/vision.go new file mode 100644 index 0000000..172b7a8 --- /dev/null +++ b/cmd/mavend/vision.go @@ -0,0 +1,282 @@ +// mavend/vision.go — core's half of image understanding (Vikunja #252, +// docs/plans/07-vision.md). +// +// The split: any surface that can receive a picture (mavweb upload, a Telegram +// photo through mavpoll, a path he names) hands the bytes to core over +// ipc.MethodDescribeImage. Core stores them content-addressed under +// media.dir, prepares a downscaled JPEG, and asks a local vision server what it +// is. The description comes back as words; nothing about the image is echoed. +// +// Off unless configured: no `media` block ⇒ nowhere to keep the bytes, so the +// method does not exist and a surface cannot make Maven accept a photo by +// merely sending one. A `media` block with no `vision` block is a real state, +// the one this box is in today: the store is wired, the method exists, the +// bytes are kept and the reply says she cannot read the picture yet. That reply +// is re-runnable by id on the day a vision model lands, which is the reason to +// keep the bytes at all. Saving the description as a note needs more than the +// read rung — see the scope check on auth.ImageNoteSource. +// +// Two things this file deliberately does not do: +// +// - No cloud vision call, ever. internal/vision refuses a non-private +// endpoint at construction; there is no config shape here that could reach +// an upstream API even if someone wanted one. +// - No automatic memory. SaveNote is opt-in per call. Glancing at a screenshot +// is not the same act as remembering it, and a 1.7B-class VLM's guess about +// a photo is not a fact worth carrying around. +package main + +import ( + "context" + "errors" + "fmt" + "log" + "path/filepath" + "sync" + "time" + + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/media" + "github.com/kami/maven/internal/router" + "github.com/kami/maven/internal/store" + "github.com/kami/maven/internal/vision" +) + +// prunePeriod — how often stored blobs are checked against media.retention. +// Hourly is far more often than needed for a 7-day retention and costs a +// directory walk over a handful of sidecars; the point is that the promise is +// kept by a loop that runs, not by an operator remembering a cron. +const prunePeriod = time.Hour + +// mediaKeeper — the blob store plus the loop that enforces its retention. The +// two are one object because a store without the loop is a directory that grows +// forever, and shipping that would break the only interesting promise this +// capability makes. +type mediaKeeper struct { + store *media.Store +} + +// openMediaStore builds the blob store from config, or returns nil when media is +// not configured. A relative dir resolves against StateDir, the same rule the db +// and socket paths follow. +func openMediaStore(cfg *config.Config) *mediaKeeper { + dir := cfg.Media.StoreDir() + if dir == "" { + return nil + } + if !filepath.IsAbs(dir) && cfg.StateDir != "" { + dir = filepath.Join(cfg.StateDir, dir) + } + st, err := media.OpenWithBudget(dir, cfg.Media.MaxBytes, cfg.Media.MaxTotalBytes, + time.Duration(cfg.Media.Retention)) + if err != nil { + log.Printf("media: %v — image and audio intake disabled", err) + return nil + } + log.Printf("media: blob store at %s, retention %s, %d of %d bytes used", + st.Dir(), st.Retention(), st.Total(), st.Budget()) + return &mediaKeeper{store: st} +} + +// runPrune deletes over-retention blobs on a loop until ctx ends. It prunes once +// immediately, so a daemon restarted after a long downtime does not sit on a +// month of stale recordings until the first tick. +func (k *mediaKeeper) runPrune(ctx context.Context) { + prune := func() { + n, err := k.store.Prune() + if err != nil { + log.Printf("media: prune: %v", err) + return + } + if n > 0 { + log.Printf("media: pruned %d blob(s) older than %s", n, k.store.Retention()) + } + } + prune() + t := time.NewTicker(prunePeriod) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + prune() + } + } +} + +// visionIntake — one image at a time: store, prepare, describe, optionally note. +type visionIntake struct { + in *vision.Intake + st *store.Store + emb router.Embedder + now func() time.Time +} + +// newVisionIntake returns nil when there is nothing to wire. keeper == nil means +// no media block, which disables the method outright; a missing or disabled +// vision block still wires the method, because storing an image and answering +// "I can't look at it yet" is more useful than pretending the surface does not +// exist — and it is exactly the state this box is in until a vision model is on +// disk. +func newVisionIntake(keeper *mediaKeeper, st *store.Store, emb router.Embedder, cfg *config.Config) *visionIntake { + if keeper == nil { + return nil + } + vc := cfg.Vision + maxDim := 0 + var provider vision.Provider = vision.Disabled{} + if vc.LooksAtImages() { + p, err := vision.NewLocal(vision.Config{ + Endpoint: vc.Endpoint, + Model: vc.Model, + Timeout: time.Duration(vc.Timeout), + MaxTokens: vc.MaxTokens, + Prompt: vc.Prompt, + }) + if err != nil { + // A public endpoint, a hostname, a bad URL. Logged once here rather + // than failing every turn, and the store still works. + log.Printf("vision: %v — she can store images but not describe them", err) + } else { + provider = p + maxDim = vc.MaxDim + log.Printf("vision: enabled against %s", p.Endpoint()) + } + } else { + log.Printf("vision: not configured — images are stored, not described") + } + return &visionIntake{ + in: vision.NewIntake(keeper.store, provider, maxDim), + st: st, + emb: emb, + now: time.Now, + } +} + +// describe handles one ipc.MethodDescribeImage call. +// +// A description failure is NOT an error out of this method when the bytes were +// stored: the caller gets the id and an empty description, which is honest ("it +// is kept, I cannot read it yet") and re-runnable. A failure to store, or bytes +// that are not an image at all, is an error — there is nothing to come back to. +func (v *visionIntake) describe(ctx context.Context, req ipc.DescribeImageReq) (ipc.DescribeImageResp, error) { + if len(req.Data) == 0 && req.ID == "" { + return ipc.DescribeImageResp{}, fmt.Errorf("describe image: neither data nor id") + } + if len(req.Data) > 0 && req.ID != "" { + // The contract says exactly one. Taking the ID branch and dropping the + // bytes silently is the worst of the three possible answers: the caller + // believes it sent a new image and nothing says otherwise. + return ipc.DescribeImageResp{}, fmt.Errorf("describe image: both data and id given, send one") + } + + var ( + res vision.Result + err error + ) + if req.ID != "" { + res, err = v.in.Rerun(ctx, req.ID, req.Question) + } else { + res, err = v.in.Accept(ctx, req.Data, sourceOrDefault(req.Source), req.Question) + } + if res.Blob.ID == "" { + // Nothing was stored: bad format, over the size cap, unwritable dir. + return ipc.DescribeImageResp{}, fmt.Errorf("describe image: %w", err) + } + + resp := ipc.DescribeImageResp{ + ID: res.Blob.ID, + Description: res.Description, + Width: res.Image.Width, + Height: res.Image.Height, + } + if err != nil { + // Bytes are safe, words are not available. The log names the blob and the + // reason; it never names what was in the picture. + if errors.Is(err, vision.ErrDisabled) { + log.Printf("vision: stored %s, no vision model configured", res.Blob) + } else { + log.Printf("vision: stored %s, describe failed: %v", res.Blob, err) + } + return resp, nil + } + + if req.SaveNote { + id, werr := v.writeNote(ctx, res) + if werr != nil { + // The description is still returned: losing the note is worse as a + // silent failure than as a log line next to a successful answer. + log.Printf("vision: note write for %s failed: %v", res.Blob, werr) + } else { + resp.NoteID = id + } + } + log.Printf("vision: described %s (%dx%d)", res.Blob, res.Image.Width, res.Image.Height) + return resp, nil +} + +// noteMarker prefixes a stored description. Without it the note reads exactly +// like something he told her, and it is not: it is a small VLM's guess about a +// picture, embedded and recalled as if it were his own words. Four characters +// of provenance in the text are cheaper than believing it later. +const noteMarker = "Со снимка: " + +// writeNote stores the description as an ordinary note so it is recallable. The +// note carries the blob id in its source, which is the only link back to the +// bytes — the note text is words about the picture, never the picture. +func (v *visionIntake) writeNote(ctx context.Context, res vision.Result) (int64, error) { + var vec []float32 + if v.emb != nil { + // EmbedPassage, not Embed: a description is text being searched FOR, and + // the e5 embedder is asymmetric. Backwards here makes it unfindable by + // the question that should have matched it. + var err error + vec, err = router.EmbedPassage(ctx, v.emb, res.Description) + if err != nil { + return 0, fmt.Errorf("embed: %w", err) + } + } + source := "media:image:" + res.Blob.ID[:12] + return v.st.WriteNote(ctx, v.now(), noteMarker+res.Description, vec, source) +} + +// sourceOrDefault labels a blob whose sender did not say where it came from. +func sourceOrDefault(s string) string { + if s == "" { + return "unknown" + } + return s +} + +// wireVision installs the IPC hook and starts the retention loop, or leaves the +// hook nil so ipc.MethodDescribeImage reports ErrUnknownMethod. Called on both +// startup paths (unlocked boot and passkey unlock) so vision behaves the same +// either way. +// +// Returns the media keeper so the meeting recorder can share it: one blob store +// with one retention loop holds both the images and the audio, which is the +// whole point of internal/media being a shared package. nil ⇒ no media block, +// and neither capability exists. +func wireVision(ctx context.Context, wg *sync.WaitGroup, srv *ipc.Server, st *store.Store, emb router.Embedder, cfg *config.Config) *mediaKeeper { + keeper := openMediaStore(cfg) + if keeper == nil { + return nil + } + // In the daemon's WaitGroup like every other loop in run: a prune deletes + // files, and shutting down in the middle of one was the single loop nobody + // waited for. + wg.Add(1) + go func() { + defer wg.Done() + keeper.runPrune(ctx) + }() + + vi := newVisionIntake(keeper, st, emb, cfg) + if vi == nil { + return keeper + } + srv.DescribeImageFn = vi.describe + return keeper +} diff --git a/cmd/mavend/vision_test.go b/cmd/mavend/vision_test.go new file mode 100644 index 0000000..ab69be0 --- /dev/null +++ b/cmd/mavend/vision_test.go @@ -0,0 +1,72 @@ +package main + +import ( + "bytes" + "context" + "image" + "image/png" + "strings" + "testing" + "time" + + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/media" + "github.com/kami/maven/internal/vision" +) + +func testIntake(t *testing.T) *visionIntake { + t.Helper() + st := newTestStore(t) + blobs, err := media.Open(t.TempDir(), 0, 0) + if err != nil { + t.Fatal(err) + } + return &visionIntake{ + in: vision.NewIntake(blobs, vision.Disabled{}, 0), + st: st, + now: time.Now, + } +} + +// The contract says exactly one of Data or ID. Taking the ID branch and +// dropping the bytes silently is the worst of the three possible answers: the +// caller believes it sent a new image and nothing says otherwise. +func TestDescribeRefusesBothDataAndID(t *testing.T) { + v := testIntake(t) + _, err := v.describe(context.Background(), ipc.DescribeImageReq{ + Data: []byte("bytes"), ID: strings.Repeat("a", 64), + }) + if err == nil { + t.Fatal("both data and id must be refused") + } + if !strings.Contains(err.Error(), "send one") { + t.Fatalf("err = %v, want it to name the contract", err) + } +} + +// Vision being off does not remove the method: the bytes are stored and the +// answer says she cannot read the picture yet, which is re-runnable by id. That +// is the state this box is in today, and three doc comments used to claim the +// opposite. +func TestVisionOffStillStores(t *testing.T) { + v := testIntake(t) + var buf bytes.Buffer + if err := png.Encode(&buf, image.NewRGBA(image.Rect(0, 0, 4, 4))); err != nil { + t.Fatal(err) + } + resp, err := v.describe(context.Background(), ipc.DescribeImageReq{Data: buf.Bytes(), Source: "web:upload"}) + if err != nil { + t.Fatalf("storing must succeed even with no vision model: %v", err) + } + if len(resp.ID) != 64 { + t.Fatalf("no blob id came back: %+v", resp) + } + if resp.Description != "" { + t.Errorf("description = %q, want none", resp.Description) + } + // And with no media block at all the method does not exist. + if vi := newVisionIntake(nil, nil, nil, &config.Config{}); vi != nil { + t.Fatal("no media block must leave the method nonexistent") + } +} diff --git a/cmd/mavend/voice.go b/cmd/mavend/voice.go index af5feb0..7bc2190 100644 --- a/cmd/mavend/voice.go +++ b/cmd/mavend/voice.go @@ -43,29 +43,19 @@ package main import ( - "bufio" "context" - "encoding/json" "errors" "fmt" "log" - "os" - "path/filepath" - "strconv" "strings" "sync" "time" - hexisclient "github.com/kami/hexis/pkg/client" "github.com/kami/maven/internal/audio" - "github.com/kami/maven/internal/config" - "github.com/kami/maven/internal/delivery" - "github.com/kami/maven/internal/delivery/voicesink" + "github.com/kami/maven/internal/crawl" "github.com/kami/maven/internal/dialogue" "github.com/kami/maven/internal/ipc" - "github.com/kami/maven/internal/llm" "github.com/kami/maven/internal/memory" - "github.com/kami/maven/internal/pattern" "github.com/kami/maven/internal/phraser" "github.com/kami/maven/internal/router" "github.com/kami/maven/internal/store" @@ -75,219 +65,8 @@ import ( "github.com/kami/maven/internal/ttsnorm" "github.com/kami/maven/internal/voice" "github.com/kami/maven/internal/weather" - "github.com/kami/maven/internal/worker" ) -// voiceWiring — everything the daemon needs to run the audio path. Held by -// cmd/mavend/main.go alongside the other wirings; closed on shutdown. -type voiceWiring struct { - server *voice.Server - sessions *voice.Sessions - voiceSink delivery.Sink - embedder router.Embedder - handler *reactiveHandler // the reactive handler for IPC Chat - // worker clients (set when configured as Remote): closed on shutdown so - // mavsttd / mavttsd don't keep a stale conn into a restarting daemon. - sttClient *worker.Client - ttsClient *worker.Client -} - -// close releases the listener + worker conns. Safe to call on nil (when -// voice is not wired — wireVoice returns nil,nil). -func (w *voiceWiring) close() { - if w == nil { - return - } - if w.embedder != nil { - _ = w.embedder.Close() - } - if w.server != nil { - _ = w.server.Close() - } - if w.sttClient != nil { - _ = w.sttClient.Close() - } - if w.ttsClient != nil { - _ = w.ttsClient.Close() - } -} - -// wireVoice builds the audio path from cfg + a CoreAPI + a router. Returns -// nil wiring + nil error when voice isn't enabled (the caller's voice sink -// stays nil; the dispatcher's ChannelVoice routing drops silently). -// -// When voice is enabled, MUST wire a voicesink into the dispatcher's Voice -// slot using w.sessions (the caller does that — see main.go). -func wireVoice(cfg *config.Config, coreAPI ipc.CoreAPI, phr phraser.Phraser, memStore memory.Store, dataStore *store.Store, eco *ecosystemWiring) (*voiceWiring, error) { - if cfg.Voice == nil || !cfg.Voice.Enabled { - return nil, nil - } - w := &voiceWiring{} - - // ----- stt (Stub in-process OR Remote via worker socket) ----- - var transcriber stt.Transcriber - if cfg.Voice.Stt != nil && cfg.Voice.Stt.Socket != "" { - c := worker.Dial(cfg.Voice.Stt.Socket) - w.sttClient = c - lang := cfg.Voice.Stt.Lang - if lang == "" { - lang = cfg.Voice.Lang - } - transcriber = stt.NewRemote(c, lang) - } else { - transcriber = stt.NewStub() - } - - // ----- tts (Stub in-process OR Remote) ----- - var synthesizer tts.Synthesizer - if cfg.Voice.Tts != nil && cfg.Voice.Tts.Socket != "" { - c := worker.Dial(cfg.Voice.Tts.Socket) - w.ttsClient = c - lang := cfg.Voice.Tts.Lang - if lang == "" { - lang = cfg.Voice.Lang - } - synthesizer = tts.NewRemote(c, lang, cfg.Voice.Tts.Voice) - } else { - synthesizer = tts.NewStub() - } - - // ----- router: embedder (ONNX when configured, floor HashEmbedder otherwise) ----- - var emb router.Embedder - if cfg.Voice.Embedder != nil { - onnx, err := router.NewONNXEmbedder( - cfg.Voice.Embedder.ModelPath, - cfg.Voice.Embedder.TokenizerPath, - cfg.Voice.Embedder.LibPath, - ) - if err != nil { - w.close() - return nil, fmt.Errorf("embedder: %w", err) - } - log.Printf("voice: onnx embedder loaded (%d dim)", onnx.Dim()) - emb = onnx - } else { - log.Printf("voice: embedder not configured, using HashEmbedder floor") - emb = router.NewHashEmbedder(1024) - } - w.embedder = emb - checkStoredEmbedder(dataStore, emb) - - // ----- tool executor (the enabled act allowlist, store-backed) ----- - // Config tools are the declarative bootstrap: seed them into the store as - // enabled (editing mavend.json IS the human enable act). Ad-hoc tools are - // enabled later through the authed mavweb surface. The executor + matcher - // both read the store live, so a newly-enabled tool is runnable without a - // daemon restart. - seedTools(coreAPI, cfg.Voice.Tools) - exec := tool.NewExecutor(coreAPI, time.Duration(cfg.Voice.ToolTimeout)) - matcher := tool.NewMatcher(coreAPI) - - // ----- weather provider (Open-Meteo when configured, Stub otherwise) ----- - var weatherProvider weather.Provider - var weatherLocation string - if cfg.Voice.Weather != nil && cfg.Voice.Weather.Provider == "open-meteo" { - weatherProvider = weather.NewOpenMeteoProvider() - weatherLocation = cfg.Voice.Weather.DefaultLocation - log.Printf("voice: weather provider: open-meteo (default location: %s)", cfg.Voice.Weather.DefaultLocation) - } else { - weatherProvider = weather.NewStubProvider() - log.Printf("voice: weather provider: stub (not configured)") - } - - // The replier uses the same llama-server as the phraser. - var llmClient *llm.Client - if lp, ok := phr.(*phraser.LLMPhraser); ok { - llmClient = llm.New(lp.BaseURL(), 60*time.Second) - } - // ----- router (the cascade; floor examples seed the classifier) ----- - // The act matcher's allowlist is exactly the enabled tool names — the - // router only matches acts the executor can run (one source of truth). - threshold := cfg.Voice.RouterThreshold - if threshold <= 0 { - threshold = config.DefaultRouterThreshold - } - // The resident model routes by default: 63.2% of held-out intents right - // against the classifier's 50.0%, at about 1s a turn instead of 30ms (see - // config.VoiceConfig.LLMRouter). The classifier always stays wired as the - // fallback, so a model error never breaks a turn. - rtr := buildRouter(emb, matcher, threshold, pickLLMRouter(cfg.Voice.UseLLMRouter(), llmClient)) - - // ----- sessions registry (shared with voicesink) ----- - sessions := voice.NewSessions() - w.sessions = sessions - - // ----- voice sink (proactive nudges: dispatcher → voicesink → tts → push to client) ----- - w.voiceSink = voicesink.New(synthesizer, sessions) - - // ----- memory (long-term vector storage) ----- - // Persistent (store-backed, survives restarts) when the daemon passes one; - // falls back to the in-memory floor otherwise (tests / no-store paths). - if memStore == nil { - memStore = memory.NewInMemoryStore() - } - - // ----- dialogue (multi-turn slot carry-over; 2-min follow-up window) ----- - // Store-backed when the daemon passes a store, so a restart mid-conversation - // keeps the thread (Vikunja #363). Sessions past their TTL are dropped on - // load, never revived. Clarify's parked question stays in memory only. - var dialogueSessions *dialogue.SessionStore - if dataStore != nil { - dialogueSessions = dialogue.NewPersistentSessionStore(2*time.Minute, dataStore) - if err := dialogueSessions.Load(context.Background(), time.Now()); err != nil { - log.Printf("dialogue: load saved sessions: %v", err) - } - } else { - dialogueSessions = dialogue.NewSessionStore(2 * time.Minute) - } - clarifyStore := dialogue.NewClarifyStore(clarifyTTL) - timeParser := router.NewPythonDateParser() - - // ----- replier (LLM-backed when the engine is on, Stub floor otherwise) ----- - replier := voice.Replier(voice.NewStubReplier()) - if llmClient != nil { - replier = newLLMReplier(llmClient, contextBlockFn(cfg, time.Now)) - } - - // ----- the handler (the reactive path; closes over stt / tts / router / coreAPI / memory) ----- - h := &reactiveHandler{ - stt: transcriber, - tts: synthesizer, - router: rtr, - embedder: emb, - api: coreAPI, - tools: exec, - matcher: matcher, - replier: replier, - phraser: phr, - now: time.Now, - weatherProvider: weatherProvider, - weatherLocation: weatherLocation, - memStore: memStore, - dataStore: dataStore, - dialogueSessions: dialogueSessions, - clarifyStore: clarifyStore, - // 0 here (unset config) ⇒ the dialogue default. - clarifyMaxAttempts: cfg.Voice.ClarifyMaxAttempts, - extractor: router.Extractor{Time: timeParser, Acts: matcher, Facts: router.DefaultFactParser{}}, - queryMinScore: cfg.Voice.QueryMinScore, - queryMinMargin: cfg.Voice.QueryMinMargin, - timeParser: timeParser, - ecosystem: eco, - } - - // ----- the server (TCP listener) ----- - srv := voice.NewServer(cfg.Voice.Bind, h, sessions) - if err := srv.Listen(); err != nil { - w.close() - return nil, fmt.Errorf("voice listen: %w", err) - } - w.server = srv - w.handler = h - - return w, nil -} - // reactiveHandler — voice.Handler implementation. One method: turn a // PushToTalkReq into a reply (audio + text). The handler is concurrency- // safe (the wired stt/tts/router/api all are); called from per-conn @@ -304,6 +83,26 @@ type reactiveHandler struct { replier voice.Replier now func() time.Time + // crawler reads a web page he names out loud (queryWeb). nil ⇒ on-demand + // page reading is off, which is the default: no `crawl` block, no fetch. + crawler *crawl.Crawler + + // feedsOn — whether any RSS feed is configured (config.Feeds). It changes + // only what she SAYS when asked and nothing is there: "ленты не настроены" + // instead of "ничего нового", which are different truths. + feedsOn bool + + // home — the Home Assistant client (Vikunja #256). nil ⇒ the house is not + // configured, which is the default: no `smarthome` block, no reads, no + // switches. Control does not go through this field — it goes through the + // act allowlist and tool.Executor, like every other mutating act. + home *homeWiring + + // netscan — the LAN scanner (Vikunja #257). nil ⇒ off, which is the + // default. A scan is a read, so it has no allowlist row; what keeps it + // safe is that its range comes from config and from nowhere else. + netscan *netWiring + weatherProvider weather.Provider weatherLocation string // default location for weather queries @@ -352,42 +151,6 @@ type reactiveHandler struct { ecosystem *ecosystemWiring // nexus + hexis + praxis clients } -// pendingHexisExec — a mutating Hexis capability parked awaiting a spoken -// confirm. The confirmation is bound to the resolved capability + canonical -// target entity so a later "да" can only execute exactly what was proposed -// (ecosystem invariant: protected actions require bound confirmation). -type pendingHexisExec struct { - capabilityID string - capName string - entityID string - displayName string - expiry time.Time -} - -// pendingRoutineConfirm — a proposed routine awaiting a spoken y/n to become -// a recurring reminder. Set by detectPattern after creating a proposal. -type pendingRoutineConfirm struct { - routineID int64 - action string - object string - interval float64 - phrase string - expiry time.Time -} - -// pendingAct — a destructive act awaiting a spoken confirm. -type pendingAct struct { - fn string - args []string - phrase string - expiry time.Time -} - -// confirmTTL — how long a parked destructive confirm stays answerable. Short: -// a confirm is a same-breath gesture; a stale prompt shouldn't fire on an -// unrelated later "да". -const confirmTTL = 90 * time.Second - // HandlePushToTalk — the full reactive round-trip. Each step's failure // surfaces as a short reply text + empty audio OR an error; the voice // server translates an error into a wire RpcError. Today the handler @@ -407,49 +170,96 @@ func (h *reactiveHandler) HandlePushToTalk(ctx context.Context, req voice.PushTo } log.Printf("voice: stt → %q", text) - // 1b. confirm turn — if a destructive act is parked, this utterance is its - // y/n answer, not a fresh command. Handled before routing so "да" doesn't - // get classified as some other intent. - if reply, handled := h.resolveConfirm(ctx, text); handled { - return h.reply(ctx, reply, nil) - } + // 2-5. the shared turn pipeline (confirm → clarify → route → dialogue → + // action → replier), identical to the text path. + replyText := h.runTurn(ctx, text, sourceVoice) - // 1b2. expired clarify — a question was parked but its TTL ran out, so the + // 6. tts — synthesise the reply text; return to the voice server which + // ships it back on the conn. + return h.reply(ctx, replyText, nil) +} + +// handleText — the core reactive path without stt/tts. Used by the IPC Chat +// endpoint (and eventually by telegram). Splits out the audio bookends from +// HandlePushToTalk so text channels share the same routing logic. +func (h *reactiveHandler) handleText(ctx context.Context, text string) string { + log.Printf("voice: handleText: %q", text) + return h.runTurn(ctx, text, sourceText) +} + +// turnSource — which channel this utterance arrived on, in the same provenance +// vocabulary facts use (internal/event). It is threaded through runTurn because +// a turn can write a fact, and a fact that lies about where it came from is +// worse than no fact: provenance is the first column read when asking why a +// daemon-wide setting is the way it is. +type turnSource string + +const ( + sourceVoice turnSource = "tap:voice" // HandlePushToTalk, a real microphone + sourceText turnSource = "tap:text" // handleText: mavweb /api/chat, telegram +) + +// runTurn — the reactive turn pipeline shared by the voice and text entry +// points: expired-clarify notice → confirm answer → clarify answer → quiet +// toggle → route → dialogue merge → clarify question → action → replier. +// Takes the already-transcribed utterance, returns the reply text; the voice +// path wraps it in stt/tts, the text path returns it as-is. +// +// The ordering is load-bearing — see the step comments. +func (h *reactiveHandler) runTurn(ctx context.Context, text string, src turnSource) string { + // 1. expired clarify — a question was parked but its TTL ran out, so the // request behind it is gone. Say that out loud (see clarify.go) and carry // on: these words are still routed as a fresh utterance below, with the // notice glued in front of whatever the fresh routing answers. Checked // BEFORE the answer path: reading a parked question drops an expired one. + // + // Taken before the confirm check, not after, because a confirm turn returns + // early. He can be asked a question, walk off, come back and say "да" to a + // confirm that is still parked; computing the notice after that return meant + // he answered the confirm and never heard that the older request was let go. expiredNotice := h.clarifyExpiredNotice() - // 1b3. clarify answer — if she asked a live question last turn, this + // 2. confirm turn — if a destructive act is parked, this utterance is its + // y/n answer, not a fresh command. Handled before routing so "да" doesn't + // get classified as some other intent. + if reply, handled := h.resolveConfirm(ctx, text); handled { + return withNotice(expiredNotice, reply) + } + + // 3. clarify answer — if she asked a live question last turn, this // utterance is its answer, not a fresh command. After the confirm check: a // y/n gate is armed by her own prompt and is the narrower claim on the // utterance. + // A live question and an expired one cannot both exist for one dialogue id, + // so the notice is empty here in practice. withNotice anyway: every exit + // from runTurn carries it, and that is what stops the next one from + // forgetting. if reply, handled := h.resolveClarifyAnswer(ctx, text); handled { - return h.reply(ctx, reply, nil) + return withNotice(expiredNotice, reply) } - // 1c. quiet-hours toggle — keyword match, not classifier-dependent. + // 4. quiet-hours toggle — keyword match, not classifier-dependent. // "тихий режим" / "quiet on" would route through the classifier // unreliably (it's a command, not a free-form query), so we match it // before routing. Same pattern as the confirm turn above. - if reply, handled := h.resolveQuietToggle(ctx, text); handled { - return h.reply(ctx, withNotice(expiredNotice, reply), nil) + if reply, handled := h.resolveQuietToggle(ctx, text, src); handled { + return withNotice(expiredNotice, reply) } - // 2. router — classify the utterance. + // 5. router — classify the utterance. dec, err := h.router.Route(ctx, text, h.now()) if err != nil { // ErrNoIntents ⇒ classifier unseeded (cold boot). reply with a // "still warming up" rather than a wire error. if errors.Is(err, router.ErrNoIntents) { - return h.reply(ctx, withNotice(expiredNotice, "я ещё не понимаю свободную речь — скоро научусь."), nil) + return withNotice(expiredNotice, "я ещё не понимаю свободную речь — скоро научусь.") } log.Printf("voice: router error: %v", err) - return h.reply(ctx, withNotice(expiredNotice, "не получилось разобрать команду."), nil) + return withNotice(expiredNotice, "не получилось разобрать команду.") } + log.Printf("voice: route result: intent=%s slots=%+v", dec.Intent, dec.Slots) - // 2b. dialogue — fill this turn's missing slots from a prior same-intent + // 6. dialogue — fill this turn's missing slots from a prior same-intent // turn (follow-ups like «напомни завтра» → «…позвонить маме»), then remember // this turn for the next follow-up. Only same-intent, non-expired, non- // clarify turns carry (see followUpMerge). Best-effort: nil store ⇒ skipped. @@ -462,82 +272,22 @@ func (h *reactiveHandler) HandlePushToTalk(ctx context.Context, req voice.PushTo } } - // 2c. clarify — she is not sure. If one named thing is missing, ask about it + // 7. clarify — she is not sure. If one named thing is missing, ask about it // and park the request (clarify.go); otherwise the replier's canned reply // stands. - if dec.Clarify { - if question, asked := h.askClarify(dec); asked { - return h.reply(ctx, withNotice(expiredNotice, question), nil) - } - } - - // 3. action — execute the decision's intent. errors here surface as - // short reply text (the user wants to know the action didn't land); - // the round-trip stays alive. - replyText := h.applyAction(ctx, dec) - - // 4. replier — phrase the reply across the router decision. - if replyText == "" { - replyText = h.replier.Reply(dec) - } - - // 5. tts — synthesise the reply text; return to the voice server which - // ships it back on the conn. - return h.reply(ctx, withNotice(expiredNotice, replyText), nil) -} - -// handleText — the core reactive path without stt/tts: confirm check → -// route → dialogue → action → replier. Used by the IPC Chat endpoint -// (and eventually by telegram). Splits out the audio bookends from -// HandlePushToTalk so text channels share the same routing logic. -func (h *reactiveHandler) handleText(ctx context.Context, text string) string { - log.Printf("voice: handleText: %q", text) - // 1b. confirm turn — if a destructive act is parked, this utterance is its - // y/n answer. Same check as HandlePushToTalk. - if reply, handled := h.resolveConfirm(ctx, text); handled { - return reply - } - - // 1b2/1b3. expired clarify then clarify answer — same order and reasons as - // HandlePushToTalk. - expiredNotice := h.clarifyExpiredNotice() - if reply, handled := h.resolveClarifyAnswer(ctx, text); handled { - return reply - } - - // 2. router — classify the utterance. - dec, err := h.router.Route(ctx, text, h.now()) - if err != nil { - if errors.Is(err, router.ErrNoIntents) { - return withNotice(expiredNotice, "я ещё не понимаю свободную речь — скоро научусь.") - } - log.Printf("voice: handleText router error: %v", err) - return withNotice(expiredNotice, "не получилось разобрать команду.") - } - log.Printf("voice: route result: intent=%s slots=%+v", dec.Intent, dec.Slots) - - // 2b. dialogue — same as HandlePushToTalk. - if h.dialogueSessions != nil { - now := h.now() - prev := h.dialogueSessions.Get(voiceDialogueID, now) - dec = followUpMerge(prev, dec, now) - if !dec.Clarify { - h.rememberTurn(prev, dec, now) - } - } - - // 2c. clarify — same as HandlePushToTalk: ask about the one missing thing. if dec.Clarify { if question, asked := h.askClarify(dec); asked { return withNotice(expiredNotice, question) } } - // 3. action — execute the decision's intent. + // 8. action — execute the decision's intent. errors here surface as + // short reply text (the user wants to know the action didn't land); + // the round-trip stays alive. replyText := h.applyAction(ctx, dec) log.Printf("voice: applyAction returned: %q", replyText) - // 4. replier — phrase the reply when applyAction returned "". + // 9. replier — phrase the reply across the router decision. if replyText == "" { replyText = h.replier.Reply(dec) } @@ -561,540 +311,12 @@ func (h *reactiveHandler) applyAction(ctx context.Context, dec router.Decision) if dec.Clarify { return "" // the Replier phrases clarify } - switch dec.Intent { - case router.IntentFact: - if !dec.Slots.HasKey { - return "не разобрала, что записать — попробуй иначе." - } - now := h.now() - req := ipc.WriteFactReq{ - Ts: now, - Kind: "self", - Key: dec.Slots.Key, - Value: dec.Slots.Value, - Source: "tap:voice", - Confidence: 1.0, - // Subject: the key doubles as the entity-resolution candidate — - // a voice-tapped fact's key is usually the thing/person it's - // about ("espresso_machine", "kate"), so queueing it for Nexus - // resolution costs one async lookup and is a no-op (not_found) - // for the abstract self-state keys (mood, water) that aren't - // entities at all. - Subject: dec.Slots.Key, - } - factID, err := h.api.WriteFact(ctx, req) - if err != nil { - log.Printf("voice: write fact: %v", err) - return "не получилось сохранить факт." - } - // Index the fact utterance in long-term memory (best-effort, must not - // fail the fact write). Facts aren't in the notes table, so this is the - // only recall path for them — "когда я пил воду?" reads back from here. - if h.memStore != nil { - if vec, err := router.EmbedPassage(ctx, h.embedder, dec.Utterance); err != nil { - log.Printf("voice: embed fact for memory: %v", err) - } else if err := h.memStore.Insert(ctx, "fact:"+dec.Slots.Key+":"+strconv.FormatInt(now.Unix(), 10), vec, map[string]string{ - "source": "voice", - "type": "fact", - "text": dec.Utterance, - "ts": strconv.FormatInt(now.Unix(), 10), - }); err != nil { - log.Printf("voice: memory insert fact: %v", err) - } - } - // Event extraction + pattern detection (best-effort, must not fail the - // fact write). If the fact describes a recognizable action, it becomes a - // normalized event; if ≥3 events for the same action+object show stable - // intervals, a proposed routine is created and parked for confirmation. - if h.dataStore != nil { - if phrase := h.detectPattern(ctx, factID, dec.Slots.Key, dec.Slots.Value, now); phrase != "" { - return phrase // "ты заправляешь ... напоминать?" - } - } - return "" // replier phrases the success reply - - case router.IntentReminder: - if !dec.Slots.HasTime { - // Stage-0 (reminder-wakeword grammar) skips the extractor, so the - // time wasn't parsed. Run the parser as a fallback. - if dec.Stage == 0 && h.timeParser != nil { - t, ok, err := h.timeParser.Parse(ctx, dec.Utterance, h.now()) - if err == nil && ok { - dec.Slots.Time = t - dec.Slots.HasTime = true - } - } - if !dec.Slots.HasTime { - return "не получилось разобрать время напоминания." - } - } - payload := `{"text":` + jsonString(dec.Utterance) + `}` - if _, err := h.api.CreateReminder(ctx, dec.Slots.Time, payload, ""); err != nil { - log.Printf("voice: create reminder: %v", err) - return "не получилось поставить напоминание." - } - return "" - - case router.IntentAct: - // tool executor: run the matched fn against the enabled allowlist. - // HasFn=false ⇒ try the matcher (for LLM-routed acts where the verb - // didn't go through the stage-0 act grammar). - if !dec.Slots.HasFn && dec.Slots.Text != "" && h.matcher != nil { - if fn, args, ok := h.matcher.Match(dec.Slots.Text); ok { - dec.Slots.Fn, dec.Slots.Args, dec.Slots.HasFn = fn, args, true - } - } - - // Praxis ecosystem tools: intercept before the system command executor. - if h.ecosystem != nil && h.ecosystem.praxis != nil && dec.Slots.HasFn { - if reply := h.handlePraxisAct(ctx, dec); reply != "" { - return reply - } - } - - // Hexis ecosystem action: if ecosystem is configured and we have a verb - // + entity text, try to resolve the entity and execute via Hexis. - if h.ecosystem != nil && h.ecosystem.hexis != nil && dec.Slots.Text != "" { - if reply := h.handleHexisAct(ctx, dec); reply != "" { - return reply - } - } - - // HasFn still false ⇒ no allowlist match: scaffold a 'proposed' tool - // the user can enable on the authed surface ("earn the right to ask"). - if !dec.Slots.HasFn { - return h.proposeGap(ctx, dec) - } - out, err := h.tools.Exec(ctx, dec.Slots.Fn, dec.Slots.Args, false) - if err != nil { - switch { - case errors.Is(err, tool.ErrNeedsConfirm): - // destructive: park it and ask. The next utterance answers. - phrase := actPhrase(dec.Slots.Fn, dec.Slots.Args) - h.park(dec.Slots.Fn, dec.Slots.Args, phrase) - return "выполнить «" + phrase + "»? скажи «да» или «нет»." - case errors.Is(err, tool.ErrNotEnabled): - return h.proposeGap(ctx, dec) - } - log.Printf("voice: tool %s: %v", dec.Slots.Fn, err) - if out != "" { - return "не получилось выполнить команду: " + firstLine(out) - } - return "не получилось выполнить команду." - } - if out != "" { - return "готово: " + firstLine(out) - } - return "готово." - - case router.IntentChat: - // Conversational: build history from dialogue session (prior user turns) - // and let the LLM respond from general knowledge + context. - history := h.chatHistory() - reply, err := h.phraser.PhraseChat(ctx, dec.Utterance, history) - if err != nil { - log.Printf("voice: chat: %v", err) - return "поговорили." - } - return reply - - case router.IntentSystem: - return h.replySystem(ctx, dec) - - case router.IntentNote: - // embed the note text with the same model the classifier uses, persist - // via CoreAPI (source=tap:voice). Semantic recall lives in `notes`, not - // facts — no predicate reads it (spec's two-memory split). - vec, err := router.EmbedPassage(ctx, h.embedder, dec.Utterance) - if err != nil { - log.Printf("voice: embed note: %v", err) - return "не получилось сохранить заметку." - } - noteTs := h.now() - noteID, err := h.api.WriteNote(ctx, noteTs, dec.Utterance, vec, "tap:voice") - if err != nil { - log.Printf("voice: write note: %v", err) - return "не получилось сохранить заметку." - } - // Insert into long-term memory (best-effort, must not fail the note write). - // text/ts in the meta make a Search hit self-describing (see bestRecall). - if h.memStore != nil { - if err := h.memStore.Insert(ctx, "note:"+strconv.FormatInt(noteID, 10), vec, map[string]string{ - "source": "voice", - "type": "note", - "text": dec.Utterance, - "ts": strconv.FormatInt(noteTs.Unix(), 10), - }); err != nil { - log.Printf("voice: memory insert: %v", err) - } - } - return "" // replier phrases the "saved" reply - - case router.IntentQuery: - // Fact-by-key lookup: when the dialogue layer resolved an anaphoric - // reference to a prior fact's key (e.g. "когда я это сделал?" after - // "запиши что я пил воду"), look up the fact's value directly. - if dec.Slots.HasKey && dec.Slots.Key != "" { - if f, err := h.api.LatestFact(ctx, dec.Slots.Key); err == nil { - if dec.Slots.HasTime { - // The query asks about timing — the fact's own timestamp - // is the answer it's looking for. Format as a natural reply. - reply := fmt.Sprintf("я записала это %s", formatTime(f.Ts)) - return reply - } - // General fact reference: describe what we know. - if dec.Utterance == "" { - return fmt.Sprintf("вот что я знаю: %s — %s", dec.Slots.Key, f.Value) - } - // The utterance still carries the question; fall through to - // normal RAG with the resolved key in context. - } - } - - // Calendar questions: "что у меня сегодня?", "планы на завтра?" - // h.now(), not time.Now(): the handler's clock is the injected one, so - // this arm can be tested at a fixed time like the rest. - if date, ok := router.ParseCalendarDate(dec.Utterance, h.now()); ok { - events, err := h.api.CalendarEvents(ctx, date, date.Add(24*time.Hour)) - if err != nil { - log.Printf("voice: calendar events: %v", err) - return "не получилось проверить календарь." - } - values := make([]string, len(events)) - for i, e := range events { - values[i] = e.Value - } - var f router.CalendarEventFormatter - return f.Format(values, date) - } - - // Weather questions - if isWeatherQuery(dec.Utterance) { - loc := extractWeatherLocation(dec.Utterance, h.weatherLocation) - ctxWT, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - w, err := h.weatherProvider.CurrentWeather(ctxWT, loc) - if errors.Is(err, weather.ErrNotConfigured) { - return "погода не настроена." - } - if err != nil { - log.Printf("voice: weather: %v", err) - return "не получилось узнать погоду." - } - return fmt.Sprintf("в %s сейчас %.0f градусов, %s.", w.Location, w.Temperature, w.Condition) - } - - vec, err := router.EmbedQuery(ctx, h.embedder, dec.Utterance) - if err != nil { - log.Printf("voice: embed query: %v", err) - return "не получилось найти ответ." - } - // Long-term memory first: ONE search over everything Maven remembers - // (notes and facts share this index) and ONE confidence gate, so the - // memory that is clearly the best match answers — a note just as much - // as a fact. - // - // This used to run only after the notes-only gate below had already - // rejected the same note at the same score, which no note could ever - // survive a second time: the branch could only return a fact (#373). - // Order, not the gate, was the bug — the set of questions Maven answers - // is unchanged, only which memory gets to answer them. - if h.memStore != nil { - if hits, herr := h.memStore.Search(ctx, vec, 3); herr == nil { - if hit, ok := bestRecall(hits, h.queryMinScore, h.queryMinMargin); ok { - text := hit.Meta["text"] - // A note is phrased in Maven's voice; a fact is read back - // as it was stored. - if hit.Meta["type"] == "note" { - if reply, perr := h.phraser.PhraseQuery(ctx, dec.Utterance, []string{text}); perr == nil && reply != "" { - return reply - } - } - return text - } - } else { - log.Printf("voice: memory search: %v", herr) - } - } - - notes, err := h.api.QueryNotes(ctx, vec, 5) - if err != nil { - log.Printf("voice: query notes: %v", err) - return "не получилось найти ответ." - } - // Notes-only pass, for notes the vector index above does not hold (an - // older note written before it existed). Same gate, notes-only - // candidates. - // - // Confidence gate: below it, say "I don't know" rather than read back - // the least-unrelated note — a confident wrong recall is worse than a - // gap (spec's "not a guesser-of-truth"). Same instinct as the loop's - // since(key)==null → don't fire. Two parts: an absolute cosine floor, - // and a margin over the runner-up, which is the part that works with - // the e5 embedder's narrow score band. See memory.Confident. - noteScores := make([]float64, len(notes)) - for i, n := range notes { - noteScores[i] = n.Score - } - if !memory.ConfidentScores(noteScores, h.queryMinScore, h.queryMinMargin) { - // Try general knowledge from the phraser before giving up - reply, err := h.phraser.PhraseQuery(ctx, dec.Utterance, nil) - if err != nil || reply == "" { - return "не знаю." - } - return reply - } - texts := make([]string, len(notes)) - for i, n := range notes { - texts[i] = n.Text - } - reply, err := h.phraser.PhraseQuery(ctx, dec.Utterance, texts) - if err != nil { - log.Printf("voice: phrase query: %v", err) - } - if reply == "" { - reply = "вот что я нашла: " + texts[0] - } - return reply + if handler, ok := actionHandlers[dec.Intent]; ok { + return handler(h, ctx, dec) } return "" } -// detectPattern extracts an event from the written fact and runs the pattern -// detector. If a stable recurring pattern is found and no proposed routine -// exists for this action+object yet, one is created and the user is prompted -// to confirm via the park() mechanism. Returns the suggestion phrase when a -// new proposal was created and parked; "" otherwise. -func (h *reactiveHandler) detectPattern(ctx context.Context, factID int64, key, value string, ts time.Time) string { - ev := pattern.Extract(factID, key, value, ts) - if ev == nil { - return "" // not an actionable event - } - if _, err := h.dataStore.CreateEvent(ctx, factID, ev.Action, ev.Object, ts); err != nil { - log.Printf("voice: create event: %v", err) - return "" - } - events, err := h.dataStore.EventsFor(ctx, ev.Action, ev.Object) - if err != nil { - log.Printf("voice: events for %s/%s: %v", ev.Action, ev.Object, err) - return "" - } - // Convert store.Events to pattern.Events for the detector. - patEvents := make([]pattern.Event, len(events)) - for i, e := range events { - patEvents[i] = pattern.Event{ - FactID: e.FactID, - Action: e.Action, - Object: e.Object, - Ts: e.Ts, - } - } - r, err := pattern.Detect(patEvents) - if err != nil { - log.Printf("voice: pattern detect: %v", err) - return "" - } - if r == nil { - return "" // not enough data or intervals too irregular - } - // Check if already proposed/accepted/dismissed for this pair. - existing, err := h.dataStore.LookupProposedRoutine(ctx, r.Action, r.Object) - if err != nil { - log.Printf("voice: lookup proposed routine: %v", err) - return "" - } - if existing != nil { - return "" // already proposed, accepted, or dismissed - } - id, err := h.dataStore.CreateProposedRoutine(ctx, r.Action, r.Object, r.IntervalDays, ts) - if err != nil { - log.Printf("voice: create proposed routine: %v", err) - return "" - } - log.Printf("voice: proposed routine: %s/%s every %.1f days", r.Action, r.Object, r.IntervalDays) - - // Park the proposal for voice confirmation. - phrase := pattern.PhraseRoutine(r) - h.mu.Lock() - h.pendingRoutine = &pendingRoutineConfirm{ - routineID: id, - action: r.Action, - object: r.Object, - interval: r.IntervalDays, - phrase: phrase, - expiry: ts.Add(confirmTTL), - } - h.mu.Unlock() - return phrase -} - -var ruWeekdays = []string{ - "воскресенье", "понедельник", "вторник", "среда", - "четверг", "пятница", "суббота", -} - -var ruMonths = []string{ - "января", "февраля", "марта", "апреля", "мая", "июня", - "июля", "августа", "сентября", "октября", "ноября", "декабря", -} - -// onlyLocalTimeReply — the honest answer when the user asks the time somewhere -// other than here. She only keeps one clock, and saying so is better than -// naming the wrong city's time. -// -// There used to be a city→time-zone table here. It was removed on purpose: the -// user only ever asks for local time, so the table was a second list of cities -// to keep in step with the weather one for no gain. -const onlyLocalTimeReply = "я знаю только местное время, про другие города пока не скажу." - -// notPlaceAfterV — words that follow "в" without naming a place, so -// mentionsUnknownPlace does not mistake them for a city. -var notPlaceAfterV = map[string]bool{ - "данный": true, "данную": true, "этот": true, "эту": true, - "котором": true, "какое": true, "какой": true, "который": true, - "общем": true, "точности": true, "курсе": true, "сутках": true, - "часах": true, "минутах": true, "секундах": true, "неделе": true, -} - -// mentionsUnknownPlace reports whether the question has a "в <слово>" phrase -// that looks like a place we do not know ("который час в киеве"). Used only to -// pick the honest "local time only" reply instead of answering local time as -// if it were the city's. -func mentionsUnknownPlace(u string) bool { - toks := strings.Fields(u) - for i := 0; i+1 < len(toks); i++ { - if toks[i] != "в" && toks[i] != "во" { - continue - } - next := strings.Trim(toks[i+1], ".,?!") - if next == "" || notPlaceAfterV[next] { - continue - } - // A number after "в" is a clock ("в 5 часов"), not a place. - if _, err := strconv.Atoi(strings.SplitN(next, ":", 2)[0]); err == nil { - continue - } - return true - } - return false -} - -// onlyNearDaysReply — she can work out today, tomorrow, the day after and -// yesterday, and nothing further. Said out loud instead of answering today's -// date for a day she did not understand. -const onlyNearDaysReply = "я считаю только сегодня, завтра, послезавтра и вчера — про другие дни пока не скажу." - -// dayWords — day references the calendar parser cannot resolve. A weekday name -// or a "через …" phrase means he asked about a specific other day. -var dayWords = []string{ - "понедельник", "вторник", "сред", "четверг", "пятниц", "суббот", "воскресен", - "через", "monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday", -} - -// mentionsUnknownDay reports whether the question names a day the calendar -// parser could not resolve. Mirror of mentionsUnknownPlace: it exists only to -// pick an honest reply over a confidently wrong one. -// -// Only called after ParseCalendarDate has already failed, so "завтра" and the -// other words it does know never reach here. -func mentionsUnknownDay(u string) bool { - for _, w := range dayWords { - if strings.Contains(u, w) { - return true - } - } - return false -} - -// ruClock renders the clock part of the time reply: "15 часов 4 минуты". -func ruClock(t time.Time) string { - h, m := t.Hour(), t.Minute() - hourWord := ruPlural(h, "час", "часа", "часов") - if m == 0 { - return fmt.Sprintf("%d %s ровно", h, hourWord) - } - return fmt.Sprintf("%d %s %d %s", h, hourWord, m, ruPlural(m, "минута", "минуты", "минут")) -} - -// dayPrefix names the day relative to now ("завтра", "вчера", …) so the date -// reply opens the way a person would say it. -func dayPrefix(now, day time.Time) string { - base := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) - switch int(day.Sub(base).Hours() / 24) { - case -1: - return "вчера" - case 0: - return "сегодня" - case 1: - return "завтра" - case 2: - return "послезавтра" - } - return "это" -} - -func ruPlural(n int, one, two, many string) string { - n = n % 100 - if n > 10 && n < 20 { - return many - } - n = n % 10 - switch n { - case 1: - return one - case 2, 3, 4: - return two - default: - return many - } -} - -// resolveQuietToggle — pre-route keyword check. Returns (reply, true) when -// the utterance is a quiet-on/off command; ("", false) otherwise. Called from -// HandlePushToTalk BEFORE the router so a classifier miscue can't drop it. -func (h *reactiveHandler) resolveQuietToggle(ctx context.Context, text string) (string, bool) { - u := strings.ToLower(strings.TrimSpace(text)) - var on, off bool - // Match as whole-token phrases so "тихий" in "тихий режим включи" still - // catches, but "тихий" alone in "очень тихий сегодня день" doesn't fire. - // The confirm turn is handled above, so "да"/"нет" won't reach here. - for _, kw := range []string{"quiet on", "quiet mode", "тихий режим", "тихий", "не шуми", "не беспокоить", "тихо"} { - if strings.Contains(u, kw) { - on = true - break - } - } - if !on { - for _, kw := range []string{"quiet off", "quiet end", "громкий режим", "шумный режим", "отмени тихий", "выключи тихий", "не тихо"} { - if strings.Contains(u, kw) { - off = true - break - } - } - } - if !on && !off { - return "", false - } - val := "false" - reply := "тихий режим выключен." - if on { - val = "true" - reply = "тихий режим включён. буду реже напоминать." - } - if _, err := h.api.WriteFact(ctx, ipc.WriteFactReq{ - Ts: h.now(), - Kind: "config", - Key: "quiet_hours", - Value: val, - Source: "tap:voice", - Confidence: 1.0, - }); err != nil { - log.Printf("voice: write quiet_hours: %v", err) - return "не получилось переключить тихий режим.", true - } - return reply, true -} - // replySystem answers system-observable queries using the handler's clock // and (in future) system interfaces. The decision's utterance is parsed // for keywords to determine what the user is asking about. @@ -1167,29 +389,6 @@ func (h *reactiveHandler) chatHistory() []dialogue.Turn { return out } -// hasDurationWords checks whether u is asking about elapsed/remaining time -// rather than the current clock — guards replySystem from replying "сейчас -// X часов" to "сколько времени прошло". Mirrors the stage0.go build filter. -func hasDurationWords(u string) bool { - s := strings.ToLower(strings.TrimSpace(u)) - // First-word duration markers (same keywords as timeQueryBuild in stage0). - first := strings.Fields(s) - if len(first) > 0 { - switch first[0] { - case "прошло", "осталось", "пройдет", "минуло", "проходит": - return true - } - } - // Broader duration keywords appearing anywhere in the utterance. - if strings.Contains(s, "прошло") || strings.Contains(s, "осталось") { - return true - } - if strings.Contains(s, " до ") { - return true - } - return false -} - // reply wraps a text reply through TTS to produce a PushToTalkResp. If TTS // fails, the response carries an empty audio + the text — the client can // still display text if it can't play. The routedChannels field is @@ -1206,741 +405,3 @@ func (h *reactiveHandler) reply(ctx context.Context, text string, _ []string) (v } return voice.PushToTalkResp{ReplyText: text, ReplyAudio: audioOut}, nil } - -// pickLLMRouter returns the LLM router when the operator asked for it and there -// is a llama-server to talk to, and nil otherwise. nil is safe: the cascade then -// routes with the classifier, so an unusable setting costs accuracy, not turns. -func pickLLMRouter(enabled bool, c *llm.Client) *router.LLMRouter { - if !enabled { - return nil - } - if c == nil { - log.Printf("voice: voice.llm_router is on but there is no llama-server to route with (the phraser is not an LLM phraser) — using the classifier instead") - return nil - } - log.Printf("voice: LLM router enabled") - return router.NewLLMRouter(c) -} - -// buildRouter constructs the reactive-path router with the given embedder -// and confidence threshold. -// - stage-0 grammars from DefaultActMatcher whose fn allowlist is exactly -// the enabled tool names (actFns) — the router only matches acts the -// executor can run. Empty ⇒ every act refuses at the matcher. -// - The embedder is provided by wireVoice: HashEmbedder (floor) when no -// embedder config is present, or the ONNX multilingual model when -// configured — same interface, one constructor change. -// - 6 bootstrap examples covering the 5 intents + one compound-capture -// placeholder. Spec calls for ~10 per intent at production; this is the -// bootstrapping floor swapped by tuning the seed set later. -// - Threshold is from voice.router_threshold config (default 0.55). -func buildRouter(emb router.Embedder, acts router.ActMatcher, threshold float64, llmR *router.LLMRouter) *router.Router { - cls := router.NewClassifier(emb) - seedClassifier(cls) - grammars := router.DefaultGrammars(acts) - grammars = append(grammars, router.SystemTimeDateGrammars()...) - grammars = append(grammars, router.ReminderGrammar()) - return router.New(router.Config{ - Grammars: grammars, - Classifier: cls, - Extractor: router.Extractor{ - Time: router.NewPythonDateParser(), - Acts: acts, - Facts: router.DefaultFactParser{}, - }, - Threshold: threshold, - LLM: llmR, - }) -} - -// seedDir is the directory containing intent seed files. Each file is named -// .txt and contains one training example per line (blank lines and -// lines starting with # are ignored). Relative to the working directory. -const seedDir = "models/seeds" - -// seedClassifier floors the embedded examples so the cold-boot path -// doesn't return ErrNoIntents. Loads examples from seedDir — one file per -// intent (act.txt, reminder.txt, fact.txt, note.txt, query.txt). When the -// classifier can't decide it falls through to Clarify — the last-resort -// path asks the user to rephrase rather than guessing wrong. -func seedClassifier(c *router.Classifier) { - intents := []router.Intent{ - router.IntentAct, - router.IntentReminder, - router.IntentFact, - router.IntentNote, - router.IntentQuery, - router.IntentChat, - router.IntentSystem, - } - total := 0 - for _, intent := range intents { - n, err := loadSeedFile(c, intent) - if err != nil { - log.Printf("voice: seed %s: %v", intent, err) - continue - } - total += n - } - log.Printf("voice: loaded %d seed examples from %s", total, seedDir) -} - -func loadSeedFile(c *router.Classifier, intent router.Intent) (int, error) { - path := filepath.Join(seedDir, string(intent)+".txt") - f, err := os.Open(path) - if err != nil { - return 0, fmt.Errorf("open %s: %w", path, err) - } - defer f.Close() - - var count int - sc := bufio.NewScanner(f) - for sc.Scan() { - line := strings.TrimSpace(sc.Text()) - if line == "" || strings.HasPrefix(line, "#") { - continue - } - if err := c.AddExample(context.Background(), intent, line); err != nil { - log.Printf("voice: seed %s: skipping %q: %v", intent, line, err) - continue - } - count++ - } - if err := sc.Err(); err != nil { - return count, fmt.Errorf("scan %s: %w", path, err) - } - return count, nil -} - -// handlePraxisAct — dispatches ecosystem tool acts through the Praxis tools API. -// Returns "" when the act is not a Praxis verb (the caller falls through to the -// system command executor). Returns a reply string otherwise. -func (h *reactiveHandler) handlePraxisAct(ctx context.Context, dec router.Decision) string { - if h.ecosystem == nil || h.ecosystem.praxis == nil { - return "" - } - px := h.ecosystem.praxis - fn := dec.Slots.Fn - - // Map verbs and Russian aliases to Praxis tool calls. - // Each case: if the verb matches, call the tool and return a user-facing reply. - switch fn { - case "list_attention", "attention", "внимание", "что требует внимания", "что нового": - items, err := px.ListAttention(ctx, 20) - if err != nil { - log.Printf("ecosystem: praxis attention: %v", err) - return "не могу сейчас узнать, что требует внимания." - } - if len(items) == 0 { - return "ничего не требует внимания." - } - h.recordPraxisTrace(ctx, "list_attention", map[string]any{"count": len(items)}) - var parts []string - for _, item := range items { - title, _ := item["title"].(string) - // importance arrives as JSON number ⇒ float64 over the HTTP contract. - importance, _ := item["importance"].(float64) - rule, _ := item["rule"].(string) - s := title - if importance > 0 { - s += fmt.Sprintf(" (важность %d", int(importance)) - if rule != "" { - s += ": " + rule - } - s += ")" - } - parts = append(parts, s) - - // Speaking an item surfaces it, it does not acknowledge it - // (ECOSYSTEM-SPEC.md §2.3: surfaced != acknowledged). Best-effort: - // a failed surface call must not block delivering the digest. - if id, ok := item["id"].(string); ok && id != "" { - if _, err := px.Surface(ctx, id); err != nil { - log.Printf("ecosystem: praxis surface %s: %v", id, err) - } - } - } - return "требует внимания: " + strings.Join(parts, "; ") - - case "acknowledge_item", "принято", "понял", "поняла": - id := dec.Slots.Value - if id == "" { - return "какой пункт отметить принятым?" - } - if _, err := px.Acknowledge(ctx, id); err != nil { - log.Printf("ecosystem: praxis acknowledge %s: %v", id, err) - return "не получилось отметить принятым." - } - h.recordPraxisTrace(ctx, "acknowledge", map[string]any{"item_id": id}) - return "принято." - - case "resolve_item", "сделано", "готово", "решено": - id := dec.Slots.Value - if id == "" { - return "какой пункт отметить сделанным?" - } - if _, err := px.Resolve(ctx, id); err != nil { - log.Printf("ecosystem: praxis resolve %s: %v", id, err) - return "не получилось отметить сделанным." - } - h.recordPraxisTrace(ctx, "resolve", map[string]any{"item_id": id}) - return "отмечено как сделано." - - case "ignore_item", "игнорировать", "неважно": - id := dec.Slots.Value - if id == "" { - return "какой пункт игнорировать?" - } - if _, err := px.Ignore(ctx, id); err != nil { - log.Printf("ecosystem: praxis ignore %s: %v", id, err) - return "не получилось проигнорировать." - } - h.recordPraxisTrace(ctx, "ignore", map[string]any{"item_id": id}) - return "проигнорировано." - - case "pin_item", "закрепить": - id := dec.Slots.Value - if id == "" { - return "какой пункт закрепить?" - } - if _, err := px.Pin(ctx, id, true); err != nil { - log.Printf("ecosystem: praxis pin %s: %v", id, err) - return "не получилось закрепить." - } - h.recordPraxisTrace(ctx, "pin", map[string]any{"item_id": id}) - return "закреплено." - - case "list_changes", "changes", "изменения", "что изменилось": - changes, err := px.ListChanges(ctx, 20) - if err != nil { - log.Printf("ecosystem: praxis changes: %v", err) - return "не могу сейчас узнать об изменениях." - } - if len(changes) == 0 { - return "нет изменений." - } - h.recordPraxisTrace(ctx, "list_changes", map[string]any{"count": len(changes)}) - var parts []string - for _, c := range changes { - title, _ := c["title"].(string) - typ, _ := c["change_type"].(string) - parts = append(parts, fmt.Sprintf("%s (%s)", title, typ)) - } - return "изменения: " + strings.Join(parts, "; ") - - default: - // Not a Praxis verb — let the caller fall through. - return "" - } -} - -// recordPraxisTrace — writes a fact recording a cross-service ecosystem call. -// The fact is stored with source "praxis:trace" so the proactive loop can -// reference it and the dashboard can display recent ecosystem activity. -func (h *reactiveHandler) recordPraxisTrace(ctx context.Context, operation string, details map[string]any) { - now := h.now() - value := operation - if len(details) > 0 { - if b, err := json.Marshal(details); err == nil { - value = operation + " " + string(b) - } - } - _, _ = h.api.WriteFact(ctx, ipc.WriteFactReq{ - Ts: now, - Kind: "system", - Key: "praxis:" + operation, - Value: value, - Source: "praxis:trace", - Confidence: 1.0, - }) -} - -// handleHexisAct — resolves entity references through Nexus and executes -// matching capabilities through Hexis. Returns a reply string when handled, -// or "" to fall through to the system command executor. -func (h *reactiveHandler) handleHexisAct(ctx context.Context, dec router.Decision) string { - if h.ecosystem == nil { - return "" - } - - // Resolve the utterance text as an entity reference through Nexus. An - // ambiguous match must stop and clarify — never guess a mutation target. - entityID, displayName, ambiguous, err := h.ecosystem.resolveEntityReference(ctx, dec.Slots.Text, nil) - if err != nil { - // A genuine Nexus dependency failure, not "no such entity" — stop here - // and report degradation rather than silently falling through to the - // local command executor (ECOSYSTEM-SPEC.md: services degrade - // independently, never a silent all-clear). - return "экосистема недоступна, попробуй ещё раз." - } - if len(ambiguous) > 0 { - return "уточни, что именно: " + strings.Join(ambiguous, ", ") + "?" - } - if entityID == "" { - return "" - } - - // Discover Hexis capabilities for this entity. A resolved entity with a - // genuine Hexis failure must not be treated as "no capabilities" and - // fall through to unrelated local execution. - caps, err := h.ecosystem.discoverCapabilities(ctx, entityID) - if err != nil { - return "экосистема недоступна, попробуй ещё раз." - } - if len(caps) == 0 { - return "" - } - - // Match the user's verb to a capability by name/description. Collect all - // matches: more than one is itself ambiguous, so we ask rather than pick - // the first (ecosystem invariant: no arbitrary target for mutation). - verb := dec.Slots.Fn - if verb == "" { - verb = dec.Slots.Text - } - verbLower := strings.ToLower(verb) - - var matches []*hexisclient.Capability - for i, c := range caps { - if strings.Contains(strings.ToLower(c.Name), verbLower) || - (c.Description != "" && strings.Contains(strings.ToLower(c.Description), verbLower)) { - matches = append(matches, &caps[i]) - } - } - if len(matches) == 0 { - return "" - } - if len(matches) > 1 { - var names []string - for _, m := range matches { - names = append(names, m.Name) - } - return "какую команду для " + displayName + ": " + strings.Join(names, ", ") + "?" - } - matched := matches[0] - - // Read-only capabilities run immediately; mutating ones are parked for an - // explicit spoken confirm bound to this capability + target. - if !matched.ReadOnly { - h.mu.Lock() - h.pendingHexis = &pendingHexisExec{ - capabilityID: matched.ID, - capName: matched.Name, - entityID: entityID, - displayName: displayName, - expiry: h.now().Add(confirmTTL), - } - h.mu.Unlock() - return "выполнить «" + matched.Name + "» для " + displayName + "? скажи «да» или «нет»." - } - - return h.execHexis(ctx, matched.ID, matched.Name, entityID, displayName) -} - -// execHexis runs a resolved capability and records a cross-service trace with -// the correlation ID. It reports command success, never operational recovery -// (Praxis observes recovery independently). -func (h *reactiveHandler) execHexis(ctx context.Context, capID, capName, entityID, displayName string) string { - correlationID, err := h.ecosystem.executeCapability(ctx, capID, entityID, nil) - if err != nil { - log.Printf("ecosystem: hexis execute error (cor=%s): %v", correlationID, err) - return "не получилось выполнить команду для " + displayName + "." - } - h.recordPraxisTrace(ctx, "hexis:"+capName, map[string]any{ - "entity_id": entityID, - "entity_name": displayName, - "capability": capName, - "correlation_id": correlationID, - }) - return "команда выполнена для " + displayName + "." -} - -// jsonString — a one-line JSON string encoder without dragging encoding/json -// into the top of this file. Used to wrap a reminder payload's text field; -// the router's reminder Slots are already absolute (DateTimeParser resolved -// relative→absolute), the payload shape is conventional {"text":...}. -func jsonString(s string) string { - return jsonStringImpl(s) -} - -// park stores a destructive act awaiting confirmation. Overwrites any prior -// pending (last-asked wins — single-user box). -func (h *reactiveHandler) park(fn string, args []string, phrase string) { - h.mu.Lock() - h.pending = &pendingAct{fn: fn, args: args, phrase: phrase, expiry: h.now().Add(confirmTTL)} - h.mu.Unlock() -} - -// resolveConfirm interprets an utterance as the answer to a parked destructive -// act OR a parked routine proposal. Returns (reply, true) when it consumed the -// utterance as a y/n answer; ("", false) when there's nothing pending (or the -// parked act expired), so the caller routes the utterance normally. An -// unrecognised answer cancels the pending and routes normally — a confirm that -// can't be answered clearly is safer abandoned than left armed. -func (h *reactiveHandler) resolveConfirm(ctx context.Context, text string) (string, bool) { - h.mu.Lock() - defer h.mu.Unlock() - - // Check routine proposal first (newer feature; checked before tool confirm - // so a routine confirm doesn't get eaten by a stale tool pending). - pr := h.pendingRoutine - if pr != nil && !h.now().After(pr.expiry) { - switch classifyConfirm(text) { - case confirmYes: - h.pendingRoutine = nil - // Only record the acceptance. The tick loop reads accepted - // routines and nudges on their own interval. Building a reminder - // here made a routine fire exactly once (Vikunja #366). - if err := h.dataStore.AcceptProposedRoutine(ctx, pr.routineID, h.now()); err != nil { - log.Printf("voice: accept proposed routine: %v", err) - return "не получилось запомнить рутину.", true - } - return "буду напоминать.", true - case confirmNo: - h.pendingRoutine = nil - if err := h.dataStore.DismissProposedRoutine(ctx, pr.routineID); err != nil { - log.Printf("voice: dismiss proposed routine: %v", err) - } - return "хорошо, не буду.", true - default: - // unclear: abandon the routine proposal, route normally. - h.pendingRoutine = nil - return "", false - } - } - // Clear expired routine if it existed. - if pr != nil { - h.pendingRoutine = nil - } - - // Check pending Hexis execution confirm. Bound to the exact capability + - // target that was proposed; a stray "да" can only run that, nothing else. - if hx := h.pendingHexis; hx != nil { - if h.now().After(hx.expiry) { - h.pendingHexis = nil - } else { - switch classifyConfirm(text) { - case confirmYes: - h.pendingHexis = nil - return h.execHexis(ctx, hx.capabilityID, hx.capName, hx.entityID, hx.displayName), true - case confirmNo: - h.pendingHexis = nil - return "отменила.", true - default: - h.pendingHexis = nil - return "", false - } - } - } - - // Check tool confirm (existing behavior). - p := h.pending - if p == nil { - return "", false - } - if h.now().After(p.expiry) { - h.pending = nil - return "", false - } - switch classifyConfirm(text) { - case confirmYes: - h.pending = nil - out, err := h.tools.Exec(ctx, p.fn, p.args, true) // confirmed - if err != nil { - log.Printf("voice: tool %s (confirmed): %v", p.fn, err) - if out != "" { - return "не получилось выполнить команду: " + firstLine(out), true - } - return "не получилось выполнить команду.", true - } - if out != "" { - return "готово: " + firstLine(out), true - } - return "готово.", true - case confirmNo: - h.pending = nil - return "отменила.", true - default: - // unclear answer: abandon the confirm, route this utterance normally. - h.pending = nil - return "", false - } -} - -// proposeGap scaffolds a 'proposed' tool for an act whose verb isn't enabled. -// maven drafts the registration (name = the verb, provenance = the utterance); -// a human enables it on the authed surface. She suggests, never enables. -func (h *reactiveHandler) proposeGap(ctx context.Context, dec router.Decision) string { - name := firstWord(stripWake(dec.Utterance)) - if name == "" { - return "не разобрала команду — попробуй иначе." - } - newly, err := h.api.ProposeTool(ctx, name, dec.Utterance, "", h.now()) - if err != nil { - log.Printf("voice: propose tool %q: %v", name, err) - return "команды «" + name + "» нет в списке разрешённых." - } - if newly { - return "команды «" + name + "» нет в списке. Предложила её добавить — включи через клиент." - } - return "команды «" + name + "» пока нет в списке — она уже предложена, включи через клиент." -} - -// confirmVerdict — the parse of a y/n confirm answer. -type confirmVerdict int - -const ( - confirmUnknown confirmVerdict = iota - confirmYes - confirmNo -) - -// classifyConfirm reads a short ru/en yes-or-no answer. Substring match on the -// stems so inflections/fillers ("да, давай", "нет, отмени") still land. -func classifyConfirm(text string) confirmVerdict { - t := strings.ToLower(strings.TrimSpace(text)) - // negatives first — "не надо" contains no "да", but check no-stems before - // yes so a leading "нет" isn't shadowed. - for _, no := range []string{"нет", "не надо", "отмен", "стоп", "no", "cancel", "stop", "don't"} { - if strings.Contains(t, no) { - return confirmNo - } - } - for _, yes := range []string{"да", "ага", "давай", "подтвер", "конечно", "yes", "yeah", "yep", "confirm", "ок", "okay", "ok"} { - if strings.Contains(t, yes) { - return confirmYes - } - } - return confirmUnknown -} - -// actPhrase renders "fn arg1 arg2" for the confirm prompt. -func actPhrase(fn string, args []string) string { - if len(args) == 0 { - return fn - } - return fn + " " + strings.Join(args, " ") -} - -// stripWake removes a leading wake token (any script the STT phonetically -// transcribes "Maven" as) so the verb is the first word. -func stripWake(u string) string { - stripped, had := router.StripWakeToken(u) - if !had { - return strings.TrimSpace(u) - } - return stripped -} - -// firstWord returns the first whitespace-delimited token (lowercased) — the -// proposed tool's name. -func firstWord(s string) string { - f := strings.Fields(s) - if len(f) == 0 { - return "" - } - return strings.ToLower(f[0]) -} - -// seedTools upserts the config-declared tools into the store as enabled. Editing -// mavend.json is a human act, so a config tool is enabled by definition; this -// makes the declarative config the reproducible bootstrap while the store stays -// the single runtime source of truth (mavweb enables ad-hoc ones on top). -func seedTools(api ipc.CoreAPI, tools []config.ToolConfig) { - ctx := context.Background() - now := time.Now() - n := 0 - for _, tc := range tools { - if tc.Name == "" || len(tc.Cmd) == 0 { - log.Printf("voice: skipping malformed tool config %+v", tc) - continue - } - if err := api.EnableTool(ctx, tc.Name, tc.Cmd, tc.Destructive, tc.Scope, now); err != nil { - log.Printf("voice: seed tool %q: %v", tc.Name, err) - continue - } - n++ - } - log.Printf("voice: seeded %d act tools from config", n) -} - -// firstLine — the first non-empty line of a tool's output, for a short spoken -// reply (the full output goes to the log, not the TTS). Trimmed to keep the -// utterance sane if a command dumps a wall of text. -func firstLine(s string) string { - for _, line := range strings.Split(s, "\n") { - line = strings.TrimSpace(line) - if line != "" { - if len(line) > 200 { - line = line[:200] - } - return line - } - } - return "" -} - -// isWeatherQuery returns true if the utterance is about weather. -func isWeatherQuery(u string) bool { - lower := strings.ToLower(u) - return strings.Contains(lower, "погод") || - strings.Contains(lower, "градус") || - strings.Contains(lower, "температур") || - strings.Contains(lower, "дожд") || - strings.Contains(lower, "холод") || - strings.Contains(lower, "тепл") || - strings.Contains(lower, "weather") || - strings.Contains(lower, "temperature") -} - -// extractWeatherLocation parses a location from the utterance, or falls back -// to the configured default. Very basic: just checks for known city names. -func extractWeatherLocation(u, defaultLoc string) string { - lower := strings.ToLower(u) - cities := map[string]string{ - "москв": "Moscow", - "moscow": "Moscow", - "питер": "Saint Petersburg", - "spb": "Saint Petersburg", - "петербур": "Saint Petersburg", - "лондон": "London", - "london": "London", - "париж": "Paris", - "paris": "Paris", - "берлин": "Berlin", - "berlin": "Berlin", - "нью-йорк": "New York", - "new york": "New York", - } - for substr, name := range cities { - if strings.Contains(lower, substr) { - return name - } - } - if defaultLoc != "" { - return defaultLoc - } - return "Moscow" -} - -// formatTime returns a human-readable Russian time string for a fact timestamp. -// Used by the query handler when answering "когда я это сделал?"-style questions. -func formatTime(t time.Time) string { - now := time.Now() - if t.After(now.Add(-2*time.Minute)) && t.Before(now.Add(2*time.Minute)) { - return "только что" - } - diff := now.Sub(t) - switch { - case diff < 10*time.Minute: - return "несколько минут назад" - case diff < 60*time.Minute: - return fmt.Sprintf("%d минут назад", int(diff.Minutes())) - case diff < 2*time.Hour: - return "час назад" - case diff < 24*time.Hour: - return fmt.Sprintf("%d часа назад", int(diff.Hours())) - default: - return t.Format("2 января 15:04") - } -} - -func jsonStringImpl(s string) string { - // minimal JSON string escape — quotes + backslash + control chars. - // adequate for the reminder payload's text field; not a general JSON - // encoder. The chroma / RAG modules (when they land) use a real json - // encoder for richer payloads. Keep it inline here so the import - // direction stays narrow. - var b []byte - b = append(b, '"') - for _, r := range s { - switch r { - case '"': - b = append(b, '\\', '"') - case '\\': - b = append(b, '\\', '\\') - case '\n': - b = append(b, '\\', 'n') - case '\r': - b = append(b, '\\', 'r') - case '\t': - b = append(b, '\\', 't') - default: - if r < 0x20 { - b = append(b, []byte(fmt.Sprintf("\\u%04x", r))...) - } else { - b = append(b, []byte(string(r))...) - } - } - } - b = append(b, '"') - return string(b) -} - -// reembedOnStart is the -reembed flag (set in run()). Opt-in on purpose: see -// runReembed. -var reembedOnStart bool - -// checkStoredEmbedder compares the embedder we just loaded with the one that -// wrote the vectors already in the DB (Vikunja #378). -// -// The two models we have both make 384-dim vectors, so a size check catches -// nothing: after a swap, recall silently compares vectors from different -// spaces and the scores are noise. So we say it out loud. Recall itself is not -// changed here — the fix is `mavend -reembed`. -func checkStoredEmbedder(dataStore *store.Store, emb router.Embedder) { - if dataStore == nil { - return - } - current := router.EmbedderID(emb) - if reembedOnStart { - runReembed(dataStore, emb, current) - return - } - stored, mismatch, err := dataStore.CheckEmbedder(context.Background(), current) - if err != nil { - log.Printf("voice: embedder marker check failed: %v", err) - return - } - if mismatch { - log.Printf("voice: WARNING embedder MISMATCH — stored vectors were written by %q but the configured embedder is %q; recall scores are noise until the notes and facts are re-embedded — run `mavend -reembed` once (Vikunja #378)", stored, current) - return - } - log.Printf("voice: embedder marker ok (%s)", current) -} - -// runReembed is the one-shot backfill behind -reembed. -// -// Why a flag and not automatic on mismatch: the embedder is ONNX on the -// laptop's CPU, so a few thousand notes is minutes of work. Doing that silently -// inside a normal start would look like the daemon hanging on boot. So the user -// runs it once, deliberately, after an embedder swap; the mismatch warning -// above tells them to. It re-embeds, logs what it did, and then the daemon -// carries on serving as usual — no separate binary, no second start needed. -func runReembed(dataStore *store.Store, emb router.Embedder, current string) { - log.Printf("voice: re-embedding stored notes and facts with %s — this can take a few minutes, do not interrupt", current) - res, err := dataStore.ReembedAll(context.Background(), current, - // EmbedPassage, not EmbedQuery: these are stored texts being searched - // FOR, which is the side they were written with. - func(ctx context.Context, text string) ([]float32, error) { - return router.EmbedPassage(ctx, emb, text) - }) - if err != nil { - log.Printf("voice: re-embed FAILED, nothing was changed and no marker was written — safe to run again: %v", err) - return - } - if res.Skipped { - log.Printf("voice: re-embed skipped — the stored vectors were already written by %s", current) - return - } - log.Printf("voice: re-embed done — %d notes in the notes table, %d notes and %d facts in the memory index, took %s; stored vectors now belong to %s", - res.Notes, res.MemNotes, res.Facts, res.Took.Round(time.Second), current) - - // A row with no text cannot be re-embedded, so its vector is still the old - // model's noise while the marker now says everything is current. Both write - // paths always store the text, so this should be zero — say it loudly - // rather than bury it in the line above if it ever isn't. - if res.NoText > 0 { - log.Printf("voice: WARNING %d stored rows had no text, so their vectors could not be re-embedded and are still noise; they will never match anything useful (Vikunja #378)", res.NoText) - } -} diff --git a/cmd/mavend/voicewire.go b/cmd/mavend/voicewire.go new file mode 100644 index 0000000..1adf273 --- /dev/null +++ b/cmd/mavend/voicewire.go @@ -0,0 +1,478 @@ +package main + +import ( + "bufio" + "context" + "fmt" + "log" + "os" + "path/filepath" + "strings" + "time" + + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/delivery" + "github.com/kami/maven/internal/delivery/voicesink" + "github.com/kami/maven/internal/dialogue" + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/llm" + "github.com/kami/maven/internal/memory" + "github.com/kami/maven/internal/phraser" + "github.com/kami/maven/internal/router" + "github.com/kami/maven/internal/store" + "github.com/kami/maven/internal/stt" + "github.com/kami/maven/internal/tool" + "github.com/kami/maven/internal/tts" + "github.com/kami/maven/internal/voice" + "github.com/kami/maven/internal/weather" + "github.com/kami/maven/internal/worker" +) + +// voiceWiring — everything the daemon needs to run the audio path. Held by +// cmd/mavend/main.go alongside the other wirings; closed on shutdown. +type voiceWiring struct { + server *voice.Server + sessions *voice.Sessions + voiceSink delivery.Sink + embedder router.Embedder + handler *reactiveHandler // the reactive handler for IPC Chat + // worker clients (set when configured as Remote): closed on shutdown so + // mavsttd / mavttsd don't keep a stale conn into a restarting daemon. + sttClient *worker.Client + ttsClient *worker.Client + // transcriber — the STT in use, exposed so the meeting recorder + // (cmd/mavend/capture.go) can reuse it. Maven has exactly one STT and does + // not grow a second one for capture: this is the same whisper.cpp worker the + // voice path talks to. + transcriber stt.Transcriber + // mcp — the MCP client, nil unless the `mcp` block configures an enabled + // server (Vikunja #251). Its tools land in the same allowlist as every + // other act, so nothing else here has to know about it. + mcp *mcpWiring + // home — the Home Assistant client, nil unless the `smarthome` block is + // enabled (Vikunja #256). Its devices land in the same allowlist as every + // other act, so nothing else here has to know about it. + home *homeWiring + // netscan — the LAN scanner, nil unless the `netscan` block is enabled + // (Vikunja #257). + netscan *netWiring +} + +// close releases the listener + worker conns. Safe to call on nil (when +// voice is not wired — wireVoice returns nil,nil). +func (w *voiceWiring) close() { + if w == nil { + return + } + if w.embedder != nil { + _ = w.embedder.Close() + } + if w.server != nil { + _ = w.server.Close() + } + if w.sttClient != nil { + _ = w.sttClient.Close() + } + if w.ttsClient != nil { + _ = w.ttsClient.Close() + } + w.mcp.close() +} + +// wireVoice builds the audio path from cfg + a CoreAPI + a router. Returns +// nil wiring + nil error when voice isn't enabled (the caller's voice sink +// stays nil; the dispatcher's ChannelVoice routing drops silently). +// +// When voice is enabled, MUST wire a voicesink into the dispatcher's Voice +// slot using w.sessions (the caller does that — see main.go). +func wireVoice(cfg *config.Config, coreAPI ipc.CoreAPI, phr phraser.Phraser, memStore memory.Store, dataStore *store.Store, eco *ecosystemWiring) (*voiceWiring, error) { + if cfg.Voice == nil || !cfg.Voice.Enabled { + return nil, nil + } + w := &voiceWiring{} + + // ----- stt (Stub in-process OR Remote via worker socket) ----- + var transcriber stt.Transcriber + if cfg.Voice.Stt != nil && cfg.Voice.Stt.Socket != "" { + c := worker.Dial(cfg.Voice.Stt.Socket) + w.sttClient = c + lang := cfg.Voice.Stt.Lang + if lang == "" { + lang = cfg.Voice.Lang + } + transcriber = stt.NewRemote(c, lang) + } else { + transcriber = stt.NewStub() + } + w.transcriber = transcriber + + // ----- tts (Stub in-process OR Remote) ----- + var synthesizer tts.Synthesizer + if cfg.Voice.Tts != nil && cfg.Voice.Tts.Socket != "" { + c := worker.Dial(cfg.Voice.Tts.Socket) + w.ttsClient = c + lang := cfg.Voice.Tts.Lang + if lang == "" { + lang = cfg.Voice.Lang + } + synthesizer = tts.NewRemote(c, lang, cfg.Voice.Tts.Voice) + } else { + synthesizer = tts.NewStub() + } + + // ----- router: embedder (ONNX when configured, floor HashEmbedder otherwise) ----- + var emb router.Embedder + if cfg.Voice.Embedder != nil { + onnx, err := router.NewONNXEmbedder( + cfg.Voice.Embedder.ModelPath, + cfg.Voice.Embedder.TokenizerPath, + cfg.Voice.Embedder.LibPath, + ) + if err != nil { + w.close() + return nil, fmt.Errorf("embedder: %w", err) + } + log.Printf("voice: onnx embedder loaded (%d dim)", onnx.Dim()) + emb = onnx + } else { + log.Printf("voice: embedder not configured, using HashEmbedder floor") + emb = router.NewHashEmbedder(1024) + } + w.embedder = emb + checkStoredEmbedder(dataStore, emb) + + // ----- tool executor (the enabled act allowlist, store-backed) ----- + // Config tools are the declarative bootstrap: seed them into the store as + // enabled (editing mavend.json IS the human enable act). Ad-hoc tools are + // enabled later through the authed mavweb surface. The executor + matcher + // both read the store live, so a newly-enabled tool is runnable without a + // daemon restart. + seedTools(coreAPI, cfg.Voice.Tools) + exec := tool.NewExecutor(coreAPI, time.Duration(cfg.Voice.ToolTimeout)) + // MCP servers (Vikunja #251): discovery PROPOSES tools into the same + // allowlist, so an MCP tool is enabled by hand on /tools like any other and + // runs through the same confirm turn. Off unless the `mcp` block configures + // an enabled server. + w.mcp = wireMCP(cfg, dataStore) + if w.mcp != nil { + exec = exec.WithMCP(w.mcp.caller()) + } + // The house (Vikunja #256): same story as MCP. Discovery PROPOSES a row per + // controllable device, always destructive, and Kami enables the ones he + // wants on /tools. Off unless the `smarthome` block is enabled. + w.home = wireSmartHome(cfg, dataStore) + if w.home != nil { + exec = exec.WithHome(w.home.caller()) + } + // The LAN scanner (Vikunja #257): a read, bounded to the configured + // subnets and rate-limited. Off unless the `netscan` block is enabled. + w.netscan = wireNetScan(cfg, coreAPI) + matcher := tool.NewMatcher(coreAPI) + + // ----- weather provider (Open-Meteo when configured, Stub otherwise) ----- + var weatherProvider weather.Provider + var weatherLocation string + if cfg.Voice.Weather != nil && cfg.Voice.Weather.Provider == "open-meteo" { + weatherProvider = weather.NewOpenMeteoProvider() + weatherLocation = cfg.Voice.Weather.DefaultLocation + log.Printf("voice: weather provider: open-meteo (default location: %s)", cfg.Voice.Weather.DefaultLocation) + } else { + weatherProvider = weather.NewStubProvider() + log.Printf("voice: weather provider: stub (not configured)") + } + + // The replier uses the same llama-server as the phraser. + var llmClient *llm.Client + if lp, ok := phr.(*phraser.LLMPhraser); ok { + // llmClientFor, not llm.New: this client must follow the phraser onto + // the new llama-server when the resident model is swapped (Vikunja #250). + llmClient = llmClientFor(lp, 60*time.Second) + } + // ----- router (the cascade; floor examples seed the classifier) ----- + // The act matcher's allowlist is exactly the enabled tool names — the + // router only matches acts the executor can run (one source of truth). + threshold := cfg.Voice.RouterThreshold + if threshold <= 0 { + threshold = config.DefaultRouterThreshold + } + // The resident model routes by default: 63.2% of held-out intents right + // against the classifier's 50.0%, at about 1s a turn instead of 30ms (see + // config.VoiceConfig.LLMRouter). The classifier always stays wired as the + // fallback, so a model error never breaks a turn. + rtr := buildRouter(emb, matcher, threshold, pickLLMRouter(cfg.Voice.UseLLMRouter(), llmClient)) + + // ----- sessions registry (shared with voicesink) ----- + sessions := voice.NewSessions() + w.sessions = sessions + + // ----- voice sink (proactive nudges: dispatcher → voicesink → tts → push to client) ----- + w.voiceSink = voicesink.New(synthesizer, sessions) + + // ----- memory (long-term vector storage) ----- + // Persistent (store-backed, survives restarts) when the daemon passes one; + // falls back to the in-memory floor otherwise (tests / no-store paths). + if memStore == nil { + memStore = memory.NewInMemoryStore() + } + + // ----- dialogue (multi-turn slot carry-over; 2-min follow-up window) ----- + // Store-backed when the daemon passes a store, so a restart mid-conversation + // keeps the thread (Vikunja #363). Sessions past their TTL are dropped on + // load, never revived. Clarify's parked question stays in memory only. + var dialogueSessions *dialogue.SessionStore + if dataStore != nil { + dialogueSessions = dialogue.NewPersistentSessionStore(2*time.Minute, dataStore) + if err := dialogueSessions.Load(context.Background(), time.Now()); err != nil { + log.Printf("dialogue: load saved sessions: %v", err) + } + } else { + dialogueSessions = dialogue.NewSessionStore(2 * time.Minute) + } + clarifyStore := dialogue.NewClarifyStore(clarifyTTL) + timeParser := router.NewPythonDateParser() + + // ----- replier (LLM-backed when the engine is on, Stub floor otherwise) ----- + replier := voice.Replier(voice.NewStubReplier()) + if llmClient != nil { + replier = newLLMReplier(llmClient, contextBlockFn(cfg, time.Now)) + } + + // ----- the handler (the reactive path; closes over stt / tts / router / coreAPI / memory) ----- + h := &reactiveHandler{ + stt: transcriber, + tts: synthesizer, + router: rtr, + embedder: emb, + api: coreAPI, + tools: exec, + matcher: matcher, + replier: replier, + phraser: phr, + now: time.Now, + feedsOn: cfg.Feeds != nil, + home: w.home, + netscan: w.netscan, + // nil unless `crawl.on_demand` is on: reading a page he names is a + // capability, and capabilities are off unless configured. + crawler: onDemandCrawler(cfg), + weatherProvider: weatherProvider, + weatherLocation: weatherLocation, + memStore: memStore, + dataStore: dataStore, + dialogueSessions: dialogueSessions, + clarifyStore: clarifyStore, + // 0 here (unset config) ⇒ the dialogue default. + clarifyMaxAttempts: cfg.Voice.ClarifyMaxAttempts, + extractor: router.Extractor{Time: timeParser, Acts: matcher, Facts: router.DefaultFactParser{}}, + queryMinScore: cfg.Voice.QueryMinScore, + queryMinMargin: cfg.Voice.QueryMinMargin, + timeParser: timeParser, + ecosystem: eco, + } + + // ----- the server (TCP listener) ----- + srv := voice.NewServer(cfg.Voice.Bind, h, sessions) + if err := srv.Listen(); err != nil { + w.close() + return nil, fmt.Errorf("voice listen: %w", err) + } + w.server = srv + w.handler = h + + return w, nil +} + +// pickLLMRouter returns the LLM router when the operator asked for it and there +// is a llama-server to talk to, and nil otherwise. nil is safe: the cascade then +// routes with the classifier, so an unusable setting costs accuracy, not turns. +func pickLLMRouter(enabled bool, c *llm.Client) *router.LLMRouter { + if !enabled { + return nil + } + if c == nil { + log.Printf("voice: voice.llm_router is on but there is no llama-server to route with (the phraser is not an LLM phraser) — using the classifier instead") + return nil + } + log.Printf("voice: LLM router enabled") + return router.NewLLMRouter(c) +} + +// buildRouter constructs the reactive-path router with the given embedder +// and confidence threshold. +// - stage-0 grammars from DefaultActMatcher whose fn allowlist is exactly +// the enabled tool names (actFns) — the router only matches acts the +// executor can run. Empty ⇒ every act refuses at the matcher. +// - The embedder is provided by wireVoice: HashEmbedder (floor) when no +// embedder config is present, or the ONNX multilingual model when +// configured — same interface, one constructor change. +// - 6 bootstrap examples covering the 5 intents + one compound-capture +// placeholder. Spec calls for ~10 per intent at production; this is the +// bootstrapping floor swapped by tuning the seed set later. +// - Threshold is from voice.router_threshold config (default 0.55). +func buildRouter(emb router.Embedder, acts router.ActMatcher, threshold float64, llmR *router.LLMRouter) *router.Router { + cls := router.NewClassifier(emb) + seedClassifier(cls) + grammars := router.DefaultGrammars(acts) + grammars = append(grammars, router.SystemTimeDateGrammars()...) + grammars = append(grammars, router.ReminderGrammar()) + return router.New(router.Config{ + Grammars: grammars, + Classifier: cls, + Extractor: router.Extractor{ + Time: router.NewPythonDateParser(), + Acts: acts, + Facts: router.DefaultFactParser{}, + }, + Threshold: threshold, + LLM: llmR, + }) +} + +// seedDir is the directory containing intent seed files. Each file is named +// .txt and contains one training example per line (blank lines and +// lines starting with # are ignored). Relative to the working directory. +const seedDir = "models/seeds" + +// seedClassifier floors the embedded examples so the cold-boot path +// doesn't return ErrNoIntents. Loads examples from seedDir — one file per +// intent (act.txt, reminder.txt, fact.txt, note.txt, query.txt). When the +// classifier can't decide it falls through to Clarify — the last-resort +// path asks the user to rephrase rather than guessing wrong. +func seedClassifier(c *router.Classifier) { + intents := []router.Intent{ + router.IntentAct, + router.IntentReminder, + router.IntentFact, + router.IntentNote, + router.IntentQuery, + router.IntentChat, + router.IntentSystem, + } + total := 0 + for _, intent := range intents { + n, err := loadSeedFile(c, intent) + if err != nil { + log.Printf("voice: seed %s: %v", intent, err) + continue + } + total += n + } + log.Printf("voice: loaded %d seed examples from %s", total, seedDir) +} + +func loadSeedFile(c *router.Classifier, intent router.Intent) (int, error) { + path := filepath.Join(seedDir, string(intent)+".txt") + f, err := os.Open(path) + if err != nil { + return 0, fmt.Errorf("open %s: %w", path, err) + } + defer f.Close() + + var count int + sc := bufio.NewScanner(f) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + if err := c.AddExample(context.Background(), intent, line); err != nil { + log.Printf("voice: seed %s: skipping %q: %v", intent, line, err) + continue + } + count++ + } + if err := sc.Err(); err != nil { + return count, fmt.Errorf("scan %s: %w", path, err) + } + return count, nil +} + +// seedTools upserts the config-declared tools into the store as enabled. Editing +// mavend.json is a human act, so a config tool is enabled by definition; this +// makes the declarative config the reproducible bootstrap while the store stays +// the single runtime source of truth (mavweb enables ad-hoc ones on top). +func seedTools(api ipc.CoreAPI, tools []config.ToolConfig) { + ctx := context.Background() + now := time.Now() + n := 0 + for _, tc := range tools { + if tc.Name == "" || len(tc.Cmd) == 0 { + log.Printf("voice: skipping malformed tool config %+v", tc) + continue + } + if err := api.EnableTool(ctx, tc.Name, tc.Cmd, tc.Destructive, tc.Scope, now); err != nil { + log.Printf("voice: seed tool %q: %v", tc.Name, err) + continue + } + n++ + } + log.Printf("voice: seeded %d act tools from config", n) +} + +// reembedOnStart is the -reembed flag (set in run()). Opt-in on purpose: see +// runReembed. +var reembedOnStart bool + +// checkStoredEmbedder compares the embedder we just loaded with the one that +// wrote the vectors already in the DB (Vikunja #378). +// +// The two models we have both make 384-dim vectors, so a size check catches +// nothing: after a swap, recall silently compares vectors from different +// spaces and the scores are noise. So we say it out loud. Recall itself is not +// changed here — the fix is `mavend -reembed`. +func checkStoredEmbedder(dataStore *store.Store, emb router.Embedder) { + if dataStore == nil { + return + } + current := router.EmbedderID(emb) + if reembedOnStart { + runReembed(dataStore, emb, current) + return + } + stored, mismatch, err := dataStore.CheckEmbedder(context.Background(), current) + if err != nil { + log.Printf("voice: embedder marker check failed: %v", err) + return + } + if mismatch { + log.Printf("voice: WARNING embedder MISMATCH — stored vectors were written by %q but the configured embedder is %q; recall scores are noise until the notes and facts are re-embedded — run `mavend -reembed` once (Vikunja #378)", stored, current) + return + } + log.Printf("voice: embedder marker ok (%s)", current) +} + +// runReembed is the one-shot backfill behind -reembed. +// +// Why a flag and not automatic on mismatch: the embedder is ONNX on the +// laptop's CPU, so a few thousand notes is minutes of work. Doing that silently +// inside a normal start would look like the daemon hanging on boot. So the user +// runs it once, deliberately, after an embedder swap; the mismatch warning +// above tells them to. It re-embeds, logs what it did, and then the daemon +// carries on serving as usual — no separate binary, no second start needed. +func runReembed(dataStore *store.Store, emb router.Embedder, current string) { + log.Printf("voice: re-embedding stored notes and facts with %s — this can take a few minutes, do not interrupt", current) + res, err := dataStore.ReembedAll(context.Background(), current, + // EmbedPassage, not EmbedQuery: these are stored texts being searched + // FOR, which is the side they were written with. + func(ctx context.Context, text string) ([]float32, error) { + return router.EmbedPassage(ctx, emb, text) + }) + if err != nil { + log.Printf("voice: re-embed FAILED, nothing was changed and no marker was written — safe to run again: %v", err) + return + } + if res.Skipped { + log.Printf("voice: re-embed skipped — the stored vectors were already written by %s", current) + return + } + log.Printf("voice: re-embed done — %d notes in the notes table, %d notes and %d facts in the memory index, took %s; stored vectors now belong to %s", + res.Notes, res.MemNotes, res.Facts, res.Took.Round(time.Second), current) + + // A row with no text cannot be re-embedded, so its vector is still the old + // model's noise while the marker now says everything is current. Both write + // paths always store the text, so this should be zero — say it loudly + // rather than bury it in the line above if it ever isn't. + if res.NoText > 0 { + log.Printf("voice: WARNING %d stored rows had no text, so their vectors could not be re-embedded and are still noise; they will never match anything useful (Vikunja #378)", res.NoText) + } +} diff --git a/cmd/mavend/weatherq.go b/cmd/mavend/weatherq.go new file mode 100644 index 0000000..344e2fa --- /dev/null +++ b/cmd/mavend/weatherq.go @@ -0,0 +1,58 @@ +// Package main — weatherq.go holds the weather-query keyword helpers: does +// this utterance ask about weather at all, and which city (if any) did it +// name. Both are plain substring/lookup matching, not NLU — extend this file +// rather than voice.go for anything in that shape. +package main + +import "strings" + +// isWeatherQuery returns true if the utterance is about weather. +func isWeatherQuery(u string) bool { + lower := strings.ToLower(u) + return strings.Contains(lower, "погод") || + strings.Contains(lower, "градус") || + strings.Contains(lower, "температур") || + strings.Contains(lower, "дожд") || + strings.Contains(lower, "холод") || + strings.Contains(lower, "тепл") || + strings.Contains(lower, "weather") || + strings.Contains(lower, "temperature") +} + +// weatherCities — the city names an utterance may name explicitly, as +// lowercase substrings mapped to the provider's spelling. This is a +// convenience for "какая погода в Лондоне", NOT a source of default truth: +// nothing here is used unless he actually said it. +var weatherCities = map[string]string{ + "москв": "Moscow", + "moscow": "Moscow", + "питер": "Saint Petersburg", + "spb": "Saint Petersburg", + "петербур": "Saint Petersburg", + "лондон": "London", + "london": "London", + "париж": "Paris", + "paris": "Paris", + "берлин": "Berlin", + "berlin": "Berlin", + "нью-йорк": "New York", + "new york": "New York", +} + +// extractWeatherLocation returns the city he named, or the configured default +// when he named none. It returns "" when he named none AND no default is +// configured — the caller must then say it does not know. +// +// It used to return "Moscow" in that case. That is a made-up answer presented +// as fact: reading out Moscow's temperature to someone who is not in Moscow is +// wrong in exactly the way maven must never be wrong. voice.weather +// .default_location is the only source of an unstated location. +func extractWeatherLocation(u, defaultLoc string) string { + lower := strings.ToLower(u) + for substr, name := range weatherCities { + if strings.Contains(lower, substr) { + return name + } + } + return defaultLoc +} diff --git a/cmd/mavmaild/main.go b/cmd/mavmaild/main.go new file mode 100644 index 0000000..cf547c2 --- /dev/null +++ b/cmd/mavmaild/main.go @@ -0,0 +1,405 @@ +// mavmaild — the mail reader module (Vikunja #246, +// docs/plans/01-email-reader.md). +// +// Every so often it opens one IMAP mailbox read-only, fetches the messages it +// has not read yet, and hands each one to core over ipc.MethodIngestMail. Core +// runs the extraction on the resident model and writes what comes back as task +// CANDIDATES he reviews on /tasks. Nothing here writes to the store, nothing +// here can create a reminder, and nothing here speaks. +// +// Why a separate daemon rather than a loop inside mavend, when extraction has +// to happen in mavend anyway: the credential. mavpoll set the precedent with the +// zenmoney token (#125) — the module that talks to a third party holds the +// secret, reads it from a FILE so it never appears in `ps`, in +// docker-compose.yml or in shell history, and core never sees it. Core learns +// that mail exists only as message text on one IPC method; it cannot connect to +// the mailbox even if it wanted to, and a compromised core yields no mail +// password. +// +// Off unless configured: without -password-file there is nothing to run, and +// the daemon says so and exits. If core has no `email` block the very first +// ingest comes back ErrUnknownMethod and this daemon stops polling instead of +// hammering a socket that will keep refusing. +// +// Mail is personal, so the log is counts and UIDs: how many messages were +// fetched, how many were bulk, how many candidates came back. No subject, no +// sender, no body, ever — reviewing a candidate is what /tasks is for. +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "log" + "os" + "os/signal" + "path/filepath" + "sort" + "strings" + "syscall" + "time" + + "github.com/kami/maven/internal/email" + "github.com/kami/maven/internal/ipc" +) + +func main() { + if err := run(os.Args[1:]); err != nil { + fmt.Fprintln(os.Stderr, "mavmaild:", err) + os.Exit(1) + } +} + +func run(args []string) error { + fs := flag.NewFlagSet("mavmaild", flag.ContinueOnError) + socket := fs.String("socket", "", "core IPC socket path (required)") + server := fs.String("imap", "", "IMAP server, host or host:993 (required)") + user := fs.String("user", "", "IMAP username (required)") + passFile := fs.String("password-file", "", "file holding the IMAP password (required — never passed as a flag value)") + mailbox := fs.String("mailbox", "INBOX", "mailbox to read, read-only") + interval := fs.Duration("interval", 15*time.Minute, "how often to read the mailbox") + lookback := fs.Duration("lookback", 72*time.Hour, "how far back to search on each poll") + // -max and -interval are one decision, not two. Every non-bulk message in a + // poll is one serialized llama-server call on core's side, and core gates + // mail extraction behind voice turns (llm.Gate), so a large batch does not + // mute Maven, it just takes a while. Raise -max only alongside whatever + // bound core is running. + max := fs.Int("max", 25, "most messages to fetch in one poll") + timeout := fs.Duration("timeout", 30*time.Second, "IMAP network timeout") + statePath := fs.String("state", "", "file remembering which UIDs were read (default: none — every poll re-reads the window)") + if err := fs.Parse(args); err != nil { + return err + } + if *socket == "" { + return fmt.Errorf("-socket is required") + } + if *server == "" || *user == "" || *passFile == "" { + return fmt.Errorf("mail reading is off unless configured: set -imap, -user and -password-file") + } + + // The password is read from a file, never taken as a flag value: an argv + // secret is visible in `ps` to every user on the box and lands in the compose + // file and the shell history. Read once at start — a rotated password means a + // restart, which is cheaper than re-reading his credential every quarter hour. + raw, err := os.ReadFile(*passFile) + if err != nil { + return fmt.Errorf("read password file: %w", err) + } + password := strings.TrimSpace(string(raw)) + if password == "" { + return fmt.Errorf("password file %s is empty", *passFile) + } + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + core, err := ipc.DialWait(*socket, 60*time.Second) + if err != nil { + return err + } + defer core.Close() + + r := &reader{ + core: core, + addr: *server, + user: *user, + mailbox: *mailbox, + lookback: *lookback, + max: *max, + timeout: *timeout, + state: newSeenState(*statePath), + } + if err := r.state.load(); err != nil { + // A missing or corrupt state file must not stop mail from being read: the + // worst case is re-reading the window, and capture dedupes on text. + log.Printf("mavmaild: state: %v (starting from an empty seen-set)", err) + } + + // The password is never logged, not even its length. + log.Printf("mavmaild: reading %s on %s every %s (lookback %s, max %d/poll)", + *mailbox, *server, *interval, *lookback, *max) + + r.pollOnce(ctx, password) // don't idle a full interval on start + t := time.NewTicker(*interval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + log.Printf("mavmaild: bye") + return nil + case <-t.C: + // Core told us mail ingestion is not configured. Nothing will change + // without a core restart, and a restart restarts us too, so the + // daemon stays up and does nothing at all. + // + // It does NOT exit. The compose service inherits restart: + // unless-stopped, which restarts a clean exit as readily as a crash, + // so exiting here produced a loop: log in to IMAP, get refused by + // core, exit, restart, log in again. Four IMAP logins an hour + // against a mailbox that has nothing to give, and Gmail and Yandex + // both rate-limit exactly that. + if r.disabled { + continue + } + r.pollOnce(ctx, password) + } + } +} + +// mailIngester — the slice of core this daemon uses. One method: hand over a +// message. It cannot write a fact, create a reminder or read the store, and the +// interface says so. +type mailIngester interface { + IngestMail(ctx context.Context, req ipc.IngestMailReq) (ipc.IngestMailResp, error) +} + +type reader struct { + core mailIngester + addr string + user string + mailbox string + lookback time.Duration + max int + timeout time.Duration + state *seenState + + // fetchMail — the read seam, nil ⇒ the real IMAP read. The tests replace + // the whole read rather than the transport: internal/email keeps its dialer + // unexported so that no code outside that package can point the reader at a + // cleartext socket and hand it the password, and this daemon is code + // outside that package. + fetchMail func(password string) ([]email.Message, error) + + // disabled — core answered ErrUnknownMethod, i.e. it has no email block. + // Written in pollOnce and read in the ticker loop, both on the one + // goroutine that run() drives, so it needs no atomic. If a second caller of + // pollOnce ever appears, this becomes a race and has to change. + disabled bool +} + +// pollOnce — one read of the mailbox, then one ingest per message. +// +// A fetch error aborts this poll and nothing else; the next tick tries again. +// An ingest error for one message does not skip the rest — one mail the model +// choked on should not hide the four behind it. +func (r *reader) pollOnce(ctx context.Context, password string) { + msgs, err := r.fetch(password) + if err != nil { + // The error may name a UID; it never names a subject or a sender. + log.Printf("mavmaild: fetch: %v", err) + if len(msgs) == 0 { + return + } + } + var junk, candidates, created int + for _, m := range msgs { + if ctx.Err() != nil { + return + } + req := ipc.IngestMailReq{ + Mailbox: r.mailbox, + UID: m.UID, + From: m.From, + Subject: m.Subject, + Date: m.Date, + Body: m.Body, + } + if m.Junk { + junk++ + // Core is TOLD, which is what its wire doc says: it counts the bulk + // message and answers Skipped without spending the model. The header + // filter already decided, so no content is sent with the verdict — + // nothing will read it. + req = ipc.IngestMailReq{Mailbox: r.mailbox, UID: m.UID, Junk: true} + } + resp, err := r.core.IngestMail(ctx, req) + if errors.Is(err, ipc.ErrUnknownMethod) { + log.Printf("mavmaild: core has no email block configured — mail ingestion is off; idling until a restart") + r.disabled = true + return + } + if err != nil { + // Not marked seen: an ingest that failed should be retried next poll. + log.Printf("mavmaild: ingest uid %d: %v", m.UID, err) + continue + } + r.state.mark(m.UID) + candidates += len(resp.TaskIDs) + created += resp.Created + } + if err := r.state.save(); err != nil { + log.Printf("mavmaild: state: %v", err) + } + log.Printf("mavmaild: %s: %d read, %d bulk, %d candidate(s), %d new", r.mailbox, len(msgs), junk, candidates, created) +} + +// fetch reads the mailbox. Messages already in the seen-set are not fetched at +// all, so a steady mailbox costs one SEARCH per poll and nothing else. +func (r *reader) fetch(password string) ([]email.Message, error) { + if r.fetchMail != nil { + return r.fetchMail(password) + } + f := email.FetchSince{ + Addr: r.addr, + User: r.user, + Mailbox: r.mailbox, + Timeout: r.timeout, + Since: time.Now().Add(-r.lookback), + Max: r.max, + Skip: r.state.seen, + // Everything below the oldest searchable UID has aged out of the + // lookback window and can never be read again. Retiring it is what keeps + // one permanently failing message from pinning the high-water mark + // forever. See seenState.retire. + OnSearch: func(uids []uint32) { + if len(uids) == 0 { + return + } + low := uids[0] + for _, u := range uids { + if u < low { + low = u + } + } + r.state.retire(low) + }, + } + return f.Run(password) +} + +// ---- seen state ------------------------------------------------------------ + +// seenState — the UIDs already handed to core, persisted so a restart does not +// re-read (and re-extract, at multi-second LLM cost) the whole lookback window. +// +// Correctness does not depend on it: ipc.CaptureTask dedupes on normalised text +// among live tasks, so a re-read produces no duplicate rows. This exists to save +// the model's time, which is why a broken state file is a log line rather than a +// failure. +// +// UIDs are per-mailbox and monotonic, so the set is kept as a high-water mark +// plus the stragglers above it. If the server ever changes UIDVALIDITY, UIDs +// reset and the window is simply re-read once — dedupe absorbs it. +// +// The high-water mark only advances through a CONTIGUOUS run, so a UID that +// never ingests successfully would pin it forever: everything above stays in +// the explicit set, and save rewrites all of it every poll. A year of that is +// a few hundred thousand entries written every quarter hour, which breaks +// nothing loudly and is exactly why it is worth catching. retire is the answer: +// a UID that has fallen out of the SEARCH SINCE window can never be fetched +// again, so there is nothing left to wait for. +type seenState struct { + path string + high uint32 + set map[uint32]bool + dirty bool +} + +func newSeenState(path string) *seenState { + return &seenState{path: path, set: map[uint32]bool{}} +} + +type seenFile struct { + High uint32 `json:"high"` + UIDs []uint32 `json:"uids,omitempty"` +} + +func (s *seenState) seen(uid uint32) bool { + return uid <= s.high || s.set[uid] +} + +func (s *seenState) mark(uid uint32) { + if s.seen(uid) { + return + } + s.set[uid] = true + s.dirty = true + // Advance the high-water mark through any contiguous run, so the explicit set + // stays small on a mailbox read in order. + for { + next := s.high + 1 + if !s.set[next] { + break + } + delete(s.set, next) + s.high = next + } +} + +// retire records that no UID below floor is reachable any more — they have +// aged out of the lookback window, so no poll will ever fetch them. The +// high-water mark can jump past the gap they were holding open, and the +// stragglers below it leave the explicit set. +// +// It never moves backwards, so a UIDVALIDITY reset (UIDs restarting low) makes +// this a no-op rather than a way to un-see a mailbox. +func (s *seenState) retire(floor uint32) { + if floor == 0 || floor-1 <= s.high { + return + } + s.high = floor - 1 + for u := range s.set { + if u <= s.high { + delete(s.set, u) + } + } + // The run above the new mark may now be contiguous with it. + for s.set[s.high+1] { + delete(s.set, s.high+1) + s.high++ + } + s.dirty = true +} + +func (s *seenState) load() error { + if s.path == "" { + return nil + } + b, err := os.ReadFile(s.path) + if errors.Is(err, os.ErrNotExist) { + return nil // first run + } + if err != nil { + return err + } + var f seenFile + if err := json.Unmarshal(b, &f); err != nil { + return fmt.Errorf("parse %s: %w", s.path, err) + } + s.high = f.High + for _, u := range f.UIDs { + s.set[u] = true + } + return nil +} + +// save writes the state atomically (temp file + rename), 0600: it is a list of +// message ids from his mailbox, which is metadata about his mail. +func (s *seenState) save() error { + if s.path == "" || !s.dirty { + return nil + } + uids := make([]uint32, 0, len(s.set)) + for u := range s.set { + uids = append(uids, u) + } + sort.Slice(uids, func(i, j int) bool { return uids[i] < uids[j] }) + b, err := json.Marshal(seenFile{High: s.high, UIDs: uids}) + if err != nil { + return err + } + tmp := s.path + ".tmp" + if err := os.MkdirAll(filepath.Dir(s.path), 0o700); err != nil { + return err + } + if err := os.WriteFile(tmp, b, 0o600); err != nil { + return err + } + if err := os.Rename(tmp, s.path); err != nil { + return err + } + s.dirty = false + return nil +} diff --git a/cmd/mavmaild/main_test.go b/cmd/mavmaild/main_test.go new file mode 100644 index 0000000..f4787dc --- /dev/null +++ b/cmd/mavmaild/main_test.go @@ -0,0 +1,315 @@ +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/kami/maven/internal/email" + "github.com/kami/maven/internal/ipc" +) + +// ---- a fake mailbox ------------------------------------------------------- +// +// It fakes the READ, not the protocol: internal/email owns the IMAP tests, and +// its dialer is unexported precisely so this package cannot substitute a +// transport. + +type fakeIMAP struct { + msgs map[uint32]string + uids []uint32 + cmds []string +} + +// fetch is the read seam the reader exposes: the daemon cannot reach +// internal/email's dialer (it is unexported so nothing outside that package can +// point the reader at a cleartext transport), so a test fakes the whole read. +// The IMAP protocol itself is covered by internal/email's own tests. +func (f *fakeIMAP) fetch(r *reader) func(string) ([]email.Message, error) { + return func(string) ([]email.Message, error) { + var out []email.Message + var low uint32 + for _, uid := range f.uids { + if low == 0 || uid < low { + low = uid + } + } + if low > 0 { + r.state.retire(low) + } + for i := len(f.uids) - 1; i >= 0; i-- { + uid := f.uids[i] + if r.state.seen(uid) { + continue + } + raw, ok := f.msgs[uid] + if !ok { + continue + } + f.cmds = append(f.cmds, fmt.Sprintf("UID FETCH %d", uid)) + msg, err := email.ParseMessage(uid, []byte(raw)) + if err != nil { + continue + } + out = append(out, msg) + if r.max > 0 && len(out) >= r.max { + break + } + } + return out, nil + } +} + +// ---- a fake core ----------------------------------------------------------- + +type fakeCore struct { + got []ipc.IngestMailReq + resp ipc.IngestMailResp + err error +} + +func (c *fakeCore) IngestMail(_ context.Context, req ipc.IngestMailReq) (ipc.IngestMailResp, error) { + c.got = append(c.got, req) + if c.err != nil { + return ipc.IngestMailResp{}, c.err + } + return c.resp, nil +} + +func mail(subject, body string, extraHeaders ...string) string { + h := "Subject: " + subject + "\r\nFrom: a@b.c\r\nContent-Type: text/plain; charset=utf-8\r\n" + for _, e := range extraHeaders { + h += e + "\r\n" + } + return h + "\r\n" + body + "\r\n" +} + +func newTestReader(t *testing.T, f *fakeIMAP, core *fakeCore, statePath string) *reader { + t.Helper() + r := &reader{ + core: core, addr: "mail.example:993", user: "kami", mailbox: "INBOX", + lookback: 72 * time.Hour, max: 25, timeout: 5 * time.Second, + state: newSeenState(statePath), + } + r.fetchMail = f.fetch(r) + return r +} + +func TestPollHandsMessagesToCore(t *testing.T) { + f := &fakeIMAP{ + uids: []uint32{1, 2}, + msgs: map[uint32]string{ + 1: mail("Счёт", "Оплатить до 5 августа."), + 2: mail("Скидки", "Sale!", "List-Unsubscribe: "), + }, + } + core := &fakeCore{resp: ipc.IngestMailResp{TaskIDs: []int64{1}, Created: 1}} + r := newTestReader(t, f, core, "") + r.pollOnce(context.Background(), "secret") + + // Two calls: the real mail with its text, and the newsletter as a verdict + // with no content at all. Core is told about bulk rather than asked, so it + // can count it without spending the model. + if len(core.got) != 2 { + t.Fatalf("core saw %d messages, want 2: %+v", len(core.got), core.got) + } + var got, bulk ipc.IngestMailReq + for _, r := range core.got { + if r.Junk { + bulk = r + } else { + got = r + } + } + if bulk.UID != 2 || !bulk.Junk { + t.Errorf("bulk req = %+v, want uid 2 flagged junk", bulk) + } + if bulk.Subject != "" || bulk.Body != "" || bulk.From != "" { + t.Errorf("a bulk verdict must carry no mail content: %+v", bulk) + } + if got.UID != 1 || got.Mailbox != "INBOX" || got.Subject != "Счёт" { + t.Errorf("ingest req = %+v", got) + } + if !strings.Contains(got.Body, "Оплатить") { + t.Errorf("body = %q", got.Body) + } +} + +// A second poll must not re-send what core already saw — extraction is a +// multi-second LLM call per message. +func TestPollSkipsSeenUIDs(t *testing.T) { + f := &fakeIMAP{uids: []uint32{5}, msgs: map[uint32]string{5: mail("Счёт", "текст")}} + core := &fakeCore{} + r := newTestReader(t, f, core, "") + r.pollOnce(context.Background(), "secret") + r.pollOnce(context.Background(), "secret") + if len(core.got) != 1 { + t.Errorf("core saw %d messages over two polls, want 1", len(core.got)) + } +} + +// An ingest that failed is NOT marked seen: the next poll retries it. +func TestPollRetriesFailedIngest(t *testing.T) { + f := &fakeIMAP{uids: []uint32{5}, msgs: map[uint32]string{5: mail("Счёт", "текст")}} + core := &fakeCore{err: fmt.Errorf("llama-server is warming up")} + r := newTestReader(t, f, core, "") + r.pollOnce(context.Background(), "secret") + core.err = nil + r.pollOnce(context.Background(), "secret") + if len(core.got) != 2 { + t.Errorf("core saw %d attempts, want 2 (a failed ingest is retried)", len(core.got)) + } +} + +// Core without an email block ⇒ stop, don't hammer the socket. +func TestPollStopsWhenCoreRefusesMail(t *testing.T) { + f := &fakeIMAP{uids: []uint32{1, 2}, msgs: map[uint32]string{1: mail("a", "b"), 2: mail("c", "d")}} + core := &fakeCore{err: fmt.Errorf("call: %w", ipc.ErrUnknownMethod)} + r := newTestReader(t, f, core, "") + r.pollOnce(context.Background(), "secret") + if !r.disabled { + t.Error("ErrUnknownMethod must disable the reader") + } + if len(core.got) != 1 { + t.Errorf("core saw %d messages, want 1 — stop at the first refusal", len(core.got)) + } +} + +func TestSeenStatePersists(t *testing.T) { + path := filepath.Join(t.TempDir(), "state", "seen.json") + f := &fakeIMAP{uids: []uint32{9}, msgs: map[uint32]string{9: mail("Счёт", "текст")}} + core := &fakeCore{} + r := newTestReader(t, f, core, path) + r.pollOnce(context.Background(), "secret") + + fi, err := os.Stat(path) + if err != nil { + t.Fatalf("state file: %v", err) + } + // A list of message ids from his mailbox is metadata about his mail. + if perm := fi.Mode().Perm(); perm != 0o600 { + t.Errorf("state file mode = %v, want 0600", perm) + } + + // A fresh reader with the same state file must not re-read the message. + core2 := &fakeCore{} + r2 := newTestReader(t, f, core2, path) + if err := r2.state.load(); err != nil { + t.Fatalf("load: %v", err) + } + r2.pollOnce(context.Background(), "secret") + if len(core2.got) != 0 { + t.Errorf("after a restart core saw %d messages, want 0", len(core2.got)) + } +} + +func TestSeenStateHighWaterMark(t *testing.T) { + s := newSeenState("") + s.mark(1) + s.mark(3) + s.mark(2) + if s.high != 3 { + t.Errorf("high = %d, want 3 (contiguous run collapses)", s.high) + } + if len(s.set) != 0 { + t.Errorf("explicit set = %v, want empty", s.set) + } + if !s.seen(2) || s.seen(4) { + t.Errorf("seen(2)=%v seen(4)=%v", s.seen(2), s.seen(4)) + } +} + +func TestSeenStateCorruptFileIsNotFatal(t *testing.T) { + path := filepath.Join(t.TempDir(), "seen.json") + if err := os.WriteFile(path, []byte("{not json"), 0o600); err != nil { + t.Fatal(err) + } + s := newSeenState(path) + if err := s.load(); err == nil { + t.Error("a corrupt state file should report an error the caller logs") + } + if s.seen(1) { + t.Error("a corrupt state file must leave an empty seen-set, not a poisoned one") + } +} + +// Off unless configured, and the credential is never a flag value. +func TestRunRequiresConfig(t *testing.T) { + if err := run([]string{}); err == nil { + t.Error("no -socket must be an error") + } + if err := run([]string{"-socket", "/tmp/nope.sock"}); err == nil { + t.Error("no mailbox configuration must be an error, not a default mailbox") + } + // There is no -password flag at all: only -password-file. + if err := run([]string{"-socket", "/x", "-imap", "h", "-user", "u", "-password", "p"}); err == nil || + !strings.Contains(err.Error(), "flag provided but not defined") { + t.Errorf("a -password flag must not exist; err = %v", err) + } +} + +func TestRunRejectsEmptyPasswordFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "pass") + if err := os.WriteFile(path, []byte(" \n"), 0o600); err != nil { + t.Fatal(err) + } + err := run([]string{"-socket", "/x/y.sock", "-imap", "h", "-user", "u", "-password-file", path}) + if err == nil || !strings.Contains(err.Error(), "empty") { + t.Errorf("an empty password file must be refused before dialling; err = %v", err) + } +} + +// A UID that never ingests pinned the high-water mark forever, because the mark +// only advances through a contiguous run. Once that UID falls out of the +// lookback window it can never be fetched again, so there is nothing left to +// wait for and everything above it can leave the explicit set. +func TestSeenStateRetiresAgedOutUIDs(t *testing.T) { + s := newSeenState("") + s.mark(1000) // 999 failed and was deliberately not marked + s.mark(1001) + if s.high != 0 || len(s.set) != 2 { + t.Fatalf("high = %d, set = %v; want the mark pinned below the gap", s.high, s.set) + } + // The next SEARCH window starts at 1000: 999 has aged out. + s.retire(1000) + if s.high != 1001 { + t.Errorf("high = %d, want 1001 once the gap is unreachable", s.high) + } + if len(s.set) != 0 { + t.Errorf("explicit set = %v, want empty", s.set) + } + if !s.seen(999) || !s.seen(1001) || s.seen(1002) { + t.Errorf("seen(999)=%v seen(1001)=%v seen(1002)=%v", s.seen(999), s.seen(1001), s.seen(1002)) + } +} + +// retire never moves the mark backwards: a UIDVALIDITY reset restarts UIDs low, +// and that must not un-see a mailbox or re-see one. +func TestSeenStateRetireNeverGoesBackwards(t *testing.T) { + s := newSeenState("") + s.mark(1) + s.mark(2) + s.retire(1) + if s.high != 2 { + t.Errorf("high = %d, want 2 unchanged", s.high) + } +} + +// A poll must not leave the state file growing with UIDs that are already +// covered by the high-water mark. +func TestPollRetiresThroughTheSearchWindow(t *testing.T) { + f := &fakeIMAP{uids: []uint32{100, 101}, msgs: map[uint32]string{100: mail("a", "b"), 101: mail("c", "d")}} + core := &fakeCore{} + r := newTestReader(t, f, core, "") + r.pollOnce(context.Background(), "secret") + if r.state.high != 101 { + t.Errorf("high = %d, want 101 — everything below the search window is unreachable", r.state.high) + } + if len(r.state.set) != 0 { + t.Errorf("explicit set = %v, want empty", r.state.set) + } +} diff --git a/cmd/mavpoll/main.go b/cmd/mavpoll/main.go index d15ae89..69c9eb4 100644 --- a/cmd/mavpoll/main.go +++ b/cmd/mavpoll/main.go @@ -9,6 +9,12 @@ // Two sources, each its own provenance (the loop's rules trust source): // - netdata → poll:netdata resource alarms (disk/mem/cert/temp) // - kuma → poll:uptimekuma service up/down (the source of truth for it) +// - zenmoney → poll:zenmoney spending/income totals (Vikunja #125) +// +// The zenmoney source is why the token lives HERE and not in core: the poller +// already owns every other third-party credential, it holds no store key, and +// core never needs to know an account exists to answer a question about a fact +// the poller wrote. It is off unless -zenmoney-token-file is given. // // Netdata needs no auth over the wg-fronted net. Kuma's /metrics needs an API // key (basic-auth); without -kuma the whole kuma path is skipped (netdata-only @@ -22,6 +28,7 @@ package main import ( "context" "encoding/json" + "errors" "flag" "fmt" "io" @@ -37,6 +44,7 @@ import ( "time" "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/zenmoney" ) func main() { @@ -52,6 +60,9 @@ func run(args []string) error { netdataURL := fs.String("netdata", "http://127.0.0.1:19999", "netdata base URL ('' to disable)") kumaURL := fs.String("kuma", "", "uptime-kuma metrics URL, e.g. http://127.0.0.1:3001/metrics ('' to disable)") kumaKey := fs.String("kuma-key", "", "uptime-kuma API key (basic-auth username)") + zenTokenFile := fs.String("zenmoney-token-file", "", "file holding the zenmoney API token ('' disables money tracking)") + zenURL := fs.String("zenmoney-url", zenmoney.DefaultBaseURL, "zenmoney API base URL (tests/self-hosted proxies)") + zenInterval := fs.Duration("zenmoney-interval", time.Hour, "how often to read zenmoney (money does not move every minute)") wgIface := fs.String("wg", "", "wireguard interface for the presence signal, e.g. wg0 or 'all' ('' to disable)") wgCmd := fs.String("wg-cmd", "wg", "wg binary (use e.g. 'sudo wg' if the poller lacks CAP_NET_ADMIN)") interval := fs.Duration("interval", 60*time.Second, "poll cadence") @@ -62,8 +73,24 @@ func run(args []string) error { if *socket == "" { return fmt.Errorf("-socket is required") } - if *netdataURL == "" && *kumaURL == "" && *wgIface == "" { - return fmt.Errorf("nothing to poll: set -netdata, -kuma and/or -wg") + if *netdataURL == "" && *kumaURL == "" && *wgIface == "" && *zenTokenFile == "" { + return fmt.Errorf("nothing to poll: set -netdata, -kuma, -wg and/or -zenmoney-token-file") + } + + // The token is read from a file, never taken as a flag value: an argv token + // is visible in `ps` to every user on the box and lands in the compose file + // and the shell history. Read once at start — a rotated token means a + // restart, which is cheaper than re-reading his credential every hour. + var zen *zenmoney.Client + if *zenTokenFile != "" { + raw, err := os.ReadFile(*zenTokenFile) + if err != nil { + return fmt.Errorf("read zenmoney token: %w", err) + } + zen, err = zenmoney.New(strings.TrimSpace(string(raw)), *zenURL, *timeout*3) + if err != nil { + return err + } } ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) @@ -83,9 +110,13 @@ func run(args []string) error { kumaKey: *kumaKey, wgIface: *wgIface, wgCmd: *wgCmd, + zen: zen, + zenEvery: *zenInterval, } - log.Printf("mavpoll: polling every %s (netdata=%q kuma=%q wg=%q)", *interval, *netdataURL, *kumaURL, *wgIface) + // The token is never logged, not even its length. + log.Printf("mavpoll: polling every %s (netdata=%q kuma=%q wg=%q zenmoney=%v every %s)", + *interval, *netdataURL, *kumaURL, *wgIface, zen != nil, *zenInterval) p.pollOnce(ctx) // fire immediately; don't idle a full interval on start t := time.NewTicker(*interval) defer t.Stop() @@ -108,6 +139,12 @@ type poller struct { kumaKey string wgIface string wgCmd string + + // zen is nil unless a token file was configured — money tracking is a + // capability, off by default like weather and telegram. + zen *zenmoney.Client + zenEvery time.Duration + zenLast time.Time } // pollOnce — one sweep of both sources. A failure in one source logs and does @@ -129,6 +166,76 @@ func (p *poller) pollOnce(ctx context.Context) { log.Printf("mavpoll: wg: %v", err) } } + // Money on its own, much slower cadence: a bank feed that updates hourly + // polled every minute is 60 pointless reads of his financial history. + if p.zen != nil && now.Sub(p.zenLast) >= p.zenEvery { + p.zenLast = now + if err := p.pollZenmoney(ctx, now); err != nil { + log.Printf("mavpoll: zenmoney: %v", err) + } + } +} + +// ---- zenmoney: spending/income totals → money facts ------------------------ + +// pollZenmoney reads today's and this month's totals and writes them as +// facts(kind=env, source=poll:zenmoney) (Vikunja #125). +// +// Two properties this function exists to hold: +// +// - An empty or failed read writes NOTHING. zenmoney.Summary.Value() refuses +// to encode a summary built from zero transactions, so a poller that cannot +// reach the API leaves the last good fact in place rather than overwriting +// it with a zero Maven would then recite as fact. +// - Nothing about the money leaves the box except the diff request itself, to +// the service that already holds his bank sessions. The totals are written +// to the store and read back only when he asks; they are never search input +// and no tick rule fires on them. +// +// Both windows are read from one diff call each. Two calls an hour against an +// API whose whole job is this is not worth caching. +// +// The write is UNCONDITIONAL, unlike every other poll in this file. The +// value-dedupe in writeIfChangedRaw only advances ts when the number moves, and +// for money that made ts mean "last changed" while the reader was asking it "as +// of when". A quiet 27 hours had core prefixing "данные от 30.07" to a figure +// that was current. The value now carries its own read stamp, so it differs +// every poll anyway and there is nothing left for the dedupe to catch. + +// moneyWindow — one fact key and the period it covers. +type moneyWindow struct { + key string + from, to time.Time +} + +func (p *poller) pollZenmoney(ctx context.Context, now time.Time) error { + dFrom, dTo := zenmoney.DayWindow(now) + mFrom, mTo := zenmoney.MonthWindow(now) + windows := []moneyWindow{ + {zenmoney.KeySpentToday, dFrom, dTo}, + {zenmoney.KeySpentMonth, mFrom, mTo}, + } + var firstErr error + for _, w := range windows { + sum, err := p.zen.Since(ctx, w.from, w.to) + if err != nil { + if firstErr == nil { + firstErr = err + } + continue + } + val, ok := sum.Value(now) + if !ok { + // Nothing read. Silence, not a zero. The last good fact stays, and + // the window stamp inside it is what stops core reciting yesterday's + // day total as today's after midnight. + continue + } + if err := p.writeMoneyFact(ctx, w.key, val, now); err != nil && firstErr == nil { + firstErr = err + } + } + return firstErr } // ---- wireguard: latest handshake → presence signal ------------------------- @@ -301,20 +408,52 @@ func (p *poller) writeIfChanged(ctx context.Context, key, source, val string, no return nil } -// isNoFact — ErrNoFact rehydrated over the wire is wrapped (fmt.Errorf %w), so -// errors.Is is the right check; keep a helper so the switch above reads clean. -func isNoFact(err error) bool { - for e := err; e != nil; { - if e == ipc.ErrNoFact { - return true - } - u, ok := e.(interface{ Unwrap() error }) - if !ok { - return false - } - e = u.Unwrap() +// writeIfChangedRaw is writeIfChanged for values that are already JSON (the +// money facts store an object, not a string). Kept separate rather than +// generalising writeIfChanged, because the string-valued env facts encoding +// their own value is the convention the rules rely on. +// +// The log line names the key and the source, never the figures: mavpoll's log +// is not the place his spending ends up. +func (p *poller) writeIfChangedRaw(ctx context.Context, key, source, jsonVal string, now time.Time) error { + prev, err := p.core.LatestFactBySource(ctx, key, source) + switch { + case err == nil && prev.Value == jsonVal: + return nil + case err != nil && err != ipc.ErrNoFact && !isNoFact(err): + return fmt.Errorf("read %s: %w", key, err) } - return false + if _, err := p.core.WriteFact(ctx, ipc.WriteFactReq{ + Ts: now, Kind: "env", Key: key, Value: jsonVal, + Source: source, Confidence: 1.0, + }); err != nil { + return fmt.Errorf("write %s: %w", key, err) + } + log.Printf("mavpoll: %s updated (%s)", key, source) + return nil +} + +// writeMoneyFact writes a money fact every poll, with no value comparison. See +// the comment above pollZenmoney for why this one does not go through +// writeIfChangedRaw. +// +// The log line names the key only, never the figures: mavpoll's log is not the +// place his spending ends up. +func (p *poller) writeMoneyFact(ctx context.Context, key, jsonVal string, now time.Time) error { + if _, err := p.core.WriteFact(ctx, ipc.WriteFactReq{ + Ts: now, Kind: "env", Key: key, Value: jsonVal, + Source: zenmoney.Source, Confidence: 1.0, + }); err != nil { + return fmt.Errorf("write %s: %w", key, err) + } + log.Printf("mavpoll: %s read (%s)", key, zenmoney.Source) + return nil +} + +// isNoFact — ErrNoFact rehydrated over the wire is wrapped (fmt.Errorf %w), so +// errors.Is is the right check. +func isNoFact(err error) bool { + return errors.Is(err, ipc.ErrNoFact) } func (p *poller) get(ctx context.Context, url, basicUser string) ([]byte, error) { diff --git a/cmd/mavpoll/main_test.go b/cmd/mavpoll/main_test.go index 2b79156..7004518 100644 --- a/cmd/mavpoll/main_test.go +++ b/cmd/mavpoll/main_test.go @@ -1,8 +1,17 @@ package main import ( + "context" "encoding/json" + "net/http" + "net/http/httptest" + "os" + "strings" "testing" + "time" + + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/zenmoney" ) func TestMaxSeverity(t *testing.T) { @@ -62,3 +71,120 @@ func TestParseMaxHandshake(t *testing.T) { } } } + +// ---- zenmoney (Vikunja #125) ---------------------------------------------- + +// factCore records the facts the poller wrote and answers "no fact yet". +type factCore struct { + ipc.UnimplementedCoreAPI + + written []ipc.WriteFactReq + prev map[string]string +} + +func (c *factCore) LatestFactBySource(_ context.Context, key, source string) (ipc.Fact, error) { + if v, ok := c.prev[key+"|"+source]; ok { + return ipc.Fact{Key: key, Source: source, Value: v}, nil + } + return ipc.Fact{}, ipc.ErrNoFact +} + +func (c *factCore) WriteFact(_ context.Context, req ipc.WriteFactReq) (int64, error) { + c.written = append(c.written, req) + return int64(len(c.written)), nil +} + +func zenFixtureServer(t *testing.T, body []byte, status int) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if status != http.StatusOK { + w.WriteHeader(status) + return + } + w.Write(body) + })) +} + +func TestPollZenmoneyWritesMoneyFacts(t *testing.T) { + body, err := os.ReadFile("../../internal/zenmoney/testdata/diff.json") + if err != nil { + t.Fatal(err) + } + srv := zenFixtureServer(t, body, http.StatusOK) + defer srv.Close() + zen, err := zenmoney.New("tok", srv.URL, time.Second) + if err != nil { + t.Fatal(err) + } + core := &factCore{} + p := &poller{core: core, zen: zen} + now := time.Date(2026, 8, 1, 21, 0, 0, 0, time.UTC) + if err := p.pollZenmoney(context.Background(), now); err != nil { + t.Fatal(err) + } + if len(core.written) != 2 { + t.Fatalf("wrote %d facts, want today + month", len(core.written)) + } + for _, f := range core.written { + if f.Kind != "env" || f.Source != zenmoney.Source { + t.Errorf("fact = %+v, want kind=env source=%s", f, zenmoney.Source) + } + if _, err := zenmoney.ParseFactValue(f.Value); err != nil { + t.Errorf("fact value %q does not decode: %v", f.Value, err) + } + } +} + +// A read that returns nothing for the window writes NOTHING. Silence, not a +// zero: an invented 0 would be recited back to him as fact. +func TestPollZenmoneyWritesNothingWhenEmpty(t *testing.T) { + srv := zenFixtureServer(t, []byte(`{"serverTimestamp":1,"instrument":[],"transaction":[]}`), http.StatusOK) + defer srv.Close() + zen, _ := zenmoney.New("tok", srv.URL, time.Second) + core := &factCore{} + p := &poller{core: core, zen: zen} + if err := p.pollZenmoney(context.Background(), time.Now()); err != nil { + t.Fatal(err) + } + if len(core.written) != 0 { + t.Errorf("wrote %+v, want no fact at all", core.written) + } +} + +// An API failure must not overwrite the last good total either. +func TestPollZenmoneyFailureWritesNothing(t *testing.T) { + srv := zenFixtureServer(t, nil, http.StatusUnauthorized) + defer srv.Close() + zen, _ := zenmoney.New("bad", srv.URL, time.Second) + core := &factCore{} + p := &poller{core: core, zen: zen} + if err := p.pollZenmoney(context.Background(), time.Now()); err == nil { + t.Error("want the 401 reported") + } + if len(core.written) != 0 { + t.Errorf("wrote %+v on a failed read", core.written) + } +} + +// Unchanged totals do not churn the facts table. +func TestWriteIfChangedRawSkipsUnchanged(t *testing.T) { + core := &factCore{prev: map[string]string{ + zenmoney.KeySpentToday + "|" + zenmoney.Source: `{"count":1}`, + }} + p := &poller{core: core} + if err := p.writeIfChangedRaw(context.Background(), zenmoney.KeySpentToday, zenmoney.Source, `{"count":1}`, time.Now()); err != nil { + t.Fatal(err) + } + if len(core.written) != 0 { + t.Errorf("wrote %+v for an unchanged value", core.written) + } +} + +// Money tracking is off unless configured: no token file, no zenmoney client, +// and the poller still refuses to start with nothing at all to poll. +func TestRunRequiresSomethingToPoll(t *testing.T) { + err := run([]string{"-socket", "/tmp/nope.sock", "-netdata", "", "-kuma", "", "-wg", ""}) + if err == nil || !strings.Contains(err.Error(), "nothing to poll") { + t.Errorf("err = %v, want a 'nothing to poll' refusal", err) + } +} diff --git a/cmd/mavsttd/golden_test.go b/cmd/mavsttd/golden_test.go new file mode 100644 index 0000000..a547e11 --- /dev/null +++ b/cmd/mavsttd/golden_test.go @@ -0,0 +1,391 @@ +package main + +// Golden-audio STT tests (Vikunja #288). +// +// These push real audio through the real whisper.cpp binding. What that +// covers, precisely, is two things: the model still transcribes known speech +// well enough for the router to act on it, and the silence gate still lets real +// speech through. A regression in either shows up in `make test` rather than in +// the owner talking to a daemon that mishears him. +// +// It is worth being exact about what is NOT covered, because this comment used +// to claim more. Nothing here resamples: audio.PCMFromWAV refuses anything that +// is not 16 kHz mono s16, the fixtures arrive at 16 kHz from ffmpeg, and there +// is no conversion step between the WAV and whisper_full. Nothing here +// exercises language selection either: the hint comes out of the manifest +// already correct and goes straight into the request, so how mavsttd chooses a +// language is untested. And a wrong model path is not caught when it is the +// default one, because a box without the model skips; an explicitly set +// MAVEN_WHISPER_MODEL that does not exist is a failure, since that is a +// mistake and not an absence. +// +// The fixtures are piper-synthesised, not recorded — see +// scripts/gen-stt-fixtures.sh. Nothing of the owner's voice is committed, and +// any fixture can be rebuilt from the script plus a voice model. +// +// Matching is deliberately tolerant. Golden transcripts are model-dependent: +// swapping ggml-small for a different whisper build moves punctuation, casing +// and the odd word ending, and an exact-string assertion would turn every +// model swap into a fixture rewrite. Each case therefore asserts two things — +// the words that carry the intent are present, and the word error rate +// against the reference stays under a per-case ceiling. + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "unicode" + + "github.com/kami/maven/internal/audio" + "github.com/kami/maven/internal/worker" +) + +// goldenModelPath — the whisper model the golden tests run against. Same file +// the Makefile's run-stt target uses. Overridable so a box that keeps its +// models elsewhere can still run these. +func goldenModelPath() string { + if p := os.Getenv("MAVEN_WHISPER_MODEL"); p != "" { + return p + } + return filepath.Join("..", "..", "models", "stt", "ggml-small.bin") +} + +type goldenCase struct { + Name string `json:"name"` + WAV string `json:"wav"` + Lang string `json:"lang"` + Text string `json:"text"` + Keywords []string `json:"keywords"` + // MeasuredWER is what this case scored when the ceiling was last set, so + // a model swap is a diff to a recorded number rather than silence. + MeasuredWER float64 `json:"measured_wer"` + MaxWER float64 `json:"max_wer"` +} + +type goldenManifest struct { + Cases []goldenCase `json:"cases"` +} + +func loadGoldenManifest(t *testing.T) goldenManifest { + t.Helper() + raw, err := os.ReadFile(filepath.Join("testdata", "golden_v1.json")) + if err != nil { + t.Fatalf("read golden manifest: %v", err) + } + var m goldenManifest + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatalf("parse golden manifest: %v", err) + } + if len(m.Cases) == 0 { + t.Fatal("golden manifest has no cases") + } + return m +} + +// normalizeTranscript lowercases, drops punctuation, folds the Russian ё onto +// е (whisper is inconsistent about it and the router does not care), and +// collapses whitespace. Everything the comparison does happens on this form. +func normalizeTranscript(s string) []string { + var b strings.Builder + for _, r := range strings.ToLower(s) { + switch { + case r == 'ё': + b.WriteRune('е') + case unicode.IsLetter(r) || unicode.IsDigit(r): + b.WriteRune(r) + default: + b.WriteRune(' ') + } + } + return strings.Fields(b.String()) +} + +// wordErrorRate is the Levenshtein distance between two word sequences, +// divided by the length of the reference. 0 means identical; it can exceed 1 +// when the hypothesis is much longer than the reference. +func wordErrorRate(ref, hyp []string) float64 { + if len(ref) == 0 { + if len(hyp) == 0 { + return 0 + } + return 1 + } + prev := make([]int, len(hyp)+1) + cur := make([]int, len(hyp)+1) + for j := range prev { + prev[j] = j + } + for i := 1; i <= len(ref); i++ { + cur[0] = i + for j := 1; j <= len(hyp); j++ { + cost := 1 + if ref[i-1] == hyp[j-1] { + cost = 0 + } + cur[j] = min(prev[j]+1, min(cur[j-1]+1, prev[j-1]+cost)) + } + prev, cur = cur, prev + } + return float64(prev[len(hyp)]) / float64(len(ref)) +} + +// missingKeywords returns the keywords absent from the hypothesis. A keyword +// matches on prefix, so a different case ending ("воды" vs "воду") does not +// fail the assertion — the router's stage-0 grammar is stem-shaped too. +func missingKeywords(keywords []string, hyp []string) []string { + var missing []string + for _, kw := range keywords { + want := normalizeTranscript(kw) + if len(want) == 0 { + continue + } + if !containsSeq(hyp, want) { + missing = append(missing, kw) + } + } + return missing +} + +func containsSeq(hyp, want []string) bool { + for i := 0; i+len(want) <= len(hyp); i++ { + ok := true + for j, w := range want { + // Prefix match, so inflection differences pass but + // distinct words do not. + if !looseWordMatch(hyp[i+j], w) { + ok = false + break + } + } + if ok { + return true + } + } + return false +} + +// looseWordMatch reports whether got is want, or an inflection of it. +// +// A shared prefix alone is not enough. "воды" retains three runes, so "водка" +// used to satisfy the ru_fact keyword and the test passed on whisper hearing +// "выпил водки". "disk" retains "dis", which "display", "distance" and +// "discuss" all match. So the hypothesis is also capped in length: a case +// ending adds a rune or two, it does not add a syllable. Short words get no +// slack at all, because there is nothing left of them after a prefix cut. +func looseWordMatch(got, want string) bool { + if got == want { + return true + } + g, w := []rune(got), []rune(want) + n := len(w) - 1 + if len(w) > 6 { + n = len(w) - 2 + } + // Words of three runes or fewer have no room for a safe prefix: require + // an exact match rather than letting "час" pass for "часть". + if n < 3 || len(g) < n { + return false + } + extra := 2 + if len(w) <= 4 { + extra = 0 + } + if len(g) > len(w)+extra { + return false + } + return string(g[:n]) == string(w[:n]) +} + +// --- the model-backed test ------------------------------------------------- + +func TestGoldenAudioTranscription(t *testing.T) { + m := loadGoldenManifest(t) + + model := goldenModelPath() + if _, err := os.Stat(model); err != nil { + // An explicit override that points at nothing is a mistake, not a box + // without the model. Skipping there made a typo look like a pass. + if os.Getenv("MAVEN_WHISPER_MODEL") != "" { + t.Fatalf("MAVEN_WHISPER_MODEL=%s does not exist: %v", model, err) + } + t.Skipf("whisper model %s absent (%v) — set MAVEN_WHISPER_MODEL or see AGENTS.md", model, err) + } + + // Same gate thresholds as mavsttd's defaults, so a regression in the + // silence gate shows up here as an empty transcript. + h, err := newWhisperHandler(model, 300, 0.01) + if err != nil { + t.Fatalf("load whisper model %s: %v", model, err) + } + defer h.Close() + + for _, c := range m.Cases { + t.Run(c.Name, func(t *testing.T) { + path := filepath.Join("testdata", c.WAV) + raw, err := os.ReadFile(path) + if err != nil { + // Not a skip. A fixture the generator failed to write is a + // broken checkout, and skipping made `make test` green on one. + t.Fatalf("fixture %s absent (%v) — run scripts/gen-stt-fixtures.sh", path, err) + } + format, pcm, err := audio.PCMFromWAV(raw) + if err != nil { + t.Fatalf("%s is not canonical 16k mono PCM: %v", path, err) + } + + resp, err := h.Transcribe(context.Background(), worker.TranscribeReq{ + Audio: audio.Audio{Format: format, Bytes: pcm}, + Lang: c.Lang, + }) + if err != nil { + t.Fatalf("transcribe %s: %v", c.WAV, err) + } + t.Logf("%s → %q (confidence %.3f)", c.WAV, resp.Text, resp.Confidence) + + if strings.TrimSpace(resp.Text) == "" { + t.Fatalf("%s transcribed to empty text — the silence gate ate real speech", c.WAV) + } + if resp.Confidence <= 0 { + t.Errorf("%s: confidence %v, want > 0", c.WAV, resp.Confidence) + } + + hyp := normalizeTranscript(resp.Text) + ref := normalizeTranscript(c.Text) + + if missing := missingKeywords(c.Keywords, hyp); len(missing) > 0 { + t.Errorf("%s: missing keywords %v in %q", c.WAV, missing, resp.Text) + } + wer := wordErrorRate(ref, hyp) + if wer > c.MaxWER { + t.Errorf("%s: WER %.2f > %.2f (measured %.2f when the ceiling was set)\n want: %q\n got: %q", + c.WAV, wer, c.MaxWER, c.MeasuredWER, c.Text, resp.Text) + } + t.Logf("%s: WER %.2f (ceiling %.2f, was %.2f)", c.WAV, wer, c.MaxWER, c.MeasuredWER) + }) + } +} + +// TestGoldenFixturesAreCanonical checks the committed audio without needing a +// model, so a fixture regenerated at the wrong sample rate fails on every box. +func TestGoldenFixturesAreCanonical(t *testing.T) { + m := loadGoldenManifest(t) + for _, c := range m.Cases { + path := filepath.Join("testdata", c.WAV) + raw, err := os.ReadFile(path) + if err != nil { + t.Errorf("fixture %s missing: %v", path, err) + continue + } + format, pcm, err := audio.PCMFromWAV(raw) + if err != nil { + t.Errorf("%s: %v", path, err) + continue + } + if !format.IsValid() { + t.Errorf("%s: format %+v is not canonical", path, format) + } + a := audio.Audio{Format: format, Bytes: pcm} + if d := a.Duration(); d < 0.5 || d > 10 { + t.Errorf("%s: duration %.2fs outside the sane 0.5–10s fixture range", path, d) + } + // The fixture must clear mavsttd's own silence gate, otherwise the + // model test below would be asserting on a gated empty string. + if reason := gateReason(pcmSamples(pcm), whisperSampleRate, 300, 0.01); reason != "" { + t.Errorf("%s: would be gated as %s", path, reason) + } + if len(c.Keywords) == 0 { + t.Errorf("%s: manifest case has no keywords", c.Name) + } + // An empty reference makes wordErrorRate return 1 for every + // hypothesis, so the WER assertion fires with nothing useful to say. + if len(normalizeTranscript(c.Text)) == 0 { + t.Errorf("%s: manifest case has no reference text", c.Name) + } + if c.Lang != "ru" && c.Lang != "en" { + t.Errorf("%s: lang %q is not one of the two languages mavsttd is run with", c.Name, c.Lang) + } + if c.MaxWER <= 0 || c.MaxWER > 1 { + t.Errorf("%s: max_wer %v outside (0,1]", c.Name, c.MaxWER) + } + if c.MeasuredWER > c.MaxWER { + t.Errorf("%s: measured_wer %v is above max_wer %v, so the ceiling was never met", c.Name, c.MeasuredWER, c.MaxWER) + } + } +} + +// --- matcher unit tests (no model, no fixtures) ---------------------------- + +func TestNormalizeTranscript(t *testing.T) { + got := normalizeTranscript(" Ещё, Раз... ") + want := []string{"еще", "раз"} + if len(got) != len(want) || got[0] != want[0] || got[1] != want[1] { + t.Fatalf("normalizeTranscript = %v, want %v", got, want) + } +} + +func TestWordErrorRate(t *testing.T) { + cases := []struct { + name string + ref, hyp string + want float64 + }{ + {"identical", "напомни мне через час", "Напомни мне через час.", 0}, + {"one substitution", "напомни мне через час", "напомни мне через день", 0.25}, + {"one deletion", "напомни мне через час", "напомни мне час", 0.25}, + {"empty hypothesis", "напомни мне", "", 1}, + {"both empty", "", "", 0}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := wordErrorRate(normalizeTranscript(c.ref), normalizeTranscript(c.hyp)) + if got != c.want { + t.Fatalf("WER = %v, want %v", got, c.want) + } + }) + } +} + +func TestMissingKeywords(t *testing.T) { + hyp := normalizeTranscript("Отметь, что я выпил воду.") + if got := missingKeywords([]string{"воды", "отметь"}, hyp); len(got) != 0 { + t.Fatalf("missingKeywords = %v, want none (inflection must not fail the match)", got) + } + if got := missingKeywords([]string{"календарю"}, hyp); len(got) != 1 { + t.Fatalf("missingKeywords = %v, want the absent keyword reported", got) + } + // A short word must match exactly — no 4-rune prefix shortcut that would + // let "час" pass for "часть". + hyp2 := normalizeTranscript("через час") + if got := missingKeywords([]string{"часть"}, hyp2); len(got) != 1 { + t.Fatalf("missingKeywords = %v, want %q reported missing", got, "часть") + } + // A prefix is not a word. These are different words that share one, and + // each of them used to satisfy the keyword it is paired with. + different := [][2]string{ + {"воды", "Я выпил водки."}, + {"disk", "check the display"}, + {"disk", "we should discuss it"}, + {"server", "a serverless function"}, + } + for _, d := range different { + if got := missingKeywords([]string{d[0]}, normalizeTranscript(d[1])); len(got) != 1 { + t.Errorf("keyword %q was satisfied by %q", d[0], d[1]) + } + } + // And the inflections still pass, which is the whole point of the loose + // match. + same := [][2]string{ + {"воды", "выпил воду"}, + {"напомни", "напомните мне"}, + {"календарю", "по календаре"}, + {"restart", "restarted the server"}, + } + for _, d := range same { + if got := missingKeywords([]string{d[0]}, normalizeTranscript(d[1])); len(got) != 0 { + t.Errorf("keyword %q was not matched by %q", d[0], d[1]) + } + } +} diff --git a/cmd/mavsttd/testdata/en_act.wav b/cmd/mavsttd/testdata/en_act.wav new file mode 100644 index 0000000..5a904ae Binary files /dev/null and b/cmd/mavsttd/testdata/en_act.wav differ diff --git a/cmd/mavsttd/testdata/golden_v1.json b/cmd/mavsttd/testdata/golden_v1.json new file mode 100644 index 0000000..5e7b4b9 --- /dev/null +++ b/cmd/mavsttd/testdata/golden_v1.json @@ -0,0 +1,42 @@ +{ + "note": "Golden STT fixtures. Audio is piper-synthesised, not recorded — see scripts/gen-stt-fixtures.sh, which reads `text` from this file and synthesises from it. This is the only source of the spoken words; regenerate with that script and do not hand-edit `wav`.", + "wer_note": "max_wer is set just above what each case actually measures against ggml-small, recorded in `measured_wer` on 2026-08-01. A flat 0.34 over a five-word reference tolerated two wrong words and left most of the range unguarded. A model swap should show up as a diff to these numbers, not as silence: rerun `make test-stt-golden`, read the logged transcript, and move both fields together.", + "cases": [ + { + "name": "ru_reminder", + "wav": "ru_reminder.wav", + "lang": "ru", + "text": "напомни мне через час позвонить маме", + "keywords": ["напомни", "час", "позвонить"], + "measured_wer": 0.0, + "max_wer": 0.1 + }, + { + "name": "ru_fact", + "wav": "ru_fact.wav", + "lang": "ru", + "text": "отметь что я выпил воды", + "keywords": ["отметь", "воды"], + "measured_wer": 0.2, + "max_wer": 0.25 + }, + { + "name": "ru_query", + "wav": "ru_query.wav", + "lang": "ru", + "text": "что у меня сегодня по календарю", + "keywords": ["сегодня", "календарю"], + "measured_wer": 0.0, + "max_wer": 0.1 + }, + { + "name": "en_act", + "wav": "en_act.wav", + "lang": "en", + "text": "restart the web server and check the disk space", + "keywords": ["restart", "server", "disk"], + "measured_wer": 0.0, + "max_wer": 0.1 + } + ] +} diff --git a/cmd/mavsttd/testdata/ru_fact.wav b/cmd/mavsttd/testdata/ru_fact.wav new file mode 100644 index 0000000..fe6a3ac Binary files /dev/null and b/cmd/mavsttd/testdata/ru_fact.wav differ diff --git a/cmd/mavsttd/testdata/ru_query.wav b/cmd/mavsttd/testdata/ru_query.wav new file mode 100644 index 0000000..3fabbda Binary files /dev/null and b/cmd/mavsttd/testdata/ru_query.wav differ diff --git a/cmd/mavsttd/testdata/ru_reminder.wav b/cmd/mavsttd/testdata/ru_reminder.wav new file mode 100644 index 0000000..b979e7b Binary files /dev/null and b/cmd/mavsttd/testdata/ru_reminder.wav differ diff --git a/cmd/mavsttd/whisper_handler.go b/cmd/mavsttd/whisper_handler.go index b8cdf93..bc58cbd 100644 --- a/cmd/mavsttd/whisper_handler.go +++ b/cmd/mavsttd/whisper_handler.go @@ -66,6 +66,19 @@ func gateReason(samples []float32, rate, minMs int, minRMS float64) string { return "" } +// pcmSamples converts canonical s16le little-endian PCM to the float32 range +// whisper wants. Shared with the golden tests: they used to carry their own +// copy, so a regression here (a /32767 divisor, a byte order slip) left the +// assertion that the fixtures clear the silence gate green. +func pcmSamples(b []byte) []float32 { + out := make([]float32, len(b)/2) + for i := range out { + s := int16(b[i*2]) | int16(b[i*2+1])<<8 + out[i] = float32(s) / 32768.0 + } + return out +} + func (h *whisperHandler) Transcribe(ctx context.Context, req worker.TranscribeReq) (worker.TranscribeResp, error) { if err := ctx.Err(); err != nil { return worker.TranscribeResp{}, fmt.Errorf("whisper: context done before transcribe: %w", err) @@ -75,12 +88,7 @@ func (h *whisperHandler) Transcribe(ctx context.Context, req worker.TranscribeRe return worker.TranscribeResp{}, fmt.Errorf("whisper: empty audio") } - nSamples := len(a.Bytes) / 2 - samples := make([]float32, nSamples) - for i := 0; i < nSamples; i++ { - s := int16(a.Bytes[i*2]) | int16(a.Bytes[i*2+1])<<8 - samples[i] = float32(s) / 32768.0 - } + samples := pcmSamples(a.Bytes) // Silence gate: drop non-speech before whisper hallucinates on it. if reason := gateReason(samples, whisperSampleRate, h.minMs, h.minRMS); reason != "" { @@ -111,7 +119,7 @@ func (h *whisperHandler) Transcribe(ctx context.Context, req worker.TranscribeRe ch := make(chan result, 1) cSamples := (*C.float)(unsafe.Pointer(&samples[0])) go func() { - ch <- result{code: int(C.whisper_full(h.ctx, params, cSamples, C.int(nSamples)))} + ch <- result{code: int(C.whisper_full(h.ctx, params, cSamples, C.int(len(samples))))} }() select { case r := <-ch: diff --git a/cmd/mavupdate/main.go b/cmd/mavupdate/main.go new file mode 100644 index 0000000..7f64914 --- /dev/null +++ b/cmd/mavupdate/main.go @@ -0,0 +1,213 @@ +// Command mavupdate deploys a new build of Maven to the box she runs on, with +// an automatic rollback when the new build does not come up (Vikunja #249). +// +// It is a CLI on purpose, and it is the ONLY trigger for the update path. +// +// The obvious design — an IPC method plus a button on the web UI behind the +// step-up passkey gate, the way /tools works — was considered and refused. A +// step-up gate protects against the wrong person clicking; it does not change +// the fact that anything reachable over the network becomes, in the event of a +// mavweb bug, a remote arbitrary-code path with a build system attached. An +// update needs shell access on the host, which is a strictly higher bar than +// the gate that guards the tool allowlist. That is deliberate and it is the +// reason there is no MethodApplyUpdate anywhere in internal/ipc. +// +// Consequently: mavend never constructs an update.Updater and nothing in the +// daemon can call Apply, nothing runs on a timer, nothing checks a release +// server, and no act, intent, tool or LLM output can reach any of this. The +// package is linked into mavend through internal/config, which validates the +// update block at startup; the guarantee is the absent caller, not an absent +// import. She cannot update herself. She can be updated, by him. +// +// mavupdate -config deploy/mavend.json list # snapshots available to roll back to +// mavupdate -config deploy/mavend.json verify # make build + make test, deploys nothing +// mavupdate -config deploy/mavend.json apply -yes # the whole thing +// mavupdate -config deploy/mavend.json rollback [id] # restore + restart (default: newest) +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "os" + "os/signal" + "syscall" + "time" + + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/update" +) + +func main() { + cfgPath := flag.String("config", "deploy/mavend.json", "path to mavend.json (the update block is read from it)") + yes := flag.Bool("yes", false, "required by `apply` and `rollback`: yes, restart the daemon") + flag.Usage = usage + flag.Parse() + + // The stdlib flag package stops parsing at the first non-flag argument, so a + // `-yes` written after the subcommand (which is how anyone would type it, and + // how the usage text shows it) lands in Args instead of the flag. Pick it out + // by hand rather than silently treating "apply -yes" as an unconfirmed apply. + var args []string + for _, a := range flag.Args() { + if a == "-yes" || a == "--yes" { + *yes = true + continue + } + args = append(args, a) + } + if len(args) == 0 { + usage() + os.Exit(2) + } + + cfg, err := config.Load(*cfgPath) + if err != nil { + die("config: %v", err) + } + if cfg.Update == nil { + die("no `update` block in %s — the update capability is off unless configured.\nSee the package comment in internal/update for what it does and does not do.", *cfgPath) + } + + logf := func(format string, a ...any) { + fmt.Fprintf(os.Stderr, "%s %s\n", time.Now().Format("15:04:05"), fmt.Sprintf(format, a...)) + } + u, err := update.New(*cfg.Update, update.WithLogger(logf)) + if err != nil { + die("%v", err) + } + + // Ctrl-C cancels the build or the health wait. It cannot cancel a rollback + // midway into leaving the box in an unknown state, because the rollback runs + // on its own context — see cmdApply. + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + switch args[0] { + case "list": + cmdList(u) + case "verify": + cmdVerify(ctx, u) + case "apply": + if !*yes { + die("apply restarts mavend and can roll her back. Re-run with -yes if that is what you want.") + } + cmdApply(ctx, u) + case "rollback": + if !*yes { + die("rollback restores the previous artifacts and restarts mavend. Re-run with -yes.") + } + id := "" + if len(args) > 1 { + id = args[1] + } + cmdRollback(ctx, u, id) + default: + usage() + os.Exit(2) + } +} + +func cmdList(u *update.Updater) { + snaps, err := u.Snapshots() + if err != nil { + die("snapshots: %v", err) + } + if len(snaps) == 0 { + fmt.Println("no snapshots yet — the first `apply` takes one before it builds anything") + return + } + fmt.Printf("%-18s %-12s %s\n", "SNAPSHOT", "COMMIT", "FILES") + for _, s := range snaps { + commit := s.Commit + if len(commit) > 12 { + commit = commit[:12] + } + if commit == "" { + commit = "-" + } + fmt.Printf("%-18s %-12s %d\n", s.ID, commit, len(s.Files)) + } + fmt.Printf("\nrollback to the newest with: mavupdate rollback -yes\n") +} + +func cmdVerify(ctx context.Context, u *update.Updater) { + steps, err := u.Verify(ctx) + report(steps) + if err != nil { + die("%v", err) + } + fmt.Println("verified: the tree builds and passes its own tests. Nothing was deployed — run `apply -yes` for that.") +} + +func cmdApply(ctx context.Context, u *update.Updater) { + res, err := u.Apply(ctx) + report(res.Steps) + summarize(res) + switch { + case err == nil: + fmt.Println("\nupdate committed: she answers on the new build.") + case errors.Is(err, update.ErrRollbackFailed): + die("\n%v\n\nSHE IS PROBABLY DOWN. The previous artifacts are in the snapshot dir; copy them\nover the install dir and restart by hand.", err) + case errors.Is(err, update.ErrRolledBack): + die("\n%v\n\nShe is answering again on the previous build. Nothing was lost; fix the change and retry.", err) + default: + die("\n%v", err) + } +} + +func cmdRollback(ctx context.Context, u *update.Updater, id string) { + res, err := u.Rollback(ctx, id) + report(res.Steps) + summarize(res) + // The standalone rollback is what he reaches for when something is already + // wrong, so a failed one needs the loud paragraph more than apply does, not + // less. + if errors.Is(err, update.ErrRollbackFailed) { + die("\n%v\n\nSHE IS PROBABLY DOWN. The previous artifacts are in the snapshot dir; copy them\nover the install dir and restart by hand.", err) + } + if err != nil && !errors.Is(err, update.ErrRolledBack) { + die("\n%v", err) + } + fmt.Printf("\nrolled back to %s; she answers on it.\n", res.SnapshotID) +} + +func report(steps []update.Step) { + for _, s := range steps { + status := "ok" + if s.Err != nil { + status = "FAILED: " + s.Err.Error() + } + fmt.Printf(" %-8s %-8s %s\n", s.Name, s.Took.Round(time.Second), status) + if s.Output != "" { + fmt.Printf("---- %s output ----\n%s\n-------------------\n", s.Name, s.Output) + } + } +} + +func summarize(res update.Result) { + fmt.Printf("\nverified=%v snapshot=%s installed=%d restarted=%v healthy=%v rolled_back=%v rollback_healthy=%v took=%s\n", + res.Verified, res.SnapshotID, len(res.Installed), res.Restarted, res.Healthy, res.RolledBack, res.RollbackHealthy, res.Took.Round(time.Second)) +} + +func usage() { + fmt.Fprint(os.Stderr, `mavupdate — deploy a new build of Maven, with rollback. + + mavupdate [-config path] list + mavupdate [-config path] verify + mavupdate [-config path] apply -yes + mavupdate [-config path] rollback [snapshot-id] -yes + +apply is: health-check the running daemon, snapshot the deployed artifacts, +make build, make test, install, restart, health-check — and restore the +snapshot if any of that fails. It never fetches code and never runs by itself. + +`) + flag.PrintDefaults() +} + +func die(format string, a ...any) { + fmt.Fprintf(os.Stderr, format+"\n", a...) + os.Exit(1) +} diff --git a/cmd/mavwaked/main.go b/cmd/mavwaked/main.go index d8ee946..65e4a55 100644 --- a/cmd/mavwaked/main.go +++ b/cmd/mavwaked/main.go @@ -12,8 +12,18 @@ // (30ms frames, 16kHz PCM) matches silero-vad's input interface exactly, so // swapping energy-threshold for ONNX-inference is a local change in vad.go. // +// While a reply is playing the capture side is muted (half-duplex): without +// it, Maven's own voice comes back in through the mic and she answers +// herself. -barge-in punches one hole in that gate — sustained energy above +// -barge-in-rms cuts playback so he can talk over her. It is off by default +// because the threshold is room-specific; see playback.go. The threshold is a +// raw frame RMS and has no reference to what the speaker actually leaks, so +// the daemon logs the mean energy of the frames it suppressed while speaking. +// Set -barge-in-rms from those numbers rather than by guessing. +// // usage: // mavwaked # default ALSA device, 127.0.0.1:9100 +// mavwaked -barge-in # let him interrupt her mid-reply // mavwaked -device hw:1,0 -addr 10.42.0.1:9100 // mavwaked -test file.wav # read from file, no arecord package main @@ -60,6 +70,9 @@ func run(args []string) error { silenceMs := flag.Int("silence-ms", defaultSilenceMs, "silence ms to end utterance") maxMs := flag.Int("max-ms", defaultMaxMs, "max utterance ms") testFile := flag.String("test", "", "read PCM from file instead of arecord (testing only)") + bargeIn := flag.Bool("barge-in", false, "cut Maven off when he talks over her (needs a room-tuned -barge-in-rms)") + bargeRMS := flag.Int("barge-in-rms", defaultBargeRMS, "RMS x10000 a frame must clear to count as barge-in") + bargeFrames := flag.Int("barge-in-frames", defaultBargeFrames, "consecutive frames over -barge-in-rms before playback is cut") flag.CommandLine.Parse(args) ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP) @@ -117,14 +130,29 @@ func run(args []string) error { defer src.Close() - return captureLoop(ctx, src, vad, vc, *lang) + var barge bargeInConfig + if *bargeIn { + barge = bargeInConfig{RMS: float64(*bargeRMS) / 10000.0, Frames: *bargeFrames} + if barge.Enabled() { + log.Printf("mavwaked: barge-in on (rms %.4f x %d frames)", barge.RMS, barge.Frames) + } else { + // The log used to say "barge-in on (rms 0.0000 x 5)" here and then + // nothing happened, because Enabled needs a positive threshold. + log.Printf("mavwaked: -barge-in was passed but rms %.4f x %d frames disables it; "+ + "both must be above zero, so barge-in is OFF", + barge.RMS, barge.Frames) + } + } + sess := newSession(vad, newAplayPlayer(), &voiceSender{vc: vc}, *lang, barge) + + return captureLoop(ctx, src, sess) } -// captureLoop reads PCM from src, runs VAD, and sends complete utterances to -// the voice server. Returns when ctx is done or src is exhausted. -func captureLoop(ctx context.Context, src io.Reader, vad *VAD, vc *voice.Client, lang string) error { +// captureLoop reads PCM from src and hands whole frames to the session. +// Returns when ctx is done or src is exhausted. +func captureLoop(ctx context.Context, src io.Reader, sess *session) error { br := bufio.NewReaderSize(src, defaultReadSize) - frameBytes := vad.FrameSamples() * 2 // 480 samples × 2 bytes = 960 bytes per 30ms + frameBytes := sess.vad.FrameSamples() * 2 // 480 samples × 2 bytes = 960 bytes per 30ms log.Printf("mavwaked: capture loop starting (frame=%d bytes, %dms)", frameBytes, defaultFrameMs) @@ -147,7 +175,7 @@ func captureLoop(ctx context.Context, src io.Reader, vad *VAD, vc *voice.Client, // Flush partial frame. partial = append(partial, buf[:n]...) if len(partial) >= frameBytes { - if err := processFrame(partial[:frameBytes], vad, vc, lang); err != nil { + if err := sess.feed(ctx, partial[:frameBytes]); err != nil { log.Printf("mavwaked: process frame: %v", err) } partial = partial[frameBytes:] @@ -165,107 +193,31 @@ func captureLoop(ctx context.Context, src io.Reader, vad *VAD, vc *voice.Client, partial = nil } - if err := processFrame(full, vad, vc, lang); err != nil { + if err := sess.feed(ctx, full); err != nil { log.Printf("mavwaked: process frame: %v", err) } } } -// processFrame feeds one 30ms PCM frame to the VAD and sends any completed -// utterance to the voice server. -func processFrame(frame []byte, vad *VAD, vc *voice.Client, lang string) error { - samples := PCMToI16(frame) - utt, state := vad.Feed(samples) +// voiceSender is the production utteranceSender: one PushToTalk round-trip +// over the voice wire. SurfaceVoice (not the default SurfacePCClient that +// c.PushToTalk uses) caps everything at L0, which is what makes an accidental +// VAD trigger safe. +type voiceSender struct{ vc *voice.Client } - if state == StateSpeech { - // Speech is in progress; nothing to send yet. - return nil - } - - if utt.Bytes == nil { - // Still in silence, or short speech that didn't trigger. - return nil - } - - // We have a complete utterance — send it to the voice server. - return sendUtterance(context.Background(), utt, vc, lang) -} - -// sendUtterance sends audio to the voice server and plays the reply. -func sendUtterance(ctx context.Context, utt audio.Audio, vc *voice.Client, lang string) error { - dur := utt.Duration() - log.Printf("mavwaked: utterance complete (%.2fs, %d bytes), sending...", - dur, len(utt.Bytes)) - - // Use SendRequest directly so we can set SurfaceVoice instead of the - // default SurfacePCClient that c.PushToTalk uses. +func (s *voiceSender) Send(ctx context.Context, utt audio.Audio, lang string) (audio.Audio, error) { var resp voice.PushToTalkResp - err := vc.SendRequest(ctx, voice.MethodPushToTalk, voice.PushToTalkReq{ + err := s.vc.SendRequest(ctx, voice.MethodPushToTalk, voice.PushToTalkReq{ Audio: utt, Lang: lang, Surface: voice.SurfaceVoice, }, &resp) if err != nil { - return fmt.Errorf("push-to-talk: %w", err) + return audio.Audio{}, fmt.Errorf("push-to-talk: %w", err) } - log.Printf("mavwaked: reply: %q (%.2fs audio)", resp.ReplyText, resp.ReplyAudio.Duration()) - - // Play the reply audio. - if len(resp.ReplyAudio.Bytes) > 0 { - go playAudio(resp.ReplyAudio) - } else { - log.Printf("mavwaked: empty reply audio (text only)") - } - if len(resp.RoutedChannels) > 0 { log.Printf("mavwaked: also routed to: %v", resp.RoutedChannels) } - - return nil -} - -// playAudio pipes PCM audio to aplay(1) for playback. Runs in a goroutine. -func playAudio(a audio.Audio) { - // Build WAV header for aplay (or pipe raw PCM with the right format flags). - cmd := exec.Command("aplay", - "-f", "S16_LE", - "-r", fmt.Sprintf("%d", a.Format.SampleRate), - "-c", fmt.Sprintf("%d", a.Format.Channels), - "-t", "raw", - ) - - stdin, err := cmd.StdinPipe() - if err != nil { - log.Printf("mavwaked: aplay stdin pipe: %v", err) - return - } - - if err := cmd.Start(); err != nil { - log.Printf("mavwaked: start aplay: %v", err) - return - } - - // Write audio to aplay's stdin. - if _, err := stdin.Write(a.Bytes); err != nil { - log.Printf("mavwaked: write to aplay: %v", err) - } - _ = stdin.Close() - - // Wait for playback to finish (with a timeout). - done := make(chan error, 1) - go func() { - done <- cmd.Wait() - }() - - select { - case err := <-done: - if err != nil { - log.Printf("mavwaked: aplay: %v", err) - } - case <-time.After(30 * time.Second): - log.Printf("mavwaked: aplay timeout, killing") - _ = cmd.Process.Kill() - <-done - } + return resp.ReplyAudio, nil } diff --git a/cmd/mavwaked/playback.go b/cmd/mavwaked/playback.go new file mode 100644 index 0000000..01fd69d --- /dev/null +++ b/cmd/mavwaked/playback.go @@ -0,0 +1,152 @@ +package main + +// Reply playback, and the half-duplex gate around it (Vikunja #287). +// +// Before this, playback was `go playAudio(reply)` — fire and forget, with no +// handle on the running aplay. Two things fell out of that, and both are +// audible: +// +// 1. Self-trigger. The capture loop keeps feeding the VAD while the speaker +// is playing, so Maven's own reply comes back in through the mic, trips +// the VAD, and is sent to the daemon as a fresh utterance. She answers +// herself. There is no acoustic echo canceller in this pipeline, so the +// only correct fix is half-duplex: while she is speaking, the capture +// side is muted. +// +// 2. No barge-in. Talking over her did nothing — there was nothing to +// cancel, because nobody held the process handle. +// +// The two are the same mechanism seen from opposite sides, so they live +// together here. Echo suppression is unconditional (it fixes a bug). Barge-in +// is off unless -barge-in is passed, because it needs a room-specific energy +// threshold: with no echo canceller, the only way to tell "he is talking over +// her" from "the mic is hearing her" is that he is louder, and how much +// louder depends on where the mic sits relative to the speaker. + +import ( + "log" + "os/exec" + "strconv" + "sync" + "time" + + "github.com/kami/maven/internal/audio" +) + +// playbackMargin is the slack over the reply's own duration before a stuck +// aplay is killed. Enough for ALSA to open the device and drain its buffer, +// short enough that a busy device does not cost her a turn. +const playbackMargin = 2 * time.Second + +// player plays one reply at a time and can be cut off mid-utterance. +type player interface { + // Play starts playback of a, replacing anything already playing, and + // returns immediately. + Play(a audio.Audio) + // Stop ends playback now. A no-op when nothing is playing. + Stop() + // Playing reports whether audio is currently going out of the speaker. + Playing() bool +} + +// aplayPlayer pipes raw PCM to aplay(1). Stop kills the child, which is what +// makes barge-in instant rather than "instant at the end of the sentence". +type aplayPlayer struct { + mu sync.Mutex + cmd *exec.Cmd + playing bool + // gen rises on every Play/Stop so a finishing playback cannot clear the + // playing flag of the one that replaced it. + gen uint64 +} + +func newAplayPlayer() *aplayPlayer { return &aplayPlayer{} } + +func (p *aplayPlayer) Play(a audio.Audio) { + if len(a.Bytes) == 0 { + return + } + p.Stop() + + cmd := exec.Command("aplay", + "-f", "S16_LE", + "-r", strconv.Itoa(a.Format.SampleRate), + "-c", strconv.Itoa(a.Format.Channels), + "-t", "raw", + ) + stdin, err := cmd.StdinPipe() + if err != nil { + log.Printf("mavwaked: aplay stdin pipe: %v", err) + return + } + if err := cmd.Start(); err != nil { + log.Printf("mavwaked: start aplay: %v", err) + _ = stdin.Close() + return + } + + p.mu.Lock() + p.gen++ + gen := p.gen + p.cmd = cmd + p.playing = true + p.mu.Unlock() + + // Bound the mute window by the reply itself. Playing() gates all capture + // now, so a wedged aplay does not merely go silent, it makes her deaf for + // as long as the flag is set. The old ceiling was a flat 30s inherited + // from the fire-and-forget version, where it only bounded a leaked + // goroutine. A reply cannot legitimately take longer than it lasts. + limit := time.Duration(a.Duration()*float64(time.Second)) + playbackMargin + + go func() { + if _, err := stdin.Write(a.Bytes); err != nil { + // Broken pipe is the expected outcome of Stop(). + log.Printf("mavwaked: write to aplay: %v", err) + } + _ = stdin.Close() + + done := make(chan error, 1) + go func() { done <- cmd.Wait() }() + select { + case err := <-done: + if err != nil { + log.Printf("mavwaked: aplay: %v", err) + } + case <-time.After(limit): + log.Printf("mavwaked: aplay did not finish %.1fs of audio within %s, killing (capture was muted the whole time)", + a.Duration(), limit) + if pr := cmd.Process; pr != nil { + _ = pr.Kill() + } + <-done + } + + p.mu.Lock() + if p.gen == gen { + p.playing = false + p.cmd = nil + } + p.mu.Unlock() + }() +} + +func (p *aplayPlayer) Stop() { + p.mu.Lock() + cmd := p.cmd + if cmd != nil { + p.gen++ + p.playing = false + p.cmd = nil + } + p.mu.Unlock() + if cmd != nil && cmd.Process != nil { + _ = cmd.Process.Kill() + } +} + +func (p *aplayPlayer) Playing() bool { + p.mu.Lock() + defer p.mu.Unlock() + return p.playing +} diff --git a/cmd/mavwaked/playback_test.go b/cmd/mavwaked/playback_test.go new file mode 100644 index 0000000..faab3fc --- /dev/null +++ b/cmd/mavwaked/playback_test.go @@ -0,0 +1,33 @@ +package main + +import ( + "testing" + + "github.com/kami/maven/internal/audio" +) + +// The real player must be safe to poke when nothing is playing — the capture +// loop calls Playing() on every 30ms frame, and Stop() lands on an idle +// player whenever a barge-in races the end of a reply. Neither may need +// aplay(1) to be installed. +func TestAplayPlayerIdleIsSafe(t *testing.T) { + p := newAplayPlayer() + if p.Playing() { + t.Fatal("a fresh player reports playing") + } + p.Stop() + p.Stop() + if p.Playing() { + t.Fatal("playing after Stop on an idle player") + } + // Empty audio is a text-only turn: nothing to play, no process to spawn. + p.Play(audio.Audio{Format: audio.PCM16kMono}) + if p.Playing() { + t.Fatal("empty audio started playback") + } +} + +func TestAplayPlayerSatisfiesPlayer(t *testing.T) { + var _ player = newAplayPlayer() + var _ player = &fakePlayer{} +} diff --git a/cmd/mavwaked/session.go b/cmd/mavwaked/session.go new file mode 100644 index 0000000..24de887 --- /dev/null +++ b/cmd/mavwaked/session.go @@ -0,0 +1,233 @@ +package main + +// The capture session: what happens to one 30ms frame, given whether Maven is +// currently speaking. Split out of main.go's processFrame so the decision is +// testable without a mic, a speaker, or a daemon (Vikunja #287). + +import ( + "context" + "log" + "time" + + "github.com/kami/maven/internal/audio" +) + +// utteranceSender ships one complete utterance to the voice server and +// returns the reply audio to play. The real one round-trips over the voice +// wire; tests substitute a recorder. +type utteranceSender interface { + Send(ctx context.Context, utt audio.Audio, lang string) (audio.Audio, error) +} + +// bargeInConfig holds the two numbers barge-in needs. Zero Frames disables +// barge-in entirely — the half-duplex gate still runs. +type bargeInConfig struct { + // RMS is the normalised energy a frame must exceed to count as him + // talking over her rather than the mic hearing her. It is deliberately + // far above the VAD's own floor: the speaker leaks into the mic at + // roughly ambient level, a person talking at the mic does not. + RMS float64 + // Frames is how many consecutive frames must clear RMS before playback + // is cut. One loud frame is a door closing; five in a row is a voice. + Frames int +} + +// Enabled reports whether barge-in should be attempted at all. +func (c bargeInConfig) Enabled() bool { return c.Frames > 0 && c.RMS > 0 } + +// session is the per-client capture state machine. +type session struct { + vad *VAD + player player + sender utteranceSender + lang string + barge bargeInConfig + + // now is the clock, swapped in tests. The round-trip backlog is measured + // in wall time, because that is the only thing that says how much room + // went into the pipe while the daemon was thinking. + now func() time.Time + + // discard is how many buffered frames still have to be thrown away + // before capture means anything again. See dispatch. + discard int + + // recent holds the last few frames seen during playback, so the ones + // that proved he was interrupting can be replayed into the VAD after the + // barge-in reset instead of being clipped off the front of his sentence. + recent [][]byte + + // loudFrames counts consecutive over-threshold frames seen while she is + // speaking. Reset whenever a frame falls back under the threshold, and + // whenever playback ends. + loudFrames int + + // counters, read by tests and logged on the way out. + suppressed int // frames dropped because she was speaking + dropped int // frames dropped as round-trip backlog + bargeIns int // times playback was cut because he spoke over her + sent int // utterances shipped to the daemon + + // loudSum and loudSeen accumulate the energy of suppressed frames, so + // the operator can read what the room actually measures and set + // -barge-in-rms from data instead of guessing. + loudSum float64 + loudSeen int +} + +func newSession(vad *VAD, p player, s utteranceSender, lang string, barge bargeInConfig) *session { + return &session{vad: vad, player: p, sender: s, lang: lang, barge: barge, now: time.Now} +} + +// frameDuration is the wall time one captured frame represents. +const frameDuration = defaultFrameMs * time.Millisecond + +// suppressLogEvery — how many suppressed frames between energy reports. 200 +// frames is six seconds of her talking, so this is roughly one line per reply. +const suppressLogEvery = 200 + +// feed processes one 30ms PCM frame. +// +// While the player is running the capture side is muted: the VAD is not fed +// and no utterance can be produced, so Maven's own reply cannot come back in +// as a new command. The one thing that gets through is barge-in — sustained +// energy well above the speaker's leak level cuts playback, and capture +// resumes on the very next frame with a clean VAD. +func (s *session) feed(ctx context.Context, frame []byte) error { + // Backlog first, before anything looks at this frame. These are frames + // the microphone captured while the round-trip blocked; they arrive in a + // burst at pipe speed and they are not a command, not an answer and not + // an interruption. + if s.discard > 0 { + s.discard-- + s.dropped++ + return nil + } + + if s.player.Playing() { + s.suppressed++ + rms := frameRMS(PCMToI16(frame)) + s.loudSum += rms + s.loudSeen++ + if s.loudSeen >= suppressLogEvery { + // The doc comment asks for energy "well above the speaker's leak + // level" and never says what that is. This is what it is. + log.Printf("mavwaked: suppressed %d frames while speaking, mean rms %.4f (barge-in threshold %.4f)", + s.loudSeen, s.loudSum/float64(s.loudSeen), s.barge.RMS) + s.loudSum, s.loudSeen = 0, 0 + } + if !s.barge.Enabled() { + return nil + } + if rms < s.barge.RMS { + s.loudFrames = 0 + s.recent = s.recent[:0] + return nil + } + s.loudFrames++ + s.keepRecent(frame) + if s.loudFrames < s.barge.Frames { + return nil + } + // He is talking over her. Cut her off, drop the VAD state that + // accumulated from the echo, and start listening for real — starting + // with the frames that proved he was talking. Those used to be + // thrown away, which clipped the first 150ms off his interruption, + // and on a short one that is the whole first word. + s.player.Stop() + s.bargeIns++ + s.loudFrames = 0 + s.vad.Reset() + log.Printf("mavwaked: barge-in — stopped playback") + s.replayRecent() + return nil + } + + // Not speaking. If we just stopped, make sure no echo-era state leaks + // into the next utterance. + if s.loudFrames != 0 { + s.loudFrames = 0 + s.vad.Reset() + } + + utt, state := s.vad.Feed(PCMToI16(frame)) + if state == StateSpeech || utt.Bytes == nil { + return nil + } + return s.dispatch(ctx, utt) +} + +// keepRecent stores a copy of one barge-in trigger frame, keeping at most +// barge.Frames of them. +func (s *session) keepRecent(frame []byte) { + if len(s.recent) >= s.barge.Frames { + copy(s.recent, s.recent[1:]) + s.recent = s.recent[:len(s.recent)-1] + } + s.recent = append(s.recent, append([]byte(nil), frame...)) +} + +// replayRecent feeds the trigger frames back into the freshly reset VAD, so +// his interruption starts where he started it. +// +// Feed cannot complete an utterance here: closing one needs silenceMs of +// trailing quiet and these frames are all above the barge-in threshold, which +// is far above the VAD floor. Any utterance it did return would be a fragment +// of a sentence he is still speaking, so it is not dispatched. +func (s *session) replayRecent() { + for _, f := range s.recent { + s.vad.Feed(PCMToI16(f)) + } + s.recent = s.recent[:0] +} + +// dispatch ships a complete utterance and plays whatever comes back. +// +// Every return path here has to deal with the backlog. Nothing reads the +// microphone while Send is in flight, so the audio piles up in arecord's pipe +// and the kernel buffer, and it arrives in a burst the moment this returns. A +// round-trip is p50 2.7s through the LLM router, which is around 90 frames of +// room, of him finishing his sentence, of the television. +// +// This used to reset the VAD on the reply path only, and for the wrong reason: +// the comment said the VAD had been accumulating during the round-trip, when +// in fact its state is exactly what Feed left it as. The two paths that had no +// reset are the ones that mattered, because neither of them starts playback +// and so neither is covered by the half-duplex gate. A text-only turn fed the +// whole backlog straight into the VAD, and a Send error did the same on every +// failed turn, so a dead socket drove a retry loop off nothing but backlog. +func (s *session) dispatch(ctx context.Context, utt audio.Audio) error { + log.Printf("mavwaked: utterance complete (%.2fs, %d bytes), sending...", utt.Duration(), len(utt.Bytes)) + start := s.now() + reply, err := s.sender.Send(ctx, utt, s.lang) + defer s.dropBacklog(start) + if err != nil { + return err + } + // Count what was shipped, not what was attempted. This used to run + // before the error check, so failed round-trips counted as sent. + s.sent++ + if len(reply.Bytes) == 0 { + log.Printf("mavwaked: empty reply audio (text only)") + return nil + } + s.player.Play(reply) + return nil +} + +// dropBacklog resets the VAD and arranges for the frames captured during the +// round-trip to be thrown away as they arrive. +// +// Discarding them is also what keeps barge-in honest. The Frames guard is +// documented as "long enough that a door or a cough does not cut her off", +// which assumes the frames are real time. Draining a backlog delivers five +// frames in microseconds, so without this she could be cut off by audio +// recorded before she started speaking. +func (s *session) dropBacklog(start time.Time) { + s.vad.Reset() + s.loudFrames = 0 + s.recent = s.recent[:0] + if elapsed := s.now().Sub(start); elapsed > 0 { + s.discard = int(elapsed / frameDuration) + } +} diff --git a/cmd/mavwaked/session_test.go b/cmd/mavwaked/session_test.go new file mode 100644 index 0000000..da6940b --- /dev/null +++ b/cmd/mavwaked/session_test.go @@ -0,0 +1,427 @@ +package main + +import ( + "context" + "errors" + "math" + "testing" + "time" + + "github.com/kami/maven/internal/audio" +) + +// fakePlayer records Play/Stop instead of shelling out to aplay. +type fakePlayer struct { + playing bool + plays int + stops int + last audio.Audio +} + +func (p *fakePlayer) Play(a audio.Audio) { p.playing = true; p.plays++; p.last = a } +func (p *fakePlayer) Stop() { p.playing = false; p.stops++ } +func (p *fakePlayer) Playing() bool { return p.playing } + +// fakeSender records what was shipped and hands back a canned reply. +type fakeSender struct { + sent []audio.Audio + reply audio.Audio + err error +} + +func (s *fakeSender) Send(_ context.Context, utt audio.Audio, _ string) (audio.Audio, error) { + s.sent = append(s.sent, utt) + return s.reply, s.err +} + +func replyAudio() audio.Audio { + return audio.Audio{Format: audio.PCM16kMono, Bytes: make([]byte, 16000)} +} + +// frameAt returns a 30ms frame whose RMS is approximately rms. +func frameAt(rms float64) []byte { + amp := rms * math.Sqrt2 * 32768 + f := make([]int16, frameSamples) + for i := range f { + f[i] = int16(amp * math.Sin(2*math.Pi*440*float64(i)/16000)) + } + return pcmBytes(f) +} + +func silentBytes() []byte { return make([]byte, frameSamples*2) } + +// newTestSession wires a session with fakes and a default VAD. +func newTestSession(barge bargeInConfig) (*session, *fakePlayer, *fakeSender) { + p := &fakePlayer{} + s := &fakeSender{reply: replyAudio()} + return newSession(NewVAD(0, 0, 0, 0), p, s, "ru", barge), p, s +} + +// speakThenPause drives a full utterance through the session: enough loud +// frames to trigger, then enough silence to end it. +func speakThenPause(t *testing.T, sess *session) { + t.Helper() + speechFrames := (defaultSpeechMs + defaultFrameMs - 1) / defaultFrameMs + silenceFrames := (defaultSilenceMs+defaultFrameMs-1)/defaultFrameMs + 2 + loud := frameAt(0.35) + for i := 0; i < speechFrames+5; i++ { + if err := sess.feed(context.Background(), loud); err != nil { + t.Fatalf("feed loud frame %d: %v", i, err) + } + } + for i := 0; i < silenceFrames; i++ { + if err := sess.feed(context.Background(), silentBytes()); err != nil { + t.Fatalf("feed silent frame %d: %v", i, err) + } + } +} + +func TestSessionSendsUtteranceAndPlaysReply(t *testing.T) { + sess, p, snd := newTestSession(bargeInConfig{}) + speakThenPause(t, sess) + + if len(snd.sent) != 1 { + t.Fatalf("sent %d utterances, want 1", len(snd.sent)) + } + if snd.sent[0].Format != audio.PCM16kMono { + t.Errorf("utterance format = %+v, want canonical", snd.sent[0].Format) + } + if p.plays != 1 { + t.Errorf("plays = %d, want 1", p.plays) + } +} + +// The bug this whole file exists for: while the speaker is running, the mic +// hears Maven and the old code shipped that back as a fresh command. +func TestSessionDoesNotHearItselfWhilePlaying(t *testing.T) { + sess, p, snd := newTestSession(bargeInConfig{}) + speakThenPause(t, sess) + if !p.Playing() { + t.Fatal("expected playback to be running after the reply") + } + + // Feed a long stretch of loud audio — Maven's own voice coming back in. + base := sess.suppressed + loud := frameAt(0.35) + for i := 0; i < 200; i++ { + if err := sess.feed(context.Background(), loud); err != nil { + t.Fatalf("feed echo frame %d: %v", i, err) + } + } + + if len(snd.sent) != 1 { + t.Fatalf("sent %d utterances, want 1 — her own reply was captured as a command", len(snd.sent)) + } + if got := sess.suppressed - base; got != 200 { + t.Errorf("suppressed %d of the 200 echo frames, want all of them", got) + } + if p.stops != 0 { + t.Errorf("stops = %d, want 0 — barge-in is off, nothing should cut her off", p.stops) + } +} + +// With barge-in off, no amount of noise stops playback. +func TestSessionBargeInDisabledByDefault(t *testing.T) { + sess, p, _ := newTestSession(bargeInConfig{}) + if sess.barge.Enabled() { + t.Fatal("zero bargeInConfig must be disabled") + } + speakThenPause(t, sess) + veryLoud := frameAt(0.6) + for i := 0; i < 50; i++ { + _ = sess.feed(context.Background(), veryLoud) + } + if p.stops != 0 || sess.bargeIns != 0 { + t.Fatalf("stops = %d, bargeIns = %d, want 0 with barge-in off", p.stops, sess.bargeIns) + } +} + +func TestSessionBargeInCutsPlayback(t *testing.T) { + barge := bargeInConfig{RMS: 0.12, Frames: 5} + sess, p, _ := newTestSession(barge) + speakThenPause(t, sess) + if !p.Playing() { + t.Fatal("expected playback after the reply") + } + + // Four loud frames must not be enough — a door closing is not a voice. + veryLoud := frameAt(0.35) + for i := 0; i < 4; i++ { + _ = sess.feed(context.Background(), veryLoud) + } + if p.stops != 0 { + t.Fatalf("playback cut after 4 frames, want it to hold until %d", barge.Frames) + } + + // The fifth cuts her off. + _ = sess.feed(context.Background(), veryLoud) + if p.stops != 1 || sess.bargeIns != 1 { + t.Fatalf("stops = %d, bargeIns = %d, want 1 and 1", p.stops, sess.bargeIns) + } + if p.Playing() { + t.Fatal("still playing after barge-in") + } +} + +// A burst that falls back under the threshold resets the counter, so noise +// spread over a whole reply never accumulates into a false barge-in. +func TestSessionBargeInNeedsConsecutiveFrames(t *testing.T) { + sess, p, _ := newTestSession(bargeInConfig{RMS: 0.12, Frames: 5}) + speakThenPause(t, sess) + + veryLoud := frameAt(0.35) + quiet := frameAt(0.02) + for i := 0; i < 20; i++ { + _ = sess.feed(context.Background(), veryLoud) + _ = sess.feed(context.Background(), veryLoud) + _ = sess.feed(context.Background(), quiet) + } + if p.stops != 0 || sess.bargeIns != 0 { + t.Fatalf("stops = %d, bargeIns = %d, want 0 — two-frame bursts must not accumulate", p.stops, sess.bargeIns) + } +} + +// Speaker leak sits near the room floor; it must never reach the barge-in bar. +func TestSessionEchoLevelAudioNeverBargesIn(t *testing.T) { + sess, p, _ := newTestSession(bargeInConfig{RMS: 0.12, Frames: 5}) + speakThenPause(t, sess) + + base := sess.suppressed + leak := frameAt(0.05) // loud enough for the VAD, far under the barge bar + for i := 0; i < 300; i++ { + _ = sess.feed(context.Background(), leak) + } + if p.stops != 0 { + t.Fatalf("stops = %d, want 0 — speaker leak must not read as barge-in", p.stops) + } + if got := sess.suppressed - base; got != 300 { + t.Errorf("suppressed %d of the 300 leak frames, want all of them", got) + } +} + +// After barge-in the VAD must start clean, so the interrupting speech is +// captured as a whole utterance rather than joined onto echo state. +func TestSessionCapturesTheInterruptingUtterance(t *testing.T) { + sess, p, snd := newTestSession(bargeInConfig{RMS: 0.12, Frames: 5}) + speakThenPause(t, sess) + + veryLoud := frameAt(0.35) + for i := 0; i < 5; i++ { + _ = sess.feed(context.Background(), veryLoud) + } + if p.stops != 1 { + t.Fatalf("expected barge-in, stops = %d", p.stops) + } + + // He keeps talking; that is a new command. + speakThenPause(t, sess) + if len(snd.sent) != 2 { + t.Fatalf("sent %d utterances, want 2 — the interruption itself must be heard", len(snd.sent)) + } + if p.plays != 2 { + t.Errorf("plays = %d, want 2", p.plays) + } +} + +// A failed round-trip must surface as an error and must not start playback. +func TestSessionSendErrorDoesNotPlay(t *testing.T) { + p := &fakePlayer{} + snd := &fakeSender{err: errors.New("boom")} + sess := newSession(NewVAD(0, 0, 0, 0), p, snd, "ru", bargeInConfig{}) + + speechFrames := (defaultSpeechMs + defaultFrameMs - 1) / defaultFrameMs + silenceFrames := (defaultSilenceMs+defaultFrameMs-1)/defaultFrameMs + 2 + loud := frameAt(0.35) + var lastErr error + for i := 0; i < speechFrames+5; i++ { + _ = sess.feed(context.Background(), loud) + } + for i := 0; i < silenceFrames; i++ { + if err := sess.feed(context.Background(), silentBytes()); err != nil { + lastErr = err + } + } + if lastErr == nil { + t.Fatal("send error was swallowed") + } + if p.plays != 0 || p.Playing() { + t.Fatalf("plays = %d, playing = %v, want no playback on a failed round-trip", p.plays, p.Playing()) + } +} + +// An empty reply (text-only turn) must leave the capture side open. +func TestSessionEmptyReplyLeavesCaptureOpen(t *testing.T) { + p := &fakePlayer{} + snd := &fakeSender{reply: audio.Audio{Format: audio.PCM16kMono}} + sess := newSession(NewVAD(0, 0, 0, 0), p, snd, "ru", bargeInConfig{}) + + speakThenPause(t, sess) + if p.plays != 0 { + t.Fatalf("plays = %d, want 0 for an empty reply", p.plays) + } + speakThenPause(t, sess) + if len(snd.sent) != 2 { + t.Fatalf("sent %d, want 2 — capture must stay open when there is no audio reply", len(snd.sent)) + } +} + +func TestBargeInConfigEnabled(t *testing.T) { + cases := []struct { + c bargeInConfig + want bool + }{ + {bargeInConfig{}, false}, + {bargeInConfig{RMS: 0.12}, false}, + {bargeInConfig{Frames: 5}, false}, + {bargeInConfig{RMS: 0.12, Frames: 5}, true}, + } + for _, tc := range cases { + if got := tc.c.Enabled(); got != tc.want { + t.Errorf("%+v.Enabled() = %v, want %v", tc.c, got, tc.want) + } + } +} + +// slowSender models the real thing: a round-trip takes wall-clock time, and +// the microphone keeps recording into a pipe nobody is reading. +type slowSender struct { + fakeSender + clock *time.Time + took time.Duration +} + +func (s *slowSender) Send(ctx context.Context, utt audio.Audio, lang string) (audio.Audio, error) { + *s.clock = s.clock.Add(s.took) + return s.fakeSender.Send(ctx, utt, lang) +} + +// newSlowSession wires a session whose round-trip takes took of wall time. +func newSlowSession(barge bargeInConfig, reply audio.Audio, err error, took time.Duration) (*session, *fakePlayer, *slowSender) { + now := time.Unix(0, 0) + p := &fakePlayer{} + snd := &slowSender{fakeSender: fakeSender{reply: reply, err: err}, clock: &now, took: took} + sess := newSession(NewVAD(0, 0, 0, 0), p, snd, "ru", barge) + sess.now = func() time.Time { return now } + return sess, p, snd +} + +// A text-only turn starts no playback, so the half-duplex gate does not cover +// it. The backlog captured during the round-trip has to be dropped anyway, or +// three seconds of room arrives at pipe speed and becomes a command. +func TestSessionDropsBacklogAfterAnEmptyReply(t *testing.T) { + sess, p, snd := newSlowSession(bargeInConfig{}, audio.Audio{Format: audio.PCM16kMono}, nil, 3*time.Second) + + speakThenPause(t, sess) + if p.plays != 0 || len(snd.sent) != 1 { + t.Fatalf("plays = %d, sent = %d; want one text-only turn", p.plays, len(snd.sent)) + } + // The tail of speakThenPause already spent a couple of them. + if want := int(3 * time.Second / frameDuration); sess.discard+sess.dropped != want { + t.Fatalf("discard %d + dropped %d frames, want %d (3s of backlog)", sess.discard, sess.dropped, want) + } + + // The burst: the whole backlog, all of it him still talking. + loud := frameAt(0.35) + for i := 0; i < sess.discard; i++ { + if err := sess.feed(context.Background(), loud); err != nil { + t.Fatal(err) + } + } + if len(snd.sent) != 1 { + t.Errorf("the backlog was sent as a second utterance (sent = %d)", len(snd.sent)) + } + if sess.dropped == 0 { + t.Error("no frames were counted as backlog") + } +} + +// Same on the error path. A dead daemon used to seed the next spurious trigger +// on every failed turn, so a dead socket drove a retry loop off backlog alone. +func TestSessionDropsBacklogAfterASendError(t *testing.T) { + sess, _, _ := newSlowSession(bargeInConfig{}, audio.Audio{}, errors.New("boom"), 3*time.Second) + + // Not speakThenPause: the dispatch returns the send error, which that + // helper treats as fatal. + loud := frameAt(0.35) + for i := 0; i < (defaultSpeechMs+defaultFrameMs-1)/defaultFrameMs+5; i++ { + _ = sess.feed(context.Background(), loud) + } + for i := 0; i < (defaultSilenceMs+defaultFrameMs-1)/defaultFrameMs+2; i++ { + _ = sess.feed(context.Background(), silentBytes()) + } + if sess.discard == 0 { + t.Fatal("a failed round-trip left the backlog to be fed into the VAD") + } +} + +// Barge-in must not be triggerable by the backlog. Those frames are him +// finishing the sentence he started before she answered, delivered in +// microseconds, and the five-frame guard assumes real time. +func TestSessionBacklogCannotBargeIn(t *testing.T) { + sess, p, _ := newSlowSession(bargeInConfig{RMS: 0.12, Frames: 5}, replyAudio(), nil, 3*time.Second) + + speakThenPause(t, sess) + if p.plays != 1 || !p.Playing() { + t.Fatalf("plays = %d, playing = %v; want the reply playing", p.plays, p.Playing()) + } + + loud := frameAt(0.35) + backlog := sess.discard + if backlog < 5 { + t.Fatalf("discard = %d, want a real backlog", backlog) + } + for i := 0; i < backlog; i++ { + if err := sess.feed(context.Background(), loud); err != nil { + t.Fatal(err) + } + } + if p.stops != 0 { + t.Fatalf("she was cut off by audio recorded before she started speaking (stops = %d)", p.stops) + } + + // Real-time speech after the backlog still interrupts her. + for i := 0; i < 5; i++ { + if err := sess.feed(context.Background(), loud); err != nil { + t.Fatal(err) + } + } + if p.stops != 1 { + t.Fatalf("stops = %d, want 1 — barge-in must still work after the backlog", p.stops) + } +} + +// The frames that proved he was interrupting are replayed into the VAD, so his +// first word is not clipped. Five trigger frames plus five real ones reach the +// 300ms speech threshold; without the replay the first five are lost and no +// utterance is produced at all. +func TestSessionReplaysTheBargeInTriggerFrames(t *testing.T) { + sess, p, snd := newTestSession(bargeInConfig{RMS: 0.12, Frames: 5}) + speakThenPause(t, sess) + + veryLoud := frameAt(0.35) + for i := 0; i < 5; i++ { + _ = sess.feed(context.Background(), veryLoud) + } + if p.stops != 1 { + t.Fatalf("expected barge-in, stops = %d", p.stops) + } + if p.Playing() { + t.Fatal("fake player still playing after Stop") + } + + speechFrames := (defaultSpeechMs + defaultFrameMs - 1) / defaultFrameMs + for i := 0; i < speechFrames-5; i++ { + if err := sess.feed(context.Background(), veryLoud); err != nil { + t.Fatal(err) + } + } + silenceFrames := (defaultSilenceMs+defaultFrameMs-1)/defaultFrameMs + 2 + for i := 0; i < silenceFrames; i++ { + if err := sess.feed(context.Background(), silentBytes()); err != nil { + t.Fatal(err) + } + } + if len(snd.sent) != 2 { + t.Fatalf("sent %d utterances, want 2 — the 150ms that triggered barge-in was clipped", len(snd.sent)) + } +} diff --git a/cmd/mavwaked/vad.go b/cmd/mavwaked/vad.go index 11f26d5..d6d3b10 100644 --- a/cmd/mavwaked/vad.go +++ b/cmd/mavwaked/vad.go @@ -23,6 +23,15 @@ const ( defaultSilenceMs = 800 // silence hold before declaring end-of-utterance defaultMaxMs = 10000 // cap single utterance at 10s defaultMinRMS = 0.01 // RMS floor (same as mavsttd) + + // Barge-in thresholds. Only used when -barge-in is passed. The RMS is + // x10000 like -min-rms, and sits an order of magnitude above the VAD's + // own floor on purpose: with no acoustic echo canceller, a frame only + // counts as "he is talking over her" if it is far louder than what the + // speaker leaks back into the mic. 5 frames is 150ms — long enough that + // a door or a cough does not cut her off mid-sentence. + defaultBargeRMS = 1200 // 0.12 normalised RMS + defaultBargeFrames = 5 ) // frameSamples — samples per 30ms frame at 16kHz. diff --git a/cmd/mavweb/ambient.go b/cmd/mavweb/ambient.go new file mode 100644 index 0000000..470ea53 --- /dev/null +++ b/cmd/mavweb/ambient.go @@ -0,0 +1,150 @@ +package main + +import ( + "crypto/subtle" + "encoding/json" + "errors" + "io" + "log" + "net/http" + "strings" + + "github.com/kami/maven/internal/calendar" + "github.com/kami/maven/internal/ipc" +) + +// POST /api/ambient — the work calendar read (Vikunja #126). +// +// Maven does not hold a work credential. A corp mail or calendar session on the +// homelab ties the box's blast radius to the employer's data, so the work +// calendar is read as a SIGNAL instead: an Android notification-listener on the +// owner's phone posts meeting notifications here over wg/LAN, and the ones that +// clearly describe a meeting become calendar events at source=ambient:notif, +// confidence below 1.0. Mail as a notification signal, not a mailbox. +// +// Off unless configured: no -ambient-token, no route. The token is a shared +// secret because the poster is a phone service, not a browser — WebAuthn has no +// answer for a background Android service. The endpoint is write-only and +// accepts exactly one shape of write; it cannot read anything back out. +// +// A notification with no recognisable clock reading stores NOTHING. Maven is +// not a guesser-of-truth, and a mailbox of noise rendered as invented meetings +// is worse than a gap. +// +// KNOWN GAP: this writes calendar_event_* and nothing else, so an ambient +// meeting is good enough to recite and not good enough to stop a nudge — +// calendar_busy is still written only by the CalDAV poller. That is backwards, +// since suppressing a nudge is the lower-risk use of a low-confidence signal. +// calendar_busy is a level rather than an event, so an ambient writer needs an +// expiry, which is its own task and not a change here. + +// ambientMaxBody bounds the request. A notification is two short lines. +const ambientMaxBody = 8 << 10 + +type ambientResp struct { + Stored bool `json:"stored"` + Key string `json:"key,omitempty"` + Reason string `json:"reason,omitempty"` +} + +// handleAmbient ingests one relayed notification. token is the configured +// shared secret; an empty token means the capability is off and the handler is +// never registered, so it is treated as a hard failure here too. +func handleAmbient(w http.ResponseWriter, r *http.Request, core ipc.CoreAPI, token string) { + if r.Method != http.MethodPost { + http.Error(w, "POST only", http.StatusMethodNotAllowed) + return + } + if token == "" { + http.Error(w, "ambient ingest disabled (no -ambient-token)", http.StatusServiceUnavailable) + return + } + if !ambientAuthorized(r, token) { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + if core == nil { + http.Error(w, "ambient ingest disabled (no -core)", http.StatusServiceUnavailable) + return + } + + var n calendar.Notification + body, err := io.ReadAll(io.LimitReader(r.Body, ambientMaxBody)) + if err != nil { + http.Error(w, "read failed", http.StatusBadRequest) + return + } + if err := json.Unmarshal(body, &n); err != nil { + http.Error(w, "bad json", http.StatusBadRequest) + return + } + if n.Posted.IsZero() { + writeAmbient(w, http.StatusBadRequest, ambientResp{Reason: "posted_at is required"}) + return + } + + ev, ok := calendar.EventFromNotification(n) + if !ok { + // Not an event. 202: the relay did its job, there is just nothing here + // worth remembering, and it must not retry. + writeAmbient(w, http.StatusAccepted, ambientResp{Reason: "no meeting time in notification"}) + return + } + + key := calendar.FactKey(ev) + val := calendar.FactValue(ev) + + // Append-only discipline, same as cmd/mavcaldav: a phone reposts the same + // notification many times, and each repost is the same event. + if prev, err := core.LatestFactBySource(r.Context(), key, calendar.SourceAmbient); err == nil && prev.Value == val { + writeAmbient(w, http.StatusOK, ambientResp{Stored: false, Key: key, Reason: "unchanged"}) + return + } else if err != nil && !errors.Is(err, ipc.ErrNoFact) { + log.Printf("ambient: read %s: %v", key, err) + http.Error(w, "read failed", http.StatusBadGateway) + return + } + + // kind=env: an observation about the world, never a self-fact — a passive + // signal does not write truth about the owner. Confidence below 1.0 is the + // honest part: this is a notification about a meeting, not a reading of a + // calendar, and the query path hedges when it recites one. + if _, err := core.WriteFact(r.Context(), ipc.WriteFactReq{ + Ts: ev.Start, + Kind: "env", + Key: key, + Value: val, + Source: calendar.SourceAmbient, + Confidence: calendar.AmbientConfidence, + }); err != nil { + log.Printf("ambient: write %s: %v", key, err) + http.Error(w, "write failed", http.StatusBadGateway) + return + } + log.Printf("ambient: %s=%s (%s, pkg=%s)", key, val, calendar.SourceAmbient, n.Package) + writeAmbient(w, http.StatusCreated, ambientResp{Stored: true, Key: key}) +} + +// ambientAuthorized accepts the token as a bearer header or as an X-Maven-Token +// header, compared in constant time. +// +// The scheme is matched case-insensitively. RFC 7235 says it is, and a phone +// client sending "bearer " used to fall through to the X-Maven-Token +// branch and get a silent 401 with nothing to see from the phone's side. +func ambientAuthorized(r *http.Request, token string) bool { + got := "" + if authz := strings.TrimSpace(r.Header.Get("Authorization")); len(authz) >= len("Bearer") && + strings.EqualFold(authz[:len("Bearer")], "Bearer") { + got = strings.TrimSpace(authz[len("Bearer"):]) + } + if got == "" { + got = strings.TrimSpace(r.Header.Get("X-Maven-Token")) + } + return subtle.ConstantTimeCompare([]byte(got), []byte(token)) == 1 +} + +func writeAmbient(w http.ResponseWriter, code int, resp ambientResp) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + json.NewEncoder(w).Encode(resp) +} diff --git a/cmd/mavweb/ambient_test.go b/cmd/mavweb/ambient_test.go new file mode 100644 index 0000000..1636a1b --- /dev/null +++ b/cmd/mavweb/ambient_test.go @@ -0,0 +1,247 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/kami/maven/internal/calendar" + "github.com/kami/maven/internal/ipc" +) + +const ambientTestToken = "s3cret" + +// ambientCore adds provenance-scoped reads to fakeCore, which the dedupe path +// needs. +type ambientCore struct { + fakeCore + latest map[string]ipc.Fact // "key|source" → fact + readErr error +} + +func (c *ambientCore) LatestFactBySource(_ context.Context, key, source string) (ipc.Fact, error) { + if c.readErr != nil { + return ipc.Fact{}, c.readErr + } + f, ok := c.latest[key+"|"+source] + if !ok { + return ipc.Fact{}, ipc.ErrNoFact + } + return f, nil +} + +func postAmbient(t *testing.T, core ipc.CoreAPI, token string, n calendar.Notification) (*httptest.ResponseRecorder, ambientResp) { + t.Helper() + body, err := json.Marshal(n) + if err != nil { + t.Fatal(err) + } + req := httptest.NewRequest(http.MethodPost, "/api/ambient", strings.NewReader(string(body))) + req.Header.Set("Authorization", "Bearer "+ambientTestToken) + rr := httptest.NewRecorder() + handleAmbient(rr, req, core, token) + var resp ambientResp + json.Unmarshal(rr.Body.Bytes(), &resp) + return rr, resp +} + +func meetingNotification() calendar.Notification { + return calendar.Notification{ + Package: "com.google.android.gm", + Title: "Планёрка", + Text: "10:00-10:30", + // Local, like a phone relaying from the box's own timezone: the fact + // key and value are stamped on the owner's clock, so a UTC reading + // here would only be testing the offset of the test machine. + Posted: time.Date(2026, 8, 3, 9, 40, 0, 0, time.Local), + } +} + +func TestHandleAmbientStoresMeeting(t *testing.T) { + core := &ambientCore{} + rr, resp := postAmbient(t, core, ambientTestToken, meetingNotification()) + + if rr.Code != http.StatusCreated { + t.Fatalf("status = %d, want 201: %s", rr.Code, rr.Body) + } + if !resp.Stored { + t.Errorf("resp = %+v, want stored", resp) + } + if len(core.writeLog) != 1 { + t.Fatalf("expected 1 fact write, got %d", len(core.writeLog)) + } + got := core.writeLog[0] + if got.Source != calendar.SourceAmbient { + t.Errorf("source = %q, want %q", got.Source, calendar.SourceAmbient) + } + if got.Confidence >= 1.0 { + t.Errorf("confidence = %v — a notification is not a calendar read", got.Confidence) + } + if got.Confidence != calendar.AmbientConfidence { + t.Errorf("confidence = %v, want %v", got.Confidence, calendar.AmbientConfidence) + } + if got.Kind != "env" { + t.Errorf("kind = %q — a passive signal never writes a self-fact", got.Kind) + } + if want := "calendar_event_20260803_"; !strings.HasPrefix(got.Key, want) { + t.Errorf("key = %q, want prefix %q", got.Key, want) + } + if got.Value != "Планёрка @ 10:00-10:30" { + t.Errorf("value = %q", got.Value) + } +} + +// A phone reposts the same notification many times. Each repost is the same +// event, and the append-only log must not fill with duplicates. +func TestHandleAmbientDedupesReposts(t *testing.T) { + core := &ambientCore{} + postAmbient(t, core, ambientTestToken, meetingNotification()) + if len(core.writeLog) != 1 { + t.Fatalf("first post did not write") + } + w := core.writeLog[0] + core.latest = map[string]ipc.Fact{w.Key + "|" + w.Source: {Value: w.Value}} + + rr, resp := postAmbient(t, core, ambientTestToken, meetingNotification()) + if rr.Code != http.StatusOK { + t.Errorf("status = %d, want 200 for an unchanged repost", rr.Code) + } + if resp.Stored { + t.Error("a repost must not be stored again") + } + if len(core.writeLog) != 1 { + t.Errorf("wrote %d facts, want 1", len(core.writeLog)) + } +} + +// The conservative half: noise stores nothing at all. +func TestHandleAmbientIgnoresNonMeetings(t *testing.T) { + core := &ambientCore{} + rr, resp := postAmbient(t, core, ambientTestToken, calendar.Notification{ + Package: "com.google.android.gm", + Title: "3 новых письма", + Posted: time.Now(), + }) + if rr.Code != http.StatusAccepted { + t.Errorf("status = %d, want 202 (accepted, nothing to store — the relay must not retry)", rr.Code) + } + if resp.Stored { + t.Error("a notification with no meeting time must store nothing") + } + if len(core.writeLog) != 0 { + t.Fatalf("wrote %d facts for a non-meeting", len(core.writeLog)) + } +} + +func TestHandleAmbientAuth(t *testing.T) { + body := `{"title":"Планёрка 10:00","posted_at":"2026-08-03T09:40:00Z"}` + + newReq := func(hdr, val string) *http.Request { + r := httptest.NewRequest(http.MethodPost, "/api/ambient", strings.NewReader(body)) + if hdr != "" { + r.Header.Set(hdr, val) + } + return r + } + + t.Run("no token rejected", func(t *testing.T) { + core := &ambientCore{} + rr := httptest.NewRecorder() + handleAmbient(rr, newReq("", ""), core, ambientTestToken) + if rr.Code != http.StatusUnauthorized { + t.Errorf("status = %d, want 401", rr.Code) + } + if len(core.writeLog) != 0 { + t.Error("an unauthorized post must not write") + } + }) + + t.Run("wrong token rejected", func(t *testing.T) { + rr := httptest.NewRecorder() + handleAmbient(rr, newReq("Authorization", "Bearer nope"), &ambientCore{}, ambientTestToken) + if rr.Code != http.StatusUnauthorized { + t.Errorf("status = %d, want 401", rr.Code) + } + }) + + // RFC 7235 says the scheme is case-insensitive. A phone sending + // "bearer " used to fall through to the X-Maven-Token branch and get a + // 401 that looked, from the phone's side, like a wrong token. + t.Run("lowercase bearer scheme accepted", func(t *testing.T) { + rr := httptest.NewRecorder() + handleAmbient(rr, newReq("Authorization", "bearer "+ambientTestToken), &ambientCore{}, ambientTestToken) + if rr.Code != http.StatusCreated { + t.Errorf("status = %d, want 201: %s", rr.Code, rr.Body) + } + }) + + // A bare token with no scheme is not a bearer header. Accepting it made the + // Authorization branch a second, undocumented X-Maven-Token. + t.Run("bare token in Authorization rejected", func(t *testing.T) { + rr := httptest.NewRecorder() + handleAmbient(rr, newReq("Authorization", ambientTestToken), &ambientCore{}, ambientTestToken) + if rr.Code != http.StatusUnauthorized { + t.Errorf("status = %d, want 401", rr.Code) + } + }) + + t.Run("X-Maven-Token accepted", func(t *testing.T) { + rr := httptest.NewRecorder() + handleAmbient(rr, newReq("X-Maven-Token", ambientTestToken), &ambientCore{}, ambientTestToken) + if rr.Code != http.StatusCreated { + t.Errorf("status = %d, want 201: %s", rr.Code, rr.Body) + } + }) + + t.Run("capability off", func(t *testing.T) { + rr := httptest.NewRecorder() + handleAmbient(rr, newReq("Authorization", "Bearer "+ambientTestToken), &ambientCore{}, "") + if rr.Code != http.StatusServiceUnavailable { + t.Errorf("status = %d, want 503 when no token is configured", rr.Code) + } + }) + + t.Run("GET rejected", func(t *testing.T) { + rr := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/api/ambient", nil) + handleAmbient(rr, r, &ambientCore{}, ambientTestToken) + if rr.Code != http.StatusMethodNotAllowed { + t.Errorf("status = %d, want 405 — the ingest is write-only", rr.Code) + } + }) +} + +func TestHandleAmbientBadInput(t *testing.T) { + t.Run("bad json", func(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, "/api/ambient", strings.NewReader("{nope")) + req.Header.Set("X-Maven-Token", ambientTestToken) + rr := httptest.NewRecorder() + handleAmbient(rr, req, &ambientCore{}, ambientTestToken) + if rr.Code != http.StatusBadRequest { + t.Errorf("status = %d, want 400", rr.Code) + } + }) + + t.Run("missing posted_at", func(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, "/api/ambient", strings.NewReader(`{"title":"Планёрка 10:00"}`)) + req.Header.Set("X-Maven-Token", ambientTestToken) + rr := httptest.NewRecorder() + handleAmbient(rr, req, &ambientCore{}, ambientTestToken) + if rr.Code != http.StatusBadRequest { + t.Errorf("status = %d, want 400", rr.Code) + } + }) + + t.Run("read error surfaces", func(t *testing.T) { + core := &ambientCore{readErr: fmt.Errorf("socket closed")} + rr, _ := postAmbient(t, core, ambientTestToken, meetingNotification()) + if rr.Code != http.StatusBadGateway { + t.Errorf("status = %d, want 502", rr.Code) + } + }) +} diff --git a/cmd/mavweb/ecosystem.go b/cmd/mavweb/ecosystem.go index 7198853..77a329c 100644 --- a/cmd/mavweb/ecosystem.go +++ b/cmd/mavweb/ecosystem.go @@ -8,6 +8,8 @@ import ( "net/http" "sync" "time" + + "github.com/kami/maven/internal/ipc" ) // The three sibling services Maven coordinates are headless JSON APIs (no web UI @@ -84,9 +86,13 @@ type ecoData struct { Nexus ecoPanel[ecoEntity] Praxis ecoPanel[ecoItem] Hexis ecoPanel[ecoCap] + Calls ecoPanel[ipc.EcosystemTrace] } -func handleEcosystem(w http.ResponseWriter, r *http.Request, urls ecoURLs) { +// handleEcosystem renders the three sibling panels plus Maven's own log of the +// calls she made to them. The call log comes from core, not from the siblings: +// it is what Maven saw, including the hops that never got an answer. +func handleEcosystem(w http.ResponseWriter, r *http.Request, urls ecoURLs, core ipc.CoreAPI) { ctx := r.Context() var d ecoData var wg sync.WaitGroup @@ -102,6 +108,15 @@ func handleEcosystem(w http.ResponseWriter, r *http.Request, urls ecoURLs) { go func() { defer wg.Done(); d.Hexis.Err = getEco(ctx, urls.hexis, "/api/v1/capabilities", &d.Hexis.Rows) }() wg.Wait() + if core == nil { + d.Calls.Err = "not configured" + } else if rows, err := core.RecentEcosystemTraces(ctx, 50); err != nil { + log.Printf("ecosystem traces: %v", err) + d.Calls.Err = "core read failed" + } else { + d.Calls.Rows = rows + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") if err := ecosystemTmpl.Execute(w, d); err != nil { log.Printf("ecosystem render: %v", err) diff --git a/cmd/mavweb/ecosystem.html b/cmd/mavweb/ecosystem.html index 1bce15f..719173e 100644 --- a/cmd/mavweb/ecosystem.html +++ b/cmd/mavweb/ecosystem.html @@ -41,11 +41,24 @@ {{end}} +
+
+

Calls what Maven asked them

+
+{{with .Calls}} +{{if .Err}}
calls — {{.Err}}
+{{else if not .Rows}}
no ecosystem calls yet.
+{{else}}
+{{range .Rows}}{{end}} +
whenserviceoperationstatusmshttpcorrelation
{{ago .Ts}}{{.Service}}{{.Operation}}{{if eq .Status "ok"}}ok{{else}}{{.Status}}{{end}}{{.DurationMs}}{{if .HTTPStatus}}{{.HTTPStatus}}{{else}}—{{end}}{{.CorrelationID}}
{{end}} +{{end}} +
+ {{template "shellBottom"}} `, + OccurredAt: time.Date(2026, 8, 1, 7, 0, 0, 0, time.UTC), + }}} + body := getEvents(t, core).Body.String() + if strings.Contains(body, "") { + t.Error("intake title was not escaped") + } + if !strings.Contains(body, "<script>") { + t.Error("intake title is missing from the page entirely") + } +} diff --git a/cmd/mavweb/handlers_test.go b/cmd/mavweb/handlers_test.go index f13b26a..6b56c51 100644 --- a/cmd/mavweb/handlers_test.go +++ b/cmd/mavweb/handlers_test.go @@ -17,11 +17,12 @@ import ( ) // fakeCore records the mutating calls handleTools makes and returns canned -// tool lists / errors. Embedding ipc.CoreAPI (nil) satisfies the large +// tool lists / errors. Embedding ipc.UnimplementedCoreAPI satisfies the large // interface — only the methods the handlers touch are overridden; any other -// call would nil-panic, which is fine since the handlers never make them. +// call returns ipc.ErrNotImplemented instead of nil-panicking, so a test that +// accidentally exercises an undeclared method fails loudly. type fakeCore struct { - ipc.CoreAPI + ipc.UnimplementedCoreAPI proposed, enabled []ipc.Tool listErr error @@ -62,6 +63,26 @@ type fakeCore struct { // for handleTrace tests tickTrace ipc.TickTrace traceErr error + + // for handleChatAPI tests + chatText string + chatErr error + + // for the MCP section of /tools + mcpServers []ipc.MCPServerStatus + mcpErr error +} + +func (f *fakeCore) MCPServers(context.Context) ([]ipc.MCPServerStatus, error) { + return f.mcpServers, f.mcpErr +} + +func (f *fakeCore) Chat(_ context.Context, text string) (string, error) { + f.chatText = text + if f.chatErr != nil { + return "", f.chatErr + } + return "поняла", nil } func (f *fakeCore) EnableTool(_ context.Context, name string, cmd []string, destructive bool, scope string, _ time.Time) error { @@ -1044,3 +1065,189 @@ func TestHandleRoutines_NilCore_503(t *testing.T) { t.Fatalf("status = %d, want 503", rr.Code) } } + +// --- handleChatAPI step-up gate (Vikunja #317) --- +// +// POST /api/chat reaches the router, the LLM and the act path, so it carries +// the same gate as POST /tools and POST /api/revert. + +func postChat(text string) *http.Request { + req := httptest.NewRequest(http.MethodPost, "/api/chat", strings.NewReader("text="+url.QueryEscape(text))) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + return req +} + +func TestHandleChatAPI_RequireStepUp_FailsClosed(t *testing.T) { + core := &fakeCore{} + rr := httptest.NewRecorder() + handleChatAPI(rr, postChat("выключи свет"), core, nil, true) + if rr.Code != http.StatusForbidden { + t.Fatalf("status = %d, want 403; body=%s", rr.Code, rr.Body.String()) + } + if core.chatText != "" { + t.Errorf("core.Chat called with %q, but -require-stepup should deny", core.chatText) + } +} + +func TestHandleChatAPI_UnassertedSession_Denied(t *testing.T) { + core := &fakeCore{} + rr := httptest.NewRecorder() + handleChatAPI(rr, postChat("выключи свет"), core, webauthn.NewPasskeySession(5*time.Minute), false) + if rr.Code != http.StatusForbidden { + t.Fatalf("status = %d, want 403", rr.Code) + } + if core.chatText != "" { + t.Errorf("core.Chat called with %q despite an unasserted session", core.chatText) + } +} + +func TestHandleChatAPI_AssertedSession_PassesGate(t *testing.T) { + core := &fakeCore{} + rr := httptest.NewRecorder() + handleChatAPI(rr, postChat("привет"), core, stepUpSession(), true) + if rr.Code != http.StatusSeeOther { + t.Fatalf("status = %d, want 303; body=%s", rr.Code, rr.Body.String()) + } + if core.chatText != "привет" { + t.Errorf("core.Chat text = %q, want %q", core.chatText, "привет") + } +} + +// Default deploy: WebAuthn unconfigured and -require-stepup off ⇒ chat keeps +// working, resting on the transport-level auth in front of mavweb. +func TestHandleChatAPI_FailOpenByDefault(t *testing.T) { + core := &fakeCore{} + rr := httptest.NewRecorder() + handleChatAPI(rr, postChat("привет"), core, nil, false) + if rr.Code != http.StatusSeeOther { + t.Fatalf("status = %d, want 303", rr.Code) + } + if core.chatText != "привет" { + t.Errorf("core.Chat text = %q, want %q", core.chatText, "привет") + } +} + +// The MCP section renders the configured servers, and a proposal that already +// knows its cmd prefills the enable form so the argv is not retyped by hand. +func TestHandleTools_GET_MCPSection(t *testing.T) { + core := &fakeCore{ + proposed: []ipc.Tool{{ + Name: "vikunja_list_tasks", Scope: "mcp:vikunja", + Cmd: []string{"mcp", "vikunja", "list_tasks"}, Destructive: true, + Utterance: "mcp vikunja/list_tasks: List tasks in a project.", + }}, + mcpServers: []ipc.MCPServerStatus{ + {Name: "vikunja", Transport: "http", Target: "http://192.168.1.104:9100/mcp", Connected: true, Server: "vikunja 0.1.0", Tools: 4}, + {Name: "files", Transport: "stdio", Target: "mcp-server-fs /srv", Err: "start: no such file"}, + }, + } + rr := httptest.NewRecorder() + handleTools(rr, httptest.NewRequest(http.MethodGet, "/tools", nil), core, nil, false) + if rr.Code != http.StatusOK { + t.Fatalf("status = %d", rr.Code) + } + body := rr.Body.String() + for _, want := range []string{ + "MCP servers", "vikunja", "192.168.1.104:9100/mcp", "vikunja 0.1.0", + "files", "no such file", + `value="mcp vikunja list_tasks"`, // the enable form is prefilled + "checked", // and pre-marked destructive (no readOnlyHint) + } { + if !strings.Contains(body, want) { + t.Errorf("missing %q in /tools output", want) + } + } +} + +// MCP off (or an older core that does not know the method) renders the section +// empty instead of breaking the page. +func TestHandleTools_GET_MCPUnavailable(t *testing.T) { + core := &fakeCore{mcpErr: ipc.ErrNotImplemented} + rr := httptest.NewRecorder() + handleTools(rr, httptest.NewRequest(http.MethodGet, "/tools", nil), core, nil, false) + if rr.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rr.Code) + } + if !strings.Contains(rr.Body.String(), "no MCP servers configured") { + t.Error("expected the empty-state copy") + } +} + +// --- voice-path step-up gate (Vikunja #317) --- +// +// POST /api/ptt and GET /ws proxy audio into mavend's voice port, which runs +// the same router, LLM and act path as POST /api/chat. They used to be +// ungated on the grounds that the voice port is only reachable inside the +// deploy, but mavweb is the thing proxying into it from outside. Speaking +// "выключи свет" is not a smaller act than typing it. + +// unreachableVoice is a closed port: a request that clears the gate fails at +// the dial with 503, which is how these tests tell "passed" from "denied". +const unreachableVoice = "127.0.0.1:1" + +func pttReq() *http.Request { + return httptest.NewRequest(http.MethodPost, "/api/ptt", strings.NewReader("PCM-ish bytes")) +} + +func TestHandlePTT_RequireStepUp_FailsClosed(t *testing.T) { + rr := httptest.NewRecorder() + handlePTT(rr, pttReq(), unreachableVoice, nil, true) + if rr.Code != http.StatusForbidden { + t.Fatalf("status = %d, want 403; body=%s", rr.Code, rr.Body.String()) + } +} + +func TestHandlePTT_UnassertedSession_Denied(t *testing.T) { + rr := httptest.NewRecorder() + handlePTT(rr, pttReq(), unreachableVoice, webauthn.NewPasskeySession(5*time.Minute), false) + if rr.Code != http.StatusForbidden { + t.Fatalf("status = %d, want 403; body=%s", rr.Code, rr.Body.String()) + } +} + +func TestHandlePTT_AssertedSession_PassesGate(t *testing.T) { + rr := httptest.NewRecorder() + handlePTT(rr, pttReq(), unreachableVoice, stepUpSession(), true) + if rr.Code == http.StatusForbidden { + t.Fatalf("status = 403 on an asserted session; body=%s", rr.Body.String()) + } + if rr.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want 503 from the dial past the gate; body=%s", rr.Code, rr.Body.String()) + } +} + +// Default deploy: WebAuthn unconfigured and -require-stepup off ⇒ push-to-talk +// keeps working, resting on the transport-level auth in front of mavweb. +func TestHandlePTT_FailOpenByDefault(t *testing.T) { + rr := httptest.NewRecorder() + handlePTT(rr, pttReq(), unreachableVoice, nil, false) + if rr.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want 503 from the dial past the gate; body=%s", rr.Code, rr.Body.String()) + } +} + +func TestHandleWS_RequireStepUp_FailsClosed(t *testing.T) { + rr := httptest.NewRecorder() + handleWS(rr, httptest.NewRequest(http.MethodGet, "/ws", nil), unreachableVoice, nil, true) + if rr.Code != http.StatusForbidden { + t.Fatalf("status = %d, want 403; body=%s", rr.Code, rr.Body.String()) + } +} + +func TestHandleWS_UnassertedSession_Denied(t *testing.T) { + rr := httptest.NewRecorder() + handleWS(rr, httptest.NewRequest(http.MethodGet, "/ws", nil), unreachableVoice, webauthn.NewPasskeySession(5*time.Minute), false) + if rr.Code != http.StatusForbidden { + t.Fatalf("status = %d, want 403; body=%s", rr.Code, rr.Body.String()) + } +} + +// Past the gate the handshake itself fails (httptest's recorder cannot be +// hijacked), which is not a 403. That is all this asserts: the gate let it by. +func TestHandleWS_AssertedSession_PassesGate(t *testing.T) { + rr := httptest.NewRecorder() + handleWS(rr, httptest.NewRequest(http.MethodGet, "/ws", nil), unreachableVoice, stepUpSession(), true) + if rr.Code == http.StatusForbidden { + t.Fatalf("status = 403 on an asserted session; body=%s", rr.Body.String()) + } +} diff --git a/cmd/mavweb/main.go b/cmd/mavweb/main.go index 0d41585..b39823e 100644 --- a/cmd/mavweb/main.go +++ b/cmd/mavweb/main.go @@ -18,6 +18,7 @@ import ( "net/url" "os" "os/signal" + "strconv" "strings" "time" @@ -25,6 +26,7 @@ import ( "github.com/kami/maven/internal/audio" "github.com/kami/maven/internal/ipc" "github.com/kami/maven/internal/pattern" + "github.com/kami/maven/internal/tasks" "github.com/kami/maven/internal/voice" "github.com/kami/maven/internal/webauthn" ) @@ -58,6 +60,9 @@ var notificationsHTML string //go:embed reminders.html var remindersHTML string +//go:embed tasks.html +var tasksHTML string + //go:embed voice.html var voiceHTML string @@ -67,6 +72,9 @@ var ecosystemHTML string //go:embed morning.html var morningHTML string +//go:embed events.html +var eventsHTML string + // ── Ethos Workstation Shell ── // // Two template pieces that wrap every page: @@ -97,9 +105,11 @@ var sidebarSections = []struct { Pages: []struct{ Label, URL, Key string }{ {Label: "Rule Trace", URL: "/trace", Key: "trace"}, {Label: "Notifications", URL: "/notifications", Key: "notifications"}, + {Label: "Tasks", URL: "/tasks", Key: "tasks"}, {Label: "Reminders", URL: "/reminders", Key: "reminders"}, {Label: "Routines", URL: "/routines", Key: "routines"}, {Label: "Morning", URL: "/morning", Key: "morning"}, + {Label: "Intake", URL: "/events", Key: "events"}, }, }, { @@ -119,6 +129,7 @@ var sidebarSections = []struct { Label: "Settings", Pages: []struct{ Label, URL, Key string }{ {Label: "Tools", URL: "/tools", Key: "tools"}, + {Label: "Model", URL: "/models", Key: "models"}, {Label: "Passkey", URL: "/auth/passkey", Key: "passkey"}, }, }, @@ -170,6 +181,8 @@ func pageIcon(key string) string { return `` case "notifications": return `` + case "tasks": + return `` case "reminders": return `` case "routines": @@ -184,6 +197,8 @@ func pageIcon(key string) string { return `` case "tools": return `` + case "models": + return `` case "passkey": return `` default: @@ -202,6 +217,8 @@ func pageTitle(key string) string { return "Rule Trace" case "notifications": return "Notifications" + case "tasks": + return "Tasks" case "reminders": return "Reminders" case "routines": @@ -216,6 +233,8 @@ func pageTitle(key string) string { return "Ecosystem" case "tools": return "Tools" + case "models": + return "Resident Model" case "passkey": return "Passkey" default: @@ -300,6 +319,10 @@ var dashTmpl = template.Must(template.New("dash").Funcs(shellFuncs()).Parse(shel // human surface is here (they ship no web UI of their own). var ecosystemTmpl = template.Must(template.New("ecosystem").Funcs(shellFuncs()).Parse(shellTopHTML + ecosystemHTML + shellBottomHTML)) +// eventsTmpl — the unified intake journal (Vikunja #283), read-only. Same +// shape as trace.html and morning.html: server-rendered, refreshed on reload. +var eventsTmpl = template.Must(template.New("events").Funcs(shellFuncs()).Parse(shellTopHTML + eventsHTML + shellBottomHTML)) + // morningTmpl — read-only view of today's checklist state per configured // morning routine (internal/morning). Same shape as trace.html: a plain // server-rendered page, refreshed on reload — no live-update loop, since @@ -314,7 +337,7 @@ func noCache(h http.Handler) http.Handler { } func main() { - addr := flag.String("addr", ":9200", "HTTP listen address") + addr := flag.String("addr", "127.0.0.1:9200", "HTTP listen address (loopback by default; pass e.g. \":9200\" or a LAN IP deliberately for wider exposure — POST /chat and /routines are state-changing)") voiceAddr := flag.String("voice", "127.0.0.1:9100", "voice server TCP addr (host:port)") // ntfyWS: the ntfy WebSocket subscribe URL the PWA connects to for in-app // nudge delivery, e.g. wss://ntfy.kvmx.ru/maven/ws?auth=. The @@ -329,14 +352,23 @@ func main() { coreSock := flag.String("core", "", "mavend IPC socket path for presence-signal ingest (empty = disabled)") pkOrigin := flag.String("webauthn-origin", "", "WebAuthn origin URL (e.g. https://maven.kvmx.ru)") pkRPID := flag.String("webauthn-rpid", "", "WebAuthn RP ID (e.g. maven.kvmx.ru)") - requireStepUp := flag.Bool("require-stepup", false, "fail closed on step-up-gated actions (/tools POST, /api/revert) when WebAuthn step-up cannot be asserted; default false preserves the historical fail-open behaviour") + requireStepUp := flag.Bool("require-stepup", false, "fail closed on step-up-gated actions (POST /tools, /routines, /models, /api/revert, /api/chat, /api/ptt and GET /ws) when WebAuthn step-up cannot be asserted; default false preserves the historical fail-open behaviour") pkFile := flag.String("passkey-file", "./passkeys.json", "path to WebAuthn credential store (JSON)") nexusURL := flag.String("nexus", "", "Nexus base URL for the /ecosystem panel (empty = not configured)") praxisURL := flag.String("praxis", "", "Praxis base URL for the /ecosystem panel (empty = not configured)") hexisURL := flag.String("hexis", "", "Hexis base URL for the /ecosystem panel (empty = not configured)") + // Shared secret for POST /api/ambient, the notification-relay ingest that + // reads the work calendar as a signal instead of holding a work credential + // (see ambient.go). Empty ⇒ the route is not registered at all. + ambientToken := flag.String("ambient-token", "", "shared secret for POST /api/ambient notification ingest (empty = ingest disabled, route not registered)") flag.Parse() var core ipc.CoreAPI + // swapConn — a second connection, for /models and nothing else. A model swap + // is a multi-minute IPC call and ipc.Client serialises everything on one + // mutex, so sharing the connection would freeze every other page for the + // length of the load. See handleModels. + var swapConn modelController if *coreSock != "" { c, err := ipc.DialWait(*coreSock, 60*time.Second) if err != nil { @@ -344,6 +376,12 @@ func main() { } defer c.Close() core = c + if sc, err := ipc.Dial(*coreSock); err != nil { + log.Printf("models: second core connection failed (%v) — /models will share the main one and a swap will block the other pages", err) + } else { + defer sc.Close() + swapConn = sc + } } mux := http.NewServeMux() @@ -361,12 +399,9 @@ func main() { handleVoice(w, r) })) - mux.HandleFunc("/ws", func(w http.ResponseWriter, r *http.Request) { - handleWS(w, r, *voiceAddr) - }) - mux.HandleFunc("/api/ptt", func(w http.ResponseWriter, r *http.Request) { - handlePTT(w, r, *voiceAddr) - }) + // /ws and /api/ptt are registered further down, next to /api/chat: they + // carry the same step-up gate and so need stepUpSession, which is only + // built once the passkey endpoints are wired. mux.HandleFunc("/api/ping", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("pong")) }) @@ -381,6 +416,14 @@ func main() { mux.HandleFunc("/api/signal", func(w http.ResponseWriter, r *http.Request) { handleSignal(w, r, core) }) + // Off unless configured: no token, no route — an unconfigured ingest is not + // a 503 waiting to be probed, it does not exist. + if *ambientToken != "" { + mux.HandleFunc("/api/ambient", func(w http.ResponseWriter, r *http.Request) { + handleAmbient(w, r, core, *ambientToken) + }) + log.Printf("mavweb: ambient notification ingest enabled at POST /api/ambient") + } mux.HandleFunc("/dash", func(w http.ResponseWriter, r *http.Request) { handleDash(w, r, core) }) @@ -396,12 +439,20 @@ func main() { mux.HandleFunc("/reminders", func(w http.ResponseWriter, r *http.Request) { handleReminders(w, r, core) }) + // /tasks — capture + review. POST is not step-up gated; see handleTasks for + // why a task write is not in the same class as /tools or /routines. + mux.HandleFunc("/tasks", func(w http.ResponseWriter, r *http.Request) { + handleTasks(w, r, core) + }) mux.HandleFunc("/morning", func(w http.ResponseWriter, r *http.Request) { handleMorning(w, r, core) }) + mux.HandleFunc("/events", func(w http.ResponseWriter, r *http.Request) { + handleEvents(w, r, core) + }) ecoURLsCfg := ecoURLs{nexus: *nexusURL, praxis: *praxisURL, hexis: *hexisURL} mux.HandleFunc("/ecosystem", func(w http.ResponseWriter, r *http.Request) { - handleEcosystem(w, r, ecoURLsCfg) + handleEcosystem(w, r, ecoURLsCfg, core) }) // ----- passkey (WebAuthn) endpoints ----- // Wired when both -core and a configured origin are present. The origin @@ -433,10 +484,29 @@ func main() { mux.HandleFunc("/auth/webauthn/assert/finish", pk.AssertFinish) } if stepUpSession == nil { + // One surface per line: these are read in a terminal at the moment + // someone is deciding whether the box is safe to expose. + surfaces := []string{ + "POST /tools defines arbitrary argv via name+cmd, which internal/tool then EXECUTES", + "POST /routines accepting schedules recurring firing", + "POST /models chooses the resident model that routes and words every turn", + "POST /api/revert voids the latest fact for a key", + "POST /api/chat reaches the router, the LLM and, through applyAction, the act path", + "POST /api/ptt the same, from audio", + "GET /ws the same, streamed", + } if *requireStepUp { - log.Printf("SECURITY: step-up verification is DISABLED (-webauthn-origin/-webauthn-rpid unset) and -require-stepup is set: POST /tools (tool enable/disable/dismiss — defines and executes arbitrary argv) and POST /api/revert will be DENIED (403). Set -webauthn-origin and -webauthn-rpid to enable passkey step-up.") + log.Printf("SECURITY: step-up verification is DISABLED (-webauthn-origin/-webauthn-rpid unset) and -require-stepup is set. These surfaces will be DENIED (403):") } else { - log.Printf("SECURITY WARNING: step-up verification is DISABLED because -webauthn-origin/-webauthn-rpid are unset. UNGUARDED SURFACES: POST /tools (defines arbitrary argv via name+cmd, which internal/tool then EXECUTES) and POST /api/revert (voids the latest fact for a key). These are protected only by whatever transport-level auth sits in front of mavweb (wg+nginx+auth) — do NOT expose -addr on a public interface. Set -webauthn-origin and -webauthn-rpid to require passkey step-up, or pass -require-stepup to fail closed instead.") + log.Printf("SECURITY WARNING: step-up verification is DISABLED (-webauthn-origin/-webauthn-rpid unset). These surfaces are UNGUARDED:") + } + for _, s := range surfaces { + log.Printf("SECURITY: %s", s) + } + if *requireStepUp { + log.Printf("SECURITY: set -webauthn-origin and -webauthn-rpid to enable passkey step-up.") + } else { + log.Printf("SECURITY: they rest on the transport-level auth in front of mavweb (wg+nginx+auth). Do NOT expose -addr on a public interface. Set -webauthn-origin and -webauthn-rpid to require passkey step-up, or pass -require-stepup to fail closed instead.") } } @@ -453,19 +523,57 @@ func main() { mux.HandleFunc("/routines", func(w http.ResponseWriter, r *http.Request) { handleRoutines(w, r, core, stepUpSession, *requireStepUp) }) + // /models — the resident-model surface (Vikunja #250). Same step-up gate as + // /tools, and for a comparable reason: which model is loaded decides how every + // utterance is routed and how every reply is worded. GET is read-only. + mux.HandleFunc("/models", func(w http.ResponseWriter, r *http.Request) { + handleModels(w, r, core, swapConn, stepUpSession, *requireStepUp) + }) - // /api/revert voids the latest fact for a key — a store mutation, so it - // sits behind the same passkey step-up as tool enable (nil session ⇒ - // WebAuthn unconfigured ⇒ transport-level auth only, same as /tools). + // State-changing routes on this server, and their gate (Vikunja #317): + // + // POST /tools step-up — defines argv that internal/tool executes + // POST /routines step-up — accepting schedules recurring firing + // POST /models step-up — replaces the model that routes and phrases + // POST /api/revert step-up — voids the latest fact for a key + // POST /api/chat step-up — reaches the router, LLM and the act path + // POST /api/ptt step-up — audio into runTurn, so the same router, + // LLM and act path as /api/chat + // GET /ws step-up — same, streamed + // POST /api/signal none — appends a presence fact, no argv, no act + // POST /api/ambient shared secret — notification relay, constant-time + // token compare, poster is a phone service + // and not a browser, so step-up cannot apply + // + // "step-up" means stepUpOK: asserted passkey when WebAuthn is configured, + // otherwise fail-open unless -require-stepup, which denies. + // + // /api/ptt and /ws used to be ungated, justified by mavend's voice port + // being reachable only inside the deploy. That argument does not hold: + // mavweb is the thing proxying into it from outside. Speaking "выключи + // свет" is not a smaller act than typing it (Vikunja #317). + // + // The gate here is per-request, which costs the hands-free case a passkey + // assertion per turn whenever WebAuthn is configured. A session-scoped + // assertion covering a run of turns is the right shape and is its own task. + // + // GET /chat only renders the page and echoes back the q/r query params the + // POST redirect set — nothing to gate. mux.HandleFunc("/chat", func(w http.ResponseWriter, r *http.Request) { handleChatPage(w, r, core) }) mux.HandleFunc("/api/chat", func(w http.ResponseWriter, r *http.Request) { - handleChatAPI(w, r, core) + handleChatAPI(w, r, core, stepUpSession, *requireStepUp) }) mux.HandleFunc("/api/revert", func(w http.ResponseWriter, r *http.Request) { handleRevert(w, r, core, stepUpSession, *requireStepUp) }) + mux.HandleFunc("/ws", func(w http.ResponseWriter, r *http.Request) { + handleWS(w, r, *voiceAddr, stepUpSession, *requireStepUp) + }) + mux.HandleFunc("/api/ptt", func(w http.ResponseWriter, r *http.Request) { + handlePTT(w, r, *voiceAddr, stepUpSession, *requireStepUp) + }) srv := &http.Server{Addr: *addr, Handler: mux} @@ -483,7 +591,11 @@ func main() { } } -func handleWS(w http.ResponseWriter, r *http.Request, voiceAddr string) { +func handleWS(w http.ResponseWriter, r *http.Request, voiceAddr string, session *webauthn.PasskeySession, requireStepUp bool) { + if !stepUpOK(session, requireStepUp) { + http.Error(w, "step-up required: assert a passkey first", http.StatusForbidden) + return + } conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{ OriginPatterns: []string{"*"}, }) @@ -643,7 +755,7 @@ const toolsHTML = `{{template "shellTop" "tools"}} {{if .Msg}}
{{.Msg}}
{{end}}

proposed {{len .Proposed}}

-{{if .Proposed}}

maven drafted these from acts she couldn't run. Fill the command (argv, space-separated) and enable.

+{{if .Proposed}}

maven drafted these from acts she couldn't run. Fill the command (argv, space-separated) and enable. A row in an mcp: scope came from an MCP server and already knows what it calls — check the command, then enable.

{{range .Proposed}} @@ -651,8 +763,8 @@ const toolsHTML = `{{template "shellTop" "tools"}} - - + + @@ -681,6 +793,19 @@ const toolsHTML = `{{template "shellTop" "tools"}}
enable proposed tools above, or ask maven to configure one
{{end}} +
+

MCP servers {{len .MCP}}

+{{if .MCP}}

servers she connects OUT to. Their tools appear above as proposals — a configured server is a place she may look, not a capability she has. A stdio target is a process on this box; an http one on a loopback or LAN address is inside the network, so treat its tools accordingly.

+
namescopefrom utteranceenable as
{{.Name}}{{.Scope}}{{.Utterance}}
+{{range .MCP}} + +{{end}}
nametransporttargetstatetools
{{.Name}}{{.Transport}}{{.Target}}{{if .Connected}}connected{{if .Server}} — {{.Server}}{{end}}{{else}}down{{if .Err}} — {{.Err}}{{end}}{{end}}{{.Tools}}
+{{else}}
+ +
no MCP servers configured
+
add an mcp.servers block to mavend.json to let her use an external tool server
+
{{end}} +
{{template "shellBottom"}}` // routinesHTML — proposed routine review surface. One row per thing maven @@ -720,6 +845,8 @@ var passkeyTmpl = template.Must(template.New("passkey").Funcs(shellFuncs()).Pars var voiceTmpl = template.Must(template.New("voice").Funcs(shellFuncs()).Parse(shellTopHTML + voiceHTML + shellBottomHTML)) +var tasksTmpl = template.Must(template.New("tasks").Funcs(shellFuncs()).Parse(shellTopHTML + tasksHTML + shellBottomHTML)) + var routinesTmpl = template.Must(template.New("routines").Funcs(shellFuncs()).Parse(shellTopHTML + routinesHTML + shellBottomHTML)) var traceTmpl = template.Must(template.New("trace").Funcs(func() template.FuncMap { @@ -790,6 +917,214 @@ func handleReminders(w http.ResponseWriter, r *http.Request, core ipc.CoreAPI) { } } +// now — the wall clock, indirected so the task page can be rendered at a fixed +// instant in a test. internal/tasks is pure and the daemon path already ranks +// through a clock it is handed; the page had no reason to be the one surface +// that could only be tested at whatever time it happened to run. +var now = time.Now + +// resolvedShown — how many finished tasks the page renders. The list is +// history, it only grows, and the rows below the first screen are read by +// nobody. +const resolvedShown = 50 + +// taskRow is one line on /tasks, with every timestamp already formatted so the +// template holds no date logic. +type taskRow struct { + ID int64 + Text string + Source string + Evidence string + Status string + Due string + Created string + Resolved string + ResolvedBy string + // Why — the ranker's reason for this row's position (Vikunja #129), in + // Russian, empty when nothing distinguished the task. Blank is the honest + // rendering: he never said this one mattered more. + Why string +} + +// handleTasks serves the task review surface (GET) and the four writes it +// offers (POST): add, confirm, done, drop. +// +// Not step-up gated, unlike /tools and /routines, and the difference is the +// point: enabling a tool defines argv Maven will execute, and accepting a +// routine hands the tick loop a new standing reason to interrupt him. A task is +// neither — nothing in the tick loop reads the tasks table, so the worst a +// weaker caller can do here is write a line onto a list he reads himself. It +// still sits behind whatever transport auth fronts mavweb, like every other +// page. +// +// "confirm" is the only interesting move: it promotes a candidate Maven derived +// from something she read into work he owns. That review step is why derived +// tasks are captured as candidates in the first place. +func handleTasks(w http.ResponseWriter, r *http.Request, core ipc.CoreAPI) { + if core == nil { + http.Error(w, "tasks disabled (no -core)", http.StatusServiceUnavailable) + return + } + ctx := r.Context() + var msg, errMsg string + if r.Method == http.MethodPost { + var err error + msg, err = applyTaskPost(ctx, core, r) + if err != nil { + log.Printf("tasks: %v", err) + errMsg = err.Error() + } + } + + all, err := core.ListTasks(ctx, "") + if err != nil { + log.Printf("tasks: list: %v", err) + http.Error(w, "tasks error: "+err.Error(), http.StatusBadGateway) + return + } + // Live rows are ordered by the same ranker the spoken list uses, so the page + // and the voice reply can never disagree about what comes first. Resolved + // rows keep store order (newest first) — ranking finished work is pointless. + var live []tasks.Item + var resolved []taskRow + resolvedTotal := 0 + for _, t := range all { + switch t.Status { + case "candidate", "open": + live = append(live, tasks.Item{ + ID: t.ID, Text: t.Text, Status: t.Status, + Created: t.CreatedTs, Due: t.Due, Weight: t.Weight, + }) + default: + resolvedTotal++ + // Finished work is history, and the history only grows. The page + // showed every row that ever existed, which is a page that gets + // slower every month for a section nobody reads past the top of. + if len(resolved) >= resolvedShown { + continue + } + resolved = append(resolved, taskRow{ + ID: t.ID, Text: t.Text, Source: t.Source, Evidence: t.Evidence, + Status: t.Status, Created: fmtTaskTime(&t.CreatedTs), + Due: fmtTaskDate(t.Due), Resolved: fmtTaskTime(t.Resolved), + ResolvedBy: t.ResolvedBy, + }) + } + } + byID := make(map[int64]ipc.Task, len(all)) + for _, t := range all { + byID[t.ID] = t + } + var cands, open []taskRow + for _, r := range tasks.Rank(live, now()) { + t := byID[r.ID] + row := taskRow{ + ID: t.ID, Text: t.Text, Source: t.Source, Evidence: t.Evidence, + Status: t.Status, Created: fmtTaskTime(&t.CreatedTs), + Due: fmtTaskDate(t.Due), Resolved: fmtTaskTime(t.Resolved), + Why: r.Reason, + } + if t.Status == "candidate" { + // A candidate's due date is Maven's reading of a mail, so its + // ranking reason is not shown as if he had set a priority. + row.Why = "" + cands = append(cands, row) + } else { + open = append(open, row) + } + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") + if err := tasksTmpl.Execute(w, struct { + Msg, Err string + Candidates []taskRow + Open []taskRow + Resolved []taskRow + ResolvedMore bool + }{msg, errMsg, cands, open, resolved, resolvedTotal > len(resolved)}); err != nil { + log.Printf("tasks render: %v", err) + } +} + +// applyTaskPost performs one write and returns the message to show. A bad +// request returns an error, which the page renders inline rather than as a +// bare 400 — this is a form surface, not an API. +func applyTaskPost(ctx context.Context, core ipc.CoreAPI, r *http.Request) (string, error) { + action := r.FormValue("action") + if action == "add" { + text := strings.TrimSpace(r.FormValue("text")) + if text == "" { + return "", errors.New("empty task text") + } + req := ipc.CaptureTaskReq{Text: text, Source: "tap:web", Status: "open", Ts: now()} + // Importance is his, stated on the form. Out-of-range values are + // clamped rather than rejected — a bad select is not worth a 400. + if v := r.FormValue("weight"); v != "" { + // strconv, not Sscanf: Sscanf("3junk", "%d") succeeds with 3, and a + // form value is not a place to accept trailing garbage. + wgt, err := strconv.Atoi(v) + if err != nil || wgt < 0 { + return "", fmt.Errorf("bad weight %q", v) + } + if wgt > tasks.MaxWeight { + wgt = tasks.MaxWeight + } + req.Weight = wgt + } + if d := r.FormValue("due"); d != "" { + due, err := time.ParseInLocation("2006-01-02", d, now().Location()) + if err != nil { + return "", fmt.Errorf("bad due date %q", d) + } + req.Due = &due + } + resp, err := core.CaptureTask(ctx, req) + if err != nil { + return "", err + } + if resp.Promoted { + return "confirmed a candidate maven had found", nil + } + if !resp.Created { + return "already on the list", nil + } + return "added task", nil + } + + id, err := strconv.ParseInt(r.FormValue("id"), 10, 64) + if err != nil { + return "", errors.New("invalid id") + } + var status, msg string + switch action { + case "confirm": + status, msg = "open", "confirmed task" + case "done": + status, msg = "done", "task done" + case "drop": + status, msg = "dropped", "dropped task" + default: + return "", fmt.Errorf("unknown action %q", action) + } + if err := core.SetTaskStatus(ctx, id, status, now(), "tap:web"); err != nil { + return "", err + } + return msg, nil +} + +func fmtTaskTime(t *time.Time) string { + if t == nil || t.IsZero() { + return "—" + } + return t.Local().Format("02 Jan 15:04") +} + +func fmtTaskDate(t *time.Time) string { + if t == nil || t.IsZero() { + return "—" + } + return t.Local().Format("02 Jan") +} + // routineRow is one line on the page: what maven noticed, in her words, and // how long ago she noticed it. type routineRow struct { @@ -935,12 +1270,63 @@ func handleMorning(w http.ResponseWriter, r *http.Request, core ipc.CoreAPI) { http.Error(w, "core read failed", http.StatusBadGateway) return } + view := morningView{Routines: status} + // The day plan (#128) shows on this page because it is the same question at + // a different scale. A plan read that fails must not take the checklist + // down with it — the page degrades to what it had before. + plan, err := core.DayPlan(ctx) + if err != nil { + log.Printf("morning: day plan: %v", err) + view.PlanErr = err.Error() + } else { + view.Plan = &plan + } w.Header().Set("Content-Type", "text/html; charset=utf-8") - if err := morningTmpl.Execute(w, status); err != nil { + if err := morningTmpl.Execute(w, view); err != nil { log.Printf("morning render: %v", err) } } +// morningView — what /morning renders: today's plan on top, the checklist +// state under it. PlanErr is set instead of Plan when the core could not build +// a plan, so the page says so rather than showing an empty day. +type morningView struct { + Plan *ipc.DayPlan + PlanErr string + Routines []ipc.MorningRoutineStatus +} + +// eventsView — what /events renders. Err is set instead of Events when the +// core could not serve the journal, so the page says why rather than showing an +// empty intake and implying nothing arrived. +type eventsView struct { + Events []ipc.IntakeEvent + Err string +} + +// eventsPageLimit — how many envelopes the page shows. The ring holds more; a +// page is for scanning what just happened, not for archaeology. +const eventsPageLimit = 200 + +func handleEvents(w http.ResponseWriter, r *http.Request, core ipc.CoreAPI) { + if core == nil { + http.Error(w, "intake journal disabled (no -core)", http.StatusServiceUnavailable) + return + } + var view eventsView + evs, err := core.RecentEvents(r.Context(), eventsPageLimit) + if err != nil { + log.Printf("events: %v", err) + view.Err = err.Error() + } else { + view.Events = evs + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") + if err := eventsTmpl.Execute(w, view); err != nil { + log.Printf("events render: %v", err) + } +} + func handleVoice(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/html; charset=utf-8") if err := voiceTmpl.Execute(w, nil); err != nil { @@ -1068,12 +1454,20 @@ func handleTools(w http.ResponseWriter, r *http.Request, core ipc.CoreAPI, sessi http.Error(w, "core read failed", http.StatusBadGateway) return } + // MCP is off by default and an older core may not know the method at all, + // so a failure here renders an empty section rather than breaking the page. + servers, err := core.MCPServers(ctx) + if err != nil { + log.Printf("tools: mcp servers: %v", err) + servers = nil + } w.Header().Set("Content-Type", "text/html; charset=utf-8") if err := toolsTmpl.Execute(w, struct { Msg string Proposed []ipc.Tool Enabled []ipc.Tool - }{msg, proposed, enabled}); err != nil { + MCP []ipc.MCPServerStatus + }{msg, proposed, enabled, servers}); err != nil { log.Printf("tools render: %v", err) } } @@ -1134,11 +1528,15 @@ func readOneFrame(r io.Reader) (*voice.Response, *voice.Push, error) { return &voice.Response{ID: raw.ID, Result: raw.Result, Error: raw.Error}, nil, nil } -func handlePTT(w http.ResponseWriter, r *http.Request, voiceAddr string) { +func handlePTT(w http.ResponseWriter, r *http.Request, voiceAddr string, session *webauthn.PasskeySession, requireStepUp bool) { if r.Method != http.MethodPost { http.Error(w, "POST only", 405) return } + if !stepUpOK(session, requireStepUp) { + http.Error(w, "step-up required: assert a passkey first", http.StatusForbidden) + return + } body, err := io.ReadAll(r.Body) if err != nil { http.Error(w, err.Error(), 400) @@ -1258,7 +1656,14 @@ func handleChatPage(w http.ResponseWriter, r *http.Request, core ipc.CoreAPI) { } // handleChatAPI processes a chat message POST and redirects back to /chat. -func handleChatAPI(w http.ResponseWriter, r *http.Request, core ipc.CoreAPI) { +// +// State-changing, and the widest surface on this server: the text reaches the +// router, the LLM, and through mavend's applyAction the whole action path +// including `act` — so it is gated on the same step-up as POST /tools and +// POST /api/revert (Vikunja #317). With WebAuthn unconfigured the gate is +// fail-open exactly like the others (see stepUpOK); with -require-stepup it +// denies, which is the point of that flag. +func handleChatAPI(w http.ResponseWriter, r *http.Request, core ipc.CoreAPI, session *webauthn.PasskeySession, requireStepUp bool) { if r.Method != http.MethodPost { http.Error(w, "POST only", http.StatusMethodNotAllowed) return @@ -1267,6 +1672,10 @@ func handleChatAPI(w http.ResponseWriter, r *http.Request, core ipc.CoreAPI) { http.Error(w, "chat disabled (no -core)", http.StatusServiceUnavailable) return } + if !stepUpOK(session, requireStepUp) { + http.Error(w, "step-up required: assert a passkey first", http.StatusForbidden) + return + } text := strings.TrimSpace(r.FormValue("text")) if text == "" { http.Redirect(w, r, "/chat", http.StatusSeeOther) diff --git a/cmd/mavweb/models.go b/cmd/mavweb/models.go new file mode 100644 index 0000000..28b598a --- /dev/null +++ b/cmd/mavweb/models.go @@ -0,0 +1,161 @@ +package main + +import ( + "context" + "errors" + "html/template" + "log" + "net/http" + "strconv" + "strings" + + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/webauthn" +) + +// The resident-model surface (Vikunja #250). +// +// GET shows which model llama-server actually has loaded and which files the +// daemon is configured to allow. POST swaps to one of them, behind the same +// step-up gate as POST /tools: the loaded model decides how every utterance is +// routed and how every reply is worded, so it is an owner action. +// +// There is nothing on this page Maven can press. The swap is an IPC method rated +// AuthStepUp in internal/auth, unreachable from an act, an intent or a timer. + +// modelController — the two non-CoreAPI methods this page needs. *ipc.Client +// satisfies it; a core without a swap allowlist answers ErrUnknownMethod, which +// the page renders as "not configured" rather than an error. +type modelController interface { + ModelStatus(ctx context.Context) (ipc.ModelStatusResp, error) + SwapModel(ctx context.Context, req ipc.SwapModelReq) (ipc.SwapModelResp, error) +} + +var modelsTmpl = template.Must(template.New("models").Funcs(shellFuncs()).Parse(shellTopHTML + modelsHTML + shellBottomHTML)) + +const modelsHTML = `{{template "shellTop" "models"}} +

Resident model

+

swapping requires step-up — assert a passkey first. The old model is unloaded before the new one is loaded (one model fits the iGPU at a time), so turns during the load are refused and fall back to the classifier.

+

a swap is not remembered. Nothing writes it down, so the next restart of the daemon — including the one mavupdate does — comes back on phraser.model_path from the config. Make it stick by editing that.

+{{if .Msg}}
{{.Msg}}
{{end}} +{{if .Err}}
{{.Err}}
{{end}} +{{if .Off}} +
+

swap not configured

+

this core has no phraser.swap_models allowlist, so there is nothing to swap to. Add the gguf paths you allow to deploy/mavend.json and restart once.

+
+{{else}} +
+

loaded now

+
+ + + + + +
model{{.Status.Model}}
file{{.Status.ModelPath}}
server{{.Status.BaseURL}}
n_ctx{{.Status.NCtx}}
n_gpu_layers{{.Status.NGpuLayers}}
+

the model name is what llama-server reports for itself, not what the config says it should be.

+
+
+

allowed models {{len .Status.Swappable}}

+{{if .Status.Swappable}}
+{{range .Status.Swappable}} +{{end}} +
file
{{.}} + +
+{{else}}
no models allowlisted
{{end}} +
+{{end}} +{{template "shellBottom"}}` + +type modelsPage struct { + Msg string + Err string + Off bool + Status ipc.ModelStatusResp +} + +// handleModels renders the model surface (GET) and applies a swap (POST). +// +// A failed swap is reported as a failure with the model that is still serving +// named, because that is the state the operator needs: the daemon rolled back +// and is answering turns, it just is not answering them with what he asked for. +// swapConn, when non-nil, is a SECOND connection to the same core, used for +// nothing but this page. ipc.Client holds its mutex for a whole roundtrip and +// neither side sets a read deadline, so a swap on the shared connection blocks +// /dash, /history, /notifications and everything else for as long as the load +// takes: a 90s drain plus a 60s launch plus a 30s probe, doubled if it rolls +// back. No browser timeout frees them, because the server side keeps reading +// the reply. On its own connection the swap only blocks the swap. +func handleModels(w http.ResponseWriter, r *http.Request, core ipc.CoreAPI, swapConn modelController, session *webauthn.PasskeySession, requireStepUp bool) { + if core == nil { + http.Error(w, "models disabled (no -core)", http.StatusServiceUnavailable) + return + } + mc, ok := swapConn, swapConn != nil + if !ok { + mc, ok = core.(modelController) + } + if !ok { + http.Error(w, "models unavailable: core connection does not support model swap", http.StatusServiceUnavailable) + return + } + ctx := r.Context() + page := modelsPage{} + + if r.Method == http.MethodPost { + if !stepUpOK(session, requireStepUp) { + http.Error(w, "step-up required: assert a passkey first", http.StatusForbidden) + return + } + path := strings.TrimSpace(r.FormValue("model_path")) + if path == "" { + http.Error(w, "model_path required", http.StatusBadRequest) + return + } + // Only the path comes off the form. n_ctx and n_gpu_layers are load + // settings the daemon keeps from what is live, and the resident model is + // a Thinking variant whose 4096-token window is sized for reasoning + // tokens (CLAUDE.md). A field nothing renders, that a hand-crafted POST + // could use to shrink the window under the router, is not worth having. + // Changing them is a config edit and a restart. + res, err := mc.SwapModel(ctx, ipc.SwapModelReq{ModelPath: path}) + switch { + case err == nil: + page.Msg = "loaded " + res.Model + " (" + strconv.FormatInt(res.TookMs, 10) + "ms)" + log.Printf("models: swapped to %s (%s) in %dms", res.ModelPath, res.Model, res.TookMs) + case errors.Is(err, ipc.ErrForbidden): + http.Error(w, "refused: that model is not in phraser.swap_models, or step-up was not asserted", http.StatusForbidden) + return + case errors.Is(err, ipc.ErrUnknownMethod): + http.Error(w, "swap not configured on this core", http.StatusServiceUnavailable) + return + case res.NoBackend: + page.Err = "swap failed AND the rollback failed — no model is loaded. She is answering from templates and routing on the classifier. Try loading a model again; a restart is not needed." + log.Printf("models: swap to %s failed and the rollback failed, no model loaded: %v", path, err) + case res.RolledBack: + page.Err = "swap failed, rolled back to " + res.Model + " — she is still answering, with the old model" + log.Printf("models: swap to %s failed, rolled back: %v", path, err) + default: + page.Err = "swap failed: " + err.Error() + log.Printf("models: swap to %s failed: %v", path, err) + } + } + + st, err := mc.ModelStatus(ctx) + if err != nil { + if errors.Is(err, ipc.ErrUnknownMethod) { + page.Off = true + } else { + log.Printf("models: status: %v", err) + http.Error(w, "core read failed", http.StatusBadGateway) + return + } + } + page.Status = st + w.Header().Set("Content-Type", "text/html; charset=utf-8") + if err := modelsTmpl.Execute(w, page); err != nil { + log.Printf("models render: %v", err) + } +} diff --git a/cmd/mavweb/models_test.go b/cmd/mavweb/models_test.go new file mode 100644 index 0000000..7810b5f --- /dev/null +++ b/cmd/mavweb/models_test.go @@ -0,0 +1,206 @@ +package main + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/webauthn" +) + +// fakeModelCore is a core that supports the two model methods. It records what +// the page asked for, so the tests can assert the gate rather than the HTML. +type fakeModelCore struct { + ipc.UnimplementedCoreAPI + + status ipc.ModelStatusResp + statusErr error + + swapResp ipc.SwapModelResp + swapErr error + swapped []ipc.SwapModelReq +} + +func (f *fakeModelCore) ModelStatus(ctx context.Context) (ipc.ModelStatusResp, error) { + return f.status, f.statusErr +} + +func (f *fakeModelCore) SwapModel(ctx context.Context, req ipc.SwapModelReq) (ipc.SwapModelResp, error) { + f.swapped = append(f.swapped, req) + return f.swapResp, f.swapErr +} + +func modelsGET(t *testing.T, core ipc.CoreAPI) *httptest.ResponseRecorder { + t.Helper() + w := httptest.NewRecorder() + handleModels(w, httptest.NewRequest(http.MethodGet, "/models", nil), core, nil, nil, false) + return w +} + +func modelsPOST(t *testing.T, core ipc.CoreAPI, session *webauthn.PasskeySession, requireStepUp bool, path string) *httptest.ResponseRecorder { + t.Helper() + r := httptest.NewRequest(http.MethodPost, "/models", strings.NewReader("model_path="+path)) + r.Header.Set("Content-Type", "application/x-www-form-urlencoded") + w := httptest.NewRecorder() + handleModels(w, r, core, nil, session, requireStepUp) + return w +} + +func TestModels_GETShowsTheLoadedModelAndTheAllowlist(t *testing.T) { + core := &fakeModelCore{status: ipc.ModelStatusResp{ + Model: "Qwen3-1.7B-UD-Q4_K_XL", + ModelPath: "/opt/maven/models/llm/qwen3.gguf", + BaseURL: "http://127.0.0.1:18099", + NCtx: 4096, + Swappable: []string{"/opt/maven/models/llm/qwen3.gguf", "/opt/maven/models/llm/qwen3-cpt.gguf"}, + }} + w := modelsGET(t, core) + if w.Code != http.StatusOK { + t.Fatalf("GET /models = %d; want 200", w.Code) + } + body := w.Body.String() + for _, want := range []string{"Qwen3-1.7B-UD-Q4_K_XL", "qwen3-cpt.gguf", "4096"} { + if !strings.Contains(body, want) { + t.Errorf("page does not mention %q", want) + } + } + if len(core.swapped) != 0 { + t.Errorf("a GET swapped the model: %v", core.swapped) + } +} + +func TestModels_POSTRequiresStepUpWhenFailingClosed(t *testing.T) { + // No WebAuthn configured (nil session) + -require-stepup ⇒ deny, exactly + // like POST /tools. Nothing reaches core. + core := &fakeModelCore{} + w := modelsPOST(t, core, nil, true, "/opt/maven/models/llm/qwen3.gguf") + if w.Code != http.StatusForbidden { + t.Fatalf("POST /models without assertable step-up = %d; want 403", w.Code) + } + if len(core.swapped) != 0 { + t.Fatalf("a denied POST still called SwapModel: %v", core.swapped) + } +} + +func TestModels_POSTSwapsAndReportsTheModelThatAnswered(t *testing.T) { + core := &fakeModelCore{ + swapResp: ipc.SwapModelResp{Model: "qwen3-cpt", ModelPath: "/m/cpt.gguf", TookMs: 4200}, + status: ipc.ModelStatusResp{Model: "qwen3-cpt", ModelPath: "/m/cpt.gguf"}, + } + w := modelsPOST(t, core, nil, false, "/m/cpt.gguf") + if w.Code != http.StatusOK { + t.Fatalf("POST /models = %d; want 200", w.Code) + } + if len(core.swapped) != 1 || core.swapped[0].ModelPath != "/m/cpt.gguf" { + t.Fatalf("SwapModel calls = %v; want one for /m/cpt.gguf", core.swapped) + } + if !strings.Contains(w.Body.String(), "loaded qwen3-cpt") { + t.Errorf("page does not report which model was loaded:\n%s", w.Body.String()) + } +} + +func TestModels_RolledBackSwapSaysSheIsStillAnswering(t *testing.T) { + core := &fakeModelCore{ + swapResp: ipc.SwapModelResp{Model: "qwen3", ModelPath: "/m/old.gguf", RolledBack: true}, + swapErr: errBrokenModel{}, + status: ipc.ModelStatusResp{Model: "qwen3", ModelPath: "/m/old.gguf"}, + } + w := modelsPOST(t, core, nil, false, "/m/cpt.gguf") + if w.Code != http.StatusOK { + t.Fatalf("POST /models after a rollback = %d; want 200 with the failure rendered", w.Code) + } + body := w.Body.String() + if !strings.Contains(body, "rolled back to qwen3") { + t.Errorf("page does not say it rolled back:\n%s", body) + } +} + +func TestModels_RefusedPathIs403(t *testing.T) { + core := &fakeModelCore{swapErr: ipc.ErrForbidden} + w := modelsPOST(t, core, nil, false, "/etc/passwd") + if w.Code != http.StatusForbidden { + t.Fatalf("POST /models with a non-allowlisted path = %d; want 403", w.Code) + } +} + +func TestModels_UnconfiguredCoreRendersOff(t *testing.T) { + core := &fakeModelCore{statusErr: ipc.ErrUnknownMethod} + w := modelsGET(t, core) + if w.Code != http.StatusOK { + t.Fatalf("GET /models against a core without the swap = %d; want 200", w.Code) + } + if !strings.Contains(w.Body.String(), "swap not configured") { + t.Errorf("page does not say the capability is off:\n%s", w.Body.String()) + } +} + +func TestModels_CoreWithoutTheMethodsIs503(t *testing.T) { + // An in-process CoreAPI (no swap methods) must not 500 the page. + w := modelsGET(t, ipc.UnimplementedCoreAPI{}) + if w.Code != http.StatusServiceUnavailable { + t.Fatalf("GET /models on a core without the methods = %d; want 503", w.Code) + } +} + +type errBrokenModel struct{} + +func (errBrokenModel) Error() string { return "llm: server did not start" } + +func TestModels_TotalFailureDoesNotSaySheIsStillAnswering(t *testing.T) { + // The load failed and so did the rollback: nothing is loaded. The page used + // to branch on RolledBack first and render "rolled back to — she is still + // answering, with the old model" over an empty model name. + core := &fakeModelCore{ + swapResp: ipc.SwapModelResp{NoBackend: true}, + swapErr: errBrokenModel{}, + status: ipc.ModelStatusResp{Model: "unknown"}, + } + w := modelsPOST(t, core, nil, false, "/m/cpt.gguf") + if w.Code != http.StatusOK { + t.Fatalf("POST /models after a total failure = %d; want 200 with the failure rendered", w.Code) + } + body := w.Body.String() + if strings.Contains(body, "still answering") { + t.Errorf("the page claims she is still answering while no model is loaded:\n%s", body) + } + if !strings.Contains(body, "no model is loaded") { + t.Errorf("the page does not name the state the operator is in:\n%s", body) + } +} + +func TestModels_POSTIgnoresLoadSettingsOffTheForm(t *testing.T) { + // n_ctx was read off a form that renders no such input, so only a + // hand-crafted POST could set it. The resident model is a Thinking variant + // whose window is sized for reasoning tokens; shrinking it from the wire is + // not a capability this page offers. + core := &fakeModelCore{swapResp: ipc.SwapModelResp{Model: "qwen3-cpt"}} + r := httptest.NewRequest(http.MethodPost, "/models", strings.NewReader("model_path=/m/cpt.gguf&n_ctx=512&n_gpu_layers=0")) + r.Header.Set("Content-Type", "application/x-www-form-urlencoded") + w := httptest.NewRecorder() + handleModels(w, r, core, nil, nil, false) + if len(core.swapped) != 1 { + t.Fatalf("SwapModel calls = %v; want one", core.swapped) + } + if got := core.swapped[0]; got.NCtx != 0 || got.NGpuLayers != 0 { + t.Errorf("swap request = %+v; want the load settings left to the daemon", got) + } +} + +func TestModels_SwapUsesItsOwnConnection(t *testing.T) { + // A swap is a multi-minute IPC call and ipc.Client serialises everything on + // one mutex, so it must not run on the connection every other page shares. + shared := &fakeModelCore{status: ipc.ModelStatusResp{Model: "qwen3"}} + swapConn := &fakeModelCore{swapResp: ipc.SwapModelResp{Model: "qwen3-cpt"}} + r := httptest.NewRequest(http.MethodPost, "/models", strings.NewReader("model_path=/m/cpt.gguf")) + r.Header.Set("Content-Type", "application/x-www-form-urlencoded") + handleModels(httptest.NewRecorder(), r, shared, swapConn, nil, false) + if len(shared.swapped) != 0 { + t.Errorf("the swap went out on the shared connection: %v", shared.swapped) + } + if len(swapConn.swapped) != 1 { + t.Errorf("the swap did not use the dedicated connection: %v", swapConn.swapped) + } +} diff --git a/cmd/mavweb/morning.html b/cmd/mavweb/morning.html index abf5ae1..3db912e 100644 --- a/cmd/mavweb/morning.html +++ b/cmd/mavweb/morning.html @@ -1,9 +1,27 @@ {{template "shellTop" "morning"}} +

Today

+{{with .Plan}} +
{{.Date.Format "02.01.2006"}}
+{{if not .Items}} +
nothing planned
+{{else}} +
+ +{{range .Items}} + + + +{{end}} +
atkindwhat
{{.At.Format "15:04"}}{{.Kind}}{{if .Uncertain}}похоже, {{end}}{{.Text}}
+{{end}} +{{end}} +{{if .PlanErr}}
plan unavailable: {{.PlanErr}}
{{end}} +

Morning Routines

-{{if not .}} +{{if not .Routines}}
no morning routines configured
{{else}} -{{range .}} +{{range .Routines}}
{{.Name}} {{if .Active}}active now{{else}}outside window{{end}} diff --git a/cmd/mavweb/passkey_prf_test.go b/cmd/mavweb/passkey_prf_test.go new file mode 100644 index 0000000..915551f --- /dev/null +++ b/cmd/mavweb/passkey_prf_test.go @@ -0,0 +1,409 @@ +package main + +import ( + "bytes" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + + "github.com/kami/maven/internal/webauthn" +) + +const prfTestOrigin = "https://maven.test" +const prfTestRPID = "maven.test" + +// fakeKeyIPC stands in for the mavend socket and records exactly what secret +// each call received — the point of the whole test file is that it is the PRF +// output and never the credential public key. +type fakeKeyIPC struct { + unlockSecret []byte + wrapSecret []byte + unlockCalls int + wrapCalls int + unlockErr error + wrapExplicit bool + // opensWith, when set, is the only secret Unlock accepts. It stands in + // for a wrapped blob on disk: everything else gets unlockErr. + opensWith []byte +} + +func (f *fakeKeyIPC) Unlock(_ context.Context, secret []byte) error { + f.unlockCalls++ + f.unlockSecret = bytes.Clone(secret) + if f.opensWith != nil { + if bytes.Equal(secret, f.opensWith) { + return nil + } + return errors.New("unwrap key: decrypt failed (wrong credential?)") + } + return f.unlockErr +} + +func (f *fakeKeyIPC) StoreEncryptionKey(_ context.Context, secret []byte, explicit bool) error { + f.wrapCalls++ + f.wrapSecret = bytes.Clone(secret) + f.wrapExplicit = explicit + return nil +} + +func b64u(b []byte) string { return base64.RawURLEncoding.EncodeToString(b) } + +// prfAuthenticator is a minimal software authenticator: a P-256 key plus the +// COSE encoding of its public half. +type prfAuthenticator struct { + key *ecdsa.PrivateKey + credID []byte + cose []byte +} + +func newPRFAuthenticator(t *testing.T) *prfAuthenticator { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generate key: %v", err) + } + x := key.PublicKey.X.FillBytes(make([]byte, 32)) + y := key.PublicKey.Y.FillBytes(make([]byte, 32)) + // COSE_Key: {1: 2 (EC2), 3: -7 (ES256), -1: 1 (P-256), -2: x, -3: y} + var c []byte + c = append(c, 0xa5) // map(5) + c = append(c, 0x01, 0x02) // 1: 2 + c = append(c, 0x03, 0x26) // 3: -7 + c = append(c, 0x20, 0x01) // -1: 1 + c = append(c, 0x21, 0x58, 0x20) // -2: bytes(32) + c = append(c, x...) + c = append(c, 0x22, 0x58, 0x20) // -3: bytes(32) + c = append(c, y...) + return &prfAuthenticator{key: key, credID: []byte("prf-cred"), cose: c} +} + +func (a *prfAuthenticator) authData(flags byte, counter uint32, attested bool) []byte { + h := sha256.Sum256([]byte(prfTestRPID)) + d := append([]byte{}, h[:]...) + d = append(d, flags) + cb := make([]byte, 4) + binary.BigEndian.PutUint32(cb, counter) + d = append(d, cb...) + if attested { + d = append(d, make([]byte, 16)...) // aaguid + l := make([]byte, 2) + binary.BigEndian.PutUint16(l, uint16(len(a.credID))) + d = append(d, l...) + d = append(d, a.credID...) + d = append(d, a.cose...) + } + return d +} + +func clientDataJSON(typ, challenge string) []byte { + b, _ := json.Marshal(map[string]string{"type": typ, "challenge": challenge, "origin": prfTestOrigin}) + return b +} + +// register drives POST /register/finish with a valid attestation. +func (a *prfAuthenticator) register(t *testing.T, h *PasskeyHandle) { + t.Helper() + _, chal, err := h.rp.CreationOptions([]byte("u"), "user") + if err != nil { + t.Fatalf("CreationOptions: %v", err) + } + // {"fmt":"none","attStmt":{},"authData":} + att := []byte{0xa3} + att = append(att, 0x63, 'f', 'm', 't', 0x64, 'n', 'o', 'n', 'e') + att = append(att, 0x67, 'a', 't', 't', 'S', 't', 'm', 't', 0xa0) + ad := a.authData(1<<6|0x05, 0, true) + att = append(att, 0x68, 'a', 'u', 't', 'h', 'D', 'a', 't', 'a') + att = append(att, 0x59, byte(len(ad)>>8), byte(len(ad))) + att = append(att, ad...) + + body, _ := json.Marshal(map[string]any{ + "challenge": chal, + "credential": map[string]any{ + "id": b64u(a.credID), + "type": "public-key", + "response": map[string]any{ + "clientDataJSON": b64u(clientDataJSON("webauthn.create", chal)), + "attestationObject": b64u(att), + }, + }, + }) + w := httptest.NewRecorder() + h.RegisterFinish(w, httptest.NewRequest(http.MethodPost, "/auth/webauthn/register/finish", bytes.NewReader(body))) + if w.Code != http.StatusOK { + t.Fatalf("RegisterFinish: %d %s", w.Code, w.Body.String()) + } +} + +// assert drives POST /assert/finish with a valid assertion and the given +// base64url PRF result. +func (a *prfAuthenticator) assert(t *testing.T, h *PasskeyHandle, prf string) *httptest.ResponseRecorder { + t.Helper() + return a.assertExplicit(t, h, prf, false) +} + +// assertExplicit is assert with control over the explicit flag the rewrite +// button sets. +func (a *prfAuthenticator) assertExplicit(t *testing.T, h *PasskeyHandle, prf string, explicit bool) *httptest.ResponseRecorder { + t.Helper() + _, chal, err := h.rp.AssertionOptions() + if err != nil { + t.Fatalf("AssertionOptions: %v", err) + } + ad := a.authData(0x05, 7, false) + cdj := clientDataJSON("webauthn.get", chal) + hash := sha256.Sum256(cdj) + sig, err := ecdsa.SignASN1(rand.Reader, a.key, append(append([]byte{}, ad...), hash[:]...)) + if err != nil { + t.Fatalf("sign: %v", err) + } + body, _ := json.Marshal(map[string]any{ + "challenge": chal, + "prf": prf, + "explicit": explicit, + "credential": map[string]any{ + "id": b64u(a.credID), + "type": "public-key", + "response": map[string]any{ + "clientDataJSON": b64u(cdj), + "authenticatorData": b64u(ad), + "signature": b64u(sig), + }, + }, + }) + w := httptest.NewRecorder() + h.AssertFinish(w, httptest.NewRequest(http.MethodPost, "/auth/webauthn/assert/finish", bytes.NewReader(body))) + return w +} + +func newPRFHandle(t *testing.T, key *fakeKeyIPC) *PasskeyHandle { + t.Helper() + store, err := newCredentialStore(filepath.Join(t.TempDir(), "passkeys.json")) + if err != nil { + t.Fatalf("credential store: %v", err) + } + return &PasskeyHandle{ + rp: webauthn.NewRP(webauthn.Config{Origin: prfTestOrigin, RPID: prfTestRPID, RPName: "maven"}), + encryptFn: key, + store: store, + session: webauthn.NewPasskeySession(0), + } +} + +// The fix for Vikunja #14: what goes over IPC is the PRF secret from the +// authenticator, not the credential public key sitting in passkeys.json. +func TestAssertSendsPRFSecretNotPublicKey(t *testing.T) { + key := &fakeKeyIPC{} + h := newPRFHandle(t, key) + auth := newPRFAuthenticator(t) + auth.register(t, h) + + // Enrolment must not wrap anything: create() yields no PRF result. + if key.wrapCalls != 0 || key.unlockCalls != 0 { + t.Fatalf("registration touched the key IPC (wrap=%d unlock=%d)", key.wrapCalls, key.unlockCalls) + } + + secret := make([]byte, 32) + for i := range secret { + secret[i] = byte(i + 1) + } + if w := auth.assert(t, h, b64u(secret)); w.Code != http.StatusOK { + t.Fatalf("AssertFinish: %d %s", w.Code, w.Body.String()) + } + + if key.unlockCalls != 1 || key.wrapCalls != 1 { + t.Fatalf("unlock=%d wrap=%d, want 1 and 1", key.unlockCalls, key.wrapCalls) + } + if !bytes.Equal(key.unlockSecret, secret) { + t.Errorf("Unlock got %x, want the PRF secret %x", key.unlockSecret, secret) + } + if !bytes.Equal(key.wrapSecret, secret) { + t.Errorf("StoreEncryptionKey got %x, want the PRF secret %x", key.wrapSecret, secret) + } + // And explicitly: not the credential public key. + pub, _, err := h.store.Lookup(b64u(auth.credID)) + if err != nil { + t.Fatalf("lookup: %v", err) + } + if bytes.Equal(key.unlockSecret, pub) { + t.Fatal("the credential public key was sent as the unlock secret") + } +} + +// An authenticator without PRF must produce no unlock attempt at all — the +// assertion still succeeds (step-up works), but cold-start unlock stays off +// rather than falling back to something weaker. +func TestAssertWithoutPRFDoesNotUnlock(t *testing.T) { + for _, prf := range []string{"", "!!!not-base64!!!", b64u(make([]byte, 32)), b64u(make([]byte, 16))} { + key := &fakeKeyIPC{} + h := newPRFHandle(t, key) + auth := newPRFAuthenticator(t) + auth.register(t, h) + + w := auth.assert(t, h, prf) + if w.Code != http.StatusOK { + t.Fatalf("prf=%q: AssertFinish %d %s", prf, w.Code, w.Body.String()) + } + if key.unlockCalls != 0 || key.wrapCalls != 0 { + t.Errorf("prf=%q: unlock=%d wrap=%d, want no key IPC at all", prf, key.unlockCalls, key.wrapCalls) + } + } +} + +// A failed unlock must not fail the assertion: step-up is independently valid, +// and a locked daemon degrades rather than breaking the login. +func TestAssertSucceedsWhenUnlockFails(t *testing.T) { + key := &fakeKeyIPC{unlockErr: errors.New("wrong credential")} + h := newPRFHandle(t, key) + auth := newPRFAuthenticator(t) + auth.register(t, h) + + secret := bytes.Repeat([]byte{3}, 32) + if w := auth.assert(t, h, b64u(secret)); w.Code != http.StatusOK { + t.Fatalf("AssertFinish: %d %s", w.Code, w.Body.String()) + } + if key.unlockCalls == 0 { + t.Error("unlock was never attempted") + } +} + +// A forged assertion must never reach the unlock path. +func TestForgedAssertionNeverUnlocks(t *testing.T) { + key := &fakeKeyIPC{} + h := newPRFHandle(t, key) + auth := newPRFAuthenticator(t) + auth.register(t, h) + + // A different key signing over the same credential id. + attacker := newPRFAuthenticator(t) + attacker.credID = auth.credID + w := attacker.assert(t, h, b64u(bytes.Repeat([]byte{4}, 32))) + if w.Code == http.StatusOK { + t.Fatal("an assertion signed by the wrong key was accepted") + } + if key.unlockCalls != 0 || key.wrapCalls != 0 { + t.Fatalf("a forged assertion reached the key IPC (unlock=%d wrap=%d)", key.unlockCalls, key.wrapCalls) + } +} + +// The browser side is the only place the PRF result exists. If the page stops +// asking for it or stops reading it back, cold-start unlock silently dies with +// nothing failing, so the page source is asserted directly. +func TestPasskeyPageRequestsAndPostsPRF(t *testing.T) { + for _, want := range []string{ + "getClientExtensionResults", + "ext.prf.results.first", + "body:JSON.stringify({challenge,prf,", + } { + if !strings.Contains(passkeyPageHTML, want) { + t.Errorf("the passkey page no longer contains %q", want) + } + } +} + +// A box enrolled before Vikunja #14 has a v1 blob wrapped under the credential +// PUBLIC key. The PRF secret cannot open it, and this handler is the only +// caller of Unlock, so without the legacy retry that box stays locked forever +// while a perfectly good passkey is asserted at it. +func TestLegacyV1BlobStillColdStarts(t *testing.T) { + key := &fakeKeyIPC{} + h := newPRFHandle(t, key) + auth := newPRFAuthenticator(t) + auth.register(t, h) + + pub, _, err := h.store.Lookup(b64u(auth.credID)) + if err != nil { + t.Fatalf("lookup: %v", err) + } + // The daemon only opens under the public key — a v1 blob. + key.opensWith = pub + + secret := bytes.Repeat([]byte{9}, 32) + if w := auth.assert(t, h, b64u(secret)); w.Code != http.StatusOK { + t.Fatalf("AssertFinish: %d %s", w.Code, w.Body.String()) + } + if key.unlockCalls != 2 { + t.Fatalf("unlock attempted %d times, want 2 (PRF, then the legacy public key)", key.unlockCalls) + } + if !bytes.Equal(key.unlockSecret, pub) { + t.Fatal("the legacy retry did not send the credential public key, so a v1 box can never cold-start again") + } +} + +// The PRF secret is tried first and, when it works, the public key is never +// sent. The legacy retry is a one-way door out of v1, not a fallback offered +// to every assertion. +func TestPRFUnlockNeverFallsBackWhenItWorks(t *testing.T) { + secret := bytes.Repeat([]byte{7}, 32) + key := &fakeKeyIPC{opensWith: secret} + h := newPRFHandle(t, key) + auth := newPRFAuthenticator(t) + auth.register(t, h) + + if w := auth.assert(t, h, b64u(secret)); w.Code != http.StatusOK { + t.Fatalf("AssertFinish: %d %s", w.Code, w.Body.String()) + } + if key.unlockCalls != 1 { + t.Fatalf("unlock attempted %d times, want 1", key.unlockCalls) + } +} + +// Wrapping the at-rest key is an explicit act, never a side effect of a +// step-up. A page POSTing a substituted prf on a routine assertion must not +// make the daemon re-wrap the database key under it. +func TestPlainAssertionAsksForNoRewrite(t *testing.T) { + key := &fakeKeyIPC{} + h := newPRFHandle(t, key) + auth := newPRFAuthenticator(t) + auth.register(t, h) + + if w := auth.assert(t, h, b64u(bytes.Repeat([]byte{5}, 32))); w.Code != http.StatusOK { + t.Fatalf("AssertFinish: %d %s", w.Code, w.Body.String()) + } + if key.wrapCalls != 1 { + t.Fatalf("wrapCalls = %d, want 1", key.wrapCalls) + } + if key.wrapExplicit { + t.Fatal("a plain step-up asked the daemon to rewrite the cold-start key") + } +} + +// The rewrite button, and only the rewrite button, sets explicit. +func TestRewriteButtonAsksForAnExplicitWrap(t *testing.T) { + key := &fakeKeyIPC{} + h := newPRFHandle(t, key) + auth := newPRFAuthenticator(t) + auth.register(t, h) + + if w := auth.assertExplicit(t, h, b64u(bytes.Repeat([]byte{6}, 32)), true); w.Code != http.StatusOK { + t.Fatalf("AssertFinish: %d %s", w.Code, w.Body.String()) + } + if !key.wrapExplicit { + t.Fatal("the explicit flag did not reach the daemon, so the rewrite button cannot work") + } +} + +// The page is the only place the explicit flag originates. If the button or +// the field goes away, rewriting a cold-start key becomes impossible with +// nothing failing. +func TestPasskeyPageHasTheRewriteButton(t *testing.T) { + for _, want := range []string{ + "rewrite cold-start key", + "explicit:!!explicit", + "async function rewrapKey()", + } { + if !strings.Contains(passkeyPageHTML, want) { + t.Errorf("the passkey page no longer contains %q", want) + } + } +} diff --git a/cmd/mavweb/tasks.html b/cmd/mavweb/tasks.html new file mode 100644 index 0000000..70f6797 --- /dev/null +++ b/cmd/mavweb/tasks.html @@ -0,0 +1,88 @@ +{{template "shellTop" "tasks"}} +

Tasks

+{{if .Msg}}
{{.Msg}}
{{end}} +{{if .Err}}
{{.Err}}
{{end}} + +
+

add

+
+ + + + + + +
+
+ +{{if .Candidates}} +
+

found, not confirmed {{len .Candidates}}

+
maven derived these from something she read. nothing counts as your work until you confirm it.
+
+ +{{range .Candidates}} + + + + + + +{{end}}
taskwhere fromduecaptured
{{.Text}}{{.Source}}{{if .Evidence}} — {{.Evidence}}{{end}}{{.Due}}{{.Created}}
+ + +
+ + +
+
+{{end}} + +
+

open {{len .Open}}

+
most pressing first — by the deadlines and the urgency you gave. nothing about a task is guessed; the only signal that is not yours is age, which lifts anything sitting here for weeks.
+{{if .Open}}
+ +{{range .Open}} + + + + + + + +{{end}}
taskwhyfromduecaptured
{{.Text}}{{.Why}}{{.Source}}{{.Due}}{{.Created}}
+ + +
+ + +
+{{else}}
+ +
no open tasks
+
add one above, or tell maven "добавь в задачи …"
+
{{end}} +
+ +{{if .Resolved}} +
+

resolved {{len .Resolved}}

+
+ +{{range .Resolved}} + + + + +{{end}}
taskstatuswhenby
{{.Text}}{{.Status}}{{.Resolved}}{{.ResolvedBy}}
+{{if .ResolvedMore}}
only the {{len .Resolved}} most recent are shown.
{{end}} +
+{{end}} +{{template "shellBottom"}} diff --git a/cmd/mavweb/tasks_test.go b/cmd/mavweb/tasks_test.go new file mode 100644 index 0000000..5f945ba --- /dev/null +++ b/cmd/mavweb/tasks_test.go @@ -0,0 +1,341 @@ +package main + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" + + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/tasks" +) + +// fakeTaskCore serves the /tasks handler: a canned list plus a log of the +// writes the page made. +type fakeTaskCore struct { + ipc.UnimplementedCoreAPI + + tasks []ipc.Task + listErr error + + captured []ipc.CaptureTaskReq + created bool + captureErr error + + statusID int64 + statusVal string + statusBy string + statusErr error + + promoted bool +} + +func (f *fakeTaskCore) ListTasks(_ context.Context, status string) ([]ipc.Task, error) { + if f.listErr != nil { + return nil, f.listErr + } + return f.tasks, nil +} + +func (f *fakeTaskCore) CaptureTask(_ context.Context, req ipc.CaptureTaskReq) (ipc.CaptureTaskResp, error) { + f.captured = append(f.captured, req) + if f.captureErr != nil { + return ipc.CaptureTaskResp{}, f.captureErr + } + return ipc.CaptureTaskResp{ID: 7, Created: f.created, Promoted: f.promoted}, nil +} + +func (f *fakeTaskCore) SetTaskStatus(_ context.Context, id int64, status string, _ time.Time, by string) error { + f.statusID, f.statusVal, f.statusBy = id, status, by + return f.statusErr +} + +func TestHandleTasksSplitsCandidatesFromOpen(t *testing.T) { + now := time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC) + resolved := now.Add(time.Hour) + core := &fakeTaskCore{tasks: []ipc.Task{ + {ID: 1, Text: "купить молоко", Source: "tap:voice", Status: "open", CreatedTs: now}, + {ID: 2, Text: "продлить страховку", Source: "email:kami", Evidence: "полис истекает", Status: "candidate", CreatedTs: now}, + {ID: 3, Text: "полить цветы", Source: "tap:web", Status: "done", CreatedTs: now, Resolved: &resolved}, + }} + rec := httptest.NewRecorder() + handleTasks(rec, httptest.NewRequest(http.MethodGet, "/tasks", nil), core) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d", rec.Code) + } + body := rec.Body.String() + for _, want := range []string{ + "купить молоко", "продлить страховку", "полить цветы", + "полис истекает", // the evidence trail is visible for review + "found, not confirmed", // candidates get their own section + } { + if !strings.Contains(body, want) { + t.Errorf("body missing %q", want) + } + } + // The candidate must offer confirm, and the open task must not. + if !strings.Contains(body, "value=confirm") { + t.Error("candidate row has no confirm action") + } +} + +func TestHandleTasksAddCaptures(t *testing.T) { + core := &fakeTaskCore{created: true} + form := url.Values{"action": {"add"}, "text": {" позвонить в банк "}, "due": {"2026-08-05"}} + req := httptest.NewRequest(http.MethodPost, "/tasks", strings.NewReader(form.Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + rec := httptest.NewRecorder() + handleTasks(rec, req, core) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d", rec.Code) + } + if len(core.captured) != 1 { + t.Fatalf("captured %d requests, want 1", len(core.captured)) + } + got := core.captured[0] + if got.Text != "позвонить в банк" { + t.Errorf("text = %q, want trimmed", got.Text) + } + if got.Source != "tap:web" { + t.Errorf("source = %q, want tap:web", got.Source) + } + if got.Status != "open" { + t.Errorf("status = %q — a task he typed himself is open, not a candidate", got.Status) + } + if got.Due == nil || got.Due.Format("2006-01-02") != "2026-08-05" { + t.Errorf("due = %v", got.Due) + } + if !strings.Contains(rec.Body.String(), "added task") { + t.Error("no confirmation message") + } +} + +func TestHandleTasksAddSaysAlreadyOnTheList(t *testing.T) { + core := &fakeTaskCore{created: false} + form := url.Values{"action": {"add"}, "text": {"купить молоко"}} + req := httptest.NewRequest(http.MethodPost, "/tasks", strings.NewReader(form.Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + rec := httptest.NewRecorder() + handleTasks(rec, req, core) + if !strings.Contains(rec.Body.String(), "already on the list") { + t.Error("a deduped capture must not claim it saved something new") + } +} + +func TestHandleTasksStatusActions(t *testing.T) { + for _, tc := range []struct{ action, want string }{ + {"confirm", "open"}, + {"done", "done"}, + {"drop", "dropped"}, + } { + core := &fakeTaskCore{} + form := url.Values{"action": {tc.action}, "id": {"42"}} + req := httptest.NewRequest(http.MethodPost, "/tasks", strings.NewReader(form.Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + handleTasks(httptest.NewRecorder(), req, core) + if core.statusID != 42 || core.statusVal != tc.want { + t.Errorf("%s → SetTaskStatus(%d, %q), want (42, %q)", tc.action, core.statusID, core.statusVal, tc.want) + } + } +} + +func TestHandleTasksRejectsBadPost(t *testing.T) { + core := &fakeTaskCore{} + form := url.Values{"action": {"explode"}, "id": {"1"}} + req := httptest.NewRequest(http.MethodPost, "/tasks", strings.NewReader(form.Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + rec := httptest.NewRecorder() + handleTasks(rec, req, core) + // The page still renders, with the error inline — and nothing was written. + if rec.Code != http.StatusOK { + t.Fatalf("status = %d", rec.Code) + } + if core.statusVal != "" || len(core.captured) != 0 { + t.Error("an unknown action must write nothing") + } + if !strings.Contains(rec.Body.String(), "unknown action") { + t.Error("error not surfaced on the page") + } +} + +func TestHandleTasksNoCore(t *testing.T) { + rec := httptest.NewRecorder() + handleTasks(rec, httptest.NewRequest(http.MethodGet, "/tasks", nil), nil) + if rec.Code != http.StatusServiceUnavailable { + t.Errorf("status = %d, want 503", rec.Code) + } +} + +// The open list is ordered by the ranker, and the reason is shown so the page +// says why a task is first instead of asking him to trust the order. +func TestHandleTasksOrdersOpenByRank(t *testing.T) { + now := time.Now() + due := now + core := &fakeTaskCore{tasks: []ipc.Task{ + {ID: 1, Text: "купить молоко", Status: "open", CreatedTs: now}, + {ID: 2, Text: "оплатить интернет", Status: "open", CreatedTs: now, Due: &due}, + }} + rec := httptest.NewRecorder() + handleTasks(rec, httptest.NewRequest(http.MethodGet, "/tasks", nil), core) + body := rec.Body.String() + if strings.Index(body, "оплатить интернет") > strings.Index(body, "купить молоко") { + t.Error("want the dated task rendered first") + } + if !strings.Contains(body, "сегодня") { + t.Error("want the ranker's reason shown in the why column") + } +} + +// A candidate is ranked into place but never carries a priority reason: its due +// date is Maven's reading of a mail, not something he stated. +func TestHandleTasksHidesCandidateReason(t *testing.T) { + now := time.Now() + due := now + core := &fakeTaskCore{tasks: []ipc.Task{ + {ID: 1, Text: "продлить страховку", Status: "candidate", CreatedTs: now, Due: &due}, + }} + rec := httptest.NewRecorder() + handleTasks(rec, httptest.NewRequest(http.MethodGet, "/tasks", nil), core) + if strings.Contains(rec.Body.String(), "сегодня") { + t.Error("a candidate must not be shown with a priority reason") + } +} + +func TestApplyTaskPostCarriesWeight(t *testing.T) { + core := &fakeTaskCore{created: true} + form := url.Values{"action": {"add"}, "text": {"оплатить интернет"}, "weight": {"3"}} + req := httptest.NewRequest(http.MethodPost, "/tasks", strings.NewReader(form.Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + handleTasks(httptest.NewRecorder(), req, core) + if len(core.captured) != 1 || core.captured[0].Weight != 3 { + t.Fatalf("captured = %+v, want weight 3", core.captured) + } +} + +// Out of range clamps rather than 400s; a non-number is a real client error. +func TestApplyTaskPostClampsWeight(t *testing.T) { + core := &fakeTaskCore{created: true} + form := url.Values{"action": {"add"}, "text": {"что-то"}, "weight": {"99"}} + req := httptest.NewRequest(http.MethodPost, "/tasks", strings.NewReader(form.Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + handleTasks(httptest.NewRecorder(), req, core) + if core.captured[0].Weight != tasks.MaxWeight { + t.Errorf("weight = %d, want the cap", core.captured[0].Weight) + } +} + +// The page ranked with the wall clock while the daemon path ranked with a clock +// it was handed, so this was the one surface that could only be tested at +// whatever time it happened to run. +func TestHandleTasksRanksAtTheInjectedClock(t *testing.T) { + fixed := time.Date(2026, 8, 1, 10, 0, 0, 0, time.FixedZone("UTC+4", 4*3600)) + old := now + now = func() time.Time { return fixed } + t.Cleanup(func() { now = old }) + + // Due tomorrow, local time, stored the way the store hands it back: UTC. + due := time.Date(2026, 8, 2, 0, 0, 0, 0, fixed.Location()).UTC() + core := &fakeTaskCore{tasks: []ipc.Task{ + {ID: 1, Text: "оплатить интернет", Status: "open", CreatedTs: fixed, Due: &due}, + }} + rec := httptest.NewRecorder() + handleTasks(rec, httptest.NewRequest(http.MethodGet, "/tasks", nil), core) + body := rec.Body.String() + if !strings.Contains(body, "завтра") { + t.Errorf("why column does not say завтра: %q", why(body)) + } + if strings.Contains(body, "сегодня") || strings.Contains(body, "просрочено") { + t.Error("a task due tomorrow was ranked as today's or overdue") + } +} + +// why is a crude excerpt of the rendered why column, for a readable failure. +func why(body string) string { + i := strings.Index(body, "") + if i < 0 { + return body + } + j := i + 200 + if j > len(body) { + j = len(body) + } + return body[i:j] +} + +// The resolved section rendered every row that ever existed. +func TestHandleTasksBoundsResolved(t *testing.T) { + base := time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC) + var rows []ipc.Task + for i := 0; i < resolvedShown+10; i++ { + ts := base.Add(time.Duration(i) * time.Minute) + rows = append(rows, ipc.Task{ + ID: int64(i + 1), Text: fmt.Sprintf("задача %d", i), Status: "done", + CreatedTs: ts, Resolved: &ts, + }) + } + core := &fakeTaskCore{tasks: rows} + rec := httptest.NewRecorder() + handleTasks(rec, httptest.NewRequest(http.MethodGet, "/tasks", nil), core) + body := rec.Body.String() + if n := strings.Count(body, "задача "); n != resolvedShown { + t.Errorf("rendered %d resolved rows, want the %d-row bound", n, resolvedShown) + } + if !strings.Contains(body, "most recent are shown") { + t.Error("the page must say it is showing only part of the history") + } +} + +// A capture over a candidate is a confirmation, not a duplicate. +func TestHandleTasksAddSaysPromoted(t *testing.T) { + core := &fakeTaskCore{promoted: true} + form := url.Values{"action": {"add"}, "text": {"продлить страховку"}} + req := httptest.NewRequest(http.MethodPost, "/tasks", strings.NewReader(form.Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + rec := httptest.NewRecorder() + handleTasks(rec, req, core) + if !strings.Contains(rec.Body.String(), "confirmed a candidate") { + t.Error("a promoted capture must not read as a duplicate") + } +} + +// Sscanf accepted "3junk" as 3, and the same call parsed the row id. +func TestApplyTaskPostRejectsTrailingGarbage(t *testing.T) { + core := &fakeTaskCore{created: true} + form := url.Values{"action": {"add"}, "text": {"что-то"}, "weight": {"3junk"}} + req := httptest.NewRequest(http.MethodPost, "/tasks", strings.NewReader(form.Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + rec := httptest.NewRecorder() + handleTasks(rec, req, core) + if len(core.captured) != 0 { + t.Errorf("captured %+v, want nothing on a malformed weight", core.captured) + } + if !strings.Contains(rec.Body.String(), "bad weight") { + t.Error("error not surfaced on the page") + } + + core = &fakeTaskCore{} + form = url.Values{"action": {"done"}, "id": {"42junk"}} + req = httptest.NewRequest(http.MethodPost, "/tasks", strings.NewReader(form.Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + handleTasks(httptest.NewRecorder(), req, core) + if core.statusID != 0 { + t.Errorf("SetTaskStatus called with id %d on a malformed id", core.statusID) + } +} + +// A resolution says what resolved it: resolved_ts recorded when and never by +// what. +func TestHandleTasksRecordsTheCaller(t *testing.T) { + core := &fakeTaskCore{} + form := url.Values{"action": {"done"}, "id": {"42"}} + req := httptest.NewRequest(http.MethodPost, "/tasks", strings.NewReader(form.Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + handleTasks(httptest.NewRecorder(), req, core) + if core.statusBy != "tap:web" { + t.Errorf("resolved by %q, want tap:web", core.statusBy) + } +} diff --git a/cmd/mavweb/webauthn.go b/cmd/mavweb/webauthn.go index b3493fc..6717e8c 100644 --- a/cmd/mavweb/webauthn.go +++ b/cmd/mavweb/webauthn.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "errors" "fmt" "log" "net/http" @@ -23,8 +24,8 @@ type assertIPC interface { // is *ipc.Client; in-process CoreAPI adapters do not implement it. When nil, // StoreEncryptionKey and Unlock are silently skipped. type keyIPC interface { - StoreEncryptionKey(ctx context.Context, publicKey []byte) error - Unlock(ctx context.Context, publicKey []byte) error + StoreEncryptionKey(ctx context.Context, secret []byte, explicit bool) error + Unlock(ctx context.Context, secret []byte) error } // PasskeyHandle holds the WebAuthn relying party, a local in-memory credential @@ -86,8 +87,10 @@ const passkeyPageHTML = `{{template "shellTop" "passkey"}}
+
+

Rewriting the cold-start key points it at the passkey you assert next. Every other enrolled passkey stops being able to unlock a cold-booted daemon.

{{template "shellBottom"}} ` func (h *PasskeyHandle) RegisterBegin(w http.ResponseWriter, r *http.Request) { @@ -140,9 +164,7 @@ func (h *PasskeyHandle) RegisterFinish(w http.ResponseWriter, r *http.Request) { http.Error(w, "bad request: "+err.Error(), http.StatusBadRequest) return } - var enrolledPublicKey []byte save := func(id string, publicKey []byte, _ []byte, _ string) error { - enrolledPublicKey = publicKey return h.store.Save(id, publicKey) } credID, err := h.rp.FinishRegistration(save, body.Challenge, body.Credential) @@ -153,19 +175,15 @@ func (h *PasskeyHandle) RegisterFinish(w http.ResponseWriter, r *http.Request) { } log.Printf("webauthn: registered credential %s", credID) - // If mavend is reachable and supports key wrapping, store the encryption - // key wrapped with this credential's public key — enables cold-start unlock. - if h.encryptFn != nil && enrolledPublicKey != nil { - ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) - defer cancel() - if err := h.encryptFn.StoreEncryptionKey(ctx, enrolledPublicKey); err != nil { - log.Printf("webauthn: store encryption key: %v", err) - // Non-fatal: enrollment still succeeded, the wrapped key can be - // created later via the same endpoint. - } else { - log.Printf("webauthn: encryption key wrapped with credential %s", credID) - } - } + // Note what does NOT happen here: the encryption key is not wrapped at + // enrolment. Wrapping needs the authenticator's PRF output, and create() + // does not produce one on most authenticators — it only reports whether + // the extension is supported. The wrapped key is written on the first + // assertion instead (see AssertFinish). + // + // This used to wrap the key under the credential *public* key, which is + // written to passkeys.json next to the wrapped blob. See the header of + // internal/webauthn/keywrap.go. json.NewEncoder(w).Encode(map[string]string{"credential_id": credID}) } @@ -189,6 +207,24 @@ func (h *PasskeyHandle) AssertFinish(w http.ResponseWriter, r *http.Request) { var body struct { Challenge string `json:"challenge"` Credential map[string]any `json:"credential"` + // PRF is the base64url WebAuthn PRF output the browser read out of + // getClientExtensionResults(). Empty when the authenticator has no + // PRF extension: cold-start unlock is then unavailable and we say so + // rather than falling back to something weaker. + // + // Known property, accepted deliberately: this value is supplied by + // the client and is NOT covered by the assertion signature. WebAuthn + // client extension outputs never are, and binding one would need a + // per-assertion salt, which would make the wrapped blob unopenable on + // the next boot. Nothing here can tell a real PRF output from 32 + // bytes a compromised page chose. What limits the damage is that the + // daemon refuses to rewrite an existing blob unless the operator + // asked for it — see Explicit below and cmd/mavend/keyfile.go. + PRF string `json:"prf"` + // Explicit marks the "rewrite cold-start key" button rather than a + // plain step-up. Only then may the daemon replace a blob that is + // already on disk. + Explicit bool `json:"explicit"` } if err := json.NewDecoder(r.Body).Decode(&body); err != nil { http.Error(w, "bad request: "+err.Error(), http.StatusBadRequest) @@ -222,26 +258,22 @@ func (h *PasskeyHandle) AssertFinish(w http.ResponseWriter, r *http.Request) { } } - // If the daemon is locked (cold-start), send the credential's public key - // over IPC so mavend can unwrap its encryption key and open the store. - // The public key comes from the local credential store (it was stored - // during enrollment). Non-fatal: if IPC doesn't support Unlock or the - // daemon is already unlocked, the call is a no-op on the server side. + // Cold-start unlock and key wrapping, both keyed on the PRF secret this + // assertion just produced. The secret is used here and dropped; it is + // never stored on this side, and it must never be logged — unlike a + // signature it does not expire, so one copy in a proxy log or a HAR file + // is permanent access to the wrapped blob. + // + // Order matters: unlock first (if the daemon is locked there is nothing to + // wrap yet), then wrap. Both are best-effort, because the assertion itself + // is valid either way. if h.encryptFn != nil { - publicKey, _, err := h.store.Lookup(credID) - if err == nil && publicKey != nil { + if secret, err := webauthn.DecodePRFResult(body.PRF); err != nil { + log.Printf("webauthn: no usable PRF secret from credential %s: %v", credID, err) + } else { ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) defer cancel() - if err := h.encryptFn.Unlock(ctx, publicKey); err != nil { - log.Printf("webauthn: unlock via credential %s: %v", credID, err) - // Non-fatal: assertion succeeded; if the daemon stays locked - // the user will see errors on subsequent pages, but the - // assertion itself is valid. - } else { - log.Printf("webauthn: daemon unlocked via credential %s", credID) - } - } else if err != nil { - log.Printf("webauthn: lookup credential %s for unlock: %v", credID, err) + h.coldStart(ctx, credID, secret, body.Explicit) } } @@ -253,3 +285,56 @@ func (h *PasskeyHandle) AssertFinish(w http.ResponseWriter, r *http.Request) { log.Printf("webauthn: asserted credential %s", credID) json.NewEncoder(w).Encode(map[string]string{"credential_id": credID}) } + +// coldStart unlocks a locked daemon with this assertion's PRF output and then +// asks it to wrap the at-rest key. Never fatal: a locked or unreachable daemon +// does not invalidate the step-up. +// +// # The legacy retry +// +// A box enrolled before Vikunja #14 has a v1 blob, wrapped under the +// credential PUBLIC key. The PRF secret cannot open it, and this handler is +// the only caller of Unlock, so without a second attempt that box could never +// cold-start again: it would sit locked while a perfectly good passkey was +// asserted, and the only way back in would be putting MAVEN_DB_KEY into the +// environment — the exact thing cold-start unlock exists to avoid. +// +// So a failed PRF unlock is retried with the public key from the credential +// store. That is not a weaker fallback being offered to new deployments: +// nothing writes v1 any more, and a v2 blob does not open under a public key +// either. It is a one-way door out of the old format, and the operator is told +// to walk through it. +func (h *PasskeyHandle) coldStart(ctx context.Context, credID string, secret []byte, explicit bool) { + legacy := false + err := h.encryptFn.Unlock(ctx, secret) + if err != nil && !errors.Is(err, ipc.ErrUnknownMethod) { + if pub, _, lerr := h.store.Lookup(credID); lerr == nil && len(pub) > 0 { + if err2 := h.encryptFn.Unlock(ctx, pub); err2 == nil { + err, legacy = nil, true + } + } + } + switch { + case errors.Is(err, ipc.ErrUnknownMethod): + // Env-key mode: the daemon was never locked and has no UnlockFn. Not + // a failure, and the old code logged it as one on every assertion. + case err != nil: + log.Printf("webauthn: unlock via credential %s failed: %v", credID, err) + case legacy: + log.Printf("SECURITY: webauthn: daemon unlocked from a LEGACY v1 wrapped key using credential %s. That blob is derived from the credential public key, which sits in passkeys.json beside it, so it protects nothing. Press \"rewrite cold-start key\" on this page to replace it with a v2 blob.", credID) + default: + log.Printf("webauthn: daemon reports unlocked, credential %s", credID) + } + + // explicit=false means "write the blob only if there is none". The daemon + // enforces that; sending the flag is the whole of this side's part in it. + switch err := h.encryptFn.StoreEncryptionKey(ctx, secret, explicit); { + case err == nil && explicit: + log.Printf("webauthn: cold-start key rewritten under credential %s", credID) + case err == nil: + case errors.Is(err, ipc.ErrUnknownMethod): + // No key to wrap: a plaintext dev store, or a daemon still locked. + default: + log.Printf("webauthn: wrap encryption key: %v", err) + } +} diff --git a/deploy/README.md b/deploy/README.md index afdeba2..fec729b 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -36,6 +36,83 @@ present (see `.dockerignore`). | `/var/lib/maven` (volume) | encrypted db at rest | | `/dev/shm` (tmpfs) | decrypted db working copy (RAM only) | +## Reading the outside world (off by default) + +`mavend.json` ships without a `feeds` block, which means no RSS/Atom feed is +fetched and no outbound request is made. Switching it on is adding the block: + +```json +"feeds": { + "poll_interval": "30m", + "max_items": 5, + "max_age": "24h", + "sources": [ + { "name": "habr", "url": "https://habr.com/ru/rss/best/daily/", + "category": "технологии", "exclude": ["реклама"] } + ] +} +``` + +What it does and does not do: + +- items are written as notes with source `rss:`, visible on `/dash`; +- **nothing is announced.** She reads them back when asked — "что нового в + лентах?", "что нового по технологиям?" — and never on arrival. There is no + severity or channel knob here on purpose; +- the fetcher is allowlisted to the hosts of the configured feeds, plus any + `allow_hosts`. It refuses non-http(s) schemes and every private address + (loopback, the LAN, the `10.42.0.0/24` wg range, cloud metadata). It caps the + response at 2 MiB and redirects at 3, and makes at most one request per host + per second. See `internal/webfetch`; +- how far each feed was read is stored as a config fact `rss:latest:`, so + a restart does not re-note yesterday's headlines. A feed whose items carry no + dates gets the same mark, and the first poll after a restart takes those items + as already read rather than writing them all again; +- `max_items` paces, it does not drop: a burst larger than the cap arrives over + the following polls, oldest first; +- feed notes are **not** part of recall. "что я говорил про X" searches what he + said; headlines are read back only by asking about the feeds. + +### Reading a page (`crawl`, also off by default) + +There is no `crawl` block either, so no page is fetched. Two halves, separately +switched: + +```json +"crawl": { + "on_demand": true, + "interval": "6h", + "max_runes": 4000, + "watches": [ + { "name": "changelog", "url": "https://example.org/changelog", "interval": "12h" } + ] +} +``` + +- `on_demand` lets her read a page he names in the utterance: "посмотри + https://example.org/x — что там?". The page becomes context for his question, + and only the URL leaves the box. Without a URL nothing is fetched, so this is + a fallback and not a habit; +- `watches` re-reads a fixed list on its interval and writes a note when the + text changed. Like the feeds, it announces nothing; +- the answer path sits behind his memory and his notes, and ahead of the model + answering from what it remembers. Kiwix is not wired into the chain yet. A + local read costs nothing, so anything local goes first; +- `robots.txt` is fetched first and obeyed with no override; a `Disallow` is a + refusal she says out loud. `Crawl-delay` is waited out before the page is + fetched, and a delay longer than the turn fails the read instead of hanging + it. A `robots.txt` that answers 5xx refuses the crawl — a broken server is + not permission; +- `allow_hosts` limits on-demand reading to those hosts and nothing else. + Watched pages' hosts are reachable by the scheduled crawler whether listed or + not, but a watch does **not** widen what he may ask her to read; +- same guarded fetcher as the feeds: allowlist/denylist, no private addresses, + size cap, redirect cap, timeout, one request per host per second; +- dedup state is the config fact `crawl:hash:`; +- like feed notes, watch notes are kept out of recall (`store.ReadSourcePrefixes`). + Text from someone else's page is not something he said, so it must not come + back as an answer to a question about him. Watch notes are visible on `/dash`. + ## Not yet verified / host-dependent This stack is correct-by-construction but has **not been build-tested here** @@ -53,3 +130,92 @@ build on the target host, most likely in one of these: work fine over the core socket. - **netdata** — `mavpoll` reaches it via `host.docker.internal`; adjust if netdata runs elsewhere. + +## Updating her (`mavupdate`, Vikunja #249) + +Off unless configured, and there is deliberately no button for it. There is no +IPC method, no web route, no timer and no act that starts an update — the trigger +is a human running `mavupdate` on the host, which needs shell access, a strictly +higher bar than the step-up passkey gate that guards `/tools`. She cannot update +herself; she can be updated. Nothing here ever fetches code: the new version is +whatever you pulled into the working tree yourself. + +Add an `update` block to `mavend.json` (mavend ignores it — only the CLI reads +it), with paths as they exist **on the host**, not inside a container: + +```json +"update": { + "source_dir": "/home/kami/apps/Maven", + "install_dir": "/home/kami/apps/Maven", + "snapshot_dir": "/var/lib/maven-snapshots", + "source_rollback": "git", + "binaries": ["mavend", "mavweb", "mavsttd", "mavttsd", "mavwaked", + "mavenclient", "mavpoll", "mavcaldav", "mavmaild", "mavupdate"], + "config_files": ["deploy/mavend.json"], + "restart_cmd": ["docker", "compose", "up", "-d", "--build", "mavend"], + "health_socket": "/run/maven-host/mavend.sock", + "health_timeout_sec": 120 +} +``` + +`snapshot_dir` must be outside both `install_dir` and `source_dir` (a restore +must not read from what the install writes, and a snapshot dir inside the tree +lands in the docker build context). `health_socket` is required: an update that +cannot check its own result cannot roll itself back, so the config is refused +without one. + +**`source_rollback` is what makes a rollback real on this deployment.** Compose +builds the image from the tree — the Dockerfile copies `cmd/` and `internal/` +and runs the build in the builder stage, and `.dockerignore` keeps the host +binaries out — so `install_dir` is the tree, `install` is a no-op, and putting +the old binaries back puts back bytes nothing reads. A rollback that only did +that would rebuild the same bad image and burn a second health timeout proving +it. With `"source_rollback": "git"` the commit is recorded before the update and +checked back out before the restart, so the restore is of the thing that +actually gets deployed. It requires a clean tree: `apply` refuses to start with +uncommitted changes, because the recorded commit would not describe what is +deployed and the forced checkout on the way back would delete the work. It also +means a rollback moves every tracked file, `deploy/mavend.json` included, so on +this deployment a config edit belongs in a commit. + +Leaving `source_rollback` out is only valid when `install_dir` holds what +actually runs. `Validate` refuses the combination of "same dir" and "no way to +put the source back" at startup rather than at the one rollback that mattered. + +**The socket has to be one the account running `mavupdate` can open.** The +compose stack keeps IPC in a named volume, whose host path +(`/var/lib/docker/volumes/maven_sockets/_data`) is under a `drwx--x--- root +root` directory, and the socket itself is 0600 owned by the container's uid +10001. A non-root `mavupdate` gets EACCES on the dial, which reports as +`update: cannot open the health socket` rather than as a daemon that will not +answer. Bind-mount the socket dir to a host path he owns and run the daemon +under his uid instead: + +```yaml +mavend: + user: "1000:1000" + volumes: + - /run/maven-host:/run/maven +``` + +Do **not** work around it with `sudo mavupdate apply`. `verify` runs `make +build` and `make test` in `source_dir`, and as root that leaves root-owned +binaries, object files and a build cache in the working tree, so the next +ordinary `make` fails. `Verify` refuses to run as root over a tree owned by +someone else for exactly that reason. + +Then: + +```sh +mavupdate -config deploy/mavend.json verify # make build + make test, deploys nothing +mavupdate -config deploy/mavend.json apply -yes # snapshot, verify, install, restart, health-check +mavupdate -config deploy/mavend.json list # what you can roll back to +mavupdate -config deploy/mavend.json rollback -yes # restore the previous artifacts and restart +``` + +`apply` refuses to start if she is not already answering — otherwise a failed +update and a box that was already broken are indistinguishable afterwards. On any +failure after the install it restores the snapshot, restarts, and checks again; +if that also fails it says so loudly and names the directory to copy back by hand. +The database is never snapshotted or rolled back (see the package comment in +`internal/update`); schema compatibility is `store.Migrate`'s job. diff --git a/deploy/ecosystem/docker-compose.yml b/deploy/ecosystem/docker-compose.yml index a49082e..0cca3ec 100644 --- a/deploy/ecosystem/docker-compose.yml +++ b/deploy/ecosystem/docker-compose.yml @@ -8,6 +8,18 @@ # # Maven's own compose joins this same network (add `ecosystem` as an external # network there) to reach nexus:9740 / praxis:8989 / hexis:9741 directly. +# +# NO RELEASE PINNING (Vikunja #354): each `build:` below points at a sibling +# WORKING TREE, so `up --build` ships whatever is checked out there, including +# uncommitted edits. Before bringing this up, check what you are about to +# deploy: +# +# for r in nexus praxis hexis; do git -C ../../../$r status --short; \ +# git -C ../../../$r log -1 --oneline; done +# +# The host nginx that fronts these is deploy/ecosystem/nginx.conf — it binds +# the wg and LAN addresses only, with allow/deny. Keep it that way: none of +# these containers has auth of its own. name: ecosystem services: diff --git a/deploy/ecosystem/nginx-upgrade-map.conf b/deploy/ecosystem/nginx-upgrade-map.conf new file mode 100644 index 0000000..1c237bb --- /dev/null +++ b/deploy/ecosystem/nginx-upgrade-map.conf @@ -0,0 +1,22 @@ +# $connection_upgrade — WebSocket upgrade helper for the maven. block +# in nginx.conf. Install this ONLY if your nginx does not already define +# $connection_upgrade somewhere in the http context. +# +# It is a separate file because nginx treats a duplicate `map` directive as a +# fatal configuration error, not a warning: if this block were inside +# nginx.conf and your setup already had one (nginx-panel and most WebSocket +# recipes ship one), the next `nginx -s reload` would fail the config test and +# nginx would refuse to come back up — taking every other site on the box down +# with it, not just maven. +# +# Check before installing: +# grep -rn 'connection_upgrade' /etc/nginx/ +# Nothing? Drop this in /etc/nginx/conf.d/ and reload. Something already there? +# Skip this file entirely; nginx.conf works as-is. +# +# Verify either way before reloading: +# nginx -t +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} diff --git a/deploy/ecosystem/nginx.conf b/deploy/ecosystem/nginx.conf index 43b863e..41a7dbc 100644 --- a/deploy/ecosystem/nginx.conf +++ b/deploy/ecosystem/nginx.conf @@ -1,13 +1,60 @@ -# Reverse-proxy the three sibling admin UIs. Drop into your nginx sites (or the -# nginx-panel app) and reload. Assumes the compose publishes each service on -# 127.0.0.1:. Add TLS (certbot / your existing cert block) per server. +# Reverse-proxy Maven's own web UI plus the three sibling admin UIs. Drop into +# your nginx sites (or the nginx-panel app) and reload. Assumes the compose +# publishes each service on 127.0.0.1:. Add TLS (certbot / your existing +# cert block) per server. # # NOTE: hexis. previously pointed at the MCP tool — repoint that # elsewhere first (the app now owns hexis.*). +# +# 10.42.0.1 and 192.168.1.104 below are THIS BOX's WireGuard and LAN +# addresses (homesrv) — these admin UIs have no auth of their own, so the +# explicit bind + allow/deny below is what keeps them off the open internet. +# On a different box, replace both addresses with that box's wg and LAN IPs. +# Do NOT "fix" a failed bind by reverting to `listen 80` (all interfaces) — +# that removes the only access control these containers have. +# +# BEFORE YOU RELOAD — two ways this file takes nginx down, both host-side. +# These blocks are for the HOST nginx, not for anything inside the compose; +# the containers publish on 127.0.0.1 and have no nginx of their own. +# +# 1. Binding an address that does not exist yet. `listen 10.42.0.1:80` fails +# with EADDRNOTAVAIL if wg0 is down, and nginx exits rather than starting +# without it — so a reboot that brings nginx up before WireGuard leaves the +# box with no web at all. Allow the bind to succeed regardless: +# sysctl -w net.ipv4.ip_nonlocal_bind=1 +# echo 'net.ipv4.ip_nonlocal_bind = 1' > /etc/sysctl.d/99-nginx-bind.conf +# Ordering nginx after the wg interface works too, but only until the next +# time the tunnel restarts. +# +# 2. A duplicate $connection_upgrade map. That is a fatal config error, so the +# map now lives in nginx-upgrade-map.conf and is installed separately — +# read the note at the top of that file first. +# +# `nginx -t` catches the second and not the first. Run it anyway, every time. +# +# BLOCK ORDER IS LOAD-BEARING. nginx serves the first block for a given listen +# address when no server_name matches, so whichever block comes first here +# answers requests with an unknown or absent Host header. That must not be +# mavweb: it is the one surface in this file that can define and run argv. The +# nexus block is marked default_server so the choice is explicit rather than a +# consequence of file order, and the maven block sits last as a second guard. +# If you add a block, do not put it above nexus. If another site file already +# claims default_server on 10.42.0.1:80 or 192.168.1.104:80, nginx refuses to +# start with "a duplicate default server" — drop the two keywords here and rely +# on the block order instead. +# +# Every server_name below is a literal for kvmx.ru even though the comments +# write maven.. This file reads like a template and is not one. server { - listen 80; + listen 10.42.0.1:80 default_server; + listen 192.168.1.104:80 default_server; server_name nexus.kvmx.ru; + + allow 10.42.0.0/24; + allow 192.168.1.0/24; + deny all; + location / { proxy_pass http://127.0.0.1:9740; proxy_set_header Host $host; @@ -18,8 +65,14 @@ server { } server { - listen 80; + listen 10.42.0.1:80; + listen 192.168.1.104:80; server_name praxis.kvmx.ru; + + allow 10.42.0.0/24; + allow 192.168.1.0/24; + deny all; + location / { proxy_pass http://127.0.0.1:8989; proxy_set_header Host $host; @@ -30,8 +83,14 @@ server { } server { - listen 80; + listen 10.42.0.1:80; + listen 192.168.1.104:80; server_name hexis.kvmx.ru; + + allow 10.42.0.0/24; + allow 192.168.1.0/24; + deny all; + location / { proxy_pass http://127.0.0.1:9741; proxy_set_header Host $host; @@ -40,3 +99,47 @@ server { proxy_set_header X-Forwarded-Proto $scheme; } } + +# maven. → mavweb (docker-compose.yml publishes it on 127.0.0.1:9201). +# Same bind + ACL as the siblings, and for a stronger reason: mavweb serves +# POST /tools, which defines argv that internal/tool EXECUTES, plus POST +# /routines, /api/revert, /api/chat, /api/ptt and /ws (Vikunja #317). Without +# -webauthn-origin/-webauthn-rpid mavweb has no auth of its own, so this block +# is the auth. If you add TLS and a basic-auth/oauth2-proxy layer, keep the +# allow/deny anyway — belt and braces on an RCE surface. +# +# WebSocket upgrade matters here: /ws carries push-to-talk audio, so the +# Upgrade/Connection headers below are required, not decoration. They reference +# $connection_upgrade, which this file does NOT define — see +# nginx-upgrade-map.conf and point 2 above. + +server { + listen 10.42.0.1:80; + listen 192.168.1.104:80; + server_name maven.kvmx.ru; + + allow 10.42.0.0/24; + allow 192.168.1.0/24; + deny all; + + # push-to-talk uploads raw PCM; the default 1m is enough for a short + # utterance but not for a long one. + client_max_body_size 32m; + # A 32m upload over a slow link outlives the 60s default on the two body + # timeouts, and nginx cuts it at exactly 60s with a 408 or a 504 that looks + # like the turn failed. Raise them with the size, not just proxy_read_timeout. + client_body_timeout 300s; + + location / { + proxy_pass http://127.0.0.1:9201; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_send_timeout 300s; # pushing the PCM upstream, same reason + proxy_read_timeout 300s; # an LLM turn can take minutes on the iGPU + } +} diff --git a/deploy/mavend.json b/deploy/mavend.json index 962e47d..0ea21ad 100644 --- a/deploy/mavend.json +++ b/deploy/mavend.json @@ -26,6 +26,45 @@ "severity_ceiling": 2 }, + "pattern_proposals": { + "notify": false, + "cooldown": "24h" + }, + + "mcp": { + "timeout": "15s", + "servers": [ + { + "name": "vikunja", + "url": "http://192.168.1.104:9100/mcp", + "allow_private": true, + "allow_tools": ["list_projects", "list_tasks", "get_task_details", "create_task"], + "max_tools": 6, + "enabled": false + } + ] + }, + + "smarthome": { + "provider": "homeassistant", + "url": "http://192.168.1.50:8123", + "token": "${HA_TOKEN}", + "domains": ["light", "switch", "sensor"], + "max_entities": 40, + "timeout": "10s", + "refresh": "15m", + "enabled": false + }, + + "netscan": { + "subnets": ["192.168.1.0/24"], + "ports": [22, 80, 443, 8080], + "timeout": "400ms", + "rate": 100, + "max_hosts": 256, + "enabled": false + }, + "nexus": { "url": "http://nexus:9740" }, "praxis": { "url": "http://praxis:8989" }, "hexis": { "url": "http://hexis:9741" }, diff --git a/docker-compose.yml b/docker-compose.yml index 0652d17..be5a6e9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -101,13 +101,53 @@ services: "-netdata", "http://127.0.0.1:19999", "-kuma", "http://127.0.0.1:3001/metrics", "-kuma-key", "uk5_mavpoll-key"] + # Money tracking (Vikunja #125) is OFF: it needs a zenmoney token, + # which mavpoll reads from a FILE so it never appears in `ps`, in + # this file, or in shell history. To enable, mount the token and + # append: "-zenmoney-token-file", "/run/secrets/zenmoney.token" + # (optionally "-zenmoney-interval", "1h"). Core never sees the + # token — the poller writes facts(kind=env, source=poll:zenmoney) + # and mavend only reads those back when he asks. depends_on: [mavend] volumes: - sockets:/run/maven + # - ./deploy/zenmoney.token:/run/secrets/zenmoney.token:ro + + # The mail reader (Vikunja #246) is OFF and commented out: it needs an IMAP + # account, and there is none on this box. mavmaild reads the password from a + # FILE so it never appears in `ps`, in this file, or in shell history — the + # same rule mavpoll follows for the zenmoney token. Core never sees the + # password: the reader hands core message text on one IPC method, and core + # writes what the model extracts as task CANDIDATES he reviews on /tasks. + # Nothing here can create a reminder, so a misread mail cannot fire. + # + # To enable: write the password to deploy/imap.password (0600, gitignored), + # add an "email": {} block to deploy/mavend.json, and uncomment this service. + # mavmaild: + # <<: *image + # command: ["mavmaild", "-socket", "/run/maven/mavend.sock", + # "-imap", "imap.example.org:993", + # "-user", "kami@example.org", + # "-password-file", "/run/secrets/imap.password", + # "-mailbox", "INBOX", + # "-interval", "15m", + # "-state", "/var/lib/mavmaild/mail-seen.json"] + # depends_on: [mavend] + # volumes: + # - sockets:/run/maven + # # Its OWN volume, not dbdata. The whole point of a separate reader is + # # that a compromise on either side does not reach the other, and dbdata + # # is the encrypted database. The reader needs one JSON file of UIDs and + # # gets a volume that holds nothing else, so neither can be restored from + # # a backup of the other. + # - maildata:/var/lib/mavmaild + # - ./deploy/imap.password:/run/secrets/imap.password:ro volumes: dbdata: sockets: + # maildata — the mail reader's seen-UID file, and nothing else. See mavmaild. + maildata: networks: default: diff --git a/docs/plans/03-memory-evaluation.md b/docs/plans/03-memory-evaluation.md index a58be76..2dfedde 100644 --- a/docs/plans/03-memory-evaluation.md +++ b/docs/plans/03-memory-evaluation.md @@ -25,3 +25,46 @@ 6. Add `/eval` API method to `ipc.CoreAPI` (or reuse `Chat` with system context) so mavweb can show evaluation history 7. Add `memory_eval` block to `deploy/mavend.json` 8. Test with synthetic store state — verify observations match expected patterns + +--- + +## Status 2026-08-01 — foundation shipped (Vikunja #248) + +**Shipped:** `internal/memeval` (not `internal/memory/eval.go` — `internal/store` +imports `internal/memory` for the vector backend, so an evaluator that reads +`store.Fact` there would close an import cycle). `Evaluator.Evaluate` reads +`RecentFacts` / `RecentNotes` / `RecentNudges`, prompts the resident model under +a GBNF grammar for at most three `{observation, confidence, suggested_action}` +objects, drops anything under `min_confidence`, deduplicates against what earlier +evaluations wrote, and records the rest as notes with source `infer:memory-eval`. +Driver: `cmd/mavend/memoryeval.go`, its own goroutine on its own ticker. Config: +the `memory_eval` block — **absent ⇒ the loop does not run**. Visibility: `/dash` +already renders notes with their source, so evaluation output is visible with no +UI change. + +**Deliberately not shipped — this is policy, not an unfinished edge:** + +- *Dispatching observations as care nudges (plan step 4).* An hourly LLM loop + with permission to speak is a machine for generating interruptions, and the + content is model-generated text about his own life. The evaluator has no + dispatcher reference at all, so it cannot reach a channel by accident. Wiring + it to `delivery.Dispatcher` is a separate decision with its own opt-in. +- *Acting on `suggested_action`.* It is recorded inside the note text and + interpreted by nobody. No reminder, routine or fact is created. +- *Writing observation embeddings.* Notes are written with a nil embedding, so + they stay out of the RAG recall pool. Feeding generated text back into the pool + it came from is how a small model starts citing its own guesses as evidence. + +**Deferred, wants a decision or another capability:** + +- *Plan step 6, the `/eval` IPC method and an evaluation-history view.* `/dash` + covers reading the output; a dedicated trace surface is worth building once + there is real output to look at, and it should probably show the prompt too. +- *`RecentEvents`.* The plan lists it; the evaluator reads facts, notes and + nudges. Detected action/object events already drive pattern proposals (#43), and + duplicating them here would mostly re-derive that. +- *Output quality is unmeasured.* There is no fixture for "did she notice + something true". The tests cover the machinery — empty store, confidence floor, + dedupe, own-notes exclusion, error handling — not the observations. Until + someone reads a week of real output on `/dash`, treat the wording and the + `min_confidence` default as unvalidated. diff --git a/docs/plans/07-vision.md b/docs/plans/07-vision.md index 07ac9c1..5483a52 100644 --- a/docs/plans/07-vision.md +++ b/docs/plans/07-vision.md @@ -1,27 +1,106 @@ # Plan: Vision — Image Understanding Capability -**Goal:** Maven can "see" — accept images (from mavweb upload, Telegram, or filesystem paths), run vision inference via a local or remote multimodal model, and answer questions about the image content or extract structured information. +**Goal:** Maven can "see" — accept images (from mavweb upload, Telegram, or filesystem paths), store them, run inference via a **local** multimodal model, and answer questions about the image content or extract text from it. -**Done when:** -- Vision model backend is configurable: local multimodal LLM (e.g., LLaVA, Qwen-VL via `llama-server` mmproj) or remote API -- `internal/vision/` package handles image preprocessing, model inference, result parsing -- Voice/text commands like "что на картинке?" or "прочитай текст с экрана" route to the vision handler -- Extracted information can be written as facts/notes through `ipc.CoreAPI` -- Telegram image messages are processed through the same pipeline +**Status (2026-08-01):** intake, storage, config seam and the provider are shipped. The +describing half is **BLOCKED on a model download** — see "What is blocked" below. -**Scope:** -- New `internal/vision/` package — image loader (Go stdlib `image` + `golang.org/x/image`), inference client -- New config block: `voice.vision` in `config.Config` — `{enabled, provider, model_path, mmproj_path, remote_url}` -- Router intent extension: new `IntentVision` or reuse `IntentQuery` with a vision flag -- Reuses `internal/llm.Client` for API-compatible backends (OpenAI-compatible vision API) -- Reuses `internal/ipc.CoreAPI` for writing extracted data +## What shipped -**Steps:** -1. Create `internal/vision/provider.go` — `Provider` interface with `Describe(image []byte, prompt string) (string, error)` and `ExtractText(image []byte) (string, error)` -2. Implement `LocalProvider` — spawns `llama-server` with mmproj, sends multimodal chat completion requests -3. Implement `RemoteProvider` — calls an OpenAI-compatible vision API endpoint, reuses `internal/llm.Client` -4. Create `internal/vision/processor.go` — image preprocessing (resize, format conversion to JPEG/PNG, base64 encoding) -5. Wire vision into `cmd/mavend/voice.go:reactiveHandler` — detect vision intent from router (new `IntentVision` or a `Slots.HasImage` flag) -6. Add IPC method `MethodDescribeImage` for programmatic access (mavweb upload, telegram bot) -7. Add vision config block to `config.Config` and wire in `cmd/mavend/main.go` -8. Test with a local multimodal model: send an image via mavweb, verify description and text extraction +| Piece | Where | +|---|---| +| Blob store (content-addressed, retention-pruned) | `internal/media/store.go` | +| Image decode / flatten / downscale / JPEG | `internal/media/image.go` | +| `Provider` seam + `Disabled` floor + `LocalProvider` | `internal/vision/vision.go` | +| Store-then-describe orchestration, re-runnable | `internal/vision/intake.go` | +| Config blocks `media` and `vision` | `internal/config/config.go` | +| IPC method `describe_image` (`AuthRead`) | `internal/ipc/{wire,api,client,server}.go`, `internal/auth/policy.go` | +| Daemon wiring + hourly retention prune | `cmd/mavend/vision.go` | + +`internal/media` is deliberately shared: hearing (#253) and speaker recognition (#255) have +the same intake problem — a blob arrives, gets stored, gets described — and they store their +audio in the same place under the same retention. + +## Design decisions worth knowing + +**Store before describe.** `Intake.Accept` writes the blob to disk *first*, then asks the +model. If the model is missing or broken — which is this box's actual state — the answer is +"it's kept, I can't read it yet" with a content-addressed id, and `Intake.Rerun(id, question)` +describes it later. Nothing is lost to a missing model. + +**No `RemoteProvider`.** The original step 3 called for "an OpenAI-compatible vision API +endpoint". Refused. The surviving hard constraint in CLAUDE.md after "never phones home" was +deprecated is *no cloud model, inference stays on the box*, and a photo of his flat is the +worst possible exception. `vision.NewLocal` therefore validates the endpoint at construction: +loopback, a private IP, or `localhost`. A hostname is refused too — it could resolve anywhere, +and resolving it would mean trusting DNS with his pictures. + +**Blobs are not in the database.** The sqlite store is small, encrypted and read every tick; +a 40 MB blob has no business there. What lands in the database is the *text* the blob produced, +as an ordinary note (`source: media:image:`), and only when the caller asks for it +(`save_note`). Glancing at a screenshot is not the same act as remembering it. + +**Images are never search input and never embedded.** Only the derived description +participates in recall, and only after he can see it as a note. + +**Retention is enforced by a loop, not by a promise.** `media.retention` defaults to 7 days +and `cmd/mavend` prunes hourly, starting at boot. A store that grows forever would be the real +failure mode of this capability. + +**No webp.** The stdlib has no webp decoder and this repo takes no new dependencies (the box +is offline). `media.SniffImage` recognises webp well enough to refuse it *by name*, so the log +says "webp is not supported" instead of "not an image". Telegram sends webp for stickers; that +is a known gap, not a mystery. + +**Text extraction is not a second method.** "прочитай текст с картинки" is a prompt. A VLM has +no separate OCR mode to select, and a second interface method would only duplicate the first. + +## What is blocked, and on what + +There is **no vision-capable gguf and no mmproj file on this box**. Checked 2026-08-01: + +``` +/mnt/hdd1/llms/{Bonsai,LFM2.5,llama3.2,ministral,nemotron3-nano,qwen3,qwen3.5} +``` + +— sixteen ggufs, all text-only, no `*mmproj*` anywhere. The resident Qwen3-1.7B is text-only +by construction, so vision needs a *second* model. The ≤1.7B ceiling in CLAUDE.md is about the +resident router/phraser, not about a second model loaded on demand — but iGPU VRAM still is, +so keep it small. + +To unblock, download one pair to `/mnt/hdd1/llms/vision/` (bind-mounted to +`/opt/maven/models/llm`), a gguf **and** its mmproj: + +- `Qwen2.5-VL-3B-Instruct` (Q4_K_M + `mmproj-F16.gguf`) — the safe default; reads Russian, and + its OCR is the best of this size class. +- `SmolVLM2-2.2B-Instruct` — smaller and faster, weaker at Cyrillic text in images. +- `moondream2` — smallest, English-only in practice. Do not bother, per the sub-500M lesson. + +Then run a second llama-server on 8081 with `--mmproj`, point `vision.endpoint` at it, and +walk the QA steps on Vikunja #252. + +## Config + +```json +"media": { "dir": "media", "retention": "168h", "max_bytes": 67108864 }, +"vision": { + "enabled": true, + "endpoint": "http://127.0.0.1:8081", + "model": "qwen2.5-vl-3b", + "max_dim": 896, + "max_tokens": 300, + "timeout": "90s" +} +``` + +Both absent by default. No `media` block ⇒ `describe_image` does not exist at all; a `media` +block with no `vision` block ⇒ images are stored and honestly not described. + +## Still open + +- **Router intent.** "что на картинке?" does not route anywhere yet. Adding an intent is + premature while nothing can answer it; the IPC method is the surface a Telegram photo or a + mavweb upload calls today. +- **Telegram photo path** in `mavpoll` (download the file, call `DescribeImage`). +- **mavweb upload page** and a `/media` listing so stored blobs are visible and deletable from + the authed surface. diff --git a/docs/plans/08-hearing.md b/docs/plans/08-hearing.md index a7cb69f..41c30a0 100644 --- a/docs/plans/08-hearing.md +++ b/docs/plans/08-hearing.md @@ -1,28 +1,121 @@ -# Plan: Hearing — Audio Stream Monitoring & Meeting Summarization +# Plan: Hearing — Meeting Capture & Summarisation -**Goal:** Maven can "hear" ambient audio from workpc — microphone input during meetings, system audio — and on demand (or on trigger) produce transcripts, summaries, or extract action items. A typical use case: "Maven, запиши встречу" starts capture, "хватит" stops it, and Maven writes a summary note. +**Goal:** "Maven, запиши встречу" starts a recording, "хватит" stops it, and she writes a +summary note. The audio stays on the box, is pruned by retention, and nothing is recorded that +nobody asked for. -**Done when:** -- `internal/audio/capture.go` — remote microphone capture client (receives PCM stream from workpc over WebSocket or the existing voice TCP protocol) -- `internal/stt/` — streaming transcription (uses existing `stt.Transcriber` interface, extended with streaming support) -- Meeting capture triggered by voice command (IntentCapture) or configurable keyword ("maven record") -- Raw audio is either streamed to STT in real-time or saved to a WAV file and transcribed after capture ends -- Transcription + LLM summary is written as a note (`source:capture:meeting`) through `ipc.CoreAPI` -- New `mavheary` module (`cmd/mavheard/`) — the workpc-side agent that captures mic/speaker audio and streams it to mavend +**Status (2026-08-01):** the recorder, the storage, the chunked transcription, the map-reduce +summariser, the config seam and the four IPC methods are shipped and tested. What is not +shipped is the workpc-side microphone agent and the router intent — see "Still open". -**Scope:** -- New `cmd/mavheard/` — workpc-side agent: captures microphone (PortAudio or ALSA `arecord`), streams over WebSocket to mavend -- `internal/audio/` extended with capture types: `MicCapture`, `SystemCapture`, `FileCapture` -- `internal/stt/stt.go` extended with `StreamingTranscriber` interface (or reuse existing with chunked input) -- Router: new `IntentCapture` intent for start/stop commands -- Reuses `internal/llm.Client` for summarization -- Reuses `internal/voice/server.go` TCP protocol for streaming audio +## What shipped -**Steps:** -1. Create `cmd/mavheard/main.go` — workpc-side daemon: captures microphone via `arecord` pipe or PortAudio, opens WebSocket or TCP connection to mavend, streams PCM frames -2. Create `internal/audio/capture.go` — `Capture` interface: `Start()`, `Stop()`, `AudioCh <-chan Audio`; implement `MicCapture` (reads from `mavheard` stream) and `FileCapture` (reads WAV) -3. Extend `internal/stt/stt.go` — add `TranscribeStream(ctx, audio <-chan Audio) (string, error)` to `Transcriber` interface; `Stub` returns empty; `Remote` forwards chunks to worker socket -4. Add `IntentCapture` to `internal/router/intent.go` — slots: `Action` ("start"/"stop"/"status"), `Duration` -5. Wire capture handler in `cmd/mavend/voice.go:reactiveHandler` — start = spawn goroutine receiving audio, stream to STT; stop = finalize, send to LLM for summarization, write note via `WriteNote` -6. Add capture config to `voice` block in `config.Config` — `{capture_enabled, capture_timeout}` -7. Test with a recorded WAV file — simulate a meeting, verify transcription + summary note is created +| Piece | Where | +|---|---| +| Session state machine: start / append / stop / abort / status | `internal/capture/capture.go` | +| Map-reduce summarisation against `n_ctx` 4096 | `internal/capture/summarize.go` | +| Audio blobs in the shared store, pruned by `media.retention` | `internal/media` (from #252) | +| Config block `capture`, off by default | `internal/config/config.go` | +| IPC `capture_start` / `capture_append` / `capture_stop` / `capture_status` | `internal/ipc/{wire,api,client,server}.go` | +| Authority: the three write methods `AuthWrite`, status `AuthRead` | `internal/auth/policy.go` | +| Daemon wiring, note write, STT reuse | `cmd/mavend/capture.go` | + +The audio lands in the same content-addressed blob store as images, under the same retention +loop, because #252 and #253 have the same intake problem and solving it twice would mean two +directories to remember to prune. + +## The refusals, and why + +**Nothing listens.** The original step 8 called for capture "triggered by voice command +(IntentCapture) **or configurable keyword ('maven record')**". The keyword half is refused. +Noticing a keyword requires listening to the room continuously, which is precisely the +behaviour this capability must not have, and the refusal is in the code rather than in a +comment: `Recorder.Append` is the only way audio enters, and it returns `ErrNoSession` unless +someone explicitly started a session. Audio arriving at an idle core is dropped, not buffered +"just in case". + +**Off unless configured, twice over.** No `media` block ⇒ nowhere to keep audio ⇒ the four +methods do not exist. No `capture` block with `enabled: true` ⇒ they still do not exist. On an +unconfigured box there is no wire path at all that begins a recording. That is the only +guarantee worth making here, and it is the reason the hooks use the nil-hook ⇒ +`ErrUnknownMethod` pattern rather than an in-handler check. + +**A forgotten session ends itself.** `max_minutes` defaults to 120 and is checked on every +append, not on a timer that could be missed. Past the cap `Append` returns `ErrExpired` +permanently, so a client that ignores the error cannot grow the recording; the audio collected +before the cap is kept and `Stop` still works. + +**"Забудь, не записывай" leaves nothing behind.** `capture_stop` with `discard: true` throws +the session away without storing, transcribing or summarising anything — not a blob with a note +saying it was abandoned. Nothing. + +**The transcript is not saved by default.** The summary is written where he will read it; the +verbatim record of what other people said in a room is a heavier thing to keep and takes a +deliberate `save_transcript: true`. The audio blob is pruned by `media.retention` either way. + +**No second STT.** Step 3 of the original plan extended the `Transcriber` interface with +streaming. Not needed and not done: whisper.cpp already runs as `mavsttd`, and `internal/capture` +takes the ordinary `stt.Transcriber` the voice path already holds (exposed as +`voiceWiring.transcriber`). Long recordings are handed over in five-minute windows — +`chunkAudio`, cut on sample boundaries — for the same reason whisper itself works in 30-second +windows: an hour of PCM in one call either times out or blocks the voice path for minutes. +Capture with voice off is refused rather than degraded, because storing hours of unreadable +audio of other people is worse than not recording. + +**Not `AuthStepUp`.** Recording people is invasive enough to argue for the top rung, and it is +still wrong: step-up needs a passkey gesture, which the voice path cannot make, so +"запиши встречу" could never work by voice — the only way he will actually use this. `AuthWrite` +plus the off-unless-configured gate is the honest combination. + +## Long audio against a 4096-token context + +The resident model is a Thinking variant at `n_ctx` 4096, so an hour of transcript does not fit +in one prompt and never will. `summarize.go` does map-reduce and nothing cleverer: split the +transcript on sentence boundaries into 3000-rune windows (about 1100 Qwen tokens of Russian, +leaving room for the persona block, the reasoning and the answer), summarise each, then +summarise the summaries. A transcript that fits in one window skips the reduce step. + +Truncation was the alternative and is rejected: a truncated meeting summary reads as complete +and is not, and he would act on it. Past `max_chunks` (40, roughly the two-hour cap) the +transcript *is* cut, and the summary says so in the note. + +Two degradations are deliberate and both are reported rather than hidden: + +- No llama-server ⇒ transcript, no summary. The words exist. +- The reduce call fails ⇒ the per-chunk summaries are returned joined. Real work, not thrown + away over the last call. + +The map and reduce prompts contain no first person at all, so the persona's feminine-form rules +have nothing to get wrong in them; the reply she actually gives him is phrased by the ordinary +replier, which does carry the persona. + +## Config + +```json +"media": { "dir": "media", "retention": "168h" }, +"capture": { + "enabled": true, + "max_minutes": 120, + "stt_window": "5m", + "chunk_runes": 3000, + "max_chunks": 40, + "save_transcript": false +} +``` + +Both absent by default. `capture` alone does nothing without `media`. + +## Still open + +- **`cmd/mavheard`** — the workpc-side microphone agent. Deferred, not refused: the core half + is the part with the invariants in it, and a mic client is straightforward once there is a + stable wire to stream at. It should be an explicit-start process, not a resident one, for the + same reason the recorder has no keyword trigger. The four IPC methods are the wire it will + use; `mavenclient` already has the mic plumbing to borrow. +- **Router intent.** "запиши встречу" / "хватит" does not route anywhere yet. It needs the + `system` intent plus slots, and it needs care: "хватит" is also how someone tells her to stop + talking, so the recorder's stop and the speech barge-in must not collide. +- **A `/dash` panel** showing a running session, so a recording is visible on a surface and not + only in a log line. +- **Speaker attribution** — who said what — is #255 and is blocked on a model; see + `docs/plans/10-speaker-recognition.md`. diff --git a/docs/plans/09-behavioral-memory.md b/docs/plans/09-behavioral-memory.md index 9313303..363c568 100644 --- a/docs/plans/09-behavioral-memory.md +++ b/docs/plans/09-behavioral-memory.md @@ -27,3 +27,36 @@ 6. Wire voice query — `"что я обычно делаю?"` routes to `IntentQuery` → behavior profile lookup → LLM-phrased answer 7. Add IPC read method `MethodGetBehaviorProfile` so mavweb can display it on `/dash` 8. Test with synthetic fact history — verify weekly schedule is correctly inferred + +--- + +## Status (2026-08-01) — partially shipped, deliberately narrowed + +Shipped on `overnight/behavior-profile`: + +- `internal/memory/behavior.go` — `BuildProfile` counts habits per weekday out of + self-facts: distinct-day counts (`MinHabitDays = 2`), a median time-of-day, and + `FormatWeekdayRU` / `FormatOverallRU` for the spoken answer. +- `internal/router/habit.go` — `ParseHabitQuery`, which requires a habit marker + ("обычно", "каждую", "привычки", …) and parses the weekday deterministically. +- `cmd/mavend/actions_query.go` — a `habits` query source, so "что я обычно делаю + по вторникам?" is answered. + +**Not shipped, and not to be shipped as written:** + +- *Step 3, LLM-generated profile stored as a fact.* The profile is COUNTED, not + generated. A 1.7B asked to summarise a year of habits produces fluent claims + about the owner's life that no row supports, and a wrong claim about him is the + most expensive kind of wrong maven can be. Counting is verifiable and cheap. +- *Step 5, incremental updates on fact write.* There is no cache to keep fresh — + the profile is recomputed on the question, so a new fact is already in the next + answer. A cached profile that can disagree with its own rows is two truths. +- *Step 4, proactive daily plan proposals via the dispatcher.* Maven is not a nag, + and a nudge at 08:00 every day proposing the day is the definition of one. The + sanctioned path from "she noticed a pattern" to "she acts on it" already exists: + `internal/pattern/detector.go` proposes a routine, and the owner accepts it on + `/routines`. It goes through him. + +Still open, if wanted later: `MethodGetBehaviorProfile` + a `/dash` panel (step 7). +The counted profile needs no new IPC method to be *asked* about — the query source +reads `RecentFacts` over the existing surface — so this is a display concern only. diff --git a/docs/plans/10-speaker-recognition.md b/docs/plans/10-speaker-recognition.md index abd5b96..63273f9 100644 --- a/docs/plans/10-speaker-recognition.md +++ b/docs/plans/10-speaker-recognition.md @@ -1,27 +1,125 @@ # Plan: Speaker Recognition -**Goal:** Maven can distinguish between different speakers on the voice channel — recognize known voices (the user, family members) and tag facts/notes/transcripts with a speaker identity. +**Goal:** Maven can tell who is speaking on the voice channel, and tag what she writes with +who said it. -**Done when:** -- Speaker embedding extractor (e.g., ECAPA-TDNN or a simple MFCC + GMM) runs on incoming voice PCM before STT -- Embedding is compared against enrolled speaker profiles (stored as vectors in the `memory_vectors` table alongside semantic memory) -- Unknown speakers are enrolled on first interaction (prompt: "кто это?") -- All voice fact/note writes are tagged with `speaker:` in the value/source metadata -- Speaker identity is available as context to the router, phraser, and replier ("ok, ") +**Status (2026-08-01, Vikunja #255):** the enrolment half is shipped. The recognising half is +**BLOCKED on a model download** — there is no speaker-embedding model on this box, and one +was not invented to fill the gap. See "Blocked, and on what" below. -**Scope:** -- New `internal/speaker/` package — enrollment, recognition, embedding extraction -- Reuses `internal/store.MemoryStore` for speaker vector storage (same `memory_vectors` table, different `source` prefix) -- Reuses `internal/audio` for PCM preprocessing -- Integration point: `cmd/mavend/voice.go:HandlePushToTalk` — speaker ID extracted before STT, passed through context +## What shipped -**Steps:** -1. Research speaker embedding approaches — simplest floor: MFCC + cosine similarity via `github.com/mjibson/go-dsp` or a pre-trained ONNX model (SpeechBrain ECAPA) -2. Create `internal/speaker/recognizer.go` — `Recognizer` interface: `Identify(pcm []float32) (SpeakerID, confidence)`, `Enroll(id, pcm)` -3. Create `internal/speaker/store.go` — speaker profile CRUD via `store.MemoryStore`: `Insert("speaker:", embedding, meta)`, `Search(embedding, k)` -4. Create `internal/speaker/enroll.go` — enrollment flow: capture N seconds of audio, extract embedding, prompt for name via TTS + STT round-trip -5. Wire into `cmd/mavend/voice.go:HandlePushToTalk` — run speaker ID on the PCM before STT; pass speaker ID through `context.Context` to `applyAction` -6. Tag all voice-written facts/notes with speaker ID — `Source` becomes `tap:voice:speaker:` or metadata field -7. Add IPC methods `MethodEnrollSpeaker`, `MethodListSpeakers`, `MethodRemoveSpeaker` -8. Add speaker config block to `voice` in `config.Config` — `{speaker_recognition: true, model_path}` -9. Test with 2+ recorded voice samples — verify correct identification and rejection of unknown speakers +| Piece | Where | State | +|---|---|---| +| `Recognizer` — identify, list, get, forget | `internal/speaker/recognizer.go` | done; `Identify` answers `ErrDisabled` until a model exists | +| Enrolment — several samples, averaged, re-normalised | `internal/speaker/enroll.go` | done | +| Profile shape, id validation, cosine similarity | `internal/speaker/speaker.go` | done | +| Profile storage as `speaker:` vectors | `internal/memory` `Catalog` + `internal/store/memory.go` | done, no schema migration | +| Config block, off by default | `internal/config` `SpeakerConfig` | done | +| `enroll_speaker` / `list_speakers` / `forget_speaker` | `internal/ipc` | done, absent unless configured | +| Authority rows | `internal/auth/policy.go` | done — enrol step-up, forget write, list read | +| Daemon wiring + honest startup log | `cmd/mavend/speaker.go` | done | +| Embedding backend | `newSpeakerEmbedder` | **BLOCKED** — returns nil, seam only | +| Tagging voice writes with the speaker | `cmd/mavend/voice.go` | not wired; nothing to tag with yet | + +## Blocked, and on what + +A voiceprint needs a speaker-embedding model. The box was searched: `/mnt/hdd1/llms` holds +sixteen ggufs across seven families and every one of them is a text model. There is no ECAPA, +no x-vector, no titanet, no wespeaker, and no `.onnx` under `/mnt/hdd1` at all. There are also +no enrolment samples, because nothing has ever recorded any. + +To unblock, two things are needed and neither can be done from inside the repo: + +1. **A model.** SpeechBrain ECAPA-TDNN exported to ONNX (`speechbrain/spkrec-ecapa-voxceleb`, + 192-dim) is the usual choice and runs on CPU in well under a second for a few seconds of + audio. Download it per the recipe in `AGENTS.md`, put it beside the other models so the + bind mount picks it up, and point `speaker.model_path` at it. +2. **An implementation of one function.** `newSpeakerEmbedder` in `cmd/mavend/speaker.go` is + the entire seam: give it an ONNX session that turns `audio.Audio` into a `[]float32` and + `Identify` starts working. Nothing else changes — not the store, not the protocol, not the + authority table, not the handlers. `internal/onnx` already loads the e5 embedder, so the + runtime wiring exists to copy. +3. **Enrolment samples**, three or more per person, recorded deliberately. + +### Why there is no fallback + +The original plan offered "a simple MFCC + GMM" as the floor. That is refused. MFCC cosine +distance is a channel and loudness detector as much as a voice detector: it will happily match +two different people who sit at the same distance from the same microphone, and it drifts when +the room changes. A general classifier that is sometimes wrong is a nuisance; a **biometric** +that is confidently wrong writes false claims about named people into his memory, and then +those claims get recalled as fact. For this capability a bad floor is worse than none, so the +shipped state is honest absence: `speaker.Disabled`, `ErrDisabled`, and a startup line saying +so. + +## The refusals, and why + +- **Unknown speakers are NOT enrolled on first interaction.** The plan's fourth "done when" + bullet asked for exactly that, with a TTS "кто это?" prompt. It is refused in + `enroll.go`'s doc comment and there is no request shape in the protocol that could express + it. Enrolling a voice is taking a biometric of a person; doing it automatically to whoever + walks past the microphone does it to guests who are not party to the exchange, and a + synthesised question into a room is not consent from whoever happens to answer. Enrolment is + an explicit act: an id, a name, and samples recorded for the purpose. +- **One sample is not enough.** Three separate utterances and nine seconds minimum. A profile + built from one sentence encodes that sentence as much as the person, and the threshold then + behaves unpredictably against everything else. +- **An unknown voice stays unknown.** Below threshold, `Identify` returns `ErrUnknown` naming + the closest profile in the error text for diagnosis, never as an answer. Guessing who is in + the room is how false memories about people get written. +- **Deletion is one authority rung below enrolment.** Everywhere else in `policy.go` the + destructive direction is gated at least as hard as the constructive one. Here that would be + backwards: getting rid of a biometric must never be the harder half. +- **The voiceprint never crosses the socket.** `ListSpeakersResp` carries ids, names, dates + and sample counts. The vector stays in core. +- **Off unless configured.** No `speaker` block ⇒ the three methods answer + `ErrUnknownMethod`. There is no wire path on a default box that takes a voiceprint. + +## Storage + +Profiles live in the existing `memory_vectors` table under the `speaker:` id prefix, as the +plan intended, so there is no migration. What that needed was a wider interface than +`memory.Store`: `memory.Catalog` adds `ByPrefix` and `Delete`. `Delete` is the load-bearing +one — a voiceprint someone asked to be rid of has to actually go, and a search-only store +cannot do that. `InMemoryStore.Insert` also became an upsert by id, matching what the +persistent store already did, so re-enrolling replaces a profile instead of stacking a second +one behind the first. + +Profiles do not collide with note or fact vectors: they are only ever read through +`ByPrefix("speaker:")`, and a note search never returns one because the prefix is not in its +query path. + +## Config + +```json +"speaker": { + "enabled": true, + "model_path": "/opt/maven/models/spk/ecapa-voxceleb.onnx", + "lib_path": "/opt/maven/lib", + "threshold": 0.7, + "min_seconds": 2.0 +} +``` + +`Recognizes()` requires both `enabled` and a `model_path`, so a half-filled block reads as off +rather than as a capability that fails every turn. With `enabled` and no model the daemon still +attaches the three methods — profiles can be created, listed and deleted — and logs that +recognition is blocked. + +## Still open + +- The embedding backend (above). Everything below waits on it. +- **Tagging voice writes.** `Profile.Source("tap:voice")` already produces + `tap:voice:speaker:kami`, which is the shape step 6 asked for, but nothing calls it yet: + with no recogniser there is no id to tag with. When the model lands, the hook is in the + voice path before STT. +- **Speaker as router/phraser context.** Same dependency. Note the persona constraint when it + arrives: Maven addresses the owner informally and speaks to him, so "ok, " needs care + for anyone who is not him. +- **An enrolment surface.** The three IPC methods exist; no page drives them. Enrolment is + step-up, so it belongs on `/dash` behind a passkey, with a per-profile forget button next to + each row — that button is the reason `list_speakers` exists. +- **A speaker column on the meeting recorder** (#253). Attributing lines in a transcript is + the obvious pairing, and it is the place where getting attribution wrong is most damaging, + so it waits for a real model too. diff --git a/docs/plans/13-rss-news-feeds.md b/docs/plans/13-rss-news-feeds.md index 9edd1b5..5a4289e 100644 --- a/docs/plans/13-rss-news-feeds.md +++ b/docs/plans/13-rss-news-feeds.md @@ -27,3 +27,28 @@ 7. Add voice query handler — `"что нового?"` queries `RecentNotes` filtered by source prefix `rss:` and phrases via `phraser.PhraseQuery` 8. Add `feeds` block to `config.Config` and `deploy/mavend.json` 9. Test with a live RSS feed (e.g., `https://news.ycombinator.com/rss`) — verify items appear in notes table + +--- + +## Shipped 2026-08-01 (#258) + +`internal/webfetch` (the guarded HTTP door: scheme, allow/deny hosts, private-address +refusal in the dialer, size cap, redirect cap, per-host rate limit), `internal/rss` +(RSS 2.0 + Atom parser, poller with durable marks and a keyword filter), +`cmd/mavend/feeds.go` (ticker, fetcher adapter, `rss:latest:` fact marks), +config block `feeds`, and the `feeds` query source with `router.ParseFeedQuery`. + +Deviations from the plan above, both deliberate: + +- **Step 5 (breaking-news nudges) was not built.** A feed that dispatches is a nag, + and the one thing Maven is not is a nag. Items are read when asked and nowhere else. + If breaking news is ever wanted, it belongs behind the existing delivery policy + (severity, quiet hours, digest), not in the poller. +- **Step 3 (embedder relevance) is a seam, not an implementation.** `rss.Ranker` + exists and is wired nil. Scoring items against an "interest profile" needs a + profile, and there is none yet; a threshold with nothing to compare against is a + random filter with a confident name. The filter that runs is the per-feed + include/exclude keyword list, which he can read and predict. + +No new dependency: stdlib `encoding/xml`, no gofeed. Stock deploy config has no +`feeds` block, so the capability is off. diff --git a/docs/plans/14-web-crawler.md b/docs/plans/14-web-crawler.md index b5c78ac..a9b22f3 100644 --- a/docs/plans/14-web-crawler.md +++ b/docs/plans/14-web-crawler.md @@ -28,3 +28,41 @@ 7. Add IPC methods `MethodTriggerCrawl(name)`, `MethodListCrawls`, `MethodGetCrawlResult(name)` 8. Add `crawls` block to `config.Config` and `deploy/mavend.json` 9. Test with a static HTML page — verify extraction matches expected values, verify scheduling fires correctly + +## Shipped 2026-08-01 (#259) + +Built as `internal/crawl` (pure: robots, extraction, watcher) plus +`cmd/mavend/crawls.go` (fetcher, ticker, dedup facts), on top of the guarded +`internal/webfetch` door added with the feed reader (#258). Off unless +configured, in two separately-switched halves: `crawl.on_demand` for a URL he +names, `crawl.watches` for a scheduled re-read. + +**Limits are code, not documentation** (`internal/webfetch`, tested one test per +limit): host allowlist/denylist, no private addresses (loopback, RFC1918 — +hence the LAN and the `10.42.0.0/24` wg range —, link-local incl. cloud +metadata, CGNAT, v6 ULA) enforced in the dialer's `Control` hook so DNS +rebinding and every redirect hop are covered, response size cap, redirect cap, +timeout, one request per host per second. `robots.txt` is fetched first, cached +per host, and a `Disallow` is refused with no override. + +Deliberate deviations from the plan above: + +- **No CSS selectors and no LLM structured extraction** (steps 2). The output is + plaintext handed to the phraser as context for the question he asked. A 1.7B + extracting a JSON price table from 4000 runes is a worse bet than reading, and + `goquery` is not vendored. +- **No `crawl` act verb and no new IPC methods** (steps 5, 7). Reading a page is + a query source (`queryWeb` in `actions_query.go`, last in the chain, behind + Kiwix once that is wired), not an action he commands. Nothing needs a new wire + method to work. +- **Notes, not facts.** A page's text is not a fact about him. Only the dedup + hash is a fact (`crawl:hash:`, kind `config`, source `poll:crawl`). +- **Nothing is dispatched.** A changed page writes a note; it does not nudge. + Not a nag. +- **No `/tools` crawl history page.** The notes and the hash facts are already + visible on `/dash`. + +**No new dependency.** The vendored tree has no `x/net/html`, no `goquery` and +no `temoto/robotstxt`, so robots parsing and HTML-to-text are stdlib +(`regexp`, `html`) — RE2 has no backreferences, hence the `pairsRE` builder in +`extract.go`. diff --git a/internal/audio/pcmwav.go b/internal/audio/pcmwav.go index 4889fa3..b9da0f8 100644 --- a/internal/audio/pcmwav.go +++ b/internal/audio/pcmwav.go @@ -94,14 +94,33 @@ func PCMFromWAV(wav []byte) (Format, []byte, error) { // header so the result can be written to disk and played with `aplay`. // Used by the reference client to write the TTS reply; not on the wire. func WAVFromPCM(format Format, pcm []byte) ([]byte, error) { + hdr, err := WAVHeader(format, len(pcm)) + if err != nil { + return nil, err + } + out := make([]byte, wavHeaderSize+len(pcm)) + copy(out, hdr) + copy(out[wavHeaderSize:], pcm) + return out, nil +} + +// WAVHeaderSize is the fixed size of the header WAVHeader writes. A caller +// spooling audio to a file reserves this many bytes up front and rewrites them +// once it knows the length. +const WAVHeaderSize = wavHeaderSize + +// WAVHeader builds just the 44-byte canonical header for n bytes of PCM. It +// exists so a long recording can be written straight to a file: holding the +// whole meeting in memory to prepend 44 bytes is what the streaming path is +// avoiding. +func WAVHeader(format Format, n int) ([]byte, error) { if !format.IsValid() { return nil, fmt.Errorf("audio: WAVFromPCM: %w: %+v", ErrNotCanonicalPCM, format) } - out := make([]byte, wavHeaderSize+len(pcm)) - copy(out[wavHeaderSize:], pcm) + out := make([]byte, wavHeaderSize) // RIFF header copy(out[0:4], []byte("RIFF")) - binary.LittleEndian.PutUint32(out[4:8], uint32(36+len(pcm))) + binary.LittleEndian.PutUint32(out[4:8], uint32(36+n)) copy(out[8:12], []byte("WAVE")) // fmt chunk copy(out[12:16], []byte("fmt ")) @@ -116,6 +135,6 @@ func WAVFromPCM(format Format, pcm []byte) ([]byte, error) { binary.LittleEndian.PutUint16(out[34:36], uint16(format.SampleBits)) // data chunk copy(out[36:40], []byte("data")) - binary.LittleEndian.PutUint32(out[40:44], uint32(len(pcm))) + binary.LittleEndian.PutUint32(out[40:44], uint32(n)) return out, nil } diff --git a/internal/auth/auth_test.go b/internal/auth/auth_test.go index 1e19edc..8de46ac 100644 --- a/internal/auth/auth_test.go +++ b/internal/auth/auth_test.go @@ -7,7 +7,6 @@ import ( "path/filepath" "strings" "testing" - "time" "github.com/kami/maven/internal/ipc" ) @@ -359,8 +358,12 @@ func TestGate_IpcServer_ChatAllowedForEnrolledCaller(t *testing.T) { // recordingAPI — a no-op CoreAPI that counts WriteFact invocations; the auth // check must reject before reaching it, otherwise the refusal leaks into the -// fake's counts and we fail. +// fake's counts and we fail. Embeds ipc.UnimplementedCoreAPI so every method +// this test doesn't exercise returns ipc.ErrNotImplemented loudly instead of +// being hand-stubbed to a canned value nobody checks. type recordingAPI struct { + ipc.UnimplementedCoreAPI + writes int chats int } @@ -369,89 +372,7 @@ func (r *recordingAPI) WriteFact(_ context.Context, _ ipc.WriteFactReq) (int64, r.writes++ return int64(r.writes), nil } -func (r *recordingAPI) LatestFact(_ context.Context, _ string) (ipc.Fact, error) { - return ipc.Fact{}, ipc.ErrNoFact -} -func (r *recordingAPI) LatestFactBySource(_ context.Context, _, _ string) (ipc.Fact, error) { - return ipc.Fact{}, ipc.ErrNoFact -} -func (r *recordingAPI) Since(_ context.Context, _ string, _ time.Time) (time.Duration, error) { - return 0, ipc.ErrNoFact -} -func (r *recordingAPI) Presence(_ context.Context) (ipc.Presence, error) { - return ipc.Presence{}, nil -} -func (r *recordingAPI) CreateReminder(_ context.Context, _ time.Time, _, _ string) (int64, error) { - return 1, nil -} -func (r *recordingAPI) MarkReminder(_ context.Context, _ int64, _ string) error { return nil } -func (r *recordingAPI) ListReminders(_ context.Context, _ int) ([]ipc.Reminder, error) { - return nil, nil -} -func (r *recordingAPI) TickTrace(_ context.Context) (ipc.TickTrace, error) { - return ipc.TickTrace{}, nil -} -func (r *recordingAPI) MorningStatus(_ context.Context) ([]ipc.MorningRoutineStatus, error) { - return nil, nil -} -func (r *recordingAPI) RecordNudge(_ context.Context, _, _, _ string, _ time.Time) (int64, error) { - return 1, nil -} -func (r *recordingAPI) ResolveNudge(_ context.Context, _ int64, _ string, _ time.Time) error { - return nil -} -func (r *recordingAPI) RecentOutcomes(_ context.Context, _ string, _ int) ([]string, error) { - return nil, nil -} -func (r *recordingAPI) RecentFacts(_ context.Context, _ int) ([]ipc.Fact, error) { - return nil, nil -} -func (r *recordingAPI) CalendarEvents(_ context.Context, _, _ time.Time) ([]ipc.Fact, error) { - return nil, nil -} -func (r *recordingAPI) RecentNudges(_ context.Context, _ int) ([]ipc.Nudge, error) { - return nil, nil -} -func (r *recordingAPI) WriteNote(_ context.Context, _ time.Time, _ string, _ []float32, _ string) (int64, error) { - return 1, nil -} -func (r *recordingAPI) QueryNotes(_ context.Context, _ []float32, _ int) ([]ipc.Note, error) { - return nil, nil -} -func (r *recordingAPI) RecentNotes(_ context.Context, _ int) ([]ipc.Note, error) { - return nil, nil -} -func (r *recordingAPI) ProposeTool(_ context.Context, _, _, _ string, _ time.Time) (bool, error) { - return false, nil -} -func (r *recordingAPI) EnableTool(_ context.Context, _ string, _ []string, _ bool, _ string, _ time.Time) error { - return nil -} -func (r *recordingAPI) DisableTool(_ context.Context, _ string) error { - return nil -} -func (r *recordingAPI) DeleteTool(_ context.Context, _ string) error { - return nil -} -func (r *recordingAPI) LookupTool(_ context.Context, _ string) (ipc.Tool, error) { - return ipc.Tool{}, ipc.ErrToolNotFound -} -func (r *recordingAPI) ListTools(_ context.Context, _ string) ([]ipc.Tool, error) { - return nil, nil -} -func (r *recordingAPI) RevertFact(_ context.Context, _ string) (int64, error) { - return 0, nil -} -func (r *recordingAPI) ListProposedRoutines(_ context.Context) ([]ipc.ProposedRoutine, error) { - return nil, nil -} -func (r *recordingAPI) AcceptProposedRoutine(_ context.Context, _ int64) error { - return nil -} -func (r *recordingAPI) DismissProposedRoutine(_ context.Context, _ int64) error { - return nil -} func (r *recordingAPI) Chat(_ context.Context, text string) (string, error) { r.chats++ return "echo: " + text, nil @@ -472,3 +393,131 @@ func mustWriteFactParams(source string) []byte { } return b } + +// TestRequirement_SwapModel — loading a different resident model is an owner +// action at the same rung as mutating the tool allowlist: it decides how every +// utterance is routed and how every reply is worded. The read side is not. +func TestRequirement_SwapModel(t *testing.T) { + if got := Requirement(ipc.MethodSwapModel); got != AuthStepUp { + t.Errorf("SwapModel authority = %v; want AuthStepUp", got) + } + if got := Requirement(ipc.MethodModelStatus); got != AuthRead { + t.Errorf("ModelStatus authority = %v; want AuthRead", got) + } + // A surface that cannot carry a passkey gesture cannot swap the model, no + // matter what it is enrolled as — this is the "never through voice" property. + voice := Scope{Surface: SurfaceVoice, Module: "voice", SourceScope: []string{"*"}} + if err := Can(ipc.MethodSwapModel, voice, nil); !errors.Is(err, ErrForbidden) { + t.Errorf("voice swapping the model = %v; want ErrForbidden", err) + } + // And with no step-up session asserted, the gate refuses even a capable surface. + noSession := &Gate{Enrollment: NewFloorEnrollment()} + if err := noSession.Check(context.Background(), ipc.MethodSwapModel, nil); !errors.Is(err, ipc.ErrForbidden) { + t.Errorf("SwapModel with no asserted step-up = %v; want ErrForbidden", err) + } +} + +// TestRequirement_ListMutation — the three methods that change what is on his +// lists sit on one rung. IngestMail joined them on 2026-08-01; it used to be +// AuthRead, which made it disagree with SetTaskStatus about the same question. +// CaptureTask stays a read: it puts one line on a list he asked for. +func TestRequirement_ListMutation(t *testing.T) { + for _, m := range []ipc.Method{ + ipc.MethodIngestMail, ipc.MethodSetTaskStatus, + } { + if got := Requirement(m); got != AuthWrite { + t.Errorf("%s authority = %v; want AuthWrite", m, got) + } + } + if got := Requirement(ipc.MethodCaptureTask); got != AuthRead { + t.Errorf("CaptureTask authority = %v; want AuthRead", got) + } + // mavmaild keeps working: AuthWrite outside WriteFact only needs enrollment. + maild := Scope{Surface: SurfaceCoreProcess, Module: "mavmaild", SourceScope: []string{"mail:*"}} + if err := Can(ipc.MethodIngestMail, maild, nil); err != nil { + t.Errorf("mavmaild ingesting mail = %v; want allowed", err) + } +} + +// TestRequirement_Capture — recording other people is a write, not a read: it +// puts audio of them on disk. The read side, "что ты записываешь?", is not. +// +// It is deliberately NOT AuthStepUp. Step-up needs a passkey gesture, which the +// voice path cannot make, so putting it there would mean "запиши встречу" could +// never work by voice. The real gate on this capability is that the methods do +// not exist at all unless the operator enabled a capture block. +func TestRequirement_Capture(t *testing.T) { + for _, m := range []ipc.Method{ + ipc.MethodCaptureStart, ipc.MethodCaptureAppend, ipc.MethodCaptureStop, + } { + if got := Requirement(m); got != AuthWrite { + t.Errorf("%s authority = %v; want AuthWrite", m, got) + } + } + if got := Requirement(ipc.MethodCaptureStatus); got != AuthRead { + t.Errorf("CaptureStatus authority = %v; want AuthRead", got) + } + // Voice can start one: it is the surface he will actually use to say + // "запиши встречу", and it carries AuthWrite. + voice := Scope{Surface: SurfaceVoice, Module: "voice", SourceScope: []string{"*"}} + if err := Can(ipc.MethodCaptureStart, voice, nil); err != nil { + t.Errorf("voice starting a capture = %v; want allowed", err) + } +} + +// TestRequirement_Speaker — a voiceprint is a biometric of a named person, so +// taking one is step-up: a deliberate act from a surface that can carry a +// passkey gesture, never something the voice path does mid-conversation. +// +// Deletion is one rung lower, and that asymmetry is the point. Everywhere else +// in the table the destructive direction is gated at least as hard as the +// constructive one; for a biometric that would be backwards, because getting +// rid of it must never be the harder half. +func TestRequirement_Speaker(t *testing.T) { + if got := Requirement(ipc.MethodEnrollSpeaker); got != AuthStepUp { + t.Errorf("EnrollSpeaker authority = %v; want AuthStepUp", got) + } + if got := Requirement(ipc.MethodForgetSpeaker); got != AuthWrite { + t.Errorf("ForgetSpeaker authority = %v; want AuthWrite", got) + } + if got := Requirement(ipc.MethodListSpeakers); got != AuthRead { + t.Errorf("ListSpeakers authority = %v; want AuthRead", got) + } + // Voice cannot enrol anybody, however the utterance is phrased. + voice := Scope{Surface: SurfaceVoice, Module: "voice", SourceScope: []string{"*"}} + if err := Can(ipc.MethodEnrollSpeaker, voice, nil); err == nil { + t.Error("voice enrolling a speaker was allowed; want refused") + } + // But it can read the roster, which is what answering "кого ты знаешь?" + // needs. + if err := Can(ipc.MethodListSpeakers, voice, nil); err != nil { + t.Errorf("voice listing speakers = %v; want allowed", err) + } +} + +// Describing an image is a read. Saving the description is a write of recall +// corpus under a source no enrollment owns, so it is held to the same +// source-scope rule WriteFact is. Before this, any AuthRead caller could put a +// small VLM's guess into what Maven knows. +func TestCan_DescribeImage_SaveNoteNeedsScope(t *testing.T) { + poller := Scope{Surface: SurfaceTelegram, Module: "poll", SourceScope: []string{"poll:healthcheck"}} + web := Scope{Surface: SurfaceAuthedPage, Module: "web", SourceScope: []string{"*"}} + + plain, err := json.Marshal(ipc.DescribeImageReq{Data: []byte("x")}) + if err != nil { + t.Fatal(err) + } + noting, err := json.Marshal(ipc.DescribeImageReq{Data: []byte("x"), SaveNote: true}) + if err != nil { + t.Fatal(err) + } + if err := Can(ipc.MethodDescribeImage, poller, plain); err != nil { + t.Errorf("describing without saving must stay a read: %v", err) + } + if err := Can(ipc.MethodDescribeImage, poller, noting); !errors.Is(err, ErrForbidden) { + t.Errorf("save_note out of scope = %v, want ErrForbidden", err) + } + if err := Can(ipc.MethodDescribeImage, web, noting); err != nil { + t.Errorf("a module scoped to everything must still be allowed: %v", err) + } +} diff --git a/internal/auth/policy.go b/internal/auth/policy.go index bafe402..ede1e77 100644 --- a/internal/auth/policy.go +++ b/internal/auth/policy.go @@ -53,8 +53,68 @@ func Requirement(m ipc.Method) Authority { // asserted — never a module or the voice/chat path. maven can propose // (MethodProposeTool, no step-up: she has no passkey) but never en/disable. return AuthStepUp + case ipc.MethodSwapModel: + // Swapping the resident model changes what routes every utterance and + // what words every reply. It is the owner's call, from a surface that can + // carry a passkey gesture — the same rung as mutating the tool allowlist, + // and for the same reason: nothing Maven says or does may reach it. + // MethodModelStatus is only the read side, so it stays at AuthRead. + return AuthStepUp + case ipc.MethodCaptureStart, ipc.MethodCaptureAppend, ipc.MethodCaptureStop: + // Recording a meeting (Vikunja #253). AuthWrite, not AuthRead: it puts + // audio of other people on disk, which is a heavier thing than reading a + // fact, and it is not something a read-only surface should be able to + // begin. Append and Stop sit on the same rung as Start deliberately — + // a surface that may not start a recording has no business feeding or + // harvesting one either. + // + // Not AuthStepUp, and this is the interesting line: step-up needs a + // passkey gesture, which the voice path cannot make. Putting it here + // would mean "запиши встречу" could never work by voice, and the real + // gate on this capability is elsewhere and stronger — the methods do not + // exist at all unless the operator enabled a capture block, and no + // recording can begin without someone saying so. + return AuthWrite + case ipc.MethodEnrollSpeaker: + // Taking a voiceprint (Vikunja #255). AuthStepUp, and unlike recording a + // meeting there is no reason to soften it: enrolment is not a thing anyone + // does by voice mid-conversation. It is a deliberate sit-down with a + // surface that can carry a passkey gesture, and it writes a biometric of a + // named person. If the gesture is inconvenient, that is the correct amount + // of friction for this particular write. + return AuthStepUp + case ipc.MethodForgetSpeaker: + // Deleting a voiceprint. One rung BELOW enrolment on purpose. Everywhere + // else in this table the destructive direction is gated at least as hard + // as the constructive one, and here that would be wrong: getting rid of a + // biometric must never be the harder half. The worst a caller at this rung + // can do is make Maven stop recognising someone, which is the state the + // box ships in anyway. + return AuthWrite case ipc.MethodWriteFact: return AuthWrite + case ipc.MethodIngestMail: + // Mail ingestion (Vikunja #246). Moved up from AuthRead on 2026-08-01, + // for consistency with SetTaskStatus rather than for a new threat: both + // answer the same question — may this module change what is on his + // lists? — and they were answering it differently. The old argument was + // that ingestion is additive and can only produce candidate tasks, which + // is still true; it is the weaker half of the argument, because a + // compromised mail reader that can fill the review page indefinitely is + // not a read. + // + // No caller loses anything: AuthWrite outside WriteFact only requires + // enrollment, which mavmaild already has, and the method does not exist + // unless the operator wired a mail block. + return AuthWrite + case ipc.MethodSetTaskStatus: + // Resolving a task is NOT additive, which is what separates it from + // capture. Capture at AuthRead can only put a line on a list he reads + // himself; SetTaskStatus at AuthRead would let any enrolled module — + // mavpoll, mavsttd — mark every open task done and clear the list out + // from under him. Same reasoning as WriteFact: a module gets to add to + // its own corner, not to erase his. + return AuthWrite case ipc.MethodAssertStepUp: return AuthRead case ipc.MethodLatestFact, @@ -65,7 +125,41 @@ func Requirement(m ipc.Method) Authority { ipc.MethodCreateReminder, ipc.MethodMarkReminder, ipc.MethodRecordNudge, - ipc.MethodResolveNudge: + ipc.MethodResolveNudge, + // Task capture (Vikunja #130). Listed explicitly rather than left to + // the default so the intent is on the record: capturing a task is a + // module write, not an allowlist mutation and not a new standing reason + // for Maven to speak — nothing in the tick loop reads tasks. It stays + // at AuthRead, the same rung as CreateReminder, which is the closest + // existing analogue. SetTaskStatus is NOT here: see the AuthWrite case + // above, because resolving is the one task move that destroys + // something. + ipc.MethodCaptureTask, + ipc.MethodListTasks, + // Looking at one image (Vikunja #252). AuthRead because of what it can + // produce: words about a picture, and optionally a note. It cannot write + // a fact, set a reminder, or touch the tool allowlist. The invasive part + // of this capability is not the authority rung — it is that the bytes are + // kept on disk, which media.retention bounds, and that they never leave + // the box, which internal/vision enforces by refusing a non-private + // endpoint. + ipc.MethodDescribeImage, + // "что ты записываешь?" — the read side of the recorder. It reports a + // label, a start time and a byte count, begins nothing and keeps nothing. + ipc.MethodCaptureStatus, + // Who is enrolled. Returns ids, names and enrolment dates — never the + // voiceprints themselves, which stay in core. Listing the people Maven can + // recognise is exactly the read a surface needs to offer a "forget" button. + ipc.MethodListSpeakers, + // The read side of the model swap: which model is resident, which ones are + // allowlisted. It loads nothing and changes nothing. + ipc.MethodModelStatus, + // The unified intake journal (Vikunja #283). AuthRead, and listed + // explicitly rather than inherited so the reasoning is on the record: it + // reports what already arrived — sources, keys, note headlines — which is + // the same material RecentFacts and RecentNotes already return at this + // rung. It writes nothing, and it holds nothing a fact read does not. + ipc.MethodRecentEvents: return AuthRead } // Unknown method ⇒ AuthRead, but ipc.dispatch returns ErrUnknownMethod @@ -102,6 +196,18 @@ func Can(m ipc.Method, scope Scope, params json.RawMessage) error { switch Requirement(m) { case AuthRead: + // Describing an image is a read. Saving the description as a note is + // not: writeNote embeds it, so it comes back in a later turn as + // something Maven knows, under the source media:image:, which no + // enrollment owns. The rung's own argument was that the method "cannot + // write a fact, set a reminder, or touch the tool allowlist" — it can + // write recall corpus, and that is what AuthWrite exists to scope. So + // the note half is held to the same source-scope rule WriteFact is. + if m == ipc.MethodDescribeImage && wantsNote(params) { + if !SourceAllowed(scope.SourceScope, ImageNoteSource) { + return fmt.Errorf("%w: source %q out of scope", ErrForbidden, ImageNoteSource) + } + } // Any enrolled module may read. Reads through the surface level the // Enrollment set (voice-L0 wouldn't be enrolled to write at all). return nil @@ -138,6 +244,26 @@ func Can(m ipc.Method, scope Scope, params json.RawMessage) error { return nil } +// ImageNoteSource is the source scope a caller needs to turn a described image +// into a note. The note itself is stored under "media:image:"; the +// scope is checked against this stem, because the id is not known until the +// bytes arrive and no enrollment could name it in advance. +const ImageNoteSource = "media:image" + +// wantsNote reports whether a DescribeImage call asked for the description to +// be remembered. Malformed params read as no: dispatch rejects them a moment +// later with a better error. +func wantsNote(raw json.RawMessage) bool { + if len(raw) == 0 { + return false + } + var p ipc.DescribeImageReq + if json.Unmarshal(raw, &p) != nil { + return false + } + return p.SaveNote +} + // SourceAllowed — true iff src is in scope (the wildcard "*" matches all). // Empty scope ⇒ fail closed. The function is pure; we keep it exported so a // future enrollment table can call into the same matching logic. diff --git a/internal/calendar/ambient.go b/internal/calendar/ambient.go new file mode 100644 index 0000000..6721b0d --- /dev/null +++ b/internal/calendar/ambient.go @@ -0,0 +1,277 @@ +package calendar + +import ( + "strings" + "time" + "unicode" +) + +// Ambient events — the work calendar read (Vikunja #126). +// +// The work calendar is not read by holding a work credential. A corp mail or +// calendar session living on the homelab ties the box's blast radius to the +// employer's data, which is the thing the task exists to refuse. What maven +// reads instead is the SIGNAL: an Android notification-listener on the owner's +// phone relays meeting notifications over wg/LAN, and maven turns the ones that +// clearly describe a meeting into calendar events. +// +// That makes the provenance honest. A notification is evidence about an event, +// not a reading of the calendar, so it is stored under SourceAmbient at +// AmbientConfidence — never indistinguishable from a real CalDAV read, and the +// query path hedges when it recites one. +// +// The parse is deliberately conservative. A notification with no recognisable +// clock reading produces nothing at all: maven is not a guesser-of-truth, and a +// mailbox full of noise turned into invented events is worse than a gap. Mail +// as a notification signal, not a mailbox. + +// Notification — one relayed Android notification. Package is the posting app +// (for the log and for the owner to see where a wrong event came from), Title +// and Text are the notification's two text lines, Posted is when the phone +// showed it. Nothing else off the notification is kept. +type Notification struct { + Package string `json:"package"` + Title string `json:"title"` + Text string `json:"text"` + Posted time.Time `json:"posted_at"` +} + +// ambientPastGrace — how far before the notification a derived start may sit +// before the event is refused. +// +// The date is not in the clock reading, so it is inferred, and the inference is +// only safe while the event is still roughly now. A 21:00 reminder reading +// "Tomorrow at 09:00" would otherwise land at 09:00 TODAY, twelve hours in the +// past, and FactKey would file that wrong meeting under today's date. Storing a +// wrong meeting is the one outcome this file exists to avoid, so anything this +// stale is dropped instead. The grace covers the ordinary case of a phone +// reposting a notification for a meeting already under way. +const ambientPastGrace = 2 * time.Hour + +// dayWords maps the words that move a notification off Posted's day. Only +// explicit ones: an offset is a claim about which day, and guessing which day +// is exactly the guess this parse refuses to make. +var dayWords = map[string]int{ + "завтра": 1, + "tomorrow": 1, + "сегодня": 0, + "today": 0, + "tonight": 0, + "послезавтра": 2, +} + +// EventFromNotification turns a notification into the event it describes, or +// reports false when it does not clearly describe one. +// +// It needs two things: a clock reading, and a summary that is not just that +// clock reading. The date comes from Posted's day, shifted by an explicit day +// word ("завтра", "tomorrow") when the notification carries one, and the result +// is refused if it lands more than ambientPastGrace in the past. A bare start +// time gets DefaultReminderDuration. +func EventFromNotification(n Notification) (Event, bool) { + if n.Posted.IsZero() { + return Event{}, false + } + line := strings.TrimSpace(n.Title + " " + n.Text) + start, end, ok := parseTimeRange(line) + if !ok { + return Event{}, false + } + summary := notificationSummary(n) + if summary == "" { + return Event{}, false + } + + y, m, d := n.Posted.AddDate(0, 0, dayOffset(line)).Date() + loc := n.Posted.Location() + s := time.Date(y, m, d, start.hour, start.min, 0, 0, loc) + // Too far in the past to be the meeting this notification is about. The day + // was inferred, so the honest reading is that the inference was wrong. + if s.Before(n.Posted.Add(-ambientPastGrace)) { + return Event{}, false + } + var e time.Time + if end != nil { + e = time.Date(y, m, d, end.hour, end.min, 0, 0, loc) + // A range that ends before it starts crossed midnight. + if !e.After(s) { + e = e.AddDate(0, 0, 1) + } + } else { + e = s.Add(DefaultReminderDuration) + } + return Event{Summary: summary, Start: s, End: e}, true +} + +// dayOffset reports how many days off Posted's day the notification puts the +// event. Words are matched whole, so "послезавтра" is not read as "завтра". +func dayOffset(line string) int { + for _, f := range strings.Fields(strings.ToLower(line)) { + f = strings.Trim(f, ".,;:!?—–-()\"'«»") + if off, ok := dayWords[f]; ok { + return off + } + } + return 0 +} + +// stripDayWords removes the day word from a summary candidate. It named the +// date, which now lives in Start, and leaving it in makes "Завтра Планёрка" +// the name of the meeting. +func stripDayWords(s string) string { + out := make([]string, 0, 8) + for _, f := range strings.Fields(s) { + if _, ok := dayWords[strings.Trim(strings.ToLower(f), ".,;:!?—–-()\"'«»")]; ok { + continue + } + out = append(out, f) + } + return strings.Join(out, " ") +} + +// notificationSummary picks the text that names the meeting: the title when it +// carries words, otherwise the body. The clock reading is stripped out — it +// already lives in the times, and FactValue renders it again. +func notificationSummary(n Notification) string { + for _, cand := range []string{n.Title, n.Text} { + s := strings.TrimSpace(stripDayWords(stripClock(cand))) + s = strings.Trim(s, " \t-–—,;:@|·") + s = strings.Join(strings.Fields(s), " ") + if hasLetters(s) { + return s + } + } + return "" +} + +type clock struct{ hour, min int } + +// parseTimeRange finds the first clock reading in s, and a second one if the +// text spells a range. Accepted separators between hours and minutes are ":" +// and "."; between the two ends of a range, "-", "–", "—" or "до". +// +// Bare hours ("в 14") are NOT accepted. Loose digits in a notification are far +// more often a count, a date or an unread badge than a meeting time, and an +// invented event is worse than no event. +func parseTimeRange(s string) (start clock, end *clock, ok bool) { + first, _, firstEnd, ok := nextClock(s, 0) + if !ok { + return clock{}, nil, false + } + sep := strings.TrimLeft(s[firstEnd:], " \t") + for _, p := range []string{"-", "–", "—", "до "} { + if !strings.HasPrefix(sep, p) { + continue + } + if second, _, _, ok2 := nextClock(strings.TrimPrefix(sep, p), 0); ok2 { + return first, &second, true + } + break + } + return first, nil, true +} + +// nextClock scans s from byte offset `from` for the first HH:MM (or HH.MM) and +// returns it with the byte range it occupied. Digits and separators are ASCII, +// so byte offsets are safe over Cyrillic text. +func nextClock(s string, from int) (c clock, start, end int, ok bool) { + for i := from; i < len(s); i++ { + if !isDigit(s[i]) { + continue + } + j := i + for j < len(s) && isDigit(s[j]) { + j++ + } + // A run longer than two digits is a year, an id or an unread count. + if j-i > 2 { + i = j + continue + } + if j >= len(s) || (s[j] != ':' && s[j] != '.') { + i = j + continue + } + k := j + 1 + for k < len(s) && isDigit(s[k]) { + k++ + } + if k-(j+1) != 2 { + i = j + continue + } + // Reject a group that is a link in a longer dotted or colon chain: + // "2026.08.15" would otherwise offer "08.15" as 08:15, and a deadline + // date invented as a meeting time is exactly the wrong kind of guess. + // A trailing ":ss" is fine — that is a time with seconds. + if i > 0 && (s[i-1] == '.' || s[i-1] == ':' || isDigit(s[i-1])) { + i = k + continue + } + if k < len(s) && s[k] == '.' && k+1 < len(s) && isDigit(s[k+1]) { + i = k + continue + } + hour, min := atoi(s[i:j]), atoi(s[j+1:k]) + if hour > 23 || min > 59 { + i = k + continue + } + return clock{hour, min}, i, k, true + } + return clock{}, 0, 0, false +} + +func isDigit(b byte) bool { return b >= '0' && b <= '9' } + +func atoi(s string) int { + n := 0 + for i := 0; i < len(s); i++ { + n = n*10 + int(s[i]-'0') + } + return n +} + +// stripClock removes every clock reading from a summary candidate, along with +// the preposition or separator that introduced it. +func stripClock(s string) string { + for { + _, start, end, ok := nextClock(s, 0) + if !ok { + return s + } + head := trimTrailingPreposition(strings.TrimRight(s[:start], "0123456789:.-–— \t")) + s = strings.TrimSpace(strings.TrimSpace(head) + " " + strings.TrimSpace(s[end:])) + } +} + +// trimTrailingPreposition drops the word that introduced a clock reading, so +// "Встреча в 14:00" becomes "Встреча" and "с 11:30 до 12:15 Созвон" does not +// keep a dangling "с". It repeats, because a range has two of them. +func trimTrailingPreposition(s string) string { + preps := []string{"в", "с", "до", "от", "at", "from", "to"} + for again := true; again; { + again = false + s = strings.TrimRight(s, " \t") + for _, p := range preps { + if s == p { + return "" + } + if strings.HasSuffix(s, " "+p) { + s = s[:len(s)-len(p)-1] + again = true + break + } + } + } + return s +} + +func hasLetters(s string) bool { + for _, r := range s { + if unicode.IsLetter(r) { + return true + } + } + return false +} diff --git a/internal/calendar/ambient_test.go b/internal/calendar/ambient_test.go new file mode 100644 index 0000000..5484ac2 --- /dev/null +++ b/internal/calendar/ambient_test.go @@ -0,0 +1,239 @@ +package calendar + +import ( + "testing" + "time" +) + +func TestEventFromNotification(t *testing.T) { + posted := time.Date(2026, 8, 3, 9, 40, 0, 0, time.FixedZone("+04", 4*3600)) + + tests := []struct { + name string + title, text string + wantOK bool + wantSummary string + wantStart string // "15:04" + wantEnd string + }{ + { + name: "range in the body", + title: "Планёрка", + text: "10:00-10:30", + wantOK: true, + wantSummary: "Планёрка", + wantStart: "10:00", wantEnd: "10:30", + }, + { + name: "russian preposition and single time", + title: "Встреча с подрядчиком в 14:00", + wantOK: true, + wantSummary: "Встреча с подрядчиком", + wantStart: "14:00", wantEnd: "14:30", + }, + { + name: "en dash range", + title: "Sprint review", + text: "Today 16:00 – 17:00, Meet", + wantOK: true, + wantSummary: "Sprint review", + wantStart: "16:00", wantEnd: "17:00", + }, + { + name: "до as a range separator", + title: "Созвон", + text: "с 11:30 до 12:15", + wantOK: true, + wantSummary: "Созвон", + wantStart: "11:30", wantEnd: "12:15", + }, + { + name: "dotted clock", + title: "Обед 13.00", + wantOK: true, + wantSummary: "Обед", + wantStart: "13:00", wantEnd: "13:30", + }, + { + name: "range crossing midnight", + title: "Ночной релиз", + text: "23:30-00:30", + wantOK: true, + wantSummary: "Ночной релиз", + wantStart: "23:30", wantEnd: "00:30", + }, + // The conservative half: no clock reading, no event. + {name: "no time at all", title: "3 новых письма", wantOK: false}, + {name: "bare hour is not a time", title: "Планёрка в 14", wantOK: false}, + {name: "unread count", title: "Входящие", text: "12 непрочитанных", wantOK: false}, + {name: "a date is not a clock", title: "Отчёт", text: "срок 2026.08.15", wantOK: false}, + {name: "time but nothing named", title: "10:00-10:30", wantOK: false}, + {name: "impossible clock", title: "Смена 99:99", wantOK: false}, + {name: "empty", wantOK: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ev, ok := EventFromNotification(Notification{ + Package: "com.google.android.gm", + Title: tt.title, + Text: tt.text, + Posted: posted, + }) + if ok != tt.wantOK { + t.Fatalf("ok = %v, want %v (event %+v)", ok, tt.wantOK, ev) + } + if !ok { + return + } + if ev.Summary != tt.wantSummary { + t.Errorf("summary = %q, want %q", ev.Summary, tt.wantSummary) + } + if got := ev.Start.Format("15:04"); got != tt.wantStart { + t.Errorf("start = %s, want %s", got, tt.wantStart) + } + if got := ev.End.Format("15:04"); got != tt.wantEnd { + t.Errorf("end = %s, want %s", got, tt.wantEnd) + } + if !ev.End.After(ev.Start) { + t.Errorf("end %v must be after start %v", ev.End, ev.Start) + } + // The event lands on the day the phone showed it, in the phone's + // location — not shifted into UTC. + if ev.Start.Location() != posted.Location() { + t.Errorf("location = %v, want %v", ev.Start.Location(), posted.Location()) + } + if y, m, d := ev.Start.Date(); y != 2026 || m != time.August || d != 3 { + t.Errorf("date = %d-%02d-%02d, want 2026-08-03", y, m, d) + } + }) + } +} + +// A notification is not always about today. A 21:00 reminder reading +// "Tomorrow at 09:00" used to be dated to the notification's own day, which put +// the meeting twelve hours in the past and filed it under today in FactKey. A +// wrong meeting stored is worse than nothing stored. +func TestEventFromNotificationDayWords(t *testing.T) { + loc := time.FixedZone("+04", 4*3600) + evening := time.Date(2026, 8, 3, 21, 0, 0, 0, loc) + + tests := []struct { + name string + title, text string + posted time.Time + wantOK bool + wantDay int // day of month + wantSummary string + }{ + { + name: "tomorrow in english", title: "Standup", text: "Tomorrow at 09:00", + posted: evening, wantOK: true, wantDay: 4, wantSummary: "Standup", + }, + { + name: "завтра in russian", title: "Планёрка", text: "завтра в 09:00", + posted: evening, wantOK: true, wantDay: 4, wantSummary: "Планёрка", + }, + { + name: "завтра in the title, summary in the body", title: "Завтра в 09:00", text: "Планёрка", + posted: evening, wantOK: true, wantDay: 4, wantSummary: "Планёрка", + }, + { + name: "послезавтра is two days, not one", title: "Ретро", text: "послезавтра 11:00", + posted: evening, wantOK: true, wantDay: 5, wantSummary: "Ретро", + }, + { + name: "сегодня stays on the posted day", title: "Созвон", text: "сегодня 21:30", + posted: evening, wantOK: true, wantDay: 3, wantSummary: "Созвон", + }, + // No day word: the 09:00 is twelve hours behind the notification, so the + // inferred day is wrong and there is nothing honest to store. + { + name: "stale morning time with no day word", title: "Standup", text: "at 09:00", + posted: evening, wantOK: false, + }, + // Inside the grace: a phone reposting the notification for a meeting + // already under way must still store it. + { + name: "meeting already running", title: "Планёрка", text: "20:30-22:00", + posted: evening, wantOK: true, wantDay: 3, wantSummary: "Планёрка", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ev, ok := EventFromNotification(Notification{ + Package: "com.google.android.calendar", + Title: tt.title, Text: tt.text, Posted: tt.posted, + }) + if ok != tt.wantOK { + t.Fatalf("ok = %v, want %v (event %+v)", ok, tt.wantOK, ev) + } + if !ok { + return + } + if got := ev.Start.Day(); got != tt.wantDay { + t.Errorf("start day = %d, want %d (start %v)", got, tt.wantDay, ev.Start) + } + if ev.Summary != tt.wantSummary { + t.Errorf("summary = %q, want %q", ev.Summary, tt.wantSummary) + } + if ev.Start.Before(tt.posted.Add(-ambientPastGrace)) { + t.Errorf("start %v is stale against posted %v", ev.Start, tt.posted) + } + }) + } +} + +// The day word named the date, which now lives in Start. Leaving it in the +// summary makes "Завтра Планёрка" the name of the meeting, and FactKey folds +// that into the key. +func TestEventFromNotificationDropsDayWordFromSummary(t *testing.T) { + ev, ok := EventFromNotification(Notification{ + Title: "Завтра Планёрка 09:00", + Posted: time.Date(2026, 8, 3, 21, 0, 0, 0, time.UTC), + }) + if !ok { + t.Fatal("expected an event") + } + if ev.Summary != "Планёрка" { + t.Fatalf("summary = %q, want %q", ev.Summary, "Планёрка") + } +} + +func TestEventFromNotificationNeedsPostedAt(t *testing.T) { + if _, ok := EventFromNotification(Notification{Title: "Планёрка 10:00"}); ok { + t.Error("a notification with no posted_at has no date to sit on") + } +} + +// An ambient event must never be indistinguishable from a calendar read. +func TestAmbientEventsAreStoredAtReducedConfidence(t *testing.T) { + ev, ok := EventFromNotification(Notification{ + Title: "Планёрка 10:00-10:30", + Posted: time.Date(2026, 8, 3, 9, 0, 0, 0, time.UTC), + }) + if !ok { + t.Fatal("expected an event") + } + if FactKey(ev) == "" || FactValue(ev) == "" { + t.Fatal("ambient events must use the shared fact encoding") + } + if AmbientConfidence >= 1.0 { + t.Fatal("ambient confidence must be below a calendar read's") + } +} + +func TestStripClock(t *testing.T) { + tests := []struct{ in, want string }{ + {"Встреча в 14:00", "Встреча"}, + {"Планёрка 10:00-10:30", "Планёрка"}, + {"с 11:30 до 12:15 Созвон", "Созвон"}, + {"Ничего", "Ничего"}, + } + for _, tt := range tests { + if got := stripClock(tt.in); got != tt.want { + t.Errorf("stripClock(%q) = %q, want %q", tt.in, got, tt.want) + } + } +} diff --git a/internal/calendar/calendar.go b/internal/calendar/calendar.go new file mode 100644 index 0000000..829a9e0 --- /dev/null +++ b/internal/calendar/calendar.go @@ -0,0 +1,170 @@ +// Package calendar is the one calendar data model the rest of maven shares: +// an Event, the iCal text it is parsed from and rendered to, and the fact +// encoding that puts it in the store. +// +// It exists because three separate features read or write the same events and +// must agree on their shape: the CalDAV read side (cmd/mavcaldav, Vikunja +// #126/#127), the write-only render target that publishes maven's own +// reminders as a calendar (#127), and the day plan that recites them (#128). +// Before this package the parse lived inline in cmd/mavcaldav and the fact key +// format was a Sprintf in two places. +// +// The package is pure: no HTTP, no store, no clock of its own. Callers own the +// impurity, the way internal/morning and internal/loop do. +package calendar + +import ( + "fmt" + "sort" + "strings" + "time" +) + +// Fact sources. A calendar event reaches the store as a +// `facts (kind=env, key=calendar_event_..., source=)` row, and +// the source is the whole provenance story: +// +// - SourcePersonal — maven's own Radicale, read AND rendered to. Canonical +// state stays in sqlite; the calendar is a render target (#127). +// - SourceWork — a work calendar, read-only by definition (#126). Nothing in +// maven ever writes to it: no code path pairs this source with a PUT. +// - SourceAmbient — inferred from an Android notification-listener relay +// rather than read from a server (#126). Confidence is below 1.0 because a +// notification is a signal about an event, not the event. +const ( + SourcePersonal = "poll:caldav" + SourceWork = "poll:caldav:work" + SourceAmbient = "ambient:notif" +) + +// AmbientConfidence — the confidence a notification-derived event is stored +// with. A parsed notification line is evidence, not a reading of the calendar, +// so it must never be indistinguishable from one (#126). +const AmbientConfidence = 0.6 + +// Sources lists every source a calendar event may legitimately carry, for the +// store query that reads the calendar back out. Ordered from most to least +// trusted. +func Sources() []string { + return []string{SourcePersonal, SourceWork, SourceAmbient} +} + +// ReadOnlySource reports whether events from this source may never be written +// back. The work calendar is read-only by definition — see #126: maven holding +// a credential that can write to an employer's calendar is the thing the task +// exists to avoid. +func ReadOnlySource(source string) bool { + return source == SourceWork || source == SourceAmbient +} + +// Event — one calendar entry. UID is the iCal UID when the event was parsed +// from a server and the identity maven renders under when it publishes one; +// Start/End are instants. All-day events are not modelled: the busy gate and +// the day plan both need a time of day, and an all-day marker answers neither. +type Event struct { + UID string + Summary string + Start time.Time + End time.Time +} + +// FactKey is the store key for an event: one key per day per summary, stable +// across polls so re-reading an unchanged calendar rewrites nothing. +// +// The date prefix is load-bearing — store.CalendarEvents selects a day range +// by key prefix, not by a timestamp column — and it is the OWNER's day, taken +// on the box clock. An event carries the zone its server stated it in, so +// keying off the event's own location would file a 21:00 Moscow meeting under +// a different date than the day plan asks for. +func FactKey(e Event) string { return FactKeyIn(e, time.Local) } + +// FactKeyIn is FactKey against an explicit location. +func FactKeyIn(e Event, loc *time.Location) string { + return fmt.Sprintf("calendar_event_%s_%s", e.Start.In(loc).Format("20060102"), safeKey(e.Summary)) +} + +// FactValue is the human-readable rendering stored as the fact value, and the +// string the day plan and the query path read back. Times are the owner's wall +// clock, for the same reason the key date is. +func FactValue(e Event) string { return FactValueIn(e, time.Local) } + +// FactValueIn is FactValue against an explicit location. +func FactValueIn(e Event, loc *time.Location) string { + return fmt.Sprintf("%s @ %s-%s", e.Summary, e.Start.In(loc).Format("15:04"), e.End.In(loc).Format("15:04")) +} + +// FactSummary strips the "@ HH:MM-HH:MM" tail FactValue appends, for a caller +// that prints the time itself. The day plan does: without this it renders +// "14:00 — Standup @ 14:00-14:30" and says the hour twice. +func FactSummary(value string) string { + i := strings.LastIndex(value, " @ ") + if i < 0 { + return value + } + tail := value[i+len(" @ "):] + if len(tail) != len("15:04-15:04") { + return value + } + for j, r := range tail { + switch j { + case 2, 8: + if r != ':' { + return value + } + case 5: + if r != '-' { + return value + } + default: + if r < '0' || r > '9' { + return value + } + } + } + return value[:i] +} + +// KeyPrefixForDay is the fact-key prefix covering one calendar day. The store +// range-scans between two of these. +func KeyPrefixForDay(day time.Time) string { + return fmt.Sprintf("calendar_event_%s", day.Format("20060102")) +} + +// Busy reports whether any event covers the instant now — the read the loop +// gate uses to suppress nudges during a meeting. +func Busy(events []Event, now time.Time) bool { + for _, e := range events { + if !now.Before(e.Start) && now.Before(e.End) { + return true + } + } + return false +} + +// Overlapping returns the events intersecting [from, to), sorted by start. +func Overlapping(events []Event, from, to time.Time) []Event { + var out []Event + for _, e := range events { + if e.End.After(from) && e.Start.Before(to) { + out = append(out, e) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].Start.Before(out[j].Start) }) + return out +} + +// safeKey makes a summary safe to use inside a fact key (ASCII alphanumerics +// and dashes). Non-Latin summaries collapse to their punctuation, which is why +// the day prefix carries the identity and this only disambiguates within a day. +func safeKey(s string) string { + var b strings.Builder + for _, r := range s { + switch { + case (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '-': + b.WriteRune(r) + case r == ' ' || r == '_': + b.WriteRune('-') + } + } + return b.String() +} diff --git a/internal/calendar/calendar_test.go b/internal/calendar/calendar_test.go new file mode 100644 index 0000000..a7ffdbb --- /dev/null +++ b/internal/calendar/calendar_test.go @@ -0,0 +1,265 @@ +package calendar + +import ( + "strings" + "testing" + "time" +) + +func TestParseICalDayKeepsOnlyToday(t *testing.T) { + now := time.Date(2026, 7, 3, 12, 0, 0, 0, time.UTC) + + body := []byte(`BEGIN:VCALENDAR +BEGIN:VEVENT +UID:a@example +DTSTART:20260703T090000Z +DTEND:20260703T100000Z +SUMMARY:Morning standup +END:VEVENT +BEGIN:VEVENT +DTSTART:20260703T140000Z +DTEND:20260703T150000Z +SUMMARY:Team sync +END:VEVENT +BEGIN:VEVENT +DTSTART:20260702T140000Z +DTEND:20260702T150000Z +SUMMARY:Yesterday retro +END:VEVENT +BEGIN:VEVENT +DTSTART:20260704T090000Z +DTEND:20260704T100000Z +SUMMARY:Tomorrow standup +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20260704 +DTEND;VALUE=DATE:20260705 +SUMMARY:All-day event +END:VEVENT +END:VCALENDAR`) + + events := ParseICalDay(body, now) + if len(events) != 2 { + t.Fatalf("got %d events, want 2 (today only, no all-day/past/future)", len(events)) + } + if events[0].Summary != "Morning standup" || events[0].UID != "a@example" { + t.Errorf("events[0] = %+v", events[0]) + } + if !events[0].Start.Equal(time.Date(2026, 7, 3, 9, 0, 0, 0, time.UTC)) { + t.Errorf("events[0].Start = %v", events[0].Start) + } + if !events[0].End.Equal(time.Date(2026, 7, 3, 10, 0, 0, 0, time.UTC)) { + t.Errorf("events[0].End = %v", events[0].End) + } + if events[1].Summary != "Team sync" { + t.Errorf("events[1].Summary = %q", events[1].Summary) + } +} + +// Regression: "today" is the owner's day, in the owner's location. Taking the +// day number off a local clock but building the boundaries in UTC made the +// evening fall outside the window on any box east of Greenwich. +func TestParseICalDayUsesOwnersDay(t *testing.T) { + plus4 := time.FixedZone("+04", 4*60*60) + // 01:00 on Aug 1 local is 21:00 on Jul 31 UTC. + now := time.Date(2026, 8, 1, 1, 0, 0, 0, plus4) + body := []byte("BEGIN:VCALENDAR\nBEGIN:VEVENT\n" + + "DTSTART:20260731T195406Z\nDTEND:20260731T235406Z\nSUMMARY:Current meeting\n" + + "END:VEVENT\nEND:VCALENDAR") + + events := ParseICalDay(body, now) + if len(events) != 1 { + t.Fatalf("got %d events, want the in-progress one", len(events)) + } + if !Busy(events, now.UTC()) { + t.Error("an event in progress right now must read as busy") + } +} + +func TestParseVEVENT(t *testing.T) { + block := "DTSTART;TZID=Europe/Moscow:20260703T130000\nDTEND:20260703T140000Z\nSUMMARY:Stand up meeting" + e, ok := parseVEVENT(block, time.UTC) + if !ok { + t.Fatal("expected a parsed event") + } + // 13:00 Moscow is 10:00Z. Reading it as 13:00Z is the bug that put the + // event three hours late in the day plan. + if !e.Start.Equal(time.Date(2026, 7, 3, 10, 0, 0, 0, time.UTC)) { + t.Errorf("start = %v", e.Start) + } + if !e.End.Equal(time.Date(2026, 7, 3, 14, 0, 0, 0, time.UTC)) { + t.Errorf("end = %v", e.End) + } + if e.Summary != "Stand up meeting" { + t.Errorf("summary = %q", e.Summary) + } + + allDay := "DTSTART;VALUE=DATE:20260703\nDTEND;VALUE=DATE:20260704\nSUMMARY:All-day" + if _, ok := parseVEVENT(allDay, time.UTC); ok { + t.Error("all-day event should be rejected") + } +} + +func TestParseDT(t *testing.T) { + plus4 := time.FixedZone("+04", 4*60*60) + tests := []struct { + name string + line string + loc *time.Location + want time.Time + wantOK bool + }{ + {"UTC", "DTEND:20260703T100000Z", plus4, time.Date(2026, 7, 3, 10, 0, 0, 0, time.UTC), true}, + {"tzid", "DTSTART;TZID=Europe/Moscow:20260703T130000", plus4, time.Date(2026, 7, 3, 10, 0, 0, 0, time.UTC), true}, + {"tzid quoted", `DTSTART;TZID="Europe/Moscow":20260703T130000`, plus4, time.Date(2026, 7, 3, 10, 0, 0, 0, time.UTC), true}, + {"tzid with other params", "DTSTART;VALUE=DATE-TIME;TZID=Asia/Tokyo:20260703T130000", plus4, time.Date(2026, 7, 3, 4, 0, 0, 0, time.UTC), true}, + // An unloadable zone falls back to the reader's own clock, not to UTC. + {"unknown tzid", "DTSTART;TZID=Mars/Olympus:20260703T130000", plus4, time.Date(2026, 7, 3, 13, 0, 0, 0, plus4), true}, + // Floating: no Z, no TZID. Local to whoever reads it. + {"floating", "DTSTART:20260703T130000", plus4, time.Date(2026, 7, 3, 13, 0, 0, 0, plus4), true}, + {"all-day", "DTSTART;VALUE=DATE:20260703", plus4, time.Time{}, false}, + {"garbage", "DTSTART:garbage", plus4, time.Time{}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, ok := parseDT(tt.line, tt.loc) + if ok != tt.wantOK { + t.Errorf("ok = %v, want %v", ok, tt.wantOK) + } + if !got.Equal(tt.want) { + t.Errorf("got %v, want %v", got, tt.want) + } + }) + } +} + +func TestSafeKey(t *testing.T) { + tests := []struct{ in, want string }{ + {"Stand up meeting", "Stand-up-meeting"}, + {"Hello_World", "Hello-World"}, + {"special@#$chars!!", "specialchars"}, + {"ALL_CAPS_123", "ALL-CAPS-123"}, + } + for _, tt := range tests { + if got := safeKey(tt.in); got != tt.want { + t.Errorf("safeKey(%q) = %q, want %q", tt.in, got, tt.want) + } + } +} + +func TestFactKeyAndValue(t *testing.T) { + e := Event{ + Summary: "Team sync", + Start: time.Date(2026, 7, 3, 14, 0, 0, 0, time.UTC), + End: time.Date(2026, 7, 3, 15, 0, 0, 0, time.UTC), + } + if got, want := FactKeyIn(e, time.UTC), "calendar_event_20260703_Team-sync"; got != want { + t.Errorf("FactKey = %q, want %q", got, want) + } + if got, want := FactValueIn(e, time.UTC), "Team sync @ 14:00-15:00"; got != want { + t.Errorf("FactValue = %q, want %q", got, want) + } + if got, want := KeyPrefixForDay(e.Start), "calendar_event_20260703"; got != want { + t.Errorf("KeyPrefixForDay = %q, want %q", got, want) + } + if !strings.HasPrefix(FactKeyIn(e, time.UTC), KeyPrefixForDay(e.Start)) { + t.Error("FactKey must start with the day prefix the store range-scans on") + } +} + +// The key date and the printed time are the owner's, not the calendar +// server's. A 23:00 Moscow event read on a +04 box belongs to the next local +// day, and filing it under the Moscow day would hide it from the day plan the +// store range-scans for. +func TestFactKeyAndValueUseTheOwnersClock(t *testing.T) { + msk := time.FixedZone("MSK", 3*60*60) + plus4 := time.FixedZone("+04", 4*60*60) + e := Event{ + Summary: "Late sync", + Start: time.Date(2026, 7, 3, 23, 30, 0, 0, msk), + End: time.Date(2026, 7, 4, 0, 30, 0, 0, msk), + } + if got, want := FactKeyIn(e, plus4), "calendar_event_20260704_Late-sync"; got != want { + t.Errorf("FactKeyIn = %q, want %q", got, want) + } + if got, want := FactValueIn(e, plus4), "Late sync @ 00:30-01:30"; got != want { + t.Errorf("FactValueIn = %q, want %q", got, want) + } +} + +func TestFactSummaryDropsTheTimeTail(t *testing.T) { + if got, want := FactSummary("Standup @ 14:00-14:30"), "Standup"; got != want { + t.Errorf("FactSummary = %q, want %q", got, want) + } + // Nothing that is not the exact tail FactValue writes is touched. + for _, in := range []string{"Coffee @ home", "Standup", "Standup @ 14:00-14:3", "Standup @ 1a:00-14:30"} { + if got := FactSummary(in); got != in { + t.Errorf("FactSummary(%q) = %q, want it unchanged", in, got) + } + } +} + +// Regression for the mirror image of the window bug: the window is local, so +// the event must be a real instant too. A 22:00 event stated in the poller's +// own zone used to parse as 22:00Z, which on a +03 box is past the end of the +// local day, and the whole evening dropped out of both the busy gate and the +// day plan. +func TestParseICalDayKeepsTheEveningInAZonedCalendar(t *testing.T) { + plus3 := time.FixedZone("+03", 3*60*60) + now := time.Date(2026, 8, 1, 12, 0, 0, 0, plus3) + body := []byte("BEGIN:VCALENDAR\nBEGIN:VEVENT\n" + + "DTSTART;TZID=Europe/Moscow:20260801T220000\nDTEND;TZID=Europe/Moscow:20260801T230000\n" + + "SUMMARY:Evening call\nEND:VEVENT\nEND:VCALENDAR") + + events := ParseICalDay(body, now) + if len(events) != 1 { + t.Fatalf("got %d events, want the evening one", len(events)) + } + if got := events[0].Start.In(plus3).Format("15:04"); got != "22:00" { + t.Errorf("start reads %s locally, want 22:00", got) + } +} + +func TestBusyAndOverlapping(t *testing.T) { + base := time.Date(2026, 7, 3, 0, 0, 0, 0, time.UTC) + events := []Event{ + {Summary: "late", Start: base.Add(15 * time.Hour), End: base.Add(16 * time.Hour)}, + {Summary: "early", Start: base.Add(9 * time.Hour), End: base.Add(10 * time.Hour)}, + } + if !Busy(events, base.Add(9*time.Hour+30*time.Minute)) { + t.Error("should be busy inside the early event") + } + if Busy(events, base.Add(12*time.Hour)) { + t.Error("should be free at noon") + } + // Half-open: the end instant is free. + if Busy(events, base.Add(10*time.Hour)) { + t.Error("the end instant should not count as busy") + } + got := Overlapping(events, base.Add(8*time.Hour), base.Add(11*time.Hour)) + if len(got) != 1 || got[0].Summary != "early" { + t.Fatalf("Overlapping = %+v", got) + } + all := Overlapping(events, base, base.AddDate(0, 0, 1)) + if len(all) != 2 || all[0].Summary != "early" { + t.Fatalf("Overlapping must sort by start: %+v", all) + } +} + +func TestSourceTrust(t *testing.T) { + if ReadOnlySource(SourcePersonal) { + t.Error("the personal calendar is the one maven may render to") + } + if !ReadOnlySource(SourceWork) { + t.Error("the work calendar must be read-only") + } + if !ReadOnlySource(SourceAmbient) { + t.Error("an ambient notification is not a writable calendar") + } + if AmbientConfidence >= 1.0 { + t.Error("ambient events must be less trusted than a calendar read") + } + if len(Sources()) != 3 { + t.Errorf("Sources() = %v", Sources()) + } +} diff --git a/internal/calendar/ical.go b/internal/calendar/ical.go new file mode 100644 index 0000000..1aa40de --- /dev/null +++ b/internal/calendar/ical.go @@ -0,0 +1,188 @@ +package calendar + +import ( + "fmt" + "strings" + "time" + + // The TZID of a DTSTART names an IANA zone, and resolving it needs the zone + // database. The deploy image has no system tzdata, so embed it: without it + // every zoned event would silently fall back to the box's own offset, which + // is the bug this package had before. + _ "time/tzdata" +) + +// ParseICal scans iCal text for VEVENT components and returns the events +// overlapping [from, to). All-day events are skipped: parseDT reports no time +// for a VALUE=DATE value, and an event with no clock reading answers neither +// the busy gate nor the day plan. +// +// from's location is the fallback zone for a floating DTSTART — one with +// neither a Z suffix nor a TZID. RFC 5545 says a floating time is local to +// wherever it is read, and here that is the box the poller runs on. +func ParseICal(body []byte, from, to time.Time) []Event { + var events []Event + text := string(body) + for { + i := strings.Index(text, "BEGIN:VEVENT") + if i < 0 { + break + } + text = text[i+len("BEGIN:VEVENT"):] + j := strings.Index(text, "END:VEVENT") + if j < 0 { + break + } + block := text[:j] + text = text[j+len("END:VEVENT"):] + + e, ok := parseVEVENT(block, from.Location()) + if !ok { + continue + } + if e.End.After(from) && e.Start.Before(to) { + events = append(events, e) + } + } + return events +} + +// ParseICalDay is ParseICal over the calendar day containing now, in now's own +// location — the window cmd/mavcaldav polls. +// +// The location matters, on both sides of the comparison. An older version took +// the day number off a local clock reading but built the boundaries in UTC, so +// east of Greenwich the window was shifted by the offset and part of the +// evening fell outside "today". Building the window locally is only half the +// fix: parseDT used to stamp a zoned DTSTART as UTC, which lost the mirror +// image of the same evening. Both sides are real instants now. +func ParseICalDay(body []byte, now time.Time) []Event { + y, m, d := now.Date() + start := time.Date(y, m, d, 0, 0, 0, 0, now.Location()) + return ParseICal(body, start, start.AddDate(0, 0, 1)) +} + +// parseVEVENT extracts UID, start, end and summary from a VEVENT block. +// Reports false for all-day events and parse failures. +func parseVEVENT(block string, loc *time.Location) (Event, bool) { + var e Event + for _, line := range strings.Split(block, "\n") { + line = strings.TrimSpace(line) + switch { + case strings.HasPrefix(line, "DTSTART"): + if t, ok := parseDT(line, loc); ok { + e.Start = t + } + case strings.HasPrefix(line, "DTEND"): + if t, ok := parseDT(line, loc); ok { + e.End = t + } + case strings.HasPrefix(line, "SUMMARY"): + e.Summary = afterColon(line) + case strings.HasPrefix(line, "UID"): + e.UID = afterColon(line) + } + } + if e.Start.IsZero() || e.End.IsZero() { + return Event{}, false + } + return e, true +} + +func afterColon(line string) string { + if i := strings.Index(line, ":"); i >= 0 { + return strings.TrimSpace(line[i+1:]) + } + return "" +} + +// parseDT parses a DTSTART/DTEND value into a real instant: +// +// - UTC: DTEND:20260703T100000Z +// - Zoned: DTSTART;TZID=Europe/Moscow:20260703T130000 +// - Floating: DTSTART:20260703T130000 (read in loc) +// - All-day: DTSTART;VALUE=DATE:20260703 (rejected) +// +// A zoned value is resolved against its own TZID, not stamped as UTC. The old +// behaviour was "the server and the poller share a timezone, and the busy gate +// only needs busy/not-busy to be right", and that stopped being enough when the +// day plan started reciting the wall clock: a 13:00 Moscow meeting read as +// 13:00Z was recited at 17:00 on a +04 box, and a 21:00 one fell out of the day +// altogether. An unknown or unloadable TZID falls back to loc, which is the +// closest thing to the reader's own wall clock we have. +func parseDT(line string, loc *time.Location) (time.Time, bool) { + if strings.Contains(line, "VALUE=DATE:") { + return time.Time{}, false + } + i := strings.LastIndex(line, ":") + if i < 0 { + return time.Time{}, false + } + if loc == nil { + loc = time.UTC + } + raw := strings.TrimSpace(line[i+1:]) + if strings.HasSuffix(raw, "Z") { + t, err := time.ParseInLocation("20060102T150405", strings.TrimSuffix(raw, "Z"), time.UTC) + if err != nil { + return time.Time{}, false + } + return t, true + } + if tz := tzidOf(line[:i]); tz != "" { + if l, err := time.LoadLocation(tz); err == nil { + loc = l + } + } + t, err := time.ParseInLocation("20060102T150405", raw, loc) + if err != nil { + return time.Time{}, false + } + return t, true +} + +// tzidOf pulls the TZID out of a property's parameter list ("DTSTART;TZID=..." +// up to the value colon). The value may be quoted, per RFC 5545 param syntax. +func tzidOf(params string) string { + for _, p := range strings.Split(params, ";")[1:] { + if !strings.HasPrefix(strings.ToUpper(p), "TZID=") { + continue + } + return strings.Trim(strings.TrimSpace(p[len("TZID="):]), `"`) + } + return "" +} + +// RenderICal wraps events in a VCALENDAR body suitable for PUTting to a CalDAV +// collection. One event per file is the CalDAV convention, so callers normally +// pass a single event. +// +// This is the write half of #127 and it only ever renders: the canonical state +// is sqlite, the calendar is a view of it. Nothing reads a rendered file back. +func RenderICal(events []Event) string { + var b strings.Builder + b.WriteString("BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//maven//local calendar//RU\r\n") + for _, e := range events { + b.WriteString("BEGIN:VEVENT\r\n") + fmt.Fprintf(&b, "UID:%s\r\n", escapeText(e.UID)) + fmt.Fprintf(&b, "DTSTAMP:%s\r\n", e.Start.UTC().Format("20060102T150405Z")) + fmt.Fprintf(&b, "DTSTART:%s\r\n", e.Start.UTC().Format("20060102T150405Z")) + fmt.Fprintf(&b, "DTEND:%s\r\n", e.End.UTC().Format("20060102T150405Z")) + fmt.Fprintf(&b, "SUMMARY:%s\r\n", escapeText(e.Summary)) + b.WriteString("END:VEVENT\r\n") + } + b.WriteString("END:VCALENDAR\r\n") + return b.String() +} + +// escapeText applies RFC 5545 TEXT escaping and strips the line breaks that +// would otherwise let a reminder payload inject iCal properties. +func escapeText(s string) string { + s = strings.ReplaceAll(s, "\\", "\\\\") + s = strings.ReplaceAll(s, ";", "\\;") + s = strings.ReplaceAll(s, ",", "\\,") + s = strings.ReplaceAll(s, "\r\n", "\\n") + s = strings.ReplaceAll(s, "\n", "\\n") + s = strings.ReplaceAll(s, "\r", "\\n") + return s +} diff --git a/internal/calendar/ical_render_test.go b/internal/calendar/ical_render_test.go new file mode 100644 index 0000000..6ca89de --- /dev/null +++ b/internal/calendar/ical_render_test.go @@ -0,0 +1,69 @@ +package calendar + +import ( + "strings" + "testing" + "time" +) + +func TestRenderICalRoundTrips(t *testing.T) { + e := ReminderEvent(7, time.Date(2026, 8, 1, 18, 30, 0, 0, time.UTC), "позвонить маме", 0) + if e.UID != "maven-reminder-7" { + t.Errorf("UID = %q", e.UID) + } + if got := e.End.Sub(e.Start); got != DefaultReminderDuration { + t.Errorf("duration = %v, want %v", got, DefaultReminderDuration) + } + if got, want := ReminderPath(7), "maven-reminder-7.ics"; got != want { + t.Errorf("ReminderPath = %q, want %q", got, want) + } + + body := RenderICal([]Event{e}) + if !strings.HasPrefix(body, "BEGIN:VCALENDAR\r\n") || !strings.HasSuffix(body, "END:VCALENDAR\r\n") { + t.Fatalf("not a VCALENDAR body:\n%s", body) + } + + back := ParseICal([]byte(body), e.Start.Add(-time.Hour), e.Start.Add(time.Hour)) + if len(back) != 1 { + t.Fatalf("got %d events back, want 1:\n%s", len(back), body) + } + if back[0].UID != e.UID || back[0].Summary != e.Summary { + t.Errorf("round trip lost identity: %+v", back[0]) + } + if !back[0].Start.Equal(e.Start) || !back[0].End.Equal(e.End) { + t.Errorf("round trip lost times: %+v", back[0]) + } +} + +func TestRenderICalIsDeterministic(t *testing.T) { + e := ReminderEvent(1, time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC), "выпить воды", 0) + if RenderICal([]Event{e}) != RenderICal([]Event{e}) { + t.Error("the same reminder must render byte-identically, or every poll re-PUTs it") + } +} + +// A reminder payload is owner-supplied text. It must not be able to close the +// VEVENT and inject properties of its own. +func TestRenderICalEscapesInjection(t *testing.T) { + e := ReminderEvent(2, time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC), + "обед\r\nEND:VEVENT\r\nBEGIN:VEVENT\r\nSUMMARY:injected", 0) + body := RenderICal([]Event{e}) + // Count line-initial occurrences: the escaped text still contains the + // characters "BEGIN:VEVENT", it just can no longer start a line. + if n := strings.Count(body, "\r\nBEGIN:VEVENT\r\n"); n != 1 { + t.Fatalf("payload injected a second VEVENT (%d):\n%s", n, body) + } + if n := strings.Count(body, "\r\nEND:VEVENT\r\n"); n != 1 { + t.Fatalf("payload closed the VEVENT early (%d):\n%s", n, body) + } + if !strings.Contains(body, `SUMMARY:обед\nEND:VEVENT`) { + t.Errorf("newlines should be escaped, not dropped:\n%s", body) + } +} + +func TestReminderEventEmptyPayload(t *testing.T) { + e := ReminderEvent(3, time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC), " ", 0) + if e.Summary != "напоминание" { + t.Errorf("Summary = %q, want the neutral RU fallback", e.Summary) + } +} diff --git a/internal/calendar/reminder.go b/internal/calendar/reminder.go new file mode 100644 index 0000000..0847c54 --- /dev/null +++ b/internal/calendar/reminder.go @@ -0,0 +1,69 @@ +package calendar + +import ( + "fmt" + "strconv" + "strings" + "time" +) + +// ReminderUIDPrefix namespaces every event maven publishes. Two reasons it is a +// fixed prefix and not a random UUID: the render is idempotent (the same +// reminder always lands on the same UID, so re-rendering overwrites instead of +// duplicating), and everything maven owns in the target collection is +// identifiable at a glance — she never touches a file she did not create. +const ReminderUIDPrefix = "maven-reminder-" + +// DefaultReminderDuration — how long a rendered reminder occupies. A reminder +// is an instant, a calendar entry is a span, so one has to be invented; 30 +// minutes reads as a block in a calendar app without swallowing the afternoon. +const DefaultReminderDuration = 30 * time.Minute + +// ReminderEvent maps a reminder to the event that represents it. id and fire +// come from the store; payload is the RU text as the owner said it, rendered +// verbatim as the summary — the calendar is a view of sqlite, not a place to +// rephrase. +func ReminderEvent(id int64, fire time.Time, payload string, dur time.Duration) Event { + if dur <= 0 { + dur = DefaultReminderDuration + } + summary := strings.TrimSpace(payload) + if summary == "" { + summary = "напоминание" + } + return Event{ + UID: fmt.Sprintf("%s%d", ReminderUIDPrefix, id), + Summary: summary, + Start: fire, + End: fire.Add(dur), + } +} + +// ReminderPath is the collection-relative filename for a rendered reminder. +// One event per resource, per the CalDAV convention. +func ReminderPath(id int64) string { + return fmt.Sprintf("%s%d.ics", ReminderUIDPrefix, id) +} + +// ReminderIDFromPath reads back what ReminderPath wrote, given an href out of a +// PROPFIND. It reports false for anything that is not a resource maven +// published, which is what keeps a reconciliation pass from touching a file it +// did not create. +func ReminderIDFromPath(href string) (int64, bool) { + name := href + if i := strings.LastIndex(name, "/"); i >= 0 { + name = name[i+1:] + } + if !strings.HasPrefix(name, ReminderUIDPrefix) || !strings.HasSuffix(name, ".ics") { + return 0, false + } + digits := name[len(ReminderUIDPrefix) : len(name)-len(".ics")] + if digits == "" { + return 0, false + } + id, err := strconv.ParseInt(digits, 10, 64) + if err != nil || id <= 0 { + return 0, false + } + return id, true +} diff --git a/internal/capture/capture.go b/internal/capture/capture.go new file mode 100644 index 0000000..9f25979 --- /dev/null +++ b/internal/capture/capture.go @@ -0,0 +1,619 @@ +// Package capture is Maven's meeting recorder (Vikunja #253, +// docs/plans/08-hearing.md). +// +// One session at a time, with an explicit start and an explicit stop: +// +// Start("встреча") → audio frames appended → Stop() → transcript → summary +// +// # Nothing here listens +// +// This is the most invasive capability in the backlog and the design is +// constrained accordingly. The constraints are the code, not a preamble: +// +// - There is no ambient path. `Session.Append` is the only way audio enters, +// and it only accepts frames while a session someone started is running. +// A keyword-triggered recorder ("maven record" heard in the room) was in the +// plan document and is refused: it requires listening in order to notice the +// keyword, which is the exact behaviour this capability must not have. +// - A session that is not stopped stops itself. MaxDuration is a hard cap +// checked on every Append AND against the wall clock in Start and Status, +// so a client that simply stops sending frames — a browser tab closed, wifi +// gone — does not leave the one session slot occupied until mavend +// restarts. +// - A session belongs to whoever started it. Start returns a token and Append +// and Stop require it, so a second surface at the same authority rung +// cannot feed or harvest a recording it did not begin. +// - Audio is stored under internal/media, which means retention prunes it and +// it never leaves the box. Both the audio blob and the transcript stay +// local; only the summary is written where he will read it. +// - The transcript is never search input for anything outside this box. It is +// text about a conversation with other people in it. +// +// # Long audio against a 4096-token context +// +// The resident model is a Thinking variant at n_ctx 4096, so an hour of meeting +// transcript does not fit in one prompt and never will. summarize.go does the +// obvious map-reduce: split the transcript on sentence boundaries into windows +// that fit, summarise each, then summarise the summaries. That is handled +// explicitly rather than by truncation, because a truncated meeting summary is +// worse than none — it looks complete and is not. +// +// # Transcription +// +// There is exactly one STT in Maven and this package does not add a second: it +// takes an stt.Transcriber, which in deploy is the whisper.cpp worker behind +// cmd/mavsttd. Long audio is transcribed in windows too (see transcribeFile), for +// the same reason whisper itself works in 30s windows — handing a worker an hour +// of PCM in one call is a request that either times out or blocks everything +// else for minutes. The windows are read back off the stored WAV one at a time, +// so the meeting is never in memory whole. +package capture + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "os" + "strings" + "sync" + "time" + + "github.com/kami/maven/internal/audio" + "github.com/kami/maven/internal/media" + "github.com/kami/maven/internal/stt" +) + +// DefaultMaxDuration — how long one capture may run before it stops itself. +// Two hours covers a long meeting and bounds the damage of a forgotten session: +// at 16 kHz mono that is about 230 MB of WAV, which is under media's +// DefaultMaxAudioBytes of 512 MiB. The two constants used to disagree — a +// 64 MiB blob cap is 35 minutes of audio against a 120 minute session cap — so +// the meeting that hit the limit was the one that failed to store. +const DefaultMaxDuration = 2 * time.Hour + +// StaleGrace — how long past MaxDuration a session may sit before Start and +// Status reap it. A frame in flight when the cap fires should not race the +// reaper, and a minute of slack costs nothing against a two-hour cap. +const StaleGrace = time.Minute + +// DefaultSTTWindow — how much audio goes to the transcriber in one call. Five +// minutes of 16 kHz mono is under 10 MB, transcribes in well under whisper's +// own timeout on this box, and keeps the worker responsive to the voice path +// between windows. +const DefaultSTTWindow = 5 * time.Minute + +// Errors callers distinguish. +var ( + // ErrDisabled — capture is not configured. A capability is off unless + // configured, and a recorder most of all. + ErrDisabled = errors.New("capture: not configured") + // ErrBusy — a session is already running. One at a time: two concurrent + // recordings would make "хватит" ambiguous. + ErrBusy = errors.New("capture: a session is already running") + // ErrNoSession — stop or append with nothing running. + ErrNoSession = errors.New("capture: nothing is being recorded") + // ErrBadFormat — a frame is not the canonical 16 kHz mono PCM shape. + ErrBadFormat = errors.New("capture: audio format not supported") + // ErrEmptyCapture — the session ended with no audio in it. + ErrEmptyCapture = errors.New("capture: nothing was recorded") + // ErrExpired — the session hit MaxDuration and was closed. Returned from + // Append so the caller stops sending; the audio collected so far is kept. + ErrExpired = errors.New("capture: session reached its time limit") + // ErrWrongSession — the token does not match the running session. The + // recording belongs to the surface that started it. + ErrWrongSession = errors.New("capture: that is not your session") +) + +// Session — one recording in progress. Not created directly; Recorder.Start +// makes it. Guarded by a mutex because frames arrive from a network goroutine +// while a status call may read from another. +type Session struct { + Label string + Started time.Time + // Token identifies this session to its owner. Append and Stop need it: the + // rung Append sits on is shared by every writing module, and a rung is not + // an owner. Without it any AuthWrite surface could call capture_stop on a + // meeting it did not start and be handed the verbatim transcript. + Token string + + mu sync.Mutex + spool *os.File // the WAV being written, header first + path string + n int64 // PCM bytes written, header excluded + format audio.Format + expired bool + closed bool +} + +// write appends one frame to the spool file. +func (s *Session) write(b []byte) error { + if s.spool == nil { + return errors.New("capture: session has no spool file") + } + n, err := s.spool.Write(b) + s.n += int64(n) + if err != nil { + return fmt.Errorf("capture: spool write: %w", err) + } + return nil +} + +// finish closes the spool file and stamps the real WAV header over the +// placeholder Start wrote. +func (s *Session) finish() error { + if s.closed { + return nil + } + s.closed = true + if s.spool == nil { + return nil + } + defer s.spool.Close() + hdr, err := audio.WAVHeader(s.format, int(s.n)) + if err != nil { + return err + } + if _, err := s.spool.WriteAt(hdr, 0); err != nil { + return fmt.Errorf("capture: spool header: %w", err) + } + return s.spool.Sync() +} + +// Duration is how much audio has been collected, from the bytes rather than the +// wall clock: a stream that dropped frames should report the audio that exists, +// not the time that passed. +func (s *Session) Duration() time.Duration { + s.mu.Lock() + defer s.mu.Unlock() + return s.duration() +} + +func (s *Session) duration() time.Duration { + return pcmDuration(s.format, s.n) +} + +// pcmDuration is how long n bytes of PCM lasts in the given format. +func pcmDuration(f audio.Format, n int64) time.Duration { + per := int64(f.SampleRate) * int64(f.Channels) * int64(f.SampleBits) / 8 + if per <= 0 { + return 0 + } + return time.Duration(float64(n) / float64(per) * float64(time.Second)) +} + +// Bytes is how much PCM has been collected. For a status line. +func (s *Session) Bytes() int { + s.mu.Lock() + defer s.mu.Unlock() + return int(s.n) +} + +// Status — what a "что записываешь?" answer needs, and what /dash shows. It is +// the read side of a running session and is safe to ask for at any time. +type Status struct { + Running bool `json:"running"` + Label string `json:"label,omitempty"` + Started time.Time `json:"started,omitempty"` + Duration time.Duration `json:"duration,omitempty"` + Bytes int `json:"bytes,omitempty"` +} + +// Recorder owns the single session slot, the blob store and the two models a +// finished capture needs. Build it with New; a zero Recorder is not usable. +type Recorder struct { + blobs *media.Store + tr stt.Transcriber + sum *Summarizer + maxDuration time.Duration + sttWindow time.Duration + staleGrace time.Duration + now func() time.Time + + mu sync.Mutex + current *Session +} + +// Config — the recorder's knobs, built from config.CaptureConfig by the daemon. +type Config struct { + // MaxDuration — hard cap on one session. 0 ⇒ DefaultMaxDuration. + MaxDuration time.Duration + // STTWindow — audio per transcription call. 0 ⇒ DefaultSTTWindow. + STTWindow time.Duration +} + +// New builds a Recorder. blobs and tr are required — a recorder with nowhere to +// put the audio, or nothing to transcribe it with, is not a recorder. sum may be +// nil: the transcript is still produced and stored, and the summary is simply +// absent, which is the honest degradation when there is no llama-server. +func New(blobs *media.Store, tr stt.Transcriber, sum *Summarizer, cfg Config) (*Recorder, error) { + if blobs == nil { + return nil, errors.New("capture: no blob store") + } + if tr == nil { + return nil, errors.New("capture: no transcriber") + } + maxDur := cfg.MaxDuration + if maxDur <= 0 { + maxDur = DefaultMaxDuration + } + window := cfg.STTWindow + if window <= 0 { + window = DefaultSTTWindow + } + return &Recorder{ + blobs: blobs, + tr: tr, + sum: sum, + maxDuration: maxDur, + sttWindow: window, + staleGrace: StaleGrace, + now: time.Now, + }, nil +} + +// MaxDuration is the configured hard cap. For the reply that tells him how long +// she will keep going if he forgets to say "хватит". +func (r *Recorder) MaxDuration() time.Duration { return r.maxDuration } + +// Start opens a session. label is what the meeting is called ("встреча с +// подрядчиком"); it ends up in the summary note so the note is findable. +// ErrBusy if one is already running — the caller says so rather than silently +// discarding the first recording. +func (r *Recorder) Start(label string) (*Session, error) { + r.mu.Lock() + defer r.mu.Unlock() + r.reapLocked() + if r.current != nil { + return nil, fmt.Errorf("%w: %q since %s", ErrBusy, r.current.Label, + r.current.Started.Format(time.Kitchen)) + } + f, err := r.blobs.SpoolFile("capture") + if err != nil { + return nil, err + } + format := audio.PCM16kMono + hdr, err := audio.WAVHeader(format, 0) + if err != nil { + f.Close() + return nil, err + } + // The header is written first and rewritten at Stop with the real length, + // so the spool file is a playable WAV rather than headerless PCM that has + // to be copied to gain 44 bytes. + if _, err := f.Write(hdr); err != nil { + f.Close() + _ = os.Remove(f.Name()) + return nil, fmt.Errorf("capture: spool header: %w", err) + } + token, err := newToken() + if err != nil { + f.Close() + _ = os.Remove(f.Name()) + return nil, err + } + s := &Session{ + Label: strings.TrimSpace(label), + Started: r.now().UTC(), + Token: token, + spool: f, + path: f.Name(), + format: format, + } + r.current = s + return s, nil +} + +// newToken mints a session token. Sixteen random bytes: it is a capability +// handed back over the same socket the call came in on, not a secret at rest. +func newToken() (string, error) { + var b [16]byte + if _, err := rand.Read(b[:]); err != nil { + return "", fmt.Errorf("capture: token: %w", err) + } + return hex.EncodeToString(b[:]), nil +} + +// reapLocked drops a session whose wall clock ran past MaxDuration. The +// frame-driven check in Append only fires while frames arrive, so a client that +// simply stopped sending — a phone whose browser tab was closed, wifi gone — +// left the slot occupied and every later Start answering ErrBusy with a meeting +// from last Tuesday. r.mu must be held. +func (r *Recorder) reapLocked() { + s := r.current + if s == nil { + return + } + if r.now().UTC().Sub(s.Started) < r.maxDuration+r.staleGrace { + return + } + s.mu.Lock() + s.expired = true + _ = s.finish() + path := s.path + s.mu.Unlock() + if path != "" { + // The audio goes with it. A recording nobody stopped is one nobody is + // waiting for, and keeping it would mean storing a meeting on the + // strength of a dropped connection. + _ = os.Remove(path) + } + r.current = nil +} + +// Append adds one frame to the running session. ErrNoSession when nothing is +// running, which is the guard that makes an ambient path impossible: a stream +// arriving at a Recorder nobody started is refused frame by frame. +// +// ErrExpired once the session is at MaxDuration. The audio collected so far is +// kept and Stop still works — the cap ends the recording, it does not throw it +// away. +func (r *Recorder) Append(token string, a audio.Audio) error { + if !a.Format.IsValid() { + return fmt.Errorf("%w: %+v", ErrBadFormat, a.Format) + } + r.mu.Lock() + r.reapLocked() + s := r.current + r.mu.Unlock() + if s == nil { + return ErrNoSession + } + if token != s.Token { + return ErrWrongSession + } + + s.mu.Lock() + defer s.mu.Unlock() + if s.expired { + return ErrExpired + } + // The session fixed its format at Start. A client that switches sample rate + // mid-session used to have its frames concatenated into the same buffer: + // duration() then read the whole thing at the original rate, the stored WAV + // header lied, and the cap fired at the wrong length. + if a.Format != s.format { + return fmt.Errorf("%w: session is %+v, frame is %+v", ErrBadFormat, s.format, a.Format) + } + if err := s.write(a.Bytes); err != nil { + return err + } + if s.duration() >= r.maxDuration { + s.expired = true + _ = s.finish() + return ErrExpired + } + return nil +} + +// Status reports the running session, or Running=false. +func (r *Recorder) Status() Status { + r.mu.Lock() + r.reapLocked() + s := r.current + r.mu.Unlock() + if s == nil { + return Status{} + } + return Status{ + Running: true, + Label: s.Label, + Started: s.Started, + Duration: s.Duration(), + Bytes: s.Bytes(), + } +} + +// Result — a finished capture. +type Result struct { + // BlobID — the stored audio, content-addressed. Empty only if storing failed. + BlobID string + // Label / Started / Duration — what was recorded and when. + Label string + Started time.Time + Duration time.Duration + // Transcript — the full text, joined across STT windows. + Transcript string + // Summary — the map-reduced summary, or empty when no summarizer was wired + // or the model failed. Empty summary with a non-empty transcript is a + // degraded success, not a failure: the words are there. + Summary string + // Chunks — how many windows the transcript was summarised in. 1 means it fit + // in one prompt. Reported so a suspiciously vague summary can be explained. + Chunks int + // StoreErr — why the audio was not kept, when it was not. The transcript is + // still produced in that case, so this is the difference between "no blob + // because storing failed" and "no blob because nothing was recorded". + StoreErr error +} + +// Stop ends the session and produces the result: store the audio, transcribe it +// in windows, summarise it in windows. The session slot is freed before any of +// the slow work starts, so a stuck model cannot block the next recording. +// +// The order matters and is the same as vision's: the audio is stored FIRST. If +// transcription fails, the recording is still on disk under media.retention, so +// the meeting is not lost to a model error. Note that re-running it is a manual +// job today: no method takes a blob id back, unlike vision's Rerun, and the blob +// prunes on the media retention like any other. +func (r *Recorder) Stop(ctx context.Context, token string) (Result, error) { + r.mu.Lock() + r.reapLocked() + s := r.current + if s != nil && token != s.Token { + r.mu.Unlock() + return Result{}, ErrWrongSession + } + r.current = nil + r.mu.Unlock() + if s == nil { + return Result{}, ErrNoSession + } + + s.mu.Lock() + err := s.finish() + path := s.path + format := s.format + n := s.n + s.mu.Unlock() + + res := Result{Label: s.Label, Started: s.Started} + if err != nil { + _ = os.Remove(path) + return res, err + } + if n == 0 { + _ = os.Remove(path) + return res, ErrEmptyCapture + } + res.Duration = pcmDuration(format, n) + + // The audio is stored first, as vision does, so a transcription or summary + // failure leaves something to run again. It moves rather than being read + // into memory: a two-hour meeting is a couple of hundred megabytes, and + // this is the process that owns the database and the resident model. + audioPath := path + blob, perr := r.blobs.PutFile(media.KindAudio, "audio/wav", "capture:meeting", path) + if perr == nil { + res.BlobID = blob.ID + audioPath = blob.Path + } else { + // Over the cap, or the store is full. Report it and KEEP GOING: this + // used to return, so the one case the audio cap actually fires on — a + // very long meeting — produced no transcript, no summary and no note, + // which is the whole point of the capability. The spool file stays + // until the transcript has been read off it. + res.StoreErr = perr + defer os.Remove(audioPath) + } + + text, terr := r.transcribeFile(ctx, audioPath, format, n) + res.Transcript = text + if terr != nil { + return res, fmt.Errorf("capture: transcribe: %w", terr) + } + if strings.TrimSpace(text) == "" { + return res, ErrEmptyCapture + } + if perr != nil { + return res, fmt.Errorf("capture: store audio: %w", perr) + } + return res, nil +} + +// Summarize runs the map-reduce over a transcript. It is separate from Stop so +// the daemon can answer the stop quickly and do the model work afterwards: a +// full map-reduce is up to forty model calls, and a voice turn that says +// "хватит" should not wait minutes for the reply. +// +// The salvaged text a failed reduce returns is assigned before the error is +// checked. Summarize hands back the per-chunk summaries with its error +// precisely so they are not lost, and the caller used to throw them away. +func (r *Recorder) Summarize(ctx context.Context, res *Result) error { + if r.sum == nil || strings.TrimSpace(res.Transcript) == "" { + return nil + } + summary, chunks, err := r.sum.Summarize(ctx, res.Label, res.Transcript) + res.Chunks = chunks + res.Summary = summary + if err != nil { + return fmt.Errorf("capture: summarize: %w", err) + } + return nil +} + +// Abort throws the running session away without transcribing or storing it. +// This is what "забудь, не записывай" must map to: a recording someone changed +// their mind about leaves nothing behind, not a blob with a note saying it was +// abandoned. Returns whether anything was running. +func (r *Recorder) Abort(token string) bool { + r.mu.Lock() + defer r.mu.Unlock() + r.reapLocked() + s := r.current + if s == nil || token != s.Token { + return false + } + r.current = nil + s.mu.Lock() + _ = s.finish() + path := s.path + s.mu.Unlock() + if path != "" { + _ = os.Remove(path) + } + return true +} + +// transcribeFile runs the transcriber over the stored WAV in windows and joins +// the text, reading one window at a time off disk so the meeting is never in +// memory whole. +// +// A window that fails is no longer fatal. It used to be, on the argument that a +// silent hole misleads — but the cost was 24 good windows thrown away for one +// whisper hiccup at minute 100. The hole is marked in the text instead, which +// keeps the words and stays honest about the gap. +func (r *Recorder) transcribeFile(ctx context.Context, path string, format audio.Format, n int64) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", fmt.Errorf("open audio: %w", err) + } + defer f.Close() + + per := windowBytes(format, r.sttWindow) + if per <= 0 || per > n { + per = n + } + total := int((n + per - 1) / per) + buf := make([]byte, per) + parts := make([]string, 0, total) + failed := 0 + for i, off := 0, int64(0); off < n; i, off = i+1, off+per { + size := per + if off+size > n { + size = n - off + } + // Never cut mid-sample: a split inside an int16 shifts every following + // sample by a byte and turns the tail of the window into noise. + if bps := int64(format.SampleBits / 8 * format.Channels); bps > 0 { + size -= size % bps + } + if size <= 0 { + break + } + if _, err := f.ReadAt(buf[:size], int64(audio.WAVHeaderSize)+off); err != nil { + return strings.Join(parts, " "), fmt.Errorf("window %d/%d: %w", i+1, total, err) + } + text, _, err := r.tr.Transcribe(ctx, audio.Audio{Format: format, Bytes: buf[:size]}) + if err != nil { + if ctx.Err() != nil { + return strings.Join(parts, " "), fmt.Errorf("window %d/%d: %w", i+1, total, err) + } + failed++ + parts = append(parts, gapMarker) + continue + } + if t := strings.TrimSpace(text); t != "" { + parts = append(parts, t) + } + } + if failed == total { + return "", fmt.Errorf("every one of %d window(s) failed", total) + } + return strings.Join(parts, " "), nil +} + +// gapMarker stands in for a window whisper could not read. Russian, because it +// is read by him in a note next to the words around it. +const gapMarker = "[…не разобрала…]" + +// windowBytes is how many PCM bytes one STT window holds. +func windowBytes(f audio.Format, window time.Duration) int64 { + bps := int64(f.SampleBits / 8 * f.Channels) + if bps <= 0 || f.SampleRate <= 0 || window <= 0 { + return 0 + } + per := int64(window.Seconds()) * int64(f.SampleRate) * bps + return per - per%bps +} diff --git a/internal/capture/capture_test.go b/internal/capture/capture_test.go new file mode 100644 index 0000000..998de21 --- /dev/null +++ b/internal/capture/capture_test.go @@ -0,0 +1,376 @@ +package capture + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/kami/maven/internal/audio" + "github.com/kami/maven/internal/media" +) + +// fakeTranscriber returns a fixed phrase per call so a windowed transcription is +// visible in the joined output. +type fakeTranscriber struct { + calls int + err error + phrase string +} + +func (f *fakeTranscriber) Transcribe(_ context.Context, a audio.Audio) (string, float64, error) { + f.calls++ + if f.err != nil { + return "", 0, f.err + } + p := f.phrase + if p == "" { + p = "окно" + } + return fmt.Sprintf("%s%d", p, f.calls), 1.0, nil +} + +// fakeCompleter records prompts and replies from a script. +type fakeCompleter struct { + replies []string + systems []string + users []string + err error +} + +func (f *fakeCompleter) Complete(_ context.Context, system, user string) (string, error) { + f.systems = append(f.systems, system) + f.users = append(f.users, user) + if f.err != nil { + return "", f.err + } + if len(f.replies) == 0 { + return "итог", nil + } + r := f.replies[0] + f.replies = f.replies[1:] + return r, nil +} + +// frame builds n seconds of silence in the canonical format. +func frame(seconds float64) audio.Audio { + n := int(seconds*16000) * 2 + return audio.Audio{Format: audio.PCM16kMono, Bytes: make([]byte, n)} +} + +func testRecorder(t *testing.T, tr *fakeTranscriber, sum *Summarizer, cfg Config) (*Recorder, *media.Store) { + t.Helper() + blobs, err := media.Open(t.TempDir(), 0, 0) + if err != nil { + t.Fatal(err) + } + r, err := New(blobs, tr, sum, cfg) + if err != nil { + t.Fatal(err) + } + return r, blobs +} + +func TestNewRequiresStoreAndTranscriber(t *testing.T) { + blobs, err := media.Open(t.TempDir(), 0, 0) + if err != nil { + t.Fatal(err) + } + if _, err := New(nil, &fakeTranscriber{}, nil, Config{}); err == nil { + t.Error("recorder built with no blob store") + } + if _, err := New(blobs, nil, nil, Config{}); err == nil { + t.Error("recorder built with no transcriber") + } +} + +// The invariant that matters most: audio arriving at a recorder nobody started +// is refused. There is no ambient path in. +func TestAppendWithoutStartIsRefused(t *testing.T) { + r, _ := testRecorder(t, &fakeTranscriber{}, nil, Config{}) + if err := r.Append("no-token", frame(1)); !errors.Is(err, ErrNoSession) { + t.Fatalf("got %v, want ErrNoSession", err) + } + if r.Status().Running { + t.Error("a refused frame started a session") + } +} + +func TestStopWithoutStartIsRefused(t *testing.T) { + r, _ := testRecorder(t, &fakeTranscriber{}, nil, Config{}) + if _, err := r.Stop(context.Background(), "no-token"); !errors.Is(err, ErrNoSession) { + t.Fatalf("got %v, want ErrNoSession", err) + } +} + +func TestOneSessionAtATime(t *testing.T) { + r, _ := testRecorder(t, &fakeTranscriber{}, nil, Config{}) + s, err := r.Start("встреча") + if err != nil { + t.Fatal(err) + } + if _, err := r.Start("вторая"); !errors.Is(err, ErrBusy) { + t.Fatalf("got %v, want ErrBusy", err) + } + if _, err := r.Stop(context.Background(), s.Token); !errors.Is(err, ErrEmptyCapture) { + t.Fatalf("empty stop: %v", err) + } + // The slot is free again after a stop, even a failed one. + if _, err := r.Start("третья"); err != nil { + t.Errorf("slot not released: %v", err) + } +} + +func TestRoundTripStoresAudioTranscriptAndSummary(t *testing.T) { + tr := &fakeTranscriber{phrase: "совещание"} + sum := NewSummarizer(&fakeCompleter{replies: []string{"— решили купить насос"}}, 0, 0, nil) + r, blobs := testRecorder(t, tr, sum, Config{}) + + s, err := r.Start("встреча с подрядчиком") + if err != nil { + t.Fatal(err) + } + for i := 0; i < 3; i++ { + if err := r.Append(s.Token, frame(2)); err != nil { + t.Fatal(err) + } + } + res, err := r.Stop(context.Background(), s.Token) + if err != nil { + t.Fatalf("stop: %v", err) + } + if err := r.Summarize(context.Background(), &res); err != nil { + t.Fatalf("summarize: %v", err) + } + if res.BlobID == "" { + t.Error("no audio blob stored") + } + blob, data, err := blobs.Read(res.BlobID) + if err != nil { + t.Fatalf("blob unreadable: %v", err) + } + if blob.Kind != media.KindAudio || blob.Source != "capture:meeting" { + t.Errorf("blob metadata = %+v", blob) + } + if string(data[:4]) != "RIFF" { + t.Error("audio was not stored as a playable WAV") + } + if res.Transcript == "" { + t.Error("no transcript") + } + if !strings.Contains(res.Summary, "насос") { + t.Errorf("summary = %q", res.Summary) + } + if !strings.Contains(res.Summary, "встреча с подрядчиком") { + t.Errorf("label missing from summary: %q", res.Summary) + } + if res.Duration != 6*time.Second { + t.Errorf("duration = %v, want 6s", res.Duration) + } +} + +// A forgotten session stops itself, and the audio collected before the cap is +// kept rather than thrown away. +func TestMaxDurationEndsTheSessionAndKeepsAudio(t *testing.T) { + tr := &fakeTranscriber{} + r, _ := testRecorder(t, tr, nil, Config{MaxDuration: 4 * time.Second}) + s, err := r.Start("длинная") + if err != nil { + t.Fatal(err) + } + if err := r.Append(s.Token, frame(3)); err != nil { + t.Fatalf("first frame: %v", err) + } + if err := r.Append(s.Token, frame(3)); !errors.Is(err, ErrExpired) { + t.Fatalf("got %v, want ErrExpired", err) + } + // Further frames keep being refused, so a client that ignores the error + // cannot grow the recording past the cap. + if err := r.Append(s.Token, frame(3)); !errors.Is(err, ErrExpired) { + t.Fatalf("post-expiry frame: %v", err) + } + res, err := r.Stop(context.Background(), s.Token) + if err != nil { + t.Fatalf("stop after expiry: %v", err) + } + if res.Duration != 6*time.Second { + t.Errorf("duration = %v, want the 6s collected before the cap", res.Duration) + } +} + +func TestAppendRejectsWrongFormat(t *testing.T) { + r, _ := testRecorder(t, &fakeTranscriber{}, nil, Config{}) + s, err := r.Start("x") + if err != nil { + t.Fatal(err) + } + bad := audio.Audio{Format: audio.Format{SampleRate: 44100, Channels: 2, SampleBits: 16, Encoding: "pcm_s16le"}, Bytes: make([]byte, 100)} + if err := r.Append(s.Token, bad); !errors.Is(err, ErrBadFormat) { + t.Fatalf("got %v, want ErrBadFormat", err) + } +} + +// "забудь, не записывай" must leave nothing behind — no blob, no transcript. +func TestAbortLeavesNothing(t *testing.T) { + tr := &fakeTranscriber{} + r, blobs := testRecorder(t, tr, nil, Config{}) + s, err := r.Start("зря начали") + if err != nil { + t.Fatal(err) + } + if err := r.Append(s.Token, frame(5)); err != nil { + t.Fatal(err) + } + if !r.Abort(s.Token) { + t.Fatal("Abort reported nothing running") + } + if r.Status().Running { + t.Error("session survived Abort") + } + list, err := blobs.List(media.KindAudio) + if err != nil { + t.Fatal(err) + } + if len(list) != 0 { + t.Errorf("Abort stored %d blob(s)", len(list)) + } + if tr.calls != 0 { + t.Errorf("Abort transcribed anyway (%d calls)", tr.calls) + } + if r.Abort(s.Token) { + t.Error("second Abort reported a session") + } +} + +func TestStatusReportsTheRunningSession(t *testing.T) { + r, _ := testRecorder(t, &fakeTranscriber{}, nil, Config{}) + if got := r.Status(); got.Running { + t.Error("idle recorder reports running") + } + s, err := r.Start("планёрка") + if err != nil { + t.Fatal(err) + } + if err := r.Append(s.Token, frame(10)); err != nil { + t.Fatal(err) + } + st := r.Status() + if !st.Running || st.Label != "планёрка" { + t.Fatalf("status = %+v", st) + } + if st.Duration != 10*time.Second { + t.Errorf("duration = %v", st.Duration) + } + if st.Bytes != 10*16000*2 { + t.Errorf("bytes = %d", st.Bytes) + } +} + +// Long audio goes to the transcriber in windows: handing a whisper worker an +// hour of PCM in one call blocks the voice path for minutes. +func TestLongAudioIsTranscribedInWindows(t *testing.T) { + tr := &fakeTranscriber{} + r, _ := testRecorder(t, tr, nil, Config{STTWindow: 2 * time.Second}) + s, err := r.Start("длинная") + if err != nil { + t.Fatal(err) + } + if err := r.Append(s.Token, frame(9)); err != nil { + t.Fatal(err) + } + res, err := r.Stop(context.Background(), s.Token) + if err != nil { + t.Fatalf("stop: %v", err) + } + if tr.calls != 5 { // 2+2+2+2+1 + t.Errorf("transcriber called %d times, want 5", tr.calls) + } + if !strings.Contains(res.Transcript, "окно5") { + t.Errorf("last window missing from transcript: %q", res.Transcript) + } +} + +// Every window failing is a transcription failure — but the audio is already +// stored and re-runnable. +func TestTranscriptionFailureKeepsTheAudio(t *testing.T) { + tr := &fakeTranscriber{err: errors.New("whisper is down")} + r, blobs := testRecorder(t, tr, nil, Config{}) + s, err := r.Start("встреча") + if err != nil { + t.Fatal(err) + } + if err := r.Append(s.Token, frame(2)); err != nil { + t.Fatal(err) + } + res, err := r.Stop(context.Background(), s.Token) + if err == nil { + t.Fatal("transcription failure was not reported") + } + if res.BlobID == "" { + t.Fatal("no blob id to retry with") + } + if _, _, err := blobs.Read(res.BlobID); err != nil { + t.Errorf("audio was not kept: %v", err) + } +} + +// No llama-server ⇒ transcript only. That is the honest degradation, not an +// error. +func TestNoSummarizerStillProducesATranscript(t *testing.T) { + r, _ := testRecorder(t, &fakeTranscriber{}, nil, Config{}) + s, err := r.Start("встреча") + if err != nil { + t.Fatal(err) + } + if err := r.Append(s.Token, frame(1)); err != nil { + t.Fatal(err) + } + res, err := r.Stop(context.Background(), s.Token) + if err != nil { + t.Fatalf("stop: %v", err) + } + if err := r.Summarize(context.Background(), &res); err != nil { + t.Fatalf("summarize with no summarizer: %v", err) + } + if res.Transcript == "" { + t.Error("no transcript") + } + if res.Summary != "" { + t.Errorf("summary appeared from nowhere: %q", res.Summary) + } +} + +// A summariser failure is a degraded success: the words exist and are returned. +func TestSummaryFailureStillReturnsTheTranscript(t *testing.T) { + sum := NewSummarizer(&fakeCompleter{err: errors.New("llama is down")}, 0, 0, nil) + r, _ := testRecorder(t, &fakeTranscriber{}, sum, Config{}) + s, err := r.Start("встреча") + if err != nil { + t.Fatal(err) + } + if err := r.Append(s.Token, frame(1)); err != nil { + t.Fatal(err) + } + res, err := r.Stop(context.Background(), s.Token) + if err != nil { + t.Fatalf("stop: %v", err) + } + if err := r.Summarize(context.Background(), &res); err == nil { + t.Fatal("summary failure was not reported") + } + if res.Transcript == "" { + t.Error("transcript lost to a summary failure") + } +} + +func TestWindowBytesNeverCutsMidSample(t *testing.T) { + if got := windowBytes(audio.PCM16kMono, 2*time.Second); got%2 != 0 || got != 2*16000*2 { + t.Fatalf("windowBytes = %d", got) + } + odd := audio.Format{SampleRate: 16000, Channels: 1, SampleBits: 16, Encoding: "pcm_s16le"} + if got := windowBytes(odd, 0); got != 0 { + t.Fatalf("a zero window must produce zero, got %d", got) + } +} diff --git a/internal/capture/session_test.go b/internal/capture/session_test.go new file mode 100644 index 0000000..de31658 --- /dev/null +++ b/internal/capture/session_test.go @@ -0,0 +1,154 @@ +package capture + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/kami/maven/internal/audio" + "github.com/kami/maven/internal/media" +) + +// A frame for a session that already ended must not land in the next one. The +// recorder used to be addressed as "whatever is running now", so a client whose +// session was reaped went on appending its microphone into a meeting somebody +// else had started. +func TestAppendWithTheWrongTokenIsRefused(t *testing.T) { + r, _ := testRecorder(t, &fakeTranscriber{}, nil, Config{}) + s, err := r.Start("первая") + if err != nil { + t.Fatal(err) + } + if err := r.Append("someone-elses-token", frame(1)); !errors.Is(err, ErrWrongSession) { + t.Fatalf("append = %v, want ErrWrongSession", err) + } + if _, err := r.Stop(context.Background(), "someone-elses-token"); !errors.Is(err, ErrWrongSession) { + t.Fatalf("stop = %v, want ErrWrongSession", err) + } + if r.Abort("someone-elses-token") { + t.Fatal("Abort discarded a session it does not own") + } + if err := r.Append(s.Token, frame(1)); err != nil { + t.Fatalf("the owner is still refused: %v", err) + } +} + +// A client that simply stops sending — a phone whose tab was closed — used to +// hold the single session slot forever, and every later Start answered ErrBusy +// with a meeting from last week. +func TestStaleSessionIsReapedByTheWallClock(t *testing.T) { + r, _ := testRecorder(t, &fakeTranscriber{}, nil, Config{MaxDuration: time.Minute}) + now := time.Now().UTC() + r.now = func() time.Time { return now } + s, err := r.Start("брошенная") + if err != nil { + t.Fatal(err) + } + if _, err := r.Start("вторая"); !errors.Is(err, ErrBusy) { + t.Fatalf("start = %v, want ErrBusy", err) + } + now = now.Add(time.Minute + StaleGrace + time.Second) + next, err := r.Start("вторая") + if err != nil { + t.Fatalf("a stale session was not reaped: %v", err) + } + if next.Token == s.Token { + t.Fatal("the new session reused the stale token") + } + if err := r.Append(s.Token, frame(1)); !errors.Is(err, ErrWrongSession) { + t.Fatalf("the reaped client can still write: %v", err) + } + // The abandoned recording is not kept: nobody is waiting for it, and storing + // it would mean keeping a meeting on the strength of a dropped connection. + if _, err := os.Stat(s.path); !os.IsNotExist(err) { + t.Fatalf("the reaped spool file survived: %v", err) + } +} + +// One window failing used to fail the whole transcription, which threw away +// every other window of an hour-long meeting. The hole is marked instead, so the +// summary cannot silently read as if nothing was missing. +func TestOneFailedWindowIsMarkedNotFatal(t *testing.T) { + r, _ := testRecorder(t, &fakeTranscriber{}, nil, Config{STTWindow: time.Second}) + r.tr = &windowTranscriber{failOn: 2} + s, err := r.Start("встреча") + if err != nil { + t.Fatal(err) + } + if err := r.Append(s.Token, frame(3)); err != nil { + t.Fatal(err) + } + res, err := r.Stop(context.Background(), s.Token) + if err != nil { + t.Fatalf("stop: %v", err) + } + if !strings.Contains(res.Transcript, gapMarker) { + t.Errorf("no gap marker in %q", res.Transcript) + } + if !strings.Contains(res.Transcript, "окно1") || !strings.Contains(res.Transcript, "окно3") { + t.Errorf("the surviving windows were dropped: %q", res.Transcript) + } +} + +// The audio not fitting the store is not a reason to lose the words. Stop used +// to return early on a store failure, so a recording over the blob cap produced +// neither a blob nor a transcript. +func TestStoreFailureStillTranscribes(t *testing.T) { + blobs, err := media.OpenWithBudget(t.TempDir(), 512, 1024, 0) + if err != nil { + t.Fatal(err) + } + tr := &fakeTranscriber{} + r, err := New(blobs, tr, nil, Config{}) + if err != nil { + t.Fatal(err) + } + s, err := r.Start("длинная встреча") + if err != nil { + t.Fatal(err) + } + if err := r.Append(s.Token, frame(2)); err != nil { + t.Fatal(err) + } + // The store failure is reported, but as a degraded success: the Result is + // filled in, and the caller keeps it rather than treating the error as + // nothing having happened. + res, err := r.Stop(context.Background(), s.Token) + if err == nil { + t.Fatal("the store failure was not reported") + } + if res.BlobID != "" { + t.Errorf("blob id = %q, want none", res.BlobID) + } + if !errors.Is(res.StoreErr, media.ErrTooLarge) { + t.Errorf("StoreErr = %v, want ErrTooLarge", res.StoreErr) + } + if res.Transcript == "" { + t.Fatal("the words were lost with the audio") + } + // The spool file is cleaned up even on the failure path. + glob, _ := filepath.Glob(filepath.Join(blobs.Dir(), "spool", "*")) + if len(glob) != 0 { + t.Errorf("spool leaked: %v", glob) + } +} + +// windowTranscriber answers per window and fails a chosen one, which is what a +// whisper timeout in the middle of a meeting looks like. +type windowTranscriber struct { + calls int + failOn int +} + +func (w *windowTranscriber) Transcribe(_ context.Context, _ audio.Audio) (string, float64, error) { + w.calls++ + if w.calls == w.failOn { + return "", 0, errors.New("whisper timed out") + } + return fmt.Sprintf("окно%d", w.calls), 1.0, nil +} diff --git a/internal/capture/summarize.go b/internal/capture/summarize.go new file mode 100644 index 0000000..d539472 --- /dev/null +++ b/internal/capture/summarize.go @@ -0,0 +1,261 @@ +package capture + +import ( + "context" + "errors" + "fmt" + "strings" + "unicode" +) + +// DefaultChunkRunes — how much transcript goes into one summarisation prompt. +// +// The resident model runs at n_ctx 4096 and is a Thinking variant, so reasoning +// tokens need room too. Russian runs roughly 2.5–3 characters per token on a +// Qwen tokenizer, so 3000 runes is about 1100 tokens of transcript, leaving the +// prompt, the persona block, the reasoning and the answer comfortable space. +// This is the same reasoning internal/crawl used to land on 4000 runes, tightened +// because a meeting transcript is denser in named entities than a web page and +// the reduce step has to fit several summaries at once. +const DefaultChunkRunes = 3000 + +// DefaultMaxChunks — how many windows one meeting may be summarised in. Forty +// chunks at 3000 runes is roughly a two-hour meeting, which is MaxDuration; past +// that the transcript is truncated and the summary says so, because forty-one +// sequential model calls on this box is half an hour of work nobody is waiting +// through. +const DefaultMaxChunks = 40 + +// ErrNoSummary — the model returned nothing usable for every chunk. +var ErrNoSummary = errors.New("capture: model produced no summary") + +// Completer is the one thing the summarizer needs from a model: text in, text +// out. It is an interface rather than an *llm.Client so this package stays pure +// and testable, and so the daemon can pass whatever it already has. +type Completer interface { + Complete(ctx context.Context, system, user string) (string, error) +} + +// Summarizer turns a transcript into something worth reading. It is map-reduce +// and nothing cleverer: summarise each window, then summarise the summaries. +// +// Truncation was the alternative and is rejected. A truncated meeting summary +// reads as complete and is not, which is worse than no summary at all — he would +// act on it. +type Summarizer struct { + llm Completer + chunkRunes int + maxChunks int + // context is the persona/context block the daemon prepends to every prompt, + // or empty. Passed in rather than built here so this package does not import + // internal/persona and the feminine self-reference rules stay in one place. + context func() string +} + +// NewSummarizer wires a summarizer. llm nil ⇒ nil Summarizer, which Recorder +// treats as "transcript only", the honest degradation with no llama-server. +// chunkRunes ≤ 0 ⇒ DefaultChunkRunes; maxChunks ≤ 0 ⇒ DefaultMaxChunks. +func NewSummarizer(llm Completer, chunkRunes, maxChunks int, contextBlock func() string) *Summarizer { + if llm == nil { + return nil + } + if chunkRunes <= 0 { + chunkRunes = DefaultChunkRunes + } + if maxChunks <= 0 { + maxChunks = DefaultMaxChunks + } + if contextBlock == nil { + contextBlock = func() string { return "" } + } + return &Summarizer{llm: llm, chunkRunes: chunkRunes, maxChunks: maxChunks, context: contextBlock} +} + +// chunkPrompt — the map step. Deliberately plain: this is not Maven speaking to +// him, it is a model condensing text, so there is no first person in it at all +// and therefore nothing for the persona's gender rules to get wrong. The reply +// she gives him afterwards is phrased by the ordinary replier, which does carry +// the persona. +const chunkPrompt = `Ты обрабатываешь фрагмент расшифровки разговора. +Сожми его до 2-4 пунктов: о чём говорили, какие решения приняли, какие задачи назвали. +Без вступлений и выводов. Только по тексту — не придумывай того, чего в нём нет. +Если во фрагменте нет ничего содержательного, ответь одним словом: пусто.` + +// reducePrompt — the reduce step. Same rules, over the chunk summaries. +const reducePrompt = `Ниже — конспекты фрагментов одной встречи, по порядку. +Собери из них один короткий итог: о чём была встреча, какие решения приняли, что кому делать. +Не повторяйся, не придумывай, не добавляй вступлений.` + +// emptyMarker — what the map step answers for a chunk with nothing in it. Such +// chunks are dropped before the reduce step rather than padding it with noise. +const emptyMarker = "пусто" + +// Summarize returns the summary and the number of chunks the transcript was +// split into. One chunk means it fit in a single prompt and the reduce step was +// skipped, which is the common case for a short meeting and saves a model call. +func (s *Summarizer) Summarize(ctx context.Context, label, transcript string) (string, int, error) { + if s == nil { + return "", 0, ErrDisabled + } + chunks := ChunkText(transcript, s.chunkRunes) + if len(chunks) == 0 { + return "", 0, ErrEmptyCapture + } + truncated := false + if len(chunks) > s.maxChunks { + chunks = chunks[:s.maxChunks] + truncated = true + } + + system := s.context() + chunkPrompt + parts := make([]string, 0, len(chunks)) + for i, c := range chunks { + out, err := s.llm.Complete(ctx, system, c) + if err != nil { + return "", len(chunks), fmt.Errorf("chunk %d/%d: %w", i+1, len(chunks), err) + } + out = strings.TrimSpace(out) + if out == "" || strings.EqualFold(out, emptyMarker) { + continue + } + parts = append(parts, out) + } + if len(parts) == 0 { + return "", len(chunks), ErrNoSummary + } + + summary := parts[0] + if len(parts) > 1 { + joined := strings.Join(parts, "\n\n") + reduced, err := s.llm.Complete(ctx, s.context()+reducePrompt, joined) + if err != nil { + // The per-chunk summaries are real work; hand them over rather than + // losing them to a failure in the last step. + return joined, len(chunks), fmt.Errorf("reduce: %w", err) + } + if r := strings.TrimSpace(reduced); r != "" { + summary = r + } else { + summary = joined + } + } + if label != "" { + summary = label + "\n\n" + summary + } + if truncated { + // Said in the note, not swallowed: a summary that silently covers the + // first hour of a three-hour meeting is the failure mode this guards. + summary += fmt.Sprintf("\n\n(расшифровка обрезана: обработано %d фрагментов из большего числа)", s.maxChunks) + } + return summary, len(chunks), nil +} + +// ChunkText splits text into windows of at most maxRunes runes, cutting on +// sentence boundaries where it can and on a word boundary otherwise. Exported +// because it is the part worth testing on its own and the part a future +// transcript viewer will want. +// +// A sentence longer than maxRunes (a transcript with no punctuation at all, +// which whisper does produce) is cut on whitespace rather than dropped or run +// past the limit. +func ChunkText(text string, maxRunes int) []string { + text = strings.TrimSpace(text) + if text == "" { + return nil + } + if maxRunes <= 0 { + maxRunes = DefaultChunkRunes + } + if len([]rune(text)) <= maxRunes { + return []string{text} + } + + var out []string + var cur []rune + flush := func() { + if s := strings.TrimSpace(string(cur)); s != "" { + out = append(out, s) + } + cur = cur[:0] + } + for _, sent := range splitSentences(text) { + sr := []rune(sent) + if len(sr) > maxRunes { + // Oversized sentence: emit what is buffered, then cut this one on + // word boundaries. + flush() + for _, piece := range splitWords(sr, maxRunes) { + out = append(out, piece) + } + continue + } + if len(cur)+len(sr) > maxRunes { + flush() + } + cur = append(cur, sr...) + } + flush() + return out +} + +// splitSentences cuts on sentence-ending punctuation followed by a space, +// keeping the punctuation with the sentence it ends. Good enough for a +// transcript: whisper emits periods and question marks, and being wrong about an +// abbreviation costs a slightly uneven chunk, nothing more. +func splitSentences(text string) []string { + runes := []rune(text) + var out []string + start := 0 + for i := 0; i < len(runes); i++ { + if runes[i] != '.' && runes[i] != '!' && runes[i] != '?' && runes[i] != '\n' { + continue + } + // Consume a run of punctuation ("?!", "...") so it stays together. + j := i + for j+1 < len(runes) && isSentenceEnd(runes[j+1]) { + j++ + } + if j+1 < len(runes) && !unicode.IsSpace(runes[j+1]) { + i = j + continue + } + end := j + 1 + for end < len(runes) && unicode.IsSpace(runes[end]) { + end++ + } + out = append(out, string(runes[start:end])) + start = end + i = end - 1 + } + if start < len(runes) { + out = append(out, string(runes[start:])) + } + return out +} + +func isSentenceEnd(r rune) bool { + return r == '.' || r == '!' || r == '?' +} + +// splitWords cuts an oversized run on whitespace, falling back to a hard cut +// when a single "word" is itself longer than the limit. +func splitWords(runes []rune, maxRunes int) []string { + var out []string + for len(runes) > maxRunes { + cut := maxRunes + for cut > 0 && !unicode.IsSpace(runes[cut]) { + cut-- + } + if cut == 0 { + cut = maxRunes + } + if s := strings.TrimSpace(string(runes[:cut])); s != "" { + out = append(out, s) + } + runes = runes[cut:] + } + if s := strings.TrimSpace(string(runes)); s != "" { + out = append(out, s) + } + return out +} diff --git a/internal/capture/summarize_test.go b/internal/capture/summarize_test.go new file mode 100644 index 0000000..e5e633a --- /dev/null +++ b/internal/capture/summarize_test.go @@ -0,0 +1,244 @@ +package capture + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" +) + +func TestNilSummarizerWithoutAModel(t *testing.T) { + if s := NewSummarizer(nil, 0, 0, nil); s != nil { + t.Fatal("a summarizer with no model is not nil") + } + var s *Summarizer + if _, _, err := s.Summarize(context.Background(), "x", "текст"); !errors.Is(err, ErrDisabled) { + t.Fatalf("got %v, want ErrDisabled", err) + } +} + +// The common case: a short meeting fits in one prompt, so there is exactly one +// model call and no reduce step. +func TestShortTranscriptSkipsTheReduceStep(t *testing.T) { + f := &fakeCompleter{replies: []string{"— договорились о смете"}} + s := NewSummarizer(f, 0, 0, nil) + out, chunks, err := s.Summarize(context.Background(), "смета", "Обсудили смету. Решили подписать.") + if err != nil { + t.Fatal(err) + } + if chunks != 1 { + t.Errorf("chunks = %d, want 1", chunks) + } + if len(f.users) != 1 { + t.Fatalf("%d model calls, want 1", len(f.users)) + } + if !strings.Contains(out, "смете") || !strings.HasPrefix(out, "смета") { + t.Errorf("summary = %q", out) + } +} + +func TestLongTranscriptIsMappedThenReduced(t *testing.T) { + f := &roleCompleter{mapReply: "часть", reduceReply: "общий итог"} + s := NewSummarizer(f, 40, 0, nil) + long := strings.Repeat("Говорили про насос и трубы. ", 12) + out, chunks, err := s.Summarize(context.Background(), "", long) + if err != nil { + t.Fatal(err) + } + if chunks < 2 { + t.Fatalf("chunks = %d, want the transcript split", chunks) + } + // One map call per chunk, then exactly one reduce. + if f.maps != chunks { + t.Errorf("%d map calls for %d chunks", f.maps, chunks) + } + if f.reduces != 1 { + t.Errorf("%d reduce calls, want 1", f.reduces) + } + if out != "общий итог" { + t.Errorf("summary = %q, want the reduced text", out) + } +} + +// Losing every per-chunk summary because the last call failed would throw away +// most of the work. +func TestReduceFailureReturnsTheJoinedParts(t *testing.T) { + f := &roleCompleter{mapReply: "часть", reduceFails: true} + s := NewSummarizer(f, 40, 0, nil) + long := strings.Repeat("Говорили про насос и трубы. ", 12) + out, _, err := s.Summarize(context.Background(), "", long) + if err == nil { + t.Fatal("reduce failure was not reported") + } + if !strings.Contains(out, "часть1") || !strings.Contains(out, "часть2") { + t.Errorf("per-chunk work was lost: %q", out) + } +} + +func TestChunkFailureIsReported(t *testing.T) { + f := &fakeCompleter{err: errors.New("llama is down")} + s := NewSummarizer(f, 0, 0, nil) + if _, _, err := s.Summarize(context.Background(), "", "текст"); err == nil { + t.Fatal("chunk failure was not reported") + } +} + +// "пусто" chunks are noise; they must not pad the reduce prompt, and a +// transcript that is entirely empty chunks is an honest ErrNoSummary rather than +// an invented summary. +func TestEmptyChunksAreDropped(t *testing.T) { + f := &roleCompleter{mapReply: "пусто", literalMap: true, reduceReply: "не должно вызываться"} + s := NewSummarizer(f, 40, 0, nil) + long := strings.Repeat("Тишина в комнате. ", 12) + if _, _, err := s.Summarize(context.Background(), "", long); !errors.Is(err, ErrNoSummary) { + t.Fatalf("got %v, want ErrNoSummary", err) + } +} + +func TestEmptyTranscriptIsRefused(t *testing.T) { + s := NewSummarizer(&fakeCompleter{}, 0, 0, nil) + if _, _, err := s.Summarize(context.Background(), "", " \n "); !errors.Is(err, ErrEmptyCapture) { + t.Fatalf("got %v, want ErrEmptyCapture", err) + } +} + +// A summary that silently covers the first fraction of a long meeting is the +// failure mode; it has to say so. +func TestTruncationIsStatedInTheSummary(t *testing.T) { + f := &fakeCompleter{replies: []string{"a", "b", "итог"}} + s := NewSummarizer(f, 30, 2, nil) + long := strings.Repeat("Говорили про насос и про трубы. ", 20) + out, chunks, err := s.Summarize(context.Background(), "", long) + if err != nil { + t.Fatal(err) + } + if chunks != 2 { + t.Errorf("chunks = %d, want the cap of 2", chunks) + } + if !strings.Contains(out, "обрезана") { + t.Errorf("truncation not stated: %q", out) + } +} + +// The persona block belongs to the daemon, not this package, and must reach the +// model when it is supplied. +func TestContextBlockIsPrependedToEveryPrompt(t *testing.T) { + f := &fakeCompleter{replies: []string{"итог"}} + s := NewSummarizer(f, 0, 0, func() string { return "ПЕРСОНА\n\n" }) + if _, _, err := s.Summarize(context.Background(), "", "Обсудили смету."); err != nil { + t.Fatal(err) + } + for i, sys := range f.systems { + if !strings.HasPrefix(sys, "ПЕРСОНА") { + t.Errorf("call %d lost the context block: %q", i, sys) + } + } +} + +// The map/reduce prompts must contain no first person at all: the persona's +// feminine forms live in the replier, and a first-person instruction here is a +// place for the model to write "я рад". +func TestPromptsHaveNoFirstPerson(t *testing.T) { + for name, p := range map[string]string{"chunk": chunkPrompt, "reduce": reducePrompt} { + for _, bad := range []string{" я ", "рад", "поняла", "мне ", "вы ", "ваш"} { + if strings.Contains(strings.ToLower(" "+p+" "), bad) { + t.Errorf("%s prompt contains %q", name, bad) + } + } + } +} + +func TestChunkTextSplitsOnSentenceBoundaries(t *testing.T) { + text := "Раз два три. Четыре пять шесть. Семь восемь девять." + got := ChunkText(text, 20) + if len(got) != 3 { + t.Fatalf("got %d chunks: %q", len(got), got) + } + for _, c := range got { + if !strings.HasSuffix(c, ".") { + t.Errorf("chunk does not end on a sentence: %q", c) + } + } +} + +func TestChunkTextPacksSentencesUpToTheLimit(t *testing.T) { + text := "Раз. Два. Три. Четыре." + got := ChunkText(text, 12) + if len(got) < 2 { + t.Fatalf("nothing was split: %q", got) + } + for _, c := range got { + if n := len([]rune(c)); n > 12 { + t.Errorf("chunk of %d runes exceeds the limit: %q", n, c) + } + } +} + +// whisper does emit long unpunctuated runs; those must be cut on whitespace, not +// dropped and not run past the context limit. +func TestChunkTextCutsUnpunctuatedRuns(t *testing.T) { + text := strings.TrimSpace(strings.Repeat("слово ", 50)) + got := ChunkText(text, 30) + if len(got) < 2 { + t.Fatalf("unpunctuated run was not split: %d chunks", len(got)) + } + total := 0 + for _, c := range got { + if n := len([]rune(c)); n > 30 { + t.Errorf("chunk of %d runes exceeds the limit", n) + } + total += strings.Count(c, "слово") + } + if total != 50 { + t.Errorf("%d of 50 words survived chunking", total) + } +} + +// A single token longer than the window must still come out, hard-cut. +func TestChunkTextHandlesOneOversizedWord(t *testing.T) { + text := strings.Repeat("я", 70) + got := ChunkText(text, 20) + if len(got) != 4 { + t.Fatalf("got %d chunks, want 4", len(got)) + } + if joined := strings.Join(got, ""); len([]rune(joined)) != 70 { + t.Errorf("%d runes survived, want 70", len([]rune(joined))) + } +} + +func TestChunkTextShortInputAndEmpty(t *testing.T) { + if got := ChunkText("коротко", 100); len(got) != 1 || got[0] != "коротко" { + t.Errorf("got %q", got) + } + if got := ChunkText(" ", 100); got != nil { + t.Errorf("blank text produced %q", got) + } +} + +// roleCompleter answers by which prompt it was handed, so a test does not have +// to predict how many chunks the text splits into. Map replies are numbered +// ("часть1", "часть2", …) unless literalMap is set. +type roleCompleter struct { + mapReply string + literalMap bool + reduceReply string + reduceFails bool + maps int + reduces int +} + +func (f *roleCompleter) Complete(_ context.Context, system, _ string) (string, error) { + if strings.Contains(system, "конспекты фрагментов") { + f.reduces++ + if f.reduceFails { + return "", errors.New("llama fell over") + } + return f.reduceReply, nil + } + f.maps++ + if f.literalMap { + return f.mapReply, nil + } + return fmt.Sprintf("%s%d", f.mapReply, f.maps), nil +} diff --git a/internal/config/config.go b/internal/config/config.go index 33f1f26..50da777 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -18,11 +18,17 @@ import ( "fmt" "os" "path/filepath" + "strings" "time" "github.com/kami/maven/internal/delivery/ntfysink" "github.com/kami/maven/internal/delivery/telegramsink" + "github.com/kami/maven/internal/mcp" "github.com/kami/maven/internal/morning" + "github.com/kami/maven/internal/netscan" + "github.com/kami/maven/internal/smarthome" + "github.com/kami/maven/internal/update" + "github.com/kami/maven/internal/vision" "github.com/robfig/cron/v3" ) @@ -108,6 +114,20 @@ type Config struct { // calls its /v1/chat/completions endpoint to phrase nudges and reminders. Phraser *PhraserConfig `json:"phraser,omitempty"` + // Update — how THIS box deploys a new build of Maven (Vikunja #249). nil ⇒ + // the update capability does not exist, which is the state to leave it in + // unless the operator has read internal/update's package comment. + // + // mavend never acts on this block: it constructs no Updater and cannot + // update itself. Validate below is the one thing the daemon does with it, so + // a broken update config is caught at startup instead of on the night it is + // needed. That validation is also why internal/update is linked into mavend + // at all — linked, with no caller, which is the property that matters. The + // block lives here because cmd/mavupdate — a CLI the owner runs on the host, + // the only trigger there is — reads the same config file to find the socket + // it health-checks. + Update *update.Config `json:"update,omitempty"` + // Voice — the client↔core surface + the stt/tts modules the daemon // wires. nil ⇒ the daemon doesn't wire voice: the TCP listener stays // down, the dispatcher's Voice slot stays nil (the routing table's @@ -140,6 +160,45 @@ type Config struct { // item. See internal/morning for the evaluation engine. Empty ⇒ disabled. MorningRoutines []MorningRoutineConfig `json:"morning_routines,omitempty"` + // PatternProposals — whether a routine the digestion tick inferred on its + // own may be announced, and how often. nil / absent ⇒ silent detection + // only: proposals are written for /routines and never announced. See + // PatternProposalConfig. + PatternProposals *PatternProposalConfig `json:"pattern_proposals,omitempty"` + + // MemoryEval — background memory evaluation (internal/memeval). nil / + // absent ⇒ no evaluation loop at all. See MemoryEvalConfig. + MemoryEval *MemoryEvalConfig `json:"memory_eval,omitempty"` + + // Email — mail ingestion (Vikunja #246). nil / absent ⇒ core refuses + // ipc.MethodIngestMail outright, so a mail reader cannot make Maven read a + // mailbox by merely existing. See EmailConfig; the IMAP host and credential + // live in the reader (cmd/mavmaild), never here. + Email *EmailConfig `json:"email,omitempty"` + + // IntakeJournal — how many entries the unified intake journal keeps + // (Vikunja #283): one envelope per thing that arrived, whatever direction it + // came from. Absent ⇒ DefaultIntakeJournal. A NEGATIVE value turns the + // journal off entirely, and then there is no decorator on the intake path at + // all. + // + // Not gated behind an "off unless configured" block like feeds or telegram, + // and the distinction is the one CLAUDE.md draws: that rule exists for + // capabilities that reach OUT — a fetch, a send, a third party. This reaches + // nowhere. It is a bounded in-memory log of writes core already performed, + // it is read only by /events and the simulator, and nothing Maven says + // depends on it. + IntakeJournal int `json:"intake_journal,omitempty"` + + // Feeds — RSS/Atom feed reading (Vikunja #258). nil / absent ⇒ no feed is + // ever fetched: reading the outside world is off unless configured, like + // the weather and telegram. See FeedsConfig. + Feeds *FeedsConfig `json:"feeds,omitempty"` + + // Crawl — reading a web page (Vikunja #259). nil / absent ⇒ Maven never + // fetches a page: not on request, not on a schedule. See CrawlConfig. + Crawl *CrawlConfig `json:"crawl,omitempty"` + // Praxis — the ecosystem attention-state service. When configured, maven // calls the Praxis HTTP tools API for attention listing and item lifecycle. // Maven never touches Praxis's database directly (ecosystem invariant: no @@ -155,24 +214,329 @@ type Config struct { // discovers and executes capabilities through Hexis for ecosystem actions. // nil ⇒ no capability-aware routing. Hexis *HexisConfig `json:"hexis,omitempty"` + + // Vision — image understanding (Vikunja #252). nil / absent ⇒ she cannot + // look at pictures at all: the intake refuses, and no vision server is + // contacted. See VisionConfig. + Vision *VisionConfig `json:"vision,omitempty"` + + // Media — where images and captured audio are kept on disk, and for how + // long. nil / absent ⇒ no blob store is wired, which is what disables both + // vision intake and meeting capture regardless of their own blocks: nothing + // in this repo holds a recording only in memory. See MediaConfig. + Media *MediaConfig `json:"media,omitempty"` + + // Capture — meeting recording and summarisation (Vikunja #253). nil / + // absent ⇒ the recorder does not exist: the start/stop methods are not + // served at all, so nothing on this box can begin a recording. This is the + // most invasive capability Maven has and it is the one most firmly off by + // default. See CaptureConfig. + Capture *CaptureConfig `json:"capture,omitempty"` + + // Speaker — voice identification (Vikunja #255). nil / absent ⇒ no + // voiceprint is ever computed and nobody can be enrolled. Enabling it needs + // a speaker-embedding model, which is not on this box. See SpeakerConfig. + Speaker *SpeakerConfig `json:"speaker,omitempty"` + + // MCP — Model Context Protocol servers Maven connects OUT to (Vikunja + // #251). nil / absent / no enabled server ⇒ no connection is made and no + // tool is discovered, like every other capability that reaches outside the + // box. She is a client here, never a server: nothing exposes her own + // capabilities to an outside caller. See MCPConfig. + MCP *MCPConfig `json:"mcp,omitempty"` + + // SmartHome — the Home Assistant instance (Vikunja #256). nil / absent / + // disabled ⇒ Maven neither reads the house nor touches it, and no house row + // exists in the act allowlist. See SmartHomeConfig. + SmartHome *SmartHomeConfig `json:"smarthome,omitempty"` + + // NetScan — the LAN scanner (Vikunja #257). nil / absent / disabled ⇒ + // Maven never puts a packet on the network looking for hosts. See + // NetScanConfig. + NetScan *NetScanConfig `json:"netscan,omitempty"` +} + +// MCPConfig — the MCP client block. Servers are dark until one has +// `"enabled": true`, and a discovered tool is only ever PROPOSED: Kami enables +// it on /tools, on the authed surface, exactly as he would a shell tool. The +// voice path can never grant a capability to itself. +type MCPConfig struct { + // Servers — the configured servers. Each needs exactly one of command + // (a subprocess on this box) or url (a streamable-HTTP endpoint). + Servers []MCPServerConfig `json:"servers,omitempty"` + + // Timeout — per-call budget for every server that does not set its own. + // 0 ⇒ mcp.DefaultTimeout (15s). A tool slower than this is not usable in a + // spoken turn. + Timeout Duration `json:"timeout,omitempty"` + + // AllowHosts / DenyHosts — the host lists for the shared webfetch door that + // url servers go through. Deny wins. Private addresses are refused + // unconditionally unless the individual server sets allow_private. + AllowHosts []string `json:"allow_hosts,omitempty"` + DenyHosts []string `json:"deny_hosts,omitempty"` + + // MaxBytes — cap on one JSON-RPC response. 0 ⇒ webfetch.DefaultMaxBytes. + MaxBytes int64 `json:"max_bytes,omitempty"` + + // HostInterval — minimum spacing between two requests to one MCP server. + // 0 ⇒ DefaultMCPHostInterval (50ms), NOT webfetch's own one-second default. + // That default was sized for a feed poll loop, and this path is in a spoken + // turn: one dial is three requests (initialize, initialized, tools/list), + // so a second of spacing is two seconds of pure sleeping per dial and up to + // another second before every tools/call leaves the box. + HostInterval Duration `json:"host_interval,omitempty"` +} + +// DefaultMCPHostInterval — see MCPConfig.HostInterval. Enough to stop a +// runaway loop hammering a server, small enough not to be heard. +const DefaultMCPHostInterval = 50 * time.Millisecond + +// SmartHomeConfig — the Home Assistant block (Vikunja #256). Dark until +// `"enabled": true`, and even then a discovered device is only ever PROPOSED +// into the act allowlist: Kami enables it on /tools, behind step-up, exactly as +// he would a shell tool. Finding a switch on the network is not the same as +// being allowed to flip it. +type SmartHomeConfig struct { + // Provider — only "homeassistant" is implemented. MQTT / Zigbee2MQTT are + // not: Home Assistant already fronts them, and a broker client is a + // dependency this vendored module tree cannot take on tonight. + Provider string `json:"provider,omitempty"` + + // URL — the instance base, "http://192.168.1.50:8123". + // + // Plain http is accepted and is what the deploy block uses. That is a + // deliberate choice, not an oversight: the instance is on the LAN behind + // wireguard, and a self-signed cert on a home box buys a warning rather + // than a guarantee. It does mean the long-lived token crosses the LAN in + // cleartext on every refresh, so the LAN is part of the trust boundary. + URL string `json:"url,omitempty"` + + // Token — a long-lived access token. Use ${HA_TOKEN} and keep the value in + // the gitignored env file, like the telegram credentials. + Token string `json:"token,omitempty"` + + // Domains — entity domains to take. Empty ⇒ the controllable domains + // EXCEPT lock (light, switch, fan, cover) plus sensor and binary_sensor + // for reads. A lock is only enumerated when it is named here, because a + // front door is not a lamp. Narrow it when the instance is large: a tool name the 1.7B + // half-remembers is a wrong act. + Domains []string `json:"domains,omitempty"` + + // MaxEntities — cap on the proposal catalogue. 0 ⇒ 40. + MaxEntities int `json:"max_entities,omitempty"` + + // Timeout — per-call budget. 0 ⇒ 10s. + Timeout Duration `json:"timeout,omitempty"` + + // Refresh — how often the entity list is re-read and new devices proposed. + // 0 ⇒ 15m, and anything under MinSmartHomeRefresh is raised to it: + // "refresh": "1s" used to pass validation and enumerate the whole instance + // every second. Discovery is idempotent, so this only ever adds rows. + Refresh Duration `json:"refresh,omitempty"` + + // Enabled — false (the default) keeps a written block dark, so it can be + // reviewed before the house is wired to a voice. + Enabled bool `json:"enabled,omitempty"` +} + +// SmartHomeClient maps the config block onto the smarthome package's own type. +// Returns ok=false when nothing is configured or it is disabled, so validation +// and daemon wiring cannot drift on the mapping. +func (c *Config) SmartHomeClient() (smarthome.Config, bool) { + if c.SmartHome == nil || !c.SmartHome.Enabled { + return smarthome.Config{}, false + } + return smarthome.Config{ + URL: c.SmartHome.URL, + Token: c.SmartHome.Token, + Domains: c.SmartHome.Domains, + MaxEntities: c.SmartHome.MaxEntities, + Timeout: time.Duration(c.SmartHome.Timeout), + }, true +} + +// NetScanConfig — the LAN scanner block (Vikunja #257). Dark until +// `"enabled": true`. +// +// The important field is Subnets, and it is the ONLY source of a scan target. +// Nothing an utterance, a router or a scanned host says can widen or move the +// range: internal/netscan.Scanner.Scan takes no target argument at all. Each +// subnet must be private and no larger than netscan.MaxPrefixHosts addresses +// (a /22), enforced at config load rather than at the first spoken scan. +type NetScanConfig struct { + // Subnets — CIDRs to scan, "192.168.1.0/24". + Subnets []string `json:"subnets,omitempty"` + + // Ports — TCP ports to try per host. Empty ⇒ 22, 80, 443, 8080. + Ports []int `json:"ports,omitempty"` + + // Timeout — per-connection budget. 0 ⇒ 400ms. + Timeout Duration `json:"timeout,omitempty"` + + // Rate — connections per second across the whole scan. 0 ⇒ 50. Low on + // purpose: a scan should look like background traffic, not a portscan. + Rate int `json:"rate,omitempty"` + + // MaxHosts — cap on addresses probed per scan. 0 ⇒ 256. + MaxHosts int `json:"max_hosts,omitempty"` + + // Enabled — false (the default) keeps a written block dark. + Enabled bool `json:"enabled,omitempty"` +} + +// NetScanner maps the config block onto the netscan package's own type. +// ok=false when absent or disabled, so validation and daemon wiring cannot +// drift on the mapping. +func (c *Config) NetScanner() (netscan.Config, bool) { + if c.NetScan == nil || !c.NetScan.Enabled { + return netscan.Config{}, false + } + return netscan.Config{ + Subnets: c.NetScan.Subnets, + Ports: c.NetScan.Ports, + Timeout: time.Duration(c.NetScan.Timeout), + Rate: c.NetScan.Rate, + MaxHosts: c.NetScan.MaxHosts, + }, true +} + +// MCPServerConfig — one MCP server. +type MCPServerConfig struct { + // Name — the local handle. It prefixes every tool this server contributes + // ("vikunja" + "list_tasks" ⇒ the allowlist row "vikunja_list_tasks") and + // becomes the store scope "mcp:", so its provenance is readable on + // /tools without opening the config. + Name string `json:"name"` + + // Command / Args / Env / Dir — a stdio server: a child process of mavend, + // on this box, under this user. argv, never a shell string. + Command string `json:"command,omitempty"` + Args []string `json:"args,omitempty"` + Env []string `json:"env,omitempty"` + Dir string `json:"dir,omitempty"` + + // URL — a streamable-HTTP endpoint. It is fetched through + // internal/webfetch, so the SSRF guard, the redirect cap, the size cap and + // the one-request-per-host-per-second limit all apply. + URL string `json:"url,omitempty"` + + // AllowPrivate — let THIS server be a loopback or LAN address. The Vikunja + // server on homesrv is "http://localhost:9100/mcp", which is refused + // without this flag. Understand what it means before setting it: a local + // server is a DIFFERENT trust level from a public one. It is inside the + // network, it usually needs no credential, and it can change things that + // matter — so an argument the router got wrong lands somewhere real. Set it + // only for a server you run yourself, and prefer allow_tools with it. + AllowPrivate bool `json:"allow_private,omitempty"` + + // AllowTools — when set, the ONLY remote tool names taken from this server. + // This is the knob that keeps the catalogue deliberate: the resident model + // is a 1.7B with a 4096-token context, and a tool name it half-remembers is + // a wrong act, so fewer and better-chosen beats complete. + AllowTools []string `json:"allow_tools,omitempty"` + + // MaxTools — cap on this server's contribution. 0 ⇒ mcp.DefaultMaxTools (12). + MaxTools int `json:"max_tools,omitempty"` + + // Timeout — per-call budget for this server. 0 ⇒ MCPConfig.Timeout. + Timeout Duration `json:"timeout,omitempty"` + + // Headers — sent verbatim on every request to a url server. This is how a + // bearer token reaches a real remote MCP server: {"Authorization": "Bearer + // ${MCP_TOKEN}"}, with the value in the gitignored env file like the + // telegram credentials. The Vikunja server on homesrv needs none only + // because it is unauthenticated on loopback. + Headers map[string]string `json:"headers,omitempty"` + + // Enabled — false (the default) keeps a configured server described but + // dark, so a block can be written and reviewed before it is switched on. + Enabled bool `json:"enabled,omitempty"` +} + +// MCPServers maps the config blocks onto the mcp package's own type. It lives +// here so config validation and daemon wiring cannot drift on the mapping. +// Returns nil when nothing is configured or nothing is enabled. +// +// Disabled servers are dropped here, which is why validation does NOT use this +// list — see allMCPServers. +func (c *Config) MCPServers() []mcp.ServerConfig { + return c.mcpServers(true) +} + +// allMCPServers is every configured server, enabled or not, for validation. +// +// Validating only the enabled ones meant a block with both command and url, or +// a bare hostname as the url, passed startup validation while it was dark. The +// doc on Enabled says a block can be written and reviewed before it is switched +// on; the review the config layer could give was the one thing skipped. Enabled +// gates the dialing, not the shape check. +func (c *Config) allMCPServers() []mcp.ServerConfig { + return c.mcpServers(false) +} + +func (c *Config) mcpServers(onlyEnabled bool) []mcp.ServerConfig { + if c.MCP == nil { + return nil + } + out := make([]mcp.ServerConfig, 0, len(c.MCP.Servers)) + for _, s := range c.MCP.Servers { + if onlyEnabled && !s.Enabled { + continue + } + timeout := time.Duration(s.Timeout) + if timeout <= 0 { + timeout = time.Duration(c.MCP.Timeout) + } + out = append(out, mcp.ServerConfig{ + Name: s.Name, + Command: s.Command, + Args: s.Args, + Env: s.Env, + Dir: s.Dir, + URL: s.URL, + AllowPrivate: s.AllowPrivate, + AllowTools: s.AllowTools, + MaxTools: s.MaxTools, + Headers: s.Headers, + Timeout: timeout, + Enabled: s.Enabled, + }) + } + if len(out) == 0 { + return nil + } + return out } // PraxisConfig — maven's connection to the Praxis attention service. type PraxisConfig struct { // URL — the Praxis HTTP API base URL (e.g. "http://localhost:9742"). URL string `json:"url,omitempty"` + + // Token — the shared bearer token sent on every request. Empty ⇒ calls + // go out unauthenticated, which is only appropriate on a loopback or + // unix-socket transport. Supports ${VAR} expansion, so the secret lives + // in deploy/telegram.env, not in the committed config. + Token string `json:"token,omitempty"` } // NexusConfig — connection to the Nexus identity service. type NexusConfig struct { // URL — the Nexus HTTP API base URL (e.g. "http://localhost:9740"). URL string `json:"url,omitempty"` + + // Token — shared bearer token; see PraxisConfig.Token. + Token string `json:"token,omitempty"` } // HexisConfig — connection to the Hexis capability execution service. type HexisConfig struct { // URL — the Hexis HTTP API base URL (e.g. "http://localhost:9741"). URL string `json:"url,omitempty"` + + // Token — shared bearer token; see PraxisConfig.Token. + Token string `json:"token,omitempty"` } // RoutineConfig — one scheduled routine. Cron is a standard 5-field expression @@ -324,6 +688,182 @@ type VoiceConfig struct { ToolTimeout Duration `json:"tool_timeout,omitempty"` } +// MediaConfig — the on-disk blob store for images and captured audio +// (internal/media). It is shared by all three senses: vision intake, meeting +// capture, and speaker enrolment samples all write here. +// +// Absent ⇒ off, and off means Maven cannot accept an image or start a recording +// at all. That default is deliberate: a capability that keeps photos and audio of +// people on disk should require someone to have typed a path. +type MediaConfig struct { + // Dir — the blob store root, created 0700. Relative paths resolve against + // StateDir. Required; an empty dir means the store is not wired. + Dir string `json:"dir,omitempty"` + + // Retention — how long a blob is kept before the tick prunes it. 0 ⇒ + // media.DefaultRetention (7 days). This is the knob that stops recordings + // of people accumulating; raising it past a few weeks should need a reason. + Retention Duration `json:"retention,omitempty"` + + // MaxBytes — per-blob cap. 0 ⇒ media.DefaultMaxBytes (64 MiB). + MaxBytes int64 `json:"max_bytes,omitempty"` + + // MaxTotalBytes — whole-store cap. 0 ⇒ media.DefaultMaxTotalBytes (4 GiB). + // The per-blob cap bounds one call; this one bounds the sum of them, which + // is what actually decides whether the disk mavend's database lives on can + // be filled from outside. + MaxTotalBytes int64 `json:"max_total_bytes,omitempty"` +} + +// StoreDir reports the configured blob directory, or "" when media is not +// wired. Safe on a nil receiver. +func (m *MediaConfig) StoreDir() string { + if m == nil { + return "" + } + return strings.TrimSpace(m.Dir) +} + +// VisionConfig — the vision provider (internal/vision, docs/plans/07-vision.md). +// +// Absent, or enabled=false, ⇒ the daemon wires vision.Disabled and every attempt +// to look at an image answers that vision is not set up. There is no cloud +// option in this block on purpose: Endpoint must be a loopback or private +// address and internal/vision refuses anything else at startup, because +// inference stays on the box and a photo of his flat is the last thing to make +// an exception for. +type VisionConfig struct { + // Enabled — may she look at images. Default false. + Enabled bool `json:"enabled,omitempty"` + + // Endpoint — base URL of a llama-server running a vision model with its + // mmproj, e.g. "http://127.0.0.1:8081". Loopback / private only. + Endpoint string `json:"endpoint,omitempty"` + + // Model — model name sent in the request. llama-server ignores it. + Model string `json:"model,omitempty"` + + // MaxDim — longest edge the image is scaled to before inference. 0 ⇒ + // media.DefaultMaxDim (896). + MaxDim int `json:"max_dim,omitempty"` + + // MaxTokens — cap on the description. 0 ⇒ vision.DefaultMaxTokens (300). + MaxTokens int `json:"max_tokens,omitempty"` + + // Timeout — per-description budget. 0 ⇒ vision.DefaultTimeout (90s). A small + // VLM on an iGPU is slow; a tight timeout here just means no answer ever. + Timeout Duration `json:"timeout,omitempty"` + + // Prompt — the default question when he only sent a picture. Empty ⇒ + // vision.DefaultPrompt (Russian, "опиши что на изображении"). + Prompt string `json:"prompt,omitempty"` +} + +// LooksAtImages reports whether vision is configured well enough to try. Safe on +// a nil receiver, and false without an endpoint — enabled with nothing to talk +// to is a misconfiguration, not a capability. +func (v *VisionConfig) LooksAtImages() bool { + return v != nil && v.Enabled && strings.TrimSpace(v.Endpoint) != "" +} + +// CaptureConfig — the meeting recorder (internal/capture, +// docs/plans/08-hearing.md). +// +// Absent, or enabled=false, ⇒ the recorder is not wired and the capture methods +// return "unknown method", so no client can start a recording however it asks. +// A media block is required too: audio is never held only in memory. +// +// There is deliberately no "auto", no keyword trigger and no duration default +// long enough to be forgotten about. Recording other people is an explicit act +// with a start, a stop, and a cap. +type CaptureConfig struct { + // Enabled — may she record a meeting when asked. Default false. + Enabled bool `json:"enabled,omitempty"` + + // MaxMinutes — hard cap on one session; it stops itself there. 0 ⇒ + // capture.DefaultMaxDuration (120 minutes). + MaxMinutes int `json:"max_minutes,omitempty"` + + // STTWindow — audio handed to whisper per call. 0 ⇒ + // capture.DefaultSTTWindow (5m). Larger windows transcribe slightly better + // and block the STT worker for longer. + STTWindow Duration `json:"stt_window,omitempty"` + + // ChunkRunes — transcript runes per summarisation prompt. 0 ⇒ + // capture.DefaultChunkRunes (3000), sized for the resident model's n_ctx of + // 4096. Raise this only if the resident model's context grows. + ChunkRunes int `json:"chunk_runes,omitempty"` + + // MaxChunks — how many windows one meeting may be summarised in before the + // transcript is truncated and the summary says so. 0 ⇒ + // capture.DefaultMaxChunks (40). + MaxChunks int `json:"max_chunks,omitempty"` + + // SaveTranscript — write the full transcript as a note alongside the + // summary. Default false, and the cost is not disk: a note is embedded and + // becomes recall corpus, so every later question can surface verbatim words + // other people said in a room. That is the reason it takes a deliberate yes. + // The audio blob is pruned by media.retention either way; the notes are not. + // + // A meeting with no summary writes its transcript regardless. The choice + // here is transcript IN ADDITION to a summary, not whether the meeting is + // remembered at all. + SaveTranscript bool `json:"save_transcript,omitempty"` +} + +// Records reports whether the recorder should be wired. Safe on a nil receiver. +func (c *CaptureConfig) Records() bool { + return c != nil && c.Enabled +} + +// MaxDuration is the configured session cap as a duration, or 0 for the +// package default. Safe on a nil receiver. +func (c *CaptureConfig) MaxDuration() time.Duration { + if c == nil || c.MaxMinutes <= 0 { + return 0 + } + return time.Duration(c.MaxMinutes) * time.Minute +} + +// SpeakerConfig — voice identification (internal/speaker, +// docs/plans/10-speaker-recognition.md). +// +// Absent, or enabled=false, ⇒ no voiceprint is computed for any turn, the +// enrolment methods do not exist, and nobody can be enrolled. A voiceprint is +// biometric data about a person, so this one is off until someone typed a model +// path on purpose. +// +// It cannot currently be turned on: there is no speaker-embedding model on this +// box. See the plan document for what to download. +type SpeakerConfig struct { + // Enabled — may she work out who is speaking. Default false. + Enabled bool `json:"enabled,omitempty"` + + // ModelPath — an ECAPA-TDNN (or equivalent) speaker-embedding ONNX model. + // Required; without it the recognizer runs disabled and says so once. + ModelPath string `json:"model_path,omitempty"` + + // LibPath — onnxruntime shared library, as for the text embedder. Empty ⇒ + // the same default the embedder block uses. + LibPath string `json:"lib_path,omitempty"` + + // Threshold — cosine similarity a match must beat. 0 ⇒ + // speaker.DefaultThreshold (0.7). Lower it and she starts calling guests by + // his name, which is the expensive direction of this error. + Threshold float64 `json:"threshold,omitempty"` + + // MinSeconds — least speech an identification will look at. 0 ⇒ + // speaker.DefaultMinSeconds (2s). + MinSeconds float64 `json:"min_seconds,omitempty"` +} + +// Recognizes reports whether voice identification should be wired. Safe on a +// nil receiver, and false without a model path — enabled with nothing to embed +// with is a misconfiguration, not a capability. +func (s *SpeakerConfig) Recognizes() bool { + return s != nil && s.Enabled && strings.TrimSpace(s.ModelPath) != "" +} + // WeatherConfig configures the weather provider for voice queries. type WeatherConfig struct { Provider string `json:"provider,omitempty"` // "open-meteo" or "" → stub @@ -352,6 +892,200 @@ type DigestConfig struct { SeverityCeiling int `json:"severity_ceiling,omitempty"` // max sev batched } +// PatternProposalConfig — announcement policy for routines the digestion tick +// inferred by itself (Vikunja #247, #43). +// +// Detection is always on and always silent by default: the tick writes a +// proposed_routines row and the /routines page shows it. Notify is what turns +// "she noticed" into "she said something", and it is OFF unless configured — +// Maven is not a nag and not autonomous, so a behaviour that speaks without +// being asked has to be switched on deliberately, like weather and telegram. +// +// When Notify is on, the announcement is still heavily restrained: +// - at most one proposal per tick, however many were detected; +// - at most one per Cooldown across all pairs (not per pair), so a batch of +// freshly-detected patterns cannot turn into a queue of interruptions; +// - through the ordinary care-class gate (quiet hours / away / snooze), at +// sev1 — the lowest severity there is. A proposal is the least urgent +// thing Maven can say. +// +// A pair is only ever announced once, because it is only ever proposed once: +// proposed_routines is UNIQUE(action, object) and the row survives dismissal. +type PatternProposalConfig struct { + // Notify — announce newly inferred routines. Default false. + Notify bool `json:"notify,omitempty"` + + // Cooldown — minimum spacing between two proposal announcements. 0 ⇒ + // DefaultProposalCooldown (24h). + Cooldown Duration `json:"cooldown,omitempty"` +} + +// AnnounceProposals reports whether inferred routines may be announced. Safe +// on a nil receiver — an absent config block means silent detection. +func (p *PatternProposalConfig) AnnounceProposals() bool { + return p != nil && p.Notify +} + +// FeedsConfig — the RSS/Atom reader (Vikunja #258, docs/plans/13-rss-news-feeds.md). +// +// Absent ⇒ off, and off means no outbound request at all. Present with an empty +// `sources` list is also off — a poller with nothing to poll is not wired. +// +// What a feed may NOT do here: speak. Items are written as notes with source +// "rss:" and read back when he asks; nothing is dispatched, nudged or +// announced on arrival. That is the "not a nag" constraint, and it is why there +// is no severity or channel field in this block to reach for. +type FeedsConfig struct { + // Sources — the feeds to read. Empty ⇒ the reader stays down. + Sources []FeedSourceConfig `json:"sources,omitempty"` + + // PollInterval — default per-feed cadence. 0 ⇒ rss.DefaultPollInterval (30m). + PollInterval Duration `json:"poll_interval,omitempty"` + + // MaxItems — most items kept from one feed in one poll. 0 ⇒ + // rss.DefaultMaxItems (5). This is the "не завали мне /dash" knob. + MaxItems int `json:"max_items,omitempty"` + + // MaxAge — on a first poll (no saved mark), how far back to take items. + // 0 ⇒ rss.DefaultMaxAge (24h), so switching a feed on imports today, not + // the archive. + MaxAge Duration `json:"max_age,omitempty"` + + // AllowHosts — when set, the reader may only connect to these hosts (and + // their subdomains). The feed URLs' own hosts are added automatically, so + // this is only needed to be stricter than that. + AllowHosts []string `json:"allow_hosts,omitempty"` + + // Timeout — per-request budget. 0 ⇒ webfetch.DefaultTimeout. + Timeout Duration `json:"timeout,omitempty"` + + // MaxBytes — response size cap. 0 ⇒ webfetch.DefaultMaxBytes (2 MiB). + MaxBytes int64 `json:"max_bytes,omitempty"` +} + +// FeedSourceConfig — one feed. +type FeedSourceConfig struct { + Name string `json:"name"` // note source is "rss:" + URL string `json:"url"` // http(s) only + Category string `json:"category,omitempty"` // "технологии" — what "что нового по X?" matches + Interval Duration `json:"interval,omitempty"` // 0 ⇒ FeedsConfig.PollInterval + Include []string `json:"include,omitempty"` // keep only items containing one of these + Exclude []string `json:"exclude,omitempty"` // drop items containing any of these +} + +// CrawlConfig — the web crawler (Vikunja #259, docs/plans/14-web-crawler.md). +// +// Absent ⇒ off, and off means no page is ever fetched. Present with neither +// `on_demand` nor a `watches` entry is also off: there would be nothing to do. +// +// The crawler is the LAST place an answer is looked for, behind the model, his +// own memory and the local Kiwix ZIMs. That ordering lives in the query-source +// chain (cmd/mavend/actions_query.go), not here, but it is the reason this block +// is small: it is a fallback, not a search engine. +// +// Only the URL leaves the box. His notes, facts, persona block and history are +// never part of a request — the crawler package cannot even read the store. +type CrawlConfig struct { + // OnDemand — may he ask her to read a page he names out loud + // ("посмотри https://… — что там пишут?"). false ⇒ the on-demand answer + // source stays off and only the watches below run. + OnDemand bool `json:"on_demand,omitempty"` + + // Watches — pages re-read on a schedule. A page whose text changed is + // written as a note (source "crawl:"); nothing is announced. + Watches []CrawlWatchConfig `json:"watches,omitempty"` + + // Interval — default watch cadence. 0 ⇒ crawl.DefaultWatchInterval (6h). + Interval Duration `json:"interval,omitempty"` + + // AllowHosts — when set, the ONLY hosts the crawler may reach (subdomains + // included). Setting this is how "she may read the arch wiki and nothing + // else" is expressed. + // + // A watched page's own host is reachable by the scheduled crawler whether + // or not it is listed here, because configuring a watch is already saying + // she may read it. That does NOT extend to on-demand reading: a watch is + // not an allowlist entry for pages he pastes. + AllowHosts []string `json:"allow_hosts,omitempty"` + + // DenyHosts — never reachable, checked first. Private addresses do not need + // to be listed: they are refused unconditionally (see internal/webfetch). + DenyHosts []string `json:"deny_hosts,omitempty"` + + // UserAgent — sent on every request AND matched against robots.txt groups. + // Empty ⇒ webfetch.DefaultUserAgent. + UserAgent string `json:"user_agent,omitempty"` + + // Timeout — per-request budget. 0 ⇒ webfetch.DefaultTimeout. + Timeout Duration `json:"timeout,omitempty"` + + // MaxBytes — response size cap. 0 ⇒ webfetch.DefaultMaxBytes (2 MiB). + MaxBytes int64 `json:"max_bytes,omitempty"` + + // MaxRunes — how much extracted text is kept. 0 ⇒ crawl.DefaultMaxRunes + // (4000), which is what fits a 4096-token context alongside a prompt. + MaxRunes int `json:"max_runes,omitempty"` +} + +// CrawlWatchConfig — one page kept an eye on. +type CrawlWatchConfig struct { + Name string `json:"name"` // note source is "crawl:" + URL string `json:"url"` + Interval Duration `json:"interval,omitempty"` // 0 ⇒ CrawlConfig.Interval +} + +// MemoryEvalConfig — the background memory-evaluation loop (Vikunja #248). +// Absent ⇒ off, like every other capability that costs something the owner did +// not ask for. Each evaluation is a full LLM round-trip on the one resident +// model, which is the same model answering him; running it hourly by default +// would put a multi-second stall in front of an occasional voice turn for a +// feature he may not want. +// +// The loop only ever writes notes (source infer:memory-eval, visible on +// /dash). It cannot speak — see internal/memeval. +type MemoryEvalConfig struct { + // Interval — how often to evaluate. 0 ⇒ DefaultMemoryEvalInterval. + Interval Duration `json:"interval,omitempty"` + + // MaxItems — recent facts / notes / nudges fed into one evaluation. + // 0 ⇒ memeval.DefaultMaxItems. + MaxItems int `json:"max_items,omitempty"` + + // MinConfidence — observations the model scores below this are dropped. + // 0 ⇒ memeval.DefaultMinConfidence. + MinConfidence float64 `json:"min_confidence,omitempty"` +} + +// EmailConfig — core's half of the email reader: how many task candidates one +// message may produce, and how long the extraction call may take. +// +// There is deliberately nothing about a mailbox here. Core does not connect to +// IMAP, does not know an account exists, and holds no mail credential — the +// reader daemon does, the same split mavpoll uses for the zenmoney token. This +// block only says "extraction is allowed, with these bounds". +type EmailConfig struct { + // MaxTasks — candidates per message. 0 ⇒ email.MaxCandidates (3). + MaxTasks int `json:"max_tasks,omitempty"` + + // Timeout — per-message extraction budget. 0 ⇒ DefaultEmailTimeout. This is + // a Thinking model reading a mail; nobody is waiting on the answer, but a + // hung llama-server must not pin the reader's connection forever. + Timeout Duration `json:"timeout,omitempty"` +} + +// DefaultEmailTimeout — extraction budget per message. +const DefaultEmailTimeout = 2 * time.Minute + +// DefaultSmartHomeRefresh — how often the house is re-enumerated for new +// devices. Slow on purpose: discovery only adds proposals, and a flat does not +// grow a new lamp every minute. +const DefaultSmartHomeRefresh = 15 * time.Minute + +// MinSmartHomeRefresh — the floor under SmartHomeConfig.Refresh. Enumerating +// every entity in the house is a full /api/states read; a misconfigured second +// would hammer the instance for proposals that are idempotent anyway. +const MinSmartHomeRefresh = time.Minute + // PhraserConfig — the LLM-backed phraser seam. The daemon spawns llama-server // as a managed subprocess and sends chat-completion requests to phrase nudge // and reminder messages. nil ⇒ the template-based Stub is used instead. @@ -375,6 +1109,21 @@ type PhraserConfig struct { // persona and invented units). Chat, query and reminder phrasing always go // through the model regardless. See phraser.Config.LLMNudges. LLMNudges bool `json:"llm_nudges,omitempty"` + + // SwapModels — the gguf files the running daemon is allowed to swap to + // without a restart (Vikunja #250). Empty (the default) means the swap + // capability does not exist: ipc.MethodSwapModel answers ErrUnknownMethod, + // exactly like an unconfigured weather or telegram block. + // + // It is an allowlist and not a directory on purpose. The request carries a + // path, and llama-server is started with it as `-m`; anything short of an + // exact match against a list a human wrote in this file would make "swap the + // model" mean "load a file of your choosing off my disk". ModelPath is + // always swappable back to whether or not it is listed. + // + // Paths must be absolute — the daemon's working directory is not the + // operator's, and a relative path here would resolve somewhere surprising. + SwapModels []string `json:"swap_models,omitempty"` } // EmbedderConfig — paths for the ONNX multilingual embedder. The daemon @@ -435,7 +1184,11 @@ const ( DefaultRepeatInterval = 5 * time.Minute DefaultAutotuneInterval = 10 * time.Minute DefaultRouterThreshold = 0.55 - DefaultQueryMinScore = 0.55 + // DefaultIntakeJournal — entries kept in the unified intake journal + // (Vikunja #283). A busy day is a few hundred intake writes, so this is + // roughly "today and yesterday" at a few hundred KB of memory. + DefaultIntakeJournal = 512 + DefaultQueryMinScore = 0.55 // Read off the margin sweep in internal/memory/recalleval on the e5 // embedder: 0.008 answers 68% of real questions (down from 72%) and cuts // false recall from 5/5 to 1/5. Every larger delta costs real recall @@ -448,6 +1201,15 @@ const ( DefaultLLMRouter = true DefaultFactEnrichmentInterval = 30 * time.Second + + // DefaultProposalCooldown — one inferred-routine announcement per day at + // most. A proposal is never urgent; if two patterns surface in the same + // hour, the second one waits, and the /routines page has it either way. + DefaultProposalCooldown = 24 * time.Hour + + // DefaultMemoryEvalInterval — the plan's cadence (1h) for the memory + // evaluation loop, applied only when the block is present at all. + DefaultMemoryEvalInterval = time.Hour ) // Load reads the JSON config at path and applies defaults. A missing file is @@ -476,6 +1238,9 @@ func Load(path string) (*Config, error) { } func (c *Config) applyDefaults() { + if c.IntakeJournal == 0 { + c.IntakeJournal = DefaultIntakeJournal + } if c.TickInterval == 0 { c.TickInterval = Duration(DefaultTickInterval) } @@ -520,6 +1285,66 @@ func (c *Config) applyDefaults() { c.Digest.SeverityCeiling = 2 } + // Absent block stays nil (⇒ silent detection). Present-but-partial gets the + // cooldown default, so `{"notify": true}` is enough to switch it on. + if c.PatternProposals != nil && c.PatternProposals.Cooldown <= 0 { + c.PatternProposals.Cooldown = Duration(DefaultProposalCooldown) + } + + // Same rule: absent stays nil (⇒ no evaluation loop), present gets defaults + // so `{}` is a valid "on with the plan's cadence". + if c.MemoryEval != nil && c.MemoryEval.Interval <= 0 { + c.MemoryEval.Interval = Duration(DefaultMemoryEvalInterval) + } + + // Same rule again: absent stays nil (⇒ mail ingestion refused), present gets + // the timeout default so `{}` is a valid "on with the defaults". + if c.Email != nil && c.Email.Timeout <= 0 { + c.Email.Timeout = Duration(DefaultEmailTimeout) + } + + // A feeds block with no sources is the same as no block: nothing to poll, + // nothing wired. Normalising it to nil keeps that "off" in one place. + if c.Feeds != nil && len(c.Feeds.Sources) == 0 { + c.Feeds = nil + } + + // Same rule for MCP: a block with no server at all is the same as no block. + // A block whose servers are all disabled is NOT normalised away, because + // validate has to see their shape — a dark block with a typo in it should + // fail at startup, which is the whole reason it can be written before it is + // switched on. wireMCP builds nothing when nothing is enabled, so "off" + // still holds. + if c.MCP != nil && len(c.MCP.Servers) == 0 { + c.MCP = nil + } + if c.MCP != nil && c.MCP.HostInterval <= 0 { + c.MCP.HostInterval = Duration(DefaultMCPHostInterval) + } + + // Same rule for the house: a block that is not enabled is the same as no + // block at all, so "off" stays in one place. + if c.SmartHome != nil && !c.SmartHome.Enabled { + c.SmartHome = nil + } + if c.SmartHome != nil && c.SmartHome.Refresh <= 0 { + c.SmartHome.Refresh = Duration(DefaultSmartHomeRefresh) + } + if c.SmartHome != nil && c.SmartHome.Refresh < Duration(MinSmartHomeRefresh) { + c.SmartHome.Refresh = Duration(MinSmartHomeRefresh) + } + + // Same rule for the scanner. + if c.NetScan != nil && !c.NetScan.Enabled { + c.NetScan = nil + } + + // Same rule for the crawler: a block that neither answers on demand nor + // watches anything has nothing to do, so it is normalised to "off". + if c.Crawl != nil && !c.Crawl.OnDemand && len(c.Crawl.Watches) == 0 { + c.Crawl = nil + } + if c.Voice != nil { if c.Voice.RouterThreshold <= 0 { c.Voice.RouterThreshold = DefaultRouterThreshold @@ -577,6 +1402,22 @@ func (c *Config) validate() error { if c.Phraser.ModelPath == "" { return errors.New("phraser.model_path is required") } + // A relative entry in the swap allowlist would resolve against the + // daemon's working directory, so the path a human reads in this file + // would not be the path llama-server is handed. Fail at startup. + for _, m := range c.Phraser.SwapModels { + if !filepath.IsAbs(m) { + return fmt.Errorf("phraser.swap_models: %q must be an absolute path", m) + } + } + } + // The update block is validated here even though mavend never acts on it: a + // half-written update config that is only noticed by cmd/mavupdate is noticed + // at the worst possible moment, halfway through deploying a new build. + if c.Update != nil { + if err := c.Update.Validate(); err != nil { + return err + } } if c.Voice != nil && c.Voice.Enabled { if c.Voice.Bind == "" { @@ -602,6 +1443,59 @@ func (c *Config) validate() error { return fmt.Errorf("routine %q: bad cron %q: %w", r.Name, r.Cron, err) } } + // An MCP block with a typo (no name, both command and url, a bare hostname + // as the url) fails here, at startup, rather than at the first turn that + // needed the tool. + if err := mcp.Validate(c.allMCPServers()); err != nil { + return err + } + // Same for the house: a missing token or a bare hostname fails at startup, + // not at the first "выключи свет". + if hc, ok := c.SmartHomeClient(); ok { + if p := c.SmartHome.Provider; p != "" && p != "homeassistant" { + return fmt.Errorf("smarthome: provider %q: only \"homeassistant\" is implemented", p) + } + if err := smarthome.Validate(hc); err != nil { + return err + } + } + // A scanner pointed at the public internet, or at a /8, fails here rather + // than after the packets have already left. + if nc, ok := c.NetScanner(); ok { + if err := netscan.Validate(nc); err != nil { + return err + } + } + // A media dir that cannot be created, or a vision endpoint that is a typo, + // used to be logged at wiring time and the capability just stayed off. A + // capability silently not existing is the hardest kind of misconfiguration + // to notice, so both fail here instead. + if c.Media != nil { + if c.Media.StoreDir() == "" { + return errors.New("media.dir is required when a media block is present") + } + if c.Media.MaxBytes < 0 || c.Media.MaxTotalBytes < 0 { + return errors.New("media: max_bytes and max_total_bytes cannot be negative") + } + if c.Media.MaxTotalBytes > 0 && c.Media.MaxBytes > c.Media.MaxTotalBytes { + return fmt.Errorf("media: max_bytes %d is above max_total_bytes %d", + c.Media.MaxBytes, c.Media.MaxTotalBytes) + } + } + if c.Vision != nil && c.Vision.Enabled { + if strings.TrimSpace(c.Vision.Endpoint) == "" { + return errors.New("vision.enabled set but vision.endpoint is empty") + } + if err := vision.ValidateEndpoint(c.Vision.Endpoint); err != nil { + return err + } + if c.Media.StoreDir() == "" { + return errors.New("vision.enabled set but there is no media block to keep the bytes in") + } + } + if c.Capture.Records() && c.Media.StoreDir() == "" { + return errors.New("capture.enabled set but there is no media block to keep the audio in") + } if len(c.MorningRoutines) > 0 { if err := morning.Validate(morningRoutinesFromConfig(c.MorningRoutines)); err != nil { return err diff --git a/internal/config/config_test.go b/internal/config/config_test.go index dae1b6b..d1b10d0 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -243,3 +243,127 @@ func TestDurationRoundTrip(t *testing.T) { t.Errorf("round-trip = %v, want %v", d2, d) } } + +// Both new opt-in capabilities follow the same rule: absent block ⇒ nil ⇒ the +// behaviour does not exist. Presence is the enable act, so a bare `{}` block is +// valid and gets the defaults filled in. +func TestOptInBlocksAbsentStayNil(t *testing.T) { + c, err := Load(writeConfig(t, `{}`)) + if err != nil { + t.Fatalf("Load: %v", err) + } + if c.PatternProposals != nil { + t.Errorf("pattern_proposals absent but got %+v", c.PatternProposals) + } + if c.PatternProposals.AnnounceProposals() { + t.Error("AnnounceProposals() true with no config block") + } + if c.MemoryEval != nil { + t.Errorf("memory_eval absent but got %+v", c.MemoryEval) + } +} + +func TestOptInBlocksGetDefaultsWhenPresent(t *testing.T) { + c, err := Load(writeConfig(t, `{"pattern_proposals":{"notify":true},"memory_eval":{}}`)) + if err != nil { + t.Fatalf("Load: %v", err) + } + if !c.PatternProposals.AnnounceProposals() { + t.Error("notify:true did not enable announcements") + } + if time.Duration(c.PatternProposals.Cooldown) != DefaultProposalCooldown { + t.Errorf("proposal cooldown = %v, want %v", c.PatternProposals.Cooldown, DefaultProposalCooldown) + } + if time.Duration(c.MemoryEval.Interval) != DefaultMemoryEvalInterval { + t.Errorf("memory eval interval = %v, want %v", c.MemoryEval.Interval, DefaultMemoryEvalInterval) + } +} + +// Notify is off even when the block exists — the block is where you tune it, +// notify:true is the act that lets her speak. +func TestPatternProposalNotifyDefaultsOff(t *testing.T) { + c, err := Load(writeConfig(t, `{"pattern_proposals":{"cooldown":"6h"}}`)) + if err != nil { + t.Fatalf("Load: %v", err) + } + if c.PatternProposals.AnnounceProposals() { + t.Error("notify defaulted to on") + } + if time.Duration(c.PatternProposals.Cooldown) != 6*time.Hour { + t.Errorf("cooldown = %v, want 6h", c.PatternProposals.Cooldown) + } +} + +// TestSwapModelsAbsentMeansOff — the swap capability does not exist unless the +// operator lists the models he allows (Vikunja #250). +func TestSwapModelsAbsentMeansOff(t *testing.T) { + c, err := Load(writeConfig(t, `{"phraser": {"model_path": "/m/qwen.gguf"}}`)) + if err != nil { + t.Fatalf("Load: %v", err) + } + if len(c.Phraser.SwapModels) != 0 { + t.Errorf("swap_models = %v; want empty when unconfigured", c.Phraser.SwapModels) + } +} + +func TestSwapModelsParsedAndMustBeAbsolute(t *testing.T) { + c, err := Load(writeConfig(t, `{"phraser": { + "model_path": "/m/qwen.gguf", + "swap_models": ["/m/qwen.gguf", "/m/qwen-cpt.gguf"] + }}`)) + if err != nil { + t.Fatalf("Load: %v", err) + } + if len(c.Phraser.SwapModels) != 2 { + t.Fatalf("swap_models = %v; want 2 entries", c.Phraser.SwapModels) + } + // A relative entry would resolve against the daemon's cwd, not the operator's. + if _, err := Load(writeConfig(t, `{"phraser": { + "model_path": "/m/qwen.gguf", + "swap_models": ["models/llm/qwen.gguf"] + }}`)); err == nil { + t.Error("Load accepted a relative swap_models entry; want a startup failure") + } +} + +// TestUpdateBlockAbsentMeansOff — mavend never updates itself; the block only +// exists so cmd/mavupdate can find the deployment it is asked to update +// (Vikunja #249). Absent is the normal state. +func TestUpdateBlockAbsentMeansOff(t *testing.T) { + c, err := Load(writeConfig(t, `{}`)) + if err != nil { + t.Fatalf("Load: %v", err) + } + if c.Update != nil { + t.Errorf("update = %+v; want nil when unconfigured", c.Update) + } +} + +func TestUpdateBlockValidatedAtStartup(t *testing.T) { + good := `{"update": { + "source_dir": "/srv/maven", + "install_dir": "/srv/maven", + "snapshot_dir": "/var/lib/maven/snapshots", + "binaries": ["mavend", "mavweb"], + "restart_cmd": ["docker", "compose", "up", "-d", "--build", "mavend"], + "health_socket": "/run/maven/mavend.sock", + "source_rollback": "git" + }}` + c, err := Load(writeConfig(t, good)) + if err != nil { + t.Fatalf("Load: %v", err) + } + if c.Update == nil || len(c.Update.Binaries) != 2 { + t.Fatalf("update block = %+v; want it parsed", c.Update) + } + // A block with no health check cannot detect its own failure, so it cannot + // roll back — refused at load, not halfway through a deploy. + noHealth := `{"update": { + "source_dir": "/srv/maven", "install_dir": "/srv/maven", + "snapshot_dir": "/var/lib/maven/snapshots", + "binaries": ["mavend"], "restart_cmd": ["true"] + }}` + if _, err := Load(writeConfig(t, noHealth)); err == nil { + t.Error("Load accepted an update block with no health_socket") + } +} diff --git a/internal/config/mcp_test.go b/internal/config/mcp_test.go new file mode 100644 index 0000000..e11de02 --- /dev/null +++ b/internal/config/mcp_test.go @@ -0,0 +1,123 @@ +package config + +import ( + "testing" + "time" +) + +func TestMCPAbsentIsOff(t *testing.T) { + c, err := Load(writeConfig(t, `{}`)) + if err != nil { + t.Fatal(err) + } + if c.MCP != nil { + t.Error("no mcp block ⇒ nil") + } + if got := c.MCPServers(); got != nil { + t.Errorf("MCPServers() = %+v, want nil", got) + } +} + +// A described-but-not-enabled server must not be wired. This is how a block can +// sit in the config file, reviewed, before it is switched on. +func TestMCPDisabledServerIsOff(t *testing.T) { + c, err := Load(writeConfig(t, `{"mcp":{"servers":[ + {"name":"vikunja","url":"http://localhost:9100/mcp","allow_private":true}]}}`)) + if err != nil { + t.Fatal(err) + } + if got := c.MCPServers(); len(got) != 0 { + t.Errorf("MCPServers() = %+v", got) + } +} + +// A block with no servers at all is the same as no block. +func TestMCPEmptyBlockNormalisesToNil(t *testing.T) { + c, err := Load(writeConfig(t, `{"mcp":{"servers":[]}}`)) + if err != nil { + t.Fatal(err) + } + if c.MCP != nil { + t.Errorf("mcp = %+v, want nil", c.MCP) + } +} + +// A server that is written but not switched on is still shape-checked. The +// review the config layer can give is the point of writing a block dark, and +// skipping it meant a typo only surfaced on the day it was enabled. +func TestMCPDisabledServerIsStillValidated(t *testing.T) { + cases := map[string]string{ + "both": `{"mcp":{"servers":[{"name":"a","command":"x","url":"http://a.test"}]}}`, + "bare host": `{"mcp":{"servers":[{"name":"a","url":"a.test"}]}}`, + "no name": `{"mcp":{"servers":[{"command":"x"}]}}`, + "duplicates": `{"mcp":{"servers":[{"name":"a","command":"x"},{"name":"a","command":"y"}]}}`, + } + for name, body := range cases { + t.Run(name, func(t *testing.T) { + if _, err := Load(writeConfig(t, body)); err == nil { + t.Fatal("a dark server with a typo must fail at startup") + } + }) + } +} + +// Headers carry a bearer token to a real remote server. +func TestMCPServerHeaders(t *testing.T) { + c, err := Load(writeConfig(t, `{"mcp":{"servers":[ + {"name":"remote","url":"https://mcp.example.test/mcp","enabled":true, + "headers":{"Authorization":"Bearer sekret"}}]}}`)) + if err != nil { + t.Fatal(err) + } + got := c.MCPServers() + if len(got) != 1 || got[0].Headers["Authorization"] != "Bearer sekret" { + t.Fatalf("headers not mapped: %+v", got) + } +} + +func TestMCPEnabledServerMapping(t *testing.T) { + c, err := Load(writeConfig(t, `{"mcp":{ + "timeout":"5s", + "servers":[ + {"name":"vikunja","url":"http://localhost:9100/mcp","allow_private":true, + "allow_tools":["list_tasks"],"max_tools":3,"enabled":true}, + {"name":"files","command":"mcp-server-fs","args":["/srv"],"timeout":"1s","enabled":true}, + {"name":"off","command":"nope"} + ]}}`)) + if err != nil { + t.Fatal(err) + } + got := c.MCPServers() + if len(got) != 2 { + t.Fatalf("servers = %+v", got) + } + if got[0].Name != "vikunja" || !got[0].AllowPrivate || got[0].MaxTools != 3 || + len(got[0].AllowTools) != 1 || got[0].Timeout != 5*time.Second { + t.Errorf("vikunja mapped wrong: %+v", got[0]) + } + if got[1].Command != "mcp-server-fs" || len(got[1].Args) != 1 || got[1].Timeout != time.Second { + t.Errorf("files mapped wrong: %+v", got[1]) + } + // allow_private is per server and must not leak to the other one. + if got[1].AllowPrivate { + t.Error("allow_private leaked between servers") + } +} + +func TestMCPBadServerFailsAtStartup(t *testing.T) { + cases := map[string]string{ + "no name": `{"mcp":{"servers":[{"command":"x","enabled":true}]}}`, + "both": `{"mcp":{"servers":[{"name":"a","command":"x","url":"http://a.test","enabled":true}]}}`, + "neither": `{"mcp":{"servers":[{"name":"a","enabled":true}]}}`, + "bad scheme": `{"mcp":{"servers":[{"name":"a","url":"unix:///run/x.sock","enabled":true}]}}`, + "duplicate": `{"mcp":{"servers":[{"name":"a","command":"x","enabled":true},{"name":"a","command":"y","enabled":true}]}}`, + "spacey name": `{"mcp":{"servers":[{"name":"a b","command":"x","enabled":true}]}}`, + } + for name, body := range cases { + t.Run(name, func(t *testing.T) { + if _, err := Load(writeConfig(t, body)); err == nil { + t.Fatal("want a startup error") + } + }) + } +} diff --git a/internal/config/senses_test.go b/internal/config/senses_test.go new file mode 100644 index 0000000..44ec12c --- /dev/null +++ b/internal/config/senses_test.go @@ -0,0 +1,260 @@ +package config + +import ( + "encoding/json" + "testing" + "time" +) + +// Absent blocks must read as off on a nil receiver: the daemon calls these +// helpers before it knows whether the operator configured anything. +func TestSensesOffByDefault(t *testing.T) { + var cfg Config + if cfg.Media.StoreDir() != "" { + t.Error("media store dir is set with no media block") + } + if cfg.Vision.LooksAtImages() { + t.Error("vision is on with no vision block") + } + if cfg.Capture.Records() { + t.Error("the recorder is on with no capture block") + } + if cfg.Capture.MaxDuration() != 0 { + t.Error("a nil capture block invented a duration") + } + if cfg.Speaker.Recognizes() { + t.Error("speaker recognition is on with no speaker block") + } +} + +// The recorder is the capability that most needs its default to be off, so it +// gets its own test rather than a line in the one above. +func TestCaptureIsOffUntilExplicitlyEnabled(t *testing.T) { + cases := []struct { + name string + c *CaptureConfig + want bool + }{ + {"absent", nil, false}, + {"present but not enabled", &CaptureConfig{MaxMinutes: 60}, false}, + {"enabled", &CaptureConfig{Enabled: true}, true}, + } + for _, c := range cases { + if got := c.c.Records(); got != c.want { + t.Errorf("%s: Records() = %v, want %v", c.name, got, c.want) + } + } +} + +func TestCaptureBlockParsesFromJSON(t *testing.T) { + raw := `{"capture":{"enabled":true,"max_minutes":45,"stt_window":"2m", + "chunk_runes":2000,"max_chunks":10,"save_transcript":true}}` + var cfg Config + if err := json.Unmarshal([]byte(raw), &cfg); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if !cfg.Capture.Records() { + t.Fatal("capture did not parse as enabled") + } + if cfg.Capture.MaxDuration() != 45*time.Minute { + t.Errorf("max duration = %v", cfg.Capture.MaxDuration()) + } + if time.Duration(cfg.Capture.STTWindow) != 2*time.Minute { + t.Errorf("stt window = %v", time.Duration(cfg.Capture.STTWindow)) + } + if cfg.Capture.ChunkRunes != 2000 || cfg.Capture.MaxChunks != 10 { + t.Errorf("summariser limits = %+v", cfg.Capture) + } + if !cfg.Capture.SaveTranscript { + t.Error("save_transcript did not parse") + } +} + +// Keeping the verbatim record of what other people said is the heavier act, so +// it is separately opt-in from recording at all. +func TestTranscriptIsNotSavedByDefault(t *testing.T) { + var cfg Config + if err := json.Unmarshal([]byte(`{"capture":{"enabled":true}}`), &cfg); err != nil { + t.Fatal(err) + } + if cfg.Capture.SaveTranscript { + t.Error("transcripts are saved without anyone asking") + } + if cfg.Capture.MaxDuration() != 0 { + t.Error("max_minutes defaulted in config instead of in the package") + } +} + +// enabled with nothing to talk to is a misconfiguration, not a capability. +func TestVisionNeedsBothEnabledAndEndpoint(t *testing.T) { + cases := []struct { + name string + v *VisionConfig + want bool + }{ + {"absent", nil, false}, + {"endpoint but not enabled", &VisionConfig{Endpoint: "http://127.0.0.1:8081"}, false}, + {"enabled but no endpoint", &VisionConfig{Enabled: true}, false}, + {"enabled, blank endpoint", &VisionConfig{Enabled: true, Endpoint: " "}, false}, + {"both", &VisionConfig{Enabled: true, Endpoint: "http://127.0.0.1:8081"}, true}, + } + for _, c := range cases { + if got := c.v.LooksAtImages(); got != c.want { + t.Errorf("%s: LooksAtImages() = %v, want %v", c.name, got, c.want) + } + } +} + +func TestSensesBlocksParseFromJSON(t *testing.T) { + raw := `{ + "db_path": "/tmp/x.db", + "socket_path": "/tmp/x.sock", + "media": {"dir": "media", "retention": "48h", "max_bytes": 1048576}, + "vision": { + "enabled": true, + "endpoint": "http://127.0.0.1:8081", + "model": "qwen2.5-vl", + "max_dim": 640, + "max_tokens": 200, + "timeout": "45s", + "prompt": "Что тут?" + } + }` + var cfg Config + if err := json.Unmarshal([]byte(raw), &cfg); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if cfg.Media.StoreDir() != "media" { + t.Errorf("media dir = %q", cfg.Media.StoreDir()) + } + if time.Duration(cfg.Media.Retention) != 48*time.Hour { + t.Errorf("retention = %v", time.Duration(cfg.Media.Retention)) + } + if cfg.Media.MaxBytes != 1<<20 { + t.Errorf("max_bytes = %d", cfg.Media.MaxBytes) + } + if !cfg.Vision.LooksAtImages() { + t.Fatal("vision did not parse as enabled") + } + if cfg.Vision.MaxDim != 640 || cfg.Vision.MaxTokens != 200 { + t.Errorf("vision limits = %+v", cfg.Vision) + } + if time.Duration(cfg.Vision.Timeout) != 45*time.Second { + t.Errorf("vision timeout = %v", time.Duration(cfg.Vision.Timeout)) + } + if cfg.Vision.Prompt != "Что тут?" { + t.Errorf("prompt = %q", cfg.Vision.Prompt) + } +} + +// A media dir set with no vision block is a valid state, and the useful one on a +// box with no vision model: images can be kept, they just cannot be described. +func TestMediaWithoutVisionIsValid(t *testing.T) { + var cfg Config + if err := json.Unmarshal([]byte(`{"media":{"dir":"/srv/media"}}`), &cfg); err != nil { + t.Fatal(err) + } + if cfg.Media.StoreDir() != "/srv/media" { + t.Errorf("dir = %q", cfg.Media.StoreDir()) + } + if cfg.Vision.LooksAtImages() { + t.Error("vision came on by itself") + } +} + +// A voiceprint is a biometric of a named person. Nothing about it turns on by +// itself: no speaker block means no recognition, and no enrolment either. +func TestSpeakerIsOffUntilExplicitlyEnabled(t *testing.T) { + var cfg Config + if err := json.Unmarshal([]byte(`{}`), &cfg); err != nil { + t.Fatal(err) + } + if cfg.Speaker.Recognizes() { + t.Error("speaker recognition came on with no config at all") + } + var empty Config + if err := json.Unmarshal([]byte(`{"speaker":{}}`), &empty); err != nil { + t.Fatal(err) + } + if empty.Speaker.Recognizes() { + t.Error("an empty speaker block enabled recognition") + } +} + +// Enabled alone is not enough: recognition needs a model, and on this box there +// is none. Recognizes() must stay false so the daemon reports the honest state +// instead of claiming a capability it cannot perform. +func TestSpeakerNeedsBothEnabledAndAModel(t *testing.T) { + var cfg Config + if err := json.Unmarshal([]byte(`{"speaker":{"enabled":true}}`), &cfg); err != nil { + t.Fatal(err) + } + if cfg.Speaker.Recognizes() { + t.Error("enabled with no model_path claimed to recognise") + } + var only Config + if err := json.Unmarshal([]byte(`{"speaker":{"model_path":"/opt/x.onnx"}}`), &only); err != nil { + t.Fatal(err) + } + if only.Speaker.Recognizes() { + t.Error("a model_path alone enabled recognition") + } +} + +func TestSpeakerBlockParsesFromJSON(t *testing.T) { + const raw = `{"speaker":{"enabled":true,"model_path":"/opt/maven/models/spk/ecapa.onnx",` + + `"lib_path":"/opt/maven/lib","threshold":0.62,"min_seconds":1.5}}` + var cfg Config + if err := json.Unmarshal([]byte(raw), &cfg); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if !cfg.Speaker.Recognizes() { + t.Fatal("speaker did not parse as enabled") + } + if cfg.Speaker.ModelPath != "/opt/maven/models/spk/ecapa.onnx" { + t.Errorf("model_path = %q", cfg.Speaker.ModelPath) + } + if cfg.Speaker.LibPath != "/opt/maven/lib" { + t.Errorf("lib_path = %q", cfg.Speaker.LibPath) + } + if cfg.Speaker.Threshold != 0.62 || cfg.Speaker.MinSeconds != 1.5 { + t.Errorf("thresholds = %+v", cfg.Speaker) + } +} + +// A typo in the vision endpoint, or a media block with no dir, used to be +// logged once at wiring time and the capability just stayed off. A capability +// that silently does not exist is the hardest misconfiguration to notice, so +// both fail at startup now. +func TestSensesBlocksAreValidatedAtStartup(t *testing.T) { + bad := map[string]string{ + "media with no dir": `{"media":{"retention":"48h"}}`, + "negative budget": `{"media":{"dir":"/srv/media","max_total_bytes":-1}}`, + "blob over the budget": `{"media":{"dir":"/srv/media","max_bytes":100,"max_total_bytes":10}}`, + "vision with no media dir": `{"vision":{"enabled":true,"endpoint":"http://127.0.0.1:8081"}}`, + "vision endpoint typo": `{"media":{"dir":"/srv/media"},"vision":{"enabled":true,"endpoint":"127.0.0.1:8081"}}`, + "vision on the wan": `{"media":{"dir":"/srv/media"},"vision":{"enabled":true,"endpoint":"http://8.8.8.8:8081"}}`, + "vision, empty endpoint": `{"media":{"dir":"/srv/media"},"vision":{"enabled":true}}`, + "capture with no store": `{"capture":{"enabled":true}}`, + } + for name, body := range bad { + t.Run(name, func(t *testing.T) { + if _, err := Load(writeConfig(t, body)); err == nil { + t.Fatal("want a startup error") + } + }) + } + good := map[string]string{ + "media alone": `{"media":{"dir":"/srv/media"}}`, + "media + vision": `{"media":{"dir":"/srv/media"},"vision":{"enabled":true,"endpoint":"http://127.0.0.1:8081"}}`, + "media + capture": `{"media":{"dir":"/srv/media"},"capture":{"enabled":true}}`, + "vision off": `{"vision":{"endpoint":"http://8.8.8.8:8081"}}`, + } + for name, body := range good { + t.Run(name, func(t *testing.T) { + if _, err := Load(writeConfig(t, body)); err != nil { + t.Fatalf("valid config refused: %v", err) + } + }) + } +} diff --git a/internal/crawl/crawl.go b/internal/crawl/crawl.go new file mode 100644 index 0000000..a01937f --- /dev/null +++ b/internal/crawl/crawl.go @@ -0,0 +1,254 @@ +// Package crawl reads a web page: fetch, robots check, HTML to text. +// +// It is the last LOCAL-FIRST step, and the ordering is the whole design. +// "Never phones home" is deprecated, but what replaced it puts local sources +// first. Where this actually sits in querySources (cmd/mavend/actions_query.go): +// after his memory and after his notes, and BEFORE the model answers from what +// it remembers. Not after the model, which is what this comment used to claim. +// +// That position is deliberate. A fetch only happens where he named a URL out +// loud, and a named URL is an instruction, not a guess; letting a 1.7B answer +// about a page it cannot read is how a small model invents contents. Kiwix +// (internal/kiwix) is not in the chain yet, so nothing here describes it. +// +// A page's text is written by whoever owns the page. It reaches PhraseQuery as +// context beside his question, and for a watch it becomes a note. It cannot +// reach a tool or an act — the query path executes nothing — but it can steer +// what she says, which is the same trust level as a mail body and lower than +// anything he said himself. +// +// What never leaves the box: his notes, his facts, the persona block, the +// conversation history. Only the URL is requested and, for the on-demand path, +// only because he said it out loud. Nothing here reads the store. +// +// The limits are not in this package — they are in internal/webfetch, which is +// the only way anything here touches a socket: http(s) only, host allow/deny, +// private-address refusal, size cap, redirect cap, per-host rate limit. What +// this package adds is politeness (robots.txt) and dedup. +package crawl + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "net/url" + "strings" + "sync" + "time" +) + +// Errors callers distinguish. +var ( + ErrRobots = errors.New("crawl: robots.txt disallows this path") + ErrNotHTML = errors.New("crawl: response is not html or text") + + // ErrFetchRefused — the fetcher would not go: a denied host, a private + // address, a scheme that is not http(s). The adapter that owns both + // packages (cmd/mavend/crawls.go) maps webfetch's sentinels onto this one, + // so this package tells "off limits" from "no robots.txt here" without + // importing webfetch and without matching on message text. + ErrFetchRefused = errors.New("crawl: the fetcher refused this url") + + // ErrFetchStatus — the server answered, badly (5xx, and anything else + // non-2xx). Separate from ErrFetchRefused because robots treats them + // differently: a broken server is not permission to crawl. + ErrFetchStatus = errors.New("crawl: the server answered with an error status") +) + +// Fetcher is the guarded HTTP door (internal/webfetch adapted by the daemon). An +// interface so this package constructs no http.Client of its own and can be +// tested without a network. +type Fetcher interface { + Get(ctx context.Context, url string) (*Response, error) +} + +// Response is the minimum a crawl needs from a fetch. +type Response struct { + URL string + ContentType string + Body []byte +} + +// Config — crawler knobs. +type Config struct { + // UserAgent is the name matched against robots.txt groups. It must be the + // same string the fetcher sends, or Maven would be claiming one identity + // and obeying the rules for another. + UserAgent string + // MaxRunes caps extracted text. 0 ⇒ DefaultMaxRunes. + MaxRunes int + // RobotsTTL — how long a parsed robots.txt is trusted. 0 ⇒ 1h. + RobotsTTL time.Duration + // Now is injectable for tests. nil ⇒ time.Now. + Now func() time.Time +} + +// Crawler fetches and extracts pages. Safe for concurrent use. +type Crawler struct { + fetch Fetcher + cfg Config + robots *robotsCache + + // mu guards last, the per-host time of the previous fetch. It is what makes + // Crawl-delay real: the fetcher's own limiter is a flat one request per + // host per second and knows nothing about what a site asked for. + mu sync.Mutex + last map[string]time.Time +} + +// robotsTimeout — the robots fetch gets its own, shorter deadline. It shares the +// caller's budget with the page fetch (30s for the on-demand path, against a 20s +// default fetch timeout each), so a slow robots.txt used to eat the page's half +// and he heard "не получилось прочитать страницу" about a site that was fine. +const robotsTimeout = 8 * time.Second + +// New builds a crawler. Returns nil when there is no fetcher, which is how the +// daemon expresses "crawling is off unless configured". +func New(fetch Fetcher, cfg Config) *Crawler { + if fetch == nil { + return nil + } + if cfg.UserAgent == "" { + cfg.UserAgent = "Maven" + } + if cfg.MaxRunes <= 0 { + cfg.MaxRunes = DefaultMaxRunes + } + if cfg.RobotsTTL <= 0 { + cfg.RobotsTTL = time.Hour + } + if cfg.Now == nil { + cfg.Now = time.Now + } + return &Crawler{fetch: fetch, cfg: cfg, robots: newRobotsCache(cfg.RobotsTTL), last: map[string]time.Time{}} +} + +// Page fetches rawURL and returns its text. It checks robots.txt first and +// refuses a disallowed path with ErrRobots — there is no override. +func (c *Crawler) Page(ctx context.Context, rawURL string) (Page, error) { + u, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil { + return Page{}, fmt.Errorf("crawl: bad url %q: %w", rawURL, err) + } + rules, err := c.rulesFor(ctx, u) + if err != nil { + return Page{}, err + } + path := u.EscapedPath() + if u.RawQuery != "" { + path += "?" + u.RawQuery + } + if !rules.Allowed(path) { + return Page{}, fmt.Errorf("%w: %s", ErrRobots, u.Path) + } + if err := c.waitCrawlDelay(ctx, u.Host, rules.Delay); err != nil { + return Page{}, err + } + resp, err := c.fetch.Get(ctx, u.String()) + c.markFetched(u.Host) + if err != nil { + return Page{}, err + } + // A PDF or an image is bytes Maven cannot read; saying so beats storing + // binary garbage as a "note". + ct := strings.ToLower(resp.ContentType) + if ct != "" && !strings.Contains(ct, "html") && !strings.Contains(ct, "text/") && + !strings.Contains(ct, "xml") && !strings.Contains(ct, "json") { + return Page{}, fmt.Errorf("%w: %s", ErrNotHTML, resp.ContentType) + } + return Extract(resp.URL, resp.Body, c.cfg.MaxRunes), nil +} + +// rulesFor consults robots.txt for u's host, reading it at most once per TTL. +// +// A robots.txt that is not there (404) means allow, per the standard. What does +// NOT mean allow: a server that answered with an error. The standard asks for +// the opposite there, and "the site is broken, so crawl it" is the wrong way to +// resolve an unknown. +func (c *Crawler) rulesFor(ctx context.Context, u *url.URL) (Rules, error) { + host := u.Host + now := c.cfg.Now() + rules, ok := c.robots.get(host, now) + if ok { + return rules, nil + } + robotsURL := u.Scheme + "://" + host + "/robots.txt" + rctx, cancel := context.WithTimeout(ctx, robotsTimeout) + defer cancel() + resp, err := c.fetch.Get(rctx, robotsURL) + c.markFetched(host) + switch { + case err == nil: + rules = ParseRobots(string(resp.Body), c.cfg.UserAgent) + case errors.Is(err, ErrFetchRefused): + // Not swallowed: if the fetcher says this host is denied or private, + // the page fetch would fail the same way, and the real reason beats a + // robots verdict we never got. + return Rules{}, err + case errors.Is(err, ErrFetchStatus) && isServerError(err): + return Rules{}, fmt.Errorf("%w: robots.txt at %s could not be read", ErrFetchStatus, host) + default: + rules = Rules{} + } + c.robots.put(host, rules, now) + return rules, nil +} + +// waitCrawlDelay honours a Crawl-delay the site asked for. The fetcher's flat +// one-per-host-per-second is the floor and cannot express "30 seconds"; without +// this, deploy/README's claim that Crawl-delay is honoured was false. +// +// It waits on ctx, so a delay longer than the caller's budget fails the read +// rather than blocking a turn. That is the honest outcome: a site that wants a +// minute between requests is not a site to answer a voice question from. +func (c *Crawler) waitCrawlDelay(ctx context.Context, host string, delay time.Duration) error { + if delay <= 0 { + return nil + } + c.mu.Lock() + last, ok := c.last[host] + c.mu.Unlock() + if !ok { + return nil + } + wait := delay - c.cfg.Now().Sub(last) + if wait <= 0 { + return nil + } + t := time.NewTimer(wait) + defer t.Stop() + select { + case <-ctx.Done(): + return fmt.Errorf("crawl: %s asks for %s between requests, longer than this read has: %w", host, delay, ctx.Err()) + case <-t.C: + return nil + } +} + +func (c *Crawler) markFetched(host string) { + c.mu.Lock() + c.last[host] = c.cfg.Now() + c.mu.Unlock() +} + +// isServerError — a 5xx rather than any other non-2xx. The adapter formats the +// status into the message, which is the only place it survives. +func isServerError(err error) bool { + s := err.Error() + for _, code := range []string{" 50", " 51", " 52", " 53"} { + if strings.Contains(s, code) { + return true + } + } + return false +} + +// Hash is the dedup key for a crawl result: the sha256 of the extracted text, +// hex, first 16 chars. Text and not raw HTML, because a page whose only change +// is a rotating ad slot or a CSRF token has not changed. +func Hash(text string) string { + sum := sha256.Sum256([]byte(strings.TrimSpace(text))) + return hex.EncodeToString(sum[:])[:16] +} diff --git a/internal/crawl/crawl_test.go b/internal/crawl/crawl_test.go new file mode 100644 index 0000000..0dbaf4d --- /dev/null +++ b/internal/crawl/crawl_test.go @@ -0,0 +1,155 @@ +package crawl + +import ( + "context" + "errors" + "strings" + "testing" + "time" +) + +// fakeFetcher serves canned pages by URL and counts requests, so a test can +// assert that robots.txt was read once and that a refusal never reached the page. +type fakeFetcher struct { + pages map[string]Response + err error + calls []string +} + +func (f *fakeFetcher) Get(_ context.Context, u string) (*Response, error) { + f.calls = append(f.calls, u) + if f.err != nil { + return nil, f.err + } + r, ok := f.pages[u] + if !ok { + return nil, errors.New("http 404") + } + if r.URL == "" { + r.URL = u + } + if r.ContentType == "" { + r.ContentType = "text/html; charset=utf-8" + } + return &r, nil +} + +const htmlPage = `Почему небо синее + +

Небо

+

Свет рассеивается на молекулах воздуха.

+

Короткие волны рассеиваются сильнее.

+
© 2026
` + +func newTestCrawler(f *fakeFetcher) *Crawler { + return New(f, Config{UserAgent: "Maven/1.0", Now: func() time.Time { return time.Unix(0, 0) }}) +} + +func TestPageExtractsText(t *testing.T) { + f := &fakeFetcher{pages: map[string]Response{ + "https://example.org/sky": {Body: []byte(htmlPage)}, + }} + page, err := newTestCrawler(f).Page(context.Background(), "https://example.org/sky") + if err != nil { + t.Fatal(err) + } + if page.Title != "Почему небо синее" { + t.Errorf("title = %q", page.Title) + } + if !strings.Contains(page.Text, "Свет рассеивается") { + t.Errorf("body text missing: %q", page.Text) + } + for _, junk := range []string{"track()", "color:red", "меню", "© 2026"} { + if strings.Contains(page.Text, junk) { + t.Errorf("%q survived extraction: %q", junk, page.Text) + } + } +} + +func TestRobotsIsCheckedAndObeyed(t *testing.T) { + f := &fakeFetcher{pages: map[string]Response{ + "https://example.org/robots.txt": {Body: []byte("User-agent: *\nDisallow: /secret\n"), ContentType: "text/plain"}, + "https://example.org/secret/x": {Body: []byte(htmlPage)}, + "https://example.org/open": {Body: []byte(htmlPage)}, + }} + c := newTestCrawler(f) + if _, err := c.Page(context.Background(), "https://example.org/secret/x"); !errors.Is(err, ErrRobots) { + t.Fatalf("error = %v, want ErrRobots", err) + } + for _, u := range f.calls { + if strings.Contains(u, "/secret") { + t.Fatal("the disallowed page was fetched anyway") + } + } + if _, err := c.Page(context.Background(), "https://example.org/open"); err != nil { + t.Fatalf("allowed page: %v", err) + } + // robots.txt was read once for the host, not once per page. + robotsReads := 0 + for _, u := range f.calls { + if strings.HasSuffix(u, "/robots.txt") { + robotsReads++ + } + } + if robotsReads != 1 { + t.Fatalf("robots.txt read %d times, want 1", robotsReads) + } +} + +// No robots.txt means allow — that is the standard, and the alternative makes +// most of the web unreadable. +func TestMissingRobotsAllows(t *testing.T) { + f := &fakeFetcher{pages: map[string]Response{ + "https://example.org/page": {Body: []byte(htmlPage)}, + }} + if _, err := newTestCrawler(f).Page(context.Background(), "https://example.org/page"); err != nil { + t.Fatalf("err = %v, want the page", err) + } +} + +// A refusal from the guarded fetcher must surface as itself, not be laundered +// into "no robots.txt, go ahead". +func TestFetcherRefusalIsNotSwallowed(t *testing.T) { + f := &fakeFetcher{err: errors.New("webfetch: refusing to connect to a private address: 127.0.0.1")} + _, err := newTestCrawler(f).Page(context.Background(), "http://127.0.0.1:9100/mcp") + if err == nil || !strings.Contains(err.Error(), "private address") { + t.Fatalf("error = %v, want the fetcher's refusal", err) + } +} + +func TestNonTextIsRefused(t *testing.T) { + f := &fakeFetcher{pages: map[string]Response{ + "https://example.org/f.pdf": {Body: []byte("%PDF-1.7"), ContentType: "application/pdf"}, + }} + if _, err := newTestCrawler(f).Page(context.Background(), "https://example.org/f.pdf"); !errors.Is(err, ErrNotHTML) { + t.Fatalf("error = %v, want ErrNotHTML", err) + } +} + +func TestMaxRunesCapsText(t *testing.T) { + long := "

" + strings.Repeat("привет ", 2000) + "

" + f := &fakeFetcher{pages: map[string]Response{"https://example.org/l": {Body: []byte(long)}}} + c := New(f, Config{MaxRunes: 50}) + page, err := c.Page(context.Background(), "https://example.org/l") + if err != nil { + t.Fatal(err) + } + if n := len([]rune(page.Text)); n > 51 { + t.Fatalf("text = %d runes, want the 50-rune cap", n) + } +} + +func TestNewWithoutFetcherIsNil(t *testing.T) { + if New(nil, Config{}) != nil { + t.Fatal("a crawler with no fetcher must be nil — crawling is off unless configured") + } +} + +func TestHashIgnoresNothingButText(t *testing.T) { + if Hash("a") == Hash("b") { + t.Fatal("different text hashed the same") + } + if Hash(" same \n") != Hash("same") { + t.Fatal("surrounding whitespace changed the hash") + } +} diff --git a/internal/crawl/extract.go b/internal/crawl/extract.go new file mode 100644 index 0000000..5828722 --- /dev/null +++ b/internal/crawl/extract.go @@ -0,0 +1,106 @@ +package crawl + +import ( + "html" + "regexp" + "strings" +) + +// HTML → text, with a regexp and no tokenizer. +// +// golang.org/x/net/html is not vendored and the network is not assumed, so this +// is stdlib. That is less of a compromise than it sounds: the unit of context +// here is a few hundred words for a 4096-token model to read, exactly like the +// Kiwix snippet, so what matters is dropping script/style/nav noise and keeping +// paragraph boundaries. A DOM would buy correctness on malformed markup that is +// then thrown away by truncation anyway. +// +// What this deliberately does NOT do: run JavaScript, follow links, or extract +// structured fields with CSS selectors or an LLM prompt. The plan's step 2 asked +// for the last of those; see docs/plans/14-web-crawler.md for why it was left +// out for now. + +var ( + // RE2 has no backreferences, so each tag pair is spelled out rather than + // captured and matched against itself. + dropRE = regexp.MustCompile(pairsRE("script", "style", "noscript", "svg", "head", "nav", "footer", "form")) + titleRE = regexp.MustCompile(`(?is)]*>(.*?)`) + h1RE = regexp.MustCompile(`(?is)]*>(.*?)`) + // Block-level tags become newlines so paragraphs survive as paragraphs. + blockRE = regexp.MustCompile(`(?is)]*>`) + tagRE = regexp.MustCompile(`(?s)<[^>]*>`) + commentRE = regexp.MustCompile(`(?s)`) + spaceRE = regexp.MustCompile(`[ \t\f\v]+`) + blankRE = regexp.MustCompile(`\n{2,}`) +) + +// pairsRE builds `(?is)…|…` for the given tags. +func pairsRE(tags ...string) string { + parts := make([]string, 0, len(tags)) + for _, t := range tags { + parts = append(parts, `<`+t+`\b[^>]*>.*?`) + } + return `(?is)` + strings.Join(parts, "|") +} + +// Page is an extracted page. +type Page struct { + URL string + Title string + Text string // plain text, paragraphs separated by single newlines +} + +// Extract turns a fetched HTML document into a Page. maxRunes caps the text (0 ⇒ +// DefaultMaxRunes); the cap is on runes, not bytes, because a Russian page cut +// at a byte boundary ends in half a letter. +func Extract(url string, body []byte, maxRunes int) Page { + if maxRunes <= 0 { + maxRunes = DefaultMaxRunes + } + s := string(body) + s = commentRE.ReplaceAllString(s, " ") + + title := firstGroup(titleRE, s) + if title == "" { + title = firstGroup(h1RE, s) + } + + s = dropRE.ReplaceAllString(s, "\n") + s = blockRE.ReplaceAllString(s, "\n") + s = tagRE.ReplaceAllString(s, " ") + s = html.UnescapeString(s) + s = spaceRE.ReplaceAllString(s, " ") + + var lines []string + for _, l := range strings.Split(s, "\n") { + if l = strings.TrimSpace(l); l != "" { + lines = append(lines, l) + } + } + text := blankRE.ReplaceAllString(strings.Join(lines, "\n"), "\n") + + return Page{URL: url, Title: title, Text: TrimRunes(text, maxRunes)} +} + +// DefaultMaxRunes — how much of a page is kept. ~4000 runes is a long answer's +// worth of context and still leaves room in a 4096-token window for the prompt +// and the reply. +const DefaultMaxRunes = 4000 + +func firstGroup(re *regexp.Regexp, s string) string { + m := re.FindStringSubmatch(s) + if len(m) < 2 { + return "" + } + t := tagRE.ReplaceAllString(m[1], " ") + return strings.TrimSpace(strings.Join(strings.Fields(html.UnescapeString(t)), " ")) +} + +// TrimRunes cuts s to at most max runes, on a rune boundary. +func TrimRunes(s string, max int) string { + r := []rune(s) + if len(r) <= max { + return s + } + return strings.TrimSpace(string(r[:max])) + "…" +} diff --git a/internal/crawl/politeness_test.go b/internal/crawl/politeness_test.go new file mode 100644 index 0000000..3e19309 --- /dev/null +++ b/internal/crawl/politeness_test.go @@ -0,0 +1,115 @@ +package crawl + +import ( + "context" + "errors" + "fmt" + "testing" + "time" +) + +// timedFetcher records when each request was made, so a test can assert a wait +// actually happened rather than that a field was parsed. +type timedFetcher struct { + pages map[string]Response + errs map[string]error + at []time.Time + urls []string +} + +func (f *timedFetcher) Get(_ context.Context, u string) (*Response, error) { + f.at = append(f.at, time.Now()) + f.urls = append(f.urls, u) + if err, ok := f.errs[u]; ok { + return nil, err + } + r, ok := f.pages[u] + if !ok { + return nil, errors.New("http 404: no such page") + } + if r.URL == "" { + r.URL = u + } + if r.ContentType == "" { + r.ContentType = "text/html" + } + return &r, nil +} + +func TestPage_HonoursCrawlDelay(t *testing.T) { + // deploy/README says Crawl-delay is honoured. It was parsed into Rules and + // never read: the only pacing was the fetcher's flat one request per host + // per second, which cannot express what a site asked for. + const delay = 120 * time.Millisecond + f := &timedFetcher{pages: map[string]Response{ + "https://example.org/robots.txt": {Body: []byte(fmt.Sprintf("User-agent: *\nCrawl-delay: %.3f\n", delay.Seconds())), ContentType: "text/plain"}, + "https://example.org/a": {Body: []byte("a")}, + }} + c := New(f, Config{UserAgent: "Maven/1.0"}) + if _, err := c.Page(context.Background(), "https://example.org/a"); err != nil { + t.Fatal(err) + } + if len(f.at) != 2 { + t.Fatalf("requests = %v; want robots.txt then the page", f.urls) + } + if gap := f.at[1].Sub(f.at[0]); gap < delay { + t.Errorf("the page was fetched %s after robots.txt; the site asked for %s", gap, delay) + } +} + +func TestPage_ACrawlDelayLongerThanTheTurnFailsInsteadOfBlocking(t *testing.T) { + f := &timedFetcher{pages: map[string]Response{ + "https://example.org/robots.txt": {Body: []byte("User-agent: *\nCrawl-delay: 30\n"), ContentType: "text/plain"}, + "https://example.org/a": {Body: []byte("a")}, + }} + c := New(f, Config{UserAgent: "Maven/1.0"}) + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) + defer cancel() + if _, err := c.Page(ctx, "https://example.org/a"); err == nil { + t.Fatal("a 30-second Crawl-delay was ignored inside a turn that cannot wait that long") + } + if len(f.urls) != 1 { + t.Errorf("requests = %v; the page must not be fetched before the wait it refused", f.urls) + } +} + +func TestPage_ABrokenRobotsServerIsNotPermissionToCrawl(t *testing.T) { + // A 404 means unrestricted, per the standard. A 500 does not: the standard + // asks for the opposite, and "the site is broken, so read it" is the wrong + // way to resolve an unknown. + f := &timedFetcher{ + pages: map[string]Response{"https://example.org/a": {Body: []byte("a")}}, + errs: map[string]error{"https://example.org/robots.txt": fmt.Errorf("%w: 503", ErrFetchStatus)}, + } + c := New(f, Config{UserAgent: "Maven/1.0"}) + if _, err := c.Page(context.Background(), "https://example.org/a"); !errors.Is(err, ErrFetchStatus) { + t.Fatalf("Page over a 503 robots.txt = %v; want a refusal", err) + } + if len(f.urls) != 1 { + t.Errorf("requests = %v; the page was read anyway", f.urls) + } +} + +func TestPage_AFetcherRefusalIsReportedAsItself(t *testing.T) { + // The old check matched three substrings of webfetch's message from a + // package that cannot import webfetch. The sentinel is mapped by the + // adapter that owns both (cmd/mavend/crawls.go). + f := &timedFetcher{errs: map[string]error{ + "https://example.org/robots.txt": fmt.Errorf("%w: host is not allowed", ErrFetchRefused), + }} + c := New(f, Config{UserAgent: "Maven/1.0"}) + if _, err := c.Page(context.Background(), "https://example.org/a"); !errors.Is(err, ErrFetchRefused) { + t.Fatalf("Page = %v; want the fetcher's own refusal, not a robots verdict", err) + } +} + +func TestParseRobots_MostSpecificAgentWinsRegardlessOfOrder(t *testing.T) { + body := "User-agent: maven\nDisallow: /private\n\nUser-agent: mav\nDisallow: /\n" + r := ParseRobots(body, "maven/1.0") + if !r.Allowed("/public") { + t.Error("the shorter agent group won by file order; the longer prefix is the more specific match") + } + if r.Allowed("/private") { + t.Error("the group that names us was not applied") + } +} diff --git a/internal/crawl/robots.go b/internal/crawl/robots.go new file mode 100644 index 0000000..2849721 --- /dev/null +++ b/internal/crawl/robots.go @@ -0,0 +1,216 @@ +package crawl + +import ( + "regexp" + "strings" + "sync" + "time" +) + +// robots.txt, parsed the small way: no wildcards beyond the two the standard +// actually defines (`*` inside a path and `$` at the end), no sitemaps, no +// crawl-delay-per-agent gymnastics. A personal assistant reading a handful of +// pages does not need a spec-complete implementation; it needs to not be rude, +// and to be auditable in one sitting. +// +// Two rules worth stating because they are choices, not accidents: +// +// - a missing or unreadable robots.txt means ALLOW. That is what the standard +// says (404 ⇒ unrestricted), and the alternative would make a site that +// simply has no robots.txt unreadable; +// - an explicit Disallow means REFUSE, and Maven does not offer an override. +// There is no "but he asked me to" flag: the page is not read. + +// Rules is a parsed robots.txt for one user-agent. +type Rules struct { + allow []string + disallow []string + // Delay is Crawl-delay in seconds when the group named one, 0 otherwise. + // The fetcher's own per-host rate limit is the floor; this can only make + // Maven slower, never faster. Enforced in Crawler.waitCrawlDelay — the + // fetcher's limiter is flat and cannot express what a site asked for. + Delay time.Duration +} + +// ParseRobots reads robots.txt and returns the rules that apply to agent. +// +// Group selection follows the standard: the most specific matching group wins, +// which here means an exact user-agent match beats `*`. Lines that are neither +// are ignored rather than guessed at. +func ParseRobots(body string, agent string) Rules { + agent = strings.ToLower(agent) + + type group struct { + agents []string + allow []string + disallow []string + delay time.Duration + } + var groups []group + var cur *group + // startNew tracks whether the next User-agent line opens a new group or + // joins the current one: consecutive User-agent lines share their rules. + startNew := true + + for _, raw := range strings.Split(body, "\n") { + line := raw + if i := strings.IndexByte(line, '#'); i >= 0 { + line = line[:i] + } + line = strings.TrimSpace(line) + if line == "" { + continue + } + key, val, ok := strings.Cut(line, ":") + if !ok { + continue + } + key = strings.ToLower(strings.TrimSpace(key)) + val = strings.TrimSpace(val) + + switch key { + case "user-agent": + if startNew || cur == nil { + groups = append(groups, group{}) + cur = &groups[len(groups)-1] + startNew = false + } + cur.agents = append(cur.agents, strings.ToLower(val)) + case "disallow": + if cur == nil { + continue + } + startNew = true + // "Disallow:" with an empty value allows everything, and is not a + // path rule at all. + if val != "" { + cur.disallow = append(cur.disallow, val) + } + case "allow": + if cur == nil { + continue + } + startNew = true + if val != "" { + cur.allow = append(cur.allow, val) + } + case "crawl-delay": + if cur == nil { + continue + } + startNew = true + if d, err := time.ParseDuration(val + "s"); err == nil && d > 0 { + cur.delay = d + } + } + } + + // Most specific wins, and specificity is the LENGTH of the matching agent + // string, not the file order. Two groups naming "mav" and "maven" used to be + // resolved by whichever came last in the file. + var star, exact *group + best := 0 + for i := range groups { + for _, a := range groups[i].agents { + if a == "*" && star == nil { + star = &groups[i] + } + // A robots.txt names "maven", we send "Maven/1.0 (…)": match on + // prefix, which is how every crawler reads this field. + if a != "*" && a != "" && strings.HasPrefix(agent, a) && len(a) > best { + best, exact = len(a), &groups[i] + } + } + } + g := exact + if g == nil { + g = star + } + if g == nil { + return Rules{} + } + return Rules{allow: g.allow, disallow: g.disallow, Delay: g.delay} +} + +// Allowed reports whether path may be fetched. Longest matching rule wins, and +// Allow beats Disallow at equal length — the standard's tie-break, and the one +// that makes "Disallow: /" plus "Allow: /public" mean what it looks like. +func (r Rules) Allowed(path string) bool { + if path == "" { + path = "/" + } + best, allowed := -1, true + for _, p := range r.disallow { + if n, ok := matchPath(p, path); ok && n > best { + best, allowed = n, false + } + } + for _, p := range r.allow { + if n, ok := matchPath(p, path); ok && n >= best { + best, allowed = n, true + } + } + return allowed +} + +// matchPath applies a robots path pattern and returns the pattern's length as +// the specificity score. `*` matches any run of characters, `$` anchors the end. +// A pattern is a PREFIX match otherwise, which is what "Disallow: /admin" means. +func matchPath(pattern, path string) (int, bool) { + score := len(pattern) + re, err := robotsRegexp(pattern) + if err != nil { + return 0, false + } + return score, re.MatchString(path) +} + +// robotsRegexp turns a robots path pattern into an anchored-at-the-start +// regexp. Everything but `*` and a trailing `$` is a literal, so the pattern is +// quoted first and the two metacharacters are put back afterwards. +func robotsRegexp(pattern string) (*regexp.Regexp, error) { + end := "" + if strings.HasSuffix(pattern, "$") { + pattern = strings.TrimSuffix(pattern, "$") + end = "$" + } + parts := strings.Split(pattern, "*") + for i, p := range parts { + parts[i] = regexp.QuoteMeta(p) + } + return regexp.Compile("^" + strings.Join(parts, ".*") + end) +} + +// robotsCache holds parsed rules per host so a crawl of ten pages on one site +// reads robots.txt once. TTL because a site may change its mind, and a daemon +// that runs for weeks would otherwise never notice. +type robotsCache struct { + ttl time.Duration + mu sync.Mutex + m map[string]robotsEntry +} + +type robotsEntry struct { + rules Rules + at time.Time +} + +func newRobotsCache(ttl time.Duration) *robotsCache { + return &robotsCache{ttl: ttl, m: map[string]robotsEntry{}} +} + +func (c *robotsCache) get(host string, now time.Time) (Rules, bool) { + c.mu.Lock() + defer c.mu.Unlock() + e, ok := c.m[host] + if !ok || now.Sub(e.at) > c.ttl { + return Rules{}, false + } + return e.rules, true +} + +func (c *robotsCache) put(host string, r Rules, now time.Time) { + c.mu.Lock() + defer c.mu.Unlock() + c.m[host] = robotsEntry{rules: r, at: now} +} diff --git a/internal/crawl/robots_test.go b/internal/crawl/robots_test.go new file mode 100644 index 0000000..93e6bb9 --- /dev/null +++ b/internal/crawl/robots_test.go @@ -0,0 +1,84 @@ +package crawl + +import ( + "testing" + "time" +) + +const robotsBody = `# a comment +User-agent: * +Disallow: /private +Disallow: /tmp/ +Crawl-delay: 5 + +User-agent: Maven +Disallow: / +Allow: /public +` + +func TestParseRobotsPicksTheMostSpecificGroup(t *testing.T) { + // The Maven group applies to us even though we send a longer UA string. + r := ParseRobots(robotsBody, "Maven/1.0 (self-hosted personal assistant)") + if r.Allowed("/anything") { + t.Error("Disallow: / in our own group was ignored") + } + if !r.Allowed("/public/page") { + t.Error("Allow: /public must beat the shorter Disallow: /") + } + + // A different agent falls into the * group. + star := ParseRobots(robotsBody, "SomeoneElse/2") + if !star.Allowed("/anything") { + t.Error("the * group disallows nothing but /private and /tmp/") + } + if star.Allowed("/private/x") || star.Allowed("/tmp/") { + t.Error("the * group's disallows were not applied") + } + if star.Delay != 5*time.Second { + t.Errorf("crawl-delay = %v, want 5s", star.Delay) + } +} + +func TestParseRobotsEmptyMeansAllowAll(t *testing.T) { + for _, body := range []string{"", "# nothing here\n", "User-agent: *\nDisallow:\n"} { + if !ParseRobots(body, "Maven").Allowed("/whatever") { + t.Errorf("body %q must allow everything", body) + } + } +} + +func TestRobotsWildcards(t *testing.T) { + r := ParseRobots("User-agent: *\nDisallow: /*.pdf$\nDisallow: /a/*/secret\n", "Maven") + if r.Allowed("/docs/manual.pdf") { + t.Error("*.pdf$ did not match") + } + if !r.Allowed("/docs/manual.pdf.html") { + t.Error("$ must anchor at the end") + } + if r.Allowed("/a/b/secret") { + t.Error("/a/*/secret did not match") + } + if !r.Allowed("/a/b/public") { + t.Error("unrelated path was refused") + } +} + +// Consecutive User-agent lines share one group, which is common in the wild. +func TestRobotsSharedGroup(t *testing.T) { + r := ParseRobots("User-agent: Googlebot\nUser-agent: Maven\nDisallow: /x\n", "Maven/1.0") + if r.Allowed("/x/y") { + t.Fatal("a shared group's rules were not applied to the second agent") + } +} + +func TestRobotsCacheTTL(t *testing.T) { + c := newRobotsCache(time.Minute) + now := time.Now() + c.put("example.com", ParseRobots("User-agent: *\nDisallow: /\n", "Maven"), now) + if _, ok := c.get("example.com", now.Add(30*time.Second)); !ok { + t.Error("a fresh entry must be served from cache") + } + if _, ok := c.get("example.com", now.Add(2*time.Minute)); ok { + t.Error("an expired entry must be re-read") + } +} diff --git a/internal/crawl/watch.go b/internal/crawl/watch.go new file mode 100644 index 0000000..704cef7 --- /dev/null +++ b/internal/crawl/watch.go @@ -0,0 +1,177 @@ +package crawl + +import ( + "context" + "fmt" + "log" + "strings" + "time" +) + +// Scheduled crawls: a page is re-read on an interval, and when its TEXT changed +// the new text is written as a note. Nothing is dispatched — same rule as the +// feed poller (Vikunja #258). A page that announced its own change would be a +// nag, and "the docs page changed" is not worth interrupting anyone for. +// +// Dedup is by content hash, so a page that re-renders identically writes nothing +// and a rotating ad slot does not count as news. + +// WatchConfig — one page to keep an eye on. +type WatchConfig struct { + Name string // note source is "crawl:" + URL string // http(s), guarded by the fetcher + Interval time.Duration // 0 ⇒ Watcher's default +} + +// Notes is core's note-writing half (same shape as ipc.CoreAPI's method). +type Notes interface { + WriteNote(ctx context.Context, ts time.Time, text string, embedding []float32, source string) (int64, error) +} + +// Hashes remembers the last text hash per watch, durably, so a restart does not +// re-note an unchanged page. The daemon backs this with config facts +// ("crawl:hash:"). +type Hashes interface { + LastHash(ctx context.Context, name string) (string, error) + SetHash(ctx context.Context, name, hash string) error +} + +// Embedder embeds a note on its way into the store. nil ⇒ no vector. +type Embedder interface { + Embed(ctx context.Context, text string) ([]float32, error) +} + +// DefaultWatchInterval — pages change slowly, and every check is a request in +// someone's log. +const DefaultWatchInterval = 6 * time.Hour + +// Watcher re-reads watched pages on their interval. +type Watcher struct { + c *Crawler + watches []WatchConfig + notes Notes + hashes Hashes + embed Embedder + interval time.Duration + nextDue map[string]time.Time +} + +// NewWatcher wires the scheduled half, or returns nil when there is nothing to +// watch. Callers check for nil: no watches, no goroutine, no request. +func NewWatcher(c *Crawler, watches []WatchConfig, notes Notes, hashes Hashes, embed Embedder, defaultInterval time.Duration) *Watcher { + if c == nil || notes == nil { + return nil + } + var valid []WatchConfig + for _, w := range watches { + if strings.TrimSpace(w.Name) == "" || strings.TrimSpace(w.URL) == "" { + log.Printf("crawl: skipping a watch with no name or no url") + continue + } + valid = append(valid, w) + } + if len(valid) == 0 { + return nil + } + if defaultInterval <= 0 { + defaultInterval = DefaultWatchInterval + } + return &Watcher{ + c: c, watches: valid, notes: notes, hashes: hashes, embed: embed, + interval: defaultInterval, nextDue: map[string]time.Time{}, + } +} + +// Watches returns the configured watches. +func (w *Watcher) Watches() []WatchConfig { return w.watches } + +// CheckDue re-reads every watch whose interval elapsed and returns how many +// notes were written. Errors are logged per watch, never returned: one dead page +// must not stop the others. +func (w *Watcher) CheckDue(ctx context.Context, now time.Time) int { + written := 0 + for _, watch := range w.watches { + if due, ok := w.nextDue[watch.Name]; ok && now.Before(due) { + continue + } + interval := watch.Interval + if interval <= 0 { + interval = w.interval + } + w.nextDue[watch.Name] = now.Add(interval) + changed, err := w.Check(ctx, watch, now) + if err != nil { + log.Printf("crawl: watch %s: %v", watch.Name, err) + continue + } + if changed { + log.Printf("crawl: watch %s: page changed, noted", watch.Name) + written++ + } + } + return written +} + +// Check re-reads one watch now and reports whether it wrote a note. +func (w *Watcher) Check(ctx context.Context, watch WatchConfig, now time.Time) (bool, error) { + page, err := w.c.Page(ctx, watch.URL) + if err != nil { + return false, err + } + // Title included: a page whose headline changed has changed. + h := Hash(page.Title + "\n" + page.Text) + if w.hashes != nil { + prev, err := w.hashes.LastHash(ctx, watch.Name) + if err != nil { + log.Printf("crawl: watch %s: read hash: %v", watch.Name, err) + } + if prev == h { + return false, nil + } + } + text := NoteText(watch, page) + var vec []float32 + if w.embed != nil { + v, err := w.embed.Embed(ctx, text) + if err != nil { + log.Printf("crawl: watch %s: embed: %v", watch.Name, err) + } else { + vec = v + } + } + if _, err := w.notes.WriteNote(ctx, now, text, vec, SourceFor(watch.Name)); err != nil { + return false, fmt.Errorf("write note: %w", err) + } + if w.hashes != nil { + if err := w.hashes.SetHash(ctx, watch.Name, h); err != nil { + log.Printf("crawl: watch %s: save hash: %v", watch.Name, err) + } + } + return true, nil +} + +// SourceFor is the note source for a watch, and SourcePrefix is what the answer +// path matches to recognise one. +func SourceFor(name string) string { return SourcePrefix + name } + +// SourcePrefix — provenance for anything read off the network on a schedule. +const SourcePrefix = "crawl:" + +// noteRunes — how much of a watched page goes into a note. Shorter than what the +// on-demand path reads: a note is a record of a change, not an archive. +const noteRunes = 800 + +// NoteText renders a watched page as a note body. +func NoteText(watch WatchConfig, page Page) string { + var b strings.Builder + if page.Title != "" { + b.WriteString(page.Title) + } else { + b.WriteString(watch.Name) + } + b.WriteString("\n") + b.WriteString(TrimRunes(page.Text, noteRunes)) + b.WriteString("\n") + b.WriteString(watch.URL) + return b.String() +} diff --git a/internal/crawl/watch_test.go b/internal/crawl/watch_test.go new file mode 100644 index 0000000..1f266fd --- /dev/null +++ b/internal/crawl/watch_test.go @@ -0,0 +1,117 @@ +package crawl + +import ( + "context" + "strings" + "testing" + "time" +) + +type note struct { + text string + source string +} + +type fakeNotes struct{ notes []note } + +func (n *fakeNotes) WriteNote(_ context.Context, _ time.Time, text string, _ []float32, source string) (int64, error) { + n.notes = append(n.notes, note{text, source}) + return int64(len(n.notes)), nil +} + +type fakeHashes struct{ m map[string]string } + +func newHashes() *fakeHashes { return &fakeHashes{m: map[string]string{}} } +func (f *fakeHashes) LastHash(_ context.Context, name string) (string, error) { + return f.m[name], nil +} +func (f *fakeHashes) SetHash(_ context.Context, name, h string) error { f.m[name] = h; return nil } + +var t0 = time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC) + +func TestWatchNotesAChangedPage(t *testing.T) { + f := &fakeFetcher{pages: map[string]Response{ + "https://example.org/docs": {Body: []byte(htmlPage)}, + }} + notes := &fakeNotes{} + hashes := newHashes() + w := NewWatcher(newTestCrawler(f), []WatchConfig{{Name: "docs", URL: "https://example.org/docs"}}, + notes, hashes, nil, time.Hour) + if w == nil { + t.Fatal("NewWatcher returned nil for a configured watch") + } + if n := w.CheckDue(context.Background(), t0); n != 1 { + t.Fatalf("first check wrote %d notes, want 1", n) + } + if notes.notes[0].source != "crawl:docs" { + t.Errorf("source = %q, want crawl:docs", notes.notes[0].source) + } + if !strings.Contains(notes.notes[0].text, "https://example.org/docs") { + t.Errorf("note does not carry the url: %q", notes.notes[0].text) + } + + // Unchanged page, interval elapsed: nothing written. + if n := w.CheckDue(context.Background(), t0.Add(2*time.Hour)); n != 0 { + t.Fatalf("an unchanged page wrote %d notes", n) + } + + // Changed page: one note. + f.pages["https://example.org/docs"] = Response{Body: []byte(strings.Replace(htmlPage, "синее", "серое", 1))} + if n := w.CheckDue(context.Background(), t0.Add(4*time.Hour)); n != 1 { + t.Fatalf("a changed page wrote %d notes, want 1", n) + } +} + +func TestWatchIntervalIsRespected(t *testing.T) { + f := &fakeFetcher{pages: map[string]Response{"https://example.org/d": {Body: []byte(htmlPage)}}} + w := NewWatcher(newTestCrawler(f), []WatchConfig{{Name: "d", URL: "https://example.org/d", Interval: time.Hour}}, + &fakeNotes{}, newHashes(), nil, 0) + w.CheckDue(context.Background(), t0) + before := len(f.calls) + w.CheckDue(context.Background(), t0.Add(time.Minute)) + if len(f.calls) != before { + t.Fatal("the page was re-read inside its interval") + } +} + +// The hash is durable so a restart does not re-note an unchanged page. +func TestWatchHashSurvivesRestart(t *testing.T) { + f := &fakeFetcher{pages: map[string]Response{"https://example.org/d": {Body: []byte(htmlPage)}}} + hashes := newHashes() + watches := []WatchConfig{{Name: "d", URL: "https://example.org/d"}} + NewWatcher(newTestCrawler(f), watches, &fakeNotes{}, hashes, nil, time.Hour).CheckDue(context.Background(), t0) + + notes2 := &fakeNotes{} + NewWatcher(newTestCrawler(f), watches, notes2, hashes, nil, time.Hour).CheckDue(context.Background(), t0.Add(time.Hour)) + if len(notes2.notes) != 0 { + t.Fatalf("a fresh watcher re-noted an unchanged page: %q", notes2.notes[0].text) + } +} + +func TestWatchDeadPageDoesNotStopTheOthers(t *testing.T) { + f := &fakeFetcher{pages: map[string]Response{"https://example.org/live": {Body: []byte(htmlPage)}}} + notes := &fakeNotes{} + w := NewWatcher(newTestCrawler(f), []WatchConfig{ + {Name: "dead", URL: "https://example.org/gone"}, + {Name: "live", URL: "https://example.org/live"}, + }, notes, newHashes(), nil, time.Hour) + if n := w.CheckDue(context.Background(), t0); n != 1 { + t.Fatalf("wrote %d notes, want 1 (the live page)", n) + } + if notes.notes[0].source != "crawl:live" { + t.Fatalf("source = %q", notes.notes[0].source) + } +} + +func TestNoWatchesMeansNoWatcher(t *testing.T) { + c := newTestCrawler(&fakeFetcher{}) + if NewWatcher(c, nil, &fakeNotes{}, nil, nil, 0) != nil { + t.Fatal("no watches must mean no watcher") + } + if NewWatcher(nil, []WatchConfig{{Name: "a", URL: "u"}}, &fakeNotes{}, nil, nil, 0) != nil { + t.Fatal("no crawler must mean no watcher") + } + if NewWatcher(c, []WatchConfig{{Name: "", URL: ""}}, &fakeNotes{}, nil, nil, 0) != nil { + t.Fatal("a watch with no name or url is not a configuration") + } +} diff --git a/internal/email/charset.go b/internal/email/charset.go new file mode 100644 index 0000000..f70718a --- /dev/null +++ b/internal/email/charset.go @@ -0,0 +1,48 @@ +package email + +// windows-1251 (and its ASCII-compatible low half) is decoded here rather than +// pulled in from x/text. +// +// The alternative was returning an error for the charset, which ParseMessage +// turns into a subject-only message. That is a live gap and not a small one: +// cp1251 is still what older Russian senders emit, and subject-only means those +// mails can never produce a task candidate. The whole of x/text/encoding is a +// large dependency for the most privacy-sensitive path in the tree, and +// windows-1251 is a 128-entry table. +// +// Only cp1251 is added. Guessing at an unknown charset stays forbidden: mojibake +// is worse than nothing, because the model extracts a task from it happily. + +// cp1251High — the 0x80..0xFF half of windows-1251. The low half is ASCII. +var cp1251High = [128]rune{ + 0x0402, 0x0403, 0x201A, 0x0453, 0x201E, 0x2026, 0x2020, 0x2021, + 0x20AC, 0x2030, 0x0409, 0x2039, 0x040A, 0x040C, 0x040B, 0x040F, + 0x0452, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, + 0xFFFD, 0x2122, 0x0459, 0x203A, 0x045A, 0x045C, 0x045B, 0x045F, + 0x00A0, 0x040E, 0x045E, 0x0408, 0x00A4, 0x0490, 0x00A6, 0x00A7, + 0x0401, 0x00A9, 0x0404, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x0407, + 0x00B0, 0x00B1, 0x0406, 0x0456, 0x0491, 0x00B5, 0x00B6, 0x00B7, + 0x0451, 0x2116, 0x0454, 0x00BB, 0x0458, 0x0405, 0x0455, 0x0457, + 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, + 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F, + 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, + 0x0428, 0x0429, 0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F, + 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, + 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F, + 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, + 0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D, 0x044E, 0x044F, +} + +// decodeCP1251 maps each byte through the table. Every byte has a defined +// meaning in this charset, so decoding cannot fail. +func decodeCP1251(b []byte) string { + out := make([]rune, 0, len(b)) + for _, c := range b { + if c < 0x80 { + out = append(out, rune(c)) + continue + } + out = append(out, cp1251High[c-0x80]) + } + return string(out) +} diff --git a/internal/email/export_test.go b/internal/email/export_test.go new file mode 100644 index 0000000..2fda570 --- /dev/null +++ b/internal/email/export_test.go @@ -0,0 +1,14 @@ +package email + +import "time" + +// WithDial sets the connection seam for a test. It lives in a _test.go file so +// the seam has no linker symbol in the shipped binary: no code outside this +// package can hand FetchSince a dialer, and therefore no code outside this +// package can point the mail reader at a cleartext transport and give it his +// password. The compiler is what enforces that, which is the whole reason the +// field is unexported. +func (f FetchSince) WithDial(d func(addr string, timeout time.Duration) (*Conn, error)) FetchSince { + f.dial = d + return f +} diff --git a/internal/email/extract.go b/internal/email/extract.go new file mode 100644 index 0000000..b68d6a8 --- /dev/null +++ b/internal/email/extract.go @@ -0,0 +1,244 @@ +package email + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/kami/maven/internal/llm" + "github.com/kami/maven/internal/persona" +) + +// Extraction — turning one mail into task CANDIDATES, and nothing else. +// +// The output of this file can only ever become rows in `tasks` with status +// "candidate" (store.TaskCandidate), written through the one intake seam +// (ipc.CaptureTaskReq, Vikunja #130). That bound is the whole design: +// +// - No reminder. A reminder FIRES; it speaks to him unprompted. A 1.7B that +// misreads "встреча была в четверг" as a future appointment would then wake +// him up about it. A candidate that is wrong is a line on a review page he +// dismisses in one click, which is the correct cost of a model being wrong +// about someone's mail. +// - No fact. A fact is a claim Maven will later recite as true. Nothing read +// out of a marketing mail deserves that standing. +// - No calendar event, no note, no action. Extraction writes candidates or +// writes nothing. +// +// The due date the model may return is stored on the candidate (tasks.due_ts), +// which no scheduler reads — it is there so the review page can sort by it. +// +// Privacy: the mail text goes to the resident model on this box and nowhere +// else. It is never search input (CLAUDE.md: "his notes and facts are never +// search input" — mail is the same class), and Evidence keeps only the subject +// line, so the review page shows him where a candidate came from without the +// store growing a copy of his mailbox. +// +// One constraint for whoever adds task context to a prompt later: a candidate's +// text is a model paraphrase of the content of his mail, and it lives in +// tasks.text. "Maven never sends his mail anywhere" holds today because nothing +// assembles a context block out of live tasks. The moment something does, mail +// content reaches whatever that block is sent to, and an outbound search would +// be sending his mailbox out a paraphrase at a time. Tasks sourced "email:" have +// to be excluded there, not here. + +// MaxCandidates — at most this many candidates per message, enforced by the +// grammar. A mail with four tasks in it is a mail he has to read himself; a +// model allowed ten will produce ten. +const MaxCandidates = 3 + +// SourcePrefix — provenance for everything this package captures. The mailbox +// name is appended: "email:INBOX". Same vocabulary as tap:voice / poll:netdata. +const SourcePrefix = "email:" + +// Candidate — one piece of work the model thinks the mail is asking for. +type Candidate struct { + Text string `json:"text"` + // Due — "YYYY-MM-DD" or empty. A date the model read out of the text, not a + // date it computed: relative wording ("до пятницы") is left in Text, because + // a small model resolving "пятница" against today's date gets it wrong often + // enough that a stored wrong date is worse than no date. + Due string `json:"due"` +} + +// Completer — the llama-server seam, same shape memeval and the router use, so +// the one resident model serves this caller too. +type Completer interface { + Complete(ctx context.Context, r llm.Req) (string, error) +} + +// Extractor reads a message and returns candidates. It holds no store and no +// writer on purpose: this type cannot persist anything, so "extraction never +// acts" is a property of the code, not of a review. +type Extractor struct { + llm Completer + // MaxCandidates — 0 ⇒ MaxCandidates. + max int + // ContextBlock — the shared persona block, optional. Extraction output is + // not spoken, so the persona matters less here than in the phraser; it is + // wired anyway so a candidate reads in her voice on the review page. + contextBlock func() string +} + +func NewExtractor(c Completer, max int, contextBlock func() string) *Extractor { + if max <= 0 || max > MaxCandidates { + max = MaxCandidates + } + return &Extractor{llm: c, max: max, contextBlock: contextBlock} +} + +// Max — the normalised candidate bound. Exported so the daemon logs what it will +// actually allow rather than what the config file said: 0 in the config means +// MaxCandidates here, and logging the raw value said "max 0" and then wrote +// three. +func (e *Extractor) Max() int { return e.max } + +// extractGrammar — GBNF pinning the answer to a bounded array of fixed-shape +// candidates. Same reasoning as memeval's evalGrammar and the router's +// routeGrammar: the shape and the length bound are what keep a small model from +// drifting into prose or spending the token budget repeating one field. +// +// The empty array is reachable, deliberately: most mail contains no task, and a +// model with no way to say "nothing" invents something. +const extractGrammar = ` +root ::= "[" ws (item ("," ws item){0,2})? ws "]" +item ::= "{" ws "\"text\"" ws ":" ws text "," ws "\"due\"" ws ":" ws due ws "}" +text ::= "\"" ([^"\\] | "\\" .){1,120} "\"" +due ::= "\"\"" | "\"" [0-9]{4} "-" [0-9]{2} "-" [0-9]{2} "\"" +ws ::= [ \t\n]* +` + +// extractSystem — the extraction prompt. +// +// Written around the two failure modes a small model has on this task: it +// summarises when asked to extract (turning a mail into "письмо от Антона"), +// and it invents an obligation from any polite closing sentence. Hence the +// insistence on a verb phrase, and the explicit permission to return []. +const extractSystem = `Ты читаешь одно письмо из его почты и достаёшь из него дела, которые письмо от него требует. + +Правила: +- Отвечай ТОЛЬКО массивом JSON. Каждый элемент: {"text": "...", "due": "ГГГГ-ММ-ДД" или ""}. +- text — короткая формулировка дела по-русски, с глаголом: "оплатить счёт за интернет", "отправить акт". Не пересказывай письмо и не описывай его. +- Дело — это то, что должен сделать ОН. Рассылка, реклама, уведомление, отчёт, письмо «просто к сведению» — дел не содержат. +- Если письмо ничего от него не требует, верни пустой массив []. Это нормальный ответ, так бывает чаще всего. +- Ничего не придумывай. Если срока в письме нет — "". +- due заполняй только когда в письме стоит конкретная дата. Слова вроде «до пятницы» оставь в text, дату не вычисляй. +- Максимум три дела. Лучше одно точное, чем три общих.` + +// Extract returns the candidates in one message. +// +// Junk is refused without an LLM call — cheapest possible defence, and the +// reason the header filter exists. An empty message (no subject, no body) is +// likewise not worth a round trip. +// +// A parse failure is an error the caller logs and moves past. It is never +// silently turned into zero candidates, because "the model went off the rails" +// and "the mail contains no task" want different reactions from a human reading +// the log. +func (e *Extractor) Extract(ctx context.Context, msg Message) ([]Candidate, error) { + if msg.Junk { + return nil, nil + } + user := renderForModel(msg) + if user == "" { + return nil, nil + } + raw, err := e.llm.Complete(ctx, llm.Req{ + System: persona.Prepend(e.contextBlock, extractSystem), + User: user, + Grammar: extractGrammar, + MaxTokens: 512, + RepeatPenalty: 1.1, + }) + if err != nil { + return nil, fmt.Errorf("email: extract: %w", err) + } + items, err := parseCandidates(raw) + if err != nil { + // The raw reply is NOT in the error: it is a transformation of his mail, + // and this error reaches the daemon log. + return nil, fmt.Errorf("email: extract: unparsable reply (%d bytes)", len(raw)) + } + out := make([]Candidate, 0, len(items)) + seen := map[string]bool{} + for _, it := range items { + it.Text = strings.TrimSpace(it.Text) + if it.Text == "" { + continue + } + key := strings.ToLower(strings.Join(strings.Fields(it.Text), " ")) + if seen[key] { + continue // the model repeating itself is not two tasks + } + seen[key] = true + if _, ok := ParseDue(it.Due); !ok { + it.Due = "" // a date the grammar allowed but the calendar does not + } + out = append(out, it) + if len(out) >= e.max { + break + } + } + return out, nil +} + +// renderForModel is the user turn: subject, sender and body, labelled. Only +// these three fields — no headers, no recipient list, no message-id, nothing +// that would let the model start reasoning about routing metadata. +func renderForModel(msg Message) string { + var b strings.Builder + if msg.From != "" { + fmt.Fprintf(&b, "От: %s\n", msg.From) + } + if msg.Subject != "" { + fmt.Fprintf(&b, "Тема: %s\n", msg.Subject) + } + if msg.Body != "" { + fmt.Fprintf(&b, "\n%s\n", msg.Body) + } + if msg.Subject == "" && msg.Body == "" { + return "" + } + return b.String() +} + +// parseCandidates decodes the reply and trims a fenced block or stray prose +// around the array. +// +// Through Extract that tolerance is unreachable: extractGrammar pins the first +// token to "[", so the model cannot emit reasoning before it. It is kept for +// callers that pass a raw reply from an ungrammared path, and the note is here +// so the next reader does not conclude that thinking output is expected. +func parseCandidates(raw string) ([]Candidate, error) { + s := strings.TrimSpace(raw) + if i := strings.Index(s, "["); i > 0 { + s = s[i:] + } + if j := strings.LastIndex(s, "]"); j >= 0 { + s = s[:j+1] + } + var out []Candidate + if err := json.Unmarshal([]byte(s), &out); err != nil { + return nil, err + } + return out, nil +} + +// ParseDue turns the model's "YYYY-MM-DD" into a time in UTC. Exported because +// the daemon-side intake stores it on the candidate. +// +// The zero-value/empty case returns ok=false rather than an error: no date is +// the common answer, not a failure. +func ParseDue(s string) (time.Time, bool) { + s = strings.TrimSpace(s) + if s == "" { + return time.Time{}, false + } + t, err := time.Parse("2006-01-02", s) + if err != nil { + return time.Time{}, false + } + return t, true +} diff --git a/internal/email/extract_test.go b/internal/email/extract_test.go new file mode 100644 index 0000000..8f0d26f --- /dev/null +++ b/internal/email/extract_test.go @@ -0,0 +1,143 @@ +package email + +import ( + "context" + "strings" + "testing" + + "github.com/kami/maven/internal/llm" +) + +// fakeLLM returns a canned reply and records the request, so a test can assert +// on the grammar and on what of the mail was sent. +type fakeLLM struct { + reply string + err error + got llm.Req + calls int +} + +func (f *fakeLLM) Complete(_ context.Context, r llm.Req) (string, error) { + f.calls++ + f.got = r + return f.reply, f.err +} + +func msgFor(subject, body string) Message { + return Message{UID: 1, From: "anton@example.org", Subject: subject, Body: body} +} + +func TestExtractCandidates(t *testing.T) { + f := &fakeLLM{reply: `[{"text":"отправить акт","due":""},{"text":"оплатить счёт","due":"2026-08-05"}]`} + e := NewExtractor(f, 0, nil) + got, err := e.Extract(context.Background(), msgFor("Акт и счёт", "Надо отправить акт и оплатить счёт до 5 августа.")) + if err != nil { + t.Fatalf("extract: %v", err) + } + if len(got) != 2 { + t.Fatalf("got %d candidates, want 2: %+v", len(got), got) + } + if got[0].Text != "отправить акт" || got[1].Due != "2026-08-05" { + t.Errorf("candidates = %+v", got) + } + if f.got.Grammar == "" { + t.Error("extraction must be grammar-constrained") + } + // The subject and body go to the model; nothing else about the message does. + if !strings.Contains(f.got.User, "Акт и счёт") || !strings.Contains(f.got.User, "оплатить счёт") { + t.Errorf("user turn = %q", f.got.User) + } +} + +func TestExtractEmptyArrayIsNotAnError(t *testing.T) { + f := &fakeLLM{reply: "[]"} + got, err := NewExtractor(f, 0, nil).Extract(context.Background(), msgFor("FYI", "Просто к сведению.")) + if err != nil || len(got) != 0 { + t.Fatalf("got (%v, %v), want (empty, nil) — no task is the normal answer", got, err) + } +} + +// Junk must never reach the model: the header filter exists so the resident +// model is not spent on newsletters. +func TestExtractSkipsJunkWithoutCallingModel(t *testing.T) { + f := &fakeLLM{reply: `[{"text":"купить всё со скидкой","due":""}]`} + msg := msgFor("Скидки", "Sale!") + msg.Junk = true + got, err := NewExtractor(f, 0, nil).Extract(context.Background(), msg) + if err != nil || got != nil { + t.Fatalf("got (%v, %v), want (nil, nil)", got, err) + } + if f.calls != 0 { + t.Errorf("model called %d times for junk, want 0", f.calls) + } +} + +func TestExtractEmptyMessageIsNotSent(t *testing.T) { + f := &fakeLLM{reply: "[]"} + if _, err := NewExtractor(f, 0, nil).Extract(context.Background(), Message{UID: 3}); err != nil { + t.Fatalf("extract: %v", err) + } + if f.calls != 0 { + t.Errorf("model called %d times for an empty message, want 0", f.calls) + } +} + +func TestExtractCaps(t *testing.T) { + f := &fakeLLM{reply: `[{"text":"a","due":""},{"text":"b","due":""},{"text":"c","due":""}]`} + got, err := NewExtractor(f, 2, nil).Extract(context.Background(), msgFor("s", "b")) + if err != nil { + t.Fatalf("extract: %v", err) + } + if len(got) != 2 { + t.Errorf("got %d, want the configured cap of 2", len(got)) + } +} + +func TestExtractDropsRepeatsAndBadDates(t *testing.T) { + f := &fakeLLM{reply: `[{"text":"Отправить акт","due":"2026-02-31"},{"text":"отправить акт","due":""},{"text":" ","due":""}]`} + got, err := NewExtractor(f, 0, nil).Extract(context.Background(), msgFor("s", "b")) + if err != nil { + t.Fatalf("extract: %v", err) + } + if len(got) != 1 { + t.Fatalf("got %d candidates, want 1 (repeat and blank dropped): %+v", len(got), got) + } + if got[0].Due != "" { + t.Errorf("due = %q, want empty — 2026-02-31 is not a date", got[0].Due) + } +} + +// A Thinking model sometimes wraps the array; and when it emits something +// unparsable the caller must hear about it rather than see "no tasks". +func TestParseCandidatesTolerance(t *testing.T) { + got, err := parseCandidates("думаю... [{\"text\":\"x\",\"due\":\"\"}] всё") + if err != nil || len(got) != 1 || got[0].Text != "x" { + t.Fatalf("got (%+v, %v)", got, err) + } + if _, err := parseCandidates("нет никакого JSON"); err == nil { + t.Error("unparsable output must be an error") + } +} + +func TestExtractParseErrorHidesMailText(t *testing.T) { + f := &fakeLLM{reply: "он просил отправить акт, вот такой ответ"} + _, err := NewExtractor(f, 0, nil).Extract(context.Background(), msgFor("Акт", "секретный текст")) + if err == nil { + t.Fatal("want an error") + } + if strings.Contains(err.Error(), "акт") || strings.Contains(err.Error(), "секретный") { + t.Errorf("error text leaks mail content: %v", err) + } +} + +func TestParseDue(t *testing.T) { + if _, ok := ParseDue(""); ok { + t.Error("empty due must be (zero, false)") + } + if got, ok := ParseDue("2026-08-05"); !ok || got.Year() != 2026 || got.Month() != 8 || got.Day() != 5 { + t.Errorf("ParseDue = (%v, %v)", got, ok) + } + if _, ok := ParseDue("05.08.2026"); ok { + t.Error("a non-ISO date must not parse") + } +} diff --git a/internal/email/fetch.go b/internal/email/fetch.go new file mode 100644 index 0000000..43d3f61 --- /dev/null +++ b/internal/email/fetch.go @@ -0,0 +1,139 @@ +package email + +import ( + "errors" + "fmt" + "time" +) + +// FetchSince is the whole read path in one call: connect, log in, examine the +// mailbox read-only, list what arrived since a date, fetch and parse the ones +// the caller has not seen, log out. +// +// It is a function rather than a long-lived object because a mail poller should +// not hold an authenticated session (and therefore his credential in a live TLS +// state) between polls. Connect, read, drop. +// +// skip decides which UIDs are already known — the poller's seen-set. max bounds +// one poll: a mailbox that received 400 messages overnight must not turn into +// 400 LLM calls, and the newest max are the ones a task could still be hiding +// in. Junk messages are returned too, flagged, so the caller can mark them seen +// without a second protocol round. +type FetchSince struct { + Addr string // host or host:993 + User string + Mailbox string // e.g. "INBOX" + Timeout time.Duration + Since time.Time + Max int + Skip func(uid uint32) bool + // OnSearch, when set, is handed the whole SEARCH result before anything is + // fetched, ascending, seen UIDs included. It is how the poller learns which + // UIDs are still inside the lookback window: anything below the lowest one + // can never be searched for again, and therefore can never be read again. + // Without that the poller cannot tell a UID it has not got to yet from one + // that has aged out of the window. + OnSearch func(uids []uint32) + + // dial — the connection seam, unexported on purpose: see dialer(). Tests + // inside this package set it through export_test.go; nothing outside can. + dial func(addr string, timeout time.Duration) (*Conn, error) +} + +// dial is the connection seam. nil means Dial (implicit TLS); the tests set it +// through export_test.go. It is unexported and there is no exported wrapper +// that takes a dialer, so no code outside this package can point the reader at +// a non-TLS transport and hand it the password. That is a property the compiler +// enforces, not a claim about the callers that happen to exist today. +func (f FetchSince) dialer() func(addr string, timeout time.Duration) (*Conn, error) { + if f.dial != nil { + return f.dial + } + return Dial +} + +// Run performs one read. password is passed here, not stored in the struct, so +// the configuration of a mailbox and the secret for it are never the same value +// sitting in the same place. +// +// One message that cannot be read does not abandon the poll: the rest of the +// mailbox is still worth reading, and returning early meant one oversized or +// unreadable message permanently hid every older message behind it, poll after +// poll. The returned error joins whatever failed, and the messages that did +// come back come back with it. +func (f FetchSince) Run(password string) ([]Message, error) { + if f.Addr == "" || f.User == "" || f.Mailbox == "" { + return nil, fmt.Errorf("email: mailbox not configured (addr/user/mailbox)") + } + // Timeout is validated like the other three fields. Zero disables every + // deadline in the path — net.Dialer{Timeout: 0} and a Conn that never calls + // SetDeadline — so a dead server parks the poller forever on a socket read, + // with his credential live in a TLS state. That is the exact thing the + // connect-read-drop shape exists to avoid. + if f.Timeout <= 0 { + return nil, fmt.Errorf("email: timeout must be positive") + } + dial := f.dialer() + c, err := dial(f.Addr, f.Timeout) + if err != nil { + return nil, err + } + defer c.Close() + if err := c.Login(f.User, password); err != nil { + return nil, err + } + defer c.Logout() + if err := c.Select(f.Mailbox); err != nil { + return nil, err + } + uids, err := c.SearchSince(f.Since) + if err != nil { + return nil, err + } + + if f.OnSearch != nil { + f.OnSearch(uids) + } + + // Newest UIDs first — IMAP hands them back ascending, and when Max clips the + // list the recent mail is what matters. + wanted := make([]uint32, 0, len(uids)) + for i := len(uids) - 1; i >= 0; i-- { + if f.Skip != nil && f.Skip(uids[i]) { + continue + } + wanted = append(wanted, uids[i]) + if f.Max > 0 && len(wanted) >= f.Max { + break + } + } + + out := make([]Message, 0, len(wanted)) + var failed []error + for _, uid := range wanted { + raw, err := c.Fetch(uid) + if errors.Is(err, ErrMessageTooLarge) { + // Too big to read is a permanent verdict, not a failure to retry: + // the message will be the same size next poll. Carried as bulk so + // the poller marks it seen and stops fetching it, exactly like a + // newsletter. Nothing is sent to the model. + out = append(out, Message{UID: uid, Junk: true, JunkReason: "oversize"}) + continue + } + if err != nil { + // The error names the UID, never the message. Collected rather than + // returned, so the messages behind this one are still read. + failed = append(failed, fmt.Errorf("email: fetch uid %d: %w", uid, err)) + continue + } + if len(raw) == 0 { + continue // vanished between SEARCH and FETCH + } + msg, err := ParseMessage(uid, raw) + if err != nil { + continue // unparsable headers — nothing to review, skip silently + } + out = append(out, msg) + } + return out, errors.Join(failed...) +} diff --git a/internal/email/fetch_test.go b/internal/email/fetch_test.go new file mode 100644 index 0000000..f8cd720 --- /dev/null +++ b/internal/email/fetch_test.go @@ -0,0 +1,117 @@ +package email + +import ( + "net" + "strings" + "testing" + "time" +) + +func TestFetchSinceRun(t *testing.T) { + mk := func(subject string) string { + return "Subject: " + subject + "\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nbody\r\n" + } + f := &fakeIMAP{ + uids: []uint32{1, 2, 3}, + msgs: map[uint32]string{1: mk("one"), 2: mk("two"), 3: mk("three")}, + } + fs := FetchSince{ + Addr: "mail.example:993", User: "kami", Mailbox: "INBOX", + Timeout: 5 * time.Second, + Since: time.Date(2026, 7, 30, 0, 0, 0, 0, time.UTC), + Max: 2, + Skip: func(uid uint32) bool { return uid == 3 }, + } + msgs, err := fs.WithDial(dialer(t, f)).Run("secret") + if err != nil { + t.Fatalf("run: %v", err) + } + // Newest first, the already-seen UID skipped, Max respected. + if len(msgs) != 2 { + t.Fatalf("got %d messages, want 2: %+v", len(msgs), msgs) + } + if msgs[0].Subject != "two" || msgs[1].Subject != "one" { + t.Errorf("subjects = %q,%q, want two,one (newest first)", msgs[0].Subject, msgs[1].Subject) + } + if strings.Contains(strings.Join(f.cmds, " "), "UID FETCH 3") { + t.Error("a skipped UID must not be fetched again") + } +} + +func TestFetchSinceRequiresConfig(t *testing.T) { + if _, err := (FetchSince{}).Run("secret"); err == nil { + t.Fatal("an unconfigured mailbox must not be read") + } +} + +// Timeout zero disables the dial timeout AND every socket deadline, so a dead +// server parks the poller forever with his credential live in a TLS state. +func TestFetchSinceRejectsZeroTimeout(t *testing.T) { + fs := FetchSince{Addr: "mail.example:993", User: "kami", Mailbox: "INBOX"} + if _, err := fs.Run("secret"); err == nil { + t.Fatal("a zero timeout must be rejected like an empty address") + } +} + +// dialer wires a client Conn to an in-process fake over net.Pipe. +func dialer(t *testing.T, f *fakeIMAP) func(string, time.Duration) (*Conn, error) { + t.Helper() + return func(addr string, timeout time.Duration) (*Conn, error) { + cli, srv := net.Pipe() + go f.serve(t, srv) + return NewConn(cli, timeout) + } +} + +// One message that cannot be read must not hide the older ones behind it. The +// old code returned on the first failure, so an oversized or unreadable UID +// blocked every message below it on every poll, forever. +func TestFetchSinceContinuesPastABadMessage(t *testing.T) { + mk := func(subject string) string { + return "Subject: " + subject + "\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nbody\r\n" + } + f := &fakeIMAP{ + uids: []uint32{1, 2, 3}, + msgs: map[uint32]string{1: mk("one"), 3: mk("three")}, + quoted: map[uint32]bool{2: true}, + } + fs := FetchSince{ + Addr: "mail.example:993", User: "kami", Mailbox: "INBOX", + Timeout: 5 * time.Second, + Since: time.Date(2026, 7, 30, 0, 0, 0, 0, time.UTC), + } + msgs, err := fs.WithDial(dialer(t, f)).Run("secret") + if err == nil { + t.Fatal("the unreadable UID must still be reported") + } + if !strings.Contains(err.Error(), "uid 2") { + t.Errorf("error should name the UID: %v", err) + } + if len(msgs) != 2 { + t.Fatalf("got %d messages, want the two readable ones: %+v", len(msgs), msgs) + } + if msgs[0].Subject != "three" || msgs[1].Subject != "one" { + t.Errorf("subjects = %q,%q, want three,one", msgs[0].Subject, msgs[1].Subject) + } +} + +// An oversized message is retired as bulk rather than retried: it will be the +// same size next poll, and the poller marks bulk seen without a model call. +func TestFetchSinceRetiresOversizedMessage(t *testing.T) { + f := &fakeIMAP{uids: []uint32{7}, oversize: map[uint32]int{7: MaxMessageBytes + 1}} + fs := FetchSince{ + Addr: "mail.example:993", User: "kami", Mailbox: "INBOX", + Timeout: 5 * time.Second, + Since: time.Date(2026, 7, 30, 0, 0, 0, 0, time.UTC), + } + msgs, err := fs.WithDial(dialer(t, f)).Run("secret") + if err != nil { + t.Fatalf("run: %v", err) + } + if len(msgs) != 1 || !msgs[0].Junk || msgs[0].JunkReason != "oversize" { + t.Fatalf("want one oversize-bulk message, got %+v", msgs) + } + if msgs[0].Body != "" || msgs[0].Subject != "" { + t.Error("nothing from an oversized message may be kept") + } +} diff --git a/internal/email/imap.go b/internal/email/imap.go new file mode 100644 index 0000000..845eb40 --- /dev/null +++ b/internal/email/imap.go @@ -0,0 +1,389 @@ +package email + +import ( + "bufio" + "crypto/tls" + "errors" + "fmt" + "io" + "net" + "regexp" + "strconv" + "strings" + "time" +) + +// A minimal IMAP4rev1 client — LOGIN, SELECT, UID SEARCH, UID FETCH with +// BODY.PEEK, LOGOUT, and nothing else. +// +// Why hand-rolled instead of go-imap: the whole surface Maven needs is five +// commands, and this is the one code path that holds his mailbox credential and +// reads his private mail. A ~200-line client with no dependencies is auditable +// in one sitting; a general-purpose IMAP library is a much larger amount of +// code doing much more than we asked, in the most sensitive place in the tree. +// If IDLE, CONDSTORE or server-side threading ever become worth having, that +// trade should be re-made deliberately. +// +// BODY.PEEK[] rather than BODY[] is load-bearing: Maven reads his mail and must +// leave no trace of having done so. Reading a message here does not mark it +// \Seen, so the unread state in his own mail client stays his. + +// DefaultIMAPPort — implicit-TLS IMAP. There is no cleartext and no STARTTLS +// path in this client: an option to send his password over a plain socket is an +// option to get it wrong once. +const DefaultIMAPPort = "993" + +// MaxMessageBytes — the largest message this client will read into memory. +// +// The literal size comes off the wire, so an unbounded read is an allocation +// the server picks: "{2147483647}" is a 2GB make() before a single byte +// arrives, and one ordinary mail with a 60MB attachment is a 60MB peak RSS on a +// box already holding a 1.7B model resident. All of it would then be thrown +// away, because plaintextBody skips attachments and the body is truncated to +// MaxBodyBytes anyway. +// +// 2 MiB is well above what prose plus quoted history plus base64 HTML needs and +// well below what hurts. A larger message is drained and reported as +// ErrMessageTooLarge rather than read. +const MaxMessageBytes = 2 << 20 + +// readChunk — how much of a literal is read between deadline refreshes. The +// per-connection timeout must stay an IDLE timeout: with one deadline around +// the whole read it becomes a whole-message budget, and a healthy but slow +// uplink then fails the same message on every poll forever. +const readChunk = 64 << 10 + +// ErrMessageTooLarge — the server announced a literal above MaxMessageBytes. +// The connection stays usable (the bytes are drained), and the caller decides +// what to do with the UID. FetchSince retires it rather than retrying it. +var ErrMessageTooLarge = errors.New("email: message larger than the read cap") + +// Conn — one authenticated IMAP connection. Not safe for concurrent use; the +// poller drives one connection at a time. +type Conn struct { + rwc io.ReadWriteCloser + r *bufio.Reader + tag int + timeout time.Duration +} + +// Dial opens an implicit-TLS connection and reads the server greeting. +func Dial(addr string, timeout time.Duration) (*Conn, error) { + host, _, err := net.SplitHostPort(addr) + if err != nil { + host, addr = addr, net.JoinHostPort(addr, DefaultIMAPPort) + } + d := &net.Dialer{Timeout: timeout} + // ServerName is set from the host we asked for: certificate verification is + // the only thing standing between his password and a MITM on the way out. + c, err := tls.DialWithDialer(d, "tcp", addr, &tls.Config{ServerName: host, MinVersion: tls.VersionTLS12}) + if err != nil { + return nil, fmt.Errorf("email: dial %s: %w", addr, err) + } + return NewConn(c, timeout) +} + +// NewConn wraps an already-open stream (the tests speak IMAP over a pipe) and +// consumes the greeting. +func NewConn(rwc io.ReadWriteCloser, timeout time.Duration) (*Conn, error) { + c := &Conn{rwc: rwc, r: bufio.NewReaderSize(rwc, 64<<10), timeout: timeout} + line, err := c.readLine() + if err != nil { + return nil, fmt.Errorf("email: greeting: %w", err) + } + if !strings.HasPrefix(line, "* OK") && !strings.HasPrefix(line, "* PREAUTH") { + c.rwc.Close() + return nil, fmt.Errorf("email: server refused connection: %s", line) + } + return c, nil +} + +func (c *Conn) Close() error { return c.rwc.Close() } + +// Login authenticates with LOGIN. The password is passed as an argument and +// never stored on the Conn: nothing in this package keeps a credential alive +// past the command that uses it, so no struct dump or panic trace can carry it. +func (c *Conn) Login(user, pass string) error { + // A credential with a line break in it is rejected here, not silently + // repaired. quote() strips CR and LF so a stray newline can never become a + // second command, but stripping alone means a password file that picked up + // a newline authenticates as a DIFFERENT string and comes back as the + // server's generic NO, which is a long debugging session. This error names + // the problem and cannot leak the value. + if strings.ContainsAny(user, "\r\n") { + return fmt.Errorf("email: login: username contains a line break") + } + if strings.ContainsAny(pass, "\r\n") { + return fmt.Errorf("email: login: password contains a line break") + } + // The command line itself is never logged (see exec) — a LOGIN line IS the + // credential. + if _, err := c.exec(fmt.Sprintf("LOGIN %s %s", quote(user), quote(pass))); err != nil { + return fmt.Errorf("email: login: %w", err) + } + return nil +} + +// Select opens a mailbox read-only. EXAMINE, not SELECT: read-only at the +// protocol level means no command in this session can change a flag, expunge a +// message, or move anything, even by mistake. +func (c *Conn) Select(mailbox string) error { + if _, err := c.exec(fmt.Sprintf("EXAMINE %s", quote(mailbox))); err != nil { + return fmt.Errorf("email: examine %s: %w", mailbox, err) + } + return nil +} + +// SearchSince returns the UIDs of messages received on or after since. An +// unlimited search is not offered: the first poll against a years-old mailbox +// would otherwise fetch everything and hand a decade of mail to the model. +// +// The IMAP SINCE key has date granularity (and compares the server's internal +// date), so the result can include messages slightly older than since. The +// caller dedupes by UID anyway, so a wider window costs one extra fetch. +func (c *Conn) SearchSince(since time.Time) ([]uint32, error) { + cmd := fmt.Sprintf("UID SEARCH SINCE %s", since.Format("2-Jan-2006")) + lines, err := c.exec(cmd) + if err != nil { + return nil, fmt.Errorf("email: search: %w", err) + } + var uids []uint32 + for _, l := range lines { + rest, ok := untagged(l, "SEARCH") + if !ok { + continue + } + for _, f := range strings.Fields(rest) { + n, err := strconv.ParseUint(f, 10, 32) + if err == nil { + uids = append(uids, uint32(n)) + } + } + } + return uids, nil +} + +var literalSize = regexp.MustCompile(`\{(\d+)\}$`) + +// Fetch returns the raw RFC 5322 bytes of one message, by UID. +// +// Returns (nil, nil) when the UID no longer exists — a message he deleted +// between SEARCH and FETCH is normal, not an error. A FETCH response that came +// back with no literal in it is NOT that case and is an error, so a message +// that exists and was readable is never dropped without a log line. +// +// A literal above MaxMessageBytes is drained without being kept and reported as +// ErrMessageTooLarge. +func (c *Conn) Fetch(uid uint32) ([]byte, error) { + tag := c.nextTag() + if err := c.send(fmt.Sprintf("%s UID FETCH %d (BODY.PEEK[])", tag, uid)); err != nil { + return nil, err + } + var raw []byte + var sawFetch, tooLarge bool + for { + line, err := c.readLine() + if err != nil { + return nil, fmt.Errorf("email: fetch %d: %w", uid, err) + } + if done, err := c.tagged(tag, line); done { + switch { + case err != nil: + return nil, fmt.Errorf("email: fetch %d: %w", uid, err) + case tooLarge: + return nil, fmt.Errorf("email: fetch %d: %w", uid, ErrMessageTooLarge) + case sawFetch && raw == nil: + // The server answered for this UID but not with a literal (a + // quoted string, say). Silently skipping it would look exactly + // like a vanished message. + return nil, fmt.Errorf("email: fetch %d: no message literal in the FETCH response", uid) + } + return raw, nil + } + if strings.HasPrefix(line, "* ") && strings.Contains(line, " FETCH ") { + sawFetch = true + } + m := literalSize.FindStringSubmatch(strings.TrimSpace(line)) + if m == nil { + continue + } + n, err := strconv.Atoi(m[1]) + if err != nil || n < 0 { + continue + } + if n > MaxMessageBytes { + // Drained rather than read: the stream has to stay aligned for the + // tagged completion, but nothing is allocated and nothing is parsed. + tooLarge = true + if err := c.discard(int64(n)); err != nil { + return nil, fmt.Errorf("email: fetch %d: drain literal: %w", uid, err) + } + continue + } + buf, err := c.readN(n) + if err != nil { + return nil, fmt.Errorf("email: fetch %d: literal: %w", uid, err) + } + if raw == nil { + raw = buf + } + } +} + +// Logout ends the session politely. A failure is not worth reporting — the +// connection is being closed either way. +func (c *Conn) Logout() { + _, _ = c.exec("LOGOUT") +} + +// ---- protocol plumbing ----------------------------------------------------- + +func (c *Conn) nextTag() string { + c.tag++ + return fmt.Sprintf("a%03d", c.tag) +} + +// exec sends one command and returns the untagged response lines. +// +// Neither the command nor the response is ever logged here. LOGIN goes through +// this function, and a debug line "sent: a001 LOGIN ..." is how a credential +// ends up in a log file forever. +func (c *Conn) exec(cmd string) ([]string, error) { + tag := c.nextTag() + if err := c.send(tag + " " + cmd); err != nil { + return nil, err + } + var lines []string + for { + line, err := c.readLine() + if err != nil { + return nil, err + } + if done, err := c.tagged(tag, line); done { + return lines, err + } + lines = append(lines, line) + // A response line may carry a literal (e.g. a header FETCH). Nothing we + // send asks for one outside Fetch, but skip it if it appears so the + // stream stays aligned. + if m := literalSize.FindStringSubmatch(strings.TrimSpace(line)); m != nil { + if n, err := strconv.Atoi(m[1]); err == nil && n > 0 { + if err := c.discard(int64(n)); err != nil { + return nil, err + } + } + } + } +} + +// tagged reports whether line completes the command with this tag, and turns a +// NO/BAD completion into an error. The error text is the server's, which never +// echoes a password. +func (c *Conn) tagged(tag, line string) (bool, error) { + if !strings.HasPrefix(line, tag+" ") { + return false, nil + } + rest := strings.TrimSpace(line[len(tag):]) + switch { + case strings.HasPrefix(rest, "OK"): + return true, nil + case strings.HasPrefix(rest, "NO"), strings.HasPrefix(rest, "BAD"): + return true, fmt.Errorf("server said: %s", rest) + default: + return true, fmt.Errorf("unexpected completion: %s", rest) + } +} + +func (c *Conn) send(line string) error { + c.setDeadline() + if _, err := io.WriteString(c.rwc, line+"\r\n"); err != nil { + return fmt.Errorf("email: write: %w", err) + } + return nil +} + +// readN reads exactly n bytes, refreshing the deadline every readChunk so the +// timeout stays an idle timeout rather than a budget for the whole literal. +func (c *Conn) readN(n int) ([]byte, error) { + buf := make([]byte, n) + for off := 0; off < n; { + end := off + readChunk + if end > n { + end = n + } + c.setDeadline() + got, err := io.ReadFull(c.r, buf[off:end]) + off += got + if err != nil { + return nil, err + } + } + return buf, nil +} + +// discard throws away n bytes of literal, same chunked deadline refresh as +// readN and no allocation proportional to n. +func (c *Conn) discard(n int64) error { + for n > 0 { + chunk := int64(readChunk) + if chunk > n { + chunk = n + } + c.setDeadline() + got, err := io.CopyN(io.Discard, c.r, chunk) + n -= got + if err != nil { + return err + } + } + return nil +} + +func (c *Conn) readLine() (string, error) { + c.setDeadline() + line, err := c.r.ReadString('\n') + if err != nil { + return "", err + } + return strings.TrimRight(line, "\r\n"), nil +} + +// setDeadline applies the per-connection timeout when the transport supports +// one. A hung IMAP server must not park the poller forever. +func (c *Conn) setDeadline() { + if c.timeout <= 0 { + return + } + if d, ok := c.rwc.(interface{ SetDeadline(time.Time) error }); ok { + _ = d.SetDeadline(time.Now().Add(c.timeout)) + } +} + +// untagged splits "* SEARCH 1 2 3" into its payload when the key matches. +// +// The key must be the whole word: a prefix test would also match a future +// extension's "* SEARCHRES", and reading its payload as UIDs is the kind of +// thing that ages badly next to an IMAP capability nobody asked for. +func untagged(line, key string) (string, bool) { + if !strings.HasPrefix(line, "* ") { + return "", false + } + rest := strings.TrimSpace(line[2:]) + if !strings.HasPrefix(rest, key) { + return "", false + } + rest = rest[len(key):] + if rest != "" && rest[0] != ' ' && rest[0] != '\t' { + return "", false + } + return strings.TrimSpace(rest), true +} + +// quote renders an IMAP quoted string. Passwords routinely contain characters +// that would otherwise end the argument early, and CR/LF are stripped rather +// than escaped because there is no legal way to send them — a credential file +// with a stray newline must not become a second command. +func quote(s string) string { + s = strings.NewReplacer("\r", "", "\n", "").Replace(s) + return `"` + strings.NewReplacer(`\`, `\\`, `"`, `\"`).Replace(s) + `"` +} diff --git a/internal/email/imap_test.go b/internal/email/imap_test.go new file mode 100644 index 0000000..55dd412 --- /dev/null +++ b/internal/email/imap_test.go @@ -0,0 +1,237 @@ +package email + +import ( + "bufio" + "errors" + "fmt" + "net" + "strconv" + "strings" + "testing" + "time" +) + +// fakeIMAP is a scripted server: enough of IMAP to exercise the client, and +// nothing more. It records the commands it received so a test can assert on the +// protocol (BODY.PEEK rather than BODY, EXAMINE rather than SELECT). +type fakeIMAP struct { + msgs map[uint32]string + uids []uint32 + cmds []string + failOn string // substring of a command to answer NO + // oversize — UIDs answered with a literal of this many bytes, which the + // server then actually sends. Used to exercise the read cap. + oversize map[uint32]int + // quoted — UIDs answered with a quoted string instead of a literal. + quoted map[uint32]bool +} + +func (f *fakeIMAP) serve(t *testing.T, c net.Conn) { + t.Helper() + defer c.Close() + fmt.Fprint(c, "* OK fake IMAP ready\r\n") + r := bufio.NewReader(c) + for { + line, err := r.ReadString('\n') + if err != nil { + return + } + line = strings.TrimRight(line, "\r\n") + parts := strings.SplitN(line, " ", 2) + if len(parts) != 2 { + return + } + tag, cmd := parts[0], parts[1] + f.cmds = append(f.cmds, cmd) + if f.failOn != "" && strings.Contains(cmd, f.failOn) { + fmt.Fprintf(c, "%s NO computer says no\r\n", tag) + continue + } + upper := strings.ToUpper(cmd) + switch { + case strings.HasPrefix(upper, "LOGIN"), strings.HasPrefix(upper, "EXAMINE"): + fmt.Fprintf(c, "%s OK done\r\n", tag) + case strings.HasPrefix(upper, "UID SEARCH"): + var ids []string + for _, u := range f.uids { + ids = append(ids, strconv.FormatUint(uint64(u), 10)) + } + fmt.Fprintf(c, "* SEARCH %s\r\n", strings.Join(ids, " ")) + fmt.Fprintf(c, "%s OK search done\r\n", tag) + case strings.HasPrefix(upper, "UID FETCH"): + uid64, _ := strconv.ParseUint(strings.Fields(cmd)[2], 10, 32) + uid := uint32(uid64) + if n, big := f.oversize[uid]; big { + fmt.Fprintf(c, "* 1 FETCH (UID %d BODY[] {%d}\r\n", uid64, n) + fmt.Fprint(c, strings.Repeat("x", n)) + fmt.Fprint(c, ")\r\n") + fmt.Fprintf(c, "%s OK fetch done\r\n", tag) + continue + } + if f.quoted[uid] { + fmt.Fprintf(c, "* 1 FETCH (UID %d BODY[] \"short\")\r\n", uid64) + fmt.Fprintf(c, "%s OK fetch done\r\n", tag) + continue + } + raw, ok := f.msgs[uid] + if ok { + fmt.Fprintf(c, "* 1 FETCH (UID %d BODY[] {%d}\r\n", uid64, len(raw)) + fmt.Fprint(c, raw) + fmt.Fprint(c, ")\r\n") + } + fmt.Fprintf(c, "%s OK fetch done\r\n", tag) + case strings.HasPrefix(upper, "LOGOUT"): + fmt.Fprint(c, "* BYE\r\n") + fmt.Fprintf(c, "%s OK bye\r\n", tag) + return + default: + fmt.Fprintf(c, "%s BAD unknown\r\n", tag) + } + } +} + +// dialFake wires a client Conn to an in-process server over net.Pipe. +func dialFake(t *testing.T, f *fakeIMAP) *Conn { + t.Helper() + cli, srv := net.Pipe() + go f.serve(t, srv) + c, err := NewConn(cli, 5*time.Second) + if err != nil { + t.Fatalf("greeting: %v", err) + } + t.Cleanup(func() { c.Close() }) + return c +} + +func TestIMAPRoundTrip(t *testing.T) { + body := "Subject: hello\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nCall the bank.\r\n" + f := &fakeIMAP{uids: []uint32{4, 9}, msgs: map[uint32]string{4: body, 9: body}} + c := dialFake(t, f) + + if err := c.Login("kami", `pa"ss\word`); err != nil { + t.Fatalf("login: %v", err) + } + if err := c.Select("INBOX"); err != nil { + t.Fatalf("select: %v", err) + } + uids, err := c.SearchSince(time.Date(2026, 8, 1, 0, 0, 0, 0, time.UTC)) + if err != nil { + t.Fatalf("search: %v", err) + } + if len(uids) != 2 || uids[0] != 4 || uids[1] != 9 { + t.Fatalf("uids = %v, want [4 9]", uids) + } + raw, err := c.Fetch(9) + if err != nil { + t.Fatalf("fetch: %v", err) + } + if string(raw) != body { + t.Errorf("fetched %q, want the literal verbatim", raw) + } + c.Logout() + + joined := strings.Join(f.cmds, "\n") + // Read-only at the protocol level, and peeking — Maven must leave no trace + // of having read his mail. + if !strings.Contains(joined, "EXAMINE") || strings.Contains(joined, "SELECT ") { + t.Errorf("want EXAMINE (read-only), got:\n%s", joined) + } + if !strings.Contains(joined, "BODY.PEEK[]") { + t.Errorf("want BODY.PEEK, got:\n%s", joined) + } + // The password must have been quoted and escaped, not truncated at the quote. + if !strings.Contains(joined, `"pa\"ss\\word"`) { + t.Errorf("password not quoted correctly:\n%s", joined) + } + // SINCE must carry the IMAP date form. + if !strings.Contains(joined, "SINCE 1-Aug-2026") { + t.Errorf("want a SINCE date, got:\n%s", joined) + } +} + +func TestIMAPServerNoIsAnError(t *testing.T) { + f := &fakeIMAP{failOn: "LOGIN"} + c := dialFake(t, f) + err := c.Login("kami", "wrong") + if err == nil { + t.Fatal("a NO completion must be an error") + } + // The error is the server's text; it must not echo the credential. + if strings.Contains(err.Error(), "wrong") { + t.Errorf("error leaks the password: %v", err) + } +} + +func TestIMAPFetchMissingUID(t *testing.T) { + f := &fakeIMAP{uids: []uint32{1}, msgs: map[uint32]string{}} + c := dialFake(t, f) + raw, err := c.Fetch(1) + if err != nil { + t.Fatalf("fetch: %v", err) + } + if raw != nil { + t.Errorf("a vanished UID should give nil, got %q", raw) + } +} + +func TestQuoteStripsNewlines(t *testing.T) { + if got := quote("pass\r\nA1 LOGOUT"); strings.ContainsAny(got, "\r\n") { + t.Errorf("quote kept a line break: %q", got) + } +} + +// A credential with a line break in it is a broken password file, not a +// password. Stripping it silently authenticates as a different string and the +// server answers its generic NO. +func TestLoginRejectsCredentialWithNewline(t *testing.T) { + f := &fakeIMAP{} + c := dialFake(t, f) + err := c.Login("kami", "s3cr3t\nA1 LOGOUT") + if err == nil { + t.Fatal("a password with a line break must be rejected") + } + if strings.Contains(err.Error(), "s3cr3t") { + t.Errorf("error leaks the password: %v", err) + } + if len(f.cmds) != 0 { + t.Errorf("nothing should have been sent, got %v", f.cmds) + } +} + +// The literal size comes off the wire. Without a cap the server picks the +// allocation, and one 60MB attachment is 60MB of peak RSS on a box holding a +// 1.7B model, all of it thrown away by plaintextBody afterwards. +func TestFetchRefusesOversizedLiteral(t *testing.T) { + f := &fakeIMAP{uids: []uint32{1}, oversize: map[uint32]int{1: MaxMessageBytes + 1}} + c := dialFake(t, f) + raw, err := c.Fetch(1) + if !errors.Is(err, ErrMessageTooLarge) { + t.Fatalf("fetch err = %v, want ErrMessageTooLarge", err) + } + if raw != nil { + t.Errorf("an oversized message must not be kept, got %d bytes", len(raw)) + } + // The stream stayed aligned: the connection is still usable. + if err := c.Select("INBOX"); err != nil { + t.Errorf("connection unusable after draining: %v", err) + } +} + +// A FETCH that answered without a literal is not a vanished message, and must +// not be skipped as silently as one. +func TestFetchNonLiteralResponseIsAnError(t *testing.T) { + f := &fakeIMAP{uids: []uint32{1}, quoted: map[uint32]bool{1: true}} + c := dialFake(t, f) + if _, err := c.Fetch(1); err == nil { + t.Fatal("a FETCH response with no literal must be reported, not dropped") + } +} + +func TestUntaggedMatchesWholeKeyOnly(t *testing.T) { + if _, ok := untagged("* SEARCHRES 1 2 3", "SEARCH"); ok { + t.Error("SEARCH must not match SEARCHRES") + } + if rest, ok := untagged("* SEARCH 1 2 3", "SEARCH"); !ok || rest != "1 2 3" { + t.Errorf("untagged = (%q, %v), want (\"1 2 3\", true)", rest, ok) + } +} diff --git a/internal/email/junk.go b/internal/email/junk.go new file mode 100644 index 0000000..4895430 --- /dev/null +++ b/internal/email/junk.go @@ -0,0 +1,71 @@ +package email + +import ( + "net/mail" + "strings" +) + +// The junk filter — the cheapest and most important half of reading mail. +// +// A mailbox is mostly machine-generated: newsletters, receipts nobody acts on, +// social notifications, marketing. Sending all of it to a 1.7B and asking "is +// there a task here" produces confident nonsense at a rate proportional to the +// volume, so junk is decided by HEADERS, before any model sees the message. +// +// The rules are all bulk-mail markers that senders set on themselves, never +// guesses about content: +// +// - List-Unsubscribe / List-Id — by definition a mailing list. If he can +// unsubscribe from it, it is not asking him to do anything. +// - Precedence: bulk|junk|list — the sender declaring itself bulk. +// - Auto-Submitted other than "no" (RFC 3834) — generated by a machine. +// - X-Spam-Flag: YES, X-Spam-Status: Yes — the spam filter upstream already +// decided; we do not second-guess it in the other direction. +// +// There is deliberately NO Gmail-category rule. One was written and removed: +// it matched X-GM-LABELS and X-Gmail-Labels against the parsed header block, +// and neither is a header. X-GM-LABELS is a Gmail FETCH data item, requested as +// "UID FETCH n (X-GM-LABELS)" and never present in the message source; +// X-Gmail-Labels only exists in a Takeout mbox export. This client asks for +// BODY.PEEK[] and nothing else, so the rule could not fire against a real +// mailbox while its doc comment promised a Promotions filter. Gmail's promotion +// mail carries List-Unsubscribe in practice and is caught by the rule above. +// Bringing the category rule back means adding the FETCH item and carrying the +// labels into classifyJunk out of band, not matching a header that never +// arrives. +// +// Deliberately NOT here: sender allow/deny lists and subject keyword matching. +// Both are configuration that ages badly and both would be a place for his +// contacts to end up in a config file. If a real correspondent's mail is being +// dropped, the fix is a rule about a header, not a list of names. +// +// A junk verdict never deletes anything and never touches a flag on the server. +// It means "do not spend the model on this", nothing more. + +// junkHeaders — headers whose mere presence marks bulk mail. +var junkPresence = []string{"List-Unsubscribe", "List-Id", "List-Post"} + +// classifyJunk returns whether the message is bulk/automated and why. The +// reason is a short header name, safe to log — it names the marker, never the +// sender or the subject. +func classifyJunk(h mail.Header) (bool, string) { + for _, name := range junkPresence { + if strings.TrimSpace(h.Get(name)) != "" { + return true, strings.ToLower(name) + } + } + switch strings.ToLower(strings.TrimSpace(h.Get("Precedence"))) { + case "bulk", "junk", "list": + return true, "precedence" + } + if v := strings.ToLower(strings.TrimSpace(h.Get("Auto-Submitted"))); v != "" && v != "no" { + return true, "auto-submitted" + } + if strings.EqualFold(strings.TrimSpace(h.Get("X-Spam-Flag")), "yes") { + return true, "x-spam-flag" + } + if v := strings.ToLower(strings.TrimSpace(h.Get("X-Spam-Status"))); strings.HasPrefix(v, "yes") { + return true, "x-spam-status" + } + return false, "" +} diff --git a/internal/email/junk_test.go b/internal/email/junk_test.go new file mode 100644 index 0000000..81645c9 --- /dev/null +++ b/internal/email/junk_test.go @@ -0,0 +1,61 @@ +package email + +import ( + "net/mail" + "strings" + "testing" +) + +func headers(t *testing.T, raw string) mail.Header { + t.Helper() + m, err := mail.ReadMessage(strings.NewReader(strings.ReplaceAll(raw, "\n", "\r\n") + "\r\n\r\nbody\r\n")) + if err != nil { + t.Fatalf("read headers: %v", err) + } + return m.Header +} + +func TestClassifyJunk(t *testing.T) { + cases := []struct { + name string + raw string + junk bool + reason string + }{ + {"personal", "From: a@b.c\nSubject: привет", false, ""}, + {"list-unsubscribe", "From: a@b.c\nList-Unsubscribe: ", true, "list-unsubscribe"}, + {"list-id", "From: a@b.c\nList-Id: ", true, "list-id"}, + {"precedence bulk", "From: a@b.c\nPrecedence: bulk", true, "precedence"}, + {"auto-submitted", "From: a@b.c\nAuto-Submitted: auto-generated", true, "auto-submitted"}, + {"auto-submitted no", "From: a@b.c\nAuto-Submitted: no", false, ""}, + {"spam flag", "From: a@b.c\nX-Spam-Flag: YES", true, "x-spam-flag"}, + {"spam status", "From: a@b.c\nX-Spam-Status: Yes, score=9.1", true, "x-spam-status"}, + {"spam status no", "From: a@b.c\nX-Spam-Status: No, score=0.1", false, ""}, + // X-GM-LABELS is a Gmail FETCH data item, not a header, so it never + // reaches classifyJunk through this client. The rule that matched it was + // removed rather than left claiming a Promotions filter that never ran. + {"gmail label header is not a rule", "From: a@b.c\nX-Gmail-Labels: Inbox,CATEGORY_PROMOTIONS", false, ""}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + junk, reason := classifyJunk(headers(t, c.raw)) + if junk != c.junk || reason != c.reason { + t.Errorf("classifyJunk = (%v, %q), want (%v, %q)", junk, reason, c.junk, c.reason) + } + }) + } +} + +func TestNewsletterFixtureIsJunk(t *testing.T) { + msg, err := ParseMessage(9, fixture(t, "newsletter.eml")) + if err != nil { + t.Fatalf("parse: %v", err) + } + if !msg.Junk { + t.Fatal("a newsletter with List-Unsubscribe + Precedence: bulk must be junk") + } + // The reason is what gets logged, so it must never carry mail content. + if strings.Contains(msg.JunkReason, "@") || strings.Contains(msg.JunkReason, "Скидки") { + t.Errorf("junk reason leaks content: %q", msg.JunkReason) + } +} diff --git a/internal/email/message.go b/internal/email/message.go new file mode 100644 index 0000000..5219d39 --- /dev/null +++ b/internal/email/message.go @@ -0,0 +1,297 @@ +// Package email is the reading half of the email reader (Vikunja #246, +// docs/plans/01-email-reader.md): a small IMAP client, a MIME-to-plaintext +// converter, and the junk filter that decides a message is not worth reading at +// all. Extraction lives in extract.go and writes nothing itself. +// +// Two constraints shape everything here, both from CLAUDE.md: +// +// - Mail is personal. Nothing in this package logs a body, a subject, or an +// address; callers get the text and decide. Mail text is never search input +// — no function here reaches the network except the IMAP connection itself. +// - Off unless configured. There is no default host, no default account, and +// no fallback that would make a mailbox get read because a field was empty. +// +// The IMAP subset is deliberately tiny (LOGIN, SELECT, UID SEARCH, UID FETCH +// with BODY.PEEK, LOGOUT). No IDLE: a poll every few minutes is what a task +// candidate needs, and IDLE would mean holding a connection and a credential +// open forever for latency nobody is waiting on. +package email + +import ( + "encoding/base64" + "fmt" + "io" + "mime" + "mime/multipart" + "mime/quotedprintable" + "net/mail" + "regexp" + "strings" +) + +// MaxBodyBytes — how much of one message body is kept. A task hides in the +// first screenful; the rest is signature, quoted history and legal boilerplate, +// and it would only spend the resident model's 4096-token context. +const MaxBodyBytes = 4000 + +// Message — one mail, reduced to the fields extraction and review need. +// +// Raw is deliberately absent: once a message is parsed the original bytes are +// dropped, so no caller can accidentally log or forward the whole mail. +type Message struct { + UID uint32 + From string + Subject string + Date string // as sent, unparsed — display only + Body string // plaintext, decoded, HTML-stripped, truncated + // Junk is set by the junk filter (see junk.go). A junk message is carried + // rather than dropped so the poller can count it and still mark it seen. + Junk bool + JunkReason string +} + +// ParseMessage turns one RFC 5322 message into a Message. +// +// It never fails on a body it cannot understand: an unparsable or +// unsupported-charset body yields an empty Body and the headers still come +// through, because a subject line alone is often the whole task ("Счёт за +// интернет"). Only a message whose headers cannot be read at all is an error. +func ParseMessage(uid uint32, raw []byte) (Message, error) { + m, err := mail.ReadMessage(strings.NewReader(string(raw))) + if err != nil { + return Message{}, fmt.Errorf("email: parse message: %w", err) + } + msg := Message{ + UID: uid, + From: decodeHeader(m.Header.Get("From")), + Subject: decodeHeader(m.Header.Get("Subject")), + Date: m.Header.Get("Date"), + } + msg.Junk, msg.JunkReason = classifyJunk(m.Header) + body, err := plaintextBody(m.Header.Get("Content-Type"), m.Header.Get("Content-Transfer-Encoding"), m.Body) + if err == nil { + msg.Body = truncate(collapse(body), MaxBodyBytes) + } + return msg, nil +} + +// plaintextBody walks the MIME tree and returns the best plaintext it can. +// +// Preference order inside a multipart: text/plain first, text/html stripped +// only when there is no plain part. multipart/mixed attachments are skipped +// wholesale — an attachment is a file, not a sentence, and reading one would +// mean parsing arbitrary formats from the network. +func plaintextBody(contentType, encoding string, body io.Reader) (string, error) { + mediaType, params, err := mime.ParseMediaType(contentType) + if contentType == "" || err != nil { + // No Content-Type at all is legal and means text/plain; a broken one is + // treated the same rather than dropping the message. + mediaType, params = "text/plain", nil + } + switch { + case strings.HasPrefix(mediaType, "multipart/"): + boundary := params["boundary"] + if boundary == "" { + return "", fmt.Errorf("email: multipart without boundary") + } + plain, html, err := multipartText(multipart.NewReader(body, boundary)) + if err != nil { + return "", err + } + if strings.TrimSpace(plain) != "" { + return plain, nil + } + return html, nil + case mediaType == "text/html": + raw, err := decodeBody(body, encoding, params["charset"]) + if err != nil { + return "", err + } + return stripHTML(raw), nil + case mediaType == "text/plain": + return decodeBody(body, encoding, params["charset"]) + default: + // A single-part non-text message (a bare PDF, say). No body, headers only. + return "", nil + } +} + +// multipartText reads one multipart level, recursing into nested multiparts, +// and returns the two buckets separately: real text/plain, and text stripped +// out of HTML. +// +// The buckets stay separate all the way up because a nested multipart can +// contribute either kind. Folding a nested level's answer into one string put +// HTML-derived text in the plain bucket, and a real text/plain sibling later in +// the message was then thrown away by the "plain is already set" guard. +func multipartText(mr *multipart.Reader) (plain, html string, err error) { + for { + part, err := mr.NextPart() + if err == io.EOF { + break + } + if err != nil { + // A truncated multipart still gives up whatever came before it. + break + } + if part.FileName() != "" { + part.Close() + continue // attachment + } + ct := part.Header.Get("Content-Type") + mediaType, params, _ := mime.ParseMediaType(ct) + switch { + case strings.HasPrefix(mediaType, "multipart/"): + var np, nh string + if b := params["boundary"]; b != "" { + np, nh, _ = multipartText(multipart.NewReader(part, b)) + } + part.Close() + if plain == "" { + plain = np + } + if html == "" { + html = nh + } + default: + text, terr := plaintextBody(ct, part.Header.Get("Content-Transfer-Encoding"), part) + part.Close() + if terr != nil || strings.TrimSpace(text) == "" { + continue + } + if mediaType == "text/html" { + if html == "" { + html = text + } + continue + } + if plain == "" { + plain = text + } + } + } + return plain, html, nil +} + +// decodeBody applies the transfer encoding, then the charset. +// +// Charset support is UTF-8 (and ASCII, its subset) plus windows-1251, which is +// decoded from a table in charset.go — see the reasoning there. Everything else +// returns an error, which ParseMessage turns into an empty body: subject-only, +// which is honest. Guessing at unknown bytes would feed the model mojibake it +// would happily extract a task from. +func decodeBody(r io.Reader, encoding, charset string) (string, error) { + switch strings.ToLower(strings.TrimSpace(encoding)) { + case "quoted-printable": + r = quotedprintable.NewReader(r) + case "base64": + r = newBase64Reader(r) + } + b, err := io.ReadAll(io.LimitReader(r, 1<<20)) + if err != nil && len(b) == 0 { + return "", fmt.Errorf("email: read body: %w", err) + } + switch cs := strings.ToLower(strings.TrimSpace(charset)); cs { + case "", "utf-8", "utf8", "us-ascii", "ascii": + return string(b), nil + case "windows-1251", "cp1251", "windows1251", "x-cp1251": + return decodeCP1251(b), nil + default: + return "", fmt.Errorf("email: unsupported charset %q", cs) + } +} + +// decodeHeader decodes RFC 2047 encoded words ("=?utf-8?B?...?="), which is how +// every Russian subject line arrives. Undecodable headers come back as-is +// rather than empty: a mangled subject is still a hint, and it is only ever +// shown to him as evidence. +func decodeHeader(v string) string { + dec := new(mime.WordDecoder) + // Same charset support as the body: an old Russian sender encodes the + // subject in windows-1251 too, and a subject is often the whole task. + dec.CharsetReader = func(charset string, r io.Reader) (io.Reader, error) { + switch strings.ToLower(strings.TrimSpace(charset)) { + case "windows-1251", "cp1251", "windows1251", "x-cp1251": + b, err := io.ReadAll(io.LimitReader(r, 1<<16)) + if err != nil && len(b) == 0 { + return nil, err + } + return strings.NewReader(decodeCP1251(b)), nil + } + return nil, fmt.Errorf("email: unsupported charset %q", charset) + } + out, err := dec.DecodeHeader(v) + if err != nil { + return collapse(v) + } + return collapse(out) +} + +var ( + scriptStyle = regexp.MustCompile(`(?is)<(script|style)\b[^>]*>.*?`) + htmlBreak = regexp.MustCompile(`(?i)<\s*(br\s*/?|/p|/div|/tr|/li|/h[1-6])\s*>`) + htmlTag = regexp.MustCompile(`(?s)<[^>]*>`) + htmlComment = regexp.MustCompile(`(?s)`) +) + +// stripHTML reduces an HTML part to text. A regex stripper, not a parser: +// x/net/html is not vendored, and the consumer is a model reading prose — a +// stray angle bracket costs nothing, whereas a new dependency for the privacy- +// sensitive path costs review. +func stripHTML(s string) string { + s = scriptStyle.ReplaceAllString(s, " ") + s = htmlComment.ReplaceAllString(s, " ") + s = htmlBreak.ReplaceAllString(s, "\n") + s = htmlTag.ReplaceAllString(s, " ") + return unescapeEntities(s) +} + +var entities = strings.NewReplacer( + " ", " ", "&", "&", "<", "<", ">", ">", + """, `"`, "'", "'", "'", "'", "—", "—", "–", "–", +) + +func unescapeEntities(s string) string { return entities.Replace(s) } + +// collapse squeezes runs of whitespace, keeping single newlines. Mail bodies +// arrive with hard-wrapped lines and blocks of blank space; the model does not +// need them and they are pure context budget. +func collapse(s string) string { + lines := strings.Split(strings.ReplaceAll(s, "\r\n", "\n"), "\n") + var out []string + blank := 0 + for _, l := range lines { + l = strings.TrimSpace(strings.Join(strings.Fields(l), " ")) + if l == "" { + blank++ + if blank > 1 { + continue + } + out = append(out, "") + continue + } + blank = 0 + out = append(out, l) + } + return strings.TrimSpace(strings.Join(out, "\n")) +} + +// truncate cuts to n bytes on a rune boundary. +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + cut := s[:n] + for len(cut) > 0 && !isRuneStart(cut[len(cut)-1]) { + cut = cut[:len(cut)-1] + } + return strings.TrimSpace(cut) + "…" +} + +func isRuneStart(b byte) bool { return b&0xC0 != 0x80 } + +// newBase64Reader — base64.NewDecoder already skips the CRLFs mail bodies wrap +// with, so this is only a named seam for decodeBody to read cleanly. +func newBase64Reader(r io.Reader) io.Reader { + return base64.NewDecoder(base64.StdEncoding, r) +} diff --git a/internal/email/message_test.go b/internal/email/message_test.go new file mode 100644 index 0000000..c961dce --- /dev/null +++ b/internal/email/message_test.go @@ -0,0 +1,151 @@ +package email + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func fixture(t *testing.T, name string) []byte { + t.Helper() + b, err := os.ReadFile(filepath.Join("testdata", name)) + if err != nil { + t.Fatalf("read fixture %s: %v", name, err) + } + return b +} + +func TestParsePlainRussian(t *testing.T) { + msg, err := ParseMessage(7, fixture(t, "plain_ru.eml")) + if err != nil { + t.Fatalf("parse: %v", err) + } + if msg.UID != 7 { + t.Errorf("uid = %d, want 7", msg.UID) + } + if want := "Нужно закрыть задачу"; msg.Subject != want { + t.Errorf("subject = %q, want %q", msg.Subject, want) + } + if !strings.Contains(msg.From, "Антон") { + t.Errorf("from = %q, want the decoded display name", msg.From) + } + if !strings.Contains(msg.Body, "Надо отправить акт до пятницы.") { + t.Errorf("body = %q, want the quoted-printable text decoded", msg.Body) + } + if msg.Junk { + t.Errorf("a personal mail must not be junk (%s)", msg.JunkReason) + } +} + +func TestParseHTMLOnlyIsStripped(t *testing.T) { + msg, err := ParseMessage(1, fixture(t, "html_only.eml")) + if err != nil { + t.Fatalf("parse: %v", err) + } + if strings.Contains(msg.Body, "<") || strings.Contains(msg.Body, "color:red") || strings.Contains(msg.Body, "x()") { + t.Errorf("body still has markup/script/style: %q", msg.Body) + } + for _, want := range []string{"Счёт за интернет: 700", "Оплатить до 5 августа."} { + if !strings.Contains(msg.Body, want) { + t.Errorf("body = %q, want it to contain %q", msg.Body, want) + } + } + //   must have become a real space, not vanished into the number. + if strings.Contains(msg.Body, " ") { + t.Errorf("entity left unescaped: %q", msg.Body) + } +} + +func TestParsePrefersPlainAndSkipsAttachments(t *testing.T) { + msg, err := ParseMessage(2, fixture(t, "mixed_attachment.eml")) + if err != nil { + t.Fatalf("parse: %v", err) + } + if got := strings.TrimSpace(msg.Body); got != "Sign the contract before Monday." { + t.Errorf("body = %q, want the text/plain alternative only", got) + } + if strings.Contains(msg.Body, "PDF") { + t.Errorf("attachment bytes leaked into the body: %q", msg.Body) + } +} + +// windows-1251 is what older Russian senders still emit. Subject-only for those +// mails meant they could never produce a task candidate. +func TestParseCP1251(t *testing.T) { + msg, err := ParseMessage(3, fixture(t, "cp1251.eml")) + if err != nil { + t.Fatalf("parse: %v", err) + } + if want := "Счёт за интернет"; msg.Subject != want { + t.Errorf("subject = %q, want %q", msg.Subject, want) + } + if want := "Оплати счёт до пятницы."; !strings.Contains(msg.Body, want) { + t.Errorf("body = %q, want it to contain %q", msg.Body, want) + } +} + +// A charset with no table here must degrade to headers-only rather than to +// mojibake the model would then extract a task from. +func TestParseUnsupportedCharsetKeepsHeaders(t *testing.T) { + msg, err := ParseMessage(3, fixture(t, "koi8r.eml")) + if err != nil { + t.Fatalf("parse: %v", err) + } + if msg.Subject != "Legacy" { + t.Errorf("subject = %q, want Legacy", msg.Subject) + } + if msg.Body != "" { + t.Errorf("body = %q, want empty for an undecodable charset", msg.Body) + } +} + +// A nested multipart/alternative that only had HTML must not fill the plain +// bucket: a real text/plain sibling later in the message is the better text and +// used to be discarded. +func TestParseNestedHTMLDoesNotShadowLaterPlain(t *testing.T) { + raw := "Subject: nested\r\n" + + "Content-Type: multipart/mixed; boundary=OUT\r\n\r\n" + + "--OUT\r\n" + + "Content-Type: multipart/alternative; boundary=IN\r\n\r\n" + + "--IN\r\n" + + "Content-Type: text/html; charset=utf-8\r\n\r\n" + + "

from the html part

\r\n" + + "--IN--\r\n" + + "--OUT\r\n" + + "Content-Type: text/plain; charset=utf-8\r\n\r\n" + + "the real plain text\r\n" + + "--OUT--\r\n" + msg, err := ParseMessage(5, []byte(raw)) + if err != nil { + t.Fatalf("parse: %v", err) + } + if got := strings.TrimSpace(msg.Body); got != "the real plain text" { + t.Errorf("body = %q, want the text/plain part to win", got) + } +} + +func TestParseTruncatesLongBody(t *testing.T) { + var b strings.Builder + b.WriteString("Subject: long\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n") + for i := 0; i < 2000; i++ { + b.WriteString("длинная строка ") + } + msg, err := ParseMessage(4, []byte(b.String())) + if err != nil { + t.Fatalf("parse: %v", err) + } + if len(msg.Body) > MaxBodyBytes+8 { + t.Errorf("body kept %d bytes, want ≤ %d", len(msg.Body), MaxBodyBytes) + } + if !strings.HasSuffix(msg.Body, "…") { + t.Errorf("truncated body should be marked: %q", msg.Body[len(msg.Body)-20:]) + } +} + +func TestCollapseSqueezesBlankLines(t *testing.T) { + got := collapse(" a b \r\n\r\n\r\n\r\n c \r\n") + if got != "a b\n\nc" { + t.Errorf("collapse = %q, want %q", got, "a b\n\nc") + } +} diff --git a/internal/email/testdata/cp1251.eml b/internal/email/testdata/cp1251.eml new file mode 100644 index 0000000..5f987cf --- /dev/null +++ b/internal/email/testdata/cp1251.eml @@ -0,0 +1,7 @@ +From: legacy@example.org +To: kami@example.org +Subject: =?windows-1251?B?0fe48iDn4CDo7fLl8O3l8g==?= +Date: Fri, 01 Aug 2026 05:00:00 +0400 +Content-Type: text/plain; charset="windows-1251" + + . diff --git a/internal/email/testdata/html_only.eml b/internal/email/testdata/html_only.eml new file mode 100644 index 0000000..deff075 --- /dev/null +++ b/internal/email/testdata/html_only.eml @@ -0,0 +1,13 @@ +From: billing@isp.example +To: kami@example.org +Subject: =?utf-8?B?0KHRh9GR0YIg0LfQsCDQuNC90YLQtdGA0L3QtdGC?= +Date: Fri, 01 Aug 2026 08:00:00 +0400 +MIME-Version: 1.0 +Content-Type: multipart/alternative; boundary="B1" + +--B1 +Content-Type: text/html; charset="utf-8" +Content-Transfer-Encoding: base64 + +PGh0bWw+PGhlYWQ+PHN0eWxlPnB7Y29sb3I6cmVkfTwvc3R5bGU+PC9oZWFkPjxib2R5PjxwPtCh0YfRkdGCINC30LAg0LjQvdGC0LXRgNC90LXRgjogNzAwJm5ic3A74oK9PC9wPjxwPtCe0L/Qu9Cw0YLQuNGC0Ywg0LTQviA1INCw0LLQs9GD0YHRgtCwLjwvcD48c2NyaXB0PngoKTwvc2NyaXB0PjwvYm9keT48L2h0bWw+ +--B1-- diff --git a/internal/email/testdata/koi8r.eml b/internal/email/testdata/koi8r.eml new file mode 100644 index 0000000..b6f18d7 --- /dev/null +++ b/internal/email/testdata/koi8r.eml @@ -0,0 +1,7 @@ +From: legacy@example.org +To: kami@example.org +Subject: Legacy +Date: Fri, 01 Aug 2026 05:00:00 +0400 +Content-Type: text/plain; charset="koi8-r" + + ޣ. diff --git a/internal/email/testdata/mixed_attachment.eml b/internal/email/testdata/mixed_attachment.eml new file mode 100644 index 0000000..a93c884 --- /dev/null +++ b/internal/email/testdata/mixed_attachment.eml @@ -0,0 +1,26 @@ +From: hr@work.example +To: kami@example.org +Subject: Contract +Date: Fri, 01 Aug 2026 07:00:00 +0400 +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="M1" + +--M1 +Content-Type: multipart/alternative; boundary="A1" + +--A1 +Content-Type: text/plain; charset="utf-8" + +Sign the contract before Monday. +--A1 +Content-Type: text/html; charset="utf-8" + +

Sign the contract before Monday.

+--A1-- +--M1 +Content-Type: application/pdf; name="contract.pdf" +Content-Disposition: attachment; filename="contract.pdf" +Content-Transfer-Encoding: base64 + +JVBERi0xLjQgbm90IHJlYWxseSBhIHBkZg== +--M1-- diff --git a/internal/email/testdata/newsletter.eml b/internal/email/testdata/newsletter.eml new file mode 100644 index 0000000..09c3d31 --- /dev/null +++ b/internal/email/testdata/newsletter.eml @@ -0,0 +1,9 @@ +From: news@shop.example +To: kami@example.org +Subject: =?utf-8?B?0KHQutC40LTQutC4INGC0L7Qu9GM0LrQviDRgdC10LPQvtC00L3Rjw==?= +Date: Fri, 01 Aug 2026 06:00:00 +0400 +List-Unsubscribe: +Precedence: bulk +Content-Type: text/plain; charset="utf-8" + +Sale! diff --git a/internal/email/testdata/plain_ru.eml b/internal/email/testdata/plain_ru.eml new file mode 100644 index 0000000..bd8c745 --- /dev/null +++ b/internal/email/testdata/plain_ru.eml @@ -0,0 +1,14 @@ +From: =?utf-8?B?0JDQvdGC0L7QvQ==?= +To: kami@example.org +Subject: =?utf-8?B?0J3Rg9C20L3QviDQt9Cw0LrRgNGL0YLRjCDQt9Cw0LTQsNGH0YM=?= +Date: Fri, 01 Aug 2026 09:12:00 +0400 +Content-Type: text/plain; charset="utf-8" +Content-Transfer-Encoding: quoted-printable +Message-ID: + +=D0=9F=D1=80=D0=B8=D0=B2=D0=B5=D1=82! =D0=9D=D0=B0=D0=B4=D0=BE =D0=BE=D1=82= +=D0=BF=D1=80=D0=B0=D0=B2=D0=B8=D1=82=D1=8C =D0=B0=D0=BA=D1=82 =D0=B4=D0=BE = +=D0=BF=D1=8F=D1=82=D0=BD=D0=B8=D1=86=D1=8B. + +-- +Anton diff --git a/internal/event/bus.go b/internal/event/bus.go new file mode 100644 index 0000000..72d0c0c --- /dev/null +++ b/internal/event/bus.go @@ -0,0 +1,135 @@ +package event + +import ( + "sync" + "time" +) + +// Bus — the in-memory intake journal: a bounded ring of recent Events plus +// zero or more subscribers. +// +// Two properties are load-bearing, both about not changing production +// behaviour when nobody is watching: +// +// - A nil *Bus is a working no-op. Publish on nil returns immediately, so +// an intake path can call b.Publish(...) unconditionally and a daemon that +// never built a bus behaves exactly as it did before. This is what let +// eight callers adopt the envelope without a config flag each. +// - Publish never blocks on a subscriber and never propagates a panic from +// one. Intake is on the request path of POST /api/ambient and of every +// fact write; a slow or broken observer must not be able to stall or kill +// a write that already succeeded. +// +// The ring is bounded because it is memory that nothing prunes otherwise. Its +// contents are a window, not a record: the durable consequence of an event is +// the fact, note or task the intake path wrote. +type Bus struct { + mu sync.Mutex + ring []Event // len == cap once full; oldest at (next % cap) + next int + n int + subs []func(Event) +} + +// DefaultCapacity — how many recent events a bus keeps. A busy day is a few +// hundred intake events (a feed poll is one per new item), so this is roughly +// "today and yesterday" at a few hundred KB. +const DefaultCapacity = 512 + +// NewBus returns a bus keeping the last capacity events. capacity <= 0 uses +// DefaultCapacity. +func NewBus(capacity int) *Bus { + if capacity <= 0 { + capacity = DefaultCapacity + } + return &Bus{ring: make([]Event, capacity)} +} + +// Publish normalizes e, drops it if it is not Valid, appends it to the ring and +// hands it to every subscriber. Safe on a nil receiver and safe from any +// goroutine. +// +// now is passed in rather than read from the clock: the whole point of #284's +// replay is that no time.Now() sits inside a path a scenario drives. +func (b *Bus) Publish(e Event, now time.Time) { + if b == nil { + return + } + e = e.Normalize(now) + if !e.Valid() { + return + } + + b.mu.Lock() + b.ring[b.next] = e + b.next = (b.next + 1) % len(b.ring) + if b.n < len(b.ring) { + b.n++ + } + subs := make([]func(Event), len(b.subs)) + copy(subs, b.subs) + b.mu.Unlock() + + for _, fn := range subs { + notify(fn, e) + } +} + +// notify calls one subscriber, swallowing a panic. A test double or a page +// renderer must not be able to take down a daemon from the intake path. +func notify(fn func(Event), e Event) { + defer func() { _ = recover() }() + fn(e) +} + +// Subscribe registers fn to be called for every subsequent event, in publish +// order. There is no unsubscribe: subscribers are wired at startup and live as +// long as the daemon. Safe on a nil receiver (the subscription is dropped, +// which is the honest outcome when there is no bus to subscribe to). +func (b *Bus) Subscribe(fn func(Event)) { + if b == nil || fn == nil { + return + } + b.mu.Lock() + defer b.mu.Unlock() + b.subs = append(b.subs, fn) +} + +// Recent returns up to limit events, most recently NOTICED first. limit <= 0 +// returns everything held. Safe on a nil receiver (returns nil). +// +// The order is the ring's insertion order, which is NoticedAt order, and it is +// deliberately not OccurredAt order: a cold feed read publishes a week of items +// in feed order, so sorting by when things happened would put a six-day-old +// item above one that arrived before it. Readers must label the column +// accordingly. +func (b *Bus) Recent(limit int) []Event { + if b == nil { + return nil + } + b.mu.Lock() + defer b.mu.Unlock() + if b.n == 0 { + return nil + } + if limit <= 0 || limit > b.n { + limit = b.n + } + out := make([]Event, 0, limit) + // next points one past the newest; walk backwards. + for i := 0; i < limit; i++ { + idx := (b.next - 1 - i + len(b.ring)*2) % len(b.ring) + out = append(out, b.ring[idx]) + } + return out +} + +// Len reports how many events the ring currently holds. Safe on nil. +func (b *Bus) Len() int { + if b == nil { + return 0 + } + b.mu.Lock() + defer b.mu.Unlock() + return b.n +} diff --git a/internal/event/event.go b/internal/event/event.go new file mode 100644 index 0000000..64a8457 --- /dev/null +++ b/internal/event/event.go @@ -0,0 +1,213 @@ +// Package event is the unified intake envelope (Vikunja #283, +// 20-07-2026-BACKLOG.md item 1). +// +// # The problem it solves +// +// Things arrive at Maven from a lot of directions: a relayed Android +// notification (POST /api/ambient), a mail the reader extracted candidates +// from (ingest_mail), an RSS item, a changed page the crawler noticed, a +// zenmoney spend, a CalDAV event, a wg handshake that means he is home, a +// photo he sent, a meeting she was asked to record. Each of those grew its own +// shape, its own storage decision and its own log line. Nothing could answer +// "what came in today, from where" without reading eight packages. +// +// An Event is that answer: one flat, source-agnostic description of "something +// arrived". It is deliberately NOT a new storage layer and NOT a new transport. +// Every intake path keeps writing exactly what it wrote before — a fact, a +// note, a candidate task — and additionally describes what it did as an Event. +// The envelope is a VIEW over intake, not a replacement for it, which is why +// adopting it did not require touching eight callers. +// +// # What it is not +// +// - Not a command. An Event is a report of something that happened; nothing +// in Maven executes one. Digestion may read them; it may not be driven by +// an event alone, because "a thing arrived" is not "a thing must be said". +// - Not durable. The bus is a bounded in-memory ring. An event's durable +// consequence is the fact/note/task the intake path already wrote; the +// envelope is the recent-history window on top. A restart losing the ring +// loses nothing that mattered. +// - Not a secret store. Body carries what the intake path was already willing +// to log or store. Nothing puts a mail body, an IMAP password or a +// voiceprint in here, and callers must keep it that way. +package event + +import ( + "encoding/json" + "strings" + "time" +) + +// Event — one thing that arrived, normalized. +// +// The field set is the one recorded in the backlog, and it is intentionally +// small: anything source-specific goes in Payload, so adding a source never +// widens the struct and never breaks a reader. +type Event struct { + // Source — provenance, in the facts vocabulary already used across the + // repo: "ambient:notif", "caldav:personal", "poll:zenmoney", "rss:", + // "crawl:", "email:", "infer:wg", "tap:voice". Same string + // the fact or note was written under, so an event and its row can be + // matched up by eye. + Source string `json:"source"` + + // Kind — what sort of thing arrived, from the closed set below. This is the + // field digestion switches on; Source is for provenance and display. + Kind string `json:"kind"` + + // EntityIDs — Nexus entity ids this event is about, when the intake path + // knew any. Usually empty: most intake happens before enrichment resolves a + // subject to an entity. + EntityIDs []string `json:"entity_ids,omitempty"` + + // Title — one short line, safe to show on a page. For a fact it is the key, + // for a note the first line, for a task the task text. + Title string `json:"title"` + + // Body — optional detail, already truncated by the caller. + Body string `json:"body,omitempty"` + + // Priority — one of PriorityLow / PriorityNormal / PriorityHigh. It is a + // hint about attention, not a delivery instruction: nothing here decides + // whether Maven speaks. That stays with internal/loop and internal/delivery, + // where the severity/presence routing table lives. + Priority string `json:"priority"` + + // OccurredAt — when the thing happened, NOT when Maven noticed it. A wg + // handshake carries the handshake instant; an RSS item carries its publish + // time. Intake paths already make this distinction when writing facts, and + // the envelope must not flatten it. + OccurredAt time.Time `json:"occurred_at"` + + // NoticedAt — when Maven saw it. This is the ring's own ordering and the + // one the page sorts and labels by. + // + // It exists because OccurredAt must stay truthful and therefore cannot be + // an arrival order. A feed's first read returns twenty items spread over a + // week and publishes them in feed order; the ambient relay writes a 09:00 + // notification about an 18:00 meeting. Ordering the journal by OccurredAt + // while walking the ring backwards made the timestamp column run forwards + // and backwards on the same page. Normalize fills it from now, always: a + // caller does not get to say when Maven noticed something. + NoticedAt time.Time `json:"noticed_at"` + + // Payload — source-specific extra, opaque here. Optional. + Payload json.RawMessage `json:"payload,omitempty"` +} + +// Kinds. Closed set: a reader may switch on these exhaustively. A new intake +// path picks the closest existing kind before it adds one — the point of the +// envelope is that digestion has a small stable input. +const ( + // KindFact — something was written to the facts table: a calendar read, a + // zenmoney window, a presence probe, a crawler watermark. + KindFact = "fact" + + // KindNote — something was written to the notes table: an RSS item, a + // changed page, a meeting transcript, an image description. + KindNote = "note" + + // KindTask — a candidate task was captured: the mail reader, the web form, + // the voice path. + KindTask = "task" + + // KindMessage — an inbound message on a reach channel. Nothing produces + // this yet (telegram is send-only today); the kind exists so the bridge, + // when it lands, is a constructor and not a schema change. + KindMessage = "message" + + // KindHealth — a service or probe reported its own state. + KindHealth = "health" +) + +// Priorities. +const ( + PriorityLow = "low" + PriorityNormal = "normal" + PriorityHigh = "high" +) + +// TitleMaxRunes / BodyMaxRunes bound what an envelope carries. The ring is +// in memory and served to a web page; a 40 KB crawled article has no business +// in either. Cut on a rune boundary — most of this text is Russian and half a +// cyrillic letter is a broken line. +const ( + TitleMaxRunes = 120 + BodyMaxRunes = 400 +) + +// Normalize returns e with its fields put in range: whitespace collapsed out +// of Title, Title and Body truncated, an unknown or empty Priority forced to +// PriorityNormal, and a zero OccurredAt filled from now. +// +// It takes now as a parameter rather than reading the clock, so the whole +// package stays pure and the simulator (Vikunja #284) can replay intake against +// a scripted clock. +func (e Event) Normalize(now time.Time) Event { + e.Title = truncateRunes(strings.Join(strings.Fields(e.Title), " "), TitleMaxRunes) + e.Body = truncateRunes(strings.TrimSpace(e.Body), BodyMaxRunes) + if !validPriority(e.Priority) { + e.Priority = PriorityNormal + } + if e.Kind == "" { + e.Kind = KindFact + } + if e.OccurredAt.IsZero() { + e.OccurredAt = now + } + // Notice time is not a caller's to set: it is the instant the bus took it. + e.NoticedAt = now + return e +} + +// Valid reports whether e carries the minimum a reader can rely on: a source, +// a known kind, a title and a time. The bus drops anything that fails — an +// envelope with no provenance is worse than no envelope, because it looks like +// evidence. +func (e Event) Valid() bool { + return e.Source != "" && validKind(e.Kind) && e.Title != "" && !e.OccurredAt.IsZero() +} + +func validKind(k string) bool { + switch k { + case KindFact, KindNote, KindTask, KindMessage, KindHealth: + return true + } + return false +} + +func validPriority(p string) bool { + switch p { + case PriorityLow, PriorityNormal, PriorityHigh: + return true + } + return false +} + +// truncateRunes cuts s to n runes, marking the cut. +func truncateRunes(s string, n int) string { + r := []rune(s) + if len(r) <= n { + return s + } + return string(r[:n]) + "…" +} + +// SourceKind guesses the Kind for a source string when the caller has not said +// otherwise. It exists so the one intake decorator in cmd/mavend does not need +// a switch per writer: the source prefix already tells you what arrived. +// +// Unknown prefixes get fallback, which is what the caller was going to write +// anyway (a WriteFact call knows it is a fact). There is deliberately no +// "email:" rule: the mail path constructs its task envelope itself, so mapping +// the prefix here would have journalled any future fact or note written under +// an email source as a captured task. +func SourceKind(source, fallback string) string { + switch { + case strings.HasPrefix(source, "rss:"), strings.HasPrefix(source, "crawl:"): + return KindNote + case strings.HasPrefix(source, "probe:"), strings.HasPrefix(source, "health:"): + return KindHealth + } + return fallback +} diff --git a/internal/event/event_test.go b/internal/event/event_test.go new file mode 100644 index 0000000..27bffa6 --- /dev/null +++ b/internal/event/event_test.go @@ -0,0 +1,225 @@ +package event + +import ( + "strings" + "sync" + "testing" + "time" +) + +var testNow = time.Date(2026, 8, 1, 9, 30, 0, 0, time.UTC) + +func TestNormalizeFillsDefaults(t *testing.T) { + got := Event{Source: "poll:zenmoney", Title: " spent today "}.Normalize(testNow) + if got.Title != "spent today" { + t.Errorf("title = %q, want collapsed whitespace", got.Title) + } + if got.Priority != PriorityNormal { + t.Errorf("priority = %q, want %q", got.Priority, PriorityNormal) + } + if got.Kind != KindFact { + t.Errorf("kind = %q, want %q", got.Kind, KindFact) + } + if !got.OccurredAt.Equal(testNow) { + t.Errorf("occurred_at = %v, want %v", got.OccurredAt, testNow) + } +} + +func TestNormalizeKeepsRealOccurredAt(t *testing.T) { + // A wg handshake carries the handshake instant, not "now". Flattening that + // would make every intake look like it happened at notice time. + real := testNow.Add(-3 * time.Hour) + got := Event{Source: "infer:wg", Title: "wg_handshake", OccurredAt: real}.Normalize(testNow) + if !got.OccurredAt.Equal(real) { + t.Errorf("occurred_at = %v, want the supplied %v", got.OccurredAt, real) + } +} + +func TestNormalizeTruncatesOnRuneBoundary(t *testing.T) { + long := strings.Repeat("я", TitleMaxRunes+50) + got := Event{Source: "rss:x", Title: long}.Normalize(testNow) + r := []rune(got.Title) + if len(r) != TitleMaxRunes+1 { // +1 for the ellipsis marker + t.Fatalf("title runes = %d, want %d", len(r), TitleMaxRunes+1) + } + if r[len(r)-1] != '…' { + t.Errorf("truncated title does not mark the cut: %q", string(r[len(r)-3:])) + } + for _, c := range r[:TitleMaxRunes] { + if c != 'я' { + t.Fatalf("truncation broke a rune: got %q", c) + } + } +} + +func TestNormalizeRejectsUnknownPriority(t *testing.T) { + got := Event{Source: "s", Title: "t", Priority: "URGENT!!"}.Normalize(testNow) + if got.Priority != PriorityNormal { + t.Errorf("priority = %q, want %q", got.Priority, PriorityNormal) + } +} + +func TestValid(t *testing.T) { + base := Event{Source: "rss:tech", Kind: KindNote, Title: "заголовок", OccurredAt: testNow} + if !base.Valid() { + t.Fatal("well-formed event reported invalid") + } + for name, mut := range map[string]func(Event) Event{ + "no source": func(e Event) Event { e.Source = ""; return e }, + "no title": func(e Event) Event { e.Title = ""; return e }, + "no time": func(e Event) Event { e.OccurredAt = time.Time{}; return e }, + "bad kind": func(e Event) Event { e.Kind = "whatever"; return e }, + } { + if mut(base).Valid() { + t.Errorf("%s: reported valid", name) + } + } +} + +func TestSourceKind(t *testing.T) { + cases := map[string]string{ + "rss:tech": KindNote, + "crawl:kernel": KindNote, + // No email rule: a WriteFact under an email source is a fact, not a + // captured task. The mail path builds its own task envelope. + "email:inbox": KindFact, + "probe:netdata": KindHealth, + "ambient:notif": KindFact, + "tap:voice": KindFact, + } + for src, want := range cases { + if got := SourceKind(src, KindFact); got != want { + t.Errorf("SourceKind(%q) = %q, want %q", src, got, want) + } + } +} + +func TestBusNilIsANoOp(t *testing.T) { + // The whole adoption story depends on this: an intake path calls Publish + // unconditionally, and a daemon with no bus behaves as it did before. + var b *Bus + b.Publish(Event{Source: "s", Kind: KindFact, Title: "t"}, testNow) + b.Subscribe(func(Event) { t.Error("nil bus delivered to a subscriber") }) + if got := b.Recent(10); got != nil { + t.Errorf("Recent on nil bus = %v, want nil", got) + } + if got := b.Len(); got != 0 { + t.Errorf("Len on nil bus = %d, want 0", got) + } +} + +func TestBusRecentIsNewestFirst(t *testing.T) { + b := NewBus(8) + for _, title := range []string{"one", "two", "three"} { + b.Publish(Event{Source: "rss:t", Kind: KindNote, Title: title}, testNow) + } + got := b.Recent(0) + if len(got) != 3 { + t.Fatalf("len = %d, want 3", len(got)) + } + want := []string{"three", "two", "one"} + for i, w := range want { + if got[i].Title != w { + t.Errorf("Recent()[%d] = %q, want %q", i, got[i].Title, w) + } + } + if lim := b.Recent(2); len(lim) != 2 || lim[0].Title != "three" { + t.Errorf("Recent(2) = %v, want the two newest", lim) + } +} + +func TestBusRingEvicts(t *testing.T) { + b := NewBus(3) + for _, title := range []string{"a", "b", "c", "d", "e"} { + b.Publish(Event{Source: "s", Kind: KindFact, Title: title}, testNow) + } + if b.Len() != 3 { + t.Fatalf("Len = %d, want the capacity 3", b.Len()) + } + got := b.Recent(0) + want := []string{"e", "d", "c"} + for i, w := range want { + if got[i].Title != w { + t.Errorf("Recent()[%d] = %q, want %q", i, got[i].Title, w) + } + } +} + +func TestBusDropsInvalid(t *testing.T) { + b := NewBus(4) + b.Publish(Event{Kind: KindFact, Title: "no source"}, testNow) + b.Publish(Event{Source: "s", Kind: KindFact}, testNow) + if b.Len() != 0 { + t.Errorf("Len = %d, want 0 — an envelope with no provenance must not be kept", b.Len()) + } +} + +func TestBusSubscriberPanicDoesNotBreakIntake(t *testing.T) { + b := NewBus(4) + var seen int + b.Subscribe(func(Event) { panic("observer is broken") }) + b.Subscribe(func(Event) { seen++ }) + b.Publish(Event{Source: "s", Kind: KindFact, Title: "t"}, testNow) + if seen != 1 { + t.Errorf("healthy subscriber called %d times, want 1", seen) + } + if b.Len() != 1 { + t.Errorf("event not recorded despite a panicking subscriber") + } +} + +func TestBusConcurrentPublish(t *testing.T) { + b := NewBus(256) + var wg sync.WaitGroup + for i := 0; i < 16; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 10; j++ { + b.Publish(Event{Source: "s", Kind: KindFact, Title: "t"}, testNow) + } + }() + } + wg.Wait() + if b.Len() != 160 { + t.Errorf("Len = %d, want 160", b.Len()) + } +} + +// The ring is insertion-ordered and the page calls itself newest first, so the +// two only agree if the column is notice time. A cold feed read publishes a +// week of items in feed order, which used to make the OccurredAt column walk +// forwards and backwards on the same page. +func TestRecentIsOrderedByNoticeTimeNotByWhenThingsHappened(t *testing.T) { + b := NewBus(8) + // Published in feed order, six days old first, and a mark stamped now. + for i, e := range []Event{ + {Source: "rss:t", Kind: KindNote, Title: "six days ago", OccurredAt: testNow.Add(-6 * 24 * time.Hour)}, + {Source: "rss:t", Kind: KindNote, Title: "two days ago", OccurredAt: testNow.Add(-2 * 24 * time.Hour)}, + {Source: "ambient:notif", Kind: KindFact, Title: "a meeting at six", OccurredAt: testNow.Add(9 * time.Hour)}, + } { + b.Publish(e, testNow.Add(time.Duration(i)*time.Second)) + } + got := b.Recent(0) + want := []string{"a meeting at six", "two days ago", "six days ago"} + for i, w := range want { + if got[i].Title != w { + t.Errorf("Recent()[%d] = %q, want %q (notice order)", i, got[i].Title, w) + } + } + // Notice time is monotone down the page even where occurrence time is not. + for i := 1; i < len(got); i++ { + if got[i].NoticedAt.After(got[i-1].NoticedAt) { + t.Errorf("NoticedAt is not descending at %d", i) + } + } + if got[0].OccurredAt.Before(got[1].OccurredAt) { + t.Fatal("this fixture is supposed to have occurrence time out of order") + } + // A caller does not get to claim when Maven noticed something. + forged := Event{Source: "s", Kind: KindFact, Title: "t", NoticedAt: testNow.Add(100 * time.Hour)} + b.Publish(forged, testNow) + if n := b.Recent(1)[0].NoticedAt; !n.Equal(testNow) { + t.Errorf("NoticedAt = %v, want the publish instant %v", n, testNow) + } +} diff --git a/internal/ipc/api.go b/internal/ipc/api.go index 6f879b8..1584055 100644 --- a/internal/ipc/api.go +++ b/internal/ipc/api.go @@ -4,6 +4,8 @@ import ( "context" "errors" "time" + + "github.com/kami/maven/internal/audio" ) // DTOs — wire-level data. Decoupled from internal/store so the protocol is @@ -22,6 +24,23 @@ type Fact struct { VoidsID *int64 `json:"voids_id,omitempty"` } +// EcosystemTrace — one hop of a cross-service ecosystem call, read by the +// monitoring surfaces. Traces live in their own store table, not in facts: +// they are written at machine rate and would otherwise crowd every bounded +// reader of facts. +type EcosystemTrace struct { + ID int64 `json:"id"` + Ts time.Time `json:"ts"` + Service string `json:"service"` + Operation string `json:"operation"` + Status string `json:"status"` + DurationMs int64 `json:"duration_ms"` + CorrelationID string `json:"correlation_id"` + CausationID string `json:"causation_id"` + HTTPStatus int `json:"http_status"` + Fields map[string]any `json:"fields,omitempty"` +} + // Bucket — presence hysteresis state: "present" | "away". type Bucket string @@ -93,6 +112,378 @@ type WriteFactReq struct { Subject string `json:"subject,omitempty"` } +// Task — one captured piece of work (Vikunja #130). Status is +// "candidate" (Maven derived it and it is unconfirmed), "open" (his work), +// "done" or "dropped". Source is provenance in the facts vocabulary: +// "tap:voice", "tap:web", "email:". Evidence is the trail a derived +// task came from, empty for anything he stated himself. +type Task struct { + ID int64 `json:"id"` + CreatedTs time.Time `json:"created_ts"` + Text string `json:"text"` + Source string `json:"source"` + Evidence string `json:"evidence,omitempty"` + ExternalID string `json:"external_id,omitempty"` + Status string `json:"status"` + Due *time.Time `json:"due,omitempty"` + Weight int `json:"weight,omitempty"` + Resolved *time.Time `json:"resolved,omitempty"` + ResolvedBy string `json:"resolved_by,omitempty"` +} + +// CaptureTaskReq — THE INTAKE SEAM. Everything that captures a task goes +// through this one shape: the voice path, the web form, and (Vikunja #246) the +// email reader, which has not been built yet. +// +// An extractor that reads mail sets Source "email:", Status +// "candidate", and Evidence to whatever makes the task reviewable (the subject +// line). It cannot set Status "open" — work Maven inferred from something she +// read is a suggestion until the owner confirms it on the /tasks page, and the +// store refuses an open capture from a derived source rather than trusting the +// caller to have read this paragraph. +// +// ExternalID is what makes re-reading free for such a source, and it is +// REQUIRED of one. Text dedupe only covers live rows, because a voice capture +// of the same errand next week is a new task. A mailbox has no such signal: it +// hands back the same immutable message forever, so a task he already finished +// would come back as a fresh candidate on the next poll. ExternalID is unique +// over every row whatever its status: message id plus the extracted span. +type CaptureTaskReq struct { + Text string `json:"text"` + Source string `json:"source"` + Evidence string `json:"evidence,omitempty"` + ExternalID string `json:"external_id,omitempty"` + Status string `json:"status,omitempty"` // "" ⇒ open + Due *time.Time `json:"due,omitempty"` + Weight int `json:"weight,omitempty"` + Ts time.Time `json:"ts"` +} + +// CaptureTaskResp — Created is false when the same live task already existed, +// in which case ID is the existing row. A caller tells the owner "уже в +// списке" rather than claiming it saved something new. +type CaptureTaskResp struct { + ID int64 `json:"id"` + Created bool `json:"created"` + // Promoted — this capture turned an existing candidate into open work. He + // stated out loud something Maven had only proposed, which is a + // confirmation, and the caller says so rather than "уже в списке". + Promoted bool `json:"promoted,omitempty"` +} + +// IngestMailReq — one message a mail reader has fetched, handed to core for +// extraction (Vikunja #246). +// +// The mail reader (cmd/mavmaild) holds the IMAP credential and core never sees +// it, the same split mavpoll uses for the zenmoney token. What crosses this +// boundary is only the message text, because extraction runs on the resident +// model and llama-server lives inside core's process. +// +// Body is already plaintext and truncated by internal/email; core does not +// re-parse MIME and never stores the body. Junk means the reader's header +// filter already classified the message as bulk — core is told rather than +// asked, so a junk message can be counted without a model call. +// +// This method is available only when core has an email block configured AND a +// llama-server phraser; otherwise it answers ErrUnknownMethod, which is what +// "off unless configured" looks like at the wire. +type IngestMailReq struct { + Mailbox string `json:"mailbox"` + UID uint32 `json:"uid"` + From string `json:"from,omitempty"` + Subject string `json:"subject,omitempty"` + Date string `json:"date,omitempty"` + Body string `json:"body,omitempty"` + Junk bool `json:"junk,omitempty"` +} + +// IngestMailResp — what core did with the message. TaskIDs are the rows +// CaptureTask returned; Created counts the ones that were new (a re-read +// mailbox dedupes to Created=0). Skipped is set when nothing was asked of the +// model at all — junk, or an empty message. +// +// Created == 0 && !Skipped therefore means the model WAS consulted and found no +// task, which is the common answer. A reader deciding whether to mark a UID +// seen should treat that the same as a success: asking again would spend the +// resident model on the same negative answer. Skipped means the same for a +// different reason. Only an error means "not read yet". +// +// Nothing here echoes the mail back. The reader logs counts. +type IngestMailResp struct { + TaskIDs []int64 `json:"task_ids,omitempty"` + Created int `json:"created"` + Skipped bool `json:"skipped,omitempty"` +} + +// DescribeImageReq — one image handed to core to look at (Vikunja #252). +// +// Data is the raw image file as received (png / jpeg / gif). Core sniffs it and +// refuses anything else; a declared content type is not part of this request +// because the sender's claim about its own bytes is not evidence. Base64 on the +// wire via the usual JSON marshal of []byte. +// +// Question is what he asked about the picture ("что тут написано?"). Empty ⇒ +// core uses its configured default prompt. +// +// Source is provenance recorded on the stored blob: "telegram", "web:upload". +// +// Exactly one of Data or ID is set, and core refuses a request carrying both: +// it used to take the ID branch and drop the bytes without a word. +// +// The method exists when core has a media store. Vision being off does NOT +// remove it: the bytes are stored and the answer says she cannot read the +// picture yet, which is re-runnable by ID once a vision model is on disk, and +// it is the state this box is in today. So a surface that gets a reply with an +// id and an empty description has not failed, it has stored something. With no +// media block the method answers ErrUnknownMethod, which is what "off unless +// configured" looks like at the wire. +type DescribeImageReq struct { + Data []byte `json:"data,omitempty"` + ID string `json:"id,omitempty"` + Source string `json:"source,omitempty"` + Question string `json:"question,omitempty"` + // SaveNote — also write the description as a note (source + // "media:image:") so it is recallable later. Default false: a + // glance at a screenshot is not automatically a memory. + // + // Setting it raises what the call needs: an embedded note is recall corpus, + // so the caller's source scope must cover auth.ImageNoteSource. Describing + // without saving stays an ordinary read. + SaveNote bool `json:"save_note,omitempty"` +} + +// DescribeImageResp — what she saw. ID is the stored blob's content address, and +// it is set even when Description is empty because the description failed: the +// bytes are on disk and the same id can be retried. NoteID is non-zero only when +// SaveNote was set and the write succeeded. +// +// The image itself is never echoed back. +type DescribeImageResp struct { + ID string `json:"id"` + Description string `json:"description,omitempty"` + Width int `json:"width,omitempty"` + Height int `json:"height,omitempty"` + NoteID int64 `json:"note_id,omitempty"` +} + +// CaptureStartReq — begin recording a meeting (Vikunja #253). +// +// Label is what the meeting is called ("встреча с подрядчиком"); it goes into +// the summary note so the note is findable later. Empty is allowed. +// +// There is no "auto", no keyword and no schedule in this request, and there will +// not be: the only way audio enters the recorder is a client that was told to +// start, appending frames it was told to append. All four capture methods answer +// ErrUnknownMethod unless the operator enabled a capture block, so a surface +// cannot start a recording by asking nicely. +type CaptureStartReq struct { + Label string `json:"label,omitempty"` +} + +// CaptureStartResp — the session that opened. MaxSeconds is the hard cap after +// which it stops itself; the caller tells him, so a forgotten recording is his +// own informed choice rather than a surprise. +type CaptureStartResp struct { + Label string `json:"label,omitempty"` + Started time.Time `json:"started"` + // Token names THIS session. Every later append, stop and discard has to + // carry it. Without it the recorder is addressed by "whatever is running + // now", and a client whose session already ended on the duration cap goes on + // appending its microphone into the next session someone else started. + Token string `json:"token"` + MaxSeconds int `json:"max_seconds"` +} + +// CaptureAppendReq — one chunk of audio for the running session. Refused with +// "nothing is being recorded" when no session is open, which is the guard that +// makes an ambient path impossible: audio arriving at an idle core is dropped on +// the floor, not buffered "just in case". +type CaptureAppendReq struct { + // Token from CaptureStartResp. A frame for a session that already ended is + // refused rather than folded into whatever is running now. + Token string `json:"token"` + Audio audio.Audio `json:"audio"` +} + +// CaptureAppendResp — how much has been collected, so a client can show a timer +// and notice the cap coming. Expired means the session hit its limit and closed; +// stop sending and call capture_stop, the audio so far is kept. +type CaptureAppendResp struct { + Seconds float64 `json:"seconds"` + Expired bool `json:"expired,omitempty"` +} + +// CaptureStopReq — end the running session. +// +// Discard throws the recording away without transcribing, storing or +// summarising anything. This is what "забудь, не записывай" maps to, and it is a +// flag rather than a separate method so the client that says "stop" and the +// client that says "stop and forget" take the same path to the same session. +type CaptureStopReq struct { + // Token from CaptureStartResp. Stopping by "whatever is running" lets a + // late client end a recording it never started. + Token string `json:"token"` + Discard bool `json:"discard,omitempty"` +} + +// CaptureStopResp — the finished capture. BlobID is the stored WAV, kept under +// media.retention like any other blob and pruned with it. +// +// A response with a Transcript and an empty Summary is the normal shape, not a +// failure: summarising a long meeting is a map-reduce of minutes, so stop +// answers with the words and the summary note is written afterwards. Summary is +// filled in only when it happened to be ready. A response with a BlobID and no +// transcript is the audio surviving a transcription failure — the same id can be +// run again by hand off the blob before media.retention prunes it — there is no +// capture method that takes a blob id, so this is not a re-run the wire offers. +// Discarded is true when nothing was kept. +type CaptureStopResp struct { + BlobID string `json:"blob_id,omitempty"` + Label string `json:"label,omitempty"` + Started time.Time `json:"started,omitempty"` + Seconds float64 `json:"seconds,omitempty"` + Transcript string `json:"transcript,omitempty"` + Summary string `json:"summary,omitempty"` + Chunks int `json:"chunks,omitempty"` + NoteID int64 `json:"note_id,omitempty"` + Discarded bool `json:"discarded,omitempty"` +} + +// CaptureStatusResp — what "что ты записываешь?" needs, and what /dash shows. +// Running=false with everything else empty is the normal state. +type CaptureStatusResp struct { + Running bool `json:"running"` + Label string `json:"label,omitempty"` + Started time.Time `json:"started,omitempty"` + Seconds float64 `json:"seconds,omitempty"` + Bytes int `json:"bytes,omitempty"` +} + +// EnrollSpeakerReq — register a voice (Vikunja #255). +// +// Samples are separate utterances recorded deliberately for this purpose, not +// audio harvested from ordinary turns. internal/speaker requires several of +// them totalling enough seconds, and refuses one long clip: a profile built +// from a single sentence encodes that sentence as much as the person. +// +// There is no "enrol whoever just spoke" request shape, and that omission is +// the point. Taking a biometric of a guest because they walked past the +// microphone is not something a wire protocol should make easy. +type EnrollSpeakerReq struct { + ID string `json:"id"` + Name string `json:"name,omitempty"` + Samples []audio.Audio `json:"samples"` +} + +// Speaker — one enrolled voice as a surface sees it. The voiceprint itself is +// never sent: a listing says who is enrolled, it does not hand out the +// biometric. +type Speaker struct { + ID string `json:"id"` + Name string `json:"name"` + Enrolled time.Time `json:"enrolled"` + Samples int `json:"samples"` + // Damaged — the stored row's metadata did not read back cleanly. The + // voiceprint is still there; the name, sample count or enrolment time is + // not trustworthy. A surface should say so rather than render a corrupt + // row as a profile enrolled from zero samples, which is what a real + // minimal enrolment looks like. + Damaged bool `json:"damaged,omitempty"` +} + +// EnrollSpeakerResp — the profile that was written. +type EnrollSpeakerResp struct { + Speaker Speaker `json:"speaker"` +} + +// ListSpeakersResp — who is enrolled, sorted by id. Enabled is false when no +// embedding model is wired, which is this box's state: the profiles can be +// listed and deleted, nothing can be recognised. +type ListSpeakersResp struct { + Speakers []Speaker `json:"speakers"` + Enabled bool `json:"enabled"` +} + +// ForgetSpeakerReq — delete one voiceprint. This is the request that must +// always work; a biometric someone asked to be rid of has to actually go. +type ForgetSpeakerReq struct { + ID string `json:"id"` +} + +// SwapModelReq — load another resident model without restarting the daemon +// (Vikunja #250). ModelPath must be one of the paths in phraser.swap_models; +// anything else is ErrForbidden, and an unconfigured allowlist makes the whole +// method ErrUnknownMethod. +// +// NGpuLayers and NCtx are zero for "keep what is loaded now", which is the +// normal case — the same laptop iGPU, a different gguf. +// +// This is an owner action. It is AuthStepUp in the authority table, it is not on +// CoreAPI, and no act, intent or timer can reach it: swapping the model is not +// something Maven does to herself. +type SwapModelReq struct { + ModelPath string `json:"model_path"` + NGpuLayers int `json:"n_gpu_layers,omitempty"` + NCtx int `json:"n_ctx,omitempty"` +} + +// SwapModelResp — what the daemon ended up serving. Model is the identity the +// new llama-server reported for itself, not an echo of the request: if the file +// was not the model the operator thought it was, this is where it shows. +// +// RolledBack is true when the requested model failed to load or would not answer +// and the previous one was put back. In that case the call also returns an error +// — the swap did not happen — and Model names the model still serving. +// +// NoBackend is the other failure and it is not a milder one: the rollback failed +// too, no model is loaded, and every phrasing path is on its template fallback +// with routing on the classifier. It is a separate field from RolledBack because +// the two need opposite words on the page. +type SwapModelResp struct { + Model string `json:"model"` + ModelPath string `json:"model_path"` + BaseURL string `json:"base_url"` + RolledBack bool `json:"rolled_back,omitempty"` + NoBackend bool `json:"no_backend,omitempty"` + TookMs int64 `json:"took_ms"` +} + +// ModelStatusResp — which model is resident and which ones may be swapped in. +// Read-only; the authed page renders it. Swappable is the configured allowlist, +// so an empty list means the capability is off. +type ModelStatusResp struct { + Model string `json:"model"` + ModelPath string `json:"model_path"` + BaseURL string `json:"base_url"` + NGpuLayers int `json:"n_gpu_layers"` + NCtx int `json:"n_ctx"` + Swappable []string `json:"swappable,omitempty"` +} + +// PingResp — the answer to MethodPing. Alive is always true (the reply itself +// is the proof); Locked says whether the daemon is still waiting for a passkey +// assertion, which is the one state where a CoreAPI read cannot tell an +// operator anything. +type PingResp struct { + Alive bool `json:"alive"` + Locked bool `json:"locked"` +} + +type listTasksReq struct { + Status string `json:"status"` // "" all | "live" | candidate|open|done|dropped +} +type listTasksResp struct { + Tasks []Task `json:"tasks"` +} +type setTaskStatusReq struct { + ID int64 `json:"id"` + Status string `json:"status"` + Ts time.Time `json:"ts"` + // By — the caller making the move, in the source vocabulary. Recorded on + // the row so a resolved task says what resolved it. + By string `json:"by,omitempty"` +} + // idReq — methods keyed by a single id. type idReq struct { ID int64 `json:"id"` @@ -130,6 +521,14 @@ type outcomesReq struct { type nReq struct { N int `json:"n"` } +type kindNReq struct { + Kind string `json:"kind"` + N int `json:"n"` +} +type sourceNReq struct { + Prefix string `json:"prefix"` + N int `json:"n"` +} type calendarEventsReq struct { From time.Time `json:"from"` To time.Time `json:"to"` @@ -180,6 +579,19 @@ type Tool struct { Updated time.Time `json:"updated"` } +// MCPServerStatus — one configured MCP server, as the web surface sees it. +// Target is the command or url; Tools is how many tools discovery kept after +// allow_tools / max_tools, not how many the server offers. +type MCPServerStatus struct { + Name string `json:"name"` + Transport string `json:"transport"` // "stdio" (a local subprocess) or "http" + Target string `json:"target"` + Connected bool `json:"connected"` + Server string `json:"server,omitempty"` // the server's own name + version + Tools int `json:"tools"` + Err string `json:"err,omitempty"` +} + // chatReq / chatResp — text chat round-trip for the IPC Chat method. type chatReq struct { Text string `json:"text"` @@ -264,11 +676,21 @@ type CoreAPI interface { ResolveNudge(ctx context.Context, id int64, outcome string, ts time.Time) error RecentOutcomes(ctx context.Context, rule string, n int) ([]string, error) RecentFacts(ctx context.Context, n int) ([]Fact, error) + RecentActiveFactsByKind(ctx context.Context, kind string, n int) ([]Fact, error) CalendarEvents(ctx context.Context, from, to time.Time) ([]Fact, error) RecentNudges(ctx context.Context, n int) ([]Nudge, error) + + // RecentEcosystemTraces reads the ecosystem call log, which lives in its + // own table so machine-rate traces never crowd out human-rate facts. + RecentEcosystemTraces(ctx context.Context, n int) ([]EcosystemTrace, error) WriteNote(ctx context.Context, ts time.Time, text string, embedding []float32, source string) (int64, error) QueryNotes(ctx context.Context, embedding []float32, k int) ([]Note, error) RecentNotes(ctx context.Context, n int) ([]Note, error) + // RecentNotesFromSource — the newest n notes whose source starts with + // prefix. Notes Maven read rather than heard (rss:, crawl:) are excluded + // from recall, so this is the only way to reach them, and it keeps the feed + // answer from being crowded out of a fixed window by his own notes. + RecentNotesFromSource(ctx context.Context, prefix string, n int) ([]Note, error) // ProposeTool drafts an inert 'proposed' tool scaffold (maven-callable); // returns whether a new proposal was written. EnableTool fills cmd + @@ -294,6 +716,18 @@ type CoreAPI interface { // loop takes the schedule from there — no reminder is created (Vikunja #366). AcceptProposedRoutine(ctx context.Context, id int64) error + // CaptureTask records a task. See CaptureTaskReq — this is the single + // intake seam for the voice path, the web form and the future email + // extractor. Idempotent per live normalised text; the response says + // whether a row was actually created. + CaptureTask(ctx context.Context, req CaptureTaskReq) (CaptureTaskResp, error) + // ListTasks returns tasks in one status, newest first. "" is every row, + // "live" is candidate + open (outstanding work). + ListTasks(ctx context.Context, status string) ([]Task, error) + // SetTaskStatus moves a task forward once: candidate→open|dropped, + // open→done|dropped. Any other move is refused. + SetTaskStatus(ctx context.Context, id int64, status string, ts time.Time, by string) error + // TickTrace returns the most recent tick's rule trace. The daemon caches // this after every tick; the store adapter returns an error (trace is not // persisted — it's a daemon-level cache). @@ -306,10 +740,55 @@ type CoreAPI interface { // TickTrace. MorningStatus(ctx context.Context) ([]MorningRoutineStatus, error) + // MCPServers reports the configured MCP servers and their health + // (Vikunja #251). Read-only introspection for /tools — there is no + // "call this tool" method on purpose: an MCP tool runs through the same + // allowlist, confirm turn and act path as any other tool, and a second + // mutation path would be a second thing to get wrong. Empty when the + // mcp config block is absent, which is the default. + MCPServers(ctx context.Context) ([]MCPServerStatus, error) + + // DayPlan returns today's ordered plan — calendar events, pending + // reminders and any morning checklist still outstanding (see + // internal/morning.BuildPlan) — plus the spoken RU rendering of it. + // Read-only: asking for the plan never dispatches or schedules anything. + // The store adapter returns an error (the plan needs the daemon's routine + // config) — same shape as TickTrace and MorningStatus. + DayPlan(ctx context.Context) (DayPlan, error) + // Chat routes a text utterance through the reactive handler's core path // (router → dialogue → action → replier) and returns the reply text. // No audio or stt/tts — for text channels (mavweb, telegram). Chat(ctx context.Context, text string) (string, error) + + // RecentEvents returns the daemon's unified intake journal, newest first + // (Vikunja #283) — one envelope per thing that arrived, whatever direction + // it came from: a relayed notification, a mail candidate, a feed item, a + // changed page, a spend, a presence probe. + // + // Read-only and daemon-cached, the same shape as TickTrace and DayPlan: + // the store adapter returns an error, because the journal is a bounded + // in-memory ring and not a table. Its contents are a window over intake, + // never the durable record — that is still the fact, note or task the + // intake path wrote. + RecentEvents(ctx context.Context, n int) ([]IntakeEvent, error) +} + +// IntakeEvent — one entry of the unified intake journal on the wire. Mirrors +// event.Event field for field; the ipc package does not import internal/event +// so the wire shape stays independent of the in-process type. +type IntakeEvent struct { + Source string `json:"source"` + Kind string `json:"kind"` + EntityIDs []string `json:"entity_ids,omitempty"` + Title string `json:"title"` + Body string `json:"body,omitempty"` + Priority string `json:"priority"` + OccurredAt time.Time `json:"occurred_at"` + // NoticedAt — when the journal took it. This is the order the ring returns + // and the one a page must sort and label by; OccurredAt is when the thing + // happened, which for a cold feed read is a week before it arrived. + NoticedAt time.Time `json:"noticed_at"` } // --- Rule trace / explanation DTOs --- @@ -359,17 +838,50 @@ type MorningRoutineStatus struct { Items []MorningRoutineItem `json:"items"` } -// storeEncryptionKeyReq — passkey credential public key for wrapping the store -// encryption key at enrollment time. Called by mavweb after RegisterFinish. -type storeEncryptionKeyReq struct { - PublicKey []byte `json:"public_key"` +// DayPlanItem — one line of the day plan. Kind is "event", "reminder" or +// "checklist"; Uncertain marks an item whose provenance is below a full +// calendar read (a meeting relayed off a phone notification), so a UI can hedge +// the same way the spoken form does. +type DayPlanItem struct { + At time.Time `json:"at"` + Text string `json:"text"` + Kind string `json:"kind"` + Uncertain bool `json:"uncertain,omitempty"` } -// unlockReq — passkey credential public key for unwrapping the store -// encryption key at cold-start. mavend reads the wrapped blob from its own -// configured path; the public key is the other half needed for unwrapping. +// DayPlan — the plan for one calendar day. Spoken is the RU sentence maven +// says when asked, rendered core-side so the voice reply and the web view can +// never drift apart. +type DayPlan struct { + Date time.Time `json:"date"` + Items []DayPlanItem `json:"items"` + Spoken string `json:"spoken"` +} + +// storeEncryptionKeyReq — the passkey-derived secret used to wrap the store +// encryption key. Called by mavweb after a verified assertion. +// +// Secret is the 32-byte WebAuthn PRF output, NOT the credential public key. +// The field used to carry the public key and that was the bug: a public key +// sits in passkeys.json next to the wrapped blob, so the blob protected +// nothing. See internal/webauthn/keywrap.go. +// +// Explicit says the operator asked for the cold-start key to be written, as +// opposed to it being a side effect of asserting a passkey. Without the flag +// the daemon writes only when no blob exists yet. Rewriting on every assertion +// is what let a page-level compromise substitute its own PRF value and have +// the daemon re-wrap the real database key under it, and what let a second +// authenticator silently replace the first one's blob. +type storeEncryptionKeyReq struct { + Secret []byte `json:"secret"` + Explicit bool `json:"explicit,omitempty"` +} + +// unlockReq — the passkey-derived secret for unwrapping the store encryption +// key at cold-start. mavend reads the wrapped blob from its own configured +// path; this is the other half. Same PRF-output contract as above. type unlockReq struct { - PublicKey []byte `json:"public_key"` + Secret []byte `json:"secret"` } // ErrToolNotFound — no tool row with this name (re-exported store sentinel for diff --git a/internal/ipc/capture_test.go b/internal/ipc/capture_test.go new file mode 100644 index 0000000..11ce16f --- /dev/null +++ b/internal/ipc/capture_test.go @@ -0,0 +1,123 @@ +package ipc + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/kami/maven/internal/audio" +) + +// The load-bearing default for the most invasive capability Maven has: on a core +// that was never configured to record, there is no wire path that starts a +// recording, feeds one, or harvests one. Every one of the four methods refuses. +func TestCapture_OffUnlessConfigured(t *testing.T) { + _, _, cli, _ := newServerWithStore(t) + ctx := context.Background() + + if _, err := cli.CaptureStart(ctx, CaptureStartReq{Label: "встреча"}); !errors.Is(err, ErrUnknownMethod) { + t.Errorf("CaptureStart error = %v, want ErrUnknownMethod", err) + } + if _, err := cli.CaptureAppend(ctx, CaptureAppendReq{}); !errors.Is(err, ErrUnknownMethod) { + t.Errorf("CaptureAppend error = %v, want ErrUnknownMethod", err) + } + if _, err := cli.CaptureStop(ctx, CaptureStopReq{}); !errors.Is(err, ErrUnknownMethod) { + t.Errorf("CaptureStop error = %v, want ErrUnknownMethod", err) + } + if _, err := cli.CaptureStatus(ctx); !errors.Is(err, ErrUnknownMethod) { + t.Errorf("CaptureStatus error = %v, want ErrUnknownMethod", err) + } +} + +// With the hooks wired, a whole session crosses the boundary intact: the label +// out, the audio in, the summary back. +func TestCapture_RoundTrip(t *testing.T) { + _, srv, cli, _ := newServerWithStore(t) + ctx := context.Background() + + started := time.Now().UTC().Truncate(time.Second) + var gotLabel string + var gotBytes int + var gotDiscard bool + + srv.CaptureStartFn = func(_ context.Context, req CaptureStartReq) (CaptureStartResp, error) { + gotLabel = req.Label + return CaptureStartResp{Label: req.Label, Started: started, MaxSeconds: 7200}, nil + } + srv.CaptureAppendFn = func(_ context.Context, req CaptureAppendReq) (CaptureAppendResp, error) { + gotBytes = len(req.Audio.Bytes) + return CaptureAppendResp{Seconds: 1.5}, nil + } + srv.CaptureStopFn = func(_ context.Context, req CaptureStopReq) (CaptureStopResp, error) { + gotDiscard = req.Discard + return CaptureStopResp{BlobID: "abc", Summary: "— решили купить насос", Chunks: 1}, nil + } + srv.CaptureStatusFn = func(context.Context) (CaptureStatusResp, error) { + return CaptureStatusResp{Running: true, Label: "встреча", Seconds: 1.5}, nil + } + + start, err := cli.CaptureStart(ctx, CaptureStartReq{Label: "встреча с подрядчиком"}) + if err != nil { + t.Fatalf("CaptureStart: %v", err) + } + if gotLabel != "встреча с подрядчиком" || start.MaxSeconds != 7200 { + t.Errorf("start = %+v (label seen: %q)", start, gotLabel) + } + if !start.Started.Equal(started) { + t.Errorf("started = %v, want %v", start.Started, started) + } + + // Audio must survive the JSON round trip byte for byte — a base64 mistake + // here would be silence in the transcript, not a visible error. + pcm := []byte{1, 2, 3, 4, 5, 6, 7, 8} + ap, err := cli.CaptureAppend(ctx, CaptureAppendReq{ + Audio: audio.Audio{Format: audio.PCM16kMono, Bytes: pcm}, + }) + if err != nil { + t.Fatalf("CaptureAppend: %v", err) + } + if gotBytes != len(pcm) { + t.Errorf("%d bytes arrived, sent %d", gotBytes, len(pcm)) + } + if ap.Seconds != 1.5 || ap.Expired { + t.Errorf("append resp = %+v", ap) + } + + st, err := cli.CaptureStatus(ctx) + if err != nil { + t.Fatalf("CaptureStatus: %v", err) + } + if !st.Running || st.Label != "встреча" { + t.Errorf("status = %+v", st) + } + + stop, err := cli.CaptureStop(ctx, CaptureStopReq{}) + if err != nil { + t.Fatalf("CaptureStop: %v", err) + } + if gotDiscard { + t.Error("a plain stop arrived as a discard") + } + if stop.BlobID != "abc" || stop.Summary == "" { + t.Errorf("stop = %+v", stop) + } +} + +// "забудь, не записывай" has to reach core as a discard, not as an ordinary +// stop that quietly keeps everything. +func TestCapture_DiscardCrossesTheWire(t *testing.T) { + _, srv, cli, _ := newServerWithStore(t) + var gotDiscard bool + srv.CaptureStopFn = func(_ context.Context, req CaptureStopReq) (CaptureStopResp, error) { + gotDiscard = req.Discard + return CaptureStopResp{Discarded: req.Discard}, nil + } + resp, err := cli.CaptureStop(context.Background(), CaptureStopReq{Discard: true}) + if err != nil { + t.Fatalf("CaptureStop: %v", err) + } + if !gotDiscard || !resp.Discarded { + t.Errorf("discard lost: sent true, core saw %v, resp %+v", gotDiscard, resp) + } +} diff --git a/internal/ipc/client.go b/internal/ipc/client.go index 208c250..ed4a3d1 100644 --- a/internal/ipc/client.go +++ b/internal/ipc/client.go @@ -55,22 +55,30 @@ var ErrAmbiguousOutcome = errors.New("ipc: mutation outcome unknown (connection // conservative (refusing to retry) is the safe default for a method added // here by omission. var readOnlyMethods = map[Method]bool{ - MethodLatestFact: true, - MethodLatestFactBySource: true, - MethodSince: true, - MethodPresence: true, - MethodListReminders: true, - MethodRecentOutcomes: true, - MethodRecentFacts: true, - MethodCalendarEvents: true, - MethodRecentNudges: true, - MethodQueryNotes: true, - MethodRecentNotes: true, - MethodLookupTool: true, - MethodListTools: true, - MethodListProposedRoutines: true, - MethodTickTrace: true, - MethodMorningStatus: true, + MethodLatestFact: true, + MethodLatestFactBySource: true, + MethodSince: true, + MethodPresence: true, + MethodListReminders: true, + MethodRecentOutcomes: true, + MethodRecentFacts: true, + MethodRecentActiveFacts: true, + MethodCalendarEvents: true, + MethodRecentNudges: true, + MethodRecentEcoTraces: true, + MethodQueryNotes: true, + MethodRecentNotes: true, + MethodLookupTool: true, + MethodListTools: true, + MethodListProposedRoutines: true, + MethodListTasks: true, + MethodTickTrace: true, + MethodMorningStatus: true, + MethodMCPServers: true, + MethodDayPlan: true, + MethodRecentEvents: true, + MethodRecentNotesFromSource: true, + MethodPing: true, } // Dial connects to a core socket at path and returns a Client. The module @@ -330,6 +338,14 @@ func (c *Client) RecentFacts(ctx context.Context, n int) ([]Fact, error) { return out, nil } +func (c *Client) RecentActiveFactsByKind(ctx context.Context, kind string, n int) ([]Fact, error) { + var out []Fact + if err := c.call(ctx, MethodRecentActiveFacts, kindNReq{Kind: kind, N: n}, &out); err != nil { + return nil, err + } + return out, nil +} + func (c *Client) CalendarEvents(ctx context.Context, from, to time.Time) ([]Fact, error) { var out []Fact if err := c.call(ctx, MethodCalendarEvents, calendarEventsReq{From: from, To: to}, &out); err != nil { @@ -338,6 +354,14 @@ func (c *Client) CalendarEvents(ctx context.Context, from, to time.Time) ([]Fact return out, nil } +func (c *Client) RecentEcosystemTraces(ctx context.Context, n int) ([]EcosystemTrace, error) { + var out []EcosystemTrace + if err := c.call(ctx, MethodRecentEcoTraces, nReq{N: n}, &out); err != nil { + return nil, err + } + return out, nil +} + func (c *Client) RecentNudges(ctx context.Context, n int) ([]Nudge, error) { var out []Nudge if err := c.call(ctx, MethodRecentNudges, nReq{N: n}, &out); err != nil { @@ -370,6 +394,14 @@ func (c *Client) RecentNotes(ctx context.Context, n int) ([]Note, error) { return out, nil } +func (c *Client) RecentNotesFromSource(ctx context.Context, prefix string, n int) ([]Note, error) { + var out []Note + if err := c.call(ctx, MethodRecentNotesFromSource, sourceNReq{Prefix: prefix, N: n}, &out); err != nil { + return nil, err + } + return out, nil +} + func (c *Client) ProposeTool(ctx context.Context, name, utterance, scope string, ts time.Time) (bool, error) { var r proposeToolResp if err := c.call(ctx, MethodProposeTool, proposeToolReq{Name: name, Scope: scope, Utterance: utterance, Ts: ts}, &r); err != nil { @@ -390,12 +422,21 @@ func (c *Client) AssertStepUp(ctx context.Context) error { return c.call(ctx, MethodAssertStepUp, nil, nil) } -func (c *Client) StoreEncryptionKey(ctx context.Context, publicKey []byte) error { - return c.call(ctx, MethodStoreEncryptionKey, storeEncryptionKeyReq{PublicKey: publicKey}, nil) +// StoreEncryptionKey wraps the daemon's at-rest key under secret, the 32-byte +// WebAuthn PRF output for the asserted credential. +// +// explicit marks an operator-requested write. False means "write it only if +// there is nothing there yet": a blob already on disk is left alone, because +// rewriting it on every assertion is how an attacker-chosen PRF value, or a +// second authenticator, replaces the one thing that opens the database. +func (c *Client) StoreEncryptionKey(ctx context.Context, secret []byte, explicit bool) error { + return c.call(ctx, MethodStoreEncryptionKey, storeEncryptionKeyReq{Secret: secret, Explicit: explicit}, nil) } -func (c *Client) Unlock(ctx context.Context, publicKey []byte) error { - return c.call(ctx, MethodUnlock, unlockReq{PublicKey: publicKey}, nil) +// Unlock hands the daemon the PRF secret so it can unwrap its at-rest key and +// open the store. Refused unless a passkey assertion was verified first. +func (c *Client) Unlock(ctx context.Context, secret []byte) error { + return c.call(ctx, MethodUnlock, unlockReq{Secret: secret}, nil) } func (c *Client) LookupTool(ctx context.Context, name string) (Tool, error) { @@ -426,6 +467,143 @@ func (c *Client) ListProposedRoutines(ctx context.Context) ([]ProposedRoutine, e return r.Routines, nil } +func (c *Client) CaptureTask(ctx context.Context, req CaptureTaskReq) (CaptureTaskResp, error) { + var r CaptureTaskResp + if err := c.call(ctx, MethodCaptureTask, req, &r); err != nil { + return CaptureTaskResp{}, err + } + return r, nil +} + +func (c *Client) ListTasks(ctx context.Context, status string) ([]Task, error) { + var r listTasksResp + if err := c.call(ctx, MethodListTasks, listTasksReq{Status: status}, &r); err != nil { + return nil, err + } + return r.Tasks, nil +} + +func (c *Client) SetTaskStatus(ctx context.Context, id int64, status string, ts time.Time, by string) error { + return c.call(ctx, MethodSetTaskStatus, setTaskStatusReq{ID: id, Status: status, Ts: ts, By: by}, nil) +} + +// IngestMail hands one fetched message to core for extraction. ErrUnknownMethod +// means core has no email block configured — the caller should stop asking, not +// retry. +func (c *Client) IngestMail(ctx context.Context, req IngestMailReq) (IngestMailResp, error) { + var r IngestMailResp + if err := c.call(ctx, MethodIngestMail, req, &r); err != nil { + return IngestMailResp{}, err + } + return r, nil +} + +// DescribeImage hands one image to core to look at (Vikunja #252). +// ErrUnknownMethod means core has no media store or vision is off — the caller +// should stop asking, not retry. A response with an ID and an empty Description +// means the bytes were stored but nothing could describe them yet, which is the +// expected state on a box with no vision model on disk. +func (c *Client) DescribeImage(ctx context.Context, req DescribeImageReq) (DescribeImageResp, error) { + var r DescribeImageResp + if err := c.call(ctx, MethodDescribeImage, req, &r); err != nil { + return DescribeImageResp{}, err + } + return r, nil +} + +// CaptureStart begins recording a meeting (Vikunja #253). ErrUnknownMethod +// means the operator has not enabled capture — the caller should say so and stop +// asking, not retry. +func (c *Client) CaptureStart(ctx context.Context, req CaptureStartReq) (CaptureStartResp, error) { + var r CaptureStartResp + if err := c.call(ctx, MethodCaptureStart, req, &r); err != nil { + return CaptureStartResp{}, err + } + return r, nil +} + +// CaptureAppend hands one chunk of audio to the running session. An error means +// the frame was not kept: either nothing is being recorded, or the session hit +// its time limit. Either way the client stops sending. +func (c *Client) CaptureAppend(ctx context.Context, req CaptureAppendReq) (CaptureAppendResp, error) { + var r CaptureAppendResp + if err := c.call(ctx, MethodCaptureAppend, req, &r); err != nil { + return CaptureAppendResp{}, err + } + return r, nil +} + +// CaptureStop ends the session. It transcribes the whole recording before +// answering, so pass a context with room; the summary is written afterwards by +// the daemon and is usually absent from the response. Set Discard to throw the +// recording away instead. Token comes from CaptureStart. +func (c *Client) CaptureStop(ctx context.Context, req CaptureStopReq) (CaptureStopResp, error) { + var r CaptureStopResp + if err := c.call(ctx, MethodCaptureStop, req, &r); err != nil { + return CaptureStopResp{}, err + } + return r, nil +} + +// CaptureStatus reports the running session, if any. +func (c *Client) CaptureStatus(ctx context.Context) (CaptureStatusResp, error) { + var r CaptureStatusResp + if err := c.call(ctx, MethodCaptureStatus, nil, &r); err != nil { + return CaptureStatusResp{}, err + } + return r, nil +} + +// EnrollSpeaker registers a voice from several deliberately recorded samples +// (Vikunja #255). ErrUnknownMethod means no speaker block is configured, which +// is the default: on an unconfigured box there is no way to take a voiceprint. +func (c *Client) EnrollSpeaker(ctx context.Context, req EnrollSpeakerReq) (EnrollSpeakerResp, error) { + var r EnrollSpeakerResp + if err := c.call(ctx, MethodEnrollSpeaker, req, &r); err != nil { + return EnrollSpeakerResp{}, err + } + return r, nil +} + +// ListSpeakers reports who is enrolled. The voiceprints themselves stay in +// core. Enabled is false when profiles exist but no embedding model is wired, +// so a surface can say "enrolled, not recognising" rather than implying Maven +// knows who is talking. +func (c *Client) ListSpeakers(ctx context.Context) (ListSpeakersResp, error) { + var r ListSpeakersResp + if err := c.call(ctx, MethodListSpeakers, nil, &r); err != nil { + return ListSpeakersResp{}, err + } + return r, nil +} + +// ForgetSpeaker deletes one voiceprint. +func (c *Client) ForgetSpeaker(ctx context.Context, id string) error { + return c.call(ctx, MethodForgetSpeaker, ForgetSpeakerReq{ID: id}, nil) +} + +// SwapModel asks core to load another resident model (Vikunja #250). +// ErrUnknownMethod means core has no phraser.swap_models allowlist configured; +// ErrForbidden means the path is not on it, or step-up was not asserted. A +// non-nil error with RolledBack set means nothing changed — the old model is +// still serving. +func (c *Client) SwapModel(ctx context.Context, req SwapModelReq) (SwapModelResp, error) { + var r SwapModelResp + if err := c.call(ctx, MethodSwapModel, req, &r); err != nil { + return SwapModelResp{}, err + } + return r, nil +} + +// ModelStatus reports the resident model and the swap allowlist. Read-only. +func (c *Client) ModelStatus(ctx context.Context) (ModelStatusResp, error) { + var r ModelStatusResp + if err := c.call(ctx, MethodModelStatus, nil, &r); err != nil { + return ModelStatusResp{}, err + } + return r, nil +} + func (c *Client) DismissProposedRoutine(ctx context.Context, id int64) error { return c.call(ctx, MethodDismissProposedRoutine, dismissProposedRoutineReq{ID: id}, nil) } @@ -450,6 +628,22 @@ func (c *Client) TickTrace(ctx context.Context) (TickTrace, error) { return t, nil } +func (c *Client) RecentEvents(ctx context.Context, n int) ([]IntakeEvent, error) { + var e []IntakeEvent + if err := c.call(ctx, MethodRecentEvents, nReq{N: n}, &e); err != nil { + return nil, err + } + return e, nil +} + +func (c *Client) MCPServers(ctx context.Context) ([]MCPServerStatus, error) { + var s []MCPServerStatus + if err := c.call(ctx, MethodMCPServers, nil, &s); err != nil { + return nil, err + } + return s, nil +} + func (c *Client) MorningStatus(ctx context.Context) ([]MorningRoutineStatus, error) { var s []MorningRoutineStatus if err := c.call(ctx, MethodMorningStatus, nil, &s); err != nil { @@ -458,6 +652,14 @@ func (c *Client) MorningStatus(ctx context.Context) ([]MorningRoutineStatus, err return s, nil } +func (c *Client) DayPlan(ctx context.Context) (DayPlan, error) { + var p DayPlan + if err := c.call(ctx, MethodDayPlan, nil, &p); err != nil { + return DayPlan{}, err + } + return p, nil +} + func (c *Client) RevertFact(ctx context.Context, key string) (int64, error) { var result struct { NewID int64 `json:"new_id"` @@ -468,5 +670,16 @@ func (c *Client) RevertFact(ctx context.Context, key string) (int64, error) { return result.NewID, nil } +// Ping asks whether the daemon is there, and whether it is locked. It is not a +// CoreAPI method: it touches no store, so it answers before the passkey +// assertion that every other read waits for. +func (c *Client) Ping(ctx context.Context) (PingResp, error) { + var r PingResp + if err := c.call(ctx, MethodPing, nil, &r); err != nil { + return PingResp{}, err + } + return r, nil +} + // Compile-time check: *Client satisfies CoreAPI. var _ CoreAPI = (*Client)(nil) diff --git a/internal/ipc/ipc_test.go b/internal/ipc/ipc_test.go index 727b8ac..885d7bd 100644 --- a/internal/ipc/ipc_test.go +++ b/internal/ipc/ipc_test.go @@ -6,6 +6,7 @@ import ( "encoding/binary" "encoding/json" "errors" + "fmt" "io" "net" "os" @@ -410,95 +411,12 @@ func TestChatViaClient(t *testing.T) { } // chatTestAPI — a minimal CoreAPI that only implements Chat for testing. -type chatTestAPI struct{} +// Embeds UnimplementedCoreAPI so every other method fails loudly with +// ErrNotImplemented instead of needing 27 hand-written no-op stubs. +type chatTestAPI struct { + UnimplementedCoreAPI +} -func (a *chatTestAPI) WriteFact(ctx context.Context, req WriteFactReq) (int64, error) { - return 0, ErrUnknownMethod -} -func (a *chatTestAPI) LatestFact(ctx context.Context, key string) (Fact, error) { - return Fact{}, ErrUnknownMethod -} -func (a *chatTestAPI) LatestFactBySource(ctx context.Context, key, source string) (Fact, error) { - return Fact{}, ErrUnknownMethod -} -func (a *chatTestAPI) Since(ctx context.Context, key string, now time.Time) (time.Duration, error) { - return 0, ErrUnknownMethod -} -func (a *chatTestAPI) Presence(ctx context.Context) (Presence, error) { - return Presence{}, ErrUnknownMethod -} -func (a *chatTestAPI) CreateReminder(ctx context.Context, fire time.Time, payload, cron string) (int64, error) { - return 0, ErrUnknownMethod -} -func (a *chatTestAPI) MarkReminder(ctx context.Context, id int64, status string) error { - return ErrUnknownMethod -} -func (a *chatTestAPI) ListReminders(ctx context.Context, n int) ([]Reminder, error) { - return nil, ErrUnknownMethod -} -func (a *chatTestAPI) RecordNudge(ctx context.Context, rule, channel, message string, ts time.Time) (int64, error) { - return 0, ErrUnknownMethod -} -func (a *chatTestAPI) ResolveNudge(ctx context.Context, id int64, outcome string, ts time.Time) error { - return ErrUnknownMethod -} -func (a *chatTestAPI) RecentOutcomes(ctx context.Context, rule string, n int) ([]string, error) { - return nil, ErrUnknownMethod -} -func (a *chatTestAPI) RecentFacts(ctx context.Context, n int) ([]Fact, error) { - return nil, ErrUnknownMethod -} -func (a *chatTestAPI) CalendarEvents(ctx context.Context, from, to time.Time) ([]Fact, error) { - return nil, ErrUnknownMethod -} -func (a *chatTestAPI) RecentNudges(ctx context.Context, n int) ([]Nudge, error) { - return nil, ErrUnknownMethod -} -func (a *chatTestAPI) WriteNote(ctx context.Context, ts time.Time, text string, embedding []float32, source string) (int64, error) { - return 0, ErrUnknownMethod -} -func (a *chatTestAPI) QueryNotes(ctx context.Context, embedding []float32, k int) ([]Note, error) { - return nil, ErrUnknownMethod -} -func (a *chatTestAPI) RecentNotes(ctx context.Context, n int) ([]Note, error) { - return nil, ErrUnknownMethod -} -func (a *chatTestAPI) ProposeTool(ctx context.Context, name, utterance, scope string, ts time.Time) (bool, error) { - return false, ErrUnknownMethod -} -func (a *chatTestAPI) EnableTool(ctx context.Context, name string, cmd []string, destructive bool, scope string, ts time.Time) error { - return ErrUnknownMethod -} -func (a *chatTestAPI) DisableTool(ctx context.Context, name string) error { - return ErrUnknownMethod -} -func (a *chatTestAPI) DeleteTool(ctx context.Context, name string) error { - return ErrUnknownMethod -} -func (a *chatTestAPI) ListProposedRoutines(ctx context.Context) ([]ProposedRoutine, error) { - return nil, ErrUnknownMethod -} -func (a *chatTestAPI) AcceptProposedRoutine(ctx context.Context, id int64) error { - return nil -} -func (a *chatTestAPI) DismissProposedRoutine(ctx context.Context, id int64) error { - return ErrUnknownMethod -} -func (a *chatTestAPI) LookupTool(ctx context.Context, name string) (Tool, error) { - return Tool{}, ErrUnknownMethod -} -func (a *chatTestAPI) ListTools(ctx context.Context, status string) ([]Tool, error) { - return nil, ErrUnknownMethod -} -func (a *chatTestAPI) RevertFact(ctx context.Context, key string) (int64, error) { - return 0, ErrUnknownMethod -} -func (a *chatTestAPI) TickTrace(ctx context.Context) (TickTrace, error) { - return TickTrace{}, ErrUnknownMethod -} -func (a *chatTestAPI) MorningStatus(ctx context.Context) ([]MorningRoutineStatus, error) { - return nil, ErrUnknownMethod -} func (a *chatTestAPI) Chat(ctx context.Context, text string) (string, error) { if text == "привет" { return "и тебе привет!", nil @@ -648,3 +566,77 @@ func mustJSON(v any) []byte { } return b } + +// TestIngestMail_OffUnlessConfigured — with no IngestMailFn set (the default, +// and what an unconfigured core looks like) the method does not exist. A mail +// reader gets a refusal it can act on rather than a silent success. +func TestIngestMail_OffUnlessConfigured(t *testing.T) { + _, _, cli, _ := newServerWithStore(t) + if _, err := cli.IngestMail(context.Background(), IngestMailReq{Mailbox: "INBOX", UID: 1}); !errors.Is(err, ErrUnknownMethod) { + t.Fatalf("IngestMail error = %v, want ErrUnknownMethod", err) + } +} + +// TestIngestMail_Hook — when the daemon wires the hook, the message crosses the +// boundary intact and the response comes back. +func TestIngestMail_Hook(t *testing.T) { + _, srv, cli, _ := newServerWithStore(t) + var got IngestMailReq + srv.IngestMailFn = func(_ context.Context, req IngestMailReq) (IngestMailResp, error) { + got = req + return IngestMailResp{TaskIDs: []int64{7}, Created: 1}, nil + } + resp, err := cli.IngestMail(context.Background(), IngestMailReq{ + Mailbox: "INBOX", UID: 12, Subject: "Счёт", Body: "Оплатить.", Junk: false, + }) + if err != nil { + t.Fatalf("IngestMail: %v", err) + } + if resp.Created != 1 || len(resp.TaskIDs) != 1 || resp.TaskIDs[0] != 7 { + t.Errorf("resp = %+v", resp) + } + if got.UID != 12 || got.Subject != "Счёт" || got.Body != "Оплатить." { + t.Errorf("req across the wire = %+v", got) + } +} + +// TestSwapModel_OffUnlessConfigured — no allowlist in the config means the +// daemon never sets the hook, and the method does not exist. That is what "off +// unless configured" looks like at the wire for the model swap (Vikunja #250). +func TestSwapModel_OffUnlessConfigured(t *testing.T) { + _, _, cli, _ := newServerWithStore(t) + if _, err := cli.SwapModel(context.Background(), SwapModelReq{ModelPath: "/m/x.gguf"}); !errors.Is(err, ErrUnknownMethod) { + t.Fatalf("SwapModel error = %v, want ErrUnknownMethod", err) + } + if _, err := cli.ModelStatus(context.Background()); !errors.Is(err, ErrUnknownMethod) { + t.Fatalf("ModelStatus error = %v, want ErrUnknownMethod", err) + } +} + +// TestSwapModel_Hook — the request crosses the boundary intact and the reported +// identity comes back. A refusal from the daemon's allowlist arrives as +// ErrForbidden, which is what a caller keys its error message off. +func TestSwapModel_Hook(t *testing.T) { + _, srv, cli, _ := newServerWithStore(t) + var got SwapModelReq + srv.SwapModelFn = func(_ context.Context, req SwapModelReq) (SwapModelResp, error) { + got = req + if req.ModelPath != "/m/allowed.gguf" { + return SwapModelResp{}, fmt.Errorf("%w: not allowlisted", ErrForbidden) + } + return SwapModelResp{Model: "allowed", ModelPath: req.ModelPath, BaseURL: "http://127.0.0.1:9", TookMs: 12}, nil + } + resp, err := cli.SwapModel(context.Background(), SwapModelReq{ModelPath: "/m/allowed.gguf", NCtx: 4096}) + if err != nil { + t.Fatalf("SwapModel: %v", err) + } + if resp.Model != "allowed" || resp.TookMs != 12 { + t.Errorf("resp = %+v", resp) + } + if got.NCtx != 4096 { + t.Errorf("req across the wire = %+v", got) + } + if _, err := cli.SwapModel(context.Background(), SwapModelReq{ModelPath: "/etc/shadow"}); !errors.Is(err, ErrForbidden) { + t.Fatalf("swap to a non-allowlisted path = %v; want ErrForbidden", err) + } +} diff --git a/internal/ipc/server.go b/internal/ipc/server.go index b316c9b..179f697 100644 --- a/internal/ipc/server.go +++ b/internal/ipc/server.go @@ -120,6 +120,18 @@ func (a *storeAPI) RecentFacts(ctx context.Context, n int) ([]Fact, error) { return out, nil } +func (a *storeAPI) RecentActiveFactsByKind(ctx context.Context, kind string, n int) ([]Fact, error) { + fs, err := a.s.RecentActiveFactsByKind(ctx, store.FactKind(kind), n) + if err != nil { + return nil, mapErr(err) + } + out := make([]Fact, len(fs)) + for i, f := range fs { + out[i] = toFact(f) + } + return out, nil +} + func (a *storeAPI) CalendarEvents(ctx context.Context, from, to time.Time) ([]Fact, error) { fs, err := a.s.CalendarEvents(ctx, from, to) if err != nil { @@ -132,6 +144,22 @@ func (a *storeAPI) CalendarEvents(ctx context.Context, from, to time.Time) ([]Fa return out, nil } +func (a *storeAPI) RecentEcosystemTraces(ctx context.Context, n int) ([]EcosystemTrace, error) { + trs, err := a.s.RecentEcosystemTraces(ctx, n) + if err != nil { + return nil, mapErr(err) + } + out := make([]EcosystemTrace, len(trs)) + for i, tr := range trs { + out[i] = EcosystemTrace{ + ID: tr.ID, Ts: tr.Ts, Service: tr.Service, Operation: tr.Operation, + Status: tr.Status, DurationMs: tr.DurationMs, CorrelationID: tr.CorrelationID, + CausationID: tr.CausationID, HTTPStatus: tr.HTTPStatus, Fields: tr.Fields, + } + } + return out, nil +} + func (a *storeAPI) RecentNudges(ctx context.Context, n int) ([]Nudge, error) { ns, err := a.s.RecentNudges(ctx, n) if err != nil { @@ -161,6 +189,18 @@ func (a *storeAPI) QueryNotes(ctx context.Context, embedding []float32, k int) ( return out, nil } +func (a *storeAPI) RecentNotesFromSource(ctx context.Context, prefix string, n int) ([]Note, error) { + ns, err := a.s.RecentNotesFromSource(ctx, prefix, n) + if err != nil { + return nil, mapErr(err) + } + out := make([]Note, len(ns)) + for i, note := range ns { + out[i] = toNote(note) + } + return out, nil +} + func (a *storeAPI) RecentNotes(ctx context.Context, n int) ([]Note, error) { ns, err := a.s.RecentNotes(ctx, n) if err != nil { @@ -211,6 +251,20 @@ func (a *storeAPI) MorningStatus(ctx context.Context) ([]MorningRoutineStatus, e return nil, errors.New("store: morning status not available via direct store API") } +// RecentEvents — same shape as TickTrace: the intake journal is a bounded ring +// in the daemon's memory, not a table, so a bare store cannot serve it. +func (a *storeAPI) RecentEvents(ctx context.Context, n int) ([]IntakeEvent, error) { + return nil, errors.New("store: intake events not available via direct store API") +} + +func (a *storeAPI) MCPServers(ctx context.Context) ([]MCPServerStatus, error) { + return nil, nil // no manager behind a bare store: nothing configured +} + +func (a *storeAPI) DayPlan(ctx context.Context) (DayPlan, error) { + return DayPlan{}, errors.New("store: day plan not available via direct store API") +} + func (a *storeAPI) ListTools(ctx context.Context, status string) ([]Tool, error) { ts, err := a.s.ListTools(ctx, status) if err != nil { @@ -227,6 +281,51 @@ func (a *storeAPI) DeleteTool(ctx context.Context, name string) error { return mapErr(a.s.DeleteTool(ctx, name)) } +func (a *storeAPI) CaptureTask(ctx context.Context, req CaptureTaskReq) (CaptureTaskResp, error) { + res, err := a.s.CaptureTask(ctx, store.Task{ + CreatedTs: req.Ts, + Text: req.Text, + Source: req.Source, + Evidence: req.Evidence, + ExternalID: req.ExternalID, + Status: req.Status, + Due: req.Due, + Weight: req.Weight, + }) + if err != nil { + return CaptureTaskResp{}, mapErr(err) + } + return CaptureTaskResp{ID: res.ID, Created: res.Created, Promoted: res.Promoted}, nil +} + +func (a *storeAPI) ListTasks(ctx context.Context, status string) ([]Task, error) { + ts, err := a.s.ListTasks(ctx, status) + if err != nil { + return nil, mapErr(err) + } + out := make([]Task, len(ts)) + for i, t := range ts { + out[i] = Task{ + ID: t.ID, + CreatedTs: t.CreatedTs, + Text: t.Text, + Source: t.Source, + Evidence: t.Evidence, + ExternalID: t.ExternalID, + Status: t.Status, + Due: t.Due, + Weight: t.Weight, + Resolved: t.ResolvedTs, + ResolvedBy: t.ResolvedBy, + } + } + return out, nil +} + +func (a *storeAPI) SetTaskStatus(ctx context.Context, id int64, status string, ts time.Time, by string) error { + return mapErr(a.s.SetTaskStatus(ctx, id, status, ts, by)) +} + func (a *storeAPI) ListProposedRoutines(ctx context.Context) ([]ProposedRoutine, error) { rs, err := a.s.ListProposedRoutines(ctx) if err != nil { @@ -370,13 +469,75 @@ type Server struct { // MethodAssertStepUp returns ErrUnknownMethod (same as pre-stepup floor). StepUp StepUpFunc - // WrapKeyFn — wraps the in-memory store encryption key with a passkey - // credential public key (HKDF-AESGCM) and writes the wrapped blob to disk. + // WrapKeyFn — wraps the in-memory store encryption key under the passkey + // PRF secret (HKDF-AESGCM) and writes the wrapped blob to disk. // Set by the daemon; nil ⇒ MethodStoreEncryptionKey returns ErrUnknownMethod. WrapKeyFn WrapKeyFunc + // IngestMailFn — extracts task candidates from one fetched message. Set by + // the daemon only when an email block is configured AND there is a + // llama-server to extract with; nil ⇒ MethodIngestMail returns + // ErrUnknownMethod, so a mail reader pointed at a core that is not + // configured for mail is refused rather than silently ignored. + // + // Like StepUp/WrapKeyFn/UnlockFn this bypasses CoreAPI: it is not a store + // operation, it needs the resident model, and it must not become a method + // every CoreAPI implementation has to carry. + IngestMailFn IngestMailFunc + + // SwapModelFn / ModelStatusFn — the on-the-fly resident model swap (Vikunja + // #250) and its read side. Set by the daemon only when phraser.swap_models + // lists at least one model AND the phraser owns a llama-server; nil ⇒ both + // methods answer ErrUnknownMethod, which is what "off unless configured" + // looks like at the wire. + // + // They bypass CoreAPI for the same reason IngestMailFn does: this is not a + // store operation, it needs the daemon's llama-server, and no other CoreAPI + // implementation should have to carry it. MethodSwapModel is AuthStepUp in + // internal/auth — owner-triggered, never an act and never a timer. + SwapModelFn SwapModelFunc + ModelStatusFn ModelStatusFunc + + // DescribeImageFn — looks at one image (Vikunja #252). Set by the daemon + // whenever a media store is configured. Vision being off does not clear it: + // the image is stored and the reply says she cannot read it yet, which is + // re-runnable by id later. nil ⇒ no media block ⇒ MethodDescribeImage + // answers ErrUnknownMethod, so a surface cannot make Maven accept a photo + // by merely sending one. + // + // It bypasses CoreAPI for the same reason IngestMailFn does: it needs a blob + // store and a vision server, neither of which is a store operation, and no + // other CoreAPI implementation should have to carry it. + DescribeImageFn DescribeImageFunc + + // Capture* — the meeting recorder (Vikunja #253). Set by the daemon only + // when a media store is configured AND capture.enabled is true; nil ⇒ all + // four methods answer ErrUnknownMethod. That is the load-bearing default for + // this capability: on an unconfigured box there is no wire path that begins a + // recording, so nothing can be recorded by accident, by a bug in a surface, + // or by a model deciding it would be helpful. + // + // They bypass CoreAPI because a recorder needs a blob store, an STT worker + // and a llama-server, none of which is a store operation. + CaptureStartFn CaptureStartFunc + CaptureAppendFn CaptureAppendFunc + CaptureStopFn CaptureStopFunc + CaptureStatusFn CaptureStatusFunc + + // Speaker* — voice identification (Vikunja #255). Set by the daemon only + // when a speaker block is configured; nil ⇒ all three methods answer + // ErrUnknownMethod, so on an unconfigured box no wire path enrols a voice. + EnrollSpeakerFn EnrollSpeakerFunc + ListSpeakersFn ListSpeakersFunc + ForgetSpeakerFn ForgetSpeakerFunc + + // LockedFn — reports whether the daemon is in locked (pre-unlock) mode. + // Read by MethodPing only. Nil ⇒ not locked, which is what an embedded or + // test Server without the unlock dance is. + LockedFn func() bool + // UnlockFn — unwraps the store encryption key from the wrapped blob using - // the passkey credential public key, opens the encrypted store, and wires + // the passkey PRF secret, opens the encrypted store, and wires // the rest of the daemon (voice, loop, delivery). Set by the daemon when // in locked mode; nil ⇒ MethodUnlock returns ErrUnknownMethod. UnlockFn UnlockFunc @@ -385,13 +546,42 @@ type Server struct { // absolute ts supplied by callers, so this isn't load-bearing for live ops. } -// WrapKeyFunc — wraps the store encryption key with the given credential -// public key and persists the wrapped blob. -type WrapKeyFunc func(ctx context.Context, publicKey []byte) error +// WrapKeyFunc — wraps the store encryption key under the passkey-derived +// secret (a 32-byte WebAuthn PRF output) and persists the wrapped blob. +// +// explicit distinguishes "the operator asked for the cold-start key to be +// written" from "a passkey was asserted". Only the first may overwrite a blob +// that is already there; see cmd/mavend/keyfile.go. +type WrapKeyFunc func(ctx context.Context, secret []byte, explicit bool) error -// UnlockFunc — unwraps the store encryption key using the given credential -// public key and completes daemon initialization. -type UnlockFunc func(ctx context.Context, publicKey []byte) error +// UnlockFunc — unwraps the store encryption key using the passkey-derived +// secret and completes daemon initialization. +type UnlockFunc func(ctx context.Context, secret []byte) error + +// SwapModelFunc — loads another resident model in place of the live one. +type SwapModelFunc func(ctx context.Context, req SwapModelReq) (SwapModelResp, error) + +// ModelStatusFunc — reports the resident model and the swap allowlist. +type ModelStatusFunc func(ctx context.Context) (ModelStatusResp, error) + +// IngestMailFunc — core-side mail extraction. Returns what was captured. +type IngestMailFunc func(ctx context.Context, req IngestMailReq) (IngestMailResp, error) + +// DescribeImageFunc — core-side image intake + description. +type DescribeImageFunc func(ctx context.Context, req DescribeImageReq) (DescribeImageResp, error) + +// CaptureStartFunc / CaptureAppendFunc / CaptureStopFunc / CaptureStatusFunc — +// the four core-side halves of the meeting recorder. +type CaptureStartFunc func(ctx context.Context, req CaptureStartReq) (CaptureStartResp, error) +type CaptureAppendFunc func(ctx context.Context, req CaptureAppendReq) (CaptureAppendResp, error) +type CaptureStopFunc func(ctx context.Context, req CaptureStopReq) (CaptureStopResp, error) +type CaptureStatusFunc func(ctx context.Context) (CaptureStatusResp, error) + +// EnrollSpeakerFunc / ListSpeakersFunc / ForgetSpeakerFunc — the core-side +// halves of voice enrolment. +type EnrollSpeakerFunc func(ctx context.Context, req EnrollSpeakerReq) (EnrollSpeakerResp, error) +type ListSpeakersFunc func(ctx context.Context) (ListSpeakersResp, error) +type ForgetSpeakerFunc func(ctx context.Context, req ForgetSpeakerReq) error // CheckFunc — the auth hook signature. Wired by the daemon (auth.Gate.Check // satisfies this); dispatch calls it once per request after param-unmarshal @@ -501,6 +691,311 @@ func (s *Server) safeDispatch(ctx context.Context, req Request) (result json.Raw return s.dispatch(ctx, req) } +// handlerFunc — one table entry's shape: unmarshal req.Params (if it wants +// any), call the matching CoreAPI method against the api passed in, marshal +// the result. api is a parameter, not a closed-over field, precisely so a +// table built once at package init never pins a stale CoreAPI — see the note +// on methodTable below about SetAPI. +type handlerFunc func(ctx context.Context, api CoreAPI, raw json.RawMessage) (json.RawMessage, error) + +// withParams adapts a (typed params, typed result) CoreAPI call into a +// handlerFunc: unmarshal into P, call fn, marshal R. On error the result is +// dropped (marshalResult's output is never read when err != nil — see +// serveConn) so every entry can uniformly return early on error without +// re-deriving what the pre-table per-arm code used to return in that case. +func withParams[P any, R any](fn func(ctx context.Context, api CoreAPI, p P) (R, error)) handlerFunc { + return func(ctx context.Context, api CoreAPI, raw json.RawMessage) (json.RawMessage, error) { + var p P + if err := unmarshalParams(raw, &p); err != nil { + return nil, err + } + r, err := fn(ctx, api, p) + if err != nil { + return nil, err + } + return marshalResult(r), nil + } +} + +// withParamsVoid is withParams for the error-only methods (mark/resolve/ +// enable/disable/...): params in, no result out, wire reply is always null. +func withParamsVoid[P any](fn func(ctx context.Context, api CoreAPI, p P) error) handlerFunc { + return func(ctx context.Context, api CoreAPI, raw json.RawMessage) (json.RawMessage, error) { + var p P + if err := unmarshalParams(raw, &p); err != nil { + return nil, err + } + return marshalResult(nil), fn(ctx, api, p) + } +} + +// withoutParams is withParams for the handful of methods that take no +// params at all (Presence, TickTrace, MorningStatus, ListProposedRoutines). +// It does NOT call unmarshalParams — matching the pre-table arms, which +// never touched req.Params for these four methods. +func withoutParams[R any](fn func(ctx context.Context, api CoreAPI) (R, error)) handlerFunc { + return func(ctx context.Context, api CoreAPI, _ json.RawMessage) (json.RawMessage, error) { + r, err := fn(ctx, api) + if err != nil { + return nil, err + } + return marshalResult(r), nil + } +} + +// methodTable — one entry per CoreAPI-backed method. Built once at package +// init, not per-Server and not per-dispatch: entries close over nothing but +// the CoreAPI method being called, and dispatch passes in the *current* +// api (loaded fresh via s.api.Load() every call, same as before the table +// existed) as an argument — so SetAPI's runtime swap (the unlock transition) +// is still honored on the very next request with no extra plumbing here. +// +// MethodAssertStepUp, MethodStoreEncryptionKey, MethodUnlock, +// MethodIngestMail, MethodSwapModel, MethodModelStatus, +// MethodDescribeImage and the four MethodCapture* methods are NOT in this +// table: they bypass CoreAPI entirely (s.StepUp / s.WrapKeyFn / s.UnlockFn / +// s.IngestMailFn / s.DescribeImageFn / s.Capture*Fn), so dispatch +// special-cases them before consulting the table. +var methodTable = map[Method]handlerFunc{ + MethodWriteFact: withParams(func(ctx context.Context, api CoreAPI, p WriteFactReq) (idResp, error) { + id, err := api.WriteFact(ctx, p) + return idResp{ID: id}, err + }), + MethodLatestFact: withParams(func(ctx context.Context, api CoreAPI, p keyReq) (Fact, error) { + return api.LatestFact(ctx, p.Key) + }), + MethodLatestFactBySource: withParams(func(ctx context.Context, api CoreAPI, p keySourceReq) (Fact, error) { + return api.LatestFactBySource(ctx, p.Key, p.Source) + }), + MethodSince: withParams(func(ctx context.Context, api CoreAPI, p sinceReq) (sinceResp, error) { + d, err := api.Since(ctx, p.Key, p.Now) + return sinceResp{Dur: d}, err + }), + MethodPresence: withoutParams(func(ctx context.Context, api CoreAPI) (Presence, error) { + return api.Presence(ctx) + }), + MethodCreateReminder: withParams(func(ctx context.Context, api CoreAPI, p createReminderReq) (idResp, error) { + id, err := api.CreateReminder(ctx, p.Fire, p.Payload, p.Cron) + return idResp{ID: id}, err + }), + MethodMarkReminder: withParamsVoid(func(ctx context.Context, api CoreAPI, p markReminderReq) error { + return api.MarkReminder(ctx, p.ID, p.Status) + }), + MethodListReminders: withParams(func(ctx context.Context, api CoreAPI, p nReq) ([]Reminder, error) { + out, err := api.ListReminders(ctx, p.N) + if err != nil { + return nil, err + } + if out == nil { + out = []Reminder{} + } + return out, nil + }), + MethodRecordNudge: withParams(func(ctx context.Context, api CoreAPI, p recordNudgeReq) (idResp, error) { + id, err := api.RecordNudge(ctx, p.Rule, p.Channel, p.Message, p.Ts) + return idResp{ID: id}, err + }), + MethodResolveNudge: withParamsVoid(func(ctx context.Context, api CoreAPI, p resolveNudgeReq) error { + return api.ResolveNudge(ctx, p.ID, p.Outcome, p.Ts) + }), + MethodRecentOutcomes: withParams(func(ctx context.Context, api CoreAPI, p outcomesReq) ([]string, error) { + out, err := api.RecentOutcomes(ctx, p.Rule, p.N) + if err != nil { + return nil, err + } + if out == nil { + out = []string{} // stable non-null on the wire + } + return out, nil + }), + MethodRecentFacts: withParams(func(ctx context.Context, api CoreAPI, p nReq) ([]Fact, error) { + out, err := api.RecentFacts(ctx, p.N) + if err != nil { + return nil, err + } + if out == nil { + out = []Fact{} + } + return out, nil + }), + MethodRecentActiveFacts: withParams(func(ctx context.Context, api CoreAPI, p kindNReq) ([]Fact, error) { + out, err := api.RecentActiveFactsByKind(ctx, p.Kind, p.N) + if err != nil { + return nil, err + } + if out == nil { + out = []Fact{} + } + return out, nil + }), + MethodCalendarEvents: withParams(func(ctx context.Context, api CoreAPI, p calendarEventsReq) ([]Fact, error) { + out, err := api.CalendarEvents(ctx, p.From, p.To) + if err != nil { + return nil, err + } + if out == nil { + out = []Fact{} + } + return out, nil + }), + MethodRecentEcoTraces: withParams(func(ctx context.Context, api CoreAPI, p nReq) ([]EcosystemTrace, error) { + out, err := api.RecentEcosystemTraces(ctx, p.N) + if err != nil { + return nil, err + } + if out == nil { + out = []EcosystemTrace{} + } + return out, nil + }), + MethodRecentNudges: withParams(func(ctx context.Context, api CoreAPI, p nReq) ([]Nudge, error) { + out, err := api.RecentNudges(ctx, p.N) + if err != nil { + return nil, err + } + if out == nil { + out = []Nudge{} + } + return out, nil + }), + MethodWriteNote: withParams(func(ctx context.Context, api CoreAPI, p writeNoteReq) (idResp, error) { + id, err := api.WriteNote(ctx, p.Ts, p.Text, p.Embedding, p.Source) + return idResp{ID: id}, err + }), + MethodQueryNotes: withParams(func(ctx context.Context, api CoreAPI, p queryNotesReq) ([]Note, error) { + out, err := api.QueryNotes(ctx, p.Embedding, p.K) + if err != nil { + return nil, err + } + if out == nil { + out = []Note{} + } + return out, nil + }), + MethodRecentNotesFromSource: withParams(func(ctx context.Context, api CoreAPI, p sourceNReq) ([]Note, error) { + out, err := api.RecentNotesFromSource(ctx, p.Prefix, p.N) + if err != nil { + return nil, err + } + if out == nil { + out = []Note{} + } + return out, nil + }), + MethodRecentNotes: withParams(func(ctx context.Context, api CoreAPI, p nReq) ([]Note, error) { + out, err := api.RecentNotes(ctx, p.N) + if err != nil { + return nil, err + } + if out == nil { + out = []Note{} + } + return out, nil + }), + MethodProposeTool: withParams(func(ctx context.Context, api CoreAPI, p proposeToolReq) (proposeToolResp, error) { + ok, err := api.ProposeTool(ctx, p.Name, p.Utterance, p.Scope, p.Ts) + return proposeToolResp{Proposed: ok}, err + }), + MethodEnableTool: withParamsVoid(func(ctx context.Context, api CoreAPI, p enableToolReq) error { + return api.EnableTool(ctx, p.Name, p.Cmd, p.Destructive, p.Scope, p.Ts) + }), + MethodDisableTool: withParamsVoid(func(ctx context.Context, api CoreAPI, p disableToolReq) error { + return api.DisableTool(ctx, p.Name) + }), + MethodLookupTool: withParams(func(ctx context.Context, api CoreAPI, p lookupToolReq) (Tool, error) { + return api.LookupTool(ctx, p.Name) + }), + MethodListTools: withParams(func(ctx context.Context, api CoreAPI, p listToolsReq) (listToolsResp, error) { + out, err := api.ListTools(ctx, p.Status) + if err != nil { + return listToolsResp{}, err + } + if out == nil { + out = []Tool{} + } + return listToolsResp{Tools: out}, nil + }), + // MethodDeleteTool shares disableToolReq — both take just a tool name. + MethodDeleteTool: withParamsVoid(func(ctx context.Context, api CoreAPI, p disableToolReq) error { + return api.DeleteTool(ctx, p.Name) + }), + MethodCaptureTask: withParams(func(ctx context.Context, api CoreAPI, p CaptureTaskReq) (CaptureTaskResp, error) { + return api.CaptureTask(ctx, p) + }), + MethodListTasks: withParams(func(ctx context.Context, api CoreAPI, p listTasksReq) (listTasksResp, error) { + out, err := api.ListTasks(ctx, p.Status) + if err != nil { + return listTasksResp{}, err + } + if out == nil { + out = []Task{} + } + return listTasksResp{Tasks: out}, nil + }), + MethodSetTaskStatus: withParamsVoid(func(ctx context.Context, api CoreAPI, p setTaskStatusReq) error { + return api.SetTaskStatus(ctx, p.ID, p.Status, p.Ts, p.By) + }), + MethodListProposedRoutines: withoutParams(func(ctx context.Context, api CoreAPI) (listProposedRoutinesResp, error) { + out, err := api.ListProposedRoutines(ctx) + if err != nil { + return listProposedRoutinesResp{}, err + } + if out == nil { + out = []ProposedRoutine{} + } + return listProposedRoutinesResp{Routines: out}, nil + }), + MethodDismissProposedRoutine: withParamsVoid(func(ctx context.Context, api CoreAPI, p dismissProposedRoutineReq) error { + return api.DismissProposedRoutine(ctx, p.ID) + }), + MethodAcceptProposedRoutine: withParamsVoid(func(ctx context.Context, api CoreAPI, p acceptProposedRoutineReq) error { + return api.AcceptProposedRoutine(ctx, p.ID) + }), + MethodRevertFact: withParams(func(ctx context.Context, api CoreAPI, p revertReq) (map[string]int64, error) { + newID, err := api.RevertFact(ctx, p.Key) + if err != nil { + return nil, err + } + return map[string]int64{"new_id": newID}, nil + }), + MethodChat: withParams(func(ctx context.Context, api CoreAPI, p chatReq) (chatResp, error) { + reply, err := api.Chat(ctx, p.Text) + return chatResp{Reply: reply}, err + }), + MethodTickTrace: withoutParams(func(ctx context.Context, api CoreAPI) (TickTrace, error) { + return api.TickTrace(ctx) + }), + // MorningStatus intentionally has no nil→[]T{} normalization here — the + // pre-table arm marshaled api.MorningStatus's result as-is (a nil slice + // serializes as JSON null), and this preserves that exact wire shape. + MethodDayPlan: withoutParams(func(ctx context.Context, api CoreAPI) (DayPlan, error) { + return api.DayPlan(ctx) + }), + MethodMorningStatus: withoutParams(func(ctx context.Context, api CoreAPI) ([]MorningRoutineStatus, error) { + return api.MorningStatus(ctx) + }), + MethodRecentEvents: withParams(func(ctx context.Context, api CoreAPI, p nReq) ([]IntakeEvent, error) { + out, err := api.RecentEvents(ctx, p.N) + if err != nil { + return nil, err + } + if out == nil { + out = []IntakeEvent{} + } + return out, nil + }), + MethodMCPServers: withoutParams(func(ctx context.Context, api CoreAPI) ([]MCPServerStatus, error) { + out, err := api.MCPServers(ctx) + if err != nil { + return nil, err + } + if out == nil { + out = []MCPServerStatus{} + } + return out, nil + }), +} + // dispatch unmarshals params for req.Method and calls the matching CoreAPI // method. Unknown method ⇒ ErrUnknownMethod; a malformed params payload ⇒ // ErrBadParams with the underlying text (local, server-side, not shipped to @@ -517,311 +1012,21 @@ func (s *Server) dispatch(ctx context.Context, req Request) (json.RawMessage, er return nil, err } } + + // These bypass CoreAPI entirely — they drive Server fields set + // directly by the daemon (StepUp / WrapKeyFn / UnlockFn), not store + // state, so they can never be table entries keyed on a CoreAPI method. switch req.Method { - case MethodWriteFact: - var p WriteFactReq - if err := unmarshalParams(req.Params, &p); err != nil { - return nil, err + case MethodPing: + // Deliberately reaches nothing: no store, no CoreAPI, no daemon + // component. That is what makes it answerable in locked mode, and it is + // the whole point — an update that restarts her into locked mode has to + // be able to tell that apart from a daemon that did not come up. + locked := false + if s.LockedFn != nil { + locked = s.LockedFn() } - id, err := api.WriteFact(ctx, p) - return marshalResult(idResp{ID: id}), err - - case MethodLatestFact: - var p keyReq - if err := unmarshalParams(req.Params, &p); err != nil { - return nil, err - } - f, err := api.LatestFact(ctx, p.Key) - if err != nil { - return nil, err - } - return marshalResult(f), nil - - case MethodLatestFactBySource: - var p keySourceReq - if err := unmarshalParams(req.Params, &p); err != nil { - return nil, err - } - f, err := api.LatestFactBySource(ctx, p.Key, p.Source) - if err != nil { - return nil, err - } - return marshalResult(f), nil - - case MethodSince: - var p sinceReq - if err := unmarshalParams(req.Params, &p); err != nil { - return nil, err - } - d, err := api.Since(ctx, p.Key, p.Now) - if err != nil { - return nil, err - } - return marshalResult(sinceResp{Dur: d}), nil - - case MethodPresence: - pres, err := api.Presence(ctx) - if err != nil { - return nil, err - } - return marshalResult(pres), nil - - case MethodCreateReminder: - var p createReminderReq - if err := unmarshalParams(req.Params, &p); err != nil { - return nil, err - } - id, err := api.CreateReminder(ctx, p.Fire, p.Payload, p.Cron) - return marshalResult(idResp{ID: id}), err - - case MethodMarkReminder: - var p markReminderReq - if err := unmarshalParams(req.Params, &p); err != nil { - return nil, err - } - err := api.MarkReminder(ctx, p.ID, p.Status) - return marshalResult(nil), err - - case MethodListReminders: - var p nReq - if err := unmarshalParams(req.Params, &p); err != nil { - return nil, err - } - out, err := api.ListReminders(ctx, p.N) - if err != nil { - return nil, err - } - if out == nil { - out = []Reminder{} - } - return marshalResult(out), nil - - case MethodRecordNudge: - var p recordNudgeReq - if err := unmarshalParams(req.Params, &p); err != nil { - return nil, err - } - id, err := api.RecordNudge(ctx, p.Rule, p.Channel, p.Message, p.Ts) - return marshalResult(idResp{ID: id}), err - - case MethodResolveNudge: - var p resolveNudgeReq - if err := unmarshalParams(req.Params, &p); err != nil { - return nil, err - } - err := api.ResolveNudge(ctx, p.ID, p.Outcome, p.Ts) - return marshalResult(nil), err - - case MethodRecentOutcomes: - var p outcomesReq - if err := unmarshalParams(req.Params, &p); err != nil { - return nil, err - } - out, err := api.RecentOutcomes(ctx, p.Rule, p.N) - if err != nil { - return nil, err - } - if out == nil { - out = []string{} // stable non-null on the wire - } - return marshalResult(out), nil - - case MethodRecentFacts: - var p nReq - if err := unmarshalParams(req.Params, &p); err != nil { - return nil, err - } - out, err := api.RecentFacts(ctx, p.N) - if err != nil { - return nil, err - } - if out == nil { - out = []Fact{} - } - return marshalResult(out), nil - - case MethodCalendarEvents: - var p calendarEventsReq - if err := unmarshalParams(req.Params, &p); err != nil { - return nil, err - } - out, err := api.CalendarEvents(ctx, p.From, p.To) - if err != nil { - return nil, err - } - if out == nil { - out = []Fact{} - } - return marshalResult(out), nil - - case MethodRecentNudges: - var p nReq - if err := unmarshalParams(req.Params, &p); err != nil { - return nil, err - } - out, err := api.RecentNudges(ctx, p.N) - if err != nil { - return nil, err - } - if out == nil { - out = []Nudge{} - } - return marshalResult(out), nil - - case MethodWriteNote: - var p writeNoteReq - if err := unmarshalParams(req.Params, &p); err != nil { - return nil, err - } - id, err := api.WriteNote(ctx, p.Ts, p.Text, p.Embedding, p.Source) - return marshalResult(idResp{ID: id}), err - - case MethodQueryNotes: - var p queryNotesReq - if err := unmarshalParams(req.Params, &p); err != nil { - return nil, err - } - out, err := api.QueryNotes(ctx, p.Embedding, p.K) - if err != nil { - return nil, err - } - if out == nil { - out = []Note{} - } - return marshalResult(out), nil - - case MethodRecentNotes: - var p nReq - if err := unmarshalParams(req.Params, &p); err != nil { - return nil, err - } - out, err := api.RecentNotes(ctx, p.N) - if err != nil { - return nil, err - } - if out == nil { - out = []Note{} - } - return marshalResult(out), nil - - case MethodProposeTool: - var p proposeToolReq - if err := unmarshalParams(req.Params, &p); err != nil { - return nil, err - } - ok, err := api.ProposeTool(ctx, p.Name, p.Utterance, p.Scope, p.Ts) - if err != nil { - return nil, err - } - return marshalResult(proposeToolResp{Proposed: ok}), nil - - case MethodEnableTool: - var p enableToolReq - if err := unmarshalParams(req.Params, &p); err != nil { - return nil, err - } - return marshalResult(nil), api.EnableTool(ctx, p.Name, p.Cmd, p.Destructive, p.Scope, p.Ts) - - case MethodDisableTool: - var p disableToolReq - if err := unmarshalParams(req.Params, &p); err != nil { - return nil, err - } - return marshalResult(nil), api.DisableTool(ctx, p.Name) - - case MethodLookupTool: - var p lookupToolReq - if err := unmarshalParams(req.Params, &p); err != nil { - return nil, err - } - t, err := api.LookupTool(ctx, p.Name) - if err != nil { - return nil, err - } - return marshalResult(t), nil - - case MethodListTools: - var p listToolsReq - if err := unmarshalParams(req.Params, &p); err != nil { - return nil, err - } - out, err := api.ListTools(ctx, p.Status) - if err != nil { - return nil, err - } - if out == nil { - out = []Tool{} - } - return marshalResult(listToolsResp{Tools: out}), nil - - case MethodDeleteTool: - var p disableToolReq - if err := unmarshalParams(req.Params, &p); err != nil { - return nil, err - } - return marshalResult(nil), api.DeleteTool(ctx, p.Name) - - case MethodListProposedRoutines: - out, err := api.ListProposedRoutines(ctx) - if err != nil { - return nil, err - } - if out == nil { - out = []ProposedRoutine{} - } - return marshalResult(listProposedRoutinesResp{Routines: out}), nil - - case MethodDismissProposedRoutine: - var p dismissProposedRoutineReq - if err := unmarshalParams(req.Params, &p); err != nil { - return nil, err - } - return marshalResult(nil), api.DismissProposedRoutine(ctx, p.ID) - - case MethodAcceptProposedRoutine: - var p acceptProposedRoutineReq - if err := unmarshalParams(req.Params, &p); err != nil { - return nil, err - } - return marshalResult(nil), api.AcceptProposedRoutine(ctx, p.ID) - - case MethodRevertFact: - var p struct { - Key string `json:"key"` - } - if err := unmarshalParams(req.Params, &p); err != nil { - return nil, err - } - newID, err := api.RevertFact(ctx, p.Key) - if err != nil { - return nil, err - } - return marshalResult(map[string]int64{"new_id": newID}), nil - - case MethodChat: - var p chatReq - if err := unmarshalParams(req.Params, &p); err != nil { - return nil, err - } - reply, err := api.Chat(ctx, p.Text) - if err != nil { - return nil, err - } - return marshalResult(chatResp{Reply: reply}), nil - - case MethodTickTrace: - t, err := api.TickTrace(ctx) - if err != nil { - return nil, err - } - return marshalResult(t), nil - - case MethodMorningStatus: - s, err := api.MorningStatus(ctx) - if err != nil { - return nil, err - } - return marshalResult(s), nil + return marshalResult(PingResp{Alive: true, Locked: locked}), nil case MethodAssertStepUp: if s.StepUp != nil { @@ -835,7 +1040,7 @@ func (s *Server) dispatch(ctx context.Context, req Request) (json.RawMessage, er if err := unmarshalParams(req.Params, &p); err != nil { return nil, err } - return marshalResult(nil), s.WrapKeyFn(ctx, p.PublicKey) + return marshalResult(nil), s.WrapKeyFn(ctx, p.Secret, p.Explicit) } return nil, fmt.Errorf("%w: %s", ErrUnknownMethod, req.Method) @@ -845,13 +1050,157 @@ func (s *Server) dispatch(ctx context.Context, req Request) (json.RawMessage, er if err := unmarshalParams(req.Params, &p); err != nil { return nil, err } - return marshalResult(nil), s.UnlockFn(ctx, p.PublicKey) + return marshalResult(nil), s.UnlockFn(ctx, p.Secret) } return nil, fmt.Errorf("%w: %s", ErrUnknownMethod, req.Method) - default: + case MethodIngestMail: + if s.IngestMailFn != nil { + var p IngestMailReq + if err := unmarshalParams(req.Params, &p); err != nil { + return nil, err + } + resp, err := s.IngestMailFn(ctx, p) + if err != nil { + return nil, err + } + return marshalResult(resp), nil + } + return nil, fmt.Errorf("%w: %s", ErrUnknownMethod, req.Method) + + case MethodSwapModel: + if s.SwapModelFn != nil { + var p SwapModelReq + if err := unmarshalParams(req.Params, &p); err != nil { + return nil, err + } + resp, err := s.SwapModelFn(ctx, p) + if err != nil { + return nil, err + } + return marshalResult(resp), nil + } + return nil, fmt.Errorf("%w: %s", ErrUnknownMethod, req.Method) + + case MethodDescribeImage: + if s.DescribeImageFn != nil { + var p DescribeImageReq + if err := unmarshalParams(req.Params, &p); err != nil { + return nil, err + } + resp, err := s.DescribeImageFn(ctx, p) + if err != nil { + return nil, err + } + return marshalResult(resp), nil + } + return nil, fmt.Errorf("%w: %s", ErrUnknownMethod, req.Method) + + case MethodCaptureStart: + if s.CaptureStartFn != nil { + var p CaptureStartReq + if err := unmarshalParams(req.Params, &p); err != nil { + return nil, err + } + resp, err := s.CaptureStartFn(ctx, p) + if err != nil { + return nil, err + } + return marshalResult(resp), nil + } + return nil, fmt.Errorf("%w: %s", ErrUnknownMethod, req.Method) + + case MethodCaptureAppend: + if s.CaptureAppendFn != nil { + var p CaptureAppendReq + if err := unmarshalParams(req.Params, &p); err != nil { + return nil, err + } + resp, err := s.CaptureAppendFn(ctx, p) + if err != nil { + return nil, err + } + return marshalResult(resp), nil + } + return nil, fmt.Errorf("%w: %s", ErrUnknownMethod, req.Method) + + case MethodCaptureStop: + if s.CaptureStopFn != nil { + var p CaptureStopReq + if err := unmarshalParams(req.Params, &p); err != nil { + return nil, err + } + resp, err := s.CaptureStopFn(ctx, p) + if err != nil { + return nil, err + } + return marshalResult(resp), nil + } + return nil, fmt.Errorf("%w: %s", ErrUnknownMethod, req.Method) + + case MethodCaptureStatus: + if s.CaptureStatusFn != nil { + resp, err := s.CaptureStatusFn(ctx) + if err != nil { + return nil, err + } + return marshalResult(resp), nil + } + return nil, fmt.Errorf("%w: %s", ErrUnknownMethod, req.Method) + + case MethodEnrollSpeaker: + if s.EnrollSpeakerFn != nil { + var p EnrollSpeakerReq + if err := unmarshalParams(req.Params, &p); err != nil { + return nil, err + } + resp, err := s.EnrollSpeakerFn(ctx, p) + if err != nil { + return nil, err + } + return marshalResult(resp), nil + } + return nil, fmt.Errorf("%w: %s", ErrUnknownMethod, req.Method) + + case MethodListSpeakers: + if s.ListSpeakersFn != nil { + resp, err := s.ListSpeakersFn(ctx) + if err != nil { + return nil, err + } + return marshalResult(resp), nil + } + return nil, fmt.Errorf("%w: %s", ErrUnknownMethod, req.Method) + + case MethodForgetSpeaker: + if s.ForgetSpeakerFn != nil { + var p ForgetSpeakerReq + if err := unmarshalParams(req.Params, &p); err != nil { + return nil, err + } + if err := s.ForgetSpeakerFn(ctx, p); err != nil { + return nil, err + } + return marshalResult(nil), nil + } + return nil, fmt.Errorf("%w: %s", ErrUnknownMethod, req.Method) + + case MethodModelStatus: + if s.ModelStatusFn != nil { + resp, err := s.ModelStatusFn(ctx) + if err != nil { + return nil, err + } + return marshalResult(resp), nil + } return nil, fmt.Errorf("%w: %s", ErrUnknownMethod, req.Method) } + + h, ok := methodTable[req.Method] + if !ok { + return nil, fmt.Errorf("%w: %s", ErrUnknownMethod, req.Method) + } + return h(ctx, api, req.Params) } func unmarshalParams(raw json.RawMessage, v any) error { diff --git a/internal/ipc/speaker_test.go b/internal/ipc/speaker_test.go new file mode 100644 index 0000000..8d2a1bf --- /dev/null +++ b/internal/ipc/speaker_test.go @@ -0,0 +1,123 @@ +package ipc + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/kami/maven/internal/audio" +) + +// The default that matters most for a biometric: on a core that was never +// configured with a speaker block, there is no wire path that takes a +// voiceprint, and none that lists the ones that might exist. +func TestSpeaker_OffUnlessConfigured(t *testing.T) { + _, _, cli, _ := newServerWithStore(t) + ctx := context.Background() + + if _, err := cli.EnrollSpeaker(ctx, EnrollSpeakerReq{ID: "kami"}); !errors.Is(err, ErrUnknownMethod) { + t.Errorf("EnrollSpeaker error = %v, want ErrUnknownMethod", err) + } + if _, err := cli.ListSpeakers(ctx); !errors.Is(err, ErrUnknownMethod) { + t.Errorf("ListSpeakers error = %v, want ErrUnknownMethod", err) + } + if err := cli.ForgetSpeaker(ctx, "kami"); !errors.Is(err, ErrUnknownMethod) { + t.Errorf("ForgetSpeaker error = %v, want ErrUnknownMethod", err) + } +} + +// Enrolment carries several samples across the boundary byte for byte — a +// profile averaged over the wrong bytes is a profile of nobody. +func TestSpeaker_EnrollCrossesTheWire(t *testing.T) { + _, srv, cli, _ := newServerWithStore(t) + ctx := context.Background() + + enrolled := time.Now().UTC().Truncate(time.Second) + var gotID, gotName string + var gotSamples [][]byte + + srv.EnrollSpeakerFn = func(_ context.Context, req EnrollSpeakerReq) (EnrollSpeakerResp, error) { + gotID, gotName = req.ID, req.Name + for _, s := range req.Samples { + gotSamples = append(gotSamples, s.Bytes) + } + return EnrollSpeakerResp{Speaker: Speaker{ + ID: req.ID, Name: req.Name, Enrolled: enrolled, Samples: len(req.Samples), + }}, nil + } + + mk := func(b byte, n int) audio.Audio { + buf := make([]byte, n) + for i := range buf { + buf[i] = b + } + return audio.Audio{Format: audio.PCM16kMono, Bytes: buf} + } + samples := []audio.Audio{mk(1, 64), mk(2, 96), mk(3, 128)} + + resp, err := cli.EnrollSpeaker(ctx, EnrollSpeakerReq{ID: "kami", Name: "Ками", Samples: samples}) + if err != nil { + t.Fatalf("EnrollSpeaker: %v", err) + } + if gotID != "kami" || gotName != "Ками" { + t.Errorf("server saw id=%q name=%q", gotID, gotName) + } + if len(gotSamples) != 3 { + t.Fatalf("server saw %d samples, want 3", len(gotSamples)) + } + for i, want := range samples { + if string(gotSamples[i]) != string(want.Bytes) { + t.Errorf("sample %d altered in transit", i) + } + } + if resp.Speaker.Samples != 3 || !resp.Speaker.Enrolled.Equal(enrolled) { + t.Errorf("profile came back wrong: %+v", resp.Speaker) + } +} + +// A listing says who is enrolled and whether recognition actually works. On +// this box the honest answer is "enrolled, not recognising", and the response +// has to be able to say so — otherwise a surface implies Maven knows who is +// talking when nothing on disk can tell. +func TestSpeaker_ListReportsDisabledRecognition(t *testing.T) { + _, srv, cli, _ := newServerWithStore(t) + ctx := context.Background() + + srv.ListSpeakersFn = func(context.Context) (ListSpeakersResp, error) { + return ListSpeakersResp{ + Speakers: []Speaker{{ID: "kami", Name: "Ками", Samples: 3}}, + Enabled: false, + }, nil + } + + resp, err := cli.ListSpeakers(ctx) + if err != nil { + t.Fatalf("ListSpeakers: %v", err) + } + if len(resp.Speakers) != 1 || resp.Speakers[0].ID != "kami" { + t.Fatalf("speakers = %+v", resp.Speakers) + } + if resp.Enabled { + t.Error("Enabled = true; the seam must be able to report that nothing recognises") + } +} + +// Deletion reaches core with the id intact and reports success. This is the +// request that must always work. +func TestSpeaker_ForgetReachesCore(t *testing.T) { + _, srv, cli, _ := newServerWithStore(t) + ctx := context.Background() + + var forgot string + srv.ForgetSpeakerFn = func(_ context.Context, req ForgetSpeakerReq) error { + forgot = req.ID + return nil + } + if err := cli.ForgetSpeaker(ctx, "гость"); err != nil { + t.Fatalf("ForgetSpeaker: %v", err) + } + if forgot != "гость" { + t.Errorf("core forgot %q, want %q", forgot, "гость") + } +} diff --git a/internal/ipc/unimplemented.go b/internal/ipc/unimplemented.go new file mode 100644 index 0000000..4aea172 --- /dev/null +++ b/internal/ipc/unimplemented.go @@ -0,0 +1,146 @@ +package ipc + +import ( + "context" + "errors" + "time" +) + +// ErrNotImplemented is returned by every UnimplementedCoreAPI method. It is +// deliberately distinct from ErrUnknownMethod (a wire-level "no such +// method exists" verdict) and from any daemon-level "locked" error: this one +// means "this method exists on CoreAPI, but the fake/adapter embedding +// UnimplementedCoreAPI never got a real implementation for it." A test that +// exercises an undeclared method fails loudly on this text instead of +// silently nil-panicking or being mistaken for a legitimate failure. +var ErrNotImplemented = errors.New("ipc: not implemented (unimplemented CoreAPI stub)") + +// UnimplementedCoreAPI is the gRPC Unimplemented*Server pattern applied to +// CoreAPI: embed it in a test double or adapter and override only the +// methods you actually exercise. Every method returns ErrNotImplemented, so +// a call that reaches an undeclared method fails loudly and specifically, +// rather than compiling to a silent no-op or nil-pointer panic. This +// replaces the old pattern of hand-writing all 30 no-op stubs per double — +// those were compiler-satisfying padding, not tests of anything. +type UnimplementedCoreAPI struct{} + +var _ CoreAPI = UnimplementedCoreAPI{} + +func (UnimplementedCoreAPI) WriteFact(ctx context.Context, req WriteFactReq) (int64, error) { + return 0, ErrNotImplemented +} +func (UnimplementedCoreAPI) LatestFact(ctx context.Context, key string) (Fact, error) { + return Fact{}, ErrNotImplemented +} +func (UnimplementedCoreAPI) LatestFactBySource(ctx context.Context, key, source string) (Fact, error) { + return Fact{}, ErrNotImplemented +} +func (UnimplementedCoreAPI) Since(ctx context.Context, key string, now time.Time) (time.Duration, error) { + return 0, ErrNotImplemented +} +func (UnimplementedCoreAPI) Presence(ctx context.Context) (Presence, error) { + return Presence{}, ErrNotImplemented +} +func (UnimplementedCoreAPI) CreateReminder(ctx context.Context, fire time.Time, payload, cron string) (int64, error) { + return 0, ErrNotImplemented +} +func (UnimplementedCoreAPI) MarkReminder(ctx context.Context, id int64, status string) error { + return ErrNotImplemented +} +func (UnimplementedCoreAPI) ListReminders(ctx context.Context, n int) ([]Reminder, error) { + return nil, ErrNotImplemented +} +func (UnimplementedCoreAPI) RecordNudge(ctx context.Context, rule, channel, message string, ts time.Time) (int64, error) { + return 0, ErrNotImplemented +} +func (UnimplementedCoreAPI) ResolveNudge(ctx context.Context, id int64, outcome string, ts time.Time) error { + return ErrNotImplemented +} +func (UnimplementedCoreAPI) RecentOutcomes(ctx context.Context, rule string, n int) ([]string, error) { + return nil, ErrNotImplemented +} +func (UnimplementedCoreAPI) RecentFacts(ctx context.Context, n int) ([]Fact, error) { + return nil, ErrNotImplemented +} +func (UnimplementedCoreAPI) RecentActiveFactsByKind(ctx context.Context, kind string, n int) ([]Fact, error) { + return nil, ErrNotImplemented +} +func (UnimplementedCoreAPI) CalendarEvents(ctx context.Context, from, to time.Time) ([]Fact, error) { + return nil, ErrNotImplemented +} +func (UnimplementedCoreAPI) RecentNudges(ctx context.Context, n int) ([]Nudge, error) { + return nil, ErrNotImplemented +} +func (UnimplementedCoreAPI) RecentEcosystemTraces(ctx context.Context, n int) ([]EcosystemTrace, error) { + return nil, ErrNotImplemented +} +func (UnimplementedCoreAPI) WriteNote(ctx context.Context, ts time.Time, text string, embedding []float32, source string) (int64, error) { + return 0, ErrNotImplemented +} +func (UnimplementedCoreAPI) QueryNotes(ctx context.Context, embedding []float32, k int) ([]Note, error) { + return nil, ErrNotImplemented +} +func (UnimplementedCoreAPI) RecentNotesFromSource(ctx context.Context, prefix string, n int) ([]Note, error) { + return nil, ErrUnknownMethod +} + +func (UnimplementedCoreAPI) RecentNotes(ctx context.Context, n int) ([]Note, error) { + return nil, ErrNotImplemented +} +func (UnimplementedCoreAPI) ProposeTool(ctx context.Context, name, utterance, scope string, ts time.Time) (bool, error) { + return false, ErrNotImplemented +} +func (UnimplementedCoreAPI) EnableTool(ctx context.Context, name string, cmd []string, destructive bool, scope string, ts time.Time) error { + return ErrNotImplemented +} +func (UnimplementedCoreAPI) DisableTool(ctx context.Context, name string) error { + return ErrNotImplemented +} +func (UnimplementedCoreAPI) DeleteTool(ctx context.Context, name string) error { + return ErrNotImplemented +} +func (UnimplementedCoreAPI) CaptureTask(ctx context.Context, req CaptureTaskReq) (CaptureTaskResp, error) { + return CaptureTaskResp{}, ErrNotImplemented +} +func (UnimplementedCoreAPI) ListTasks(ctx context.Context, status string) ([]Task, error) { + return nil, ErrNotImplemented +} +func (UnimplementedCoreAPI) SetTaskStatus(ctx context.Context, id int64, status string, ts time.Time, by string) error { + return ErrNotImplemented +} +func (UnimplementedCoreAPI) ListProposedRoutines(ctx context.Context) ([]ProposedRoutine, error) { + return nil, ErrNotImplemented +} +func (UnimplementedCoreAPI) DismissProposedRoutine(ctx context.Context, id int64) error { + return ErrNotImplemented +} +func (UnimplementedCoreAPI) AcceptProposedRoutine(ctx context.Context, id int64) error { + return ErrNotImplemented +} +func (UnimplementedCoreAPI) LookupTool(ctx context.Context, name string) (Tool, error) { + return Tool{}, ErrNotImplemented +} +func (UnimplementedCoreAPI) ListTools(ctx context.Context, status string) ([]Tool, error) { + return nil, ErrNotImplemented +} +func (UnimplementedCoreAPI) RevertFact(ctx context.Context, key string) (int64, error) { + return 0, ErrNotImplemented +} +func (UnimplementedCoreAPI) TickTrace(ctx context.Context) (TickTrace, error) { + return TickTrace{}, ErrNotImplemented +} +func (UnimplementedCoreAPI) MorningStatus(ctx context.Context) ([]MorningRoutineStatus, error) { + return nil, ErrNotImplemented +} +func (UnimplementedCoreAPI) RecentEvents(ctx context.Context, n int) ([]IntakeEvent, error) { + return nil, ErrNotImplemented +} +func (UnimplementedCoreAPI) MCPServers(ctx context.Context) ([]MCPServerStatus, error) { + return nil, ErrNotImplemented +} +func (UnimplementedCoreAPI) DayPlan(ctx context.Context) (DayPlan, error) { + return DayPlan{}, ErrNotImplemented +} +func (UnimplementedCoreAPI) Chat(ctx context.Context, text string) (string, error) { + return "", ErrNotImplemented +} diff --git a/internal/ipc/unlock_test.go b/internal/ipc/unlock_test.go new file mode 100644 index 0000000..212f0f2 --- /dev/null +++ b/internal/ipc/unlock_test.go @@ -0,0 +1,177 @@ +package ipc + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "testing" +) + +// The wire must carry the PRF secret, not the credential public key. This is +// the field rename that fixes Vikunja #14: a v1 deployment sent "public_key", +// and the value it sent was in passkeys.json next to the wrapped blob. +func TestUnlockWireCarriesSecret(t *testing.T) { + secret := bytes.Repeat([]byte{7}, 32) + for _, p := range []any{unlockReq{Secret: secret}, storeEncryptionKeyReq{Secret: secret}} { + b, err := json.Marshal(p) + if err != nil { + t.Fatalf("marshal %T: %v", p, err) + } + var m map[string]any + if err := json.Unmarshal(b, &m); err != nil { + t.Fatalf("unmarshal %T: %v", p, err) + } + if _, ok := m["secret"]; !ok { + t.Errorf("%T has no \"secret\" field: %s", p, b) + } + if _, ok := m["public_key"]; ok { + t.Errorf("%T still sends \"public_key\": %s", p, b) + } + } +} + +// The secret must reach the daemon hook byte-for-byte through the socket. +func TestUnlockDeliversSecretToHook(t *testing.T) { + _, srv, cli, _ := newServerWithStore(t) + + secret := make([]byte, 32) + for i := range secret { + secret[i] = byte(i + 1) + } + var gotUnlock, gotWrap []byte + var gotExplicit bool + srv.UnlockFn = func(_ context.Context, s []byte) error { gotUnlock = bytes.Clone(s); return nil } + srv.WrapKeyFn = func(_ context.Context, s []byte, explicit bool) error { + gotWrap = bytes.Clone(s) + gotExplicit = explicit + return nil + } + + ctx := context.Background() + if err := cli.Unlock(ctx, secret); err != nil { + t.Fatalf("Unlock: %v", err) + } + if !bytes.Equal(gotUnlock, secret) { + t.Errorf("UnlockFn got %x, want %x", gotUnlock, secret) + } + if err := cli.StoreEncryptionKey(ctx, secret, true); err != nil { + t.Fatalf("StoreEncryptionKey: %v", err) + } + if !bytes.Equal(gotWrap, secret) { + t.Errorf("WrapKeyFn got %x, want %x", gotWrap, secret) + } + // The explicit flag rides the same request. Without it the daemon cannot + // tell "he asked for the cold-start key to be rewritten" from "a passkey + // was asserted", and rewrites the blob on every step-up. + if !gotExplicit { + t.Error("WrapKeyFn got explicit=false, want the flag to cross the wire") + } + if err := cli.StoreEncryptionKey(ctx, secret, false); err != nil { + t.Fatalf("StoreEncryptionKey: %v", err) + } + if gotExplicit { + t.Error("WrapKeyFn got explicit=true for an implicit wrap") + } +} + +// A refusal from the daemon hook — a wrong passkey, or no prior assertion — +// must surface to the caller as an error, never be swallowed into success. +func TestUnlockPropagatesRefusal(t *testing.T) { + _, srv, cli, _ := newServerWithStore(t) + srv.UnlockFn = func(context.Context, []byte) error { + return errors.New("unlock: no verified passkey assertion (assert first)") + } + if err := cli.Unlock(context.Background(), bytes.Repeat([]byte{9}, 32)); err == nil { + t.Fatal("a refused unlock reported success") + } +} + +// Without the hooks wired — the normal, unencrypted deployment — both methods +// answer ErrUnknownMethod rather than pretending to have done something. +func TestUnlockUnwiredIsUnknownMethod(t *testing.T) { + _, _, cli, _ := newServerWithStore(t) + ctx := context.Background() + if err := cli.Unlock(ctx, bytes.Repeat([]byte{1}, 32)); err == nil { + t.Error("Unlock succeeded with no UnlockFn wired") + } + if err := cli.StoreEncryptionKey(ctx, bytes.Repeat([]byte{1}, 32), false); err == nil { + t.Error("StoreEncryptionKey succeeded with no WrapKeyFn wired") + } +} + +// Locked mode: Server.Check is the whole authorization surface, and it must +// default-deny everything except the two methods the unlock flow needs. +func TestLockedCheckDefaultDenies(t *testing.T) { + _, srv, cli, _ := newServerWithStore(t) + + locked := errors.New("locked") + srv.Check = func(_ context.Context, m Method, _ json.RawMessage) error { + switch m { + case MethodAssertStepUp, MethodUnlock: + return nil + default: + return locked + } + } + unlocked := false + srv.UnlockFn = func(context.Context, []byte) error { unlocked = true; return nil } + srv.StepUp = func(context.Context) error { return nil } + srv.WrapKeyFn = func(context.Context, []byte, bool) error { return nil } + + ctx := context.Background() + // A store method must be refused while locked. + if _, err := cli.RecentNotes(ctx, 5); err == nil { + t.Error("a store read went through while locked") + } + // Key wrapping is NOT on the allowlist: a locked daemon has no key to wrap. + if err := cli.StoreEncryptionKey(ctx, bytes.Repeat([]byte{2}, 32), false); err == nil { + t.Error("StoreEncryptionKey was allowed while locked") + } + // The unlock flow itself must still work. + if err := cli.AssertStepUp(ctx); err != nil { + t.Errorf("AssertStepUp refused while locked: %v", err) + } + if err := cli.Unlock(ctx, bytes.Repeat([]byte{3}, 32)); err != nil { + t.Errorf("Unlock refused while locked: %v", err) + } + if !unlocked { + t.Error("UnlockFn never ran") + } +} + +// A locked daemon has to be able to say it is alive. Every CoreAPI method is +// refused before unlock, so a health check built on one of those cannot tell a +// daemon waiting for a passkey apart from a daemon that failed to start. That +// is what turned a good update into the manual-recovery case in +// internal/update. MethodPing reaches no store, so it answers either way. +func TestPingAnswersWhileLocked(t *testing.T) { + _, srv, cli, _ := newServerWithStore(t) + srv.LockedFn = func() bool { return true } + locked := errors.New("daemon locked") + srv.Check = func(_ context.Context, m Method, _ json.RawMessage) error { + switch m { + case MethodAssertStepUp, MethodUnlock, MethodPing: + return nil + default: + return locked + } + } + ctx := context.Background() + p, err := cli.Ping(ctx) + if err != nil { + t.Fatalf("Ping while locked: %v", err) + } + if !p.Alive || !p.Locked { + t.Errorf("Ping = %+v; want alive and locked", p) + } + // And the read it replaces is still refused, which is the whole point. + if _, err := cli.Presence(ctx); err == nil { + t.Error("Presence answered while locked") + } + + srv.LockedFn = func() bool { return false } + if p, err := cli.Ping(ctx); err != nil || p.Locked { + t.Errorf("Ping after unlock = %+v, %v; want alive and not locked", p, err) + } +} diff --git a/internal/ipc/wire.go b/internal/ipc/wire.go index c67cec1..c55a7ec 100644 --- a/internal/ipc/wire.go +++ b/internal/ipc/wire.go @@ -25,11 +25,14 @@ const ( MethodResolveNudge Method = "resolve_nudge" MethodRecentOutcomes Method = "recent_outcomes" MethodRecentFacts Method = "recent_facts" + MethodRecentActiveFacts Method = "recent_active_facts_by_kind" MethodCalendarEvents Method = "calendar_events" MethodRecentNudges Method = "recent_nudges" + MethodRecentEcoTraces Method = "recent_ecosystem_traces" MethodWriteNote Method = "write_note" MethodQueryNotes Method = "query_notes" MethodRecentNotes Method = "recent_notes" + MethodRecentNotesFromSource Method = "recent_notes_source" MethodProposeTool Method = "propose_tool" MethodEnableTool Method = "enable_tool" MethodDisableTool Method = "disable_tool" @@ -45,7 +48,32 @@ const ( MethodRevertFact Method = "revert_fact" MethodTickTrace Method = "tick_trace" MethodMorningStatus Method = "morning_status" + MethodMCPServers Method = "mcp_servers" + MethodDayPlan Method = "day_plan" MethodChat Method = "chat" + MethodCaptureTask Method = "capture_task" + MethodListTasks Method = "list_tasks" + MethodSetTaskStatus Method = "set_task_status" + MethodIngestMail Method = "ingest_mail" + MethodSwapModel Method = "swap_model" + MethodModelStatus Method = "model_status" + MethodDescribeImage Method = "describe_image" + MethodCaptureStart Method = "capture_start" + MethodCaptureAppend Method = "capture_append" + MethodCaptureStop Method = "capture_stop" + MethodCaptureStatus Method = "capture_status" + MethodEnrollSpeaker Method = "enroll_speaker" + MethodListSpeakers Method = "list_speakers" + MethodForgetSpeaker Method = "forget_speaker" + MethodRecentEvents Method = "recent_events" + + // MethodPing — liveness, and the only method that answers in locked mode + // without a passkey assertion. It reaches no store, takes no arguments and + // returns whether the daemon is locked, so an operator tool can tell "she is + // up and waiting for a passkey" apart from "she is not there at all". + // Everything else about her state needs the store, and the store needs the + // key. + MethodPing Method = "ping" ) // Request — one frame from module to core. Params is the JSON-encoded argument diff --git a/internal/llm/client.go b/internal/llm/client.go index 7760296..e8293cc 100644 --- a/internal/llm/client.go +++ b/internal/llm/client.go @@ -10,18 +10,115 @@ import ( "encoding/json" "fmt" "net/http" + "sync" "time" ) +// SwapGate — admission control for a completion. Enter blocks or refuses while the +// resident model is being swapped, and the returned release says the request is +// done. The phraser implements it: a swap kills the running llama-server, so +// every holder of a base URL has to be counted before the kill, not just the +// phrasing paths. +// +// Without this the drain saw only the phraser's own calls. The LLM router, the +// replier, the mail extractor and the memory evaluator all reach llama-server +// through this client, so a swap could report zero requests in flight and kill +// the server out from under a routing decision. The turn then finished on the +// new model, which is the "half of one model and half of another" the swap is +// supposed to make impossible. +// +// Distinct from Gate, which is about priority between a voice turn and a +// background job. This one is about the model underneath them changing. A +// request passes the priority gate first and this one second, so nothing waits +// for a quiet window while counted as in flight against the drain. +type SwapGate interface { + Enter() (release func(), err error) +} + type Client struct { + // mu guards base and gate. The base URL can change when the daemon swaps the + // resident model (Vikunja #250) and every holder of this client — the LLM + // router, the replier, the mail extractor — must follow without being + // rebuilt. A swap re-points the client, it does not replace it. + // + // On the deploy shape the new server binds the same fixed port the killed + // one released (startLlamaProc passes the port out of phraser.listen), so + // SetBaseURL is normally a no-op and the gate is the part doing the work. + // The re-pointing stays because nothing guarantees the port: a phraser + // listening on :0, or a future swap that moves the server, changes the base. + mu sync.RWMutex base string + swap SwapGate http *http.Client + + // gate / background — priority on the single llama-server slot. Set once + // at wiring time (SetGate), read on every request. nil gate ⇒ no gating, + // which is what every test and every non-daemon caller gets. + gate *Gate + background bool +} + +// SetGate gives this client a priority on the shared llama-server slot. Call it +// immediately after New, before the client is handed to anything: the fields are +// read under the same lock as base, but the intent is one-time wiring, not a +// knob to turn at runtime. +// +// background = false means "he is waiting for this" and never blocks. +// background = true means the request yields to voice turns and runs one at a +// time. See Gate. +func (c *Client) SetGate(g *Gate, background bool) { + c.mu.Lock() + c.gate, c.background = g, background + c.mu.Unlock() +} + +func (c *Client) gateFor() (*Gate, bool) { + c.mu.RLock() + defer c.mu.RUnlock() + return c.gate, c.background +} + +// SetSwapGate installs the swap admission gate. Nil (the default, and what the +// eval harness and the tests use) means no gating. +func (c *Client) SetSwapGate(g SwapGate) { + c.mu.Lock() + c.swap = g + c.mu.Unlock() +} + +func (c *Client) enter() (func(), error) { + c.mu.RLock() + g := c.swap + c.mu.RUnlock() + if g == nil { + return func() {}, nil + } + return g.Enter() } func New(baseURL string, timeout time.Duration) *Client { return &Client{base: baseURL, http: &http.Client{Timeout: timeout}} } +// SetBaseURL re-points the client at another llama-server. Safe to call while +// requests are in flight: a request that already read the old base finishes +// against the old base (or fails, and every caller of Complete has a fallback), +// and the next one uses the new base. It is deliberately NOT a queue-and-retry — +// the phraser quiesces around a swap, so the window is small and a lost turn +// degrades to the classifier rather than hanging. +func (c *Client) SetBaseURL(base string) { + c.mu.Lock() + c.base = base + c.mu.Unlock() +} + +// BaseURL is the server this client currently talks to. +func (c *Client) BaseURL() string { + c.mu.RLock() + defer c.mu.RUnlock() + return c.base +} + type Req struct { System string User string @@ -55,6 +152,26 @@ type resp struct { } func (c *Client) Complete(ctx context.Context, r Req) (string, error) { + // Priority first: a background request can sit here for a while, and it + // must not be counted against the swap drain while it waits. + if g, background := c.gateFor(); g != nil { + if background { + release, err := g.AcquireBackground(ctx) + if err != nil { + return "", err + } + defer release() + } else { + defer g.Foreground()() + } + } + // Then the swap drain, which counts what is actually about to hit the + // server it is going to kill. + release, err := c.enter() + if err != nil { + return "", err + } + defer release() b, _ := json.Marshal(body{ Messages: []msg{{Role: "system", Content: r.System}, {Role: "user", Content: r.User}}, MaxTokens: r.MaxTokens, @@ -63,7 +180,7 @@ func (c *Client) Complete(ctx context.Context, r Req) (string, error) { RepeatPenalty: r.RepeatPenalty, Stop: r.Stop, }) - req, err := http.NewRequestWithContext(ctx, "POST", c.base+"/v1/chat/completions", bytes.NewReader(b)) + req, err := http.NewRequestWithContext(ctx, "POST", c.BaseURL()+"/v1/chat/completions", bytes.NewReader(b)) if err != nil { return "", err } diff --git a/internal/llm/client_test.go b/internal/llm/client_test.go index 68f174d..9526c72 100644 --- a/internal/llm/client_test.go +++ b/internal/llm/client_test.go @@ -57,3 +57,37 @@ func TestComplete(t *testing.T) { t.Errorf("got %q, want %q", got, "ok") } } + +// TestSetBaseURL — a model swap re-points every holder of the client rather than +// rebuilding the router, the replier and the extractors (Vikunja #250). +func TestSetBaseURL(t *testing.T) { + var hit string + srvA := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hit = "A" + w.Write([]byte(`{"choices":[{"message":{"content":"a"}}]}`)) + })) + defer srvA.Close() + srvB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hit = "B" + w.Write([]byte(`{"choices":[{"message":{"content":"b"}}]}`)) + })) + defer srvB.Close() + + c := New(srvA.URL, 5*time.Second) + if _, err := c.Complete(context.Background(), Req{User: "x"}); err != nil { + t.Fatalf("Complete against A: %v", err) + } + if hit != "A" { + t.Fatalf("first request went to %q; want A", hit) + } + c.SetBaseURL(srvB.URL) + if got := c.BaseURL(); got != srvB.URL { + t.Errorf("BaseURL = %q; want %q", got, srvB.URL) + } + if _, err := c.Complete(context.Background(), Req{User: "x"}); err != nil { + t.Fatalf("Complete against B: %v", err) + } + if hit != "B" { + t.Errorf("request after the swap went to %q; want B", hit) + } +} diff --git a/internal/llm/gate.go b/internal/llm/gate.go new file mode 100644 index 0000000..58a3d75 --- /dev/null +++ b/internal/llm/gate.go @@ -0,0 +1,121 @@ +package llm + +import ( + "context" + "sync" + "time" +) + +// Gate — priority access to the one llama-server slot. +// +// llama-server is started without -np, so it serves one request at a time and +// everything else queues. That is fine while every caller is a voice turn, and +// it stops being fine the moment a background job joins: mail extraction reads +// up to 4000 characters on a Thinking 1.7B with a two minute budget, and a turn +// that arrives during one waits for however much of that budget is left. The +// router degrades to the classifier cascade on error, so he would get the 36.8% +// floor while his mail is being read, and the phraser has no floor at all and +// simply waits. +// +// So background work asks the gate first: +// +// - at most ONE background request is in flight, whatever the queue depth +// upstream. A first poll of a mailbox with 40 unseen messages cannot +// serialise 40 extractions ahead of anything. +// - a background request waits while any foreground request is in flight, and +// for Quiet after the last one finished. The quiet window is what stops an +// extraction starting in the gap between the router call and the phraser +// call of the same turn. +// +// Foreground requests never wait. This is not a fair queue and must not become +// one: the point is that the thing he is waiting for wins every time. +// +// It bounds only what goes through an *llm.Client built with SetGate. The +// phraser's own HTTP path is not gated, and a turn that reaches the phraser +// without touching the router is not marked. Every real turn routes first, so +// the marking is good enough to keep extraction out of the way; it is a +// courtesy gate, not a scheduler. +type Gate struct { + mu sync.Mutex + // fg — foreground requests in flight. + fg int + // last — when a foreground request last started or finished. + last time.Time + // bg — one token, so only one background request runs at a time. + bg chan struct{} + + quiet time.Duration + poll time.Duration + now func() time.Time +} + +// NewGate returns a gate that holds background work back for quiet after the +// last foreground request. quiet <= 0 means "wait only while one is in flight". +func NewGate(quiet time.Duration) *Gate { + return &Gate{ + bg: make(chan struct{}, 1), + quiet: quiet, + poll: 50 * time.Millisecond, + now: time.Now, + } +} + +// Foreground marks a request as the thing he is waiting for. It never blocks. +// The returned function must be called when the request finishes. +func (g *Gate) Foreground() func() { + if g == nil { + return func() {} + } + g.mu.Lock() + g.fg++ + g.last = g.now() + g.mu.Unlock() + return func() { + g.mu.Lock() + g.fg-- + g.last = g.now() + g.mu.Unlock() + } +} + +// AcquireBackground blocks until the slot is free enough for background work, +// or ctx is done. The returned release function must be called when the request +// finishes; it is nil on error. +func (g *Gate) AcquireBackground(ctx context.Context) (func(), error) { + if g == nil { + return func() {}, nil + } + select { + case g.bg <- struct{}{}: + case <-ctx.Done(): + return nil, ctx.Err() + } + release := func() { <-g.bg } + for { + if g.clear() { + return release, nil + } + t := time.NewTimer(g.poll) + select { + case <-t.C: + case <-ctx.Done(): + t.Stop() + release() + return nil, ctx.Err() + } + } +} + +// clear reports whether no foreground request is in flight and the quiet window +// since the last one has passed. +func (g *Gate) clear() bool { + g.mu.Lock() + defer g.mu.Unlock() + if g.fg > 0 { + return false + } + if g.quiet <= 0 || g.last.IsZero() { + return true + } + return g.now().Sub(g.last) >= g.quiet +} diff --git a/internal/llm/gate_test.go b/internal/llm/gate_test.go new file mode 100644 index 0000000..7275108 --- /dev/null +++ b/internal/llm/gate_test.go @@ -0,0 +1,101 @@ +package llm + +import ( + "context" + "testing" + "time" +) + +// Background work must not start while he is waiting on a turn. llama-server +// serves one request at a time, so an extraction that starts first holds the +// slot for its whole budget. +func TestGateBackgroundWaitsForForeground(t *testing.T) { + g := NewGate(0) + g.poll = time.Millisecond + done := g.Foreground() + + started := make(chan struct{}) + go func() { + release, err := g.AcquireBackground(context.Background()) + if err != nil { + t.Errorf("acquire: %v", err) + return + } + close(started) + release() + }() + + select { + case <-started: + t.Fatal("background work started while a foreground request was in flight") + case <-time.After(20 * time.Millisecond): + } + done() + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("background work never started after the foreground request finished") + } +} + +// Only one background request at a time, whatever the queue depth upstream. A +// first poll of a mailbox with 40 unseen messages must not put 40 extractions +// on the slot. +func TestGateOneBackgroundAtATime(t *testing.T) { + g := NewGate(0) + g.poll = time.Millisecond + first, err := g.AcquireBackground(context.Background()) + if err != nil { + t.Fatalf("first: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + if _, err := g.AcquireBackground(ctx); err == nil { + t.Fatal("a second background request ran alongside the first") + } + first() + second, err := g.AcquireBackground(context.Background()) + if err != nil { + t.Fatalf("second after release: %v", err) + } + second() +} + +// The quiet window covers the gap between the router call and the phraser call +// of one turn, so an extraction cannot slip in mid-turn. +func TestGateQuietWindow(t *testing.T) { + now := time.Now() + g := NewGate(time.Minute) + g.poll = time.Millisecond + g.now = func() time.Time { return now } + g.Foreground()() + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + if _, err := g.AcquireBackground(ctx); err == nil { + t.Fatal("background work started inside the quiet window") + } + now = now.Add(2 * time.Minute) + release, err := g.AcquireBackground(context.Background()) + if err != nil { + t.Fatalf("acquire after the quiet window: %v", err) + } + release() +} + +// Foreground never waits, whatever else is in flight. +func TestGateForegroundNeverBlocks(t *testing.T) { + g := NewGate(time.Minute) + release, err := g.AcquireBackground(context.Background()) + if err != nil { + t.Fatalf("acquire: %v", err) + } + defer release() + done := make(chan struct{}) + go func() { g.Foreground()(); close(done) }() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("a foreground request waited behind background work") + } +} diff --git a/internal/loop/gate_test.go b/internal/loop/gate_test.go index ed0cb1c..8c66f45 100644 --- a/internal/loop/gate_test.go +++ b/internal/loop/gate_test.go @@ -286,6 +286,54 @@ func TestRemindersStillHonourSnooze(t *testing.T) { } } +// ---------------------------- digest eligibility ------------------------------ + +// A Sev2 care candidate (break) suppressed for a genuine restraint reason is +// worth resurfacing later. +func TestDigestEligibleSev2SuppressedByRestraint(t *testing.T) { + for _, reason := range []string{"quiet_hours", "calendar_busy", "presence"} { + if !DigestEligible(Sev2, reason) { + t.Errorf("sev2 blocked by %q: want digest-eligible", reason) + } + } +} + +// A Sev1 care candidate (water/meal) never digests — a biological timer +// nudge is stale by the time anyone could resurface it, so it just drops. +func TestDigestEligibleSev1NeverDigests(t *testing.T) { + for _, reason := range []string{"quiet_hours", "calendar_busy", "presence"} { + if DigestEligible(Sev1, reason) { + t.Errorf("sev1 blocked by %q: want drop, got digest-eligible", reason) + } + } +} + +// Ops severities are never blocked by these reasons in practice (Gate only +// applies quiet_hours/calendar_busy/presence to care severities), but the +// boundary itself must refuse to digest a high severity even if asked — +// alarms bypass the gate and deliver now, unchanged, never delayed. +func TestDigestEligibleNeverDigestsHighSeverity(t *testing.T) { + for _, sev := range []Severity{Sev3, Sev4} { + for _, reason := range []string{"quiet_hours", "calendar_busy", "presence"} { + if DigestEligible(sev, reason) { + t.Errorf("sev%d blocked by %q: high severity must never digest", sev, reason) + } + } + } +} + +// cooldown and snooze are not "suppression" in the digest sense — cooldown +// means it was already said recently, snooze means the user asked to not +// hear about it. Neither should resurface later just because the severity +// matches. +func TestDigestEligibleExcludesCooldownAndSnooze(t *testing.T) { + for _, reason := range []string{"cooldown", "snooze", "inert_no_data", "predicate", ""} { + if DigestEligible(Sev2, reason) { + t.Errorf("sev2 blocked by %q: should not be digest-eligible", reason) + } + } +} + // GAP — the gate reads State.SnoozeUntil, but the Gatherer hard-codes it to nil // (internal/loop/gather.go:153), so snooze is dead in the running daemon: the // unit tests above pass while nothing can ever populate the map. This asserts diff --git a/internal/loop/loop.go b/internal/loop/loop.go index ae572d9..2070908 100644 --- a/internal/loop/loop.go +++ b/internal/loop/loop.go @@ -105,6 +105,36 @@ func Tick(s State, rules []Rule) *Candidate { return fire } +// DigestEligible decides digest-vs-drop for a care candidate the gate +// suppressed this tick (see ExplainGate's blockedBy). Pure — no I/O, no +// state, just the two facts that matter: why it was suppressed, and how +// insistent it was. +// +// Only genuine RESTRAINT blocks are eligible at all — quiet_hours, +// calendar_busy, presence(away). cooldown and snooze are not suppression in +// this sense: cooldown means "you already heard this recently" (resurfacing +// it later would be an actual repeat, not a rescue) and snooze is the user +// explicitly saying "not this" (digesting it anyway would defeat the ask). +// Ops severities (Sev3/4) never reach here — the gate never blocks them for +// these reasons in the first place (see Gate), and even if a future rule +// dropped Sev3+ into "care", digest still refuses them: alarms bypass the +// gate on purpose and must never be silently delayed into a bundle. +// +// Within care (Sev1–2), the boundary is severity itself: Sev1 (water, meal — +// biological timers with no "still relevant later" property; a water nudge +// from 3 hours into quiet hours is just wrong by morning) drops. Sev2 +// (break — "you worked through a long stretch without a break while I +// couldn't reach you") is information that stays true and useful after the +// fact, so it digests. +func DigestEligible(sev Severity, blockedBy string) bool { + switch blockedBy { + case "quiet_hours", "calendar_busy", "presence": + default: + return false + } + return sev == Sev2 +} + // ReminderDecision — a due reminder the daemon should deliver now. // NOT gated by the universal Gate (per spec: "wake me 7" fires in quiet hours; // that's the point). Snooze is the one part of restraint that still applies. diff --git a/internal/mcp/allowlist.go b/internal/mcp/allowlist.go new file mode 100644 index 0000000..235b0c5 --- /dev/null +++ b/internal/mcp/allowlist.go @@ -0,0 +1,98 @@ +package mcp + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "regexp" + "strconv" + "strings" +) + +// CmdPrefix is the reserved first argv element that marks an allowlist row as +// an MCP call rather than a process. An MCP tool row looks like +// +// name: "vikunja_list_tasks" cmd: ["mcp", "vikunja", "list_tasks"] +// +// which is why there is no new column and no migration: the store, the /tools +// page, ProposeTool, EnableTool, DisableTool, the act matcher and the confirm +// turn all keep working unchanged. The executor is the only place that has to +// know the difference, and it is one branch on Cmd[0]. +// +// The rest of the allowlist discipline is inherited whole: a row that is not +// status='enabled' does not run, and a row marked destructive does not run on +// first hearing. Nothing here can enable itself — discovery only proposes. +const CmdPrefix = "mcp" + +// Cmd builds the argv encoding for a discovered tool. +func Cmd(server, tool string) []string { return []string{CmdPrefix, server, tool} } + +// ParseCmd recognises an MCP allowlist row. ok=false for an ordinary process +// tool, which is what almost every row is. +func ParseCmd(cmd []string) (server, tool string, ok bool) { + if len(cmd) != 3 || cmd[0] != CmdPrefix { + return "", "", false + } + if cmd[1] == "" || cmd[2] == "" { + return "", "", false + } + return cmd[1], cmd[2], true +} + +var notName = regexp.MustCompile(`[^a-z0-9_]+`) + +// LocalName is the allowlist name for a discovered tool: the server handle, an +// underscore, the remote name, lowercased and stripped of anything that is not +// a word character. Namespacing by server is what keeps two servers that both +// offer "search" from colliding, and what makes the provenance of a row on the +// /tools page obvious without opening the diff. +func LocalName(server, tool string) string { + clean := func(s string) string { + return strings.Trim(notName.ReplaceAllString(strings.ToLower(strings.TrimSpace(s)), "_"), "_") + } + s, t := clean(server), clean(tool) + switch { + case s == "": + return t + case t == "": + return s + } + return s + "_" + t +} + +// Scope is the store scope for a server's rows, so the /tools page can group +// them and a human can tell at a glance where a capability came from. +func Scope(server string) string { return "mcp:" + server } + +// Fingerprint is the declared shape of a discovered tool: its name, its +// description, its input schema and its readOnlyHint, hashed. +// +// It exists because an allowlist row cannot pin an MCP tool's behaviour. The +// row's cmd is ["mcp", server, tool], a reference to a name the REMOTE server +// owns and may redefine — the row does not have to change for the tool to +// become something else. The fingerprint is what Kami actually approved, so a +// later discovery can tell "same tool" from "same name". +// +// The schema is canonicalised through a decode and re-encode, so a server that +// reorders its JSON keys or changes its whitespace does not read as a +// redefinition. Unparseable schema bytes are hashed as they arrived. +func Fingerprint(t Tool) string { + schema := "" + if len(t.InputSchema) > 0 { + var any any + if json.Unmarshal(t.InputSchema, &any) == nil { + if raw, err := json.Marshal(any); err == nil { + schema = string(raw) + } + } + if schema == "" { + schema = string(t.InputSchema) + } + } + h := sha256.New() + for _, part := range []string{t.Name, t.Description, schema, strconv.FormatBool(t.ReadOnly)} { + h.Write([]byte(part)) + h.Write([]byte{0}) + } + return hex.EncodeToString(h.Sum(nil)) +} diff --git a/internal/mcp/allowlist_test.go b/internal/mcp/allowlist_test.go new file mode 100644 index 0000000..fd74ae2 --- /dev/null +++ b/internal/mcp/allowlist_test.go @@ -0,0 +1,32 @@ +package mcp + +import ( + "encoding/json" + "testing" +) + +// The fingerprint must cover everything the approval was given for, and must +// not move when only the JSON spelling of the schema does. +func TestFingerprintCoversTheDeclaredShape(t *testing.T) { + base := Tool{Name: "list_tasks", Description: "list them", ReadOnly: true, + InputSchema: json.RawMessage(`{"type":"object","properties":{}}`)} + same := base + same.InputSchema = json.RawMessage("{\n \"properties\": {},\n \"type\": \"object\"\n}") + if Fingerprint(base) != Fingerprint(same) { + t.Error("reformatting the schema must not read as a redefinition") + } + for name, mut := range map[string]func(*Tool){ + "description": func(x *Tool) { x.Description = "delete them" }, + "schema": func(x *Tool) { x.InputSchema = json.RawMessage(`{"required":["id"]}`) }, + "readonly": func(x *Tool) { x.ReadOnly = false }, + "name": func(x *Tool) { x.Name = "delete_tasks" }, + } { + t.Run(name, func(t *testing.T) { + got := base + mut(&got) + if Fingerprint(got) == Fingerprint(base) { + t.Error("a redefinition must change the fingerprint") + } + }) + } +} diff --git a/internal/mcp/client.go b/internal/mcp/client.go new file mode 100644 index 0000000..826c3f1 --- /dev/null +++ b/internal/mcp/client.go @@ -0,0 +1,267 @@ +package mcp + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + "sync/atomic" +) + +// Errors callers distinguish. +var ( + // ErrClosed — the transport is gone (subprocess died, client closed). + ErrClosed = errors.New("mcp: connection is closed") + // ErrNotInitialized — a call was made before the initialize handshake. + ErrNotInitialized = errors.New("mcp: not initialized") + // ErrToolFailed — the server ran the tool and reported an error result. + ErrToolFailed = errors.New("mcp: tool reported an error") +) + +// Tool is one tool a server offers, in the form Maven cares about. +// +// ReadOnly comes from the server's own readOnlyHint annotation and decides +// whether the allowlist row is marked destructive: no hint, or a false one, +// means "assume it mutates", which routes the call through the confirm turn. +// Guessing wrong in that direction only costs a question. +type Tool struct { + Server string + Name string + Description string + InputSchema json.RawMessage + ReadOnly bool +} + +// Resource is one resource a server offers. Contents are fetched separately — +// listing is cheap, reading is not. +type Resource struct { + Server string + URI string + Name string + MIMEType string +} + +// ServerInfo is what came back from the handshake. +type ServerInfo struct { + Name string `json:"name"` + Version string `json:"version"` + ProtocolVersion string `json:"-"` +} + +// Client is one connected MCP server. Safe for concurrent use. +type Client struct { + name string + tr transport + next atomic.Int64 + + mu sync.Mutex + info ServerInfo + ready bool +} + +// newClient wraps a transport. Callers use Dial* in manager.go. +func newClient(name string, tr transport) *Client { + return &Client{name: name, tr: tr} +} + +// Name — the local name of this server (the config key, not the server's own). +func (c *Client) Name() string { return c.name } + +// Info — what the server said about itself during the handshake. +func (c *Client) Info() ServerInfo { + c.mu.Lock() + defer c.mu.Unlock() + return c.info +} + +// Initialize performs the MCP handshake and sends notifications/initialized. +// Capabilities we declare are empty on purpose: Maven consumes, she does not +// offer sampling or roots back to the server. +func (c *Client) Initialize(ctx context.Context) error { + var out struct { + ProtocolVersion string `json:"protocolVersion"` + ServerInfo ServerInfo `json:"serverInfo"` + } + err := c.call(ctx, "initialize", map[string]any{ + "protocolVersion": ProtocolVersion, + "capabilities": map[string]any{}, + "clientInfo": map[string]any{"name": "maven", "version": "1.0"}, + }, &out) + if err != nil { + return err + } + if strings.TrimSpace(out.ProtocolVersion) == "" { + return fmt.Errorf("mcp: %s: handshake returned no protocol version", c.name) + } + out.ServerInfo.ProtocolVersion = out.ProtocolVersion + c.mu.Lock() + c.info, c.ready = out.ServerInfo, true + c.mu.Unlock() + // Best effort: a stateless HTTP server may not care, and a failure here is + // not worth dropping a working connection over. + _ = c.tr.Notify(ctx, "notifications/initialized", map[string]any{}) + return nil +} + +// ListTools discovers the server's tools. +func (c *Client) ListTools(ctx context.Context) ([]Tool, error) { + if !c.initialized() { + return nil, ErrNotInitialized + } + var out struct { + Tools []struct { + Name string `json:"name"` + Description string `json:"description"` + InputSchema json.RawMessage `json:"inputSchema"` + Annotations *struct { + ReadOnlyHint bool `json:"readOnlyHint"` + } `json:"annotations"` + } `json:"tools"` + } + if err := c.call(ctx, "tools/list", map[string]any{}, &out); err != nil { + return nil, err + } + tools := make([]Tool, 0, len(out.Tools)) + for _, t := range out.Tools { + if strings.TrimSpace(t.Name) == "" { + continue + } + tools = append(tools, Tool{ + Server: c.name, + Name: t.Name, + Description: strings.TrimSpace(t.Description), + InputSchema: t.InputSchema, + ReadOnly: t.Annotations != nil && t.Annotations.ReadOnlyHint, + }) + } + return tools, nil +} + +// CallTool runs one tool and returns its text content, joined by newlines. +// Non-text content (images, blobs) is dropped: everything downstream of here +// is a spoken or written sentence. +// +// args is exactly what the router produced. Nothing else — no history, no +// notes, no persona — is in scope here, by construction. +func (c *Client) CallTool(ctx context.Context, name string, args map[string]any) (string, error) { + if !c.initialized() { + return "", ErrNotInitialized + } + if args == nil { + args = map[string]any{} + } + var out struct { + IsError bool `json:"isError"` + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + } `json:"content"` + } + if err := c.call(ctx, "tools/call", map[string]any{"name": name, "arguments": args}, &out); err != nil { + return "", err + } + var parts []string + for _, ct := range out.Content { + if ct.Type == "text" && strings.TrimSpace(ct.Text) != "" { + parts = append(parts, strings.TrimSpace(ct.Text)) + } + } + text := strings.Join(parts, "\n") + if out.IsError { + return text, fmt.Errorf("%w: %s/%s: %s", ErrToolFailed, c.name, name, text) + } + return text, nil +} + +// ListResources discovers the server's resources. A server without the +// resources capability answers with an error; that is not fatal, the caller +// gets an empty list. +func (c *Client) ListResources(ctx context.Context) ([]Resource, error) { + if !c.initialized() { + return nil, ErrNotInitialized + } + var out struct { + Resources []struct { + URI string `json:"uri"` + Name string `json:"name"` + MIMEType string `json:"mimeType"` + } `json:"resources"` + } + if err := c.call(ctx, "resources/list", map[string]any{}, &out); err != nil { + return nil, err + } + res := make([]Resource, 0, len(out.Resources)) + for _, r := range out.Resources { + if strings.TrimSpace(r.URI) == "" { + continue + } + res = append(res, Resource{Server: c.name, URI: r.URI, Name: r.Name, MIMEType: r.MIMEType}) + } + return res, nil +} + +// ReadResource returns a resource's text contents, joined by newlines. This is +// the RAG-hint path: the text can be pasted into a router or phraser prompt. +func (c *Client) ReadResource(ctx context.Context, uri string) (string, error) { + if !c.initialized() { + return "", ErrNotInitialized + } + var out struct { + Contents []struct { + Text string `json:"text"` + } `json:"contents"` + } + if err := c.call(ctx, "resources/read", map[string]any{"uri": uri}, &out); err != nil { + return "", err + } + var parts []string + for _, ct := range out.Contents { + if strings.TrimSpace(ct.Text) != "" { + parts = append(parts, strings.TrimSpace(ct.Text)) + } + } + return strings.Join(parts, "\n"), nil +} + +// Close drops the connection. +func (c *Client) Close() error { + c.mu.Lock() + c.ready = false + c.mu.Unlock() + return c.tr.Close() +} + +func (c *Client) initialized() bool { + c.mu.Lock() + defer c.mu.Unlock() + return c.ready +} + +// alive reports whether the underlying transport can still carry a call. HTTP +// is stateless, so it is always alive; a dead subprocess is not. +func (c *Client) alive() bool { + if s, ok := c.tr.(*stdioTransport); ok { + return s.alive() + } + return true +} + +func (c *Client) call(ctx context.Context, method string, params any, out any) error { + req := &rpcRequest{JSONRPC: "2.0", ID: c.next.Add(1), Method: method, Params: params} + resp, err := c.tr.Call(ctx, req) + if err != nil { + return fmt.Errorf("mcp: %s: %s: %w", c.name, method, err) + } + if resp.Error != nil { + return fmt.Errorf("mcp: %s: %s: %w", c.name, method, resp.Error) + } + if out == nil || len(resp.Result) == 0 { + return nil + } + if err := json.Unmarshal(resp.Result, out); err != nil { + return fmt.Errorf("mcp: %s: %s: decode result: %w", c.name, method, err) + } + return nil +} diff --git a/internal/mcp/http.go b/internal/mcp/http.go new file mode 100644 index 0000000..c4d96e4 --- /dev/null +++ b/internal/mcp/http.go @@ -0,0 +1,184 @@ +package mcp + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "sync" +) + +// Poster is the HTTP seam: internal/webfetch.Fetcher satisfies it. The +// transport takes it as an interface so a test can serve a fake without a +// listener, and so that the ONLY implementation wired in production is the +// guarded fetcher — an MCP endpoint cannot get a bare http.Client this way. +type Poster interface { + Post(ctx context.Context, rawURL, contentType string, body []byte, hdr map[string]string) (*PostResponse, error) +} + +// PostResponse is the shape webfetch returns, restated here so this package +// does not depend on it structurally. +type PostResponse struct { + Status int + ContentType string + Body []byte + Header map[string]string +} + +// httpTransport speaks streamable HTTP: every request is a POST to one +// endpoint, and the reply is either a JSON object or a text/event-stream frame +// carrying one. Both are accepted — servers pick per response, and the two the +// LAN runs disagree about which. +type httpTransport struct { + poster Poster + url string + extra map[string]string // static headers, e.g. an Authorization bearer + + mu sync.Mutex + session string // Mcp-Session-Id, echoed back when the server issues one +} + +func newHTTPTransport(post Poster, endpoint string, extra map[string]string) *httpTransport { + return &httpTransport{poster: post, url: endpoint, extra: extra} +} + +func (t *httpTransport) Call(ctx context.Context, req *rpcRequest) (*rpcResponse, error) { + body, err := t.send(ctx, req) + if err != nil { + return nil, err + } + frame, err := decodeFrame(body, req.ID) + if err != nil { + return nil, err + } + var resp rpcResponse + if err := json.Unmarshal(frame, &resp); err != nil { + return nil, fmt.Errorf("mcp: decode response: %w", err) + } + // The id check the stdio transport already did. Without it a server that + // sends a request of its own (sampling/createMessage, roots/list) mid-stream + // has that request accepted as the answer: it unmarshals into an rpcResponse + // with neither result nor error, and the call reports success with nothing + // in it. An empty string and no error is the one answer that lies — the act + // is logged as done and the tool never ran. + if resp.ID == nil || *resp.ID != req.ID { + return nil, fmt.Errorf("mcp: response id mismatch (wanted %d)", req.ID) + } + if resp.Error == nil && len(resp.Result) == 0 { + return nil, errors.New("mcp: response carries neither result nor error") + } + return &resp, nil +} + +func (t *httpTransport) Notify(ctx context.Context, method string, params any) error { + _, err := t.send(ctx, &rpcRequest{JSONRPC: "2.0", Method: method, Params: params}) + return err +} + +func (t *httpTransport) send(ctx context.Context, req *rpcRequest) ([]byte, error) { + req.JSONRPC = "2.0" + raw, err := json.Marshal(req) + if err != nil { + return nil, err + } + hdr := map[string]string{} + // Configured headers first, so nothing here can be overwritten by them: + // a real remote server needs a bearer token, and the Vikunja one on + // loopback is only reachable without one because it is unauthenticated. + for k, v := range t.extra { + hdr[k] = v + } + hdr["Accept"] = "application/json, text/event-stream" + t.mu.Lock() + if t.session != "" { + hdr["Mcp-Session-Id"] = t.session + } + t.mu.Unlock() + + resp, err := t.poster.Post(ctx, t.url, "application/json", raw, hdr) + if err != nil { + return nil, err + } + if sid := headerGet(resp.Header, "Mcp-Session-Id"); sid != "" { + t.mu.Lock() + t.session = sid + t.mu.Unlock() + } + return resp.Body, nil +} + +func (t *httpTransport) Close() error { + t.mu.Lock() + t.session = "" + t.mu.Unlock() + return nil +} + +func headerGet(h map[string]string, key string) string { + if h == nil { + return "" + } + if v, ok := h[key]; ok { + return v + } + lower := strings.ToLower(key) + for k, v := range h { + if strings.ToLower(k) == lower { + return v + } + } + return "" +} + +// decodeFrame pulls the JSON object out of a body that is either raw JSON or +// SSE. For SSE we take the last data: payload that parses AND carries our own +// id with a result or an error in it. Matching on the presence of an "id" key +// alone is not enough: a JSON-RPC request from the server has one too. +func decodeFrame(body []byte, id int64) ([]byte, error) { + trimmed := bytes.TrimSpace(body) + if len(trimmed) == 0 { + return nil, errors.New("mcp: empty response body") + } + if trimmed[0] == '{' || trimmed[0] == '[' { + return trimmed, nil + } + var last []byte + sc := bufio.NewScanner(bytes.NewReader(trimmed)) + sc.Buffer(make([]byte, 0, 64<<10), maxLine) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if !strings.HasPrefix(line, "data:") { + continue + } + payload := strings.TrimSpace(strings.TrimPrefix(line, "data:")) + if payload == "" { + continue + } + var probe struct { + ID *int64 `json:"id"` + Result json.RawMessage `json:"result"` + Error json.RawMessage `json:"error"` + Method string `json:"method"` + } + if json.Unmarshal([]byte(payload), &probe) != nil { + continue + } + if probe.Method != "" || probe.ID == nil || *probe.ID != id { + continue + } + if len(probe.Result) == 0 && len(probe.Error) == 0 { + continue + } + last = []byte(payload) + } + if err := sc.Err(); err != nil { + return nil, fmt.Errorf("mcp: read event stream: %w", err) + } + if last == nil { + return nil, fmt.Errorf("mcp: no JSON-RPC response for id %d in event stream", id) + } + return last, nil +} diff --git a/internal/mcp/jsonrpc.go b/internal/mcp/jsonrpc.go new file mode 100644 index 0000000..45222cf --- /dev/null +++ b/internal/mcp/jsonrpc.go @@ -0,0 +1,72 @@ +// Package mcp is Maven's Model Context Protocol CLIENT. She is a host: she +// connects OUT to MCP servers, discovers the tools and resources they offer, +// and hands them to the parts of her that already exist for this — the tool +// allowlist in the store, the confirm turn for anything that mutates, the +// stage-3 gate that makes an uncertain act ask instead of run. +// +// She is not an MCP server. Nothing here exposes her own capabilities to an +// outside caller; docs/plans/06-mcp-support.md asks for the host direction only. +// +// Boundaries, in code rather than in prose: +// +// - OFF unless configured. No mcp_servers block ⇒ no manager, no goroutine, +// no socket. +// - A remote server is reached through internal/webfetch, so the SSRF guard, +// the size cap, the redirect cap and the per-host rate limit all apply to +// an MCP endpoint exactly as they do to a news feed. Reaching a loopback +// or LAN server means explicitly setting allow_private on THAT server — +// a different trust level, spelled out per server rather than globally. +// - Only the tool name and the arguments the router produced are sent. This +// package never sees his notes, facts, history or the persona block, and +// has no API through which a caller could pass them. +// - Discovery proposes, it does not enable. A discovered tool lands as a +// 'proposed' row; a human enables it on the authed surface. +package mcp + +import ( + "context" + "encoding/json" + "fmt" +) + +// ProtocolVersion — the spec revision we ask for in the initialize handshake. +// A server that answers with a different one is accepted (the spec says the +// client may proceed if it can support what came back); we only refuse when it +// answers with no version at all, which means it is not an MCP server. +const ProtocolVersion = "2025-06-18" + +// rpcRequest / rpcResponse — JSON-RPC 2.0. Deliberately hand-rolled: the wire +// format is four fields, and the repo vendors its dependencies, so pulling a +// library in for this would cost more than it saves. +type rpcRequest struct { + JSONRPC string `json:"jsonrpc"` + ID int64 `json:"id,omitempty"` + Method string `json:"method"` + Params any `json:"params,omitempty"` +} + +type rpcResponse struct { + JSONRPC string `json:"jsonrpc"` + ID *int64 `json:"id"` + Result json.RawMessage `json:"result,omitempty"` + Error *rpcError `json:"error,omitempty"` +} + +type rpcError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +func (e *rpcError) Error() string { return fmt.Sprintf("mcp: rpc error %d: %s", e.Code, e.Message) } + +// transport carries one JSON-RPC conversation. Implementations: stdioTransport +// (a subprocess on this box) and httpTransport (streamable HTTP, guarded by +// webfetch). Both must be safe for concurrent use by the Client. +type transport interface { + // Call sends a request and returns the matching response. + Call(ctx context.Context, req *rpcRequest) (*rpcResponse, error) + // Notify sends a notification (no id, no reply expected). + Notify(ctx context.Context, method string, params any) error + // Close releases the transport (kills the subprocess, drops the session). + Close() error +} diff --git a/internal/mcp/manager.go b/internal/mcp/manager.go new file mode 100644 index 0000000..d8a7dc4 --- /dev/null +++ b/internal/mcp/manager.go @@ -0,0 +1,642 @@ +package mcp + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +// Defaults for a server block. Small numbers on purpose — see MaxTools. +const ( + // DefaultTimeout bounds one JSON-RPC call. A tool that takes longer than + // this is not usable in a spoken turn anyway. + DefaultTimeout = 15 * time.Second + // DefaultMaxTools caps how many tools ONE server may contribute. The + // resident model is a 1.7B with a 4096-token context: a catalogue of forty + // tool names does not fit in its head, and a name it half-remembers is a + // wrong act. Twelve per server is already generous. + DefaultMaxTools = 12 + // DefaultReconnectEvery is how long the manager waits before re-dialing a + // server whose connection died. It is the FIRST wait: every consecutive + // failure doubles it, up to MaxReconnectEvery. + DefaultReconnectEvery = 30 * time.Second + // MaxReconnectEvery caps the backoff. Without one, a permanently + // misconfigured stdio server is exec'd once a minute forever, which is a + // process spawn per minute in the logs and nothing that ever gets better. + MaxReconnectEvery = 30 * time.Minute + // DefaultMaxDescription bounds one tool description. It is written by a + // server Maven does not control and it lands in two places that cannot + // absorb an arbitrary blob: the resident model's 4096-token context, and a + // table cell on /tools. + DefaultMaxDescription = 400 +) + +var ( + // ErrNoServer — the named server is not configured. + ErrNoServer = errors.New("mcp: no such server") + // ErrNotConnected — the server is configured but nothing is dialed. Held + // apart from ErrNoServer so a caller can say "that tool is not connected" + // instead of drafting a proposal for a tool that already exists. + ErrNotConnected = errors.New("mcp: server is not connected") + // ErrToolGone — the server no longer offers this tool. An enabled row can + // outlive the tool it names; this is what the act path sees when it does. + ErrToolGone = errors.New("mcp: server no longer offers this tool") +) + +// ServerConfig is one configured MCP server. Off unless present. +// +// Exactly one of Command (a subprocess on this box) or URL (a remote or +// loopback HTTP endpoint) must be set. +type ServerConfig struct { + // Name is the local handle. It prefixes every tool this server + // contributes, so it must be short and a valid identifier-ish word. + Name string `json:"name"` + // Command + Args + Env + Dir describe a stdio server: a child process of + // mavend, on this box, under this user. argv, never a shell string. + Command string `json:"command,omitempty"` + Args []string `json:"args,omitempty"` + Env []string `json:"env,omitempty"` + Dir string `json:"dir,omitempty"` + // URL is a streamable-HTTP endpoint. It goes through internal/webfetch, so + // it inherits the SSRF guard, the size cap and the per-host rate limit. + URL string `json:"url,omitempty"` + // AllowPrivate lets THIS server be a loopback or LAN address + // (http://localhost:9100/mcp is the Vikunja server on homesrv). It is a + // per-server hole in the private-address guard and it is not the same trust + // level as a public endpoint: whatever is behind it is inside the network, + // so an argument the router got wrong reaches something that matters. Set + // it only for a server you run. + AllowPrivate bool `json:"allow_private,omitempty"` + // AllowTools, when non-empty, is the ONLY set of remote tool names taken + // from this server. This is the knob for keeping the catalogue small and + // deliberate rather than "whatever the server grew this week". + AllowTools []string `json:"allow_tools,omitempty"` + // MaxTools caps the contribution (0 ⇒ DefaultMaxTools). + MaxTools int `json:"max_tools,omitempty"` + // Headers are sent verbatim on every request to a url server. This is how + // a bearer token reaches a real remote server; the Vikunja one on loopback + // needs none only because it is unauthenticated. + Headers map[string]string `json:"-"` + // Timeout bounds one call (0 ⇒ DefaultTimeout). + Timeout time.Duration `json:"-"` + // Enabled=false keeps a configured server described but dark. + Enabled bool `json:"enabled"` +} + +// PosterFactory builds the HTTP door for one server. It is a factory rather +// than a single shared Poster because allow_private is per server: the fetcher +// that may reach http://localhost:9100/mcp must NOT be the same fetcher another +// server's public URL goes through, or one loopback exemption would quietly +// unlock the LAN for all of them. +type PosterFactory func(cfg ServerConfig) (Poster, error) + +// Manager owns the connections. Nothing here starts unless at least one server +// is configured and enabled. +type Manager struct { + newPoster PosterFactory + mu sync.Mutex + conns map[string]*conn + order []string +} + +type conn struct { + cfg ServerConfig + client *Client + tools []Tool + lastErr error + lastTry time.Time + dialedAt time.Time + fails int // consecutive dial failures, for the backoff +} + +// backoff is how long this connection waits before the next dial attempt: +// DefaultReconnectEvery doubled per consecutive failure, capped. +func (c *conn) backoff() time.Duration { + d := DefaultReconnectEvery + for i := 1; i < c.fails && d < MaxReconnectEvery; i++ { + d *= 2 + } + if d > MaxReconnectEvery { + d = MaxReconnectEvery + } + return d +} + +// NewManager builds a manager for the enabled servers in cfgs. newPoster is +// the guarded HTTP door factory for url servers; pass nil only when no url +// server is configured (a nil factory with a url server is reported per server +// at dial time rather than fatally, so one bad block never stops the daemon). +// +// Dialing is lazy: NewManager validates and records, Connect dials. +func NewManager(newPoster PosterFactory, cfgs []ServerConfig) (*Manager, error) { + m := &Manager{newPoster: newPoster, conns: map[string]*conn{}} + for _, c := range cfgs { + if !c.Enabled { + continue + } + if err := validate(c); err != nil { + return nil, err + } + if _, dup := m.conns[c.Name]; dup { + return nil, fmt.Errorf("mcp: duplicate server name %q", c.Name) + } + if c.Timeout <= 0 { + c.Timeout = DefaultTimeout + } + if c.MaxTools <= 0 { + c.MaxTools = DefaultMaxTools + } + m.conns[c.Name] = &conn{cfg: c} + m.order = append(m.order, c.Name) + } + sort.Strings(m.order) + return m, nil +} + +// Validate checks a set of server blocks without dialling anything, so a typo +// fails at startup rather than at the first turn that needed the tool. +func Validate(cfgs []ServerConfig) error { + seen := map[string]bool{} + for _, c := range cfgs { + if err := validate(c); err != nil { + return err + } + if seen[c.Name] { + return fmt.Errorf("mcp: duplicate server name %q", c.Name) + } + seen[c.Name] = true + } + return nil +} + +func validate(c ServerConfig) error { + if strings.TrimSpace(c.Name) == "" { + return errors.New("mcp: server needs a name") + } + if strings.ContainsAny(c.Name, " \t/:") { + return fmt.Errorf("mcp: server name %q must be one word without spaces, slashes or colons", c.Name) + } + hasCmd, hasURL := c.Command != "", c.URL != "" + if hasCmd == hasURL { + return fmt.Errorf("mcp: server %q needs exactly one of command or url", c.Name) + } + if hasURL && !strings.HasPrefix(c.URL, "http://") && !strings.HasPrefix(c.URL, "https://") { + return fmt.Errorf("mcp: server %q url must be http or https", c.Name) + } + return nil +} + +// Servers — the configured, enabled server names, sorted. +func (m *Manager) Servers() []string { + m.mu.Lock() + defer m.mu.Unlock() + return append([]string(nil), m.order...) +} + +// Empty reports whether nothing is configured. The daemon uses it to skip +// wiring entirely. +func (m *Manager) Empty() bool { + m.mu.Lock() + defer m.mu.Unlock() + return len(m.conns) == 0 +} + +// Connect dials every configured server, handshakes, and discovers tools. +// A server that fails is recorded and retried later by Refresh — one bad +// server never blocks the others, and never blocks boot. +func (m *Manager) Connect(ctx context.Context) { + for _, name := range m.Servers() { + if err := m.dial(ctx, name); err != nil { + log.Printf("mcp: %s: %v", name, err) + } + } +} + +func (m *Manager) dial(ctx context.Context, name string) error { + m.mu.Lock() + c, ok := m.conns[name] + if !ok { + m.mu.Unlock() + return ErrNoServer + } + cfg := c.cfg + c.lastTry = time.Now() + m.mu.Unlock() + + var tr transport + var err error + if cfg.Command != "" { + tr, err = newStdioTransport(ctx, append([]string{cfg.Command}, cfg.Args...), cfg.Env, cfg.Dir) + } else if m.newPoster == nil { + err = fmt.Errorf("server %q has a url but no http door was wired", name) + } else { + var poster Poster + if poster, err = m.newPoster(cfg); err == nil { + tr = newHTTPTransport(poster, cfg.URL, cfg.Headers) + } + } + if err != nil { + m.fail(name, err) + return err + } + + cl := newClient(name, tr) + ictx, cancel := context.WithTimeout(ctx, cfg.Timeout) + defer cancel() + if err := cl.Initialize(ictx); err != nil { + _ = cl.Close() + m.fail(name, err) + return err + } + tools, err := cl.ListTools(ictx) + if err != nil { + // A server with no tools capability is still a usable resource server. + log.Printf("mcp: %s: list tools: %v", name, err) + tools = nil + } + tools = filterTools(cfg, tools) + + m.mu.Lock() + if old := m.conns[name].client; old != nil { + _ = old.Close() + } + m.conns[name].client = cl + m.conns[name].tools = tools + m.conns[name].lastErr = nil + m.conns[name].fails = 0 + m.conns[name].dialedAt = time.Now() + m.mu.Unlock() + log.Printf("mcp: %s connected (%s %s), %d tool(s)", name, cl.Info().Name, cl.Info().Version, len(tools)) + return nil +} + +func (m *Manager) fail(name string, err error) { + m.mu.Lock() + defer m.mu.Unlock() + if c := m.conns[name]; c != nil { + c.lastErr = err + c.client = nil + c.tools = nil + c.fails++ + } +} + +// filterTools applies AllowTools and MaxTools, drops nameless entries and +// truncates descriptions. +// +// Over the cap WITHOUT allow_tools, the whole contribution is dropped. Taking +// the first N of a sorted list was deterministic but it handed the choice of +// which N to the server: a thirteenth tool named "aaa_" would push a tool that +// had already been discovered, proposed and maybe enabled out of the +// catalogue. Determinism was not the property worth buying. With allow_tools +// set, Kami named the tools, so the cap trims a list he chose. +func filterTools(cfg ServerConfig, in []Tool) []Tool { + sort.Slice(in, func(i, j int) bool { return in[i].Name < in[j].Name }) + out := make([]Tool, 0, len(in)) + for _, t := range in { + if len(cfg.AllowTools) > 0 && !contains(cfg.AllowTools, t.Name) { + continue + } + t.Description = truncate(t.Description, DefaultMaxDescription) + out = append(out, t) + } + if cfg.MaxTools > 0 && len(out) > cfg.MaxTools { + if len(cfg.AllowTools) == 0 { + log.Printf("mcp: %s offers %d tools, over the cap of %d — taking NONE of them, set allow_tools to choose or raise max_tools", + cfg.Name, len(out), cfg.MaxTools) + return nil + } + log.Printf("mcp: %s: allow_tools names %d tools, over the cap of %d — taking the first %d", + cfg.Name, len(out), cfg.MaxTools, cfg.MaxTools) + out = out[:cfg.MaxTools] + } + return out +} + +// truncate bounds a server-written string. The ellipsis is there so a reader +// on /tools can tell the text was cut rather than written that way. +func truncate(s string, max int) string { + r := []rune(s) + if len(r) <= max { + return s + } + return strings.TrimSpace(string(r[:max])) + "…" +} + +func contains(hay []string, needle string) bool { + for _, h := range hay { + if h == needle { + return true + } + } + return false +} + +// Refresh re-dials any server that is down, if enough time has passed since the +// last attempt. Call it from the daemon's periodic tick — it is cheap when +// everything is up. +// The health check itself is done OUTSIDE m.mu, the way Resources already +// does it. alive() reaches into the transport, and a transport waiting on a +// silent subprocess would otherwise hold m.mu for as long as it waits, which +// blocks Tools, Status and Call for every other server too. +func (m *Manager) Refresh(ctx context.Context) { + now := time.Now() + type candidate struct { + name string + cl *Client + ready bool + } + var cands []candidate + m.mu.Lock() + for _, name := range m.order { + c := m.conns[name] + cands = append(cands, candidate{name: name, cl: c.client, ready: now.Sub(c.lastTry) >= c.backoff()}) + } + m.mu.Unlock() + var stale []string + for _, c := range cands { + if !c.ready { + continue + } + if c.cl == nil || !c.cl.alive() { + stale = append(stale, c.name) + } + } + for _, name := range stale { + if err := m.dial(ctx, name); err != nil { + log.Printf("mcp: %s: reconnect: %v", name, err) + } + } +} + +// Tools — every discovered tool across connected servers, sorted by +// server then name. +func (m *Manager) Tools() []Tool { + m.mu.Lock() + defer m.mu.Unlock() + var out []Tool + for _, name := range m.order { + out = append(out, m.conns[name].tools...) + } + return out +} + +// Connected — the names of servers that are dialed right now. A caller that +// wants to act on a tool's ABSENCE needs this: a tool missing from Tools() +// because its server is down is not a tool the server withdrew. +func (m *Manager) Connected() []string { + m.mu.Lock() + defer m.mu.Unlock() + var out []string + for _, name := range m.order { + if m.conns[name].client != nil { + out = append(out, name) + } + } + return out +} + +// Status is one server's health, for the web surface. +type Status struct { + Name string + Transport string // "stdio" or "http" + Target string // command or url + Connected bool + Server string // the server's own name+version + Tools int + Err string +} + +// Status reports every configured server. +func (m *Manager) Status() []Status { + m.mu.Lock() + defer m.mu.Unlock() + out := make([]Status, 0, len(m.order)) + for _, name := range m.order { + c := m.conns[name] + s := Status{Name: name, Tools: len(c.tools)} + if c.cfg.Command != "" { + s.Transport, s.Target = "stdio", strings.Join(append([]string{c.cfg.Command}, c.cfg.Args...), " ") + } else { + s.Transport, s.Target = "http", c.cfg.URL + } + if c.client != nil { + s.Connected = true + s.Server = strings.TrimSpace(c.client.Info().Name + " " + c.client.Info().Version) + } + if c.lastErr != nil { + s.Err = c.lastErr.Error() + } + out = append(out, s) + } + return out +} + +// Call runs server's tool with args. Args come from the router and nothing +// else; there is no path here through which a note or a fact could travel. +func (m *Manager) Call(ctx context.Context, server, tool string, args map[string]any) (string, error) { + m.mu.Lock() + c := m.conns[server] + m.mu.Unlock() + if c == nil { + return "", fmt.Errorf("%w: %s", ErrNoServer, server) + } + m.mu.Lock() + cl, timeout, known := c.client, c.cfg.Timeout, false + for _, t := range c.tools { + if t.Name == tool { + known = true + break + } + } + m.mu.Unlock() + if cl == nil { + return "", fmt.Errorf("%w: %s", ErrNotConnected, server) + } + // The discovered-and-filtered set is the second allowlist: even an enabled + // store row cannot reach a tool the server stopped offering, or one + // allow_tools excludes. + if !known { + return "", fmt.Errorf("%w: %s/%s", ErrToolGone, server, tool) + } + cctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + return cl.CallTool(cctx, tool, args) +} + +// Resources lists resources across connected servers. +func (m *Manager) Resources(ctx context.Context) []Resource { + m.mu.Lock() + clients := make([]*Client, 0, len(m.order)) + for _, name := range m.order { + if cl := m.conns[name].client; cl != nil { + clients = append(clients, cl) + } + } + m.mu.Unlock() + var out []Resource + for _, cl := range clients { + rs, err := cl.ListResources(ctx) + if err != nil { + continue // no resources capability; not an error worth logging per tick + } + out = append(out, rs...) + } + return out +} + +// ReadResource reads one resource from one server. +func (m *Manager) ReadResource(ctx context.Context, server, uri string) (string, error) { + m.mu.Lock() + c := m.conns[server] + var cl *Client + var timeout time.Duration + if c != nil { + cl, timeout = c.client, c.cfg.Timeout + } + m.mu.Unlock() + if cl == nil { + return "", fmt.Errorf("%w: %s", ErrNoServer, server) + } + cctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + return cl.ReadResource(cctx, uri) +} + +// Close shuts every connection down. +func (m *Manager) Close() error { + m.mu.Lock() + defer m.mu.Unlock() + for _, name := range m.order { + if cl := m.conns[name].client; cl != nil { + _ = cl.Close() + m.conns[name].client = nil + } + } + return nil +} + +// ErrNeedsArgs — the tool requires arguments that a voice verb cannot supply. +var ErrNeedsArgs = errors.New("mcp: tool needs named arguments") + +// CallPositional is the voice path's way in. The router gives an act a verb and +// a tail of positional words; an MCP tool wants a named-argument object. There +// is no general mapping between those two, and inventing one is exactly the +// improvisation this codebase refuses, so the rule is deliberately narrow: +// +// - a tool with no required properties runs with no arguments (a spare tail +// is ignored — "покажи проекты пожалуйста" should still list projects); +// - a READ-ONLY tool NAMED IN allow_tools, with exactly one required +// property, of type string or integer/number, gets the tail bound to it; +// - anything else is refused with ErrNeedsArgs. Such a tool is still callable +// with explicit arguments from the authed surface, where a human types +// them. +// +// The refusal is the point, and the read-only condition on it was learned the +// hard way while testing against the Vikunja server: `update_task` requires +// only `task_id` and takes every other field as optional, so calling it with +// one guessed argument and no others BLANKED the fields it did not receive. A +// mutating tool therefore never gets a guessed argument — the one thing a +// partially-filled write can do is destroy what it did not mention. A mutating +// tool with nothing required is still fine: nothing was guessed, and it still +// goes through the confirm turn. +// +// The allow_tools condition is the second half, and it is there because +// readOnlyHint is the SERVER's claim about itself. It already buys one +// exemption (destructive=false, so no confirm turn); letting it buy argument +// binding as well means one lie converts a spoken utterance into an +// unconfirmed, argument-carrying write. A server advertising delete_project +// with readOnlyHint true and the description "show a project and its tasks" +// would be enough. So the binding half rests on something local instead: a +// name Kami typed into mavend.json. The tool name is not a defence — the +// router picks tools by name similarity and the description a human reads is +// server-written too. +func (m *Manager) CallPositional(ctx context.Context, server, tool string, args []string) (string, error) { + m.mu.Lock() + c := m.conns[server] + var schema json.RawMessage + found, readOnly, bindable := false, false, false + configured, connected := c != nil, false + if c != nil { + connected = c.client != nil + for _, t := range c.tools { + if t.Name == tool { + schema, readOnly, found = t.InputSchema, t.ReadOnly, true + bindable = contains(c.cfg.AllowTools, tool) + break + } + } + } + m.mu.Unlock() + if !found { + if !configured { + return "", fmt.Errorf("%w: %s", ErrNoServer, server) + } + if !connected { + return "", fmt.Errorf("%w: %s", ErrNotConnected, server) + } + return "", fmt.Errorf("%w: %s/%s", ErrToolGone, server, tool) + } + named, err := bindPositional(schema, args, readOnly && bindable) + if err != nil { + return "", err + } + return m.Call(ctx, server, tool, named) +} + +// bindPositional implements the rule documented on CallPositional. bind is the +// caller's verdict on whether a guessed argument is allowed at all: read-only +// AND named in allow_tools. +func bindPositional(schema json.RawMessage, args []string, bind bool) (map[string]any, error) { + var s struct { + Required []string `json:"required"` + Properties map[string]struct { + Type string `json:"type"` + } `json:"properties"` + } + if len(schema) > 0 { + if err := json.Unmarshal(schema, &s); err != nil { + return nil, fmt.Errorf("mcp: unreadable input schema: %w", err) + } + } + switch len(s.Required) { + case 0: + return map[string]any{}, nil + case 1: + name := s.Required[0] + prop, described := s.Properties[name] + if !described { + // required names it, properties does not describe it. The zero + // value would make it a string, which is a guess about a guess. + return nil, fmt.Errorf("%w: %q, which the schema never describes", ErrNeedsArgs, name) + } + if !bind { + return nil, fmt.Errorf("%w: %q, and a guessed argument goes only to a read-only tool named in allow_tools", ErrNeedsArgs, name) + } + tail := strings.TrimSpace(strings.Join(args, " ")) + if tail == "" { + return nil, fmt.Errorf("%w: %q", ErrNeedsArgs, name) + } + switch prop.Type { + case "string", "": + return map[string]any{name: tail}, nil + case "integer", "number": + n, err := strconv.ParseFloat(tail, 64) + if err != nil { + return nil, fmt.Errorf("%w: %q wants a number, got %q", ErrNeedsArgs, name, tail) + } + return map[string]any{name: n}, nil + default: + return nil, fmt.Errorf("%w: %q is a %s", ErrNeedsArgs, name, prop.Type) + } + default: + return nil, fmt.Errorf("%w: %s", ErrNeedsArgs, strings.Join(s.Required, ", ")) + } +} diff --git a/internal/mcp/mcp_test.go b/internal/mcp/mcp_test.go new file mode 100644 index 0000000..9c8c966 --- /dev/null +++ b/internal/mcp/mcp_test.go @@ -0,0 +1,643 @@ +package mcp + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + "testing" + "time" +) + +// fakePoster answers POSTs from a canned handler, in either JSON or SSE form. +type fakePoster struct { + mu sync.Mutex + handler func(method string, params json.RawMessage) (any, *rpcError) + sse bool + session string + seen []map[string]string // headers of each request, for the session test + calls []string +} + +func (f *fakePoster) Post(_ context.Context, _, _ string, body []byte, hdr map[string]string) (*PostResponse, error) { + var req struct { + ID *int64 `json:"id"` + Method string `json:"method"` + Params json.RawMessage `json:"params"` + } + if err := json.Unmarshal(body, &req); err != nil { + return nil, err + } + f.mu.Lock() + f.seen = append(f.seen, hdr) + f.calls = append(f.calls, req.Method) + f.mu.Unlock() + + if req.ID == nil { // notification + return &PostResponse{Status: 202, Body: []byte(`{}`)}, nil + } + result, rerr := f.handler(req.Method, req.Params) + resp := map[string]any{"jsonrpc": "2.0", "id": *req.ID} + if rerr != nil { + resp["error"] = map[string]any{"code": rerr.Code, "message": rerr.Message} + } else { + resp["result"] = result + } + raw, _ := json.Marshal(resp) + out := &PostResponse{Status: 200, Body: raw, ContentType: "application/json", Header: map[string]string{}} + if f.sse { + out.ContentType = "text/event-stream" + out.Body = []byte("event: message\ndata: {\"jsonrpc\":\"2.0\",\"method\":\"notifications/progress\"}\n\nevent: message\ndata: " + string(raw) + "\n\n") + } + if f.session != "" { + out.Header["Mcp-Session-Id"] = f.session + } + return out, nil +} + +// echoServer is a handler with two tools, one read-only and one not. +func echoServer() func(string, json.RawMessage) (any, *rpcError) { + return func(method string, params json.RawMessage) (any, *rpcError) { + switch method { + case "initialize": + return map[string]any{ + "protocolVersion": ProtocolVersion, + "serverInfo": map[string]any{"name": "fake", "version": "0.1"}, + }, nil + case "tools/list": + return map[string]any{"tools": []any{ + map[string]any{ + "name": "read_thing", "description": "reads", + "inputSchema": map[string]any{"type": "object"}, + "annotations": map[string]any{"readOnlyHint": true}, + }, + map[string]any{"name": "break_thing", "description": "mutates"}, + }}, nil + case "tools/call": + var p struct { + Name string `json:"name"` + Args map[string]any `json:"arguments"` + } + _ = json.Unmarshal(params, &p) + if p.Name == "break_thing" { + return map[string]any{"isError": true, "content": []any{ + map[string]any{"type": "text", "text": "не вышло"}}}, nil + } + return map[string]any{"content": []any{ + map[string]any{"type": "text", "text": fmt.Sprintf("%s:%v", p.Name, p.Args["q"])}, + map[string]any{"type": "image", "text": "ignored"}, + }}, nil + case "resources/list": + return map[string]any{"resources": []any{ + map[string]any{"uri": "note://one", "name": "one", "mimeType": "text/plain"}, + map[string]any{"uri": "", "name": "nameless"}, + }}, nil + case "resources/read": + return map[string]any{"contents": []any{map[string]any{"text": "тело ресурса"}}}, nil + } + return nil, &rpcError{Code: -32601, Message: "method not found"} + } +} + +func dialFake(t *testing.T, p *fakePoster) *Client { + t.Helper() + c := newClient("fake", newHTTPTransport(p, "http://example.test/mcp", nil)) + if err := c.Initialize(context.Background()); err != nil { + t.Fatalf("initialize: %v", err) + } + return c +} + +func TestHandshakeAndDiscovery(t *testing.T) { + for _, sse := range []bool{false, true} { + name := "json" + if sse { + name = "sse" + } + t.Run(name, func(t *testing.T) { + p := &fakePoster{handler: echoServer(), sse: sse} + c := dialFake(t, p) + if got := c.Info().Name; got != "fake" { + t.Fatalf("server name = %q", got) + } + if got := c.Info().ProtocolVersion; got != ProtocolVersion { + t.Fatalf("protocol = %q", got) + } + tools, err := c.ListTools(context.Background()) + if err != nil { + t.Fatalf("list tools: %v", err) + } + if len(tools) != 2 { + t.Fatalf("tools = %+v", tools) + } + byName := map[string]Tool{} + for _, tl := range tools { + byName[tl.Name] = tl + } + if !byName["read_thing"].ReadOnly { + t.Error("read_thing should be read-only (readOnlyHint true)") + } + // The important direction: no annotation ⇒ assume it mutates. + if byName["break_thing"].ReadOnly { + t.Error("break_thing has no readOnlyHint, must NOT be treated as read-only") + } + if byName["read_thing"].Server != "fake" { + t.Error("tool should carry its server handle") + } + }) + } +} + +func TestCallToolTextOnly(t *testing.T) { + c := dialFake(t, &fakePoster{handler: echoServer()}) + out, err := c.CallTool(context.Background(), "read_thing", map[string]any{"q": "привет"}) + if err != nil { + t.Fatalf("call: %v", err) + } + if out != "read_thing:привет" { + t.Fatalf("out = %q (non-text content must be dropped)", out) + } +} + +func TestCallToolErrorResult(t *testing.T) { + c := dialFake(t, &fakePoster{handler: echoServer()}) + out, err := c.CallTool(context.Background(), "break_thing", nil) + if err == nil { + t.Fatal("isError result must surface as an error") + } + if out != "не вышло" { + t.Fatalf("text should still come back, got %q", out) + } +} + +func TestResources(t *testing.T) { + c := dialFake(t, &fakePoster{handler: echoServer()}) + rs, err := c.ListResources(context.Background()) + if err != nil { + t.Fatalf("list resources: %v", err) + } + if len(rs) != 1 || rs[0].URI != "note://one" { + t.Fatalf("resources = %+v (a uri-less entry must be dropped)", rs) + } + body, err := c.ReadResource(context.Background(), "note://one") + if err != nil { + t.Fatalf("read: %v", err) + } + if body != "тело ресурса" { + t.Fatalf("body = %q", body) + } +} + +func TestCallBeforeInitializeRefused(t *testing.T) { + c := newClient("fake", newHTTPTransport(&fakePoster{handler: echoServer()}, "http://example.test/mcp", nil)) + if _, err := c.CallTool(context.Background(), "read_thing", nil); err != ErrNotInitialized { + t.Fatalf("err = %v, want ErrNotInitialized", err) + } +} + +func TestSessionIDEchoed(t *testing.T) { + p := &fakePoster{handler: echoServer(), session: "sess-1"} + c := dialFake(t, p) + if _, err := c.ListTools(context.Background()); err != nil { + t.Fatal(err) + } + p.mu.Lock() + defer p.mu.Unlock() + last := p.seen[len(p.seen)-1] + if last["Mcp-Session-Id"] != "sess-1" { + t.Fatalf("session header not echoed: %+v", last) + } + if !strings.Contains(last["Accept"], "text/event-stream") { + t.Fatalf("Accept must offer both forms: %q", last["Accept"]) + } +} + +func TestHandshakeWithoutProtocolVersionRefused(t *testing.T) { + p := &fakePoster{handler: func(m string, _ json.RawMessage) (any, *rpcError) { + return map[string]any{"serverInfo": map[string]any{"name": "not-mcp"}}, nil + }} + c := newClient("x", newHTTPTransport(p, "http://example.test/mcp", nil)) + if err := c.Initialize(context.Background()); err == nil { + t.Fatal("a reply with no protocolVersion is not an MCP server") + } +} + +func TestRPCErrorSurfaces(t *testing.T) { + c := dialFake(t, &fakePoster{handler: echoServer()}) + if _, err := c.callRaw(context.Background(), "nope/nope"); err == nil { + t.Fatal("want an rpc error") + } else if !strings.Contains(err.Error(), "method not found") { + t.Fatalf("err = %v", err) + } +} + +// callRaw is a test-only shim so the rpc-error path can be exercised without a +// typed wrapper for a method the server does not implement. +func (c *Client) callRaw(ctx context.Context, method string) (any, error) { + var out any + err := c.call(ctx, method, map[string]any{}, &out) + return out, err +} + +func TestDecodeFrame(t *testing.T) { + cases := []struct { + name, in, want string + wantErr bool + }{ + {name: "plain json", in: `{"id":2,"result":{}}`, want: `{"id":2,"result":{}}`}, + {name: "sse single", in: "event: message\ndata: {\"id\":2,\"result\":1}\n\n", want: `{"id":2,"result":1}`}, + { + name: "sse picks the response not the notification", + in: "data: {\"method\":\"notifications/progress\"}\n\ndata: {\"id\":2,\"result\":2}\n\n", + want: `{"id":2,"result":2}`, + }, + {name: "empty", in: " ", wantErr: true}, + {name: "sse with no response", in: "data: {\"method\":\"x\"}\n\n", wantErr: true}, + { + // A JSON-RPC REQUEST from the server has an id too. Taking it as + // the response gave a frame with neither result nor error, which + // the client reported as an empty success: the act logged as done + // and the tool never run. + name: "sse server request is not a response", + in: "data: {\"id\":2,\"method\":\"sampling/createMessage\",\"params\":{}}\n\n", + wantErr: true, + }, + { + name: "sse response for another id", + in: "data: {\"id\":9,\"result\":1}\n\n", + wantErr: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := decodeFrame([]byte(tc.in), 2) + if tc.wantErr { + if err == nil { + t.Fatalf("want error, got %q", got) + } + return + } + if err != nil { + t.Fatal(err) + } + if string(got) != tc.want { + t.Fatalf("got %q want %q", got, tc.want) + } + }) + } +} + +func TestValidate(t *testing.T) { + cases := []struct { + name string + cfg ServerConfig + wantErr bool + }{ + {name: "stdio ok", cfg: ServerConfig{Name: "a", Command: "echo"}}, + {name: "http ok", cfg: ServerConfig{Name: "a", URL: "http://x.test/mcp"}}, + {name: "no name", cfg: ServerConfig{Command: "echo"}, wantErr: true}, + {name: "spacey name", cfg: ServerConfig{Name: "a b", Command: "echo"}, wantErr: true}, + {name: "neither", cfg: ServerConfig{Name: "a"}, wantErr: true}, + {name: "both", cfg: ServerConfig{Name: "a", Command: "echo", URL: "http://x.test"}, wantErr: true}, + {name: "bad scheme", cfg: ServerConfig{Name: "a", URL: "file:///etc/passwd"}, wantErr: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := Validate([]ServerConfig{tc.cfg}) + if (err != nil) != tc.wantErr { + t.Fatalf("err = %v, wantErr = %v", err, tc.wantErr) + } + }) + } + if err := Validate([]ServerConfig{{Name: "a", Command: "x"}, {Name: "a", Command: "y"}}); err == nil { + t.Error("duplicate names must be refused") + } +} + +func TestManagerOffWhenNothingEnabled(t *testing.T) { + m, err := NewManager(nil, []ServerConfig{{Name: "a", Command: "echo"}}) // Enabled=false + if err != nil { + t.Fatal(err) + } + if !m.Empty() { + t.Fatal("a server that is not enabled must not be wired") + } + m.Connect(context.Background()) + if got := m.Tools(); len(got) != 0 { + t.Fatalf("tools = %+v", got) + } +} + +func TestManagerDiscoversAndCalls(t *testing.T) { + p := &fakePoster{handler: echoServer()} + m, err := NewManager(func(ServerConfig) (Poster, error) { return p, nil }, + []ServerConfig{{Name: "fake", URL: "http://example.test/mcp", Enabled: true}}) + if err != nil { + t.Fatal(err) + } + m.Connect(context.Background()) + defer m.Close() + + tools := m.Tools() + if len(tools) != 2 { + t.Fatalf("tools = %+v", tools) + } + out, err := m.Call(context.Background(), "fake", "read_thing", map[string]any{"q": "да"}) + if err != nil { + t.Fatalf("call: %v", err) + } + if out != "read_thing:да" { + t.Fatalf("out = %q", out) + } + // The discovered set is a second allowlist. + if _, err := m.Call(context.Background(), "fake", "not_offered", nil); err == nil { + t.Error("a tool the server does not offer must be refused") + } + if _, err := m.Call(context.Background(), "other", "read_thing", nil); err == nil { + t.Error("an unconfigured server must be refused") + } + st := m.Status() + if len(st) != 1 || !st[0].Connected || st[0].Transport != "http" || st[0].Tools != 2 { + t.Fatalf("status = %+v", st) + } +} + +func TestManagerAllowToolsAndMaxTools(t *testing.T) { + p := &fakePoster{handler: echoServer()} + mk := func(cfg ServerConfig) *Manager { + cfg.Name, cfg.URL, cfg.Enabled = "fake", "http://example.test/mcp", true + m, err := NewManager(func(ServerConfig) (Poster, error) { return p, nil }, []ServerConfig{cfg}) + if err != nil { + t.Fatal(err) + } + m.Connect(context.Background()) + return m + } + m := mk(ServerConfig{AllowTools: []string{"read_thing"}}) + defer m.Close() + if got := m.Tools(); len(got) != 1 || got[0].Name != "read_thing" { + t.Fatalf("allow_tools ignored: %+v", got) + } + if _, err := m.Call(context.Background(), "fake", "break_thing", nil); err == nil { + t.Error("a tool excluded by allow_tools must be unreachable") + } + // Over the cap with no allow_tools: NOTHING is taken. Trimming a sorted + // list handed the server the choice of which tools survive — a new tool + // named "aaa_" would push an already-approved one out of the catalogue. + m2 := mk(ServerConfig{MaxTools: 1}) + defer m2.Close() + if got := m2.Tools(); len(got) != 0 { + t.Fatalf("over the cap without allow_tools must contribute nothing, got %+v", got) + } + // With allow_tools, Kami chose the list, so the cap trims his list. + m3 := mk(ServerConfig{MaxTools: 1, AllowTools: []string{"break_thing", "read_thing"}}) + defer m3.Close() + if got := m3.Tools(); len(got) != 1 || got[0].Name != "break_thing" { + t.Fatalf("max_tools over allow_tools: %+v", got) + } +} + +func TestManagerURLServerWithoutHTTPDoor(t *testing.T) { + m, err := NewManager(nil, []ServerConfig{{Name: "fake", URL: "http://example.test/mcp", Enabled: true}}) + if err != nil { + t.Fatal(err) + } + m.Connect(context.Background()) + st := m.Status() + if len(st) != 1 || st[0].Connected || st[0].Err == "" { + t.Fatalf("a url server with no poster must be recorded as failed: %+v", st) + } +} + +func TestManagerReconnectAfterFailure(t *testing.T) { + var mu sync.Mutex + fail := true + m, err := NewManager(func(ServerConfig) (Poster, error) { + mu.Lock() + defer mu.Unlock() + if fail { + return nil, fmt.Errorf("down") + } + return &fakePoster{handler: echoServer()}, nil + }, []ServerConfig{{Name: "fake", URL: "http://example.test/mcp", Enabled: true}}) + if err != nil { + t.Fatal(err) + } + defer m.Close() + m.Connect(context.Background()) + if m.Status()[0].Connected { + t.Fatal("should be down") + } + mu.Lock() + fail = false + mu.Unlock() + // Refresh honours the backoff, so pretend the last attempt was long ago. + m.mu.Lock() + m.conns["fake"].lastTry = time.Now().Add(-2 * DefaultReconnectEvery) + m.mu.Unlock() + m.Refresh(context.Background()) + if !m.Status()[0].Connected { + t.Fatalf("should have reconnected: %+v", m.Status()) + } +} + +func TestLocalNameAndCmd(t *testing.T) { + cases := [][3]string{ + {"vikunja", "list_tasks", "vikunja_list_tasks"}, + {"Vikunja", "Get Task Details", "vikunja_get_task_details"}, + {"fs", "read-file", "fs_read_file"}, + {"", "search", "search"}, + } + for _, c := range cases { + if got := LocalName(c[0], c[1]); got != c[2] { + t.Errorf("LocalName(%q,%q) = %q want %q", c[0], c[1], got, c[2]) + } + } + server, tool, ok := ParseCmd(Cmd("vikunja", "list_tasks")) + if !ok || server != "vikunja" || tool != "list_tasks" { + t.Fatalf("ParseCmd round-trip: %q %q %v", server, tool, ok) + } + for _, bad := range [][]string{nil, {"systemctl", "restart", "nginx"}, {"mcp", "vikunja"}, {"mcp", "", "x"}} { + if _, _, ok := ParseCmd(bad); ok { + t.Errorf("ParseCmd(%v) must not claim an ordinary tool row", bad) + } + } + if Scope("vikunja") != "mcp:vikunja" { + t.Error("scope") + } +} + +func TestBindPositional(t *testing.T) { + cases := []struct { + name, schema string + args []string + mutating bool + want map[string]any + wantErr bool + }{ + { + name: "no required runs with nothing", + // A spare tail is fine: "покажи проекты пожалуйста" still lists them. + schema: `{"type":"object","properties":{},"required":[]}`, + args: []string{"пожалуйста"}, + want: map[string]any{}, + }, + { + name: "empty schema", + schema: ``, + want: map[string]any{}, + }, + { + name: "one required string gets the tail", + schema: `{"properties":{"q":{"type":"string"}},"required":["q"]}`, + args: []string{"почему", "небо", "синее"}, + want: map[string]any{"q": "почему небо синее"}, + }, + { + name: "one required string with no tail", + schema: `{"properties":{"q":{"type":"string"}},"required":["q"]}`, + wantErr: true, + }, + { + name: "one required integer parses", + schema: `{"properties":{"task_id":{"type":"integer"}},"required":["task_id"]}`, + args: []string{"251"}, + want: map[string]any{"task_id": float64(251)}, + }, + { + name: "one required integer with words", + schema: `{"properties":{"task_id":{"type":"integer"}},"required":["task_id"]}`, + args: []string{"двести", "пятьдесят", "один"}, + wantErr: true, + }, + { + name: "two required is refused rather than guessed", + schema: `{"properties":{"a":{"type":"string"},"b":{"type":"string"}},"required":["a","b"]}`, + args: []string{"что-то"}, + wantErr: true, + }, + { + name: "one required object is refused", + schema: `{"properties":{"payload":{"type":"object"}},"required":["payload"]}`, + args: []string{"что-то"}, + wantErr: true, + }, + { + // Learned from Vikunja's update_task: required ["task_id"], every + // other field optional, so one guessed argument blanks the rest. + name: "one required on a mutating tool is refused", + schema: `{"properties":{"task_id":{"type":"integer"}},"required":["task_id"]}`, + args: []string{"251"}, + mutating: true, + wantErr: true, + }, + { + // Nothing was guessed, so there is nothing to get wrong. It still + // goes through the confirm turn upstream. + name: "no required on a mutating tool still runs", + schema: `{"properties":{},"required":[]}`, + mutating: true, + want: map[string]any{}, + }, + { + name: "unreadable schema", + schema: `not json`, + wantErr: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := bindPositional(json.RawMessage(tc.schema), tc.args, !tc.mutating) + if tc.wantErr { + if err == nil { + t.Fatalf("want an error, got %v", got) + } + return + } + if err != nil { + t.Fatal(err) + } + if fmt.Sprint(got) != fmt.Sprint(tc.want) { + t.Fatalf("got %v want %v", got, tc.want) + } + }) + } +} + +func TestCallPositionalThroughManager(t *testing.T) { + p := &fakePoster{handler: echoServer()} + m, err := NewManager(func(ServerConfig) (Poster, error) { return p, nil }, + []ServerConfig{{Name: "fake", URL: "http://example.test/mcp", Enabled: true}}) + if err != nil { + t.Fatal(err) + } + m.Connect(context.Background()) + defer m.Close() + // echoServer's tools declare no required properties. + out, err := m.CallPositional(context.Background(), "fake", "read_thing", []string{"хвост"}) + if err != nil { + t.Fatalf("call: %v", err) + } + if out != "read_thing:" { + t.Fatalf("out = %q", out) + } + if _, err := m.CallPositional(context.Background(), "fake", "absent", nil); err == nil { + t.Error("an unknown tool must be refused") + } +} + +// A guessed argument may only be bound for a tool Kami named in allow_tools. +// readOnlyHint alone was the old rule, and readOnlyHint is written by the same +// server that named the tool: a server advertising delete_project as read-only +// got an unconfirmed argument-carrying call. +func TestBindPositionalNeedsAllowTools(t *testing.T) { + schema := json.RawMessage(`{"required":["query"],"properties":{"query":{"type":"string"}}}`) + if _, err := bindPositional(schema, []string{"tea"}, false); !errors.Is(err, ErrNeedsArgs) { + t.Fatalf("err = %v, want ErrNeedsArgs when the tool is not in allow_tools", err) + } + got, err := bindPositional(schema, []string{"tea"}, true) + if err != nil || got["query"] != "tea" { + t.Fatalf("bind = %v, %v", got, err) + } +} + +// required names a property the schema never describes. Falling through to the +// zero value made it a string, which is a guess about a guess. +func TestBindPositionalRefusesUndescribedProperty(t *testing.T) { + schema := json.RawMessage(`{"required":["query"],"properties":{}}`) + _, err := bindPositional(schema, []string{"tea"}, true) + if !errors.Is(err, ErrNeedsArgs) { + t.Fatalf("err = %v, want ErrNeedsArgs", err) + } + if !strings.Contains(err.Error(), "never describes") { + t.Fatalf("err = %v, want it to name the schema gap", err) + } +} + +// A server that cannot be dialled must be retried more and more slowly. At a +// flat one minute a permanently misconfigured stdio server was re-exec'd 1440 +// times a day forever. +func TestReconnectBackoffGrows(t *testing.T) { + c := &conn{} + prev := time.Duration(0) + for i := 1; i <= 12; i++ { + c.fails = i + d := c.backoff() + if d < prev { + t.Fatalf("backoff shrank at %d failures: %v after %v", i, d, prev) + } + if d > MaxReconnectEvery { + t.Fatalf("backoff %v exceeds the cap %v", d, MaxReconnectEvery) + } + prev = d + } + if prev != MaxReconnectEvery { + t.Fatalf("backoff never reached the cap: %v", prev) + } + c.fails = 1 + if c.backoff() != DefaultReconnectEvery { + t.Fatalf("first retry = %v, want %v", c.backoff(), DefaultReconnectEvery) + } +} diff --git a/internal/mcp/stdio.go b/internal/mcp/stdio.go new file mode 100644 index 0000000..86e60e3 --- /dev/null +++ b/internal/mcp/stdio.go @@ -0,0 +1,217 @@ +package mcp + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "sync" +) + +// maxLine bounds one JSON-RPC frame from a subprocess. A tool result bigger +// than this is a misbehaving server, not something to buffer. +// +// The bound is enforced by bufio.Scanner's own buffer limit, not by measuring +// the line after it was assembled. Measuring afterwards is not a bound: a +// server that emits 500 MB with no newline would have all 500 MB in mavend's +// heap before the check could reject it, which on the deploy target is an OOM +// kill of the core daemon. +const maxLine = 1 << 20 // 1 MiB + +// stdioTransport speaks newline-delimited JSON-RPC to a child process. This is +// the local transport: the server runs on this box, under this user, and gets +// no network guard because it never touches the network on our behalf. +// +// Args are argv, never a shell string — the same discipline internal/tool +// keeps, for the same reason. +// +// Reading happens on its own goroutine, feeding frames down a channel. That is +// what makes a call abandonable: bufio never observes a context, so a server +// that accepts a request and then writes nothing at all would otherwise block +// the reader forever with the transport lock held, and every other server in +// the manager behind it. +type stdioTransport struct { + cmd *exec.Cmd + in io.WriteCloser + lines chan []byte + stop chan struct{} // closed by Close, so the reader can give up + + // callMu serialises whole calls, so two callers cannot consume each + // other's frames off the shared channel. It is deliberately NOT the lock + // alive() takes: a hung call must not make the manager's health check + // block on it. + callMu sync.Mutex + + mu sync.Mutex + dead bool + readErr error +} + +func newStdioTransport(ctx context.Context, argv []string, env []string, dir string) (*stdioTransport, error) { + if len(argv) == 0 { + return nil, errors.New("mcp: stdio server needs a command") + } + cmd := exec.Command(argv[0], argv[1:]...) + cmd.Dir = dir + if len(env) > 0 { + cmd.Env = append(os.Environ(), env...) + } + cmd.Stderr = os.Stderr + in, err := cmd.StdinPipe() + if err != nil { + return nil, fmt.Errorf("mcp: stdin pipe: %w", err) + } + out, err := cmd.StdoutPipe() + if err != nil { + return nil, fmt.Errorf("mcp: stdout pipe: %w", err) + } + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("mcp: start %q: %w", argv[0], err) + } + t := &stdioTransport{ + cmd: cmd, + in: in, + lines: make(chan []byte), + stop: make(chan struct{}), + } + go t.readLoop(out) + return t, nil +} + +// readLoop pushes one frame per line onto t.lines until the pipe ends. The +// scanner's own buffer limit is the frame bound: a line longer than maxLine +// ends the scan with bufio.ErrTooLong having buffered at most maxLine, rather +// than assembling the whole thing first and rejecting it afterwards. +func (t *stdioTransport) readLoop(out io.Reader) { + defer close(t.lines) + sc := bufio.NewScanner(out) + sc.Buffer(make([]byte, 0, 64<<10), maxLine) + for sc.Scan() { + line := bytes.TrimSpace(sc.Bytes()) + if len(line) == 0 { + continue + } + frame := append([]byte(nil), line...) + select { + case t.lines <- frame: + case <-t.stop: + return + } + } + err := sc.Err() + switch { + case errors.Is(err, bufio.ErrTooLong): + err = fmt.Errorf("mcp: frame exceeds %d bytes", maxLine) + case err == nil: + err = io.EOF + } + t.mu.Lock() + t.readErr = err + t.mu.Unlock() +} + +func (t *stdioTransport) Call(ctx context.Context, req *rpcRequest) (*rpcResponse, error) { + t.callMu.Lock() + defer t.callMu.Unlock() + if !t.alive() { + return nil, ErrClosed + } + t.mu.Lock() + err := t.write(req) + t.mu.Unlock() + if err != nil { + _ = t.Close() + return nil, err + } + // Read until the frame with our id turns up; anything else on the pipe is + // a notification or a server-initiated request we do not answer. + for { + select { + case <-ctx.Done(): + // A server that took the request and answered nothing is not a + // server this connection can be reused with: the next call would + // read into a pipe whose state we no longer know. Drop it and let + // the manager re-dial. + _ = t.Close() + return nil, ctx.Err() + case line, ok := <-t.lines: + if !ok { + t.mu.Lock() + rerr := t.readErr + t.mu.Unlock() + _ = t.Close() + if rerr == nil { + rerr = ErrClosed + } + return nil, fmt.Errorf("mcp: read: %w", rerr) + } + var resp rpcResponse + if err := json.Unmarshal(line, &resp); err != nil { + continue // not a response frame; ignore rather than break the turn + } + if resp.ID == nil || *resp.ID != req.ID { + continue + } + return &resp, nil + } + } +} + +func (t *stdioTransport) Notify(ctx context.Context, method string, params any) error { + t.mu.Lock() + defer t.mu.Unlock() + if t.dead { + return ErrClosed + } + return t.write(&rpcRequest{JSONRPC: "2.0", Method: method, Params: params}) +} + +// write must be called with t.mu held. +func (t *stdioTransport) write(req *rpcRequest) error { + req.JSONRPC = "2.0" + raw, err := json.Marshal(req) + if err != nil { + return err + } + if _, err := t.in.Write(append(raw, '\n')); err != nil { + return fmt.Errorf("mcp: write %s: %w", req.Method, err) + } + return nil +} + +// Close is idempotent: a call that abandoned a silent pipe calls it, and so +// does the manager. +func (t *stdioTransport) Close() error { + t.mu.Lock() + if t.dead { + t.mu.Unlock() + return nil + } + t.dead = true + close(t.stop) + if t.in != nil { + _ = t.in.Close() + } + t.mu.Unlock() + if t.cmd.Process != nil { + _ = t.cmd.Process.Kill() + _ = t.cmd.Wait() + } + return nil +} + +// alive reports whether the transport can still carry a call. The manager uses +// it to decide on a reconnect instead of retrying into a dead pipe. +func (t *stdioTransport) alive() bool { + t.mu.Lock() + defer t.mu.Unlock() + if t.dead { + return false + } + return t.readErr == nil +} diff --git a/internal/mcp/stdio_test.go b/internal/mcp/stdio_test.go new file mode 100644 index 0000000..b75b47d --- /dev/null +++ b/internal/mcp/stdio_test.go @@ -0,0 +1,216 @@ +package mcp + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "strings" + "testing" + "time" +) + +// The stdio transport is tested against a real subprocess — this test binary, +// re-executed with MAVEN_MCP_FAKE set, acting as a minimal MCP server. No +// python, no fixture file, no network. +func TestMain(m *testing.M) { + if os.Getenv("MAVEN_MCP_FAKE") != "" { + fakeStdioServer() + return + } + os.Exit(m.Run()) +} + +func fakeStdioServer() { + h := echoServer() + sc := bufio.NewScanner(os.Stdin) + out := bufio.NewWriter(os.Stdout) + defer out.Flush() + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if line == "" { + continue + } + var req struct { + ID *int64 `json:"id"` + Method string `json:"method"` + Params json.RawMessage `json:"params"` + } + if json.Unmarshal([]byte(line), &req) != nil { + continue + } + if req.ID == nil { + // A notification gets no reply, but we emit an unrelated + // notification so the client's frame-skipping is exercised. + _, _ = out.WriteString("{\"jsonrpc\":\"2.0\",\"method\":\"notifications/message\"}\n") + _ = out.Flush() + continue + } + // "mute" answers the handshake and then goes silent: a python server + // that hit an unhandled exception in its own read loop but did not + // exit is the ordinary way to get here. + if os.Getenv("MAVEN_MCP_FAKE") == "mute" && req.Method == "tools/call" { + select {} // never answer, never exit + } + // "flood" writes one enormous line with no newline in it. + if os.Getenv("MAVEN_MCP_FAKE") == "flood" && req.Method == "tools/call" { + for i := 0; i < 64; i++ { + _, _ = out.Write(make([]byte, 1<<20)) + } + _ = out.Flush() + continue + } + result, rerr := h(req.Method, req.Params) + resp := map[string]any{"jsonrpc": "2.0", "id": *req.ID} + if rerr != nil { + resp["error"] = map[string]any{"code": rerr.Code, "message": rerr.Message} + } else { + resp["result"] = result + } + raw, _ := json.Marshal(resp) + _, _ = out.Write(append(raw, '\n')) + _ = out.Flush() + if os.Getenv("MAVEN_MCP_FAKE") == "die" && req.Method == "tools/list" { + return // hang up, so the reconnect path has something to see + } + } +} + +func stdioManager(t *testing.T, mode string) *Manager { + t.Helper() + self, err := os.Executable() + if err != nil { + t.Skipf("no executable path: %v", err) + } + if _, err := exec.LookPath(self); err != nil && !strings.Contains(self, "/") { + t.Skip("test binary not executable") + } + m, err := NewManager(nil, []ServerConfig{{ + Name: "fake", + Command: self, + Env: []string{"MAVEN_MCP_FAKE=" + mode}, + Enabled: true, + }}) + if err != nil { + t.Fatal(err) + } + m.Connect(context.Background()) + return m +} + +func TestStdioTransportEndToEnd(t *testing.T) { + m := stdioManager(t, "1") + defer m.Close() + st := m.Status() + if len(st) != 1 || !st[0].Connected { + t.Fatalf("status = %+v", st) + } + if st[0].Transport != "stdio" { + t.Fatalf("transport = %q", st[0].Transport) + } + if got := len(m.Tools()); got != 2 { + t.Fatalf("tools = %d", got) + } + out, err := m.Call(context.Background(), "fake", "read_thing", map[string]any{"q": "стдио"}) + if err != nil { + t.Fatalf("call: %v", err) + } + if out != "read_thing:стдио" { + t.Fatalf("out = %q", out) + } + res := m.Resources(context.Background()) + if len(res) != 1 || res[0].URI != "note://one" { + t.Fatalf("resources = %+v", res) + } + body, err := m.ReadResource(context.Background(), "fake", "note://one") + if err != nil { + t.Fatal(err) + } + if body != "тело ресурса" { + t.Fatalf("body = %q", body) + } +} + +func TestStdioServerThatDiesIsNotUsable(t *testing.T) { + m := stdioManager(t, "die") + defer m.Close() + // The server hung up after tools/list; the next call must fail cleanly + // rather than hang or panic. + if _, err := m.Call(context.Background(), "fake", "read_thing", nil); err == nil { + t.Fatal("a call into a dead server must error") + } +} + +func TestStdioMissingCommand(t *testing.T) { + m, err := NewManager(nil, []ServerConfig{{ + Name: "nope", Command: "/nonexistent/mcp-server-that-is-not-there", Enabled: true, + }}) + if err != nil { + t.Fatal(err) + } + m.Connect(context.Background()) + st := m.Status() + if st[0].Connected || st[0].Err == "" { + t.Fatalf("a missing binary must be recorded, not fatal: %+v", st) + } + if got := len(m.Tools()); got != 0 { + t.Fatalf("tools = %d", got) + } + if !strings.Contains(fmt.Sprint(st[0].Err), "start") { + t.Logf("err = %q", st[0].Err) + } +} + +// A stdio server that accepts a call and then answers nothing must not wedge +// the manager. Before the read moved onto its own goroutine, the read held the +// transport lock, Refresh took that lock through alive() while holding the +// manager lock, and from then on Tools, Status and Call blocked for EVERY +// server — including turns that touch no MCP tool at all. +func TestStdioSilentServerDoesNotWedgeTheManager(t *testing.T) { + m := stdioManager(t, "mute") + defer m.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer cancel() + done := make(chan error, 1) + go func() { + _, err := m.Call(ctx, "fake", "read_thing", nil) + done <- err + }() + select { + case err := <-done: + if err == nil { + t.Fatal("a call into a silent server must fail, not succeed") + } + case <-time.After(5 * time.Second): + t.Fatal("the call never returned: the context is not observed during the read") + } + + // The manager must still answer while (and after) that call was stuck. + ready := make(chan struct{}) + go func() { + m.Refresh(context.Background()) + m.Tools() + m.Status() + close(ready) + }() + select { + case <-ready: + case <-time.After(5 * time.Second): + t.Fatal("Refresh/Tools/Status deadlocked behind the hung call") + } +} + +// One frame is bounded by the reader's buffer, not measured after the whole +// thing has already been assembled in mavend's heap. +func TestStdioOversizedFrameIsRefused(t *testing.T) { + m := stdioManager(t, "flood") + defer m.Close() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if _, err := m.Call(ctx, "fake", "read_thing", nil); err == nil { + t.Fatal("a 64 MiB frame must be refused") + } +} diff --git a/internal/mcp/webfetchdoor.go b/internal/mcp/webfetchdoor.go new file mode 100644 index 0000000..2623592 --- /dev/null +++ b/internal/mcp/webfetchdoor.go @@ -0,0 +1,56 @@ +package mcp + +import ( + "context" + "fmt" + + "github.com/kami/maven/internal/webfetch" +) + +// WebfetchDoor builds the PosterFactory used in production: one guarded +// webfetch.Fetcher per url server, with that server's allow_private and the +// shared host lists and limits. +// +// One fetcher PER server is the point. allow_private is a hole in the +// private-address guard, and a hole punched for the Vikunja server on loopback +// must not become a hole for some public endpoint that happens to redirect at +// the LAN. Rate limiting is per fetcher too, which is the right shape here: +// separate servers are separate hosts. +// +// A server WITH allow_private also gets redirects switched off. Across servers +// the per-fetcher split holds the line; within the one server that has the +// flag it did not, because allow_private disables the dialer guard on every +// hop: http://localhost:9100/mcp answering 302 to +// http://169.254.169.254/latest/meta-data/ was followed, up to MaxRedirects. A +// local MCP endpoint has no business redirecting, so refusing costs nothing. +func WebfetchDoor(limits webfetch.Config) PosterFactory { + return func(cfg ServerConfig) (Poster, error) { + c := limits + c.AllowPrivate = cfg.AllowPrivate + if cfg.AllowPrivate { + c.MaxRedirects = -1 // negative ⇒ no redirects followed + } + if c.Timeout <= 0 && cfg.Timeout > 0 { + c.Timeout = cfg.Timeout + } + return fetcherPoster{webfetch.New(c)}, nil + } +} + +// fetcherPoster adapts webfetch.Fetcher to Poster. It exists so this package +// does not have to know webfetch's Response type, and so a test can substitute +// a fake without a listener. +type fetcherPoster struct{ f *webfetch.Fetcher } + +func (p fetcherPoster) Post(ctx context.Context, rawURL, contentType string, body []byte, hdr map[string]string) (*PostResponse, error) { + resp, err := p.f.Post(ctx, rawURL, contentType, body, hdr) + if err != nil { + return nil, fmt.Errorf("mcp: post %s: %w", rawURL, err) + } + return &PostResponse{ + Status: resp.Status, + ContentType: resp.ContentType, + Body: resp.Body, + Header: resp.Header, + }, nil +} diff --git a/internal/media/image.go b/internal/media/image.go new file mode 100644 index 0000000..905eccc --- /dev/null +++ b/internal/media/image.go @@ -0,0 +1,225 @@ +package media + +import ( + "bytes" + "encoding/base64" + "errors" + "fmt" + "image" + "image/gif" + "image/jpeg" + "image/png" + "strings" +) + +// DefaultMaxDim — the longest edge an image is scaled down to before it goes to +// a vision model. 896 is the tile size the current crop of small +// vision-language models (Qwen2.5-VL, SmolVLM, moondream) work in; sending a +// 12-megapixel phone photo instead just costs the box minutes of prefill for +// tiles that get pooled away anyway. +const DefaultMaxDim = 896 + +// JPEGQuality for the re-encode. 85 is the usual "no visible artefacts" point, +// and the re-encode exists to shrink the payload, not to archive it — the +// original bytes stay in the blob store untouched. +const JPEGQuality = 85 + +// DefaultMaxPixels — the largest source image this build will decode, counted +// in pixels rather than in compressed bytes. A byte cap is not a memory bound +// for an image: a 20000x20000 PNG of flat colour compresses to a few hundred +// kilobytes and decodes to 400 million pixels, which is 1.6 GB of heap in the +// process that owns the database and the socket. 40 megapixels is well past any +// phone camera and two orders of magnitude short of an OOM. +const DefaultMaxPixels = 40 << 20 + +// ErrTooManyPixels — the image header declares more pixels than this build +// will decode. Separate from ErrUnsupportedImage because the format is fine and +// the size is not, and the log line should say which. +var ErrTooManyPixels = errors.New("media: image has too many pixels") + +// ErrUnsupportedImage — the bytes are not an image format this build can +// decode. Notably webp: the stdlib has no webp decoder and this repo takes no +// new dependencies, so a webp arriving from Telegram is refused here with a +// clear error rather than handed to a model as garbage. +var ErrUnsupportedImage = errors.New("media: unsupported image format") + +// SniffImage identifies image bytes by magic number and returns the mime. It +// exists because a caller-declared content type is a claim, and the store's file +// extension (and the vision provider's data URI) should follow the bytes. +// +// Returns ErrUnsupportedImage for anything unrecognised, including webp — which +// is recognised well enough to name in the error, so the log says "webp is not +// supported" instead of "not an image". +func SniffImage(data []byte) (string, error) { + switch { + case len(data) >= 3 && data[0] == 0xFF && data[1] == 0xD8 && data[2] == 0xFF: + return "image/jpeg", nil + case len(data) >= 8 && string(data[:8]) == "\x89PNG\r\n\x1a\n": + return "image/png", nil + case len(data) >= 6 && (string(data[:6]) == "GIF87a" || string(data[:6]) == "GIF89a"): + return "image/gif", nil + case len(data) >= 12 && string(data[:4]) == "RIFF" && string(data[8:12]) == "WEBP": + return "", fmt.Errorf("%w: webp (no decoder in this build)", ErrUnsupportedImage) + } + return "", ErrUnsupportedImage +} + +// Image — an image prepared for a vision model: JPEG bytes, downscaled, with +// the dimensions it ended up at. It is deliberately a separate type from Blob: +// a Blob is what he sent, an Image is what the model sees, and the two are not +// the same bytes. +type Image struct { + JPEG []byte + Width int + Height int + // Source names where the original came from ("telegram", "web:upload"), + // carried through only so a log line can say what was looked at. + Source string +} + +// DataURI renders the image as a `data:image/jpeg;base64,...` URI, which is how +// every OpenAI-compatible multimodal endpoint takes an image. The string is +// large (roughly 4/3 of the JPEG); nothing caches it. +func (im Image) DataURI() string { + return "data:image/jpeg;base64," + base64.StdEncoding.EncodeToString(im.JPEG) +} + +// PrepareImage decodes data, scales it so its longest edge is at most maxDim +// (never up — a small image is left alone), and re-encodes it as JPEG. +// maxDim ≤ 0 ⇒ DefaultMaxDim. +// +// An image with an alpha channel is composited onto white rather than having +// alpha dropped to black, because the common case is a screenshot or a +// transparent-background diagram, and text on black-on-black is unreadable to +// the model for no reason. +func PrepareImage(data []byte, source string, maxDim int) (Image, error) { + if len(data) == 0 { + return Image{}, ErrEmpty + } + if maxDim <= 0 { + maxDim = DefaultMaxDim + } + mime, err := SniffImage(data) + if err != nil { + return Image{}, err + } + // The header is read before the pixels. Deciding after the decode is not a + // decision: by then the whole bitmap is already in the heap. + cfg, err := decodeConfig(data, mime) + if err != nil { + return Image{}, fmt.Errorf("media: read %s header: %w", mime, err) + } + if px := int64(cfg.Width) * int64(cfg.Height); px > DefaultMaxPixels { + return Image{}, fmt.Errorf("%w: %dx%d is %d, cap is %d", + ErrTooManyPixels, cfg.Width, cfg.Height, px, int64(DefaultMaxPixels)) + } + src, err := decode(data, mime) + if err != nil { + return Image{}, fmt.Errorf("media: decode %s: %w", mime, err) + } + + dst := flattenAndScale(src, maxDim) + var buf bytes.Buffer + if err := jpeg.Encode(&buf, dst, &jpeg.Options{Quality: JPEGQuality}); err != nil { + return Image{}, fmt.Errorf("media: encode jpeg: %w", err) + } + b := dst.Bounds() + return Image{JPEG: buf.Bytes(), Width: b.Dx(), Height: b.Dy(), Source: source}, nil +} + +func decodeConfig(data []byte, mime string) (image.Config, error) { + r := bytes.NewReader(data) + switch strings.ToLower(mime) { + case "image/jpeg": + return jpeg.DecodeConfig(r) + case "image/png": + return png.DecodeConfig(r) + case "image/gif": + return gif.DecodeConfig(r) + } + return image.Config{}, ErrUnsupportedImage +} + +func decode(data []byte, mime string) (image.Image, error) { + r := bytes.NewReader(data) + switch strings.ToLower(mime) { + case "image/jpeg": + return jpeg.Decode(r) + case "image/png": + return png.Decode(r) + case "image/gif": + return gif.Decode(r) + } + return nil, ErrUnsupportedImage +} + +// flattenAndScale composites onto white and box-scales down to maxDim. The +// scaler is a plain area average over the source pixels mapping to each +// destination pixel — nearest-neighbour would alias small text into noise, +// which defeats the point of reading a screenshot, and an area average is a +// dozen lines against pulling in golang.org/x/image on an offline box. +// +// It reads the source through At and allocates only the destination. Flattening +// into a full-size RGBA first doubled the peak: a 40-megapixel photo already +// costs 160 MB decoded, and the intermediate made it 320 MB before MaxDim had +// any chance to help. +func flattenAndScale(src image.Image, maxDim int) *image.RGBA { + sb := src.Bounds() + sw, sh := sb.Dx(), sb.Dy() + dw, dh := fit(sw, sh, maxDim) + + dst := image.NewRGBA(image.Rect(0, 0, dw, dh)) + for y := 0; y < dh; y++ { + y0, y1 := y*sh/dh, (y+1)*sh/dh + if y1 <= y0 { + y1 = y0 + 1 + } + for x := 0; x < dw; x++ { + x0, x1 := x*sw/dw, (x+1)*sw/dw + if x1 <= x0 { + x1 = x0 + 1 + } + var r, g, b, n uint64 + for sy := y0; sy < y1; sy++ { + for sx := x0; sx < x1; sx++ { + // At returns premultiplied 16-bit. Compositing over white + // is then c + (1-alpha), which is the same answer the + // draw.Over pass used to give, one pixel at a time. + cr, cg, cb, ca := src.At(sb.Min.X+sx, sb.Min.Y+sy).RGBA() + inv := uint64(0xFFFF - ca) + r += uint64(cr) + inv + g += uint64(cg) + inv + b += uint64(cb) + inv + n++ + } + } + o := dst.PixOffset(x, y) + dst.Pix[o] = uint8(r / n >> 8) + dst.Pix[o+1] = uint8(g / n >> 8) + dst.Pix[o+2] = uint8(b / n >> 8) + dst.Pix[o+3] = 0xFF + } + } + return dst +} + +// fit returns the largest w×h with the same aspect ratio whose longest edge is +// at most maxDim, never enlarging. Both edges are clamped to at least 1 so a +// 2000×1 strip does not scale to zero height. +func fit(w, h, maxDim int) (int, int) { + if w <= maxDim && h <= maxDim { + return w, h + } + if w >= h { + nh := h * maxDim / w + if nh < 1 { + nh = 1 + } + return maxDim, nh + } + nw := w * maxDim / h + if nw < 1 { + nw = 1 + } + return nw, maxDim +} diff --git a/internal/media/image_test.go b/internal/media/image_test.go new file mode 100644 index 0000000..fe2802b --- /dev/null +++ b/internal/media/image_test.go @@ -0,0 +1,260 @@ +package media + +import ( + "bytes" + "encoding/binary" + "errors" + "hash/crc32" + "image" + "image/color" + "image/gif" + "image/jpeg" + "image/png" + "strings" + "testing" +) + +// pngBytes builds a w×h test image: left half red, right half a light grey, so +// a downscale that averages produces a predictable mid value and a scaler that +// silently returns the wrong region is visible. +func pngBytes(t *testing.T, w, h int) []byte { + t.Helper() + img := image.NewRGBA(image.Rect(0, 0, w, h)) + for y := 0; y < h; y++ { + for x := 0; x < w; x++ { + if x < w/2 { + img.Set(x, y, color.RGBA{255, 0, 0, 255}) + } else { + img.Set(x, y, color.RGBA{200, 200, 200, 255}) + } + } + } + var buf bytes.Buffer + if err := png.Encode(&buf, img); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func TestSniffImage(t *testing.T) { + cases := []struct { + name string + data []byte + want string + }{ + {"png", pngBytes(t, 4, 4), "image/png"}, + {"jpeg", jpegBytes(t, 4, 4), "image/jpeg"}, + {"gif", gifBytes(t, 4, 4), "image/gif"}, + } + for _, c := range cases { + got, err := SniffImage(c.data) + if err != nil { + t.Errorf("%s: %v", c.name, err) + continue + } + if got != c.want { + t.Errorf("%s: got %q want %q", c.name, got, c.want) + } + } +} + +// webp is common from Telegram and there is no stdlib decoder, so it must be +// refused by name rather than mis-sniffed or fed to a model as noise. +func TestSniffRefusesWebpByName(t *testing.T) { + webp := append([]byte("RIFF\x00\x00\x00\x00WEBP"), make([]byte, 8)...) + _, err := SniffImage(webp) + if !errors.Is(err, ErrUnsupportedImage) { + t.Fatalf("got %v, want ErrUnsupportedImage", err) + } + if !strings.Contains(err.Error(), "webp") { + t.Errorf("error does not name the format: %v", err) + } +} + +func TestSniffRefusesGarbage(t *testing.T) { + for _, data := range [][]byte{nil, []byte("hello"), []byte("\x00\x01\x02\x03")} { + if _, err := SniffImage(data); !errors.Is(err, ErrUnsupportedImage) { + t.Errorf("SniffImage(%q) = %v", data, err) + } + } +} + +func TestPrepareImageDownscalesLongestEdge(t *testing.T) { + im, err := PrepareImage(pngBytes(t, 2000, 1000), "web:upload", 500) + if err != nil { + t.Fatalf("prepare: %v", err) + } + if im.Width != 500 || im.Height != 250 { + t.Errorf("got %dx%d, want 500x250", im.Width, im.Height) + } + if _, err := jpeg.Decode(bytes.NewReader(im.JPEG)); err != nil { + t.Errorf("output is not decodable jpeg: %v", err) + } + if im.Source != "web:upload" { + t.Errorf("source lost: %q", im.Source) + } +} + +// Tall images scale on the other axis; a scaler that only handles landscape is +// the classic version of this bug. +func TestPrepareImageHandlesPortrait(t *testing.T) { + im, err := PrepareImage(pngBytes(t, 400, 1600), "telegram", 800) + if err != nil { + t.Fatalf("prepare: %v", err) + } + if im.Height != 800 || im.Width != 200 { + t.Errorf("got %dx%d, want 200x800", im.Width, im.Height) + } +} + +func TestPrepareImageNeverEnlarges(t *testing.T) { + im, err := PrepareImage(pngBytes(t, 64, 32), "telegram", 896) + if err != nil { + t.Fatalf("prepare: %v", err) + } + if im.Width != 64 || im.Height != 32 { + t.Errorf("got %dx%d, want the original 64x32", im.Width, im.Height) + } +} + +// A degenerate strip must not scale to zero on the short axis — jpeg.Encode +// fails on a zero-height image, which would turn a weird screenshot into a +// hard error. +func TestPrepareImageClampsDegenerateAspect(t *testing.T) { + im, err := PrepareImage(pngBytes(t, 2000, 2), "web:upload", 100) + if err != nil { + t.Fatalf("prepare: %v", err) + } + if im.Height < 1 || im.Width != 100 { + t.Errorf("got %dx%d", im.Width, im.Height) + } +} + +// Transparent pixels composite onto white, not black: the common case is a +// screenshot or a diagram, and dark-on-black is unreadable to the model. +func TestPrepareImageFlattensAlphaOntoWhite(t *testing.T) { + img := image.NewRGBA(image.Rect(0, 0, 8, 8)) // fully transparent + var buf bytes.Buffer + if err := png.Encode(&buf, img); err != nil { + t.Fatal(err) + } + im, err := PrepareImage(buf.Bytes(), "web:upload", 8) + if err != nil { + t.Fatalf("prepare: %v", err) + } + decoded, err := jpeg.Decode(bytes.NewReader(im.JPEG)) + if err != nil { + t.Fatal(err) + } + r, g, b, _ := decoded.At(4, 4).RGBA() + if r>>8 < 240 || g>>8 < 240 || b>>8 < 240 { + t.Errorf("transparent pixel became rgb(%d,%d,%d), want near-white", r>>8, g>>8, b>>8) + } +} + +func TestPrepareImageRejectsEmpty(t *testing.T) { + if _, err := PrepareImage(nil, "x", 0); !errors.Is(err, ErrEmpty) { + t.Errorf("got %v, want ErrEmpty", err) + } +} + +func TestDataURIIsAJPEGDataURI(t *testing.T) { + im, err := PrepareImage(pngBytes(t, 16, 16), "x", 0) + if err != nil { + t.Fatal(err) + } + uri := im.DataURI() + if !strings.HasPrefix(uri, "data:image/jpeg;base64,") { + t.Fatalf("bad prefix: %.40s", uri) + } + if len(uri) <= len("data:image/jpeg;base64,") { + t.Error("data uri carries no payload") + } +} + +func jpegBytes(t *testing.T, w, h int) []byte { + t.Helper() + img := image.NewRGBA(image.Rect(0, 0, w, h)) + var buf bytes.Buffer + if err := jpeg.Encode(&buf, img, nil); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func gifBytes(t *testing.T, w, h int) []byte { + t.Helper() + img := image.NewPaletted(image.Rect(0, 0, w, h), []color.Color{color.Black, color.White}) + var buf bytes.Buffer + if err := gif.Encode(&buf, img, nil); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +// A decode bomb is a small file. Nothing bounded pixels before decoding, so a +// 20000x20000 PNG of flat colour — a few hundred kilobytes on the wire, well +// under the byte cap — decoded to 1.6 GB and then allocated another 1.6 GB to +// flatten, in the process that owns the database and the socket. +func TestPrepareImageRefusesADecodeBomb(t *testing.T) { + // The header is what is checked, so the test writes a real header and + // truncated pixel data: reaching the decode at all is the failure. + var buf bytes.Buffer + if err := png.Encode(&buf, image.NewGray(image.Rect(0, 0, 1, 1))); err != nil { + t.Fatal(err) + } + bomb := forgePNGSize(t, buf.Bytes(), 20000, 20000) + _, err := PrepareImage(bomb, "telegram", 0) + if !errors.Is(err, ErrTooManyPixels) { + t.Fatalf("err = %v, want ErrTooManyPixels", err) + } + // A phone photo is not a bomb. + if _, err := PrepareImage(pngBytes(t, 64, 48), "telegram", 0); err != nil { + t.Fatalf("an ordinary image was refused: %v", err) + } +} + +// forgePNGSize rewrites the IHDR width and height (and its CRC) of a valid PNG, +// which is how a header claiming 400 megapixels is produced without writing +// 400 megapixels. +func forgePNGSize(t *testing.T, src []byte, w, h uint32) []byte { + t.Helper() + out := append([]byte(nil), src...) + // 8 byte signature, 4 byte length, 4 byte "IHDR", then width and height. + const ihdr = 8 + 4 + 4 + binary.BigEndian.PutUint32(out[ihdr:], w) + binary.BigEndian.PutUint32(out[ihdr+4:], h) + crc := crc32.ChecksumIEEE(out[8+4 : ihdr+13]) + binary.BigEndian.PutUint32(out[ihdr+13:], crc) + return out +} + +// Transparency still composites onto white, which is what makes a screenshot +// readable. The old code did that with a full-size intermediate; the scaler +// walks the source instead and must give the same answer. +func TestPrepareImageFlattensOntoWhite(t *testing.T) { + img := image.NewRGBA(image.Rect(0, 0, 8, 8)) + // Fully transparent everywhere: over white, that is white. + data := encodePNG(t, img) + out, err := PrepareImage(data, "test", 4) + if err != nil { + t.Fatal(err) + } + dec, err := jpeg.Decode(bytes.NewReader(out.JPEG)) + if err != nil { + t.Fatal(err) + } + r, g, b, _ := dec.At(2, 2).RGBA() + if r>>8 < 240 || g>>8 < 240 || b>>8 < 240 { + t.Fatalf("transparent pixel came out %d,%d,%d, want white", r>>8, g>>8, b>>8) + } +} + +func encodePNG(t *testing.T, img image.Image) []byte { + t.Helper() + var buf bytes.Buffer + if err := png.Encode(&buf, img); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} diff --git a/internal/media/media.go b/internal/media/media.go new file mode 100644 index 0000000..d297557 --- /dev/null +++ b/internal/media/media.go @@ -0,0 +1,119 @@ +// Package media is the intake for everything Maven sees or hears that is not +// text: a photo he sends her, a meeting she was asked to record, a voice sample +// used to enrol a speaker. All three senses (vision, hearing, speaker +// recognition) share one problem — a blob arrives, it has to be stored, and +// something has to describe it — so the storing half lives here once instead of +// three times. +// +// # What this package is +// +// A content-addressed blob store on the local filesystem. Put returns a Blob +// keyed by the sha256 of its bytes, so the same photo sent twice is one file. +// Each blob gets a sidecar `.json` with its kind, mime, size, source and +// creation time; the sidecar is the whole index, because at personal scale a +// directory walk is cheaper than another sqlite table and the store has to be +// readable with `ls` when something goes wrong. +// +// Blobs are NOT in the sqlite database. The database is small, encrypted, and +// read on every tick; a 40 MB meeting recording has no business in it. What +// goes in the database is the *text* a blob produced — a transcript, a +// description — written as an ordinary note, which is the durable artefact and +// the only part worth recalling later. +// +// # Invariants (these are the point of the package, not decoration) +// +// - Nothing is captured that was not asked for. This package never records; +// it stores what a caller hands it, and every caller is an explicit act +// with a start and a stop. There is no ambient path in, and none may be +// added: see the refusal recorded in docs/plans/08-hearing.md. +// - A blob never leaves the box. No provider in this repo may upload one, and +// the vision provider refuses a non-private endpoint for exactly that +// reason (internal/vision). +// - A blob is never search input and never embedded. His photos and the audio +// of his meetings are not corpus. Only text derived from them, once he can +// see it as a note, participates in recall. +// - Storage is bounded. Retention is a config knob with a default, Prune +// enforces it, and an unpruned store is a bug: audio of people accumulating +// forever on disk is the failure mode this capability has to avoid. +// +// # Layout +// +// ///. the bytes +// ///.json the sidecar metadata +// +// `aa` is the first two hex chars of the digest — one fan-out level, enough to +// keep a directory listing usable after a few thousand blobs. +package media + +import ( + "errors" + "fmt" + "time" +) + +// Kind — what a blob is. Two values today; the kind is a directory name and a +// retention bucket, so adding a third is additive. +type Kind string + +const ( + // KindImage — a still image (png / jpeg / gif / webp bytes as received). + KindImage Kind = "image" + // KindAudio — raw PCM in the canonical internal/audio format, or a WAV + // container. Meeting captures and enrolment samples both land here. + KindAudio Kind = "audio" +) + +// Valid reports whether k is a kind this package will store. An unknown kind is +// refused at Put rather than creating a stray directory. +func (k Kind) Valid() bool { return k == KindImage || k == KindAudio } + +// Errors callers distinguish. ErrNotFound is the only one a caller usually +// handles; the rest mean the call was wrong. +var ( + // ErrNotFound — no blob with that id in this store. + ErrNotFound = errors.New("media: not found") + // ErrEmpty — Put was handed zero bytes. Storing an empty capture would + // leave a sidecar claiming a recording exists when it does not. + ErrEmpty = errors.New("media: empty payload") + // ErrTooLarge — the payload is over the store's cap. The cap exists so a + // runaway capture cannot fill the disk that mavend's database lives on. + ErrTooLarge = errors.New("media: payload too large") + // ErrBadKind — unknown Kind. + ErrBadKind = errors.New("media: unknown kind") + // ErrBadID — the id is not a 64-char lowercase hex digest, so it cannot + // have come from this store and must not be turned into a path. + ErrBadID = errors.New("media: malformed id") +) + +// Blob — one stored item. ID is the sha256 of the bytes in lowercase hex, which +// makes it both the primary key and the dedupe mechanism. Path is absolute and +// local; it is a debugging affordance and the argument a subprocess (whisper, +// llama-server) is pointed at, never something handed to a network client. +type Blob struct { + ID string `json:"id"` + Kind Kind `json:"kind"` + MIME string `json:"mime"` + Size int64 `json:"size"` + Source string `json:"source"` // provenance: "telegram", "web:upload", "capture:meeting", "enroll" + Created time.Time `json:"created"` // UTC + Path string `json:"-"` // filled by the store; not part of the sidecar +} + +// Age is how long ago the blob was stored, measured against now. Prune uses it; +// it is exported because the /media surface will want to show it. +func (b Blob) Age(now time.Time) time.Duration { return now.Sub(b.Created) } + +// String is a one-line summary for logs. Deliberately does not include Path: +// a log line is not the place to spell out where his meeting audio lives. +func (b Blob) String() string { + return fmt.Sprintf("%s %s %dB from %s", b.Kind, shortID(b.ID), b.Size, b.Source) +} + +// shortID trims a digest to something readable in a log line. Twelve hex chars +// is unambiguous at personal scale and short enough to fit next to the rest. +func shortID(id string) string { + if len(id) <= 12 { + return id + } + return id[:12] +} diff --git a/internal/media/store.go b/internal/media/store.go new file mode 100644 index 0000000..32d3fc4 --- /dev/null +++ b/internal/media/store.go @@ -0,0 +1,663 @@ +package media + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" +) + +// DefaultMaxBytes — the per-blob cap when a store is built without one. 64 MiB +// is about an hour of 16 kHz mono PCM, which is also the hearing capture's own +// ceiling; a single item bigger than that is a mistake, not a meeting. +const DefaultMaxBytes int64 = 64 << 20 + +// DefaultMaxAudioBytes — the per-blob cap for audio. Separate from +// DefaultMaxBytes because the two kinds are not the same size of thing: an +// image over 64 MiB is a mistake, and a two-hour meeting at 16 kHz mono is +// about 230 MB of PCM by design. With one shared cap, capture's own +// DefaultMaxDuration of two hours and this store's 64 MiB contradicted each +// other, and the meeting that hit the limit was the one that failed to store. +const DefaultMaxAudioBytes int64 = 512 << 20 + +// DefaultRetention — how long a blob is kept when no retention is configured. +// Seven days is long enough to re-run a transcription that came out wrong and +// short enough that "she has a month of my meetings on disk" is never true. +const DefaultRetention = 7 * 24 * time.Hour + +// DefaultMaxTotalBytes — the whole-store budget when one is not configured. The +// per-blob cap bounds one call and nothing bounded the sum of them: 64 MiB per +// call, an unlimited number of calls, and a seven-day window fills the disk +// mavend's database lives on. Content addressing does not help, because one +// flipped pixel is a different digest. 4 GiB is roughly sixty meetings or a few +// thousand photos inside the window. +const DefaultMaxTotalBytes int64 = 4 << 30 + +// ErrStoreFull — the store is at its total-bytes budget. Distinct from +// ErrTooLarge: the payload is a reasonable size and there is no room for it, so +// the answer is to prune or raise the budget, not to send something smaller. +var ErrStoreFull = errors.New("media: store is full") + +// Store — a content-addressed blob directory. Zero value is not usable; build +// one with Open, which creates the directory 0700. The store holds no lock and +// no cache: every operation is a filesystem call, and two writers of the same +// bytes produce the same file, so concurrent Puts do not need coordinating. +type Store struct { + dir string + maxBytes int64 + maxAudio int64 + maxTotal int64 + retention time.Duration + now func() time.Time + + // total is the running sum of stored blob bytes, seeded by Open with a + // directory walk and kept up to date by Put, Delete and Prune. It is a + // cache of something the filesystem already knows: re-walking on every Put + // would be correct too and would make an image intake O(store size). + totalMu sync.Mutex + total int64 +} + +// Open prepares a blob store rooted at dir. maxBytes ≤ 0 ⇒ DefaultMaxBytes; +// retention ≤ 0 ⇒ DefaultRetention. The directory (and every kind subdirectory +// created later) is 0700: these are recordings of people, and the daemon's user +// is the only reader. +func Open(dir string, maxBytes int64, retention time.Duration) (*Store, error) { + return OpenWithBudget(dir, maxBytes, 0, retention) +} + +// OpenWithBudget is Open with the whole-store budget spelled out. maxTotal ≤ 0 +// ⇒ DefaultMaxTotalBytes. +func OpenWithBudget(dir string, maxBytes, maxTotal int64, retention time.Duration) (*Store, error) { + if strings.TrimSpace(dir) == "" { + return nil, errors.New("media: empty dir") + } + abs, err := filepath.Abs(dir) + if err != nil { + return nil, fmt.Errorf("media: resolve dir: %w", err) + } + if err := os.MkdirAll(abs, 0o700); err != nil { + return nil, fmt.Errorf("media: create dir: %w", err) + } + if maxBytes <= 0 { + maxBytes = DefaultMaxBytes + } + maxAudio := DefaultMaxAudioBytes + if maxBytes > maxAudio { + maxAudio = maxBytes + } + if maxTotal <= 0 { + maxTotal = DefaultMaxTotalBytes + } + if maxTotal < maxAudio { + maxAudio = maxTotal + } + if maxTotal < maxBytes { + return nil, fmt.Errorf("media: max_total_bytes %d is below the per-blob cap %d", maxTotal, maxBytes) + } + if retention <= 0 { + retention = DefaultRetention + } + s := &Store{dir: abs, maxBytes: maxBytes, maxAudio: maxAudio, maxTotal: maxTotal, + retention: retention, now: time.Now} + s.total = s.measure() + return s, nil +} + +// measure sums what is already on disk, so a restart does not start the budget +// over at zero. +func (s *Store) measure() int64 { + var total int64 + spool := filepath.Join(s.dir, "spool") + _ = filepath.WalkDir(s.dir, func(path string, d fs.DirEntry, err error) error { + if err == nil && d.IsDir() && path == spool { + // Spool files are not blobs yet and PutFile counts them when they + // become one. Counting them here too would double them. + return filepath.SkipDir + } + if err != nil || d.IsDir() || strings.HasSuffix(path, ".json") { + return nil //nolint:nilerr // an unreadable corner is not worth refusing to boot over + } + if info, err := d.Info(); err == nil { + total += info.Size() + } + return nil + }) + return total +} + +// Total is the number of blob bytes currently stored, and Budget the cap Put +// checks it against. Both are exported so the daemon can log how close it is. +func (s *Store) Total() int64 { + s.totalMu.Lock() + defer s.totalMu.Unlock() + return s.total +} + +// Budget is the whole-store cap. +func (s *Store) Budget() int64 { return s.maxTotal } + +// Dir is the store root. Exported for logs and for pointing a subprocess at a +// path under it. +func (s *Store) Dir() string { return s.dir } + +// Retention is the configured age limit Prune enforces. +func (s *Store) Retention() time.Duration { return s.retention } + +// Put stores data and returns its Blob. The id is the sha256 of data, so +// storing the same bytes twice is idempotent: the second call rewrites the +// sidecar (keeping the ORIGINAL creation time, so a re-send cannot extend +// retention indefinitely) and returns the same id. +// +// mime is recorded as given and used only to pick a file extension; nothing +// dispatches on it. Callers that need the mime to be trustworthy sniff it +// first — see SniffImage. +func (s *Store) Put(kind Kind, mime, source string, data []byte) (Blob, error) { + if !kind.Valid() { + return Blob{}, ErrBadKind + } + if len(data) == 0 { + return Blob{}, ErrEmpty + } + if cap := s.capFor(kind); int64(len(data)) > cap { + return Blob{}, fmt.Errorf("%w: %d > %d", ErrTooLarge, len(data), cap) + } + sum := sha256.Sum256(data) + id := hex.EncodeToString(sum[:]) + + blobPath, metaPath, err := s.paths(kind, id, mime) + if err != nil { + return Blob{}, err + } + if err := os.MkdirAll(filepath.Dir(blobPath), 0o700); err != nil { + return Blob{}, fmt.Errorf("media: create bucket: %w", err) + } + + b := Blob{ID: id, Kind: kind, MIME: mime, Size: int64(len(data)), Source: source, + Created: s.now().UTC(), Path: blobPath} + + // A blob already here keeps its first-seen time. Re-sending the same photo + // every hour must not keep it alive past retention. + if prev, err := readMeta(metaPath); err == nil && !prev.Created.IsZero() { + b.Created = prev.Created + } + + // A blob already on disk costs nothing more, so dedupe is checked before + // the budget rather than after it. + _, already := os.Stat(blobPath) + if already != nil { + s.totalMu.Lock() + room := s.total+b.Size <= s.maxTotal + if room { + s.total += b.Size + } + s.totalMu.Unlock() + if !room { + return Blob{}, fmt.Errorf("%w: %d stored, %d budget, %d more asked for", + ErrStoreFull, s.Total(), s.maxTotal, b.Size) + } + } + + // The sidecar goes first. Written second, a full disk or a crash between + // the two left the bytes on disk with no sidecar, and List only sees + // sidecars, so Prune could never collect them: Put returned an error and an + // image nobody knew about became permanent. + if err := writeMeta(metaPath, b); err != nil { + return Blob{}, err + } + if err := writeFile(blobPath, data); err != nil { + _ = os.Remove(metaPath) + if already != nil { + s.totalMu.Lock() + s.total -= b.Size + s.totalMu.Unlock() + } + return Blob{}, err + } + return b, nil +} + +// capFor is the per-blob cap for a kind. Audio has its own, larger one. +func (s *Store) capFor(kind Kind) int64 { + if kind == KindAudio { + return s.maxAudio + } + return s.maxBytes +} + +// PutFile stores a file that is already on disk, by moving it into place rather +// than reading it into memory. It exists for meeting audio: a two-hour capture +// is a couple of hundred megabytes, and Put's []byte means that much heap in +// the process that owns the database, twice over while the WAV is built. +// +// src is consumed: on success it has been renamed into the store, and on a +// duplicate it is removed. On failure it is left where it is, so a caller that +// still needs the bytes can fall back to reading them. +func (s *Store) PutFile(kind Kind, mime, source, src string) (Blob, error) { + if !kind.Valid() { + return Blob{}, ErrBadKind + } + info, err := os.Stat(src) + if err != nil { + return Blob{}, fmt.Errorf("media: stat spool: %w", err) + } + if info.Size() == 0 { + return Blob{}, ErrEmpty + } + if cap := s.capFor(kind); info.Size() > cap { + return Blob{}, fmt.Errorf("%w: %d > %d", ErrTooLarge, info.Size(), cap) + } + id, err := hashFile(src) + if err != nil { + return Blob{}, err + } + blobPath, metaPath, err := s.paths(kind, id, mime) + if err != nil { + return Blob{}, err + } + if err := os.MkdirAll(filepath.Dir(blobPath), 0o700); err != nil { + return Blob{}, fmt.Errorf("media: create bucket: %w", err) + } + b := Blob{ID: id, Kind: kind, MIME: mime, Size: info.Size(), Source: source, + Created: s.now().UTC(), Path: blobPath} + if prev, err := readMeta(metaPath); err == nil && !prev.Created.IsZero() { + b.Created = prev.Created + } + _, already := os.Stat(blobPath) + if already != nil { + s.totalMu.Lock() + room := s.total+b.Size <= s.maxTotal + if room { + s.total += b.Size + } + s.totalMu.Unlock() + if !room { + return Blob{}, fmt.Errorf("%w: %d stored, %d budget, %d more asked for", + ErrStoreFull, s.Total(), s.maxTotal, b.Size) + } + } + if err := writeMeta(metaPath, b); err != nil { + return Blob{}, err + } + if already == nil { + // Same bytes already here. Drop the spool copy. + _ = os.Remove(src) + return b, nil + } + if err := os.Chmod(src, 0o600); err != nil { + return Blob{}, fmt.Errorf("media: chmod spool: %w", err) + } + if err := os.Rename(src, blobPath); err != nil { + _ = os.Remove(metaPath) + s.totalMu.Lock() + s.total -= b.Size + s.totalMu.Unlock() + return Blob{}, fmt.Errorf("media: move spool: %w", err) + } + return b, nil +} + +// SpoolFile creates an empty file under the store, outside the kind +// directories, for a caller that is writing a blob incrementally. Prune never +// looks at it and List never reports it; PutFile is what turns it into a blob. +// The caller owns removing it if it never gets that far. +func (s *Store) SpoolFile(prefix string) (*os.File, error) { + dir := filepath.Join(s.dir, "spool") + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, fmt.Errorf("media: create spool: %w", err) + } + f, err := os.CreateTemp(dir, prefix+"-*") + if err != nil { + return nil, fmt.Errorf("media: spool: %w", err) + } + if err := f.Chmod(0o600); err != nil { + f.Close() + return nil, fmt.Errorf("media: chmod spool: %w", err) + } + return f, nil +} + +// hashFile streams the digest so the id costs one buffer rather than the whole +// file. +func hashFile(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", fmt.Errorf("media: open spool: %w", err) + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", fmt.Errorf("media: hash spool: %w", err) + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +// Get returns the blob's metadata without reading its bytes. +func (s *Store) Get(id string) (Blob, error) { + if !validID(id) { + return Blob{}, ErrBadID + } + for _, kind := range []Kind{KindImage, KindAudio} { + metaPath := filepath.Join(s.dir, string(kind), id[:2], id+".json") + b, err := readMeta(metaPath) + if err != nil { + continue + } + p, err := s.locate(kind, id) + if err != nil { + continue + } + b.Path = p + return b, nil + } + return Blob{}, ErrNotFound +} + +// Read returns the blob's bytes together with its metadata. This is the only +// way out of the store, and it is a local read: nothing in this package can +// send bytes anywhere. +func (s *Store) Read(id string) (Blob, []byte, error) { + b, err := s.Get(id) + if err != nil { + return Blob{}, nil, err + } + data, err := os.ReadFile(b.Path) + if err != nil { + return Blob{}, nil, fmt.Errorf("media: read %s: %w", shortID(id), err) + } + return b, data, nil +} + +// List returns every blob of the given kind, newest first. An empty kind lists +// both. It walks the directory; at personal volumes (tens to hundreds of items +// inside the retention window) that is cheap, and it means the sidecars are the +// single source of truth with no index to fall out of sync. +func (s *Store) List(kind Kind) ([]Blob, error) { + kinds := []Kind{KindImage, KindAudio} + if kind != "" { + if !kind.Valid() { + return nil, ErrBadKind + } + kinds = []Kind{kind} + } + var out []Blob + for _, k := range kinds { + root := filepath.Join(s.dir, string(k)) + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil // kind never used; not an error + } + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".json") { + return nil + } + b, err := readMeta(path) + if err != nil { + return nil // a corrupt sidecar is skipped, not fatal + } + if p, err := s.locate(b.Kind, b.ID); err == nil { + b.Path = p + } + out = append(out, b) + return nil + }) + if err != nil { + return nil, fmt.Errorf("media: list %s: %w", k, err) + } + } + sort.Slice(out, func(i, j int) bool { + if out[i].Created.Equal(out[j].Created) { + return out[i].ID < out[j].ID + } + return out[i].Created.After(out[j].Created) + }) + return out, nil +} + +// Delete removes a blob and its sidecar. Missing is not an error: the caller +// asked for it gone and it is gone. +func (s *Store) Delete(id string) error { + if !validID(id) { + return ErrBadID + } + for _, kind := range []Kind{KindImage, KindAudio} { + bucket := filepath.Join(s.dir, string(kind), id[:2]) + entries, err := os.ReadDir(bucket) + if err != nil { + continue + } + for _, e := range entries { + if !strings.HasPrefix(e.Name(), id) { + continue + } + path := filepath.Join(bucket, e.Name()) + var size int64 + if info, err := e.Info(); err == nil && !strings.HasSuffix(e.Name(), ".json") { + size = info.Size() + } + if err := os.Remove(path); err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("media: delete %s: %w", shortID(id), err) + } + if size > 0 { + s.totalMu.Lock() + s.total -= size + if s.total < 0 { + s.total = 0 + } + s.totalMu.Unlock() + } + } + } + return nil +} + +// Prune deletes every blob older than the store's retention and reports how +// many went. It is the enforcement half of the retention promise; a caller that +// never runs it has a store that grows without bound, which is why the daemon +// runs it on the digestion tick rather than leaving it to a cron the operator +// might not add. +func (s *Store) Prune() (int, error) { + blobs, err := s.List("") + if err != nil { + return 0, err + } + now := s.now() + deleted := 0 + known := map[string]bool{} + for _, b := range blobs { + known[b.ID] = true + if b.Age(now) <= s.retention { + continue + } + if err := s.Delete(b.ID); err != nil { + return deleted, err + } + delete(known, b.ID) + deleted++ + } + n, err := s.pruneOrphans(known, now) + return deleted + n, err +} + +// pruneOrphans collects blob files with no readable sidecar. List walks +// sidecars, so those files were invisible to retention and stayed on disk +// forever: audio of people accumulating is the exact failure this package +// exists to prevent, and a half-finished Put from an older build is enough to +// produce one. They are only collected once they are older than retention, so a +// Put racing a Prune does not lose its bytes. +func (s *Store) pruneOrphans(known map[string]bool, now time.Time) (int, error) { + deleted := 0 + for _, kind := range []Kind{KindImage, KindAudio} { + root := filepath.Join(s.dir, string(kind)) + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil + } + return err + } + if d.IsDir() || strings.HasSuffix(path, ".json") { + return nil + } + name := d.Name() + id, _, _ := strings.Cut(name, ".") + if known[id] { + return nil + } + info, err := d.Info() + if err != nil { + return nil //nolint:nilerr // gone underneath us is the outcome we wanted + } + if now.Sub(info.ModTime()) <= s.retention { + return nil + } + if err := os.Remove(path); err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + s.totalMu.Lock() + s.total -= info.Size() + if s.total < 0 { + s.total = 0 + } + s.totalMu.Unlock() + deleted++ + return nil + }) + if err != nil { + return deleted, fmt.Errorf("media: prune %s: %w", kind, err) + } + } + return deleted, nil +} + +// paths returns the blob and sidecar paths for an id. +func (s *Store) paths(kind Kind, id, mime string) (blobPath, metaPath string, err error) { + if !validID(id) { + return "", "", ErrBadID + } + bucket := filepath.Join(s.dir, string(kind), id[:2]) + return filepath.Join(bucket, id+extFor(mime, kind)), filepath.Join(bucket, id+".json"), nil +} + +// locate finds the stored bytes for an id whose extension we do not know, +// because the extension came from the mime at Put time. +func (s *Store) locate(kind Kind, id string) (string, error) { + if !validID(id) { + return "", ErrBadID + } + bucket := filepath.Join(s.dir, string(kind), id[:2]) + entries, err := os.ReadDir(bucket) + if err != nil { + return "", ErrNotFound + } + for _, e := range entries { + name := e.Name() + if strings.HasPrefix(name, id) && !strings.HasSuffix(name, ".json") { + return filepath.Join(bucket, name), nil + } + } + return "", ErrNotFound +} + +// validID guards every path built from an id. Without it a caller-supplied id +// is a path traversal: Get("../../etc/passwd") would read outside the store. +func validID(id string) bool { + if len(id) != 64 { + return false + } + for i := 0; i < len(id); i++ { + c := id[i] + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return false + } + } + return true +} + +// extFor maps a mime to a file extension, defaulting per kind. The extension is +// cosmetic — the id is the key — but it is what makes the store browsable and +// lets a subprocess that sniffs by name (piper, some image tools) cope. +func extFor(mime string, kind Kind) string { + switch strings.ToLower(strings.TrimSpace(mime)) { + case "image/jpeg", "image/jpg": + return ".jpg" + case "image/png": + return ".png" + case "image/gif": + return ".gif" + case "image/webp": + // Unreachable for images today: SniffImage refuses webp before + // anything reaches Put, because this build has no webp decoder. Kept + // so the mapping is right on the day one arrives. + return ".webp" + case "audio/wav", "audio/x-wav", "audio/wave": + return ".wav" + case "audio/l16", "audio/pcm": + return ".pcm" + } + if kind == KindImage { + return ".bin" + } + return ".pcm" +} + +// writeFile writes data 0600 via a temp file in the same directory, so a +// crash mid-write cannot leave a truncated blob under a digest that claims +// to describe the whole thing. +func writeFile(path string, data []byte) error { + tmp, err := os.CreateTemp(filepath.Dir(path), ".tmp-*") + if err != nil { + return fmt.Errorf("media: temp: %w", err) + } + defer os.Remove(tmp.Name()) + if err := tmp.Chmod(0o600); err != nil { + tmp.Close() + return fmt.Errorf("media: chmod: %w", err) + } + if _, err := tmp.Write(data); err != nil { + tmp.Close() + return fmt.Errorf("media: write: %w", err) + } + if err := tmp.Close(); err != nil { + return fmt.Errorf("media: close: %w", err) + } + if err := os.Rename(tmp.Name(), path); err != nil { + return fmt.Errorf("media: rename: %w", err) + } + return nil +} + +func writeMeta(path string, b Blob) error { + data, err := json.Marshal(b) + if err != nil { + return fmt.Errorf("media: marshal meta: %w", err) + } + return writeFile(path, data) +} + +func readMeta(path string) (Blob, error) { + data, err := os.ReadFile(path) + if err != nil { + return Blob{}, err + } + var b Blob + if err := json.Unmarshal(data, &b); err != nil { + return Blob{}, err + } + if !validID(b.ID) || !b.Kind.Valid() { + return Blob{}, errors.New("media: corrupt sidecar") + } + b.Created = b.Created.UTC() + return b, nil +} diff --git a/internal/media/store_test.go b/internal/media/store_test.go new file mode 100644 index 0000000..fccb67f --- /dev/null +++ b/internal/media/store_test.go @@ -0,0 +1,342 @@ +package media + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func testStore(t *testing.T) *Store { + t.Helper() + s, err := Open(t.TempDir(), 0, 0) + if err != nil { + t.Fatalf("open: %v", err) + } + return s +} + +func TestPutAndRead(t *testing.T) { + s := testStore(t) + b, err := s.Put(KindImage, "image/png", "web:upload", []byte("pretend png")) + if err != nil { + t.Fatalf("put: %v", err) + } + if len(b.ID) != 64 { + t.Fatalf("id is not a sha256 hex digest: %q", b.ID) + } + if b.Size != int64(len("pretend png")) { + t.Errorf("size = %d", b.Size) + } + if !strings.HasSuffix(b.Path, ".png") { + t.Errorf("extension not taken from mime: %s", b.Path) + } + got, data, err := s.Read(b.ID) + if err != nil { + t.Fatalf("read: %v", err) + } + if string(data) != "pretend png" { + t.Errorf("data = %q", data) + } + if got.Source != "web:upload" || got.Kind != KindImage { + t.Errorf("metadata not round-tripped: %+v", got) + } +} + +// The same bytes twice must be one file, and must NOT get a fresh creation +// time — otherwise re-sending a photo keeps it alive past retention forever. +func TestPutIsIdempotentAndKeepsFirstSeenTime(t *testing.T) { + s := testStore(t) + base := time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC) + s.now = func() time.Time { return base } + + first, err := s.Put(KindAudio, "audio/wav", "capture:meeting", []byte("pcm")) + if err != nil { + t.Fatalf("put: %v", err) + } + s.now = func() time.Time { return base.Add(72 * time.Hour) } + second, err := s.Put(KindAudio, "audio/wav", "capture:meeting", []byte("pcm")) + if err != nil { + t.Fatalf("re-put: %v", err) + } + if first.ID != second.ID { + t.Fatalf("same bytes produced two ids") + } + if !second.Created.Equal(base) { + t.Errorf("re-put moved created time to %v, want %v", second.Created, base) + } + list, err := s.List(KindAudio) + if err != nil { + t.Fatalf("list: %v", err) + } + if len(list) != 1 { + t.Errorf("got %d blobs, want 1", len(list)) + } +} + +func TestPutRejects(t *testing.T) { + s, err := Open(t.TempDir(), 8, 0) + if err != nil { + t.Fatal(err) + } + if _, err := s.Put(KindImage, "image/png", "x", nil); !errors.Is(err, ErrEmpty) { + t.Errorf("empty payload: %v", err) + } + if _, err := s.Put("video", "video/mp4", "x", []byte("ab")); !errors.Is(err, ErrBadKind) { + t.Errorf("bad kind: %v", err) + } + if _, err := s.Put(KindImage, "image/png", "x", []byte("way too many bytes")); !errors.Is(err, ErrTooLarge) { + t.Errorf("over cap: %v", err) + } +} + +// A caller-supplied id becomes a path, so a traversal attempt must be refused +// before it touches the filesystem rather than escaping the store root. +func TestMalformedIDIsRefused(t *testing.T) { + s := testStore(t) + for _, id := range []string{"", "../../etc/passwd", strings.Repeat("z", 64), strings.Repeat("a", 63)} { + if _, err := s.Get(id); !errors.Is(err, ErrBadID) && !errors.Is(err, ErrNotFound) { + t.Errorf("Get(%q) = %v, want a refusal", id, err) + } + if _, _, err := s.Read(id); err == nil { + t.Errorf("Read(%q) succeeded", id) + } + if err := s.Delete(id); err == nil && id != "" { + // Delete of a well-formed but absent id is fine; these are not + // well-formed. + t.Errorf("Delete(%q) succeeded", id) + } + } +} + +func TestGetMissingIsNotFound(t *testing.T) { + s := testStore(t) + if _, err := s.Get(strings.Repeat("a", 64)); !errors.Is(err, ErrNotFound) { + t.Errorf("got %v, want ErrNotFound", err) + } +} + +func TestPruneEnforcesRetention(t *testing.T) { + s, err := Open(t.TempDir(), 0, 48*time.Hour) + if err != nil { + t.Fatal(err) + } + now := time.Date(2026, 8, 1, 0, 0, 0, 0, time.UTC) + + s.now = func() time.Time { return now.Add(-96 * time.Hour) } + old, _ := s.Put(KindAudio, "audio/wav", "capture:meeting", []byte("old meeting")) + s.now = func() time.Time { return now.Add(-1 * time.Hour) } + fresh, _ := s.Put(KindImage, "image/png", "telegram", []byte("recent photo")) + + s.now = func() time.Time { return now } + n, err := s.Prune() + if err != nil { + t.Fatalf("prune: %v", err) + } + if n != 1 { + t.Errorf("pruned %d, want 1", n) + } + if _, err := s.Get(old.ID); !errors.Is(err, ErrNotFound) { + t.Errorf("stale blob survived prune: %v", err) + } + if _, err := s.Get(fresh.ID); err != nil { + t.Errorf("fresh blob was pruned: %v", err) + } +} + +func TestListIsNewestFirstAcrossKinds(t *testing.T) { + s := testStore(t) + base := time.Date(2026, 8, 1, 0, 0, 0, 0, time.UTC) + s.now = func() time.Time { return base } + _, _ = s.Put(KindImage, "image/png", "telegram", []byte("one")) + s.now = func() time.Time { return base.Add(time.Hour) } + newest, _ := s.Put(KindAudio, "audio/wav", "capture:meeting", []byte("two")) + + all, err := s.List("") + if err != nil { + t.Fatalf("list: %v", err) + } + if len(all) != 2 { + t.Fatalf("got %d, want 2", len(all)) + } + if all[0].ID != newest.ID { + t.Errorf("list is not newest-first") + } +} + +// Recordings of people are 0700/0600 and nothing else. +func TestPermissionsAreOwnerOnly(t *testing.T) { + dir := t.TempDir() + s, err := Open(filepath.Join(dir, "blobs"), 0, 0) + if err != nil { + t.Fatal(err) + } + b, err := s.Put(KindAudio, "audio/wav", "capture:meeting", []byte("pcm")) + if err != nil { + t.Fatal(err) + } + di, err := os.Stat(s.Dir()) + if err != nil { + t.Fatal(err) + } + if di.Mode().Perm() != 0o700 { + t.Errorf("store dir mode = %o, want 700", di.Mode().Perm()) + } + fi, err := os.Stat(b.Path) + if err != nil { + t.Fatal(err) + } + if fi.Mode().Perm() != 0o600 { + t.Errorf("blob mode = %o, want 600", fi.Mode().Perm()) + } +} + +func TestDeleteRemovesBytesAndSidecar(t *testing.T) { + s := testStore(t) + b, _ := s.Put(KindImage, "image/png", "telegram", []byte("bytes")) + if err := s.Delete(b.ID); err != nil { + t.Fatalf("delete: %v", err) + } + if _, err := os.Stat(b.Path); !os.IsNotExist(err) { + t.Errorf("bytes survived delete") + } + if _, err := s.Get(b.ID); !errors.Is(err, ErrNotFound) { + t.Errorf("sidecar survived delete: %v", err) + } +} + +func TestOpenRejectsEmptyDir(t *testing.T) { + if _, err := Open(" ", 0, 0); err == nil { + t.Error("empty dir accepted") + } +} + +// A blob whose sidecar is missing was invisible to List, so Prune never saw it +// and the bytes stayed on disk forever. Put produced exactly that state, by +// writing the blob first and the sidecar second. +func TestPruneCollectsASidecarlessBlob(t *testing.T) { + s := testStore(t) + b, err := s.Put(KindImage, "image/png", "web:upload", []byte("orphan")) + if err != nil { + t.Fatal(err) + } + meta := filepath.Join(s.dir, string(KindImage), b.ID[:2], b.ID+".json") + if err := os.Remove(meta); err != nil { + t.Fatal(err) + } + // Age the file past retention, the same way a real orphan gets there. + old := time.Now().Add(-2 * DefaultRetention) + if err := os.Chtimes(b.Path, old, old); err != nil { + t.Fatal(err) + } + n, err := s.Prune() + if err != nil { + t.Fatal(err) + } + if n != 1 { + t.Fatalf("pruned %d, want the orphan collected", n) + } + if _, err := os.Stat(b.Path); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("the orphaned bytes are still on disk: %v", err) + } +} + +// A young orphan is left alone: a Put racing a Prune must not lose its bytes. +func TestPruneLeavesAYoungOrphan(t *testing.T) { + s := testStore(t) + b, err := s.Put(KindImage, "image/png", "web:upload", []byte("fresh")) + if err != nil { + t.Fatal(err) + } + if err := os.Remove(filepath.Join(s.dir, string(KindImage), b.ID[:2], b.ID+".json")); err != nil { + t.Fatal(err) + } + if n, err := s.Prune(); err != nil || n != 0 { + t.Fatalf("prune = %d, %v; want the fresh orphan kept", n, err) + } +} + +// Put writes the sidecar first, so a failure writing the bytes leaves nothing +// at all rather than an uncollectable blob. +func TestPutLeavesNothingWhenTheBytesCannotBeWritten(t *testing.T) { + s := testStore(t) + data := []byte("will not land") + sum := sha256.Sum256(data) + id := hex.EncodeToString(sum[:]) + bucket := filepath.Join(s.dir, string(KindImage), id[:2]) + if err := os.MkdirAll(bucket, 0o700); err != nil { + t.Fatal(err) + } + // A directory where the blob file needs to be: rename onto it fails, which + // is the same shape as a full disk one step later. + if err := os.Mkdir(filepath.Join(bucket, id+".png"), 0o700); err != nil { + t.Fatal(err) + } + if _, err := s.Put(KindImage, "image/png", "web:upload", data); err == nil { + t.Fatal("put must fail") + } + if _, err := os.Stat(filepath.Join(bucket, id+".json")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("a sidecar was left behind claiming a blob that does not exist: %v", err) + } + if s.Total() != 0 { + t.Errorf("total = %d, want the failed put not counted", s.Total()) + } +} + +// The per-blob cap bounds one call and nothing bounded their sum. 64 MiB per +// call times unlimited calls inside a seven-day window fills the disk mavend's +// database lives on. +func TestPutRefusesPastTheStoreBudget(t *testing.T) { + s, err := OpenWithBudget(t.TempDir(), 16, 48, 0) + if err != nil { + t.Fatal(err) + } + for i, want := range []bool{true, true, true, false} { + data := []byte(strings.Repeat(string(rune('a'+i)), 16)) + _, err := s.Put(KindImage, "image/png", "web:upload", data) + if ok := err == nil; ok != want { + t.Fatalf("put %d: err = %v, want ok=%v", i, err, want) + } + if !want && !errors.Is(err, ErrStoreFull) { + t.Fatalf("put %d: err = %v, want ErrStoreFull", i, err) + } + } + // The same bytes again cost nothing, so they are not refused. + if _, err := s.Put(KindImage, "image/png", "web:upload", []byte(strings.Repeat("a", 16))); err != nil { + t.Fatalf("a re-send of stored bytes was refused: %v", err) + } + // Deleting frees the budget again. + blobs, err := s.List(KindImage) + if err != nil { + t.Fatal(err) + } + if err := s.Delete(blobs[0].ID); err != nil { + t.Fatal(err) + } + if _, err := s.Put(KindImage, "image/png", "web:upload", []byte(strings.Repeat("z", 16))); err != nil { + t.Fatalf("budget was not released on delete: %v", err) + } +} + +// A restart must not start the budget over at zero. +func TestOpenSeedsTheBudgetFromDisk(t *testing.T) { + dir := t.TempDir() + s, err := OpenWithBudget(dir, 16, 48, 0) + if err != nil { + t.Fatal(err) + } + if _, err := s.Put(KindImage, "image/png", "web:upload", []byte(strings.Repeat("a", 16))); err != nil { + t.Fatal(err) + } + again, err := OpenWithBudget(dir, 16, 48, 0) + if err != nil { + t.Fatal(err) + } + if again.Total() != 16 { + t.Fatalf("total after reopen = %d, want 16", again.Total()) + } +} diff --git a/internal/memeval/eval.go b/internal/memeval/eval.go new file mode 100644 index 0000000..d1eb3ad --- /dev/null +++ b/internal/memeval/eval.go @@ -0,0 +1,372 @@ +// Package memeval is background memory evaluation (Vikunja #248, +// docs/plans/03-memory-evaluation.md). +// +// It lives beside internal/memory rather than inside it because +// internal/store imports internal/memory for the vector-store backend, and an +// evaluator has to read store.Fact / store.Note / store.Nudge — putting it in +// internal/memory would close that import cycle. +// +// Every so often Maven reads back her own recent memory — facts, notes, the +// nudges she sent — and asks the resident model what it notices: a habit that +// stopped, a gap, something worth saying later. What comes back is written as +// notes with source EvalNoteSource and nothing else happens. That restraint is +// the design, not an unfinished edge: +// +// - She does not speak here. There is no dispatcher, no channel, no nudge. +// An observation is a thought she wrote down; he reads it on /dash when he +// wants to. "Not a nag, not autonomous" (CLAUDE.md) is easy to violate with +// exactly this feature — an hourly loop with an LLM in it and permission to +// talk is a machine for generating interruptions — so the loop has no way +// to reach him at all. Turning observations into nudges is a separate +// decision with a separate opt-in, and it is deliberately NOT in this file. +// - She does not act. No reminder is created, no routine proposed, no fact +// written. The model's suggested_action is recorded as text inside the note +// and interpreted by nobody. +// - She says nothing about an empty store. No memory ⇒ no LLM call ⇒ no +// "observations" invented out of two facts. A 1.7B asked to find a pattern +// will always find one; the defence is not asking. +// +// Everything the evaluator writes is attributable: source is EvalNoteSource, so +// an inferred observation can never be mistaken for something he said, and the +// whole batch is one SQL delete away if the output turns out to be noise. +package memeval + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + "github.com/kami/maven/internal/llm" + "github.com/kami/maven/internal/persona" + "github.com/kami/maven/internal/store" +) + +// EvalNoteSource — the source stamped on every note the evaluator writes. +// Same infer:* convention as the rest of the derived facts. +const EvalNoteSource = "infer:memory-eval" + +// DefaultMinConfidence — an observation below this is dropped. The model is +// asked for its own confidence and small models are badly calibrated, so this +// is a coarse filter, not a probability: it exists to throw away the guesses +// the model itself hedged on. +const DefaultMinConfidence = 0.7 + +// DefaultMaxItems — how much recent memory goes into one evaluation, per +// store. 30 facts + 30 notes + 30 nudges is a few thousand tokens of the 4096 +// context the resident Thinking model runs with, which leaves room for its +// reasoning tokens. Raising this trades reasoning room for history. +const DefaultMaxItems = 30 + +// MaxObservations — the model may return at most this many observations per +// evaluation, enforced by the grammar. A cap here is also a noise cap: an +// evaluation that "notices" ten things has noticed nothing. +const MaxObservations = 3 + +// Observation — one thing the evaluator noticed. +type Observation struct { + Text string `json:"observation"` + Conf float64 `json:"confidence"` + // Action — what the model thinks should happen with this. Recorded, never + // executed: see the file comment. One of "note", "propose", "notify". + Action string `json:"suggested_action"` +} + +// Completer — the llama-server seam, same shape router.Completer uses so the +// one resident model serves this caller too. +type Completer interface { + Complete(ctx context.Context, r llm.Req) (string, error) +} + +// Reader — the slice of the store an evaluation reads. Narrow on purpose: the +// evaluator gets recent memory and nothing else. No entity graph, no presence, +// no config facts. +type Reader interface { + RecentFacts(ctx context.Context, n int) ([]store.Fact, error) + // RecentNotesExcludingSource feeds the snapshot, RecentNotesBySource feeds + // the dedupe. Both are source-scoped in SQL rather than filtered here: a + // plain RecentNotes read makes each window a budget over ALL note writers, + // so her own hourly observations shrink the snapshot and ordinary notes + // push old observations out of the dedupe set. + RecentNotesExcludingSource(ctx context.Context, source string, n int) ([]store.Note, error) + RecentNotesBySource(ctx context.Context, source string, n int) ([]store.Note, error) + RecentNudges(ctx context.Context, n int) ([]store.Nudge, error) +} + +// NoteWriter — where observations land. Embeddings are passed nil: an +// observation is written for a human to read on /dash, not to be recalled by +// similarity. Feeding LLM-generated text back into the RAG pool it was +// generated from is how a small model starts citing its own guesses as +// evidence. +type NoteWriter interface { + WriteNote(ctx context.Context, ts time.Time, text string, embedding []float32, source string) (int64, error) +} + +// Config — evaluator tuning. Zero values are replaced by the Default* +// constants, so the zero Config is the sane one. +type Config struct { + MaxItems int + MinConfidence float64 + + // ContextBlock — the shared persona block (internal/persona), re-evaluated + // per call so the clock in it is current. Prepended to the system prompt so + // observations come out in Maven's voice: feminine self-reference, informal + // "ты". nil is allowed; the base prompt still carries the address rules. + ContextBlock func() string +} + +// Evaluator reads recent memory and records what the model notices. +type Evaluator struct { + read Reader + write NoteWriter + llm Completer + cfg Config +} + +func NewEvaluator(r Reader, w NoteWriter, c Completer, cfg Config) *Evaluator { + if cfg.MaxItems <= 0 { + cfg.MaxItems = DefaultMaxItems + } + if cfg.MinConfidence <= 0 { + cfg.MinConfidence = DefaultMinConfidence + } + return &Evaluator{read: r, write: w, llm: c, cfg: cfg} +} + +// evalGrammar — GBNF pinning the reply to a bounded JSON array of fixed-shape +// observations. Same reasoning as the router's routeGrammar: the enum and the +// length bound are what stop a small model from drifting into free text or +// filling the token budget with one repeated field. +const evalGrammar = ` +root ::= "[" ws (obs ("," ws obs){0,2})? ws "]" +obs ::= "{" ws "\"observation\"" ws ":" ws text "," ws "\"confidence\"" ws ":" ws conf "," ws "\"suggested_action\"" ws ":" ws act ws "}" +text ::= "\"" ([^"\\] | "\\" .){1,200} "\"" +conf ::= "0" "." [0-9]{1,2} | "1" ("." "0")? +act ::= "\"note\"" | "\"propose\"" | "\"notify\"" +ws ::= [ \t\n]* +` + +// evalSystem — the evaluation prompt. Two things it insists on, both learned +// from the phraser: state the observation as something she noticed rather than +// an instruction, and say nothing when there is nothing (the model is given an +// explicit way to return an empty array, because a model with no exit returns +// filler). +const evalSystem = `Ты просматриваешь свою собственную память: недавние факты, заметки и напоминания, которые ты отправляла. +Найди то, что действительно заметно: привычка, которая прервалась; пробел в записях; повторяющаяся закономерность. + +Правила: +- Отвечай ТОЛЬКО массивом JSON. Каждый элемент: {"observation": "...", "confidence": 0.0-1.0, "suggested_action": "note"|"propose"|"notify"}. +- observation — короткая фраза по-русски о том, что ты заметила. О себе — в женском роде ("я заметила"). К нему — на "ты". +- Не выдумывай. Если в памяти нет ничего заметного, верни пустой массив []. +- Не давай советов и не приказывай. Ты замечаешь, а не требуешь. +- confidence — насколько ты уверена, что это настоящая закономерность, а не совпадение. +- Максимум три наблюдения. Лучше одно точное, чем три общих.` + +// Evaluate runs one evaluation and returns the observations it recorded. +// +// Returns (nil, nil) — not an error — for every ordinary "nothing to say" +// outcome: an empty store, an empty array from the model, everything below the +// confidence floor, or every observation already recorded earlier. Only a real +// read/LLM/write failure is an error, and the caller (a background ticker) logs +// it and waits for the next interval. +func (e *Evaluator) Evaluate(ctx context.Context, now time.Time) ([]Observation, error) { + snap, err := e.snapshot(ctx) + if err != nil { + return nil, err + } + if snap == "" { + return nil, nil // nothing recorded ⇒ nothing to notice, and no LLM call + } + + raw, err := e.llm.Complete(ctx, llm.Req{ + System: persona.Prepend(e.cfg.ContextBlock, evalSystem), + User: snap, + Grammar: evalGrammar, + MaxTokens: 512, + RepeatPenalty: 1.1, + }) + if err != nil { + return nil, fmt.Errorf("memory eval: complete: %w", err) + } + obs, err := parseObservations(raw) + if err != nil { + return nil, fmt.Errorf("memory eval: parse %q: %w", truncate(raw, 120), err) + } + + // Dedupe against what earlier evaluations already wrote. Without this an + // hourly loop over a slowly-changing store writes the same sentence every + // hour until /dash is nothing but the evaluator talking to itself. + seen, err := e.recordedTexts(ctx) + if err != nil { + return nil, err + } + + var kept []Observation + for _, o := range obs { + o.Text = strings.TrimSpace(o.Text) + if o.Text == "" || o.Conf < e.cfg.MinConfidence { + continue + } + norm := normalizeObservation(o.Text) + if seen[norm] { + continue + } + seen[norm] = true + if _, err := e.write.WriteNote(ctx, now, formatNote(o), nil, EvalNoteSource); err != nil { + return kept, fmt.Errorf("memory eval: write note: %w", err) + } + kept = append(kept, o) + } + return kept, nil +} + +// formatNote — the stored text. The suggested action is kept as a visible +// suffix rather than a column: it is the model's opinion about what to do next, +// and the only consumer is a human reading /dash. +func formatNote(o Observation) string { + if o.Action == "" { + return o.Text + } + return fmt.Sprintf("%s [%s]", o.Text, o.Action) +} + +// DedupeWindow — how many of her OWN past observations the dedupe looks back +// over. Wider than MaxItems because the point is to remember saying it, not to +// summarize it. Counted in eval notes only: when this was a plain recent-notes +// read the window was really a budget over every note writer, so a few hundred +// ordinary notes pushed an observation out of sight and the next evaluation was +// free to write the same sentence again. +const DedupeWindow = 200 + +// recordedTexts — the normalized text of every observation earlier evaluations +// wrote, for dedupe. +func (e *Evaluator) recordedTexts(ctx context.Context) (map[string]bool, error) { + notes, err := e.read.RecentNotesBySource(ctx, EvalNoteSource, DedupeWindow) + if err != nil { + return nil, fmt.Errorf("memory eval: recent notes: %w", err) + } + seen := make(map[string]bool, len(notes)) + for _, n := range notes { + seen[normalizeObservation(stripAction(n.Text))] = true + } + return seen, nil +} + +// actions — the suggested_action enum, as the grammar constrains it. +var actions = []string{"note", "propose", "notify"} + +// stripAction removes the "[action]" suffix formatNote appended, and only that. +// Matching any bracketed tail would eat the end of an observation that happens +// to finish on a bracketed clause, which changes its dedupe key and lets the +// same sentence through twice. +func stripAction(text string) string { + for _, a := range actions { + if s, ok := strings.CutSuffix(text, " ["+a+"]"); ok { + return s + } + } + return text +} + +// normalizeObservation — dedupe key. Case- and whitespace-insensitive, which +// catches the realistic repeat (the model re-emitting the same sentence with a +// different comma) without pretending to do semantic dedupe. +func normalizeObservation(s string) string { + return strings.Join(strings.Fields(strings.ToLower(s)), " ") +} + +// snapshot renders recent memory as the user turn. Returns "" when there is +// nothing in any store — the caller treats that as "do not ask the model". +// +// Notes written by earlier evaluations are excluded, in SQL. Feeding her own +// observations back in is how "я заметила, что ты не записывал еду" becomes +// evidence for noticing it again, three evaluations deep. Excluding them after +// the read was worse than not excluding them: hourly evaluation makes her own +// notes the majority of the newest rows within weeks, so asking for MaxItems +// and dropping hers left the model a handful of real notes to look at. +func (e *Evaluator) snapshot(ctx context.Context) (string, error) { + n := e.cfg.MaxItems + facts, err := e.read.RecentFacts(ctx, n) + if err != nil { + return "", fmt.Errorf("memory eval: recent facts: %w", err) + } + notes, err := e.read.RecentNotesExcludingSource(ctx, EvalNoteSource, n) + if err != nil { + return "", fmt.Errorf("memory eval: recent notes: %w", err) + } + nudges, err := e.read.RecentNudges(ctx, n) + if err != nil { + return "", fmt.Errorf("memory eval: recent nudges: %w", err) + } + + var b strings.Builder + wrote := false + if len(facts) > 0 { + b.WriteString("Факты:\n") + for _, f := range facts { + fmt.Fprintf(&b, "- %s %s=%s (%s)\n", f.Ts.Format("2006-01-02 15:04"), f.Key, truncate(f.Value, 80), f.Source) + wrote = true + } + } + if len(notes) > 0 { + b.WriteString("\nЗаметки:\n") + for _, nt := range notes { + fmt.Fprintf(&b, "- %s %s\n", nt.Ts.Format("2006-01-02 15:04"), truncate(nt.Text, 160)) + wrote = true + } + } + if len(nudges) > 0 { + b.WriteString("\nНапоминания, которые ты отправляла:\n") + for _, nd := range nudges { + outcome := nd.Outcome + if outcome == "" { + outcome = "?" + } + fmt.Fprintf(&b, "- %s %s → %s (%s)\n", nd.Ts.Format("2006-01-02 15:04"), nd.Rule, outcome, nd.Channel) + wrote = true + } + } + if !wrote { + // Only her own past observations, or nothing at all. Either way there is + // no new memory to evaluate. + return "", nil + } + b.WriteString("\nЧто ты замечаешь?") + return b.String(), nil +} + +// parseObservations reads the model's array. Tolerates the leading/trailing +// prose a Thinking model sometimes emits around JSON by taking the outermost +// bracketed span, the same tolerance the router's parser has. +func parseObservations(raw string) ([]Observation, error) { + s := strings.TrimSpace(raw) + if i := strings.Index(s, "["); i >= 0 { + if j := strings.LastIndex(s, "]"); j > i { + s = s[i : j+1] + } + } + if s == "" { + return nil, nil + } + var obs []Observation + if err := json.Unmarshal([]byte(s), &obs); err != nil { + return nil, err + } + if len(obs) > MaxObservations { + // The grammar bounds this; a grammar-less server or a future prompt + // change must not be able to flood /dash. + sort.SliceStable(obs, func(i, j int) bool { return obs[i].Conf > obs[j].Conf }) + obs = obs[:MaxObservations] + } + return obs, nil +} + +func truncate(s string, n int) string { + r := []rune(s) + if len(r) <= n { + return s + } + return string(r[:n]) + "…" +} diff --git a/internal/memeval/eval_test.go b/internal/memeval/eval_test.go new file mode 100644 index 0000000..bcd794d --- /dev/null +++ b/internal/memeval/eval_test.go @@ -0,0 +1,364 @@ +package memeval + +import ( + "context" + "database/sql" + "errors" + "fmt" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/kami/maven/internal/llm" + "github.com/kami/maven/internal/store" +) + +// fakeLLM — canned replies, one per call, and a record of what it was asked. +type fakeLLM struct { + replies []string + calls []llm.Req + err error +} + +func (f *fakeLLM) Complete(_ context.Context, r llm.Req) (string, error) { + f.calls = append(f.calls, r) + if f.err != nil { + return "", f.err + } + if len(f.replies) == 0 { + return "[]", nil + } + out := f.replies[0] + f.replies = f.replies[1:] + return out, nil +} + +func newTestStore(t *testing.T) *store.Store { + t.Helper() + st, err := store.Open(context.Background(), filepath.Join(t.TempDir(), "memeval_test.db")) + if err != nil { + t.Fatalf("store.Open: %v", err) + } + t.Cleanup(func() { _ = st.Close() }) + return st +} + +func refNow() time.Time { return time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC) } + +// seedMemory writes a little of everything the evaluator reads. +func seedMemory(t *testing.T, st *store.Store, ctx context.Context, now time.Time) { + t.Helper() + for i := 0; i < 3; i++ { + ts := now.Add(-time.Duration(i+1) * 24 * time.Hour) + if _, err := st.WriteFact(ctx, ts, store.KindSelf, "water_ml", "500", "tap:desk", 1.0, sql.NullInt64{}); err != nil { + t.Fatalf("write fact: %v", err) + } + } + if _, err := st.WriteNote(ctx, now.Add(-2*time.Hour), "купить корм для кота", nil, "tap:voice"); err != nil { + t.Fatalf("write note: %v", err) + } + if _, err := st.RecordNudge(ctx, "water", "voice", "пора выпить воды", now.Add(-time.Hour)); err != nil { + t.Fatalf("record nudge: %v", err) + } +} + +// TestEvaluateEmptyStoreAsksNothing — the "shuts up when uncertain" floor. An +// empty store must not even reach the model: a small model asked to find a +// pattern in nothing will invent one. +func TestEvaluateEmptyStoreAsksNothing(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + f := &fakeLLM{} + ev := NewEvaluator(st, st, f, Config{}) + + obs, err := ev.Evaluate(ctx, refNow()) + if err != nil { + t.Fatalf("Evaluate: %v", err) + } + if len(obs) != 0 { + t.Fatalf("observations on an empty store = %d, want 0", len(obs)) + } + if len(f.calls) != 0 { + t.Fatalf("LLM called %d times on an empty store, want 0", len(f.calls)) + } +} + +// TestEvaluateWritesHighConfidenceObservations — the happy path. Confident +// observations are written as notes stamped infer:memory-eval, and the low +// ones are dropped. +func TestEvaluateWritesHighConfidenceObservations(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := refNow() + seedMemory(t, st, ctx, now) + + f := &fakeLLM{replies: []string{`[ + {"observation":"ты три дня не записывал еду","confidence":0.9,"suggested_action":"notify"}, + {"observation":"может быть, ты стал меньше пить воды","confidence":0.3,"suggested_action":"note"} + ]`}} + ev := NewEvaluator(st, st, f, Config{}) + + obs, err := ev.Evaluate(ctx, now) + if err != nil { + t.Fatalf("Evaluate: %v", err) + } + if len(obs) != 1 { + t.Fatalf("kept %d observations, want 1 (the 0.3 one is below the floor): %+v", len(obs), obs) + } + if obs[0].Text != "ты три дня не записывал еду" { + t.Errorf("kept the wrong observation: %q", obs[0].Text) + } + + notes, err := st.RecentNotes(ctx, 50) + if err != nil { + t.Fatalf("RecentNotes: %v", err) + } + var written []store.Note + for _, n := range notes { + if n.Source == EvalNoteSource { + written = append(written, n) + } + } + if len(written) != 1 { + t.Fatalf("notes with source %s = %d, want 1", EvalNoteSource, len(written)) + } + if !strings.Contains(written[0].Text, "ты три дня не записывал еду") { + t.Errorf("note text = %q", written[0].Text) + } + if !strings.Contains(written[0].Text, "[notify]") { + t.Errorf("note text = %q, want the suggested action recorded", written[0].Text) + } + + // The prompt must carry the memory it is evaluating, and must not carry a + // grammar-free request. + if len(f.calls) != 1 { + t.Fatalf("LLM calls = %d, want 1", len(f.calls)) + } + if !strings.Contains(f.calls[0].User, "water_ml") { + t.Errorf("prompt does not mention the seeded facts:\n%s", f.calls[0].User) + } + if f.calls[0].Grammar == "" { + t.Error("evaluation ran without a grammar") + } +} + +// TestEvaluateDeduplicatesAcrossRuns — the failure mode that would make this +// feature unusable: an hourly loop over a store that barely changes writing the +// same sentence every hour until /dash is nothing but the evaluator. +func TestEvaluateDeduplicatesAcrossRuns(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := refNow() + seedMemory(t, st, ctx, now) + + same := `[{"observation":"ты три дня не записывал еду","confidence":0.9,"suggested_action":"note"}]` + spaced := `[{"observation":"Ты три дня не записывал еду","confidence":0.95,"suggested_action":"note"}]` + f := &fakeLLM{replies: []string{same, same, spaced}} + ev := NewEvaluator(st, st, f, Config{}) + + for i := 0; i < 3; i++ { + if _, err := ev.Evaluate(ctx, now.Add(time.Duration(i)*time.Hour)); err != nil { + t.Fatalf("Evaluate %d: %v", i, err) + } + } + + notes, err := st.RecentNotes(ctx, 50) + if err != nil { + t.Fatalf("RecentNotes: %v", err) + } + n := 0 + for _, nt := range notes { + if nt.Source == EvalNoteSource { + n++ + } + } + if n != 1 { + t.Fatalf("eval notes after three identical evaluations = %d, want 1", n) + } +} + +// TestEvaluateIgnoresOwnNotes — her own observations must not become input. +// Otherwise "я заметила X" is evidence for noticing X again, three evaluations +// deep. With nothing but eval notes in the store there is no new memory, so the +// model is not asked at all. +func TestEvaluateIgnoresOwnNotes(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := refNow() + if _, err := st.WriteNote(ctx, now.Add(-time.Hour), "я заметила, что ты мало пьёшь [note]", nil, EvalNoteSource); err != nil { + t.Fatalf("write note: %v", err) + } + + f := &fakeLLM{} + ev := NewEvaluator(st, st, f, Config{}) + obs, err := ev.Evaluate(ctx, now) + if err != nil { + t.Fatalf("Evaluate: %v", err) + } + if len(obs) != 0 || len(f.calls) != 0 { + t.Fatalf("observations=%d llm calls=%d, want 0/0 — own notes are not memory to evaluate", len(obs), len(f.calls)) + } +} + +// TestEvaluateEmptyArrayIsNotAnError — "nothing to say" is the expected outcome +// most of the time and must not be logged as a failure. +func TestEvaluateEmptyArrayIsNotAnError(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := refNow() + seedMemory(t, st, ctx, now) + + ev := NewEvaluator(st, st, &fakeLLM{replies: []string{"[]"}}, Config{}) + obs, err := ev.Evaluate(ctx, now) + if err != nil { + t.Fatalf("Evaluate: %v", err) + } + if len(obs) != 0 { + t.Fatalf("observations = %d, want 0", len(obs)) + } +} + +// TestEvaluateLLMErrorIsReported — a broken llama-server is an error the caller +// logs; it must not silently write anything. +func TestEvaluateLLMErrorIsReported(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := refNow() + seedMemory(t, st, ctx, now) + + ev := NewEvaluator(st, st, &fakeLLM{err: errors.New("connection refused")}, Config{}) + if _, err := ev.Evaluate(ctx, now); err == nil { + t.Fatal("want an error when the model is unreachable") + } + notes, err := st.RecentNotes(ctx, 50) + if err != nil { + t.Fatalf("RecentNotes: %v", err) + } + for _, n := range notes { + if n.Source == EvalNoteSource { + t.Fatalf("wrote a note despite an LLM failure: %q", n.Text) + } + } +} + +// TestParseObservationsTolerantAndBounded — Thinking models wrap JSON in prose, +// and no reply may exceed MaxObservations even if the grammar is bypassed. +func TestParseObservationsTolerantAndBounded(t *testing.T) { + obs, err := parseObservations(`hmm вот: [{"observation":"a","confidence":0.9,"suggested_action":"note"}] всё`) + if err != nil { + t.Fatalf("parse: %v", err) + } + if len(obs) != 1 || obs[0].Text != "a" { + t.Fatalf("got %+v, want one observation 'a'", obs) + } + + var b strings.Builder + b.WriteString("[") + for i := 0; i < MaxObservations+3; i++ { + if i > 0 { + b.WriteString(",") + } + b.WriteString(`{"observation":"x","confidence":0.5,"suggested_action":"note"}`) + } + b.WriteString("]") + obs, err = parseObservations(b.String()) + if err != nil { + t.Fatalf("parse: %v", err) + } + if len(obs) != MaxObservations { + t.Fatalf("parsed %d observations, want the %d cap", len(obs), MaxObservations) + } +} + +// TestDedupeSurvivesOrdinaryNotes — the dedupe window is a count of HER notes, +// not of all notes. With a plain recent-notes read, DedupeWindow ordinary notes +// written after an observation pushed it out of sight and the same sentence was +// written again on the next evaluation. +func TestDedupeSurvivesOrdinaryNotes(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := refNow() + seedMemory(t, st, ctx, now) + + same := `[{"observation":"ты три дня не записывал еду","confidence":0.9,"suggested_action":"note"}]` + f := &fakeLLM{replies: []string{same, same}} + ev := NewEvaluator(st, st, f, Config{}) + + if _, err := ev.Evaluate(ctx, now); err != nil { + t.Fatalf("Evaluate: %v", err) + } + // A few months of ordinary use between the two evaluations. + for i := 0; i < DedupeWindow*2; i++ { + if _, err := st.WriteNote(ctx, now.Add(time.Duration(i+1)*time.Minute), + fmt.Sprintf("обычная заметка %d", i), nil, "tap:voice"); err != nil { + t.Fatalf("write note: %v", err) + } + } + if _, err := ev.Evaluate(ctx, now.Add(24*time.Hour)); err != nil { + t.Fatalf("Evaluate: %v", err) + } + + own, err := st.RecentNotesBySource(ctx, EvalNoteSource, 100) + if err != nil { + t.Fatalf("RecentNotesBySource: %v", err) + } + if len(own) != 1 { + t.Fatalf("eval notes = %d, want 1 — the repeat was not deduped", len(own)) + } +} + +// TestSnapshotBudgetIsNotEatenByOwnNotes — MaxItems notes must be MaxItems of +// HIS notes. Reading MaxItems rows and then discarding hers left the model a +// handful of real notes once hourly evaluation had run for a few weeks. +func TestSnapshotBudgetIsNotEatenByOwnNotes(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := refNow() + + // His notes first, then a long run of hers on top of them. + for i := 0; i < 5; i++ { + if _, err := st.WriteNote(ctx, now.Add(-time.Duration(100-i)*time.Hour), + fmt.Sprintf("его заметка %d", i), nil, "tap:voice"); err != nil { + t.Fatalf("write note: %v", err) + } + } + for i := 0; i < 50; i++ { + if _, err := st.WriteNote(ctx, now.Add(-time.Duration(50-i)*time.Hour), + fmt.Sprintf("я заметила кое-что %d [note]", i), nil, EvalNoteSource); err != nil { + t.Fatalf("write note: %v", err) + } + } + + f := &fakeLLM{replies: []string{"[]"}} + ev := NewEvaluator(st, st, f, Config{MaxItems: 5}) + if _, err := ev.Evaluate(ctx, now); err != nil { + t.Fatalf("Evaluate: %v", err) + } + if len(f.calls) != 1 { + t.Fatalf("LLM calls = %d, want 1", len(f.calls)) + } + prompt := f.calls[0].User + if strings.Contains(prompt, "я заметила") { + t.Errorf("her own observations reached the prompt:\n%s", prompt) + } + for i := 0; i < 5; i++ { + if !strings.Contains(prompt, fmt.Sprintf("его заметка %d", i)) { + t.Errorf("his note %d missing from the prompt:\n%s", i, prompt) + } + } +} + +// TestStripActionKeepsBracketedTail — the dedupe key strips the recorded +// action and nothing else. Cutting at the last " [" ate the end of an +// observation that itself ends on a bracketed clause, so the same sentence +// hashed two ways. +func TestStripActionKeepsBracketedTail(t *testing.T) { + text := "ты не пил воду [со вторника]" + if got := stripAction(formatNote(Observation{Text: text, Action: "note"})); got != text { + t.Errorf("stripAction = %q, want %q", got, text) + } + if got := stripAction(text); got != text { + t.Errorf("stripAction = %q, want it untouched", got) + } +} diff --git a/internal/memory/behavior.go b/internal/memory/behavior.go new file mode 100644 index 0000000..8ed3260 --- /dev/null +++ b/internal/memory/behavior.go @@ -0,0 +1,439 @@ +package memory + +import ( + "fmt" + "sort" + "strings" + "time" +) + +// Behavioural memory — "what do I usually do?" (Vikunja #254). +// +// The profile is COUNTED, not generated. docs/plans/09-behavioral-memory.md +// asks for an LLM to write a behaviour profile daily and store it as a fact; +// this does not do that, on purpose. A 1.7B asked to summarise a year of habits +// will produce fluent claims about the owner's life that no row in the store +// supports, and a wrong claim about him is the most expensive kind of wrong +// maven can be. Counting distinct days per weekday is verifiable, cheap enough +// to run on the question, and cannot invent a habit he does not have. +// +// Recomputed on read rather than cached as a fact for the same reason the store +// is append-only: a cached profile can disagree with the rows it came from, and +// then there are two truths. The plan's step 5 ("profile updates on fact write") +// exists to keep a cache fresh; there is no cache, so a new fact is already in +// the next answer. +// +// It is also read-only and unprompted-free. The plan's step 4 — a morning +// dispatcher nudge proposing the day — is deliberately NOT here: maven is not a +// nag, and proposing plans at 08:00 every day is the definition of one. Pattern +// inference that leads to a routine the owner accepts already exists in +// internal/pattern with the proposal queue on /routines; that is the sanctioned +// path from "she noticed" to "she acts", and it goes through him. + +// Observation — one thing the owner was recorded doing, reduced to what a habit +// needs: when, and what. Facts arrive as store/ipc rows; the caller maps them +// so this package stays free of both. +type Observation struct { + At time.Time + Key string + Kind string // "self" | "env" | "config" +} + +// Activity — one recurring thing, as counted. Days is the number of DISTINCT +// days it was observed on, which is the number that decides whether something +// is a habit; Count can be inflated by one busy day. +// +// TypicalAt is the median time of day it happens at, rounded to the minute — a +// median and not a mean, so one 03:00 outlier does not move "он обычно пьёт +// воду утром" into the night. It is a CIRCULAR median: the clock wraps, and a +// plain median of minutes-since-midnight reports 12:00 for a man who goes to +// bed at 23:50. +// +// HasTypical is false when the times are spread too widely for any of them to +// be typical (see maxTypicalSpread). She then names the habit without a time +// instead of naming a time she cannot support. +type Activity struct { + Key string + Days int + Count int + TypicalAt time.Duration + HasTypical bool +} + +// Profile — the counted behaviour model. +// +// Weekly holds only the activities that DISTINGUISH a weekday: things he does +// on Tuesdays and not on most other days. Everyday holds the ones that recur +// across the week, and All holds both. The split exists because the two answer +// different questions, and conflating them produced the failure that named +// this: asked what he does on Saturdays, maven replied "ты пьёшь воду". +type Profile struct { + Since time.Time + Until time.Time + Weekly map[time.Weekday][]Activity + Everyday []Activity + All []Activity +} + +// EverydaySpan — the number of weekdays an activity must be a habit on before +// it stops counting as characteristic of any one of them. Six of seven, not +// five: a weekday-only rhythm spans exactly five, and "по будням ты +// тренируешься" is a real answer about Tuesday. Six days a week is not. +const EverydaySpan = 6 + +// MinHabitDays — how many distinct days an activity must appear on before maven +// will call it usual. Two is the smallest number that can distinguish a habit +// from a one-off; below that she says she does not know yet, which is true. +const MinHabitDays = 2 + +// nonBehaviouralKeyPrefixes — keys that are machinery or one-shot records, not +// behaviour. Calendar events carry the day in the key so they can never repeat; +// cooldown and quiet rows are maven's own tuning state, not his habits. +// quiet is listed with its separators rather than bare: as a five-letter +// prefix it would also swallow any future self-fact key that merely starts +// with those letters. +var nonBehaviouralKeyPrefixes = []string{ + "calendar_event_", + "cooldown:", + "quiet_", + "quiet:", + "behavior_profile", +} + +// nonBehaviouralKeys — exact keys, for the ones with no separator to anchor on. +var nonBehaviouralKeys = map[string]bool{"quiet": true} + +// BuildProfile counts habits out of observations. now bounds the window's upper +// end and supplies the location every day boundary is taken in — a habit is +// "on Tuesdays" in the owner's timezone or it is nothing. +// +// Only self-facts count. An env row is the world (weather, a relayed meeting), +// and a config row is maven's own state; neither says anything about what he +// usually does. +func BuildProfile(obs []Observation, now time.Time) Profile { + loc := now.Location() + p := Profile{Until: now, Weekly: map[time.Weekday][]Activity{}} + + type bucket struct { + days map[string]struct{} + count int + mins []int + } + // key → bucket, and (weekday, key) → bucket. + all := map[string]*bucket{} + weekly := map[time.Weekday]map[string]*bucket{} + + for _, o := range obs { + if o.Kind != "self" { + continue + } + key := canonicalize(o.Key) + if key == "" || nonBehavioural(key) { + continue + } + at := o.At.In(loc) + if at.IsZero() || at.After(now) { + continue + } + if p.Since.IsZero() || at.Before(p.Since) { + p.Since = at + } + day := at.Format("2006-01-02") + minute := at.Hour()*60 + at.Minute() + + bump := func(m map[string]*bucket) { + b := m[key] + if b == nil { + b = &bucket{days: map[string]struct{}{}} + m[key] = b + } + b.days[day] = struct{}{} + b.count++ + b.mins = append(b.mins, minute) + } + bump(all) + wd := at.Weekday() + if weekly[wd] == nil { + weekly[wd] = map[string]*bucket{} + } + bump(weekly[wd]) + } + + harvest := func(m map[string]*bucket) []Activity { + var out []Activity + for key, b := range m { + if len(b.days) < MinHabitDays { + continue + } + mid, known := circularMedianMinutes(b.mins) + out = append(out, Activity{ + Key: key, + Days: len(b.days), + Count: b.count, + TypicalAt: time.Duration(mid) * time.Minute, + HasTypical: known, + }) + } + // Most-established first, then earliest in the day, then by key so the + // same history always reads back the same way. + sort.Slice(out, func(i, j int) bool { + if out[i].Days != out[j].Days { + return out[i].Days > out[j].Days + } + if out[i].TypicalAt != out[j].TypicalAt { + return out[i].TypicalAt < out[j].TypicalAt + } + return out[i].Key < out[j].Key + }) + return out + } + + p.All = harvest(all) + + // How many weekdays each key is a habit on. An activity that recurs on + // most days of the week is a daily habit, and naming it as an answer to + // "что я обычно делаю по субботам?" is a non-answer: "ты пьёшь воду" is + // true of Saturday and of every other day, so it says nothing about + // Saturday. Those are held in Everyday and read back separately. + span := map[string]int{} + harvested := map[time.Weekday][]Activity{} + for wd, m := range weekly { + acts := harvest(m) + harvested[wd] = acts + for _, a := range acts { + span[a.Key]++ + } + } + for wd, acts := range harvested { + var distinct []Activity + for _, a := range acts { + if span[a.Key] >= EverydaySpan { + continue + } + distinct = append(distinct, a) + } + if len(distinct) > 0 { + p.Weekly[wd] = distinct + } + } + for _, a := range p.All { + if span[a.Key] >= EverydaySpan { + p.Everyday = append(p.Everyday, a) + } + } + return p +} + +// canonicalize maps a fact key onto the key the profile counts it under. +// Lowercased, trimmed, and separators folded to "_" before the alias lookup, +// so "Выпил воды", "выпил-воды" and "выпил_воды" are one habit and not three. +// An unlisted key counts as itself. +func canonicalize(key string) string { + k := strings.ToLower(strings.TrimSpace(key)) + k = strings.NewReplacer(" ", "_", "-", "_").Replace(k) + k = strings.Trim(k, "_") + if c, ok := canonicalKey[k]; ok { + return c + } + return k +} + +func nonBehavioural(key string) bool { + if nonBehaviouralKeys[key] { + return true + } + for _, p := range nonBehaviouralKeyPrefixes { + if strings.HasPrefix(key, p) { + return true + } + } + return false +} + +// minutesPerDay — the modulus every clock time is taken in. +const minutesPerDay = 24 * 60 + +// maxTypicalSpread — how far apart the observations of one activity may sit, +// once rotated onto the shortest arc, before "usually at X" stops being a +// claim about anything. Half a day: wider than that and the values cover the +// clock, so no point on it is typical. +const maxTypicalSpread = minutesPerDay / 2 + +// circularMedianMinutes — the median time of day, on a clock rather than on a +// number line. Reports (0, false) when the values are too spread out to have a +// middle. +// +// A plain median of minutes-since-midnight is wrong for anything that straddles +// midnight, which is exactly the activity most likely to: bedtimes of 23:40, +// 23:50, 00:10 and 00:20 average out to 720 minutes, and she says "обычно ты +// спишь около 12:00". The fix is to find the rotation of the sorted values with +// the shortest span — the arc the observations actually occupy — take the +// ordinary median inside it, and wrap the answer back into the day. +func circularMedianMinutes(xs []int) (int, bool) { + if len(xs) == 0 { + return 0, false + } + s := make([]int, len(xs)) + for i, x := range xs { + s[i] = ((x % minutesPerDay) + minutesPerDay) % minutesPerDay + } + sort.Ints(s) + + // Each rotation cuts the day at one observation and unwraps the values + // before the cut onto the following day. The cut with the smallest span is + // the one where no observation is on the far side of midnight from the rest. + best, bestSpan := 0, minutesPerDay+1 + for i := range s { + span := s[(i+len(s)-1)%len(s)] - s[i] + if i > 0 { + span += minutesPerDay + } + if span < bestSpan { + best, bestSpan = i, span + } + } + if bestSpan > maxTypicalSpread { + return 0, false + } + rot := make([]int, 0, len(s)) + for i := 0; i < len(s); i++ { + v := s[(best+i)%len(s)] + if best+i >= len(s) { + v += minutesPerDay + } + rot = append(rot, v) + } + m := medianInt(rot) % minutesPerDay + return m, true +} + +// medianInt — the middle value, averaging the two middles on an even count. +func medianInt(xs []int) int { + if len(xs) == 0 { + return 0 + } + s := make([]int, len(xs)) + copy(s, xs) + sort.Ints(s) + mid := len(s) / 2 + if len(s)%2 == 1 { + return s[mid] + } + return (s[mid-1] + s[mid]) / 2 +} + +// weekdayRU and activityRU are loaded from the embedded behavior_ru.json; +// see behavior_ru.go. + +// FormatWeekdayRU reads back what DISTINGUISHES a given weekday. +// Second person singular and informal, as she speaks TO him. +// +// When nothing distinguishes it, she says so and names the daily habits as +// daily habits instead of passing them off as an answer about that day. The +// previous version had no such distinction and answered "что я делаю по +// субботам?" with "ты пьёшь воду" — true, useless, and phrased as if Saturday +// were the reason. +func (p Profile) FormatWeekdayRU(wd time.Weekday) string { + day := weekdayRU[int(wd)%7] + acts := p.Weekly[wd] + if len(acts) > 0 { + return fmt.Sprintf("по %s ты обычно %s.", day, joinActivities(acts)) + } + if len(p.Everyday) > 0 { + return fmt.Sprintf("по %s у тебя нет ничего особенного — то же, что и в остальные дни: %s.", + day, joinActivities(p.Everyday)) + } + return fmt.Sprintf("по %s я пока не вижу у тебя ничего постоянного.", day) +} + +// FormatWeekendRU reads back what distinguishes Saturday and Sunday. +// +// The two days are answered separately rather than pooled: "по выходным" is a +// question about both, and a habit he has on Saturdays only is the interesting +// half of the answer, not noise to average away. +func (p Profile) FormatWeekendRU() string { + sat, sun := p.Weekly[time.Saturday], p.Weekly[time.Sunday] + switch { + case len(sat) > 0 && len(sun) > 0: + return fmt.Sprintf("по субботам ты обычно %s, по воскресеньям — %s.", + joinActivities(sat), joinActivities(sun)) + case len(sat) > 0: + return fmt.Sprintf("по субботам ты обычно %s, а по воскресеньям ничего постоянного.", + joinActivities(sat)) + case len(sun) > 0: + return fmt.Sprintf("по воскресеньям ты обычно %s, а по субботам ничего постоянного.", + joinActivities(sun)) + case len(p.Everyday) > 0: + return fmt.Sprintf("по выходным у тебя нет ничего особенного — то же, что и в остальные дни: %s.", + joinActivities(p.Everyday)) + } + return "по выходным я пока не вижу у тебя ничего постоянного." +} + +// FormatOverallRU reads back the habits that hold across the whole week, and +// says over what stretch of records it is claiming them. +// +// The period is spoken because "обычно" without one is an unfalsifiable claim +// about his life: the same sentence comes out of three days of taps and out of +// a year of them, and only one of those is worth believing. +func (p Profile) FormatOverallRU() string { + if len(p.All) == 0 { + return "я ещё не набрала достаточно записей, чтобы говорить о привычках." + } + return fmt.Sprintf("обычно ты %s — %s.", joinActivities(p.All), p.spanRU()) +} + +// spanRU — "по записям за последние N дней", or a vaguer phrase when the window +// is too short to name in days. +func (p Profile) spanRU() string { + if p.Since.IsZero() || !p.Until.After(p.Since) { + return "по записям за сегодня" + } + days := int(p.Until.Sub(p.Since).Hours()/24) + 1 + return fmt.Sprintf("по записям за последние %d %s", days, pluralDaysRU(days)) +} + +// pluralDaysRU — the Russian count form of "день" for n. +func pluralDaysRU(n int) string { + switch { + case n%100 >= 11 && n%100 <= 14: + return "дней" + case n%10 == 1: + return "день" + case n%10 >= 2 && n%10 <= 4: + return "дня" + default: + return "дней" + } +} + +// maxRecited bounds a spoken profile. A list of fifteen habits read aloud is +// not an answer; the most established few are. +const maxRecited = 5 + +func joinActivities(acts []Activity) string { + if len(acts) > maxRecited { + acts = acts[:maxRecited] + } + parts := make([]string, len(acts)) + for i, a := range acts { + gloss, ok := activityRU[a.Key] + if !ok { + // No gloss: quote the key instead of reading it as a verb. The keys + // come from the model, so an unglossed one is as likely to be + // "выпил_воды" as a noun, and "обычно ты выпил_воды около 09:00" is + // not a sentence. + gloss = fmt.Sprintf("отмечаешь «%s»", strings.ReplaceAll(a.Key, "_", " ")) + } + if !a.HasTypical { + parts[i] = gloss + continue + } + parts[i] = fmt.Sprintf("%s около %02d:%02d", gloss, + int(a.TypicalAt.Hours()), int(a.TypicalAt.Minutes())%60) + } + if len(parts) == 1 { + return parts[0] + } + return strings.Join(parts[:len(parts)-1], ", ") + " и " + parts[len(parts)-1] +} diff --git a/internal/memory/behavior_ru.go b/internal/memory/behavior_ru.go new file mode 100644 index 0000000..49fbe65 --- /dev/null +++ b/internal/memory/behavior_ru.go @@ -0,0 +1,55 @@ +package memory + +import ( + _ "embed" + "encoding/json" + "fmt" +) + +// The Russian read-back vocabulary lives in behavior_ru.json, not in Go source. +// +// It is data, not logic: a weekday name and a verb gloss are wording choices +// that change independently of anything the counter does, and having them as +// literals scattered through behavior.go meant every phrasing tweak was a code +// diff. go:embed keeps the single-binary deploy — the JSON is compiled in, so +// there is no file to install alongside mavend and no way for the two to drift. +// +// Parsed once at init. A malformed file is a panic, deliberately: it can only +// happen if the embedded asset is broken at build time, and a daemon that comes +// up with an empty vocabulary would answer with bare fact keys. + +//go:embed behavior_ru.json +var behaviorRUJSON []byte + +type behaviorRU struct { + Weekdays []string `json:"weekdays"` + Activities map[string]string `json:"activities"` + KeyAliases map[string][]string `json:"key_aliases"` +} + +var ( + // weekdayRU — dative plural, indexed by time.Weekday. + weekdayRU []string + // activityRU — canonical fact key to second-person-singular verb phrase. + activityRU map[string]string + // canonicalKey — observed fact key to the canonical key it counts as. + canonicalKey map[string]string +) + +func init() { + var v behaviorRU + if err := json.Unmarshal(behaviorRUJSON, &v); err != nil { + panic(fmt.Sprintf("memory: behavior_ru.json: %v", err)) + } + if len(v.Weekdays) != 7 { + panic(fmt.Sprintf("memory: behavior_ru.json: want 7 weekdays, got %d", len(v.Weekdays))) + } + weekdayRU, activityRU = v.Weekdays, v.Activities + canonicalKey = make(map[string]string) + for canonical, variants := range v.KeyAliases { + canonicalKey[canonical] = canonical + for _, variant := range variants { + canonicalKey[variant] = canonical + } + } +} diff --git a/internal/memory/behavior_ru.json b/internal/memory/behavior_ru.json new file mode 100644 index 0000000..a53147c --- /dev/null +++ b/internal/memory/behavior_ru.json @@ -0,0 +1,51 @@ +{ + "_comment": [ + "Russian read-back vocabulary for the counted behaviour profile.", + "Embedded into the binary by behavior_ru.go — there is no runtime file to", + "ship or lose. Editing wording here does not need a Go change, which is the", + "whole point: these are strings for a person, not program logic.", + "", + "weekdays are indexed by time.Weekday (0 = Sunday) and are in the dative", + "plural, because both 'по вторникам' and 'по средам' need that form.", + "", + "activities glosses a fact key as a second-person-singular verb phrase. An", + "unknown key is not glossed and is quoted rather than guessed at: inventing", + "a Russian phrase for a key maven does not recognise puts words in his", + "mouth, and reading the raw key inline produces 'обычно ты выпил_воды'.", + "", + "key_aliases maps the fact keys the router actually emits onto the canonical", + "key the profile counts. The key of a self fact comes straight out of the", + "LLM with no allowlist behind it, so 'я выпил воду' and 'попил воды' arrive", + "as different keys, split one habit into two, and drop both below the", + "day threshold. Additive and lowercase; an unlisted key counts as itself." + ], + "weekdays": [ + "воскресеньям", + "понедельникам", + "вторникам", + "средам", + "четвергам", + "пятницам", + "субботам" + ], + "activities": { + "water": "пьёшь воду", + "meal": "ешь", + "sleep": "спишь", + "break": "делаешь перерыв", + "shower": "принимаешь душ", + "walk": "гуляешь", + "pills": "пьёшь витамины", + "workout": "тренируешься" + }, + "key_aliases": { + "water": ["вода", "воду", "воды", "водичка", "попил", "попил_воды", "выпил_воды", "выпил_воду", "drink_water", "drank_water"], + "meal": ["еда", "еду", "поел", "поесть", "завтрак", "обед", "ужин", "food", "ate", "breakfast", "lunch", "dinner"], + "sleep": ["сон", "спать", "лег", "лёг", "уснул", "заснул", "bedtime", "slept"], + "break": ["перерыв", "отдых", "пауза", "rest", "pause"], + "shower": ["душ", "принял_душ", "помылся", "мытьё", "мылся"], + "walk": ["прогулка", "гулял", "погулял", "выгулял", "walked", "walking"], + "pills": ["витамины", "таблетки", "таблетка", "лекарство", "vitamins", "meds", "medicine"], + "workout": ["тренировка", "тренировался", "потренировался", "зал", "спорт", "exercise", "gym", "training"] + } +} diff --git a/internal/memory/behavior_test.go b/internal/memory/behavior_test.go new file mode 100644 index 0000000..e2ee61c --- /dev/null +++ b/internal/memory/behavior_test.go @@ -0,0 +1,369 @@ +package memory + +import ( + "strings" + "testing" + "time" + "unicode" +) + +// habitHistory — n weeks of the same weekday, at the given local time. +func habitHistory(key string, wd time.Weekday, hh, mm, weeks int, from time.Time) []Observation { + var out []Observation + d := from + for d.Weekday() != wd { + d = d.AddDate(0, 0, -1) + } + for i := 0; i < weeks; i++ { + day := d.AddDate(0, 0, -7*i) + out = append(out, Observation{ + At: time.Date(day.Year(), day.Month(), day.Day(), hh, mm, 0, 0, from.Location()), + Key: key, + Kind: "self", + }) + } + return out +} + +func behaviorNow() time.Time { + // A Monday, so "по вторникам" is a past weekday and not today. + return time.Date(2026, 8, 3, 20, 0, 0, 0, time.UTC) +} + +func TestBuildProfileCountsWeekdayHabits(t *testing.T) { + now := behaviorNow() + obs := append( + habitHistory("workout", time.Tuesday, 19, 0, 4, now), + habitHistory("water", time.Tuesday, 9, 0, 3, now)..., + ) + p := BuildProfile(obs, now) + + tue := p.Weekly[time.Tuesday] + if len(tue) != 2 { + t.Fatalf("got %d tuesday activities, want 2: %+v", len(tue), tue) + } + // Most-established first. + if tue[0].Key != "workout" || tue[0].Days != 4 { + t.Errorf("first = %+v, want workout on 4 days", tue[0]) + } + if tue[0].TypicalAt != 19*time.Hour { + t.Errorf("typical at %v, want 19:00", tue[0].TypicalAt) + } + if len(p.Weekly[time.Wednesday]) != 0 { + t.Errorf("wednesday must be empty: %+v", p.Weekly[time.Wednesday]) + } + if len(p.All) != 2 { + t.Errorf("the week-wide list should hold both: %+v", p.All) + } +} + +// A one-off is not a habit. Saying "ты обычно X" off a single row is a +// confidently wrong claim about his life. +func TestBuildProfileNeedsMoreThanOneDay(t *testing.T) { + now := behaviorNow() + obs := habitHistory("workout", time.Tuesday, 19, 0, 1, now) + // Three rows, same day — a busy Tuesday, not a habit. + obs = append(obs, Observation{At: obs[0].At.Add(time.Hour), Key: "workout", Kind: "self"}) + obs = append(obs, Observation{At: obs[0].At.Add(2 * time.Hour), Key: "workout", Kind: "self"}) + + p := BuildProfile(obs, now) + if len(p.All) != 0 || len(p.Weekly) != 0 { + t.Fatalf("one day of rows must produce no habit: %+v / %+v", p.All, p.Weekly) + } + if got := p.FormatOverallRU(); !strings.Contains(got, "не набрала достаточно") { + t.Errorf("empty profile reads %q", got) + } +} + +// Only self-facts describe him. Env rows are the world and config rows are +// maven's own tuning state; counting either as a habit would be a category +// error the owner would then be told about. +func TestBuildProfileIgnoresNonSelfAndMachineryKeys(t *testing.T) { + now := behaviorNow() + var obs []Observation + for _, o := range habitHistory("water", time.Tuesday, 9, 0, 3, now) { + o.Kind = "env" + obs = append(obs, o) + } + for _, o := range habitHistory("cooldown:water", time.Tuesday, 9, 0, 3, now) { + obs = append(obs, o) // kind=self, but a machinery key + } + for _, o := range habitHistory("calendar_event_20260804_standup", time.Tuesday, 10, 0, 3, now) { + obs = append(obs, o) + } + if p := BuildProfile(obs, now); len(p.All) != 0 { + t.Fatalf("nothing here is a habit of his: %+v", p.All) + } +} + +// The median, not the mean: one 03:00 outlier must not move a morning habit +// into the night. +func TestBuildProfileTypicalTimeIsMedian(t *testing.T) { + now := behaviorNow() + obs := habitHistory("water", time.Tuesday, 9, 0, 4, now) + obs = append(obs, Observation{At: obs[0].At.AddDate(0, 0, -28).Add(-6 * time.Hour), Key: "water", Kind: "self"}) + p := BuildProfile(obs, now) + if len(p.All) != 1 { + t.Fatalf("got %+v", p.All) + } + if p.All[0].TypicalAt != 9*time.Hour { + t.Errorf("typical at %v, want 09:00 despite the outlier", p.All[0].TypicalAt) + } +} + +func TestProfileFormatRUPersona(t *testing.T) { + now := behaviorNow() + obs := append( + habitHistory("workout", time.Tuesday, 19, 0, 4, now), + habitHistory("water", time.Tuesday, 9, 5, 3, now)..., + ) + p := BuildProfile(obs, now) + + got := p.FormatWeekdayRU(time.Tuesday) + want := "по вторникам ты обычно тренируешься около 19:00 и пьёшь воду около 09:05." + if got != want { + t.Errorf("got %q\nwant %q", got, want) + } + if empty := p.FormatWeekdayRU(time.Thursday); !strings.Contains(empty, "ничего постоянного") { + t.Errorf("an unknown weekday reads %q", empty) + } + // Persona: she addresses him informally, never in the masculine about + // herself, and never with a pet name. + for _, s := range []string{got, p.FormatOverallRU(), p.FormatWeekdayRU(time.Thursday)} { + // Whole words: "ничего" contains "его", and a substring test would + // call a correct sentence a persona violation. + for _, tok := range strings.FieldsFunc(strings.ToLower(s), func(r rune) bool { + return !unicode.IsLetter(r) + }) { + switch tok { + case "рад", "понял", "вы", "ваш", "ваши", "милый", "дорогой", "он", "его": + t.Errorf("%q uses %q", s, tok) + } + } + } +} + +// An unrecognised key is read back verbatim rather than glossed into something +// maven made up. +func TestProfileUnknownKeyReadBackVerbatim(t *testing.T) { + now := behaviorNow() + p := BuildProfile(habitHistory("починил кран", time.Tuesday, 12, 0, 2, now), now) + if got := p.FormatOverallRU(); !strings.Contains(got, "починил кран") { + t.Errorf("got %q", got) + } +} + +// A future-dated row is a clock problem, not a habit. +func TestBuildProfileIgnoresFutureRows(t *testing.T) { + now := behaviorNow() + obs := habitHistory("water", time.Tuesday, 9, 0, 3, now.AddDate(0, 2, 0)) + if p := BuildProfile(obs, now); len(p.All) != 0 { + t.Fatalf("future rows counted: %+v", p.All) + } +} + +// TestWeekdayProfileExcludesEverydayHabits — the "You drink water" case. +// Drinking water every day is not what he does on Saturdays, and answering +// with it makes the weekday question pointless. +func TestWeekdayProfileExcludesEverydayHabits(t *testing.T) { + now := time.Date(2026, 8, 1, 20, 0, 0, 0, time.UTC) // a Saturday + var obs []Observation + // water: twice a day, every day, for three weeks. + for d := 1; d <= 21; d++ { + day := now.AddDate(0, 0, -d) + obs = append(obs, + Observation{At: time.Date(day.Year(), day.Month(), day.Day(), 9, 0, 0, 0, time.UTC), Key: "water", Kind: "self"}, + Observation{At: time.Date(day.Year(), day.Month(), day.Day(), 18, 0, 0, 0, time.UTC), Key: "water", Kind: "self"}) + } + // workout: Saturdays only. + for _, d := range []int{7, 14, 21} { + day := now.AddDate(0, 0, -d) + obs = append(obs, Observation{ + At: time.Date(day.Year(), day.Month(), day.Day(), 11, 0, 0, 0, time.UTC), Key: "workout", Kind: "self"}) + } + + p := BuildProfile(obs, now) + + sat := p.Weekly[time.Saturday] + if len(sat) != 1 || sat[0].Key != "workout" { + t.Fatalf("Saturday should be characterised by workout alone, got %+v", sat) + } + if len(p.Everyday) != 1 || p.Everyday[0].Key != "water" { + t.Fatalf("water should be an everyday habit, got %+v", p.Everyday) + } + + got := p.FormatWeekdayRU(time.Saturday) + if !strings.Contains(got, "тренируешься") { + t.Fatalf("Saturday readout should name the workout: %q", got) + } + if strings.Contains(got, "воду") { + t.Fatalf("Saturday readout must not recite the everyday habit: %q", got) + } + + // A day with nothing of its own says so rather than reciting water as if + // Wednesday were the reason for it. + wed := p.FormatWeekdayRU(time.Wednesday) + if !strings.Contains(wed, "ничего особенного") || !strings.Contains(wed, "воду") { + t.Fatalf("plain weekday readout should say the day is unremarkable and name the daily habits: %q", wed) + } +} + +// TestTypicalTimeIsCircular — the median is over a clock, not a number line. +// Bedtimes either side of midnight used to average to midday, which is the +// exact error the median was chosen to avoid, on the one activity most likely +// to cross the boundary. +func TestTypicalTimeIsCircular(t *testing.T) { + now := behaviorNow() + var obs []Observation + for i, mins := range []int{23*60 + 40, 23*60 + 50, 10, 20} { + day := now.AddDate(0, 0, -(i + 1)) + obs = append(obs, Observation{ + At: time.Date(day.Year(), day.Month(), day.Day(), 0, mins, 0, 0, now.Location()), + Key: "sleep", + Kind: "self", + }) + } + p := BuildProfile(obs, now) + if len(p.All) != 1 { + t.Fatalf("got %+v, want one activity", p.All) + } + got := p.All[0].TypicalAt + if !p.All[0].HasTypical { + t.Fatal("a four-observation cluster has a typical time") + } + if got < 23*time.Hour+55*time.Minute && got > 5*time.Minute { + t.Errorf("typical bedtime = %v, want just either side of midnight", got) + } + if s := p.FormatOverallRU(); strings.Contains(s, "около 12:00") { + t.Errorf("read back as %q", s) + } +} + +// Times spread across the whole clock have no typical value, and she must not +// name one. +func TestNoTypicalTimeWhenSpreadWide(t *testing.T) { + now := behaviorNow() + var obs []Observation + for i, hh := range []int{2, 9, 16, 21} { + day := now.AddDate(0, 0, -(i + 1)) + obs = append(obs, Observation{ + At: time.Date(day.Year(), day.Month(), day.Day(), hh, 0, 0, 0, now.Location()), + Key: "water", + Kind: "self", + }) + } + p := BuildProfile(obs, now) + if len(p.All) != 1 { + t.Fatalf("got %+v, want one activity", p.All) + } + if p.All[0].HasTypical { + t.Errorf("times spanning %v were given a typical value", p.All[0].TypicalAt) + } + if s := p.FormatOverallRU(); strings.Contains(s, "около") { + t.Errorf("read back with a time she cannot support: %q", s) + } +} + +// TestKeysAreCanonicalisedBeforeCounting — the fact key comes out of the LLM +// with no allowlist behind it, so the same habit arrives spelled several ways. +// Counted separately, each spelling sits below MinHabitDays and the habit +// vanishes. +func TestKeysAreCanonicalisedBeforeCounting(t *testing.T) { + now := behaviorNow() + var obs []Observation + for i, key := range []string{"water", "Воду", "выпил воды", "попил_воды"} { + day := now.AddDate(0, 0, -(i + 1)) + obs = append(obs, Observation{ + At: time.Date(day.Year(), day.Month(), day.Day(), 9, 0, 0, 0, now.Location()), + Key: key, + Kind: "self", + }) + } + p := BuildProfile(obs, now) + if len(p.All) != 1 { + t.Fatalf("got %+v, want one activity — the spellings are one habit", p.All) + } + if p.All[0].Key != "water" || p.All[0].Days != 4 { + t.Errorf("got %+v, want water on 4 days", p.All[0]) + } + if s := p.FormatOverallRU(); !strings.Contains(s, "пьёшь воду") { + t.Errorf("read back as %q, want the glossed canonical key", s) + } +} + +// An unglossed key is quoted, not read as a verb. "обычно ты выпил_воды около +// 09:00" is what reciting the raw key produced. +func TestUnglossedKeyIsQuoted(t *testing.T) { + now := behaviorNow() + p := BuildProfile(habitHistory("починил_кран", time.Tuesday, 12, 0, 2, now), now) + got := p.FormatOverallRU() + if !strings.Contains(got, "отмечаешь «починил кран»") { + t.Errorf("got %q", got) + } +} + +// She says over what stretch of records "обычно" is claimed. Without it the +// same sentence comes out of three days and out of a year. +func TestOverallNamesThePeriod(t *testing.T) { + now := behaviorNow() + p := BuildProfile(habitHistory("water", time.Tuesday, 9, 0, 3, now), now) + got := p.FormatOverallRU() + if !strings.Contains(got, "по записям за последние 21 день") { + t.Errorf("got %q, want the period spoken", got) + } +} + +// The no-data weekday answer is about him, not about her. "у меня пока нет +// ничего постоянного" answers a question nobody asked. +func TestEmptyWeekdayAnswerIsAboutHim(t *testing.T) { + p := BuildProfile(nil, behaviorNow()) + got := p.FormatWeekdayRU(time.Wednesday) + if strings.Contains(got, "у меня") { + t.Errorf("got %q", got) + } + if !strings.Contains(got, "у тебя") { + t.Errorf("got %q, want an answer about him", got) + } +} + +// "quiet" is machinery, but only as a whole key. As a bare five-letter prefix +// it silently swallowed any future self-fact key starting with those letters. +func TestQuietPrefixDoesNotSwallowRealKeys(t *testing.T) { + now := behaviorNow() + p := BuildProfile(habitHistory("quietude", time.Tuesday, 8, 0, 3, now), now) + if len(p.All) != 1 { + t.Fatalf("got %+v, want the key counted", p.All) + } + p = BuildProfile(habitHistory("quiet_hours", time.Tuesday, 8, 0, 3, now), now) + if len(p.All) != 0 { + t.Fatalf("got %+v, want maven's own tuning state dropped", p.All) + } +} + +func TestPluralDaysRU(t *testing.T) { + for _, c := range []struct { + n int + want string + }{{1, "день"}, {2, "дня"}, {5, "дней"}, {11, "дней"}, {21, "день"}, {22, "дня"}, {114, "дней"}} { + if got := pluralDaysRU(c.n); got != c.want { + t.Errorf("pluralDaysRU(%d) = %q, want %q", c.n, got, c.want) + } + } +} + +// "по выходным" is a question about two days, answered as two days. +func TestFormatWeekendRU(t *testing.T) { + now := behaviorNow() + obs := append( + habitHistory("workout", time.Saturday, 11, 0, 3, now), + habitHistory("walk", time.Sunday, 15, 0, 3, now)..., + ) + got := BuildProfile(obs, now).FormatWeekendRU() + if !strings.Contains(got, "по субботам") || !strings.Contains(got, "по воскресеньям") { + t.Errorf("got %q, want both weekend days named", got) + } + empty := BuildProfile(nil, now).FormatWeekendRU() + if strings.Contains(empty, "у меня") { + t.Errorf("got %q", empty) + } +} diff --git a/internal/memory/store.go b/internal/memory/store.go index 33dfa3f..5077e47 100644 --- a/internal/memory/store.go +++ b/internal/memory/store.go @@ -3,9 +3,24 @@ package memory import ( "context" "sort" + "strings" "sync" ) +// NonRecallPrefix — rows whose id starts with this are excluded from Search by +// every backend. Speaker voiceprints live in the same vector table as notes and +// facts (internal/speaker writes them under this prefix), and they are not +// recall material: a voiceprint has no text to read back and surfacing one as a +// note hit leaks a name attached to a biometric. +// +// Reading them through Catalog.ByPrefix was documented as what keeps them out +// of recall. It is not. It controls how speaker code reads its own rows and +// says nothing about what Search scores. What actually kept them out was that +// cosine returns 0 on a width mismatch, so a 192-dim voiceprint scored 0 +// against a 384-dim query. That is a coincidence of two model choices — some +// x-vector exports are 384-dim — and not an invariant. This is the invariant. +const NonRecallPrefix = "speaker:" + // Result is a single search hit. type Result struct { ID string @@ -19,6 +34,33 @@ type Store interface { Search(ctx context.Context, vec []float32, topK int) ([]Result, error) } +// Record is a stored vector read back whole — id, vector and metadata — as +// opposed to Result, which is a search hit and carries a score instead of the +// vector. +type Record struct { + ID string + Vec []float32 + Meta map[string]string +} + +// Catalog is a Store that can also be enumerated by id prefix and deleted from. +// +// Search is not enough for every user of the vector table. Speaker profiles +// (internal/speaker) need to list exactly their own rows without scoring +// anything, because listing enrolled voices is not a similarity question, and +// they need Delete because a voiceprint is data about a person and "forget this +// voice" has to actually remove it. Note and fact recall use plain Store and are +// unaffected. +type Catalog interface { + Store + // ByPrefix returns every row whose id starts with prefix, in no particular + // order. An empty prefix returns everything. + ByPrefix(ctx context.Context, prefix string) ([]Record, error) + // Delete removes one row by id. Deleting a row that is not there is not an + // error: the caller asked for it to be gone and it is gone. + Delete(ctx context.Context, id string) error +} + // item is a single stored vector with metadata. type item struct { id string @@ -32,14 +74,61 @@ type InMemoryStore struct { items []item } +// compile-time check: InMemoryStore satisfies Catalog. +var _ Catalog = (*InMemoryStore)(nil) + func NewInMemoryStore() *InMemoryStore { return &InMemoryStore{} } +// Insert upserts by id, matching the persistent store.MemoryStore: a repeated +// id replaces the prior row rather than accumulating a second copy. Re-indexing +// a note is an update, and re-enrolling a voice must replace the old voiceprint +// rather than leave it searchable. func (s *InMemoryStore) Insert(_ context.Context, id string, vec []float32, meta map[string]string) error { s.mu.Lock() + defer s.mu.Unlock() + for i := range s.items { + if s.items[i].id == id { + s.items[i] = item{id: id, vec: vec, meta: meta} + return nil + } + } s.items = append(s.items, item{id: id, vec: vec, meta: meta}) - s.mu.Unlock() + return nil +} + +// ByPrefix implements Catalog. +func (s *InMemoryStore) ByPrefix(_ context.Context, prefix string) ([]Record, error) { + s.mu.RLock() + defer s.mu.RUnlock() + var out []Record + for _, it := range s.items { + if !strings.HasPrefix(it.id, prefix) { + continue + } + // Copy the metadata too. Returning it.meta by reference let a caller + // mutating the returned map edit the stored row, and the persistent + // backend unmarshals fresh, so the two disagreed. + meta := make(map[string]string, len(it.meta)) + for k, v := range it.meta { + meta[k] = v + } + out = append(out, Record{ID: it.id, Vec: append([]float32(nil), it.vec...), Meta: meta}) + } + return out, nil +} + +// Delete implements Catalog. +func (s *InMemoryStore) Delete(_ context.Context, id string) error { + s.mu.Lock() + defer s.mu.Unlock() + for i := range s.items { + if s.items[i].id == id { + s.items = append(s.items[:i], s.items[i+1:]...) + return nil + } + } return nil } @@ -59,6 +148,9 @@ func (s *InMemoryStore) Search(_ context.Context, vec []float32, topK int) ([]Re scores := make([]scored, 0, len(s.items)) for _, it := range s.items { + if strings.HasPrefix(it.id, NonRecallPrefix) { + continue + } score := cosine(vec, it.vec) scores = append(scores, scored{id: it.id, score: score, meta: it.meta}) } diff --git a/internal/memory/store_test.go b/internal/memory/store_test.go index f066cc7..62c69a7 100644 --- a/internal/memory/store_test.go +++ b/internal/memory/store_test.go @@ -2,6 +2,7 @@ package memory import ( "context" + "fmt" "math" "testing" ) @@ -40,8 +41,9 @@ func TestTopKTruncation(t *testing.T) { s := NewInMemoryStore() ctx := context.Background() + // Distinct ids: Insert upserts by id, so ten rows need ten ids. for i := 0; i < 10; i++ { - s.Insert(ctx, "", []float32{float32(i) / 10, 0, 0}, nil) + s.Insert(ctx, fmt.Sprintf("n%d", i), []float32{float32(i) / 10, 0, 0}, nil) } results, err := s.Search(ctx, []float32{1, 0, 0}, 3) @@ -78,3 +80,75 @@ func TestCosineEdgeCases(t *testing.T) { t.Errorf("dot(1,2;1,2) = %f, want 5", c) } } + +// The in-memory backend has to hide voiceprints from recall exactly like the +// persistent one, or a test passing here says nothing about the daemon. +func TestInMemorySearchSkipsVoiceprints(t *testing.T) { + ctx := context.Background() + s := NewInMemoryStore() + if err := s.Insert(ctx, "note:1", []float32{0, 1}, map[string]string{"text": "заметка"}); err != nil { + t.Fatal(err) + } + if err := s.Insert(ctx, NonRecallPrefix+"kami", []float32{1, 0}, map[string]string{"name": "Ками"}); err != nil { + t.Fatal(err) + } + got, err := s.Search(ctx, []float32{1, 0}, 10) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].ID != "note:1" { + t.Fatalf("Search = %+v, want just the note", got) + } + recs, err := s.ByPrefix(ctx, NonRecallPrefix) + if err != nil || len(recs) != 1 { + t.Fatalf("ByPrefix = %+v, %v; want the voiceprint", recs, err) + } +} + +// ByPrefix hands back a copy of the metadata. It used to return the stored map +// by reference, so a caller editing a returned Record silently edited the row, +// and the persistent backend did not behave that way. +func TestInMemoryByPrefixCopiesMeta(t *testing.T) { + ctx := context.Background() + s := NewInMemoryStore() + if err := s.Insert(ctx, "speaker:kami", []float32{1, 0}, map[string]string{"name": "Ками"}); err != nil { + t.Fatal(err) + } + recs, err := s.ByPrefix(ctx, "speaker:") + if err != nil || len(recs) != 1 { + t.Fatalf("ByPrefix = %+v, %v", recs, err) + } + recs[0].Meta["name"] = "не Ками" + + again, err := s.ByPrefix(ctx, "speaker:") + if err != nil { + t.Fatal(err) + } + if again[0].Meta["name"] != "Ками" { + t.Errorf("the stored row was edited through the returned map: %q", again[0].Meta["name"]) + } +} + +// Insert upserts. This is not only a speaker-profile concern: every user of the +// in-memory store used to accumulate a second row for a re-indexed id, and the +// stale copy stayed searchable. +func TestInMemoryInsertUpserts(t *testing.T) { + ctx := context.Background() + s := NewInMemoryStore() + if err := s.Insert(ctx, "note:1", []float32{1, 0}, map[string]string{"text": "старое"}); err != nil { + t.Fatal(err) + } + if err := s.Insert(ctx, "note:1", []float32{0, 1}, map[string]string{"text": "новое"}); err != nil { + t.Fatal(err) + } + got, err := s.Search(ctx, []float32{1, 0}, 10) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 { + t.Fatalf("Search returned %d rows, want 1 (the old copy is still searchable)", len(got)) + } + if got[0].Meta["text"] != "новое" { + t.Errorf("row = %q, want the replacement", got[0].Meta["text"]) + } +} diff --git a/internal/morning/morning.go b/internal/morning/morning.go index 8a94db7..beabb0b 100644 --- a/internal/morning/morning.go +++ b/internal/morning/morning.go @@ -140,6 +140,29 @@ func Evaluate(r Routine, facts map[string]store.Fact, now time.Time) Status { return st } +// Outstanding reports the items of a routine that today has no evidence for, +// whether or not the window is still open. Evaluate answers "what is missing +// right now" and goes silent the moment the window closes; the day plan asks a +// different question, "what did today still not get done", and a skipped +// routine is exactly what it is worth telling him. Nothing before the window +// opens is outstanding yet, so the morning routine is not a complaint at 06:00. +func Outstanding(r Routine, facts map[string]store.Fact, now time.Time) []Item { + if !appliesToday(r, now) { + return nil + } + start, ok := todayAt(r.WindowStart, now) + if !ok || now.Before(start) { + return nil + } + var missing []Item + for _, it := range r.Items { + if !evidenced(it, facts, start, now) { + missing = append(missing, it) + } + } + return missing +} + // Due returns the routines that have reached their nudge time today with at // least one item still missing, and records `now` in `last` for each one // returned so it fires at most once per calendar day. The caller owns diff --git a/internal/morning/plan.go b/internal/morning/plan.go new file mode 100644 index 0000000..ba2c699 --- /dev/null +++ b/internal/morning/plan.go @@ -0,0 +1,180 @@ +package morning + +import ( + "fmt" + "sort" + "strings" + "time" + + "github.com/kami/maven/internal/store" +) + +// The day plan (Vikunja #128). +// +// It lives here, with the morning routine engine, because it is the same +// question asked at a different scale: the routine knows what is still missing +// from a window, the plan knows what the whole day holds. A parallel system +// would have to re-read the same facts and re-decide what "today" means. +// +// It is pure, like the rest of this package: the daemon reads the calendar, +// the reminders and the checklist facts, and BuildPlan puts them in order. +// +// It is also NOT a nag. A plan she can recite when asked is the whole feature; +// nothing here fires, schedules or announces. Unprompted delivery stays with +// the existing morning nudge and the dispatcher's policy. + +// PlanKind — where a plan line came from. It survives into the reply and the +// web view because the three read differently: an event is something happening +// to the owner, a reminder is something he asked for, a checklist item is +// something he has not done yet. +type PlanKind string + +const ( + PlanEvent PlanKind = "event" + PlanReminder PlanKind = "reminder" + PlanChecklist PlanKind = "checklist" +) + +// PlanEntry — one timed thing on the day, as the daemon read it out of the +// store. Text is rendered verbatim; the plan does not rephrase. +// +// Uncertain marks provenance below a full-confidence read — a work meeting +// relayed off a phone notification (#126). It travels through to the reply so +// she hedges instead of reciting a guess as fact. +type PlanEntry struct { + At time.Time + Text string + Kind PlanKind + Uncertain bool +} + +// Plan — the ordered day. Date is the calendar day it describes. Rest marks a +// plan trimmed by After, which changes what an empty one means: a day with +// nothing on it and a day whose last item has passed are different answers. +type Plan struct { + Date time.Time + Items []PlanEntry + Rest bool +} + +// BuildPlan orders everything known about the day Now falls on: calendar +// events, pending reminders, and one line per morning routine that still has +// unfinished items. +// +// Entries outside that calendar day are dropped — a plan for today that +// includes tomorrow's meeting is wrong in a way that is worse than terse. +// Ordering is by time, then by kind, then by text, so the same day always reads +// the same way. +func BuildPlan(routines []Routine, facts map[string]store.Fact, events, reminders []PlanEntry, now time.Time) Plan { + y, m, d := now.Date() + dayStart := time.Date(y, m, d, 0, 0, 0, 0, now.Location()) + dayEnd := dayStart.AddDate(0, 0, 1) + + p := Plan{Date: dayStart} + for _, group := range [][]PlanEntry{events, reminders} { + for _, e := range group { + at := e.At.In(now.Location()) + if at.Before(dayStart) || !at.Before(dayEnd) { + continue + } + if strings.TrimSpace(e.Text) == "" { + continue + } + e.At = at + p.Items = append(p.Items, e) + } + } + p.Items = append(p.Items, checklistEntries(routines, facts, now)...) + + sort.SliceStable(p.Items, func(i, j int) bool { + a, b := p.Items[i], p.Items[j] + if !a.At.Equal(b.At) { + return a.At.Before(b.At) + } + if a.Kind != b.Kind { + return a.Kind < b.Kind + } + return a.Text < b.Text + }) + return p +} + +// checklistEntries renders one line per routine with work left in it, placed at +// the routine's nudge time — where the checklist actually matters in the day. +// A routine that does not apply today, has not opened yet, or is already +// complete contributes nothing: the plan says what is left, not what was done. +// +// A closed window still counts. Asked at 14:00 with the morning routine +// unfinished, the plan used to say nothing about it, because Evaluate reports +// Active only inside the window. What he skipped is the one thing the plan can +// tell him that the calendar cannot, and the entry sorts to its nudge time, not +// to the moment of asking. +func checklistEntries(routines []Routine, facts map[string]store.Fact, now time.Time) []PlanEntry { + var out []PlanEntry + for _, r := range routines { + missing := Outstanding(r, facts, now) + if len(missing) == 0 { + continue + } + labels := make([]string, 0, len(missing)) + for _, it := range missing { + label := it.Label + if label == "" { + label = it.Key + } + labels = append(labels, label) + } + at := r.NudgeAt + if at == "" { + at = r.WindowEnd + } + when, ok := todayAt(at, now) + if !ok { + continue + } + out = append(out, PlanEntry{ + At: when, + Text: fmt.Sprintf("%s — осталось: %s", r.Name, strings.Join(labels, ", ")), + Kind: PlanChecklist, + }) + } + return out +} + +// After returns the part of the plan that has not happened yet — the answer to +// "что дальше?" as opposed to "какие планы на сегодня?". The Date is kept, so an +// empty result still knows which day it is empty for. +func (p Plan) After(now time.Time) Plan { + out := Plan{Date: p.Date, Rest: true} + for _, it := range p.Items { + if it.At.Before(now) { + continue + } + out.Items = append(out.Items, it) + } + return out +} + +// FormatRU renders the plan as maven says it. Feminine self-reference, +// informal address, no pet names — and no exhortation: she reads the day back, +// she does not tell him to get on with it. +func (p Plan) FormatRU() string { + if len(p.Items) == 0 { + // "что дальше?" after the last item of the day. The day was not empty, + // it is over, and saying it was empty is a false statement about a day + // he just lived. + if p.Rest { + return "на сегодня больше ничего не запланировано." + } + return fmt.Sprintf("на %s ничего не запланировано.", p.Date.Format("02.01.2006")) + } + parts := make([]string, len(p.Items)) + for i, it := range p.Items { + line := fmt.Sprintf("%s — %s", it.At.Format("15:04"), it.Text) + if it.Uncertain { + line = "похоже, " + line + } + parts[i] = line + } + return fmt.Sprintf("план на %s: %s.", p.Date.Format("02.01.2006"), strings.Join(parts, "; ")) +} diff --git a/internal/morning/plan_test.go b/internal/morning/plan_test.go new file mode 100644 index 0000000..c7b282e --- /dev/null +++ b/internal/morning/plan_test.go @@ -0,0 +1,217 @@ +package morning + +import ( + "strings" + "testing" + "time" + + "github.com/kami/maven/internal/store" +) + +// planAt is at() for the plan tests' day (2026-08-03, a Monday); the existing +// at() in morning_test.go is pinned to a different date. +func planAt(now time.Time, hh, mm int) time.Time { + y, m, d := now.Date() + return time.Date(y, m, d, hh, mm, 0, 0, now.Location()) +} + +func planFixture(t *testing.T) (Plan, time.Time) { + t.Helper() + now := time.Date(2026, 8, 3, 9, 0, 0, 0, time.UTC) + routines := []Routine{{ + Name: "утро", + WindowStart: "07:00", + WindowEnd: "11:00", + NudgeAt: "10:30", + Items: []Item{ + {Key: "water", FactKey: "drank_water", Label: "выпить воды"}, + {Key: "pills", FactKey: "took_pills", Label: "витамины"}, + }, + }} + facts := map[string]store.Fact{ + "drank_water": {Ts: planAt(now, 8, 0)}, + } + events := []PlanEntry{ + {At: planAt(now, 14, 0), Text: "Планёрка @ 14:00-14:30", Kind: PlanEvent, Uncertain: true}, + {At: planAt(now, 10, 0), Text: "Standup @ 10:00-10:30", Kind: PlanEvent}, + } + reminders := []PlanEntry{ + {At: planAt(now, 18, 30), Text: "позвонить маме", Kind: PlanReminder}, + } + return BuildPlan(routines, facts, events, reminders, now), now +} + +func TestBuildPlanOrdersTheDay(t *testing.T) { + p, now := planFixture(t) + + if !p.Date.Equal(planAt(now, 0, 0)) { + t.Errorf("Date = %v, want midnight of now's day", p.Date) + } + want := []struct { + hhmm string + kind PlanKind + }{ + {"10:00", PlanEvent}, + {"10:30", PlanChecklist}, + {"14:00", PlanEvent}, + {"18:30", PlanReminder}, + } + if len(p.Items) != len(want) { + t.Fatalf("got %d items, want %d: %+v", len(p.Items), len(want), p.Items) + } + for i, w := range want { + if got := p.Items[i].At.Format("15:04"); got != w.hhmm { + t.Errorf("item %d at %s, want %s", i, got, w.hhmm) + } + if p.Items[i].Kind != w.kind { + t.Errorf("item %d kind %q, want %q", i, p.Items[i].Kind, w.kind) + } + } +} + +// The checklist line says what is LEFT. An item already evidenced today must +// not be read back as outstanding. +func TestBuildPlanChecklistListsOnlyMissing(t *testing.T) { + p, _ := planFixture(t) + var line string + for _, it := range p.Items { + if it.Kind == PlanChecklist { + line = it.Text + } + } + if line == "" { + t.Fatal("no checklist line in the plan") + } + if !strings.Contains(line, "витамины") { + t.Errorf("missing item not listed: %q", line) + } + if strings.Contains(line, "выпить воды") { + t.Errorf("a completed item must not be read back as outstanding: %q", line) + } + if !strings.HasPrefix(line, "утро — осталось:") { + t.Errorf("line = %q", line) + } +} + +func TestBuildPlanSkipsCompleteAndInactiveRoutines(t *testing.T) { + now := time.Date(2026, 8, 3, 9, 0, 0, 0, time.UTC) + routines := []Routine{ + { + Name: "утро", WindowStart: "07:00", WindowEnd: "11:00", + Items: []Item{{Key: "water", FactKey: "drank_water", Label: "выпить воды"}}, + }, + { + // Not in its window at 09:00. + Name: "вечер", WindowStart: "20:00", WindowEnd: "23:00", + Items: []Item{{Key: "walk", FactKey: "walked", Label: "прогулка"}}, + }, + } + facts := map[string]store.Fact{"drank_water": {Ts: planAt(now, 8, 0)}} + p := BuildPlan(routines, facts, nil, nil, now) + if len(p.Items) != 0 { + t.Fatalf("a complete routine and an out-of-window one must contribute nothing: %+v", p.Items) + } + if got, want := p.FormatRU(), "на 03.08.2026 ничего не запланировано."; got != want { + t.Errorf("got %q\nwant %q", got, want) + } +} + +// A plan for today that includes tomorrow's meeting is worse than terse. +func TestBuildPlanDropsOtherDays(t *testing.T) { + now := time.Date(2026, 8, 3, 9, 0, 0, 0, time.UTC) + events := []PlanEntry{ + {At: planAt(now, 10, 0), Text: "today", Kind: PlanEvent}, + {At: planAt(now, 10, 0).AddDate(0, 0, 1), Text: "tomorrow", Kind: PlanEvent}, + {At: planAt(now, 10, 0).AddDate(0, 0, -1), Text: "yesterday", Kind: PlanEvent}, + {At: planAt(now, 12, 0), Text: " ", Kind: PlanEvent}, + } + p := BuildPlan(nil, nil, events, nil, now) + if len(p.Items) != 1 || p.Items[0].Text != "today" { + t.Fatalf("got %+v", p.Items) + } +} + +func TestPlanFormatRU(t *testing.T) { + p, _ := planFixture(t) + got := p.FormatRU() + want := "план на 03.08.2026: 10:00 — Standup @ 10:00-10:30; " + + "10:30 — утро — осталось: витамины; " + + "похоже, 14:00 — Планёрка @ 14:00-14:30; " + + "18:30 — позвонить маме." + if got != want { + t.Errorf("got %q\nwant %q", got, want) + } + // Persona: she recites, she does not exhort, and she never speaks of + // herself in the masculine or addresses him formally. + for _, bad := range []string{"рад ", "понял", "вы ", "ваш", "милый", "дорогой", "давай же", "не забудь"} { + if strings.Contains(strings.ToLower(got), bad) { + t.Errorf("plan text contains %q: %q", bad, got) + } + } +} + +func TestPlanAfter(t *testing.T) { + p, now := planFixture(t) + rest := p.After(planAt(now, 11, 0)) + if len(rest.Items) != 2 { + t.Fatalf("got %d items, want the 14:00 and 18:30 ones: %+v", len(rest.Items), rest.Items) + } + if !rest.Date.Equal(p.Date) { + t.Error("After must keep the date, so an empty rest-of-day still knows which day") + } + empty := p.After(planAt(now, 23, 0)) + if len(empty.Items) != 0 { + t.Errorf("got %+v", empty.Items) + } + // An empty rest-of-day is not an empty day. Saying "на 03.08.2026 ничего + // не запланировано" at 23:00 denies the day he just lived. + if got, want := empty.FormatRU(), "на сегодня больше ничего не запланировано."; got != want { + t.Errorf("empty rest-of-day reads %q, want %q", got, want) + } +} + +// The plan says what today still has not got done, and a closed window does not +// make a skipped routine untrue. Evaluate reports Active only inside the +// window, so keying the checklist line off it meant the one thing the plan can +// tell him that the calendar cannot went silent at 11:00. +func TestBuildPlanKeepsAClosedWindowOutstanding(t *testing.T) { + now := time.Date(2026, 8, 3, 14, 0, 0, 0, time.UTC) + routines := []Routine{{ + Name: "утро", WindowStart: "07:00", WindowEnd: "11:00", NudgeAt: "10:30", + Items: []Item{ + {Key: "water", FactKey: "drank_water", Label: "выпить воды"}, + {Key: "pills", FactKey: "took_pills", Label: "витамины"}, + }, + }} + facts := map[string]store.Fact{"drank_water": {Ts: planAt(now, 8, 0)}} + + p := BuildPlan(routines, facts, nil, nil, now) + if len(p.Items) != 1 { + t.Fatalf("got %+v, want the unfinished morning routine", p.Items) + } + it := p.Items[0] + if it.Kind != PlanChecklist { + t.Errorf("kind = %q", it.Kind) + } + // Placed at the nudge time, so it sorts to the top of the day rather than + // to the moment of asking. + if got := it.At.Format("15:04"); got != "10:30" { + t.Errorf("placed at %s, want 10:30", got) + } + if !strings.Contains(it.Text, "витамины") || strings.Contains(it.Text, "выпить воды") { + t.Errorf("line = %q", it.Text) + } +} + +// A routine whose window has not opened yet is not outstanding. Nothing has +// been skipped at 06:00. +func TestBuildPlanIgnoresAnUnopenedWindow(t *testing.T) { + now := time.Date(2026, 8, 3, 6, 0, 0, 0, time.UTC) + routines := []Routine{{ + Name: "утро", WindowStart: "07:00", WindowEnd: "11:00", + Items: []Item{{Key: "water", FactKey: "drank_water", Label: "выпить воды"}}, + }} + if p := BuildPlan(routines, nil, nil, nil, now); len(p.Items) != 0 { + t.Fatalf("got %+v", p.Items) + } +} diff --git a/internal/netscan/netscan.go b/internal/netscan/netscan.go new file mode 100644 index 0000000..b2e28fa --- /dev/null +++ b/internal/netscan/netscan.go @@ -0,0 +1,423 @@ +// Package netscan discovers hosts on the LAN Maven is configured to look at +// (Vikunja #257, docs/plans/12-bluetooth-network-scan.md). +// +// A scan is a read, but an unbounded scanner on a home network is noisy and is +// trivially pointed somewhere it should not go, so the whole package is built +// around four rules: +// +// - The target range NEVER comes from an utterance, a router, an LLM or a +// device. Scan takes no target argument at all: it reads only the CIDRs in +// the config block. There is deliberately no exported way to scan an +// arbitrary range, so no amount of prompt injection or a rogue reply from a +// scanned host can retarget it. +// - Every configured CIDR must be private (RFC1918 / CGNAT / link-local) and +// no larger than MaxPrefixHosts addresses. Scanning the public internet +// from his flat is not a thing Maven does, and /8 is not a home LAN. +// - Rate-limited. Connections leave at a fixed rate, so a scan looks like +// background traffic rather than a portscan to anything watching. +// - Bounded in total. MaxHosts, a per-connection timeout and the caller's +// context all cap the work; a scan that runs long returns what it has. +// +// It is a TCP-connect scan (net.DialTimeout) and an ARP-table read. No raw +// sockets, no SYN scan, no privileges: mavend does not run as root and this +// does not ask it to. +package netscan + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "net" + "net/netip" + "os" + "sort" + "strings" + "sync" + "time" +) + +// DefaultPorts — what a scan looks at when the config names nothing. Chosen to +// answer "what is this box" on a home network, not to find a way in. +var DefaultPorts = []int{22, 80, 443, 8080} + +const ( + // DefaultTimeout — per-connection budget. Short: on a LAN a live host + // answers in single-digit milliseconds, and a filtered port never answers. + DefaultTimeout = 400 * time.Millisecond + // DefaultRate — connections per second across the whole scan. A default + // /24 at four ports is 1016 probes, so this is also what decides whether + // the shipped configuration fits inside the caller's budget: at 100/s it + // takes about ten seconds. Lowering it means a truncated scan, which is + // reported rather than hidden, but it is still a worse answer. + DefaultRate = 100 + // MaxRate — the highest configurable rate. The dial loop floors the ticker + // interval at a millisecond, so anything above this was already a lie; say + // so at config load instead of silently clamping. + MaxRate = 1000 + // DefaultMaxHosts — cap on addresses probed in one scan. + DefaultMaxHosts = 256 + // MaxPrefixHosts — the largest CIDR that may be configured, in addresses. + // 1024 is a /22: generous for a flat, and far short of anything that would + // take minutes or wake up a neighbour's IDS. + MaxPrefixHosts = 1024 + // maxParallel — in-flight dials. The rate limiter is the real throttle; + // this only stops a slow subnet from piling up file descriptors. + maxParallel = 16 + // arpFile — the kernel's ARP cache. Reading it is free and needs no packet. + arpFile = "/proc/net/arp" +) + +var ( + // ErrNotConfigured — no netscan block, or it is disabled. + ErrNotConfigured = errors.New("netscan: not configured") + // ErrNoSubnets — enabled with nothing to scan. + ErrNoSubnets = errors.New("netscan: no subnets configured") +) + +// Config — the bounds of every scan. There is nothing here that can be +// overridden at call time. +type Config struct { + // Subnets — the ONLY ranges that are ever probed, as CIDRs. Each must be + // private and no bigger than MaxPrefixHosts. + Subnets []string + // Ports — TCP ports to try on each host. Empty ⇒ DefaultPorts. + Ports []int + // Timeout — per-connection budget. 0 ⇒ DefaultTimeout. + Timeout time.Duration + // Rate — connections per second. 0 ⇒ DefaultRate. + Rate int + // MaxHosts — cap on addresses probed per scan. 0 ⇒ DefaultMaxHosts. + MaxHosts int +} + +// Host is one machine the scan saw. +type Host struct { + // Addr — the IP. + Addr string + // MAC — from the ARP cache, empty when the kernel has no entry. + MAC string + // Ports — open TCP ports, ascending. + Ports []int +} + +// Up reports whether anything at all answered for this host. +func (h Host) Up() bool { return len(h.Ports) > 0 || h.MAC != "" } + +// Validate rejects a block that cannot safely run, at config-load time rather +// than at the first spoken scan. This is the guard the whole package rides on: +// if it passes, every later scan is inside these bounds by construction. +func Validate(c Config) error { + if len(c.Subnets) == 0 { + return ErrNoSubnets + } + for _, s := range c.Subnets { + p, err := netip.ParsePrefix(strings.TrimSpace(s)) + if err != nil { + return fmt.Errorf("netscan: subnet %q: %w", s, err) + } + if !p.Addr().Is4() { + return fmt.Errorf("netscan: subnet %q: only IPv4 is scanned", s) + } + if !isPrivate(p.Addr()) { + return fmt.Errorf("netscan: subnet %q is not a private range: Maven does not scan the public internet", s) + } + if n := prefixHosts(p); n > MaxPrefixHosts { + return fmt.Errorf("netscan: subnet %q covers %d addresses, limit is %d: narrow the prefix", s, n, MaxPrefixHosts) + } + } + for _, port := range c.Ports { + if port < 1 || port > 65535 { + return fmt.Errorf("netscan: port %d out of range", port) + } + } + if c.Rate < 0 || c.MaxHosts < 0 || c.Timeout < 0 { + return errors.New("netscan: rate, max_hosts and timeout must not be negative") + } + if c.Rate > MaxRate { + return fmt.Errorf("netscan: rate %d is above the ceiling of %d connections per second", c.Rate, MaxRate) + } + return nil +} + +// isPrivate — RFC1918, CGNAT and link-local. Loopback counts: scanning this box +// is harmless and is how the tests run. +func isPrivate(a netip.Addr) bool { + if a.IsLoopback() || a.IsPrivate() || a.IsLinkLocalUnicast() { + return true + } + // 100.64.0.0/10, the carrier-grade NAT range Tailscale hands out. + cgnat := netip.MustParsePrefix("100.64.0.0/10") + return cgnat.Contains(a) +} + +// prefixHosts — addresses covered by a v4 prefix. +func prefixHosts(p netip.Prefix) int { + bits := 32 - p.Bits() + if bits >= 31 { + return MaxPrefixHosts + 1 + } + return 1 << bits +} + +// Scanner probes the configured subnets. Build it with New; the config it holds +// is the config it was validated with, and nothing mutates it afterwards. +type Scanner struct { + cfg Config + // dial is the connect seam; tests swap it. + dial func(ctx context.Context, addr string, timeout time.Duration) bool + // arp is the ARP-cache seam; tests swap it. + arp func() (map[string]string, error) +} + +// New builds a scanner. Validate first — this does not. +func New(cfg Config) *Scanner { + if len(cfg.Ports) == 0 { + cfg.Ports = append([]int(nil), DefaultPorts...) + } + if cfg.Timeout <= 0 { + cfg.Timeout = DefaultTimeout + } + if cfg.Rate <= 0 { + cfg.Rate = DefaultRate + } + if cfg.MaxHosts <= 0 { + cfg.MaxHosts = DefaultMaxHosts + } + return &Scanner{cfg: cfg, dial: dialTCP, arp: readARP} +} + +// expand lists the scannable addresses of one CIDR, skipping the network and +// broadcast address. +func expand(cidr string) []netip.Addr { + p, err := netip.ParsePrefix(strings.TrimSpace(cidr)) + if err != nil { + return nil + } + p = p.Masked() + first := p.Addr() + var out []netip.Addr + for a := first; p.Contains(a); a = a.Next() { + // Skip the network address; the broadcast address is skipped by + // looking one ahead. + if a == first && p.Bits() < 31 { + continue + } + if p.Bits() < 31 && !p.Contains(a.Next()) { + continue + } + out = append(out, a) + } + return out +} + +// targets expands the configured subnets into addresses, capped at MaxHosts, +// and reports whether the cap cut anything off. Deterministic order, so two +// scans of an unchanged network read the same. +// +// The subnets are taken round-robin rather than in order. Consuming MaxHosts +// from the first subnet used to leave a second configured LAN 99% unprobed, +// with nothing logged: he named two ranges and got an answer about one. +// Round-robin spends the budget evenly, so every named range is represented +// and the shortfall is reported instead. +func (s *Scanner) targets() ([]netip.Addr, bool) { + lists := make([][]netip.Addr, 0, len(s.cfg.Subnets)) + total := 0 + for _, cidr := range s.cfg.Subnets { + l := expand(cidr) + if len(l) == 0 { + continue + } + lists = append(lists, l) + total += len(l) + } + out := make([]netip.Addr, 0, min(total, s.cfg.MaxHosts)) + for i := 0; len(out) < s.cfg.MaxHosts; i++ { + took := false + for _, l := range lists { + if i >= len(l) { + continue + } + if len(out) >= s.cfg.MaxHosts { + break + } + out = append(out, l[i]) + took = true + } + if !took { + break + } + } + return out, len(out) < total +} + +// Result is one scan's outcome. +type Result struct { + // Hosts — what answered, ascending by address. + Hosts []Host + // Truncated — the scan did not cover every configured address, because + // MaxHosts cut the target list or the caller's context expired mid-run. + // Callers MUST NOT present a truncated result as the state of the network: + // "нашла 6 устройств" is a claim about the LAN, and a scan that stopped at + // .238 has not earned it. + Truncated bool +} + +// Scan probes every configured address and returns the hosts that answered. +// +// It takes no target: the range is the configured one, always. Callers pass a +// context and nothing else, which is the point — see the package comment. +func (s *Scanner) Scan(ctx context.Context) (Result, error) { + if len(s.cfg.Subnets) == 0 { + return Result{}, ErrNoSubnets + } + arp, err := s.arp() + if err != nil { + // A missing /proc/net/arp costs MAC addresses, not the scan. + arp = map[string]string{} + } + + // One token per connection, at Rate per second, shared by every worker. + interval := time.Second / time.Duration(s.cfg.Rate) + if interval <= 0 { + interval = time.Millisecond + } + tick := time.NewTicker(interval) + defer tick.Stop() + + type result struct { + addr string + ports []int + } + targets, truncated := s.targets() + // One slot per PROBE, not per host: a worker sends once per open port, so + // a subnet with more open ports than addresses used to fill a host-sized + // buffer and wedge. Nothing drains this channel until wg.Wait returns, and + // the sends carry no select on ctx.Done, so that was a permanent hang of + // the calling turn plus a leak of every worker. + results := make(chan result, len(targets)*len(s.cfg.Ports)) + sem := make(chan struct{}, maxParallel) + var wg sync.WaitGroup + +scan: + for _, a := range targets { + addr := a.String() + for _, port := range s.cfg.Ports { + // Checked before the select as well as inside it: select picks + // randomly among ready cases, so at a high rate the ticker would + // sometimes win over an already-canceled context and let one more + // probe out. + if ctx.Err() != nil { + truncated = true + break scan + } + select { + case <-ctx.Done(): + truncated = true + break scan + case <-tick.C: + } + sem <- struct{}{} + wg.Add(1) + go func(addr string, port int) { + defer wg.Done() + defer func() { <-sem }() + if s.dial(ctx, net.JoinHostPort(addr, itoa(port)), s.cfg.Timeout) { + results <- result{addr: addr, ports: []int{port}} + } + }(addr, port) + } + } + wg.Wait() + close(results) + + byAddr := map[string]*Host{} + for r := range results { + h := byAddr[r.addr] + if h == nil { + h = &Host{Addr: r.addr} + byAddr[r.addr] = h + } + h.Ports = append(h.Ports, r.ports...) + } + // A host in the ARP cache is up even with every port closed — it answered + // an ARP request, which is the cheapest liveness signal there is. + for _, a := range targets { + addr := a.String() + mac, ok := arp[addr] + if !ok { + continue + } + if byAddr[addr] == nil { + byAddr[addr] = &Host{Addr: addr} + } + byAddr[addr].MAC = mac + } + + out := make([]Host, 0, len(byAddr)) + for _, h := range byAddr { + sort.Ints(h.Ports) + if !h.Up() { + continue + } + out = append(out, *h) + } + sort.Slice(out, func(i, j int) bool { + ai, _ := netip.ParseAddr(out[i].Addr) + aj, _ := netip.ParseAddr(out[j].Addr) + return ai.Less(aj) + }) + return Result{Hosts: out, Truncated: truncated}, nil +} + +func itoa(n int) string { return fmt.Sprintf("%d", n) } + +func dialTCP(ctx context.Context, addr string, timeout time.Duration) bool { + d := net.Dialer{Timeout: timeout} + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + c, err := d.DialContext(ctx, "tcp", addr) + if err != nil { + return false + } + _ = c.Close() + return true +} + +func readARP() (map[string]string, error) { + f, err := os.Open(arpFile) + if err != nil { + return nil, err + } + defer f.Close() + return parseARP(f) +} + +// parseARP reads the kernel's ARP table. Incomplete entries (all-zero MAC, +// flags 0x0) are dropped: they mean "we asked and nobody answered", which is +// the opposite of a discovered host. +func parseARP(r io.Reader) (map[string]string, error) { + out := map[string]string{} + sc := bufio.NewScanner(r) + first := true + for sc.Scan() { + if first { // header row + first = false + continue + } + f := strings.Fields(sc.Text()) + if len(f) < 4 { + continue + } + ip, flags, mac := f[0], f[2], f[3] + if flags == "0x0" || mac == "00:00:00:00:00:00" { + continue + } + if _, err := netip.ParseAddr(ip); err != nil { + continue + } + out[ip] = mac + } + return out, sc.Err() +} diff --git a/internal/netscan/netscan_test.go b/internal/netscan/netscan_test.go new file mode 100644 index 0000000..772e5e5 --- /dev/null +++ b/internal/netscan/netscan_test.go @@ -0,0 +1,294 @@ +package netscan + +import ( + "context" + "errors" + "net/netip" + "strings" + "sync" + "testing" + "time" +) + +func TestValidateBounds(t *testing.T) { + ok := []Config{ + {Subnets: []string{"192.168.1.0/24"}}, + {Subnets: []string{"10.0.0.0/24", "172.16.5.0/28"}, Ports: []int{22, 80}}, + {Subnets: []string{"127.0.0.1/32"}}, + {Subnets: []string{"100.64.1.0/24"}}, // CGNAT / tailnet + } + for _, c := range ok { + if err := Validate(c); err != nil { + t.Errorf("Validate(%v) = %v, want nil", c.Subnets, err) + } + } + + bad := map[string]Config{ + "nothing to scan": {}, + "public range": {Subnets: []string{"8.8.8.0/24"}}, + "whole internet": {Subnets: []string{"0.0.0.0/0"}}, + "a slash-8 is not a flat": {Subnets: []string{"10.0.0.0/8"}}, + "a /16 is too big": {Subnets: []string{"192.168.0.0/16"}}, + "not a cidr": {Subnets: []string{"192.168.1.1"}}, + "ipv6": {Subnets: []string{"fd00::/120"}}, + "garbage": {Subnets: []string{"выключи свет"}}, + "bad port": {Subnets: []string{"192.168.1.0/24"}, Ports: []int{0}}, + "huge port": {Subnets: []string{"192.168.1.0/24"}, Ports: []int{70000}}, + "negative rate": {Subnets: []string{"192.168.1.0/24"}, Rate: -1}, + "rate past the ceiling": {Subnets: []string{"192.168.1.0/24"}, Rate: MaxRate + 1}, + } + for name, c := range bad { + if err := Validate(c); err == nil { + t.Errorf("Validate(%s) = nil, want an error", strings.ReplaceAll(name, "\n", " ")) + } + } + if !errors.Is(Validate(Config{}), ErrNoSubnets) { + t.Error("an empty block should report ErrNoSubnets") + } +} + +// The whole safety story: a scanner probes its configured range and nothing +// else. There is no API that takes a target, so this test asserts the negative +// by watching every address the dialer was handed. +func TestScanOnlyTouchesConfiguredSubnet(t *testing.T) { + s := New(Config{Subnets: []string{"192.168.9.0/29"}, Ports: []int{80}, Rate: 10000}) + inside := netip.MustParsePrefix("192.168.9.0/29") + + var mu sync.Mutex + var seen []string + s.dial = func(_ context.Context, addr string, _ time.Duration) bool { + mu.Lock() + seen = append(seen, addr) + mu.Unlock() + return addr == "192.168.9.3:80" + } + s.arp = func() (map[string]string, error) { return map[string]string{}, nil } + + res, err := s.Scan(context.Background()) + if err != nil { + t.Fatalf("Scan: %v", err) + } + hosts := res.Hosts + if len(hosts) != 1 || hosts[0].Addr != "192.168.9.3" || len(hosts[0].Ports) != 1 { + t.Fatalf("hosts = %+v", hosts) + } + // A /29 is 8 addresses; network (.0) and broadcast (.7) are skipped. + if len(seen) != 6 { + t.Errorf("probed %d addresses, want 6 (a /29 minus network and broadcast): %v", len(seen), seen) + } + for _, a := range seen { + host, _, _ := strings.Cut(a, ":") + ip, err := netip.ParseAddr(host) + if err != nil || !inside.Contains(ip) { + t.Errorf("probed %q, which is outside the configured subnet", a) + } + } +} + +func TestScanHonoursMaxHosts(t *testing.T) { + s := New(Config{Subnets: []string{"192.168.9.0/24"}, Ports: []int{80}, Rate: 10000, MaxHosts: 3}) + var mu sync.Mutex + n := 0 + s.dial = func(_ context.Context, _ string, _ time.Duration) bool { + mu.Lock() + n++ + mu.Unlock() + return false + } + s.arp = func() (map[string]string, error) { return nil, nil } + if _, err := s.Scan(context.Background()); err != nil { + t.Fatal(err) + } + if n != 3 { + t.Errorf("dialed %d times, want 3 (MaxHosts)", n) + } +} + +// The rate limiter must actually gate: 6 probes at 200/s cannot finish in less +// than ~25ms. Asserted loosely, since a CI box is not a stopwatch. +func TestScanIsRateLimited(t *testing.T) { + s := New(Config{Subnets: []string{"192.168.9.0/29"}, Ports: []int{80}, Rate: 200}) + s.dial = func(context.Context, string, time.Duration) bool { return false } + s.arp = func() (map[string]string, error) { return nil, nil } + start := time.Now() + if _, err := s.Scan(context.Background()); err != nil { + t.Fatal(err) + } + if el := time.Since(start); el < 20*time.Millisecond { + t.Errorf("6 probes at 200/s took %v: the rate limiter is not gating", el) + } +} + +func TestScanStopsOnCanceledContext(t *testing.T) { + s := New(Config{Subnets: []string{"192.168.9.0/24"}, Ports: []int{80}, Rate: 10000}) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + s.dial = func(context.Context, string, time.Duration) bool { + t.Error("a canceled scan still dialed") + return false + } + s.arp = func() (map[string]string, error) { return nil, nil } + if _, err := s.Scan(ctx); err != nil { + t.Fatal(err) + } +} + +// A host with every port closed but an ARP entry is still up. A host outside +// the configured range must not be reported even if the kernel knows it — +// otherwise the ARP cache, which is populated by the network rather than by +// Maven, would widen the answer past what he configured. +func TestARPFillsMACWithinTheConfiguredRangeOnly(t *testing.T) { + s := New(Config{Subnets: []string{"192.168.9.0/29"}, Ports: []int{80}, Rate: 10000}) + s.dial = func(context.Context, string, time.Duration) bool { return false } + s.arp = func() (map[string]string, error) { + return map[string]string{ + "192.168.9.2": "aa:bb:cc:dd:ee:ff", + "10.9.9.9": "11:22:33:44:55:66", + }, nil + } + res, err := s.Scan(context.Background()) + if err != nil { + t.Fatal(err) + } + hosts := res.Hosts + if len(hosts) != 1 { + t.Fatalf("hosts = %+v", hosts) + } + if hosts[0].Addr != "192.168.9.2" || hosts[0].MAC != "aa:bb:cc:dd:ee:ff" { + t.Errorf("host = %+v", hosts[0]) + } + if !hosts[0].Up() { + t.Error("an ARP entry with no open port is still a live host") + } +} + +const arpFixture = `IP address HW type Flags HW address Mask Device +192.168.1.1 0x1 0x2 3c:84:6a:11:22:33 * wlp1s0 +192.168.1.50 0x1 0x2 b8:27:eb:44:55:66 * wlp1s0 +192.168.1.77 0x1 0x0 00:00:00:00:00:00 * wlp1s0 +not-an-ip 0x1 0x2 de:ad:be:ef:00:01 * wlp1s0 +short line +` + +func TestParseARP(t *testing.T) { + got, err := parseARP(strings.NewReader(arpFixture)) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 { + t.Fatalf("got %d entries, want 2: %v", len(got), got) + } + if got["192.168.1.1"] != "3c:84:6a:11:22:33" || got["192.168.1.50"] != "b8:27:eb:44:55:66" { + t.Errorf("entries = %v", got) + } + if _, ok := got["192.168.1.77"]; ok { + t.Error("an incomplete ARP entry (flags 0x0) is not a discovered host") + } +} + +func TestNewAppliesDefaults(t *testing.T) { + s := New(Config{Subnets: []string{"192.168.1.0/24"}}) + if len(s.cfg.Ports) != len(DefaultPorts) || s.cfg.Rate != DefaultRate || + s.cfg.MaxHosts != DefaultMaxHosts || s.cfg.Timeout != DefaultTimeout { + t.Errorf("defaults not applied: %+v", s.cfg) + } + // The defaults must not alias the package slice, or a second scanner could + // rewrite DefaultPorts through it. + s.cfg.Ports[0] = 9999 + if DefaultPorts[0] == 9999 { + t.Error("New aliased DefaultPorts") + } +} + +// A dense subnet must not wedge the scan. The results channel used to be sized +// by the number of HOSTS while a worker sends once per open PORT, so a range +// where the open ports outnumber the addresses filled the buffer, blocked a +// worker inside wg.Wait, and hung Scan forever. Nothing drains the channel +// before wg.Wait returns and the sends carry no ctx.Done case, so the caller's +// deadline did not rescue it either. +// +// Six addresses, eight ports, everything open: 48 sends against a buffer that +// used to hold 6. Against the old code this test does not fail, it hangs, so +// the scan runs on its own goroutine with a deadline around it. +func TestScanDoesNotWedgeWhenPortsOutnumberHosts(t *testing.T) { + ports := []int{22, 80, 443, 8080, 8443, 9000, 9100, 9200} + s := New(Config{Subnets: []string{"192.168.9.0/29"}, Ports: ports, Rate: MaxRate}) + s.dial = func(context.Context, string, time.Duration) bool { return true } + s.arp = func() (map[string]string, error) { return nil, nil } + + done := make(chan Result, 1) + go func() { + res, err := s.Scan(context.Background()) + if err != nil { + t.Error(err) + } + done <- res + }() + select { + case res := <-done: + if len(res.Hosts) != 6 { + t.Fatalf("hosts = %d, want 6: %+v", len(res.Hosts), res.Hosts) + } + for _, h := range res.Hosts { + if len(h.Ports) != len(ports) { + t.Errorf("%s reported %d open ports, want %d", h.Addr, len(h.Ports), len(ports)) + } + } + case <-time.After(10 * time.Second): + t.Fatal("Scan did not return: the results channel is sized by hosts, not by probes") + } +} + +// MaxHosts is spent evenly across the configured subnets. Taking it in order +// meant a second configured LAN got whatever the first left over, which for a +// pair of /24s under the default cap was two addresses out of 254. +func TestTargetsSpreadAcrossSubnets(t *testing.T) { + s := New(Config{Subnets: []string{"192.168.1.0/24", "192.168.2.0/24"}, MaxHosts: 20}) + targets, truncated := s.targets() + if !truncated { + t.Error("508 addresses under a cap of 20 is a truncated target list") + } + if len(targets) != 20 { + t.Fatalf("targets = %d, want 20", len(targets)) + } + var first, second int + for _, a := range targets { + switch { + case netip.MustParsePrefix("192.168.1.0/24").Contains(a): + first++ + case netip.MustParsePrefix("192.168.2.0/24").Contains(a): + second++ + } + } + if first != 10 || second != 10 { + t.Errorf("split %d/%d across the two subnets, want 10/10", first, second) + } +} + +// A run cut short by the caller's deadline reports itself as truncated, so the +// spoken answer can stop claiming to describe the whole network. +func TestScanReportsTruncation(t *testing.T) { + s := New(Config{Subnets: []string{"192.168.9.0/24"}, Ports: []int{80}, Rate: 200}) + s.dial = func(context.Context, string, time.Duration) bool { return false } + s.arp = func() (map[string]string, error) { return nil, nil } + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + res, err := s.Scan(ctx) + if err != nil { + t.Fatal(err) + } + if !res.Truncated { + t.Error("a scan stopped by the deadline must report Truncated") + } + + full := New(Config{Subnets: []string{"192.168.9.0/29"}, Ports: []int{80}, Rate: MaxRate}) + full.dial = func(context.Context, string, time.Duration) bool { return false } + full.arp = func() (map[string]string, error) { return nil, nil } + res, err = full.Scan(context.Background()) + if err != nil { + t.Fatal(err) + } + if res.Truncated { + t.Error("a scan that covered every configured address is not truncated") + } +} diff --git a/internal/pattern/detector.go b/internal/pattern/detector.go index ffb92c2..741d81f 100644 --- a/internal/pattern/detector.go +++ b/internal/pattern/detector.go @@ -3,6 +3,7 @@ package pattern import ( "fmt" "math" + "sort" "strings" ) @@ -11,23 +12,57 @@ import ( type ProposedRoutine struct { Action string Object string - IntervalDays float64 // mean interval in days (float for sub-day precision) + IntervalDays float64 // median of the on-pattern intervals, in days N int // number of events used } -// MaxIntervalRatio is the maximum ratio between the longest and shortest -// interval for a pattern to be considered stable. ±50% variance allowed. +// MaxIntervalRatio — how far an interval may sit from the median and still +// count as on-pattern. 1.5 means a 7-day rhythm accepts gaps between ~4.7 and +// ~10.5 days. +// +// It is applied per interval against the MEDIAN, not to the longest/shortest +// pair. The old extremes test asked "is every gap similar to every other gap", +// which is a different and much more brittle question: 7, 7, 7, 7, 20 is four +// clean weeks and one holiday, and max/min = 2.9 threw the whole thing away. +// One missed week should not erase a habit. const MaxIntervalRatio = 1.5 +// MinOnPatternFraction — how much of the history must sit inside the band +// before a rhythm is a rhythm. A strict majority: with the median as the +// centre, half the intervals are inside it by construction, so anything at or +// below 0.5 would accept noise. 5, 8, 10, 3 has a median of 6.5 and only two +// of four gaps in band, so it stays what it is — irregular, no routine. +// +// At the MinEvents floor (three intervals) 0.7 demands all three, which is +// right: four events is already the cheapest bar and there is no room in it to +// also forgive an outlier. Tolerance starts at five intervals, where 4/5 passes. +const MinOnPatternFraction = 0.7 + // MinEvents is the minimum number of events needed to detect a pattern. -// With N events, there are N-1 intervals; we need at least 2 intervals -// before proposing anything. -const MinEvents = 3 +// With N events there are N-1 intervals, so 4 events means 3 intervals. +// +// This used to be 3 (two intervals), which is not a pattern — it is a +// coincidence with a mean. Two gaps of similar length happen constantly: +// water the plants on a Sunday, again the next Sunday, once more the Sunday +// after, and a detector with a ±50% band calls that a weekly routine. The +// cost of being wrong is asymmetric now that the digestion tick scans all of +// history on its own schedule and can announce what it finds: a false +// positive is something the owner has to read and dismiss, and a dismissal +// is permanent, so one bad guess burns that action+object pair forever. +// Three intervals is the cheapest bar that makes a run distinguishable from +// a repeat. False negatives cost one more observation and nothing else. +const MinEvents = 4 // Detect checks whether a sequence of events for the same action+object // forms a stable recurring pattern. Returns a ProposedRoutine when: -// - At least MinEvents events exist (≥2 intervals) -// - The ratio longest/shortest interval ≤ MaxIntervalRatio +// - At least MinEvents events exist (≥3 intervals) +// - At least MinOnPatternFraction of the intervals sit within +// MaxIntervalRatio of the median interval +// +// The reported IntervalDays is the median of the ON-PATTERN intervals only. +// Outliers are excluded from the number as well as from the test, so a habit +// interrupted by a two-week holiday is still reported as weekly rather than as +// "every 9.6 days" — a figure that describes neither the habit nor the gap. // // Returns nil when there aren't enough events or the intervals are too // irregular — false negatives are harmless. The only dangerous mistake @@ -41,10 +76,6 @@ func Detect(events []Event) (*ProposedRoutine, error) { nIntervals := len(events) - 1 intervals := make([]float64, nIntervals) - var sum float64 - var min float64 = math.MaxFloat64 - var max float64 - for i := 0; i < nIntervals; i++ { diff := events[i+1].Ts.Sub(events[i].Ts) days := diff.Hours() / 24.0 @@ -54,32 +85,51 @@ func Detect(events []Event) (*ProposedRoutine, error) { return nil, nil } intervals[i] = days - sum += days - if days < min { - min = days - } - if days > max { - max = days - } } - // Stability check: the most extreme intervals shouldn't differ by - // more than MaxIntervalRatio. A ratio of 1.5 means a 7-day pattern - // can have intervals between ~5.6 and ~8.4 days. - if min > 0 && max/min > MaxIntervalRatio { + center := medianFloat(intervals) + if center <= 0 { + return nil, nil + } + + // Keep the intervals that sit inside the band around the median. The + // bound is symmetric in ratio terms, not in days: half the median below, + // the median times the ratio above. + var onPattern []float64 + for _, d := range intervals { + if d <= center*MaxIntervalRatio && d >= center/MaxIntervalRatio { + onPattern = append(onPattern, d) + } + } + if float64(len(onPattern))/float64(nIntervals) < MinOnPatternFraction { return nil, nil // too irregular } - mean := sum / float64(nIntervals) - return &ProposedRoutine{ Action: events[0].Action, Object: events[0].Object, - IntervalDays: math.Round(mean*10) / 10, // round to 1 decimal + IntervalDays: math.Round(medianFloat(onPattern)*10) / 10, // round to 1 decimal N: len(events), }, nil } +// medianFloat — the middle value, averaging the two middles on an even count. +// Sorts a copy: the caller's interval order is the event order and stays that +// way. +func medianFloat(xs []float64) float64 { + if len(xs) == 0 { + return 0 + } + s := make([]float64, len(xs)) + copy(s, xs) + sort.Float64s(s) + mid := len(s) / 2 + if len(s)%2 == 1 { + return s[mid] + } + return (s[mid-1] + s[mid]) / 2 +} + // PhraseRoutine generates a human-readable suggestion string for a // detected routine. Returns a Russian phrase like // "ты заправляешь поилку раз в 7 дней — напоминать?" diff --git a/internal/pattern/detector_test.go b/internal/pattern/detector_test.go index c50d2eb..056a3f6 100644 --- a/internal/pattern/detector_test.go +++ b/internal/pattern/detector_test.go @@ -6,12 +6,13 @@ import ( ) func TestDetectEnoughEvents(t *testing.T) { - // 3 events with 7-day intervals → stable pattern + // MinEvents events with 7-day intervals → stable pattern base := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC) events := []Event{ {Action: "refill", Object: "cat_water", Ts: base}, {Action: "refill", Object: "cat_water", Ts: base.Add(7 * 24 * time.Hour)}, {Action: "refill", Object: "cat_water", Ts: base.Add(14 * 24 * time.Hour)}, + {Action: "refill", Object: "cat_water", Ts: base.Add(21 * 24 * time.Hour)}, } r, err := Detect(events) @@ -24,8 +25,8 @@ func TestDetectEnoughEvents(t *testing.T) { if r.Action != "refill" || r.Object != "cat_water" { t.Fatalf("action/object: want refill/cat_water, got %s/%s", r.Action, r.Object) } - if r.N != 3 { - t.Fatalf("want N=3, got %d", r.N) + if r.N != 4 { + t.Fatalf("want N=4, got %d", r.N) } // ~7 days if r.IntervalDays < 6.9 || r.IntervalDays > 7.1 { @@ -33,19 +34,28 @@ func TestDetectEnoughEvents(t *testing.T) { } } +// TestDetectNotEnoughEvents — two intervals are a coincidence, not a routine +// (Vikunja #43). Three same-day-of-week events used to be enough to propose a +// weekly reminder; MinEvents is 4 now so a repeat has to happen a third time +// before Maven calls it a pattern. func TestDetectNotEnoughEvents(t *testing.T) { base := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC) - events := []Event{ - {Action: "refill", Object: "cat_water", Ts: base}, - {Action: "refill", Object: "cat_water", Ts: base.Add(7 * 24 * time.Hour)}, - } - - r, err := Detect(events) - if err != nil { - t.Fatalf("Detect: %v", err) - } - if r != nil { - t.Fatal("want nil for <3 events") + for _, n := range []int{1, 2, MinEvents - 1} { + events := make([]Event, n) + for i := range events { + events[i] = Event{ + Action: "refill", + Object: "cat_water", + Ts: base.Add(time.Duration(i) * 7 * 24 * time.Hour), + } + } + r, err := Detect(events) + if err != nil { + t.Fatalf("Detect(%d events): %v", n, err) + } + if r != nil { + t.Fatalf("Detect(%d events) proposed %+v, want nil below MinEvents=%d", n, r, MinEvents) + } } } @@ -68,12 +78,13 @@ func TestDetectEmpty(t *testing.T) { } func TestDetectIrregularRejects(t *testing.T) { - // 3 events but wildly irregular: 1 day, then 14 days → ratio 14 > 1.5 + // wildly irregular: 1 day, then 14 days → ratio 14 > 1.5 base := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC) events := []Event{ {Action: "refill", Object: "cat_water", Ts: base}, {Action: "refill", Object: "cat_water", Ts: base.Add(1 * 24 * time.Hour)}, {Action: "refill", Object: "cat_water", Ts: base.Add(15 * 24 * time.Hour)}, + {Action: "refill", Object: "cat_water", Ts: base.Add(16 * 24 * time.Hour)}, } r, err := Detect(events) @@ -117,6 +128,7 @@ func TestDetectSameTimestamp(t *testing.T) { {Action: "refill", Object: "cat_water", Ts: base}, {Action: "refill", Object: "cat_water", Ts: base}, {Action: "refill", Object: "cat_water", Ts: base.Add(7 * 24 * time.Hour)}, + {Action: "refill", Object: "cat_water", Ts: base.Add(14 * 24 * time.Hour)}, } r, err := Detect(events) @@ -149,3 +161,58 @@ func TestPhraseRoutine(t *testing.T) { }) } } + +// evAt builds a run of events at the given day offsets. +func evAt(offsets ...float64) []Event { + base := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC) + out := make([]Event, len(offsets)) + for i, d := range offsets { + out[i] = Event{Action: "refill", Object: "cat_water", + Ts: base.Add(time.Duration(d * float64(24*time.Hour)))} + } + return out +} + +// TestDetectMedianBandNotExtremes — the stability test used to be +// longest/shortest, so a single outlier vetoed an otherwise clean rhythm and +// the reported interval was a mean dragged toward that outlier. Both are +// median-based now. +func TestDetectMedianBandNotExtremes(t *testing.T) { + cases := []struct { + name string + days []float64 + want float64 // 0 means "expect no routine" + }{ + // Four clean weeks and one holiday. max/min was 20/7 = 2.9, rejected. + {"weekly with one long gap", []float64{0, 7, 14, 21, 28, 48}, 7}, + // The reviewer's case: 5, 8, 10, 3. Median 6.5, only two gaps in band. + {"genuinely irregular", []float64{0, 5, 13, 23, 26}, 0}, + // A short gap outlier is treated the same as a long one. + {"weekly with one short gap", []float64{0, 7, 14, 15, 22, 29}, 7}, + // Two outliers out of five is past the fraction. + {"too many outliers", []float64{0, 7, 14, 34, 41, 61}, 0}, + // At the MinEvents floor there is no outlier budget at all. + {"floor rejects one outlier", []float64{0, 7, 14, 34}, 0}, + {"floor accepts a clean run", []float64{0, 7, 14, 21}, 7}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r, err := Detect(evAt(tc.days...)) + if err != nil { + t.Fatalf("Detect: %v", err) + } + if tc.want == 0 { + if r != nil { + t.Fatalf("want no routine, got interval %.1f", r.IntervalDays) + } + return + } + if r == nil { + t.Fatal("want a routine, got nil") + } + if r.IntervalDays != tc.want { + t.Fatalf("interval: want %.1f, got %.1f", tc.want, r.IntervalDays) + } + }) + } +} diff --git a/internal/phraser/llmphraser.go b/internal/phraser/llmphraser.go index 558bec2..4006009 100644 --- a/internal/phraser/llmphraser.go +++ b/internal/phraser/llmphraser.go @@ -27,14 +27,46 @@ var listenRE = regexp.MustCompile(`listening on (https?://\S+)`) type LLMPhraser struct { cfg Config client *http.Client - port string - cmd *exec.Cmd - cancel context.CancelFunc - wg sync.WaitGroup // tmpl — the hand-written Russian nudges. Default path for nudges; see // Config.LLMNudges. nil only if the template file failed to load. tmpl *NudgeTemplates + + // spawnCtx — the parent of every llama-server this phraser starts, i.e. the + // daemon's own context. Deliberately NOT the per-request context of the call + // that asked for a model swap: that one is cancelled the moment the request + // returns, which would kill the model it had just loaded. + spawnCtx context.Context + cancel context.CancelFunc + + // launch / probe — the two side effects of a swap, injectable so the swap + // logic is testable without a real llama-server and a real model file. + // launch is nil when this phraser does not own its server (NewLLMPhraserAt), + // which is also what makes Swap refuse there. + launch func(ctx context.Context, cfg Config) (backend, error) + probe func(ctx context.Context, base string) (string, error) + + // swapMu — single-flight around Swap. Held for the whole swap, including the + // model load, so two concurrent swap requests can never both be loading. + swapMu sync.Mutex + + // mu guards everything below: the live backend, the swap gate and the + // in-flight request count. See acquire/quiesce in swap.go. + mu sync.Mutex + be backend + live liveModel + swapping bool + inflight int + observers []func(baseURL string) +} + +// liveModel — what is actually loaded right now. Distinct from Config, which +// stays immutable after construction: a swap changes these three fields and +// nothing else, so no reader of cfg (prompts, grammar, timeouts) races a swap. +type liveModel struct { + ModelPath string + NGpuLayers int + NCtx int } type Config struct { @@ -85,15 +117,21 @@ func DefaultConfig(modelPath string) Config { func NewLLMPhraser(ctx context.Context, cfg Config) (*LLMPhraser, error) { ctx, cancel := context.WithCancel(ctx) p := &LLMPhraser{ - cfg: cfg, - client: &http.Client{Timeout: cfg.Timeout}, - cancel: cancel, - tmpl: loadNudgeTemplates(), + cfg: cfg, + client: &http.Client{Timeout: cfg.Timeout}, + tmpl: loadNudgeTemplates(), + spawnCtx: ctx, + cancel: cancel, + launch: spawnLlamaServer, + probe: defaultProbe, + live: liveModel{ModelPath: cfg.ModelPath, NGpuLayers: cfg.NGpuLayers, NCtx: cfg.NCtx}, } - if err := p.start(ctx); err != nil { + be, err := p.launch(ctx, cfg) + if err != nil { cancel() return nil, err } + p.be = be return p, nil } @@ -106,11 +144,17 @@ func NewLLMPhraser(ctx context.Context, cfg Config) (*LLMPhraser, error) { // still uses NewLLMPhraser and still owns its own child process. func NewLLMPhraserAt(baseURL string, cfg Config) *LLMPhraser { return &LLMPhraser{ - cfg: cfg, - client: &http.Client{Timeout: cfg.Timeout}, - port: strings.TrimSuffix(baseURL, "/"), - cancel: func() {}, - tmpl: loadNudgeTemplates(), + cfg: cfg, + client: &http.Client{Timeout: cfg.Timeout}, + tmpl: loadNudgeTemplates(), + spawnCtx: context.Background(), + cancel: func() {}, + probe: defaultProbe, + // launch stays nil: we did not start this server, so we must not stop it. + // Swap therefore refuses here (ErrSwapNotOwned) instead of killing a + // server another process depends on. + be: borrowedBackend(strings.TrimSuffix(baseURL, "/")), + live: liveModel{ModelPath: cfg.ModelPath, NGpuLayers: cfg.NGpuLayers, NCtx: cfg.NCtx}, } } @@ -126,16 +170,66 @@ func loadNudgeTemplates() *NudgeTemplates { return nt } -func (p *LLMPhraser) start(ctx context.Context) error { +// backend — one llama-server this phraser talks to. Two implementations: a +// llamaProc we spawned and must reap, and a borrowedBackend someone else owns. +type backend interface { + BaseURL() string + Close() error +} + +// borrowedBackend — a server started and owned by someone else (the phrasing +// scorer's shared llama-server). Closing it is a no-op by construction. +type borrowedBackend string + +func (b borrowedBackend) BaseURL() string { return string(b) } +func (b borrowedBackend) Close() error { return nil } + +// llamaProc — a llama-server child process plus the goroutine reading its +// stderr. Close kills and reaps it; see the Pdeathsig note in spawnLlamaServer. +type llamaProc struct { + base string + cmd *exec.Cmd + cancel context.CancelFunc + wg sync.WaitGroup +} + +func (l *llamaProc) BaseURL() string { return l.base } + +func (l *llamaProc) Close() error { + l.cancel() + if l.cmd != nil && l.cmd.Process != nil { + _ = l.cmd.Process.Kill() + _ = l.cmd.Wait() // reap the process — without Wait, the child becomes a zombie + } + l.wg.Wait() + return nil +} + +// spawnLlamaServer starts one llama-server for cfg and waits until it says which +// address it is listening on. ctx owns the process lifetime, so it must be the +// daemon's context, not a request's. +func spawnLlamaServer(ctx context.Context, cfg Config) (backend, error) { + ctx, cancel := context.WithCancel(ctx) + p, err := startLlamaProc(ctx, cfg) + if err != nil { + cancel() + return nil, err + } + p.cancel = cancel + return p, nil +} + +func startLlamaProc(ctx context.Context, cfg Config) (*llamaProc, error) { + p := &llamaProc{} args := []string{ - "-m", p.cfg.ModelPath, + "-m", cfg.ModelPath, "--host", "127.0.0.1", - "--port", extractPort(p.cfg.Listen), - "-c", fmt.Sprintf("%d", p.cfg.NCtx), - "-ngl", fmt.Sprintf("%d", p.cfg.NGpuLayers), + "--port", extractPort(cfg.Listen), + "-c", fmt.Sprintf("%d", cfg.NCtx), + "-ngl", fmt.Sprintf("%d", cfg.NGpuLayers), "--no-webui", } - cmd := exec.CommandContext(ctx, p.cfg.BinPath, args...) + cmd := exec.CommandContext(ctx, cfg.BinPath, args...) // Pdeathsig: the kernel SIGKILLs llama-server the moment mavend dies — by // ANY means, including SIGKILL/OOM/panic where our Close() never runs. Without // it a hard-killed mavend orphans its llama-server (reparented to init, keeps @@ -148,12 +242,12 @@ func (p *LLMPhraser) start(ctx context.Context) error { stderr, err := cmd.StderrPipe() if err != nil { - return fmt.Errorf("llm: stderr pipe: %w", err) + return nil, fmt.Errorf("llm: stderr pipe: %w", err) } if err := cmd.Start(); err != nil { stderr.Close() - return fmt.Errorf("llm: start: %w", err) + return nil, fmt.Errorf("llm: start: %w", err) } portCh := make(chan string, 1) @@ -186,32 +280,44 @@ func (p *LLMPhraser) start(ctx context.Context) error { select { case addr := <-portCh: - p.port = addr - return nil + p.base = addr + return p, nil case err := <-errCh: _ = cmd.Process.Kill() _ = cmd.Wait() - return fmt.Errorf("llm: server output: %w", err) + return nil, fmt.Errorf("llm: server output: %w", err) case <-ctx.Done(): _ = cmd.Process.Kill() _ = cmd.Wait() - return ctx.Err() + return nil, ctx.Err() case <-time.After(60 * time.Second): _ = cmd.Process.Kill() _ = cmd.Wait() - return fmt.Errorf("llm: server did not start within 60s") + return nil, fmt.Errorf("llm: server did not start within 60s") } } -func (p *LLMPhraser) BaseURL() string { return p.port } +// BaseURL is the llama-server this phraser talks to right now. It changes when +// the model is swapped, so callers that cache it must register an observer +// (OnSwap) rather than keeping the string forever. +func (p *LLMPhraser) BaseURL() string { + p.mu.Lock() + defer p.mu.Unlock() + if p.be == nil { + return "" + } + return p.be.BaseURL() +} func (p *LLMPhraser) Close() error { p.cancel() - if p.cmd != nil && p.cmd.Process != nil { - _ = p.cmd.Process.Kill() - _ = p.cmd.Wait() // reap the process — without Wait, the child becomes a zombie + p.mu.Lock() + be := p.be + p.be = nil + p.mu.Unlock() + if be != nil { + return be.Close() } - p.wg.Wait() return nil } @@ -350,6 +456,11 @@ func chatSystemPrompt(block func() string) string { // the LLM completion endpoint. Like chatWithSystem but for an arbitrary message // slice — the caller owns the system prompt placement. func (p *LLMPhraser) chatWithMessages(ctx context.Context, msgs []chatMsg, maxTokens int) (string, error) { + base, release, err := p.acquire() + if err != nil { + return "", err + } + defer release() req := chatReq{ Messages: msgs, Temperature: 0.7, @@ -360,7 +471,7 @@ func (p *LLMPhraser) chatWithMessages(ctx context.Context, msgs []chatMsg, maxTo if err != nil { return "", fmt.Errorf("llm: marshal: %w", err) } - httpReq, err := http.NewRequestWithContext(ctx, "POST", p.port+"/v1/chat/completions", bytes.NewReader(body)) + httpReq, err := http.NewRequestWithContext(ctx, "POST", base+"/v1/chat/completions", bytes.NewReader(body)) if err != nil { return "", fmt.Errorf("llm: request: %w", err) } @@ -493,6 +604,11 @@ func (p *LLMPhraser) chat(ctx context.Context, userPrompt string) (string, error } func (p *LLMPhraser) chatWithSystem(ctx context.Context, system, user string, maxTokens int) (string, error) { + base, release, err := p.acquire() + if err != nil { + return "", err + } + defer release() req := chatReq{ Messages: []chatMsg{ {Role: "system", Content: system}, @@ -507,7 +623,7 @@ func (p *LLMPhraser) chatWithSystem(ctx context.Context, system, user string, ma return "", fmt.Errorf("llm: marshal: %w", err) } - httpReq, err := http.NewRequestWithContext(ctx, "POST", p.port+"/v1/chat/completions", bytes.NewReader(body)) + httpReq, err := http.NewRequestWithContext(ctx, "POST", base+"/v1/chat/completions", bytes.NewReader(body)) if err != nil { return "", fmt.Errorf("llm: request: %w", err) } diff --git a/internal/phraser/swap.go b/internal/phraser/swap.go new file mode 100644 index 0000000..e98a798 --- /dev/null +++ b/internal/phraser/swap.go @@ -0,0 +1,357 @@ +package phraser + +import ( + "context" + "errors" + "fmt" + "log" + "time" + + "github.com/kami/maven/internal/llm" +) + +// Swapping the resident model without restarting the daemon (Vikunja #250). +// +// Three properties this file exists to hold, in order of importance: +// +// 1. NEVER two models resident at once. The deploy target is a laptop iGPU +// with the whole 1.7B offloaded to it (`n_gpu_layers: 99`); loading a second +// model beside the first is how you OOM the box, and a blue/green swap that +// "keeps the old one warm until the new one answers" does exactly that. So +// the old server is killed FIRST and the new one loaded after. The cost of +// that ordering is a window with no model at all, which is why: +// +// 2. A swap is atomic from a turn's point of view. An in-flight turn finishes +// on the old model — Swap waits for the last one to return before killing +// anything. A turn that arrives during the swap is REFUSED immediately with +// ErrSwapping rather than blocked: every phrasing path already has a +// fallback (templates, "вот что я нашла", the classifier for routing), so a +// fast refusal degrades one turn instead of hanging it for the length of a +// model load. No turn ever gets half of one model and half of another. +// "Every path" means every path: the router, the replier, the mail +// extractor and the memory evaluator do not call acquire, they call +// llm.Client.Complete, so LLMPhraser implements llm.SwapGate and the client +// enters through the same counter. +// +// 3. A failed load rolls back to the model that was working. The new server is +// probed (it must say which model it loaded) before it is published; if the +// launch or the probe fails, the previous config is relaunched and the +// phraser goes back to serving. Only if the rollback ALSO fails is the +// phraser left without a backend, and then it says so loudly and every turn +// degrades rather than breaks. +// +// Not here, deliberately: nothing calls Swap on a timer, and no act or intent can +// reach it. It is an IPC method behind the step-up gate, i.e. owner-triggered. + +var ( + // ErrSwapping — a turn arrived while the model was being swapped. Callers + // treat it like any other LLM error and use their fallback. + ErrSwapping = errors.New("phraser: model swap in progress") + + // ErrSwapNotOwned — this phraser did not start its llama-server, so it must + // not stop one (NewLLMPhraserAt: the eval harness shares a server). + ErrSwapNotOwned = errors.New("phraser: llama-server is not ours to swap") + + // ErrNoBackend — no model is loaded at all. Only reachable after a failed + // swap whose rollback also failed. + ErrNoBackend = errors.New("phraser: no llama-server loaded") + + // ErrSwapBusy — a turn was still running when the drain deadline expired, so + // the swap was abandoned. Nothing was killed; ask again. + ErrSwapBusy = errors.New("phraser: turns still in flight, swap abandoned") +) + +// SwapSpec — what to load. Zero NGpuLayers/NCtx keep whatever is live, so the +// common case ("same settings, different gguf") is one field. +type SwapSpec struct { + ModelPath string + NGpuLayers int + NCtx int +} + +// SwapResult — what happened. Model is the identity the NEW server reported, so +// it is evidence rather than an echo of the request: if the file at ModelPath is +// not what the operator thought it was, this is where that shows up. +// RolledBack is true only when a model is serving again. NoBackend is the other +// failure, and it is the worse one: the rollback failed too and nothing is +// loaded. They are separate flags because the operator surface reads them, and +// "rolled back" spelled over a dead daemon reads as reassurance. +type SwapResult struct { + Model string + BaseURL string + ModelPath string + RolledBack bool + NoBackend bool + Took time.Duration +} + +// drainTimeout — how long Swap waits for in-flight turns before giving up. A +// turn is at most Config.Timeout (30s in deploy) plus the model's own latency; +// 90s covers a slow Thinking generation without wedging the caller forever. +const drainTimeout = 90 * time.Second + +// probeTimeout — how long the new server gets to answer "which model do you +// have". The load itself is bounded by spawnLlamaServer's own 60s wait. +const probeTimeout = 30 * time.Second + +// defaultProbe asks the server which model it has loaded. This is the health +// check: a server that answers /v1/models has finished loading weights and is +// serving, and its answer is the identity we report back. +func defaultProbe(ctx context.Context, base string) (string, error) { + return llm.ModelID(ctx, base) +} + +// OnSwap registers a callback fired with the new base URL every time the live +// backend changes, including after a rollback. Holders of an *llm.Client (the +// LLM router, the replier, the mail extractor) register SetBaseURL here so a +// swap re-points them without rebuilding the router or the handler. +// +// Callbacks run with no lock held, in registration order. +func (p *LLMPhraser) OnSwap(fn func(baseURL string)) { + if fn == nil { + return + } + p.mu.Lock() + p.observers = append(p.observers, fn) + p.mu.Unlock() +} + +// LiveModel is the model file currently loaded (and its load settings). Empty +// ModelPath means no model is loaded. +func (p *LLMPhraser) LiveModel() (path string, nGpuLayers, nCtx int) { + p.mu.Lock() + defer p.mu.Unlock() + return p.live.ModelPath, p.live.NGpuLayers, p.live.NCtx +} + +// acquire reserves a slot for one request and returns the base URL to use. +// Every request path must call it and must call the returned release exactly +// once — that count is what Swap drains. +func (p *LLMPhraser) acquire() (string, func(), error) { + p.mu.Lock() + defer p.mu.Unlock() + if p.swapping { + return "", nil, ErrSwapping + } + if p.be == nil { + return "", nil, ErrNoBackend + } + p.inflight++ + base := p.be.BaseURL() + var once bool + return base, func() { + p.mu.Lock() + if !once { + once = true + p.inflight-- + } + p.mu.Unlock() + }, nil +} + +// Enter implements llm.SwapGate so every holder of an *llm.Client is drained by a +// swap, not only the phrasing paths in this package. +// +// The router, the replier, the mail extractor and the memory evaluator do not +// call acquire; they call llm.Client.Complete. Before this existed quiesce could +// see zero requests in flight while the router was mid-generation and kill the +// server under it. A refusal here is the same ErrSwapping the phrasing paths +// get, and every caller of Complete already falls back. +func (p *LLMPhraser) Enter() (func(), error) { + _, release, err := p.acquire() + return release, err +} + +// Swap loads another model in place of the live one. See the file comment for +// the properties it guarantees. Returns the new model's reported identity, or +// an error plus RolledBack=true when the old model was put back. +// +// ctx bounds the drain and the probe. It does NOT own the new server's lifetime +// — that is the daemon's context, captured at construction — so a swap survives +// the request that asked for it. +func (p *LLMPhraser) Swap(ctx context.Context, spec SwapSpec) (SwapResult, error) { + if spec.ModelPath == "" { + return SwapResult{}, fmt.Errorf("phraser: swap needs a model path") + } + p.swapMu.Lock() + defer p.swapMu.Unlock() + + if p.launch == nil { + return SwapResult{}, ErrSwapNotOwned + } + + started := time.Now() + oldLive := p.liveSnapshot() + newLive := liveModel{ + ModelPath: spec.ModelPath, + NGpuLayers: pickInt(spec.NGpuLayers, oldLive.NGpuLayers), + NCtx: pickInt(spec.NCtx, oldLive.NCtx), + } + if newLive == oldLive && p.BaseURL() != "" { + // Already serving exactly this. Report the live identity rather than + // pointlessly unloading and reloading the same weights. + base := p.BaseURL() + id, err := p.probeWith(ctx, base) + if err != nil { + return SwapResult{}, err + } + return SwapResult{Model: id, BaseURL: base, ModelPath: oldLive.ModelPath, Took: time.Since(started)}, nil + } + + if err := p.quiesce(ctx); err != nil { + return SwapResult{}, err + } + defer p.resume() + + // Property 1: the old model leaves the GPU before the new one arrives. + p.mu.Lock() + old := p.be + p.be = nil + p.mu.Unlock() + if old != nil { + _ = old.Close() + } + + be, err := p.loadAndProbe(ctx, newLive) + if err != nil { + log.Printf("phraser: swap to %s FAILED (%v) — rolling back to %s", newLive.ModelPath, err, oldLive.ModelPath) + rb, rbErr := p.loadAndProbe(ctx, oldLive) + if rbErr != nil { + // Nothing is loaded, so LiveModel must stop naming a gguf: the page + // would show a file next to an unknown model and read as half-working. + p.mu.Lock() + p.live = liveModel{} + p.mu.Unlock() + log.Printf("phraser: ROLLBACK to %s ALSO FAILED (%v) — no model is loaded, every phrasing path is on its fallback and routing is on the classifier. Swap is still wired, so another attempt can recover without restarting the daemon", oldLive.ModelPath, rbErr) + return SwapResult{NoBackend: true, Took: time.Since(started)}, + fmt.Errorf("phraser: swap failed (%w) and rollback failed too: %v", err, rbErr) + } + p.publish(rb, oldLive) + return SwapResult{ + Model: rb.id, BaseURL: rb.be.BaseURL(), ModelPath: oldLive.ModelPath, + RolledBack: true, Took: time.Since(started), + }, + fmt.Errorf("phraser: swap to %s failed, rolled back to %s: %w", newLive.ModelPath, oldLive.ModelPath, err) + } + p.publish(be, newLive) + log.Printf("phraser: model swapped to %s (%s) at %s in %s", newLive.ModelPath, be.id, be.be.BaseURL(), time.Since(started).Round(time.Millisecond)) + return SwapResult{ + Model: be.id, BaseURL: be.be.BaseURL(), ModelPath: newLive.ModelPath, + Took: time.Since(started), + }, nil +} + +// loaded — a started server plus the identity it reported. +type loaded struct { + be backend + id string +} + +// loadAndProbe starts a server for lm and verifies it answers. A server that +// starts but will not say what it loaded is treated as a failed load and is +// killed here — publishing it would hand every turn to a backend we could not +// confirm. +func (p *LLMPhraser) loadAndProbe(ctx context.Context, lm liveModel) (loaded, error) { + cfg := p.cfg + cfg.ModelPath = lm.ModelPath + cfg.NGpuLayers = lm.NGpuLayers + cfg.NCtx = lm.NCtx + // p.spawnCtx, not ctx: the process must outlive the request asking for it. + be, err := p.launch(p.spawnCtx, cfg) + if err != nil { + return loaded{}, err + } + id, err := p.probeWith(ctx, be.BaseURL()) + if err != nil { + _ = be.Close() + return loaded{}, fmt.Errorf("phraser: %s started but would not answer: %w", lm.ModelPath, err) + } + return loaded{be: be, id: id}, nil +} + +func (p *LLMPhraser) probeWith(ctx context.Context, base string) (string, error) { + probe := p.probe + if probe == nil { + probe = defaultProbe + } + pctx, cancel := context.WithTimeout(ctx, probeTimeout) + defer cancel() + return probe(pctx, base) +} + +// quiesce closes the door on new turns and waits for the ones already running. +// Polling rather than a sync.Cond: the wait happens once per swap, a 25ms poll +// is invisible next to a model load, and a poll cannot deadlock on a release +// path that panicked. +func (p *LLMPhraser) quiesce(ctx context.Context) error { + p.mu.Lock() + if p.swapping { + p.mu.Unlock() + return ErrSwapping + } + p.swapping = true + inflight := p.inflight + p.mu.Unlock() + if inflight == 0 { + return nil + } + + deadline := time.Now().Add(drainTimeout) + for { + select { + case <-ctx.Done(): + p.resume() + return ctx.Err() + case <-time.After(25 * time.Millisecond): + } + p.mu.Lock() + inflight = p.inflight + p.mu.Unlock() + if inflight == 0 { + return nil + } + if time.Now().After(deadline) { + // Nothing has been killed yet, so abandoning is free: reopen the door + // and let the operator try again rather than cutting a live turn off + // mid-generation. + p.resume() + return fmt.Errorf("%w (%d still running after %s)", ErrSwapBusy, inflight, drainTimeout) + } + } +} + +func (p *LLMPhraser) resume() { + p.mu.Lock() + p.swapping = false + p.mu.Unlock() +} + +// publish makes l the live backend and tells everyone holding a base URL. +func (p *LLMPhraser) publish(l loaded, lm liveModel) { + p.mu.Lock() + p.be = l.be + p.live = lm + obs := make([]func(string), len(p.observers)) + copy(obs, p.observers) + p.mu.Unlock() + base := l.be.BaseURL() + for _, fn := range obs { + fn(base) + } +} + +func (p *LLMPhraser) liveSnapshot() liveModel { + p.mu.Lock() + defer p.mu.Unlock() + return p.live +} + +// pickInt returns v when the caller set it, and fallback otherwise. 0 is the +// "unset" value: -1 already means "offload every layer" and deploy uses 99, so +// nothing legitimate asks for exactly zero GPU layers through this path. +func pickInt(v, fallback int) int { + if v == 0 { + return fallback + } + return v +} diff --git a/internal/phraser/swap_gate_test.go b/internal/phraser/swap_gate_test.go new file mode 100644 index 0000000..8b8917a --- /dev/null +++ b/internal/phraser/swap_gate_test.go @@ -0,0 +1,127 @@ +package phraser + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/kami/maven/internal/llm" +) + +// The drain has to cover every holder of the base URL, not only the phrasing +// paths in this package. The LLM router, the replier, the mail extractor and the +// memory evaluator all reach llama-server through llm.Client, and a swap that +// does not count them kills the server mid-turn. + +// blockingLLM — a completion endpoint that does not answer until the test says +// so. It stands in for a router call that is generating when the swap arrives. +func blockingLLM(t *testing.T, release <-chan struct{}) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-release + w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`)) + })) + t.Cleanup(srv.Close) + return srv +} + +func (p *LLMPhraser) inflightCount() int { + p.mu.Lock() + defer p.mu.Unlock() + return p.inflight +} + +func TestSwap_WaitsForARouterCallThatWentThroughLLMClient(t *testing.T) { + fl := &fakeFleet{models: map[string]string{"/m/old.gguf": "old", "/m/new.gguf": "new"}} + p := newSwapPhraser(t, fl, "/m/old.gguf") + + release := make(chan struct{}) + c := llm.New(blockingLLM(t, release).URL, 5*time.Second) + c.SetSwapGate(p) + + completed := make(chan error, 1) + go func() { + _, err := c.Complete(context.Background(), llm.Req{System: "s", User: "u"}) + completed <- err + }() + deadline := time.Now().Add(2 * time.Second) + for p.inflightCount() == 0 { + if time.Now().After(deadline) { + t.Fatal("the llm.Client request never registered with the phraser gate") + } + time.Sleep(5 * time.Millisecond) + } + + swapped := make(chan error, 1) + go func() { _, e := p.Swap(context.Background(), SwapSpec{ModelPath: "/m/new.gguf"}); swapped <- e }() + + select { + case e := <-swapped: + t.Fatalf("the swap finished while a router call was still generating (%v); the old server was killed under it", e) + case <-time.After(200 * time.Millisecond): + } + + close(release) + if e := <-completed; e != nil { + t.Fatalf("the in-flight call did not finish on the old model: %v", e) + } + if e := <-swapped; e != nil { + t.Fatalf("Swap after the drain: %v", e) + } +} + +func TestSwap_RefusesARouterCallThatArrivesMidSwap(t *testing.T) { + fl := &fakeFleet{models: map[string]string{"/m/old.gguf": "old", "/m/new.gguf": "new"}} + p := newSwapPhraser(t, fl, "/m/old.gguf") + + // An open server: the refusal has to come from the gate, not from a stall. + open := make(chan struct{}) + close(open) + c := llm.New(blockingLLM(t, open).URL, 5*time.Second) + c.SetSwapGate(p) + + // Hold the door shut the way quiesce does. + _, held, err := p.acquire() + if err != nil { + t.Fatal(err) + } + defer held() + go p.Swap(context.Background(), SwapSpec{ModelPath: "/m/new.gguf"}) + + deadline := time.Now().Add(2 * time.Second) + for { + _, err := c.Complete(context.Background(), llm.Req{User: "u"}) + if errors.Is(err, ErrSwapping) { + return + } + if time.Now().After(deadline) { + t.Fatalf("a router call during a swap was not refused (last error: %v)", err) + } + time.Sleep(10 * time.Millisecond) + } +} + +func TestSwap_TotalFailureIsNotReportedAsARollback(t *testing.T) { + fl := &fakeFleet{models: map[string]string{"/m/old.gguf": "old"}} + p := newSwapPhraser(t, fl, "/m/old.gguf") + fl.mu.Lock() + delete(fl.models, "/m/old.gguf") + fl.mu.Unlock() + + res, err := p.Swap(context.Background(), SwapSpec{ModelPath: "/m/broken.gguf"}) + if err == nil { + t.Fatal("Swap returned nil when both the load and the rollback failed") + } + if res.RolledBack { + t.Error("a total failure set RolledBack; the page then says she is still answering with the old model, and she is not answering at all") + } + if !res.NoBackend { + t.Error("a total failure did not set NoBackend, so nothing distinguishes it from a rolled-back swap") + } + if path, _, _ := p.LiveModel(); path != "" { + t.Errorf("LiveModel = %q after a total failure; nothing is loaded, and naming a gguf makes the page read as half-working", path) + } +} diff --git a/internal/phraser/swap_test.go b/internal/phraser/swap_test.go new file mode 100644 index 0000000..aea6f39 --- /dev/null +++ b/internal/phraser/swap_test.go @@ -0,0 +1,311 @@ +package phraser + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" +) + +// fakeModel — a stand-in llama-server. It answers /v1/models with its own name +// and /v1/chat/completions with a phrasing-contract reply that names itself, so +// a test can tell WHICH model answered a turn — the property the swap is about. +type fakeModel struct { + srv *httptest.Server + name string + closed atomic.Bool +} + +func newFakeModel(t *testing.T, name string) *fakeModel { + t.Helper() + f := &fakeModel{name: name} + mux := http.NewServeMux() + mux.HandleFunc("/v1/models", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{"data":[{"id":"/models/` + name + `.gguf"}]}`)) + }) + mux.HandleFunc("/v1/chat/completions", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{"choices":[{"message":{"content":"{\"response\":\"` + name + `\",\"mood\":\"neutral\"}"}}]}`)) + }) + f.srv = httptest.NewServer(mux) + t.Cleanup(f.srv.Close) + return f +} + +func (f *fakeModel) BaseURL() string { return f.srv.URL } +func (f *fakeModel) Close() error { f.closed.Store(true); return nil } + +// fakeFleet is the injected launcher: it hands out a prepared fakeModel per +// model path, and refuses paths the test did not prepare (that is what a bad +// gguf looks like from here). It also asserts the invariant that matters on a +// laptop iGPU: never two servers alive at the same time. +type fakeFleet struct { + mu sync.Mutex + models map[string]string // model path → fake name + live int + maxLive int + launch int +} + +func (fl *fakeFleet) launcher(t *testing.T) func(context.Context, Config) (backend, error) { + return func(ctx context.Context, cfg Config) (backend, error) { + fl.mu.Lock() + name, ok := fl.models[cfg.ModelPath] + fl.launch++ + if !ok { + fl.mu.Unlock() + return nil, errors.New("no such model file: " + cfg.ModelPath) + } + fl.live++ + if fl.live > fl.maxLive { + fl.maxLive = fl.live + } + fl.mu.Unlock() + f := newFakeModel(t, name) + return &fleetBackend{fleet: fl, model: f}, nil + } +} + +type fleetBackend struct { + fleet *fakeFleet + model *fakeModel + once sync.Once +} + +func (b *fleetBackend) BaseURL() string { return b.model.BaseURL() } +func (b *fleetBackend) Close() error { + b.once.Do(func() { + b.fleet.mu.Lock() + b.fleet.live-- + b.fleet.mu.Unlock() + }) + return b.model.Close() +} + +// newSwapPhraser builds an LLMPhraser with an injected launcher, so the swap +// path is exercised without a gguf or a GPU. +func newSwapPhraser(t *testing.T, fl *fakeFleet, modelPath string) *LLMPhraser { + t.Helper() + cfg := DefaultConfig(modelPath) + cfg.Timeout = 5 * time.Second + p := &LLMPhraser{ + cfg: cfg, + client: &http.Client{Timeout: cfg.Timeout}, + spawnCtx: context.Background(), + cancel: func() {}, + launch: fl.launcher(t), + probe: defaultProbe, + live: liveModel{ModelPath: modelPath, NGpuLayers: cfg.NGpuLayers, NCtx: cfg.NCtx}, + } + be, err := p.launch(p.spawnCtx, cfg) + if err != nil { + t.Fatalf("initial launch: %v", err) + } + p.be = be + t.Cleanup(func() { p.Close() }) + return p +} + +func TestSwap_LoadsNewModelAndRepointsHolders(t *testing.T) { + fl := &fakeFleet{models: map[string]string{"/m/old.gguf": "old", "/m/new.gguf": "new"}} + p := newSwapPhraser(t, fl, "/m/old.gguf") + + // A holder of the base URL (the LLM router's client, in the daemon). + var seen []string + p.OnSwap(func(base string) { seen = append(seen, base) }) + + before, err := p.PhraseChat(context.Background(), "привет", nil) + if err != nil || before != "old" { + t.Fatalf("before swap: %q, %v; want the old model to answer", before, err) + } + + res, err := p.Swap(context.Background(), SwapSpec{ModelPath: "/m/new.gguf"}) + if err != nil { + t.Fatalf("Swap: %v", err) + } + if res.Model != "new" { + t.Errorf("res.Model = %q; want the identity the NEW server reported (%q)", res.Model, "new") + } + if res.RolledBack { + t.Errorf("res.RolledBack = true on a successful swap") + } + after, err := p.PhraseChat(context.Background(), "привет", nil) + if err != nil || after != "new" { + t.Fatalf("after swap: %q, %v; want the new model to answer", after, err) + } + if path, _, _ := p.LiveModel(); path != "/m/new.gguf" { + t.Errorf("LiveModel = %q; want /m/new.gguf", path) + } + if len(seen) != 1 || seen[0] != p.BaseURL() { + t.Errorf("observers saw %v; want exactly one call with the new base %q", seen, p.BaseURL()) + } + if fl.maxLive > 1 { + t.Errorf("%d servers were alive at once; the iGPU only fits one model", fl.maxLive) + } +} + +func TestSwap_FailedLoadRollsBackToTheWorkingModel(t *testing.T) { + fl := &fakeFleet{models: map[string]string{"/m/old.gguf": "old"}} + p := newSwapPhraser(t, fl, "/m/old.gguf") + + res, err := p.Swap(context.Background(), SwapSpec{ModelPath: "/m/broken.gguf"}) + if err == nil { + t.Fatal("Swap to a model that will not load returned nil error") + } + if !res.RolledBack { + t.Errorf("res.RolledBack = false; a failed swap must say it rolled back") + } + if res.Model != "old" { + t.Errorf("res.Model = %q; want the old model back", res.Model) + } + // The point of the rollback: turns keep working. + got, err := p.PhraseChat(context.Background(), "привет", nil) + if err != nil || got != "old" { + t.Fatalf("after rollback: %q, %v; want the old model serving again", got, err) + } + if path, _, _ := p.LiveModel(); path != "/m/old.gguf" { + t.Errorf("LiveModel = %q; want the old model", path) + } + if fl.maxLive > 1 { + t.Errorf("%d servers alive at once during a rollback", fl.maxLive) + } +} + +func TestSwap_ProbeFailureIsTreatedAsAFailedLoad(t *testing.T) { + // A server that starts but will not say what it loaded must never be + // published — we would be serving turns from a backend we cannot confirm. + fl := &fakeFleet{models: map[string]string{"/m/old.gguf": "old", "/m/mute.gguf": "mute"}} + p := newSwapPhraser(t, fl, "/m/old.gguf") + // Fail the probe once — for the newly launched server — and let the + // rollback's probe through. + calls := 0 + p.probe = func(ctx context.Context, base string) (string, error) { + calls++ + if calls == 1 { + return "", errors.New("no answer from the new server") + } + return defaultProbe(ctx, base) + } + + _, err := p.Swap(context.Background(), SwapSpec{ModelPath: "/m/mute.gguf"}) + if err == nil { + t.Fatal("Swap published a server that failed its probe") + } + if path, _, _ := p.LiveModel(); path != "/m/old.gguf" { + t.Errorf("LiveModel = %q; want the old model after a failed probe", path) + } +} + +func TestSwap_RollbackFailureLeavesNoBackendAndDegrades(t *testing.T) { + fl := &fakeFleet{models: map[string]string{"/m/old.gguf": "old"}} + p := newSwapPhraser(t, fl, "/m/old.gguf") + // Make the rollback fail too: the old file "disappears" mid-swap. + fl.mu.Lock() + delete(fl.models, "/m/old.gguf") + fl.mu.Unlock() + + _, err := p.Swap(context.Background(), SwapSpec{ModelPath: "/m/broken.gguf"}) + if err == nil { + t.Fatal("Swap returned nil when both the load and the rollback failed") + } + // Nothing is loaded, and the request path says so rather than panicking. + if _, _, aerr := p.acquire(); !errors.Is(aerr, ErrNoBackend) { + t.Errorf("acquire error = %v; want ErrNoBackend", aerr) + } + // Phrasing degrades to its fallback instead of failing the turn. + got, err := p.PhraseChat(context.Background(), "привет", nil) + if err != nil { + t.Fatalf("PhraseChat after a total failure returned an error: %v", err) + } + if got == "" { + t.Error("PhraseChat returned empty; the fallback must still say something") + } +} + +func TestSwap_WaitsForInFlightTurnAndRefusesNewOnes(t *testing.T) { + fl := &fakeFleet{models: map[string]string{"/m/old.gguf": "old", "/m/new.gguf": "new"}} + p := newSwapPhraser(t, fl, "/m/old.gguf") + + // Hold one turn open by taking a slot directly — the same slot every + // request path takes. + base, release, err := p.acquire() + if err != nil { + t.Fatalf("acquire: %v", err) + } + if base == "" { + t.Fatal("acquire returned an empty base URL") + } + + swapped := make(chan error, 1) + go func() { _, e := p.Swap(context.Background(), SwapSpec{ModelPath: "/m/new.gguf"}); swapped <- e }() + + // While the swap waits to drain, a NEW turn is refused immediately rather + // than blocked for the length of a model load. + deadline := time.Now().Add(2 * time.Second) + for { + _, rel, aerr := p.acquire() + if rel != nil { + rel() + } + if errors.Is(aerr, ErrSwapping) { + break + } + if time.Now().After(deadline) { + t.Fatalf("new turns were never refused during a swap (last error: %v)", aerr) + } + time.Sleep(10 * time.Millisecond) + } + + // The swap cannot have completed while our turn was still in flight. + select { + case e := <-swapped: + t.Fatalf("Swap finished before the in-flight turn released: %v", e) + case <-time.After(50 * time.Millisecond): + } + + release() + if e := <-swapped; e != nil { + t.Fatalf("Swap after drain: %v", e) + } + got, err := p.PhraseChat(context.Background(), "привет", nil) + if err != nil || got != "new" { + t.Fatalf("after swap: %q, %v; want the new model", got, err) + } +} + +func TestSwap_RefusedWhenWeDoNotOwnTheServer(t *testing.T) { + // NewLLMPhraserAt points at a shared server the eval harness owns. Swapping + // there would kill a server another process depends on. + p := NewLLMPhraserAt("http://127.0.0.1:1/", DefaultConfig("/m/old.gguf")) + if _, err := p.Swap(context.Background(), SwapSpec{ModelPath: "/m/new.gguf"}); !errors.Is(err, ErrSwapNotOwned) { + t.Fatalf("Swap on a borrowed server = %v; want ErrSwapNotOwned", err) + } +} + +func TestSwap_SameModelIsANoOp(t *testing.T) { + fl := &fakeFleet{models: map[string]string{"/m/old.gguf": "old"}} + p := newSwapPhraser(t, fl, "/m/old.gguf") + launchesBefore := fl.launch + + res, err := p.Swap(context.Background(), SwapSpec{ModelPath: "/m/old.gguf"}) + if err != nil { + t.Fatalf("Swap to the live model: %v", err) + } + if res.Model != "old" { + t.Errorf("res.Model = %q; want old", res.Model) + } + if fl.launch != launchesBefore { + t.Errorf("%d extra launches; swapping to the live model must not reload weights", fl.launch-launchesBefore) + } +} + +func TestSwap_EmptyModelPathRefused(t *testing.T) { + fl := &fakeFleet{models: map[string]string{"/m/old.gguf": "old"}} + p := newSwapPhraser(t, fl, "/m/old.gguf") + if _, err := p.Swap(context.Background(), SwapSpec{}); err == nil { + t.Fatal("Swap with no model path returned nil error") + } +} diff --git a/internal/router/calendar.go b/internal/router/calendar.go index 72890ea..de182d0 100644 --- a/internal/router/calendar.go +++ b/internal/router/calendar.go @@ -4,16 +4,146 @@ import ( "fmt" "strings" "time" + "unicode" ) // CalendarEventFormatter formats calendar events into a Russian reply string. type CalendarEventFormatter struct{} -// Format returns a Russian reply for the given calendar events on the given date. -func (CalendarEventFormatter) Format(events []string, date time.Time) string { +// CalendarEntry — one event to recite. Uncertain marks an event maven did not +// read off a calendar server: the work calendar arrives as relayed phone +// notifications (Vikunja #126), stored below full confidence, and she says so +// rather than reciting a guess as fact. +type CalendarEntry struct { + Text string + Uncertain bool +} + +// dayPlanWords — the tokens that ask for the day as a whole rather than for a +// calendar listing. Whole words, not substrings: "планёрка" is a MEETING, and a +// notification about one must not be mistaken for a request for the plan. +var dayPlanWords = []string{ + "план", "плана", "плану", "плане", "планом", + "планы", "планов", "планам", "планах", + "расписание", "расписании", "распорядок", "распорядке", + "plan", "plans", "schedule", "agenda", +} + +// otherDayWords — a span that is not the clock's own day. The plan can only be +// built for today, so an utterance naming another day, a weekday, a week or a +// weekend belongs to the calendar listing instead. Claiming it here would +// answer today and stamp it with today's date, which is a wrong answer where +// falling through is only a terse one. +// +// The weekday names are here as a refusal, not as a feature. "какие планы на +// понедельник?" carries no other-day token in the сегодня family and does carry +// "планы", so the plan used to claim it and recite today. +var otherDayWords = []string{ + "завтра", "послезавтра", "вчера", "позавчера", + "tomorrow", "yesterday", + "понедельник", "вторник", "среду", "среда", "четверг", "пятницу", "пятница", + "субботу", "суббота", "воскресенье", + "понедельника", "вторника", "четверга", "пятницы", "субботы", "воскресенья", + "monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday", + "неделю", "неделя", "недели", "неделе", + "выходные", "выходных", "выходным", + "месяц", "месяца", "месяце", + "week", "weekend", "month", +} + +// IsDayPlanQuery reports whether an utterance asks for today's plan (Vikunja +// #128) — "какие планы на сегодня?", "что у меня по плану?", "что дальше?". +// +// Deliberately narrow. The calendar listing already answers "что у меня +// сегодня?" and a plan that hijacks every date-bearing question would bury the +// events under checklist lines. Only a plan-shaped ask, and only about today. +func IsDayPlanQuery(text string) bool { + // A habit question is never a day plan, whatever words it shares with one. + // "какие у меня обычно планы по вторникам?" carries "планы", so the plan + // source claimed it and answered today's calendar stamped with today's + // date, and the habit source never ran. Deciding it here rather than by + // reordering the source table keeps one matcher from depending on the + // other's position in a slice. + if _, ok := ParseHabitQuery(text); ok { + return false + } + toks := planTokens(text) + for _, t := range toks { + for _, w := range otherDayWords { + if t == w { + return false + } + } + } + for _, t := range toks { + for _, w := range dayPlanWords { + if t == w { + return true + } + } + } + // "что дальше?" / "what's next?" — the rest of the day, with no plan word + // in it. Both tokens rather than adjacency, because "what's" splits into + // "what" and "s" and because "и что потом дальше" is the same question. + return (hasTok(toks, "что") && hasTok(toks, "дальше")) || + (hasTok(toks, "what") && hasTok(toks, "next")) +} + +// IsRestOfDayQuery reports whether the utterance asks for what is left of the +// day rather than for the whole of it — "что дальше?" and its English form. +// +// Tokenized for the same reason IsDayPlanQuery is: the substring form matched +// "дальше" inside longer words and "next" inside "nextcloud", and the two +// predicates deciding the same utterance differently is worse than either +// being wrong on its own. +func IsRestOfDayQuery(text string) bool { + toks := planTokens(text) + return hasTok(toks, "дальше") || hasTok(toks, "next") +} + +func hasTok(toks []string, w string) bool { + for _, t := range toks { + if t == w { + return true + } + } + return false +} + +// planTokens lowercases and splits on everything that is not a letter or a +// digit, so "планы?" and "что-дальше" tokenize like the plain words do. +func planTokens(text string) []string { + return strings.FieldsFunc(strings.ToLower(text), func(r rune) bool { + return !unicode.IsLetter(r) && !unicode.IsDigit(r) + }) +} + +// Format returns a Russian reply for the given calendar events on the given +// date. Every event is treated as certain — use FormatEntries when provenance +// differs between them. +func (f CalendarEventFormatter) Format(events []string, date time.Time) string { + entries := make([]CalendarEntry, len(events)) + for i, e := range events { + entries[i] = CalendarEntry{Text: e} + } + return f.FormatEntries(entries, date) +} + +// FormatEntries returns a Russian reply, hedging the entries maven is not sure +// about. "похоже" and not "возможно": the notification did arrive, what is +// uncertain is whether it describes the meeting correctly. +func (CalendarEventFormatter) FormatEntries(entries []CalendarEntry, date time.Time) string { dateStr := date.Format("02.01.2006") - if len(events) == 0 { + if len(entries) == 0 { return fmt.Sprintf("на %s ничего нет.", dateStr) } - return fmt.Sprintf("на %s: %s", dateStr, strings.Join(events, "; ")) + parts := make([]string, len(entries)) + for i, e := range entries { + if e.Uncertain { + parts[i] = "похоже, " + e.Text + continue + } + parts[i] = e.Text + } + return fmt.Sprintf("на %s: %s", dateStr, strings.Join(parts, "; ")) } diff --git a/internal/router/calendar_test.go b/internal/router/calendar_test.go index ba5867f..7937a94 100644 --- a/internal/router/calendar_test.go +++ b/internal/router/calendar_test.go @@ -24,3 +24,95 @@ func TestCalendarEventFormatter(t *testing.T) { t.Errorf("multiple: got %q", got) } } + +func TestCalendarEventFormatterHedgesUncertainEntries(t *testing.T) { + f := CalendarEventFormatter{} + date := time.Date(2026, 7, 6, 0, 0, 0, 0, time.UTC) + + // An event relayed off a phone notification is not a calendar read, and she + // says so instead of reciting a guess as fact. + got := f.FormatEntries([]CalendarEntry{ + {Text: "Standup @ 10:00-10:30"}, + {Text: "Планёрка @ 14:00-14:30", Uncertain: true}, + }, date) + want := "на 06.07.2026: Standup @ 10:00-10:30; похоже, Планёрка @ 14:00-14:30" + if got != want { + t.Errorf("got %q\nwant %q", got, want) + } + + // Format is FormatEntries with everything certain. + if got := f.FormatEntries(nil, date); got != "на 06.07.2026 ничего нет." { + t.Errorf("empty: got %q", got) + } +} + +func TestIsDayPlanQuery(t *testing.T) { + yes := []string{ + "какие планы на сегодня?", + "что у меня по плану", + "расскажи план", + "мой распорядок на сегодня", + "расписание?", + "что дальше?", + "what's next", + "what is my plan today", + } + for _, s := range yes { + if !IsDayPlanQuery(s) { + t.Errorf("IsDayPlanQuery(%q) = false, want true", s) + } + } + + no := []string{ + // The calendar listing owns these. + "что у меня сегодня?", + "какие планы на завтра?", + "план на послезавтра", + "что было вчера", + // "планёрка" is a meeting, not a request for the plan. + "когда планёрка?", + "запиши планёрку на 14:00", + "какая погода?", + "", + } + for _, s := range no { + if IsDayPlanQuery(s) { + t.Errorf("IsDayPlanQuery(%q) = true, want false", s) + } + } +} + +// The plan is built for the clock's own day. A weekday, a week or a weekend +// carries no сегодня-family token, so the plan used to claim the utterance and +// recite today under today's date. Refusing is the right answer until the plan +// can build a day that is not the clock's own. +func TestIsDayPlanQueryRefusesOtherSpans(t *testing.T) { + for _, s := range []string{ + "какие планы на понедельник?", + "планы на пятницу", + "какие планы на неделю?", + "планы на выходные", + "какие планы на месяц?", + "what are my plans for friday?", + "my plan for the week", + } { + if IsDayPlanQuery(s) { + t.Errorf("IsDayPlanQuery(%q) = true, want false", s) + } + } +} + +// The rest-of-day test tokenizes like IsDayPlanQuery does. The substring form +// it replaced fired on any word containing "next" or "дальше". +func TestIsRestOfDayQuery(t *testing.T) { + for _, s := range []string{"что дальше?", "и что потом, дальше?", "what's next", "NEXT"} { + if !IsRestOfDayQuery(s) { + t.Errorf("IsRestOfDayQuery(%q) = false, want true", s) + } + } + for _, s := range []string{"какие планы на сегодня?", "проверь nextcloud", "дальшесъезд", ""} { + if IsRestOfDayQuery(s) { + t.Errorf("IsRestOfDayQuery(%q) = true, want false", s) + } + } +} diff --git a/internal/router/feeds.go b/internal/router/feeds.go new file mode 100644 index 0000000..f88b16e --- /dev/null +++ b/internal/router/feeds.go @@ -0,0 +1,142 @@ +package router + +import "strings" + +// Feed queries — "что нового в лентах?", "что нового по технологиям?" +// (Vikunja #258). +// +// Deterministic matching, like the calendar, plan and habit matchers above it: +// the LLM router says this is a query, and this decides whether it is a question +// about the feeds. A model deciding that would occasionally answer "что нового?" +// out of world knowledge, which is the one thing a feed reader exists to avoid. + +// FeedQuery — a parsed "what's new" question. Category is the topic he named +// ("технологии"), empty when he asked about the feeds in general. +type FeedQuery struct { + Category string +} + +// feedNouns — the words that name the feeds themselves. One of these is enough, +// with an ask, to make the turn a feed question. +var feedNouns = []string{ + "лента", "ленте", "ленты", "лентах", "лентам", + "новости", "новостей", "новостях", "новостям", + "feed", "feeds", "news", "headlines", +} + +// vagueNouns — the newness words that are NOT about the feeds by themselves. +// +// "что нового?" is the most common opener in the language and it is a greeting, +// not a request for headlines. It used to match here, so the shipping daemon — +// which has no feeds block — answered "я пока не читаю ленты, они не настроены", +// a configuration status in reply to hello. With feeds on it answered "в лентах +// пока ничего нового", which is no better. A vague noun claims the turn only +// when the utterance narrows it: a named topic ("что нового по технологиям"), or +// a feed noun somewhere in it ("что нового в лентах"). +var vagueNouns = []string{"новое", "нового", "новенького", "new"} + +// newnessMarkers — the "что нового" half. "нового" alone is in feedNouns +// because it carries the question on its own ("что нового?"); a bare "лента" +// needs the ask, which is what askMarkers below is for. +var askMarkers = []string{ + "что", "какие", "какое", "расскажи", "почитай", "прочитай", "покажи", + "what", "any", "tell", "show", "read", +} + +// ParseFeedQuery reports whether an utterance asks what is new in the feeds, and +// which topic if it names one after "по"/"об"/"про"/"about". +// +// A feed noun and an ask are required. "у меня новая лента в инстаграме" is a +// statement and must not be read as a request to recite headlines. A vague +// newness word counts as the noun only when a topic is named — see vagueNouns +// for why the bare "что нового?" must fall through. +func ParseFeedQuery(text string) (FeedQuery, bool) { + toks := planTokens(text) + noun, vague, ask := false, false, false + for _, t := range toks { + for _, n := range feedNouns { + if t == n { + noun = true + break + } + } + for _, n := range vagueNouns { + if t == n { + vague = true + break + } + } + for _, a := range askMarkers { + if t == a { + ask = true + break + } + } + } + if !ask { + return FeedQuery{}, false + } + cat := feedCategory(toks) + if !noun && !(vague && cat != "") { + return FeedQuery{}, false + } + return FeedQuery{Category: cat}, true +} + +// categoryPreps — the prepositions a topic follows. Russian marks the topic with +// a preposition ("по технологиям", "про политику"), so the word after one is the +// category; there is no stemming here, and the match against the configured +// category is a prefix comparison for exactly that reason. +// +// "о" is not in the list. It is one rune and it turns up as filler, a typo and +// half of "о'кей", so any utterance carrying a stray "о" produced a category of +// whatever word came next and she answered "по этой теме в лентах пока ничего" +// to a question that named no theme. "об" and "про" carry the same meaning and +// cannot be mistaken for anything else. +var categoryPreps = map[string]bool{"по": true, "об": true, "про": true, "about": true, "on": true} + +func feedCategory(toks []string) string { + for i, t := range toks { + if categoryPreps[t] && i+1 < len(toks) { + next := toks[i+1] + // "по новостям" names no topic, it repeats the noun. + for _, n := range append(append([]string{}, feedNouns...), vagueNouns...) { + if next == n { + return "" + } + } + return next + } + } + return "" +} + +// CategoryMatches reports whether a feed note's own category tag is the one he +// named. Russian inflects the topic ("технологиям" vs the configured +// "технологии"), and there is no stemmer in this repo, so the comparison is on a +// common prefix — long enough that "полит" and "погод" stay apart, short enough +// to survive a case ending. +// +// tag is the note's stored category (rss.NoteCategory), NOT the whole note. It +// used to be the whole note, which meant "что нового про погоду" matched any +// tech headline whose link happened to contain "pogod". +func CategoryMatches(tag, category string) bool { + if category == "" { + return true + } + stem := categoryStem(category) + if stem == "" { + return false + } + return strings.Contains(strings.ToLower(tag), stem) +} + +// categoryStem cuts a word down to the part inflection leaves alone. 5 runes is +// the compromise: shorter words are used whole. +func categoryStem(word string) string { + r := []rune(strings.ToLower(strings.TrimSpace(word))) + if len(r) > 5 { + r = r[:5] + } + return string(r) +} diff --git a/internal/router/feeds_test.go b/internal/router/feeds_test.go new file mode 100644 index 0000000..4ae6c31 --- /dev/null +++ b/internal/router/feeds_test.go @@ -0,0 +1,58 @@ +package router + +import "testing" + +func TestParseFeedQuery(t *testing.T) { + cases := []struct { + text string + ok bool + category string + }{ + {"что нового в лентах?", true, ""}, + {"что нового по технологиям", true, "технологиям"}, + {"какие новости?", true, ""}, + {"что нового по технологиям?", true, "технологиям"}, + {"расскажи новости про политику", true, "политику"}, + {"что нового по новостям", true, ""}, + {"what's new in the feeds?", true, ""}, + {"any news about kubernetes", true, "kubernetes"}, + // "что нового?" is a greeting. Claiming it made the shipping daemon + // answer hello with "я пока не читаю ленты — они не настроены". + {"что нового?", false, ""}, + {"ну что нового", false, ""}, + // A stray "о" is not a topic marker. + {"что нового в лентах, о боже", true, ""}, + // Statements, not requests. + {"у меня новая лента в инстаграме", false, ""}, + {"новости меня утомили", false, ""}, + {"напомни полить цветы", false, ""}, + {"", false, ""}, + } + for _, c := range cases { + q, ok := ParseFeedQuery(c.text) + if ok != c.ok { + t.Errorf("ParseFeedQuery(%q) ok = %v, want %v", c.text, ok, c.ok) + continue + } + if ok && q.Category != c.category { + t.Errorf("ParseFeedQuery(%q) category = %q, want %q", c.text, q.Category, c.category) + } + } +} + +func TestCategoryMatches(t *testing.T) { + // The inflected form he says must match the form the config spells. + if !CategoryMatches("технологии", "технологиям") { + t.Error("inflected category did not match") + } + if CategoryMatches("технологии", "политику") { + t.Error("unrelated category matched") + } + // The tag, not the note. The link in a tech headline is not a weather report. + if CategoryMatches("технологии", "погоду") { + t.Error("a tech note matched a weather question") + } + if !CategoryMatches("anything", "") { + t.Error("an empty category must match everything") + } +} diff --git a/internal/router/habit.go b/internal/router/habit.go new file mode 100644 index 0000000..f9e25d4 --- /dev/null +++ b/internal/router/habit.go @@ -0,0 +1,88 @@ +package router + +import "time" + +// Habit queries — "что я обычно делаю по вторникам?" (Vikunja #254). +// +// Deterministic matching, like the calendar and plan matchers: the LLM router +// classifies the intent, but WHICH weekday was asked about is a lookup, not a +// generation. A model that answers "по вторникам" for a question about Thursday +// gives a confidently wrong account of the owner's own life. + +// HabitQuery — a parsed "what do I usually do" question. Weekday is set only +// when the utterance names one; otherwise the answer covers the whole week. +// Weekend is set for "по выходным", which names two days rather than one. +type HabitQuery struct { + Weekday time.Weekday + HasWeekday bool + Weekend bool +} + +// habitMarkers — the words that make a question about habit rather than about +// today. Without one of these, "что я делаю" is a question about right now, and +// the recall path owns it. +var habitMarkers = []string{ + "обычно", "обычное", "чаще", "постоянно", "привычки", "привычка", "привычках", + "регулярно", "каждый", "каждую", "каждое", "usually", "habits", "habit", + "typically", "normally", +} + +// weekdayWords — every form of a weekday name maven needs to recognise, +// including the "по …ам" plural the question is usually phrased in. +var weekdayWords = map[string]time.Weekday{ + "понедельник": time.Monday, "понедельникам": time.Monday, + "вторник": time.Tuesday, "вторникам": time.Tuesday, + "среда": time.Wednesday, "среду": time.Wednesday, "средам": time.Wednesday, + "четверг": time.Thursday, "четвергам": time.Thursday, + "пятница": time.Friday, "пятницу": time.Friday, "пятницам": time.Friday, + "суббота": time.Saturday, "субботу": time.Saturday, "субботам": time.Saturday, + "воскресенье": time.Sunday, "воскресеньям": time.Sunday, + "воскресенья": time.Sunday, "воскресенью": time.Sunday, + "воскресеньем": time.Sunday, "воскресеньях": time.Sunday, + "monday": time.Monday, "mondays": time.Monday, + "tuesday": time.Tuesday, "tuesdays": time.Tuesday, + "wednesday": time.Wednesday, "wednesdays": time.Wednesday, + "thursday": time.Thursday, "thursdays": time.Thursday, + "friday": time.Friday, "fridays": time.Friday, + "saturday": time.Saturday, "saturdays": time.Saturday, + "sunday": time.Sunday, "sundays": time.Sunday, +} + +// weekendWords — the weekend as one unit. "что я обычно делаю по выходным?" +// has a habit marker and names days, but no weekday name is in it, so it used +// to fall through to the whole-week profile and answer about Tuesdays too. +var weekendWords = map[string]bool{ + "выходным": true, "выходные": true, "выходных": true, "выходной": true, + "weekend": true, "weekends": true, +} + +// ParseHabitQuery reports whether an utterance asks what the owner usually +// does, and on which weekday if it names one. +// +// A habit marker is required. "что я делаю в среду?" without one is a question +// about this coming Wednesday — the calendar's job — and answering it with a +// statistical average would be answering a different question. +func ParseHabitQuery(text string) (HabitQuery, bool) { + toks := planTokens(text) + marked := false + for _, t := range toks { + for _, m := range habitMarkers { + if t == m { + marked = true + break + } + } + } + if !marked { + return HabitQuery{}, false + } + for _, t := range toks { + if wd, ok := weekdayWords[t]; ok { + return HabitQuery{Weekday: wd, HasWeekday: true}, true + } + if weekendWords[t] { + return HabitQuery{Weekend: true}, true + } + } + return HabitQuery{}, true +} diff --git a/internal/router/habit_test.go b/internal/router/habit_test.go new file mode 100644 index 0000000..14af976 --- /dev/null +++ b/internal/router/habit_test.go @@ -0,0 +1,90 @@ +package router + +import ( + "testing" + "time" +) + +func TestParseHabitQuery(t *testing.T) { + tests := []struct { + in string + ok bool + wd time.Weekday + hasWD bool + }{ + {"что я обычно делаю по вторникам?", true, time.Tuesday, true}, + {"что я обычно делаю?", true, 0, false}, + {"какие у меня привычки", true, 0, false}, + {"что я каждую пятницу делаю", true, time.Friday, true}, + {"what do i usually do on mondays?", true, time.Monday, true}, + // No habit marker: this is a question about the coming Wednesday, and + // the calendar owns it. Answering with an average answers the wrong + // question. + {"что я делаю в среду?", false, 0, false}, + {"что у меня сегодня?", false, 0, false}, + {"какие планы на сегодня?", false, 0, false}, + {"", false, 0, false}, + } + for _, tt := range tests { + q, ok := ParseHabitQuery(tt.in) + if ok != tt.ok { + t.Errorf("ParseHabitQuery(%q) ok = %v, want %v", tt.in, ok, tt.ok) + continue + } + if !ok { + continue + } + if q.HasWeekday != tt.hasWD { + t.Errorf("ParseHabitQuery(%q) hasWeekday = %v, want %v", tt.in, q.HasWeekday, tt.hasWD) + continue + } + if q.HasWeekday && q.Weekday != tt.wd { + t.Errorf("ParseHabitQuery(%q) weekday = %v, want %v", tt.in, q.Weekday, tt.wd) + } + } +} + +// TestHabitQueryBeatsDayPlan — "какие у меня обычно планы по вторникам?" is a +// habit question that happens to carry a plan word. The day plan claimed it +// first and answered today's calendar stamped with today's date, and the habit +// source never ran. +func TestHabitQueryBeatsDayPlan(t *testing.T) { + for _, q := range []string{ + "какие у меня обычно планы по вторникам?", + "что обычно по плану в среду?", + "какие планы обычно по выходным?", + } { + if IsDayPlanQuery(q) { + t.Errorf("%q was claimed as a day plan", q) + } + if _, ok := ParseHabitQuery(q); !ok { + t.Errorf("%q is not parsed as a habit question", q) + } + } + // A plan question without a habit marker still belongs to the day plan. + for _, q := range []string{"какие планы на сегодня?", "что у меня по плану?"} { + if !IsDayPlanQuery(q) { + t.Errorf("%q must still be a day plan", q) + } + } +} + +// TestHabitQueryWeekendAndSundayForms — "по выходным" names days but no +// weekday, so it used to be answered with the whole-week profile. Sunday had +// only its dative plural listed. +func TestHabitQueryWeekendAndSundayForms(t *testing.T) { + q, ok := ParseHabitQuery("что я обычно делаю по выходным?") + if !ok || !q.Weekend || q.HasWeekday { + t.Errorf("weekend query parsed as %+v (ok=%v)", q, ok) + } + for _, s := range []string{ + "что я обычно делаю в воскресенье?", + "чем я обычно занят по воскресеньям?", + "что обычно бывает в воскресенья?", + } { + q, ok := ParseHabitQuery(s) + if !ok || !q.HasWeekday || q.Weekday != time.Sunday { + t.Errorf("%q parsed as %+v (ok=%v)", s, q, ok) + } + } +} diff --git a/internal/router/llmrouter.go b/internal/router/llmrouter.go index 33c47c5..eec8014 100644 --- a/internal/router/llmrouter.go +++ b/internal/router/llmrouter.go @@ -118,6 +118,39 @@ const routeRepeatPenalty = 1.15 // Route return ok=false so the caller drops to the classifier cascade. const routeIntentUnknown = "unknown" +// llmFullConfidence / llmThinConfidence — Vikunja #359. Confidence used to be +// hardcoded to 1.0 for every LLM decision, so the stage-3 gate in router.go +// never had anything to bite on and the LLM path could never produce a +// Clarify: on the 77-case RU fixture, 6/6 want_clarify cases were missed by +// EVERY model in the 31-07-2026 bake-off (0.8B through 2B) — proof this was a +// code bug, not a capability ceiling. +// +// The fix does not touch the prompt (routeSystem is under +// llm/check_prompt_parity.py in the training workspace; changing its text +// creates a parity break that has to be fixed there too — see Vikunja #362). +// Instead it reads structural signal that is already free: +// - a single-token utterance is thin evidence for anything a grammar +// didn't already catch at stage 0 — "вода" and "бэкап" alone don't say +// fact-vs-query or act-vs-report; +// - a fact with no key, or an act that never resolves to an allowlisted fn +// (checked in router.go, after slot-fill has had its say), is a decision +// with a hole in the one slot that makes it actionable. +// +// A model self-reporting confidence in the JSON was considered and rejected: +// a sub-2B is not calibrated (nothing stops it saying "confident" on exactly +// the cases it gets wrong today), and true logprobs would need a response +// field internal/llm.Client's Complete does not currently return — see +// internal/llm/client.go. +// +// llmThinConfidence sits below config.DefaultRouterThreshold (0.55) so the +// existing stage-3 gate in Router.Route treats it exactly like a low-scoring +// classifier result — same lane, same daemon-side clarify machinery +// (cmd/mavend/clarify.go), no new consumer to build. +const ( + llmFullConfidence = 1.0 + llmThinConfidence = 0.3 +) + type routeAction struct { Intent string `json:"intent"` Key string `json:"key"` @@ -160,7 +193,15 @@ func (lr *LLMRouter) Route(ctx context.Context, utterance string, now time.Time) if a.Intent == routeIntentUnknown { return Decision{}, false, nil } - d := Decision{Utterance: utterance, Stage: 1, Confidence: 1.0} + d := Decision{Utterance: utterance, Stage: 1, Confidence: llmFullConfidence} + // A single-token utterance is thin evidence: the model had nothing to + // disambiguate on ("вода" is a fact-or-query coin flip, "бэкап" an + // act-or-report one) and stage 0 would already have won on anything + // that pattern-matches cleanly. Flag it now; router.go's stage-3 gate + // (Router.Route) decides whether that trips Clarify. + if len(strings.Fields(utterance)) <= 1 { + d.Confidence = llmThinConfidence + } switch Intent(a.Intent) { case IntentFact: d.Intent = IntentFact diff --git a/internal/router/llmrouter_test.go b/internal/router/llmrouter_test.go index 658dab7..49279c1 100644 --- a/internal/router/llmrouter_test.go +++ b/internal/router/llmrouter_test.go @@ -258,4 +258,101 @@ func TestLLMFactGetsKeyFromParser(t *testing.T) { if !d.Slots.HasKey || d.Slots.Key != "water" { t.Fatalf("want key=water, got %+v", d.Slots) } + if d.Clarify { + t.Fatalf("the parser resolved the key, this must not clarify: %+v", d) + } +} + +// --- confidence / stage-3 gate on the LLM path (Vikunja #359) ----------------- + +// A single-token utterance is thin evidence on its own — "вода" alone is a +// fact/query coin flip. The gate must ask rather than guess confidently. +func TestLLMRouterSingleTokenTripsClarify(t *testing.T) { + r := newLLMTestRouter(t, `{"intent":"query","text":"вода"}`) + d, err := r.Route(context.Background(), "вода", refNow()) + if err != nil { + t.Fatalf("route: %v", err) + } + if !d.Clarify { + t.Fatalf("a bare single-token decision must clarify, got %+v", d) + } +} + +// A multi-word utterance with a clean answer must not be punished — the +// whole point is not trading the confident cases away for clarify coverage. +func TestLLMRouterMultiWordStaysConfident(t *testing.T) { + r := newLLMTestRouter(t, `{"intent":"reminder","text":"позвонить маме"}`) + d, err := r.Route(context.Background(), "напомни позвонить маме", refNow()) + if err != nil { + t.Fatalf("route: %v", err) + } + if d.Clarify { + t.Fatalf("a clean multi-word decision must not clarify: %+v", d) + } + if d.Confidence != llmFullConfidence { + t.Fatalf("want full confidence, got %v", d.Confidence) + } +} + +// "бэкап" alone: the model guesses act, but nothing on the allowlist matches +// "бэкап" as a verb — that must not fire a tool blind. +func TestLLMRouterActWithoutFnTripsClarify(t *testing.T) { + r := newLLMTestRouter(t, `{"intent":"act","verb":"бэкап"}`) + d, err := r.Route(context.Background(), "бэкап сделай пожалуйста расписание", refNow()) + if err != nil { + t.Fatalf("route: %v", err) + } + if d.Slots.HasFn { + t.Fatalf("test setup drifted: %q now resolves to an fn", d.Slots.Fn) + } + if !d.Clarify { + t.Fatalf("an unresolved act must clarify rather than guess: %+v", d) + } +} + +// An act that DOES resolve to an allowlisted fn must stay confident even +// though its own verb is single-word-ish in spirit — guard against the fn +// check firing on the happy path. +func TestLLMRouterActWithFnStaysConfident(t *testing.T) { + r := newLLMTestRouter(t, `{"intent":"act","verb":"restart nginx"}`) + d, err := r.Route(context.Background(), "слушай, restart nginx пожалуйста", refNow()) + if err != nil { + t.Fatalf("route: %v", err) + } + if !d.Slots.HasFn { + t.Fatalf("test setup drifted, want fn resolved: %+v", d.Slots) + } + if d.Clarify { + t.Fatalf("a resolved act must not clarify: %+v", d) + } +} + +// A fact where NEITHER the model NOR the deterministic parser can name a key +// must clarify instead of silently writing under an empty/guessed key. +func TestLLMRouterFactWithoutKeyTripsClarify(t *testing.T) { + r := newLLMTestRouter(t, `{"intent":"fact","value":"что-то"}`) + d, err := r.Route(context.Background(), "у меня какая-то фигня случилась вот прямо только что", refNow()) + if err != nil { + t.Fatalf("route: %v", err) + } + if d.Slots.HasKey { + t.Fatalf("test setup drifted: parser now resolves a key for this utterance") + } + if !d.Clarify { + t.Fatalf("a keyless fact must clarify rather than guess: %+v", d) + } +} + +// The whole point of #359: the classifier cascade cannot be traded away for +// clarify coverage. A multi-word fact the parser CAN key must stay confident +// through the full Router.Route path, not just the raw LLMRouter. +func TestRouterLLMFactWithResolvedKeyStaysConfident(t *testing.T) { + r := newLLMTestRouter(t, `{"intent":"fact","text":"я выпил воду"}`) + d, err := r.Route(context.Background(), "я выпил воду", refNow()) + if err != nil { + t.Fatalf("route: %v", err) + } + if d.Clarify { + t.Fatalf("a fact the parser could key must not clarify: %+v", d) + } } diff --git a/internal/router/money.go b/internal/router/money.go new file mode 100644 index 0000000..6d5917a --- /dev/null +++ b/internal/router/money.go @@ -0,0 +1,104 @@ +package router + +import "strings" + +// Money questions, matched deterministically (Vikunja #125). +// +// No new intent, for the same reason as tasks: the intent enum is a contract +// with the relabelling prompt. "сколько я потратил?" is a query; which figure +// it asks for is a lookup, not something to ask a 1.7B — and a model asked to +// invent a spending total will happily do it. + +// MoneyWindow — which period a money question asks about. +type MoneyWindow int + +const ( + MoneyNone MoneyWindow = iota + MoneyToday + MoneyMonth + // MoneyUnsupported — a money question over a window nothing is stored for + // ("вчера", "на прошлой неделе"). Claimed, not answered: the poller keeps + // today and the month, and answering a question about yesterday with the + // month-to-date total is worse than saying she does not keep it. + MoneyUnsupported +) + +// MoneyQuery — a parsed money question. Income is set when he asked what he +// EARNED rather than what he spent; the two read the same fact and differ only +// in which half of it leads the answer. +type MoneyQuery struct { + Window MoneyWindow + Income bool +} + +// incomeNouns — the words that make a money question be about income. +var incomeNouns = []string{"заработал", "заработала", "получил", "доход", "доходы", "earned", "income"} + +// moneyNouns — the words that make a question be about his money. +var moneyNouns = []string{ + "потратил", "потратила", "тратил", "траты", "трат", "расходы", "расходов", + "заработал", "заработала", "доход", "доходы", "потрачено", "денег", + "spend", "spent", "expenses", "earned", "income", +} + +// ParseMoneyQuery reports whether an utterance asks about spending or income, +// and over which window. Defaults to the month: "сколько я потратил?" without a +// period is the month-to-date question, which is the one worth answering. +// +// Narrow on purpose. A money noun alone is not enough — "я потратил весь день +// на это" is him talking about his day, so an amount word or an explicit +// question word has to be there too. The two evidence halves are INDEPENDENT: +// "траты" and "расходы" used to sit in both lists, so either word alone +// satisfied the whole gate and "у меня в этом месяце большие траты", a +// statement, came back with a figure. +func ParseMoneyQuery(text string) (MoneyQuery, bool) { + toks := planTokens(text) + if len(toks) == 0 { + return MoneyQuery{}, false + } + hasNoun := false + for _, t := range toks { + for _, n := range moneyNouns { + if t == n { + hasNoun = true + } + } + } + if !hasNoun { + return MoneyQuery{}, false + } + // "весь день", "время", "силы" — spending that is not money. + for _, t := range toks { + switch t { + case "день", "дня", "время", "времени", "силы", "сил", "нервы": + return MoneyQuery{}, false + } + } + asking := hasTok(toks, "сколько") || hasTok(toks, "какие") || hasTok(toks, "покажи") || + hasTok(toks, "how") || hasTok(toks, "much") || hasTok(toks, "my") || + hasTok(toks, "мои") + if !asking { + return MoneyQuery{}, false + } + income := false + for _, t := range toks { + for _, n := range incomeNouns { + if t == n { + income = true + } + } + } + lower := strings.ToLower(text) + switch { + // Windows nothing is stored for, named explicitly so they are refused + // rather than silently answered with the month. + case hasTok(toks, "вчера") || hasTok(toks, "позавчера") || strings.Contains(lower, "yesterday"), + hasTok(toks, "неделю") || hasTok(toks, "неделе") || hasTok(toks, "неделя") || + strings.Contains(lower, "week"), + hasTok(toks, "год") || hasTok(toks, "году") || strings.Contains(lower, "year"): + return MoneyQuery{Window: MoneyUnsupported, Income: income}, true + case hasTok(toks, "сегодня") || strings.Contains(lower, "today"): + return MoneyQuery{Window: MoneyToday, Income: income}, true + } + return MoneyQuery{Window: MoneyMonth, Income: income}, true +} diff --git a/internal/router/money_test.go b/internal/router/money_test.go new file mode 100644 index 0000000..e063845 --- /dev/null +++ b/internal/router/money_test.go @@ -0,0 +1,39 @@ +package router + +import "testing" + +func TestParseMoneyQuery(t *testing.T) { + cases := []struct { + in string + window MoneyWindow + ok bool + }{ + {"сколько я потратил сегодня?", MoneyToday, true}, + {"сколько я потратил в этом месяце?", MoneyMonth, true}, + {"сколько я потратил?", MoneyMonth, true}, // month-to-date by default + {"покажи мои траты", MoneyMonth, true}, + {"какие у меня расходы за месяц", MoneyMonth, true}, + {"how much did I spend today", MoneyToday, true}, + {"сколько я заработал в этом месяце", MoneyMonth, true}, + // Windows nothing is stored for are claimed and refused, never answered + // with the month-to-date figure. + {"сколько я потратил вчера?", MoneyUnsupported, true}, + {"сколько я потратил на прошлой неделе?", MoneyUnsupported, true}, + {"how much did I spend yesterday", MoneyUnsupported, true}, + // Not about money. + {"я потратил весь день на это", MoneyNone, false}, + // A statement, not a question: the noun and the ask must be independent + // evidence, and "траты" used to satisfy both halves on its own. + {"у меня в этом месяце большие траты", MoneyNone, false}, + {"потратил много сил", MoneyNone, false}, + {"какая погода?", MoneyNone, false}, + {"я купил молоко", MoneyNone, false}, + {"", MoneyNone, false}, + } + for _, c := range cases { + q, ok := ParseMoneyQuery(c.in) + if ok != c.ok || q.Window != c.window { + t.Errorf("ParseMoneyQuery(%q) = (%v, %v), want (%v, %v)", c.in, q.Window, ok, c.window, c.ok) + } + } +} diff --git a/internal/router/router.go b/internal/router/router.go index 4b8fd18..bf9b430 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -89,6 +89,7 @@ func (r *Router) Route(ctx context.Context, utterance string, now time.Time) (De if d, ok, err := r.llm.Route(ctx, utterance, now); err == nil && ok { d.Utterance = utterance r.fillSlots(ctx, &d, now) + r.gateLLMDecision(&d) return d, nil } else if err != nil { log.Printf("router: llm route fell back to classifier: %v", err) @@ -152,6 +153,35 @@ func (r *Router) fillSlots(ctx context.Context, d *Decision, now time.Time) { // Stage stays 1: it says who decided the route, and that was the LLM. } +// gateLLMDecision — stage 3 for the LLM path (Vikunja #359). This used to be +// the classifier's job alone (see the threshold check at the bottom of +// Route): the LLM branch returned straight from fillSlots and never touched +// r.threshold at all, so a hardcoded Confidence: 1.0 in llmrouter.go could +// never gate. Two more structural holes are checked here, after fillSlots +// has had a chance to fill them from the deterministic parsers — checking +// before fillSlots would flag e.g. every keyless fact the fact parser goes +// on to resolve (TestLLMFactGetsKeyFromParser): +// - a fact with no key even after the parser tried — nothing to write, or +// worse, a confident write under the wrong key; +// - an act that never resolved to an allowlisted fn — a confident guess +// here means either silently doing nothing or, if the daemon is lax, +// running something never on the allowlist. Don't guess; ask. +// +// Anything below threshold gets the exact same Clarify=true treatment the +// classifier path already produces — same field, same daemon-side consumer +// (cmd/mavend/clarify.go), nothing new to wire. +func (r *Router) gateLLMDecision(d *Decision) { + if d.Intent == IntentFact && !d.Slots.HasKey && d.Confidence > llmThinConfidence { + d.Confidence = llmThinConfidence + } + if d.Intent == IntentAct && !d.Slots.HasFn && d.Confidence > llmThinConfidence { + d.Confidence = llmThinConfidence + } + if d.Confidence < r.threshold { + d.Clarify = true + } +} + // CorrectMisroute — the user corrected a bad classification. Appends a new // example for the corrected intent (append-only — grows the classifier, no // retrain). Same shape as nudges.outcome tuning cooldowns: more reliable over diff --git a/internal/router/task.go b/internal/router/task.go new file mode 100644 index 0000000..c85059b --- /dev/null +++ b/internal/router/task.go @@ -0,0 +1,203 @@ +package router + +import "strings" + +// Task capture and task listing, matched deterministically (Vikunja #130). +// +// No new intent. The router's intent enum is a contract shared with the +// relabelling prompt in the training workspace (`llm/check_prompt_parity.py` +// enforces it), so adding an eighth intent would mean retraining before a task +// could be captured at all. A task phrased out loud is a note-shaped or +// query-shaped utterance with an explicit marker in it, and the marker is a +// lookup — the same reasoning the calendar, plan and habit matchers already +// follow. What the model classifies is unchanged; what these functions decide +// is which store the turn lands in. + +// The phrase tables this file matches against — taskCapturePrefixes, +// urgencyMarkers, taskListWords, taskListWordsShortcut — are loaded from the +// embedded task_phrases.json. See task_phrases.go. + +// TaskCapture — a parsed capture: the task itself, plus the importance he +// stated out loud if he stated one (Vikunja #129). Weight 0 means he said +// nothing about importance, which the ranker treats as exactly that — no +// urgency is inferred from the wording. +type TaskCapture struct { + Text string + Weight int +} + +// ParseTaskCapture reports whether an utterance explicitly files a task, and +// returns the task text with the marker stripped. A marker with nothing after it +// is not a capture (there is no task in "добавь в задачи") — the caller falls +// through to whatever it would otherwise have done with the turn. +func ParseTaskCapture(text string) (TaskCapture, bool) { + trimmed := strings.TrimSpace(text) + lower := strings.ToLower(trimmed) + best := "" + for _, p := range taskCapturePrefixes { + if strings.HasPrefix(lower, p) && len(p) > len(best) { + best = p + } + } + if best == "" { + return TaskCapture{}, false + } + // Cut on the rune length of the matched prefix. ToLower does not change the + // byte length of Russian or English letters, so the index carries over. + rest := strings.TrimSpace(trimmed[len(best):]) + rest = strings.TrimLeft(rest, ":—- ") + rest = strings.TrimSpace(rest) + // The question mark goes too. Whisper punctuates dictated Russian, and + // "добавь в задачи позвонить в банк?" must not store the mark or carry it + // into the dedupe key. + rest = strings.TrimRight(rest, ".!?") + rest, weight := stripUrgency(rest) + if rest == "" { + return TaskCapture{}, false + } + return TaskCapture{Text: rest, Weight: weight}, true +} + +// urgencyIntensifiers — words that may sit between the edge and the marker. +// "очень срочно оплатить интернет" is the marker at the edge with one word in +// front of it, and it means exactly what "срочно оплатить интернет" means. +var urgencyIntensifiers = []string{"очень", "прям", "прямо", "really", "very", "super"} + +// urgencyEdgeTrim — punctuation to ignore around an edge token and to clean off +// the remainder afterwards. +const urgencyEdgeTrim = " .,;:!?—-" + +// stripUrgency pulls a leading or trailing urgency word out of the task text +// and returns the weight it implies. Only at the edges: "срочно оплатить +// интернет" and "оплатить интернет срочно" are the same instruction, while +// "позвонить в срочную помощь" is a task whose text happens to contain the +// stem, and cutting a word out of the middle of it would mangle the task. +// +// Matched as a TOKEN, not as a fixed prefix or suffix string. The old shape +// required exactly one space before a trailing marker, so "оплатить интернет, +// срочно" — which is what whisper produces from dictated Russian — kept weight +// 0 and stored the comma and the word as part of the task, polluting the dedupe +// key with the very flag he was trying to set. +// +// The word is removed from the text, because the list should read "оплатить +// интернет (важно)" and not "важно оплатить интернет (важно)". +func stripUrgency(text string) (string, int) { + fields := strings.Fields(text) + if len(fields) == 0 { + return text, 0 + } + for _, m := range urgencyMarkers { + // Strongest marker first (task_phrases.go sorts them), leading edge + // before trailing, so a text carrying both keeps the stronger one. + if lo, hi, ok := urgencySpan(fields, m.Word); ok { + rest := strings.Join(append(append([]string{}, fields[:lo]...), fields[hi+1:]...), " ") + rest = strings.Trim(rest, urgencyEdgeTrim) + if rest == "" { + // Nothing but the marker — no task in it. + return "", 0 + } + return rest, m.Weight + } + } + return text, 0 +} + +// urgencySpan finds the marker at either edge, allowing intensifiers between +// the edge and the marker, and returns the inclusive token range to cut. +func urgencySpan(fields []string, word string) (lo, hi int, ok bool) { + for i := 0; i < len(fields); i++ { + if isUrgencyToken(fields[i], word) { + return 0, i, true + } + if !isIntensifier(fields[i]) { + break + } + } + for i := len(fields) - 1; i >= 0; i-- { + if isUrgencyToken(fields[i], word) { + return i, len(fields) - 1, true + } + if !isIntensifier(fields[i]) { + break + } + } + return 0, 0, false +} + +func isUrgencyToken(tok, word string) bool { + return strings.Trim(strings.ToLower(tok), urgencyEdgeTrim) == word +} + +func isIntensifier(tok string) bool { + t := strings.Trim(strings.ToLower(tok), urgencyEdgeTrim) + for _, w := range urgencyIntensifiers { + if t == w { + return true + } + } + return false +} + +// IsTaskListQuery reports whether an utterance asks for the outstanding task +// list — "какие у меня задачи?", "что мне нужно сделать?", "список дел". +// +// Narrow on purpose. "как дела?" is a greeting, not a query about work, and it +// contains a task noun; it is excluded explicitly. Anything that mentions a +// task noun without asking for the list falls through to ordinary recall. +func IsTaskListQuery(text string) bool { + toks := planTokens(text) + if len(toks) == 0 { + return false + } + // "как дела" — the greeting. Excluded before anything else matches. + if hasTok(toks, "как") && (hasTok(toks, "дела") || hasTok(toks, "делишки")) { + return false + } + // "что мне нужно сделать" / "чем мне заняться" — no task noun at all, so + // the pronoun is what carries the meaning. Without it these rules claimed + // every question with a verb in them: "что нужно сделать чтобы перезапустить + // сервер?" and "what does docker do?" both answered "задач нет." from ahead + // of recall and the model, which is the failure the source ordering exists + // to avoid, pointed the other way. + // + // A "с"/"со" object excludes them too: "что мне сделать с этим файлом" has + // the pronoun and is still a question about a file. + if !hasTok(toks, "с") && !hasTok(toks, "со") { + if hasTok(toks, "мне") && (hasTok(toks, "что") || hasTok(toks, "чем")) && + (hasTok(toks, "сделать") || hasTok(toks, "делать") || hasTok(toks, "заняться")) { + return true + } + if hasTok(toks, "what") && hasTok(toks, "do") && hasTok(toks, "i") && !hasTok(toks, "you") { + return true + } + } + hasNoun := false + for _, t := range toks { + for _, w := range taskListWords { + if t == w { + hasNoun = true + } + } + } + if !hasNoun { + return false + } + // A task noun plus any of: a question word, "список", or a bare + // one/two-word ask ("задачи", "мои задачи"). + if hasTok(toks, "какие") || hasTok(toks, "какая") || hasTok(toks, "что") || + hasTok(toks, "сколько") || hasTok(toks, "список") || hasTok(toks, "покажи") || + hasTok(toks, "напомни") || hasTok(toks, "my") || hasTok(toks, "list") || + hasTok(toks, "show") { + return true + } + if len(toks) <= 2 { + for _, t := range toks { + for _, w := range taskListWordsShortcut { + if t == w { + return true + } + } + } + } + return false +} diff --git a/internal/router/task_phrases.go b/internal/router/task_phrases.go new file mode 100644 index 0000000..c7113f7 --- /dev/null +++ b/internal/router/task_phrases.go @@ -0,0 +1,59 @@ +package router + +import ( + _ "embed" + "encoding/json" + "fmt" + "sort" +) + +// The task vocabulary lives in task_phrases.json, not in Go source. See the +// comment block inside that file for what each list means and why the entries +// that are NOT in it were left out. +// +// go:embed, so the single-binary deploy is unchanged: the JSON is compiled into +// mavend and there is nothing to install beside it. Parsed once at init; a +// malformed asset panics at startup rather than silently disabling task capture, +// which would look like the feature quietly not working. + +//go:embed task_phrases.json +var taskPhrasesJSON []byte + +type urgencyMarker struct { + Word string `json:"word"` + Weight int `json:"weight"` +} + +type taskPhrases struct { + CapturePrefixes []string `json:"capture_prefixes"` + UrgencyMarkers []urgencyMarker `json:"urgency_markers"` + ListNouns []string `json:"list_nouns"` + ListShortcutNouns []string `json:"list_shortcut_nouns"` +} + +var ( + taskCapturePrefixes []string + urgencyMarkers []urgencyMarker + taskListWords []string + taskListWordsShortcut []string +) + +func init() { + var v taskPhrases + if err := json.Unmarshal(taskPhrasesJSON, &v); err != nil { + panic(fmt.Sprintf("router: task_phrases.json: %v", err)) + } + if len(v.CapturePrefixes) == 0 || len(v.ListNouns) == 0 { + panic("router: task_phrases.json: capture_prefixes and list_nouns must be non-empty") + } + // Strongest urgency first, so stripUrgency finds "срочно" before "важно" + // in an utterance carrying both. The file is written in that order already; + // sorting here means a careless edit cannot silently downgrade a task. + sort.SliceStable(v.UrgencyMarkers, func(i, j int) bool { + return v.UrgencyMarkers[i].Weight > v.UrgencyMarkers[j].Weight + }) + taskCapturePrefixes = v.CapturePrefixes + urgencyMarkers = v.UrgencyMarkers + taskListWords = v.ListNouns + taskListWordsShortcut = v.ListShortcutNouns +} diff --git a/internal/router/task_phrases.json b/internal/router/task_phrases.json new file mode 100644 index 0000000..f70eff3 --- /dev/null +++ b/internal/router/task_phrases.json @@ -0,0 +1,59 @@ +{ + "_comment": [ + "The task capture and task-listing vocabulary. Embedded by task_phrases.go.", + "", + "These are lexicon, not logic: which words mean 'put this on the list' is a", + "fact about how he speaks, and it changes as he uses the thing. Keeping them", + "in Go meant every new phrasing was a source diff.", + "", + "capture_prefixes must be LEADING phrases. 'добавь в задачи купить молоко' is", + "a capture; 'я не добавил молоко в список' is him talking, and only position", + "tells them apart. There is deliberately no 'надо'/'нужно' entry: 'надо бы", + "поспать' is a mood, not a task, and guessing would fill the list with them.", + "", + "urgency markers carry the weight he stated out loud. Two rungs only:", + "'срочно' is a deadline he has not named and 'важно' is a preference. A third", + "shade would be a distinction he never makes.", + "", + "list_nouns are the nouns that make a question be about the list.", + "list_shortcut_nouns are the subset that stand alone as a whole utterance", + "('задачи'), which the longer nouns do not ('дел')." + ], + "capture_prefixes": [ + "добавь в задачи", + "добавь в тудушки", + "добавь в список задач", + "добавь в список дел", + "добавь в список", + "добавь задачу", + "запиши в задачи", + "запиши задачу", + "новая задача", + "поставь задачу", + "в задачи", + "add a task", + "add task", + "add to my tasks", + "add to tasks", + "new task" + ], + "urgency_markers": [ + { "word": "срочно", "weight": 3 }, + { "word": "urgent", "weight": 3 }, + { "word": "важно", "weight": 2 }, + { "word": "important", "weight": 2 } + ], + "list_nouns": [ + "задачи", + "задачах", + "задач", + "задачам", + "дела", + "делах", + "дел", + "tasks", + "todo", + "todos" + ], + "list_shortcut_nouns": ["задачи", "задач", "tasks"] +} diff --git a/internal/router/task_test.go b/internal/router/task_test.go new file mode 100644 index 0000000..75503f5 --- /dev/null +++ b/internal/router/task_test.go @@ -0,0 +1,87 @@ +package router + +import "testing" + +func TestParseTaskCapture(t *testing.T) { + cases := []struct { + in string + text string + weight int + ok bool + }{ + {"добавь в задачи купить молоко", "купить молоко", 0, true}, + {"Добавь в список дел: позвонить в банк", "позвонить в банк", 0, true}, + {"запиши задачу починить кран.", "починить кран", 0, true}, + {"новая задача — оплатить интернет", "оплатить интернет", 0, true}, + {"add a task buy milk", "buy milk", 0, true}, + // Urgency he stated out loud, leading or trailing, stripped from the text. + {"добавь в задачи срочно оплатить интернет", "оплатить интернет", 3, true}, + {"добавь в задачи оплатить интернет срочно", "оплатить интернет", 3, true}, + {"новая задача важно позвонить маме", "позвонить маме", 2, true}, + // The stem inside the task text is part of the task, not a marker. + {"добавь в задачи позвонить в срочную помощь", "позвонить в срочную помощь", 0, true}, + // Whisper punctuates dictated Russian. The marker used to be missed as + // soon as anything sat next to it, and then it stayed in the task text + // and in the dedupe key — the exact task he was trying to flag. + {"добавь в задачи оплатить интернет, срочно", "оплатить интернет", 3, true}, + {"добавь в задачи очень срочно оплатить интернет", "оплатить интернет", 3, true}, + {"добавь в задачи оплатить интернет — важно", "оплатить интернет", 2, true}, + // A dictated question mark is not part of the task. + {"добавь в задачи позвонить в банк?", "позвонить в банк", 0, true}, + // The phrasings he uses that the prefix list did not have. + {"поставь задачу вынести мусор", "вынести мусор", 0, true}, + {"добавь в тудушки купить лампочки", "купить лампочки", 0, true}, + // A marker with nothing after it files nothing. + {"добавь в задачи", "", 0, false}, + {"новая задача", "", 0, false}, + {"добавь в задачи срочно", "", 0, false}, + // Not a capture: he is talking, not filing. + {"надо бы поспать", "", 0, false}, + {"я не добавил молоко в список", "", 0, false}, + {"какие у меня задачи?", "", 0, false}, + {"", "", 0, false}, + } + for _, c := range cases { + got, ok := ParseTaskCapture(c.in) + if ok != c.ok || got.Text != c.text || got.Weight != c.weight { + t.Errorf("ParseTaskCapture(%q) = (%+v, %v), want (%q, w=%d, %v)", c.in, got, ok, c.text, c.weight, c.ok) + } + } +} + +func TestIsTaskListQuery(t *testing.T) { + yes := []string{ + "какие у меня задачи?", + "что мне нужно сделать?", + "покажи список дел", + "сколько у меня задач?", + "задачи", + "мои задачи", + "what should I do", + "что мне делать?", + } + for _, s := range yes { + if !IsTaskListQuery(s) { + t.Errorf("IsTaskListQuery(%q) = false, want true", s) + } + } + no := []string{ + "как дела?", + // No task noun and no pronoun: these fired ahead of recall and the + // model, and answered a question about a file or a server with + // "задач нет." + "что нужно сделать чтобы перезапустить сервер?", + "что мне сделать с этим файлом?", + "what does docker do?", + "what do you do?", + "какая погода?", + "напомни мне позвонить маме в шесть", + "я сделал зарядку", + "", + } + for _, s := range no { + if IsTaskListQuery(s) { + t.Errorf("IsTaskListQuery(%q) = true, want false", s) + } + } +} diff --git a/internal/router/url.go b/internal/router/url.go new file mode 100644 index 0000000..8cdea09 --- /dev/null +++ b/internal/router/url.go @@ -0,0 +1,39 @@ +package router + +import ( + "regexp" + "strings" +) + +// Finding a URL in an utterance (Vikunja #259). +// +// This is deliberately strict: a scheme is required. "посмотри на example.org" +// is not treated as a fetch request, because a bare dotted word is also how +// people say file names, versions and Russian abbreviations, and the cost of a +// false positive here is an outbound request nobody asked for. +// +// Note where this runs: an utterance from STT. Whisper will mangle a spoken URL, +// which is fine — the URL that survives is one he pasted into the web chat, and +// a mangled one simply fails to match. +var urlRE = regexp.MustCompile(`(?i)\bhttps?://[^\s<>"']+`) + +// FirstURL returns the first http(s) URL in text. +// +// Trailing punctuation is trimmed: he ends sentences, and "…/page." is not a +// path component. A closing bracket is only trimmed when it has no opener, +// because a wikipedia URL legitimately ends in one. +func FirstURL(text string) (string, bool) { + m := urlRE.FindString(text) + if m == "" { + return "", false + } + m = strings.TrimRight(m, ".,;:!?…") + if strings.HasSuffix(m, ")") && strings.Count(m, "(") == 0 { + m = strings.TrimSuffix(m, ")") + } + // A scheme with nothing after it is not a URL. + if rest := strings.SplitN(m, "//", 2); len(rest) < 2 || rest[1] == "" { + return "", false + } + return m, true +} diff --git a/internal/router/url_test.go b/internal/router/url_test.go new file mode 100644 index 0000000..6710773 --- /dev/null +++ b/internal/router/url_test.go @@ -0,0 +1,34 @@ +package router + +import "testing" + +func TestFirstURL(t *testing.T) { + cases := []struct { + text string + want string + }{ + {"посмотри https://example.org/page — что там?", "https://example.org/page"}, + {"почитай http://example.org/a/b?x=1 и скажи", "http://example.org/a/b?x=1"}, + {"вот ссылка: https://example.org/page.", "https://example.org/page"}, + {"https://ru.wikipedia.org/wiki/Небо_(значения)", "https://ru.wikipedia.org/wiki/Небо_(значения)"}, + // No scheme ⇒ no fetch. A bare dotted word is not an instruction to + // reach out to the network. + {"посмотри на example.org", ""}, + {"открой файл config.json", ""}, + {"что нового?", ""}, + {"https://", ""}, + {"", ""}, + } + for _, c := range cases { + got, ok := FirstURL(c.text) + if c.want == "" { + if ok { + t.Errorf("FirstURL(%q) = %q, want no match", c.text, got) + } + continue + } + if !ok || got != c.want { + t.Errorf("FirstURL(%q) = %q, %v; want %q", c.text, got, ok, c.want) + } + } +} diff --git a/internal/rss/feed.go b/internal/rss/feed.go new file mode 100644 index 0000000..06edfa3 --- /dev/null +++ b/internal/rss/feed.go @@ -0,0 +1,183 @@ +// Package rss reads RSS 2.0 and Atom feeds, and does nothing else with them. +// +// Parsing and polling are split from delivery on purpose: a feed is a source +// Maven can be ASKED about, not a thing that speaks. Nothing in this package +// dispatches, nudges or notifies — the poller writes notes, and the answer path +// reads them when he asks "что нового в лентах?". "Not a nag" is the oldest +// constraint in the spec, and a news feed is the single most tempting way to +// break it. +// +// Stdlib only (encoding/xml). Feeds are XML from strangers, so the parser takes +// what it recognises and ignores the rest rather than failing a whole feed over +// one malformed item. +package rss + +import ( + "encoding/xml" + "fmt" + "html" + "io" + "regexp" + "strings" + "time" +) + +// Item is one feed entry, normalised across RSS and Atom. +type Item struct { + Title string + Link string + Summary string // plain text, tags stripped, entities decoded + Published time.Time // zero when the feed did not say + ID string // guid / atom id, falling back to the link +} + +// Feed is a parsed document. +type Feed struct { + Title string + Items []Item +} + +// feedDoc covers both dialects in one struct. RSS puts items under +// channel>item, Atom puts entries at the top level, and the field names barely +// overlap — so both sets are declared and whichever the document filled in wins. +type feedDoc struct { + ChannelTitle string `xml:"channel>title"` + AtomTitle string `xml:"title"` + + Items []struct { + Title string `xml:"title"` + Link string `xml:"link"` + Description string `xml:"description"` + Encoded string `xml:"encoded"` // content:encoded + GUID string `xml:"guid"` + PubDate string `xml:"pubDate"` + Date string `xml:"date"` // dc:date + } `xml:"channel>item"` + + Entries []struct { + Title string `xml:"title"` + Links []struct { + Href string `xml:"href,attr"` + Rel string `xml:"rel,attr"` + } `xml:"link"` + Summary string `xml:"summary"` + Content string `xml:"content"` + ID string `xml:"id"` + Updated string `xml:"updated"` + Published string `xml:"published"` + } `xml:"entry"` +} + +// Parse reads a feed document. +func Parse(r io.Reader) (Feed, error) { + var doc feedDoc + dec := xml.NewDecoder(r) + // Strict=false buys tolerance of the malformed markup feeds are full of: + // unclosed tags, stray entities. It has nothing to do with charsets. + // + // Charsets are handled by not handling them: CharsetReader stays nil, so a + // feed declaring windows-1251 fails to parse rather than being read as + // UTF-8. That is the behaviour we want — a charset we cannot decode is a + // feed we do not read, which beats mojibake in his notes. + dec.Strict = false + if err := dec.Decode(&doc); err != nil { + return Feed{}, fmt.Errorf("rss: bad xml: %w", err) + } + + f := Feed{Title: strings.TrimSpace(doc.ChannelTitle)} + if f.Title == "" { + f.Title = strings.TrimSpace(doc.AtomTitle) + } + for _, it := range doc.Items { + item := Item{ + Title: PlainText(it.Title), + Link: strings.TrimSpace(it.Link), + Summary: PlainText(firstNonEmpty(it.Description, it.Encoded)), + Published: parseTime(firstNonEmpty(it.PubDate, it.Date)), + ID: strings.TrimSpace(firstNonEmpty(it.GUID, it.Link)), + } + if item.Title != "" || item.Link != "" { + f.Items = append(f.Items, item) + } + } + for _, e := range doc.Entries { + link := "" + for _, l := range e.Links { + if l.Rel == "" || l.Rel == "alternate" { + link = strings.TrimSpace(l.Href) + break + } + } + if link == "" && len(e.Links) > 0 { + link = strings.TrimSpace(e.Links[0].Href) + } + item := Item{ + Title: PlainText(e.Title), + Link: link, + Summary: PlainText(firstNonEmpty(e.Summary, e.Content)), + Published: parseTime(firstNonEmpty(e.Published, e.Updated)), + ID: strings.TrimSpace(firstNonEmpty(e.ID, link)), + } + if item.Title != "" || item.Link != "" { + f.Items = append(f.Items, item) + } + } + return f, nil +} + +func firstNonEmpty(vals ...string) string { + for _, v := range vals { + if strings.TrimSpace(v) != "" { + return v + } + } + return "" +} + +// timeLayouts — RFC1123/822 for RSS, RFC3339 for Atom, plus the near-misses +// real feeds ship (no seconds, numeric zone where a name is expected). +var timeLayouts = []string{ + time.RFC1123Z, + time.RFC1123, + time.RFC822Z, + time.RFC822, + time.RFC3339, + "2006-01-02T15:04:05Z0700", + "2006-01-02 15:04:05", + "2006-01-02", + "Mon, 02 Jan 2006 15:04:05 -0700", + "Mon, 2 Jan 2006 15:04:05 -0700", + "Mon, 2 Jan 2006 15:04:05 MST", +} + +// parseTime returns the zero time on anything it cannot read. An undated item +// is still an item; the poller dedupes by ID, so a missing date costs nothing. +func parseTime(s string) time.Time { + s = strings.TrimSpace(s) + if s == "" { + return time.Time{} + } + for _, l := range timeLayouts { + if t, err := time.Parse(l, s); err == nil { + return t.UTC() + } + } + return time.Time{} +} + +var ( + // RE2 has no backreferences, so the two tags are spelled out rather than + // captured and matched against themselves. + scriptRE = regexp.MustCompile(`(?is)]*>.*?|]*>.*?`) + tagRE = regexp.MustCompile(`(?s)<[^>]*>`) +) + +// PlainText strips markup and decodes entities — feed summaries are HTML, and +// what reaches a note (and possibly the TTS) must be text. Exported because the +// crawler's extractor needs exactly this on a bigger input. +func PlainText(s string) string { + s = scriptRE.ReplaceAllString(s, " ") + s = tagRE.ReplaceAllString(s, " ") + s = html.UnescapeString(s) + return strings.TrimSpace(strings.Join(strings.Fields(s), " ")) +} diff --git a/internal/rss/feed_test.go b/internal/rss/feed_test.go new file mode 100644 index 0000000..04a87f3 --- /dev/null +++ b/internal/rss/feed_test.go @@ -0,0 +1,112 @@ +package rss + +import ( + "strings" + "testing" + "time" +) + +const rss2 = ` + + + Хабр + + Новая уязвимость в ядре + https://example.org/a + <p>Патч уже <b>вышел</b>.</p> + tag:example.org,a + Mon, 28 Jul 2026 10:00:00 +0000 + + + Без даты + https://example.org/b + + +` + +const atom = ` + + Example Atom + + Release 2.0 + + + urn:uuid:1 + 2026-07-30T12:30:00Z + Ships & works + +` + +func TestParseRSS2(t *testing.T) { + f, err := Parse(strings.NewReader(rss2)) + if err != nil { + t.Fatal(err) + } + if f.Title != "Хабр" { + t.Fatalf("title = %q", f.Title) + } + if len(f.Items) != 2 { + t.Fatalf("items = %d, want 2", len(f.Items)) + } + it := f.Items[0] + if it.Title != "Новая уязвимость в ядре" { + t.Errorf("title = %q", it.Title) + } + if it.Summary != "Патч уже вышел ." && it.Summary != "Патч уже вышел." { + t.Errorf("summary = %q — tags must be stripped and entities decoded", it.Summary) + } + if it.ID != "tag:example.org,a" { + t.Errorf("id = %q", it.ID) + } + if want := time.Date(2026, 7, 28, 10, 0, 0, 0, time.UTC); !it.Published.Equal(want) { + t.Errorf("published = %v, want %v", it.Published, want) + } + if !f.Items[1].Published.IsZero() { + t.Errorf("undated item got a date: %v", f.Items[1].Published) + } + if f.Items[1].ID != "https://example.org/b" { + t.Errorf("id falls back to the link, got %q", f.Items[1].ID) + } +} + +func TestParseAtom(t *testing.T) { + f, err := Parse(strings.NewReader(atom)) + if err != nil { + t.Fatal(err) + } + if f.Title != "Example Atom" || len(f.Items) != 1 { + t.Fatalf("feed = %+v", f) + } + it := f.Items[0] + if it.Link != "https://example.com/rel" { + t.Errorf("link = %q, want the alternate link", it.Link) + } + if it.Summary != "Ships & works" { + t.Errorf("summary = %q", it.Summary) + } + if want := time.Date(2026, 7, 30, 12, 30, 0, 0, time.UTC); !it.Published.Equal(want) { + t.Errorf("published = %v, want %v", it.Published, want) + } +} + +func TestParseGarbage(t *testing.T) { + if _, err := Parse(strings.NewReader("not a feed")); err == nil { + t.Fatal("want an error on a non-feed document") + } + // A feed with an item that has neither title nor link contributes nothing + // rather than an empty note. + f, err := Parse(strings.NewReader(`x`)) + if err != nil { + t.Fatal(err) + } + if len(f.Items) != 0 { + t.Fatalf("items = %d, want 0", len(f.Items)) + } +} + +func TestPlainTextDropsScript(t *testing.T) { + got := PlainText(`

hi

there`) + if got != "hi there" { + t.Fatalf("got %q", got) + } +} diff --git a/internal/rss/marks_test.go b/internal/rss/marks_test.go new file mode 100644 index 0000000..6d25ace --- /dev/null +++ b/internal/rss/marks_test.go @@ -0,0 +1,114 @@ +package rss + +import ( + "context" + "fmt" + "strings" + "testing" + "time" +) + +// undatedFeed — a feed whose items carry no pubDate. Plenty of real ones do not. +func undatedFeed(titles ...string) string { + var b strings.Builder + b.WriteString(`u`) + for _, t := range titles { + fmt.Fprintf(&b, `%shttps://example.org/%s%s`, t, t, t) + } + b.WriteString(``) + return b.String() +} + +// datedFeed — newest first, one hour apart, the standard shape. +func datedFeed(base time.Time, n int) string { + var b strings.Builder + b.WriteString(`d`) + for i := 0; i < n; i++ { + ts := base.Add(-time.Duration(i) * time.Hour) + fmt.Fprintf(&b, `item-%dhttps://example.org/%dg%d%s`, + i, i, i, ts.Format(time.RFC1123Z)) + } + b.WriteString(``) + return b.String() +} + +func TestPoll_UndatedFeedIsNotReNotedAfterARestart(t *testing.T) { + // The seen-IDs map dies with the process, so before the durable mark was + // consulted every boot re-noted the whole front page, stamped now, on top of + // the recent-notes window. A crash loop made that a flood. + feed := FeedConfig{Name: "u", URL: "https://example.org/rss"} + fetch := &fakeFetch{body: undatedFeed("a", "b", "c")} + marks := newMarks() + + notes1 := &fakeNotes{} + p1 := NewPoller([]FeedConfig{feed}, fetch, notes1, marks, nil, nil, Config{}) + p1.PollDue(context.Background(), now) + if len(notes1.notes) != 3 { + t.Fatalf("first boot wrote %d notes, want 3", len(notes1.notes)) + } + if marks.m["u"].IsZero() { + t.Fatal("an undated feed left no mark, so the next process cannot tell it has been read") + } + + // Restart. Same process-lifetime dedup map, gone. + notes2 := &fakeNotes{} + p2 := NewPoller([]FeedConfig{feed}, fetch, notes2, marks, nil, nil, Config{}) + p2.PollDue(context.Background(), now.Add(time.Hour)) + if len(notes2.notes) != 0 { + t.Fatalf("a restart re-noted %d undated items: %v", len(notes2.notes), notes2.notes) + } + // And the same process still notices something genuinely new. + fetch.body = undatedFeed("a", "b", "c", "d") + p2.PollDue(context.Background(), now.Add(2*time.Hour)) + if len(notes2.notes) != 1 { + t.Fatalf("a new undated item after the resync wrote %d notes, want 1", len(notes2.notes)) + } +} + +func TestPoll_BurstLargerThanMaxItemsIsPacedNotDropped(t *testing.T) { + // max_items reads as pacing in the config doc. Marking the newest item + // written put everything below the cap behind the mark, permanently. + feed := FeedConfig{Name: "d", URL: "https://example.org/rss"} + fetch := &fakeFetch{body: datedFeed(now.Add(-time.Minute), 12)} + notes := &fakeNotes{} + marks := newMarks() + p := NewPoller([]FeedConfig{feed}, fetch, notes, marks, nil, nil, Config{MaxItems: 5, MaxAge: 48 * time.Hour}) + + at := now + for i := 0; i < 3; i++ { + if _, err := p.PollFeed(context.Background(), feed, at); err != nil { + t.Fatal(err) + } + at = at.Add(time.Hour) + } + seen := map[string]bool{} + for _, n := range notes.notes { + title := strings.SplitN(n.text, "\n", 2)[0] + if seen[title] { + t.Errorf("item %q was noted twice", title) + } + seen[title] = true + } + if len(seen) != 12 { + t.Errorf("after three polls of a 12-item burst she has %d of them; the rest were dropped for good", len(seen)) + } +} + +func TestNoteHeadlineAndCategory(t *testing.T) { + text := NoteText(FeedConfig{Category: "технологии"}, Item{ + Title: "Новая уязвимость", Summary: "Патч вышел", Link: "https://example.org/a", + }) + if got := NoteHeadline(text); got != "Новая уязвимость" { + t.Errorf("NoteHeadline = %q; she reads the brackets out loud", got) + } + if got := NoteCategory(text); got != "технологии" { + t.Errorf("NoteCategory = %q, want технологии", got) + } + // A feed's own leading tag stays part of the title. + if got := NoteHeadline("[перевод] Что-то"); got != "[перевод] Что-то" { + t.Errorf("NoteHeadline stripped the feed's own tag: %q", got) + } + if got := NoteCategory("Без категории"); got != "" { + t.Errorf("NoteCategory on an untagged note = %q, want empty", got) + } +} diff --git a/internal/rss/poller.go b/internal/rss/poller.go new file mode 100644 index 0000000..24e3049 --- /dev/null +++ b/internal/rss/poller.go @@ -0,0 +1,437 @@ +package rss + +import ( + "context" + "fmt" + "log" + "sort" + "strings" + "time" +) + +// FeedConfig — one feed to read. A feed with no Name or no URL is ignored. +type FeedConfig struct { + Name string // short id; the note source is "rss:" + URL string // http(s) only, enforced by the fetcher + Category string // free text ("технологии"), used to answer "что по X?" + Interval time.Duration // 0 ⇒ the poller's default + Include []string // when non-empty, keep only items matching one of these + Exclude []string // drop items matching any of these, even if included +} + +// Fetcher is the guarded HTTP door (internal/webfetch). An interface so the +// poller is testable without a network and so it CANNOT fetch by any other +// means: no http.Client is constructed in this package. +type Fetcher interface { + Get(ctx context.Context, url string) (*Body, error) +} + +// Body is the minimum the poller needs from a response. +type Body struct{ Bytes []byte } + +// Notes is core's note-writing half. Same shape as ipc.CoreAPI's method, so the +// daemon passes its API straight in. +type Notes interface { + WriteNote(ctx context.Context, ts time.Time, text string, embedding []float32, source string) (int64, error) +} + +// Marks remembers how far a feed was read. Durable, because the alternative is +// re-writing yesterday's headlines as fresh notes after every restart. The +// daemon backs this with config facts (key "rss:latest:"). +type Marks interface { + LastMark(ctx context.Context, feed string) (time.Time, error) + SetMark(ctx context.Context, feed string, at time.Time) error +} + +// Embedder embeds a note on its way into the store so recall can find it. nil ⇒ +// notes are written without a vector (still readable by the recent-notes path). +type Embedder interface { + Embed(ctx context.Context, text string) ([]float32, error) +} + +// Ranker is the relevance seam. The plan called for scoring each item against +// an interest profile built from his notes; that profile does not exist yet, and +// a threshold over an embedder with no profile to compare to is a random filter +// with a confident name. So the seam is here, nil in the daemon, and the filter +// that actually runs is the per-feed keyword one — a rule he can read and +// predict. When there IS a profile, implement this and pass it. +// +// Note what a Ranker must NOT be: anything that sends his notes outward. The +// scoring happens locally against a local embedder; the feed item is the input, +// his memory is never the payload. +type Ranker interface { + Relevant(ctx context.Context, text string) (bool, error) +} + +// Config — poller-wide settings. +type Config struct { + DefaultInterval time.Duration // 0 ⇒ DefaultPollInterval + MaxItems int // most notes written per feed per poll; 0 ⇒ DefaultMaxItems + MaxAge time.Duration // ignore items older than this on a cold start; 0 ⇒ DefaultMaxAge +} + +// Defaults chosen to be quiet: a feed read every half hour, at most a handful of +// items kept, and a cold start that does not import a month of history. +const ( + DefaultPollInterval = 30 * time.Minute + DefaultMaxItems = 5 + DefaultMaxAge = 24 * time.Hour +) + +// Poller reads feeds on a schedule and writes what survives filtering as notes. +type Poller struct { + feeds []FeedConfig + fetch Fetcher + notes Notes + marks Marks + embed Embedder + ranker Ranker + cfg Config + nextDue map[string]time.Time + seen map[string]map[string]bool // feed → item ID, for items with no date + polled map[string]bool // feed → polled at least once in THIS process +} + +// NewPoller wires a poller. Returns nil when there is nothing to poll — a +// capability is off unless configured, and callers check for nil. +func NewPoller(feeds []FeedConfig, fetch Fetcher, notes Notes, marks Marks, embed Embedder, ranker Ranker, cfg Config) *Poller { + var valid []FeedConfig + for _, f := range feeds { + if strings.TrimSpace(f.Name) == "" || strings.TrimSpace(f.URL) == "" { + // Name the offender. A silent skip in a list of six feeds is a + // config typo nobody finds. + log.Printf("rss: skipping feed %q (%q): a feed needs both a name and a url", f.Name, f.URL) + continue + } + valid = append(valid, f) + } + if len(valid) == 0 || fetch == nil || notes == nil { + return nil + } + if cfg.DefaultInterval <= 0 { + cfg.DefaultInterval = DefaultPollInterval + } + if cfg.MaxItems <= 0 { + cfg.MaxItems = DefaultMaxItems + } + if cfg.MaxAge <= 0 { + cfg.MaxAge = DefaultMaxAge + } + return &Poller{ + feeds: valid, fetch: fetch, notes: notes, marks: marks, + embed: embed, ranker: ranker, cfg: cfg, + nextDue: map[string]time.Time{}, + seen: map[string]map[string]bool{}, + polled: map[string]bool{}, + } +} + +// Feeds returns the configured feeds (the answer path lists categories). +func (p *Poller) Feeds() []FeedConfig { return p.feeds } + +// PollDue reads every feed whose interval has elapsed and returns how many +// notes were written. Errors are logged per feed, never returned: one dead feed +// must not stop the others, and there is nobody waiting on this. +func (p *Poller) PollDue(ctx context.Context, now time.Time) int { + written := 0 + for _, f := range p.feeds { + if due, ok := p.nextDue[f.Name]; ok && now.Before(due) { + continue + } + interval := f.Interval + if interval <= 0 { + interval = p.cfg.DefaultInterval + } + p.nextDue[f.Name] = now.Add(interval) + n, err := p.PollFeed(ctx, f, now) + if err != nil { + // The URL is configured by him and not a secret, so it is loggable; + // item titles are not logged, only counts. + log.Printf("rss: feed %s: %v", f.Name, err) + continue + } + if n > 0 { + log.Printf("rss: feed %s: %d new item(s) noted", f.Name, n) + } + written += n + } + return written +} + +// PollFeed reads one feed now, regardless of its schedule. +func (p *Poller) PollFeed(ctx context.Context, f FeedConfig, now time.Time) (int, error) { + body, err := p.fetch.Get(ctx, f.URL) + if err != nil { + return 0, err + } + feed, err := Parse(strings.NewReader(string(body.Bytes))) + if err != nil { + return 0, err + } + + mark, durable := p.mark(ctx, f.Name, now) + // resync — the first poll of this feed since the process started, on a feed + // we have read before. Undated items are deduped by an in-memory ID set that + // dies with the process, so on this poll they are all "unseen" again and + // would all be re-noted. See fresh. + resync := durable && !p.polled[f.Name] + p.polled[f.Name] = true + + // Gather first, cap second, and write OLDEST first. + // + // The old loop walked the feed newest-first and stopped at MaxItems, then + // marked the newest item it had written. Feeds are newest-first, so with + // twenty new items and a cap of five it wrote the five newest and moved the + // mark past all twenty: items six through twenty were older than the mark on + // the next poll and were dropped for good. max_items reads as a pacing knob + // in the config doc, and that made it a silent loss. Writing the oldest five + // and marking the newest of THOSE is pacing: the rest arrive over the polls + // that follow, in order, each one exactly once. + var cands []Item + sawUndated := false + for _, it := range feed.Items { + if it.Published.IsZero() { + sawUndated = true + } + if !p.fresh(f, it, mark, now, resync) { + continue + } + if !Matches(f, it) { + continue + } + if p.ranker != nil { + ok, err := p.ranker.Relevant(ctx, it.Title+" "+it.Summary) + if err != nil { + log.Printf("rss: feed %s: relevance: %v", f.Name, err) + } else if !ok { + continue + } + } + cands = append(cands, it) + } + sort.SliceStable(cands, func(i, j int) bool { + return itemTime(cands[i], now).Before(itemTime(cands[j], now)) + }) + if len(cands) > p.cfg.MaxItems { + cands = cands[:p.cfg.MaxItems] + } + + newest := time.Time{} + written := 0 + for _, it := range cands { + if err := p.write(ctx, f, it, now); err != nil { + return written, err + } + written++ + if it.Published.After(newest) { + newest = it.Published + } + } + p.advance(ctx, f.Name, mark, newest, sawUndated, now) + return written, nil +} + +// itemTime — an item's own date, or now when the feed did not give one. Undated +// items sort last, which is the only defensible guess: they were seen now. +func itemTime(it Item, now time.Time) time.Time { + if it.Published.IsZero() { + return now + } + return it.Published +} + +// advance moves the durable mark to the newest item actually WRITTEN. Because +// the cap is applied to the oldest candidates (see PollFeed), that is never +// ahead of an item still waiting to be read. +// +// A feed whose items carry no dates gets the mark set to now instead. Nothing +// else would ever set it, and the mark's existence is what tells the next +// process that this feed has been read before. +func (p *Poller) advance(ctx context.Context, feed string, mark, newest time.Time, sawUndated bool, now time.Time) { + if p.marks == nil { + return + } + at := newest + if at.IsZero() && sawUndated { + at = now + } + if at.IsZero() || !at.After(mark) { + return + } + if err := p.marks.SetMark(ctx, feed, at); err != nil { + log.Printf("rss: feed %s: save mark: %v", feed, err) + } +} + +// mark — how far this feed was read, and whether that came from the durable +// store. A feed with no mark starts MaxAge ago, so a first poll takes today's +// headlines instead of the whole archive; durable is false in that case, and it +// is what tells PollFeed the difference between "never read" and "read by an +// earlier process". +func (p *Poller) mark(ctx context.Context, feed string, now time.Time) (time.Time, bool) { + cold := now.Add(-p.cfg.MaxAge) + if p.marks == nil { + return cold, false + } + at, err := p.marks.LastMark(ctx, feed) + if err != nil || at.IsZero() { + return cold, false + } + return at, true +} + +// fresh — two dedup rules, because feeds are inconsistent about dates. A dated +// item must be newer than the mark; an undated one is kept once per process by +// ID. +// +// The ID set does not survive a restart, and on its own that re-notes an undated +// feed's whole front page on every boot — five notes, then five more, all stamped +// `now`, sitting at the top of the recent-notes window and crowding out the notes +// he actually made. A crash loop turns it into a flood. So on the first poll after +// a restart of a feed we have read before (resync), undated items are recorded as +// seen and NOT written. The cost is the undated items that appeared while the +// daemon was down. That is a bounded loss, and the alternative is an unbounded one. +func (p *Poller) fresh(f FeedConfig, it Item, mark, now time.Time, resync bool) bool { + if !it.Published.IsZero() { + if !it.Published.After(mark) { + return false + } + // A feed that dates its items in the future (or a clock skew) must not + // win the mark and mute everything after it. + return !it.Published.After(now.Add(time.Hour)) + } + id := it.ID + if id == "" { + id = it.Title + } + if p.seen[f.Name] == nil { + p.seen[f.Name] = map[string]bool{} + } + if p.seen[f.Name][id] { + return false + } + p.seen[f.Name][id] = true + return !resync +} + +// write stores one item as a note. Source "rss:" is what the answer path +// filters on, and what makes a feed note distinguishable from something he said. +func (p *Poller) write(ctx context.Context, f FeedConfig, it Item, now time.Time) error { + text := NoteText(f, it) + var vec []float32 + if p.embed != nil { + v, err := p.embed.Embed(ctx, text) + if err != nil { + log.Printf("rss: feed %s: embed: %v", f.Name, err) + } else { + vec = v + } + } + ts := it.Published + if ts.IsZero() { + ts = now + } + if _, err := p.notes.WriteNote(ctx, ts, text, vec, SourceFor(f.Name)); err != nil { + return fmt.Errorf("write note: %w", err) + } + return nil +} + +// SourceFor is the note source for a feed. +func SourceFor(feed string) string { return "rss:" + feed } + +// SourcePrefix — what the answer path matches to find feed notes. +const SourcePrefix = "rss:" + +// NoteText renders an item as the note body. The category is included because +// "что нового по технологиям?" is answered by reading notes, and a note has to +// carry enough to be recognised as belonging to that category. +func NoteText(f FeedConfig, it Item) string { + var b strings.Builder + b.WriteString(it.Title) + if f.Category != "" { + fmt.Fprintf(&b, " [%s]", f.Category) + } + if it.Summary != "" { + b.WriteString("\n") + b.WriteString(trimRunes(it.Summary, 500)) + } + if it.Link != "" { + b.WriteString("\n") + b.WriteString(it.Link) + } + return b.String() +} + +// NoteHeadline is the part of a feed note she reads out: the first line with the +// category tag taken off. The tag is bookkeeping for the answer path, and piper +// says brackets out loud — "Заголовок [технологии]" is what he heard before. +func NoteHeadline(text string) string { + line := text + if i := strings.IndexByte(line, '\n'); i >= 0 { + line = line[:i] + } + line = strings.TrimSpace(line) + if head, _, ok := splitTag(line); ok { + return head + } + return line +} + +// NoteCategory is the category tag a feed note carries, empty when it has none. +// Matching a topic against THIS rather than against the whole note is what keeps +// "что нового про погоду" from matching a tech headline whose link happens to +// contain "pogod". +func NoteCategory(text string) string { + line := text + if i := strings.IndexByte(line, '\n'); i >= 0 { + line = line[:i] + } + _, tag, _ := splitTag(strings.TrimSpace(line)) + return tag +} + +// splitTag pulls a trailing "[...]" off a headline. Only a trailing one: a title +// that opens with "[перевод]" is the feed's own word, not ours. +func splitTag(line string) (head, tag string, ok bool) { + if !strings.HasSuffix(line, "]") { + return line, "", false + } + i := strings.LastIndexByte(line, '[') + if i < 0 { + return line, "", false + } + return strings.TrimSpace(line[:i]), strings.TrimSpace(line[i+1 : len(line)-1]), true +} + +// trimRunes cuts on a rune boundary — a note is Russian as often as English and +// half a cyrillic letter is a broken note. +func trimRunes(s string, max int) string { + r := []rune(s) + if len(r) <= max { + return s + } + return strings.TrimSpace(string(r[:max])) + "…" +} + +// Matches applies the per-feed keyword filter: keep when Include is empty or one +// include matches, drop when any exclude matches. Case-insensitive substring, +// which for Russian is the honest choice — no stemmer here, so "выборы" does not +// match "выборах", and a filter he writes is a filter he can predict. +func Matches(f FeedConfig, it Item) bool { + hay := strings.ToLower(it.Title + " " + it.Summary) + for _, x := range f.Exclude { + if x = strings.ToLower(strings.TrimSpace(x)); x != "" && strings.Contains(hay, x) { + return false + } + } + if len(f.Include) == 0 { + return true + } + for _, in := range f.Include { + if in = strings.ToLower(strings.TrimSpace(in)); in != "" && strings.Contains(hay, in) { + return true + } + } + return false +} diff --git a/internal/rss/poller_test.go b/internal/rss/poller_test.go new file mode 100644 index 0000000..f071399 --- /dev/null +++ b/internal/rss/poller_test.go @@ -0,0 +1,210 @@ +package rss + +import ( + "context" + "errors" + "strings" + "testing" + "time" +) + +type fakeFetch struct { + body string + err error + calls int + urls []string +} + +func (f *fakeFetch) Get(_ context.Context, url string) (*Body, error) { + f.calls++ + f.urls = append(f.urls, url) + if f.err != nil { + return nil, f.err + } + return &Body{Bytes: []byte(f.body)}, nil +} + +type writtenNote struct { + ts time.Time + text string + source string + vec []float32 +} + +type fakeNotes struct{ notes []writtenNote } + +func (n *fakeNotes) WriteNote(_ context.Context, ts time.Time, text string, vec []float32, source string) (int64, error) { + n.notes = append(n.notes, writtenNote{ts, text, source, vec}) + return int64(len(n.notes)), nil +} + +type fakeMarks struct{ m map[string]time.Time } + +func newMarks() *fakeMarks { return &fakeMarks{m: map[string]time.Time{}} } +func (f *fakeMarks) LastMark(_ context.Context, feed string) (time.Time, error) { + return f.m[feed], nil +} +func (f *fakeMarks) SetMark(_ context.Context, feed string, at time.Time) error { + f.m[feed] = at + return nil +} + +var now = time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC) + +func TestPollWritesNotesWithSource(t *testing.T) { + fetch := &fakeFetch{body: rss2} + notes := &fakeNotes{} + marks := newMarks() + p := NewPoller([]FeedConfig{{Name: "habr", URL: "https://example.org/rss", Category: "технологии"}}, + fetch, notes, marks, nil, nil, Config{}) + if p == nil { + t.Fatal("NewPoller returned nil for a configured feed") + } + n := p.PollDue(context.Background(), now) + if n != 2 || len(notes.notes) != 2 { + t.Fatalf("wrote %d notes (returned %d), want 2", len(notes.notes), n) + } + if notes.notes[0].source != "rss:habr" { + t.Errorf("source = %q, want rss:habr", notes.notes[0].source) + } + if !strings.Contains(notes.notes[0].text, "технологии") { + t.Errorf("note does not carry its category: %q", notes.notes[0].text) + } + if !strings.Contains(notes.notes[0].text, "https://example.org/a") { + t.Errorf("note does not carry its link: %q", notes.notes[0].text) + } + // The undated item is stamped with now, the dated one with its own date. + if !notes.notes[1].ts.Equal(now) { + t.Errorf("undated item ts = %v, want now", notes.notes[1].ts) + } +} + +// The whole point of a mark: polling twice must not re-note the same headlines. +func TestSecondPollIsQuiet(t *testing.T) { + fetch := &fakeFetch{body: rss2} + notes := &fakeNotes{} + p := NewPoller([]FeedConfig{{Name: "habr", URL: "u", Interval: time.Minute}}, fetch, notes, newMarks(), nil, nil, Config{}) + p.PollDue(context.Background(), now) + before := len(notes.notes) + p.PollDue(context.Background(), now.Add(2*time.Minute)) + if len(notes.notes) != before { + t.Fatalf("second poll wrote %d extra notes", len(notes.notes)-before) + } +} + +// A mark that survives a restart is the durable half; simulate one by building a +// fresh poller over the same marks. +func TestMarkSurvivesRestart(t *testing.T) { + marks := newMarks() + fetch := &fakeFetch{body: rss2} + notes := &fakeNotes{} + feeds := []FeedConfig{{Name: "habr", URL: "u"}} + NewPoller(feeds, fetch, notes, marks, nil, nil, Config{}).PollDue(context.Background(), now) + if len(notes.notes) != 2 { + t.Fatalf("first run wrote %d", len(notes.notes)) + } + notes2 := &fakeNotes{} + NewPoller(feeds, fetch, notes2, marks, nil, nil, Config{}).PollDue(context.Background(), now.Add(time.Hour)) + // The dated item is behind the mark. The undated one has no date to compare, + // so it comes back — accepted and documented in fresh(): an undated feed is + // deduped per process, not forever. + for _, n := range notes2.notes { + if strings.Contains(n.text, "уязвимость") { + t.Fatalf("dated item re-noted after restart: %q", n.text) + } + } +} + +func TestIntervalIsRespected(t *testing.T) { + fetch := &fakeFetch{body: rss2} + p := NewPoller([]FeedConfig{{Name: "habr", URL: "u", Interval: time.Hour}}, fetch, &fakeNotes{}, newMarks(), nil, nil, Config{}) + p.PollDue(context.Background(), now) + p.PollDue(context.Background(), now.Add(time.Minute)) + if fetch.calls != 1 { + t.Fatalf("fetched %d times inside one interval, want 1", fetch.calls) + } + p.PollDue(context.Background(), now.Add(2*time.Hour)) + if fetch.calls != 2 { + t.Fatalf("fetched %d times, want 2 after the interval elapsed", fetch.calls) + } +} + +func TestColdStartIgnoresOldItems(t *testing.T) { + old := `Староеl` + + `Mon, 01 Jun 2026 10:00:00 +0000` + notes := &fakeNotes{} + p := NewPoller([]FeedConfig{{Name: "f", URL: "u"}}, &fakeFetch{body: old}, notes, newMarks(), nil, nil, Config{MaxAge: 24 * time.Hour}) + if n := p.PollDue(context.Background(), now); n != 0 { + t.Fatalf("cold start imported %d old items, want 0", n) + } +} + +func TestMaxItemsCap(t *testing.T) { + var b strings.Builder + b.WriteString("") + for i := 0; i < 10; i++ { + b.WriteString("t") + b.WriteByte(byte('0' + i)) + b.WriteString("https://example.org/") + b.WriteByte(byte('0' + i)) + b.WriteString("") + } + b.WriteString("") + notes := &fakeNotes{} + p := NewPoller([]FeedConfig{{Name: "f", URL: "u"}}, &fakeFetch{body: b.String()}, notes, newMarks(), nil, nil, Config{MaxItems: 3}) + if n := p.PollDue(context.Background(), now); n != 3 { + t.Fatalf("wrote %d notes, want the cap of 3", n) + } +} + +func TestKeywordFilter(t *testing.T) { + f := FeedConfig{Include: []string{"ядр"}, Exclude: []string{"реклама"}} + if !Matches(f, Item{Title: "Новое ядро"}) { + t.Error("include did not match") + } + if Matches(f, Item{Title: "Новое ядро", Summary: "Реклама внутри"}) { + t.Error("exclude must win over include") + } + if Matches(f, Item{Title: "Погода"}) { + t.Error("non-matching item passed the include filter") + } + if !Matches(FeedConfig{}, Item{Title: "что угодно"}) { + t.Error("an unfiltered feed must keep everything") + } +} + +type fakeRanker struct{ keep bool } + +func (r fakeRanker) Relevant(context.Context, string) (bool, error) { return r.keep, nil } + +func TestRankerCanDropEverything(t *testing.T) { + notes := &fakeNotes{} + p := NewPoller([]FeedConfig{{Name: "f", URL: "u"}}, &fakeFetch{body: rss2}, notes, newMarks(), nil, fakeRanker{false}, Config{}) + if n := p.PollDue(context.Background(), now); n != 0 { + t.Fatalf("ranker rejected everything but %d notes were written", n) + } +} + +func TestFetchErrorIsSurvivable(t *testing.T) { + notes := &fakeNotes{} + p := NewPoller([]FeedConfig{ + {Name: "dead", URL: "u1"}, + {Name: "live", URL: "u2"}, + }, &fakeFetch{err: errors.New("boom")}, notes, newMarks(), nil, nil, Config{}) + if n := p.PollDue(context.Background(), now); n != 0 { + t.Fatalf("n = %d", n) + } + // Both feeds were attempted: one dead feed does not abort the round. + if p.nextDue["live"].IsZero() { + t.Fatal("the second feed was never attempted") + } +} + +func TestNoFeedsMeansNoPoller(t *testing.T) { + if p := NewPoller(nil, &fakeFetch{}, &fakeNotes{}, nil, nil, nil, Config{}); p != nil { + t.Fatal("NewPoller must return nil when nothing is configured") + } + if p := NewPoller([]FeedConfig{{Name: "", URL: ""}}, &fakeFetch{}, &fakeNotes{}, nil, nil, nil, Config{}); p != nil { + t.Fatal("a feed with no name or url is not a configuration") + } +} diff --git a/internal/smarthome/ha.go b/internal/smarthome/ha.go new file mode 100644 index 0000000..ec6d54c --- /dev/null +++ b/internal/smarthome/ha.go @@ -0,0 +1,308 @@ +package smarthome + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strings" + "time" +) + +// DefaultTimeout — per-call budget. A house that takes longer than this to +// answer is not usable in a spoken turn. +const DefaultTimeout = 10 * time.Second + +// DefaultMaxEntities — cap on how many entities become allowlist proposals. +// The resident model is a 1.7B with a 4096-token context: a tool name it +// half-remembers is a wrong act, so a bounded, deliberate catalogue beats a +// complete one. +const DefaultMaxEntities = 40 + +// maxBody — cap on one /api/states response. A Home Assistant with hundreds of +// entities would otherwise stream megabytes into a daemon that wants forty +// names. +const maxBody = 4 << 20 + +// Config — what a Home Assistant instance needs to be reachable. +type Config struct { + // URL — the base, "http://homeassistant.local:8123". No trailing path. + URL string + // Token — a long-lived access token. Sent as a bearer header and never + // logged. + Token string + // Domains — the entity domains to take. Empty ⇒ every domain in the + // controllable table EXCEPT lock, plus sensor/binary_sensor for reads. + // A lock is only enumerated when it is named here. + Domains []string + // MaxEntities — 0 ⇒ DefaultMaxEntities. + MaxEntities int + // Timeout — 0 ⇒ DefaultTimeout. + Timeout time.Duration +} + +// Validate rejects a block that cannot work, at config-load time rather than at +// the first spoken act. +func Validate(c Config) error { + if c.URL == "" { + return errors.New("smarthome: url is required") + } + u, err := url.Parse(c.URL) + if err != nil { + return fmt.Errorf("smarthome: url: %w", err) + } + if u.Scheme != "http" && u.Scheme != "https" { + return fmt.Errorf("smarthome: url scheme %q: want http or https", u.Scheme) + } + if u.Host == "" { + return errors.New("smarthome: url has no host") + } + if c.Token == "" { + return errors.New("smarthome: token is required") + } + return nil +} + +// Client is a Home Assistant REST client. Read (States) and one write +// (CallService); no WebSocket, because a spoken turn is request/response and an +// event stream is a second failure mode for no gain yet. +type Client struct { + cfg Config + http *http.Client +} + +// NewClient builds a client. Validate first — this does not. +func NewClient(cfg Config) *Client { + if cfg.Timeout <= 0 { + cfg.Timeout = DefaultTimeout + } + if cfg.MaxEntities <= 0 { + cfg.MaxEntities = DefaultMaxEntities + } + return &Client{cfg: cfg, http: &http.Client{Timeout: cfg.Timeout}} +} + +// SetHTTPClient swaps the transport. Tests use it; nothing else should. +func (c *Client) SetHTTPClient(h *http.Client) { c.http = h } + +// wanted reports whether an entity's domain is one Maven takes. The config list +// wins when set; otherwise every controllable domain plus the two read-only +// sensor domains. +func (c *Client) wanted(domain string) bool { + if len(c.cfg.Domains) > 0 { + for _, d := range c.cfg.Domains { + if d == domain { + return true + } + } + return false + } + if _, ok := controllable[domain]; ok { + // A deadbolt is a different class of object from a lamp, so "lock" is + // not in the implicit set: a bare url+token block must not auto-propose + // an unlock row for every door in the flat. Naming it in domains is the + // operator saying he meant it. + return domain != "lock" + } + return domain == "sensor" || domain == "binary_sensor" +} + +type haState struct { + EntityID string `json:"entity_id"` + State string `json:"state"` + Attributes json.RawMessage `json:"attributes"` +} + +type haAttrs struct { + FriendlyName string `json:"friendly_name"` + Unit string `json:"unit_of_measurement"` +} + +// capEntities cuts a state list to at most max entries, taking a controllable +// entity before any sensor and then round-robin across domains. +// +// The cap used to be applied to a globally id-sorted list, and entity ids sort +// by domain prefix: binary_sensor < cover < fan < light < lock < sensor < +// switch. A stock Home Assistant carries dozens of binary_sensor rows before it +// carries anything else, so forty slots went entirely to connectivity and +// update-available sensors. propose then found zero controllable entities, and +// homeSummary, reading the same list, said "всё выключено" with the lights on. +// +// The cap itself stays. The resident model is a 1.7B with a 4096-token context +// and a tool name it half-remembers is a wrong act, so a bounded deliberate +// catalogue still beats a complete one. What changes is which forty: every +// switch and light before any sensor, and an even spread inside each group so +// one crowded domain cannot starve the others. The result is sorted by id, so +// /tools reads the same across restarts. +func capEntities(all []Entity, max int) []Entity { + if max <= 0 || len(all) <= max { + sort.Slice(all, func(i, j int) bool { return all[i].ID < all[j].ID }) + return all + } + byDomain := map[string][]Entity{} + for _, e := range all { + byDomain[e.Domain] = append(byDomain[e.Domain], e) + } + var control, read []string + for d := range byDomain { + sort.Slice(byDomain[d], func(i, j int) bool { return byDomain[d][i].ID < byDomain[d][j].ID }) + if _, ok := controllable[d]; ok { + control = append(control, d) + } else { + read = append(read, d) + } + } + sort.Strings(control) + sort.Strings(read) + + out := make([]Entity, 0, max) + take := func(domains []string) { + for i := 0; len(out) < max; i++ { + took := false + for _, d := range domains { + l := byDomain[d] + if i >= len(l) || len(out) >= max { + continue + } + out = append(out, l[i]) + took = true + } + if !took { + return + } + } + } + take(control) + take(read) + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out +} + +// States reads every entity Maven cares about, sorted by id and capped at +// MaxEntities so the catalogue is deterministic across restarts — a proposal +// list that reshuffles itself would make /tools unreadable. See capEntities for +// what the cap keeps. +func (c *Client) States(ctx context.Context) ([]Entity, error) { + body, err := c.do(ctx, http.MethodGet, "/api/states", nil) + if err != nil { + return nil, err + } + var raw []haState + if err := json.Unmarshal(body, &raw); err != nil { + return nil, fmt.Errorf("smarthome: decode states: %w", err) + } + out := make([]Entity, 0, len(raw)) + for _, s := range raw { + domain := DomainOf(s.EntityID) + if domain == "" || !c.wanted(domain) { + continue + } + e := Entity{ID: s.EntityID, Domain: domain, Name: s.EntityID, State: s.State} + if len(s.Attributes) > 0 { + var a haAttrs + // Attributes are free-form per integration; a shape we cannot read + // costs the friendly name, not the entity. + if err := json.Unmarshal(s.Attributes, &a); err == nil { + if a.FriendlyName != "" { + e.Name = a.FriendlyName + } + e.Unit = a.Unit + } + } + out = append(out, e) + } + return capEntities(out, c.cfg.MaxEntities), nil +} + +// CallService performs one service call against one entity and returns a short +// Russian confirmation. +// +// The entity id and service are NOT taken from the utterance: they come from +// the allowlist row that Kami enabled, so the router can only pick a row, never +// compose a target. That is the whole reason control is encoded in the cmd +// column instead of parsed out of speech. +func (c *Client) CallService(ctx context.Context, entityID, service string) (string, error) { + domain := DomainOf(entityID) + if domain == "" { + return "", ErrUnknownEntity + } + svcs := Services(domain) + if len(svcs) == 0 { + return "", ErrNotControllable + } + known := false + for _, s := range svcs { + if s.Name == service { + known = true + break + } + } + if !known { + return "", fmt.Errorf("%w: %s has no service %q", ErrNotControllable, domain, service) + } + payload, err := json.Marshal(map[string]string{"entity_id": entityID}) + if err != nil { + return "", fmt.Errorf("smarthome: encode call: %w", err) + } + path := "/api/services/" + url.PathEscape(domain) + "/" + url.PathEscape(service) + body, err := c.do(ctx, http.MethodPost, path, payload) + if err != nil { + return "", err + } + // Home Assistant answers a service call with the states it changed. An + // entity that was removed since discovery, or one whose integration is + // offline, gets 200 and an empty array. Reporting "готово" for that is + // Maven asserting something false about the physical world: he says + // "выключи свет", she says done, the light stays on. + var changed []haState + if err := json.Unmarshal(body, &changed); err != nil { + // A shape we cannot read is not evidence of failure. HA has answered + // 2xx, so report the call as made rather than inventing a fault. + return "готово", nil + } + if len(changed) == 0 { + return "", fmt.Errorf("%w: %s did not change anything", ErrUnknownEntity, entityID) + } + return "готово", nil +} + +// do issues one authenticated request and returns the (capped) body. +func (c *Client) do(ctx context.Context, method, path string, body []byte) ([]byte, error) { + if c.cfg.URL == "" || c.cfg.Token == "" { + return nil, ErrNotConfigured + } + target := strings.TrimRight(c.cfg.URL, "/") + path + var rdr io.Reader + if body != nil { + rdr = bytes.NewReader(body) + } + req, err := http.NewRequestWithContext(ctx, method, target, rdr) + if err != nil { + return nil, fmt.Errorf("smarthome: request: %w", err) + } + req.Header.Set("Authorization", "Bearer "+c.cfg.Token) + req.Header.Set("Accept", "application/json") + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + resp, err := c.http.Do(req) + if err != nil { + return nil, fmt.Errorf("smarthome: %s %s: %w", method, path, err) + } + defer resp.Body.Close() + out, err := io.ReadAll(io.LimitReader(resp.Body, maxBody)) + if err != nil { + return nil, fmt.Errorf("smarthome: read %s: %w", path, err) + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + // The body of an error can contain the instance's own detail; the token + // never appears in it, but keep it to one line anyway. + return nil, fmt.Errorf("smarthome: %s %s: http %d", method, path, resp.StatusCode) + } + return out, nil +} diff --git a/internal/smarthome/smarthome.go b/internal/smarthome/smarthome.go new file mode 100644 index 0000000..d156c2e --- /dev/null +++ b/internal/smarthome/smarthome.go @@ -0,0 +1,121 @@ +// Package smarthome talks to a Home Assistant instance so Maven can read what +// the house is doing and change it (Vikunja #256, +// docs/plans/11-smarthome-integration.md). +// +// The shape of this package is copied deliberately from internal/mcp: a +// controllable entity becomes a PROPOSED row in the existing act allowlist, +// encoded in the columns that already exist — cmd +// ["smarthome", "", ""], scope "smarthome:". So +// ProposeTool/EnableTool/DisableTool, tool.Matcher and the confirm turn need no +// change, and turning a light off in his flat goes through exactly the same +// gate as `restart nginx`. +// +// Two rules that are not negotiable here: +// +// - Discovery only ever PROPOSES. Finding a switch on the network is not the +// same as being allowed to flip it; Kami enables it on /tools, behind +// step-up. +// - Every control row is destructive=true. There is no read-only way to turn +// the heating off. That means a spoken act always gets the confirm turn, +// which is the point. +// +// MQTT / Zigbee2MQTT (steps 2 and 5 of the plan) are NOT here: they need a +// broker client dependency and the module cache in this repo is vendored, and +// there is no broker on this network to test one against. Home Assistant's REST +// API is stdlib-only and already fronts Zigbee2MQTT when it is present. +package smarthome + +import ( + "errors" + "strings" +) + +var ( + // ErrNotConfigured — no smarthome block, or it is disabled. + ErrNotConfigured = errors.New("smarthome: not configured") + // ErrUnknownEntity — the entity vanished between discovery and the call. + ErrUnknownEntity = errors.New("smarthome: unknown entity") + // ErrNotControllable — the entity's domain has no service Maven will call. + ErrNotControllable = errors.New("smarthome: entity is not controllable") +) + +// cmdPrefix marks an allowlist row as a Home Assistant service call rather than +// a process. It is never run as a binary — tool.Executor branches on it before +// it ever reaches exec. +const cmdPrefix = "smarthome" + +// Entity is one thing in the house, as Home Assistant sees it. +type Entity struct { + // ID — the Home Assistant entity_id, "light.living_room". + ID string + // Domain — the part before the dot. Decides which services apply. + Domain string + // Name — friendly_name when the instance has one, else ID. + Name string + // State — "on", "off", "22.5", … + State string + // Unit — unit_of_measurement, for sensors. + Unit string +} + +// Service is one thing Maven can do to an entity. +type Service struct { + // Name — the Home Assistant service, "turn_on". + Name string + // Verb — the local suffix used to build the allowlist row name. + Verb string +} + +// controllable maps a domain to the services Maven will expose for it. A domain +// that is not in this table gets no control row at all — the list is an +// allowlist, not a default, so a new HA integration cannot quietly hand her a +// verb nobody reviewed. set_temperature and set_brightness take a value and are +// deliberately absent: a spoken number that the router got wrong is a wrong act +// on real hardware, and on/off is the whole of what a voice turn can defend. +var controllable = map[string][]Service{ + "light": {{Name: "turn_on", Verb: "on"}, {Name: "turn_off", Verb: "off"}}, + "switch": {{Name: "turn_on", Verb: "on"}, {Name: "turn_off", Verb: "off"}}, + "fan": {{Name: "turn_on", Verb: "on"}, {Name: "turn_off", Verb: "off"}}, + "cover": {{Name: "open_cover", Verb: "open"}, {Name: "close_cover", Verb: "close"}}, + "lock": {{Name: "lock", Verb: "lock"}, {Name: "unlock", Verb: "unlock"}}, +} + +// Services returns the services exposed for an entity, nil when its domain is +// not controllable (a sensor, a person, a weather entity: readable, not +// flippable). +func Services(domain string) []Service { return controllable[domain] } + +// DomainOf splits "light.living_room" into "light". Empty when the id has no +// dot, which Home Assistant guarantees it does. +func DomainOf(entityID string) string { + i := strings.IndexByte(entityID, '.') + if i <= 0 { + return "" + } + return entityID[:i] +} + +// LocalName is the allowlist row name for one entity+service. Prefixed so a +// house row is recognisable on /tools without opening the config, and so it +// cannot collide with a shell tool Kami named himself. +func LocalName(entityID, verb string) string { + return "home_" + strings.ReplaceAll(entityID, ".", "_") + "_" + verb +} + +// Scope is the store scope for an entity's domain. +func Scope(domain string) string { return cmdPrefix + ":" + domain } + +// Cmd is the allowlist cmd column for an entity+service. +func Cmd(entityID, service string) []string { return []string{cmdPrefix, entityID, service} } + +// ParseCmd recognises a Home Assistant row. ok=false ⇒ an ordinary process row, +// and the caller execs it as it always did. +func ParseCmd(cmd []string) (entityID, service string, ok bool) { + if len(cmd) != 3 || cmd[0] != cmdPrefix { + return "", "", false + } + if cmd[1] == "" || cmd[2] == "" { + return "", "", false + } + return cmd[1], cmd[2], true +} diff --git a/internal/smarthome/smarthome_test.go b/internal/smarthome/smarthome_test.go new file mode 100644 index 0000000..34bac98 --- /dev/null +++ b/internal/smarthome/smarthome_test.go @@ -0,0 +1,291 @@ +package smarthome + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// statesFixture is a trimmed /api/states response from a Home Assistant with +// one light, one switch, one sensor and two entities Maven must ignore. +const statesFixture = `[ + {"entity_id":"light.living_room","state":"on","attributes":{"friendly_name":"Гостиная"}}, + {"entity_id":"switch.kettle","state":"off","attributes":{"friendly_name":"Чайник"}}, + {"entity_id":"sensor.bedroom_temp","state":"22.5","attributes":{"unit_of_measurement":"°C"}}, + {"entity_id":"person.kami","state":"home","attributes":{}}, + {"entity_id":"automation.wake","state":"on","attributes":[]} +]` + +func newTestClient(t *testing.T, h http.HandlerFunc) (*Client, *httptest.Server) { + t.Helper() + srv := httptest.NewServer(h) + t.Cleanup(srv.Close) + c := NewClient(Config{URL: srv.URL, Token: "tok"}) + return c, srv +} + +func TestStatesFiltersAndNames(t *testing.T) { + var auth string + c, _ := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + auth = r.Header.Get("Authorization") + if r.URL.Path != "/api/states" { + t.Errorf("path = %q", r.URL.Path) + } + _, _ = w.Write([]byte(statesFixture)) + }) + got, err := c.States(context.Background()) + if err != nil { + t.Fatalf("States: %v", err) + } + if auth != "Bearer tok" { + t.Errorf("Authorization = %q", auth) + } + // person and automation are neither controllable nor sensors. + want := []string{"light.living_room", "sensor.bedroom_temp", "switch.kettle"} + if len(got) != len(want) { + t.Fatalf("got %d entities, want %d: %+v", len(got), len(want), got) + } + for i, id := range want { + if got[i].ID != id { + t.Errorf("entity %d = %q, want %q (sorted by id)", i, got[i].ID, id) + } + } + if got[0].Name != "Гостиная" || got[0].Domain != "light" || got[0].State != "on" { + t.Errorf("light = %+v", got[0]) + } + if got[1].Unit != "°C" { + t.Errorf("sensor unit = %q", got[1].Unit) + } + // An attributes value of the wrong shape must not lose the entity. + if got[2].Name != "Чайник" { + t.Errorf("switch name = %q", got[2].Name) + } +} + +func TestStatesRespectsConfiguredDomainsAndCap(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(statesFixture)) + })) + defer srv.Close() + + c := NewClient(Config{URL: srv.URL, Token: "t", Domains: []string{"switch"}}) + got, err := c.States(context.Background()) + if err != nil { + t.Fatalf("States: %v", err) + } + if len(got) != 1 || got[0].ID != "switch.kettle" { + t.Fatalf("domain filter: %+v", got) + } + + c = NewClient(Config{URL: srv.URL, Token: "t", MaxEntities: 2}) + got, err = c.States(context.Background()) + if err != nil { + t.Fatalf("States: %v", err) + } + if len(got) != 2 { + t.Fatalf("cap: got %d entities, want 2", len(got)) + } +} + +func TestCallServicePostsEntityID(t *testing.T) { + var path, body string + c, _ := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + path = r.URL.Path + b := make([]byte, 256) + n, _ := r.Body.Read(b) + body = string(b[:n]) + _, _ = w.Write([]byte(`[{"entity_id":"light.living_room","state":"off"}]`)) + }) + out, err := c.CallService(context.Background(), "light.living_room", "turn_off") + if err != nil { + t.Fatalf("CallService: %v", err) + } + if out != "готово" { + t.Errorf("out = %q", out) + } + if path != "/api/services/light/turn_off" { + t.Errorf("path = %q", path) + } + if !strings.Contains(body, `"entity_id":"light.living_room"`) { + t.Errorf("body = %q", body) + } +} + +// A service that is not in the domain's table never leaves the box. The +// allowlist is the gate, and it is enforced on the way out too, so a corrupted +// or hand-edited cmd column cannot reach an arbitrary Home Assistant service. +func TestCallServiceRefusesUnknownServiceAndDomain(t *testing.T) { + called := false + c, _ := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + called = true + _, _ = w.Write([]byte(`[]`)) + }) + for _, tc := range []struct { + entity, service string + want error + }{ + {"light.living_room", "delete_everything", ErrNotControllable}, + {"sensor.bedroom_temp", "turn_on", ErrNotControllable}, + {"nodot", "turn_on", ErrUnknownEntity}, + } { + if _, err := c.CallService(context.Background(), tc.entity, tc.service); !errors.Is(err, tc.want) { + t.Errorf("CallService(%q,%q) err = %v, want %v", tc.entity, tc.service, err, tc.want) + } + } + if called { + t.Error("a refused call still reached the network") + } +} + +func TestHTTPErrorIsAnError(t *testing.T) { + c, _ := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + }) + if _, err := c.States(context.Background()); err == nil { + t.Fatal("want error on 401") + } +} + +func TestUnconfiguredClientRefuses(t *testing.T) { + c := NewClient(Config{}) + if _, err := c.States(context.Background()); !errors.Is(err, ErrNotConfigured) { + t.Fatalf("err = %v, want ErrNotConfigured", err) + } +} + +func TestValidate(t *testing.T) { + ok := Config{URL: "http://ha.lan:8123", Token: "t"} + if err := Validate(ok); err != nil { + t.Fatalf("Validate(ok): %v", err) + } + for name, c := range map[string]Config{ + "no url": {Token: "t"}, + "no token": {URL: "http://ha.lan:8123"}, + "bad scheme": {URL: "ftp://ha.lan", Token: "t"}, + "no host": {URL: "http://", Token: "t"}, + "not a url": {URL: "://x", Token: "t"}, + "bare string": {URL: "ha.lan:8123", Token: "t"}, + } { + if err := Validate(c); err == nil { + t.Errorf("Validate(%s) = nil, want error", name) + } + } +} + +func TestAllowlistEncoding(t *testing.T) { + cmd := Cmd("light.living_room", "turn_off") + id, svc, ok := ParseCmd(cmd) + if !ok || id != "light.living_room" || svc != "turn_off" { + t.Fatalf("ParseCmd(%v) = %q,%q,%v", cmd, id, svc, ok) + } + // Anything that is not exactly a three-element smarthome row stays a + // process row, or the executor would swallow a real shell tool. + for _, bad := range [][]string{ + nil, + {"smarthome"}, + {"smarthome", "light.x"}, + {"smarthome", "light.x", "turn_on", "extra"}, + {"smarthome", "", "turn_on"}, + {"smarthome", "light.x", ""}, + {"systemctl", "restart", "nginx"}, + } { + if _, _, ok := ParseCmd(bad); ok { + t.Errorf("ParseCmd(%v) = ok, want not a smarthome row", bad) + } + } + if got := LocalName("light.living_room", "off"); got != "home_light_living_room_off" { + t.Errorf("LocalName = %q", got) + } + if got := Scope("light"); got != "smarthome:light" { + t.Errorf("Scope = %q", got) + } + if Services("light") == nil || Services("sensor") != nil { + t.Error("Services: light must be controllable and sensor must not") + } + if DomainOf("light.x") != "light" || DomainOf("nodot") != "" || DomainOf(".x") != "" { + t.Error("DomainOf") + } +} + +// Home Assistant answers a service call with the states it changed, and an +// entity that has been removed or whose integration is offline gets 200 and an +// empty array. "готово" for that is Maven asserting something false about the +// physical world: he says выключи свет, she says done, the light stays on. +func TestCallServiceOnAnEntityThatChangedNothing(t *testing.T) { + c, _ := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`[]`)) + }) + if _, err := c.CallService(context.Background(), "light.living_room", "turn_off"); !errors.Is(err, ErrUnknownEntity) { + t.Errorf("err = %v, want ErrUnknownEntity for a call that changed nothing", err) + } + // A response shape we cannot parse is not evidence of failure: HA answered + // 2xx, so the call is reported as made. + c2, _ := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"result":"ok"}`)) + }) + if out, err := c2.CallService(context.Background(), "light.living_room", "turn_off"); err != nil || out != "готово" { + t.Errorf("out,err = %q,%v", out, err) + } +} + +// The cap must not be spent on sensors. Entity ids sort by domain prefix and +// binary_sensor sorts first, so a globally sorted truncation handed all forty +// slots to connectivity sensors: propose found nothing controllable and +// homeSummary said "всё выключено" with the lights on. +func TestCapKeepsControllableEntitiesFirst(t *testing.T) { + var all []Entity + for i := 0; i < 40; i++ { + all = append(all, Entity{ID: fmt.Sprintf("binary_sensor.b%02d", i), Domain: "binary_sensor", State: "off"}) + } + for i := 0; i < 30; i++ { + all = append(all, Entity{ID: fmt.Sprintf("sensor.s%02d", i), Domain: "sensor", State: "1"}) + } + for i := 0; i < 4; i++ { + all = append(all, Entity{ID: fmt.Sprintf("switch.w%d", i), Domain: "switch", State: "on"}) + } + for i := 0; i < 3; i++ { + all = append(all, Entity{ID: fmt.Sprintf("light.l%d", i), Domain: "light", State: "on"}) + } + got := capEntities(all, 40) + if len(got) != 40 { + t.Fatalf("kept %d entities, want 40", len(got)) + } + kept := map[string]int{} + for _, e := range got { + kept[e.Domain]++ + } + if kept["switch"] != 4 || kept["light"] != 3 { + t.Errorf("kept %d switches and %d lights, want all 4 and all 3: %v", kept["switch"], kept["light"], kept) + } + // The remainder still carries readable sensors, spread across both sensor + // domains rather than exhausting the one that sorts first. + if kept["sensor"] == 0 || kept["binary_sensor"] == 0 { + t.Errorf("the read domains were starved: %v", kept) + } + // Deterministic across restarts: /tools has to read the same each time. + for i := 1; i < len(got); i++ { + if got[i-1].ID >= got[i].ID { + t.Fatalf("output is not sorted by id at %d", i) + } + } +} + +// A deadbolt is a different class of object from a lamp. A bare url+token block +// must not auto-propose an unlock row for every door in the flat. +func TestLockIsNotInTheDefaultDomains(t *testing.T) { + c := NewClient(Config{URL: "http://x", Token: "t"}) + if c.wanted("lock") { + t.Error("lock is enumerated without being named in domains") + } + if !c.wanted("light") || !c.wanted("sensor") { + t.Error("the ordinary default domains were lost") + } + named := NewClient(Config{URL: "http://x", Token: "t", Domains: []string{"lock"}}) + if !named.wanted("lock") { + t.Error("lock named in domains is still not enumerated") + } +} diff --git a/internal/speaker/enroll.go b/internal/speaker/enroll.go new file mode 100644 index 0000000..d5461f7 --- /dev/null +++ b/internal/speaker/enroll.go @@ -0,0 +1,109 @@ +package speaker + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + "github.com/kami/maven/internal/audio" +) + +// Enroll registers a voice under an id and a spoken name. +// +// Several separate samples are required (MinEnrollSamples, MinEnrollSeconds +// total): a profile built from one sentence encodes that sentence as much as the +// person, and the resulting threshold behaviour is unpredictable. The samples +// are embedded individually and the voiceprints averaged, then re-normalised. +// +// Re-enrolling an existing id REPLACES the profile. That is the intended way to +// improve a weak one, and it is why the store upserts by id. +// +// # The refused step +// +// The plan document's fourth bullet reads "unknown speakers are enrolled on +// first interaction (prompt: 'кто это?')". That is refused. Enrolling a voice is +// taking a biometric of a person; doing it automatically the first time someone +// walks past the microphone is doing it to guests, without them being part of +// the exchange, and a TTS question into a room is not consent from whoever +// happens to answer. Enrolment here is an explicit act: an id, a name, and +// samples deliberately recorded for the purpose. An unknown voice stays unknown, +// which the rest of the system is built to cope with. +func (r *Recognizer) Enroll(ctx context.Context, id, name string, samples []audio.Audio) (Profile, error) { + id = NormalizeID(id) + if !ValidID(id) { + return Profile{}, fmt.Errorf("%w: %q", ErrBadID, id) + } + name = strings.TrimSpace(name) + if name == "" { + name = id + } + if len(samples) < MinEnrollSamples { + return Profile{}, fmt.Errorf("%w: %d sample(s), need %d separate ones", + ErrTooShort, len(samples), MinEnrollSamples) + } + + var total float64 + for i, s := range samples { + if !s.Format.IsValid() { + return Profile{}, fmt.Errorf("%w: sample %d: %+v", ErrBadFormat, i+1, s.Format) + } + total += seconds(s) + } + if total < MinEnrollSeconds { + return Profile{}, fmt.Errorf("%w: %.1fs total, need %.1fs", + ErrTooShort, total, MinEnrollSeconds) + } + + // Embed first, store second. A model failure halfway through must not leave + // a half-built profile that would then be matched against. + var ( + sum []float32 + dim int + ) + for i, s := range samples { + vec, err := r.embed(ctx, s) + if err != nil { + return Profile{}, fmt.Errorf("speaker: enroll %q sample %d: %w", id, i+1, err) + } + if sum == nil { + sum = make([]float32, len(vec)) + dim = len(vec) + } else if len(vec) != dim { + // One model, one width. A mixed-width average would be nonsense. + return Profile{}, fmt.Errorf("%w: sample %d is %d wide, expected %d", + ErrBadVector, i+1, len(vec), dim) + } + for j, f := range vec { + sum[j] += f + } + } + mean, err := Normalize(sum) + if err != nil { + // Samples that cancel each other out to zero are not one voice. + return Profile{}, fmt.Errorf("speaker: enroll %q: %w", id, err) + } + + p := Profile{ + ID: id, + Name: name, + Enrolled: r.now().UTC(), + Samples: len(samples), + Dim: dim, + Vec: mean, + } + meta := map[string]string{ + "name": p.Name, + "samples": strconv.Itoa(p.Samples), + "enrolled": p.Enrolled.Format(time.RFC3339), + // kind marks the row for anything walking the vector table, so a future + // export or debug page can tell a voiceprint from a note embedding + // without parsing the id. + "kind": "speaker", + } + if err := r.cat.Insert(ctx, Prefix+id, mean, meta); err != nil { + return Profile{}, fmt.Errorf("speaker: enroll %q: %w", id, err) + } + return p, nil +} diff --git a/internal/speaker/recognizer.go b/internal/speaker/recognizer.go new file mode 100644 index 0000000..d881de5 --- /dev/null +++ b/internal/speaker/recognizer.go @@ -0,0 +1,219 @@ +package speaker + +import ( + "context" + "fmt" + "sort" + "strconv" + "time" + + "github.com/kami/maven/internal/audio" + "github.com/kami/maven/internal/memory" +) + +// Recognizer holds the embedder and the enrolled profiles. +// +// The profiles live in the shared vector table under the "speaker:" id prefix, +// which is what the plan asked for and what keeps them inside the encrypted +// store rather than in a sidecar file. They are read through memory.Catalog +// (ByPrefix / Delete) rather than Search, because "who is enrolled" is not a +// similarity question and note recall must never rank a voiceprint. +type Recognizer struct { + emb Embedder + cat memory.Catalog + threshold float64 + minSec float64 + now func() time.Time +} + +// Config — the recognizer's knobs, built from config.SpeakerConfig. +type Config struct { + // Threshold — cosine similarity a match must beat. 0 ⇒ DefaultThreshold. + Threshold float64 + // MinSeconds — least speech an identification will look at. 0 ⇒ + // DefaultMinSeconds. + MinSeconds float64 +} + +// New builds a Recognizer. emb nil ⇒ Disabled, which is this box's state and +// makes every Identify answer ErrDisabled while enrolment and listing still +// behave sensibly (they refuse for the same reason, with the same error). +func New(emb Embedder, cat memory.Catalog, cfg Config) (*Recognizer, error) { + if cat == nil { + return nil, fmt.Errorf("speaker: no profile store") + } + if emb == nil { + emb = Disabled{} + } + th := cfg.Threshold + if th <= 0 { + th = DefaultThreshold + } + min := cfg.MinSeconds + if min <= 0 { + min = DefaultMinSeconds + } + return &Recognizer{emb: emb, cat: cat, threshold: th, minSec: min, now: time.Now}, nil +} + +// Enabled reports whether an embedding model is actually wired. Surfaces use it +// to say "recognition is off" once instead of failing every turn. +func (r *Recognizer) Enabled() bool { + _, disabled := r.emb.(Disabled) + return !disabled +} + +// Threshold is the configured match floor, for a status line. +func (r *Recognizer) Threshold() float64 { return r.threshold } + +// Identify names the voice in a. ErrUnknown when nothing is close enough, which +// is a normal answer and not a failure: a guest is a guest, and the caller +// carries on with no speaker attached rather than guessing. +// +// Identification never decides whether Maven listens. It annotates the turn. +func (r *Recognizer) Identify(ctx context.Context, a audio.Audio) (Match, error) { + if !a.Format.IsValid() { + return Match{}, fmt.Errorf("%w: %+v", ErrBadFormat, a.Format) + } + if seconds(a) < r.minSec { + return Match{}, fmt.Errorf("%w: %.1fs, need %.1fs", ErrTooShort, seconds(a), r.minSec) + } + vec, err := r.embed(ctx, a) + if err != nil { + return Match{}, err + } + profiles, err := r.List(ctx) + if err != nil { + return Match{}, err + } + if len(profiles) == 0 { + return Match{}, ErrNoProfiles + } + + best := Match{Score: -2} + for _, p := range profiles { + if s := Similarity(vec, p.Vec); s > best.Score { + best = Match{Profile: p, Score: s} + } + } + if best.Score < r.threshold { + // The closest profile is reported in the error for a log line, because + // "не узнала, ближе всего Ками на 0.61" is what makes a threshold + // tunable. The caller must not use it as an identification. + return Match{}, fmt.Errorf("%w (closest %s at %.2f, need %.2f)", + ErrUnknown, best.Profile.ID, best.Score, r.threshold) + } + return best, nil +} + +// List returns every enrolled profile, sorted by id so a listing is stable. +func (r *Recognizer) List(ctx context.Context) ([]Profile, error) { + recs, err := r.cat.ByPrefix(ctx, Prefix) + if err != nil { + return nil, fmt.Errorf("speaker: list: %w", err) + } + out := make([]Profile, 0, len(recs)) + for _, rec := range recs { + out = append(out, profileFromRecord(rec)) + } + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out, nil +} + +// Get returns one profile by id. +func (r *Recognizer) Get(ctx context.Context, id string) (Profile, error) { + id = NormalizeID(id) + if !ValidID(id) { + return Profile{}, fmt.Errorf("%w: %q", ErrBadID, id) + } + recs, err := r.cat.ByPrefix(ctx, Prefix+id) + if err != nil { + return Profile{}, fmt.Errorf("speaker: get: %w", err) + } + for _, rec := range recs { + if rec.ID == Prefix+id { + return profileFromRecord(rec), nil + } + } + return Profile{}, fmt.Errorf("%w: %q", ErrNotFound, id) +} + +// Forget deletes a profile. This is the one operation that must always work: +// a voiceprint is data about a person, and "перестань узнавать её" has to +// actually remove it, not mark it inactive. +// +// Forgetting a profile that is not there is not an error, matching +// memory.Catalog.Delete. It used to read the row first and answer ErrNotFound, +// which meant the layer documented as the one that must always work was the +// layer reintroducing a failure: a surface retrying a forget after a partial +// failure got an error on the second try, for a voiceprint that was already +// gone. The caller asked for it to be gone and it is gone. +func (r *Recognizer) Forget(ctx context.Context, id string) error { + id = NormalizeID(id) + if !ValidID(id) { + return fmt.Errorf("%w: %q", ErrBadID, id) + } + if err := r.cat.Delete(ctx, Prefix+id); err != nil { + return fmt.Errorf("speaker: forget %q: %w", id, err) + } + return nil +} + +// embed runs the model and normalises the result. +func (r *Recognizer) embed(ctx context.Context, a audio.Audio) ([]float32, error) { + raw, err := r.emb.Embed(ctx, a) + if err != nil { + return nil, err + } + vec, err := Normalize(raw) + if err != nil { + return nil, err + } + return vec, nil +} + +// profileFromRecord reads a stored row back into a Profile. A row with +// unreadable metadata still yields a usable voiceprint — the vector is the part +// that matters, and losing a name should not lose the enrolment. +// +// It also says when the metadata did not read cleanly. Without that, a row +// whose samples count is "12x" and whose name is missing came back as a +// plausible profile called by its own id with 0 samples, which is exactly what +// a real minimal enrolment looks like. Damaged is the difference between "he +// enrolled badly" and "this row is broken". +func profileFromRecord(rec memory.Record) Profile { + p := Profile{ + ID: trimPrefix(rec.ID), + Vec: rec.Vec, + Dim: len(rec.Vec), + Name: rec.Meta["name"], + } + if s := rec.Meta["samples"]; s != "" { + n, err := strconv.Atoi(s) + if err != nil || n < 0 { + p.Damaged = true + } else { + p.Samples = n + } + } + if ts := rec.Meta["enrolled"]; ts != "" { + t, err := time.Parse(time.RFC3339, ts) + if err != nil { + p.Damaged = true + } else { + p.Enrolled = t + } + } + if p.Name == "" { + p.Name = p.ID + p.Damaged = true + } + return p +} + +func trimPrefix(id string) string { + if len(id) > len(Prefix) && id[:len(Prefix)] == Prefix { + return id[len(Prefix):] + } + return id +} diff --git a/internal/speaker/speaker.go b/internal/speaker/speaker.go new file mode 100644 index 0000000..2fc9b28 --- /dev/null +++ b/internal/speaker/speaker.go @@ -0,0 +1,241 @@ +// Package speaker is voice identification (Vikunja #255, +// docs/plans/10-speaker-recognition.md). +// +// The shape is the same seam internal/vision uses: an Embedder turns audio into +// a voiceprint, a Recognizer compares one against the enrolled profiles, and a +// Disabled floor refuses politely when nothing is wired. On this box nothing is +// wired, and that is the honest state — see "Blocked" below. +// +// # A voiceprint is not like the other vectors +// +// Everything else in the vector table is something he wrote or said. A speaker +// profile is biometric data about a person, quite possibly a person who never +// asked for Maven to exist. The rules that follow from that are in the code: +// +// - Enrolment is explicit and named. There is no "enrol the unknown voice +// automatically" path; see the refusal in enroll.go. +// - A profile is deletable, individually, and Forget really removes the row. +// - Below the threshold the answer is "I do not know", never the closest +// guess. A misattributed fact is worse than an unattributed one. +// - Nothing here gates whether Maven listens or answers. Identification +// annotates a turn; it never authorises one, and an unrecognised voice is +// not turned away. +// - Voiceprints never leave the box. They live in the encrypted store with +// everything else and are never search input to anything external. +// +// # Blocked +// +// There is no speaker-embedding model on this box: no ECAPA-TDNN, no x-vector, +// no wespeaker or titanet ONNX anywhere under /mnt/hdd1 or models/ (checked +// 2026-08-01; the only ONNX files are the e5 text embedder and the piper voice). +// There are also no enrolment samples. So Recognizer runs against Disabled and +// every Identify answers ErrDisabled until a model lands. +// +// "Recognition is blocked but enrolment works" is not true and this package +// used to imply it. Enroll embeds every sample before it stores anything +// (enroll.go, "Embed first, store second"), so with no model it fails on the +// first sample with ErrDisabled and nothing is ever stored. List then returns +// an empty list forever and Forget has nothing to delete. Without an embedder +// all three operations are no-ops, and mavend does not wire the wire methods +// at all in that state. +// +// The MFCC + GMM "simplest floor" in the plan document is refused rather than +// deferred. A hand-rolled spectral distance would identify people confidently +// and wrongly, and its output would be written into facts as "Ками said this". +// For a biometric, a bad floor is worse than none: no answer is honest, and a +// wrong answer is a false memory about a person. +package speaker + +import ( + "context" + "errors" + "math" + "strings" + "time" + + "github.com/kami/maven/internal/audio" + "github.com/kami/maven/internal/memory" +) + +// Prefix — the id prefix speaker profiles carry in the shared vector table. +// It is what ByPrefix enumerates, and what every Store implementation filters +// out of Search so note recall cannot rank a voiceprint. The constant lives in +// internal/memory because the store layer has to know it and cannot import this +// package. +const Prefix = memory.NonRecallPrefix + +// DefaultThreshold — cosine similarity a match must beat to be a match. +// +// 0.7 is the usual operating point for ECAPA-style embeddings on clean speech +// and it is deliberately on the strict side here. The two error directions are +// not symmetric: refusing to name a voice costs a "не узнала", while naming the +// wrong person writes his wife's remark into a fact attributed to him. +const DefaultThreshold = 0.7 + +// DefaultMinSeconds — how much speech an identification needs. Under about two +// seconds a voiceprint is mostly noise and the similarity score is not worth +// reading. +const DefaultMinSeconds = 2.0 + +// MinEnrollSamples / MinEnrollSeconds — what enrolment requires. Several +// separate utterances, not one long one: a profile built from a single sentence +// encodes that sentence's prosody as much as the voice. +const ( + MinEnrollSamples = 3 + MinEnrollSeconds = 9.0 +) + +// Errors callers distinguish. +var ( + // ErrDisabled — no embedding model is wired. The state of this box. + ErrDisabled = errors.New("speaker: recognition is not configured") + // ErrTooShort — not enough speech to say anything about. + ErrTooShort = errors.New("speaker: not enough audio") + // ErrUnknown — audio embedded fine, but no enrolled profile is close + // enough. Not an error in the sense of something being broken: it is the + // correct answer for a guest, and the caller should carry on without a + // speaker rather than treat the turn as failed. + ErrUnknown = errors.New("speaker: voice not recognised") + // ErrNoProfiles — nobody is enrolled yet. + ErrNoProfiles = errors.New("speaker: nobody is enrolled") + // ErrNotFound — no profile with that id. + ErrNotFound = errors.New("speaker: no such profile") + // ErrBadID — an id that is empty or carries characters an id should not. + ErrBadID = errors.New("speaker: invalid profile id") + // ErrBadFormat — audio that is not the canonical 16 kHz mono PCM shape. + ErrBadFormat = errors.New("speaker: audio format not supported") + // ErrBadVector — an embedder returned something unusable (empty, or all + // zeroes, which normalises to nothing and would match everything equally). + ErrBadVector = errors.New("speaker: embedder returned an unusable vector") +) + +// Embedder turns speech into a voiceprint. Implementations are expected to +// return an L2-normalised vector, because the whole store compares by dot +// product; Normalize is applied anyway rather than trusted. +// +// This is the seam a downloaded ECAPA-TDNN ONNX model plugs into. It is an +// interface rather than a concrete ONNX type so the package is testable with no +// model on disk, which is the only way it could be tested here at all. +type Embedder interface { + Embed(ctx context.Context, a audio.Audio) ([]float32, error) + // Dim is the vector width, used to reject a profile recorded with a + // different model rather than silently scoring it as zero. + Dim() int +} + +// Disabled is the floor: no model, no answers, no guesses. +type Disabled struct{} + +// Embed always fails with ErrDisabled. +func (Disabled) Embed(context.Context, audio.Audio) ([]float32, error) { return nil, ErrDisabled } + +// Dim is 0 for the disabled embedder. +func (Disabled) Dim() int { return 0 } + +// Profile — one enrolled voice. +// +// Name is what she calls the person out loud ("Ками"). ID is the stable handle +// used in sources and metadata. Samples records how many utterances the +// voiceprint was averaged from, so a profile enrolled from the bare minimum is +// visibly weaker than one built from ten. +type Profile struct { + ID string `json:"id"` + Name string `json:"name"` + Enrolled time.Time `json:"enrolled"` + Samples int `json:"samples"` + Dim int `json:"dim"` + + // Damaged marks a row whose stored metadata did not read back cleanly — + // an unparsable sample count or timestamp, or a missing name. The + // voiceprint is still usable, but the row should be listed as damaged + // rather than as a plausible profile enrolled from nothing. + Damaged bool `json:"damaged,omitempty"` + + // Vec is the voiceprint. Not serialised to any surface: a listing tells him + // who is enrolled, it does not hand out the biometric itself. + Vec []float32 `json:"-"` +} + +// Source is what a fact or note written during this speaker's turn is tagged +// with, e.g. "tap:voice:speaker:kami". Attribution belongs in the source rather +// than in the text, so it can be corrected or dropped later without rewriting +// what was said. +func (p Profile) Source(base string) string { + if p.ID == "" { + return base + } + return base + ":" + Prefix + p.ID +} + +// Match — an identification result. Score is cosine similarity in [-1, 1]. +type Match struct { + Profile Profile + Score float64 +} + +// ValidID reports whether an id is usable as a profile handle. Deliberately +// narrow: lowercase letters, digits, dash and underscore. Ids end up in note +// sources and in vector-table keys, so a permissive id would be a way to write +// into a neighbouring key space. +func ValidID(id string) bool { + if id == "" || len(id) > 64 { + return false + } + for _, r := range id { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '-', r == '_': + default: + return false + } + } + return true +} + +// NormalizeID lowercases and trims a proposed id before validating it, so +// "Ками" typed as "Kami " does not fail for a reason nobody can see. +func NormalizeID(id string) string { + return strings.ToLower(strings.TrimSpace(id)) +} + +// Normalize returns an L2-normalised copy of v, or ErrBadVector when there is +// nothing to normalise. A zero vector is refused rather than passed on: it +// scores 0 against everything, which reads as "no match" but for the wrong +// reason and would hide a broken embedder. +func Normalize(v []float32) ([]float32, error) { + if len(v) == 0 { + return nil, ErrBadVector + } + var sum float64 + for _, f := range v { + if math.IsNaN(float64(f)) || math.IsInf(float64(f), 0) { + return nil, ErrBadVector + } + sum += float64(f) * float64(f) + } + norm := math.Sqrt(sum) + if norm == 0 { + return nil, ErrBadVector + } + out := make([]float32, len(v)) + for i, f := range v { + out[i] = float32(float64(f) / norm) + } + return out, nil +} + +// Similarity is the cosine similarity of two L2-normalised vectors. Different +// widths score 0: a profile enrolled with another model must not accidentally +// match, and 0 is below every sane threshold. +func Similarity(a, b []float32) float64 { + if len(a) != len(b) || len(a) == 0 { + return 0 + } + var sum float64 + for i := range a { + sum += float64(a[i]) * float64(b[i]) + } + return sum +} + +// seconds is the playback length of a frame, for the minimum-audio checks. +func seconds(a audio.Audio) float64 { return a.Duration() } diff --git a/internal/speaker/speaker_test.go b/internal/speaker/speaker_test.go new file mode 100644 index 0000000..5af12b1 --- /dev/null +++ b/internal/speaker/speaker_test.go @@ -0,0 +1,447 @@ +package speaker + +import ( + "context" + "errors" + "math" + "strings" + "testing" + "time" + + "github.com/kami/maven/internal/audio" + "github.com/kami/maven/internal/memory" +) + +// fakeEmbedder returns a fixed vector per "voice", so a test can enrol one +// person and present another without a model. Wobble adds a small perturbation +// so repeated samples of one voice are close but not identical, which is what a +// real embedder produces. +type fakeEmbedder struct { + vec []float32 + err error + calls int + wobble float32 +} + +func (f *fakeEmbedder) Embed(_ context.Context, _ audio.Audio) ([]float32, error) { + f.calls++ + if f.err != nil { + return nil, f.err + } + out := append([]float32(nil), f.vec...) + if f.wobble != 0 && len(out) > 1 { + out[0] += f.wobble * float32(f.calls) + out[1] -= f.wobble * float32(f.calls) + } + return out, nil +} + +func (f *fakeEmbedder) Dim() int { return len(f.vec) } + +// speech builds n seconds of the canonical audio shape. +func speech(sec float64) audio.Audio { + return audio.Audio{Format: audio.PCM16kMono, Bytes: make([]byte, int(sec*16000)*2)} +} + +func newRec(t *testing.T, emb Embedder) (*Recognizer, memory.Catalog) { + t.Helper() + cat := memory.NewInMemoryStore() + r, err := New(emb, cat, Config{}) + if err != nil { + t.Fatal(err) + } + return r, cat +} + +func enrolSamples(n int, sec float64) []audio.Audio { + out := make([]audio.Audio, n) + for i := range out { + out[i] = speech(sec) + } + return out +} + +// The state of this box: no model on disk. Every identification refuses rather +// than guessing, and it says why. +func TestDisabledRefusesEverything(t *testing.T) { + r, _ := newRec(t, nil) + if r.Enabled() { + t.Error("a recognizer with no model reports itself enabled") + } + if _, err := r.Identify(context.Background(), speech(5)); !errors.Is(err, ErrDisabled) { + t.Errorf("Identify = %v, want ErrDisabled", err) + } + if _, err := r.Enroll(context.Background(), "kami", "Ками", enrolSamples(3, 4)); !errors.Is(err, ErrDisabled) { + t.Errorf("Enroll = %v, want ErrDisabled", err) + } + // Listing still works: knowing that nobody is enrolled needs no model. + got, err := r.List(context.Background()) + if err != nil || len(got) != 0 { + t.Errorf("List = %v, %v", got, err) + } +} + +func TestNewRequiresAProfileStore(t *testing.T) { + if _, err := New(nil, nil, Config{}); err == nil { + t.Error("built a recognizer with nowhere to keep profiles") + } +} + +func TestEnrollThenIdentify(t *testing.T) { + emb := &fakeEmbedder{vec: []float32{1, 0, 0, 0}, wobble: 0.01} + r, _ := newRec(t, emb) + ctx := context.Background() + + p, err := r.Enroll(ctx, "Kami ", "Ками", enrolSamples(3, 4)) + if err != nil { + t.Fatalf("enroll: %v", err) + } + if p.ID != "kami" { + t.Errorf("id = %q, want the normalised %q", p.ID, "kami") + } + if p.Name != "Ками" || p.Samples != 3 || p.Dim != 4 { + t.Errorf("profile = %+v", p) + } + + m, err := r.Identify(ctx, speech(5)) + if err != nil { + t.Fatalf("identify: %v", err) + } + if m.Profile.ID != "kami" || m.Profile.Name != "Ками" { + t.Errorf("match = %+v", m) + } + if m.Score < r.Threshold() { + t.Errorf("score %.3f is below the threshold it supposedly passed", m.Score) + } +} + +// The error direction that matters. Naming the wrong person writes a false +// memory about them, so a voice that is not close enough gets no name at all. +func TestUnfamiliarVoiceIsNotGuessed(t *testing.T) { + emb := &fakeEmbedder{vec: []float32{1, 0, 0, 0}} + r, _ := newRec(t, emb) + ctx := context.Background() + if _, err := r.Enroll(ctx, "kami", "Ками", enrolSamples(3, 4)); err != nil { + t.Fatal(err) + } + + // A different voice: orthogonal voiceprint, similarity 0. + emb.vec = []float32{0, 1, 0, 0} + m, err := r.Identify(ctx, speech(5)) + if !errors.Is(err, ErrUnknown) { + t.Fatalf("Identify = %v, want ErrUnknown", err) + } + if m.Profile.ID != "" { + t.Errorf("a refused identification still handed back %q", m.Profile.ID) + } + // The log line needs the near miss to make the threshold tunable. + if !contains(err.Error(), "kami") { + t.Errorf("error does not name the closest profile: %v", err) + } +} + +// Just under the threshold is still unknown. A boundary this important gets its +// own test rather than being implied. +func TestThresholdIsAFloorNotASuggestion(t *testing.T) { + cat := memory.NewInMemoryStore() + emb := &fakeEmbedder{vec: []float32{1, 0}} + r, err := New(emb, cat, Config{Threshold: 0.9}) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + if _, err := r.Enroll(ctx, "kami", "Ками", enrolSamples(3, 4)); err != nil { + t.Fatal(err) + } + // cos ≈ 0.866, comfortably similar and still not similar enough. + emb.vec = []float32{0.866, 0.5} + if _, err := r.Identify(ctx, speech(5)); !errors.Is(err, ErrUnknown) { + t.Fatalf("0.866 against a 0.9 threshold = %v, want ErrUnknown", err) + } +} + +func TestShortAudioIsRefusedBeforeTheModelRuns(t *testing.T) { + emb := &fakeEmbedder{vec: []float32{1, 0}} + r, _ := newRec(t, emb) + if _, err := r.Identify(context.Background(), speech(0.5)); !errors.Is(err, ErrTooShort) { + t.Fatalf("got %v, want ErrTooShort", err) + } + if emb.calls != 0 { + t.Error("a half-second of audio was sent to the model anyway") + } +} + +func TestWrongAudioFormatIsRefused(t *testing.T) { + r, _ := newRec(t, &fakeEmbedder{vec: []float32{1, 0}}) + bad := audio.Audio{ + Format: audio.Format{SampleRate: 44100, Channels: 2, SampleBits: 16, Encoding: "pcm_s16le"}, + Bytes: make([]byte, 44100*4*5), + } + if _, err := r.Identify(context.Background(), bad); !errors.Is(err, ErrBadFormat) { + t.Fatalf("got %v, want ErrBadFormat", err) + } +} + +func TestIdentifyWithNobodyEnrolled(t *testing.T) { + r, _ := newRec(t, &fakeEmbedder{vec: []float32{1, 0}}) + if _, err := r.Identify(context.Background(), speech(5)); !errors.Is(err, ErrNoProfiles) { + t.Fatalf("got %v, want ErrNoProfiles", err) + } +} + +// Enrolment is an explicit act with real samples behind it, not a byproduct of +// someone speaking once. +func TestEnrollmentRequiresSeveralRealSamples(t *testing.T) { + r, _ := newRec(t, &fakeEmbedder{vec: []float32{1, 0}}) + ctx := context.Background() + cases := []struct { + name string + samples []audio.Audio + }{ + {"one long sample", enrolSamples(1, 30)}, + {"two samples", enrolSamples(2, 10)}, + {"three samples but seconds of audio", enrolSamples(3, 1)}, + {"none at all", nil}, + } + for _, c := range cases { + if _, err := r.Enroll(ctx, "kami", "Ками", c.samples); !errors.Is(err, ErrTooShort) { + t.Errorf("%s: %v, want ErrTooShort", c.name, err) + } + } +} + +func TestEnrollRejectsBadIDs(t *testing.T) { + r, _ := newRec(t, &fakeEmbedder{vec: []float32{1, 0}}) + for _, id := range []string{"", " ", "../etc/passwd", "speaker:kami", "имя", "a/b", "x y"} { + if _, err := r.Enroll(context.Background(), id, "n", enrolSamples(3, 4)); !errors.Is(err, ErrBadID) { + t.Errorf("id %q accepted or wrong error: %v", id, err) + } + } +} + +// Re-enrolling replaces the voiceprint. Leaving the old one searchable would +// mean a person's rejected profile keeps matching them. +func TestReEnrollReplaces(t *testing.T) { + emb := &fakeEmbedder{vec: []float32{1, 0, 0}} + r, _ := newRec(t, emb) + ctx := context.Background() + if _, err := r.Enroll(ctx, "kami", "Ками", enrolSamples(3, 4)); err != nil { + t.Fatal(err) + } + emb.vec = []float32{0, 1, 0} + if _, err := r.Enroll(ctx, "kami", "Ками", enrolSamples(4, 4)); err != nil { + t.Fatal(err) + } + list, err := r.List(ctx) + if err != nil { + t.Fatal(err) + } + if len(list) != 1 { + t.Fatalf("%d profiles after re-enrolling one person", len(list)) + } + if list[0].Samples != 4 { + t.Errorf("sample count = %d, want the new 4", list[0].Samples) + } + // The new voiceprint is the one that matches. + if m, err := r.Identify(ctx, speech(5)); err != nil || m.Score < 0.99 { + t.Errorf("identify after re-enrol: %v (score %.3f)", err, m.Score) + } +} + +// "Перестань узнавать её" has to actually delete the biometric. +func TestForgetRemovesTheVoiceprint(t *testing.T) { + emb := &fakeEmbedder{vec: []float32{1, 0}} + r, cat := newRec(t, emb) + ctx := context.Background() + if _, err := r.Enroll(ctx, "guest", "Гостья", enrolSamples(3, 4)); err != nil { + t.Fatal(err) + } + if err := r.Forget(ctx, "Guest "); err != nil { + t.Fatalf("forget: %v", err) + } + recs, err := cat.ByPrefix(ctx, Prefix) + if err != nil { + t.Fatal(err) + } + if len(recs) != 0 { + t.Errorf("%d row(s) survived Forget", len(recs)) + } + if _, err := r.Get(ctx, "guest"); !errors.Is(err, ErrNotFound) { + t.Errorf("Get after Forget = %v, want ErrNotFound", err) + } + // Forgetting twice is not an error. A surface retrying after a partial + // failure must not be told the voice it asked to remove is missing. + if err := r.Forget(ctx, "guest"); err != nil { + t.Errorf("second Forget = %v, want nil", err) + } +} + +// A row whose metadata is corrupt lists as damaged, not as a plausible profile +// enrolled from zero samples. Those two used to look identical. +func TestCorruptMetadataListsAsDamaged(t *testing.T) { + r, cat := newRec(t, &fakeEmbedder{vec: []float32{1, 0}}) + ctx := context.Background() + if err := cat.Insert(ctx, Prefix+"kami", []float32{1, 0}, map[string]string{ + "kind": "speaker", + "samples": "12x", + "enrolled": "yesterday", + }); err != nil { + t.Fatal(err) + } + ps, err := r.List(ctx) + if err != nil { + t.Fatal(err) + } + if len(ps) != 1 { + t.Fatalf("List returned %d profiles, want 1", len(ps)) + } + p := ps[0] + if !p.Damaged { + t.Errorf("profile %+v is not marked damaged", p) + } + if p.Samples != 0 || !p.Enrolled.IsZero() { + t.Errorf("unreadable metadata was parsed anyway: samples %d, enrolled %v", p.Samples, p.Enrolled) + } + if len(p.Vec) != 2 { + t.Errorf("the voiceprint was dropped: %v", p.Vec) + } +} + +// A clean row is not damaged. Without this the flag could be set always and +// the test above would still pass. +func TestCleanProfileIsNotDamaged(t *testing.T) { + r, _ := newRec(t, &fakeEmbedder{vec: []float32{1, 0}}) + ctx := context.Background() + if _, err := r.Enroll(ctx, "kami", "Ками", enrolSamples(3, 4)); err != nil { + t.Fatal(err) + } + ps, err := r.List(ctx) + if err != nil { + t.Fatal(err) + } + if len(ps) != 1 || ps[0].Damaged { + t.Fatalf("List = %+v, want one undamaged profile", ps) + } +} + +// Voiceprints share the vector table with note and fact embeddings, so the +// prefix has to actually partition it. +func TestProfilesDoNotCollideWithNoteVectors(t *testing.T) { + emb := &fakeEmbedder{vec: []float32{1, 0}} + r, cat := newRec(t, emb) + ctx := context.Background() + if err := cat.Insert(ctx, "note:1", []float32{1, 0}, map[string]string{"text": "заметка"}); err != nil { + t.Fatal(err) + } + if _, err := r.Enroll(ctx, "kami", "Ками", enrolSamples(3, 4)); err != nil { + t.Fatal(err) + } + list, err := r.List(ctx) + if err != nil { + t.Fatal(err) + } + if len(list) != 1 || list[0].ID != "kami" { + t.Errorf("listing picked up a non-speaker row: %+v", list) + } + // And an identical note vector is never returned as a match. + m, err := r.Identify(ctx, speech(5)) + if err != nil { + t.Fatal(err) + } + if m.Profile.ID != "kami" { + t.Errorf("matched %q", m.Profile.ID) + } +} + +func TestEmbedderFailurePropagates(t *testing.T) { + emb := &fakeEmbedder{vec: []float32{1, 0}, err: errors.New("onnx fell over")} + r, _ := newRec(t, emb) + if _, err := r.Identify(context.Background(), speech(5)); err == nil { + t.Error("a model failure was reported as a successful identification") + } + if _, err := r.Enroll(context.Background(), "kami", "К", enrolSamples(3, 4)); err == nil { + t.Error("a model failure produced a profile") + } +} + +// A zero vector scores 0 against everything, which reads as "no match" for the +// wrong reason and would hide a broken model. +func TestUnusableVectorsAreRefused(t *testing.T) { + r, _ := newRec(t, &fakeEmbedder{vec: []float32{0, 0, 0}}) + if _, err := r.Enroll(context.Background(), "kami", "К", enrolSamples(3, 4)); !errors.Is(err, ErrBadVector) { + t.Errorf("zero vector: %v, want ErrBadVector", err) + } + if _, err := Normalize(nil); !errors.Is(err, ErrBadVector) { + t.Errorf("empty: %v", err) + } + if _, err := Normalize([]float32{float32(nan())}); !errors.Is(err, ErrBadVector) { + t.Errorf("NaN: %v", err) + } +} + +func TestNormalizeProducesAUnitVector(t *testing.T) { + v, err := Normalize([]float32{3, 4}) + if err != nil { + t.Fatal(err) + } + if got := Similarity(v, v); got < 0.999 || got > 1.001 { + t.Errorf("self-similarity = %f, want 1", got) + } +} + +// A profile enrolled with another model must not accidentally match. +func TestDifferentWidthsScoreZero(t *testing.T) { + if got := Similarity([]float32{1, 0}, []float32{1, 0, 0}); got != 0 { + t.Errorf("mismatched widths scored %f", got) + } +} + +// Attribution belongs in the source, so it can be corrected without rewriting +// what was said. +func TestProfileSource(t *testing.T) { + p := Profile{ID: "kami"} + if got := p.Source("tap:voice"); got != "tap:voice:speaker:kami" { + t.Errorf("source = %q", got) + } + var anon Profile + if got := anon.Source("tap:voice"); got != "tap:voice" { + t.Errorf("unattributed source = %q, want the base unchanged", got) + } +} + +func TestValidID(t *testing.T) { + for _, ok := range []string{"kami", "guest-2", "a_b", "x"} { + if !ValidID(ok) { + t.Errorf("%q rejected", ok) + } + } + for _, bad := range []string{"", "Kami", "имя", "a b", "a/b", "a:b", "..", strings.Repeat("a", 65)} { + if ValidID(bad) { + t.Errorf("%q accepted", bad) + } + } +} + +func TestProfileMetadataSurvivesARoundTrip(t *testing.T) { + emb := &fakeEmbedder{vec: []float32{1, 0}} + r, _ := newRec(t, emb) + r.now = func() time.Time { return time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC) } + ctx := context.Background() + if _, err := r.Enroll(ctx, "kami", "Ками", enrolSamples(3, 4)); err != nil { + t.Fatal(err) + } + got, err := r.Get(ctx, "kami") + if err != nil { + t.Fatal(err) + } + if got.Name != "Ками" || got.Samples != 3 { + t.Errorf("profile = %+v", got) + } + if !got.Enrolled.Equal(time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC)) { + t.Errorf("enrolled = %v", got.Enrolled) + } +} + +func contains(s, sub string) bool { return strings.Contains(s, sub) } + +func nan() float64 { return math.NaN() } diff --git a/internal/store/digest.go b/internal/store/digest.go new file mode 100644 index 0000000..457b50e --- /dev/null +++ b/internal/store/digest.go @@ -0,0 +1,149 @@ +package store + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "time" +) + +// DigestStatus — the lifecycle state of a digest entry. Go has no enum type; +// the idiom is a defined type plus constants, which is what this is. The point +// is not ceremony: with bare strings nothing stopped a rule name or a body +// hash being passed where a status belongs, and every one of these values +// reaches SQL. A defined type makes that a compile error. +type DigestStatus string + +// pending = enqueued, waiting for a drain. drained = spoken as part of a +// bundle. expired = the tick loop's expiry sweep found it past its expires_ts +// before a drain happened — dropped, not delivered late. +const ( + DigestPending DigestStatus = "pending" + DigestDrained DigestStatus = "drained" + DigestExpired DigestStatus = "expired" +) + +// DigestEntry — one gate-suppressed care candidate durably held for later +// bundled delivery. +type DigestEntry struct { + ID int64 + Rule string + Severity int + Body string + CreatedTs time.Time + ExpiresTs time.Time +} + +// DigestBodyHash is the dedupe key for a digest entry: same rule, same +// wording ⇒ the same suppressed nudge repeating across ticks, and he should +// hear it once, not once per tick it kept getting suppressed. +func DigestBodyHash(rule, body string) string { + sum := sha256.Sum256([]byte(rule + "\x00" + body)) + return hex.EncodeToString(sum[:8]) +} + +// EnqueueDigestEntry durably records a suppressed care candidate worth +// resurfacing later. If a pending entry with the same rule+body already +// exists, this is a no-op that returns the existing id and deduped=true — +// the same suppressed nudge repeating across ticks must not pile up into +// several copies of itself in the eventual bundle. +func (s *Store) EnqueueDigestEntry(ctx context.Context, rule string, severity int, body string, now, expiresAt time.Time) (id int64, deduped bool, err error) { + hash := DigestBodyHash(rule, body) + var existing int64 + err = s.db.QueryRowContext(ctx, + `SELECT id FROM digest_entries WHERE status = ? AND rule = ? AND body_hash = ? LIMIT 1`, + DigestPending, rule, hash).Scan(&existing) + if err == nil { + return existing, true, nil + } + + res, err := s.db.ExecContext(ctx, + `INSERT INTO digest_entries (rule, severity, body, body_hash, status, created_ts, expires_ts) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + rule, severity, body, hash, DigestPending, now.UnixMilli(), expiresAt.UnixMilli()) + if err != nil { + return 0, false, fmt.Errorf("enqueue digest entry: %w", err) + } + id, err = res.LastInsertId() + if err != nil { + return 0, false, fmt.Errorf("enqueue digest entry: last insert id: %w", err) + } + return id, false, nil +} + +// PendingDigestEntries returns the live (not yet expired) pending entries, +// oldest first — the order they were suppressed in, which is also the order +// a bundled readout should mention them. +func (s *Store) PendingDigestEntries(ctx context.Context, now time.Time) ([]DigestEntry, error) { + rows, err := s.db.QueryContext(ctx, + `SELECT id, rule, severity, body, created_ts, expires_ts + FROM digest_entries WHERE status = ? AND expires_ts > ? ORDER BY created_ts ASC`, + DigestPending, now.UnixMilli()) + if err != nil { + return nil, fmt.Errorf("pending digest entries: %w", err) + } + defer rows.Close() + + var out []DigestEntry + for rows.Next() { + var e DigestEntry + var created, expires int64 + if err := rows.Scan(&e.ID, &e.Rule, &e.Severity, &e.Body, &created, &expires); err != nil { + return nil, fmt.Errorf("pending digest entries: scan: %w", err) + } + e.CreatedTs = time.UnixMilli(created) + e.ExpiresTs = time.UnixMilli(expires) + out = append(out, e) + } + return out, rows.Err() +} + +// ExpireStaleDigestEntries marks pending entries whose expires_ts has passed +// as expired — stale information (yesterday's battery warning) is noise, not +// news, so it is dropped rather than delivered late. Called once per tick, +// mirroring ReconcileStaleDeliveryAttempts's "sweep, don't guess" shape. +// Returns the count expired, for logging. +func (s *Store) ExpireStaleDigestEntries(ctx context.Context, now time.Time) (int, error) { + res, err := s.db.ExecContext(ctx, + `UPDATE digest_entries SET status = ? WHERE status = ? AND expires_ts <= ?`, + DigestExpired, DigestPending, now.UnixMilli()) + if err != nil { + return 0, fmt.Errorf("expire stale digest entries: %w", err) + } + n, err := res.RowsAffected() + if err != nil { + return 0, fmt.Errorf("expire stale digest entries: rows affected: %w", err) + } + return int(n), nil +} + +// DrainDigestEntries marks the given entries drained — they were folded into +// a bundle that was successfully dispatched. Called only after a successful +// send, same rule as the delivery outbox: a failed dispatch must not mark +// entries drained, or the bundle is lost along with the failed send. +func (s *Store) DrainDigestEntries(ctx context.Context, ids []int64, now time.Time) error { + if len(ids) == 0 { + return nil + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("drain digest entries: begin: %w", err) + } + defer func() { _ = tx.Rollback() }() + stmt, err := tx.PrepareContext(ctx, + `UPDATE digest_entries SET status = ? WHERE id = ? AND status = ?`) + if err != nil { + return fmt.Errorf("drain digest entries: prepare: %w", err) + } + defer stmt.Close() + for _, id := range ids { + if _, err := stmt.ExecContext(ctx, DigestDrained, id, DigestPending); err != nil { + return fmt.Errorf("drain digest entry %d: %w", id, err) + } + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("drain digest entries: commit: %w", err) + } + return nil +} diff --git a/internal/store/digest_test.go b/internal/store/digest_test.go new file mode 100644 index 0000000..9c78392 --- /dev/null +++ b/internal/store/digest_test.go @@ -0,0 +1,202 @@ +package store + +import ( + "context" + "testing" + "time" +) + +// TestDigestEntryRoundTrips — a suppressed care candidate lands durably and +// comes back out of PendingDigestEntries with its severity and body intact. +func TestDigestEntryRoundTrips(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + now := time.Now() + + id, deduped, err := s.EnqueueDigestEntry(ctx, "break", 2, "ты долго не отдыхала", now, now.Add(24*time.Hour)) + if err != nil { + t.Fatalf("enqueue: %v", err) + } + if deduped { + t.Fatal("first enqueue must not report deduped") + } + if id == 0 { + t.Fatal("want a nonzero id") + } + + entries, err := s.PendingDigestEntries(ctx, now) + if err != nil { + t.Fatalf("pending: %v", err) + } + if len(entries) != 1 || entries[0].ID != id { + t.Fatalf("want 1 pending entry with id %d, got %+v", id, entries) + } + if entries[0].Rule != "break" || entries[0].Severity != 2 || entries[0].Body != "ты долго не отдыхала" { + t.Fatalf("entry contents wrong: %+v", entries[0]) + } +} + +// TestDigestEntrySurvivesRestart — durability is the whole point: a fresh +// Store handle on the same file must see the same pending entry, exactly +// like the delivery outbox's crash-recovery promise. +func TestDigestEntrySurvivesRestart(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + now := time.Now() + + s1, err := Open(ctx, dir+"/m.db") + if err != nil { + t.Fatalf("open: %v", err) + } + id, _, err := s1.EnqueueDigestEntry(ctx, "break", 2, "перерыв", now, now.Add(24*time.Hour)) + if err != nil { + t.Fatalf("enqueue: %v", err) + } + if err := s1.Close(); err != nil { + t.Fatalf("close: %v", err) + } + + // simulated restart: a brand new Store handle on the same file. + s2, err := Open(ctx, dir+"/m.db") + if err != nil { + t.Fatalf("reopen: %v", err) + } + defer func() { _ = s2.Close() }() + + entries, err := s2.PendingDigestEntries(ctx, now) + if err != nil { + t.Fatalf("pending after restart: %v", err) + } + if len(entries) != 1 || entries[0].ID != id { + t.Fatalf("digest entry did not survive restart: %+v", entries) + } +} + +// TestDigestEntryDedupesSameRuleAndBody — the same suppressed nudge +// repeating across ticks (quiet hours holding for hours) must not pile up +// into several copies of itself; he hears it once. +func TestDigestEntryDedupesSameRuleAndBody(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + now := time.Now() + + id1, deduped1, err := s.EnqueueDigestEntry(ctx, "break", 2, "перерыв нужен", now, now.Add(24*time.Hour)) + if err != nil { + t.Fatalf("first enqueue: %v", err) + } + if deduped1 { + t.Fatal("first enqueue should not be deduped") + } + + for i := 0; i < 2; i++ { + id2, deduped2, err := s.EnqueueDigestEntry(ctx, "break", 2, "перерыв нужен", now.Add(time.Minute), now.Add(25*time.Hour)) + if err != nil { + t.Fatalf("repeat enqueue: %v", err) + } + if !deduped2 { + t.Fatal("repeat enqueue of the same rule+body should report deduped") + } + if id2 != id1 { + t.Fatalf("deduped enqueue should return the original id: want %d got %d", id1, id2) + } + } + + entries, err := s.PendingDigestEntries(ctx, now) + if err != nil { + t.Fatalf("pending: %v", err) + } + if len(entries) != 1 { + t.Fatalf("want exactly 1 pending entry after 3 enqueues of the same nudge, got %d", len(entries)) + } +} + +// TestDigestEntryExpiresRatherThanDeliversLate — a stale entry (past its +// expires_ts) must not surface in PendingDigestEntries, and the sweep should +// mark it expired instead of leaving it around to be delivered late. +func TestDigestEntryExpiresRatherThanDeliversLate(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + created := time.Now() + expiresAt := created.Add(time.Hour) + + id, _, err := s.EnqueueDigestEntry(ctx, "water", 1, "стакан воды", created, expiresAt) + if err != nil { + t.Fatalf("enqueue: %v", err) + } + + afterExpiry := expiresAt.Add(time.Minute) + + // even before the sweep runs, a stale entry must not be handed back as + // pending — "not yet swept" must not mean "still deliverable". + entries, err := s.PendingDigestEntries(ctx, afterExpiry) + if err != nil { + t.Fatalf("pending: %v", err) + } + if len(entries) != 0 { + t.Fatalf("stale entry must not be returned as pending, got %+v", entries) + } + + n, err := s.ExpireStaleDigestEntries(ctx, afterExpiry) + if err != nil { + t.Fatalf("expire sweep: %v", err) + } + if n != 1 { + t.Fatalf("want 1 entry expired, got %d", n) + } + + var status DigestStatus + if err := s.db.QueryRowContext(ctx, `SELECT status FROM digest_entries WHERE id = ?`, id).Scan(&status); err != nil { + t.Fatalf("read back: %v", err) + } + if status != DigestExpired { + t.Fatalf("status: want %q, got %q", DigestExpired, status) + } + + // idempotent: a second sweep finds nothing new. + n2, err := s.ExpireStaleDigestEntries(ctx, afterExpiry.Add(time.Hour)) + if err != nil { + t.Fatalf("second sweep: %v", err) + } + if n2 != 0 { + t.Fatalf("second sweep should find nothing, got %d", n2) + } +} + +// TestDigestEntryDrainMarksDrainedNotDeleted — draining is bookkeeping, not +// deletion: the row survives as an audit trail of what she actually said. +func TestDigestEntryDrainMarksDrainedNotDeleted(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + now := time.Now() + + id1, _, err := s.EnqueueDigestEntry(ctx, "break", 2, "перерыв", now, now.Add(24*time.Hour)) + if err != nil { + t.Fatalf("enqueue 1: %v", err) + } + id2, _, err := s.EnqueueDigestEntry(ctx, "break2", 2, "другое", now, now.Add(24*time.Hour)) + if err != nil { + t.Fatalf("enqueue 2: %v", err) + } + + if err := s.DrainDigestEntries(ctx, []int64{id1, id2}, now.Add(time.Hour)); err != nil { + t.Fatalf("drain: %v", err) + } + + entries, err := s.PendingDigestEntries(ctx, now.Add(time.Hour)) + if err != nil { + t.Fatalf("pending: %v", err) + } + if len(entries) != 0 { + t.Fatalf("drained entries must not still be pending, got %+v", entries) + } + + for _, id := range []int64{id1, id2} { + var status DigestStatus + if err := s.db.QueryRowContext(ctx, `SELECT status FROM digest_entries WHERE id = ?`, id).Scan(&status); err != nil { + t.Fatalf("read back %d: %v", id, err) + } + if status != DigestDrained { + t.Fatalf("entry %d status: want %q, got %q", id, DigestDrained, status) + } + } +} diff --git a/internal/store/ecotraces.go b/internal/store/ecotraces.go new file mode 100644 index 0000000..c39a953 --- /dev/null +++ b/internal/store/ecotraces.go @@ -0,0 +1,107 @@ +package store + +import ( + "context" + "encoding/json" + "fmt" + "time" +) + +// ecosystemTraceRetention is how many trace rows are kept. Traces are +// diagnostics with a short useful life, and they arrive at machine rate, so +// the table is bounded rather than append-only. The facts table is the audit +// trail; this one is not. +const ecosystemTraceRetention = 5000 + +// EcosystemTrace is one hop of a cross-service call: which service, which +// operation, how it ended, how long it took, and the ids that stitch the hops +// of one turn together. Fields carries the hop-specific detail (entity id, +// capability, failure class) as a JSON object. +type EcosystemTrace struct { + ID int64 `json:"id"` + Ts time.Time `json:"ts"` + Service string `json:"service"` + Operation string `json:"operation"` + Status string `json:"status"` + DurationMs int64 `json:"duration_ms"` + CorrelationID string `json:"correlation_id"` + CausationID string `json:"causation_id"` + HTTPStatus int `json:"http_status"` + Fields map[string]any `json:"fields"` +} + +// WriteEcosystemTrace appends one trace row and keeps the table bounded. +func (s *Store) WriteEcosystemTrace(ctx context.Context, tr EcosystemTrace) (int64, error) { + fields := "{}" + if len(tr.Fields) > 0 { + b, err := json.Marshal(tr.Fields) + if err != nil { + return 0, fmt.Errorf("marshal trace fields: %w", err) + } + fields = string(b) + } + res, err := s.db.ExecContext(ctx, ` + INSERT INTO ecosystem_traces + (ts, service, operation, status, duration_ms, correlation_id, causation_id, http_status, fields) + VALUES (?,?,?,?,?,?,?,?,?)`, + tr.Ts.UnixMilli(), tr.Service, tr.Operation, tr.Status, tr.DurationMs, + tr.CorrelationID, tr.CausationID, tr.HTTPStatus, fields) + if err != nil { + return 0, fmt.Errorf("write ecosystem trace: %w", err) + } + id, err := res.LastInsertId() + if err != nil { + return 0, fmt.Errorf("last insert id: %w", err) + } + // Prune rarely: the cost of the delete is not worth paying on every hop, + // and the bound is a ceiling, not an exact size. + if id%256 == 0 { + if err := s.PruneEcosystemTraces(ctx, ecosystemTraceRetention); err != nil { + return id, err + } + } + return id, nil +} + +// PruneEcosystemTraces drops all but the newest keep rows. +func (s *Store) PruneEcosystemTraces(ctx context.Context, keep int) error { + if keep <= 0 { + return nil + } + _, err := s.db.ExecContext(ctx, ` + DELETE FROM ecosystem_traces + WHERE id <= (SELECT MAX(id) FROM ecosystem_traces) - ?`, keep) + if err != nil { + return fmt.Errorf("prune ecosystem traces: %w", err) + } + return nil +} + +// RecentEcosystemTraces returns the newest n traces, newest first. +func (s *Store) RecentEcosystemTraces(ctx context.Context, n int) ([]EcosystemTrace, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT id, ts, service, operation, status, duration_ms, correlation_id, causation_id, http_status, fields + FROM ecosystem_traces + ORDER BY id DESC + LIMIT ?`, n) + if err != nil { + return nil, fmt.Errorf("recent ecosystem traces: %w", err) + } + defer rows.Close() + var out []EcosystemTrace + for rows.Next() { + var tr EcosystemTrace + var tsMilli int64 + var fields string + if err := rows.Scan(&tr.ID, &tsMilli, &tr.Service, &tr.Operation, &tr.Status, + &tr.DurationMs, &tr.CorrelationID, &tr.CausationID, &tr.HTTPStatus, &fields); err != nil { + return nil, err + } + tr.Ts = time.UnixMilli(tsMilli).UTC() + if fields != "" { + _ = json.Unmarshal([]byte(fields), &tr.Fields) + } + out = append(out, tr) + } + return out, rows.Err() +} diff --git a/internal/store/events.go b/internal/store/events.go index e3dd66a..f40d4e1 100644 --- a/internal/store/events.go +++ b/internal/store/events.go @@ -35,6 +35,36 @@ func (s *Store) CreateEvent(ctx context.Context, factID int64, action, object st return id, nil } +// EventPair identifies one action+object grouping in the events table — the +// unit the pattern detector reasons about. +type EventPair struct { + Action string + Object string +} + +// DistinctEventPairs returns every distinct action+object pair that has at +// least one event, in no particular order. This is what lets the proactive +// digestion tick run the pattern detector over everything accumulated so far +// instead of only the pair touched by the utterance that just landed +// (Vikunja #43) — the tick has no "current utterance," so it has to ask the +// store what to look at. +func (s *Store) DistinctEventPairs(ctx context.Context) ([]EventPair, error) { + rows, err := s.db.QueryContext(ctx, `SELECT DISTINCT action, object FROM events`) + if err != nil { + return nil, fmt.Errorf("distinct event pairs: %w", err) + } + defer rows.Close() + var out []EventPair + for rows.Next() { + var p EventPair + if err := rows.Scan(&p.Action, &p.Object); err != nil { + return nil, err + } + out = append(out, p) + } + return out, rows.Err() +} + // EventsFor returns all events matching action+object, ordered by ts ascending // (oldest first — the order the pattern detector needs for interval computation). func (s *Store) EventsFor(ctx context.Context, action, object string) ([]Event, error) { diff --git a/internal/store/facts.go b/internal/store/facts.go index 5388eb2..ffdfd43 100644 --- a/internal/store/facts.go +++ b/internal/store/facts.go @@ -6,7 +6,10 @@ import ( "encoding/json" "errors" "fmt" + "strings" "time" + + "github.com/kami/maven/internal/calendar" ) // WriteFact appends a fact row. confidence must be 1.0 for taps and (0,1) for @@ -79,17 +82,76 @@ func (s *Store) RecentFacts(ctx context.Context, n int) ([]Fact, error) { return out, rows.Err() } -// CalendarEvents returns caldav facts whose key date falls within [from, to). +// RecentActiveFactsByKind — the newest n facts of one kind, newest first, with +// the retracted ones left out. "Active" means two exclusions: a row some later +// row voids, and the void marker VoidLatestFact writes to retract it. A +// correction still counts, because a correction is a value he stands behind. +// +// It exists because the facts table is shared and the noisy writers are not the +// interesting ones. A caller that reads n recent rows and then keeps the self +// ones has a window whose real length is set by how often the pollers write: +// one WireGuard peer alone rehandshakes every couple of minutes, which is +// enough env rows to reduce a 2000-row window to under three days. Filtering in +// SQL makes the bound mean what the caller thinks it means. +// +// Voided rows are excluded here and included by RecentFacts on purpose. The +// dash shows the audit trail, because you want to SEE a correction. A reader +// that asks what he usually does must not count something he explicitly took +// back. +func (s *Store) RecentActiveFactsByKind(ctx context.Context, kind FactKind, n int) ([]Fact, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT id, ts, kind, key, value, source, confidence, voids_id + FROM facts + WHERE kind = ? + AND NOT (voids_id IS NOT NULL AND value = '"voided"') + AND id NOT IN (SELECT voids_id FROM facts WHERE voids_id IS NOT NULL) + ORDER BY ts DESC, id DESC LIMIT ?`, string(kind), n) + if err != nil { + return nil, fmt.Errorf("recent facts by kind: %w", err) + } + defer rows.Close() + var out []Fact + for rows.Next() { + f, err := scanFact(rows) + if err != nil { + return nil, err + } + out = append(out, f) + } + return out, rows.Err() +} + +// CalendarEvents returns calendar facts whose key date falls within [from, to). // Calendar event keys have the format calendar_event_YYYYMMDD_. +// +// Every calendar source is included, not just the personal CalDAV poll: the work +// calendar arrives as ambient:notif notifications (Vikunja #126) and belongs in +// the same answer. The source stays on each Fact, along with its confidence, so +// the caller can hedge a reading it did not get from a calendar server — +// filtering by source here would have thrown that judgement away. +// +// One row per event, not one per write. The facts table is append-only, so a +// standup moved from 14:00 to 16:00 leaves two rows under the same key, and the +// day plan used to recite both as if the owner had two meetings. Voided rows +// are excluded, the latest row wins within a source, and the best-evidenced +// source wins across them — a calendar read beats the notification relay that +// guessed at the same meeting. func (s *Store) CalendarEvents(ctx context.Context, from, to time.Time) ([]Fact, error) { - prefixFrom := fmt.Sprintf("calendar_event_%s", from.Format("20060102")) - prefixTo := fmt.Sprintf("calendar_event_%s", to.Format("20060102")) + prefixFrom := calendar.KeyPrefixForDay(from) + prefixTo := calendar.KeyPrefixForDay(to) + sources := calendar.Sources() + args := make([]any, 0, len(sources)+2) + for _, src := range sources { + args = append(args, src) + } + args = append(args, prefixFrom, prefixTo) rows, err := s.db.QueryContext(ctx, ` SELECT id, ts, kind, key, value, source, confidence, voids_id FROM facts - WHERE source = 'poll:caldav' + WHERE source IN (`+placeholders(len(sources))+`) AND key >= ? AND key < ? - ORDER BY key`, prefixFrom, prefixTo) + AND id NOT IN (SELECT voids_id FROM facts WHERE voids_id IS NOT NULL) + ORDER BY key, ts, id`, args...) if err != nil { return nil, fmt.Errorf("calendar events: %w", err) } @@ -102,7 +164,46 @@ func (s *Store) CalendarEvents(ctx context.Context, from, to time.Time) ([]Fact, } out = append(out, f) } - return out, rows.Err() + if err := rows.Err(); err != nil { + return nil, err + } + return latestPerCalendarKey(out), nil +} + +// latestPerCalendarKey reduces the append-only rows for one day to one row per +// event key. Input must be ordered by key then oldest-first, so the last row +// seen for a key and source is that source's current value. +func latestPerCalendarKey(in []Fact) []Fact { + type slot struct { + bySource map[string]Fact + order []string + } + var keys []string + byKey := map[string]*slot{} + for _, f := range in { + s, ok := byKey[f.Key] + if !ok { + s = &slot{bySource: map[string]Fact{}} + byKey[f.Key] = s + keys = append(keys, f.Key) + } + if _, seen := s.bySource[f.Source]; !seen { + s.order = append(s.order, f.Source) + } + s.bySource[f.Source] = f + } + out := make([]Fact, 0, len(keys)) + for _, k := range keys { + s := byKey[k] + best := s.bySource[s.order[0]] + for _, src := range s.order[1:] { + if s.bySource[src].Confidence > best.Confidence { + best = s.bySource[src] + } + } + out = append(out, best) + } + return out } // LatestFactBySource — provenance-scoped. A rule on `service_down` trusts only @@ -254,3 +355,8 @@ func scanFact(r rowScanner) (Fact, error) { f.VoidsID = voids return f, nil } + +// placeholders renders n comma-separated SQL bind markers. +func placeholders(n int) string { + return strings.TrimSuffix(strings.Repeat("?,", n), ",") +} diff --git a/internal/store/memory.go b/internal/store/memory.go index f2b43b1..bc83648 100644 --- a/internal/store/memory.go +++ b/internal/store/memory.go @@ -8,6 +8,7 @@ import ( "fmt" "math" "sort" + "strings" "time" "github.com/kami/maven/internal/memory" @@ -37,8 +38,10 @@ func (s *Store) VectorMemory() *MemoryStore { return &MemoryStore{db: s.db} } -// compile-time check: MemoryStore satisfies the memory.Store interface. +// compile-time check: MemoryStore satisfies the memory.Store interface, and the +// wider Catalog that speaker profiles need (enumerate by prefix, delete by id). var _ memory.Store = (*MemoryStore)(nil) +var _ memory.Catalog = (*MemoryStore)(nil) // Insert upserts a vector by id: a repeated id replaces the prior row rather // than accumulating duplicates (the note/fact ids are stable and unique, so a @@ -61,11 +64,17 @@ func (m *MemoryStore) Insert(ctx context.Context, id string, vec []float32, meta // Search returns the topK nearest rows by cosine similarity. A full scan; see // the type doc for why that's fine at this scale. +// +// Rows under memory.NonRecallPrefix are excluded in SQL. They are speaker +// voiceprints sharing this table, and note recall must not rank them; see that +// constant for why the previous arrangement only appeared to do this. func (m *MemoryStore) Search(ctx context.Context, vec []float32, topK int) ([]memory.Result, error) { if topK <= 0 { topK = 10 } - rows, err := m.db.QueryContext(ctx, `SELECT id, vec, meta FROM memory_vectors`) + rows, err := m.db.QueryContext(ctx, + `SELECT id, vec, meta FROM memory_vectors WHERE id NOT LIKE ? ESCAPE '\'`, + escapeLike(memory.NonRecallPrefix)+"%") if err != nil { return nil, fmt.Errorf("memory: scan: %w", err) } @@ -95,6 +104,56 @@ func (m *MemoryStore) Search(ctx context.Context, vec []float32, topK int) ([]me return out, nil } +// ByPrefix returns every row whose id starts with prefix, vectors included. +// +// This is not a similarity query and deliberately does not score anything: +// listing the enrolled voices is a question about which rows exist, and asking +// it through Search would mean inventing a query vector to rank them by. The +// prefix is matched with LIKE against an escaped pattern, so a profile id +// containing % or _ cannot widen the match. +func (m *MemoryStore) ByPrefix(ctx context.Context, prefix string) ([]memory.Record, error) { + pattern := escapeLike(prefix) + "%" + rows, err := m.db.QueryContext(ctx, + `SELECT id, vec, meta FROM memory_vectors WHERE id LIKE ? ESCAPE '\'`, pattern) + if err != nil { + return nil, fmt.Errorf("memory: by prefix %q: %w", prefix, err) + } + defer rows.Close() + + var out []memory.Record + for rows.Next() { + var id, metaJSON string + var blob []byte + if err := rows.Scan(&id, &blob, &metaJSON); err != nil { + return nil, fmt.Errorf("memory: row: %w", err) + } + meta := map[string]string{} + if err := json.Unmarshal([]byte(metaJSON), &meta); err != nil { + return nil, fmt.Errorf("memory: unmarshal meta for %q: %w", id, err) + } + out = append(out, memory.Record{ID: id, Vec: decodeVec(blob), Meta: meta}) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("memory: rows: %w", err) + } + return out, nil +} + +// Delete removes one vector by id. A row that is not there is not an error — +// "forget this voice" is satisfied either way. +func (m *MemoryStore) Delete(ctx context.Context, id string) error { + if _, err := m.db.ExecContext(ctx, `DELETE FROM memory_vectors WHERE id = ?`, id); err != nil { + return fmt.Errorf("memory: delete %q: %w", id, err) + } + return nil +} + +// escapeLike neutralises the LIKE wildcards in a literal prefix. +func escapeLike(s string) string { + r := strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`) + return r.Replace(s) +} + // encodeVec serializes a float32 slice as little-endian IEEE-754 bytes (4 bytes // per element) for the BLOB column. func encodeVec(v []float32) []byte { diff --git a/internal/store/memory_test.go b/internal/store/memory_test.go index 3e57243..8ba2b09 100644 --- a/internal/store/memory_test.go +++ b/internal/store/memory_test.go @@ -3,7 +3,10 @@ package store import ( "context" "path/filepath" + "strings" "testing" + + "github.com/kami/maven/internal/memory" ) func newMemTestStore(t *testing.T) *Store { @@ -101,3 +104,38 @@ func TestMemoryStorePersistsAcrossReopen(t *testing.T) { t.Fatalf("memory did not survive reopen: %v", got) } } + +// A voiceprint sharing the vector table must never rank as a note hit. The +// dimensions match here on purpose: what used to hide these rows was cosine +// returning 0 on a width mismatch, which is a property of two model choices and +// not of the store. +func TestMemoryStoreSearchSkipsVoiceprints(t *testing.T) { + ctx := context.Background() + m := newMemTestStore(t).VectorMemory() + + if err := m.Insert(ctx, "note:1", []float32{0, 1, 0}, map[string]string{"text": "заметка"}); err != nil { + t.Fatal(err) + } + if err := m.Insert(ctx, memory.NonRecallPrefix+"kami", []float32{1, 0, 0}, map[string]string{"name": "Ками"}); err != nil { + t.Fatal(err) + } + + got, err := m.Search(ctx, []float32{1, 0, 0}, 10) + if err != nil { + t.Fatalf("Search: %v", err) + } + for _, r := range got { + if strings.HasPrefix(r.ID, memory.NonRecallPrefix) { + t.Fatalf("recall returned a voiceprint: %+v", r) + } + } + if len(got) != 1 || got[0].ID != "note:1" { + t.Fatalf("Search = %+v, want just the note", got) + } + + // The row is still there for the speaker code that owns it. + recs, err := m.ByPrefix(ctx, memory.NonRecallPrefix) + if err != nil || len(recs) != 1 { + t.Fatalf("ByPrefix = %+v, %v; want the voiceprint", recs, err) + } +} diff --git a/internal/store/migrations.go b/internal/store/migrations.go index 8bc86eb..a266efe 100644 --- a/internal/store/migrations.go +++ b/internal/store/migrations.go @@ -112,6 +112,102 @@ ALTER TABLE reminders ADD COLUMN next_fire_ts INTEGER;`, // #2 DROP TABLE delivery_attempts; ALTER TABLE delivery_attempts_v12 RENAME TO delivery_attempts; CREATE INDEX IF NOT EXISTS idx_delivery_attempts_status ON delivery_attempts (status);`, + + // #13 — durable digest outbox (Vikunja #281). A care nudge the restraint + // gate suppresses (quiet hours / away / calendar-busy) is not necessarily + // lost: if it's worth resurfacing, it lands here instead, and gets spoken + // as one bundle at the next moment speaking is appropriate. body_hash + // dedupes repeat suppressions of the "same" nudge; expires_ts bounds how + // stale an entry may get before it's worthless and must be dropped rather + // than delivered late. + `CREATE TABLE IF NOT EXISTS digest_entries ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + rule TEXT NOT NULL, + severity INTEGER NOT NULL, + body TEXT NOT NULL, + body_hash TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending','drained','expired')), + created_ts INTEGER NOT NULL, + expires_ts INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_digest_entries_status ON digest_entries (status);`, + + // #14 — the task capture store (Vikunja #130). Deliberately NOT facts: + // a fact is a claim about the world that gets superseded, a task is a + // piece of work with a lifecycle (captured → open → done), and the + // prioritiser needs to read the live set cheaply. + // + // status: 'candidate' is a task Maven derived from something she read + // (mail, later) and has NOT been confirmed by the owner; 'open' is a task + // he actually stated (or confirmed). Nothing schedules or announces off + // this table — capture is not a nag. + // + // norm is the normalised dedupe key. The unique index is PARTIAL, over + // live rows only: re-capturing "купить молоко" after last week's one is + // done must work, while the same mail arriving twice must not produce two + // rows. + `CREATE TABLE IF NOT EXISTS tasks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + created_ts INTEGER NOT NULL, + text TEXT NOT NULL, + norm TEXT NOT NULL, + source TEXT NOT NULL, + evidence TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL DEFAULT 'open' CHECK (status IN ('candidate','open','done','dropped')), + due_ts INTEGER, + weight INTEGER NOT NULL DEFAULT 0, + resolved_ts INTEGER + ); + CREATE UNIQUE INDEX IF NOT EXISTS idx_tasks_live_norm ON tasks (norm) WHERE status IN ('candidate','open'); + CREATE INDEX IF NOT EXISTS idx_tasks_status ON tasks (status, created_ts DESC);`, + + // #15 — external identity and resolution attribution for tasks. + // + // ext_id is the identity of the thing a derived task was extracted FROM + // (message id plus the extracted span), and its unique index covers EVERY + // row, not just the live ones. The live-only norm index is right for + // voice, where him saying the errand again is the recurrence signal. It is + // wrong for a mailbox: mavmaild is a read-only reader, nothing marks a + // message read, so a task he already finished would be re-extracted from + // the same immutable text on the next poll and land back on his list as a + // fresh candidate, forever. + // + // resolved_by records which caller moved the task. resolved_ts said when + // and never by what, so a wrong resolution left no trace at all. + `ALTER TABLE tasks ADD COLUMN ext_id TEXT; + ALTER TABLE tasks ADD COLUMN resolved_by TEXT NOT NULL DEFAULT ''; + CREATE UNIQUE INDEX IF NOT EXISTS idx_tasks_ext_id ON tasks (ext_id) WHERE ext_id IS NOT NULL;`, + // #16 — ecosystem call traces (Vikunja #273). Deliberately NOT facts. + // Traces are written at machine rate, one act turn produces three or four, + // while facts are written at human rate. Sharing the facts table made every + // bounded reader of facts (the habit profile's 2000-row window, memeval's + // prompt snapshot, /dash's 50 and /history's 200) read mostly traces after + // a day of ecosystem use, pushing the rows that matter out of range. + // Retention is enforced on write (PruneEcosystemTraces) because nothing + // here is an audit trail: a trace answers "did this hop work" for as long + // as anyone is still asking. + `CREATE TABLE IF NOT EXISTS ecosystem_traces ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts INTEGER NOT NULL, + service TEXT NOT NULL, + operation TEXT NOT NULL, + status TEXT NOT NULL, + duration_ms INTEGER NOT NULL DEFAULT 0, + correlation_id TEXT NOT NULL DEFAULT '', + causation_id TEXT NOT NULL DEFAULT '', + http_status INTEGER NOT NULL DEFAULT 0, + fields TEXT NOT NULL DEFAULT '{}' + ); + CREATE INDEX IF NOT EXISTS idx_eco_traces_ts ON ecosystem_traces (ts DESC); + CREATE INDEX IF NOT EXISTS idx_eco_traces_correlation ON ecosystem_traces (correlation_id);`, + `ALTER TABLE tools ADD COLUMN fingerprint TEXT NOT NULL DEFAULT '';`, + // #17 — what a discovered tool WAS when it was approved (Vikunja #251). + // An MCP row's cmd is ["mcp", server, tool], which is a late-bound + // reference: it names a tool on a server the remote end owns and it pins + // no behaviour at all. A server upgraded, or taken over, can redefine + // list_tasks into something that writes without the row changing by one + // byte. The fingerprint is the declared shape at approval time, so a + // redefinition is a re-approval instead of a silent upgrade. } // migrate applies every migration with a number greater than the DB's current diff --git a/internal/store/notes.go b/internal/store/notes.go index b2b4179..07b16cd 100644 --- a/internal/store/notes.go +++ b/internal/store/notes.go @@ -6,6 +6,7 @@ import ( "fmt" "math" "sort" + "strings" "time" ) @@ -42,7 +43,8 @@ func (s *Store) WriteNote(ctx context.Context, ts time.Time, text string, embedd // or an ANN index only when note count or latency actually bites — at personal // scale (hundreds–thousands) a full scan is sub-millisecond. func (s *Store) QueryNotes(ctx context.Context, embedding []float32, k int) ([]Note, error) { - rows, err := s.db.QueryContext(ctx, `SELECT id, ts, text, embedding, source FROM notes`) + rows, err := s.db.QueryContext(ctx, + `SELECT id, ts, text, embedding, source FROM notes WHERE `+notHisWordsSQL) if err != nil { return nil, fmt.Errorf("query notes: %w", err) } @@ -76,12 +78,88 @@ func (s *Store) QueryNotes(ctx context.Context, embedding []float32, k int) ([]N return out, nil } +// ReadSourcePrefixes — note sources that are text Maven READ somewhere, not +// text he said or wrote: RSS items (internal/rss) and crawled pages +// (internal/crawl). +// +// They are notes because that is where fetched text lands, and they are kept out +// of recall because recall answers questions about HIM. "что я говорил про +// переезд" must not be answered out of a stranger's web page that happened to +// land near in the vector space, and the fallback line for that is "вот что я +// нашла: " followed by the stranger's words. Ask for them by source instead — +// that is what RecentNotesFromSource is for. +var ReadSourcePrefixes = []string{"rss:", "crawl:"} + +// notHisWordsSQL — the WHERE clause that drops the read sources. Built from the +// list above so adding a source is one line. +var notHisWordsSQL = buildNotHisWordsSQL() + +func buildNotHisWordsSQL() string { + var b strings.Builder + b.WriteString("(source IS NULL OR (") + for i, p := range ReadSourcePrefixes { + if i > 0 { + b.WriteString(" AND ") + } + fmt.Fprintf(&b, "source NOT LIKE '%s%%'", p) + } + b.WriteString("))") + return b.String() +} + +// RecentNotesFromSource returns the newest n notes whose source starts with +// prefix, newest first. The answer path for feeds and watches uses it: scanning +// the last 200 notes of ANY source meant a busy day of voice notes pushed the +// newest headline out of the window, and she said "в лентах пока ничего нового" +// while the poller was working fine. +func (s *Store) RecentNotesFromSource(ctx context.Context, prefix string, n int) ([]Note, error) { + rows, err := s.db.QueryContext(ctx, + `SELECT id, ts, text, source FROM notes WHERE source LIKE ? ORDER BY ts DESC LIMIT ?`, + prefix+"%", n) + if err != nil { + return nil, fmt.Errorf("recent notes by source: %w", err) + } + defer rows.Close() + var out []Note + for rows.Next() { + var nt Note + var tsMilli int64 + if err := rows.Scan(&nt.ID, &tsMilli, &nt.Text, &nt.Source); err != nil { + return nil, err + } + nt.Ts = time.UnixMilli(tsMilli).UTC() + out = append(out, nt) + } + return out, rows.Err() +} + // RecentNotes returns the newest n notes, newest first — a browse view (no // embedding math; Score stays 0). This is the read surface for /dash: notes // captured by voice are otherwise only reachable through semantic query. func (s *Store) RecentNotes(ctx context.Context, n int) ([]Note, error) { + return s.recentNotesWhere(ctx, "", n) +} + +// RecentNotesBySource returns the newest n notes written by one source, newest +// first. The filter is in SQL, not in the caller, because a caller that reads n +// rows and then keeps the ones it wants has a window measured in OTHER writers' +// traffic: once n newer notes from anywhere have landed, the rows it was +// looking for are gone. Anything that needs "the last n of mine" wants this. +func (s *Store) RecentNotesBySource(ctx context.Context, source string, n int) ([]Note, error) { + return s.recentNotesWhere(ctx, "WHERE source = ?", n, source) +} + +// RecentNotesExcludingSource returns the newest n notes NOT written by source. +// Same reasoning inverted: a reader that wants n notes he wrote must not have +// its budget eaten by rows it is about to discard. +func (s *Store) RecentNotesExcludingSource(ctx context.Context, source string, n int) ([]Note, error) { + return s.recentNotesWhere(ctx, "WHERE source <> ?", n, source) +} + +func (s *Store) recentNotesWhere(ctx context.Context, where string, n int, args ...any) ([]Note, error) { + args = append(args, n) rows, err := s.db.QueryContext(ctx, - `SELECT id, ts, text, source FROM notes ORDER BY ts DESC LIMIT ?`, n) + `SELECT id, ts, text, source FROM notes `+where+` ORDER BY ts DESC LIMIT ?`, args...) if err != nil { return nil, fmt.Errorf("recent notes: %w", err) } diff --git a/internal/store/notes_test.go b/internal/store/notes_test.go index 068eb40..df7efa8 100644 --- a/internal/store/notes_test.go +++ b/internal/store/notes_test.go @@ -36,3 +36,43 @@ func TestQueryNotesRanksByCosine(t *testing.T) { t.Errorf("scores not descending: %.3f then %.3f", got[0].Score, got[1].Score) } } + +// Recall answers questions about HIM. A feed item and a crawled page are text +// Maven read somewhere, and letting them into the nearest-neighbour pool means +// "что я говорил про переезд" can be answered with a stranger's sentence, under +// the line "вот что я нашла: ". +func TestQueryNotesLeavesOutWhatSheOnlyRead(t *testing.T) { + ctx := context.Background() + st := newTestStore(t) + now := time.Now() + // The read sources sit exactly on the query vector; his own note is further + // away, so ranking alone would put them first. + for _, n := range []struct{ text, source string }{ + {"переезд в новую квартиру описан тут", "rss:habr"}, + {"страница про переезд", "crawl:changelog"}, + } { + if _, err := st.WriteNote(ctx, now, n.text, []float32{1, 0, 0}, n.source); err != nil { + t.Fatal(err) + } + } + if _, err := st.WriteNote(ctx, now, "переезд в субботу", []float32{0.8, 0.6, 0}, "tap:voice"); err != nil { + t.Fatal(err) + } + + got, err := st.QueryNotes(ctx, []float32{1, 0, 0}, 5) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].Source != "tap:voice" { + t.Fatalf("recall returned %+v; want only what he said himself", got) + } + + // They are still reachable, by source. + feed, err := st.RecentNotesFromSource(ctx, "rss:", 10) + if err != nil { + t.Fatal(err) + } + if len(feed) != 1 || feed[0].Source != "rss:habr" { + t.Fatalf("RecentNotesFromSource(rss:) = %+v; want the one feed note", feed) + } +} diff --git a/internal/store/proposed_routines.go b/internal/store/proposed_routines.go index 741fd0a..67e43f5 100644 --- a/internal/store/proposed_routines.go +++ b/internal/store/proposed_routines.go @@ -51,9 +51,10 @@ var ( // keep finding the pattern, and every re-propose is refused here. Maven is not // a nag. // -// TODO(vikunja#46): the detector currently only writes here from the voice -// path. Once digestion runs the detector on its own tick, that tick should -// call this too, so a pattern gets noticed even with nobody at the mic. +// Vikunja #43: this is called both from the voice fact-write path (for the +// immediate spoken confirmation) and from the digestion tick's proactive +// scan (cmd/mavend/tick.go's detectPatterns, via patterns.go's +// detectAndPropose), so a pattern gets noticed even with nobody at the mic. func (s *Store) CreateProposedRoutine(ctx context.Context, action, object string, intervalDays float64, ts time.Time) (int64, error) { res, err := s.db.ExecContext(ctx, `INSERT INTO proposed_routines (action, object, interval_days, status, created_ts) diff --git a/internal/store/reminders.go b/internal/store/reminders.go index 2fb4447..e3788c5 100644 --- a/internal/store/reminders.go +++ b/internal/store/reminders.go @@ -26,6 +26,15 @@ type Reminder struct { Collapsed []Reminder } +// Reminder lifecycle states. Named for the same reason DigestStatus is: a +// caller filtering on the string literal "pending" is one typo away from a +// filter that silently matches nothing. +const ( + ReminderPending = "pending" + ReminderFired = "fired" + ReminderCancelled = "cancelled" +) + var ( ErrReminderNotFound = errors.New("store: reminder not found") ErrReminderState = errors.New("store: reminder not in a mutable state") @@ -93,6 +102,36 @@ func (s *Store) DueReminders(ctx context.Context, now time.Time) ([]Reminder, er return out, rows.Err() } +// PendingReminders returns the pending reminders whose next fire time falls in +// [from, to), earliest first. +// +// The day plan used to take the newest 500 rows out of ListReminders, which +// orders by creation, and then filter them by day. A reminder stated long ago +// for today fell off the end of that scan while a reminder stated this morning +// for next year stayed on it. Bounding by fire time drops what is out of range +// instead of what is old. +func (s *Store) PendingReminders(ctx context.Context, from, to time.Time) ([]Reminder, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT id, created_ts, fire_ts, next_fire_ts, payload, status, cron + FROM reminders + WHERE status = ? AND next_fire_ts >= ? AND next_fire_ts < ? + ORDER BY next_fire_ts ASC, id ASC`, + ReminderPending, from.UnixMilli(), to.UnixMilli()) + if err != nil { + return nil, fmt.Errorf("pending reminders: %w", err) + } + defer rows.Close() + var out []Reminder + for rows.Next() { + r, err := scanReminder(rows) + if err != nil { + return nil, err + } + out = append(out, r) + } + return out, rows.Err() +} + // MarkReminder sets a reminder's status. Only valid transitions: pending→fired, // pending→cancelled. Anything else is a programming error. func (s *Store) MarkReminder(ctx context.Context, id int64, status string) error { diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 3e915b2..cdb684b 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -8,6 +8,8 @@ import ( "path/filepath" "testing" "time" + + "github.com/kami/maven/internal/calendar" ) func newTestStore(t *testing.T) *Store { @@ -378,3 +380,153 @@ func TestCalendarEvents(t *testing.T) { t.Fatalf("expected 0 events on July 8, got %d", len(events)) } } + +// A rescheduled meeting keeps its key and appends a row. The query must return +// the current value, not the history: reciting both told the owner he had two +// standups when one had been moved. +func TestCalendarEventsReturnsOneRowPerEvent(t *testing.T) { + store := newTestStore(t) + defer store.Close() + + ctx := context.Background() + day := time.Date(2026, 8, 3, 0, 0, 0, 0, time.UTC) + const key = "calendar_event_20260803_Standup" + + store.WriteFact(ctx, day.Add(14*time.Hour), KindEnv, key, + `"Standup @ 14:00-14:30"`, calendar.SourcePersonal, 1.0, sql.NullInt64{}) + store.WriteFact(ctx, day.Add(16*time.Hour), KindEnv, key, + `"Standup @ 16:00-16:30"`, calendar.SourcePersonal, 1.0, sql.NullInt64{}) + // The notification relay guessed at the same meeting. A calendar read is + // better evidence, so the hedged row must not displace it. + store.WriteFact(ctx, day.Add(17*time.Hour), KindEnv, key, + `"Standup @ 17:00-17:30"`, calendar.SourceAmbient, calendar.AmbientConfidence, sql.NullInt64{}) + + events, err := store.CalendarEvents(ctx, day, day.AddDate(0, 0, 1)) + if err != nil { + t.Fatalf("CalendarEvents: %v", err) + } + if len(events) != 1 { + t.Fatalf("got %d rows, want the current one only: %+v", len(events), events) + } + if events[0].Value != `"Standup @ 16:00-16:30"` { + t.Errorf("value = %q, want the latest calendar read", events[0].Value) + } +} + +// A voided calendar fact is gone, not history to recite. +func TestCalendarEventsSkipsVoidedRows(t *testing.T) { + store := newTestStore(t) + defer store.Close() + + ctx := context.Background() + day := time.Date(2026, 8, 3, 0, 0, 0, 0, time.UTC) + id, err := store.WriteFact(ctx, day.Add(14*time.Hour), KindEnv, "calendar_event_20260803_Cancelled", + `"Cancelled @ 14:00-14:30"`, calendar.SourcePersonal, 1.0, sql.NullInt64{}) + if err != nil { + t.Fatalf("WriteFact: %v", err) + } + if _, err := store.WriteFact(ctx, day.Add(15*time.Hour), KindEnv, "calendar_event_20260803_Cancelled", + `"Cancelled @ 14:00-14:30"`, calendar.SourcePersonal, 1.0, sql.NullInt64{Int64: id, Valid: true}); err != nil { + t.Fatalf("WriteFact void: %v", err) + } + + events, err := store.CalendarEvents(ctx, day, day.AddDate(0, 0, 1)) + if err != nil { + t.Fatalf("CalendarEvents: %v", err) + } + for _, e := range events { + if e.ID == id { + t.Fatalf("voided row %d came back: %+v", id, e) + } + } +} + +// The work calendar arrives as relayed phone notifications, not a CalDAV read +// (Vikunja #126). Those events belong in the same day's answer, and their +// provenance has to survive the query so the caller can hedge them. +func TestCalendarEventsIncludesAmbientSource(t *testing.T) { + store := newTestStore(t) + defer store.Close() + + ctx := context.Background() + day := time.Date(2026, 8, 3, 0, 0, 0, 0, time.UTC) + + store.WriteFact(ctx, day.Add(10*time.Hour), KindEnv, "calendar_event_20260803_Aaa-personal", + `"Aaa personal @ 10:00-10:30"`, calendar.SourcePersonal, 1.0, sql.NullInt64{}) + store.WriteFact(ctx, day.Add(14*time.Hour), KindEnv, "calendar_event_20260803_Bbb-work", + `"Bbb work @ 14:00-14:30"`, calendar.SourceAmbient, calendar.AmbientConfidence, sql.NullInt64{}) + // A fact that merely looks like one must still be excluded by source. + store.WriteFact(ctx, day.Add(16*time.Hour), KindEnv, "calendar_event_20260803_Ccc-forged", + `"Ccc forged @ 16:00-16:30"`, "tap:voice", 1.0, sql.NullInt64{}) + + events, err := store.CalendarEvents(ctx, day, day.AddDate(0, 0, 1)) + if err != nil { + t.Fatalf("CalendarEvents: %v", err) + } + if len(events) != 2 { + t.Fatalf("got %d events, want the personal and the ambient one: %+v", len(events), events) + } + bySource := map[string]Fact{} + for _, e := range events { + bySource[e.Source] = e + } + if _, ok := bySource[calendar.SourcePersonal]; !ok { + t.Error("the personal CalDAV event is missing") + } + amb, ok := bySource[calendar.SourceAmbient] + if !ok { + t.Fatal("the ambient work event is missing") + } + if amb.Confidence >= 1.0 { + t.Errorf("ambient confidence = %v, must stay below a calendar read's", amb.Confidence) + } + if _, ok := bySource["tap:voice"]; ok { + t.Error("a non-calendar source must not be read as a calendar event") + } +} + +// TestRecentActiveFactsByKind — the behaviour profile's window. Env rows must +// not spend it, and a retracted row must not be counted as something he did. +func TestRecentActiveFactsByKind(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + now := time.Now().UTC().Truncate(time.Millisecond) + + // A noisy env writer, the way mavpoll writes handshakes. + for i := 0; i < 50; i++ { + if _, err := s.WriteFact(ctx, now.Add(-time.Duration(i)*time.Minute), KindEnv, + "wg_handshake", "1", "poll:wireguard", 1.0, sql.NullInt64{}); err != nil { + t.Fatalf("write env fact: %v", err) + } + } + if _, err := s.WriteFact(ctx, now.Add(-2*time.Hour), KindSelf, "workout", "done", "tap:voice", 1.0, sql.NullInt64{}); err != nil { + t.Fatalf("write self fact: %v", err) + } + if _, err := s.WriteFact(ctx, now.Add(-3*time.Hour), KindSelf, "walk", "done", "tap:voice", 1.0, sql.NullInt64{}); err != nil { + t.Fatalf("write self fact: %v", err) + } + if _, _, err := s.VoidLatestFact(ctx, "walk", "tap:voice", now.Add(-time.Hour)); err != nil { + t.Fatalf("VoidLatestFact: %v", err) + } + + got, err := s.RecentActiveFactsByKind(ctx, KindSelf, 10) + if err != nil { + t.Fatalf("RecentActiveFactsByKind: %v", err) + } + if len(got) != 1 { + t.Fatalf("got %d facts, want 1 (workout): %+v", len(got), got) + } + if got[0].Key != "workout" { + t.Errorf("key = %q, want workout", got[0].Key) + } + + // The window is spent on self rows only: a limit smaller than the env + // traffic still returns the taps. + got, err = s.RecentActiveFactsByKind(ctx, KindSelf, 1) + if err != nil { + t.Fatalf("RecentActiveFactsByKind: %v", err) + } + if len(got) != 1 || got[0].Key != "workout" { + t.Fatalf("got %+v, want the newest self fact", got) + } +} diff --git a/internal/store/tasks.go b/internal/store/tasks.go new file mode 100644 index 0000000..9552554 --- /dev/null +++ b/internal/store/tasks.go @@ -0,0 +1,382 @@ +package store + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + "time" + "unicode" +) + +// Tasks — the capture store (Vikunja #130). One row per piece of work, with a +// lifecycle instead of a valid-time: captured, then either done or dropped. +// +// Why not facts: a fact is a claim about the world and a correction supersedes +// it (append-only, voids_id). A task is not a claim — it is work, it has a +// state that moves forward once, and the read the prioritiser needs is "every +// live task right now", which over an append-only log would mean replaying +// history on every question. +// +// Nothing in this file schedules, fires or announces anything. Capture is a +// store, not a trigger: a task exists to be answered when asked about, and the +// owner's reminders remain the only thing that speaks unprompted. +const ( + // TaskCandidate — Maven derived this task from something she read (mail, + // once the email reader exists) and the owner has not confirmed it. A + // candidate is inert: it is listed as a candidate and never counted as work + // he agreed to. + TaskCandidate = "candidate" + // TaskOpen — work the owner stated himself, or a candidate he confirmed. + TaskOpen = "open" + // TaskDone — finished. + TaskDone = "done" + // TaskDropped — declined, or a candidate rejected. Kept for provenance, so + // the same mail cannot resurrect it silently; nothing re-proposes a + // dropped task. + TaskDropped = "dropped" +) + +// Task — one captured piece of work. +// +// Source is provenance in the same vocabulary facts use: "tap:voice" for +// something he said, "tap:web" for the review page, "email:" for a +// mail-derived candidate. Evidence is the free-text trail a derived task came +// from (a subject line), empty for anything he stated himself — it is what +// makes a candidate reviewable instead of mysterious. +// +// Due is optional. Weight is an explicit importance hint (0 = none), which the +// prioritiser reads; capture never invents one. +// ExternalID is the identity of the thing this task was derived FROM — a +// message id plus the extracted span, for a source that re-reads the same +// immutable text forever. Empty for anything he stated himself. +// +// ResolvedBy names the caller that moved the task to its terminal state, in the +// source vocabulary. Empty while the task is live. +type Task struct { + ID int64 + CreatedTs time.Time + Text string + Source string + Evidence string + ExternalID string + Status string + Due *time.Time + Weight int + ResolvedTs *time.Time + ResolvedBy string +} + +// CaptureResult — what CaptureTask did. Created is a new row. Promoted is an +// existing candidate this capture turned into open work: he stated out loud a +// task Maven had only proposed, which is a confirmation, and the caller says so +// instead of "уже в списке". +type CaptureResult struct { + ID int64 + Created bool + Promoted bool +} + +var ( + ErrTaskNotFound = errors.New("store: task not found") + ErrTaskEmpty = errors.New("store: task text is empty") + ErrTaskStatus = errors.New("store: invalid task status") +) + +// liveTaskStatuses — the two statuses that count as outstanding work. +var liveTaskStatuses = []string{TaskCandidate, TaskOpen} + +// derivedSourcePrefixes — provenance that means "Maven read this somewhere", +// as opposed to "he said it". A task from one of these is a candidate and +// nothing else; see CaptureTask. +var derivedSourcePrefixes = []string{"email:"} + +// IsDerivedSource reports whether a task source means Maven inferred the task +// from something she read rather than being told it. +func IsDerivedSource(source string) bool { + for _, p := range derivedSourcePrefixes { + if strings.HasPrefix(source, p) { + return true + } + } + return false +} + +// CaptureTask inserts a task, or returns the existing one when the same work is +// already there. The result says which happened, so a caller can tell the owner +// "уже в списке" instead of pretending it wrote something. +// +// Two dedupe keys, because voice and mail have different intake semantics: +// +// - ExternalID, unique over EVERY row whatever its status. A source that +// re-reads the same immutable text forever must never resurrect work he has +// already finished. This is the property the email intake depends on: it +// may call CaptureTask for every message it extracts from, as often as it +// likes, without growing the list. +// - The normalised text among LIVE rows only (the partial unique index in +// migration #14), for anything with no external identity. A weekly errand +// captured again once the last one is done must produce a new row, because +// him saying it again IS the recurrence signal. +// +// A capture with Status open over an existing candidate PROMOTES it. Stating +// the work out loud is a confirmation, and leaving it a candidate would have +// Maven read it back as something he never confirmed. +// +// A derived source may only ever capture a candidate. The doc on the intake +// seam said "must NOT set Status open"; this is where that stops being an +// honour system, so a compromised reader cannot file work he never reviewed. +func (s *Store) CaptureTask(ctx context.Context, t Task) (CaptureResult, error) { + text := strings.TrimSpace(t.Text) + if text == "" { + return CaptureResult{}, ErrTaskEmpty + } + status := t.Status + if status == "" { + status = TaskOpen + } + if status != TaskCandidate && status != TaskOpen { + // Capturing straight into a resolved state is meaningless — a task is + // captured live and moved later. + return CaptureResult{}, fmt.Errorf("%w: capture status %q", ErrTaskStatus, status) + } + if status == TaskOpen && IsDerivedSource(t.Source) { + return CaptureResult{}, fmt.Errorf("%w: derived source %q may only capture a candidate", ErrTaskStatus, t.Source) + } + norm := NormalizeTaskText(text) + created2 := t.CreatedTs + if created2.IsZero() { + created2 = time.Now() + } + var due sql.NullInt64 + if t.Due != nil { + due = sql.NullInt64{Int64: t.Due.UnixMilli(), Valid: true} + } + var ext sql.NullString + if e := strings.TrimSpace(t.ExternalID); e != "" { + ext = sql.NullString{String: e, Valid: true} + } + + // Untargeted DO NOTHING: either unique index may be the one that fires, and + // the lookup below sorts out which. + res, err := s.db.ExecContext(ctx, + `INSERT INTO tasks (created_ts, text, norm, source, evidence, ext_id, status, due_ts, weight) + VALUES (?,?,?,?,?,?,?,?,?) + ON CONFLICT DO NOTHING`, + created2.UnixMilli(), text, norm, t.Source, t.Evidence, ext, status, due, t.Weight) + if err != nil { + return CaptureResult{}, fmt.Errorf("capture task: %w", err) + } + if n, err := res.RowsAffected(); err != nil { + return CaptureResult{}, fmt.Errorf("capture task: rows affected: %w", err) + } else if n > 0 { + id, err := res.LastInsertId() + if err != nil { + return CaptureResult{}, fmt.Errorf("capture task: last insert id: %w", err) + } + return CaptureResult{ID: id, Created: true}, nil + } + + // Something already holds one of the keys. External identity first: that + // row may be resolved, in which case the answer is "already handled", not a + // new task. + var existing Task + if ext.Valid { + existing, err = s.lookupTaskByExternalID(ctx, ext.String) + if err != nil && !errors.Is(err, ErrTaskNotFound) { + return CaptureResult{}, err + } + } + if existing.ID == 0 { + existing, err = s.lookupLiveTaskByNorm(ctx, norm) + if err != nil { + return CaptureResult{}, err + } + } + if status == TaskOpen && existing.Status == TaskCandidate { + if err := s.SetTaskStatus(ctx, existing.ID, TaskOpen, created2, t.Source); err != nil { + return CaptureResult{}, fmt.Errorf("capture task: promote candidate: %w", err) + } + return CaptureResult{ID: existing.ID, Promoted: true}, nil + } + return CaptureResult{ID: existing.ID}, nil +} + +// lookupTaskByExternalID finds a task by the identity of what it was derived +// from, in ANY status. Resolved rows count: the whole point of the key is that +// re-reading the mail that produced a finished task produces nothing. +func (s *Store) lookupTaskByExternalID(ctx context.Context, ext string) (Task, error) { + row := s.db.QueryRowContext(ctx, taskSelect+` WHERE ext_id = ?`, ext) + t, err := scanTask(row) + if errors.Is(err, sql.ErrNoRows) { + return Task{}, ErrTaskNotFound + } + if err != nil { + return Task{}, fmt.Errorf("lookup task by external id: %w", err) + } + return t, nil +} + +// lookupLiveTaskByNorm finds the outstanding task with this normalised text. +func (s *Store) lookupLiveTaskByNorm(ctx context.Context, norm string) (Task, error) { + row := s.db.QueryRowContext(ctx, taskSelect+` + WHERE norm = ? AND status IN ('candidate','open')`, norm) + t, err := scanTask(row) + if errors.Is(err, sql.ErrNoRows) { + return Task{}, ErrTaskNotFound + } + if err != nil { + return Task{}, fmt.Errorf("lookup live task: %w", err) + } + return t, nil +} + +const taskSelect = `SELECT id, created_ts, text, source, evidence, COALESCE(ext_id,''), status, due_ts, weight, resolved_ts, resolved_by FROM tasks` + +// LookupTask returns one task by id. +func (s *Store) LookupTask(ctx context.Context, id int64) (Task, error) { + row := s.db.QueryRowContext(ctx, taskSelect+` WHERE id = ?`, id) + t, err := scanTask(row) + if errors.Is(err, sql.ErrNoRows) { + return Task{}, fmt.Errorf("%w: id=%d", ErrTaskNotFound, id) + } + if err != nil { + return Task{}, fmt.Errorf("lookup task: %w", err) + } + return t, nil +} + +// MaxTaskRows — the hard bound on one ListTasks read. The live set is a list a +// person keeps by hand and never approaches this; the resolved set grows for as +// long as the box runs, and an unbounded read of it is a page that gets slower +// every month. Newest first, so the bound drops the oldest finished work. +const MaxTaskRows = 500 + +// ListTasks returns tasks in one status, newest first, at most MaxTaskRows of +// them. An empty status returns every row; "live" returns candidate + open, +// which is what every read path that means "outstanding work" wants. +func (s *Store) ListTasks(ctx context.Context, status string) ([]Task, error) { + q := taskSelect + var args []any + switch status { + case "": + case "live": + q += ` WHERE status IN (?,?)` + args = append(args, liveTaskStatuses[0], liveTaskStatuses[1]) + default: + q += ` WHERE status = ?` + args = append(args, status) + } + q += fmt.Sprintf(` ORDER BY created_ts DESC, id DESC LIMIT %d`, MaxTaskRows) + + rows, err := s.db.QueryContext(ctx, q, args...) + if err != nil { + return nil, fmt.Errorf("list tasks: %w", err) + } + defer rows.Close() + var out []Task + for rows.Next() { + t, err := scanTask(rows) + if err != nil { + return nil, fmt.Errorf("list tasks: %w", err) + } + out = append(out, t) + } + return out, rows.Err() +} + +// SetTaskStatus moves a task once, forward. The legal moves are: +// +// candidate → open (the owner confirms a derived task) +// candidate → dropped (he rejects it) +// open → done (finished) +// open → dropped (abandoned) +// +// Anything else — including re-resolving a resolved task — is refused with +// ErrTaskNotFound-wrapped detail, the same one-way shape proposed_routines and +// tools use: an answered question is not answered twice. +// +// Resolving frees the NORM dedupe key, which is the point: the work can recur +// when he says it again. It does not free an external identity — see +// CaptureTask for why a re-read mailbox must not resurrect finished work. +// +// by names the caller making the move, in the source vocabulary ("tap:web", +// "tap:voice"). It is recorded on the row, so a task that turns up resolved +// says what resolved it. +func (s *Store) SetTaskStatus(ctx context.Context, id int64, status string, ts time.Time, by string) error { + var from []string + switch status { + case TaskOpen: + from = []string{TaskCandidate} + case TaskDone: + from = []string{TaskOpen} + case TaskDropped: + from = []string{TaskCandidate, TaskOpen} + default: + return fmt.Errorf("%w: %q", ErrTaskStatus, status) + } + + // resolved_ts is only meaningful for a terminal state; confirming a + // candidate leaves it null (the task is still live). + var resolved sql.NullInt64 + if status == TaskDone || status == TaskDropped { + resolved = sql.NullInt64{Int64: ts.UnixMilli(), Valid: true} + } + + q := `UPDATE tasks SET status = ?, resolved_ts = ?, resolved_by = ? WHERE id = ? AND status IN (?` + + strings.Repeat(",?", len(from)-1) + `)` + args := []any{status, resolved, by, id} + for _, f := range from { + args = append(args, f) + } + res, err := s.db.ExecContext(ctx, q, args...) + if err != nil { + return fmt.Errorf("set task status: %w", err) + } + n, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("set task status: rows affected: %w", err) + } + if n == 0 { + return fmt.Errorf("%w: id=%d not in %v", ErrTaskNotFound, id, from) + } + return nil +} + +// NormalizeTaskText is the dedupe key: lowercased, punctuation dropped, +// whitespace collapsed. Exported because the intake seam (and its tests) needs +// to reason about what will and will not be treated as the same task. +// +// Deliberately shallow — no stemming, no synonyms. Russian morphology would +// need a real lemmatiser to do better, and a normaliser that guesses would +// silently swallow two different tasks. This only catches the case that +// actually happens: the same sentence arriving twice with different casing or +// punctuation. +func NormalizeTaskText(s string) string { + var b strings.Builder + space := true // leading space collapses to nothing + for _, r := range strings.ToLower(s) { + switch { + case unicode.IsLetter(r) || unicode.IsDigit(r): + b.WriteRune(r) + space = false + case !space: + b.WriteRune(' ') + space = true + } + } + return strings.TrimSpace(b.String()) +} + +func scanTask(sc scanner) (Task, error) { + var t Task + var created int64 + var due, resolved sql.NullInt64 + if err := sc.Scan(&t.ID, &created, &t.Text, &t.Source, &t.Evidence, &t.ExternalID, &t.Status, &due, &t.Weight, &resolved, &t.ResolvedBy); err != nil { + return Task{}, err + } + t.CreatedTs = time.UnixMilli(created).UTC() + t.Due = millisToTime(due) + t.ResolvedTs = millisToTime(resolved) + return t, nil +} diff --git a/internal/store/tasks_test.go b/internal/store/tasks_test.go new file mode 100644 index 0000000..dadc48e --- /dev/null +++ b/internal/store/tasks_test.go @@ -0,0 +1,376 @@ +package store + +import ( + "context" + "errors" + "fmt" + "testing" + "time" +) + +func TestCaptureTaskDedupesLiveWork(t *testing.T) { + ctx := context.Background() + st := newTestStore(t) + now := time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC) + + first, err := st.CaptureTask(ctx, Task{Text: "купить молоко", Source: "tap:voice", CreatedTs: now}) + if err != nil { + t.Fatal(err) + } + if !first.Created { + t.Fatal("first capture must create a row") + } + + // Same work, different casing and punctuation — one task, not two. + again, err := st.CaptureTask(ctx, Task{Text: "Купить молоко!", Source: "tap:web", CreatedTs: now}) + if err != nil { + t.Fatal(err) + } + if again.Created { + t.Error("second capture of the same live work must not create a row") + } + if again.ID != first.ID { + t.Errorf("dedupe returned id %d, want the existing %d", again.ID, first.ID) + } + + live, err := st.ListTasks(ctx, "live") + if err != nil { + t.Fatal(err) + } + if len(live) != 1 { + t.Fatalf("live tasks = %d, want 1", len(live)) + } +} + +func TestCaptureTaskAfterDoneIsANewTask(t *testing.T) { + ctx := context.Background() + st := newTestStore(t) + now := time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC) + + first, err := st.CaptureTask(ctx, Task{Text: "полить цветы", Source: "tap:voice", CreatedTs: now}) + if err != nil { + t.Fatal(err) + } + id := first.ID + if err := st.SetTaskStatus(ctx, id, TaskDone, now.Add(time.Hour), "tap:web"); err != nil { + t.Fatal(err) + } + // The dedupe key is free again: a recurring errand must be capturable. + second, err := st.CaptureTask(ctx, Task{Text: "полить цветы", Source: "tap:voice", CreatedTs: now.AddDate(0, 0, 7)}) + if err != nil { + t.Fatal(err) + } + id2 := second.ID + if !second.Created || id2 == id { + t.Fatalf("re-capture after done: created=%v id=%d (previous %d)", second.Created, id2, id) + } + live, err := st.ListTasks(ctx, "live") + if err != nil { + t.Fatal(err) + } + if len(live) != 1 || live[0].ID != id2 { + t.Fatalf("live = %+v, want only the new task %d", live, id2) + } +} + +func TestCaptureTaskCandidateKeepsEvidence(t *testing.T) { + ctx := context.Background() + st := newTestStore(t) + now := time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC) + due := now.Add(48 * time.Hour) + + res, err := st.CaptureTask(ctx, Task{ + Text: "продлить страховку", + Source: "email:kami", + Evidence: "Re: страховой полис истекает", + Status: TaskCandidate, + Due: &due, + Weight: 2, + CreatedTs: now, + }) + if err != nil { + t.Fatal(err) + } + got, err := st.LookupTask(ctx, res.ID) + if err != nil { + t.Fatal(err) + } + if got.Status != TaskCandidate { + t.Errorf("status = %q, want candidate", got.Status) + } + if got.Evidence != "Re: страховой полис истекает" { + t.Errorf("evidence = %q", got.Evidence) + } + if got.Due == nil || !got.Due.Equal(due.UTC()) { + t.Errorf("due = %v, want %v", got.Due, due.UTC()) + } + if got.Weight != 2 { + t.Errorf("weight = %d, want 2", got.Weight) + } + if got.ResolvedTs != nil { + t.Errorf("resolved_ts = %v on a live task, want nil", got.ResolvedTs) + } +} + +func TestSetTaskStatusMovesOnceForwardOnly(t *testing.T) { + ctx := context.Background() + st := newTestStore(t) + now := time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC) + + res, err := st.CaptureTask(ctx, Task{Text: "записаться к врачу", Source: "email:kami", Status: TaskCandidate, CreatedTs: now}) + if err != nil { + t.Fatal(err) + } + cand := res.ID + // candidate → done is not a legal move: he has to confirm it first. + if err := st.SetTaskStatus(ctx, cand, TaskDone, now, "tap:web"); !errors.Is(err, ErrTaskNotFound) { + t.Errorf("candidate→done err = %v, want ErrTaskNotFound", err) + } + if err := st.SetTaskStatus(ctx, cand, TaskOpen, now, "tap:web"); err != nil { + t.Fatal(err) + } + if err := st.SetTaskStatus(ctx, cand, TaskDone, now.Add(time.Hour), "tap:web"); err != nil { + t.Fatal(err) + } + // Already resolved — a second resolve must not move it again. + if err := st.SetTaskStatus(ctx, cand, TaskDropped, now.Add(2*time.Hour), "tap:web"); !errors.Is(err, ErrTaskNotFound) { + t.Errorf("second resolve err = %v, want ErrTaskNotFound", err) + } + got, err := st.LookupTask(ctx, cand) + if err != nil { + t.Fatal(err) + } + if got.Status != TaskDone { + t.Errorf("status = %q, want done", got.Status) + } + if got.ResolvedTs == nil || !got.ResolvedTs.Equal(now.Add(time.Hour).UTC()) { + t.Errorf("resolved_ts = %v, want %v", got.ResolvedTs, now.Add(time.Hour).UTC()) + } +} + +func TestSetTaskStatusRejectsUnknownStatus(t *testing.T) { + ctx := context.Background() + st := newTestStore(t) + res, err := st.CaptureTask(ctx, Task{Text: "что-то", Source: "tap:web"}) + if err != nil { + t.Fatal(err) + } + id := res.ID + if err := st.SetTaskStatus(ctx, id, "candidate", time.Now(), "tap:web"); !errors.Is(err, ErrTaskStatus) { + t.Errorf("→candidate err = %v, want ErrTaskStatus", err) + } + if err := st.SetTaskStatus(ctx, id, "urgent", time.Now(), "tap:web"); !errors.Is(err, ErrTaskStatus) { + t.Errorf("→urgent err = %v, want ErrTaskStatus", err) + } +} + +func TestCaptureTaskRejectsEmptyText(t *testing.T) { + ctx := context.Background() + st := newTestStore(t) + if _, err := st.CaptureTask(ctx, Task{Text: " ", Source: "tap:voice"}); !errors.Is(err, ErrTaskEmpty) { + t.Errorf("err = %v, want ErrTaskEmpty", err) + } +} + +func TestListTasksFiltersByStatus(t *testing.T) { + ctx := context.Background() + st := newTestStore(t) + now := time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC) + + open1, _ := st.CaptureTask(ctx, Task{Text: "первая", Source: "tap:voice", CreatedTs: now}) + _, _ = st.CaptureTask(ctx, Task{Text: "вторая", Source: "email:kami", Status: TaskCandidate, CreatedTs: now.Add(time.Minute)}) + done, _ := st.CaptureTask(ctx, Task{Text: "третья", Source: "tap:voice", CreatedTs: now.Add(2 * time.Minute)}) + if err := st.SetTaskStatus(ctx, done.ID, TaskDone, now.Add(time.Hour), "tap:web"); err != nil { + t.Fatal(err) + } + + cands, err := st.ListTasks(ctx, TaskCandidate) + if err != nil { + t.Fatal(err) + } + if len(cands) != 1 || cands[0].Text != "вторая" { + t.Fatalf("candidates = %+v", cands) + } + opens, err := st.ListTasks(ctx, TaskOpen) + if err != nil { + t.Fatal(err) + } + if len(opens) != 1 || opens[0].ID != open1.ID { + t.Fatalf("open = %+v", opens) + } + all, err := st.ListTasks(ctx, "") + if err != nil { + t.Fatal(err) + } + if len(all) != 3 { + t.Fatalf("all = %d, want 3", len(all)) + } + // Newest first. + if all[0].Text != "третья" { + t.Errorf("first = %q, want newest ('третья')", all[0].Text) + } +} + +func TestNormalizeTaskText(t *testing.T) { + cases := []struct{ in, want string }{ + {"Купить молоко!", "купить молоко"}, + {" купить МОЛОКО ", "купить молоко"}, + {"позвонить в банк (важно)", "позвонить в банк важно"}, + {"", ""}, + } + for _, c := range cases { + if got := NormalizeTaskText(c.in); got != c.want { + t.Errorf("NormalizeTaskText(%q) = %q, want %q", c.in, got, c.want) + } + } +} + +// A mail the reader keeps seeing must not resurrect work he already finished. +// The norm key alone frees on resolve, which is right for voice and wrong for a +// mailbox: mavmaild never marks anything read, so the same message is extracted +// again on every poll, forever. +func TestCaptureTaskExternalIDSurvivesResolution(t *testing.T) { + ctx := context.Background() + st := newTestStore(t) + now := time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC) + mail := Task{ + Text: "продлить страховку", Source: "email:kami", Status: TaskCandidate, + ExternalID: "email:kami#412:продлить страховку", CreatedTs: now, + } + + first, err := st.CaptureTask(ctx, mail) + if err != nil { + t.Fatal(err) + } + if !first.Created { + t.Fatal("first capture must create a row") + } + // He confirms it and does it. + if err := st.SetTaskStatus(ctx, first.ID, TaskOpen, now.Add(time.Hour), "tap:web"); err != nil { + t.Fatal(err) + } + if err := st.SetTaskStatus(ctx, first.ID, TaskDone, now.Add(2*time.Hour), "tap:web"); err != nil { + t.Fatal(err) + } + + // The next poll reads the same message again. + mail.CreatedTs = now.AddDate(0, 0, 1) + again, err := st.CaptureTask(ctx, mail) + if err != nil { + t.Fatal(err) + } + if again.Created { + t.Error("re-reading the same mail created a second task after the first was done") + } + if again.ID != first.ID { + t.Errorf("id = %d, want the resolved row %d", again.ID, first.ID) + } + live, err := st.ListTasks(ctx, "live") + if err != nil { + t.Fatal(err) + } + if len(live) != 0 { + t.Fatalf("live = %+v, want nothing: he already did this", live) + } +} + +// Stating out loud a task Maven only proposed is a confirmation. Leaving it a +// candidate had her read it straight back as something he had not confirmed. +func TestCaptureTaskPromotesCandidate(t *testing.T) { + ctx := context.Background() + st := newTestStore(t) + now := time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC) + + cand, err := st.CaptureTask(ctx, Task{ + Text: "продлить страховку", Source: "email:kami", Status: TaskCandidate, + ExternalID: "email:kami#7:продлить страховку", CreatedTs: now, + }) + if err != nil { + t.Fatal(err) + } + spoken, err := st.CaptureTask(ctx, Task{ + Text: "продлить страховку", Source: "tap:voice", Status: TaskOpen, + CreatedTs: now.Add(time.Minute), + }) + if err != nil { + t.Fatal(err) + } + if spoken.Created { + t.Error("capture over a live candidate must not create a second row") + } + if !spoken.Promoted { + t.Error("capture with status open over a candidate must promote it") + } + got, err := st.LookupTask(ctx, cand.ID) + if err != nil { + t.Fatal(err) + } + if got.Status != TaskOpen { + t.Errorf("status = %q, want open", got.Status) + } + if got.ResolvedTs != nil { + t.Errorf("resolved_ts = %v, want nil: the task is still live", got.ResolvedTs) + } +} + +// A derived source may only ever file a candidate. The intake seam documented +// this and nothing enforced it, so a caller could skip review entirely. +func TestCaptureTaskRefusesOpenFromDerivedSource(t *testing.T) { + ctx := context.Background() + st := newTestStore(t) + _, err := st.CaptureTask(ctx, Task{Text: "оплатить счёт", Source: "email:kami", Status: TaskOpen}) + if !errors.Is(err, ErrTaskStatus) { + t.Errorf("err = %v, want ErrTaskStatus", err) + } +} + +// resolved_ts said when a task was resolved and never by what. +func TestSetTaskStatusRecordsWho(t *testing.T) { + ctx := context.Background() + st := newTestStore(t) + now := time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC) + res, err := st.CaptureTask(ctx, Task{Text: "выкинуть мусор", Source: "tap:voice", CreatedTs: now}) + if err != nil { + t.Fatal(err) + } + if err := st.SetTaskStatus(ctx, res.ID, TaskDone, now.Add(time.Hour), "tap:voice"); err != nil { + t.Fatal(err) + } + got, err := st.LookupTask(ctx, res.ID) + if err != nil { + t.Fatal(err) + } + if got.ResolvedBy != "tap:voice" { + t.Errorf("resolved_by = %q, want tap:voice", got.ResolvedBy) + } +} + +// The resolved history only grows; an unbounded read of it is a page that gets +// slower every month. +func TestListTasksIsBounded(t *testing.T) { + ctx := context.Background() + st := newTestStore(t) + now := time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC) + for i := 0; i < MaxTaskRows+5; i++ { + res, err := st.CaptureTask(ctx, Task{ + Text: fmt.Sprintf("задача %d", i), Source: "tap:voice", + CreatedTs: now.Add(time.Duration(i) * time.Minute), + }) + if err != nil { + t.Fatal(err) + } + if err := st.SetTaskStatus(ctx, res.ID, TaskDone, now.Add(time.Hour), "tap:web"); err != nil { + t.Fatal(err) + } + } + all, err := st.ListTasks(ctx, "") + if err != nil { + t.Fatal(err) + } + if len(all) != MaxTaskRows { + t.Fatalf("all = %d rows, want the %d-row bound", len(all), MaxTaskRows) + } + if all[0].Text != fmt.Sprintf("задача %d", MaxTaskRows+4) { + t.Errorf("first = %q, want the newest", all[0].Text) + } +} diff --git a/internal/store/tools.go b/internal/store/tools.go index 2c86af8..bf34901 100644 --- a/internal/store/tools.go +++ b/internal/store/tools.go @@ -56,6 +56,171 @@ func (s *Store) ProposeTool(ctx context.Context, name, utterance, scope string, return n > 0, nil } +// ProposeMCPTool is ProposeTool for a tool discovered on an MCP server +// (Vikunja #251): the proposal already knows what it would run, so cmd and +// destructive are written with it and Kami only has to press enable. +// +// It is still a PROPOSAL. Discovery cannot grant a capability — that is the +// whole reason a server can be configured without its tools becoming live. +// Like ProposeTool it never touches an existing row, so re-discovery on every +// restart is idempotent and cannot silently re-arm a tool that was disabled or +// change the cmd of one already enabled. +// +// The row is NOT what protects him, and it is worth being exact about that. +// cmd is ["mcp", server, tool]: a late-bound reference to a name the remote +// server owns. The tool it points at can be redefined on the far end without +// the row changing at all, so "the cmd cannot change" is true and beside the +// point. fingerprint is what closes that: it records the declared shape (name, +// description, input schema, readOnlyHint) at the time the proposal was +// written, and ReconcileMCPTool compares against it on every later discovery. +// Pass "" for a row with nothing to fingerprint (a Home Assistant device). +func (s *Store) ProposeMCPTool(ctx context.Context, name, scope string, cmd []string, destructive bool, utterance, fingerprint string, ts time.Time) (bool, error) { + if len(cmd) == 0 { + return false, ErrToolCmd + } + if scope == "" { + scope = "homelab" + } + raw, err := json.Marshal(cmd) + if err != nil { + return false, fmt.Errorf("propose mcp tool: %w", err) + } + d := 0 + if destructive { + d = 1 + } + res, err := s.db.ExecContext(ctx, ` + INSERT INTO tools (name, scope, cmd, destructive, status, utterance, fingerprint, created_ts, updated_ts) + VALUES (?, ?, ?, ?, 'proposed', ?, ?, ?, ?) + ON CONFLICT(name) DO NOTHING`, + name, scope, string(raw), d, utterance, fingerprint, ts.UnixMilli(), ts.UnixMilli()) + if err != nil { + return false, fmt.Errorf("propose mcp tool: %w", err) + } + n, err := res.RowsAffected() + if err != nil { + return false, fmt.Errorf("propose mcp tool: rows affected: %w", err) + } + return n > 0, nil +} + +// ProposeSmartHomeTool is ProposeTool for a controllable device discovered on +// the Home Assistant instance (Vikunja #256). Like ProposeMCPTool the proposal +// already knows what it would run, so cmd is written with it and Kami only has +// to press enable. +// +// It is still a PROPOSAL, and destructive is not a parameter: there is no +// read-only way to turn a lamp off, so every house row carries the confirm +// turn. Re-discovery on every refresh is idempotent — an existing row is never +// touched, so a device he disabled stays disabled. +func (s *Store) ProposeSmartHomeTool(ctx context.Context, name, scope string, cmd []string, utterance string, ts time.Time) (bool, error) { + return s.ProposeMCPTool(ctx, name, scope, cmd, true, utterance, "", ts) +} + +// ToolChange — what ReconcileMCPTool did to an existing row. +type ToolChange struct { + // Changed — the discovered shape differs from the approved one. + Changed bool + // Demoted — the row was enabled and is now 'proposed' again, so the + // capability is off until a human looks at it a second time. + Demoted bool + // Escalated — destructive went from 0 to 1. It never goes the other way. + Escalated bool +} + +// ReconcileMCPTool compares a freshly discovered tool against the row that was +// approved, and escalates when they disagree. +// +// The failure this exists for: day 1 the server offers list_tasks with +// readOnlyHint true, so the row is proposed non-destructive and Kami enables +// it. Day 30 the server is upgraded, or taken over, and list_tasks now writes. +// Insert-or-skip does nothing on that discovery — the row is still enabled, +// still destructive=0 — and the confirm turn never fires, because the flag was +// frozen against a claim the server has since withdrawn. +// +// So: a differing fingerprint drops the row back to 'proposed' and rewrites the +// provenance, and a tool that stopped claiming read-only gets destructive=1. +// destructive is only ever raised, never lowered: relaxing it on the say-so of +// the same server that changed underneath us would undo the point. +// +// A row with an empty stored fingerprint predates this and simply adopts the +// discovered one — an upgrade is not a redefinition. +func (s *Store) ReconcileMCPTool(ctx context.Context, name, fingerprint string, destructive bool, utterance string, ts time.Time) (ToolChange, error) { + var ( + stored string + status string + wasDest int + ) + err := s.db.QueryRowContext(ctx, + `SELECT fingerprint, status, destructive FROM tools WHERE name = ?`, name). + Scan(&stored, &status, &wasDest) + if errors.Is(err, sql.ErrNoRows) { + return ToolChange{}, ErrToolNotFound + } + if err != nil { + return ToolChange{}, fmt.Errorf("reconcile mcp tool: %w", err) + } + var ch ToolChange + if stored == "" { + if _, err := s.db.ExecContext(ctx, + `UPDATE tools SET fingerprint = ?, updated_ts = ? WHERE name = ?`, + fingerprint, ts.UnixMilli(), name); err != nil { + return ToolChange{}, fmt.Errorf("reconcile mcp tool: %w", err) + } + return ch, nil + } + if stored == fingerprint { + return ch, nil + } + ch.Changed = true + ch.Demoted = status == "enabled" + d := wasDest + if destructive && wasDest == 0 { + d, ch.Escalated = 1, true + } + if _, err := s.db.ExecContext(ctx, ` + UPDATE tools + SET fingerprint = ?, destructive = ?, status = 'proposed', utterance = ?, updated_ts = ? + WHERE name = ?`, + fingerprint, d, utterance, ts.UnixMilli(), name); err != nil { + return ToolChange{}, fmt.Errorf("reconcile mcp tool: %w", err) + } + return ch, nil +} + +// WithdrawTool disarms a row whose remote tool no longer exists: it drops back +// to 'proposed' and its provenance says why. +// +// Nothing else retracted a proposal, so a tool a server stopped offering kept +// its row forever, and an ENABLED one stayed enabled and failed at call time +// with an internal string the act path does not match. /tools is where he would +// go to find out and it was the one place that did not say. Returns whether the +// row was still enabled. +func (s *Store) WithdrawTool(ctx context.Context, name, utterance string, ts time.Time) (bool, error) { + res, err := s.db.ExecContext(ctx, ` + UPDATE tools SET status = 'proposed', utterance = ?, updated_ts = ? + WHERE name = ? AND status = 'enabled'`, + utterance, ts.UnixMilli(), name) + if err != nil { + return false, fmt.Errorf("withdraw tool: %w", err) + } + n, err := res.RowsAffected() + if err != nil { + return false, fmt.Errorf("withdraw tool: rows affected: %w", err) + } + if n > 0 { + return true, nil + } + // Not enabled: still refresh the provenance so the proposed row says it. + _, err = s.db.ExecContext(ctx, + `UPDATE tools SET utterance = ?, updated_ts = ? WHERE name = ?`, + utterance, ts.UnixMilli(), name) + if err != nil { + return false, fmt.Errorf("withdraw tool: %w", err) + } + return false, nil +} + // EnableTool fills cmd + destructive and flips status to 'enabled'. This is the // human "enable" act (the authed surface calls it); it upserts so enabling a // name that was never proposed still works. An empty cmd is refused — an diff --git a/internal/store/tools_test.go b/internal/store/tools_test.go index 1111e6b..375d5b0 100644 --- a/internal/store/tools_test.go +++ b/internal/store/tools_test.go @@ -2,6 +2,7 @@ package store import ( "context" + "errors" "testing" "time" ) @@ -60,3 +61,199 @@ func TestToolLifecycle(t *testing.T) { t.Fatalf("disable absent must be no-op: %v", err) } } + +// A discovered MCP tool arrives as a proposal that already knows its cmd, so +// enabling it is one click rather than one retyped argv. +func TestProposeMCPTool(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + now := time.Now() + cmd := []string{"mcp", "vikunja", "list_tasks"} + + fresh, err := s.ProposeMCPTool(ctx, "vikunja_list_tasks", "mcp:vikunja", cmd, false, "mcp vikunja/list_tasks: List tasks", "fp1", now) + if err != nil { + t.Fatal(err) + } + if !fresh { + t.Fatal("first proposal should be new") + } + got, err := s.LookupTool(ctx, "vikunja_list_tasks") + if err != nil { + t.Fatal(err) + } + if got.Status != "proposed" { + t.Fatalf("status = %q — discovery must never enable", got.Status) + } + if len(got.Cmd) != 3 || got.Cmd[0] != "mcp" || got.Cmd[2] != "list_tasks" { + t.Fatalf("cmd = %v", got.Cmd) + } + if got.Scope != "mcp:vikunja" || got.Utterance == "" { + t.Fatalf("provenance lost: %+v", got) + } + + // Re-discovery on the next boot is idempotent. + fresh, err = s.ProposeMCPTool(ctx, "vikunja_list_tasks", "mcp:vikunja", cmd, true, "changed", "fp1", now) + if err != nil { + t.Fatal(err) + } + if fresh { + t.Error("re-proposing an existing row must report nothing new") + } + + // And it must not re-arm or rewrite a row a human already acted on. + if err := s.EnableTool(ctx, "vikunja_list_tasks", cmd, false, "mcp:vikunja", now); err != nil { + t.Fatal(err) + } + if _, err := s.ProposeMCPTool(ctx, "vikunja_list_tasks", "mcp:vikunja", []string{"mcp", "vikunja", "delete_task"}, true, "x", "fp2", now); err != nil { + t.Fatal(err) + } + got, err = s.LookupTool(ctx, "vikunja_list_tasks") + if err != nil { + t.Fatal(err) + } + if got.Status != "enabled" || got.Cmd[2] != "list_tasks" || got.Destructive { + t.Fatalf("an enabled row was modified by discovery: %+v", got) + } +} + +func TestProposeMCPToolNeedsCmd(t *testing.T) { + s := newTestStore(t) + if _, err := s.ProposeMCPTool(context.Background(), "x", "mcp:y", nil, false, "", "", time.Now()); !errors.Is(err, ErrToolCmd) { + t.Fatalf("err = %v, want ErrToolCmd", err) + } +} + +// A server that redefines a tool Kami already approved must have to ask again. +// The row stores cmd ["mcp", server, tool], a late-bound reference to a name +// the far end owns, so before the fingerprint a server could turn an enabled +// read-only list_tasks into something that writes and Maven would keep running +// it without a confirm turn. +func TestReconcileMCPToolDemotesARedefinedTool(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + now := time.Now() + cmd := []string{"mcp", "vikunja", "list_tasks"} + if _, err := s.ProposeMCPTool(ctx, "vikunja_list_tasks", "mcp:vikunja", cmd, false, "read only", "fp1", now); err != nil { + t.Fatal(err) + } + if err := s.EnableTool(ctx, "vikunja_list_tasks", cmd, false, "mcp:vikunja", now); err != nil { + t.Fatal(err) + } + + // Same shape ⇒ nothing happens. Discovery runs every minute and must be + // idempotent. + ch, err := s.ReconcileMCPTool(ctx, "vikunja_list_tasks", "fp1", false, "read only", now) + if err != nil { + t.Fatal(err) + } + if ch.Changed { + t.Fatalf("an unchanged tool must not be touched: %+v", ch) + } + if got, _ := s.LookupTool(ctx, "vikunja_list_tasks"); got.Status != "enabled" { + t.Fatalf("status = %q, want it left enabled", got.Status) + } + + // It stopped claiming read-only and its schema moved. + ch, err = s.ReconcileMCPTool(ctx, "vikunja_list_tasks", "fp2", true, "now writes", now) + if err != nil { + t.Fatal(err) + } + if !ch.Changed || !ch.Demoted || !ch.Escalated { + t.Fatalf("change = %+v, want changed+demoted+escalated", ch) + } + got, err := s.LookupTool(ctx, "vikunja_list_tasks") + if err != nil { + t.Fatal(err) + } + if got.Status != "proposed" { + t.Errorf("status = %q, want a redefined tool back in the queue", got.Status) + } + if !got.Destructive { + t.Error("a tool that stopped claiming read-only must gain the confirm turn") + } + if got.Utterance != "now writes" { + t.Errorf("utterance = %q, want what the server says today", got.Utterance) + } + + // destructive is only ever raised. The server that changed underneath us + // does not get to relax it by claiming read-only next time. + if _, err := s.ReconcileMCPTool(ctx, "vikunja_list_tasks", "fp3", false, "read only again", now); err != nil { + t.Fatal(err) + } + if got, _ = s.LookupTool(ctx, "vikunja_list_tasks"); !got.Destructive { + t.Error("destructive was relaxed by the server") + } +} + +// A row written before fingerprints exist simply adopts one. An upgrade is not +// a redefinition and must not disable everything Kami approved. +func TestReconcileMCPToolAdoptsAnEmptyFingerprint(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + now := time.Now() + cmd := []string{"mcp", "vikunja", "list_tasks"} + if _, err := s.ProposeMCPTool(ctx, "vikunja_list_tasks", "mcp:vikunja", cmd, false, "x", "", now); err != nil { + t.Fatal(err) + } + if err := s.EnableTool(ctx, "vikunja_list_tasks", cmd, false, "mcp:vikunja", now); err != nil { + t.Fatal(err) + } + ch, err := s.ReconcileMCPTool(ctx, "vikunja_list_tasks", "fp1", false, "x", now) + if err != nil { + t.Fatal(err) + } + if ch.Changed { + t.Fatalf("adopting must be silent: %+v", ch) + } + if got, _ := s.LookupTool(ctx, "vikunja_list_tasks"); got.Status != "enabled" { + t.Fatalf("status = %q, want still enabled after the upgrade", got.Status) + } + // And now it is pinned. + if ch, _ = s.ReconcileMCPTool(ctx, "vikunja_list_tasks", "fp2", false, "y", now); !ch.Changed { + t.Fatal("the adopted fingerprint must be enforced on the next pass") + } +} + +func TestReconcileMCPToolUnknownRow(t *testing.T) { + s := newTestStore(t) + if _, err := s.ReconcileMCPTool(context.Background(), "nope", "fp", false, "", time.Now()); !errors.Is(err, ErrToolNotFound) { + t.Fatalf("err = %v, want ErrToolNotFound", err) + } +} + +// A tool the server stopped offering must be disarmed and must say why. It used +// to stay enabled and fail at call time with an internal string, and /tools — +// the one place he would look — did not mention it. +func TestWithdrawTool(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + now := time.Now() + cmd := []string{"mcp", "vikunja", "list_tasks"} + if _, err := s.ProposeMCPTool(ctx, "vikunja_list_tasks", "mcp:vikunja", cmd, false, "x", "fp1", now); err != nil { + t.Fatal(err) + } + if err := s.EnableTool(ctx, "vikunja_list_tasks", cmd, false, "mcp:vikunja", now); err != nil { + t.Fatal(err) + } + was, err := s.WithdrawTool(ctx, "vikunja_list_tasks", "gone", now) + if err != nil { + t.Fatal(err) + } + if !was { + t.Error("withdrawing an enabled tool must report that it was enabled") + } + got, err := s.LookupTool(ctx, "vikunja_list_tasks") + if err != nil { + t.Fatal(err) + } + if got.Status != "proposed" || got.Utterance != "gone" { + t.Fatalf("row = %+v, want proposed and saying why", got) + } + // Withdrawing again is not an error and does not claim it was enabled. + if was, err = s.WithdrawTool(ctx, "vikunja_list_tasks", "still gone", now); err != nil || was { + t.Fatalf("second withdraw = %v, %v", was, err) + } + if got, _ = s.LookupTool(ctx, "vikunja_list_tasks"); got.Utterance != "still gone" { + t.Errorf("utterance = %q, want the provenance refreshed anyway", got.Utterance) + } +} diff --git a/internal/tasks/rank.go b/internal/tasks/rank.go new file mode 100644 index 0000000..e2f88fd --- /dev/null +++ b/internal/tasks/rank.go @@ -0,0 +1,270 @@ +// Package tasks ranks captured work (Vikunja #129). +// +// The ordering is COMPUTED, not generated. Asking a 1.7B model which of his +// tasks matters most would produce a fluent opinion about his life with no +// basis in anything, and a confidently wrong priority is worse than no +// priority at all — the same reasoning as the behaviour profile in +// internal/memory, which counts instead of summarising. +// +// So: four signals, all of them things he told her, and a reason string naming +// the one that decided each row. Nothing here invents urgency. A task with no +// due date and no weight scores nothing and sits where its age puts it, which +// is the honest answer to "which of these matters?" when he never said. +// +// Ranking is a READ. It sorts and renders; it never writes, schedules or +// announces. Maven is not a nag: a task rising to the top of this list is not a +// reason to speak, only the order she recites in when asked. +package tasks + +import ( + "fmt" + "sort" + "strings" + "time" +) + +// Status values, mirroring internal/store so a caller can rank ipc.Task rows +// without importing the store. +const ( + StatusCandidate = "candidate" + StatusOpen = "open" +) + +// Item — one task to rank. The subset of a task that ranking depends on; +// callers map their own row type onto it. +type Item struct { + ID int64 + Text string + Status string + Created time.Time + Due *time.Time + Weight int +} + +// Ranked — one task with its score and the reason that decided it. +type Ranked struct { + Item + Score float64 + // Reason — the dominant signal, in Russian, for the page and the spoken + // list. Empty when nothing distinguished this task: no due date, no + // weight, not old. Saying "потому что" about a task he never prioritised + // would be making something up. + Reason string +} + +// Scoring weights. Deliberately coarse round numbers: this is a knob, not +// math, and the only property that has to hold is the ordering between classes +// (overdue beats today beats this week beats undated). +const ( + scoreOverdue = 100 // he already missed it + scoreOverduePer = 5 // per further day late, capped + scoreOverdueCap = 40 + scoreDueToday = 60 + scoreDueTomorrow = 40 + scoreDueWeek = 20 + // Above scoreAgeCap on purpose: a dated task must outrank an undated one + // however long the undated one has sat, or the class ordering this block + // claims is inverted by age alone. + scoreDueLater = 12 + scorePerWeight = 15 // "срочно" / "важно" / the web form's select + scorePerWeekOld = 1 // so nothing rots at the bottom forever + scoreAgeCap = 10 + // MaxWeight — the highest importance hint capture accepts. Three rungs is + // as many as anyone can rank by hand honestly. + MaxWeight = 3 +) + +// Rank scores every item and returns them ordered: confirmed work first, then +// candidates, each by score descending, oldest first on a tie. +// +// Candidates never outrank open work, whatever their due date. A task Maven +// derived from something she read is a suggestion until he confirms it, and +// putting her guess above his own stated work would be reading his priorities +// back to him wrong. +func Rank(items []Item, now time.Time) []Ranked { + out := make([]Ranked, 0, len(items)) + for _, it := range items { + score, reason := score(it, now) + out = append(out, Ranked{Item: it, Score: score, Reason: reason}) + } + sort.SliceStable(out, func(i, j int) bool { + ci, cj := out[i].Status == StatusCandidate, out[j].Status == StatusCandidate + if ci != cj { + return !ci // open before candidate + } + if out[i].Score != out[j].Score { + return out[i].Score > out[j].Score + } + return out[i].Created.Before(out[j].Created) // oldest first, FIFO + }) + return out +} + +// score — the per-item scoring function. Returns the score and the dominant +// reason. Deadline beats weight when both are present: a date is a fact about +// the world, a weight is how he felt when he filed it. +func score(it Item, now time.Time) (float64, string) { + var total float64 + reason := "" + + if it.Due != nil { + days := dayDelta(*it.Due, now) + switch { + case days < 0: + late := -days + bonus := float64(late * scoreOverduePer) + if bonus > scoreOverdueCap { + bonus = scoreOverdueCap + } + total += scoreOverdue + bonus + reason = "просрочено" + if late == 1 { + reason = "просрочено на день" + } else if late > 1 { + reason = fmt.Sprintf("просрочено на %d дн.", late) + } + case days == 0: + total += scoreDueToday + reason = "сегодня" + case days == 1: + total += scoreDueTomorrow + reason = "завтра" + case days <= 7: + total += scoreDueWeek + reason = fmt.Sprintf("через %d дн.", days) + default: + total += scoreDueLater + } + } + + w := it.Weight + if w > MaxWeight { + w = MaxWeight + } + if w > 0 { + total += float64(w * scorePerWeight) + if reason == "" { + // The rungs get their own words. The reason string is the one place + // the ranking explains itself, and reading "важно" back at a task + // he flagged "срочно" reports a word he did not say. + reason = "важно" + if w >= MaxWeight { + reason = "срочно" + } + } + } + + if !it.Created.IsZero() { + weeks := int(now.Sub(it.Created).Hours() / (24 * 7)) + if weeks > 0 { + age := float64(weeks * scorePerWeekOld) + if age > scoreAgeCap { + age = scoreAgeCap + } + total += age + if reason == "" && weeks >= 2 { + reason = "давно в списке" + } + } + } + return total, reason +} + +// dayDelta — calendar days from now to due, in NOW's location. Whole days, not +// hours: a task due today is due today whether it is 09:00 or 23:00, and an +// hours-based comparison would call this evening's task "overdue" all afternoon. +// +// The location has to come from now. A due date read back from the store is a +// UTC instant (store.scanTask ends in time.UnixMilli(...).UTC()), so taking the +// location from it compared calendar days in UTC while the page rendered the +// same date in local time. East of Greenwich that is off by one all morning: a +// task due tomorrow read "сегодня", and on its due date it read "просрочено на +// день" and scored 105 instead of 60, one table cell away from a due column +// that said otherwise. +func dayDelta(due, now time.Time) int { + loc := now.Location() + d := due.In(loc) + dd := time.Date(d.Year(), d.Month(), d.Day(), 0, 0, 0, 0, loc) + nn := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, loc) + return int(dd.Sub(nn).Hours() / 24) +} + +// SpokenLimit — how many tasks the spoken list names before it summarises the +// rest. A recital of twenty items is noise; five is a list he can hold. +const SpokenLimit = 5 + +// FormatRU renders a ranked list the way Maven says it. Confirmed work first, +// with the reason attached where there is one; candidates named as +// unconfirmed, never recited as his work. +// +// One renderer for the voice reply and the web page, for the same reason +// DayPlan.Spoken is built core-side: two formatters drift, and then she says +// one order and shows another. +func FormatRU(ranked []Ranked) string { + var open, cands []Ranked + for _, r := range ranked { + if r.Status == StatusCandidate { + cands = append(cands, r) + } else { + open = append(open, r) + } + } + if len(open) == 0 && len(cands) == 0 { + return "задач нет." + } + + var b strings.Builder + if len(open) > 0 { + b.WriteString("сначала: ") + b.WriteString(joinRU(open, SpokenLimit, true)) + b.WriteString(".") + } + if len(cands) > 0 { + if b.Len() > 0 { + b.WriteString(" ") + } + b.WriteString("ещё я нашла, но ты не подтвердил: ") + b.WriteString(joinRU(cands, SpokenLimit, false)) + b.WriteString(".") + } + return b.String() +} + +// joinRU lists up to limit tasks, then says how many are left. withReasons +// attaches the parenthesised reason — candidates are listed bare, since their +// due dates are Maven's reading of a mail and not something he stated. +func joinRU(rs []Ranked, limit int, withReasons bool) string { + shown := rs + rest := 0 + if len(rs) > limit { + shown, rest = rs[:limit], len(rs)-limit + } + parts := make([]string, 0, len(shown)) + for _, r := range shown { + if withReasons && r.Reason != "" { + parts = append(parts, r.Text+" ("+r.Reason+")") + } else { + parts = append(parts, r.Text) + } + } + s := strings.Join(parts, "; ") + if rest > 0 { + // With the noun. Spoken, a bare number trails off mid-sentence. + s += fmt.Sprintf("; и ещё %d %s", rest, pluralTasksRU(rest)) + } + return s +} + +// pluralTasksRU — the right form of "задача" for a count. Russian needs three. +func pluralTasksRU(n int) string { + if n%100 >= 11 && n%100 <= 14 { + return "задач" + } + switch n % 10 { + case 1: + return "задача" + case 2, 3, 4: + return "задачи" + } + return "задач" +} diff --git a/internal/tasks/rank_test.go b/internal/tasks/rank_test.go new file mode 100644 index 0000000..3364e27 --- /dev/null +++ b/internal/tasks/rank_test.go @@ -0,0 +1,245 @@ +package tasks + +import ( + "strings" + "testing" + "time" +) + +func at(y int, m time.Month, d int) *time.Time { + t := time.Date(y, m, d, 0, 0, 0, 0, time.UTC) + return &t +} + +func now() time.Time { return time.Date(2026, 8, 1, 14, 0, 0, 0, time.UTC) } + +func texts(rs []Ranked) []string { + out := make([]string, len(rs)) + for i, r := range rs { + out[i] = r.Text + } + return out +} + +func TestRankOrdersByDeadline(t *testing.T) { + items := []Item{ + {ID: 1, Text: "через неделю", Status: StatusOpen, Due: at(2026, 8, 7), Created: now()}, + {ID: 2, Text: "просрочено", Status: StatusOpen, Due: at(2026, 7, 28), Created: now()}, + {ID: 3, Text: "без срока", Status: StatusOpen, Created: now()}, + {ID: 4, Text: "сегодня", Status: StatusOpen, Due: at(2026, 8, 1), Created: now()}, + {ID: 5, Text: "завтра", Status: StatusOpen, Due: at(2026, 8, 2), Created: now()}, + } + got := texts(Rank(items, now())) + want := []string{"просрочено", "сегодня", "завтра", "через неделю", "без срока"} + for i := range want { + if got[i] != want[i] { + t.Fatalf("order = %v, want %v", got, want) + } + } +} + +func TestRankCandidatesNeverOutrankOpenWork(t *testing.T) { + items := []Item{ + {ID: 1, Text: "его задача", Status: StatusOpen, Created: now()}, + // Everything about this one screams urgent — and it is still a guess. + {ID: 2, Text: "из письма", Status: StatusCandidate, Due: at(2026, 7, 1), Weight: 3, Created: now()}, + } + got := Rank(items, now()) + if got[0].Text != "его задача" { + t.Errorf("order = %v, want his own work first", texts(got)) + } +} + +func TestRankWeightLiftsUndatedWork(t *testing.T) { + items := []Item{ + {ID: 1, Text: "обычная", Status: StatusOpen, Created: now()}, + {ID: 2, Text: "важная", Status: StatusOpen, Weight: 2, Created: now()}, + } + got := Rank(items, now()) + if got[0].Text != "важная" { + t.Errorf("order = %v, want the weighted task first", texts(got)) + } + if got[0].Reason != "важно" { + t.Errorf("reason = %q, want важно", got[0].Reason) + } + // A deadline still beats a weight: a date is a fact, a weight is a feeling. + items = append(items, Item{ID: 3, Text: "сегодня", Status: StatusOpen, Due: at(2026, 8, 1), Created: now()}) + got = Rank(items, now()) + if got[0].Text != "сегодня" { + t.Errorf("order = %v, want the dated task first", texts(got)) + } +} + +func TestRankOldestFirstOnATie(t *testing.T) { + old := now().AddDate(0, 0, -3) + items := []Item{ + {ID: 1, Text: "новая", Status: StatusOpen, Created: now()}, + {ID: 2, Text: "старая", Status: StatusOpen, Created: old}, + } + got := Rank(items, now()) + if got[0].Text != "старая" { + t.Errorf("order = %v, want FIFO on equal urgency", texts(got)) + } +} + +func TestRankNoInventedReason(t *testing.T) { + got := Rank([]Item{{ID: 1, Text: "что-то", Status: StatusOpen, Created: now()}}, now()) + if got[0].Reason != "" { + t.Errorf("reason = %q — nothing distinguished this task, so there is nothing to say", got[0].Reason) + } + if got[0].Score != 0 { + t.Errorf("score = %v, want 0", got[0].Score) + } +} + +func TestRankAgeIsCappedAndNamed(t *testing.T) { + items := []Item{ + {ID: 1, Text: "прошлогодняя", Status: StatusOpen, Created: now().AddDate(-1, 0, 0)}, + {ID: 2, Text: "трёхнедельная", Status: StatusOpen, Created: now().AddDate(0, 0, -21)}, + } + got := Rank(items, now()) + if got[0].Score != scoreAgeCap { + t.Errorf("oldest score = %v, want the cap %v", got[0].Score, float64(scoreAgeCap)) + } + if got[0].Reason != "давно в списке" { + t.Errorf("reason = %q", got[0].Reason) + } +} + +// A task due at 23:00 today is due today, not overdue since this morning. +func TestRankDueTodayIsNotOverdue(t *testing.T) { + due := time.Date(2026, 8, 1, 23, 0, 0, 0, time.UTC) + got := Rank([]Item{{ID: 1, Text: "вечером", Status: StatusOpen, Due: &due, Created: now()}}, now()) + if got[0].Reason != "сегодня" { + t.Errorf("reason = %q, want сегодня", got[0].Reason) + } +} + +func TestRankOverdueDaysAreCounted(t *testing.T) { + got := Rank([]Item{ + {ID: 1, Text: "вчера", Status: StatusOpen, Due: at(2026, 7, 31), Created: now()}, + {ID: 2, Text: "давно", Status: StatusOpen, Due: at(2026, 7, 20), Created: now()}, + }, now()) + if got[0].Text != "давно" { + t.Errorf("order = %v, want the later-overdue task first", texts(got)) + } + if got[0].Reason != "просрочено на 12 дн." { + t.Errorf("reason = %q", got[0].Reason) + } + if got[1].Reason != "просрочено на день" { + t.Errorf("reason = %q", got[1].Reason) + } +} + +func TestFormatRUNamesReasonsAndSeparatesCandidates(t *testing.T) { + ranked := Rank([]Item{ + {ID: 1, Text: "оплатить интернет", Status: StatusOpen, Due: at(2026, 8, 1), Created: now()}, + {ID: 2, Text: "купить молоко", Status: StatusOpen, Created: now()}, + {ID: 3, Text: "продлить страховку", Status: StatusCandidate, Due: at(2026, 7, 1), Created: now()}, + }, now()) + got := FormatRU(ranked) + if !strings.HasPrefix(got, "сначала: оплатить интернет (сегодня)") { + t.Errorf("reply = %q", got) + } + if !strings.Contains(got, "не подтвердил: продлить страховку") { + t.Errorf("candidate not named as unconfirmed: %q", got) + } + // A candidate's due date is Maven's reading of a mail, not his statement. + if strings.Contains(got, "продлить страховку (") { + t.Errorf("a candidate must be listed without a reason: %q", got) + } + // Persona: nothing masculine, no pet names, informal address only. + for _, bad := range []string{"рад ", "понял ", "милый", "дорогой", "вам", "ваши"} { + if strings.Contains(got, bad) { + t.Errorf("reply %q contains %q", got, bad) + } + } +} + +func TestFormatRUCapsTheSpokenList(t *testing.T) { + var items []Item + for i := 0; i < SpokenLimit+3; i++ { + items = append(items, Item{ID: int64(i), Text: "задача", Status: StatusOpen, Created: now()}) + } + got := FormatRU(Rank(items, now())) + if !strings.Contains(got, "и ещё 3") { + t.Errorf("reply = %q, want the tail summarised", got) + } + if strings.Count(got, "задача") != SpokenLimit { + t.Errorf("reply = %q, want exactly %d named", got, SpokenLimit) + } +} + +func TestFormatRUEmpty(t *testing.T) { + if got := FormatRU(nil); got != "задач нет." { + t.Errorf("reply = %q", got) + } +} + +// A due date read back from the store is a UTC instant, so comparing calendar +// days in ITS location put every date a day out east of Greenwich: the row said +// "сегодня" for a task due tomorrow, and "просрочено на день" on the due date +// itself while the due column one cell over said otherwise. +func TestRankComparesDaysInTheCallersLocation(t *testing.T) { + tz := time.FixedZone("UTC+4", 4*3600) + // Entered on the web form as 2026-08-02 local, stored and read back as UTC. + due := time.Date(2026, 8, 2, 0, 0, 0, 0, tz).UTC() + local := time.Date(2026, 8, 1, 10, 0, 0, 0, tz) + + got := Rank([]Item{{ID: 1, Text: "оплатить интернет", Status: StatusOpen, Due: &due, Created: local}}, local) + if got[0].Reason != "завтра" { + t.Errorf("reason = %q, want завтра on the day before", got[0].Reason) + } + // The morning of the due date itself. + onTheDay := time.Date(2026, 8, 2, 10, 0, 0, 0, tz) + got = Rank([]Item{{ID: 1, Text: "оплатить интернет", Status: StatusOpen, Due: &due, Created: local}}, onTheDay) + if got[0].Reason != "сегодня" { + t.Errorf("reason = %q, want сегодня on the due date", got[0].Reason) + } + if got[0].Score != scoreDueToday { + t.Errorf("score = %v, want %v", got[0].Score, float64(scoreDueToday)) + } +} + +// "срочно" and "важно" are two rungs and the read-back said "важно" for both, +// which reports a word he did not say. +func TestRankNamesTheUrgencyHeStated(t *testing.T) { + got := Rank([]Item{ + {ID: 1, Text: "оплатить интернет", Status: StatusOpen, Weight: 3, Created: now()}, + {ID: 2, Text: "починить кран", Status: StatusOpen, Weight: 2, Created: now()}, + }, now()) + if got[0].Reason != "срочно" { + t.Errorf("reason = %q, want срочно", got[0].Reason) + } + if got[1].Reason != "важно" { + t.Errorf("reason = %q, want важно", got[1].Reason) + } +} + +// The package doc guarantees a class ordering. Age used to invert it: an +// undated task at the age cap outscored a dated one three weeks out. +func TestRankDatedWorkBeatsAgeAlone(t *testing.T) { + got := Rank([]Item{ + {ID: 1, Text: "старьё", Status: StatusOpen, Created: now().AddDate(0, 0, -70)}, + {ID: 2, Text: "через три недели", Status: StatusOpen, Due: at(2026, 8, 22), Created: now()}, + }, now()) + if got[0].Text != "через три недели" { + t.Errorf("order = %v, want the dated task first", texts(got)) + } +} + +// A bare "и ещё 5" trails off when spoken. +func TestFormatRUTailCarriesTheNoun(t *testing.T) { + var items []Item + for i := 0; i < SpokenLimit+3; i++ { + items = append(items, Item{ID: int64(i), Text: "дело", Status: StatusOpen, Created: now()}) + } + if got := FormatRU(Rank(items, now())); !strings.Contains(got, "и ещё 3 задачи") { + t.Errorf("reply = %q, want the count with its noun", got) + } + for n, want := range map[int]string{1: "задача", 2: "задачи", 5: "задач", 11: "задач", 21: "задача"} { + if got := pluralTasksRU(n); got != want { + t.Errorf("pluralTasksRU(%d) = %q, want %q", n, got, want) + } + } +} diff --git a/internal/tool/tool.go b/internal/tool/tool.go index abe0977..4a1b017 100644 --- a/internal/tool/tool.go +++ b/internal/tool/tool.go @@ -13,6 +13,16 @@ // - Args are passed as argv, NEVER through a shell. STT text lands as // positional arguments to Cmd; there is no `sh -c`, so "restart nginx; // rm -rf" can't inject — the tail is one argv element to the named binary. +// - An enabled row whose cmd is ["smarthome", "", ""] is +// a Home Assistant service call instead of a process (Vikunja #256), by +// exactly the same trick and under exactly the same rules. Control rows are +// always destructive, so flipping something in his flat always costs a +// confirm turn. +// - An enabled row whose cmd is ["mcp", "", ""] is a call to a +// configured MCP server instead of a process (Vikunja #251). It goes +// through every rule above unchanged — enabled, and confirmed if it +// mutates — because the store is still the allowlist; only the dispatch at +// the bottom of Exec differs. // - Destructive tools don't run on first hearing: Exec returns ErrNeedsConfirm // and the handler runs a confirm turn ("выполнить X? да/нет"); only a // confirmed re-Exec runs them. A gate assumes a fully-formed action, which @@ -31,7 +41,9 @@ import ( "time" "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/mcp" "github.com/kami/maven/internal/router" + "github.com/kami/maven/internal/smarthome" ) // API — the narrow slice of ipc.CoreAPI the executor and matcher need. Backed @@ -48,14 +60,39 @@ var ( ErrNotEnabled = errors.New("tool not on the enabled allowlist") // ErrNeedsConfirm — the fn is enabled but destructive; needs a confirm turn. ErrNeedsConfirm = errors.New("destructive tool needs confirmation") + // ErrNotConnected — the row is enabled and well formed, but the thing it + // dispatches to is not wired: the mcp block was dropped from the config + // while enabled MCP rows remained, or the same for the house. Held apart + // from ErrNotEnabled because the act path turns that one into a fresh + // proposal, and drafting a new proposal for a tool that already exists and + // is enabled is a lie about what is wrong. + ErrNotConnected = errors.New("tool is enabled but its backend is not connected") ) +// MCPCaller is the seam for an act that is an MCP tool call rather than a +// process (Vikunja #251). internal/mcp.Manager satisfies it via CallPositional. +// nil ⇒ MCP is not configured, and an MCP row refuses to run rather than +// silently doing nothing. +type MCPCaller interface { + CallPositional(ctx context.Context, server, tool string, args []string) (string, error) +} + +// HomeCaller is the seam for an act that is a Home Assistant service call +// rather than a process (Vikunja #256). internal/smarthome.Client satisfies it. +// nil ⇒ the house is not configured, and a house row refuses to run rather than +// silently doing nothing. +type HomeCaller interface { + CallService(ctx context.Context, entityID, service string) (string, error) +} + // Executor runs enabled tools. run is the exec seam (default: real process); // tests swap it. timeout bounds each invocation. type Executor struct { api API timeout time.Duration run func(ctx context.Context, argv []string) (string, error) + mcp MCPCaller + home HomeCaller } // NewExecutor builds the executor. timeout<=0 defaults to 30s. @@ -66,6 +103,21 @@ func NewExecutor(api API, timeout time.Duration) *Executor { return &Executor{api: api, timeout: timeout, run: runProcess} } +// WithMCP attaches the MCP caller. Called once at wiring time when the mcp +// config block is present; without it, a row whose cmd is ["mcp", …] refuses. +func (e *Executor) WithMCP(m MCPCaller) *Executor { + e.mcp = m + return e +} + +// WithHome attaches the Home Assistant caller. Called once at wiring time when +// the smarthome block is enabled; without it, a row whose cmd is +// ["smarthome", …] refuses. +func (e *Executor) WithHome(h HomeCaller) *Executor { + e.home = h + return e +} + // Exec looks up name in the store and runs Cmd+args as argv (no shell). // confirmed=true is the second turn of a destructive act (the user said "да"); // it bypasses the ErrNeedsConfirm gate. Non-enabled ⇒ ErrNotEnabled; a @@ -84,6 +136,40 @@ func (e *Executor) Exec(ctx context.Context, name string, args []string, confirm if t.Destructive && !confirmed { return "", ErrNeedsConfirm } + // An MCP row is a call to a configured server, not a process. Everything + // above still applied: it had to be enabled, and a mutating one had to be + // confirmed. Only the dispatch differs. + if server, remote, ok := mcp.ParseCmd(t.Cmd); ok { + if e.mcp == nil { + return "", fmt.Errorf("%w: %s is an MCP tool and no mcp block is configured", ErrNotConnected, name) + } + ctx, cancel := context.WithTimeout(ctx, e.timeout) + defer cancel() + return e.mcp.CallPositional(ctx, server, remote, args) + } + // A house row is a Home Assistant service call, not a process (Vikunja + // #256). Same story: enabled, and confirmed — every control row is + // destructive, because there is no read-only way to turn the heating off. + // The spoken args are dropped on purpose: the entity and the service come + // from the row Kami enabled, so a router that misheard can pick the wrong + // row but can never compose a target of its own. + if entityID, service, ok := smarthome.ParseCmd(t.Cmd); ok { + if e.home == nil { + return "", fmt.Errorf("%w: %s is a house tool and no smarthome block is configured", ErrNotConnected, name) + } + // The confirm turn on a house row is structural, not a column. The + // proposal is written destructive=true, but /tools writes the checkbox + // straight through on enable (destructive=excluded.destructive), so + // unticking it once turned home_lock_front_door_unlock into a row that + // ran on first hearing. Nothing any surface writes can remove the + // second turn from a physical device. + if !confirmed { + return "", ErrNeedsConfirm + } + ctx, cancel := context.WithTimeout(ctx, e.timeout) + defer cancel() + return e.home.CallService(ctx, entityID, service) + } argv := append(append([]string(nil), t.Cmd...), args...) if len(argv) == 0 { return "", ErrNotEnabled diff --git a/internal/tool/tool_test.go b/internal/tool/tool_test.go index a28b4bb..5b63a64 100644 --- a/internal/tool/tool_test.go +++ b/internal/tool/tool_test.go @@ -85,3 +85,217 @@ func TestExec(t *testing.T) { t.Fatal("proposed tool must not match (not enabled)") } } + +// fakeMCP records what the executor asked it to call. +type fakeMCP struct { + server, tool string + args []string + out string + err error + calls int +} + +func (f *fakeMCP) CallPositional(_ context.Context, server, tool string, args []string) (string, error) { + f.calls++ + f.server, f.tool, f.args = server, tool, args + return f.out, f.err +} + +// An MCP row dispatches to the caller instead of a process, and the process +// seam is never touched. +func TestExecMCPRowDispatchesToMCP(t *testing.T) { + api := fakeAPI{tools: map[string]ipc.Tool{ + "vikunja_list_tasks": { + Name: "vikunja_list_tasks", Status: "enabled", Scope: "mcp:vikunja", + Cmd: []string{"mcp", "vikunja", "list_tasks"}, + }, + }} + m := &fakeMCP{out: "две задачи"} + ran := false + e := NewExecutor(api, time.Second).WithMCP(m) + e.run = func(context.Context, []string) (string, error) { ran = true; return "", nil } + + out, err := e.Exec(context.Background(), "vikunja_list_tasks", []string{"мавен"}, false) + if err != nil { + t.Fatalf("exec: %v", err) + } + if out != "две задачи" { + t.Fatalf("out = %q", out) + } + if ran { + t.Fatal("an MCP row must not be executed as a process") + } + if m.server != "vikunja" || m.tool != "list_tasks" || len(m.args) != 1 || m.args[0] != "мавен" { + t.Fatalf("dispatched wrong: %+v", m) + } +} + +// The allowlist rules still apply to an MCP row: destructive means a confirm +// turn first, and nothing is called until the second turn. +func TestExecMCPRowStillNeedsConfirm(t *testing.T) { + api := fakeAPI{tools: map[string]ipc.Tool{ + "vikunja_delete_task": { + Name: "vikunja_delete_task", Status: "enabled", Destructive: true, + Cmd: []string{"mcp", "vikunja", "delete_task"}, + }, + }} + m := &fakeMCP{out: "удалила"} + e := NewExecutor(api, time.Second).WithMCP(m) + if _, err := e.Exec(context.Background(), "vikunja_delete_task", nil, false); !errors.Is(err, ErrNeedsConfirm) { + t.Fatalf("err = %v, want ErrNeedsConfirm", err) + } + if m.calls != 0 { + t.Fatal("a destructive MCP tool must not reach the server before confirmation") + } + if _, err := e.Exec(context.Background(), "vikunja_delete_task", nil, true); err != nil { + t.Fatalf("confirmed exec: %v", err) + } + if m.calls != 1 { + t.Fatalf("calls = %d", m.calls) + } +} + +// A proposed MCP row does not run, exactly like a proposed shell tool. +func TestExecMCPRowNotEnabled(t *testing.T) { + api := fakeAPI{tools: map[string]ipc.Tool{ + "vikunja_list_tasks": {Name: "vikunja_list_tasks", Status: "proposed", Cmd: []string{"mcp", "vikunja", "list_tasks"}}, + }} + m := &fakeMCP{} + e := NewExecutor(api, time.Second).WithMCP(m) + if _, err := e.Exec(context.Background(), "vikunja_list_tasks", nil, false); !errors.Is(err, ErrNotEnabled) { + t.Fatalf("err = %v", err) + } + if m.calls != 0 { + t.Fatal("a proposal must not call anything") + } +} + +// With MCP unconfigured, an MCP row refuses rather than trying to exec "mcp". +func TestExecMCPRowWithoutCallerRefuses(t *testing.T) { + api := fakeAPI{tools: map[string]ipc.Tool{ + "vikunja_list_tasks": {Name: "vikunja_list_tasks", Status: "enabled", Cmd: []string{"mcp", "vikunja", "list_tasks"}}, + }} + ran := false + e := NewExecutor(api, time.Second) + e.run = func(context.Context, []string) (string, error) { ran = true; return "", nil } + err := func() error { _, e2 := e.Exec(context.Background(), "vikunja_list_tasks", nil, false); return e2 }() + // ErrNotConnected, NOT ErrNotEnabled: the act path turns ErrNotEnabled into + // a fresh proposal, and drafting a proposal for a row that already exists + // and is enabled answers the wrong question. + if !errors.Is(err, ErrNotConnected) { + t.Fatalf("err = %v, want ErrNotConnected", err) + } + if errors.Is(err, ErrNotEnabled) { + t.Fatal("an enabled row with a missing backend must not read as not-enabled") + } + if ran { + t.Fatal(`"mcp" must never be run as a binary`) + } +} + +// fakeHome records what the executor asked the house to do. +type fakeHome struct { + entity, service string + calls int +} + +func (f *fakeHome) CallService(_ context.Context, entityID, service string) (string, error) { + f.calls++ + f.entity, f.service = entityID, service + return "готово", nil +} + +// A house row goes through the same allowlist and the same confirm turn as any +// other act, and it is never exec'd as a binary (Vikunja #256). +func TestExecSmartHomeRow(t *testing.T) { + api := fakeAPI{tools: map[string]ipc.Tool{ + "home_light_x_off": { + Name: "home_light_x_off", Scope: "smarthome:light", + Cmd: []string{"smarthome", "light.x", "turn_off"}, Destructive: true, Status: "enabled", + }, + "home_draft": { + Name: "home_draft", Scope: "smarthome:light", + Cmd: []string{"smarthome", "light.y", "turn_on"}, Destructive: true, Status: "proposed", + }, + }} + ran := false + newExec := func(h HomeCaller) *Executor { + e := NewExecutor(api, time.Second) + e.run = func(context.Context, []string) (string, error) { ran = true; return "", nil } + if h != nil { + e = e.WithHome(h) + } + return e + } + + // No house configured ⇒ the row refuses rather than being exec'd. + if _, err := newExec(nil).Exec(context.Background(), "home_light_x_off", nil, true); !errors.Is(err, ErrNotConnected) { + t.Fatalf("unconfigured house: err = %v, want ErrNotConnected", err) + } + if ran { + t.Fatal(`"smarthome" was run as a binary`) + } + + // Configured, but not confirmed ⇒ the confirm turn, before any call. + fh := &fakeHome{} + if _, err := newExec(fh).Exec(context.Background(), "home_light_x_off", nil, false); !errors.Is(err, ErrNeedsConfirm) { + t.Fatalf("err = %v, want ErrNeedsConfirm", err) + } + if fh.calls != 0 { + t.Fatal("an unconfirmed house act reached the house") + } + + // A merely proposed row never runs, confirmed or not. + if _, err := newExec(fh).Exec(context.Background(), "home_draft", nil, true); !errors.Is(err, ErrNotEnabled) { + t.Fatalf("proposed row: err = %v, want ErrNotEnabled", err) + } + if fh.calls != 0 { + t.Fatal("a proposed house row reached the house") + } + + // Confirmed ⇒ the service call, with the entity from the ROW and the + // spoken tail dropped. + out, err := newExec(fh).Exec(context.Background(), "home_light_x_off", []string{"light.somewhere_else"}, true) + if err != nil { + t.Fatalf("Exec: %v", err) + } + if out != "готово" { + t.Errorf("out = %q", out) + } + if fh.entity != "light.x" || fh.service != "turn_off" { + t.Errorf("called %s/%s: the target must come from the enabled row, never from the utterance", fh.entity, fh.service) + } + if ran { + t.Fatal(`"smarthome" was run as a binary`) + } +} + +// The confirm turn on a house row survives the destructive column being wrong. +// ProposeSmartHomeTool writes destructive=true, but /tools reads the checkbox +// from the form and EnableTool writes destructive=excluded.destructive, so +// unticking it once turned home_lock_front_door_unlock into a row that opened +// the front door on first hearing. The guarantee has to be structural. +func TestExecSmartHomeRowConfirmsEvenWhenNotMarkedDestructive(t *testing.T) { + api := fakeAPI{tools: map[string]ipc.Tool{ + "home_lock_front_door_unlock": { + Name: "home_lock_front_door_unlock", Scope: "smarthome:lock", + Cmd: []string{"smarthome", "lock.front_door", "unlock"}, + // The column Kami unticked on /tools. + Destructive: false, Status: "enabled", + }, + }} + fh := &fakeHome{} + e := NewExecutor(api, time.Second).WithHome(fh) + if _, err := e.Exec(context.Background(), "home_lock_front_door_unlock", nil, false); !errors.Is(err, ErrNeedsConfirm) { + t.Fatalf("err = %v, want ErrNeedsConfirm", err) + } + if fh.calls != 0 { + t.Fatal("the front door was unlocked without a confirm turn") + } + if _, err := e.Exec(context.Background(), "home_lock_front_door_unlock", nil, true); err != nil { + t.Fatalf("confirmed: %v", err) + } + if fh.calls != 1 { + t.Fatalf("calls = %d, want 1 after the confirm turn", fh.calls) + } +} diff --git a/internal/update/apply.go b/internal/update/apply.go new file mode 100644 index 0000000..e839d70 --- /dev/null +++ b/internal/update/apply.go @@ -0,0 +1,282 @@ +package update + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +// Result — the full account of one Apply. Every field is filled in on the +// failure paths too, because "what state is my box in" is the only question that +// matters after a failed update. +type Result struct { + Verified bool + SnapshotID string // the rollback target; named even when the rollback failed + Installed []string + Restarted bool + Healthy bool + RolledBack bool + // RollbackHealthy — whether she answered again after the restore. False with + // RolledBack true is the manual-recovery case. + RollbackHealthy bool + Steps []Step + Took time.Duration +} + +// Apply is the whole update, in the only order that is safe. +// +// It is called by a human running cmd/mavupdate on the box. Nothing else calls +// it: no timer, no IPC method, no web route, no act. See the package comment. +func (u *Updater) Apply(ctx context.Context) (Result, error) { + start := u.now() + res := Result{} + defer func() { res.Took = u.now().Sub(start) }() + + // 0. She has to be answering before we start. Otherwise a failed update and + // a box that was already broken look identical afterwards, and the rollback + // has no baseline to prove itself against. + u.log("preflight: checking the running daemon") + if err := u.health(ctx, u.cfg.HealthSocket); err != nil { + if errors.Is(err, ErrHealthDial) { + // Not the daemon's fault and not fixed by fixing the daemon. The + // usual cause is the socket path: under a docker volume it sits in + // /var/lib/docker, which the operator's account cannot traverse. + return res, err + } + return res, fmt.Errorf("%w: %v", ErrUnhealthyBefore, err) + } + + // 0b. If the source is part of what a rollback has to put back, it must be + // in a state that can be described and restored. A dirty tree is neither: + // the commit recorded in the snapshot does not say what is deployed, and a + // forced checkout on the way back would delete his uncommitted work. + commit := u.gitHead(ctx) + if u.cfg.SourceRollback == "git" { + if commit == "" { + return res, fmt.Errorf("%w: source_rollback is \"git\" but %s has no readable git HEAD", ErrSourceRollback, u.cfg.SourceDir) + } + if dirty, err := u.gitDirty(ctx); err != nil { + return res, fmt.Errorf("%w: %v", ErrDirtyTree, err) + } else if dirty { + return res, fmt.Errorf("%w: %s", ErrDirtyTree, u.cfg.SourceDir) + } + } + + // 1. Snapshot what is deployed now, BEFORE the build. + // + // The order matters and it is not the obvious one. `make build` writes its + // binaries into the working tree, and on the docker deployment the working + // tree IS the install dir — so snapshotting after the build would snapshot + // the new artifacts and leave nothing to roll back to. The snapshot is the + // only thing standing between a bad build and a box that needs a screwdriver, + // so it is taken first, while the deployed bytes are still the old ones. + names := append(append([]string{}, u.cfg.Binaries...), u.cfg.ConfigFiles...) + snap, err := u.store.Save(u.cfg.InstallDir, names, commit, "pre-update") + if err != nil { + return res, err + } + res.SnapshotID = snap.ID + u.log("snapshot: %s (%d files) in %s", snap.ID, len(snap.Files), snap.Dir()) + + // 2. Build and test before anything is deployed. A broken tree costs time + // and nothing else — but `make build` has already overwritten the binaries in + // the tree, so restore them: otherwise a later restart by hand would deploy + // code that failed its own tests. Nothing has been restarted, so this is a + // file restore with no restart and no health check. + steps, err := u.Verify(ctx) + res.Steps = append(res.Steps, steps...) + if err != nil { + // RolledBack stays false here on purpose. Nothing was installed and + // nothing was restarted, so there is no rollback to report; a compile + // error printing rolled_back=true sends the operator looking for a + // restart that never happened. The log line carries what was done. + if rerr := u.restoreBinaries(snap); rerr != nil { + u.log("verify failed and the artifacts could not be put back: %v — the previous ones are in %s", rerr, snap.Dir()) + } else { + u.log("verify failed; the previously deployed artifacts are back in place, she was never restarted") + } + return res, err + } + res.Verified = true + + // 3. Install. Per-file temp+rename, so an interruption leaves whole files. + // Config is snapshotted but never overwritten — an update does not get to + // replace the operator's config. + installed, err := u.install() + res.Installed = installed + if err != nil { + // Files may be half-swapped across the set, so restore before returning + // even though nothing has been restarted yet. + u.log("install failed: %v — restoring", err) + return u.rollback(ctx, snap, res, err) + } + u.log("install: %d artifact(s) into %s", len(installed), u.cfg.InstallDir) + + // 4. Restart, then 5. prove she answers. + if err := u.restart(ctx, &res); err != nil { + return u.rollback(ctx, snap, res, err) + } + u.log("restart: ok, waiting for her to answer (up to %s)", u.cfg.healthTimeout()) + if err := u.waitHealthy(ctx, u.cfg.healthTimeout()); err != nil { + return u.rollback(ctx, snap, res, err) + } + res.Healthy = true + u.log("health: she answers on %s — update committed", u.cfg.HealthSocket) + + if err := u.store.Prune(u.cfg.KeepSnapshots); err != nil { + u.log("prune: %v (harmless)", err) + } + return res, nil +} + +// Rollback restores a snapshot by id (empty = the newest) and restarts. Exposed +// separately so the operator can undo an update that verified, restarted and +// answered a Presence call but is wrong in a way no health check can see. +func (u *Updater) Rollback(ctx context.Context, id string) (Result, error) { + var snap Snapshot + var err error + if id == "" { + snaps, lerr := u.store.List() + if lerr != nil { + return Result{}, lerr + } + if len(snaps) == 0 { + return Result{}, errors.New("update: no snapshots to roll back to") + } + snap = snaps[0] + } else if snap, err = u.store.Load(id); err != nil { + return Result{}, err + } + res := Result{SnapshotID: snap.ID} + return u.rollback(ctx, snap, res, errors.New("operator asked for a rollback")) +} + +// rollback restores the snapshot and restarts, then reports whether that worked. +// It depends on nothing that the update changed: file copies out of the snapshot +// dir and the same restart command. No build, no migration, no cooperation from +// the code being replaced. +func (u *Updater) rollback(ctx context.Context, snap Snapshot, res Result, cause error) (Result, error) { + // A rollback interrupted halfway is the one outcome worse than the failure + // that triggered it, so it does not inherit the caller's cancellation: a + // Ctrl-C during the health wait must not abandon the restore mid-restart. + ctx = context.WithoutCancel(ctx) + res.RolledBack = true + u.log("rollback: restoring snapshot %s over %s", snap.ID, u.cfg.InstallDir) + if err := u.restoreBinaries(snap); err != nil { + u.log("rollback: RESTORE FAILED: %v", err) + return res, fmt.Errorf("%w: %v (after %v); the previous artifacts are in %s — copy them back by hand", ErrRollbackFailed, err, cause, snap.Dir()) + } + // On a deployment that rebuilds from source, putting the binaries back is + // the part that changes nothing. The source is what the restart deploys, so + // it goes back too, and it goes back before the restart that reads it. + if err := u.restoreSource(ctx, snap); err != nil { + u.log("rollback: SOURCE CHECKOUT FAILED: %v", err) + return res, fmt.Errorf("%w: %v (after %v); the tree is still on the new commit, so a restart would redeploy it — `git -C %s checkout --force %s` by hand", ErrRollbackFailed, err, cause, u.cfg.SourceDir, snap.Commit) + } + // A restore with no restart leaves the failed process running, so a failed + // restart here is still the manual-recovery case. + if err := u.restart(ctx, &res); err != nil { + u.log("rollback: RESTART FAILED: %v", err) + return res, fmt.Errorf("%w: restored %s but the restart failed: %v (after %v)", ErrRollbackFailed, snap.ID, err, cause) + } + if err := u.waitHealthy(ctx, u.cfg.healthTimeout()); err != nil { + u.log("rollback: she still does not answer: %v", err) + return res, fmt.Errorf("%w: restored %s and restarted but she does not answer: %v (after %v)", ErrRollbackFailed, snap.ID, err, cause) + } + res.RollbackHealthy = true + u.log("rollback: she answers again on the previous build (%s)", snap.ID) + return res, fmt.Errorf("%w to %s: %v", ErrRolledBack, snap.ID, cause) +} + +// install copies the freshly built binaries from SourceDir into InstallDir. +// +// When the two are the same directory — the docker deployment builds the image +// from the working tree — this is a no-op by design rather than by accident: the +// artifacts are already where they belong and the restart command rebuilds the +// image from them. +func (u *Updater) install() ([]string, error) { + if filepath.Clean(u.cfg.SourceDir) == filepath.Clean(u.cfg.InstallDir) { + return u.cfg.Binaries, nil + } + var done []string + for _, name := range u.cfg.Binaries { + src := filepath.Join(u.cfg.SourceDir, name) + fi, err := os.Stat(src) + if err != nil { + return done, fmt.Errorf("update: install %s: %w (did `make build` produce it?)", name, err) + } + if _, err := copyFile(src, filepath.Join(u.cfg.InstallDir, name), fi.Mode().Perm()); err != nil { + return done, fmt.Errorf("update: install %s: %w", name, err) + } + done = append(done, name) + } + return done, nil +} + +func (u *Updater) restart(ctx context.Context, res *Result) error { + u.log("restart: %v", u.cfg.RestartCmd) + out, err := u.run(ctx, u.cfg.SourceDir, u.cfg.RestartCmd) + if err != nil { + res.Steps = append(res.Steps, Step{Name: "restart", Argv: u.cfg.RestartCmd, Err: err, Output: tail(out, 4000)}) + return fmt.Errorf("update: restart %v: %w", u.cfg.RestartCmd, err) + } + res.Restarted = true + res.Steps = append(res.Steps, Step{Name: "restart", Argv: u.cfg.RestartCmd}) + return nil +} + +// restoreBinaries puts back the built artifacts and nothing else. +// +// ConfigFiles are snapshotted and deliberately not restored. The Config doc +// says an update never replaces the operator's config, and a rollback that +// quietly reverted deploy/mavend.json would undo edits made since the last +// apply — a phraser.model_path change among them, which is how the resident +// model gets swapped. The copies stay in the snapshot dir for him to take by +// hand if the config is what he wants back. +func (u *Updater) restoreBinaries(snap Snapshot) error { + return snap.RestoreOnly(u.cfg.InstallDir, u.cfg.Binaries) +} + +// restoreSource puts the working tree back on the commit the snapshot was taken +// at, for the deployments where that is what the restart command deploys. A +// no-op for every other shape. +func (u *Updater) restoreSource(ctx context.Context, snap Snapshot) error { + if u.cfg.SourceRollback != "git" { + return nil + } + if snap.Commit == "" { + return fmt.Errorf("snapshot %s records no commit, so there is nothing to check out", snap.ID) + } + u.log("rollback: checking %s back out to %s", u.cfg.SourceDir, snap.Commit) + // --force because the failed build left artifacts in the tree. Safe only + // because Apply refused to start on a dirty tree, so nothing uncommitted of + // his is in reach. + out, err := u.run(ctx, u.cfg.SourceDir, []string{"git", "checkout", "--force", snap.Commit}) + if err != nil { + return fmt.Errorf("git checkout %s: %v: %s", snap.Commit, err, tail(out, 1000)) + } + return nil +} + +// gitDirty reports whether the working tree has uncommitted changes. +func (u *Updater) gitDirty(ctx context.Context) (bool, error) { + out, err := u.run(ctx, u.cfg.SourceDir, []string{"git", "status", "--porcelain"}) + if err != nil { + return false, fmt.Errorf("git status in %s: %v", u.cfg.SourceDir, err) + } + return strings.TrimSpace(out) != "", nil +} + +// gitHead records which commit produced a snapshot, for the operator's benefit. +// Best-effort: a tree without git is not a reason to refuse to snapshot. +func (u *Updater) gitHead(ctx context.Context) string { + out, err := u.run(ctx, u.cfg.SourceDir, []string{"git", "rev-parse", "HEAD"}) + if err != nil { + return "" + } + return strings.TrimSpace(out) +} diff --git a/internal/update/health.go b/internal/update/health.go new file mode 100644 index 0000000..621f78a --- /dev/null +++ b/internal/update/health.go @@ -0,0 +1,100 @@ +package update + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/kami/maven/internal/ipc" +) + +// The health check is the whole basis for rolling back, so it has to mean +// something. "The process is running" does not: mavend can be up with a dead +// store, a socket it never bound, or a config it failed to parse. What is +// checked instead is that she answers a real read over the real IPC socket — +// which exercises the socket, the dispatch table and the store in one call. +// +// Presence is the method used because it is read-only (safe to retry), needs no +// arguments, and touches the store. It cannot write anything, so a health check +// never leaves a trace in her memory. +// +// A locked daemon is healthy. With a passkey enrolled and no env key mavend +// boots locked and refuses every CoreAPI method until an assertion arrives, so +// a Presence read there fails for a daemon that came up perfectly. Rolling back +// on that would turn a good update into the manual-recovery case, and the +// rollback would boot locked too. MethodPing reaches no store and answers in +// locked mode, so it is asked first: an answer of "locked" is proof of life and +// is where the check stops. + +// ErrHealthDial — the socket could not be opened at all. Kept apart from a +// failed read because the two have different causes and different fixes: on the +// docker deployment the socket lives under /var/lib/docker, which is +// drwx--x--- root root, so a non-root operator gets EACCES before reaching +// mavend. "She is not answering" would be the wrong thing to tell him. +var ErrHealthDial = errors.New("update: cannot open the health socket") + +// DialHealth connects to the mavend socket and proves someone is serving it. +func DialHealth(ctx context.Context, socket string) error { + c, err := ipc.Dial(socket) + if err != nil { + return fmt.Errorf("%w %s: %v", ErrHealthDial, socket, err) + } + defer c.Close() + // Liveness first, because it is the only question a locked daemon can + // answer. ErrUnknownMethod means an older mavend on the other end, which is + // exactly the case during a rollback to a build from before ping existed — + // fall through to the store read rather than calling that a failure. + switch p, perr := c.Ping(ctx); { + case perr == nil && p.Locked: + return nil + case perr != nil && !errors.Is(perr, ipc.ErrUnknownMethod): + return fmt.Errorf("update: health ping: %w", perr) + } + if _, err := c.Presence(ctx); err != nil { + return fmt.Errorf("update: health read: %w", err) + } + return nil +} + +// waitHealthy retries the health check until it passes or the timeout elapses. +// A restart is not instantaneous — she loads a 1.7B on boot — so the first few +// failures are expected and are not a reason to roll back. +func (u *Updater) waitHealthy(ctx context.Context, timeout time.Duration) error { + deadline := u.now().Add(timeout) + delay := 500 * time.Millisecond + var last error + for { + // Cap the attempt at whatever is left of the budget, not a flat 10s: an + // attempt starting at 89s of a 90s timeout would otherwise run to 99s, + // and the caller asked for 90. + attempt := 10 * time.Second + if left := deadline.Sub(u.now()); left < attempt { + attempt = left + } + if attempt <= 0 { + if last == nil { + last = context.DeadlineExceeded + } + return fmt.Errorf("update: not healthy after %s: %w", timeout, last) + } + attemptCtx, cancel := context.WithTimeout(ctx, attempt) + err := u.health(attemptCtx, u.cfg.HealthSocket) + cancel() + if err == nil { + return nil + } + last = err + if u.now().After(deadline) { + return fmt.Errorf("update: not healthy after %s: %w", timeout, last) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(delay): + } + if delay < 5*time.Second { + delay *= 2 + } + } +} diff --git a/internal/update/snapshot.go b/internal/update/snapshot.go new file mode 100644 index 0000000..f591f9e --- /dev/null +++ b/internal/update/snapshot.go @@ -0,0 +1,282 @@ +package update + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "time" +) + +// A snapshot is a byte-for-byte copy of the deployed artifacts plus a manifest +// of their sha256 sums, taken before an install. +// +// It is copies, not hardlinks and not a git stash, for one reason: the restore +// path must work when everything else is broken. A hardlink into the install dir +// would be clobbered by the very install it exists to undo, and a git-based +// undo needs a toolchain, a clean tree, and a rebuild — three things a failed +// update is likely to have taken away. Copying two dozen megabytes of Go +// binaries costs a second and needs nothing but the filesystem. +// +// The sums are what make a restore verifiable rather than hopeful: Restore +// re-hashes every file it writes, so "the old bytes are back" is checked, not +// assumed. + +// FileRec — one file in a snapshot. +type FileRec struct { + Name string `json:"name"` // relative name inside the install dir + SHA256 string `json:"sha256"` // of the snapshotted bytes + Mode os.FileMode `json:"mode"` + Size int64 `json:"size"` +} + +// Snapshot — the manifest. Written last, so a directory without a readable +// manifest.json is an aborted snapshot and is never offered as a rollback target. +type Snapshot struct { + ID string `json:"id"` // sortable timestamp, also the directory name + CreatedAt time.Time `json:"created_at"` + Commit string `json:"commit,omitempty"` // git HEAD of the tree that produced it, when known + Note string `json:"note,omitempty"` + Files []FileRec `json:"files"` + + dir string // absolute path, filled in by List/Load +} + +// Dir — where this snapshot's file copies live. +func (s Snapshot) Dir() string { return s.dir } + +const manifestName = "manifest.json" + +// Store is a directory of snapshots. +type Store struct { + Dir string + now func() time.Time +} + +func (st *Store) clock() time.Time { + if st.now != nil { + return st.now() + } + return time.Now() +} + +// Save copies names (relative to srcDir) into a new snapshot and writes the +// manifest. A name that does not exist is skipped rather than fatal: the first +// ever run happens on a box where some artifact may legitimately be missing, and +// refusing to snapshot then would mean refusing to update. +func (st *Store) Save(srcDir string, names []string, commit, note string) (Snapshot, error) { + ts := st.clock().UTC() + snap := Snapshot{ + ID: ts.Format("20060102-150405"), + CreatedAt: ts, + Commit: commit, + Note: note, + } + snap.dir = filepath.Join(st.Dir, snap.ID) + if err := os.MkdirAll(snap.dir, 0o700); err != nil { + return Snapshot{}, fmt.Errorf("update: snapshot dir: %w", err) + } + for _, name := range names { + src := filepath.Join(srcDir, name) + fi, err := os.Stat(src) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + continue + } + return Snapshot{}, fmt.Errorf("update: snapshot %s: %w", name, err) + } + if fi.IsDir() { + return Snapshot{}, fmt.Errorf("update: snapshot %s: is a directory (only files are deployable artifacts)", name) + } + dst := filepath.Join(snap.dir, name) + if err := os.MkdirAll(filepath.Dir(dst), 0o700); err != nil { + return Snapshot{}, err + } + sum, err := copyFile(src, dst, fi.Mode().Perm()) + if err != nil { + return Snapshot{}, fmt.Errorf("update: snapshot %s: %w", name, err) + } + snap.Files = append(snap.Files, FileRec{Name: name, SHA256: sum, Mode: fi.Mode().Perm(), Size: fi.Size()}) + } + if len(snap.Files) == 0 { + os.RemoveAll(snap.dir) + return Snapshot{}, fmt.Errorf("update: snapshot of %s is empty — none of the listed artifacts exist", srcDir) + } + // Manifest last: its presence is what makes the snapshot usable. + blob, err := json.MarshalIndent(snap, "", " ") + if err != nil { + return Snapshot{}, err + } + if err := os.WriteFile(filepath.Join(snap.dir, manifestName), blob, 0o600); err != nil { + return Snapshot{}, fmt.Errorf("update: snapshot manifest: %w", err) + } + return snap, nil +} + +// List returns the complete snapshots, newest first. +func (st *Store) List() ([]Snapshot, error) { + ents, err := os.ReadDir(st.Dir) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + return nil, err + } + var out []Snapshot + for _, e := range ents { + if !e.IsDir() { + continue + } + s, err := st.Load(e.Name()) + if err != nil { + continue // aborted or hand-mangled: not a rollback target + } + out = append(out, s) + } + sort.Slice(out, func(i, j int) bool { return out[i].ID > out[j].ID }) + return out, nil +} + +// Load reads one snapshot's manifest. +func (st *Store) Load(id string) (Snapshot, error) { + dir := filepath.Join(st.Dir, id) + blob, err := os.ReadFile(filepath.Join(dir, manifestName)) + if err != nil { + return Snapshot{}, err + } + var s Snapshot + if err := json.Unmarshal(blob, &s); err != nil { + return Snapshot{}, fmt.Errorf("update: manifest %s: %w", id, err) + } + s.dir = dir + return s, nil +} + +// Restore copies a snapshot's files back over dstDir and verifies every write +// against the manifest sum. Only the named files are touched; anything else in +// dstDir is left alone. +// +// This is the function the whole package exists to be able to run. It uses the +// filesystem and nothing else — no toolchain, no build, no cooperation from the +// code being replaced. +func (s Snapshot) Restore(dstDir string) error { return s.RestoreOnly(dstDir, nil) } + +// RestoreOnly is Restore limited to the named files. A nil list means all of +// them. The caller uses it to put binaries back without putting config back: +// see Updater.restoreBinaries. +func (s Snapshot) RestoreOnly(dstDir string, names []string) error { + if s.dir == "" { + return errors.New("update: snapshot has no directory (load it through the store)") + } + var want map[string]bool + if names != nil { + want = make(map[string]bool, len(names)) + for _, n := range names { + want[n] = true + } + } + for _, f := range s.Files { + if want != nil && !want[f.Name] { + continue + } + src := filepath.Join(s.dir, f.Name) + sum, err := hashFile(src) + if err != nil { + return fmt.Errorf("update: restore %s: %w", f.Name, err) + } + if sum != f.SHA256 { + return fmt.Errorf("update: restore %s: snapshot is corrupt (sha256 %s, manifest says %s)", f.Name, sum, f.SHA256) + } + dst := filepath.Join(dstDir, f.Name) + if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + return err + } + got, err := copyFile(src, dst, f.Mode) + if err != nil { + return fmt.Errorf("update: restore %s: %w", f.Name, err) + } + if got != f.SHA256 { + return fmt.Errorf("update: restore %s: wrote the wrong bytes (sha256 %s)", f.Name, got) + } + } + return nil +} + +// Prune keeps the newest keep snapshots and removes the rest. The newest is +// never pruned regardless of keep — it is the rollback target. +func (st *Store) Prune(keep int) error { + if keep < 1 { + keep = 1 + } + snaps, err := st.List() + if err != nil { + return err + } + for _, s := range snaps[min(keep, len(snaps)):] { + if err := os.RemoveAll(s.dir); err != nil { + return err + } + } + return nil +} + +// copyFile writes src to dst atomically (temp + rename, so a reader never sees a +// half file and an interrupted copy leaves the old one intact) and returns the +// sha256 of what was written. +func copyFile(src, dst string, mode os.FileMode) (string, error) { + in, err := os.Open(src) + if err != nil { + return "", err + } + defer in.Close() + if mode == 0 { + mode = 0o644 + } + tmp, err := os.CreateTemp(filepath.Dir(dst), ".update-*") + if err != nil { + return "", err + } + tmpName := tmp.Name() + defer os.Remove(tmpName) // no-op once the rename succeeds + h := sha256.New() + if _, err := io.Copy(io.MultiWriter(tmp, h), in); err != nil { + tmp.Close() + return "", err + } + // fsync before the rename: a binary that is renamed into place but whose + // bytes are still in the page cache is exactly the file a power cut turns + // into an unbootable daemon. + if err := tmp.Sync(); err != nil { + tmp.Close() + return "", err + } + if err := tmp.Chmod(mode); err != nil { + tmp.Close() + return "", err + } + if err := tmp.Close(); err != nil { + return "", err + } + if err := os.Rename(tmpName, dst); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +func hashFile(p string) (string, error) { + f, err := os.Open(p) + if err != nil { + return "", err + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/internal/update/source_test.go b/internal/update/source_test.go new file mode 100644 index 0000000..e3914eb --- /dev/null +++ b/internal/update/source_test.go @@ -0,0 +1,193 @@ +package update + +import ( + "context" + "errors" + "path/filepath" + "strings" + "testing" +) + +// The deployment the README documents builds the image from the working tree: +// source_dir and install_dir are the same path, the Dockerfile copies cmd/ and +// internal/ and runs the build inside the builder stage, and .dockerignore +// keeps the host binaries out. Restoring binaries there restores bytes nothing +// reads. These tests pin the two acceptable outcomes: the source goes back, or +// the config is refused. + +// dockerBox — a fakeBox wired the way the README wires the docker deployment. +func dockerBox(t *testing.T, sourceRollback string) (*fakeBox, Config) { + t.Helper() + b := newFakeBox(t) + // The source IS the deployment. The old commit's source is what the running + // image was built from. + b.byCommit["old-commit"] = "GOOD-SOURCE" + b.commit = "old-commit" + write(t, filepath.Join(b.root, "src", "source.go"), "BAD-SOURCE") + + cfg := b.cfg() + cfg.InstallDir = cfg.SourceDir + cfg.ConfigFiles = nil + cfg.SourceRollback = sourceRollback + // The artifacts the docker shape snapshots live in the tree. + write(t, filepath.Join(b.root, "src", "mavend"), "OLD-BUILD") + return b, cfg +} + +func TestValidate_RefusesABuildFromSourceDeploymentThatCannotRollBack(t *testing.T) { + _, cfg := dockerBox(t, "") + err := cfg.Validate() + if !errors.Is(err, ErrSourceRollback) { + t.Fatalf("Validate = %v; want ErrSourceRollback — a binary snapshot rolls back nothing when the restart rebuilds from the tree", err) + } + if _, err := New(cfg); !errors.Is(err, ErrSourceRollback) { + t.Fatalf("New = %v; want the same refusal", err) + } +} + +func TestApply_RollbackPutsTheSourceBackBeforeTheRestart(t *testing.T) { + b, cfg := dockerBox(t, "git") + u, err := New(cfg, WithRunner(b.run), WithHealth(b.health)) + if err != nil { + t.Fatal(err) + } + // The new build compiles and passes, and then does not come up — the case + // this whole package exists for. + first := true + b.healthFn = func() error { + if first { + first = false + return nil // preflight + } + if len(b.builtFromSource) > 0 && b.builtFromSource[len(b.builtFromSource)-1] == "GOOD-SOURCE" { + return nil // she answers again once the good source is deployed + } + return errors.New("she does not answer") + } + + res, err := u.Apply(context.Background()) + if !errors.Is(err, ErrRolledBack) { + t.Fatalf("Apply = %v; want ErrRolledBack", err) + } + if !res.RollbackHealthy { + t.Fatalf("result = %+v; want a healthy rollback", res) + } + if len(b.builtFromSource) != 2 { + t.Fatalf("restarts = %v; want the bad one and the rolled-back one", b.builtFromSource) + } + if b.builtFromSource[1] != "GOOD-SOURCE" { + t.Errorf("the rollback restarted on %q; want the previous commit's source — restoring binaries alone redeploys the bad commit", b.builtFromSource[1]) + } + // And the checkout came before the restart, not after it. + var checkoutAt, restartAt = -1, -1 + for i, c := range b.ran { + if strings.HasPrefix(c, "git checkout") { + checkoutAt = i + } + if c == "restart-the-thing" { + restartAt = i + } + } + if checkoutAt < 0 || restartAt < checkoutAt { + t.Errorf("commands ran = %v; want the checkout before the last restart", b.ran) + } +} + +func TestApply_RefusesADirtyTreeWhenTheSourceIsTheRollbackTarget(t *testing.T) { + b, cfg := dockerBox(t, "git") + b.dirty = true + u, err := New(cfg, WithRunner(b.run), WithHealth(b.health)) + if err != nil { + t.Fatal(err) + } + if _, err := u.Apply(context.Background()); !errors.Is(err, ErrDirtyTree) { + t.Fatalf("Apply on a dirty tree = %v; want ErrDirtyTree", err) + } + for _, c := range b.ran { + if strings.HasPrefix(c, "make") || c == "restart-the-thing" { + t.Errorf("a refused update still ran %q", c) + } + } +} + +func TestApply_VerifyFailureDoesNotClaimARollback(t *testing.T) { + b := newFakeBox(t) + b.testErr = errors.New("exit status 1") + res, err := b.updater(t).Apply(context.Background()) + if !errors.Is(err, ErrVerifyFailed) { + t.Fatalf("Apply = %v; want ErrVerifyFailed", err) + } + if res.RolledBack { + t.Error("a compile error reported rolled_back=true; nothing was installed and nothing was restarted") + } +} + +func TestRollback_LeavesHisConfigAlone(t *testing.T) { + b := newFakeBox(t) + if _, err := b.updater(t).Apply(context.Background()); err != nil { + t.Fatalf("setup Apply: %v", err) + } + // He edits the config after the update — a phraser.model_path change, say. + cfgPath := filepath.Join(b.root, "install", "mavend.json") + write(t, cfgPath, `{"tick_interval":"90s"}`) + + if _, err := b.updater(t).Rollback(context.Background(), ""); err != nil && !errors.Is(err, ErrRolledBack) { + t.Fatalf("Rollback: %v", err) + } + if got := read(t, cfgPath); !strings.Contains(got, "90s") { + t.Errorf("config after rollback = %q; a rollback must not revert his config", got) + } + if got := b.deployed(); got != "OLD-BUILD" { + t.Errorf("deployed binary = %q; want the binaries rolled back", got) + } +} + +func TestVerify_RefusesToBuildHisTreeAsRoot(t *testing.T) { + b := newFakeBox(t) + u := b.updater(t, WithIDs(func(string) (int, uint32, error) { return 0, 1000, nil })) + if _, err := u.Verify(context.Background()); !errors.Is(err, ErrRootOnHisTree) { + t.Fatalf("Verify as root over a uid-1000 tree = %v; want ErrRootOnHisTree", err) + } + if len(b.ran) != 0 { + t.Errorf("the refusal still ran %v — root-owned artifacts would break his next make", b.ran) + } + // Root's own tree is fine, and so is a normal user. + for _, ids := range []func(string) (int, uint32, error){ + func(string) (int, uint32, error) { return 0, 0, nil }, + func(string) (int, uint32, error) { return 1000, 1000, nil }, + } { + if _, err := b.updater(t, WithIDs(ids)).Verify(context.Background()); err != nil { + t.Errorf("Verify refused a legitimate build: %v", err) + } + } +} + +func TestValidate_RefusesSnapshotsInsideTheSourceTree(t *testing.T) { + cfg := (&fakeBox{root: t.TempDir()}).cfg() + cfg.SnapshotDir = filepath.Join(cfg.SourceDir, "snaps") + if err := cfg.Validate(); err == nil { + t.Error("snapshot_dir inside source_dir was accepted; it lands in the docker build context") + } + cfg = (&fakeBox{root: t.TempDir()}).cfg() + cfg.SourceRollback = "svn" + if err := cfg.Validate(); err == nil { + t.Error("an unknown source_rollback was accepted") + } +} + +func TestTail_CutsOnARuneBoundary(t *testing.T) { + s := strings.Repeat("x", 20) + "--- FAIL: TestПривет" + got := tail(s, 10) + if !utf8ValidString(got) { + t.Errorf("tail(%q) = %q; a cut mid-rune shows as a replacement character", s, got) + } +} + +func utf8ValidString(s string) bool { + for _, r := range s { + if r == 0xFFFD { + return false + } + } + return true +} diff --git a/internal/update/update.go b/internal/update/update.go new file mode 100644 index 0000000..5dc53fa --- /dev/null +++ b/internal/update/update.go @@ -0,0 +1,373 @@ +// Package update applies a new build of Maven to the box she runs on, with a +// verified-before-committed install and an automatic rollback (Vikunja #249). +// +// # What this package refuses to be +// +// This is the highest-risk capability in the backlog — code that changes the +// running system — so the refusals are as much of the design as the features, +// and they are enforced here rather than described in a doc: +// +// - It is never automatic and never on a timer. There is no checker, no +// channel, no "check for updates" call and nothing that fires from the tick +// loop. Apply runs exactly when a human runs cmd/mavupdate on the box. +// - The daemon cannot update itself. mavend never constructs an Updater and +// nothing in the daemon can call Apply: there is no IPC method and no web +// route that reaches this package, so no act, no intent, no tool and no LLM +// output can start an update. (The package IS linked into mavend, via +// internal/config, which calls Config.Validate so a bad update block is +// caught at daemon startup rather than on the night it is needed. Linked is +// not reachable — the guarantee is the absent caller, not an absent +// import.) The trigger needs shell access to the host, which is a strictly +// higher bar than the step-up passkey gate that guards /tools — an update +// is not a thing to expose to anything reachable over the network. +// - It does not fetch code. Nothing here talks to a release server, a +// registry, or GitHub. The new version is whatever is in the working tree +// the operator points it at, which he pulled himself. Downloading and +// running code on the strength of a checksum in the same download is not a +// property we can verify on one box. +// - It does not supervise its own death. The plan asked for an in-process +// crash-loop detector; a process cannot reliably notice that it keeps +// dying, and one that thinks it can is worse than nothing. Restart-on-crash +// belongs to whatever starts mavend (compose `restart: unless-stopped`, +// systemd `Restart=`). What this package guarantees instead is narrower and +// real: within one Apply, the new build is proven to answer before the old +// one is considered replaced, and if it does not answer the old bytes go +// back and are proven to answer again. +// +// # The order of operations, and why +// +// Apply is: health-check the CURRENT daemon → snapshot → build → test → +// install → restart → health-check → rollback on any failure. +// +// The first health check is not ceremony. If she is already not answering, a +// failed update and a broken box are indistinguishable afterwards, and the +// rollback has nothing to prove itself against — so Apply refuses to start. +// +// The snapshot comes before the build, not after, and the comment in apply.go +// spells out why: `make build` writes into the working tree, which on the +// docker deployment IS the install dir, so a snapshot taken after it would +// snapshot the new artifacts. +// +// Build and test run BEFORE anything is written to the install dir, so a broken +// tree costs nothing but time. Install is per-file write-temp-then-rename, so a +// crash mid-install leaves whole files, not half ones. +// +// The rollback path deliberately depends on nothing that just changed: it copies +// byte-for-byte from a snapshot taken before the install and re-runs the same +// restart command. It does not ask the new binary to do anything, does not run +// a migration, and does not need the update to have gotten far enough to leave +// a working anything behind. +// +// # What the snapshot has to cover +// +// A rollback is only real if it puts back the thing the restart command +// deploys. For a bare-metal layout that is the binaries in InstallDir. For the +// docker layout it is not: the image is built from the source tree, and the +// host binaries never enter it. Restoring binaries there rebuilds the same bad +// image and burns a second health timeout proving it. So a deployment that +// rebuilds from source must say how the source is put back +// (Config.SourceRollback), and one that cannot say is refused by Validate +// rather than discovering it during the one rollback that mattered. +// +// # What an update is not +// +// It is not turn-safe. Nothing quiesces the daemon first: the restart kills the +// process mid-utterance if one is in flight. The model swap in +// internal/phraser drains, because a swap is a routine operation on a running +// box; an update is a deliberate restart and the operator picked the moment. +// +// # What is out of scope on purpose +// +// The database is not snapshotted or rolled back. It is encrypted, live, and +// often larger than the disk headroom; a store rolled back under a schema that +// already migrated forward loses writes silently, which is worse than a failed +// update. Schema compatibility is store.Migrate's job. A snapshot here is the +// deployable artifacts only: binaries and config. +package update + +import ( + "context" + "errors" + "fmt" + "os/exec" + "path/filepath" + "strings" + "time" +) + +var ( + // ErrNotConfigured — no update block in the config. The capability does not + // exist unless the operator described his own deployment. + ErrNotConfigured = errors.New("update: not configured") + + // ErrUnhealthyBefore — the daemon was already not answering when Apply + // started. Refused: see the package comment. + ErrUnhealthyBefore = errors.New("update: the running daemon is not healthy — refusing to update on top of a broken box") + + // ErrVerifyFailed — build or test failed. Nothing was installed. + ErrVerifyFailed = errors.New("update: verification failed") + + // ErrRolledBack — the new build was installed and did not come up healthy, + // so the previous snapshot was restored. Wraps the underlying failure. + ErrRolledBack = errors.New("update: rolled back") + + // ErrSourceRollback — the deployment rebuilds from source, and nothing in + // the config says how to put the source back. Refused at Validate: see + // Config.SourceRollback. + ErrSourceRollback = errors.New("update: this deployment rebuilds from source and has no way to roll the source back") + + // ErrDirtyTree — source_rollback is "git" and the working tree has + // uncommitted changes, so the recorded commit does not describe what is + // deployed and a checkout would throw work away. Refused before anything is + // built. + ErrDirtyTree = errors.New("update: the source tree has uncommitted changes — commit or stash them first") + + // ErrRootOnHisTree — running as root over a tree owned by somebody else. + // Refused: Verify runs `make build` and `make test` in SourceDir, and as + // root that leaves root-owned binaries, object files and a build cache in + // his working tree. His next ordinary `make` then fails, so one root apply + // breaks the normal build. This fires easily, because the documented health + // socket lives under a root-only directory and sudo is the obvious way past + // that. + ErrRootOnHisTree = errors.New("update: refusing to build someone else's tree as root — it would leave root-owned artifacts and break his next make") + + // ErrRollbackFailed — the worst case: the new build failed AND the restore + // did not bring her back. The operator has to fix the box by hand; the + // snapshot directory is named in the result so he knows what to copy. + ErrRollbackFailed = errors.New("update: ROLLBACK FAILED — manual recovery required") +) + +// Config — the operator's description of his own deployment. Every path is +// absolute and validated; nothing is guessed, because guessing wrong here means +// overwriting the wrong file. +type Config struct { + // SourceDir — the git working tree to build. The operator pulls it himself; + // this package never fetches. + SourceDir string `json:"source_dir"` + + // InstallDir — where the built binaries are copied to. On the docker + // deployment this is the tree the image is built from, so it is usually the + // same as SourceDir and Install is a no-op copy; on a bare-metal deployment + // it is /opt/maven/bin. + InstallDir string `json:"install_dir"` + + // SnapshotDir — where the pre-install copies live. Must not be inside + // InstallDir or SourceDir: a restore reading from a directory the install is + // writing to is not a restore, and a snapshot dir inside the source tree + // lands in the docker build context and in whatever make and git do there. + SnapshotDir string `json:"snapshot_dir"` + + // SourceRollback — how the SOURCE is put back when the deployment rebuilds + // from it. "" means it is not, which is only valid when the built binaries + // are what gets deployed. + // + // This exists because of what a rollback has to undo, which is not always + // the binaries. When RestartCmd is `docker compose up -d --build`, the image + // is built by the Dockerfile from cmd/ and internal/, and the host binaries + // are excluded by .dockerignore. Restoring them then restores bytes nothing + // reads: the restart rebuilds the same bad image from the same bad source, + // and the box stays down through two health timeouts for no reason. + // + // "git" makes the source part of the snapshot: the commit is recorded before + // the update and a rollback checks it back out before restarting. It + // requires a clean tree, because a recorded commit does not describe a dirty + // one and a forced checkout would throw uncommitted work away. + // + // Validate refuses a build-from-source deployment (SourceDir == InstallDir) + // that leaves this empty, rather than letting the operator find out during + // the one rollback he needed. + SourceRollback string `json:"source_rollback,omitempty"` + + // Binaries — the artifact names to snapshot and install, relative to + // SourceDir (built) and InstallDir (deployed). Listed explicitly rather than + // globbed so a stray file in the tree never gets deployed. + Binaries []string `json:"binaries"` + + // ConfigFiles — extra files to snapshot alongside the binaries, relative to + // InstallDir. Snapshotted and never written back: not by an install, and + // not by a rollback either. The operator's config is not something an update + // gets to replace, and a rollback that reverted it would silently undo every + // edit since the last apply. The copies are in the snapshot dir if he wants + // one back. + // + // The exception is source_rollback "git": a checkout moves every tracked + // file, config included. That is the same rollback the deployment needs to + // work at all, so on that shape a config edit belongs in a commit. + ConfigFiles []string `json:"config_files,omitempty"` + + // RestartCmd — how this deployment restarts mavend, e.g. + // ["docker","compose","up","-d","--build","mavend"] or + // ["systemctl","restart","mavend"]. Run in SourceDir. Required: there is no + // portable default and picking one would mean restarting the wrong thing. + RestartCmd []string `json:"restart_cmd"` + + // HealthSocket — mavend's IPC socket, used to prove she answers after a + // restart. Required: without a health check there is no signal to roll back + // on, and an update that cannot detect its own failure is not what this + // package is for. + HealthSocket string `json:"health_socket"` + + // HealthTimeoutSec — how long to wait for the restarted daemon to answer. + // Default 90s; she loads a 1.7B on boot, so this is not a couple of seconds. + HealthTimeoutSec int `json:"health_timeout_sec,omitempty"` + + // VerifyTimeoutMin — cap on `make build` + `make test`. Default 20m. + VerifyTimeoutMin int `json:"verify_timeout_min,omitempty"` + + // KeepSnapshots — how many snapshots to retain. Default 5, minimum 1: the + // most recent one is the rollback target and is never pruned. + KeepSnapshots int `json:"keep_snapshots,omitempty"` +} + +// Validate — fail at startup, not halfway through an install. +func (c Config) Validate() error { + if c.SourceDir == "" || c.InstallDir == "" || c.SnapshotDir == "" { + return errors.New("update: source_dir, install_dir and snapshot_dir are all required") + } + for _, p := range []string{c.SourceDir, c.InstallDir, c.SnapshotDir} { + if !filepath.IsAbs(p) { + return fmt.Errorf("update: %q must be an absolute path", p) + } + } + if within(c.SnapshotDir, c.InstallDir) { + return fmt.Errorf("update: snapshot_dir %q is inside install_dir %q — a restore must not read from what the install writes", c.SnapshotDir, c.InstallDir) + } + if within(c.SnapshotDir, c.SourceDir) { + return fmt.Errorf("update: snapshot_dir %q is inside source_dir %q — snapshots would land in the build context, and in whatever make and git do to that tree", c.SnapshotDir, c.SourceDir) + } + switch c.SourceRollback { + case "", "git": + default: + return fmt.Errorf("update: source_rollback %q is not a thing — use \"git\" or leave it out", c.SourceRollback) + } + if c.buildsFromSource() && c.SourceRollback == "" { + return fmt.Errorf("%w: source_dir and install_dir are both %q, so the restart deploys the tree and a restore of the binaries would undo nothing. Set \"source_rollback\": \"git\", or split the layout so install_dir holds what actually runs", ErrSourceRollback, c.SourceDir) + } + if len(c.Binaries) == 0 { + return errors.New("update: binaries is empty — nothing to install") + } + for _, b := range append(append([]string{}, c.Binaries...), c.ConfigFiles...) { + if filepath.IsAbs(b) || strings.Contains(b, "..") { + return fmt.Errorf("update: %q must be a plain relative name", b) + } + } + if len(c.RestartCmd) == 0 { + return errors.New("update: restart_cmd is required — there is no safe default for restarting someone else's deployment") + } + if c.HealthSocket == "" { + return errors.New("update: health_socket is required — an update that cannot check its own result cannot roll back on failure") + } + return nil +} + +func (c Config) withDefaults() Config { + if c.HealthTimeoutSec <= 0 { + c.HealthTimeoutSec = 90 + } + if c.VerifyTimeoutMin <= 0 { + c.VerifyTimeoutMin = 20 + } + if c.KeepSnapshots < 1 { + c.KeepSnapshots = 5 + } + return c +} + +// buildsFromSource — the deployment whose restart command rebuilds from the +// tree, which is what SourceDir == InstallDir means in practice (install is a +// no-op copy and the artifacts that matter are produced inside the image). +func (c Config) buildsFromSource() bool { + return filepath.Clean(c.SourceDir) == filepath.Clean(c.InstallDir) +} + +func (c Config) healthTimeout() time.Duration { + return time.Duration(c.HealthTimeoutSec) * time.Second +} + +func (c Config) verifyTimeout() time.Duration { + return time.Duration(c.VerifyTimeoutMin) * time.Minute +} + +// within reports whether p is dir or lives under it. +func within(p, dir string) bool { + p, dir = filepath.Clean(p), filepath.Clean(dir) + if p == dir { + return true + } + rel, err := filepath.Rel(dir, p) + return err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) +} + +// Runner runs one command and returns its combined output. Injected so the +// tests can drive build/test/restart failures without a toolchain, a container +// or a real daemon to break. +type Runner func(ctx context.Context, dir string, argv []string) (string, error) + +// ExecRunner is the real one. +func ExecRunner(ctx context.Context, dir string, argv []string) (string, error) { + cmd := exec.CommandContext(ctx, argv[0], argv[1:]...) + cmd.Dir = dir + out, err := cmd.CombinedOutput() + return string(out), err +} + +// HealthCheck proves the daemon at socket answers. Injected for the same reason +// as Runner. +type HealthCheck func(ctx context.Context, socket string) error + +// Logger receives one line per step. The CLI prints these as they happen: an +// update that goes quiet for four minutes during `make test` reads as a hang. +type Logger func(format string, args ...any) + +// Updater is the whole capability. Construct with New and call Apply or +// Rollback; there is no background goroutine and nothing starts on its own. +type Updater struct { + cfg Config + store *Store + run Runner + health HealthCheck + log Logger + now func() time.Time + // ids reports the running euid and the owner of a directory. Injected so + // the root-build refusal is testable without a second account. + ids func(dir string) (int, uint32, error) +} + +// New builds an Updater. Every seam has a real default; the tests replace them. +func New(cfg Config, opts ...Option) (*Updater, error) { + if err := cfg.Validate(); err != nil { + return nil, err + } + u := &Updater{ + cfg: cfg.withDefaults(), + store: &Store{Dir: cfg.SnapshotDir}, + run: ExecRunner, + health: DialHealth, + log: func(string, ...any) {}, + now: time.Now, + ids: realIDs, + } + for _, o := range opts { + o(u) + } + u.store.now = u.now + return u, nil +} + +// Option — a constructor seam. +type Option func(*Updater) + +func WithRunner(r Runner) Option { return func(u *Updater) { u.run = r } } +func WithHealth(h HealthCheck) Option { return func(u *Updater) { u.health = h } } +func WithLogger(l Logger) Option { return func(u *Updater) { u.log = l } } +func WithClock(f func() time.Time) Option { + return func(u *Updater) { u.now = f } +} + +// WithIDs replaces the euid/owner lookup behind the root-build refusal. +func WithIDs(f func(dir string) (int, uint32, error)) Option { + return func(u *Updater) { u.ids = f } +} + +// Snapshots lists what is available to roll back to, newest first. +func (u *Updater) Snapshots() ([]Snapshot, error) { return u.store.List() } diff --git a/internal/update/update_test.go b/internal/update/update_test.go new file mode 100644 index 0000000..f5193af --- /dev/null +++ b/internal/update/update_test.go @@ -0,0 +1,476 @@ +package update + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// The tests drive the whole orchestration against a fake box: a directory tree +// standing in for the install dir, an injected Runner standing in for +// make/git/docker, and an injected HealthCheck standing in for mavend. That is +// what makes the failure paths — the ones that matter — testable at all: you +// cannot ask a real deployment to fail its health check on demand, and the +// rollback path is exactly the path nobody exercises by hand. + +type fakeBox struct { + t *testing.T + root string + + // what the fake `make build` writes into the source tree + newBytes string + // scripted failures + buildErr error + testErr error + restartErr error + + // health: fails until the Nth call, then follows healthy + healthErrs int // remaining failures to serve + healthy bool + healthChecks int + // healthFn overrides the scripted behaviour entirely, for tests whose + // verdict depends on what the last restart actually deployed. + healthFn func() error + // deployedAtRestart records the installed bytes each time restart runs, so a + // test can prove the rollback put the old bytes back BEFORE restarting. + deployedAtRestart []string + // builtFromSource records the source the restart would have built an image + // from, each time it runs. + builtFromSource []string + + ran []string + + // The git half, for the deployment whose restart rebuilds from the tree. + // commit is HEAD; byCommit is what each commit's source says; dirty makes + // `git status --porcelain` report uncommitted work. + commit string + byCommit map[string]string + dirty bool +} + +func newFakeBox(t *testing.T) *fakeBox { + t.Helper() + root := t.TempDir() + for _, d := range []string{"src", "install", "snapshots"} { + if err := os.MkdirAll(filepath.Join(root, d), 0o755); err != nil { + t.Fatal(err) + } + } + // The currently deployed build, and a config file next to it. + write(t, filepath.Join(root, "install", "mavend"), "OLD-BUILD") + write(t, filepath.Join(root, "install", "mavend.json"), `{"tick_interval":"60s"}`) + // The source tree already contains a stale binary; `make build` overwrites it. + write(t, filepath.Join(root, "src", "mavend"), "STALE") + return &fakeBox{ + t: t, root: root, newBytes: "NEW-BUILD", healthy: true, + commit: "cafebabecafebabecafebabecafebabecafebabe", + byCommit: map[string]string{}, + } +} + +func (b *fakeBox) cfg() Config { + return Config{ + SourceDir: filepath.Join(b.root, "src"), + InstallDir: filepath.Join(b.root, "install"), + SnapshotDir: filepath.Join(b.root, "snapshots"), + Binaries: []string{"mavend"}, + ConfigFiles: []string{"mavend.json"}, + RestartCmd: []string{"restart-the-thing"}, + HealthSocket: filepath.Join(b.root, "mavend.sock"), + HealthTimeoutSec: 1, + KeepSnapshots: 3, + } +} + +func (b *fakeBox) run(ctx context.Context, dir string, argv []string) (string, error) { + b.ran = append(b.ran, strings.Join(argv, " ")) + switch strings.Join(argv, " ") { + case "make build": + if b.buildErr != nil { + return "ld: undefined reference to everything", b.buildErr + } + // A real build writes its artifacts into the working tree — the behaviour + // the snapshot-before-build ordering exists to survive. + write(b.t, filepath.Join(b.root, "src", "mavend"), b.newBytes) + return "built", nil + case "make test": + if b.testErr != nil { + return "--- FAIL: TestSomething", b.testErr + } + return "ok", nil + case "git rev-parse HEAD": + return b.commit + "\n", nil + case "git status --porcelain": + if b.dirty { + return " M internal/router/router.go\n", nil + } + return "", nil + case "restart-the-thing": + b.deployedAtRestart = append(b.deployedAtRestart, read(b.t, filepath.Join(b.root, "install", "mavend"))) + // The docker shape: the restart rebuilds the image from the tree, so + // what it deploys is the source, not any binary on the host. + if src, err := os.ReadFile(filepath.Join(b.root, "src", "source.go")); err == nil { + b.builtFromSource = append(b.builtFromSource, string(src)) + } + if b.restartErr != nil { + return "no such container", b.restartErr + } + return "restarted", nil + } + if len(argv) == 4 && argv[0] == "git" && argv[1] == "checkout" && argv[2] == "--force" { + content, ok := b.byCommit[argv[3]] + if !ok { + return "error: pathspec did not match", errors.New("exit status 1") + } + b.commit = argv[3] + write(b.t, filepath.Join(b.root, "src", "source.go"), content) + return "HEAD is now at " + argv[3], nil + } + return "", errors.New("unexpected command: " + strings.Join(argv, " ")) +} + +func (b *fakeBox) health(ctx context.Context, socket string) error { + b.healthChecks++ + if b.healthFn != nil { + return b.healthFn() + } + if b.healthErrs > 0 { + b.healthErrs-- + return errors.New("connection refused") + } + if !b.healthy { + return errors.New("she does not answer") + } + return nil +} + +func (b *fakeBox) updater(t *testing.T, extra ...Option) *Updater { + t.Helper() + opts := append([]Option{WithRunner(b.run), WithHealth(b.health)}, extra...) + u, err := New(b.cfg(), opts...) + if err != nil { + t.Fatal(err) + } + return u +} + +func (b *fakeBox) deployed() string { return read(b.t, filepath.Join(b.root, "install", "mavend")) } + +func write(t *testing.T, path, content string) { + t.Helper() + if err := os.WriteFile(path, []byte(content), 0o755); err != nil { + t.Fatal(err) + } +} + +func read(t *testing.T, path string) string { + t.Helper() + b, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +func TestApply_HappyPath(t *testing.T) { + b := newFakeBox(t) + res, err := b.updater(t).Apply(context.Background()) + if err != nil { + t.Fatalf("Apply: %v", err) + } + if !res.Verified || !res.Restarted || !res.Healthy || res.RolledBack { + t.Fatalf("result = %+v; want verified+restarted+healthy and no rollback", res) + } + if got := b.deployed(); got != "NEW-BUILD" { + t.Errorf("deployed binary = %q; want the new build", got) + } + // The order is the property: health, snapshot, build, test, install, restart. + want := []string{"git rev-parse HEAD", "make build", "make test", "restart-the-thing"} + if strings.Join(b.ran, "|") != strings.Join(want, "|") { + t.Errorf("commands ran = %v; want %v", b.ran, want) + } + if res.SnapshotID == "" { + t.Error("no snapshot was taken") + } +} + +func TestApply_RefusesWhenSheIsAlreadyDown(t *testing.T) { + // A box that is already broken has no baseline for the rollback to prove + // itself against, so the update never starts. + b := newFakeBox(t) + b.healthy = false + res, err := b.updater(t).Apply(context.Background()) + if !errors.Is(err, ErrUnhealthyBefore) { + t.Fatalf("Apply on an unhealthy box = %v; want ErrUnhealthyBefore", err) + } + if len(b.ran) != 0 { + t.Errorf("a refused update still ran %v", b.ran) + } + if res.SnapshotID != "" { + t.Error("a refused update still took a snapshot") + } +} + +func TestApply_TestFailureDeploysNothingAndPutsTheTreeBack(t *testing.T) { + b := newFakeBox(t) + b.testErr = errors.New("exit status 1") + res, err := b.updater(t).Apply(context.Background()) + if !errors.Is(err, ErrVerifyFailed) { + t.Fatalf("Apply with failing tests = %v; want ErrVerifyFailed", err) + } + if res.Verified { + t.Error("result claims verified after a failing test suite") + } + for _, c := range b.ran { + if c == "restart-the-thing" { + t.Fatal("a failed verification restarted the daemon") + } + } + if got := b.deployed(); got != "OLD-BUILD" { + t.Errorf("deployed binary = %q; want the old build untouched", got) + } + // The failing output is kept so the operator can see why. + var found bool + for _, s := range res.Steps { + if s.Name == "test" && strings.Contains(s.Output, "FAIL") { + found = true + } + } + if !found { + t.Error("the failing test output was not retained") + } +} + +func TestApply_BuildFailureIsCaughtBeforeTheTests(t *testing.T) { + b := newFakeBox(t) + b.buildErr = errors.New("exit status 2") + if _, err := b.updater(t).Apply(context.Background()); !errors.Is(err, ErrVerifyFailed) { + t.Fatalf("Apply with a failing build = %v; want ErrVerifyFailed", err) + } + for _, c := range b.ran { + if c == "make test" { + t.Error("ran the test suite after the build failed") + } + } +} + +func TestApply_UnhealthyAfterRestartRollsBackToTheOldBytes(t *testing.T) { + // The case the package exists for: everything verifies, the new build + // installs, and then she does not come up. + b := newFakeBox(t) + b.healthErrs = 1 // the preflight check passes, then she stops answering + b.healthy = false + u := b.updater(t) + // Once the rollback restores the old build, she answers again. + restored := false + u.health = func(ctx context.Context, socket string) error { + b.healthChecks++ + if b.deployed() == "OLD-BUILD" && restored { + return nil + } + if b.healthChecks == 1 { + return nil // preflight: the old build is up + } + if b.deployed() == "OLD-BUILD" { + restored = true + return nil + } + return errors.New("she does not answer on the new build") + } + res, err := u.Apply(context.Background()) + if !errors.Is(err, ErrRolledBack) { + t.Fatalf("Apply with a dead new build = %v; want ErrRolledBack", err) + } + if !res.RolledBack || !res.RollbackHealthy || res.Healthy { + t.Fatalf("result = %+v; want rolled back and healthy again on the old build", res) + } + if got := b.deployed(); got != "OLD-BUILD" { + t.Errorf("deployed binary after the rollback = %q; want OLD-BUILD", got) + } + // And the restore happened BEFORE the second restart, not after it. + if len(b.deployedAtRestart) != 2 { + t.Fatalf("restarts = %v; want two (the update and the rollback)", b.deployedAtRestart) + } + if b.deployedAtRestart[0] != "NEW-BUILD" || b.deployedAtRestart[1] != "OLD-BUILD" { + t.Errorf("bytes in place at each restart = %v; want [NEW-BUILD OLD-BUILD]", b.deployedAtRestart) + } +} + +func TestApply_RestartFailureRollsBack(t *testing.T) { + b := newFakeBox(t) + b.restartErr = errors.New("exit status 1") + res, err := b.updater(t).Apply(context.Background()) + // The rollback's own restart fails too, so this is the manual-recovery case — + // and it says so instead of reporting a tidy rollback. + if !errors.Is(err, ErrRollbackFailed) { + t.Fatalf("Apply with a broken restart command = %v; want ErrRollbackFailed", err) + } + if !res.RolledBack || res.RollbackHealthy { + t.Fatalf("result = %+v; want rolled back but not healthy", res) + } + if got := b.deployed(); got != "OLD-BUILD" { + t.Errorf("deployed binary = %q; want the old bytes restored even so", got) + } +} + +func TestApply_RollbackNeedsNoBuildAndNoNewCode(t *testing.T) { + // The rollback must not depend on the toolchain, the source tree, or the + // code it is replacing. Prove it: delete the source tree's binary and make + // every command except the restart fail, then roll back. + b := newFakeBox(t) + if _, err := b.updater(t).Apply(context.Background()); err != nil { + t.Fatalf("setup Apply: %v", err) + } + if b.deployed() != "NEW-BUILD" { + t.Fatal("setup did not deploy") + } + os.RemoveAll(filepath.Join(b.root, "src")) + if err := os.MkdirAll(filepath.Join(b.root, "src"), 0o755); err != nil { + t.Fatal(err) + } + b.buildErr = errors.New("no toolchain here") + b.testErr = errors.New("no toolchain here") + b.ran = nil + + res, err := b.updater(t).Rollback(context.Background(), "") + if err != nil && !errors.Is(err, ErrRolledBack) { + t.Fatalf("Rollback: %v", err) + } + if !res.RollbackHealthy { + t.Fatalf("result = %+v; want a healthy rollback", res) + } + if got := b.deployed(); got != "OLD-BUILD" { + t.Errorf("deployed binary = %q; want OLD-BUILD", got) + } + for _, c := range b.ran { + if strings.HasPrefix(c, "make") { + t.Errorf("the rollback ran %q — it must not need a build", c) + } + } +} + +func TestApply_ConfigIsSnapshottedButNeverOverwritten(t *testing.T) { + b := newFakeBox(t) + // A config in the source tree must not be deployed over the operator's. + write(t, filepath.Join(b.root, "src", "mavend.json"), `{"tick_interval":"1s"}`) + if _, err := b.updater(t).Apply(context.Background()); err != nil { + t.Fatalf("Apply: %v", err) + } + if got := read(t, filepath.Join(b.root, "install", "mavend.json")); !strings.Contains(got, "60s") { + t.Errorf("installed config = %q; an update must not replace his config", got) + } + snaps, err := b.updater(t).Snapshots() + if err != nil || len(snaps) == 0 { + t.Fatalf("Snapshots: %v %v", snaps, err) + } + var names []string + for _, f := range snaps[0].Files { + names = append(names, f.Name) + } + if len(names) != 2 { + t.Errorf("snapshot files = %v; want the binary and the config", names) + } + if snaps[0].Commit == "" { + t.Error("the snapshot did not record which commit produced it") + } +} + +func TestRollback_CorruptSnapshotIsRefusedNotRestored(t *testing.T) { + b := newFakeBox(t) + if _, err := b.updater(t).Apply(context.Background()); err != nil { + t.Fatalf("setup Apply: %v", err) + } + snaps, _ := b.updater(t).Snapshots() + // Something ate the snapshot. Restoring it would deploy garbage. + write(t, filepath.Join(snaps[0].Dir(), "mavend"), "CORRUPT") + _, err := b.updater(t).Rollback(context.Background(), snaps[0].ID) + if !errors.Is(err, ErrRollbackFailed) || !strings.Contains(err.Error(), "corrupt") { + t.Fatalf("Rollback of a corrupt snapshot = %v; want a refusal naming the corruption", err) + } + if got := b.deployed(); got != "NEW-BUILD" { + t.Errorf("deployed binary = %q; a refused restore must change nothing", got) + } +} + +func TestRollback_NoSnapshots(t *testing.T) { + b := newFakeBox(t) + if _, err := b.updater(t).Rollback(context.Background(), ""); err == nil { + t.Error("Rollback with no snapshots succeeded; want an error") + } +} + +func TestPrune_KeepsTheNewestAsTheRollbackTarget(t *testing.T) { + b := newFakeBox(t) + st := &Store{Dir: filepath.Join(b.root, "snapshots")} + base := time.Date(2026, 8, 1, 3, 0, 0, 0, time.UTC) + for i := 0; i < 4; i++ { + i := i + st.now = func() time.Time { return base.Add(time.Duration(i) * time.Minute) } + if _, err := st.Save(filepath.Join(b.root, "install"), []string{"mavend"}, "", ""); err != nil { + t.Fatal(err) + } + } + if err := st.Prune(0); err != nil { // 0 is clamped to 1, never to zero + t.Fatal(err) + } + snaps, err := st.List() + if err != nil { + t.Fatal(err) + } + if len(snaps) != 1 { + t.Fatalf("kept %d snapshots; want 1", len(snaps)) + } + if snaps[0].ID != "20260801-030300" { + t.Errorf("kept %s; want the newest", snaps[0].ID) + } +} + +func TestList_IgnoresSnapshotsWithNoManifest(t *testing.T) { + // An interrupted snapshot has files but no manifest. It must never be offered + // as a rollback target — restoring a half-copied binary is the worst outcome + // in the package. + b := newFakeBox(t) + dir := filepath.Join(b.root, "snapshots", "20260801-000000") + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + write(t, filepath.Join(dir, "mavend"), "HALF") + snaps, err := (&Store{Dir: filepath.Join(b.root, "snapshots")}).List() + if err != nil { + t.Fatal(err) + } + if len(snaps) != 0 { + t.Errorf("List returned %d snapshots; want none (no manifest)", len(snaps)) + } +} + +func TestConfigValidate(t *testing.T) { + ok := (&fakeBox{root: t.TempDir()}).cfg() + if err := ok.Validate(); err != nil { + t.Fatalf("valid config rejected: %v", err) + } + bad := map[string]func(c Config) Config{ + "relative source": func(c Config) Config { c.SourceDir = "src"; return c }, + "no restart command": func(c Config) Config { c.RestartCmd = nil; return c }, + "no health socket": func(c Config) Config { c.HealthSocket = ""; return c }, + "no binaries": func(c Config) Config { c.Binaries = nil; return c }, + "escaping artifact name": func(c Config) Config { c.Binaries = []string{"../../etc/passwd"}; return c }, + "absolute artifact name": func(c Config) Config { c.Binaries = []string{"/usr/bin/mavend"}; return c }, + "snapshots inside install": func(c Config) Config { c.SnapshotDir = filepath.Join(c.InstallDir, "snaps"); return c }, + } + for name, mutate := range bad { + if err := mutate(ok).Validate(); err == nil { + t.Errorf("%s was accepted; want a startup failure", name) + } + } + // And New refuses an invalid config outright rather than half-configuring. + if _, err := New(mutate(ok, "no health socket", bad)); err == nil { + t.Error("New accepted a config with no health socket") + } +} + +func mutate(c Config, key string, m map[string]func(Config) Config) Config { return m[key](c) } diff --git a/internal/update/verify.go b/internal/update/verify.go new file mode 100644 index 0000000..44a402a --- /dev/null +++ b/internal/update/verify.go @@ -0,0 +1,104 @@ +package update + +import ( + "context" + "errors" + "fmt" + "os" + "syscall" + "time" + "unicode/utf8" +) + +// Verification is "does this tree build and does it pass its own tests", run +// before a single byte is written to the install dir. +// +// It is `make build` and `make test`, not `go build`: the CGO daemons need the +// vendored toolchain and the whisper/piper include and library paths wired +// through the Makefile, and a bare `go build` on them fails in a way that has +// nothing to do with the change being deployed. `make test` is the -race suite +// with the CGO env set, and it is the only evidence available on a single box +// that the new code does what the old code did. +// +// This is not a substitute for a second environment. A test suite that passes +// says the code is self-consistent; it does not say the new build will start +// against this machine's actual models, sockets and encrypted store. That is +// what the post-restart health check is for, and it is why the install is +// reversible rather than merely careful. + +// Step — one verification or orchestration step and how it went. Kept so the CLI +// can print a truthful account of what was done, including on the failure path. +type Step struct { + Name string + Argv []string + Took time.Duration + Err error + Output string // combined output, only retained for failures +} + +// Verify runs the build and the test suite in SourceDir. +func (u *Updater) Verify(ctx context.Context) ([]Step, error) { + if err := u.refuseRootBuild(); err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(ctx, u.cfg.verifyTimeout()) + defer cancel() + var steps []Step + for _, argv := range [][]string{{"make", "build"}, {"make", "test"}} { + u.log("verify: %v (this takes a while)", argv) + start := u.now() + out, err := u.run(ctx, u.cfg.SourceDir, argv) + st := Step{Name: argv[len(argv)-1], Argv: argv, Took: u.now().Sub(start), Err: err} + if err != nil { + st.Output = tail(out, 4000) + } + steps = append(steps, st) + if err != nil { + u.log("verify: %v FAILED after %s", argv, st.Took.Round(time.Second)) + return steps, fmt.Errorf("%w: %v: %v", ErrVerifyFailed, argv, err) + } + u.log("verify: %v ok in %s", argv, st.Took.Round(time.Second)) + } + return steps, nil +} + +// refuseRootBuild stops a sudo'd apply from building in a tree it does not own. +// +// The seam is injected so the tests can drive both sides without a second uid. +func (u *Updater) refuseRootBuild() error { + uid, owner, err := u.ids(u.cfg.SourceDir) + if err != nil || uid != 0 || owner == 0 { + return nil // not root, or root's own tree, or we cannot tell + } + return fmt.Errorf("%w: %s is owned by uid %d", ErrRootOnHisTree, u.cfg.SourceDir, owner) +} + +// realIDs — the running uid and the owner of dir. Split out for the tests. +func realIDs(dir string) (uid int, owner uint32, err error) { + fi, err := os.Stat(dir) + if err != nil { + return 0, 0, err + } + st, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return 0, 0, errors.New("update: cannot read directory ownership") + } + return os.Geteuid(), st.Uid, nil +} + +// tail keeps the last n bytes — a failing `make test` prints far more than is +// useful, and the failure is always at the end. +// +// The cut is nudged forward to a rune boundary. Russian test names and fixture +// strings are the common case in this tree, and a slice landing mid-rune starts +// the log with a replacement character. +func tail(s string, n int) string { + if len(s) <= n { + return s + } + cut := len(s) - n + for cut < len(s) && !utf8.RuneStart(s[cut]) { + cut++ + } + return "…" + s[cut:] +} diff --git a/internal/vision/intake.go b/internal/vision/intake.go new file mode 100644 index 0000000..ac2d53d --- /dev/null +++ b/internal/vision/intake.go @@ -0,0 +1,111 @@ +package vision + +import ( + "context" + "fmt" + "strings" + + "github.com/kami/maven/internal/media" +) + +// Intake is the whole path from "bytes arrived" to "here is what she saw", +// in one place, so that every surface that can receive an image — a Telegram +// photo, a mavweb upload, a file path he names — goes through the same steps in +// the same order: +// +// 1. sniff the bytes (the sender's declared content type is not trusted); +// 2. store them content-addressed, so the same photo twice is one file and the +// original is still on disk if the description came out wrong; +// 3. prepare a downscaled JPEG for the model; +// 4. describe it. +// +// Step 2 happens BEFORE step 4 deliberately. If the vision model is missing or +// broken — which is today's actual state on this box — the image is still safely +// stored and describable later, and the failure is "I can't look at it yet", not +// "it's gone". +// +// Writing the description as a note is NOT done here. That needs the store and +// the embedder and belongs to the daemon; Intake returns the text and lets the +// caller decide whether it becomes a note, a reply, or both. +type Intake struct { + store *media.Store + provider Provider + maxDim int +} + +// NewIntake wires an intake. provider may be Disabled — storing still works, +// which is the point. maxDim ≤ 0 ⇒ media.DefaultMaxDim. +func NewIntake(store *media.Store, provider Provider, maxDim int) *Intake { + if provider == nil { + provider = Disabled{} + } + return &Intake{store: store, provider: provider, maxDim: maxDim} +} + +// Result — what an intake produced. Blob is always set when Store succeeded, so +// a caller that got an error from the description still knows what was kept and +// can retry against the same id later. +type Result struct { + Blob media.Blob + Image media.Image + Description string +} + +// Accept stores data and describes it. source is provenance recorded on the +// blob ("telegram", "web:upload"); question is what he asked about the image, or +// empty for the default "what is this". +// +// A description failure is returned alongside a populated Result: the caller +// gets the blob id for the log and the reply, and the error to explain why there +// are no words yet. +func (in *Intake) Accept(ctx context.Context, data []byte, source, question string) (Result, error) { + if in == nil || in.store == nil { + return Result{}, fmt.Errorf("vision: intake not wired") + } + mime, err := media.SniffImage(data) + if err != nil { + return Result{}, err + } + blob, err := in.store.Put(media.KindImage, mime, source, data) + if err != nil { + return Result{}, err + } + im, err := media.PrepareImage(data, source, in.maxDim) + if err != nil { + return Result{Blob: blob}, err + } + res := Result{Blob: blob, Image: im} + text, err := in.provider.Describe(ctx, im, question) + if err != nil { + return res, err + } + res.Description = strings.TrimSpace(text) + return res, nil +} + +// Rerun describes an already-stored image again — a different question, or the +// first successful attempt after the model finally landed on disk. It is the +// reason step 2 comes before step 4. +func (in *Intake) Rerun(ctx context.Context, id, question string) (Result, error) { + if in == nil || in.store == nil { + return Result{}, fmt.Errorf("vision: intake not wired") + } + blob, data, err := in.store.Read(id) + if err != nil { + return Result{}, err + } + if blob.Kind != media.KindImage { + return Result{Blob: blob}, fmt.Errorf("vision: %s is %s, not an image", id[:12], blob.Kind) + } + im, err := media.PrepareImage(data, blob.Source, in.maxDim) + if err != nil { + return Result{Blob: blob}, err + } + res := Result{Blob: blob, Image: im} + text, err := in.provider.Describe(ctx, im, question) + if err != nil { + return res, err + } + res.Description = strings.TrimSpace(text) + return res, nil +} diff --git a/internal/vision/intake_test.go b/internal/vision/intake_test.go new file mode 100644 index 0000000..5284225 --- /dev/null +++ b/internal/vision/intake_test.go @@ -0,0 +1,147 @@ +package vision + +import ( + "bytes" + "context" + "errors" + "image" + "image/png" + "testing" + + "github.com/kami/maven/internal/media" +) + +type fakeProvider struct { + reply string + err error + seen int + lastQ string + lastDim int +} + +func (f *fakeProvider) Describe(_ context.Context, im media.Image, prompt string) (string, error) { + f.seen++ + f.lastQ = prompt + f.lastDim = im.Width + return f.reply, f.err +} + +func pngPayload(t *testing.T, w, h int) []byte { + t.Helper() + var buf bytes.Buffer + if err := png.Encode(&buf, image.NewRGBA(image.Rect(0, 0, w, h))); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func testIntake(t *testing.T, p Provider) (*Intake, *media.Store) { + t.Helper() + s, err := media.Open(t.TempDir(), 0, 0) + if err != nil { + t.Fatal(err) + } + return NewIntake(s, p, 64), s +} + +func TestAcceptStoresThenDescribes(t *testing.T) { + fp := &fakeProvider{reply: "кот на подоконнике"} + in, store := testIntake(t, fp) + + res, err := in.Accept(context.Background(), pngPayload(t, 200, 100), "telegram", "кто это?") + if err != nil { + t.Fatalf("accept: %v", err) + } + if res.Description != "кот на подоконнике" { + t.Errorf("description = %q", res.Description) + } + if fp.lastQ != "кто это?" { + t.Errorf("question not passed through: %q", fp.lastQ) + } + if fp.lastDim != 64 { + t.Errorf("image not downscaled to maxDim: width %d", fp.lastDim) + } + // The sniffed mime wins over anything a sender claimed. + got, _, err := store.Read(res.Blob.ID) + if err != nil { + t.Fatalf("blob not stored: %v", err) + } + if got.MIME != "image/png" || got.Source != "telegram" { + t.Errorf("blob metadata = %+v", got) + } +} + +// The ordering promise: with no vision model on the box — today's real state — +// the image is still on disk and the id is still reported, so it can be +// described later instead of being lost. +func TestAcceptKeepsBlobWhenDescribeFails(t *testing.T) { + in, store := testIntake(t, Disabled{}) + res, err := in.Accept(context.Background(), pngPayload(t, 32, 32), "web:upload", "") + if !errors.Is(err, ErrDisabled) { + t.Fatalf("got %v, want ErrDisabled", err) + } + if res.Blob.ID == "" { + t.Fatal("no blob id reported on a description failure") + } + if _, _, err := store.Read(res.Blob.ID); err != nil { + t.Errorf("blob was not kept: %v", err) + } +} + +func TestRerunDescribesAStoredBlob(t *testing.T) { + fp := &fakeProvider{reply: "текст: ошибка E24"} + in, _ := testIntake(t, fp) + first, err := in.Accept(context.Background(), pngPayload(t, 40, 40), "telegram", "") + if err != nil { + t.Fatal(err) + } + res, err := in.Rerun(context.Background(), first.Blob.ID, "прочитай текст") + if err != nil { + t.Fatalf("rerun: %v", err) + } + if res.Description != "текст: ошибка E24" { + t.Errorf("description = %q", res.Description) + } + if fp.lastQ != "прочитай текст" { + t.Errorf("new question not used: %q", fp.lastQ) + } + if fp.seen != 2 { + t.Errorf("provider called %d times, want 2", fp.seen) + } +} + +func TestRerunRefusesAudioBlob(t *testing.T) { + in, store := testIntake(t, &fakeProvider{reply: "x"}) + b, err := store.Put(media.KindAudio, "audio/wav", "capture:meeting", []byte("pcm bytes")) + if err != nil { + t.Fatal(err) + } + if _, err := in.Rerun(context.Background(), b.ID, ""); err == nil { + t.Error("audio blob was accepted as an image") + } +} + +func TestRerunUnknownID(t *testing.T) { + in, _ := testIntake(t, &fakeProvider{}) + if _, err := in.Rerun(context.Background(), "nope", ""); err == nil { + t.Error("malformed id accepted") + } +} + +func TestAcceptRefusesNonImage(t *testing.T) { + in, _ := testIntake(t, &fakeProvider{}) + if _, err := in.Accept(context.Background(), []byte("this is a text file"), "web:upload", ""); !errors.Is(err, media.ErrUnsupportedImage) { + t.Errorf("got %v, want ErrUnsupportedImage", err) + } +} + +func TestNilProviderDegradesToDisabled(t *testing.T) { + s, err := media.Open(t.TempDir(), 0, 0) + if err != nil { + t.Fatal(err) + } + in := NewIntake(s, nil, 0) + if _, err := in.Accept(context.Background(), pngPayload(t, 8, 8), "x", ""); !errors.Is(err, ErrDisabled) { + t.Errorf("got %v, want ErrDisabled", err) + } +} diff --git a/internal/vision/vision.go b/internal/vision/vision.go new file mode 100644 index 0000000..a1a4667 --- /dev/null +++ b/internal/vision/vision.go @@ -0,0 +1,305 @@ +// Package vision is Maven's image-understanding seam (Vikunja #252, +// docs/plans/07-vision.md). +// +// One interface, Provider, with one method: describe an image, in words, in +// Russian, with an optional question about it. Text extraction is not a second +// method — "прочитай текст с картинки" is a prompt, and a vision-language model +// does not have a separate OCR mode to select. +// +// # What is deliberately NOT here +// +// The plan document called for a `RemoteProvider` calling "an OpenAI-compatible +// vision API endpoint". That step is refused: CLAUDE.md's surviving hard +// constraint after "never phones home" was deprecated is *no cloud model, +// inference stays on the box*, and a photo of his flat is the single worst thing +// to make an exception for. Endpoint is therefore checked at construction and +// must be a loopback or private address — a public host is a config error, not a +// deployment option. That check is the reason this package does not simply reuse +// internal/llm.Client. +// +// # State on this box, honestly +// +// The resident model is Qwen3-1.7B, which is text-only, and as of 2026-08-01 +// there is no vision-capable gguf and no mmproj file anywhere under +// /mnt/hdd1/llms. So LocalProvider is written, tested against a fake server, and +// currently has nothing real to talk to: the describing half is BLOCKED on a +// model download (see docs/plans/07-vision.md for the candidates and the +// recipe). What works today without any download is the intake — an image +// arrives, is stored, is prepared — and the config seam that turns the rest on. +// +// Provider is nil-safe through Disabled, and vision is OFF unless configured, +// like the weather and telegram. +package vision + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strings" + "time" + + "github.com/kami/maven/internal/media" + "github.com/kami/maven/internal/webfetch" +) + +// DefaultTimeout — budget for one description. A small VLM doing prefill over +// an 896px image on a Vega iGPU is slow; 90s is generous because nobody is +// holding a conversation open on this path — the answer arrives as a reply or a +// note, and a too-tight timeout just means it never arrives at all. +const DefaultTimeout = 90 * time.Second + +// DefaultMaxTokens — cap on the description. A paragraph is what a spoken +// answer can carry; a page is not. +const DefaultMaxTokens = 300 + +// DefaultPrompt — what she is asked when he did not ask anything specific, +// only sent a picture. Russian, because that is the channel language, and +// feminine self-reference is not needed here (the prompt is an instruction, the +// persona block is added by the caller that phrases the reply). +const DefaultPrompt = "Опиши, что на этом изображении. Коротко, 2-3 предложения. Если на нём есть текст, приведи его." + +// Errors callers distinguish. +var ( + // ErrDisabled — vision is not configured. Returned by Disabled, which is + // what the daemon wires when the config block is absent. + ErrDisabled = errors.New("vision: not configured") + // ErrNotPrivate — the configured endpoint is not on this box or its + // network. Refused at construction; see the package comment. + ErrNotPrivate = errors.New("vision: endpoint must be a local or private address") + // ErrEmptyReply — the model returned nothing usable. + ErrEmptyReply = errors.New("vision: empty description") +) + +// Provider — the image-understanding contract. Describe takes an image already +// prepared by internal/media (decoded, downscaled, JPEG) and a prompt; an empty +// prompt means DefaultPrompt. +type Provider interface { + Describe(ctx context.Context, im media.Image, prompt string) (string, error) +} + +// Disabled — the floor Provider. Every call fails with ErrDisabled, which the +// caller turns into "я не умею смотреть картинки — зрение не настроено". It +// exists so that no call site needs a nil check and switching vision off cannot +// crash a turn. +type Disabled struct{} + +// Describe always fails. The signature matches Provider. +func (Disabled) Describe(context.Context, media.Image, string) (string, error) { + return "", ErrDisabled +} + +// MaxReplyBytes bounds what is read back from the vision server. A description +// is words; anything past a megabyte is a broken endpoint. +const MaxReplyBytes = 1 << 20 + +// Config — how to reach the local vision server. Built from +// config.VisionConfig by the daemon; kept separate so this package does not +// import internal/config. +type Config struct { + // Endpoint — base URL of a llama-server started with a vision model and its + // mmproj (`llama-server -m model.gguf --mmproj mmproj.gguf`). Must be + // loopback or private. The path is appended by the provider; give it + // "http://127.0.0.1:8081". + Endpoint string + // Model — the model name to send. llama-server ignores it; it matters if the + // endpoint is something else OpenAI-shaped on the same box. + Model string + // Timeout — per-description budget. 0 ⇒ DefaultTimeout. + Timeout time.Duration + // MaxTokens — cap on the reply. 0 ⇒ DefaultMaxTokens. + MaxTokens int + // Prompt — the default question. Empty ⇒ DefaultPrompt. + Prompt string +} + +// LocalProvider talks to a llama-server on this box over its +// /v1/chat/completions endpoint, sending the image as a data URI content part. +// It is the only real Provider, and it is a plain HTTP client: no subprocess +// spawning, because the daemon already owns llama-server lifecycle for the +// resident model and a second managed process is a bigger change than this task. +type LocalProvider struct { + endpoint string + model string + prompt string + maxTokens int + http *http.Client +} + +// NewLocal builds a LocalProvider, refusing a non-private endpoint. A bad URL +// or a public host is an error at construction so the daemon logs it once at +// startup instead of failing every turn. +func NewLocal(cfg Config) (*LocalProvider, error) { + base := strings.TrimRight(strings.TrimSpace(cfg.Endpoint), "/") + if base == "" { + return nil, errors.New("vision: empty endpoint") + } + if err := checkPrivate(base); err != nil { + return nil, err + } + timeout := cfg.Timeout + if timeout <= 0 { + timeout = DefaultTimeout + } + maxTokens := cfg.MaxTokens + if maxTokens <= 0 { + maxTokens = DefaultMaxTokens + } + prompt := strings.TrimSpace(cfg.Prompt) + if prompt == "" { + prompt = DefaultPrompt + } + return &LocalProvider{ + endpoint: base, + model: cfg.Model, + prompt: prompt, + maxTokens: maxTokens, + http: &http.Client{ + Timeout: timeout, + // No redirects. checkPrivate validates the configured literal and + // nothing validated a hop, so a 302 from the local llama-server + // would send the photo, as a data URI in the POST body, wherever + // the redirect named. "No provider in this repo may upload a blob" + // has to be true of the second request as well as the first. + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + }, nil +} + +// Endpoint is the server this provider talks to. For logs and /dash. +func (p *LocalProvider) Endpoint() string { return p.endpoint } + +// ValidateEndpoint reports whether a configured endpoint is one this package +// would accept. Exported so config validation fails at startup on a typo, +// rather than logging once at wiring time and leaving the capability quietly +// off. +func ValidateEndpoint(raw string) error { + return checkPrivate(strings.TrimRight(strings.TrimSpace(raw), "/")) +} + +// checkPrivate refuses any endpoint that is not on this box or its LAN. A +// hostname that is not an IP literal is refused too: "vision.example.com" could +// resolve anywhere, and resolving it here would be trusting DNS with his photos. +// localhost is the one name allowed, because it is the common case. +func checkPrivate(raw string) error { + u, err := url.Parse(raw) + if err != nil { + return fmt.Errorf("vision: parse endpoint: %w", err) + } + if u.Scheme != "http" && u.Scheme != "https" { + return fmt.Errorf("vision: endpoint scheme %q not supported", u.Scheme) + } + host := u.Hostname() + if host == "" { + return errors.New("vision: endpoint has no host") + } + if strings.EqualFold(host, "localhost") { + return nil + } + ip := net.ParseIP(host) + if ip == nil { + return fmt.Errorf("%w: %q is a name, not an address", ErrNotPrivate, host) + } + if !webfetch.IsPrivateIP(ip) { + return fmt.Errorf("%w: %s", ErrNotPrivate, host) + } + return nil +} + +// chat request shapes. Content is the OpenAI multimodal array form: a text part +// and an image_url part whose url is a data URI. +type textPart struct { + Type string `json:"type"` + Text string `json:"text"` +} +type imageURL struct { + URL string `json:"url"` +} +type imagePart struct { + Type string `json:"type"` + ImageURL imageURL `json:"image_url"` +} +type chatReq struct { + Model string `json:"model,omitempty"` + Messages []any `json:"messages"` + MaxTokens int `json:"max_tokens,omitempty"` + Temp float64 `json:"temperature"` +} +type userMsg struct { + Role string `json:"role"` + Content []any `json:"content"` +} +type chatResp struct { + Choices []struct { + Message struct { + Content string `json:"content"` + ReasoningContent string `json:"reasoning_content"` + } `json:"message"` + } `json:"choices"` +} + +// Describe sends the image and prompt and returns the model's answer. An empty +// prompt uses the configured default. Errors are wrapped, never fatal: the +// caller says she could not make out the picture and the turn continues. +func (p *LocalProvider) Describe(ctx context.Context, im media.Image, prompt string) (string, error) { + if len(im.JPEG) == 0 { + return "", media.ErrEmpty + } + q := strings.TrimSpace(prompt) + if q == "" { + q = p.prompt + } + body, err := json.Marshal(chatReq{ + Model: p.model, + MaxTokens: p.maxTokens, + Messages: []any{userMsg{Role: "user", Content: []any{ + textPart{Type: "text", Text: q}, + imagePart{Type: "image_url", ImageURL: imageURL{URL: im.DataURI()}}, + }}}, + }) + if err != nil { + return "", fmt.Errorf("vision: marshal: %w", err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + p.endpoint+"/v1/chat/completions", bytes.NewReader(body)) + if err != nil { + return "", fmt.Errorf("vision: request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := p.http.Do(req) + if err != nil { + return "", fmt.Errorf("vision: post: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("vision: status %d", resp.StatusCode) + } + var out chatResp + // Capped: the decoder would otherwise read whatever the endpoint sends, and + // a local server answering with a stuck stream should not cost the daemon + // its memory. A description is a few hundred tokens. + if err := json.NewDecoder(io.LimitReader(resp.Body, MaxReplyBytes)).Decode(&out); err != nil { + return "", fmt.Errorf("vision: decode: %w", err) + } + if len(out.Choices) == 0 { + return "", ErrEmptyReply + } + text := strings.TrimSpace(out.Choices[0].Message.Content) + if text == "" { + // Same fallback as internal/llm: a Thinking model sometimes puts the + // whole answer in reasoning_content and leaves content empty. + text = strings.TrimSpace(out.Choices[0].Message.ReasoningContent) + } + if text == "" { + return "", ErrEmptyReply + } + return text, nil +} diff --git a/internal/vision/vision_test.go b/internal/vision/vision_test.go new file mode 100644 index 0000000..a65cfc5 --- /dev/null +++ b/internal/vision/vision_test.go @@ -0,0 +1,268 @@ +package vision + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "image" + "image/png" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/kami/maven/internal/media" +) + +func testImage(t *testing.T) media.Image { + t.Helper() + var buf bytes.Buffer + if err := png.Encode(&buf, image.NewRGBA(image.Rect(0, 0, 32, 32))); err != nil { + t.Fatal(err) + } + im, err := media.PrepareImage(buf.Bytes(), "test", 32) + if err != nil { + t.Fatal(err) + } + return im +} + +func TestDisabledAlwaysRefuses(t *testing.T) { + _, err := Disabled{}.Describe(context.Background(), testImage(t), "что тут?") + if !errors.Is(err, ErrDisabled) { + t.Fatalf("got %v, want ErrDisabled", err) + } +} + +// The whole reason this package has its own HTTP client instead of reusing +// internal/llm.Client: a vision endpoint that is not on this box is refused. +func TestNewLocalRefusesNonPrivateEndpoints(t *testing.T) { + bad := []string{ + "https://api.openai.com", + "http://8.8.8.8:8080", + "https://vision.example.com", // a name could resolve anywhere + "ftp://127.0.0.1:8080", // wrong scheme + "", // nothing to talk to + } + for _, ep := range bad { + if _, err := NewLocal(Config{Endpoint: ep}); err == nil { + t.Errorf("NewLocal(%q) was accepted", ep) + } + } +} + +func TestNewLocalAcceptsLocalEndpoints(t *testing.T) { + for _, ep := range []string{"http://127.0.0.1:8081", "http://localhost:8081/", "http://192.168.1.104:8081", "http://[::1]:8081"} { + p, err := NewLocal(Config{Endpoint: ep}) + if err != nil { + t.Errorf("NewLocal(%q): %v", ep, err) + continue + } + if strings.HasSuffix(p.Endpoint(), "/") { + t.Errorf("trailing slash kept: %q", p.Endpoint()) + } + } +} + +func TestDescribeSendsImageAsDataURIAndReturnsText(t *testing.T) { + var gotBody map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/chat/completions" { + t.Errorf("path = %s", r.URL.Path) + } + raw, _ := io.ReadAll(r.Body) + if err := json.Unmarshal(raw, &gotBody); err != nil { + t.Errorf("unmarshal request: %v", err) + } + _, _ = w.Write([]byte(`{"choices":[{"message":{"content":" На картинке кот "}}]}`)) + })) + defer srv.Close() + + p, err := NewLocal(Config{Endpoint: srv.URL, Model: "qwen-vl"}) + if err != nil { + t.Fatal(err) + } + text, err := p.Describe(context.Background(), testImage(t), "кто на фото?") + if err != nil { + t.Fatalf("describe: %v", err) + } + if text != "На картинке кот" { + t.Errorf("text = %q (should be trimmed)", text) + } + + msgs, ok := gotBody["messages"].([]any) + if !ok || len(msgs) != 1 { + t.Fatalf("messages = %#v", gotBody["messages"]) + } + parts, ok := msgs[0].(map[string]any)["content"].([]any) + if !ok || len(parts) != 2 { + t.Fatalf("content parts = %#v", msgs[0]) + } + if got := parts[0].(map[string]any)["text"]; got != "кто на фото?" { + t.Errorf("prompt = %v", got) + } + url := parts[1].(map[string]any)["image_url"].(map[string]any)["url"].(string) + if !strings.HasPrefix(url, "data:image/jpeg;base64,") { + t.Errorf("image not sent as a jpeg data uri: %.40s", url) + } +} + +func TestDescribeUsesDefaultPromptWhenNoQuestion(t *testing.T) { + var sentPrompt string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body struct { + Messages []struct { + Content []struct { + Text string `json:"text"` + } `json:"content"` + } `json:"messages"` + } + _ = json.NewDecoder(r.Body).Decode(&body) + sentPrompt = body.Messages[0].Content[0].Text + _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ок"}}]}`)) + })) + defer srv.Close() + + p, err := NewLocal(Config{Endpoint: srv.URL, Prompt: "Опиши по-русски."}) + if err != nil { + t.Fatal(err) + } + if _, err := p.Describe(context.Background(), testImage(t), " "); err != nil { + t.Fatal(err) + } + if sentPrompt != "Опиши по-русски." { + t.Errorf("prompt = %q", sentPrompt) + } +} + +// A Thinking model sometimes leaves content empty and puts the answer in +// reasoning_content; internal/llm has the same fallback and vision needs it too. +func TestDescribeFallsBackToReasoningContent(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"","reasoning_content":"схема платы"}}]}`)) + })) + defer srv.Close() + p, _ := NewLocal(Config{Endpoint: srv.URL}) + text, err := p.Describe(context.Background(), testImage(t), "") + if err != nil { + t.Fatal(err) + } + if text != "схема платы" { + t.Errorf("text = %q", text) + } +} + +func TestDescribeErrors(t *testing.T) { + t.Run("no choices", func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"choices":[]}`)) + })) + defer srv.Close() + p, _ := NewLocal(Config{Endpoint: srv.URL}) + if _, err := p.Describe(context.Background(), testImage(t), ""); !errors.Is(err, ErrEmptyReply) { + t.Errorf("got %v, want ErrEmptyReply", err) + } + }) + t.Run("server error", func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + p, _ := NewLocal(Config{Endpoint: srv.URL}) + if _, err := p.Describe(context.Background(), testImage(t), ""); err == nil { + t.Error("500 was not an error") + } + }) + t.Run("empty image", func(t *testing.T) { + p, _ := NewLocal(Config{Endpoint: "http://127.0.0.1:1"}) + if _, err := p.Describe(context.Background(), media.Image{}, ""); !errors.Is(err, media.ErrEmpty) { + t.Errorf("got %v, want media.ErrEmpty", err) + } + }) + t.Run("context cancelled", func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + time.Sleep(200 * time.Millisecond) + _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"поздно"}}]}`)) + })) + defer srv.Close() + p, _ := NewLocal(Config{Endpoint: srv.URL}) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + if _, err := p.Describe(ctx, testImage(t), ""); err == nil { + t.Error("cancelled context returned no error") + } + }) +} + +// checkPrivate validates the configured literal and used to validate nothing +// else. A 302 from the local llama-server would have sent the photo, as a data +// URI in the POST body, wherever the redirect named. +func TestLocalProviderDoesNotFollowARedirect(t *testing.T) { + var elsewhere int32 + away := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&elsewhere, 1) + _, _ = io.WriteString(w, `{"choices":[{"message":{"content":"leaked"}}]}`) + })) + defer away.Close() + local := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, away.URL+"/v1/chat/completions", http.StatusFound) + })) + defer local.Close() + + p, err := NewLocal(Config{Endpoint: local.URL}) + if err != nil { + t.Fatal(err) + } + im := media.Image{JPEG: []byte{0xFF, 0xD8, 0xFF}} + if _, err := p.Describe(context.Background(), im, "что это"); err == nil { + t.Fatal("a redirected describe must fail, not follow") + } + if n := atomic.LoadInt32(&elsewhere); n != 0 { + t.Fatalf("the image was sent to the redirect target %d time(s)", n) + } +} + +// The reply is read through a cap. A stuck endpoint should not cost the daemon +// its memory. +func TestLocalProviderCapsTheReply(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"choices":[{"message":{"content":"`) + for written := 0; written < MaxReplyBytes+(1<<20); written += 1 << 16 { + if _, err := io.WriteString(w, strings.Repeat("a", 1<<16)); err != nil { + return + } + } + })) + defer srv.Close() + p, err := NewLocal(Config{Endpoint: srv.URL}) + if err != nil { + t.Fatal(err) + } + im := media.Image{JPEG: []byte{0xFF, 0xD8, 0xFF}} + if _, err := p.Describe(context.Background(), im, ""); err == nil { + t.Fatal("an unbounded reply must fail rather than being read whole") + } +} + +// ValidateEndpoint is what config calls at startup, and it must agree with the +// constructor. +func TestValidateEndpointMatchesTheConstructor(t *testing.T) { + for _, raw := range []string{"http://127.0.0.1:8081", "http://localhost:8081/"} { + if err := ValidateEndpoint(raw); err != nil { + t.Errorf("ValidateEndpoint(%q) = %v", raw, err) + } + } + for _, raw := range []string{"http://8.8.8.8:8081", "http://vision.example.com", "ftp://127.0.0.1"} { + if err := ValidateEndpoint(raw); err == nil { + t.Errorf("ValidateEndpoint(%q) accepted a non-private endpoint", raw) + } + if _, err := NewLocal(Config{Endpoint: raw}); err == nil { + t.Errorf("NewLocal(%q) accepted what ValidateEndpoint should refuse", raw) + } + } +} diff --git a/internal/webauthn/keywrap.go b/internal/webauthn/keywrap.go index 1070862..7129ea4 100644 --- a/internal/webauthn/keywrap.go +++ b/internal/webauthn/keywrap.go @@ -1,24 +1,59 @@ -// Key wrapping for cold-start unlock. +// Key wrapping for cold-start unlock (Vikunja #14). // -// The at-rest AES-256 key is wrapped with a key derived from the passkey -// credential public key (stable across assertions) via HKDF-SHA256, then -// AES-256-GCM. The wrapped blob is stored on disk; at cold-start the passkey -// assertion provides the credential public key to unwrap it. +// The at-rest AES-256 key is never on disk in the clear. It is wrapped with a +// key derived from a secret only the authenticator can produce, so a cold boot +// needs the physical passkey and nothing else opens the store. // -// The passkey credential is a P-256 ECDSA public key. Its raw uncompressed -// bytes (65 bytes, 0x04 || X || Y) are the HKDF input — high-entropy, stable. +// # What the secret must be // -// Blob format: salt (16) || nonce (12) || AES-256-GCM ciphertext. -// No file magic — the caller (mavend) owns the file path. +// The WebAuthn PRF extension. On assertion, the authenticator evaluates a +// keyed pseudo-random function over a fixed salt and hands back 32 bytes that +// are stable for the credential, unpredictable to everyone else, and never +// leave the device except as that output. That is the only thing in WebAuthn +// that yields a *secret* rather than a signature, and it is what makes the +// wrapped blob worth wrapping. +// +// # What it must NOT be, and used to be +// +// v1 of this file derived the wrapping key from the credential *public* key, +// on the reasoning that it is high-entropy and stable across assertions. Both +// are true and neither matters: a public key is public. mavweb writes it +// verbatim to passkeys.json, normally in the same state dir as the wrapped +// blob, so anyone holding both files recovered the database key offline with +// no authenticator involved. A v1 blob is a plaintext key with extra steps. +// +// v1 blobs are still readable, so an existing deployment opens and can be +// re-wrapped, and UnwrapKey reports which format it read so the caller can +// say so out loud. Nothing writes v1 any more. Reading one needs the +// credential public key, which only cmd/mavweb still has: its assertion +// handler retries a failed PRF unwrap with it, because otherwise a box +// enrolled before v2 could never cold-start again. +// +// # What the wrapped blob's security rests on +// +// The PRF secret is stable for the lifetime of the credential and it reaches +// mavend inside an HTTP request body. Unlike a signature it does not expire. +// One copy in a proxy log, a devtools HAR, or a crash dump is permanent +// offline access to whatever this blob wraps. Nothing on this path may log the +// secret, and nothing does. +// +// # Blob format +// +// v2: "MVNKW2\x00" (7) || salt (16) || nonce (12) || AES-256-GCM ciphertext +// v1: salt (16) || nonce (12) || AES-256-GCM ciphertext (legacy, read-only) +// +// The magic doubles as the version discriminator: v1 had none, so anything +// that does not start with it is v1 by elimination. A random 16-byte v1 salt +// colliding with the magic is a 2^-56 event, and the GCM tag catches it. package webauthn import ( "crypto/aes" "crypto/cipher" - "crypto/hmac" + "crypto/hkdf" "crypto/rand" "crypto/sha256" - "encoding/binary" + "crypto/subtle" "errors" "fmt" "io" @@ -31,143 +66,186 @@ const ( nonceLen = 12 // keyLen — AES-256 key length. keyLen = 32 - // wrapInfo — HKDF info string for domain separation. - wrapInfo = "maven-passkey-keywrap-v1" + // secretLen — required length of the PRF output used as key material. + // WebAuthn PRF results are 32 bytes. Requiring exactly that is not + // pedantry: it is the structural guard that stops a COSE credential + // public key (77+ bytes) being passed here again by accident. + secretLen = 32 + + // wrapInfoV2 — HKDF info string. Carries the version so a v1 and a v2 + // derivation can never collide even given the same input. + wrapInfoV2 = "maven-passkey-keywrap-v2" + // wrapInfoV1 — the legacy info string, kept only to read old blobs. + wrapInfoV1 = "maven-passkey-keywrap-v1" ) +// blobMagicV2 prefixes every v2 blob. +var blobMagicV2 = []byte("MVNKW2\x00") + var ( ErrKeyWrap = errors.New("webauthn: key wrap failed") ErrKeyUnwrap = errors.New("webauthn: key unwrap failed (wrong credential?)") ErrBlobTooLong = errors.New("webauthn: wrapped blob too long") + // ErrSecretLen is returned when the caller passes something that is not a + // 32-byte PRF output — most likely a credential public key. + ErrSecretLen = errors.New("webauthn: wrapping secret must be a 32-byte PRF output") ) -// WrapKey derives a wrapping key from credPublicKey via HKDF-SHA256 and -// AES-GCM-wraps plaintextKey. Returns the blob: salt || nonce || ciphertext. -// plaintextKey must be exactly 32 bytes (AES-256). -func WrapKey(plaintextKey, credPublicKey []byte) ([]byte, error) { +// BlobVersion identifies which format a blob was read as. +type BlobVersion int + +const ( + // BlobV1 is the legacy public-key-derived format. Readable, never written. + BlobV1 BlobVersion = 1 + // BlobV2 is the PRF-derived format. + BlobV2 BlobVersion = 2 +) + +func (v BlobVersion) String() string { + switch v { + case BlobV1: + return "v1 (legacy, public-key derived — NOT SECRET)" + case BlobV2: + return "v2 (PRF derived)" + } + return "unknown" +} + +// maxBlobLen — sanity limit; a real blob is 67 bytes. +const maxBlobLen = 1 << 20 + +// WrapKey wraps plaintextKey (32 bytes, AES-256) under a key derived from +// secret via HKDF-SHA256, and returns a v2 blob. +// +// secret must be the 32-byte WebAuthn PRF output for the enrolled credential. +// Anything else is refused — see the file header for why passing a credential +// public key here is the bug this replaces. +func WrapKey(plaintextKey, secret []byte) ([]byte, error) { if len(plaintextKey) != keyLen { return nil, fmt.Errorf("%w: plaintext key must be %d bytes", ErrKeyWrap, keyLen) } - if len(credPublicKey) == 0 { - return nil, fmt.Errorf("%w: empty credential public key", ErrKeyWrap) + if err := checkSecret(secret); err != nil { + return nil, fmt.Errorf("%w: %v", ErrKeyWrap, err) } salt := make([]byte, saltLen) if _, err := io.ReadFull(rand.Reader, salt); err != nil { return nil, fmt.Errorf("%w: salt: %v", ErrKeyWrap, err) } - - wrapKey := hkdfSHA256(credPublicKey, salt, []byte(wrapInfo), keyLen) - nonce := make([]byte, nonceLen) if _, err := io.ReadFull(rand.Reader, nonce); err != nil { return nil, fmt.Errorf("%w: nonce: %v", ErrKeyWrap, err) } - block, err := aes.NewCipher(wrapKey) + gcm, err := gcmFor(secret, salt, wrapInfoV2) if err != nil { - return nil, fmt.Errorf("%w: aes: %v", ErrKeyWrap, err) - } - gcm, err := cipher.NewGCM(block) - if err != nil { - return nil, fmt.Errorf("%w: gcm: %v", ErrKeyWrap, err) + return nil, fmt.Errorf("%w: %v", ErrKeyWrap, err) } - // Seal appends ciphertext+tag to nonce (which becomes nonce||ct). - ct := gcm.Seal(nil, nonce, plaintextKey, nil) + // The magic is authenticated as additional data, so a v2 blob cannot be + // stripped of its header and re-read as a v1 blob. + ct := gcm.Seal(nil, nonce, plaintextKey, blobMagicV2) - out := make([]byte, 0, saltLen+nonceLen+len(ct)) + out := make([]byte, 0, len(blobMagicV2)+saltLen+nonceLen+len(ct)) + out = append(out, blobMagicV2...) out = append(out, salt...) out = append(out, nonce...) out = append(out, ct...) return out, nil } -// UnwrapKey extracts the salt from blob, re-derives the wrapping key from -// credPublicKey, and AES-GCM-unwraps. Returns the plaintext 32-byte AES key. -func UnwrapKey(blob, credPublicKey []byte) ([]byte, error) { - if len(blob) < saltLen+nonceLen+1 { - return nil, fmt.Errorf("%w: blob too short (%d)", ErrKeyUnwrap, len(blob)) +// UnwrapKey recovers the plaintext AES-256 key from blob. +// +// It reads both formats and reports which one it got, so the caller can warn +// that a v1 blob offers no real protection. For a v2 blob, secret must be the +// 32-byte PRF output; for a v1 blob it is the credential public key, whatever +// length that happens to be. +func UnwrapKey(blob, secret []byte) ([]byte, BlobVersion, error) { + if len(blob) > maxBlobLen { + return nil, 0, ErrBlobTooLong } - if len(blob) > 1<<20 { // 1MB sanity limit - return nil, ErrBlobTooLong - } - if len(credPublicKey) == 0 { - return nil, fmt.Errorf("%w: empty credential public key", ErrKeyUnwrap) + if len(secret) == 0 { + return nil, 0, fmt.Errorf("%w: empty secret", ErrKeyUnwrap) } - salt := blob[:saltLen] - nonce := blob[saltLen : saltLen+nonceLen] - ct := blob[saltLen+nonceLen:] + if len(blob) >= len(blobMagicV2) && subtle.ConstantTimeCompare(blob[:len(blobMagicV2)], blobMagicV2) == 1 { + key, err := unwrap(blob[len(blobMagicV2):], secret, wrapInfoV2, blobMagicV2, secretLen) + return key, BlobV2, err + } + key, err := unwrap(blob, secret, wrapInfoV1, nil, 0) + return key, BlobV1, err +} - wrapKey := hkdfSHA256(credPublicKey, salt, []byte(wrapInfo), keyLen) +// unwrap does the shared salt||nonce||ct work. wantSecretLen of 0 means any +// non-empty secret is accepted (the v1 case, where it is a public key). +func unwrap(body, secret []byte, info string, aad []byte, wantSecretLen int) ([]byte, error) { + if len(body) < saltLen+nonceLen+1 { + return nil, fmt.Errorf("%w: blob too short (%d)", ErrKeyUnwrap, len(body)) + } + // The v2 side is held to exactly what WrapKey demands, all-zero included. + // Letting the two ends disagree about what a valid secret is would leave + // a blob that can be opened by material that could never have sealed it. + if wantSecretLen > 0 { + if len(secret) != wantSecretLen { + return nil, fmt.Errorf("%w: %v", ErrKeyUnwrap, ErrSecretLen) + } + if err := checkSecret(secret); err != nil { + return nil, fmt.Errorf("%w: %v", ErrKeyUnwrap, err) + } + } - block, err := aes.NewCipher(wrapKey) + salt := body[:saltLen] + nonce := body[saltLen : saltLen+nonceLen] + ct := body[saltLen+nonceLen:] + + gcm, err := gcmFor(secret, salt, info) if err != nil { - return nil, fmt.Errorf("%w: aes: %v", ErrKeyUnwrap, err) + return nil, fmt.Errorf("%w: %v", ErrKeyUnwrap, err) } - gcm, err := cipher.NewGCM(block) - if err != nil { - return nil, fmt.Errorf("%w: gcm: %v", ErrKeyUnwrap, err) - } - - plain, err := gcm.Open(nil, nonce, ct, nil) + plain, err := gcm.Open(nil, nonce, ct, aad) if err != nil { return nil, fmt.Errorf("%w: decrypt failed (wrong credential?)", ErrKeyUnwrap) } + if len(plain) != keyLen { + return nil, fmt.Errorf("%w: unwrapped key is %d bytes, want %d", ErrKeyUnwrap, len(plain), keyLen) + } return plain, nil } -// hkdfSHA256 implements HKDF-SHA256 (RFC 5869) using only stdlib. +// gcmFor derives the wrapping key with HKDF-SHA256 and returns a GCM AEAD. // -// Input: -// - secret: the input key material (credential public key bytes) -// - salt: random salt (16 bytes) -// - info: optional context string for domain separation -// - length: desired output length in bytes -// -// Output: length bytes of derived key material. -// -// HKDF is extract-then-expand. We use HMAC-SHA256 for both steps. This avoids -// importing golang.org/x/crypto/hkdf — a ~30-line function vs a new dep. The -// tradeoff is no constant-time guarantees on the extract step beyond HMAC's; -// acceptable here because the input is already high-entropy key material (a -// P-256 public key), not a low-entropy passphrase. -func hkdfSHA256(secret, salt, info []byte, length int) []byte { - // Step 1: Extract — PRK = HMAC-SHA256(salt, secret) - // If salt is nil/empty, use a zero-filled block (RFC 5869 §2.2). - if salt == nil { - salt = make([]byte, sha256.Size) +// This uses the standard library's crypto/hkdf rather than the hand-rolled +// HKDF this file used to carry. That implementation keyed the expand step with +// the salt instead of the PRK — self-consistent, so wrap and unwrap agreed, +// but not RFC 5869 and not the domain separation it claimed to provide. +func gcmFor(secret, salt []byte, info string) (cipher.AEAD, error) { + wrapKey, err := hkdf.Key(sha256.New, secret, salt, info, keyLen) + if err != nil { + return nil, fmt.Errorf("hkdf: %v", err) } - mac := hmac.New(sha256.New, salt) - mac.Write(secret) - prk := mac.Sum(nil) - - // Step 2: Expand — produce length bytes via T(i) = HMAC-SHA256(PRK, T(i-1) || info || i) - // Where T(0) = empty, i is a byte counter starting at 1. - out := make([]byte, 0, length) - block := make([]byte, 0, sha256.Size+len(info)+1) - var t []byte // T(i-1) - for counter := byte(1); len(out) < length; counter++ { - block = block[:0] - block = append(block, t...) - block = append(block, info...) - block = append(block, counter) - - mac.Reset() - mac.Write(block) - t = mac.Sum(prk[:0]) // reuse prk buffer — mac.Sum appends to its arg - // t now starts with prk[:0] (empty) followed by the HMAC result. - // Since we need just the HMAC result (sha256.Size bytes), re-slice. - t = t[len(t)-sha256.Size:] - out = append(out, t...) + block, err := aes.NewCipher(wrapKey) + if err != nil { + return nil, fmt.Errorf("aes: %v", err) } - return out[:length] + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, fmt.Errorf("gcm: %v", err) + } + return gcm, nil } -// encodeUint32 — big-endian uint32 for the blob format header, if needed. -func encodeUint32(v uint32) []byte { - var b [4]byte - binary.BigEndian.PutUint32(b[:], v) - return b[:] +func checkSecret(secret []byte) error { + if len(secret) != secretLen { + return fmt.Errorf("%w (got %d bytes)", ErrSecretLen, len(secret)) + } + // An all-zero PRF result means the authenticator returned nothing useful; + // wrapping under it would produce a blob anyone can open. + var acc byte + for _, b := range secret { + acc |= b + } + if acc == 0 { + return fmt.Errorf("%w (all zero)", ErrSecretLen) + } + return nil } diff --git a/internal/webauthn/keywrap_test.go b/internal/webauthn/keywrap_test.go new file mode 100644 index 0000000..c3d2816 --- /dev/null +++ b/internal/webauthn/keywrap_test.go @@ -0,0 +1,246 @@ +package webauthn + +import ( + "bytes" + "crypto/rand" + "errors" + "io" + "testing" +) + +func testSecret(t *testing.T) []byte { + t.Helper() + s := make([]byte, secretLen) + if _, err := io.ReadFull(rand.Reader, s); err != nil { + t.Fatalf("rand: %v", err) + } + s[0] |= 1 // never all-zero + return s +} + +func testKey(t *testing.T) []byte { + t.Helper() + k := make([]byte, keyLen) + if _, err := io.ReadFull(rand.Reader, k); err != nil { + t.Fatalf("rand: %v", err) + } + return k +} + +func TestWrapUnwrapRoundTrip(t *testing.T) { + key, secret := testKey(t), testSecret(t) + + blob, err := WrapKey(key, secret) + if err != nil { + t.Fatalf("WrapKey: %v", err) + } + if !bytes.HasPrefix(blob, blobMagicV2) { + t.Fatalf("blob does not start with the v2 magic: %x", blob[:8]) + } + // The plaintext key must not be recoverable by reading the file. + if bytes.Contains(blob, key) { + t.Fatal("the wrapped blob contains the plaintext key verbatim") + } + + got, version, err := UnwrapKey(blob, secret) + if err != nil { + t.Fatalf("UnwrapKey: %v", err) + } + if version != BlobV2 { + t.Errorf("version = %v, want v2", version) + } + if !bytes.Equal(got, key) { + t.Errorf("unwrapped key differs from the wrapped one") + } +} + +// Fresh salt and nonce per wrap: two blobs of the same key under the same +// secret must not be byte-identical, or the file leaks that nothing changed. +func TestWrapKeyIsNotDeterministic(t *testing.T) { + key, secret := testKey(t), testSecret(t) + a, err := WrapKey(key, secret) + if err != nil { + t.Fatalf("WrapKey: %v", err) + } + b, err := WrapKey(key, secret) + if err != nil { + t.Fatalf("WrapKey: %v", err) + } + if bytes.Equal(a, b) { + t.Fatal("two wraps of the same key produced identical blobs") + } +} + +// The failure mode that matters most: a wrong passkey must not unlock. +func TestUnwrapWithWrongSecretFails(t *testing.T) { + key := testKey(t) + blob, err := WrapKey(key, testSecret(t)) + if err != nil { + t.Fatalf("WrapKey: %v", err) + } + got, _, err := UnwrapKey(blob, testSecret(t)) + if err == nil { + t.Fatal("a different secret unwrapped the blob") + } + if !errors.Is(err, ErrKeyUnwrap) { + t.Errorf("err = %v, want ErrKeyUnwrap", err) + } + if got != nil { + t.Error("key material returned alongside an error") + } +} + +// One flipped bit anywhere must fail the GCM tag, including in the salt and +// nonce — those are not authenticated by the tag but they change the +// derivation, so the tag fails anyway. +func TestUnwrapRejectsTamperedBlob(t *testing.T) { + key, secret := testKey(t), testSecret(t) + blob, err := WrapKey(key, secret) + if err != nil { + t.Fatalf("WrapKey: %v", err) + } + for i := range blob { + bad := bytes.Clone(blob) + bad[i] ^= 0x01 + if _, _, err := UnwrapKey(bad, secret); err == nil { + t.Fatalf("byte %d of %d could be flipped and the blob still opened", i, len(blob)) + } + } +} + +func TestUnwrapRejectsTruncatedBlob(t *testing.T) { + key, secret := testKey(t), testSecret(t) + blob, err := WrapKey(key, secret) + if err != nil { + t.Fatalf("WrapKey: %v", err) + } + for _, n := range []int{0, 1, len(blobMagicV2), len(blobMagicV2) + saltLen, len(blob) - 1} { + if _, _, err := UnwrapKey(blob[:n], secret); err == nil { + t.Errorf("a %d-byte blob unwrapped", n) + } + } +} + +// A v2 blob must not be downgradeable to v1 by stripping its header: the magic +// is GCM additional data, so the tag fails once it is gone. +func TestV2BlobCannotBeStrippedToV1(t *testing.T) { + key, secret := testKey(t), testSecret(t) + blob, err := WrapKey(key, secret) + if err != nil { + t.Fatalf("WrapKey: %v", err) + } + if _, _, err := UnwrapKey(blob[len(blobMagicV2):], secret); err == nil { + t.Fatal("a header-stripped v2 blob was accepted as v1") + } +} + +// v1 blobs still open, and report themselves as v1 so the daemon can warn. +// wrapV1 reproduces the legacy writer this file no longer has. +func wrapV1(t *testing.T, key, secret []byte) []byte { + t.Helper() + salt := make([]byte, saltLen) + nonce := make([]byte, nonceLen) + if _, err := io.ReadFull(rand.Reader, salt); err != nil { + t.Fatalf("rand: %v", err) + } + if _, err := io.ReadFull(rand.Reader, nonce); err != nil { + t.Fatalf("rand: %v", err) + } + gcm, err := gcmFor(secret, salt, wrapInfoV1) + if err != nil { + t.Fatalf("gcmFor: %v", err) + } + out := append([]byte{}, salt...) + out = append(out, nonce...) + return append(out, gcm.Seal(nil, nonce, key, nil)...) +} + +func TestUnwrapReadsLegacyV1(t *testing.T) { + key := testKey(t) + // v1 was keyed on the credential public key: not 32 bytes, and that is + // deliberately still accepted on the read path. + pub := make([]byte, 77) + if _, err := io.ReadFull(rand.Reader, pub); err != nil { + t.Fatalf("rand: %v", err) + } + blob := wrapV1(t, key, pub) + + got, version, err := UnwrapKey(blob, pub) + if err != nil { + t.Fatalf("UnwrapKey(v1): %v", err) + } + if version != BlobV1 { + t.Errorf("version = %v, want v1", version) + } + if !bytes.Equal(got, key) { + t.Error("v1 round-trip lost the key") + } + if _, _, err := UnwrapKey(blob, pub[:76]); err == nil { + t.Error("a truncated public key opened the v1 blob") + } +} + +// The structural guard against the bug this replaces: a COSE public key is not +// 32 bytes, so it can never be used to write a new blob. +func TestWrapKeyRefusesNonPRFSecret(t *testing.T) { + key := testKey(t) + cases := map[string][]byte{ + "nil": nil, + "empty": {}, + "short": make([]byte, 16), + "cose public key": make([]byte, 77), + "all-zero 32 byte": make([]byte, 32), + } + for name, secret := range cases { + t.Run(name, func(t *testing.T) { + if _, err := WrapKey(key, secret); err == nil { + t.Fatalf("WrapKey accepted a %s secret", name) + } + }) + } +} + +func TestWrapKeyRefusesWrongKeyLength(t *testing.T) { + secret := testSecret(t) + for _, n := range []int{0, 16, 31, 33, 64} { + if _, err := WrapKey(make([]byte, n), secret); err == nil { + t.Errorf("WrapKey accepted a %d-byte plaintext key", n) + } + } +} + +// A v2 blob demands exactly 32 bytes on the read path too, so a caller cannot +// go back to passing a public key. +func TestUnwrapV2RefusesNonPRFSecret(t *testing.T) { + blob, err := WrapKey(testKey(t), testSecret(t)) + if err != nil { + t.Fatalf("WrapKey: %v", err) + } + if _, _, err := UnwrapKey(blob, make([]byte, 77)); !errors.Is(err, ErrKeyUnwrap) { + t.Fatalf("err = %v, want ErrKeyUnwrap for a 77-byte secret", err) + } + if _, _, err := UnwrapKey(blob, nil); err == nil { + t.Fatal("an empty secret unwrapped a v2 blob") + } +} + +func TestUnwrapRejectsOversizeBlob(t *testing.T) { + if _, _, err := UnwrapKey(make([]byte, maxBlobLen+1), testSecret(t)); !errors.Is(err, ErrBlobTooLong) { + t.Fatalf("err = %v, want ErrBlobTooLong", err) + } +} + +// WrapKey refuses an all-zero secret because a blob wrapped under one is a +// blob anyone can open. The v2 unwrap side must refuse it for the same reason: +// if the two ends disagree about what a valid secret is, a blob can be opened +// by material that could never have sealed it. +func TestUnwrapV2RefusesAnAllZeroSecret(t *testing.T) { + key := bytes.Repeat([]byte{1}, 32) + blob, err := WrapKey(key, bytes.Repeat([]byte{2}, 32)) + if err != nil { + t.Fatalf("WrapKey: %v", err) + } + if _, _, err := UnwrapKey(blob, make([]byte, 32)); !errors.Is(err, ErrKeyUnwrap) { + t.Fatalf("UnwrapKey with an all-zero secret = %v, want refusal", err) + } +} diff --git a/internal/webauthn/prf.go b/internal/webauthn/prf.go new file mode 100644 index 0000000..21655e6 --- /dev/null +++ b/internal/webauthn/prf.go @@ -0,0 +1,70 @@ +package webauthn + +import ( + "crypto/sha256" + "encoding/base64" + "errors" + "fmt" +) + +// The WebAuthn PRF extension is where cold-start unlock gets its secret +// (Vikunja #14). The authenticator evaluates a keyed PRF over a salt we +// choose and returns 32 bytes that are: +// +// - stable — the same credential and the same salt always give the same +// bytes, which is what lets a blob wrapped today be opened tomorrow; +// - secret — they never leave the authenticator except as this output, so +// unlike the credential public key they are not sitting in passkeys.json; +// - bound to user verification — the assertion that produces them required +// a gesture, so the bytes cannot be harvested silently. +// +// The salt is fixed and public. It is a domain separator, not a secret: it +// makes maven's PRF output different from any other relying party's use of +// the same credential. + +// prfSaltInput — the string hashed into the 32-byte evaluation salt. Changing +// it invalidates every wrapped key file in existence, which is why it is a +// constant and not configuration. +const prfSaltInput = "maven-coldstart-unlock-v1" + +// PRFSalt returns the fixed 32-byte PRF evaluation salt. +func PRFSalt() []byte { + sum := sha256.Sum256([]byte(prfSaltInput)) + return sum[:] +} + +// ErrNoPRF is returned when a browser reports no PRF result — either the +// authenticator does not implement the extension, or the platform stripped +// it. Cold-start unlock is unavailable for that credential, and the correct +// response is to say so rather than to fall back to something weaker. +var ErrNoPRF = errors.New("webauthn: authenticator returned no PRF result (cold-start unlock unavailable)") + +// DecodePRFResult parses the base64url PRF output the browser read out of +// getClientExtensionResults().prf.results.first and checks it is usable as +// wrapping key material. +// +// The browser is not trusted to send something sensible: a short, empty, or +// all-zero result would silently produce a blob that anyone can open, so all +// three are refused here rather than at the crypto layer. +func DecodePRFResult(b64 string) ([]byte, error) { + if b64 == "" { + return nil, ErrNoPRF + } + secret, err := decodeB64Any(b64) + if err != nil { + return nil, fmt.Errorf("webauthn: prf result: %w", err) + } + if err := checkSecret(secret); err != nil { + return nil, err + } + return secret, nil +} + +// decodeB64Any accepts padded or unpadded base64url — browsers differ, and +// the JS helper on the passkey page strips padding. +func decodeB64Any(s string) ([]byte, error) { + if b, err := base64.RawURLEncoding.DecodeString(s); err == nil { + return b, nil + } + return base64.URLEncoding.DecodeString(s) +} diff --git a/internal/webauthn/prf_test.go b/internal/webauthn/prf_test.go new file mode 100644 index 0000000..11038e0 --- /dev/null +++ b/internal/webauthn/prf_test.go @@ -0,0 +1,114 @@ +package webauthn + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "errors" + "testing" +) + +// The salt is the identity of every wrapped key file ever written. If it +// changes, every deployment's blob becomes unopenable, so it is pinned here. +func TestPRFSaltIsStable(t *testing.T) { + salt := PRFSalt() + if len(salt) != 32 { + t.Fatalf("salt is %d bytes, want 32", len(salt)) + } + if got := base64.RawURLEncoding.EncodeToString(salt); got != base64.RawURLEncoding.EncodeToString(PRFSalt()) { + t.Fatal("PRFSalt is not deterministic") + } + // Mutating the returned slice must not affect the next caller. + salt[0] ^= 0xff + if bytes.Equal(salt, PRFSalt()) { + t.Fatal("PRFSalt returned shared backing state") + } +} + +func TestDecodePRFResult(t *testing.T) { + raw := make([]byte, 32) + for i := range raw { + raw[i] = byte(i + 1) + } + for _, enc := range []string{ + base64.RawURLEncoding.EncodeToString(raw), + base64.URLEncoding.EncodeToString(raw), + } { + got, err := DecodePRFResult(enc) + if err != nil { + t.Fatalf("DecodePRFResult(%q): %v", enc, err) + } + if !bytes.Equal(got, raw) { + t.Errorf("decoded %x, want %x", got, raw) + } + } +} + +// No PRF must be a distinguishable, named failure — never a silent fallback to +// some other secret. +func TestDecodePRFResultNoPRF(t *testing.T) { + if _, err := DecodePRFResult(""); !errors.Is(err, ErrNoPRF) { + t.Fatalf("err = %v, want ErrNoPRF", err) + } +} + +func TestDecodePRFResultRejectsUnusable(t *testing.T) { + zeros := base64.RawURLEncoding.EncodeToString(make([]byte, 32)) + short := base64.RawURLEncoding.EncodeToString(make([]byte, 16)) + long := base64.RawURLEncoding.EncodeToString(make([]byte, 64)) + for name, in := range map[string]string{ + "not base64": "!!!!", + "all zero": zeros, + "too short": short, + "too long": long, + } { + t.Run(name, func(t *testing.T) { + if _, err := DecodePRFResult(in); err == nil { + t.Fatalf("accepted a %s PRF result", name) + } + }) + } +} + +// Both option builders must ask for PRF, or the browser never produces a +// secret and cold-start unlock silently never works. +func TestOptionsRequestPRF(t *testing.T) { + rp := NewRP(Config{Origin: "http://localhost:8080", RPID: "localhost", RPName: "maven"}) + + create, _, err := rp.CreationOptions([]byte("u"), "u") + if err != nil { + t.Fatalf("CreationOptions: %v", err) + } + if _, ok := extPRF(t, create)["prf"]; !ok { + t.Error("creation options do not request the prf extension") + } + + assert, _, err := rp.AssertionOptions() + if err != nil { + t.Fatalf("AssertionOptions: %v", err) + } + prf, ok := extPRF(t, assert)["prf"].(map[string]any) + if !ok { + t.Fatal("assertion options do not request the prf extension") + } + eval, _ := prf["eval"].(map[string]any) + first, _ := eval["first"].(string) + if first != base64.RawURLEncoding.EncodeToString(PRFSalt()) { + t.Errorf("prf.eval.first = %q, want the fixed salt", first) + } +} + +func extPRF(t *testing.T, opts any) map[string]any { + t.Helper() + b, err := json.Marshal(opts) + if err != nil { + t.Fatalf("marshal options: %v", err) + } + var m struct { + Extensions map[string]any `json:"extensions"` + } + if err := json.Unmarshal(b, &m); err != nil { + t.Fatalf("unmarshal options: %v", err) + } + return m.Extensions +} diff --git a/internal/webauthn/webauthn.go b/internal/webauthn/webauthn.go index 21d4fb8..0a366a4 100644 --- a/internal/webauthn/webauthn.go +++ b/internal/webauthn/webauthn.go @@ -124,6 +124,13 @@ func (rp *RP) CreationOptions(userID []byte, userName string) (map[string]any, s "timeout": 60000, "attestation": "none", "excludeCredentials": []any{}, + // PRF: ask the authenticator at enrollment time whether it can + // produce a per-credential secret. Nothing is wrapped here — the + // browser reports support back and mavweb decides whether cold-start + // unlock is available for this credential. See internal/webauthn/prf.go. + "extensions": map[string]any{ + "prf": map[string]any{}, + }, }, challengeB64, nil } @@ -193,6 +200,15 @@ func (rp *RP) AssertionOptions() (map[string]any, string, error) { "rpId": rp.cfg.RPID, "allowCredentials": []any{}, "userVerification": "required", + // PRF evaluation over the fixed cold-start salt. The 32 bytes that + // come back are the ONLY thing that can unwrap the database key. + "extensions": map[string]any{ + "prf": map[string]any{ + "eval": map[string]any{ + "first": base64.RawURLEncoding.EncodeToString(PRFSalt()), + }, + }, + }, }, challengeB64, nil } diff --git a/internal/webfetch/webfetch.go b/internal/webfetch/webfetch.go new file mode 100644 index 0000000..5f58050 --- /dev/null +++ b/internal/webfetch/webfetch.go @@ -0,0 +1,348 @@ +// Package webfetch is the one door Maven uses to read something off the +// network, and it is a narrow one. +// +// "Never phones home" stopped being a hard constraint on 2026-07-31, but what +// replaced it is not "she may fetch anything": local sources come first (Kiwix +// on the box), external fetching is off unless configured, and only the +// utterance ever leaves — never his notes, facts or history. That policy is +// enforced by the callers. What THIS package enforces is the part that must be +// code rather than a paragraph in a plan, because it protects the homelab from +// its own assistant: +// +// - http/https only — no file://, no ftp://, no gopher; +// - no private address, ever: loopback, RFC1918 (which is what makes the +// 10.42.0.0/24 wireguard tunnel and the 192.168.1.0/24 LAN unreachable), +// link-local incl. the 169.254.169.254 cloud metadata address, CGNAT, +// unique-local v6. Checked in the dialer's Control hook, so it holds for +// every address the resolver returns AND for every hop of a redirect +// chain — a DNS name that resolves to 127.0.0.1 is refused at connect +// time, which a pre-flight lookup could not promise (rebinding); +// - an allowlist, when one is configured, and a denylist that always wins; +// - a response size cap, a total timeout, a redirect cap; +// - one request per host per interval, so a poll loop with a bug is slow +// rather than an outbound flood. +// +// Everything above is on by default with sane numbers: a zero Config is a +// usable, conservative fetcher. There is no cache and no retry — a feed poll +// or a page read that fails is simply not answered this round. +package webfetch + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strings" + "sync" + "syscall" + "time" +) + +// Defaults. Small on purpose: this reads feeds and article pages, not ISOs. +const ( + DefaultTimeout = 20 * time.Second + DefaultMaxBytes = 2 << 20 // 2 MiB + DefaultMaxRedirects = 3 + DefaultHostInterval = time.Second + DefaultUserAgent = "Maven/1.0 (self-hosted personal assistant)" +) + +// Errors callers distinguish. Everything else is wrapped transport error. +var ( + ErrScheme = errors.New("webfetch: only http and https are allowed") + ErrBlocked = errors.New("webfetch: host is not allowed") + ErrPrivate = errors.New("webfetch: refusing to connect to a private address") + ErrTooLarge = errors.New("webfetch: response exceeds the size cap") + ErrRedirects = errors.New("webfetch: too many redirects") + ErrStatus = errors.New("webfetch: non-2xx status") +) + +// Config are the limits. Every zero value means "the default above", so +// Config{} is safe; the only field that changes behaviour by being empty is +// AllowHosts (empty ⇒ any public host that is not denied). +type Config struct { + // AllowHosts — when non-empty, the ONLY hosts that may be fetched. An + // entry matches the host itself and its subdomains ("example.com" allows + // "news.example.com"). This is the knob to reach for when a capability + // should read two feeds and nothing else. + AllowHosts []string + // DenyHosts — same matching, checked first and always winning. + DenyHosts []string + + Timeout time.Duration // whole request, including redirects and body read + MaxBytes int64 // response body cap + MaxRedirects int // 0 ⇒ default; negative ⇒ no redirects followed + HostInterval time.Duration // minimum spacing between requests to one host + UserAgent string + + // AllowPrivate disables the private-address guard. It exists for tests + // (httptest listens on 127.0.0.1) and for an explicitly configured + // on-box mirror. Nothing in deploy/mavend.json sets it, and it should + // stay that way: with it on, any URL Maven is handed becomes an SSRF + // probe of the LAN and the wireguard range. + AllowPrivate bool +} + +// Response is a fetched body, already bounded by MaxBytes. +type Response struct { + URL string // final URL after redirects + Status int + ContentType string + Body []byte + // Header — the response headers, one value each (the first). Populated for + // every request; MCP needs Mcp-Session-Id, nothing else reads it. + Header map[string]string +} + +// Fetcher performs guarded GETs. Safe for concurrent use; the per-host rate +// limiter is shared, which is the point of sharing one Fetcher. +type Fetcher struct { + cfg Config + http *http.Client + + mu sync.Mutex + last map[string]time.Time // host → when we last dialed it +} + +// New builds a fetcher from cfg, filling in defaults. +func New(cfg Config) *Fetcher { + if cfg.Timeout <= 0 { + cfg.Timeout = DefaultTimeout + } + if cfg.MaxBytes <= 0 { + cfg.MaxBytes = DefaultMaxBytes + } + if cfg.MaxRedirects == 0 { + cfg.MaxRedirects = DefaultMaxRedirects + } + if cfg.HostInterval <= 0 { + cfg.HostInterval = DefaultHostInterval + } + if cfg.UserAgent == "" { + cfg.UserAgent = DefaultUserAgent + } + f := &Fetcher{cfg: cfg, last: map[string]time.Time{}} + + dialer := &net.Dialer{Timeout: 10 * time.Second} + if !cfg.AllowPrivate { + // The guard lives here rather than in a pre-flight net.LookupHost so + // that it sees the address actually being connected to: every A/AAAA + // the resolver handed back, on every redirect hop, with no window in + // which the name could be re-pointed at the LAN. + dialer.Control = func(_, address string, _ syscall.RawConn) error { + host, _, err := net.SplitHostPort(address) + if err != nil { + return err + } + ip := net.ParseIP(host) + if ip == nil || IsPrivateIP(ip) { + return fmt.Errorf("%w: %s", ErrPrivate, host) + } + return nil + } + } + f.http = &http.Client{ + Timeout: cfg.Timeout, + Transport: &http.Transport{DialContext: dialer.DialContext}, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) > f.cfg.MaxRedirects { + return ErrRedirects + } + // A redirect is a fresh URL and gets the full check: an allowed + // host must not be able to bounce us onto a denied one. + return f.checkURL(req.URL) + }, + } + return f +} + +// Get fetches rawURL. The body is capped: a larger response is an error, not a +// truncation, because half an XML document is worse than none. +func (f *Fetcher) Get(ctx context.Context, rawURL string) (*Response, error) { + return f.do(ctx, http.MethodGet, rawURL, nil, nil) +} + +// Post sends body to rawURL and returns the reply, under exactly the same +// guards as Get: scheme rule, host lists, the dialer's private-address check on +// every hop, the size cap and the per-host rate limit. +// +// It exists for JSON-RPC over HTTP (internal/mcp), which cannot be expressed as +// a GET. That an outbound request now carries a body does not widen the +// address policy one bit — a POST to the LAN is refused for the same reason a +// GET is, unless AllowPrivate was set for that specific fetcher. +// +// hdr is merged over the defaults; a caller may not override User-Agent or +// Accept-Encoding, because identity encoding and an honest UA are part of the +// contract with whatever is on the other end. +func (f *Fetcher) Post(ctx context.Context, rawURL, contentType string, body []byte, hdr map[string]string) (*Response, error) { + if contentType == "" { + contentType = "application/json" + } + if hdr == nil { + hdr = map[string]string{} + } + merged := make(map[string]string, len(hdr)+1) + for k, v := range hdr { + merged[k] = v + } + merged["Content-Type"] = contentType + return f.do(ctx, http.MethodPost, rawURL, body, merged) +} + +func (f *Fetcher) do(ctx context.Context, method, rawURL string, body []byte, hdr map[string]string) (*Response, error) { + u, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil { + return nil, fmt.Errorf("webfetch: bad url %q: %w", rawURL, err) + } + if err := f.checkURL(u); err != nil { + return nil, err + } + if err := f.waitTurn(ctx, u.Hostname()); err != nil { + return nil, err + } + + var rdr io.Reader + if body != nil { + rdr = bytes.NewReader(body) + } + req, err := http.NewRequestWithContext(ctx, method, u.String(), rdr) + if err != nil { + return nil, err + } + for k, v := range hdr { + req.Header.Set(k, v) + } + req.Header.Set("User-Agent", f.cfg.UserAgent) + req.Header.Set("Accept-Encoding", "identity") + + resp, err := f.http.Do(req) + if err != nil { + // http.Client wraps our sentinels in *url.Error; unwrap so callers can + // still tell "blocked" from "the network is down". + for _, sentinel := range []error{ErrPrivate, ErrBlocked, ErrRedirects, ErrScheme} { + if errors.Is(err, sentinel) { + return nil, err + } + } + return nil, err + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(io.LimitReader(resp.Body, f.cfg.MaxBytes+1)) + if err != nil { + return nil, err + } + if int64(len(respBody)) > f.cfg.MaxBytes { + return nil, fmt.Errorf("%w (%d bytes)", ErrTooLarge, f.cfg.MaxBytes) + } + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return nil, fmt.Errorf("%w: %d", ErrStatus, resp.StatusCode) + } + out := &Response{ + URL: resp.Request.URL.String(), + Status: resp.StatusCode, + ContentType: resp.Header.Get("Content-Type"), + Body: respBody, + Header: map[string]string{}, + } + for k := range resp.Header { + out.Header[k] = resp.Header.Get(k) + } + return out, nil +} + +// checkURL applies the scheme rule and the host lists. The address rule is the +// dialer's job (see New). +func (f *Fetcher) checkURL(u *url.URL) error { + switch u.Scheme { + case "http", "https": + default: + return fmt.Errorf("%w: %q", ErrScheme, u.Scheme) + } + host := strings.ToLower(u.Hostname()) + if host == "" { + return fmt.Errorf("%w: no host", ErrBlocked) + } + if HostMatches(host, f.cfg.DenyHosts) { + return fmt.Errorf("%w: %s is denied", ErrBlocked, host) + } + if len(f.cfg.AllowHosts) > 0 && !HostMatches(host, f.cfg.AllowHosts) { + return fmt.Errorf("%w: %s is not on the allowlist", ErrBlocked, host) + } + // A literal private address is refused here as well as in the dialer, so + // the error is the specific one even when no connection is attempted. + if !f.cfg.AllowPrivate { + if ip := net.ParseIP(host); ip != nil && IsPrivateIP(ip) { + return fmt.Errorf("%w: %s", ErrPrivate, host) + } + } + return nil +} + +// waitTurn blocks until this host's rate-limit interval has elapsed. It holds +// no lock while sleeping, so two hosts never wait on each other. +func (f *Fetcher) waitTurn(ctx context.Context, host string) error { + for { + f.mu.Lock() + now := time.Now() + earliest := f.last[host].Add(f.cfg.HostInterval) + if !now.Before(earliest) { + f.last[host] = now + f.mu.Unlock() + return nil + } + f.mu.Unlock() + wait := time.NewTimer(earliest.Sub(now)) + select { + case <-ctx.Done(): + wait.Stop() + return ctx.Err() + case <-wait.C: + } + } +} + +// HostMatches reports whether host equals one of pats or is a subdomain of one. +// Exported because the crawler applies the same rule to links it decides not to +// follow, before it ever builds a request. +func HostMatches(host string, pats []string) bool { + host = strings.ToLower(strings.TrimSuffix(host, ".")) + for _, p := range pats { + p = strings.ToLower(strings.TrimSpace(strings.TrimPrefix(p, "*."))) + if p == "" { + continue + } + if host == p || strings.HasSuffix(host, "."+p) { + return true + } + } + return false +} + +// cgnat is 100.64.0.0/10 — carrier NAT, not covered by net.IP's helpers and not +// somewhere a personal assistant has business connecting. +var cgnat = &net.IPNet{IP: net.IPv4(100, 64, 0, 0).To4(), Mask: net.CIDRMask(10, 32)} + +// IsPrivateIP reports whether ip is somewhere Maven must never reach out to: +// the box itself, the LAN, the wireguard range (10.42.0.0/24 ⊂ 10/8), the cloud +// metadata address (169.254.169.254 ⊂ link-local), or anything unroutable. +func IsPrivateIP(ip net.IP) bool { + if ip.IsLoopback() || ip.IsPrivate() || ip.IsUnspecified() || + ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || + ip.IsInterfaceLocalMulticast() || ip.IsMulticast() { + return true + } + if v4 := ip.To4(); v4 != nil && cgnat.Contains(v4) { + return true + } + // IPv4-mapped/compatible forms of the above are handled by To4() inside the + // stdlib helpers; what is left is v6 unique-local (fc00::/7). + if len(ip) == net.IPv6len && ip.To4() == nil && ip[0]&0xfe == 0xfc { + return true + } + return false +} diff --git a/internal/webfetch/webfetch_test.go b/internal/webfetch/webfetch_test.go new file mode 100644 index 0000000..fafdabb --- /dev/null +++ b/internal/webfetch/webfetch_test.go @@ -0,0 +1,299 @@ +package webfetch + +import ( + "context" + "errors" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +// The limits in this package are the reason a crawler is allowed to exist on +// this box at all, so each one has a test that fails loudly if it is removed. + +func TestPrivateAddressesAreRefused(t *testing.T) { + // The wireguard range (10.42.0.0/24), the LAN (192.168.1.0/24) and the + // cloud metadata address are the three that matter here; the rest come + // along for free. + for _, s := range []string{ + "127.0.0.1", "127.1.2.3", "10.42.0.7", "10.0.0.5", "192.168.1.104", + "172.16.4.4", "169.254.169.254", "100.64.1.1", "0.0.0.0", + "::1", "fc00::1", "fd12:3456::1", "fe80::1", + } { + if !IsPrivateIP(net.ParseIP(s)) { + t.Errorf("IsPrivateIP(%s) = false, want true", s) + } + } + for _, s := range []string{"8.8.8.8", "1.1.1.1", "93.184.216.34", "2606:2800:220:1::1"} { + if IsPrivateIP(net.ParseIP(s)) { + t.Errorf("IsPrivateIP(%s) = true, want false", s) + } + } +} + +func TestGetRefusesPrivateLiteral(t *testing.T) { + f := New(Config{}) + for _, u := range []string{ + "http://127.0.0.1:8034/search", + "http://10.42.0.1/", + "http://192.168.1.104/dash", + "http://[::1]:9100/mcp", + } { + if _, err := f.Get(context.Background(), u); !errors.Is(err, ErrPrivate) { + t.Errorf("Get(%s) error = %v, want ErrPrivate", u, err) + } + } +} + +// A hostname that resolves into private space must fail too — that is the +// rebinding case, and it is why the check lives in the dialer. +func TestGetRefusesPrivateResolution(t *testing.T) { + f := New(Config{}) + if _, err := f.Get(context.Background(), "http://localhost:8034/"); !errors.Is(err, ErrPrivate) { + t.Fatalf("Get(localhost) error = %v, want ErrPrivate", err) + } +} + +func TestGetRefusesNonHTTPSchemes(t *testing.T) { + f := New(Config{}) + for _, u := range []string{"file:///etc/passwd", "ftp://example.com/x", "gopher://example.com"} { + if _, err := f.Get(context.Background(), u); !errors.Is(err, ErrScheme) { + t.Errorf("Get(%s) error = %v, want ErrScheme", u, err) + } + } +} + +// testFetcher — a fetcher pointed at an httptest server, which necessarily +// listens on loopback. AllowPrivate is the test-only escape hatch. +func testFetcher(t *testing.T, cfg Config) *Fetcher { + t.Helper() + cfg.AllowPrivate = true + if cfg.HostInterval == 0 { + cfg.HostInterval = time.Nanosecond + } + return New(cfg) +} + +func TestAllowAndDenyLists(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("ok")) + })) + defer srv.Close() + + f := testFetcher(t, Config{AllowHosts: []string{"example.com"}}) + if _, err := f.Get(context.Background(), srv.URL); !errors.Is(err, ErrBlocked) { + t.Fatalf("off-allowlist host: error = %v, want ErrBlocked", err) + } + f = testFetcher(t, Config{DenyHosts: []string{"127.0.0.1"}}) + if _, err := f.Get(context.Background(), srv.URL); !errors.Is(err, ErrBlocked) { + t.Fatalf("denied host: error = %v, want ErrBlocked", err) + } + f = testFetcher(t, Config{AllowHosts: []string{"127.0.0.1"}}) + if _, err := f.Get(context.Background(), srv.URL); err != nil { + t.Fatalf("allowlisted host: %v", err) + } +} + +func TestHostMatchesSubdomains(t *testing.T) { + pats := []string{"example.com", "*.news.org"} + for _, h := range []string{"example.com", "news.example.com", "a.b.example.com", "news.org", "feeds.news.org"} { + if !HostMatches(h, pats) { + t.Errorf("HostMatches(%q) = false, want true", h) + } + } + for _, h := range []string{"notexample.com", "example.com.evil.net", "org"} { + if HostMatches(h, pats) { + t.Errorf("HostMatches(%q) = true, want false", h) + } + } +} + +func TestSizeCap(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(strings.Repeat("x", 5000))) + })) + defer srv.Close() + + f := testFetcher(t, Config{MaxBytes: 100}) + if _, err := f.Get(context.Background(), srv.URL); !errors.Is(err, ErrTooLarge) { + t.Fatalf("error = %v, want ErrTooLarge", err) + } + f = testFetcher(t, Config{MaxBytes: 6000}) + resp, err := f.Get(context.Background(), srv.URL) + if err != nil { + t.Fatalf("under the cap: %v", err) + } + if len(resp.Body) != 5000 { + t.Fatalf("body = %d bytes, want 5000", len(resp.Body)) + } +} + +func TestRedirectCap(t *testing.T) { + var srv *httptest.Server + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, srv.URL+"/again", http.StatusFound) + })) + defer srv.Close() + + f := testFetcher(t, Config{MaxRedirects: 2}) + if _, err := f.Get(context.Background(), srv.URL); !errors.Is(err, ErrRedirects) { + t.Fatalf("error = %v, want ErrRedirects", err) + } +} + +// A redirect off the allowlist is the interesting redirect: the first hop is +// permitted, the second must not be. +func TestRedirectRecheckedAgainstDenylist(t *testing.T) { + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("secret")) + })) + defer target.Close() + hop := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, target.URL, http.StatusFound) + })) + defer hop.Close() + + // Reach the hop under the name "localhost" and allow only that name; the + // redirect lands on the same box under its literal address, which the + // allowlist does not cover. Without the CheckRedirect hook this fetch + // succeeds and returns "secret". + f := testFetcher(t, Config{AllowHosts: []string{"localhost"}}) + viaName := strings.Replace(hop.URL, "127.0.0.1", "localhost", 1) + if _, err := f.Get(context.Background(), viaName); !errors.Is(err, ErrBlocked) { + t.Fatalf("error = %v, want ErrBlocked", err) + } +} + +func TestPerHostRateLimit(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("ok")) + })) + defer srv.Close() + + f := testFetcher(t, Config{HostInterval: 60 * time.Millisecond}) + start := time.Now() + for i := 0; i < 3; i++ { + if _, err := f.Get(context.Background(), srv.URL); err != nil { + t.Fatalf("request %d: %v", i, err) + } + } + if elapsed := time.Since(start); elapsed < 120*time.Millisecond { + t.Fatalf("three requests took %s, want at least 120ms of spacing", elapsed) + } +} + +func TestRateLimitHonoursContext(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + defer srv.Close() + + f := testFetcher(t, Config{HostInterval: 10 * time.Second}) + if _, err := f.Get(context.Background(), srv.URL); err != nil { + t.Fatalf("first request: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + if _, err := f.Get(ctx, srv.URL); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("error = %v, want DeadlineExceeded", err) + } +} + +func TestNon2xxIsAnError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "nope", http.StatusInternalServerError) + })) + defer srv.Close() + f := testFetcher(t, Config{}) + if _, err := f.Get(context.Background(), srv.URL); !errors.Is(err, ErrStatus) { + t.Fatalf("error = %v, want ErrStatus", err) + } +} + +func TestUserAgentIsSent(t *testing.T) { + got := make(chan string, 1) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got <- r.Header.Get("User-Agent") + })) + defer srv.Close() + f := testFetcher(t, Config{UserAgent: "Maven/test"}) + if _, err := f.Get(context.Background(), srv.URL); err != nil { + t.Fatal(err) + } + if ua := <-got; ua != "Maven/test" { + t.Fatalf("user-agent = %q", ua) + } +} + +func TestPostSendsBodyAndHeaders(t *testing.T) { + type seen struct { + method, ctype, accept, ua, custom string + body []byte + } + ch := make(chan seen, 1) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + ch <- seen{r.Method, r.Header.Get("Content-Type"), r.Header.Get("Accept"), + r.Header.Get("User-Agent"), r.Header.Get("X-Thing"), b} + w.Header().Set("Mcp-Session-Id", "sess-9") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"ok":true}`)) + })) + defer srv.Close() + f := testFetcher(t, Config{UserAgent: "Maven/test"}) + resp, err := f.Post(context.Background(), srv.URL, "application/json", + []byte(`{"jsonrpc":"2.0"}`), map[string]string{"Accept": "text/event-stream", "X-Thing": "1"}) + if err != nil { + t.Fatal(err) + } + if string(resp.Body) != `{"ok":true}` { + t.Fatalf("body = %q", resp.Body) + } + if resp.Header["Mcp-Session-Id"] != "sess-9" { + t.Fatalf("response headers not surfaced: %+v", resp.Header) + } + s := <-ch + if s.method != http.MethodPost { + t.Fatalf("method = %s", s.method) + } + if string(s.body) != `{"jsonrpc":"2.0"}` { + t.Fatalf("request body = %q", s.body) + } + if s.ctype != "application/json" { + t.Fatalf("content-type = %q", s.ctype) + } + if s.accept != "text/event-stream" || s.custom != "1" { + t.Fatalf("caller headers dropped: %+v", s) + } + if s.ua != "Maven/test" { + t.Fatalf("user-agent = %q — a caller must not be able to override it", s.ua) + } +} + +// The whole point of routing MCP through webfetch: a POST is guarded exactly +// like a GET. A body does not buy a caller a way onto the LAN. +func TestPostRefusesPrivateAddress(t *testing.T) { + f := New(Config{}) // no AllowPrivate + _, err := f.Post(context.Background(), "http://127.0.0.1:9100/mcp", "application/json", []byte(`{}`), nil) + if !errors.Is(err, ErrPrivate) { + t.Fatalf("error = %v, want ErrPrivate", err) + } +} + +func TestPostRefusesNonHTTPScheme(t *testing.T) { + f := New(Config{}) + if _, err := f.Post(context.Background(), "file:///etc/passwd", "application/json", nil, nil); !errors.Is(err, ErrScheme) { + t.Fatalf("error = %v, want ErrScheme", err) + } +} + +func TestPostObeysDenylist(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + defer srv.Close() + f := testFetcher(t, Config{DenyHosts: []string{"127.0.0.1"}}) + if _, err := f.Post(context.Background(), srv.URL, "application/json", []byte(`{}`), nil); !errors.Is(err, ErrBlocked) { + t.Fatalf("error = %v, want ErrBlocked", err) + } +} diff --git a/internal/zenmoney/client.go b/internal/zenmoney/client.go new file mode 100644 index 0000000..dcb6624 --- /dev/null +++ b/internal/zenmoney/client.go @@ -0,0 +1,340 @@ +// Package zenmoney reads spending and income from ZenMoney's /v8/diff/ API +// (Vikunja #125). +// +// Trust boundary: ZenMoney, not Maven. They already hold his bank sessions — +// this package only reads back what they have, over a token that lives in the +// poller module and is never handed to core. Nothing here writes to ZenMoney; +// diff is called read-only (an empty change set in, a change set out). +// +// Two rules the code exists to enforce: +// +// - NEVER invent a number. Every figure in a Summary is a sum of amounts the +// API returned. A request that fails, or returns nothing, produces no +// summary and therefore no fact — silence, not a zero. A confidently wrong +// "ты потратил 0" is worse than no answer. +// - His money is never search input. This package holds no notes, no +// utterances and no persona text, and it has no path to the external search +// capability. The only thing that leaves the box here is the diff request +// itself, to the service that already has the data. +package zenmoney + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "sort" + "strings" + "sync" + "time" +) + +// DefaultBaseURL — ZenMoney's API root. Overridable so the tests can point at +// an httptest server replaying a recorded response. +const DefaultBaseURL = "https://api.zenmoney.ru" + +// Client is a ZenMoney diff reader. The token is held here, in the poller's +// address space; core never receives it and never learns it exists. +type Client struct { + BaseURL string + Token string + HTTP *http.Client + + // instruments — id → short title, fetched once from a cursor-zero diff and + // kept for the process lifetime. Currencies do not change; the reason this + // cache exists is that a windowed diff only returns objects changed since + // the cursor, so a day window almost never carries the instrument rows the + // transactions in it point at. + mu sync.Mutex + instruments map[int64]string +} + +// New returns a client with a bounded HTTP timeout. An empty token is a +// programming error the caller must catch — the capability is off unless +// configured, so a client is only ever built when a token was supplied. +func New(token, baseURL string, timeout time.Duration) (*Client, error) { + if strings.TrimSpace(token) == "" { + return nil, fmt.Errorf("zenmoney: empty token") + } + if baseURL == "" { + baseURL = DefaultBaseURL + } + if timeout <= 0 { + timeout = 20 * time.Second + } + return &Client{ + BaseURL: strings.TrimRight(baseURL, "/"), + Token: token, + HTTP: &http.Client{Timeout: timeout}, + }, nil +} + +// diffRequest — the smallest body /v8/diff/ accepts. serverTimestamp is the +// incremental cursor: the server returns objects changed at or after it. +type diffRequest struct { + CurrentClientTimestamp int64 `json:"currentClientTimestamp"` + ServerTimestamp int64 `json:"serverTimestamp"` +} + +// diffResponse — only the fields spending needs. ZenMoney returns a dozen more +// object types (tags, merchants, budgets, reminders); decoding them would mean +// holding more of his financial life in memory than the question needs. +type diffResponse struct { + ServerTimestamp int64 `json:"serverTimestamp"` + Instrument []instrument `json:"instrument"` + Transaction []transaction `json:"transaction"` +} + +type instrument struct { + ID int64 `json:"id"` + ShortTitle string `json:"shortTitle"` +} + +type transaction struct { + ID string `json:"id"` + Date string `json:"date"` // "2026-07-15" + Deleted bool `json:"deleted"` + Income float64 `json:"income"` + Outcome float64 `json:"outcome"` + IncomeInstrument int64 `json:"incomeInstrument"` + OutcomeInstrmnt int64 `json:"outcomeInstrument"` + IncomeAccount string `json:"incomeAccount"` + OutcomeAccount string `json:"outcomeAccount"` +} + +// Money — an amount in one currency. Kept as the currency's own short title +// ("RUB", "EUR") rather than converted: ZenMoney's rates are a snapshot, and +// converting would turn a figure he can check against his bank into one he +// cannot. +type Money struct { + Currency string `json:"currency"` + Amount float64 `json:"amount"` +} + +// Summary — what was spent and earned over a window, per currency, plus how +// many transactions it was computed from. Count is the honesty check: a +// summary built from zero transactions is not "you spent nothing", it is "there +// was nothing to read", and callers treat it as no answer. +type Summary struct { + From, To time.Time + Spent []Money `json:"spent"` + Earned []Money `json:"earned"` + Count int `json:"count"` + // ServerTimestamp — the cursor the API returned, for the caller to log or + // carry. Not used as an incremental cursor for summaries; see Since. + ServerTimestamp int64 `json:"-"` +} + +// Since returns the summary of transactions dated in [from, to). +// +// The diff cursor is set to `from` so the server only sends objects changed +// since then, which for a "this month" window is everything filed this month. +// The caveat, deliberately accepted: a transaction he EDITED this month but +// dated last month arrives too, and is then excluded by date — so editing old +// records cannot inflate this month's total. +// +// The reverse case is real and undercounts: a transaction dated inside the +// window but last CHANGED before `from` never arrives. A planned transaction +// entered last month and dated this month is exactly that, and it goes missing +// from the total. Widening the cursor would mean pulling his whole history +// every poll, so the total is "what was filed or touched in the window", and +// that is the honest reading of it. +func (c *Client) Since(ctx context.Context, from, to time.Time) (Summary, error) { + resp, err := c.diff(ctx, from.Unix()) + if err != nil { + return Summary{}, err + } + names := c.currencyNames(ctx, resp) + return summarize(resp, names, from, to), nil +} + +// currencyNames resolves instrument ids to short titles. The window's own diff +// first, then — only if a transaction in it points at an instrument the window +// did not carry — one cursor-zero diff, cached for the process lifetime. +// +// A failed instrument fetch is not an error: the summary is still every number +// the API returned, and an amount whose currency cannot be named is dropped +// from the spoken string rather than read out as "1749.5 ?". +func (c *Client) currencyNames(ctx context.Context, resp diffResponse) map[int64]string { + names := map[int64]string{} + for _, in := range resp.Instrument { + names[in.ID] = in.ShortTitle + } + missing := false + for _, t := range resp.Transaction { + if t.Deleted { + continue + } + for _, id := range []int64{t.OutcomeInstrmnt, t.IncomeInstrument} { + if id != 0 && names[id] == "" { + missing = true + } + } + } + if !missing { + return names + } + for id, title := range c.allInstruments(ctx) { + if names[id] == "" { + names[id] = title + } + } + return names +} + +// allInstruments fetches every instrument once, from a cursor-zero diff, and +// caches it. The response also carries transactions, which are decoded and +// dropped: this is the one call in the package that reads more of his financial +// life than the question needs, and it happens at most once per process. +func (c *Client) allInstruments(ctx context.Context) map[int64]string { + c.mu.Lock() + defer c.mu.Unlock() + if c.instruments != nil { + return c.instruments + } + resp, err := c.diff(ctx, 0) + if err != nil { + // Not cached: a network failure is not a fact about his currencies. + return nil + } + c.instruments = make(map[int64]string, len(resp.Instrument)) + for _, in := range resp.Instrument { + c.instruments[in.ID] = in.ShortTitle + } + return c.instruments +} + +func (c *Client) diff(ctx context.Context, serverTimestamp int64) (diffResponse, error) { + body, err := json.Marshal(diffRequest{ + CurrentClientTimestamp: time.Now().Unix(), + ServerTimestamp: serverTimestamp, + }) + if err != nil { + return diffResponse{}, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.BaseURL+"/v8/diff/", bytes.NewReader(body)) + if err != nil { + return diffResponse{}, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+c.Token) + hc := c.HTTP + if hc == nil { + hc = &http.Client{Timeout: 20 * time.Second} + } + res, err := hc.Do(req) + if err != nil { + return diffResponse{}, err + } + defer res.Body.Close() + raw, err := io.ReadAll(io.LimitReader(res.Body, 32<<20)) + if err != nil { + return diffResponse{}, err + } + if res.StatusCode != http.StatusOK { + // The status only. The body of a failed diff can echo account data, and + // this string reaches the log. + return diffResponse{}, fmt.Errorf("zenmoney diff: %s", res.Status) + } + var out diffResponse + if err := json.Unmarshal(raw, &out); err != nil { + return diffResponse{}, fmt.Errorf("zenmoney diff: decode: %w", err) + } + return out, nil +} + +// summarize sums the transactions dated inside the window. +// +// Excluded, in order: deleted rows (ZenMoney tombstones rather than removes), +// transfers and currency exchanges (income and outcome both non-zero — moving +// his own money between his own accounts is not spending), and anything dated +// outside the window. +func summarize(resp diffResponse, cur map[int64]string, from, to time.Time) Summary { + spent := map[string]float64{} + earned := map[string]float64{} + count := 0 + for _, t := range resp.Transaction { + if t.Deleted { + continue + } + d, err := time.ParseInLocation("2006-01-02", t.Date, from.Location()) + if err != nil { + continue // an undated row is not a number we can place + } + if d.Before(from) || !d.Before(to) { + continue + } + if t.Income > 0 && t.Outcome > 0 { + continue // transfer / exchange + } + switch { + case t.Outcome > 0: + spent[currency(cur, t.OutcomeInstrmnt)] += t.Outcome + count++ + case t.Income > 0: + earned[currency(cur, t.IncomeInstrument)] += t.Income + count++ + } + } + return Summary{ + From: from, To: to, + Spent: sortMoney(spent), Earned: sortMoney(earned), + Count: count, ServerTimestamp: resp.ServerTimestamp, + } +} + +// UnknownCurrency — the label for an instrument id nothing could name. It +// survives into the Summary so a caller can see that a bucket exists; the +// renderer drops it rather than reading "?" aloud as a currency. +const UnknownCurrency = "?" + +// currency names the instrument, or says it does not know. An unknown id keeps +// its bucket in the Summary rather than being folded into a named one: a sum is +// only checkable against his bank if every amount in it is in one currency. +func currency(names map[int64]string, id int64) string { + if s := names[id]; s != "" { + return s + } + return UnknownCurrency +} + +// sortMoney gives the amounts a stable order (largest first) so the rendered +// string and the written fact do not churn between polls. +func sortMoney(m map[string]float64) []Money { + out := make([]Money, 0, len(m)) + for c, a := range m { + out = append(out, Money{Currency: c, Amount: a}) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Amount != out[j].Amount { + return out[i].Amount > out[j].Amount + } + return out[i].Currency < out[j].Currency + }) + return out +} + +// Empty reports whether the summary rests on no transactions at all. Callers +// must treat an empty summary as "nothing to say", never as a zero: the +// difference between "he spent nothing" and "the read returned nothing" is the +// difference between an answer and an invented one. +func (s Summary) Empty() bool { return s.Count == 0 } + +// MonthWindow — the first instant of now's month, and now's own day-end +// exclusive bound, in now's location. The window a "сколько я потратил в этом +// месяце?" question means. +func MonthWindow(now time.Time) (from, to time.Time) { + loc := now.Location() + from = time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, loc) + to = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, loc).AddDate(0, 0, 1) + return from, to +} + +// DayWindow — today, in now's location. +func DayWindow(now time.Time) (from, to time.Time) { + loc := now.Location() + from = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, loc) + return from, from.AddDate(0, 0, 1) +} diff --git a/internal/zenmoney/client_test.go b/internal/zenmoney/client_test.go new file mode 100644 index 0000000..a30e142 --- /dev/null +++ b/internal/zenmoney/client_test.go @@ -0,0 +1,274 @@ +package zenmoney + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "time" +) + +// fixtureServer replays testdata/diff.json and records the request, so the +// tests can assert the wire contract (Bearer token, POST, /v8/diff/) without a +// ZenMoney account. +func fixtureServer(t *testing.T, got *diffRequest, auth *string) *httptest.Server { + t.Helper() + body, err := os.ReadFile("testdata/diff.json") + if err != nil { + t.Fatal(err) + } + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + t.Errorf("method = %s, want POST", r.Method) + } + if r.URL.Path != "/v8/diff/" { + t.Errorf("path = %s, want /v8/diff/", r.URL.Path) + } + if auth != nil { + *auth = r.Header.Get("Authorization") + } + if got != nil { + if err := json.NewDecoder(r.Body).Decode(got); err != nil { + t.Errorf("decode request: %v", err) + } + } + w.Header().Set("Content-Type", "application/json") + w.Write(body) + })) +} + +func aug(day int) time.Time { return time.Date(2026, 8, day, 0, 0, 0, 0, time.UTC) } + +func TestSinceSumsSpendingPerCurrency(t *testing.T) { + var req diffRequest + var auth string + srv := fixtureServer(t, &req, &auth) + defer srv.Close() + + c, err := New("tok", srv.URL, time.Second) + if err != nil { + t.Fatal(err) + } + s, err := c.Since(context.Background(), aug(1), aug(6)) + if err != nil { + t.Fatal(err) + } + if auth != "Bearer tok" { + t.Errorf("Authorization = %q", auth) + } + if req.ServerTimestamp != aug(1).Unix() { + t.Errorf("serverTimestamp = %d, want the window start", req.ServerTimestamp) + } + // 1500 + 249.5 RUB spent, 12 EUR spent, 3000 RUB in. The transfer (t4), the + // deleted row (t6) and July's salary (t3) are all excluded. + want := map[string]float64{"RUB": 1749.5, "EUR": 12} + if len(s.Spent) != 2 { + t.Fatalf("spent = %+v, want two currencies", s.Spent) + } + for _, m := range s.Spent { + if want[m.Currency] != m.Amount { + t.Errorf("spent %s = %v, want %v", m.Currency, m.Amount, want[m.Currency]) + } + } + if len(s.Earned) != 1 || s.Earned[0].Amount != 3000 || s.Earned[0].Currency != "RUB" { + t.Errorf("earned = %+v, want 3000 RUB (July's salary is outside the window)", s.Earned) + } + if s.Count != 4 { + t.Errorf("count = %d, want 4 counted transactions", s.Count) + } + // Largest first, so the fact value does not churn between polls. + if s.Spent[0].Currency != "RUB" { + t.Errorf("spent order = %+v, want the largest amount first", s.Spent) + } +} + +// A window with nothing in it is NOT a zero. No transactions means no answer, +// and the caller must be able to tell the difference. +func TestSinceEmptyWindowIsNotAZero(t *testing.T) { + srv := fixtureServer(t, nil, nil) + defer srv.Close() + c, _ := New("tok", srv.URL, time.Second) + s, err := c.Since(context.Background(), time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC), time.Date(2026, 9, 30, 0, 0, 0, 0, time.UTC)) + if err != nil { + t.Fatal(err) + } + if !s.Empty() { + t.Fatalf("summary = %+v, want empty", s) + } + if _, ok := s.Value(time.Now()); ok { + t.Error("an empty summary must not produce a fact value") + } +} + +func TestSinceReportsHTTPFailureWithoutTheBody(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, `{"account":"acc-card","secret":"leaky"}`, http.StatusUnauthorized) + })) + defer srv.Close() + c, _ := New("tok", srv.URL, time.Second) + _, err := c.Since(context.Background(), aug(1), aug(6)) + if err == nil { + t.Fatal("want an error on 401") + } + if strings.Contains(err.Error(), "acc-card") || strings.Contains(err.Error(), "leaky") { + t.Errorf("error %q echoes the response body — it reaches the log", err) + } +} + +func TestNewRequiresAToken(t *testing.T) { + if _, err := New(" ", "", 0); err == nil { + t.Error("want an error for an empty token — the capability is off unless configured") + } +} + +func TestMonthAndDayWindows(t *testing.T) { + now := time.Date(2026, 8, 15, 21, 30, 0, 0, time.UTC) + from, to := MonthWindow(now) + if from != time.Date(2026, 8, 1, 0, 0, 0, 0, time.UTC) || to != time.Date(2026, 8, 16, 0, 0, 0, 0, time.UTC) { + t.Errorf("month window = %v..%v", from, to) + } + from, to = DayWindow(now) + if from != time.Date(2026, 8, 15, 0, 0, 0, 0, time.UTC) || to != time.Date(2026, 8, 16, 0, 0, 0, 0, time.UTC) { + t.Errorf("day window = %v..%v", from, to) + } +} + +func TestFactValueRoundTripAndFormat(t *testing.T) { + s := Summary{Spent: []Money{{"RUB", 1749.5}}, Earned: []Money{{"RUB", 3000}}, Count: 3} + raw, ok := s.Value(time.Date(2026, 8, 1, 22, 0, 0, 0, time.UTC)) + if !ok { + t.Fatal("want a fact value") + } + v, err := ParseFactValue(raw) + if err != nil { + t.Fatal(err) + } + got := v.FormatRU("в этом месяце") + if !strings.Contains(got, "1749.5 RUB") || !strings.Contains(got, "3000 RUB") { + t.Errorf("reply = %q, want the exact figures", got) + } + // Persona: informal, feminine, no commentary on his spending. + for _, bad := range []string{"вы", "ваш", "милый", "дорогой", "рад ", "слишком", "много"} { + if strings.Contains(got, bad) { + t.Errorf("reply %q contains %q", got, bad) + } + } + if strings.Contains(got, "он ") { + t.Errorf("reply %q talks about him in the third person", got) + } +} + +// An empty fact value renders to nothing, so a caller cannot accidentally +// speak a zero. +func TestFormatRUEmptyRendersNothing(t *testing.T) { + if got := (FactValue{}).FormatRU("сегодня"); got != "" { + t.Errorf("reply = %q, want empty", got) + } +} + +func TestFormatAmountKeepsTheTruth(t *testing.T) { + for in, want := range map[float64]string{1500: "1500", 249.5: "249.5", 0.99: "0.99", 1749.55: "1749.55"} { + if got := formatAmount(in); got != want { + t.Errorf("formatAmount(%v) = %q, want %q", in, got, want) + } + } +} + +// The day window rolls over at midnight and the first spend of the new day may +// be hours away, so the last good money_today fact keeps a fresh ts while +// covering yesterday. Only the window stamp inside the value can tell. +func TestFactValueCoversDay(t *testing.T) { + from, _ := DayWindow(time.Date(2026, 8, 1, 22, 0, 0, 0, time.UTC)) + s := Summary{From: from, Spent: []Money{{"RUB", 1749.5}}, Count: 1} + raw, ok := s.Value(time.Date(2026, 8, 1, 22, 0, 0, 0, time.UTC)) + if !ok { + t.Fatal("want a fact value") + } + v, err := ParseFactValue(raw) + if err != nil { + t.Fatal(err) + } + if !v.CoversDay(time.Date(2026, 8, 1, 23, 59, 0, 0, time.UTC)) { + t.Error("the same day must be covered") + } + if v.CoversDay(time.Date(2026, 8, 2, 9, 0, 0, 0, time.UTC)) { + t.Error("yesterday's day total must not count as today's") + } + if (FactValue{}).CoversDay(time.Date(2026, 8, 2, 9, 0, 0, 0, time.UTC)) { + t.Error("a value with no window stamp must fail closed") + } + if v.AsOf.IsZero() { + t.Error("the value must carry when it was read, not only when it changed") + } +} + +// The instrument rows are only in the diff when they changed since the cursor, +// which for a day window they usually have not. An amount whose currency +// nothing could name is dropped from the spoken string rather than read out +// as "1749.5 ?". +func TestUnknownCurrencyIsNotSpoken(t *testing.T) { + v := FactValue{Spent: []Money{{UnknownCurrency, 1749.5}}, Count: 1} + if got := v.FormatRU("сегодня"); got != "" { + t.Errorf("reply = %q, want nothing said about an unlabelled amount", got) + } + v = FactValue{Spent: []Money{{"RUB", 100}, {UnknownCurrency, 1749.5}}, Count: 2} + got := v.FormatRU("сегодня") + if strings.Contains(got, UnknownCurrency) { + t.Errorf("reply = %q, want no %q currency", got, UnknownCurrency) + } + if !strings.Contains(got, "100 RUB") { + t.Errorf("reply = %q, want the amount that does have a currency", got) + } +} + +// A day diff cursored at midnight usually carries no instrument rows at all. +// The client fetches them once from a cursor-zero diff instead of labelling +// every amount "?". +func TestSinceResolvesCurrencyFromASeparateDiff(t *testing.T) { + body, err := os.ReadFile("testdata/diff.json") + if err != nil { + t.Fatal(err) + } + var full diffResponse + if err := json.Unmarshal(body, &full); err != nil { + t.Fatal(err) + } + windowed := diffResponse{ServerTimestamp: full.ServerTimestamp, Transaction: full.Transaction} + instrumentsOnly := diffResponse{ServerTimestamp: full.ServerTimestamp, Instrument: full.Instrument} + zeroCursorCalls := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req diffRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Errorf("decode request: %v", err) + } + out := windowed + if req.ServerTimestamp == 0 { + zeroCursorCalls++ + out = instrumentsOnly + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(out) + })) + defer srv.Close() + + c, _ := New("tok", srv.URL, time.Second) + s, err := c.Since(context.Background(), aug(1), aug(6)) + if err != nil { + t.Fatal(err) + } + for _, m := range s.Spent { + if m.Currency == UnknownCurrency { + t.Fatalf("spent = %+v, want every amount named", s.Spent) + } + } + // Cached for the process lifetime: a second window does not refetch. + if _, err := c.Since(context.Background(), aug(1), aug(6)); err != nil { + t.Fatal(err) + } + if zeroCursorCalls != 1 { + t.Errorf("cursor-zero diffs = %d, want exactly 1", zeroCursorCalls) + } +} diff --git a/internal/zenmoney/render.go b/internal/zenmoney/render.go new file mode 100644 index 0000000..5a991ef --- /dev/null +++ b/internal/zenmoney/render.go @@ -0,0 +1,165 @@ +package zenmoney + +import ( + "encoding/json" + "fmt" + "strings" + "time" +) + +// Fact keys the poller writes, all under source "poll:zenmoney". Two windows, +// because they are the two questions he actually asks; a per-category +// breakdown would mean storing what he bought, and the store is not a ledger. +const ( + KeySpentToday = "money_today" + KeySpentMonth = "money_month" +) + +// Source — the provenance every money fact carries. The loop's rules trust +// source, and nothing in Maven has a rule on these keys: they are read when he +// asks, never a reason to speak. Maven is not a nag, least of all about money. +const Source = "poll:zenmoney" + +// FactValue — the JSON stored in a money fact. +// +// From and AsOf are both here because the fact's own Ts can express neither. +// +// - From is the first instant of the window the figure covers. The day fact +// is only true for the day it was read on, and after midnight the poller +// has nothing new to write until the first spend of the new day, so the +// previous day's total sits there as the latest money_today looking +// perfectly fresh. Without From, "сколько я потратил сегодня?" at 09:00 +// answered with yesterday's spending. +// - AsOf is when the figure was last READ, not when it last changed. The +// poller used to skip a write when the value was byte-identical, so a quiet +// stretch left Ts pointing at the last time the number moved and the answer +// came back prefixed "данные от 30.07" while being current and correct. +type FactValue struct { + Spent []Money `json:"spent"` + Earned []Money `json:"earned"` + Count int `json:"count"` + From time.Time `json:"from,omitempty"` + AsOf time.Time `json:"as_of,omitempty"` +} + +// Value encodes the summary for the facts table, stamped with the instant it +// was read. Returns ok=false for an empty summary: no transactions read means +// no fact written, so that a failed or empty poll can never be recited back to +// him as a zero. +func (s Summary) Value(asOf time.Time) (string, bool) { + if s.Empty() { + return "", false + } + b, err := json.Marshal(FactValue{ + Spent: s.Spent, Earned: s.Earned, Count: s.Count, + From: s.From, AsOf: asOf, + }) + if err != nil { + return "", false + } + return string(b), true +} + +// CoversDay reports whether this value's window starts at now's midnight, in +// now's location. A day total whose window has rolled over is not a stale +// figure to be prefixed with a date, it is an answer to a different question, +// and it must not be spoken as today's. +// +// A value written before From existed has a zero From and fails the check, +// which is the safe direction: the next poll rewrites it. +func (v FactValue) CoversDay(now time.Time) bool { + if v.From.IsZero() { + return false + } + f := v.From.In(now.Location()) + return f.Year() == now.Year() && f.Month() == now.Month() && f.Day() == now.Day() +} + +// ParseFactValue decodes a stored money fact. +func ParseFactValue(raw string) (FactValue, error) { + var v FactValue + if err := json.Unmarshal([]byte(raw), &v); err != nil { + return FactValue{}, err + } + return v, nil +} + +// FormatRU renders a money fact the way Maven says it — feminine, informal, +// and only about numbers that came from ZenMoney. window is the Russian phrase +// for the period ("сегодня", "в этом месяце"). +// +// No commentary. She reports the figure and stops: an opinion about his +// spending is exactly the nagging Maven is not for. +func (v FactValue) FormatRU(window string) string { + return v.formatRU(window, false) +} + +// FormatIncomeRU is FormatRU with the income read first, for a question that +// asked about income ("сколько я заработал в этом месяце?"). Same figures, same +// refusal to comment; only the order of the two halves differs, so the number +// he asked for is the number she says first. +func (v FactValue) FormatIncomeRU(window string) string { + return v.formatRU(window, true) +} + +func (v FactValue) formatRU(window string, incomeFirst bool) string { + if v.Count == 0 { + return "" + } + spent, earned := "", "" + if len(v.Spent) > 0 { + if s := joinMoney(v.Spent); s != "" { + spent = "потратил " + s + } + } + if len(v.Earned) > 0 { + if s := joinMoney(v.Earned); s != "" { + earned = "получил " + s + } + } + order := []string{spent, earned} + if incomeFirst { + order = []string{earned, spent} + } + var parts []string + for _, p := range order { + if p != "" { + parts = append(parts, p) + } + } + if len(parts) == 0 { + return "" + } + return window + " ты " + strings.Join(parts, ", ") + "." +} + +// joinMoney renders the amounts, DROPPING any whose currency could not be +// named. "сегодня ты потратил 1749.5 ?." is not something to read aloud, and a +// figure with no currency on it is not a figure he can check against his bank. +// An amount silently missing is the lesser wrong: the alternative is speaking a +// number whose units Maven does not know. +func joinMoney(ms []Money) string { + parts := make([]string, 0, len(ms)) + for _, m := range ms { + if m.Currency == UnknownCurrency || m.Currency == "" { + continue + } + parts = append(parts, fmt.Sprintf("%s %s", formatAmount(m.Amount), m.Currency)) + } + return strings.Join(parts, " и ") +} + +// formatAmount — whole units when the amount is whole, two decimals otherwise. +// Never rounded to something prettier than the truth. +func formatAmount(a float64) string { + if a == float64(int64(a)) { + return fmt.Sprintf("%d", int64(a)) + } + return strings.TrimRight(strings.TrimRight(fmt.Sprintf("%.2f", a), "0"), ".") +} + +// StaleAfter — how old a money fact may be and still be worth reciting. The +// poller is off unless configured and can be down; answering with last week's +// total as if it were today's would be a lie by omission, so a stale fact is +// reported as stale. +const StaleAfter = 26 * time.Hour diff --git a/internal/zenmoney/testdata/diff.json b/internal/zenmoney/testdata/diff.json new file mode 100644 index 0000000..d6a9009 --- /dev/null +++ b/internal/zenmoney/testdata/diff.json @@ -0,0 +1,35 @@ +{ + "serverTimestamp": 1785312000, + "instrument": [ + {"id": 2, "title": "Российский рубль", "shortTitle": "RUB", "symbol": "₽", "rate": 1}, + {"id": 3, "title": "Евро", "shortTitle": "EUR", "symbol": "€", "rate": 100} + ], + "account": [ + {"id": "acc-card", "title": "карта", "instrument": 2}, + {"id": "acc-cash", "title": "наличные", "instrument": 2}, + {"id": "acc-eur", "title": "евро", "instrument": 3} + ], + "transaction": [ + {"id": "t1", "date": "2026-08-01", "changed": 1785300000, "income": 0, "outcome": 1500, + "incomeInstrument": 2, "outcomeInstrument": 2, "incomeAccount": "acc-card", "outcomeAccount": "acc-card", + "payee": "пятёрочка", "deleted": false}, + {"id": "t2", "date": "2026-08-01", "changed": 1785300001, "income": 0, "outcome": 249.5, + "incomeInstrument": 2, "outcomeInstrument": 2, "incomeAccount": "acc-card", "outcomeAccount": "acc-card", + "payee": "метро", "deleted": false}, + {"id": "t3", "date": "2026-07-20", "changed": 1785300002, "income": 120000, "outcome": 0, + "incomeInstrument": 2, "outcomeInstrument": 2, "incomeAccount": "acc-card", "outcomeAccount": "acc-card", + "payee": "зарплата", "deleted": false}, + {"id": "t4", "date": "2026-08-02", "changed": 1785300003, "income": 5000, "outcome": 5000, + "incomeInstrument": 2, "outcomeInstrument": 2, "incomeAccount": "acc-cash", "outcomeAccount": "acc-card", + "payee": "", "comment": "снял наличные", "deleted": false}, + {"id": "t5", "date": "2026-08-03", "changed": 1785300004, "income": 0, "outcome": 12, + "incomeInstrument": 3, "outcomeInstrument": 3, "incomeAccount": "acc-eur", "outcomeAccount": "acc-eur", + "payee": "hosting", "deleted": false}, + {"id": "t6", "date": "2026-08-04", "changed": 1785300005, "income": 0, "outcome": 999, + "incomeInstrument": 2, "outcomeInstrument": 2, "incomeAccount": "acc-card", "outcomeAccount": "acc-card", + "payee": "удалённая", "deleted": true}, + {"id": "t7", "date": "2026-08-05", "changed": 1785300006, "income": 3000, "outcome": 0, + "incomeInstrument": 2, "outcomeInstrument": 2, "incomeAccount": "acc-card", "outcomeAccount": "acc-card", + "payee": "возврат", "deleted": false} + ] +} diff --git a/mavwaked b/mavwaked deleted file mode 100755 index a74d8de..0000000 Binary files a/mavwaked and /dev/null differ diff --git a/models/stt b/models/stt new file mode 120000 index 0000000..b983fa3 --- /dev/null +++ b/models/stt @@ -0,0 +1 @@ +/home/kami/apps/Maven/models/stt \ No newline at end of file diff --git a/models/tts b/models/tts new file mode 120000 index 0000000..66782fb --- /dev/null +++ b/models/tts @@ -0,0 +1 @@ +/home/kami/apps/Maven/models/tts \ No newline at end of file diff --git a/scripts/gen-stt-fixtures.sh b/scripts/gen-stt-fixtures.sh new file mode 100755 index 0000000..5f430dc --- /dev/null +++ b/scripts/gen-stt-fixtures.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# gen-stt-fixtures.sh — regenerate the golden STT audio fixtures. +# +# The fixtures in cmd/mavsttd/testdata/*.wav are SYNTHESISED, not recorded. +# They come out of the same piper voices maven speaks with, so nothing of the +# owner's voice is committed and every fixture is reproducible from this +# script plus the voice model. They are also small: 16 kHz mono s16le, a +# couple of seconds each. +# +# Usage: +# scripts/gen-stt-fixtures.sh +# +# The spoken text is NOT written here. It is read out of +# cmd/mavsttd/testdata/golden_v1.json, which is the same file the test scores +# against. It used to live in both places, so editing this script and running +# make stt-fixtures left the manifest describing audio that no longer existed — +# and at a WER ceiling of 0.34 over a five-word reference, a one-word drift +# passed silently. Punctuation does not matter: normalizeTranscript strips it. +# +# Voices are picked up from, in order, $PIPER_VOICE_RU / $PIPER_VOICE_EN, then +# the repo's models/tts, then ~/esp-server/voices. The English voice is not +# vendored; if it is missing the English fixture is skipped and the existing +# one is left alone. +set -euo pipefail + +root="$(cd "$(dirname "$0")/.." && pwd)" +out="$root/cmd/mavsttd/testdata" +piper="${PIPER_BIN:-$root/deps/piper/piper}" +espeak="${PIPER_ESPEAK:-$root/deps/piper/espeak-ng-data}" + +pick_voice() { + for c in "$@"; do + [ -f "$c" ] && { echo "$c"; return 0; } + done + return 1 +} + +ru="$(pick_voice "${PIPER_VOICE_RU:-}" "$root/models/tts/ru_RU-irina-medium.onnx" "$HOME/esp-server/voices/ru_RU-irina-medium.onnx")" || { + echo "no russian piper voice found" >&2 + exit 1 +} +en="$(pick_voice "${PIPER_VOICE_EN:-}" "$root/models/tts/en_US-lessac-medium.onnx" "$HOME/esp-server/voices/en_US-lessac-medium.onnx")" || en="" + +manifest="$root/cmd/mavsttd/testdata/golden_v1.json" +command -v jq >/dev/null || { echo "jq is required to read $manifest" >&2; exit 1; } +[ -f "$manifest" ] || { echo "missing $manifest" >&2; exit 1; } + +# case_text — the reference transcript for one manifest case. +case_text() { + local name="$1" text + text="$(jq -r --arg n "$name" '.cases[] | select(.name==$n) | .text' "$manifest")" + [ -n "$text" ] && [ "$text" != "null" ] || { echo "no case named $name in $manifest" >&2; exit 1; } + printf '%s' "$text" +} + +# case_wav — the file name the manifest expects for one case. +case_wav() { + local name="$1" wav + wav="$(jq -r --arg n "$name" '.cases[] | select(.name==$n) | .wav' "$manifest")" + [ -n "$wav" ] && [ "$wav" != "null" ] || { echo "no case named $name in $manifest" >&2; exit 1; } + printf '%s' "$wav" +} + +# synth +# piper emits raw 22050 Hz s16le on stdout; ffmpeg resamples to the canonical +# 16 kHz mono and writes a plain 44-byte-header WAV (-fflags bitexact keeps +# ffmpeg's encoder LIST chunk out, so the bytes are stable across ffmpeg +# builds and internal/audio.PCMFromWAV reads them without scanning). +synth() { + local voice="$1" dest="$2" text="$3" + printf '%s' "$text" | LD_LIBRARY_PATH="$(dirname "$piper")" "$piper" \ + --model "$voice" --config "$voice.json" \ + --espeak_data "$espeak" --output_raw --quiet | + ffmpeg -hide_banner -loglevel error -y \ + -f s16le -ar 22050 -ac 1 -i - \ + -af "adelay=200,apad=pad_dur=0.2" \ + -ar 16000 -ac 1 -c:a pcm_s16le -fflags bitexact "$dest" + echo "wrote $dest ($(stat -c%s "$dest") bytes)" +} + +for name in ru_reminder ru_fact ru_query; do + synth "$ru" "$out/$(case_wav "$name")" "$(case_text "$name")" +done + +if [ -n "$en" ]; then + # Keep the English line in the manifest free of words piper spells out + # letter by letter — "nginx" comes out of lessac as "engine X", which is a + # TTS artefact and would make the fixture assert on the wrong thing. + synth "$en" "$out/$(case_wav en_act)" "$(case_text en_act)" +else + echo "no english piper voice found — skipping en_act.wav" >&2 +fi + +echo "fixtures regenerated; expected transcripts live in $out/golden_v1.json" diff --git a/vendor/github.com/kami/hexis/pkg/client/capability.go b/vendor/github.com/kami/hexis/pkg/client/capability.go new file mode 100644 index 0000000..99ff964 --- /dev/null +++ b/vendor/github.com/kami/hexis/pkg/client/capability.go @@ -0,0 +1,62 @@ +package client + +import "time" + +// Capability is THE wire shape for a Hexis capability. +// +// There is exactly one definition of it, here, and every producer in this +// repository serializes through it: the HTTP handler (GET/POST +// /api/v1/capabilities, GET /api/v1/capabilities/{id}), the MCP adapter +// (hexis.list_capabilities), and this client's decode path. It lives in +// pkg/client rather than internal/ so that external consumers get the shape +// without vendoring internal packages; internal/wire holds the +// domain.Capability -> Capability conversion. +// +// Compatibility note — `id` and `capability_id` are BOTH emitted, deliberately. +// They always carry the same value. Maven's vendored consumer decodes `id` +// (cmd/mavend/voice.go matches on Capability.ID); the ECOSYSTEM-SPEC.md §4.1 +// schema and the rest of the Hexis API name the column `capability_id`. Hexis +// is mid-rollout of bearer auth on /api/v1/, which is already one breaking +// change for that consumer; dropping either alias here would stack a second, +// silent one on top. Both stay until every consumer is confirmed to read +// `capability_id`, at which point `id` can be removed in a deliberate, +// announced change. Do not "clean this up" incidentally. +type Capability struct { + // CapabilityID is the canonical field (ECOSYSTEM-SPEC.md §4.1). + CapabilityID string `json:"capability_id"` + // ID is a deprecated alias for CapabilityID, kept for wire compatibility. + // Always identical to CapabilityID. Prefer CapabilityID in new code. + ID string `json:"id"` + + Name string `json:"name"` + Description string `json:"description,omitempty"` + TargetTypes []string `json:"target_types"` + TargetEntityID string `json:"target_entity_id,omitempty"` + Provider string `json:"provider"` + Operation string `json:"operation"` + Risk string `json:"risk,omitempty"` + ReadOnly bool `json:"read_only"` + ExpectedSideEffects string `json:"expected_side_effects,omitempty"` + + // RequiresConfirmation and Enabled are server-derived from the risk tier + // and are never settable by a caller. listCapabilities used to omit both, + // which left clients unable to tell a callable capability from one that + // would be rejected with 403; the unified shape always carries them. + RequiresConfirmation bool `json:"requires_confirmation"` + Enabled bool `json:"enabled"` + TimeoutSeconds int `json:"timeout_seconds,omitempty"` + + Attributes map[string]any `json:"attributes,omitempty"` + CreatedAt time.Time `json:"created_at,omitempty"` + UpdatedAt time.Time `json:"updated_at,omitempty"` + Version int64 `json:"version,omitempty"` +} + +// EffectiveID returns the capability ID, tolerating a peer that sends only one +// of the two aliases. +func (c Capability) EffectiveID() string { + if c.CapabilityID != "" { + return c.CapabilityID + } + return c.ID +} diff --git a/vendor/github.com/kami/hexis/pkg/client/client.go b/vendor/github.com/kami/hexis/pkg/client/client.go index ec06047..41dda1c 100644 --- a/vendor/github.com/kami/hexis/pkg/client/client.go +++ b/vendor/github.com/kami/hexis/pkg/client/client.go @@ -7,6 +7,8 @@ import ( "fmt" "io" "net/http" + "net/url" + "strconv" "time" ) @@ -42,6 +44,7 @@ func causationIDFrom(ctx context.Context) string { type Client struct { baseURL string httpClient *http.Client + token string } func New(baseURL string) *Client { @@ -51,6 +54,12 @@ func New(baseURL string) *Client { } } +// WithToken sets the shared bearer token sent on every /api/v1/ request. +func (c *Client) WithToken(token string) *Client { + c.token = token + return c +} + func (c *Client) do(ctx context.Context, method, path string, body, result any) error { var reqBody io.Reader if body != nil { @@ -67,6 +76,9 @@ func (c *Client) do(ctx context.Context, method, path string, body, result any) } req.Header.Set("Content-Type", "application/json") req.Header.Set("X-Hexis-Version", APIVersion) + if c.token != "" { + req.Header.Set("Authorization", "Bearer "+c.token) + } if id := correlationIDFrom(ctx); id != "" { req.Header.Set("X-Correlation-ID", id) } @@ -97,8 +109,37 @@ func (c *Client) do(ctx context.Context, method, path string, body, result any) return nil } -type Capability struct { - ID string `json:"id"` +// Capability is defined in capability.go — the single wire shape shared by +// the HTTP handler, the MCP adapter and this client. + +type Execution struct { + // Seq is the pagination cursor for Executions; see the `since` parameter. + // Zero on single-execution reads. + Seq int64 `json:"seq,omitempty"` + ID string `json:"id"` + CapabilityID string `json:"capability_id"` + TargetEntityID string `json:"target_entity_id"` + Status string `json:"status"` + Result map[string]any `json:"result,omitempty"` + Error string `json:"error,omitempty"` + RequestedBy map[string]string `json:"requested_by,omitempty"` + CorrelationID string `json:"correlation_id,omitempty"` + CausationID string `json:"causation_id,omitempty"` + IdempotencyKey string `json:"idempotency_key,omitempty"` +} + +type ExecuteRequest struct { + CapabilityID string `json:"capability_id"` + TargetEntityID string `json:"target_entity_id"` + Arguments map[string]any `json:"arguments,omitempty"` + RequestedBy map[string]string `json:"requested_by,omitempty"` + Origin map[string]string `json:"origin,omitempty"` + IdempotencyKey string `json:"idempotency_key,omitempty"` + CorrelationID string `json:"correlation_id,omitempty"` + CausationID string `json:"causation_id,omitempty"` +} + +type CreateCapabilityRequest struct { Name string `json:"name"` Description string `json:"description,omitempty"` TargetTypes []string `json:"target_types"` @@ -110,47 +151,11 @@ type Capability struct { ExpectedSideEffects string `json:"expected_side_effects,omitempty"` } -type Execution struct { - ID string `json:"id"` - CapabilityID string `json:"capability_id"` - TargetEntityID string `json:"target_entity_id"` - Status string `json:"status"` - Result map[string]any `json:"result,omitempty"` - Error string `json:"error,omitempty"` - RequestedBy map[string]string `json:"requested_by,omitempty"` - CorrelationID string `json:"correlation_id,omitempty"` - CausationID string `json:"causation_id,omitempty"` - IdempotencyKey string `json:"idempotency_key,omitempty"` -} - -type ExecuteRequest struct { - CapabilityID string `json:"capability_id"` - TargetEntityID string `json:"target_entity_id"` - Arguments map[string]any `json:"arguments,omitempty"` - RequestedBy map[string]string `json:"requested_by,omitempty"` - Origin map[string]string `json:"origin,omitempty"` - IdempotencyKey string `json:"idempotency_key,omitempty"` - CorrelationID string `json:"correlation_id,omitempty"` - CausationID string `json:"causation_id,omitempty"` -} - -type CreateCapabilityRequest struct { - Name string `json:"name"` - Description string `json:"description,omitempty"` - TargetTypes []string `json:"target_types"` - TargetEntityID string `json:"target_entity_id,omitempty"` - Provider string `json:"provider"` - Operation string `json:"operation"` - Risk string `json:"risk,omitempty"` - ReadOnly bool `json:"read_only"` - ExpectedSideEffects string `json:"expected_side_effects,omitempty"` -} - func (c *Client) Capabilities(ctx context.Context, entityID string) ([]Capability, error) { var result []Capability path := "/api/v1/capabilities" if entityID != "" { - path += "?entity_id=" + entityID + path += "?entity_id=" + url.QueryEscape(entityID) } if err := c.do(ctx, http.MethodGet, path, nil, &result); err != nil { return nil, err @@ -190,6 +195,33 @@ func (c *Client) GetExecution(ctx context.Context, id string) (*Execution, error return &result, nil } +// Executions returns execution history, newest last, ordered by ascending +// `seq` (ECOSYSTEM-SPEC.md §4.5). +// +// entityID, when non-empty, filters to executions against that target entity. +// since is an exclusive cursor: pass 0 for the first page, then the Seq of the +// last element returned. The server caps a page at 100 rows, so a full page +// means "call again with the new cursor". +func (c *Client) Executions(ctx context.Context, entityID string, since int64) ([]Execution, error) { + q := url.Values{} + if entityID != "" { + q.Set("entity_id", entityID) + } + if since > 0 { + q.Set("since", strconv.FormatInt(since, 10)) + } + path := "/api/v1/executions" + if len(q) > 0 { + path += "?" + q.Encode() + } + + var result []Execution + if err := c.do(ctx, http.MethodGet, path, nil, &result); err != nil { + return nil, err + } + return result, nil +} + func (c *Client) Health(ctx context.Context) error { return c.do(ctx, http.MethodGet, "/health", nil, nil) }