ee7bec11e3
The extraction seam landed on the previous branch but nothing fed it. This adds the daemon that does: every interval it opens one mailbox read-only (EXAMINE + BODY.PEEK, so reading leaves no \Seen behind), fetches the UIDs it has not handed over yet, and posts each message to core over ingest_mail. Core runs the model and writes task candidates; this daemon writes nothing and cannot create a reminder. It is a separate daemon because of the credential. mavpoll set the precedent with the zenmoney token (#125): the module talking to the third party holds the secret, reads it from a file so it never lands in argv, in docker-compose.yml or in shell history, and core never sees it. There is deliberately no -password flag, and a test asserts that. Off unless configured at both ends: without -password-file the daemon refuses to start, and if core has no email block the first ingest returns ErrUnknownMethod, which disables the reader instead of hammering a socket that will keep refusing. A seen-UID state file (0600, atomic write) keeps a restart from re-extracting the whole lookback window; correctness does not depend on it, since capture dedupes on normalised text. Logs are counts and UIDs — no subject, sender or body. Verified with an in-process IMAP server and a fake core: bulk mail is filtered before core is asked, seen UIDs are not re-fetched, a failed ingest is retried next poll, ErrUnknownMethod stops at the first message, and state survives a restart. The live half is untested by design — no IMAP credential exists on this box; setup is written up as QA steps. Vikunja #246
255 lines
7.7 KiB
Go
255 lines
7.7 KiB
Go
package main
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"fmt"
|
|
"net"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/kami/maven/internal/email"
|
|
"github.com/kami/maven/internal/ipc"
|
|
)
|
|
|
|
// ---- a scripted IMAP server, same shape internal/email's tests use ---------
|
|
|
|
type fakeIMAP struct {
|
|
msgs map[uint32]string
|
|
uids []uint32
|
|
cmds []string
|
|
}
|
|
|
|
func (f *fakeIMAP) serve(c net.Conn) {
|
|
defer c.Close()
|
|
fmt.Fprint(c, "* OK fake ready\r\n")
|
|
r := bufio.NewReader(c)
|
|
for {
|
|
line, err := r.ReadString('\n')
|
|
if err != nil {
|
|
return
|
|
}
|
|
parts := strings.SplitN(strings.TrimRight(line, "\r\n"), " ", 2)
|
|
if len(parts) != 2 {
|
|
return
|
|
}
|
|
tag, cmd := parts[0], parts[1]
|
|
f.cmds = append(f.cmds, cmd)
|
|
upper := strings.ToUpper(cmd)
|
|
switch {
|
|
case strings.HasPrefix(upper, "LOGIN"), strings.HasPrefix(upper, "EXAMINE"):
|
|
fmt.Fprintf(c, "%s OK\r\n", tag)
|
|
case strings.HasPrefix(upper, "UID SEARCH"):
|
|
var ids []string
|
|
for _, u := range f.uids {
|
|
ids = append(ids, strconv.FormatUint(uint64(u), 10))
|
|
}
|
|
fmt.Fprintf(c, "* SEARCH %s\r\n%s OK\r\n", strings.Join(ids, " "), tag)
|
|
case strings.HasPrefix(upper, "UID FETCH"):
|
|
uid, _ := strconv.ParseUint(strings.Fields(cmd)[2], 10, 32)
|
|
if raw, ok := f.msgs[uint32(uid)]; ok {
|
|
fmt.Fprintf(c, "* 1 FETCH (UID %d BODY[] {%d}\r\n%s)\r\n", uid, len(raw), raw)
|
|
}
|
|
fmt.Fprintf(c, "%s OK\r\n", tag)
|
|
case strings.HasPrefix(upper, "LOGOUT"):
|
|
fmt.Fprintf(c, "* BYE\r\n%s OK\r\n", tag)
|
|
return
|
|
default:
|
|
fmt.Fprintf(c, "%s BAD\r\n", tag)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (f *fakeIMAP) dial(_ string, timeout time.Duration) (*email.Conn, error) {
|
|
cli, srv := net.Pipe()
|
|
go f.serve(srv)
|
|
return email.NewConn(cli, timeout)
|
|
}
|
|
|
|
// ---- a fake core -----------------------------------------------------------
|
|
|
|
type fakeCore struct {
|
|
got []ipc.IngestMailReq
|
|
resp ipc.IngestMailResp
|
|
err error
|
|
}
|
|
|
|
func (c *fakeCore) IngestMail(_ context.Context, req ipc.IngestMailReq) (ipc.IngestMailResp, error) {
|
|
c.got = append(c.got, req)
|
|
if c.err != nil {
|
|
return ipc.IngestMailResp{}, c.err
|
|
}
|
|
return c.resp, nil
|
|
}
|
|
|
|
func mail(subject, body string, extraHeaders ...string) string {
|
|
h := "Subject: " + subject + "\r\nFrom: a@b.c\r\nContent-Type: text/plain; charset=utf-8\r\n"
|
|
for _, e := range extraHeaders {
|
|
h += e + "\r\n"
|
|
}
|
|
return h + "\r\n" + body + "\r\n"
|
|
}
|
|
|
|
func newTestReader(t *testing.T, f *fakeIMAP, core *fakeCore, statePath string) *reader {
|
|
t.Helper()
|
|
return &reader{
|
|
core: core, addr: "mail.example:993", user: "kami", mailbox: "INBOX",
|
|
lookback: 72 * time.Hour, max: 25, timeout: 5 * time.Second,
|
|
state: newSeenState(statePath),
|
|
dial: f.dial,
|
|
}
|
|
}
|
|
|
|
func TestPollHandsMessagesToCore(t *testing.T) {
|
|
f := &fakeIMAP{
|
|
uids: []uint32{1, 2},
|
|
msgs: map[uint32]string{
|
|
1: mail("Счёт", "Оплатить до 5 августа."),
|
|
2: mail("Скидки", "Sale!", "List-Unsubscribe: <mailto:u@x>"),
|
|
},
|
|
}
|
|
core := &fakeCore{resp: ipc.IngestMailResp{TaskIDs: []int64{1}, Created: 1}}
|
|
r := newTestReader(t, f, core, "")
|
|
r.pollOnce(context.Background(), "secret")
|
|
|
|
// The newsletter is filtered before core is asked: only the real mail crosses.
|
|
if len(core.got) != 1 {
|
|
t.Fatalf("core saw %d messages, want 1 (the bulk one must not cross): %+v", len(core.got), core.got)
|
|
}
|
|
got := core.got[0]
|
|
if got.UID != 1 || got.Mailbox != "INBOX" || got.Subject != "Счёт" {
|
|
t.Errorf("ingest req = %+v", got)
|
|
}
|
|
if !strings.Contains(got.Body, "Оплатить") {
|
|
t.Errorf("body = %q", got.Body)
|
|
}
|
|
}
|
|
|
|
// A second poll must not re-send what core already saw — extraction is a
|
|
// multi-second LLM call per message.
|
|
func TestPollSkipsSeenUIDs(t *testing.T) {
|
|
f := &fakeIMAP{uids: []uint32{5}, msgs: map[uint32]string{5: mail("Счёт", "текст")}}
|
|
core := &fakeCore{}
|
|
r := newTestReader(t, f, core, "")
|
|
r.pollOnce(context.Background(), "secret")
|
|
r.pollOnce(context.Background(), "secret")
|
|
if len(core.got) != 1 {
|
|
t.Errorf("core saw %d messages over two polls, want 1", len(core.got))
|
|
}
|
|
}
|
|
|
|
// An ingest that failed is NOT marked seen: the next poll retries it.
|
|
func TestPollRetriesFailedIngest(t *testing.T) {
|
|
f := &fakeIMAP{uids: []uint32{5}, msgs: map[uint32]string{5: mail("Счёт", "текст")}}
|
|
core := &fakeCore{err: fmt.Errorf("llama-server is warming up")}
|
|
r := newTestReader(t, f, core, "")
|
|
r.pollOnce(context.Background(), "secret")
|
|
core.err = nil
|
|
r.pollOnce(context.Background(), "secret")
|
|
if len(core.got) != 2 {
|
|
t.Errorf("core saw %d attempts, want 2 (a failed ingest is retried)", len(core.got))
|
|
}
|
|
}
|
|
|
|
// Core without an email block ⇒ stop, don't hammer the socket.
|
|
func TestPollStopsWhenCoreRefusesMail(t *testing.T) {
|
|
f := &fakeIMAP{uids: []uint32{1, 2}, msgs: map[uint32]string{1: mail("a", "b"), 2: mail("c", "d")}}
|
|
core := &fakeCore{err: fmt.Errorf("call: %w", ipc.ErrUnknownMethod)}
|
|
r := newTestReader(t, f, core, "")
|
|
r.pollOnce(context.Background(), "secret")
|
|
if !r.disabled {
|
|
t.Error("ErrUnknownMethod must disable the reader")
|
|
}
|
|
if len(core.got) != 1 {
|
|
t.Errorf("core saw %d messages, want 1 — stop at the first refusal", len(core.got))
|
|
}
|
|
}
|
|
|
|
func TestSeenStatePersists(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "state", "seen.json")
|
|
f := &fakeIMAP{uids: []uint32{9}, msgs: map[uint32]string{9: mail("Счёт", "текст")}}
|
|
core := &fakeCore{}
|
|
r := newTestReader(t, f, core, path)
|
|
r.pollOnce(context.Background(), "secret")
|
|
|
|
fi, err := os.Stat(path)
|
|
if err != nil {
|
|
t.Fatalf("state file: %v", err)
|
|
}
|
|
// A list of message ids from his mailbox is metadata about his mail.
|
|
if perm := fi.Mode().Perm(); perm != 0o600 {
|
|
t.Errorf("state file mode = %v, want 0600", perm)
|
|
}
|
|
|
|
// A fresh reader with the same state file must not re-read the message.
|
|
core2 := &fakeCore{}
|
|
r2 := newTestReader(t, f, core2, path)
|
|
if err := r2.state.load(); err != nil {
|
|
t.Fatalf("load: %v", err)
|
|
}
|
|
r2.pollOnce(context.Background(), "secret")
|
|
if len(core2.got) != 0 {
|
|
t.Errorf("after a restart core saw %d messages, want 0", len(core2.got))
|
|
}
|
|
}
|
|
|
|
func TestSeenStateHighWaterMark(t *testing.T) {
|
|
s := newSeenState("")
|
|
s.mark(1)
|
|
s.mark(3)
|
|
s.mark(2)
|
|
if s.high != 3 {
|
|
t.Errorf("high = %d, want 3 (contiguous run collapses)", s.high)
|
|
}
|
|
if len(s.set) != 0 {
|
|
t.Errorf("explicit set = %v, want empty", s.set)
|
|
}
|
|
if !s.seen(2) || s.seen(4) {
|
|
t.Errorf("seen(2)=%v seen(4)=%v", s.seen(2), s.seen(4))
|
|
}
|
|
}
|
|
|
|
func TestSeenStateCorruptFileIsNotFatal(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "seen.json")
|
|
if err := os.WriteFile(path, []byte("{not json"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
s := newSeenState(path)
|
|
if err := s.load(); err == nil {
|
|
t.Error("a corrupt state file should report an error the caller logs")
|
|
}
|
|
if s.seen(1) {
|
|
t.Error("a corrupt state file must leave an empty seen-set, not a poisoned one")
|
|
}
|
|
}
|
|
|
|
// Off unless configured, and the credential is never a flag value.
|
|
func TestRunRequiresConfig(t *testing.T) {
|
|
if err := run([]string{}); err == nil {
|
|
t.Error("no -socket must be an error")
|
|
}
|
|
if err := run([]string{"-socket", "/tmp/nope.sock"}); err == nil {
|
|
t.Error("no mailbox configuration must be an error, not a default mailbox")
|
|
}
|
|
// There is no -password flag at all: only -password-file.
|
|
if err := run([]string{"-socket", "/x", "-imap", "h", "-user", "u", "-password", "p"}); err == nil ||
|
|
!strings.Contains(err.Error(), "flag provided but not defined") {
|
|
t.Errorf("a -password flag must not exist; err = %v", err)
|
|
}
|
|
}
|
|
|
|
func TestRunRejectsEmptyPasswordFile(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "pass")
|
|
if err := os.WriteFile(path, []byte(" \n"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
err := run([]string{"-socket", "/x/y.sock", "-imap", "h", "-user", "u", "-password-file", path})
|
|
if err == nil || !strings.Contains(err.Error(), "empty") {
|
|
t.Errorf("an empty password file must be refused before dialling; err = %v", err)
|
|
}
|
|
}
|