From f42d1594ef88105f4c3eae77434b808d2714307c Mon Sep 17 00:00:00 2001 From: kami Date: Sat, 1 Aug 2026 03:06:55 +0400 Subject: [PATCH] Turn a mail into task candidates, and into nothing else (#246) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The extraction half. internal/email.Extractor asks the resident Qwen3-1.7B, under a GBNF grammar, what one message requires of him, and returns at most three short candidates with an optional date. Everything it can produce is a row in `tasks` with status "candidate", written through the intake seam #130 built for exactly this (Source "email:", Evidence = the subject line). No reminder, no fact, no note, no calendar event. That bound is the design: a reminder FIRES, so a 1.7B misreading "встреча была в четверг" as a future appointment would wake him up about it, whereas a wrong candidate is a line he dismisses in one click. A due date the model read out of the mail is stored on the candidate, where no scheduler reads it — the review page sorts by it. Relative wording ("до пятницы") is deliberately left in the text rather than resolved to a date the model would get wrong. The prompt is written against the two things a small model does here: it summarises when asked to extract, and it invents an obligation out of a polite closing line. Hence the demand for a verb phrase, and an explicit empty array — most mail contains no task, and a model with no way to say "nothing" says something. Wiring: core owns extraction because llama-server lives in core's process, so the reader hands messages over a new ipc.MethodIngestMail. It is a Server hook (like StepUp/UnlockFn), not a CoreAPI method — not a store operation, and no CoreAPI implementation should have to carry it. The hook stays nil without an `email` config block or without a llama-server phraser, so the method answers ErrUnknownMethod: off unless configured, twice over. There is no keyword fallback on purpose — "the subject became a task" is a mailbox rendered as a to-do list, not extraction. Privacy: junk is refused before the model is called, mail text is never search input, extraction errors carry byte counts rather than the reply, the stored evidence is a truncated subject, and the log line names the mailbox and the UID only. --- cmd/mavend/mail.go | 158 +++++++++++++++++++++++ cmd/mavend/mail_test.go | 182 ++++++++++++++++++++++++++ cmd/mavend/main.go | 8 ++ internal/auth/policy.go | 7 +- internal/config/config.go | 32 +++++ internal/email/extract.go | 226 +++++++++++++++++++++++++++++++++ internal/email/extract_test.go | 143 +++++++++++++++++++++ internal/ipc/api.go | 38 ++++++ internal/ipc/client.go | 11 ++ internal/ipc/ipc_test.go | 33 +++++ internal/ipc/server.go | 37 +++++- internal/ipc/wire.go | 1 + 12 files changed, 871 insertions(+), 5 deletions(-) create mode 100644 cmd/mavend/mail.go create mode 100644 cmd/mavend/mail_test.go create mode 100644 internal/email/extract.go create mode 100644 internal/email/extract_test.go diff --git a/cmd/mavend/mail.go b/cmd/mavend/mail.go new file mode 100644 index 0000000..2e6e796 --- /dev/null +++ b/cmd/mavend/mail.go @@ -0,0 +1,158 @@ +// mavend/mail.go — core's half of the email reader (Vikunja #246, +// docs/plans/01-email-reader.md). +// +// The split: cmd/mavmaild holds the IMAP credential, connects to the mailbox +// and converts messages to plaintext; it hands each message to core over +// ipc.MethodIngestMail. Core runs the extraction on the resident model — +// llama-server lives in this process, spawned by the phraser — and writes what +// comes back through the one task intake seam. +// +// What this file may produce is exactly one thing: rows in `tasks` with status +// "candidate". No fact, no reminder, no note, no nudge, no calendar event. A +// 1.7B misreading a mail can therefore put a wrong line on a review page and +// nothing else; it can never make Maven speak, and it can never make her +// recite something out of an advert as true. +// +// Off unless configured twice over: no `email` block in mavend.json ⇒ the IPC +// method does not exist; no llama-server phraser ⇒ same. A reader pointed at a +// core that is not set up for mail gets ErrUnknownMethod rather than silence. +package main + +import ( + "context" + "fmt" + "log" + "time" + + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/email" + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/llm" + "github.com/kami/maven/internal/phraser" + "github.com/kami/maven/internal/store" +) + +// evidenceMaxChars — how much of the subject line is kept as a candidate's +// evidence. Enough to recognise the mail on /tasks, not enough to turn the task +// list into a copy of his mailbox. +const evidenceMaxChars = 160 + +// mailIntake — extraction + capture for one message at a time. +type mailIntake struct { + st *store.Store + ex *email.Extractor + timeout time.Duration + now func() time.Time +} + +// newMailIntake returns nil when mail ingestion must not be available, which is +// the default. Both preconditions are real: +// +// - no cfg.Email ⇒ not configured, and a capability is off unless configured; +// - no llama-server phraser ⇒ nothing to extract with. There is deliberately +// no keyword fallback: "the subject line became a task" is not extraction, +// it is a mailbox rendered as a to-do list, and it would fill the review +// page faster than he could clear it. +func newMailIntake(st *store.Store, phr phraser.Phraser, cfg *config.Config) *mailIntake { + if cfg.Email == nil { + return nil + } + lp, ok := phr.(*phraser.LLMPhraser) + if !ok { + log.Printf("mail intake: configured but no llama-server phraser — mail ingestion disabled") + return nil + } + timeout := time.Duration(cfg.Email.Timeout) + if timeout <= 0 { + timeout = config.DefaultEmailTimeout + } + ex := email.NewExtractor(llm.New(lp.BaseURL(), timeout), cfg.Email.MaxTasks, contextBlockFn(cfg, time.Now)) + log.Printf("mail intake: enabled (max %d candidates per message, timeout %s)", cfg.Email.MaxTasks, timeout) + return &mailIntake{st: st, ex: ex, timeout: timeout, now: time.Now} +} + +// ingest handles one ipc.MethodIngestMail call. +// +// Junk and empty messages are answered Skipped without touching the model — the +// reader's header filter is what keeps the resident model off newsletters. +// +// Every candidate is captured with Status "candidate", Source "email:" +// and the subject as Evidence. CaptureTask dedupes on normalised text among +// live rows, so a mailbox re-read after a restart produces Created=0 rather +// than a second copy of every task. +func (m *mailIntake) ingest(ctx context.Context, req ipc.IngestMailReq) (ipc.IngestMailResp, error) { + msg := email.Message{ + UID: req.UID, + From: req.From, + Subject: req.Subject, + Date: req.Date, + Body: req.Body, + Junk: req.Junk, + } + if msg.Junk || (msg.Subject == "" && msg.Body == "") { + return ipc.IngestMailResp{Skipped: true}, nil + } + + ctx, cancel := context.WithTimeout(ctx, m.timeout) + defer cancel() + cands, err := m.ex.Extract(ctx, msg) + if err != nil { + // The error from internal/email never carries mail text; keep it that way + // by not adding the subject here. + return ipc.IngestMailResp{}, fmt.Errorf("mail intake: uid %d: %w", req.UID, err) + } + if len(cands) == 0 { + return ipc.IngestMailResp{}, nil + } + + source := email.SourcePrefix + req.Mailbox + evidence := truncateRunes(req.Subject, evidenceMaxChars) + now := m.now() + var resp ipc.IngestMailResp + for _, c := range cands { + t := store.Task{ + CreatedTs: now, + Text: c.Text, + Source: source, + Evidence: evidence, + // The one status this path may ever write. Anything Maven derived from + // something she read is a suggestion until he confirms it on /tasks. + Status: store.TaskCandidate, + } + if due, ok := email.ParseDue(c.Due); ok { + t.Due = &due + } + id, created, err := m.st.CaptureTask(ctx, t) + if err != nil { + return resp, fmt.Errorf("mail intake: capture: %w", err) + } + resp.TaskIDs = append(resp.TaskIDs, id) + if created { + resp.Created++ + } + } + // Counts only: the log line names the mailbox and the UID, never the subject, + // the sender or the task text. Reviewing a candidate is what /tasks is for. + log.Printf("mail intake: %s uid %d → %d candidate(s), %d new", source, req.UID, len(cands), resp.Created) + return resp, nil +} + +// wireMailIntake installs the IPC hook, or leaves it nil so the method reports +// ErrUnknownMethod. Called on both startup paths (unlocked boot and passkey +// unlock) so mail behaves the same either way. +func wireMailIntake(srv *ipc.Server, st *store.Store, phr phraser.Phraser, cfg *config.Config) { + mi := newMailIntake(st, phr, cfg) + if mi == nil { + return + } + srv.IngestMailFn = mi.ingest +} + +// truncateRunes cuts a string to n runes, marking the cut. +func truncateRunes(s string, n int) string { + r := []rune(s) + if len(r) <= n { + return s + } + return string(r[:n]) + "…" +} diff --git a/cmd/mavend/mail_test.go b/cmd/mavend/mail_test.go new file mode 100644 index 0000000..d06bb70 --- /dev/null +++ b/cmd/mavend/mail_test.go @@ -0,0 +1,182 @@ +package main + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/email" + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/llm" + "github.com/kami/maven/internal/store" +) + +// mailLLM — a canned extraction reply. +type mailLLM struct { + reply string + calls int +} + +func (m *mailLLM) Complete(_ context.Context, _ llm.Req) (string, error) { + m.calls++ + return m.reply, nil +} + +func newTestIntake(t *testing.T, reply string) (*mailIntake, *store.Store, *mailLLM) { + t.Helper() + st := newTestStore(t) + fake := &mailLLM{reply: reply} + return &mailIntake{ + st: st, + ex: email.NewExtractor(fake, 0, nil), + timeout: 5 * time.Second, + now: func() time.Time { return time.Date(2026, 8, 1, 10, 0, 0, 0, time.UTC) }, + }, st, fake +} + +func ingestReq() ipc.IngestMailReq { + return ipc.IngestMailReq{ + Mailbox: "INBOX", UID: 42, + From: "billing@isp.example", + Subject: "Счёт за интернет", + Body: "Оплатите счёт до 5 августа.", + } +} + +// The one property that matters: a mail-derived task is a candidate, attributed +// to the mailbox, with the subject as reviewable evidence — and nothing else is +// written. +func TestIngestCapturesCandidates(t *testing.T) { + mi, st, _ := newTestIntake(t, `[{"text":"оплатить счёт за интернет","due":"2026-08-05"}]`) + resp, err := mi.ingest(context.Background(), ingestReq()) + if err != nil { + t.Fatalf("ingest: %v", err) + } + if resp.Created != 1 || len(resp.TaskIDs) != 1 { + t.Fatalf("resp = %+v, want one created task", resp) + } + tasks, err := st.ListTasks(context.Background(), "") + if err != nil { + t.Fatalf("list: %v", err) + } + if len(tasks) != 1 { + t.Fatalf("got %d tasks, want 1", len(tasks)) + } + got := tasks[0] + if got.Status != store.TaskCandidate { + t.Errorf("status = %q, want %q — mail may only produce candidates", got.Status, store.TaskCandidate) + } + if got.Source != "email:INBOX" { + t.Errorf("source = %q, want email:INBOX", got.Source) + } + if got.Evidence != "Счёт за интернет" { + t.Errorf("evidence = %q, want the subject line", got.Evidence) + } + if got.Due == nil || got.Due.Format("2006-01-02") != "2026-08-05" { + t.Errorf("due = %v, want 2026-08-05", got.Due) + } + // Nothing else may have been written: no reminder, no fact. + rem, err := st.ListReminders(context.Background(), 10) + if err != nil { + t.Fatalf("list reminders: %v", err) + } + if len(rem) != 0 { + t.Errorf("mail created %d reminders; a misread mail must never be able to fire", len(rem)) + } +} + +// Re-reading a mailbox must not grow the list — CaptureTask dedupes among live +// rows, and the intake relies on exactly that. +func TestIngestSameMailTwiceIsIdempotent(t *testing.T) { + mi, st, _ := newTestIntake(t, `[{"text":"оплатить счёт","due":""}]`) + if _, err := mi.ingest(context.Background(), ingestReq()); err != nil { + t.Fatalf("first ingest: %v", err) + } + resp, err := mi.ingest(context.Background(), ingestReq()) + if err != nil { + t.Fatalf("second ingest: %v", err) + } + if resp.Created != 0 || len(resp.TaskIDs) != 1 { + t.Errorf("resp = %+v, want the existing row and Created=0", resp) + } + tasks, _ := st.ListTasks(context.Background(), "") + if len(tasks) != 1 { + t.Errorf("got %d tasks after two reads, want 1", len(tasks)) + } +} + +func TestIngestJunkSkipsTheModel(t *testing.T) { + mi, st, fake := newTestIntake(t, `[{"text":"купить со скидкой","due":""}]`) + req := ingestReq() + req.Junk = true + resp, err := mi.ingest(context.Background(), req) + if err != nil { + t.Fatalf("ingest: %v", err) + } + if !resp.Skipped || resp.Created != 0 { + t.Errorf("resp = %+v, want skipped", resp) + } + if fake.calls != 0 { + t.Errorf("model called %d times for junk, want 0", fake.calls) + } + if tasks, _ := st.ListTasks(context.Background(), ""); len(tasks) != 0 { + t.Errorf("junk produced %d tasks, want 0", len(tasks)) + } +} + +func TestIngestEmptyMessageSkipped(t *testing.T) { + mi, _, fake := newTestIntake(t, "[]") + resp, err := mi.ingest(context.Background(), ipc.IngestMailReq{Mailbox: "INBOX", UID: 1}) + if err != nil || !resp.Skipped { + t.Fatalf("resp = %+v, err = %v; want skipped", resp, err) + } + if fake.calls != 0 { + t.Errorf("model called %d times for an empty message, want 0", fake.calls) + } +} + +func TestIngestNoTasksWritesNothing(t *testing.T) { + mi, st, _ := newTestIntake(t, "[]") + resp, err := mi.ingest(context.Background(), ingestReq()) + if err != nil { + t.Fatalf("ingest: %v", err) + } + if resp.Created != 0 || len(resp.TaskIDs) != 0 || resp.Skipped { + t.Errorf("resp = %+v, want nothing captured and not skipped", resp) + } + if tasks, _ := st.ListTasks(context.Background(), ""); len(tasks) != 0 { + t.Errorf("got %d tasks, want 0", len(tasks)) + } +} + +func TestIngestTruncatesEvidence(t *testing.T) { + mi, st, _ := newTestIntake(t, `[{"text":"дело","due":""}]`) + req := ingestReq() + req.Subject = strings.Repeat("щ", 400) + if _, err := mi.ingest(context.Background(), req); err != nil { + t.Fatalf("ingest: %v", err) + } + tasks, _ := st.ListTasks(context.Background(), "") + if len(tasks) != 1 { + t.Fatalf("got %d tasks, want 1", len(tasks)) + } + if n := len([]rune(tasks[0].Evidence)); n > evidenceMaxChars+1 { + t.Errorf("evidence kept %d runes, want ≤ %d", n, evidenceMaxChars) + } +} + +// Off unless configured: no email block ⇒ no intake, so the IPC method does not +// exist at all. +func TestNewMailIntakeOffWithoutConfig(t *testing.T) { + st := newTestStore(t) + if mi := newMailIntake(st, nil, &config.Config{}); mi != nil { + t.Error("no email block must mean no mail intake") + } + // Configured but with a non-LLM phraser: still off — there is no fallback + // extraction, by design. + if mi := newMailIntake(st, nil, &config.Config{Email: &config.EmailConfig{}}); mi != nil { + t.Error("without a llama-server phraser there is nothing to extract with") + } +} diff --git a/cmd/mavend/main.go b/cmd/mavend/main.go index e532810..5d1cf2c 100644 --- a/cmd/mavend/main.go +++ b/cmd/mavend/main.go @@ -320,6 +320,13 @@ func run(args []string) error { srv.StepUp = func(ctx context.Context) error { return passkeySess.Assert(ctx, auth.Scope{}) } + // Mail ingestion (Vikunja #246): the hook stays nil unless an email block is + // configured and there is a llama-server to extract with, in which case + // ipc.MethodIngestMail reports ErrUnknownMethod. + if !locked { + wireMailIntake(srv, st, phr, cfg) + } + // WrapKeyFn — wraps the env key with a passkey credential public key and // persists the wrapped blob. Only wired when the daemon has the key in // memory (env key mode). Called by mavweb after passkey enrollment. @@ -457,6 +464,7 @@ func run(args []string) error { } srv.SetAPI(newAPI) srv.Check = (&auth.Gate{Enrollment: auth.NewFloorEnrollment(), Session: passkeySess}).Check + wireMailIntake(srv, st, phr, cfg) // Start voice server. if voiceW != nil { diff --git a/internal/auth/policy.go b/internal/auth/policy.go index 33873a3..429edc3 100644 --- a/internal/auth/policy.go +++ b/internal/auth/policy.go @@ -74,7 +74,12 @@ func Requirement(m ipc.Method) Authority { // existing analogue. ipc.MethodCaptureTask, ipc.MethodListTasks, - ipc.MethodSetTaskStatus: + ipc.MethodSetTaskStatus, + // Mail ingestion (Vikunja #246). AuthRead because of what the method can + // produce: candidate tasks and nothing else. It cannot write a fact, set a + // reminder, or touch the tool allowlist, so a compromised mail reader can + // at worst put junk on a review page he clears in one click. + ipc.MethodIngestMail: return AuthRead } // Unknown method ⇒ AuthRead, but ipc.dispatch returns ErrUnknownMethod diff --git a/internal/config/config.go b/internal/config/config.go index 8883f61..5becbb8 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -150,6 +150,12 @@ type Config struct { // absent ⇒ no evaluation loop at all. See MemoryEvalConfig. MemoryEval *MemoryEvalConfig `json:"memory_eval,omitempty"` + // Email — mail ingestion (Vikunja #246). nil / absent ⇒ core refuses + // ipc.MethodIngestMail outright, so a mail reader cannot make Maven read a + // mailbox by merely existing. See EmailConfig; the IMAP host and credential + // live in the reader (cmd/mavmaild), never here. + Email *EmailConfig `json:"email,omitempty"` + // Praxis — the ecosystem attention-state service. When configured, maven // calls the Praxis HTTP tools API for attention listing and item lifecycle. // Maven never touches Praxis's database directly (ecosystem invariant: no @@ -418,6 +424,26 @@ type MemoryEvalConfig struct { MinConfidence float64 `json:"min_confidence,omitempty"` } +// EmailConfig — core's half of the email reader: how many task candidates one +// message may produce, and how long the extraction call may take. +// +// There is deliberately nothing about a mailbox here. Core does not connect to +// IMAP, does not know an account exists, and holds no mail credential — the +// reader daemon does, the same split mavpoll uses for the zenmoney token. This +// block only says "extraction is allowed, with these bounds". +type EmailConfig struct { + // MaxTasks — candidates per message. 0 ⇒ email.MaxCandidates (3). + MaxTasks int `json:"max_tasks,omitempty"` + + // Timeout — per-message extraction budget. 0 ⇒ DefaultEmailTimeout. This is + // a Thinking model reading a mail; nobody is waiting on the answer, but a + // hung llama-server must not pin the reader's connection forever. + Timeout Duration `json:"timeout,omitempty"` +} + +// DefaultEmailTimeout — extraction budget per message. +const DefaultEmailTimeout = 2 * time.Minute + // PhraserConfig — the LLM-backed phraser seam. The daemon spawns llama-server // as a managed subprocess and sends chat-completion requests to phrase nudge // and reminder messages. nil ⇒ the template-based Stub is used instead. @@ -607,6 +633,12 @@ func (c *Config) applyDefaults() { c.MemoryEval.Interval = Duration(DefaultMemoryEvalInterval) } + // Same rule again: absent stays nil (⇒ mail ingestion refused), present gets + // the timeout default so `{}` is a valid "on with the defaults". + if c.Email != nil && c.Email.Timeout <= 0 { + c.Email.Timeout = Duration(DefaultEmailTimeout) + } + if c.Voice != nil { if c.Voice.RouterThreshold <= 0 { c.Voice.RouterThreshold = DefaultRouterThreshold diff --git a/internal/email/extract.go b/internal/email/extract.go new file mode 100644 index 0000000..d96723a --- /dev/null +++ b/internal/email/extract.go @@ -0,0 +1,226 @@ +package email + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/kami/maven/internal/llm" + "github.com/kami/maven/internal/persona" +) + +// Extraction — turning one mail into task CANDIDATES, and nothing else. +// +// The output of this file can only ever become rows in `tasks` with status +// "candidate" (store.TaskCandidate), written through the one intake seam +// (ipc.CaptureTaskReq, Vikunja #130). That bound is the whole design: +// +// - No reminder. A reminder FIRES; it speaks to him unprompted. A 1.7B that +// misreads "встреча была в четверг" as a future appointment would then wake +// him up about it. A candidate that is wrong is a line on a review page he +// dismisses in one click, which is the correct cost of a model being wrong +// about someone's mail. +// - No fact. A fact is a claim Maven will later recite as true. Nothing read +// out of a marketing mail deserves that standing. +// - No calendar event, no note, no action. Extraction writes candidates or +// writes nothing. +// +// The due date the model may return is stored on the candidate (tasks.due_ts), +// which no scheduler reads — it is there so the review page can sort by it. +// +// Privacy: the mail text goes to the resident model on this box and nowhere +// else. It is never search input (CLAUDE.md: "his notes and facts are never +// search input" — mail is the same class), and Evidence keeps only the subject +// line, so the review page shows him where a candidate came from without the +// store growing a copy of his mailbox. + +// MaxCandidates — at most this many candidates per message, enforced by the +// grammar. A mail with four tasks in it is a mail he has to read himself; a +// model allowed ten will produce ten. +const MaxCandidates = 3 + +// SourcePrefix — provenance for everything this package captures. The mailbox +// name is appended: "email:INBOX". Same vocabulary as tap:voice / poll:netdata. +const SourcePrefix = "email:" + +// Candidate — one piece of work the model thinks the mail is asking for. +type Candidate struct { + Text string `json:"text"` + // Due — "YYYY-MM-DD" or empty. A date the model read out of the text, not a + // date it computed: relative wording ("до пятницы") is left in Text, because + // a small model resolving "пятница" against today's date gets it wrong often + // enough that a stored wrong date is worse than no date. + Due string `json:"due"` +} + +// Completer — the llama-server seam, same shape memeval and the router use, so +// the one resident model serves this caller too. +type Completer interface { + Complete(ctx context.Context, r llm.Req) (string, error) +} + +// Extractor reads a message and returns candidates. It holds no store and no +// writer on purpose: this type cannot persist anything, so "extraction never +// acts" is a property of the code, not of a review. +type Extractor struct { + llm Completer + // MaxCandidates — 0 ⇒ MaxCandidates. + max int + // ContextBlock — the shared persona block, optional. Extraction output is + // not spoken, so the persona matters less here than in the phraser; it is + // wired anyway so a candidate reads in her voice on the review page. + contextBlock func() string +} + +func NewExtractor(c Completer, max int, contextBlock func() string) *Extractor { + if max <= 0 || max > MaxCandidates { + max = MaxCandidates + } + return &Extractor{llm: c, max: max, contextBlock: contextBlock} +} + +// extractGrammar — GBNF pinning the answer to a bounded array of fixed-shape +// candidates. Same reasoning as memeval's evalGrammar and the router's +// routeGrammar: the shape and the length bound are what keep a small model from +// drifting into prose or spending the token budget repeating one field. +// +// The empty array is reachable, deliberately: most mail contains no task, and a +// model with no way to say "nothing" invents something. +const extractGrammar = ` +root ::= "[" ws (item ("," ws item){0,2})? ws "]" +item ::= "{" ws "\"text\"" ws ":" ws text "," ws "\"due\"" ws ":" ws due ws "}" +text ::= "\"" ([^"\\] | "\\" .){1,120} "\"" +due ::= "\"\"" | "\"" [0-9]{4} "-" [0-9]{2} "-" [0-9]{2} "\"" +ws ::= [ \t\n]* +` + +// extractSystem — the extraction prompt. +// +// Written around the two failure modes a small model has on this task: it +// summarises when asked to extract (turning a mail into "письмо от Антона"), +// and it invents an obligation from any polite closing sentence. Hence the +// insistence on a verb phrase, and the explicit permission to return []. +const extractSystem = `Ты читаешь одно письмо из его почты и достаёшь из него дела, которые письмо от него требует. + +Правила: +- Отвечай ТОЛЬКО массивом JSON. Каждый элемент: {"text": "...", "due": "ГГГГ-ММ-ДД" или ""}. +- text — короткая формулировка дела по-русски, с глаголом: "оплатить счёт за интернет", "отправить акт". Не пересказывай письмо и не описывай его. +- Дело — это то, что должен сделать ОН. Рассылка, реклама, уведомление, отчёт, письмо «просто к сведению» — дел не содержат. +- Если письмо ничего от него не требует, верни пустой массив []. Это нормальный ответ, так бывает чаще всего. +- Ничего не придумывай. Если срока в письме нет — "". +- due заполняй только когда в письме стоит конкретная дата. Слова вроде «до пятницы» оставь в text, дату не вычисляй. +- Максимум три дела. Лучше одно точное, чем три общих.` + +// Extract returns the candidates in one message. +// +// Junk is refused without an LLM call — cheapest possible defence, and the +// reason the header filter exists. An empty message (no subject, no body) is +// likewise not worth a round trip. +// +// A parse failure is an error the caller logs and moves past. It is never +// silently turned into zero candidates, because "the model went off the rails" +// and "the mail contains no task" want different reactions from a human reading +// the log. +func (e *Extractor) Extract(ctx context.Context, msg Message) ([]Candidate, error) { + if msg.Junk { + return nil, nil + } + user := renderForModel(msg) + if user == "" { + return nil, nil + } + raw, err := e.llm.Complete(ctx, llm.Req{ + System: persona.Prepend(e.contextBlock, extractSystem), + User: user, + Grammar: extractGrammar, + MaxTokens: 512, + RepeatPenalty: 1.1, + }) + if err != nil { + return nil, fmt.Errorf("email: extract: %w", err) + } + items, err := parseCandidates(raw) + if err != nil { + // The raw reply is NOT in the error: it is a transformation of his mail, + // and this error reaches the daemon log. + return nil, fmt.Errorf("email: extract: unparsable reply (%d bytes)", len(raw)) + } + out := make([]Candidate, 0, len(items)) + seen := map[string]bool{} + for _, it := range items { + it.Text = strings.TrimSpace(it.Text) + if it.Text == "" { + continue + } + key := strings.ToLower(strings.Join(strings.Fields(it.Text), " ")) + if seen[key] { + continue // the model repeating itself is not two tasks + } + seen[key] = true + if _, ok := ParseDue(it.Due); !ok { + it.Due = "" // a date the grammar allowed but the calendar does not + } + out = append(out, it) + if len(out) >= e.max { + break + } + } + return out, nil +} + +// renderForModel is the user turn: subject, sender and body, labelled. Only +// these three fields — no headers, no recipient list, no message-id, nothing +// that would let the model start reasoning about routing metadata. +func renderForModel(msg Message) string { + var b strings.Builder + if msg.From != "" { + fmt.Fprintf(&b, "От: %s\n", msg.From) + } + if msg.Subject != "" { + fmt.Fprintf(&b, "Тема: %s\n", msg.Subject) + } + if msg.Body != "" { + fmt.Fprintf(&b, "\n%s\n", msg.Body) + } + if msg.Subject == "" && msg.Body == "" { + return "" + } + return b.String() +} + +// parseCandidates decodes the grammar-constrained reply, tolerating the +// wrappers a Thinking model sometimes leaves around it (a fenced block, or +// leading reasoning before the array). +func parseCandidates(raw string) ([]Candidate, error) { + s := strings.TrimSpace(raw) + if i := strings.Index(s, "["); i > 0 { + s = s[i:] + } + if j := strings.LastIndex(s, "]"); j >= 0 { + s = s[:j+1] + } + var out []Candidate + if err := json.Unmarshal([]byte(s), &out); err != nil { + return nil, err + } + return out, nil +} + +// ParseDue turns the model's "YYYY-MM-DD" into a time in UTC. Exported because +// the daemon-side intake stores it on the candidate. +// +// The zero-value/empty case returns ok=false rather than an error: no date is +// the common answer, not a failure. +func ParseDue(s string) (time.Time, bool) { + s = strings.TrimSpace(s) + if s == "" { + return time.Time{}, false + } + t, err := time.Parse("2006-01-02", s) + if err != nil { + return time.Time{}, false + } + return t, true +} diff --git a/internal/email/extract_test.go b/internal/email/extract_test.go new file mode 100644 index 0000000..8f0d26f --- /dev/null +++ b/internal/email/extract_test.go @@ -0,0 +1,143 @@ +package email + +import ( + "context" + "strings" + "testing" + + "github.com/kami/maven/internal/llm" +) + +// fakeLLM returns a canned reply and records the request, so a test can assert +// on the grammar and on what of the mail was sent. +type fakeLLM struct { + reply string + err error + got llm.Req + calls int +} + +func (f *fakeLLM) Complete(_ context.Context, r llm.Req) (string, error) { + f.calls++ + f.got = r + return f.reply, f.err +} + +func msgFor(subject, body string) Message { + return Message{UID: 1, From: "anton@example.org", Subject: subject, Body: body} +} + +func TestExtractCandidates(t *testing.T) { + f := &fakeLLM{reply: `[{"text":"отправить акт","due":""},{"text":"оплатить счёт","due":"2026-08-05"}]`} + e := NewExtractor(f, 0, nil) + got, err := e.Extract(context.Background(), msgFor("Акт и счёт", "Надо отправить акт и оплатить счёт до 5 августа.")) + if err != nil { + t.Fatalf("extract: %v", err) + } + if len(got) != 2 { + t.Fatalf("got %d candidates, want 2: %+v", len(got), got) + } + if got[0].Text != "отправить акт" || got[1].Due != "2026-08-05" { + t.Errorf("candidates = %+v", got) + } + if f.got.Grammar == "" { + t.Error("extraction must be grammar-constrained") + } + // The subject and body go to the model; nothing else about the message does. + if !strings.Contains(f.got.User, "Акт и счёт") || !strings.Contains(f.got.User, "оплатить счёт") { + t.Errorf("user turn = %q", f.got.User) + } +} + +func TestExtractEmptyArrayIsNotAnError(t *testing.T) { + f := &fakeLLM{reply: "[]"} + got, err := NewExtractor(f, 0, nil).Extract(context.Background(), msgFor("FYI", "Просто к сведению.")) + if err != nil || len(got) != 0 { + t.Fatalf("got (%v, %v), want (empty, nil) — no task is the normal answer", got, err) + } +} + +// Junk must never reach the model: the header filter exists so the resident +// model is not spent on newsletters. +func TestExtractSkipsJunkWithoutCallingModel(t *testing.T) { + f := &fakeLLM{reply: `[{"text":"купить всё со скидкой","due":""}]`} + msg := msgFor("Скидки", "Sale!") + msg.Junk = true + got, err := NewExtractor(f, 0, nil).Extract(context.Background(), msg) + if err != nil || got != nil { + t.Fatalf("got (%v, %v), want (nil, nil)", got, err) + } + if f.calls != 0 { + t.Errorf("model called %d times for junk, want 0", f.calls) + } +} + +func TestExtractEmptyMessageIsNotSent(t *testing.T) { + f := &fakeLLM{reply: "[]"} + if _, err := NewExtractor(f, 0, nil).Extract(context.Background(), Message{UID: 3}); err != nil { + t.Fatalf("extract: %v", err) + } + if f.calls != 0 { + t.Errorf("model called %d times for an empty message, want 0", f.calls) + } +} + +func TestExtractCaps(t *testing.T) { + f := &fakeLLM{reply: `[{"text":"a","due":""},{"text":"b","due":""},{"text":"c","due":""}]`} + got, err := NewExtractor(f, 2, nil).Extract(context.Background(), msgFor("s", "b")) + if err != nil { + t.Fatalf("extract: %v", err) + } + if len(got) != 2 { + t.Errorf("got %d, want the configured cap of 2", len(got)) + } +} + +func TestExtractDropsRepeatsAndBadDates(t *testing.T) { + f := &fakeLLM{reply: `[{"text":"Отправить акт","due":"2026-02-31"},{"text":"отправить акт","due":""},{"text":" ","due":""}]`} + got, err := NewExtractor(f, 0, nil).Extract(context.Background(), msgFor("s", "b")) + if err != nil { + t.Fatalf("extract: %v", err) + } + if len(got) != 1 { + t.Fatalf("got %d candidates, want 1 (repeat and blank dropped): %+v", len(got), got) + } + if got[0].Due != "" { + t.Errorf("due = %q, want empty — 2026-02-31 is not a date", got[0].Due) + } +} + +// A Thinking model sometimes wraps the array; and when it emits something +// unparsable the caller must hear about it rather than see "no tasks". +func TestParseCandidatesTolerance(t *testing.T) { + got, err := parseCandidates("думаю... [{\"text\":\"x\",\"due\":\"\"}] всё") + if err != nil || len(got) != 1 || got[0].Text != "x" { + t.Fatalf("got (%+v, %v)", got, err) + } + if _, err := parseCandidates("нет никакого JSON"); err == nil { + t.Error("unparsable output must be an error") + } +} + +func TestExtractParseErrorHidesMailText(t *testing.T) { + f := &fakeLLM{reply: "он просил отправить акт, вот такой ответ"} + _, err := NewExtractor(f, 0, nil).Extract(context.Background(), msgFor("Акт", "секретный текст")) + if err == nil { + t.Fatal("want an error") + } + if strings.Contains(err.Error(), "акт") || strings.Contains(err.Error(), "секретный") { + t.Errorf("error text leaks mail content: %v", err) + } +} + +func TestParseDue(t *testing.T) { + if _, ok := ParseDue(""); ok { + t.Error("empty due must be (zero, false)") + } + if got, ok := ParseDue("2026-08-05"); !ok || got.Year() != 2026 || got.Month() != 8 || got.Day() != 5 { + t.Errorf("ParseDue = (%v, %v)", got, ok) + } + if _, ok := ParseDue("05.08.2026"); ok { + t.Error("a non-ISO date must not parse") + } +} diff --git a/internal/ipc/api.go b/internal/ipc/api.go index c89d520..28d7284 100644 --- a/internal/ipc/api.go +++ b/internal/ipc/api.go @@ -138,6 +138,44 @@ type CaptureTaskResp struct { Created bool `json:"created"` } +// IngestMailReq — one message a mail reader has fetched, handed to core for +// extraction (Vikunja #246). +// +// The mail reader (cmd/mavmaild) holds the IMAP credential and core never sees +// it, the same split mavpoll uses for the zenmoney token. What crosses this +// boundary is only the message text, because extraction runs on the resident +// model and llama-server lives inside core's process. +// +// Body is already plaintext and truncated by internal/email; core does not +// re-parse MIME and never stores the body. Junk means the reader's header +// filter already classified the message as bulk — core is told rather than +// asked, so a junk message can be counted without a model call. +// +// This method is available only when core has an email block configured AND a +// llama-server phraser; otherwise it answers ErrUnknownMethod, which is what +// "off unless configured" looks like at the wire. +type IngestMailReq struct { + Mailbox string `json:"mailbox"` + UID uint32 `json:"uid"` + From string `json:"from,omitempty"` + Subject string `json:"subject,omitempty"` + Date string `json:"date,omitempty"` + Body string `json:"body,omitempty"` + Junk bool `json:"junk,omitempty"` +} + +// IngestMailResp — what core did with the message. TaskIDs are the rows +// CaptureTask returned; Created counts the ones that were new (a re-read +// mailbox dedupes to Created=0). Skipped is set when nothing was asked of the +// model at all — junk, or an empty message. +// +// Nothing here echoes the mail back. The reader logs counts. +type IngestMailResp struct { + TaskIDs []int64 `json:"task_ids,omitempty"` + Created int `json:"created"` + Skipped bool `json:"skipped,omitempty"` +} + type listTasksReq struct { Status string `json:"status"` // "" all | "live" | candidate|open|done|dropped } diff --git a/internal/ipc/client.go b/internal/ipc/client.go index 7048795..3602bbb 100644 --- a/internal/ipc/client.go +++ b/internal/ipc/client.go @@ -448,6 +448,17 @@ func (c *Client) SetTaskStatus(ctx context.Context, id int64, status string, ts return c.call(ctx, MethodSetTaskStatus, setTaskStatusReq{ID: id, Status: status, Ts: ts}, nil) } +// IngestMail hands one fetched message to core for extraction. ErrUnknownMethod +// means core has no email block configured — the caller should stop asking, not +// retry. +func (c *Client) IngestMail(ctx context.Context, req IngestMailReq) (IngestMailResp, error) { + var r IngestMailResp + if err := c.call(ctx, MethodIngestMail, req, &r); err != nil { + return IngestMailResp{}, err + } + return r, nil +} + func (c *Client) DismissProposedRoutine(ctx context.Context, id int64) error { return c.call(ctx, MethodDismissProposedRoutine, dismissProposedRoutineReq{ID: id}, nil) } diff --git a/internal/ipc/ipc_test.go b/internal/ipc/ipc_test.go index 8e26091..86f6a9d 100644 --- a/internal/ipc/ipc_test.go +++ b/internal/ipc/ipc_test.go @@ -565,3 +565,36 @@ func mustJSON(v any) []byte { } return b } + +// TestIngestMail_OffUnlessConfigured — with no IngestMailFn set (the default, +// and what an unconfigured core looks like) the method does not exist. A mail +// reader gets a refusal it can act on rather than a silent success. +func TestIngestMail_OffUnlessConfigured(t *testing.T) { + _, _, cli, _ := newServerWithStore(t) + if _, err := cli.IngestMail(context.Background(), IngestMailReq{Mailbox: "INBOX", UID: 1}); !errors.Is(err, ErrUnknownMethod) { + t.Fatalf("IngestMail error = %v, want ErrUnknownMethod", err) + } +} + +// TestIngestMail_Hook — when the daemon wires the hook, the message crosses the +// boundary intact and the response comes back. +func TestIngestMail_Hook(t *testing.T) { + _, srv, cli, _ := newServerWithStore(t) + var got IngestMailReq + srv.IngestMailFn = func(_ context.Context, req IngestMailReq) (IngestMailResp, error) { + got = req + return IngestMailResp{TaskIDs: []int64{7}, Created: 1}, nil + } + resp, err := cli.IngestMail(context.Background(), IngestMailReq{ + Mailbox: "INBOX", UID: 12, Subject: "Счёт", Body: "Оплатить.", Junk: false, + }) + if err != nil { + t.Fatalf("IngestMail: %v", err) + } + if resp.Created != 1 || len(resp.TaskIDs) != 1 || resp.TaskIDs[0] != 7 { + t.Errorf("resp = %+v", resp) + } + if got.UID != 12 || got.Subject != "Счёт" || got.Body != "Оплатить." { + t.Errorf("req across the wire = %+v", got) + } +} diff --git a/internal/ipc/server.go b/internal/ipc/server.go index 59043f7..1137153 100644 --- a/internal/ipc/server.go +++ b/internal/ipc/server.go @@ -421,6 +421,17 @@ type Server struct { // Set by the daemon; nil ⇒ MethodStoreEncryptionKey returns ErrUnknownMethod. WrapKeyFn WrapKeyFunc + // IngestMailFn — extracts task candidates from one fetched message. Set by + // the daemon only when an email block is configured AND there is a + // llama-server to extract with; nil ⇒ MethodIngestMail returns + // ErrUnknownMethod, so a mail reader pointed at a core that is not + // configured for mail is refused rather than silently ignored. + // + // Like StepUp/WrapKeyFn/UnlockFn this bypasses CoreAPI: it is not a store + // operation, it needs the resident model, and it must not become a method + // every CoreAPI implementation has to carry. + IngestMailFn IngestMailFunc + // UnlockFn — unwraps the store encryption key from the wrapped blob using // the passkey credential public key, opens the encrypted store, and wires // the rest of the daemon (voice, loop, delivery). Set by the daemon when @@ -439,6 +450,9 @@ type WrapKeyFunc func(ctx context.Context, publicKey []byte) error // public key and completes daemon initialization. type UnlockFunc func(ctx context.Context, publicKey []byte) error +// IngestMailFunc — core-side mail extraction. Returns what was captured. +type IngestMailFunc func(ctx context.Context, req IngestMailReq) (IngestMailResp, error) + // CheckFunc — the auth hook signature. Wired by the daemon (auth.Gate.Check // satisfies this); dispatch calls it once per request after param-unmarshal // independence (it gets the raw params, may unmarshal what it needs — ipc @@ -606,9 +620,10 @@ func withoutParams[R any](fn func(ctx context.Context, api CoreAPI) (R, error)) // existed) as an argument — so SetAPI's runtime swap (the unlock transition) // is still honored on the very next request with no extra plumbing here. // -// MethodAssertStepUp, MethodStoreEncryptionKey and MethodUnlock are NOT in -// this table: they bypass CoreAPI entirely (s.StepUp / s.WrapKeyFn / -// s.UnlockFn), so dispatch special-cases them before consulting the table. +// MethodAssertStepUp, MethodStoreEncryptionKey, MethodUnlock and +// MethodIngestMail are NOT in this table: they bypass CoreAPI entirely +// (s.StepUp / s.WrapKeyFn / s.UnlockFn / s.IngestMailFn), so dispatch +// special-cases them before consulting the table. var methodTable = map[Method]handlerFunc{ MethodWriteFact: withParams(func(ctx context.Context, api CoreAPI, p WriteFactReq) (idResp, error) { id, err := api.WriteFact(ctx, p) @@ -816,7 +831,7 @@ func (s *Server) dispatch(ctx context.Context, req Request) (json.RawMessage, er } } - // These three bypass CoreAPI entirely — they drive Server fields set + // These bypass CoreAPI entirely — they drive Server fields set // directly by the daemon (StepUp / WrapKeyFn / UnlockFn), not store // state, so they can never be table entries keyed on a CoreAPI method. switch req.Method { @@ -845,6 +860,20 @@ func (s *Server) dispatch(ctx context.Context, req Request) (json.RawMessage, er return marshalResult(nil), s.UnlockFn(ctx, p.PublicKey) } return nil, fmt.Errorf("%w: %s", ErrUnknownMethod, req.Method) + + case MethodIngestMail: + if s.IngestMailFn != nil { + var p IngestMailReq + if err := unmarshalParams(req.Params, &p); err != nil { + return nil, err + } + resp, err := s.IngestMailFn(ctx, p) + if err != nil { + return nil, err + } + return marshalResult(resp), nil + } + return nil, fmt.Errorf("%w: %s", ErrUnknownMethod, req.Method) } h, ok := methodTable[req.Method] diff --git a/internal/ipc/wire.go b/internal/ipc/wire.go index a3d89c5..3d1ea08 100644 --- a/internal/ipc/wire.go +++ b/internal/ipc/wire.go @@ -50,6 +50,7 @@ const ( MethodCaptureTask Method = "capture_task" MethodListTasks Method = "list_tasks" MethodSetTaskStatus Method = "set_task_status" + MethodIngestMail Method = "ingest_mail" ) // Request — one frame from module to core. Params is the JSON-encoded argument