From b4646155b4e5271230e63a0b745d90f7157e0530 Mon Sep 17 00:00:00 2001 From: kami Date: Sat, 1 Aug 2026 02:59:24 +0400 Subject: [PATCH] Read a mailbox read-only, in a client small enough to audit (#246) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit internal/email is the reading half of the email reader: a ~200-line IMAP client (LOGIN, EXAMINE, UID SEARCH SINCE, UID FETCH BODY.PEEK, LOGOUT), a MIME-to-plaintext converter, and a header-only junk filter. Two protocol choices are the design, not shortcuts. EXAMINE instead of SELECT means the session is read-only at the protocol level, so no command in it can flip a flag or expunge anything by mistake. BODY.PEEK instead of BODY means reading a message does not mark it \Seen — Maven reads his mail and leaves no trace of having done so, and the unread state in his own client stays his. Hand-rolled rather than go-imap because this is the one path that holds his mailbox credential and reads his private mail: five commands with no dependencies is auditable in a sitting. No IDLE and no cleartext/STARTTLS either — an option to send his password over a plain socket is an option to get it wrong once. Junk is decided by headers alone, before any model is involved: List-Unsubscribe/List-Id, Precedence: bulk, Auto-Submitted, the spam headers, and Gmail's own category labels. Sender lists and subject keywords are deliberately absent — they age badly and they would put his contacts in a config file. A junk verdict only means "do not spend the model on this"; nothing is deleted and no server flag is touched. Nothing here logs a body, a subject or an address, the junk reason names a header rather than content, and an undecodable charset degrades to headers-only instead of feeding the model mojibake. Verified against recorded .eml fixtures and an in-process fake IMAP server. --- internal/email/fetch.go | 96 +++++++ internal/email/fetch_test.go | 50 ++++ internal/email/imap.go | 280 +++++++++++++++++++ internal/email/imap_test.go | 162 +++++++++++ internal/email/junk.go | 80 ++++++ internal/email/junk_test.go | 59 ++++ internal/email/message.go | 258 +++++++++++++++++ internal/email/message_test.go | 111 ++++++++ internal/email/testdata/cp1251.eml | 7 + internal/email/testdata/html_only.eml | 13 + internal/email/testdata/mixed_attachment.eml | 26 ++ internal/email/testdata/newsletter.eml | 9 + internal/email/testdata/plain_ru.eml | 14 + 13 files changed, 1165 insertions(+) create mode 100644 internal/email/fetch.go create mode 100644 internal/email/fetch_test.go create mode 100644 internal/email/imap.go create mode 100644 internal/email/imap_test.go create mode 100644 internal/email/junk.go create mode 100644 internal/email/junk_test.go create mode 100644 internal/email/message.go create mode 100644 internal/email/message_test.go create mode 100644 internal/email/testdata/cp1251.eml create mode 100644 internal/email/testdata/html_only.eml create mode 100644 internal/email/testdata/mixed_attachment.eml create mode 100644 internal/email/testdata/newsletter.eml create mode 100644 internal/email/testdata/plain_ru.eml diff --git a/internal/email/fetch.go b/internal/email/fetch.go new file mode 100644 index 0000000..f41ed31 --- /dev/null +++ b/internal/email/fetch.go @@ -0,0 +1,96 @@ +package email + +import ( + "fmt" + "time" +) + +// FetchSince is the whole read path in one call: connect, log in, examine the +// mailbox read-only, list what arrived since a date, fetch and parse the ones +// the caller has not seen, log out. +// +// It is a function rather than a long-lived object because a mail poller should +// not hold an authenticated session (and therefore his credential in a live TLS +// state) between polls. Connect, read, drop. +// +// skip decides which UIDs are already known — the poller's seen-set. max bounds +// one poll: a mailbox that received 400 messages overnight must not turn into +// 400 LLM calls, and the newest max are the ones a task could still be hiding +// in. Junk messages are returned too, flagged, so the caller can mark them seen +// without a second protocol round. +type FetchSince struct { + Addr string // host or host:993 + User string + Mailbox string // e.g. "INBOX" + Timeout time.Duration + Since time.Time + Max int + Skip func(uid uint32) bool + + // dial is the connection seam. nil means Dial (implicit TLS); the tests set + // it to an in-process fake. Unexported so no configuration path can point + // the reader at a non-TLS transport. + dial func(addr string, timeout time.Duration) (*Conn, error) +} + +// Run performs one read. password is passed here, not stored in the struct, so +// the configuration of a mailbox and the secret for it are never the same value +// sitting in the same place. +func (f FetchSince) Run(password string) ([]Message, error) { + if f.Addr == "" || f.User == "" || f.Mailbox == "" { + return nil, fmt.Errorf("email: mailbox not configured (addr/user/mailbox)") + } + dial := f.dial + if dial == nil { + dial = Dial + } + c, err := dial(f.Addr, f.Timeout) + if err != nil { + return nil, err + } + defer c.Close() + if err := c.Login(f.User, password); err != nil { + return nil, err + } + defer c.Logout() + if err := c.Select(f.Mailbox); err != nil { + return nil, err + } + uids, err := c.SearchSince(f.Since) + if err != nil { + return nil, err + } + + // Newest UIDs first — IMAP hands them back ascending, and when Max clips the + // list the recent mail is what matters. + wanted := make([]uint32, 0, len(uids)) + for i := len(uids) - 1; i >= 0; i-- { + if f.Skip != nil && f.Skip(uids[i]) { + continue + } + wanted = append(wanted, uids[i]) + if f.Max > 0 && len(wanted) >= f.Max { + break + } + } + + out := make([]Message, 0, len(wanted)) + for _, uid := range wanted { + raw, err := c.Fetch(uid) + if err != nil { + // One unreadable message does not abandon the poll; the rest of the + // mailbox is still worth reading. The error names the UID, not the + // message. + return out, fmt.Errorf("email: fetch uid %d: %w", uid, err) + } + if len(raw) == 0 { + continue // vanished between SEARCH and FETCH + } + msg, err := ParseMessage(uid, raw) + if err != nil { + continue // unparsable headers — nothing to review, skip silently + } + out = append(out, msg) + } + return out, nil +} diff --git a/internal/email/fetch_test.go b/internal/email/fetch_test.go new file mode 100644 index 0000000..f9c246e --- /dev/null +++ b/internal/email/fetch_test.go @@ -0,0 +1,50 @@ +package email + +import ( + "net" + "strings" + "testing" + "time" +) + +func TestFetchSinceRun(t *testing.T) { + mk := func(subject string) string { + return "Subject: " + subject + "\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nbody\r\n" + } + f := &fakeIMAP{ + uids: []uint32{1, 2, 3}, + msgs: map[uint32]string{1: mk("one"), 2: mk("two"), 3: mk("three")}, + } + fs := FetchSince{ + Addr: "mail.example:993", User: "kami", Mailbox: "INBOX", + Timeout: 5 * time.Second, + Since: time.Date(2026, 7, 30, 0, 0, 0, 0, time.UTC), + Max: 2, + Skip: func(uid uint32) bool { return uid == 3 }, + dial: func(addr string, timeout time.Duration) (*Conn, error) { + cli, srv := net.Pipe() + go f.serve(t, srv) + return NewConn(cli, timeout) + }, + } + msgs, err := fs.Run("secret") + if err != nil { + t.Fatalf("run: %v", err) + } + // Newest first, the already-seen UID skipped, Max respected. + if len(msgs) != 2 { + t.Fatalf("got %d messages, want 2: %+v", len(msgs), msgs) + } + if msgs[0].Subject != "two" || msgs[1].Subject != "one" { + t.Errorf("subjects = %q,%q, want two,one (newest first)", msgs[0].Subject, msgs[1].Subject) + } + if strings.Contains(strings.Join(f.cmds, " "), "UID FETCH 3") { + t.Error("a skipped UID must not be fetched again") + } +} + +func TestFetchSinceRequiresConfig(t *testing.T) { + if _, err := (FetchSince{}).Run("secret"); err == nil { + t.Fatal("an unconfigured mailbox must not be read") + } +} diff --git a/internal/email/imap.go b/internal/email/imap.go new file mode 100644 index 0000000..44f2150 --- /dev/null +++ b/internal/email/imap.go @@ -0,0 +1,280 @@ +package email + +import ( + "bufio" + "crypto/tls" + "fmt" + "io" + "net" + "regexp" + "strconv" + "strings" + "time" +) + +// A minimal IMAP4rev1 client — LOGIN, SELECT, UID SEARCH, UID FETCH with +// BODY.PEEK, LOGOUT, and nothing else. +// +// Why hand-rolled instead of go-imap: the whole surface Maven needs is five +// commands, and this is the one code path that holds his mailbox credential and +// reads his private mail. A ~200-line client with no dependencies is auditable +// in one sitting; a general-purpose IMAP library is a much larger amount of +// code doing much more than we asked, in the most sensitive place in the tree. +// If IDLE, CONDSTORE or server-side threading ever become worth having, that +// trade should be re-made deliberately. +// +// BODY.PEEK[] rather than BODY[] is load-bearing: Maven reads his mail and must +// leave no trace of having done so. Reading a message here does not mark it +// \Seen, so the unread state in his own mail client stays his. + +// DefaultIMAPPort — implicit-TLS IMAP. There is no cleartext and no STARTTLS +// path in this client: an option to send his password over a plain socket is an +// option to get it wrong once. +const DefaultIMAPPort = "993" + +// Conn — one authenticated IMAP connection. Not safe for concurrent use; the +// poller drives one connection at a time. +type Conn struct { + rwc io.ReadWriteCloser + r *bufio.Reader + tag int + timeout time.Duration +} + +// Dial opens an implicit-TLS connection and reads the server greeting. +func Dial(addr string, timeout time.Duration) (*Conn, error) { + host, _, err := net.SplitHostPort(addr) + if err != nil { + host, addr = addr, net.JoinHostPort(addr, DefaultIMAPPort) + } + d := &net.Dialer{Timeout: timeout} + // ServerName is set from the host we asked for: certificate verification is + // the only thing standing between his password and a MITM on the way out. + c, err := tls.DialWithDialer(d, "tcp", addr, &tls.Config{ServerName: host, MinVersion: tls.VersionTLS12}) + if err != nil { + return nil, fmt.Errorf("email: dial %s: %w", addr, err) + } + return NewConn(c, timeout) +} + +// NewConn wraps an already-open stream (the tests speak IMAP over a pipe) and +// consumes the greeting. +func NewConn(rwc io.ReadWriteCloser, timeout time.Duration) (*Conn, error) { + c := &Conn{rwc: rwc, r: bufio.NewReaderSize(rwc, 64<<10), timeout: timeout} + line, err := c.readLine() + if err != nil { + return nil, fmt.Errorf("email: greeting: %w", err) + } + if !strings.HasPrefix(line, "* OK") && !strings.HasPrefix(line, "* PREAUTH") { + c.rwc.Close() + return nil, fmt.Errorf("email: server refused connection: %s", line) + } + return c, nil +} + +func (c *Conn) Close() error { return c.rwc.Close() } + +// Login authenticates with LOGIN. The password is passed as an argument and +// never stored on the Conn: nothing in this package keeps a credential alive +// past the command that uses it, so no struct dump or panic trace can carry it. +func (c *Conn) Login(user, pass string) error { + // The command line itself is never logged (see exec) — a LOGIN line IS the + // credential. + if _, err := c.exec(fmt.Sprintf("LOGIN %s %s", quote(user), quote(pass))); err != nil { + return fmt.Errorf("email: login: %w", err) + } + return nil +} + +// Select opens a mailbox read-only. EXAMINE, not SELECT: read-only at the +// protocol level means no command in this session can change a flag, expunge a +// message, or move anything, even by mistake. +func (c *Conn) Select(mailbox string) error { + if _, err := c.exec(fmt.Sprintf("EXAMINE %s", quote(mailbox))); err != nil { + return fmt.Errorf("email: examine %s: %w", mailbox, err) + } + return nil +} + +// SearchSince returns the UIDs of messages received on or after since. An +// unlimited search is not offered: the first poll against a years-old mailbox +// would otherwise fetch everything and hand a decade of mail to the model. +// +// The IMAP SINCE key has date granularity (and compares the server's internal +// date), so the result can include messages slightly older than since. The +// caller dedupes by UID anyway, so a wider window costs one extra fetch. +func (c *Conn) SearchSince(since time.Time) ([]uint32, error) { + cmd := fmt.Sprintf("UID SEARCH SINCE %s", since.Format("2-Jan-2006")) + lines, err := c.exec(cmd) + if err != nil { + return nil, fmt.Errorf("email: search: %w", err) + } + var uids []uint32 + for _, l := range lines { + rest, ok := untagged(l, "SEARCH") + if !ok { + continue + } + for _, f := range strings.Fields(rest) { + n, err := strconv.ParseUint(f, 10, 32) + if err == nil { + uids = append(uids, uint32(n)) + } + } + } + return uids, nil +} + +var literalSize = regexp.MustCompile(`\{(\d+)\}$`) + +// Fetch returns the raw RFC 5322 bytes of one message, by UID. +// +// Returns (nil, nil) when the UID no longer exists — a message he deleted +// between SEARCH and FETCH is normal, not an error. +func (c *Conn) Fetch(uid uint32) ([]byte, error) { + tag := c.nextTag() + if err := c.send(fmt.Sprintf("%s UID FETCH %d (BODY.PEEK[])", tag, uid)); err != nil { + return nil, err + } + var raw []byte + for { + line, err := c.readLine() + if err != nil { + return nil, fmt.Errorf("email: fetch %d: %w", uid, err) + } + if done, err := c.tagged(tag, line); done { + if err != nil { + return nil, fmt.Errorf("email: fetch %d: %w", uid, err) + } + return raw, nil + } + m := literalSize.FindStringSubmatch(strings.TrimSpace(line)) + if m == nil { + continue + } + n, err := strconv.Atoi(m[1]) + if err != nil { + continue + } + buf := make([]byte, n) + if _, err := io.ReadFull(c.r, buf); err != nil { + return nil, fmt.Errorf("email: fetch %d: literal: %w", uid, err) + } + if raw == nil { + raw = buf + } + } +} + +// Logout ends the session politely. A failure is not worth reporting — the +// connection is being closed either way. +func (c *Conn) Logout() { + _, _ = c.exec("LOGOUT") +} + +// ---- protocol plumbing ----------------------------------------------------- + +func (c *Conn) nextTag() string { + c.tag++ + return fmt.Sprintf("a%03d", c.tag) +} + +// exec sends one command and returns the untagged response lines. +// +// Neither the command nor the response is ever logged here. LOGIN goes through +// this function, and a debug line "sent: a001 LOGIN ..." is how a credential +// ends up in a log file forever. +func (c *Conn) exec(cmd string) ([]string, error) { + tag := c.nextTag() + if err := c.send(tag + " " + cmd); err != nil { + return nil, err + } + var lines []string + for { + line, err := c.readLine() + if err != nil { + return nil, err + } + if done, err := c.tagged(tag, line); done { + return lines, err + } + lines = append(lines, line) + // A response line may carry a literal (e.g. a header FETCH). Nothing we + // send asks for one outside Fetch, but skip it if it appears so the + // stream stays aligned. + if m := literalSize.FindStringSubmatch(strings.TrimSpace(line)); m != nil { + if n, err := strconv.Atoi(m[1]); err == nil { + if _, err := io.CopyN(io.Discard, c.r, int64(n)); err != nil { + return nil, err + } + } + } + } +} + +// tagged reports whether line completes the command with this tag, and turns a +// NO/BAD completion into an error. The error text is the server's, which never +// echoes a password. +func (c *Conn) tagged(tag, line string) (bool, error) { + if !strings.HasPrefix(line, tag+" ") { + return false, nil + } + rest := strings.TrimSpace(line[len(tag):]) + switch { + case strings.HasPrefix(rest, "OK"): + return true, nil + case strings.HasPrefix(rest, "NO"), strings.HasPrefix(rest, "BAD"): + return true, fmt.Errorf("server said: %s", rest) + default: + return true, fmt.Errorf("unexpected completion: %s", rest) + } +} + +func (c *Conn) send(line string) error { + c.setDeadline() + if _, err := io.WriteString(c.rwc, line+"\r\n"); err != nil { + return fmt.Errorf("email: write: %w", err) + } + return nil +} + +func (c *Conn) readLine() (string, error) { + c.setDeadline() + line, err := c.r.ReadString('\n') + if err != nil { + return "", err + } + return strings.TrimRight(line, "\r\n"), nil +} + +// setDeadline applies the per-connection timeout when the transport supports +// one. A hung IMAP server must not park the poller forever. +func (c *Conn) setDeadline() { + if c.timeout <= 0 { + return + } + if d, ok := c.rwc.(interface{ SetDeadline(time.Time) error }); ok { + _ = d.SetDeadline(time.Now().Add(c.timeout)) + } +} + +// untagged splits "* SEARCH 1 2 3" into its payload when the key matches. +func untagged(line, key string) (string, bool) { + if !strings.HasPrefix(line, "* ") { + return "", false + } + rest := strings.TrimSpace(line[2:]) + if !strings.HasPrefix(rest, key) { + return "", false + } + return strings.TrimSpace(rest[len(key):]), true +} + +// quote renders an IMAP quoted string. Passwords routinely contain characters +// that would otherwise end the argument early, and CR/LF are stripped rather +// than escaped because there is no legal way to send them — a credential file +// with a stray newline must not become a second command. +func quote(s string) string { + s = strings.NewReplacer("\r", "", "\n", "").Replace(s) + return `"` + strings.NewReplacer(`\`, `\\`, `"`, `\"`).Replace(s) + `"` +} diff --git a/internal/email/imap_test.go b/internal/email/imap_test.go new file mode 100644 index 0000000..cd82d08 --- /dev/null +++ b/internal/email/imap_test.go @@ -0,0 +1,162 @@ +package email + +import ( + "bufio" + "fmt" + "net" + "strconv" + "strings" + "testing" + "time" +) + +// fakeIMAP is a scripted server: enough of IMAP to exercise the client, and +// nothing more. It records the commands it received so a test can assert on the +// protocol (BODY.PEEK rather than BODY, EXAMINE rather than SELECT). +type fakeIMAP struct { + msgs map[uint32]string + uids []uint32 + cmds []string + failOn string // substring of a command to answer NO +} + +func (f *fakeIMAP) serve(t *testing.T, c net.Conn) { + t.Helper() + defer c.Close() + fmt.Fprint(c, "* OK fake IMAP ready\r\n") + r := bufio.NewReader(c) + for { + line, err := r.ReadString('\n') + if err != nil { + return + } + line = strings.TrimRight(line, "\r\n") + parts := strings.SplitN(line, " ", 2) + if len(parts) != 2 { + return + } + tag, cmd := parts[0], parts[1] + f.cmds = append(f.cmds, cmd) + if f.failOn != "" && strings.Contains(cmd, f.failOn) { + fmt.Fprintf(c, "%s NO computer says no\r\n", tag) + continue + } + upper := strings.ToUpper(cmd) + switch { + case strings.HasPrefix(upper, "LOGIN"), strings.HasPrefix(upper, "EXAMINE"): + fmt.Fprintf(c, "%s OK done\r\n", tag) + case strings.HasPrefix(upper, "UID SEARCH"): + var ids []string + for _, u := range f.uids { + ids = append(ids, strconv.FormatUint(uint64(u), 10)) + } + fmt.Fprintf(c, "* SEARCH %s\r\n", strings.Join(ids, " ")) + fmt.Fprintf(c, "%s OK search done\r\n", tag) + case strings.HasPrefix(upper, "UID FETCH"): + uid64, _ := strconv.ParseUint(strings.Fields(cmd)[2], 10, 32) + raw, ok := f.msgs[uint32(uid64)] + if ok { + fmt.Fprintf(c, "* 1 FETCH (UID %d BODY[] {%d}\r\n", uid64, len(raw)) + fmt.Fprint(c, raw) + fmt.Fprint(c, ")\r\n") + } + fmt.Fprintf(c, "%s OK fetch done\r\n", tag) + case strings.HasPrefix(upper, "LOGOUT"): + fmt.Fprint(c, "* BYE\r\n") + fmt.Fprintf(c, "%s OK bye\r\n", tag) + return + default: + fmt.Fprintf(c, "%s BAD unknown\r\n", tag) + } + } +} + +// dialFake wires a client Conn to an in-process server over net.Pipe. +func dialFake(t *testing.T, f *fakeIMAP) *Conn { + t.Helper() + cli, srv := net.Pipe() + go f.serve(t, srv) + c, err := NewConn(cli, 5*time.Second) + if err != nil { + t.Fatalf("greeting: %v", err) + } + t.Cleanup(func() { c.Close() }) + return c +} + +func TestIMAPRoundTrip(t *testing.T) { + body := "Subject: hello\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nCall the bank.\r\n" + f := &fakeIMAP{uids: []uint32{4, 9}, msgs: map[uint32]string{4: body, 9: body}} + c := dialFake(t, f) + + if err := c.Login("kami", `pa"ss\word`); err != nil { + t.Fatalf("login: %v", err) + } + if err := c.Select("INBOX"); err != nil { + t.Fatalf("select: %v", err) + } + uids, err := c.SearchSince(time.Date(2026, 8, 1, 0, 0, 0, 0, time.UTC)) + if err != nil { + t.Fatalf("search: %v", err) + } + if len(uids) != 2 || uids[0] != 4 || uids[1] != 9 { + t.Fatalf("uids = %v, want [4 9]", uids) + } + raw, err := c.Fetch(9) + if err != nil { + t.Fatalf("fetch: %v", err) + } + if string(raw) != body { + t.Errorf("fetched %q, want the literal verbatim", raw) + } + c.Logout() + + joined := strings.Join(f.cmds, "\n") + // Read-only at the protocol level, and peeking — Maven must leave no trace + // of having read his mail. + if !strings.Contains(joined, "EXAMINE") || strings.Contains(joined, "SELECT ") { + t.Errorf("want EXAMINE (read-only), got:\n%s", joined) + } + if !strings.Contains(joined, "BODY.PEEK[]") { + t.Errorf("want BODY.PEEK, got:\n%s", joined) + } + // The password must have been quoted and escaped, not truncated at the quote. + if !strings.Contains(joined, `"pa\"ss\\word"`) { + t.Errorf("password not quoted correctly:\n%s", joined) + } + // SINCE must carry the IMAP date form. + if !strings.Contains(joined, "SINCE 1-Aug-2026") { + t.Errorf("want a SINCE date, got:\n%s", joined) + } +} + +func TestIMAPServerNoIsAnError(t *testing.T) { + f := &fakeIMAP{failOn: "LOGIN"} + c := dialFake(t, f) + err := c.Login("kami", "wrong") + if err == nil { + t.Fatal("a NO completion must be an error") + } + // The error is the server's text; it must not echo the credential. + if strings.Contains(err.Error(), "wrong") { + t.Errorf("error leaks the password: %v", err) + } +} + +func TestIMAPFetchMissingUID(t *testing.T) { + f := &fakeIMAP{uids: []uint32{1}, msgs: map[uint32]string{}} + c := dialFake(t, f) + raw, err := c.Fetch(1) + if err != nil { + t.Fatalf("fetch: %v", err) + } + if raw != nil { + t.Errorf("a vanished UID should give nil, got %q", raw) + } +} + +func TestQuoteStripsNewlines(t *testing.T) { + if got := quote("pass\r\nA1 LOGOUT"); strings.ContainsAny(got, "\r\n") { + t.Errorf("quote kept a line break: %q", got) + } +} diff --git a/internal/email/junk.go b/internal/email/junk.go new file mode 100644 index 0000000..dc64167 --- /dev/null +++ b/internal/email/junk.go @@ -0,0 +1,80 @@ +package email + +import ( + "net/mail" + "strings" +) + +// The junk filter — the cheapest and most important half of reading mail. +// +// A mailbox is mostly machine-generated: newsletters, receipts nobody acts on, +// social notifications, marketing. Sending all of it to a 1.7B and asking "is +// there a task here" produces confident nonsense at a rate proportional to the +// volume, so junk is decided by HEADERS, before any model sees the message. +// +// The rules are all bulk-mail markers that senders set on themselves, never +// guesses about content: +// +// - List-Unsubscribe / List-Id — by definition a mailing list. If he can +// unsubscribe from it, it is not asking him to do anything. +// - Precedence: bulk|junk|list — the sender declaring itself bulk. +// - Auto-Submitted other than "no" (RFC 3834) — generated by a machine. +// - X-Spam-Flag: YES, X-Spam-Status: Yes — the spam filter upstream already +// decided; we do not second-guess it in the other direction. +// - X-GM-LABELS / X-Gmail-Labels containing a Gmail category — Gmail's own +// Promotions/Social/Forums/Spam classification, when the server sends it. +// +// Deliberately NOT here: sender allow/deny lists and subject keyword matching. +// Both are configuration that ages badly and both would be a place for his +// contacts to end up in a config file. If a real correspondent's mail is being +// dropped, the fix is a rule about a header, not a list of names. +// +// A junk verdict never deletes anything and never touches a flag on the server. +// It means "do not spend the model on this", nothing more. + +// junkHeaders — headers whose mere presence marks bulk mail. +var junkPresence = []string{"List-Unsubscribe", "List-Id", "List-Post"} + +// gmailCategories — Gmail's category labels, lowercased as they appear in +// X-GM-LABELS. "important" and "inbox" are labels too, and are NOT categories. +// Matching is by these exact tokens (substring is fine — they are namespaced +// and cannot appear in a hand-made label by accident), so a user label named +// "Social Club" is not mistaken for Gmail's Social category. +var gmailCategories = []string{ + "category_promotions", "category_social", "category_forums", "category_updates", + `\spam`, `\junk`, +} + +// classifyJunk returns whether the message is bulk/automated and why. The +// reason is a short header name, safe to log — it names the marker, never the +// sender or the subject. +func classifyJunk(h mail.Header) (bool, string) { + for _, name := range junkPresence { + if strings.TrimSpace(h.Get(name)) != "" { + return true, strings.ToLower(name) + } + } + switch strings.ToLower(strings.TrimSpace(h.Get("Precedence"))) { + case "bulk", "junk", "list": + return true, "precedence" + } + if v := strings.ToLower(strings.TrimSpace(h.Get("Auto-Submitted"))); v != "" && v != "no" { + return true, "auto-submitted" + } + if strings.EqualFold(strings.TrimSpace(h.Get("X-Spam-Flag")), "yes") { + return true, "x-spam-flag" + } + if v := strings.ToLower(strings.TrimSpace(h.Get("X-Spam-Status"))); strings.HasPrefix(v, "yes") { + return true, "x-spam-status" + } + labels := strings.ToLower(h.Get("X-GM-LABELS") + " " + h.Get("X-Gmail-Labels")) + for _, c := range gmailCategories { + if c == "" { + continue + } + if strings.Contains(labels, c) { + return true, "gmail-category" + } + } + return false, "" +} diff --git a/internal/email/junk_test.go b/internal/email/junk_test.go new file mode 100644 index 0000000..d8e8722 --- /dev/null +++ b/internal/email/junk_test.go @@ -0,0 +1,59 @@ +package email + +import ( + "net/mail" + "strings" + "testing" +) + +func headers(t *testing.T, raw string) mail.Header { + t.Helper() + m, err := mail.ReadMessage(strings.NewReader(strings.ReplaceAll(raw, "\n", "\r\n") + "\r\n\r\nbody\r\n")) + if err != nil { + t.Fatalf("read headers: %v", err) + } + return m.Header +} + +func TestClassifyJunk(t *testing.T) { + cases := []struct { + name string + raw string + junk bool + reason string + }{ + {"personal", "From: a@b.c\nSubject: привет", false, ""}, + {"list-unsubscribe", "From: a@b.c\nList-Unsubscribe: ", true, "list-unsubscribe"}, + {"list-id", "From: a@b.c\nList-Id: ", true, "list-id"}, + {"precedence bulk", "From: a@b.c\nPrecedence: bulk", true, "precedence"}, + {"auto-submitted", "From: a@b.c\nAuto-Submitted: auto-generated", true, "auto-submitted"}, + {"auto-submitted no", "From: a@b.c\nAuto-Submitted: no", false, ""}, + {"spam flag", "From: a@b.c\nX-Spam-Flag: YES", true, "x-spam-flag"}, + {"spam status", "From: a@b.c\nX-Spam-Status: Yes, score=9.1", true, "x-spam-status"}, + {"spam status no", "From: a@b.c\nX-Spam-Status: No, score=0.1", false, ""}, + {"gmail promo", "From: a@b.c\nX-Gmail-Labels: Inbox,CATEGORY_PROMOTIONS", true, "gmail-category"}, + {"user label", "From: a@b.c\nX-Gmail-Labels: Social Club,Important", false, ""}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + junk, reason := classifyJunk(headers(t, c.raw)) + if junk != c.junk || reason != c.reason { + t.Errorf("classifyJunk = (%v, %q), want (%v, %q)", junk, reason, c.junk, c.reason) + } + }) + } +} + +func TestNewsletterFixtureIsJunk(t *testing.T) { + msg, err := ParseMessage(9, fixture(t, "newsletter.eml")) + if err != nil { + t.Fatalf("parse: %v", err) + } + if !msg.Junk { + t.Fatal("a newsletter with List-Unsubscribe + Precedence: bulk must be junk") + } + // The reason is what gets logged, so it must never carry mail content. + if strings.Contains(msg.JunkReason, "@") || strings.Contains(msg.JunkReason, "Скидки") { + t.Errorf("junk reason leaks content: %q", msg.JunkReason) + } +} diff --git a/internal/email/message.go b/internal/email/message.go new file mode 100644 index 0000000..883be39 --- /dev/null +++ b/internal/email/message.go @@ -0,0 +1,258 @@ +// Package email is the reading half of the email reader (Vikunja #246, +// docs/plans/01-email-reader.md): a small IMAP client, a MIME-to-plaintext +// converter, and the junk filter that decides a message is not worth reading at +// all. Extraction lives in extract.go and writes nothing itself. +// +// Two constraints shape everything here, both from CLAUDE.md: +// +// - Mail is personal. Nothing in this package logs a body, a subject, or an +// address; callers get the text and decide. Mail text is never search input +// — no function here reaches the network except the IMAP connection itself. +// - Off unless configured. There is no default host, no default account, and +// no fallback that would make a mailbox get read because a field was empty. +// +// The IMAP subset is deliberately tiny (LOGIN, SELECT, UID SEARCH, UID FETCH +// with BODY.PEEK, LOGOUT). No IDLE: a poll every few minutes is what a task +// candidate needs, and IDLE would mean holding a connection and a credential +// open forever for latency nobody is waiting on. +package email + +import ( + "encoding/base64" + "fmt" + "io" + "mime" + "mime/multipart" + "mime/quotedprintable" + "net/mail" + "regexp" + "strings" +) + +// MaxBodyBytes — how much of one message body is kept. A task hides in the +// first screenful; the rest is signature, quoted history and legal boilerplate, +// and it would only spend the resident model's 4096-token context. +const MaxBodyBytes = 4000 + +// Message — one mail, reduced to the fields extraction and review need. +// +// Raw is deliberately absent: once a message is parsed the original bytes are +// dropped, so no caller can accidentally log or forward the whole mail. +type Message struct { + UID uint32 + From string + Subject string + Date string // as sent, unparsed — display only + Body string // plaintext, decoded, HTML-stripped, truncated + // Junk is set by the junk filter (see junk.go). A junk message is carried + // rather than dropped so the poller can count it and still mark it seen. + Junk bool + JunkReason string +} + +// ParseMessage turns one RFC 5322 message into a Message. +// +// It never fails on a body it cannot understand: an unparsable or +// unsupported-charset body yields an empty Body and the headers still come +// through, because a subject line alone is often the whole task ("Счёт за +// интернет"). Only a message whose headers cannot be read at all is an error. +func ParseMessage(uid uint32, raw []byte) (Message, error) { + m, err := mail.ReadMessage(strings.NewReader(string(raw))) + if err != nil { + return Message{}, fmt.Errorf("email: parse message: %w", err) + } + msg := Message{ + UID: uid, + From: decodeHeader(m.Header.Get("From")), + Subject: decodeHeader(m.Header.Get("Subject")), + Date: m.Header.Get("Date"), + } + msg.Junk, msg.JunkReason = classifyJunk(m.Header) + body, err := plaintextBody(m.Header.Get("Content-Type"), m.Header.Get("Content-Transfer-Encoding"), m.Body) + if err == nil { + msg.Body = truncate(collapse(body), MaxBodyBytes) + } + return msg, nil +} + +// plaintextBody walks the MIME tree and returns the best plaintext it can. +// +// Preference order inside a multipart: text/plain first, text/html stripped +// only when there is no plain part. multipart/mixed attachments are skipped +// wholesale — an attachment is a file, not a sentence, and reading one would +// mean parsing arbitrary formats from the network. +func plaintextBody(contentType, encoding string, body io.Reader) (string, error) { + mediaType, params, err := mime.ParseMediaType(contentType) + if contentType == "" || err != nil { + // No Content-Type at all is legal and means text/plain; a broken one is + // treated the same rather than dropping the message. + mediaType, params = "text/plain", nil + } + switch { + case strings.HasPrefix(mediaType, "multipart/"): + boundary := params["boundary"] + if boundary == "" { + return "", fmt.Errorf("email: multipart without boundary") + } + return multipartText(multipart.NewReader(body, boundary)) + case mediaType == "text/html": + raw, err := decodeBody(body, encoding, params["charset"]) + if err != nil { + return "", err + } + return stripHTML(raw), nil + case mediaType == "text/plain": + return decodeBody(body, encoding, params["charset"]) + default: + // A single-part non-text message (a bare PDF, say). No body, headers only. + return "", nil + } +} + +// multipartText reads one multipart level, recursing into nested multiparts. +// Returns the plain part if any part yielded one, else the stripped HTML. +func multipartText(mr *multipart.Reader) (string, error) { + var plain, html string + for { + part, err := mr.NextPart() + if err == io.EOF { + break + } + if err != nil { + // A truncated multipart still gives up whatever came before it. + break + } + if part.FileName() != "" { + part.Close() + continue // attachment + } + ct := part.Header.Get("Content-Type") + mediaType, _, _ := mime.ParseMediaType(ct) + text, err := plaintextBody(ct, part.Header.Get("Content-Transfer-Encoding"), part) + part.Close() + if err != nil || strings.TrimSpace(text) == "" { + continue + } + if mediaType == "text/html" && !strings.HasPrefix(mediaType, "multipart/") { + if html == "" { + html = text + } + continue + } + if plain == "" { + plain = text + } + } + if strings.TrimSpace(plain) != "" { + return plain, nil + } + return html, nil +} + +// decodeBody applies the transfer encoding, then the charset. +// +// Charset support is UTF-8 (and ASCII, its subset) only, on purpose: x/text's +// encoding tables are not vendored here, and guessing at windows-1251 bytes +// would feed the model mojibake it would happily extract a task from. An +// unsupported charset returns an error, which ParseMessage turns into an empty +// body — subject-only, which is honest. +func decodeBody(r io.Reader, encoding, charset string) (string, error) { + switch strings.ToLower(strings.TrimSpace(encoding)) { + case "quoted-printable": + r = quotedprintable.NewReader(r) + case "base64": + r = newBase64Reader(r) + } + b, err := io.ReadAll(io.LimitReader(r, 1<<20)) + if err != nil && len(b) == 0 { + return "", fmt.Errorf("email: read body: %w", err) + } + switch cs := strings.ToLower(strings.TrimSpace(charset)); cs { + case "", "utf-8", "utf8", "us-ascii", "ascii": + return string(b), nil + default: + return "", fmt.Errorf("email: unsupported charset %q", cs) + } +} + +// decodeHeader decodes RFC 2047 encoded words ("=?utf-8?B?...?="), which is how +// every Russian subject line arrives. Undecodable headers come back as-is +// rather than empty: a mangled subject is still a hint, and it is only ever +// shown to him as evidence. +func decodeHeader(v string) string { + dec := new(mime.WordDecoder) + out, err := dec.DecodeHeader(v) + if err != nil { + return collapse(v) + } + return collapse(out) +} + +var ( + scriptStyle = regexp.MustCompile(`(?is)<(script|style)\b[^>]*>.*?`) + htmlBreak = regexp.MustCompile(`(?i)<\s*(br\s*/?|/p|/div|/tr|/li|/h[1-6])\s*>`) + htmlTag = regexp.MustCompile(`(?s)<[^>]*>`) + htmlComment = regexp.MustCompile(`(?s)`) +) + +// stripHTML reduces an HTML part to text. A regex stripper, not a parser: +// x/net/html is not vendored, and the consumer is a model reading prose — a +// stray angle bracket costs nothing, whereas a new dependency for the privacy- +// sensitive path costs review. +func stripHTML(s string) string { + s = scriptStyle.ReplaceAllString(s, " ") + s = htmlComment.ReplaceAllString(s, " ") + s = htmlBreak.ReplaceAllString(s, "\n") + s = htmlTag.ReplaceAllString(s, " ") + return unescapeEntities(s) +} + +var entities = strings.NewReplacer( + " ", " ", "&", "&", "<", "<", ">", ">", + """, `"`, "'", "'", "'", "'", "—", "—", "–", "–", +) + +func unescapeEntities(s string) string { return entities.Replace(s) } + +// collapse squeezes runs of whitespace, keeping single newlines. Mail bodies +// arrive with hard-wrapped lines and blocks of blank space; the model does not +// need them and they are pure context budget. +func collapse(s string) string { + lines := strings.Split(strings.ReplaceAll(s, "\r\n", "\n"), "\n") + var out []string + blank := 0 + for _, l := range lines { + l = strings.TrimSpace(strings.Join(strings.Fields(l), " ")) + if l == "" { + blank++ + if blank > 1 { + continue + } + out = append(out, "") + continue + } + blank = 0 + out = append(out, l) + } + return strings.TrimSpace(strings.Join(out, "\n")) +} + +// truncate cuts to n bytes on a rune boundary. +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + cut := s[:n] + for len(cut) > 0 && !isRuneStart(cut[len(cut)-1]) { + cut = cut[:len(cut)-1] + } + return strings.TrimSpace(cut) + "…" +} + +func isRuneStart(b byte) bool { return b&0xC0 != 0x80 } + +// newBase64Reader — base64.NewDecoder already skips the CRLFs mail bodies wrap +// with, so this is only a named seam for decodeBody to read cleanly. +func newBase64Reader(r io.Reader) io.Reader { + return base64.NewDecoder(base64.StdEncoding, r) +} diff --git a/internal/email/message_test.go b/internal/email/message_test.go new file mode 100644 index 0000000..0ef9028 --- /dev/null +++ b/internal/email/message_test.go @@ -0,0 +1,111 @@ +package email + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func fixture(t *testing.T, name string) []byte { + t.Helper() + b, err := os.ReadFile(filepath.Join("testdata", name)) + if err != nil { + t.Fatalf("read fixture %s: %v", name, err) + } + return b +} + +func TestParsePlainRussian(t *testing.T) { + msg, err := ParseMessage(7, fixture(t, "plain_ru.eml")) + if err != nil { + t.Fatalf("parse: %v", err) + } + if msg.UID != 7 { + t.Errorf("uid = %d, want 7", msg.UID) + } + if want := "Нужно закрыть задачу"; msg.Subject != want { + t.Errorf("subject = %q, want %q", msg.Subject, want) + } + if !strings.Contains(msg.From, "Антон") { + t.Errorf("from = %q, want the decoded display name", msg.From) + } + if !strings.Contains(msg.Body, "Надо отправить акт до пятницы.") { + t.Errorf("body = %q, want the quoted-printable text decoded", msg.Body) + } + if msg.Junk { + t.Errorf("a personal mail must not be junk (%s)", msg.JunkReason) + } +} + +func TestParseHTMLOnlyIsStripped(t *testing.T) { + msg, err := ParseMessage(1, fixture(t, "html_only.eml")) + if err != nil { + t.Fatalf("parse: %v", err) + } + if strings.Contains(msg.Body, "<") || strings.Contains(msg.Body, "color:red") || strings.Contains(msg.Body, "x()") { + t.Errorf("body still has markup/script/style: %q", msg.Body) + } + for _, want := range []string{"Счёт за интернет: 700", "Оплатить до 5 августа."} { + if !strings.Contains(msg.Body, want) { + t.Errorf("body = %q, want it to contain %q", msg.Body, want) + } + } + //   must have become a real space, not vanished into the number. + if strings.Contains(msg.Body, " ") { + t.Errorf("entity left unescaped: %q", msg.Body) + } +} + +func TestParsePrefersPlainAndSkipsAttachments(t *testing.T) { + msg, err := ParseMessage(2, fixture(t, "mixed_attachment.eml")) + if err != nil { + t.Fatalf("parse: %v", err) + } + if got := strings.TrimSpace(msg.Body); got != "Sign the contract before Monday." { + t.Errorf("body = %q, want the text/plain alternative only", got) + } + if strings.Contains(msg.Body, "PDF") { + t.Errorf("attachment bytes leaked into the body: %q", msg.Body) + } +} + +// An unsupported charset must degrade to headers-only rather than to mojibake +// the model would then extract a task from. +func TestParseUnsupportedCharsetKeepsHeaders(t *testing.T) { + msg, err := ParseMessage(3, fixture(t, "cp1251.eml")) + if err != nil { + t.Fatalf("parse: %v", err) + } + if msg.Subject != "Legacy" { + t.Errorf("subject = %q, want Legacy", msg.Subject) + } + if msg.Body != "" { + t.Errorf("body = %q, want empty for an undecodable charset", msg.Body) + } +} + +func TestParseTruncatesLongBody(t *testing.T) { + var b strings.Builder + b.WriteString("Subject: long\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n") + for i := 0; i < 2000; i++ { + b.WriteString("длинная строка ") + } + msg, err := ParseMessage(4, []byte(b.String())) + if err != nil { + t.Fatalf("parse: %v", err) + } + if len(msg.Body) > MaxBodyBytes+8 { + t.Errorf("body kept %d bytes, want ≤ %d", len(msg.Body), MaxBodyBytes) + } + if !strings.HasSuffix(msg.Body, "…") { + t.Errorf("truncated body should be marked: %q", msg.Body[len(msg.Body)-20:]) + } +} + +func TestCollapseSqueezesBlankLines(t *testing.T) { + got := collapse(" a b \r\n\r\n\r\n\r\n c \r\n") + if got != "a b\n\nc" { + t.Errorf("collapse = %q, want %q", got, "a b\n\nc") + } +} diff --git a/internal/email/testdata/cp1251.eml b/internal/email/testdata/cp1251.eml new file mode 100644 index 0000000..ca1a016 --- /dev/null +++ b/internal/email/testdata/cp1251.eml @@ -0,0 +1,7 @@ +From: legacy@example.org +To: kami@example.org +Subject: Legacy +Date: Fri, 01 Aug 2026 05:00:00 +0400 +Content-Type: text/plain; charset="windows-1251" + +Ï diff --git a/internal/email/testdata/html_only.eml b/internal/email/testdata/html_only.eml new file mode 100644 index 0000000..deff075 --- /dev/null +++ b/internal/email/testdata/html_only.eml @@ -0,0 +1,13 @@ +From: billing@isp.example +To: kami@example.org +Subject: =?utf-8?B?0KHRh9GR0YIg0LfQsCDQuNC90YLQtdGA0L3QtdGC?= +Date: Fri, 01 Aug 2026 08:00:00 +0400 +MIME-Version: 1.0 +Content-Type: multipart/alternative; boundary="B1" + +--B1 +Content-Type: text/html; charset="utf-8" +Content-Transfer-Encoding: base64 + +PGh0bWw+PGhlYWQ+PHN0eWxlPnB7Y29sb3I6cmVkfTwvc3R5bGU+PC9oZWFkPjxib2R5PjxwPtCh0YfRkdGCINC30LAg0LjQvdGC0LXRgNC90LXRgjogNzAwJm5ic3A74oK9PC9wPjxwPtCe0L/Qu9Cw0YLQuNGC0Ywg0LTQviA1INCw0LLQs9GD0YHRgtCwLjwvcD48c2NyaXB0PngoKTwvc2NyaXB0PjwvYm9keT48L2h0bWw+ +--B1-- diff --git a/internal/email/testdata/mixed_attachment.eml b/internal/email/testdata/mixed_attachment.eml new file mode 100644 index 0000000..a93c884 --- /dev/null +++ b/internal/email/testdata/mixed_attachment.eml @@ -0,0 +1,26 @@ +From: hr@work.example +To: kami@example.org +Subject: Contract +Date: Fri, 01 Aug 2026 07:00:00 +0400 +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="M1" + +--M1 +Content-Type: multipart/alternative; boundary="A1" + +--A1 +Content-Type: text/plain; charset="utf-8" + +Sign the contract before Monday. +--A1 +Content-Type: text/html; charset="utf-8" + +

Sign the contract before Monday.

+--A1-- +--M1 +Content-Type: application/pdf; name="contract.pdf" +Content-Disposition: attachment; filename="contract.pdf" +Content-Transfer-Encoding: base64 + +JVBERi0xLjQgbm90IHJlYWxseSBhIHBkZg== +--M1-- diff --git a/internal/email/testdata/newsletter.eml b/internal/email/testdata/newsletter.eml new file mode 100644 index 0000000..09c3d31 --- /dev/null +++ b/internal/email/testdata/newsletter.eml @@ -0,0 +1,9 @@ +From: news@shop.example +To: kami@example.org +Subject: =?utf-8?B?0KHQutC40LTQutC4INGC0L7Qu9GM0LrQviDRgdC10LPQvtC00L3Rjw==?= +Date: Fri, 01 Aug 2026 06:00:00 +0400 +List-Unsubscribe: +Precedence: bulk +Content-Type: text/plain; charset="utf-8" + +Sale! diff --git a/internal/email/testdata/plain_ru.eml b/internal/email/testdata/plain_ru.eml new file mode 100644 index 0000000..bd8c745 --- /dev/null +++ b/internal/email/testdata/plain_ru.eml @@ -0,0 +1,14 @@ +From: =?utf-8?B?0JDQvdGC0L7QvQ==?= +To: kami@example.org +Subject: =?utf-8?B?0J3Rg9C20L3QviDQt9Cw0LrRgNGL0YLRjCDQt9Cw0LTQsNGH0YM=?= +Date: Fri, 01 Aug 2026 09:12:00 +0400 +Content-Type: text/plain; charset="utf-8" +Content-Transfer-Encoding: quoted-printable +Message-ID: + +=D0=9F=D1=80=D0=B8=D0=B2=D0=B5=D1=82! =D0=9D=D0=B0=D0=B4=D0=BE =D0=BE=D1=82= +=D0=BF=D1=80=D0=B0=D0=B2=D0=B8=D1=82=D1=8C =D0=B0=D0=BA=D1=82 =D0=B4=D0=BE = +=D0=BF=D1=8F=D1=82=D0=BD=D0=B8=D1=86=D1=8B. + +-- +Anton