Read a mailbox read-only, in a client small enough to audit (#246)
internal/email is the reading half of the email reader: a ~200-line IMAP client (LOGIN, EXAMINE, UID SEARCH SINCE, UID FETCH BODY.PEEK, LOGOUT), a MIME-to-plaintext converter, and a header-only junk filter. Two protocol choices are the design, not shortcuts. EXAMINE instead of SELECT means the session is read-only at the protocol level, so no command in it can flip a flag or expunge anything by mistake. BODY.PEEK instead of BODY means reading a message does not mark it \Seen — Maven reads his mail and leaves no trace of having done so, and the unread state in his own client stays his. Hand-rolled rather than go-imap because this is the one path that holds his mailbox credential and reads his private mail: five commands with no dependencies is auditable in a sitting. No IDLE and no cleartext/STARTTLS either — an option to send his password over a plain socket is an option to get it wrong once. Junk is decided by headers alone, before any model is involved: List-Unsubscribe/List-Id, Precedence: bulk, Auto-Submitted, the spam headers, and Gmail's own category labels. Sender lists and subject keywords are deliberately absent — they age badly and they would put his contacts in a config file. A junk verdict only means "do not spend the model on this"; nothing is deleted and no server flag is touched. Nothing here logs a body, a subject or an address, the junk reason names a header rather than content, and an undecodable charset degrades to headers-only instead of feeding the model mojibake. Verified against recorded .eml fixtures and an in-process fake IMAP server.
This commit is contained in:
kami
@@ -0,0 +1,258 @@
|
||||
// Package email is the reading half of the email reader (Vikunja #246,
|
||||
// docs/plans/01-email-reader.md): a small IMAP client, a MIME-to-plaintext
|
||||
// converter, and the junk filter that decides a message is not worth reading at
|
||||
// all. Extraction lives in extract.go and writes nothing itself.
|
||||
//
|
||||
// Two constraints shape everything here, both from CLAUDE.md:
|
||||
//
|
||||
// - Mail is personal. Nothing in this package logs a body, a subject, or an
|
||||
// address; callers get the text and decide. Mail text is never search input
|
||||
// — no function here reaches the network except the IMAP connection itself.
|
||||
// - Off unless configured. There is no default host, no default account, and
|
||||
// no fallback that would make a mailbox get read because a field was empty.
|
||||
//
|
||||
// The IMAP subset is deliberately tiny (LOGIN, SELECT, UID SEARCH, UID FETCH
|
||||
// with BODY.PEEK, LOGOUT). No IDLE: a poll every few minutes is what a task
|
||||
// candidate needs, and IDLE would mean holding a connection and a credential
|
||||
// open forever for latency nobody is waiting on.
|
||||
package email
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"mime/multipart"
|
||||
"mime/quotedprintable"
|
||||
"net/mail"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// MaxBodyBytes — how much of one message body is kept. A task hides in the
|
||||
// first screenful; the rest is signature, quoted history and legal boilerplate,
|
||||
// and it would only spend the resident model's 4096-token context.
|
||||
const MaxBodyBytes = 4000
|
||||
|
||||
// Message — one mail, reduced to the fields extraction and review need.
|
||||
//
|
||||
// Raw is deliberately absent: once a message is parsed the original bytes are
|
||||
// dropped, so no caller can accidentally log or forward the whole mail.
|
||||
type Message struct {
|
||||
UID uint32
|
||||
From string
|
||||
Subject string
|
||||
Date string // as sent, unparsed — display only
|
||||
Body string // plaintext, decoded, HTML-stripped, truncated
|
||||
// Junk is set by the junk filter (see junk.go). A junk message is carried
|
||||
// rather than dropped so the poller can count it and still mark it seen.
|
||||
Junk bool
|
||||
JunkReason string
|
||||
}
|
||||
|
||||
// ParseMessage turns one RFC 5322 message into a Message.
|
||||
//
|
||||
// It never fails on a body it cannot understand: an unparsable or
|
||||
// unsupported-charset body yields an empty Body and the headers still come
|
||||
// through, because a subject line alone is often the whole task ("Счёт за
|
||||
// интернет"). Only a message whose headers cannot be read at all is an error.
|
||||
func ParseMessage(uid uint32, raw []byte) (Message, error) {
|
||||
m, err := mail.ReadMessage(strings.NewReader(string(raw)))
|
||||
if err != nil {
|
||||
return Message{}, fmt.Errorf("email: parse message: %w", err)
|
||||
}
|
||||
msg := Message{
|
||||
UID: uid,
|
||||
From: decodeHeader(m.Header.Get("From")),
|
||||
Subject: decodeHeader(m.Header.Get("Subject")),
|
||||
Date: m.Header.Get("Date"),
|
||||
}
|
||||
msg.Junk, msg.JunkReason = classifyJunk(m.Header)
|
||||
body, err := plaintextBody(m.Header.Get("Content-Type"), m.Header.Get("Content-Transfer-Encoding"), m.Body)
|
||||
if err == nil {
|
||||
msg.Body = truncate(collapse(body), MaxBodyBytes)
|
||||
}
|
||||
return msg, nil
|
||||
}
|
||||
|
||||
// plaintextBody walks the MIME tree and returns the best plaintext it can.
|
||||
//
|
||||
// Preference order inside a multipart: text/plain first, text/html stripped
|
||||
// only when there is no plain part. multipart/mixed attachments are skipped
|
||||
// wholesale — an attachment is a file, not a sentence, and reading one would
|
||||
// mean parsing arbitrary formats from the network.
|
||||
func plaintextBody(contentType, encoding string, body io.Reader) (string, error) {
|
||||
mediaType, params, err := mime.ParseMediaType(contentType)
|
||||
if contentType == "" || err != nil {
|
||||
// No Content-Type at all is legal and means text/plain; a broken one is
|
||||
// treated the same rather than dropping the message.
|
||||
mediaType, params = "text/plain", nil
|
||||
}
|
||||
switch {
|
||||
case strings.HasPrefix(mediaType, "multipart/"):
|
||||
boundary := params["boundary"]
|
||||
if boundary == "" {
|
||||
return "", fmt.Errorf("email: multipart without boundary")
|
||||
}
|
||||
return multipartText(multipart.NewReader(body, boundary))
|
||||
case mediaType == "text/html":
|
||||
raw, err := decodeBody(body, encoding, params["charset"])
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return stripHTML(raw), nil
|
||||
case mediaType == "text/plain":
|
||||
return decodeBody(body, encoding, params["charset"])
|
||||
default:
|
||||
// A single-part non-text message (a bare PDF, say). No body, headers only.
|
||||
return "", nil
|
||||
}
|
||||
}
|
||||
|
||||
// multipartText reads one multipart level, recursing into nested multiparts.
|
||||
// Returns the plain part if any part yielded one, else the stripped HTML.
|
||||
func multipartText(mr *multipart.Reader) (string, error) {
|
||||
var plain, html string
|
||||
for {
|
||||
part, err := mr.NextPart()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
// A truncated multipart still gives up whatever came before it.
|
||||
break
|
||||
}
|
||||
if part.FileName() != "" {
|
||||
part.Close()
|
||||
continue // attachment
|
||||
}
|
||||
ct := part.Header.Get("Content-Type")
|
||||
mediaType, _, _ := mime.ParseMediaType(ct)
|
||||
text, err := plaintextBody(ct, part.Header.Get("Content-Transfer-Encoding"), part)
|
||||
part.Close()
|
||||
if err != nil || strings.TrimSpace(text) == "" {
|
||||
continue
|
||||
}
|
||||
if mediaType == "text/html" && !strings.HasPrefix(mediaType, "multipart/") {
|
||||
if html == "" {
|
||||
html = text
|
||||
}
|
||||
continue
|
||||
}
|
||||
if plain == "" {
|
||||
plain = text
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(plain) != "" {
|
||||
return plain, nil
|
||||
}
|
||||
return html, nil
|
||||
}
|
||||
|
||||
// decodeBody applies the transfer encoding, then the charset.
|
||||
//
|
||||
// Charset support is UTF-8 (and ASCII, its subset) only, on purpose: x/text's
|
||||
// encoding tables are not vendored here, and guessing at windows-1251 bytes
|
||||
// would feed the model mojibake it would happily extract a task from. An
|
||||
// unsupported charset returns an error, which ParseMessage turns into an empty
|
||||
// body — subject-only, which is honest.
|
||||
func decodeBody(r io.Reader, encoding, charset string) (string, error) {
|
||||
switch strings.ToLower(strings.TrimSpace(encoding)) {
|
||||
case "quoted-printable":
|
||||
r = quotedprintable.NewReader(r)
|
||||
case "base64":
|
||||
r = newBase64Reader(r)
|
||||
}
|
||||
b, err := io.ReadAll(io.LimitReader(r, 1<<20))
|
||||
if err != nil && len(b) == 0 {
|
||||
return "", fmt.Errorf("email: read body: %w", err)
|
||||
}
|
||||
switch cs := strings.ToLower(strings.TrimSpace(charset)); cs {
|
||||
case "", "utf-8", "utf8", "us-ascii", "ascii":
|
||||
return string(b), nil
|
||||
default:
|
||||
return "", fmt.Errorf("email: unsupported charset %q", cs)
|
||||
}
|
||||
}
|
||||
|
||||
// decodeHeader decodes RFC 2047 encoded words ("=?utf-8?B?...?="), which is how
|
||||
// every Russian subject line arrives. Undecodable headers come back as-is
|
||||
// rather than empty: a mangled subject is still a hint, and it is only ever
|
||||
// shown to him as evidence.
|
||||
func decodeHeader(v string) string {
|
||||
dec := new(mime.WordDecoder)
|
||||
out, err := dec.DecodeHeader(v)
|
||||
if err != nil {
|
||||
return collapse(v)
|
||||
}
|
||||
return collapse(out)
|
||||
}
|
||||
|
||||
var (
|
||||
scriptStyle = regexp.MustCompile(`(?is)<(script|style)\b[^>]*>.*?</\s*(script|style)\s*>`)
|
||||
htmlBreak = regexp.MustCompile(`(?i)<\s*(br\s*/?|/p|/div|/tr|/li|/h[1-6])\s*>`)
|
||||
htmlTag = regexp.MustCompile(`(?s)<[^>]*>`)
|
||||
htmlComment = regexp.MustCompile(`(?s)<!--.*?-->`)
|
||||
)
|
||||
|
||||
// stripHTML reduces an HTML part to text. A regex stripper, not a parser:
|
||||
// x/net/html is not vendored, and the consumer is a model reading prose — a
|
||||
// stray angle bracket costs nothing, whereas a new dependency for the privacy-
|
||||
// sensitive path costs review.
|
||||
func stripHTML(s string) string {
|
||||
s = scriptStyle.ReplaceAllString(s, " ")
|
||||
s = htmlComment.ReplaceAllString(s, " ")
|
||||
s = htmlBreak.ReplaceAllString(s, "\n")
|
||||
s = htmlTag.ReplaceAllString(s, " ")
|
||||
return unescapeEntities(s)
|
||||
}
|
||||
|
||||
var entities = strings.NewReplacer(
|
||||
" ", " ", "&", "&", "<", "<", ">", ">",
|
||||
""", `"`, "'", "'", "'", "'", "—", "—", "–", "–",
|
||||
)
|
||||
|
||||
func unescapeEntities(s string) string { return entities.Replace(s) }
|
||||
|
||||
// collapse squeezes runs of whitespace, keeping single newlines. Mail bodies
|
||||
// arrive with hard-wrapped lines and blocks of blank space; the model does not
|
||||
// need them and they are pure context budget.
|
||||
func collapse(s string) string {
|
||||
lines := strings.Split(strings.ReplaceAll(s, "\r\n", "\n"), "\n")
|
||||
var out []string
|
||||
blank := 0
|
||||
for _, l := range lines {
|
||||
l = strings.TrimSpace(strings.Join(strings.Fields(l), " "))
|
||||
if l == "" {
|
||||
blank++
|
||||
if blank > 1 {
|
||||
continue
|
||||
}
|
||||
out = append(out, "")
|
||||
continue
|
||||
}
|
||||
blank = 0
|
||||
out = append(out, l)
|
||||
}
|
||||
return strings.TrimSpace(strings.Join(out, "\n"))
|
||||
}
|
||||
|
||||
// truncate cuts to n bytes on a rune boundary.
|
||||
func truncate(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
cut := s[:n]
|
||||
for len(cut) > 0 && !isRuneStart(cut[len(cut)-1]) {
|
||||
cut = cut[:len(cut)-1]
|
||||
}
|
||||
return strings.TrimSpace(cut) + "…"
|
||||
}
|
||||
|
||||
func isRuneStart(b byte) bool { return b&0xC0 != 0x80 }
|
||||
|
||||
// newBase64Reader — base64.NewDecoder already skips the CRLFs mail bodies wrap
|
||||
// with, so this is only a named seam for decodeBody to read cleanly.
|
||||
func newBase64Reader(r io.Reader) io.Reader {
|
||||
return base64.NewDecoder(base64.StdEncoding, r)
|
||||
}
|
||||
Reference in New Issue
Block a user