// Package email is the reading half of the email reader (Vikunja #246, // docs/plans/01-email-reader.md): a small IMAP client, a MIME-to-plaintext // converter, and the junk filter that decides a message is not worth reading at // all. Extraction lives in extract.go and writes nothing itself. // // Two constraints shape everything here, both from CLAUDE.md: // // - Mail is personal. Nothing in this package logs a body, a subject, or an // address; callers get the text and decide. Mail text is never search input // — no function here reaches the network except the IMAP connection itself. // - Off unless configured. There is no default host, no default account, and // no fallback that would make a mailbox get read because a field was empty. // // The IMAP subset is deliberately tiny (LOGIN, SELECT, UID SEARCH, UID FETCH // with BODY.PEEK, LOGOUT). No IDLE: a poll every few minutes is what a task // candidate needs, and IDLE would mean holding a connection and a credential // open forever for latency nobody is waiting on. package email import ( "encoding/base64" "fmt" "io" "mime" "mime/multipart" "mime/quotedprintable" "net/mail" "regexp" "strings" ) // MaxBodyBytes — how much of one message body is kept. A task hides in the // first screenful; the rest is signature, quoted history and legal boilerplate, // and it would only spend the resident model's 4096-token context. const MaxBodyBytes = 4000 // Message — one mail, reduced to the fields extraction and review need. // // Raw is deliberately absent: once a message is parsed the original bytes are // dropped, so no caller can accidentally log or forward the whole mail. type Message struct { UID uint32 From string Subject string Date string // as sent, unparsed — display only Body string // plaintext, decoded, HTML-stripped, truncated // Junk is set by the junk filter (see junk.go). A junk message is carried // rather than dropped so the poller can count it and still mark it seen. Junk bool JunkReason string } // ParseMessage turns one RFC 5322 message into a Message. // // It never fails on a body it cannot understand: an unparsable or // unsupported-charset body yields an empty Body and the headers still come // through, because a subject line alone is often the whole task ("Счёт за // интернет"). Only a message whose headers cannot be read at all is an error. func ParseMessage(uid uint32, raw []byte) (Message, error) { m, err := mail.ReadMessage(strings.NewReader(string(raw))) if err != nil { return Message{}, fmt.Errorf("email: parse message: %w", err) } msg := Message{ UID: uid, From: decodeHeader(m.Header.Get("From")), Subject: decodeHeader(m.Header.Get("Subject")), Date: m.Header.Get("Date"), } msg.Junk, msg.JunkReason = classifyJunk(m.Header) body, err := plaintextBody(m.Header.Get("Content-Type"), m.Header.Get("Content-Transfer-Encoding"), m.Body) if err == nil { msg.Body = truncate(collapse(body), MaxBodyBytes) } return msg, nil } // plaintextBody walks the MIME tree and returns the best plaintext it can. // // Preference order inside a multipart: text/plain first, text/html stripped // only when there is no plain part. multipart/mixed attachments are skipped // wholesale — an attachment is a file, not a sentence, and reading one would // mean parsing arbitrary formats from the network. func plaintextBody(contentType, encoding string, body io.Reader) (string, error) { mediaType, params, err := mime.ParseMediaType(contentType) if contentType == "" || err != nil { // No Content-Type at all is legal and means text/plain; a broken one is // treated the same rather than dropping the message. mediaType, params = "text/plain", nil } switch { case strings.HasPrefix(mediaType, "multipart/"): boundary := params["boundary"] if boundary == "" { return "", fmt.Errorf("email: multipart without boundary") } return multipartText(multipart.NewReader(body, boundary)) case mediaType == "text/html": raw, err := decodeBody(body, encoding, params["charset"]) if err != nil { return "", err } return stripHTML(raw), nil case mediaType == "text/plain": return decodeBody(body, encoding, params["charset"]) default: // A single-part non-text message (a bare PDF, say). No body, headers only. return "", nil } } // multipartText reads one multipart level, recursing into nested multiparts. // Returns the plain part if any part yielded one, else the stripped HTML. func multipartText(mr *multipart.Reader) (string, error) { var plain, html string for { part, err := mr.NextPart() if err == io.EOF { break } if err != nil { // A truncated multipart still gives up whatever came before it. break } if part.FileName() != "" { part.Close() continue // attachment } ct := part.Header.Get("Content-Type") mediaType, _, _ := mime.ParseMediaType(ct) text, err := plaintextBody(ct, part.Header.Get("Content-Transfer-Encoding"), part) part.Close() if err != nil || strings.TrimSpace(text) == "" { continue } if mediaType == "text/html" && !strings.HasPrefix(mediaType, "multipart/") { if html == "" { html = text } continue } if plain == "" { plain = text } } if strings.TrimSpace(plain) != "" { return plain, nil } return html, nil } // decodeBody applies the transfer encoding, then the charset. // // Charset support is UTF-8 (and ASCII, its subset) only, on purpose: x/text's // encoding tables are not vendored here, and guessing at windows-1251 bytes // would feed the model mojibake it would happily extract a task from. An // unsupported charset returns an error, which ParseMessage turns into an empty // body — subject-only, which is honest. func decodeBody(r io.Reader, encoding, charset string) (string, error) { switch strings.ToLower(strings.TrimSpace(encoding)) { case "quoted-printable": r = quotedprintable.NewReader(r) case "base64": r = newBase64Reader(r) } b, err := io.ReadAll(io.LimitReader(r, 1<<20)) if err != nil && len(b) == 0 { return "", fmt.Errorf("email: read body: %w", err) } switch cs := strings.ToLower(strings.TrimSpace(charset)); cs { case "", "utf-8", "utf8", "us-ascii", "ascii": return string(b), nil default: return "", fmt.Errorf("email: unsupported charset %q", cs) } } // decodeHeader decodes RFC 2047 encoded words ("=?utf-8?B?...?="), which is how // every Russian subject line arrives. Undecodable headers come back as-is // rather than empty: a mangled subject is still a hint, and it is only ever // shown to him as evidence. func decodeHeader(v string) string { dec := new(mime.WordDecoder) out, err := dec.DecodeHeader(v) if err != nil { return collapse(v) } return collapse(out) } var ( scriptStyle = regexp.MustCompile(`(?is)<(script|style)\b[^>]*>.*?`) htmlBreak = regexp.MustCompile(`(?i)<\s*(br\s*/?|/p|/div|/tr|/li|/h[1-6])\s*>`) htmlTag = regexp.MustCompile(`(?s)<[^>]*>`) htmlComment = regexp.MustCompile(`(?s)`) ) // stripHTML reduces an HTML part to text. A regex stripper, not a parser: // x/net/html is not vendored, and the consumer is a model reading prose — a // stray angle bracket costs nothing, whereas a new dependency for the privacy- // sensitive path costs review. func stripHTML(s string) string { s = scriptStyle.ReplaceAllString(s, " ") s = htmlComment.ReplaceAllString(s, " ") s = htmlBreak.ReplaceAllString(s, "\n") s = htmlTag.ReplaceAllString(s, " ") return unescapeEntities(s) } var entities = strings.NewReplacer( " ", " ", "&", "&", "<", "<", ">", ">", """, `"`, "'", "'", "'", "'", "—", "—", "–", "–", ) func unescapeEntities(s string) string { return entities.Replace(s) } // collapse squeezes runs of whitespace, keeping single newlines. Mail bodies // arrive with hard-wrapped lines and blocks of blank space; the model does not // need them and they are pure context budget. func collapse(s string) string { lines := strings.Split(strings.ReplaceAll(s, "\r\n", "\n"), "\n") var out []string blank := 0 for _, l := range lines { l = strings.TrimSpace(strings.Join(strings.Fields(l), " ")) if l == "" { blank++ if blank > 1 { continue } out = append(out, "") continue } blank = 0 out = append(out, l) } return strings.TrimSpace(strings.Join(out, "\n")) } // truncate cuts to n bytes on a rune boundary. func truncate(s string, n int) string { if len(s) <= n { return s } cut := s[:n] for len(cut) > 0 && !isRuneStart(cut[len(cut)-1]) { cut = cut[:len(cut)-1] } return strings.TrimSpace(cut) + "…" } func isRuneStart(b byte) bool { return b&0xC0 != 0x80 } // newBase64Reader — base64.NewDecoder already skips the CRLFs mail bodies wrap // with, so this is only a named seam for decodeBody to read cleanly. func newBase64Reader(r io.Reader) io.Reader { return base64.NewDecoder(base64.StdEncoding, r) }