Maven/internal/email/message.go

// Package email is the reading half of the email reader (Vikunja #246,
// docs/plans/01-email-reader.md): a small IMAP client, a MIME-to-plaintext
// converter, and the junk filter that decides a message is not worth reading at
// all. Extraction lives in extract.go and writes nothing itself.
//
// Two constraints shape everything here, both from CLAUDE.md:
//
//   - Mail is personal. Nothing in this package logs a body, a subject, or an
//     address; callers get the text and decide. Mail text is never search input
//     — no function here reaches the network except the IMAP connection itself.
//   - Off unless configured. There is no default host, no default account, and
//     no fallback that would make a mailbox get read because a field was empty.
//
// The IMAP subset is deliberately tiny (LOGIN, SELECT, UID SEARCH, UID FETCH
// with BODY.PEEK, LOGOUT). No IDLE: a poll every few minutes is what a task
// candidate needs, and IDLE would mean holding a connection and a credential
// open forever for latency nobody is waiting on.
package email

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"io"
	"mime"
	"mime/multipart"
	"mime/quotedprintable"
	"net/mail"
	"regexp"
	"strings"
)

// MaxBodyBytes — how much of one message body is kept. A task hides in the
// first screenful; the rest is signature, quoted history and legal boilerplate,
// and it would only spend the resident model's 4096-token context.
const MaxBodyBytes = 4000

// Message — one mail, reduced to the fields extraction and review need.
//
// Raw is deliberately absent: once a message is parsed the original bytes are
// dropped, so no caller can accidentally log or forward the whole mail.
type Message struct {
	UID     uint32
	From    string
	Subject string
	Date    string // as sent, unparsed — display only
	Body    string // plaintext, decoded, HTML-stripped, truncated
	// Junk is set by the junk filter (see junk.go). A junk message is carried
	// rather than dropped so the poller can count it and still mark it seen.
	Junk       bool
	JunkReason string
}

// ParseMessage turns one RFC 5322 message into a Message.
//
// It never fails on a body it cannot understand: an unparsable or
// unsupported-charset body yields an empty Body and the headers still come
// through, because a subject line alone is often the whole task ("Счёт за
// интернет"). Only a message whose headers cannot be read at all is an error.
func ParseMessage(uid uint32, raw []byte) (Message, error) {
	// bytes.NewReader, not strings.NewReader(string(raw)): the conversion copied
	// the whole message, and MaxMessageBytes lets that be 2 MiB per mail on a box
	// already holding the resident model.
	m, err := mail.ReadMessage(bytes.NewReader(raw))
	if err != nil {
		return Message{}, fmt.Errorf("email: parse message: %w", err)
	}
	msg := Message{
		UID:     uid,
		From:    decodeHeader(m.Header.Get("From")),
		Subject: decodeHeader(m.Header.Get("Subject")),
		Date:    m.Header.Get("Date"),
	}
	msg.Junk, msg.JunkReason = classifyJunk(m.Header)
	body, err := plaintextBody(m.Header.Get("Content-Type"), m.Header.Get("Content-Transfer-Encoding"), m.Body)
	if err == nil {
		msg.Body = truncate(collapse(body), MaxBodyBytes)
	}
	return msg, nil
}

// plaintextBody walks the MIME tree and returns the best plaintext it can.
//
// Preference order inside a multipart: text/plain first, text/html stripped
// only when there is no plain part. multipart/mixed attachments are skipped
// wholesale — an attachment is a file, not a sentence, and reading one would
// mean parsing arbitrary formats from the network.
func plaintextBody(contentType, encoding string, body io.Reader) (string, error) {
	return plaintextBodyAt(contentType, encoding, body, 0)
}

// MaxMIMEDepth — how deep the MIME tree is walked.
//
// The nesting comes off the wire, so the recursion depth is the sender's to
// pick: a boundary line is a few bytes, and one message inside MaxMessageBytes
// can declare tens of thousands of multipart levels. Real mail is three deep
// (mixed, then alternative, then related), so a message past this is malformed
// or hostile and truncating the walk costs a body nobody was going to read.
const MaxMIMEDepth = 12

// plaintextBodyAt is plaintextBody carrying the current nesting depth.
func plaintextBodyAt(contentType, encoding string, body io.Reader, depth int) (string, error) {
	if depth > MaxMIMEDepth {
		return "", nil
	}
	mediaType, params, err := mime.ParseMediaType(contentType)
	if contentType == "" || err != nil {
		// No Content-Type at all is legal and means text/plain; a broken one is
		// treated the same rather than dropping the message.
		mediaType, params = "text/plain", nil
	}
	switch {
	case strings.HasPrefix(mediaType, "multipart/"):
		boundary := params["boundary"]
		if boundary == "" {
			return "", fmt.Errorf("email: multipart without boundary")
		}
		plain, html, err := multipartText(multipart.NewReader(body, boundary), depth+1)
		if err != nil {
			return "", err
		}
		if strings.TrimSpace(plain) != "" {
			return plain, nil
		}
		return html, nil
	case mediaType == "text/html":
		raw, err := decodeBody(body, encoding, params["charset"])
		if err != nil {
			return "", err
		}
		return stripHTML(raw), nil
	case mediaType == "text/plain":
		return decodeBody(body, encoding, params["charset"])
	default:
		// A single-part non-text message (a bare PDF, say). No body, headers only.
		return "", nil
	}
}

// multipartText reads one multipart level, recursing into nested multiparts,
// and returns the two buckets separately: real text/plain, and text stripped
// out of HTML.
//
// The buckets stay separate all the way up because a nested multipart can
// contribute either kind. Folding a nested level's answer into one string put
// HTML-derived text in the plain bucket, and a real text/plain sibling later in
// the message was then thrown away by the "plain is already set" guard.
func multipartText(mr *multipart.Reader, depth int) (plain, html string, err error) {
	for {
		part, err := mr.NextPart()
		if err == io.EOF {
			break
		}
		if err != nil {
			// A truncated multipart still gives up whatever came before it.
			break
		}
		if part.FileName() != "" {
			part.Close()
			continue // attachment
		}
		ct := part.Header.Get("Content-Type")
		mediaType, params, _ := mime.ParseMediaType(ct)
		switch {
		case strings.HasPrefix(mediaType, "multipart/"):
			var np, nh string
			if b := params["boundary"]; b != "" && depth <= MaxMIMEDepth {
				np, nh, _ = multipartText(multipart.NewReader(part, b), depth+1)
			}
			part.Close()
			if plain == "" {
				plain = np
			}
			if html == "" {
				html = nh
			}
		default:
			text, terr := plaintextBodyAt(ct, part.Header.Get("Content-Transfer-Encoding"), part, depth)
			part.Close()
			if terr != nil || strings.TrimSpace(text) == "" {
				continue
			}
			if mediaType == "text/html" {
				if html == "" {
					html = text
				}
				continue
			}
			if plain == "" {
				plain = text
			}
		}
	}
	return plain, html, nil
}

// decodeBody applies the transfer encoding, then the charset.
//
// Charset support is UTF-8 (and ASCII, its subset) plus windows-1251, which is
// decoded from a table in charset.go — see the reasoning there. Everything else
// returns an error, which ParseMessage turns into an empty body: subject-only,
// which is honest. Guessing at unknown bytes would feed the model mojibake it
// would happily extract a task from.
func decodeBody(r io.Reader, encoding, charset string) (string, error) {
	switch strings.ToLower(strings.TrimSpace(encoding)) {
	case "quoted-printable":
		r = quotedprintable.NewReader(r)
	case "base64":
		r = newBase64Reader(r)
	}
	b, err := io.ReadAll(io.LimitReader(r, 1<<20))
	if err != nil && len(b) == 0 {
		return "", fmt.Errorf("email: read body: %w", err)
	}
	switch cs := strings.ToLower(strings.TrimSpace(charset)); cs {
	case "", "utf-8", "utf8", "us-ascii", "ascii":
		return string(b), nil
	case "windows-1251", "cp1251", "windows1251", "x-cp1251":
		return decodeCP1251(b), nil
	default:
		return "", fmt.Errorf("email: unsupported charset %q", cs)
	}
}

// decodeHeader decodes RFC 2047 encoded words ("=?utf-8?B?...?="), which is how
// every Russian subject line arrives. Undecodable headers come back as-is
// rather than empty: a mangled subject is still a hint, and it is only ever
// shown to him as evidence.
func decodeHeader(v string) string {
	dec := new(mime.WordDecoder)
	// Same charset support as the body: an old Russian sender encodes the
	// subject in windows-1251 too, and a subject is often the whole task.
	dec.CharsetReader = func(charset string, r io.Reader) (io.Reader, error) {
		switch strings.ToLower(strings.TrimSpace(charset)) {
		case "windows-1251", "cp1251", "windows1251", "x-cp1251":
			b, err := io.ReadAll(io.LimitReader(r, 1<<16))
			if err != nil && len(b) == 0 {
				return nil, err
			}
			return strings.NewReader(decodeCP1251(b)), nil
		}
		return nil, fmt.Errorf("email: unsupported charset %q", charset)
	}
	out, err := dec.DecodeHeader(v)
	if err != nil {
		return collapse(v)
	}
	return collapse(out)
}

var (
	scriptStyle = regexp.MustCompile(`(?is)<(script|style)\b[^>]*>.*?</\s*(script|style)\s*>`)
	htmlBreak   = regexp.MustCompile(`(?i)<\s*(br\s*/?|/p|/div|/tr|/li|/h[1-6])\s*>`)
	htmlTag     = regexp.MustCompile(`(?s)<[^>]*>`)
	htmlComment = regexp.MustCompile(`(?s)<!--.*?-->`)
)

// stripHTML reduces an HTML part to text. A regex stripper, not a parser:
// x/net/html is not vendored, and the consumer is a model reading prose — a
// stray angle bracket costs nothing, whereas a new dependency for the privacy-
// sensitive path costs review.
func stripHTML(s string) string {
	s = scriptStyle.ReplaceAllString(s, " ")
	s = htmlComment.ReplaceAllString(s, " ")
	s = htmlBreak.ReplaceAllString(s, "\n")
	s = htmlTag.ReplaceAllString(s, " ")
	return unescapeEntities(s)
}

var entities = strings.NewReplacer(
	"&nbsp;", " ", "&amp;", "&", "&lt;", "<", "&gt;", ">",
	"&quot;", `"`, "&#39;", "'", "&apos;", "'", "&mdash;", "—", "&ndash;", "–",
)

func unescapeEntities(s string) string { return entities.Replace(s) }

// collapse squeezes runs of whitespace, keeping single newlines. Mail bodies
// arrive with hard-wrapped lines and blocks of blank space; the model does not
// need them and they are pure context budget.
func collapse(s string) string {
	lines := strings.Split(strings.ReplaceAll(s, "\r\n", "\n"), "\n")
	var out []string
	blank := 0
	for _, l := range lines {
		l = strings.TrimSpace(strings.Join(strings.Fields(l), " "))
		if l == "" {
			blank++
			if blank > 1 {
				continue
			}
			out = append(out, "")
			continue
		}
		blank = 0
		out = append(out, l)
	}
	return strings.TrimSpace(strings.Join(out, "\n"))
}

// truncate cuts to n bytes on a rune boundary.
func truncate(s string, n int) string {
	if len(s) <= n {
		return s
	}
	cut := s[:n]
	for len(cut) > 0 && !isRuneStart(cut[len(cut)-1]) {
		cut = cut[:len(cut)-1]
	}
	return strings.TrimSpace(cut) + "…"
}

func isRuneStart(b byte) bool { return b&0xC0 != 0x80 }

// newBase64Reader — base64.NewDecoder already skips the CRLFs mail bodies wrap
// with, so this is only a named seam for decodeBody to read cleanly.
func newBase64Reader(r io.Reader) io.Reader {
	return base64.NewDecoder(base64.StdEncoding, r)
}