diff --git a/internal/email/charset.go b/internal/email/charset.go new file mode 100644 index 0000000..f70718a --- /dev/null +++ b/internal/email/charset.go @@ -0,0 +1,48 @@ +package email + +// windows-1251 (and its ASCII-compatible low half) is decoded here rather than +// pulled in from x/text. +// +// The alternative was returning an error for the charset, which ParseMessage +// turns into a subject-only message. That is a live gap and not a small one: +// cp1251 is still what older Russian senders emit, and subject-only means those +// mails can never produce a task candidate. The whole of x/text/encoding is a +// large dependency for the most privacy-sensitive path in the tree, and +// windows-1251 is a 128-entry table. +// +// Only cp1251 is added. Guessing at an unknown charset stays forbidden: mojibake +// is worse than nothing, because the model extracts a task from it happily. + +// cp1251High — the 0x80..0xFF half of windows-1251. The low half is ASCII. +var cp1251High = [128]rune{ + 0x0402, 0x0403, 0x201A, 0x0453, 0x201E, 0x2026, 0x2020, 0x2021, + 0x20AC, 0x2030, 0x0409, 0x2039, 0x040A, 0x040C, 0x040B, 0x040F, + 0x0452, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, + 0xFFFD, 0x2122, 0x0459, 0x203A, 0x045A, 0x045C, 0x045B, 0x045F, + 0x00A0, 0x040E, 0x045E, 0x0408, 0x00A4, 0x0490, 0x00A6, 0x00A7, + 0x0401, 0x00A9, 0x0404, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x0407, + 0x00B0, 0x00B1, 0x0406, 0x0456, 0x0491, 0x00B5, 0x00B6, 0x00B7, + 0x0451, 0x2116, 0x0454, 0x00BB, 0x0458, 0x0405, 0x0455, 0x0457, + 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, + 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F, + 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, + 0x0428, 0x0429, 0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F, + 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, + 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F, + 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, + 0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D, 0x044E, 0x044F, +} + +// decodeCP1251 maps each byte through the table. Every byte has a defined +// meaning in this charset, so decoding cannot fail. +func decodeCP1251(b []byte) string { + out := make([]rune, 0, len(b)) + for _, c := range b { + if c < 0x80 { + out = append(out, rune(c)) + continue + } + out = append(out, cp1251High[c-0x80]) + } + return string(out) +} diff --git a/internal/email/junk.go b/internal/email/junk.go index dc64167..4895430 100644 --- a/internal/email/junk.go +++ b/internal/email/junk.go @@ -21,8 +21,18 @@ import ( // - Auto-Submitted other than "no" (RFC 3834) — generated by a machine. // - X-Spam-Flag: YES, X-Spam-Status: Yes — the spam filter upstream already // decided; we do not second-guess it in the other direction. -// - X-GM-LABELS / X-Gmail-Labels containing a Gmail category — Gmail's own -// Promotions/Social/Forums/Spam classification, when the server sends it. +// +// There is deliberately NO Gmail-category rule. One was written and removed: +// it matched X-GM-LABELS and X-Gmail-Labels against the parsed header block, +// and neither is a header. X-GM-LABELS is a Gmail FETCH data item, requested as +// "UID FETCH n (X-GM-LABELS)" and never present in the message source; +// X-Gmail-Labels only exists in a Takeout mbox export. This client asks for +// BODY.PEEK[] and nothing else, so the rule could not fire against a real +// mailbox while its doc comment promised a Promotions filter. Gmail's promotion +// mail carries List-Unsubscribe in practice and is caught by the rule above. +// Bringing the category rule back means adding the FETCH item and carrying the +// labels into classifyJunk out of band, not matching a header that never +// arrives. // // Deliberately NOT here: sender allow/deny lists and subject keyword matching. // Both are configuration that ages badly and both would be a place for his @@ -35,16 +45,6 @@ import ( // junkHeaders — headers whose mere presence marks bulk mail. var junkPresence = []string{"List-Unsubscribe", "List-Id", "List-Post"} -// gmailCategories — Gmail's category labels, lowercased as they appear in -// X-GM-LABELS. "important" and "inbox" are labels too, and are NOT categories. -// Matching is by these exact tokens (substring is fine — they are namespaced -// and cannot appear in a hand-made label by accident), so a user label named -// "Social Club" is not mistaken for Gmail's Social category. -var gmailCategories = []string{ - "category_promotions", "category_social", "category_forums", "category_updates", - `\spam`, `\junk`, -} - // classifyJunk returns whether the message is bulk/automated and why. The // reason is a short header name, safe to log — it names the marker, never the // sender or the subject. @@ -67,14 +67,5 @@ func classifyJunk(h mail.Header) (bool, string) { if v := strings.ToLower(strings.TrimSpace(h.Get("X-Spam-Status"))); strings.HasPrefix(v, "yes") { return true, "x-spam-status" } - labels := strings.ToLower(h.Get("X-GM-LABELS") + " " + h.Get("X-Gmail-Labels")) - for _, c := range gmailCategories { - if c == "" { - continue - } - if strings.Contains(labels, c) { - return true, "gmail-category" - } - } return false, "" } diff --git a/internal/email/junk_test.go b/internal/email/junk_test.go index d8e8722..81645c9 100644 --- a/internal/email/junk_test.go +++ b/internal/email/junk_test.go @@ -31,8 +31,10 @@ func TestClassifyJunk(t *testing.T) { {"spam flag", "From: a@b.c\nX-Spam-Flag: YES", true, "x-spam-flag"}, {"spam status", "From: a@b.c\nX-Spam-Status: Yes, score=9.1", true, "x-spam-status"}, {"spam status no", "From: a@b.c\nX-Spam-Status: No, score=0.1", false, ""}, - {"gmail promo", "From: a@b.c\nX-Gmail-Labels: Inbox,CATEGORY_PROMOTIONS", true, "gmail-category"}, - {"user label", "From: a@b.c\nX-Gmail-Labels: Social Club,Important", false, ""}, + // X-GM-LABELS is a Gmail FETCH data item, not a header, so it never + // reaches classifyJunk through this client. The rule that matched it was + // removed rather than left claiming a Promotions filter that never ran. + {"gmail label header is not a rule", "From: a@b.c\nX-Gmail-Labels: Inbox,CATEGORY_PROMOTIONS", false, ""}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { diff --git a/internal/email/message.go b/internal/email/message.go index 883be39..5219d39 100644 --- a/internal/email/message.go +++ b/internal/email/message.go @@ -94,7 +94,14 @@ func plaintextBody(contentType, encoding string, body io.Reader) (string, error) if boundary == "" { return "", fmt.Errorf("email: multipart without boundary") } - return multipartText(multipart.NewReader(body, boundary)) + plain, html, err := multipartText(multipart.NewReader(body, boundary)) + if err != nil { + return "", err + } + if strings.TrimSpace(plain) != "" { + return plain, nil + } + return html, nil case mediaType == "text/html": raw, err := decodeBody(body, encoding, params["charset"]) if err != nil { @@ -109,10 +116,15 @@ func plaintextBody(contentType, encoding string, body io.Reader) (string, error) } } -// multipartText reads one multipart level, recursing into nested multiparts. -// Returns the plain part if any part yielded one, else the stripped HTML. -func multipartText(mr *multipart.Reader) (string, error) { - var plain, html string +// multipartText reads one multipart level, recursing into nested multiparts, +// and returns the two buckets separately: real text/plain, and text stripped +// out of HTML. +// +// The buckets stay separate all the way up because a nested multipart can +// contribute either kind. Folding a nested level's answer into one string put +// HTML-derived text in the plain bucket, and a real text/plain sibling later in +// the message was then thrown away by the "plain is already set" guard. +func multipartText(mr *multipart.Reader) (plain, html string, err error) { for { part, err := mr.NextPart() if err == io.EOF { @@ -127,35 +139,47 @@ func multipartText(mr *multipart.Reader) (string, error) { continue // attachment } ct := part.Header.Get("Content-Type") - mediaType, _, _ := mime.ParseMediaType(ct) - text, err := plaintextBody(ct, part.Header.Get("Content-Transfer-Encoding"), part) - part.Close() - if err != nil || strings.TrimSpace(text) == "" { - continue - } - if mediaType == "text/html" && !strings.HasPrefix(mediaType, "multipart/") { - if html == "" { - html = text + mediaType, params, _ := mime.ParseMediaType(ct) + switch { + case strings.HasPrefix(mediaType, "multipart/"): + var np, nh string + if b := params["boundary"]; b != "" { + np, nh, _ = multipartText(multipart.NewReader(part, b)) + } + part.Close() + if plain == "" { + plain = np + } + if html == "" { + html = nh + } + default: + text, terr := plaintextBody(ct, part.Header.Get("Content-Transfer-Encoding"), part) + part.Close() + if terr != nil || strings.TrimSpace(text) == "" { + continue + } + if mediaType == "text/html" { + if html == "" { + html = text + } + continue + } + if plain == "" { + plain = text } - continue - } - if plain == "" { - plain = text } } - if strings.TrimSpace(plain) != "" { - return plain, nil - } - return html, nil + return plain, html, nil } // decodeBody applies the transfer encoding, then the charset. // -// Charset support is UTF-8 (and ASCII, its subset) only, on purpose: x/text's -// encoding tables are not vendored here, and guessing at windows-1251 bytes -// would feed the model mojibake it would happily extract a task from. An -// unsupported charset returns an error, which ParseMessage turns into an empty -// body — subject-only, which is honest. +// Charset support is UTF-8 (and ASCII, its subset) plus windows-1251, which is +// decoded from a table in charset.go — see the reasoning there. Everything else +// returns an error, which ParseMessage turns into an empty body: subject-only, +// which is honest. Guessing at unknown bytes would feed the model mojibake it +// would happily extract a task from. func decodeBody(r io.Reader, encoding, charset string) (string, error) { switch strings.ToLower(strings.TrimSpace(encoding)) { case "quoted-printable": @@ -170,6 +194,8 @@ func decodeBody(r io.Reader, encoding, charset string) (string, error) { switch cs := strings.ToLower(strings.TrimSpace(charset)); cs { case "", "utf-8", "utf8", "us-ascii", "ascii": return string(b), nil + case "windows-1251", "cp1251", "windows1251", "x-cp1251": + return decodeCP1251(b), nil default: return "", fmt.Errorf("email: unsupported charset %q", cs) } @@ -181,6 +207,19 @@ func decodeBody(r io.Reader, encoding, charset string) (string, error) { // shown to him as evidence. func decodeHeader(v string) string { dec := new(mime.WordDecoder) + // Same charset support as the body: an old Russian sender encodes the + // subject in windows-1251 too, and a subject is often the whole task. + dec.CharsetReader = func(charset string, r io.Reader) (io.Reader, error) { + switch strings.ToLower(strings.TrimSpace(charset)) { + case "windows-1251", "cp1251", "windows1251", "x-cp1251": + b, err := io.ReadAll(io.LimitReader(r, 1<<16)) + if err != nil && len(b) == 0 { + return nil, err + } + return strings.NewReader(decodeCP1251(b)), nil + } + return nil, fmt.Errorf("email: unsupported charset %q", charset) + } out, err := dec.DecodeHeader(v) if err != nil { return collapse(v) diff --git a/internal/email/message_test.go b/internal/email/message_test.go index 0ef9028..c961dce 100644 --- a/internal/email/message_test.go +++ b/internal/email/message_test.go @@ -70,13 +70,28 @@ func TestParsePrefersPlainAndSkipsAttachments(t *testing.T) { } } -// An unsupported charset must degrade to headers-only rather than to mojibake -// the model would then extract a task from. -func TestParseUnsupportedCharsetKeepsHeaders(t *testing.T) { +// windows-1251 is what older Russian senders still emit. Subject-only for those +// mails meant they could never produce a task candidate. +func TestParseCP1251(t *testing.T) { msg, err := ParseMessage(3, fixture(t, "cp1251.eml")) if err != nil { t.Fatalf("parse: %v", err) } + if want := "Счёт за интернет"; msg.Subject != want { + t.Errorf("subject = %q, want %q", msg.Subject, want) + } + if want := "Оплати счёт до пятницы."; !strings.Contains(msg.Body, want) { + t.Errorf("body = %q, want it to contain %q", msg.Body, want) + } +} + +// A charset with no table here must degrade to headers-only rather than to +// mojibake the model would then extract a task from. +func TestParseUnsupportedCharsetKeepsHeaders(t *testing.T) { + msg, err := ParseMessage(3, fixture(t, "koi8r.eml")) + if err != nil { + t.Fatalf("parse: %v", err) + } if msg.Subject != "Legacy" { t.Errorf("subject = %q, want Legacy", msg.Subject) } @@ -85,6 +100,31 @@ func TestParseUnsupportedCharsetKeepsHeaders(t *testing.T) { } } +// A nested multipart/alternative that only had HTML must not fill the plain +// bucket: a real text/plain sibling later in the message is the better text and +// used to be discarded. +func TestParseNestedHTMLDoesNotShadowLaterPlain(t *testing.T) { + raw := "Subject: nested\r\n" + + "Content-Type: multipart/mixed; boundary=OUT\r\n\r\n" + + "--OUT\r\n" + + "Content-Type: multipart/alternative; boundary=IN\r\n\r\n" + + "--IN\r\n" + + "Content-Type: text/html; charset=utf-8\r\n\r\n" + + "

from the html part

\r\n" + + "--IN--\r\n" + + "--OUT\r\n" + + "Content-Type: text/plain; charset=utf-8\r\n\r\n" + + "the real plain text\r\n" + + "--OUT--\r\n" + msg, err := ParseMessage(5, []byte(raw)) + if err != nil { + t.Fatalf("parse: %v", err) + } + if got := strings.TrimSpace(msg.Body); got != "the real plain text" { + t.Errorf("body = %q, want the text/plain part to win", got) + } +} + func TestParseTruncatesLongBody(t *testing.T) { var b strings.Builder b.WriteString("Subject: long\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n") diff --git a/internal/email/testdata/cp1251.eml b/internal/email/testdata/cp1251.eml index ca1a016..5f987cf 100644 --- a/internal/email/testdata/cp1251.eml +++ b/internal/email/testdata/cp1251.eml @@ -1,7 +1,7 @@ From: legacy@example.org To: kami@example.org -Subject: Legacy +Subject: =?windows-1251?B?0fe48iDn4CDo7fLl8O3l8g==?= Date: Fri, 01 Aug 2026 05:00:00 +0400 Content-Type: text/plain; charset="windows-1251" -Ï + . diff --git a/internal/email/testdata/koi8r.eml b/internal/email/testdata/koi8r.eml new file mode 100644 index 0000000..b6f18d7 --- /dev/null +++ b/internal/email/testdata/koi8r.eml @@ -0,0 +1,7 @@ +From: legacy@example.org +To: kami@example.org +Subject: Legacy +Date: Fri, 01 Aug 2026 05:00:00 +0400 +Content-Type: text/plain; charset="koi8-r" + + ޣ.