Maven/internal/rss/feed.go

// Package rss reads RSS 2.0 and Atom feeds, and does nothing else with them.
//
// Parsing and polling are split from delivery on purpose: a feed is a source
// Maven can be ASKED about, not a thing that speaks. Nothing in this package
// dispatches, nudges or notifies — the poller writes notes, and the answer path
// reads them when he asks "что нового в лентах?". "Not a nag" is the oldest
// constraint in the spec, and a news feed is the single most tempting way to
// break it.
//
// Stdlib only (encoding/xml). Feeds are XML from strangers, so the parser takes
// what it recognises and ignores the rest rather than failing a whole feed over
// one malformed item.
package rss

import (
	"encoding/xml"
	"fmt"
	"html"
	"io"
	"regexp"
	"strings"
	"time"
)

// Item is one feed entry, normalised across RSS and Atom.
type Item struct {
	Title     string
	Link      string
	Summary   string    // plain text, tags stripped, entities decoded
	Published time.Time // zero when the feed did not say
	ID        string    // guid / atom id, falling back to the link
}

// Feed is a parsed document.
type Feed struct {
	Title string
	Items []Item
}

// feedDoc covers both dialects in one struct. RSS puts items under
// channel>item, Atom puts entries at the top level, and the field names barely
// overlap — so both sets are declared and whichever the document filled in wins.
type feedDoc struct {
	ChannelTitle string `xml:"channel>title"`
	AtomTitle    string `xml:"title"`

	Items []struct {
		Title       string `xml:"title"`
		Link        string `xml:"link"`
		Description string `xml:"description"`
		Encoded     string `xml:"encoded"` // content:encoded
		GUID        string `xml:"guid"`
		PubDate     string `xml:"pubDate"`
		Date        string `xml:"date"` // dc:date
	} `xml:"channel>item"`

	Entries []struct {
		Title string `xml:"title"`
		Links []struct {
			Href string `xml:"href,attr"`
			Rel  string `xml:"rel,attr"`
		} `xml:"link"`
		Summary   string `xml:"summary"`
		Content   string `xml:"content"`
		ID        string `xml:"id"`
		Updated   string `xml:"updated"`
		Published string `xml:"published"`
	} `xml:"entry"`
}

// Parse reads a feed document.
func Parse(r io.Reader) (Feed, error) {
	var doc feedDoc
	dec := xml.NewDecoder(r)
	// Strict=false buys tolerance of the malformed markup feeds are full of:
	// unclosed tags, stray entities. It has nothing to do with charsets.
	//
	// Charsets are handled by not handling them: CharsetReader stays nil, so a
	// feed declaring windows-1251 fails to parse rather than being read as
	// UTF-8. That is the behaviour we want — a charset we cannot decode is a
	// feed we do not read, which beats mojibake in his notes.
	dec.Strict = false
	if err := dec.Decode(&doc); err != nil {
		return Feed{}, fmt.Errorf("rss: bad xml: %w", err)
	}

	f := Feed{Title: strings.TrimSpace(doc.ChannelTitle)}
	if f.Title == "" {
		f.Title = strings.TrimSpace(doc.AtomTitle)
	}
	for _, it := range doc.Items {
		item := Item{
			Title:     PlainText(it.Title),
			Link:      strings.TrimSpace(it.Link),
			Summary:   PlainText(firstNonEmpty(it.Description, it.Encoded)),
			Published: parseTime(firstNonEmpty(it.PubDate, it.Date)),
			ID:        strings.TrimSpace(firstNonEmpty(it.GUID, it.Link)),
		}
		if item.Title != "" || item.Link != "" {
			f.Items = append(f.Items, item)
		}
	}
	for _, e := range doc.Entries {
		link := ""
		for _, l := range e.Links {
			if l.Rel == "" || l.Rel == "alternate" {
				link = strings.TrimSpace(l.Href)
				break
			}
		}
		if link == "" && len(e.Links) > 0 {
			link = strings.TrimSpace(e.Links[0].Href)
		}
		item := Item{
			Title:     PlainText(e.Title),
			Link:      link,
			Summary:   PlainText(firstNonEmpty(e.Summary, e.Content)),
			Published: parseTime(firstNonEmpty(e.Published, e.Updated)),
			ID:        strings.TrimSpace(firstNonEmpty(e.ID, link)),
		}
		if item.Title != "" || item.Link != "" {
			f.Items = append(f.Items, item)
		}
	}
	return f, nil
}

func firstNonEmpty(vals ...string) string {
	for _, v := range vals {
		if strings.TrimSpace(v) != "" {
			return v
		}
	}
	return ""
}

// timeLayouts — RFC1123/822 for RSS, RFC3339 for Atom, plus the near-misses
// real feeds ship (no seconds, numeric zone where a name is expected).
var timeLayouts = []string{
	time.RFC1123Z,
	time.RFC1123,
	time.RFC822Z,
	time.RFC822,
	time.RFC3339,
	"2006-01-02T15:04:05Z0700",
	"2006-01-02 15:04:05",
	"2006-01-02",
	"Mon, 02 Jan 2006 15:04:05 -0700",
	"Mon, 2 Jan 2006 15:04:05 -0700",
	"Mon, 2 Jan 2006 15:04:05 MST",
}

// parseTime returns the zero time on anything it cannot read. An undated item
// is still an item; the poller dedupes by ID, so a missing date costs nothing.
func parseTime(s string) time.Time {
	s = strings.TrimSpace(s)
	if s == "" {
		return time.Time{}
	}
	for _, l := range timeLayouts {
		if t, err := time.Parse(l, s); err == nil {
			return t.UTC()
		}
	}
	return time.Time{}
}

var (
	// RE2 has no backreferences, so the two tags are spelled out rather than
	// captured and matched against themselves.
	scriptRE = regexp.MustCompile(`(?is)<script\b[^>]*>.*?</script>|<style\b[^>]*>.*?</style>`)
	tagRE    = regexp.MustCompile(`(?s)<[^>]*>`)
)

// PlainText strips markup and decodes entities — feed summaries are HTML, and
// what reaches a note (and possibly the TTS) must be text. Exported so a caller
// holding raw feed markup can reduce it the same way; crawl/extract.go does the
// bigger job on a whole document and does not go through here.
func PlainText(s string) string {
	s = scriptRE.ReplaceAllString(s, " ")
	s = tagRE.ReplaceAllString(s, " ")
	s = html.UnescapeString(s)
	return strings.TrimSpace(strings.Join(strings.Fields(s), " "))
}