// Package rss reads RSS 2.0 and Atom feeds, and does nothing else with them.
//
// Parsing and polling are split from delivery on purpose: a feed is a source
// Maven can be ASKED about, not a thing that speaks. Nothing in this package
// dispatches, nudges or notifies — the poller writes notes, and the answer path
// reads them when he asks "что нового в лентах?". "Not a nag" is the oldest
// constraint in the spec, and a news feed is the single most tempting way to
// break it.
//
// Stdlib only (encoding/xml). Feeds are XML from strangers, so the parser takes
// what it recognises and ignores the rest rather than failing a whole feed over
// one malformed item.
package rss
import (
"encoding/xml"
"fmt"
"html"
"io"
"regexp"
"strings"
"time"
)
// Item is one feed entry, normalised across RSS and Atom.
type Item struct {
Title string
Link string
Summary string // plain text, tags stripped, entities decoded
Published time.Time // zero when the feed did not say
ID string // guid / atom id, falling back to the link
}
// Feed is a parsed document.
type Feed struct {
Title string
Items []Item
}
// feedDoc covers both dialects in one struct. RSS puts items under
// channel>item, Atom puts entries at the top level, and the field names barely
// overlap — so both sets are declared and whichever the document filled in wins.
type feedDoc struct {
ChannelTitle string `xml:"channel>title"`
AtomTitle string `xml:"title"`
Items []struct {
Title string `xml:"title"`
Link string `xml:"link"`
Description string `xml:"description"`
Encoded string `xml:"encoded"` // content:encoded
GUID string `xml:"guid"`
PubDate string `xml:"pubDate"`
Date string `xml:"date"` // dc:date
} `xml:"channel>item"`
Entries []struct {
Title string `xml:"title"`
Links []struct {
Href string `xml:"href,attr"`
Rel string `xml:"rel,attr"`
} `xml:"link"`
Summary string `xml:"summary"`
Content string `xml:"content"`
ID string `xml:"id"`
Updated string `xml:"updated"`
Published string `xml:"published"`
} `xml:"entry"`
}
// Parse reads a feed document.
func Parse(r io.Reader) (Feed, error) {
var doc feedDoc
dec := xml.NewDecoder(r)
// Strict=false buys tolerance of the malformed markup feeds are full of:
// unclosed tags, stray entities. It has nothing to do with charsets.
//
// Charsets are handled by not handling them: CharsetReader stays nil, so a
// feed declaring windows-1251 fails to parse rather than being read as
// UTF-8. That is the behaviour we want — a charset we cannot decode is a
// feed we do not read, which beats mojibake in his notes.
dec.Strict = false
if err := dec.Decode(&doc); err != nil {
return Feed{}, fmt.Errorf("rss: bad xml: %w", err)
}
f := Feed{Title: strings.TrimSpace(doc.ChannelTitle)}
if f.Title == "" {
f.Title = strings.TrimSpace(doc.AtomTitle)
}
for _, it := range doc.Items {
item := Item{
Title: PlainText(it.Title),
Link: strings.TrimSpace(it.Link),
Summary: PlainText(firstNonEmpty(it.Description, it.Encoded)),
Published: parseTime(firstNonEmpty(it.PubDate, it.Date)),
ID: strings.TrimSpace(firstNonEmpty(it.GUID, it.Link)),
}
if item.Title != "" || item.Link != "" {
f.Items = append(f.Items, item)
}
}
for _, e := range doc.Entries {
link := ""
for _, l := range e.Links {
if l.Rel == "" || l.Rel == "alternate" {
link = strings.TrimSpace(l.Href)
break
}
}
if link == "" && len(e.Links) > 0 {
link = strings.TrimSpace(e.Links[0].Href)
}
item := Item{
Title: PlainText(e.Title),
Link: link,
Summary: PlainText(firstNonEmpty(e.Summary, e.Content)),
Published: parseTime(firstNonEmpty(e.Published, e.Updated)),
ID: strings.TrimSpace(firstNonEmpty(e.ID, link)),
}
if item.Title != "" || item.Link != "" {
f.Items = append(f.Items, item)
}
}
return f, nil
}
func firstNonEmpty(vals ...string) string {
for _, v := range vals {
if strings.TrimSpace(v) != "" {
return v
}
}
return ""
}
// timeLayouts — RFC1123/822 for RSS, RFC3339 for Atom, plus the near-misses
// real feeds ship (no seconds, numeric zone where a name is expected).
var timeLayouts = []string{
time.RFC1123Z,
time.RFC1123,
time.RFC822Z,
time.RFC822,
time.RFC3339,
"2006-01-02T15:04:05Z0700",
"2006-01-02 15:04:05",
"2006-01-02",
"Mon, 02 Jan 2006 15:04:05 -0700",
"Mon, 2 Jan 2006 15:04:05 -0700",
"Mon, 2 Jan 2006 15:04:05 MST",
}
// parseTime returns the zero time on anything it cannot read. An undated item
// is still an item; the poller dedupes by ID, so a missing date costs nothing.
func parseTime(s string) time.Time {
s = strings.TrimSpace(s)
if s == "" {
return time.Time{}
}
for _, l := range timeLayouts {
if t, err := time.Parse(l, s); err == nil {
return t.UTC()
}
}
return time.Time{}
}
var (
// RE2 has no backreferences, so the two tags are spelled out rather than
// captured and matched against themselves.
scriptRE = regexp.MustCompile(`(?is)|`)
tagRE = regexp.MustCompile(`(?s)<[^>]*>`)
)
// PlainText strips markup and decodes entities — feed summaries are HTML, and
// what reaches a note (and possibly the TTS) must be text. Exported so a caller
// holding raw feed markup can reduce it the same way; crawl/extract.go does the
// bigger job on a whole document and does not go through here.
func PlainText(s string) string {
s = scriptRE.ReplaceAllString(s, " ")
s = tagRE.ReplaceAllString(s, " ")
s = html.UnescapeString(s)
return strings.TrimSpace(strings.Join(strings.Fields(s), " "))
}