// Package rss reads RSS 2.0 and Atom feeds, and does nothing else with them. // // Parsing and polling are split from delivery on purpose: a feed is a source // Maven can be ASKED about, not a thing that speaks. Nothing in this package // dispatches, nudges or notifies — the poller writes notes, and the answer path // reads them when he asks "что нового в лентах?". "Not a nag" is the oldest // constraint in the spec, and a news feed is the single most tempting way to // break it. // // Stdlib only (encoding/xml). Feeds are XML from strangers, so the parser takes // what it recognises and ignores the rest rather than failing a whole feed over // one malformed item. package rss import ( "encoding/xml" "fmt" "html" "io" "regexp" "strings" "time" ) // Item is one feed entry, normalised across RSS and Atom. type Item struct { Title string Link string Summary string // plain text, tags stripped, entities decoded Published time.Time // zero when the feed did not say ID string // guid / atom id, falling back to the link } // Feed is a parsed document. type Feed struct { Title string Items []Item } // feedDoc covers both dialects in one struct. RSS puts items under // channel>item, Atom puts entries at the top level, and the field names barely // overlap — so both sets are declared and whichever the document filled in wins. type feedDoc struct { ChannelTitle string `xml:"channel>title"` AtomTitle string `xml:"title"` Items []struct { Title string `xml:"title"` Link string `xml:"link"` Description string `xml:"description"` Encoded string `xml:"encoded"` // content:encoded GUID string `xml:"guid"` PubDate string `xml:"pubDate"` Date string `xml:"date"` // dc:date } `xml:"channel>item"` Entries []struct { Title string `xml:"title"` Links []struct { Href string `xml:"href,attr"` Rel string `xml:"rel,attr"` } `xml:"link"` Summary string `xml:"summary"` Content string `xml:"content"` ID string `xml:"id"` Updated string `xml:"updated"` Published string `xml:"published"` } `xml:"entry"` } // Parse reads a feed document. func Parse(r io.Reader) (Feed, error) { var doc feedDoc dec := xml.NewDecoder(r) // Strict=false buys tolerance of the malformed markup feeds are full of: // unclosed tags, stray entities. It has nothing to do with charsets. // // Charsets are handled by not handling them: CharsetReader stays nil, so a // feed declaring windows-1251 fails to parse rather than being read as // UTF-8. That is the behaviour we want — a charset we cannot decode is a // feed we do not read, which beats mojibake in his notes. dec.Strict = false if err := dec.Decode(&doc); err != nil { return Feed{}, fmt.Errorf("rss: bad xml: %w", err) } f := Feed{Title: strings.TrimSpace(doc.ChannelTitle)} if f.Title == "" { f.Title = strings.TrimSpace(doc.AtomTitle) } for _, it := range doc.Items { item := Item{ Title: PlainText(it.Title), Link: strings.TrimSpace(it.Link), Summary: PlainText(firstNonEmpty(it.Description, it.Encoded)), Published: parseTime(firstNonEmpty(it.PubDate, it.Date)), ID: strings.TrimSpace(firstNonEmpty(it.GUID, it.Link)), } if item.Title != "" || item.Link != "" { f.Items = append(f.Items, item) } } for _, e := range doc.Entries { link := "" for _, l := range e.Links { if l.Rel == "" || l.Rel == "alternate" { link = strings.TrimSpace(l.Href) break } } if link == "" && len(e.Links) > 0 { link = strings.TrimSpace(e.Links[0].Href) } item := Item{ Title: PlainText(e.Title), Link: link, Summary: PlainText(firstNonEmpty(e.Summary, e.Content)), Published: parseTime(firstNonEmpty(e.Published, e.Updated)), ID: strings.TrimSpace(firstNonEmpty(e.ID, link)), } if item.Title != "" || item.Link != "" { f.Items = append(f.Items, item) } } return f, nil } func firstNonEmpty(vals ...string) string { for _, v := range vals { if strings.TrimSpace(v) != "" { return v } } return "" } // timeLayouts — RFC1123/822 for RSS, RFC3339 for Atom, plus the near-misses // real feeds ship (no seconds, numeric zone where a name is expected). var timeLayouts = []string{ time.RFC1123Z, time.RFC1123, time.RFC822Z, time.RFC822, time.RFC3339, "2006-01-02T15:04:05Z0700", "2006-01-02 15:04:05", "2006-01-02", "Mon, 02 Jan 2006 15:04:05 -0700", "Mon, 2 Jan 2006 15:04:05 -0700", "Mon, 2 Jan 2006 15:04:05 MST", } // parseTime returns the zero time on anything it cannot read. An undated item // is still an item; the poller dedupes by ID, so a missing date costs nothing. func parseTime(s string) time.Time { s = strings.TrimSpace(s) if s == "" { return time.Time{} } for _, l := range timeLayouts { if t, err := time.Parse(l, s); err == nil { return t.UTC() } } return time.Time{} } var ( // RE2 has no backreferences, so the two tags are spelled out rather than // captured and matched against themselves. scriptRE = regexp.MustCompile(`(?is)]*>.*?|]*>.*?`) tagRE = regexp.MustCompile(`(?s)<[^>]*>`) ) // PlainText strips markup and decodes entities — feed summaries are HTML, and // what reaches a note (and possibly the TTS) must be text. Exported because the // crawler's extractor needs exactly this on a bigger input. func PlainText(s string) string { s = scriptRE.ReplaceAllString(s, " ") s = tagRE.ReplaceAllString(s, " ") s = html.UnescapeString(s) return strings.TrimSpace(strings.Join(strings.Fields(s), " ")) }