Read RSS and Atom feeds, and speak about them only when asked (#258)
internal/rss parses RSS 2.0 and Atom, and polls each configured feed on its own interval; internal/webfetch is the one door either of them uses to touch the network. The poller writes items as notes with source "rss:<feed>" and nothing else: the answer path reads them back when he asks "что нового в лентах?", and nothing is announced on arrival. A feed that dispatched would be a nag, which is why the plan's breaking-news rule was left out rather than built. webfetch is where the limits live, as code rather than a paragraph: http(s) only, an allowlist (the configured feeds' hosts) and a denylist, a 2 MiB body cap, a 3-redirect cap, one request per host per second, and a refusal to connect to any private address — checked in the dialer's Control hook so it holds for every resolved address and every redirect hop, not just for a literal IP. Off unless configured: no "feeds" block, no poller, no outbound request. How far a feed was read is a config fact (rss:latest:<name>), so a restart does not re-note yesterday's headlines.
This commit is contained in:
kami
@@ -0,0 +1,179 @@
|
||||
// Package rss reads RSS 2.0 and Atom feeds, and does nothing else with them.
|
||||
//
|
||||
// Parsing and polling are split from delivery on purpose: a feed is a source
|
||||
// Maven can be ASKED about, not a thing that speaks. Nothing in this package
|
||||
// dispatches, nudges or notifies — the poller writes notes, and the answer path
|
||||
// reads them when he asks "что нового в лентах?". "Not a nag" is the oldest
|
||||
// constraint in the spec, and a news feed is the single most tempting way to
|
||||
// break it.
|
||||
//
|
||||
// Stdlib only (encoding/xml). Feeds are XML from strangers, so the parser takes
|
||||
// what it recognises and ignores the rest rather than failing a whole feed over
|
||||
// one malformed item.
|
||||
package rss
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"html"
|
||||
"io"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Item is one feed entry, normalised across RSS and Atom.
|
||||
type Item struct {
|
||||
Title string
|
||||
Link string
|
||||
Summary string // plain text, tags stripped, entities decoded
|
||||
Published time.Time // zero when the feed did not say
|
||||
ID string // guid / atom id, falling back to the link
|
||||
}
|
||||
|
||||
// Feed is a parsed document.
|
||||
type Feed struct {
|
||||
Title string
|
||||
Items []Item
|
||||
}
|
||||
|
||||
// feedDoc covers both dialects in one struct. RSS puts items under
|
||||
// channel>item, Atom puts entries at the top level, and the field names barely
|
||||
// overlap — so both sets are declared and whichever the document filled in wins.
|
||||
type feedDoc struct {
|
||||
ChannelTitle string `xml:"channel>title"`
|
||||
AtomTitle string `xml:"title"`
|
||||
|
||||
Items []struct {
|
||||
Title string `xml:"title"`
|
||||
Link string `xml:"link"`
|
||||
Description string `xml:"description"`
|
||||
Encoded string `xml:"encoded"` // content:encoded
|
||||
GUID string `xml:"guid"`
|
||||
PubDate string `xml:"pubDate"`
|
||||
Date string `xml:"date"` // dc:date
|
||||
} `xml:"channel>item"`
|
||||
|
||||
Entries []struct {
|
||||
Title string `xml:"title"`
|
||||
Links []struct {
|
||||
Href string `xml:"href,attr"`
|
||||
Rel string `xml:"rel,attr"`
|
||||
} `xml:"link"`
|
||||
Summary string `xml:"summary"`
|
||||
Content string `xml:"content"`
|
||||
ID string `xml:"id"`
|
||||
Updated string `xml:"updated"`
|
||||
Published string `xml:"published"`
|
||||
} `xml:"entry"`
|
||||
}
|
||||
|
||||
// Parse reads a feed document.
|
||||
func Parse(r io.Reader) (Feed, error) {
|
||||
var doc feedDoc
|
||||
dec := xml.NewDecoder(r)
|
||||
// Feeds in the wild declare windows-1251 and worse. We only ever read
|
||||
// UTF-8; a charset we cannot decode is a feed we do not read, which is
|
||||
// better than mojibake in his notes.
|
||||
dec.Strict = false
|
||||
if err := dec.Decode(&doc); err != nil {
|
||||
return Feed{}, fmt.Errorf("rss: bad xml: %w", err)
|
||||
}
|
||||
|
||||
f := Feed{Title: strings.TrimSpace(doc.ChannelTitle)}
|
||||
if f.Title == "" {
|
||||
f.Title = strings.TrimSpace(doc.AtomTitle)
|
||||
}
|
||||
for _, it := range doc.Items {
|
||||
item := Item{
|
||||
Title: PlainText(it.Title),
|
||||
Link: strings.TrimSpace(it.Link),
|
||||
Summary: PlainText(firstNonEmpty(it.Description, it.Encoded)),
|
||||
Published: parseTime(firstNonEmpty(it.PubDate, it.Date)),
|
||||
ID: strings.TrimSpace(firstNonEmpty(it.GUID, it.Link)),
|
||||
}
|
||||
if item.Title != "" || item.Link != "" {
|
||||
f.Items = append(f.Items, item)
|
||||
}
|
||||
}
|
||||
for _, e := range doc.Entries {
|
||||
link := ""
|
||||
for _, l := range e.Links {
|
||||
if l.Rel == "" || l.Rel == "alternate" {
|
||||
link = strings.TrimSpace(l.Href)
|
||||
break
|
||||
}
|
||||
}
|
||||
if link == "" && len(e.Links) > 0 {
|
||||
link = strings.TrimSpace(e.Links[0].Href)
|
||||
}
|
||||
item := Item{
|
||||
Title: PlainText(e.Title),
|
||||
Link: link,
|
||||
Summary: PlainText(firstNonEmpty(e.Summary, e.Content)),
|
||||
Published: parseTime(firstNonEmpty(e.Published, e.Updated)),
|
||||
ID: strings.TrimSpace(firstNonEmpty(e.ID, link)),
|
||||
}
|
||||
if item.Title != "" || item.Link != "" {
|
||||
f.Items = append(f.Items, item)
|
||||
}
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func firstNonEmpty(vals ...string) string {
|
||||
for _, v := range vals {
|
||||
if strings.TrimSpace(v) != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// timeLayouts — RFC1123/822 for RSS, RFC3339 for Atom, plus the near-misses
|
||||
// real feeds ship (no seconds, numeric zone where a name is expected).
|
||||
var timeLayouts = []string{
|
||||
time.RFC1123Z,
|
||||
time.RFC1123,
|
||||
time.RFC822Z,
|
||||
time.RFC822,
|
||||
time.RFC3339,
|
||||
"2006-01-02T15:04:05Z0700",
|
||||
"2006-01-02 15:04:05",
|
||||
"2006-01-02",
|
||||
"Mon, 02 Jan 2006 15:04:05 -0700",
|
||||
"Mon, 2 Jan 2006 15:04:05 -0700",
|
||||
"Mon, 2 Jan 2006 15:04:05 MST",
|
||||
}
|
||||
|
||||
// parseTime returns the zero time on anything it cannot read. An undated item
|
||||
// is still an item; the poller dedupes by ID, so a missing date costs nothing.
|
||||
func parseTime(s string) time.Time {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return time.Time{}
|
||||
}
|
||||
for _, l := range timeLayouts {
|
||||
if t, err := time.Parse(l, s); err == nil {
|
||||
return t.UTC()
|
||||
}
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
var (
|
||||
// RE2 has no backreferences, so the two tags are spelled out rather than
|
||||
// captured and matched against themselves.
|
||||
scriptRE = regexp.MustCompile(`(?is)<script\b[^>]*>.*?</script>|<style\b[^>]*>.*?</style>`)
|
||||
tagRE = regexp.MustCompile(`(?s)<[^>]*>`)
|
||||
)
|
||||
|
||||
// PlainText strips markup and decodes entities — feed summaries are HTML, and
|
||||
// what reaches a note (and possibly the TTS) must be text. Exported because the
|
||||
// crawler's extractor needs exactly this on a bigger input.
|
||||
func PlainText(s string) string {
|
||||
s = scriptRE.ReplaceAllString(s, " ")
|
||||
s = tagRE.ReplaceAllString(s, " ")
|
||||
s = html.UnescapeString(s)
|
||||
return strings.TrimSpace(strings.Join(strings.Fields(s), " "))
|
||||
}
|
||||
Reference in New Issue
Block a user