Read RSS and Atom feeds, and speak about them only when asked (#258)
internal/rss parses RSS 2.0 and Atom, and polls each configured feed on its own interval; internal/webfetch is the one door either of them uses to touch the network. The poller writes items as notes with source "rss:<feed>" and nothing else: the answer path reads them back when he asks "что нового в лентах?", and nothing is announced on arrival. A feed that dispatched would be a nag, which is why the plan's breaking-news rule was left out rather than built. webfetch is where the limits live, as code rather than a paragraph: http(s) only, an allowlist (the configured feeds' hosts) and a denylist, a 2 MiB body cap, a 3-redirect cap, one request per host per second, and a refusal to connect to any private address — checked in the dialer's Control hook so it holds for every resolved address and every redirect hop, not just for a literal IP. Off unless configured: no "feeds" block, no poller, no outbound request. How far a feed was read is a config fact (rss:latest:<name>), so a restart does not re-note yesterday's headlines.
This commit is contained in:
kami
@@ -0,0 +1,179 @@
|
||||
// Package rss reads RSS 2.0 and Atom feeds, and does nothing else with them.
|
||||
//
|
||||
// Parsing and polling are split from delivery on purpose: a feed is a source
|
||||
// Maven can be ASKED about, not a thing that speaks. Nothing in this package
|
||||
// dispatches, nudges or notifies — the poller writes notes, and the answer path
|
||||
// reads them when he asks "что нового в лентах?". "Not a nag" is the oldest
|
||||
// constraint in the spec, and a news feed is the single most tempting way to
|
||||
// break it.
|
||||
//
|
||||
// Stdlib only (encoding/xml). Feeds are XML from strangers, so the parser takes
|
||||
// what it recognises and ignores the rest rather than failing a whole feed over
|
||||
// one malformed item.
|
||||
package rss
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"html"
|
||||
"io"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Item is one feed entry, normalised across RSS and Atom.
|
||||
type Item struct {
|
||||
Title string
|
||||
Link string
|
||||
Summary string // plain text, tags stripped, entities decoded
|
||||
Published time.Time // zero when the feed did not say
|
||||
ID string // guid / atom id, falling back to the link
|
||||
}
|
||||
|
||||
// Feed is a parsed document.
|
||||
type Feed struct {
|
||||
Title string
|
||||
Items []Item
|
||||
}
|
||||
|
||||
// feedDoc covers both dialects in one struct. RSS puts items under
|
||||
// channel>item, Atom puts entries at the top level, and the field names barely
|
||||
// overlap — so both sets are declared and whichever the document filled in wins.
|
||||
type feedDoc struct {
|
||||
ChannelTitle string `xml:"channel>title"`
|
||||
AtomTitle string `xml:"title"`
|
||||
|
||||
Items []struct {
|
||||
Title string `xml:"title"`
|
||||
Link string `xml:"link"`
|
||||
Description string `xml:"description"`
|
||||
Encoded string `xml:"encoded"` // content:encoded
|
||||
GUID string `xml:"guid"`
|
||||
PubDate string `xml:"pubDate"`
|
||||
Date string `xml:"date"` // dc:date
|
||||
} `xml:"channel>item"`
|
||||
|
||||
Entries []struct {
|
||||
Title string `xml:"title"`
|
||||
Links []struct {
|
||||
Href string `xml:"href,attr"`
|
||||
Rel string `xml:"rel,attr"`
|
||||
} `xml:"link"`
|
||||
Summary string `xml:"summary"`
|
||||
Content string `xml:"content"`
|
||||
ID string `xml:"id"`
|
||||
Updated string `xml:"updated"`
|
||||
Published string `xml:"published"`
|
||||
} `xml:"entry"`
|
||||
}
|
||||
|
||||
// Parse reads a feed document.
|
||||
func Parse(r io.Reader) (Feed, error) {
|
||||
var doc feedDoc
|
||||
dec := xml.NewDecoder(r)
|
||||
// Feeds in the wild declare windows-1251 and worse. We only ever read
|
||||
// UTF-8; a charset we cannot decode is a feed we do not read, which is
|
||||
// better than mojibake in his notes.
|
||||
dec.Strict = false
|
||||
if err := dec.Decode(&doc); err != nil {
|
||||
return Feed{}, fmt.Errorf("rss: bad xml: %w", err)
|
||||
}
|
||||
|
||||
f := Feed{Title: strings.TrimSpace(doc.ChannelTitle)}
|
||||
if f.Title == "" {
|
||||
f.Title = strings.TrimSpace(doc.AtomTitle)
|
||||
}
|
||||
for _, it := range doc.Items {
|
||||
item := Item{
|
||||
Title: PlainText(it.Title),
|
||||
Link: strings.TrimSpace(it.Link),
|
||||
Summary: PlainText(firstNonEmpty(it.Description, it.Encoded)),
|
||||
Published: parseTime(firstNonEmpty(it.PubDate, it.Date)),
|
||||
ID: strings.TrimSpace(firstNonEmpty(it.GUID, it.Link)),
|
||||
}
|
||||
if item.Title != "" || item.Link != "" {
|
||||
f.Items = append(f.Items, item)
|
||||
}
|
||||
}
|
||||
for _, e := range doc.Entries {
|
||||
link := ""
|
||||
for _, l := range e.Links {
|
||||
if l.Rel == "" || l.Rel == "alternate" {
|
||||
link = strings.TrimSpace(l.Href)
|
||||
break
|
||||
}
|
||||
}
|
||||
if link == "" && len(e.Links) > 0 {
|
||||
link = strings.TrimSpace(e.Links[0].Href)
|
||||
}
|
||||
item := Item{
|
||||
Title: PlainText(e.Title),
|
||||
Link: link,
|
||||
Summary: PlainText(firstNonEmpty(e.Summary, e.Content)),
|
||||
Published: parseTime(firstNonEmpty(e.Published, e.Updated)),
|
||||
ID: strings.TrimSpace(firstNonEmpty(e.ID, link)),
|
||||
}
|
||||
if item.Title != "" || item.Link != "" {
|
||||
f.Items = append(f.Items, item)
|
||||
}
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func firstNonEmpty(vals ...string) string {
|
||||
for _, v := range vals {
|
||||
if strings.TrimSpace(v) != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// timeLayouts — RFC1123/822 for RSS, RFC3339 for Atom, plus the near-misses
|
||||
// real feeds ship (no seconds, numeric zone where a name is expected).
|
||||
var timeLayouts = []string{
|
||||
time.RFC1123Z,
|
||||
time.RFC1123,
|
||||
time.RFC822Z,
|
||||
time.RFC822,
|
||||
time.RFC3339,
|
||||
"2006-01-02T15:04:05Z0700",
|
||||
"2006-01-02 15:04:05",
|
||||
"2006-01-02",
|
||||
"Mon, 02 Jan 2006 15:04:05 -0700",
|
||||
"Mon, 2 Jan 2006 15:04:05 -0700",
|
||||
"Mon, 2 Jan 2006 15:04:05 MST",
|
||||
}
|
||||
|
||||
// parseTime returns the zero time on anything it cannot read. An undated item
|
||||
// is still an item; the poller dedupes by ID, so a missing date costs nothing.
|
||||
func parseTime(s string) time.Time {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return time.Time{}
|
||||
}
|
||||
for _, l := range timeLayouts {
|
||||
if t, err := time.Parse(l, s); err == nil {
|
||||
return t.UTC()
|
||||
}
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
var (
|
||||
// RE2 has no backreferences, so the two tags are spelled out rather than
|
||||
// captured and matched against themselves.
|
||||
scriptRE = regexp.MustCompile(`(?is)<script\b[^>]*>.*?</script>|<style\b[^>]*>.*?</style>`)
|
||||
tagRE = regexp.MustCompile(`(?s)<[^>]*>`)
|
||||
)
|
||||
|
||||
// PlainText strips markup and decodes entities — feed summaries are HTML, and
|
||||
// what reaches a note (and possibly the TTS) must be text. Exported because the
|
||||
// crawler's extractor needs exactly this on a bigger input.
|
||||
func PlainText(s string) string {
|
||||
s = scriptRE.ReplaceAllString(s, " ")
|
||||
s = tagRE.ReplaceAllString(s, " ")
|
||||
s = html.UnescapeString(s)
|
||||
return strings.TrimSpace(strings.Join(strings.Fields(s), " "))
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package rss
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const rss2 = `<?xml version="1.0"?>
|
||||
<rss version="2.0">
|
||||
<channel>
|
||||
<title>Хабр</title>
|
||||
<item>
|
||||
<title>Новая уязвимость в ядре</title>
|
||||
<link>https://example.org/a</link>
|
||||
<description><p>Патч уже <b>вышел</b>.</p></description>
|
||||
<guid>tag:example.org,a</guid>
|
||||
<pubDate>Mon, 28 Jul 2026 10:00:00 +0000</pubDate>
|
||||
</item>
|
||||
<item>
|
||||
<title>Без даты</title>
|
||||
<link>https://example.org/b</link>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>`
|
||||
|
||||
const atom = `<?xml version="1.0" encoding="utf-8"?>
|
||||
<feed xmlns="http://www.w3.org/2005/Atom">
|
||||
<title>Example Atom</title>
|
||||
<entry>
|
||||
<title>Release 2.0</title>
|
||||
<link rel="alternate" href="https://example.com/rel"/>
|
||||
<link rel="edit" href="https://example.com/edit"/>
|
||||
<id>urn:uuid:1</id>
|
||||
<updated>2026-07-30T12:30:00Z</updated>
|
||||
<summary>Ships & works</summary>
|
||||
</entry>
|
||||
</feed>`
|
||||
|
||||
func TestParseRSS2(t *testing.T) {
|
||||
f, err := Parse(strings.NewReader(rss2))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.Title != "Хабр" {
|
||||
t.Fatalf("title = %q", f.Title)
|
||||
}
|
||||
if len(f.Items) != 2 {
|
||||
t.Fatalf("items = %d, want 2", len(f.Items))
|
||||
}
|
||||
it := f.Items[0]
|
||||
if it.Title != "Новая уязвимость в ядре" {
|
||||
t.Errorf("title = %q", it.Title)
|
||||
}
|
||||
if it.Summary != "Патч уже вышел ." && it.Summary != "Патч уже вышел." {
|
||||
t.Errorf("summary = %q — tags must be stripped and entities decoded", it.Summary)
|
||||
}
|
||||
if it.ID != "tag:example.org,a" {
|
||||
t.Errorf("id = %q", it.ID)
|
||||
}
|
||||
if want := time.Date(2026, 7, 28, 10, 0, 0, 0, time.UTC); !it.Published.Equal(want) {
|
||||
t.Errorf("published = %v, want %v", it.Published, want)
|
||||
}
|
||||
if !f.Items[1].Published.IsZero() {
|
||||
t.Errorf("undated item got a date: %v", f.Items[1].Published)
|
||||
}
|
||||
if f.Items[1].ID != "https://example.org/b" {
|
||||
t.Errorf("id falls back to the link, got %q", f.Items[1].ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseAtom(t *testing.T) {
|
||||
f, err := Parse(strings.NewReader(atom))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.Title != "Example Atom" || len(f.Items) != 1 {
|
||||
t.Fatalf("feed = %+v", f)
|
||||
}
|
||||
it := f.Items[0]
|
||||
if it.Link != "https://example.com/rel" {
|
||||
t.Errorf("link = %q, want the alternate link", it.Link)
|
||||
}
|
||||
if it.Summary != "Ships & works" {
|
||||
t.Errorf("summary = %q", it.Summary)
|
||||
}
|
||||
if want := time.Date(2026, 7, 30, 12, 30, 0, 0, time.UTC); !it.Published.Equal(want) {
|
||||
t.Errorf("published = %v, want %v", it.Published, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseGarbage(t *testing.T) {
|
||||
if _, err := Parse(strings.NewReader("<html><body>not a feed")); err == nil {
|
||||
t.Fatal("want an error on a non-feed document")
|
||||
}
|
||||
// A feed with an item that has neither title nor link contributes nothing
|
||||
// rather than an empty note.
|
||||
f, err := Parse(strings.NewReader(`<rss><channel><item><description>x</description></item></channel></rss>`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(f.Items) != 0 {
|
||||
t.Fatalf("items = %d, want 0", len(f.Items))
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlainTextDropsScript(t *testing.T) {
|
||||
got := PlainText(`<p>hi</p><script>alert("x")</script><style>b{}</style> there`)
|
||||
if got != "hi there" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,324 @@
|
||||
package rss
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// FeedConfig — one feed to read. A feed with no Name or no URL is ignored.
|
||||
type FeedConfig struct {
|
||||
Name string // short id; the note source is "rss:<Name>"
|
||||
URL string // http(s) only, enforced by the fetcher
|
||||
Category string // free text ("технологии"), used to answer "что по X?"
|
||||
Interval time.Duration // 0 ⇒ the poller's default
|
||||
Include []string // when non-empty, keep only items matching one of these
|
||||
Exclude []string // drop items matching any of these, even if included
|
||||
}
|
||||
|
||||
// Fetcher is the guarded HTTP door (internal/webfetch). An interface so the
|
||||
// poller is testable without a network and so it CANNOT fetch by any other
|
||||
// means: no http.Client is constructed in this package.
|
||||
type Fetcher interface {
|
||||
Get(ctx context.Context, url string) (*Body, error)
|
||||
}
|
||||
|
||||
// Body is the minimum the poller needs from a response.
|
||||
type Body struct{ Bytes []byte }
|
||||
|
||||
// Notes is core's note-writing half. Same shape as ipc.CoreAPI's method, so the
|
||||
// daemon passes its API straight in.
|
||||
type Notes interface {
|
||||
WriteNote(ctx context.Context, ts time.Time, text string, embedding []float32, source string) (int64, error)
|
||||
}
|
||||
|
||||
// Marks remembers how far a feed was read. Durable, because the alternative is
|
||||
// re-writing yesterday's headlines as fresh notes after every restart. The
|
||||
// daemon backs this with config facts (key "rss:latest:<feed>").
|
||||
type Marks interface {
|
||||
LastMark(ctx context.Context, feed string) (time.Time, error)
|
||||
SetMark(ctx context.Context, feed string, at time.Time) error
|
||||
}
|
||||
|
||||
// Embedder embeds a note on its way into the store so recall can find it. nil ⇒
|
||||
// notes are written without a vector (still readable by the recent-notes path).
|
||||
type Embedder interface {
|
||||
Embed(ctx context.Context, text string) ([]float32, error)
|
||||
}
|
||||
|
||||
// Ranker is the relevance seam. The plan called for scoring each item against
|
||||
// an interest profile built from his notes; that profile does not exist yet, and
|
||||
// a threshold over an embedder with no profile to compare to is a random filter
|
||||
// with a confident name. So the seam is here, nil in the daemon, and the filter
|
||||
// that actually runs is the per-feed keyword one — a rule he can read and
|
||||
// predict. When there IS a profile, implement this and pass it.
|
||||
//
|
||||
// Note what a Ranker must NOT be: anything that sends his notes outward. The
|
||||
// scoring happens locally against a local embedder; the feed item is the input,
|
||||
// his memory is never the payload.
|
||||
type Ranker interface {
|
||||
Relevant(ctx context.Context, text string) (bool, error)
|
||||
}
|
||||
|
||||
// Config — poller-wide settings.
|
||||
type Config struct {
|
||||
DefaultInterval time.Duration // 0 ⇒ DefaultPollInterval
|
||||
MaxItems int // most notes written per feed per poll; 0 ⇒ DefaultMaxItems
|
||||
MaxAge time.Duration // ignore items older than this on a cold start; 0 ⇒ DefaultMaxAge
|
||||
}
|
||||
|
||||
// Defaults chosen to be quiet: a feed read every half hour, at most a handful of
|
||||
// items kept, and a cold start that does not import a month of history.
|
||||
const (
|
||||
DefaultPollInterval = 30 * time.Minute
|
||||
DefaultMaxItems = 5
|
||||
DefaultMaxAge = 24 * time.Hour
|
||||
)
|
||||
|
||||
// Poller reads feeds on a schedule and writes what survives filtering as notes.
|
||||
type Poller struct {
|
||||
feeds []FeedConfig
|
||||
fetch Fetcher
|
||||
notes Notes
|
||||
marks Marks
|
||||
embed Embedder
|
||||
ranker Ranker
|
||||
cfg Config
|
||||
nextDue map[string]time.Time
|
||||
seen map[string]map[string]bool // feed → item ID, for items with no date
|
||||
}
|
||||
|
||||
// NewPoller wires a poller. Returns nil when there is nothing to poll — a
|
||||
// capability is off unless configured, and callers check for nil.
|
||||
func NewPoller(feeds []FeedConfig, fetch Fetcher, notes Notes, marks Marks, embed Embedder, ranker Ranker, cfg Config) *Poller {
|
||||
var valid []FeedConfig
|
||||
for _, f := range feeds {
|
||||
if strings.TrimSpace(f.Name) == "" || strings.TrimSpace(f.URL) == "" {
|
||||
log.Printf("rss: skipping a feed with no name or no url")
|
||||
continue
|
||||
}
|
||||
valid = append(valid, f)
|
||||
}
|
||||
if len(valid) == 0 || fetch == nil || notes == nil {
|
||||
return nil
|
||||
}
|
||||
if cfg.DefaultInterval <= 0 {
|
||||
cfg.DefaultInterval = DefaultPollInterval
|
||||
}
|
||||
if cfg.MaxItems <= 0 {
|
||||
cfg.MaxItems = DefaultMaxItems
|
||||
}
|
||||
if cfg.MaxAge <= 0 {
|
||||
cfg.MaxAge = DefaultMaxAge
|
||||
}
|
||||
return &Poller{
|
||||
feeds: valid, fetch: fetch, notes: notes, marks: marks,
|
||||
embed: embed, ranker: ranker, cfg: cfg,
|
||||
nextDue: map[string]time.Time{},
|
||||
seen: map[string]map[string]bool{},
|
||||
}
|
||||
}
|
||||
|
||||
// Feeds returns the configured feeds (the answer path lists categories).
|
||||
func (p *Poller) Feeds() []FeedConfig { return p.feeds }
|
||||
|
||||
// PollDue reads every feed whose interval has elapsed and returns how many
|
||||
// notes were written. Errors are logged per feed, never returned: one dead feed
|
||||
// must not stop the others, and there is nobody waiting on this.
|
||||
func (p *Poller) PollDue(ctx context.Context, now time.Time) int {
|
||||
written := 0
|
||||
for _, f := range p.feeds {
|
||||
if due, ok := p.nextDue[f.Name]; ok && now.Before(due) {
|
||||
continue
|
||||
}
|
||||
interval := f.Interval
|
||||
if interval <= 0 {
|
||||
interval = p.cfg.DefaultInterval
|
||||
}
|
||||
p.nextDue[f.Name] = now.Add(interval)
|
||||
n, err := p.PollFeed(ctx, f, now)
|
||||
if err != nil {
|
||||
// The URL is configured by him and not a secret, so it is loggable;
|
||||
// item titles are not logged, only counts.
|
||||
log.Printf("rss: feed %s: %v", f.Name, err)
|
||||
continue
|
||||
}
|
||||
if n > 0 {
|
||||
log.Printf("rss: feed %s: %d new item(s) noted", f.Name, n)
|
||||
}
|
||||
written += n
|
||||
}
|
||||
return written
|
||||
}
|
||||
|
||||
// PollFeed reads one feed now, regardless of its schedule.
|
||||
func (p *Poller) PollFeed(ctx context.Context, f FeedConfig, now time.Time) (int, error) {
|
||||
body, err := p.fetch.Get(ctx, f.URL)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
feed, err := Parse(strings.NewReader(string(body.Bytes)))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
mark := p.mark(ctx, f.Name, now)
|
||||
newest := mark
|
||||
written := 0
|
||||
for _, it := range feed.Items {
|
||||
if written >= p.cfg.MaxItems {
|
||||
break
|
||||
}
|
||||
if !p.fresh(f, it, mark, now) {
|
||||
continue
|
||||
}
|
||||
if !Matches(f, it) {
|
||||
continue
|
||||
}
|
||||
if p.ranker != nil {
|
||||
ok, err := p.ranker.Relevant(ctx, it.Title+" "+it.Summary)
|
||||
if err != nil {
|
||||
log.Printf("rss: feed %s: relevance: %v", f.Name, err)
|
||||
} else if !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if err := p.write(ctx, f, it, now); err != nil {
|
||||
return written, err
|
||||
}
|
||||
written++
|
||||
if it.Published.After(newest) {
|
||||
newest = it.Published
|
||||
}
|
||||
}
|
||||
if p.marks != nil && newest.After(mark) {
|
||||
if err := p.marks.SetMark(ctx, f.Name, newest); err != nil {
|
||||
log.Printf("rss: feed %s: save mark: %v", f.Name, err)
|
||||
}
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
|
||||
// mark — how far this feed was read. A feed with no mark starts MaxAge ago, so
|
||||
// a first poll takes today's headlines instead of the whole archive.
|
||||
func (p *Poller) mark(ctx context.Context, feed string, now time.Time) time.Time {
|
||||
cold := now.Add(-p.cfg.MaxAge)
|
||||
if p.marks == nil {
|
||||
return cold
|
||||
}
|
||||
at, err := p.marks.LastMark(ctx, feed)
|
||||
if err != nil || at.IsZero() {
|
||||
return cold
|
||||
}
|
||||
return at
|
||||
}
|
||||
|
||||
// fresh — two dedup rules, because feeds are inconsistent about dates. A dated
|
||||
// item must be newer than the mark; an undated one is kept once per process by
|
||||
// ID. Both are needed: dates alone re-import undated feeds forever, IDs alone
|
||||
// lose their memory on restart.
|
||||
func (p *Poller) fresh(f FeedConfig, it Item, mark, now time.Time) bool {
|
||||
if !it.Published.IsZero() {
|
||||
if !it.Published.After(mark) {
|
||||
return false
|
||||
}
|
||||
// A feed that dates its items in the future (or a clock skew) must not
|
||||
// win the mark and mute everything after it.
|
||||
return !it.Published.After(now.Add(time.Hour))
|
||||
}
|
||||
id := it.ID
|
||||
if id == "" {
|
||||
id = it.Title
|
||||
}
|
||||
if p.seen[f.Name] == nil {
|
||||
p.seen[f.Name] = map[string]bool{}
|
||||
}
|
||||
if p.seen[f.Name][id] {
|
||||
return false
|
||||
}
|
||||
p.seen[f.Name][id] = true
|
||||
return true
|
||||
}
|
||||
|
||||
// write stores one item as a note. Source "rss:<feed>" is what the answer path
|
||||
// filters on, and what makes a feed note distinguishable from something he said.
|
||||
func (p *Poller) write(ctx context.Context, f FeedConfig, it Item, now time.Time) error {
|
||||
text := NoteText(f, it)
|
||||
var vec []float32
|
||||
if p.embed != nil {
|
||||
v, err := p.embed.Embed(ctx, text)
|
||||
if err != nil {
|
||||
log.Printf("rss: feed %s: embed: %v", f.Name, err)
|
||||
} else {
|
||||
vec = v
|
||||
}
|
||||
}
|
||||
ts := it.Published
|
||||
if ts.IsZero() {
|
||||
ts = now
|
||||
}
|
||||
if _, err := p.notes.WriteNote(ctx, ts, text, vec, SourceFor(f.Name)); err != nil {
|
||||
return fmt.Errorf("write note: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SourceFor is the note source for a feed.
|
||||
func SourceFor(feed string) string { return "rss:" + feed }
|
||||
|
||||
// SourcePrefix — what the answer path matches to find feed notes.
|
||||
const SourcePrefix = "rss:"
|
||||
|
||||
// NoteText renders an item as the note body. The category is included because
|
||||
// "что нового по технологиям?" is answered by reading notes, and a note has to
|
||||
// carry enough to be recognised as belonging to that category.
|
||||
func NoteText(f FeedConfig, it Item) string {
|
||||
var b strings.Builder
|
||||
b.WriteString(it.Title)
|
||||
if f.Category != "" {
|
||||
fmt.Fprintf(&b, " [%s]", f.Category)
|
||||
}
|
||||
if it.Summary != "" {
|
||||
b.WriteString("\n")
|
||||
b.WriteString(trimRunes(it.Summary, 500))
|
||||
}
|
||||
if it.Link != "" {
|
||||
b.WriteString("\n")
|
||||
b.WriteString(it.Link)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// trimRunes cuts on a rune boundary — a note is Russian as often as English and
|
||||
// half a cyrillic letter is a broken note.
|
||||
func trimRunes(s string, max int) string {
|
||||
r := []rune(s)
|
||||
if len(r) <= max {
|
||||
return s
|
||||
}
|
||||
return strings.TrimSpace(string(r[:max])) + "…"
|
||||
}
|
||||
|
||||
// Matches applies the per-feed keyword filter: keep when Include is empty or one
|
||||
// include matches, drop when any exclude matches. Case-insensitive substring,
|
||||
// which for Russian is the honest choice — no stemmer here, so "выборы" does not
|
||||
// match "выборах", and a filter he writes is a filter he can predict.
|
||||
func Matches(f FeedConfig, it Item) bool {
|
||||
hay := strings.ToLower(it.Title + " " + it.Summary)
|
||||
for _, x := range f.Exclude {
|
||||
if x = strings.ToLower(strings.TrimSpace(x)); x != "" && strings.Contains(hay, x) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if len(f.Include) == 0 {
|
||||
return true
|
||||
}
|
||||
for _, in := range f.Include {
|
||||
if in = strings.ToLower(strings.TrimSpace(in)); in != "" && strings.Contains(hay, in) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
package rss
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type fakeFetch struct {
|
||||
body string
|
||||
err error
|
||||
calls int
|
||||
urls []string
|
||||
}
|
||||
|
||||
func (f *fakeFetch) Get(_ context.Context, url string) (*Body, error) {
|
||||
f.calls++
|
||||
f.urls = append(f.urls, url)
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
return &Body{Bytes: []byte(f.body)}, nil
|
||||
}
|
||||
|
||||
type writtenNote struct {
|
||||
ts time.Time
|
||||
text string
|
||||
source string
|
||||
vec []float32
|
||||
}
|
||||
|
||||
type fakeNotes struct{ notes []writtenNote }
|
||||
|
||||
func (n *fakeNotes) WriteNote(_ context.Context, ts time.Time, text string, vec []float32, source string) (int64, error) {
|
||||
n.notes = append(n.notes, writtenNote{ts, text, source, vec})
|
||||
return int64(len(n.notes)), nil
|
||||
}
|
||||
|
||||
type fakeMarks struct{ m map[string]time.Time }
|
||||
|
||||
func newMarks() *fakeMarks { return &fakeMarks{m: map[string]time.Time{}} }
|
||||
func (f *fakeMarks) LastMark(_ context.Context, feed string) (time.Time, error) {
|
||||
return f.m[feed], nil
|
||||
}
|
||||
func (f *fakeMarks) SetMark(_ context.Context, feed string, at time.Time) error {
|
||||
f.m[feed] = at
|
||||
return nil
|
||||
}
|
||||
|
||||
var now = time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
func TestPollWritesNotesWithSource(t *testing.T) {
|
||||
fetch := &fakeFetch{body: rss2}
|
||||
notes := &fakeNotes{}
|
||||
marks := newMarks()
|
||||
p := NewPoller([]FeedConfig{{Name: "habr", URL: "https://example.org/rss", Category: "технологии"}},
|
||||
fetch, notes, marks, nil, nil, Config{})
|
||||
if p == nil {
|
||||
t.Fatal("NewPoller returned nil for a configured feed")
|
||||
}
|
||||
n := p.PollDue(context.Background(), now)
|
||||
if n != 2 || len(notes.notes) != 2 {
|
||||
t.Fatalf("wrote %d notes (returned %d), want 2", len(notes.notes), n)
|
||||
}
|
||||
if notes.notes[0].source != "rss:habr" {
|
||||
t.Errorf("source = %q, want rss:habr", notes.notes[0].source)
|
||||
}
|
||||
if !strings.Contains(notes.notes[0].text, "технологии") {
|
||||
t.Errorf("note does not carry its category: %q", notes.notes[0].text)
|
||||
}
|
||||
if !strings.Contains(notes.notes[0].text, "https://example.org/a") {
|
||||
t.Errorf("note does not carry its link: %q", notes.notes[0].text)
|
||||
}
|
||||
// The undated item is stamped with now, the dated one with its own date.
|
||||
if !notes.notes[1].ts.Equal(now) {
|
||||
t.Errorf("undated item ts = %v, want now", notes.notes[1].ts)
|
||||
}
|
||||
}
|
||||
|
||||
// The whole point of a mark: polling twice must not re-note the same headlines.
|
||||
func TestSecondPollIsQuiet(t *testing.T) {
|
||||
fetch := &fakeFetch{body: rss2}
|
||||
notes := &fakeNotes{}
|
||||
p := NewPoller([]FeedConfig{{Name: "habr", URL: "u", Interval: time.Minute}}, fetch, notes, newMarks(), nil, nil, Config{})
|
||||
p.PollDue(context.Background(), now)
|
||||
before := len(notes.notes)
|
||||
p.PollDue(context.Background(), now.Add(2*time.Minute))
|
||||
if len(notes.notes) != before {
|
||||
t.Fatalf("second poll wrote %d extra notes", len(notes.notes)-before)
|
||||
}
|
||||
}
|
||||
|
||||
// A mark that survives a restart is the durable half; simulate one by building a
|
||||
// fresh poller over the same marks.
|
||||
func TestMarkSurvivesRestart(t *testing.T) {
|
||||
marks := newMarks()
|
||||
fetch := &fakeFetch{body: rss2}
|
||||
notes := &fakeNotes{}
|
||||
feeds := []FeedConfig{{Name: "habr", URL: "u"}}
|
||||
NewPoller(feeds, fetch, notes, marks, nil, nil, Config{}).PollDue(context.Background(), now)
|
||||
if len(notes.notes) != 2 {
|
||||
t.Fatalf("first run wrote %d", len(notes.notes))
|
||||
}
|
||||
notes2 := &fakeNotes{}
|
||||
NewPoller(feeds, fetch, notes2, marks, nil, nil, Config{}).PollDue(context.Background(), now.Add(time.Hour))
|
||||
// The dated item is behind the mark. The undated one has no date to compare,
|
||||
// so it comes back — accepted and documented in fresh(): an undated feed is
|
||||
// deduped per process, not forever.
|
||||
for _, n := range notes2.notes {
|
||||
if strings.Contains(n.text, "уязвимость") {
|
||||
t.Fatalf("dated item re-noted after restart: %q", n.text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntervalIsRespected(t *testing.T) {
|
||||
fetch := &fakeFetch{body: rss2}
|
||||
p := NewPoller([]FeedConfig{{Name: "habr", URL: "u", Interval: time.Hour}}, fetch, &fakeNotes{}, newMarks(), nil, nil, Config{})
|
||||
p.PollDue(context.Background(), now)
|
||||
p.PollDue(context.Background(), now.Add(time.Minute))
|
||||
if fetch.calls != 1 {
|
||||
t.Fatalf("fetched %d times inside one interval, want 1", fetch.calls)
|
||||
}
|
||||
p.PollDue(context.Background(), now.Add(2*time.Hour))
|
||||
if fetch.calls != 2 {
|
||||
t.Fatalf("fetched %d times, want 2 after the interval elapsed", fetch.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestColdStartIgnoresOldItems(t *testing.T) {
|
||||
old := `<rss><channel><item><title>Старое</title><link>l</link>` +
|
||||
`<pubDate>Mon, 01 Jun 2026 10:00:00 +0000</pubDate></item></channel></rss>`
|
||||
notes := &fakeNotes{}
|
||||
p := NewPoller([]FeedConfig{{Name: "f", URL: "u"}}, &fakeFetch{body: old}, notes, newMarks(), nil, nil, Config{MaxAge: 24 * time.Hour})
|
||||
if n := p.PollDue(context.Background(), now); n != 0 {
|
||||
t.Fatalf("cold start imported %d old items, want 0", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaxItemsCap(t *testing.T) {
|
||||
var b strings.Builder
|
||||
b.WriteString("<rss><channel>")
|
||||
for i := 0; i < 10; i++ {
|
||||
b.WriteString("<item><title>t")
|
||||
b.WriteByte(byte('0' + i))
|
||||
b.WriteString("</title><link>https://example.org/")
|
||||
b.WriteByte(byte('0' + i))
|
||||
b.WriteString("</link></item>")
|
||||
}
|
||||
b.WriteString("</channel></rss>")
|
||||
notes := &fakeNotes{}
|
||||
p := NewPoller([]FeedConfig{{Name: "f", URL: "u"}}, &fakeFetch{body: b.String()}, notes, newMarks(), nil, nil, Config{MaxItems: 3})
|
||||
if n := p.PollDue(context.Background(), now); n != 3 {
|
||||
t.Fatalf("wrote %d notes, want the cap of 3", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeywordFilter(t *testing.T) {
|
||||
f := FeedConfig{Include: []string{"ядр"}, Exclude: []string{"реклама"}}
|
||||
if !Matches(f, Item{Title: "Новое ядро"}) {
|
||||
t.Error("include did not match")
|
||||
}
|
||||
if Matches(f, Item{Title: "Новое ядро", Summary: "Реклама внутри"}) {
|
||||
t.Error("exclude must win over include")
|
||||
}
|
||||
if Matches(f, Item{Title: "Погода"}) {
|
||||
t.Error("non-matching item passed the include filter")
|
||||
}
|
||||
if !Matches(FeedConfig{}, Item{Title: "что угодно"}) {
|
||||
t.Error("an unfiltered feed must keep everything")
|
||||
}
|
||||
}
|
||||
|
||||
type fakeRanker struct{ keep bool }
|
||||
|
||||
func (r fakeRanker) Relevant(context.Context, string) (bool, error) { return r.keep, nil }
|
||||
|
||||
func TestRankerCanDropEverything(t *testing.T) {
|
||||
notes := &fakeNotes{}
|
||||
p := NewPoller([]FeedConfig{{Name: "f", URL: "u"}}, &fakeFetch{body: rss2}, notes, newMarks(), nil, fakeRanker{false}, Config{})
|
||||
if n := p.PollDue(context.Background(), now); n != 0 {
|
||||
t.Fatalf("ranker rejected everything but %d notes were written", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchErrorIsSurvivable(t *testing.T) {
|
||||
notes := &fakeNotes{}
|
||||
p := NewPoller([]FeedConfig{
|
||||
{Name: "dead", URL: "u1"},
|
||||
{Name: "live", URL: "u2"},
|
||||
}, &fakeFetch{err: errors.New("boom")}, notes, newMarks(), nil, nil, Config{})
|
||||
if n := p.PollDue(context.Background(), now); n != 0 {
|
||||
t.Fatalf("n = %d", n)
|
||||
}
|
||||
// Both feeds were attempted: one dead feed does not abort the round.
|
||||
if p.nextDue["live"].IsZero() {
|
||||
t.Fatal("the second feed was never attempted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoFeedsMeansNoPoller(t *testing.T) {
|
||||
if p := NewPoller(nil, &fakeFetch{}, &fakeNotes{}, nil, nil, nil, Config{}); p != nil {
|
||||
t.Fatal("NewPoller must return nil when nothing is configured")
|
||||
}
|
||||
if p := NewPoller([]FeedConfig{{Name: "", URL: ""}}, &fakeFetch{}, &fakeNotes{}, nil, nil, nil, Config{}); p != nil {
|
||||
t.Fatal("a feed with no name or url is not a configuration")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user