Read RSS and Atom feeds, and speak about them only when asked (#258)

internal/rss parses RSS 2.0 and Atom, and polls each configured feed on its own
interval; internal/webfetch is the one door either of them uses to touch the
network. The poller writes items as notes with source "rss:<feed>" and nothing
else: the answer path reads them back when he asks "что нового в лентах?", and
nothing is announced on arrival. A feed that dispatched would be a nag, which is
why the plan's breaking-news rule was left out rather than built.

webfetch is where the limits live, as code rather than a paragraph: http(s)
only, an allowlist (the configured feeds' hosts) and a denylist, a 2 MiB body
cap, a 3-redirect cap, one request per host per second, and a refusal to connect
to any private address — checked in the dialer's Control hook so it holds for
every resolved address and every redirect hop, not just for a literal IP.

Off unless configured: no "feeds" block, no poller, no outbound request. How far
a feed was read is a config fact (rss:latest:<name>), so a restart does not
re-note yesterday's headlines.
This commit is contained in:
kami
2026-08-01 03:27:45 +04:00
parent ee7bec11e3
commit cb3641e7bb
17 changed files with 2069 additions and 0 deletions
+109
View File
@@ -0,0 +1,109 @@
package router
import "strings"
// Feed queries — "что нового в лентах?", "что нового по технологиям?"
// (Vikunja #258).
//
// Deterministic matching, like the calendar, plan and habit matchers above it:
// the LLM router says this is a query, and this decides whether it is a question
// about the feeds. A model deciding that would occasionally answer "что нового?"
// out of world knowledge, which is the one thing a feed reader exists to avoid.
// FeedQuery — a parsed "what's new" question. Category is the topic he named
// ("технологии"), empty when he asked about the feeds in general.
type FeedQuery struct {
Category string
}
// feedNouns — the words that make a question about the feeds themselves.
var feedNouns = []string{
"лента", "ленте", "ленты", "лентах", "лентам",
"новости", "новостей", "новостях", "новостям",
"новое", "нового", "новенького",
"feed", "feeds", "news", "headlines",
}
// newnessMarkers — the "что нового" half. "нового" alone is in feedNouns
// because it carries the question on its own ("что нового?"); a bare "лента"
// needs the ask, which is what askMarkers below is for.
var askMarkers = []string{
"что", "какие", "какое", "расскажи", "почитай", "прочитай", "покажи",
"what", "any", "tell", "show", "read",
}
// ParseFeedQuery reports whether an utterance asks what is new in the feeds, and
// which topic if it names one after "по"/"о"/"про"/"about".
//
// Both a feed noun and an ask are required. "у меня новая лента в инстаграме" is
// a statement and must not be read as a request to recite headlines.
func ParseFeedQuery(text string) (FeedQuery, bool) {
toks := planTokens(text)
noun, ask := false, false
for _, t := range toks {
for _, n := range feedNouns {
if t == n {
noun = true
break
}
}
for _, a := range askMarkers {
if t == a {
ask = true
break
}
}
}
if !noun || !ask {
return FeedQuery{}, false
}
return FeedQuery{Category: feedCategory(toks)}, true
}
// categoryPreps — the prepositions a topic follows. Russian marks the topic with
// a preposition ("по технологиям", "про политику"), so the word after one is the
// category; there is no stemming here, and the match against the configured
// category is a prefix comparison for exactly that reason.
var categoryPreps = map[string]bool{"по": true, "о": true, "об": true, "про": true, "about": true, "on": true}
func feedCategory(toks []string) string {
for i, t := range toks {
if categoryPreps[t] && i+1 < len(toks) {
next := toks[i+1]
// "по новостям" names no topic, it repeats the noun.
for _, n := range feedNouns {
if next == n {
return ""
}
}
return next
}
}
return ""
}
// CategoryMatches reports whether a note's text plausibly belongs to the
// category he named. Russian inflects the topic ("технологиям" vs the configured
// "технологии"), and there is no stemmer in this repo, so the comparison is on a
// common prefix — long enough that "полит" and "погод" stay apart, short enough
// to survive a case ending.
func CategoryMatches(text, category string) bool {
if category == "" {
return true
}
stem := categoryStem(category)
if stem == "" {
return false
}
return strings.Contains(strings.ToLower(text), stem)
}
// categoryStem cuts a word down to the part inflection leaves alone. 5 runes is
// the compromise: shorter words are used whole.
func categoryStem(word string) string {
r := []rune(strings.ToLower(strings.TrimSpace(word)))
if len(r) > 5 {
r = r[:5]
}
return string(r)
}
+48
View File
@@ -0,0 +1,48 @@
package router
import "testing"
func TestParseFeedQuery(t *testing.T) {
cases := []struct {
text string
ok bool
category string
}{
{"что нового в лентах?", true, ""},
{"что нового?", true, ""},
{"какие новости?", true, ""},
{"что нового по технологиям?", true, "технологиям"},
{"расскажи новости про политику", true, "политику"},
{"что нового по новостям", true, ""},
{"what's new in the feeds?", true, ""},
{"any news about kubernetes", true, "kubernetes"},
// Statements, not requests.
{"у меня новая лента в инстаграме", false, ""},
{"новости меня утомили", false, ""},
{"напомни полить цветы", false, ""},
{"", false, ""},
}
for _, c := range cases {
q, ok := ParseFeedQuery(c.text)
if ok != c.ok {
t.Errorf("ParseFeedQuery(%q) ok = %v, want %v", c.text, ok, c.ok)
continue
}
if ok && q.Category != c.category {
t.Errorf("ParseFeedQuery(%q) category = %q, want %q", c.text, q.Category, c.category)
}
}
}
func TestCategoryMatches(t *testing.T) {
// The inflected form he says must match the form the config spells.
if !CategoryMatches("Новый релиз [технологии]", "технологиям") {
t.Error("inflected category did not match")
}
if CategoryMatches("Новый релиз [технологии]", "политику") {
t.Error("unrelated category matched")
}
if !CategoryMatches("anything", "") {
t.Error("an empty category must match everything")
}
}