Read RSS and Atom feeds, and speak about them only when asked (#258)

internal/rss parses RSS 2.0 and Atom, and polls each configured feed on its own interval; internal/webfetch is the one door either of them uses to touch the network. The poller writes items as notes with source "rss:<feed>" and nothing else: the answer path reads them back when he asks "что нового в лентах?", and nothing is announced on arrival. A feed that dispatched would be a nag, which is why the plan's breaking-news rule was left out rather than built. webfetch is where the limits live, as code rather than a paragraph: http(s) only, an allowlist (the configured feeds' hosts) and a denylist, a 2 MiB body cap, a 3-redirect cap, one request per host per second, and a refusal to connect to any private address — checked in the dialer's Control hook so it holds for every resolved address and every redirect hop, not just for a literal IP. Off unless configured: no "feeds" block, no poller, no outbound request. How far a feed was read is a config fact (rss:latest:<name>), so a restart does not re-note yesterday's headlines.
2026-08-01 03:27:45 +04:00
parent ee7bec11e3
commit cb3641e7bb
17 changed files with 2069 additions and 0 deletions
@@ -0,0 +1,301 @@
+// Package webfetch is the one door Maven uses to read something off the
+// network, and it is a narrow one.
+//
+// "Never phones home" stopped being a hard constraint on 2026-07-31, but what
+// replaced it is not "she may fetch anything": local sources come first (Kiwix
+// on the box), external fetching is off unless configured, and only the
+// utterance ever leaves — never his notes, facts or history. That policy is
+// enforced by the callers. What THIS package enforces is the part that must be
+// code rather than a paragraph in a plan, because it protects the homelab from
+// its own assistant:
+//
+//   - http/https only — no file://, no ftp://, no gopher;
+//   - no private address, ever: loopback, RFC1918 (which is what makes the
+//     10.42.0.0/24 wireguard tunnel and the 192.168.1.0/24 LAN unreachable),
+//     link-local incl. the 169.254.169.254 cloud metadata address, CGNAT,
+//     unique-local v6. Checked in the dialer's Control hook, so it holds for
+//     every address the resolver returns AND for every hop of a redirect
+//     chain — a DNS name that resolves to 127.0.0.1 is refused at connect
+//     time, which a pre-flight lookup could not promise (rebinding);
+//   - an allowlist, when one is configured, and a denylist that always wins;
+//   - a response size cap, a total timeout, a redirect cap;
+//   - one request per host per interval, so a poll loop with a bug is slow
+//     rather than an outbound flood.
+//
+// Everything above is on by default with sane numbers: a zero Config is a
+// usable, conservative fetcher. There is no cache and no retry — a feed poll
+// or a page read that fails is simply not answered this round.
+package webfetch
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+)
+
+// Defaults. Small on purpose: this reads feeds and article pages, not ISOs.
+const (
+	DefaultTimeout      = 20 * time.Second
+	DefaultMaxBytes     = 2 << 20 // 2 MiB
+	DefaultMaxRedirects = 3
+	DefaultHostInterval = time.Second
+	DefaultUserAgent    = "Maven/1.0 (self-hosted personal assistant)"
+)
+
+// Errors callers distinguish. Everything else is wrapped transport error.
+var (
+	ErrScheme    = errors.New("webfetch: only http and https are allowed")
+	ErrBlocked   = errors.New("webfetch: host is not allowed")
+	ErrPrivate   = errors.New("webfetch: refusing to connect to a private address")
+	ErrTooLarge  = errors.New("webfetch: response exceeds the size cap")
+	ErrRedirects = errors.New("webfetch: too many redirects")
+	ErrStatus    = errors.New("webfetch: non-2xx status")
+)
+
+// Config are the limits. Every zero value means "the default above", so
+// Config{} is safe; the only field that changes behaviour by being empty is
+// AllowHosts (empty ⇒ any public host that is not denied).
+type Config struct {
+	// AllowHosts — when non-empty, the ONLY hosts that may be fetched. An
+	// entry matches the host itself and its subdomains ("example.com" allows
+	// "news.example.com"). This is the knob to reach for when a capability
+	// should read two feeds and nothing else.
+	AllowHosts []string
+	// DenyHosts — same matching, checked first and always winning.
+	DenyHosts []string
+
+	Timeout      time.Duration // whole request, including redirects and body read
+	MaxBytes     int64         // response body cap
+	MaxRedirects int           // 0 ⇒ default; negative ⇒ no redirects followed
+	HostInterval time.Duration // minimum spacing between requests to one host
+	UserAgent    string
+
+	// AllowPrivate disables the private-address guard. It exists for tests
+	// (httptest listens on 127.0.0.1) and for an explicitly configured
+	// on-box mirror. Nothing in deploy/mavend.json sets it, and it should
+	// stay that way: with it on, any URL Maven is handed becomes an SSRF
+	// probe of the LAN and the wireguard range.
+	AllowPrivate bool
+}
+
+// Response is a fetched body, already bounded by MaxBytes.
+type Response struct {
+	URL         string // final URL after redirects
+	Status      int
+	ContentType string
+	Body        []byte
+}
+
+// Fetcher performs guarded GETs. Safe for concurrent use; the per-host rate
+// limiter is shared, which is the point of sharing one Fetcher.
+type Fetcher struct {
+	cfg  Config
+	http *http.Client
+
+	mu   sync.Mutex
+	last map[string]time.Time // host → when we last dialed it
+}
+
+// New builds a fetcher from cfg, filling in defaults.
+func New(cfg Config) *Fetcher {
+	if cfg.Timeout <= 0 {
+		cfg.Timeout = DefaultTimeout
+	}
+	if cfg.MaxBytes <= 0 {
+		cfg.MaxBytes = DefaultMaxBytes
+	}
+	if cfg.MaxRedirects == 0 {
+		cfg.MaxRedirects = DefaultMaxRedirects
+	}
+	if cfg.HostInterval <= 0 {
+		cfg.HostInterval = DefaultHostInterval
+	}
+	if cfg.UserAgent == "" {
+		cfg.UserAgent = DefaultUserAgent
+	}
+	f := &Fetcher{cfg: cfg, last: map[string]time.Time{}}
+
+	dialer := &net.Dialer{Timeout: 10 * time.Second}
+	if !cfg.AllowPrivate {
+		// The guard lives here rather than in a pre-flight net.LookupHost so
+		// that it sees the address actually being connected to: every A/AAAA
+		// the resolver handed back, on every redirect hop, with no window in
+		// which the name could be re-pointed at the LAN.
+		dialer.Control = func(_, address string, _ syscall.RawConn) error {
+			host, _, err := net.SplitHostPort(address)
+			if err != nil {
+				return err
+			}
+			ip := net.ParseIP(host)
+			if ip == nil || IsPrivateIP(ip) {
+				return fmt.Errorf("%w: %s", ErrPrivate, host)
+			}
+			return nil
+		}
+	}
+	f.http = &http.Client{
+		Timeout:   cfg.Timeout,
+		Transport: &http.Transport{DialContext: dialer.DialContext},
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			if len(via) > f.cfg.MaxRedirects {
+				return ErrRedirects
+			}
+			// A redirect is a fresh URL and gets the full check: an allowed
+			// host must not be able to bounce us onto a denied one.
+			return f.checkURL(req.URL)
+		},
+	}
+	return f
+}
+
+// Get fetches rawURL. The body is capped: a larger response is an error, not a
+// truncation, because half an XML document is worse than none.
+func (f *Fetcher) Get(ctx context.Context, rawURL string) (*Response, error) {
+	u, err := url.Parse(strings.TrimSpace(rawURL))
+	if err != nil {
+		return nil, fmt.Errorf("webfetch: bad url %q: %w", rawURL, err)
+	}
+	if err := f.checkURL(u); err != nil {
+		return nil, err
+	}
+	if err := f.waitTurn(ctx, u.Hostname()); err != nil {
+		return nil, err
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("User-Agent", f.cfg.UserAgent)
+	req.Header.Set("Accept-Encoding", "identity")
+
+	resp, err := f.http.Do(req)
+	if err != nil {
+		// http.Client wraps our sentinels in *url.Error; unwrap so callers can
+		// still tell "blocked" from "the network is down".
+		for _, sentinel := range []error{ErrPrivate, ErrBlocked, ErrRedirects, ErrScheme} {
+			if errors.Is(err, sentinel) {
+				return nil, err
+			}
+		}
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(io.LimitReader(resp.Body, f.cfg.MaxBytes+1))
+	if err != nil {
+		return nil, err
+	}
+	if int64(len(body)) > f.cfg.MaxBytes {
+		return nil, fmt.Errorf("%w (%d bytes)", ErrTooLarge, f.cfg.MaxBytes)
+	}
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		return nil, fmt.Errorf("%w: %d", ErrStatus, resp.StatusCode)
+	}
+	return &Response{
+		URL:         resp.Request.URL.String(),
+		Status:      resp.StatusCode,
+		ContentType: resp.Header.Get("Content-Type"),
+		Body:        body,
+	}, nil
+}
+
+// checkURL applies the scheme rule and the host lists. The address rule is the
+// dialer's job (see New).
+func (f *Fetcher) checkURL(u *url.URL) error {
+	switch u.Scheme {
+	case "http", "https":
+	default:
+		return fmt.Errorf("%w: %q", ErrScheme, u.Scheme)
+	}
+	host := strings.ToLower(u.Hostname())
+	if host == "" {
+		return fmt.Errorf("%w: no host", ErrBlocked)
+	}
+	if HostMatches(host, f.cfg.DenyHosts) {
+		return fmt.Errorf("%w: %s is denied", ErrBlocked, host)
+	}
+	if len(f.cfg.AllowHosts) > 0 && !HostMatches(host, f.cfg.AllowHosts) {
+		return fmt.Errorf("%w: %s is not on the allowlist", ErrBlocked, host)
+	}
+	// A literal private address is refused here as well as in the dialer, so
+	// the error is the specific one even when no connection is attempted.
+	if !f.cfg.AllowPrivate {
+		if ip := net.ParseIP(host); ip != nil && IsPrivateIP(ip) {
+			return fmt.Errorf("%w: %s", ErrPrivate, host)
+		}
+	}
+	return nil
+}
+
+// waitTurn blocks until this host's rate-limit interval has elapsed. It holds
+// no lock while sleeping, so two hosts never wait on each other.
+func (f *Fetcher) waitTurn(ctx context.Context, host string) error {
+	for {
+		f.mu.Lock()
+		now := time.Now()
+		earliest := f.last[host].Add(f.cfg.HostInterval)
+		if !now.Before(earliest) {
+			f.last[host] = now
+			f.mu.Unlock()
+			return nil
+		}
+		f.mu.Unlock()
+		wait := time.NewTimer(earliest.Sub(now))
+		select {
+		case <-ctx.Done():
+			wait.Stop()
+			return ctx.Err()
+		case <-wait.C:
+		}
+	}
+}
+
+// HostMatches reports whether host equals one of pats or is a subdomain of one.
+// Exported because the crawler applies the same rule to links it decides not to
+// follow, before it ever builds a request.
+func HostMatches(host string, pats []string) bool {
+	host = strings.ToLower(strings.TrimSuffix(host, "."))
+	for _, p := range pats {
+		p = strings.ToLower(strings.TrimSpace(strings.TrimPrefix(p, "*.")))
+		if p == "" {
+			continue
+		}
+		if host == p || strings.HasSuffix(host, "."+p) {
+			return true
+		}
+	}
+	return false
+}
+
+// cgnat is 100.64.0.0/10 — carrier NAT, not covered by net.IP's helpers and not
+// somewhere a personal assistant has business connecting.
+var cgnat = &net.IPNet{IP: net.IPv4(100, 64, 0, 0).To4(), Mask: net.CIDRMask(10, 32)}
+
+// IsPrivateIP reports whether ip is somewhere Maven must never reach out to:
+// the box itself, the LAN, the wireguard range (10.42.0.0/24 ⊂ 10/8), the cloud
+// metadata address (169.254.169.254 ⊂ link-local), or anything unroutable.
+func IsPrivateIP(ip net.IP) bool {
+	if ip.IsLoopback() || ip.IsPrivate() || ip.IsUnspecified() ||
+		ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
+		ip.IsInterfaceLocalMulticast() || ip.IsMulticast() {
+		return true
+	}
+	if v4 := ip.To4(); v4 != nil && cgnat.Contains(v4) {
+		return true
+	}
+	// IPv4-mapped/compatible forms of the above are handled by To4() inside the
+	// stdlib helpers; what is left is v6 unique-local (fc00::/7).
+	if len(ip) == net.IPv6len && ip.To4() == nil && ip[0]&0xfe == 0xfc {
+		return true
+	}
+	return false
+}
@@ -0,0 +1,227 @@
+package webfetch
+
+import (
+	"context"
+	"errors"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+)
+
+// The limits in this package are the reason a crawler is allowed to exist on
+// this box at all, so each one has a test that fails loudly if it is removed.
+
+func TestPrivateAddressesAreRefused(t *testing.T) {
+	// The wireguard range (10.42.0.0/24), the LAN (192.168.1.0/24) and the
+	// cloud metadata address are the three that matter here; the rest come
+	// along for free.
+	for _, s := range []string{
+		"127.0.0.1", "127.1.2.3", "10.42.0.7", "10.0.0.5", "192.168.1.104",
+		"172.16.4.4", "169.254.169.254", "100.64.1.1", "0.0.0.0",
+		"::1", "fc00::1", "fd12:3456::1", "fe80::1",
+	} {
+		if !IsPrivateIP(net.ParseIP(s)) {
+			t.Errorf("IsPrivateIP(%s) = false, want true", s)
+		}
+	}
+	for _, s := range []string{"8.8.8.8", "1.1.1.1", "93.184.216.34", "2606:2800:220:1::1"} {
+		if IsPrivateIP(net.ParseIP(s)) {
+			t.Errorf("IsPrivateIP(%s) = true, want false", s)
+		}
+	}
+}
+
+func TestGetRefusesPrivateLiteral(t *testing.T) {
+	f := New(Config{})
+	for _, u := range []string{
+		"http://127.0.0.1:8034/search",
+		"http://10.42.0.1/",
+		"http://192.168.1.104/dash",
+		"http://[::1]:9100/mcp",
+	} {
+		if _, err := f.Get(context.Background(), u); !errors.Is(err, ErrPrivate) {
+			t.Errorf("Get(%s) error = %v, want ErrPrivate", u, err)
+		}
+	}
+}
+
+// A hostname that resolves into private space must fail too — that is the
+// rebinding case, and it is why the check lives in the dialer.
+func TestGetRefusesPrivateResolution(t *testing.T) {
+	f := New(Config{})
+	if _, err := f.Get(context.Background(), "http://localhost:8034/"); !errors.Is(err, ErrPrivate) {
+		t.Fatalf("Get(localhost) error = %v, want ErrPrivate", err)
+	}
+}
+
+func TestGetRefusesNonHTTPSchemes(t *testing.T) {
+	f := New(Config{})
+	for _, u := range []string{"file:///etc/passwd", "ftp://example.com/x", "gopher://example.com"} {
+		if _, err := f.Get(context.Background(), u); !errors.Is(err, ErrScheme) {
+			t.Errorf("Get(%s) error = %v, want ErrScheme", u, err)
+		}
+	}
+}
+
+// testFetcher — a fetcher pointed at an httptest server, which necessarily
+// listens on loopback. AllowPrivate is the test-only escape hatch.
+func testFetcher(t *testing.T, cfg Config) *Fetcher {
+	t.Helper()
+	cfg.AllowPrivate = true
+	if cfg.HostInterval == 0 {
+		cfg.HostInterval = time.Nanosecond
+	}
+	return New(cfg)
+}
+
+func TestAllowAndDenyLists(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte("ok"))
+	}))
+	defer srv.Close()
+
+	f := testFetcher(t, Config{AllowHosts: []string{"example.com"}})
+	if _, err := f.Get(context.Background(), srv.URL); !errors.Is(err, ErrBlocked) {
+		t.Fatalf("off-allowlist host: error = %v, want ErrBlocked", err)
+	}
+	f = testFetcher(t, Config{DenyHosts: []string{"127.0.0.1"}})
+	if _, err := f.Get(context.Background(), srv.URL); !errors.Is(err, ErrBlocked) {
+		t.Fatalf("denied host: error = %v, want ErrBlocked", err)
+	}
+	f = testFetcher(t, Config{AllowHosts: []string{"127.0.0.1"}})
+	if _, err := f.Get(context.Background(), srv.URL); err != nil {
+		t.Fatalf("allowlisted host: %v", err)
+	}
+}
+
+func TestHostMatchesSubdomains(t *testing.T) {
+	pats := []string{"example.com", "*.news.org"}
+	for _, h := range []string{"example.com", "news.example.com", "a.b.example.com", "news.org", "feeds.news.org"} {
+		if !HostMatches(h, pats) {
+			t.Errorf("HostMatches(%q) = false, want true", h)
+		}
+	}
+	for _, h := range []string{"notexample.com", "example.com.evil.net", "org"} {
+		if HostMatches(h, pats) {
+			t.Errorf("HostMatches(%q) = true, want false", h)
+		}
+	}
+}
+
+func TestSizeCap(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte(strings.Repeat("x", 5000)))
+	}))
+	defer srv.Close()
+
+	f := testFetcher(t, Config{MaxBytes: 100})
+	if _, err := f.Get(context.Background(), srv.URL); !errors.Is(err, ErrTooLarge) {
+		t.Fatalf("error = %v, want ErrTooLarge", err)
+	}
+	f = testFetcher(t, Config{MaxBytes: 6000})
+	resp, err := f.Get(context.Background(), srv.URL)
+	if err != nil {
+		t.Fatalf("under the cap: %v", err)
+	}
+	if len(resp.Body) != 5000 {
+		t.Fatalf("body = %d bytes, want 5000", len(resp.Body))
+	}
+}
+
+func TestRedirectCap(t *testing.T) {
+	var srv *httptest.Server
+	srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, srv.URL+"/again", http.StatusFound)
+	}))
+	defer srv.Close()
+
+	f := testFetcher(t, Config{MaxRedirects: 2})
+	if _, err := f.Get(context.Background(), srv.URL); !errors.Is(err, ErrRedirects) {
+		t.Fatalf("error = %v, want ErrRedirects", err)
+	}
+}
+
+// A redirect off the allowlist is the interesting redirect: the first hop is
+// permitted, the second must not be.
+func TestRedirectRecheckedAgainstDenylist(t *testing.T) {
+	target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte("secret"))
+	}))
+	defer target.Close()
+	hop := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, target.URL, http.StatusFound)
+	}))
+	defer hop.Close()
+
+	// Reach the hop under the name "localhost" and allow only that name; the
+	// redirect lands on the same box under its literal address, which the
+	// allowlist does not cover. Without the CheckRedirect hook this fetch
+	// succeeds and returns "secret".
+	f := testFetcher(t, Config{AllowHosts: []string{"localhost"}})
+	viaName := strings.Replace(hop.URL, "127.0.0.1", "localhost", 1)
+	if _, err := f.Get(context.Background(), viaName); !errors.Is(err, ErrBlocked) {
+		t.Fatalf("error = %v, want ErrBlocked", err)
+	}
+}
+
+func TestPerHostRateLimit(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte("ok"))
+	}))
+	defer srv.Close()
+
+	f := testFetcher(t, Config{HostInterval: 60 * time.Millisecond})
+	start := time.Now()
+	for i := 0; i < 3; i++ {
+		if _, err := f.Get(context.Background(), srv.URL); err != nil {
+			t.Fatalf("request %d: %v", i, err)
+		}
+	}
+	if elapsed := time.Since(start); elapsed < 120*time.Millisecond {
+		t.Fatalf("three requests took %s, want at least 120ms of spacing", elapsed)
+	}
+}
+
+func TestRateLimitHonoursContext(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+	defer srv.Close()
+
+	f := testFetcher(t, Config{HostInterval: 10 * time.Second})
+	if _, err := f.Get(context.Background(), srv.URL); err != nil {
+		t.Fatalf("first request: %v", err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+	defer cancel()
+	if _, err := f.Get(ctx, srv.URL); !errors.Is(err, context.DeadlineExceeded) {
+		t.Fatalf("error = %v, want DeadlineExceeded", err)
+	}
+}
+
+func TestNon2xxIsAnError(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "nope", http.StatusInternalServerError)
+	}))
+	defer srv.Close()
+	f := testFetcher(t, Config{})
+	if _, err := f.Get(context.Background(), srv.URL); !errors.Is(err, ErrStatus) {
+		t.Fatalf("error = %v, want ErrStatus", err)
+	}
+}
+
+func TestUserAgentIsSent(t *testing.T) {
+	got := make(chan string, 1)
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		got <- r.Header.Get("User-Agent")
+	}))
+	defer srv.Close()
+	f := testFetcher(t, Config{UserAgent: "Maven/test"})
+	if _, err := f.Get(context.Background(), srv.URL); err != nil {
+		t.Fatal(err)
+	}
+	if ua := <-got; ua != "Maven/test" {
+		t.Fatalf("user-agent = %q", ua)
+	}
+}