Read a web page when he names one, and watch a few on a timer (#259)

The network fallback behind the local sources, off unless configured.

internal/crawl is pure: a stdlib robots.txt parser (group specificity,
wildcards, Crawl-delay, cached per host), HTML-to-plaintext extraction, and a
watcher that notes a watched page only when its text changed. It has no store
access and no net/http; cmd/mavend/crawls.go is the impure half.

Every limit is code and tested: the guarded fetcher from #258 enforces the host
allowlist/denylist, refuses private addresses in the dialer Control hook (so DNS
rebinding and each redirect hop are covered), caps size and redirects, times out,
and spaces requests per host. A robots.txt Disallow is refused with no override.

On demand, reading is a query source placed last in the chain, after his memory,
his notes, and the local Kiwix ZIMs once those are wired: no URL in the
utterance means no fetch, and only the URL ever leaves the box. Scheduled
watches write notes and announce nothing.

The vendored tree has no x/net/html, goquery or temoto/robotstxt, so the parsers
are stdlib. No new dependency.

internal/crawl/watch_test.go

@@ -0,0 +1,117 @@
+package crawl
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+)
+
+type note struct {
+	text   string
+	source string
+}
+
+type fakeNotes struct{ notes []note }
+
+func (n *fakeNotes) WriteNote(_ context.Context, _ time.Time, text string, _ []float32, source string) (int64, error) {
+	n.notes = append(n.notes, note{text, source})
+	return int64(len(n.notes)), nil
+}
+
+type fakeHashes struct{ m map[string]string }
+
+func newHashes() *fakeHashes { return &fakeHashes{m: map[string]string{}} }
+func (f *fakeHashes) LastHash(_ context.Context, name string) (string, error) {
+	return f.m[name], nil
+}
+func (f *fakeHashes) SetHash(_ context.Context, name, h string) error { f.m[name] = h; return nil }
+
+var t0 = time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC)
+
+func TestWatchNotesAChangedPage(t *testing.T) {
+	f := &fakeFetcher{pages: map[string]Response{
+		"https://example.org/docs": {Body: []byte(htmlPage)},
+	}}
+	notes := &fakeNotes{}
+	hashes := newHashes()
+	w := NewWatcher(newTestCrawler(f), []WatchConfig{{Name: "docs", URL: "https://example.org/docs"}},
+		notes, hashes, nil, time.Hour)
+	if w == nil {
+		t.Fatal("NewWatcher returned nil for a configured watch")
+	}
+	if n := w.CheckDue(context.Background(), t0); n != 1 {
+		t.Fatalf("first check wrote %d notes, want 1", n)
+	}
+	if notes.notes[0].source != "crawl:docs" {
+		t.Errorf("source = %q, want crawl:docs", notes.notes[0].source)
+	}
+	if !strings.Contains(notes.notes[0].text, "https://example.org/docs") {
+		t.Errorf("note does not carry the url: %q", notes.notes[0].text)
+	}
+
+	// Unchanged page, interval elapsed: nothing written.
+	if n := w.CheckDue(context.Background(), t0.Add(2*time.Hour)); n != 0 {
+		t.Fatalf("an unchanged page wrote %d notes", n)
+	}
+
+	// Changed page: one note.
+	f.pages["https://example.org/docs"] = Response{Body: []byte(strings.Replace(htmlPage, "синее", "серое", 1))}
+	if n := w.CheckDue(context.Background(), t0.Add(4*time.Hour)); n != 1 {
+		t.Fatalf("a changed page wrote %d notes, want 1", n)
+	}
+}
+
+func TestWatchIntervalIsRespected(t *testing.T) {
+	f := &fakeFetcher{pages: map[string]Response{"https://example.org/d": {Body: []byte(htmlPage)}}}
+	w := NewWatcher(newTestCrawler(f), []WatchConfig{{Name: "d", URL: "https://example.org/d", Interval: time.Hour}},
+		&fakeNotes{}, newHashes(), nil, 0)
+	w.CheckDue(context.Background(), t0)
+	before := len(f.calls)
+	w.CheckDue(context.Background(), t0.Add(time.Minute))
+	if len(f.calls) != before {
+		t.Fatal("the page was re-read inside its interval")
+	}
+}
+
+// The hash is durable so a restart does not re-note an unchanged page.
+func TestWatchHashSurvivesRestart(t *testing.T) {
+	f := &fakeFetcher{pages: map[string]Response{"https://example.org/d": {Body: []byte(htmlPage)}}}
+	hashes := newHashes()
+	watches := []WatchConfig{{Name: "d", URL: "https://example.org/d"}}
+	NewWatcher(newTestCrawler(f), watches, &fakeNotes{}, hashes, nil, time.Hour).CheckDue(context.Background(), t0)
+
+	notes2 := &fakeNotes{}
+	NewWatcher(newTestCrawler(f), watches, notes2, hashes, nil, time.Hour).CheckDue(context.Background(), t0.Add(time.Hour))
+	if len(notes2.notes) != 0 {
+		t.Fatalf("a fresh watcher re-noted an unchanged page: %q", notes2.notes[0].text)
+	}
+}
+
+func TestWatchDeadPageDoesNotStopTheOthers(t *testing.T) {
+	f := &fakeFetcher{pages: map[string]Response{"https://example.org/live": {Body: []byte(htmlPage)}}}
+	notes := &fakeNotes{}
+	w := NewWatcher(newTestCrawler(f), []WatchConfig{
+		{Name: "dead", URL: "https://example.org/gone"},
+		{Name: "live", URL: "https://example.org/live"},
+	}, notes, newHashes(), nil, time.Hour)
+	if n := w.CheckDue(context.Background(), t0); n != 1 {
+		t.Fatalf("wrote %d notes, want 1 (the live page)", n)
+	}
+	if notes.notes[0].source != "crawl:live" {
+		t.Fatalf("source = %q", notes.notes[0].source)
+	}
+}
+
+func TestNoWatchesMeansNoWatcher(t *testing.T) {
+	c := newTestCrawler(&fakeFetcher{})
+	if NewWatcher(c, nil, &fakeNotes{}, nil, nil, 0) != nil {
+		t.Fatal("no watches must mean no watcher")
+	}
+	if NewWatcher(nil, []WatchConfig{{Name: "a", URL: "u"}}, &fakeNotes{}, nil, nil, 0) != nil {
+		t.Fatal("no crawler must mean no watcher")
+	}
+	if NewWatcher(c, []WatchConfig{{Name: "", URL: ""}}, &fakeNotes{}, nil, nil, 0) != nil {
+		t.Fatal("a watch with no name or url is not a configuration")
+	}
+}