Read a web page when he names one, and watch a few on a timer (#259)

The network fallback behind the local sources, off unless configured.

internal/crawl is pure: a stdlib robots.txt parser (group specificity,
wildcards, Crawl-delay, cached per host), HTML-to-plaintext extraction, and a
watcher that notes a watched page only when its text changed. It has no store
access and no net/http; cmd/mavend/crawls.go is the impure half.

Every limit is code and tested: the guarded fetcher from #258 enforces the host
allowlist/denylist, refuses private addresses in the dialer Control hook (so DNS
rebinding and each redirect hop are covered), caps size and redirects, times out,
and spaces requests per host. A robots.txt Disallow is refused with no override.

On demand, reading is a query source placed last in the chain, after his memory,
his notes, and the local Kiwix ZIMs once those are wired: no URL in the
utterance means no fetch, and only the URL ever leaves the box. Scheduled
watches write notes and announce nothing.

The vendored tree has no x/net/html, goquery or temoto/robotstxt, so the parsers
are stdlib. No new dependency.

internal/crawl/robots_test.go

@@ -0,0 +1,84 @@
+package crawl
+
+import (
+	"testing"
+	"time"
+)
+
+const robotsBody = `# a comment
+User-agent: *
+Disallow: /private
+Disallow: /tmp/
+Crawl-delay: 5
+
+User-agent: Maven
+Disallow: /
+Allow: /public
+`
+
+func TestParseRobotsPicksTheMostSpecificGroup(t *testing.T) {
+	// The Maven group applies to us even though we send a longer UA string.
+	r := ParseRobots(robotsBody, "Maven/1.0 (self-hosted personal assistant)")
+	if r.Allowed("/anything") {
+		t.Error("Disallow: / in our own group was ignored")
+	}
+	if !r.Allowed("/public/page") {
+		t.Error("Allow: /public must beat the shorter Disallow: /")
+	}
+
+	// A different agent falls into the * group.
+	star := ParseRobots(robotsBody, "SomeoneElse/2")
+	if !star.Allowed("/anything") {
+		t.Error("the * group disallows nothing but /private and /tmp/")
+	}
+	if star.Allowed("/private/x") || star.Allowed("/tmp/") {
+		t.Error("the * group's disallows were not applied")
+	}
+	if star.Delay != 5*time.Second {
+		t.Errorf("crawl-delay = %v, want 5s", star.Delay)
+	}
+}
+
+func TestParseRobotsEmptyMeansAllowAll(t *testing.T) {
+	for _, body := range []string{"", "# nothing here\n", "User-agent: *\nDisallow:\n"} {
+		if !ParseRobots(body, "Maven").Allowed("/whatever") {
+			t.Errorf("body %q must allow everything", body)
+		}
+	}
+}
+
+func TestRobotsWildcards(t *testing.T) {
+	r := ParseRobots("User-agent: *\nDisallow: /*.pdf$\nDisallow: /a/*/secret\n", "Maven")
+	if r.Allowed("/docs/manual.pdf") {
+		t.Error("*.pdf$ did not match")
+	}
+	if !r.Allowed("/docs/manual.pdf.html") {
+		t.Error("$ must anchor at the end")
+	}
+	if r.Allowed("/a/b/secret") {
+		t.Error("/a/*/secret did not match")
+	}
+	if !r.Allowed("/a/b/public") {
+		t.Error("unrelated path was refused")
+	}
+}
+
+// Consecutive User-agent lines share one group, which is common in the wild.
+func TestRobotsSharedGroup(t *testing.T) {
+	r := ParseRobots("User-agent: Googlebot\nUser-agent: Maven\nDisallow: /x\n", "Maven/1.0")
+	if r.Allowed("/x/y") {
+		t.Fatal("a shared group's rules were not applied to the second agent")
+	}
+}
+
+func TestRobotsCacheTTL(t *testing.T) {
+	c := newRobotsCache(time.Minute)
+	now := time.Now()
+	c.put("example.com", ParseRobots("User-agent: *\nDisallow: /\n", "Maven"), now)
+	if _, ok := c.get("example.com", now.Add(30*time.Second)); !ok {
+		t.Error("a fresh entry must be served from cache")
+	}
+	if _, ok := c.get("example.com", now.Add(2*time.Minute)); ok {
+		t.Error("an expired entry must be re-read")
+	}
+}