kami
2c1b0eede0
The network fallback behind the local sources, off unless configured. internal/crawl is pure: a stdlib robots.txt parser (group specificity, wildcards, Crawl-delay, cached per host), HTML-to-plaintext extraction, and a watcher that notes a watched page only when its text changed. It has no store access and no net/http; cmd/mavend/crawls.go is the impure half. Every limit is code and tested: the guarded fetcher from #258 enforces the host allowlist/denylist, refuses private addresses in the dialer Control hook (so DNS rebinding and each redirect hop are covered), caps size and redirects, times out, and spaces requests per host. A robots.txt Disallow is refused with no override. On demand, reading is a query source placed last in the chain, after his memory, his notes, and the local Kiwix ZIMs once those are wired: no URL in the utterance means no fetch, and only the URL ever leaves the box. Scheduled watches write notes and announce nothing. The vendored tree has no x/net/html, goquery or temoto/robotstxt, so the parsers are stdlib. No new dependency.
85 lines
2.4 KiB
Go
85 lines
2.4 KiB
Go
package crawl
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
const robotsBody = `# a comment
|
|
User-agent: *
|
|
Disallow: /private
|
|
Disallow: /tmp/
|
|
Crawl-delay: 5
|
|
|
|
User-agent: Maven
|
|
Disallow: /
|
|
Allow: /public
|
|
`
|
|
|
|
func TestParseRobotsPicksTheMostSpecificGroup(t *testing.T) {
|
|
// The Maven group applies to us even though we send a longer UA string.
|
|
r := ParseRobots(robotsBody, "Maven/1.0 (self-hosted personal assistant)")
|
|
if r.Allowed("/anything") {
|
|
t.Error("Disallow: / in our own group was ignored")
|
|
}
|
|
if !r.Allowed("/public/page") {
|
|
t.Error("Allow: /public must beat the shorter Disallow: /")
|
|
}
|
|
|
|
// A different agent falls into the * group.
|
|
star := ParseRobots(robotsBody, "SomeoneElse/2")
|
|
if !star.Allowed("/anything") {
|
|
t.Error("the * group disallows nothing but /private and /tmp/")
|
|
}
|
|
if star.Allowed("/private/x") || star.Allowed("/tmp/") {
|
|
t.Error("the * group's disallows were not applied")
|
|
}
|
|
if star.Delay != 5*time.Second {
|
|
t.Errorf("crawl-delay = %v, want 5s", star.Delay)
|
|
}
|
|
}
|
|
|
|
func TestParseRobotsEmptyMeansAllowAll(t *testing.T) {
|
|
for _, body := range []string{"", "# nothing here\n", "User-agent: *\nDisallow:\n"} {
|
|
if !ParseRobots(body, "Maven").Allowed("/whatever") {
|
|
t.Errorf("body %q must allow everything", body)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRobotsWildcards(t *testing.T) {
|
|
r := ParseRobots("User-agent: *\nDisallow: /*.pdf$\nDisallow: /a/*/secret\n", "Maven")
|
|
if r.Allowed("/docs/manual.pdf") {
|
|
t.Error("*.pdf$ did not match")
|
|
}
|
|
if !r.Allowed("/docs/manual.pdf.html") {
|
|
t.Error("$ must anchor at the end")
|
|
}
|
|
if r.Allowed("/a/b/secret") {
|
|
t.Error("/a/*/secret did not match")
|
|
}
|
|
if !r.Allowed("/a/b/public") {
|
|
t.Error("unrelated path was refused")
|
|
}
|
|
}
|
|
|
|
// Consecutive User-agent lines share one group, which is common in the wild.
|
|
func TestRobotsSharedGroup(t *testing.T) {
|
|
r := ParseRobots("User-agent: Googlebot\nUser-agent: Maven\nDisallow: /x\n", "Maven/1.0")
|
|
if r.Allowed("/x/y") {
|
|
t.Fatal("a shared group's rules were not applied to the second agent")
|
|
}
|
|
}
|
|
|
|
func TestRobotsCacheTTL(t *testing.T) {
|
|
c := newRobotsCache(time.Minute)
|
|
now := time.Now()
|
|
c.put("example.com", ParseRobots("User-agent: *\nDisallow: /\n", "Maven"), now)
|
|
if _, ok := c.get("example.com", now.Add(30*time.Second)); !ok {
|
|
t.Error("a fresh entry must be served from cache")
|
|
}
|
|
if _, ok := c.get("example.com", now.Add(2*time.Minute)); ok {
|
|
t.Error("an expired entry must be re-read")
|
|
}
|
|
}
|