kami
327726a06a
The on-demand crawler was built over allow_hosts plus every watched host. webfetch reads a non-empty allow list as these and nothing else, so a config with one watch and no allow_hosts at all silently narrowed on-demand reading to the watched site. Every other url he pasted came back as a flat refusal with nothing in the log to explain it. The two crawlers now take two host lists from one crawlHosts helper. Crawl-delay was parsed into Rules and never read. The only pacing was the fetcher's flat one request per host per second, which cannot express what a site asked for, and deploy/README claimed the field was honoured. Page now waits it out between the robots fetch and the page fetch, and a delay longer than the turn fails the read instead of hanging it. A robots.txt that failed was treated as no rules, so a site whose server was having a bad minute became a site with no restrictions. A 5xx now refuses the crawl. A 404 still means unrestricted, which is what the standard says. The refusal check matched substrings of webfetch's message text from a package that cannot import webfetch, so a reworded error would have silently turned into a robots verdict. internal/crawl now exports ErrFetchRefused and ErrFetchStatus and the adapter in cmd/mavend maps the webfetch sentinels onto them. Robots group selection picks the longest matching agent prefix instead of the first one in file order. queryWeb passed a claim it could not serve when no crawler was configured, so an unconfigured deployment answered a web question with an apology instead of falling through to the model. Found in review of #67.
116 lines
4.2 KiB
Go
116 lines
4.2 KiB
Go
package crawl
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// timedFetcher records when each request was made, so a test can assert a wait
|
|
// actually happened rather than that a field was parsed.
|
|
type timedFetcher struct {
|
|
pages map[string]Response
|
|
errs map[string]error
|
|
at []time.Time
|
|
urls []string
|
|
}
|
|
|
|
func (f *timedFetcher) Get(_ context.Context, u string) (*Response, error) {
|
|
f.at = append(f.at, time.Now())
|
|
f.urls = append(f.urls, u)
|
|
if err, ok := f.errs[u]; ok {
|
|
return nil, err
|
|
}
|
|
r, ok := f.pages[u]
|
|
if !ok {
|
|
return nil, errors.New("http 404: no such page")
|
|
}
|
|
if r.URL == "" {
|
|
r.URL = u
|
|
}
|
|
if r.ContentType == "" {
|
|
r.ContentType = "text/html"
|
|
}
|
|
return &r, nil
|
|
}
|
|
|
|
func TestPage_HonoursCrawlDelay(t *testing.T) {
|
|
// deploy/README says Crawl-delay is honoured. It was parsed into Rules and
|
|
// never read: the only pacing was the fetcher's flat one request per host
|
|
// per second, which cannot express what a site asked for.
|
|
const delay = 120 * time.Millisecond
|
|
f := &timedFetcher{pages: map[string]Response{
|
|
"https://example.org/robots.txt": {Body: []byte(fmt.Sprintf("User-agent: *\nCrawl-delay: %.3f\n", delay.Seconds())), ContentType: "text/plain"},
|
|
"https://example.org/a": {Body: []byte("<html><body>a</body></html>")},
|
|
}}
|
|
c := New(f, Config{UserAgent: "Maven/1.0"})
|
|
if _, err := c.Page(context.Background(), "https://example.org/a"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(f.at) != 2 {
|
|
t.Fatalf("requests = %v; want robots.txt then the page", f.urls)
|
|
}
|
|
if gap := f.at[1].Sub(f.at[0]); gap < delay {
|
|
t.Errorf("the page was fetched %s after robots.txt; the site asked for %s", gap, delay)
|
|
}
|
|
}
|
|
|
|
func TestPage_ACrawlDelayLongerThanTheTurnFailsInsteadOfBlocking(t *testing.T) {
|
|
f := &timedFetcher{pages: map[string]Response{
|
|
"https://example.org/robots.txt": {Body: []byte("User-agent: *\nCrawl-delay: 30\n"), ContentType: "text/plain"},
|
|
"https://example.org/a": {Body: []byte("<html><body>a</body></html>")},
|
|
}}
|
|
c := New(f, Config{UserAgent: "Maven/1.0"})
|
|
ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond)
|
|
defer cancel()
|
|
if _, err := c.Page(ctx, "https://example.org/a"); err == nil {
|
|
t.Fatal("a 30-second Crawl-delay was ignored inside a turn that cannot wait that long")
|
|
}
|
|
if len(f.urls) != 1 {
|
|
t.Errorf("requests = %v; the page must not be fetched before the wait it refused", f.urls)
|
|
}
|
|
}
|
|
|
|
func TestPage_ABrokenRobotsServerIsNotPermissionToCrawl(t *testing.T) {
|
|
// A 404 means unrestricted, per the standard. A 500 does not: the standard
|
|
// asks for the opposite, and "the site is broken, so read it" is the wrong
|
|
// way to resolve an unknown.
|
|
f := &timedFetcher{
|
|
pages: map[string]Response{"https://example.org/a": {Body: []byte("<html><body>a</body></html>")}},
|
|
errs: map[string]error{"https://example.org/robots.txt": fmt.Errorf("%w: 503", ErrFetchStatus)},
|
|
}
|
|
c := New(f, Config{UserAgent: "Maven/1.0"})
|
|
if _, err := c.Page(context.Background(), "https://example.org/a"); !errors.Is(err, ErrFetchStatus) {
|
|
t.Fatalf("Page over a 503 robots.txt = %v; want a refusal", err)
|
|
}
|
|
if len(f.urls) != 1 {
|
|
t.Errorf("requests = %v; the page was read anyway", f.urls)
|
|
}
|
|
}
|
|
|
|
func TestPage_AFetcherRefusalIsReportedAsItself(t *testing.T) {
|
|
// The old check matched three substrings of webfetch's message from a
|
|
// package that cannot import webfetch. The sentinel is mapped by the
|
|
// adapter that owns both (cmd/mavend/crawls.go).
|
|
f := &timedFetcher{errs: map[string]error{
|
|
"https://example.org/robots.txt": fmt.Errorf("%w: host is not allowed", ErrFetchRefused),
|
|
}}
|
|
c := New(f, Config{UserAgent: "Maven/1.0"})
|
|
if _, err := c.Page(context.Background(), "https://example.org/a"); !errors.Is(err, ErrFetchRefused) {
|
|
t.Fatalf("Page = %v; want the fetcher's own refusal, not a robots verdict", err)
|
|
}
|
|
}
|
|
|
|
func TestParseRobots_MostSpecificAgentWinsRegardlessOfOrder(t *testing.T) {
|
|
body := "User-agent: maven\nDisallow: /private\n\nUser-agent: mav\nDisallow: /\n"
|
|
r := ParseRobots(body, "maven/1.0")
|
|
if !r.Allowed("/public") {
|
|
t.Error("the shorter agent group won by file order; the longer prefix is the more specific match")
|
|
}
|
|
if r.Allowed("/private") {
|
|
t.Error("the group that names us was not applied")
|
|
}
|
|
}
|