package crawl import ( "context" "errors" "fmt" "testing" "time" ) // timedFetcher records when each request was made, so a test can assert a wait // actually happened rather than that a field was parsed. type timedFetcher struct { pages map[string]Response errs map[string]error at []time.Time urls []string } func (f *timedFetcher) Get(_ context.Context, u string) (*Response, error) { f.at = append(f.at, time.Now()) f.urls = append(f.urls, u) if err, ok := f.errs[u]; ok { return nil, err } r, ok := f.pages[u] if !ok { return nil, errors.New("http 404: no such page") } if r.URL == "" { r.URL = u } if r.ContentType == "" { r.ContentType = "text/html" } return &r, nil } func TestPage_HonoursCrawlDelay(t *testing.T) { // deploy/README says Crawl-delay is honoured. It was parsed into Rules and // never read: the only pacing was the fetcher's flat one request per host // per second, which cannot express what a site asked for. const delay = 120 * time.Millisecond f := &timedFetcher{pages: map[string]Response{ "https://example.org/robots.txt": {Body: []byte(fmt.Sprintf("User-agent: *\nCrawl-delay: %.3f\n", delay.Seconds())), ContentType: "text/plain"}, "https://example.org/a": {Body: []byte("a")}, }} c := New(f, Config{UserAgent: "Maven/1.0"}) if _, err := c.Page(context.Background(), "https://example.org/a"); err != nil { t.Fatal(err) } if len(f.at) != 2 { t.Fatalf("requests = %v; want robots.txt then the page", f.urls) } if gap := f.at[1].Sub(f.at[0]); gap < delay { t.Errorf("the page was fetched %s after robots.txt; the site asked for %s", gap, delay) } } func TestPage_ACrawlDelayLongerThanTheTurnFailsInsteadOfBlocking(t *testing.T) { f := &timedFetcher{pages: map[string]Response{ "https://example.org/robots.txt": {Body: []byte("User-agent: *\nCrawl-delay: 30\n"), ContentType: "text/plain"}, "https://example.org/a": {Body: []byte("a")}, }} c := New(f, Config{UserAgent: "Maven/1.0"}) ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) defer cancel() if _, err := c.Page(ctx, "https://example.org/a"); err == nil { t.Fatal("a 30-second Crawl-delay was ignored inside a turn that cannot wait that long") } if len(f.urls) != 1 { t.Errorf("requests = %v; the page must not be fetched before the wait it refused", f.urls) } } func TestPage_ABrokenRobotsServerIsNotPermissionToCrawl(t *testing.T) { // A 404 means unrestricted, per the standard. A 500 does not: the standard // asks for the opposite, and "the site is broken, so read it" is the wrong // way to resolve an unknown. f := &timedFetcher{ pages: map[string]Response{"https://example.org/a": {Body: []byte("a")}}, errs: map[string]error{"https://example.org/robots.txt": &StatusError{Code: 503}}, } c := New(f, Config{UserAgent: "Maven/1.0"}) if _, err := c.Page(context.Background(), "https://example.org/a"); !errors.Is(err, ErrFetchStatus) { t.Fatalf("Page over a 503 robots.txt = %v; want a refusal", err) } if len(f.urls) != 1 { t.Errorf("requests = %v; the page was read anyway", f.urls) } } func TestPage_AFetcherRefusalIsReportedAsItself(t *testing.T) { // The old check matched three substrings of webfetch's message from a // package that cannot import webfetch. The sentinel is mapped by the // adapter that owns both (cmd/mavend/crawls.go). f := &timedFetcher{errs: map[string]error{ "https://example.org/robots.txt": fmt.Errorf("%w: host is not allowed", ErrFetchRefused), }} c := New(f, Config{UserAgent: "Maven/1.0"}) if _, err := c.Page(context.Background(), "https://example.org/a"); !errors.Is(err, ErrFetchRefused) { t.Fatalf("Page = %v; want the fetcher's own refusal, not a robots verdict", err) } } func TestParseRobots_MostSpecificAgentWinsRegardlessOfOrder(t *testing.T) { body := "User-agent: maven\nDisallow: /private\n\nUser-agent: mav\nDisallow: /\n" r := ParseRobots(body, "maven/1.0") if !r.Allowed("/public") { t.Error("the shorter agent group won by file order; the longer prefix is the more specific match") } if r.Allowed("/private") { t.Error("the group that names us was not applied") } }