claude
04584fb2da
A non-2xx reply was read in full first and only then rejected. Two costs followed. A 500 with a large error page pulled up to MaxBytes off the wire for nothing. An error page over the cap returned ErrTooLarge, which names the size and hides the status the server actually sent. The status is a typed error now. webfetch.StatusError carries the code and unwraps to ErrStatus, so errors.Is keeps working and errors.As reads the number. crawl.StatusError is the same shape on the other side of the seam, and cmd/mavend/crawls.go carries the code across. That removes the string grep in crawl.isServerError, which decided whether a failed robots.txt blocks a crawl by looking for " 50" in an error message it did not own. A reworded error would have turned a 503 robots.txt into permission to crawl. It reads the code now. Two comments corrected. webfetch.HostMatches said the crawler calls it and nothing outside the package does. rss.PlainText said the crawler's extractor goes through it and crawl/extract.go has its own pass. The rss poller parses the feed straight off the byte slice instead of copying a document that can run to a megabyte through a string. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
116 lines
4.2 KiB
Go
116 lines
4.2 KiB
Go
package crawl
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// timedFetcher records when each request was made, so a test can assert a wait
|
|
// actually happened rather than that a field was parsed.
|
|
type timedFetcher struct {
|
|
pages map[string]Response
|
|
errs map[string]error
|
|
at []time.Time
|
|
urls []string
|
|
}
|
|
|
|
func (f *timedFetcher) Get(_ context.Context, u string) (*Response, error) {
|
|
f.at = append(f.at, time.Now())
|
|
f.urls = append(f.urls, u)
|
|
if err, ok := f.errs[u]; ok {
|
|
return nil, err
|
|
}
|
|
r, ok := f.pages[u]
|
|
if !ok {
|
|
return nil, errors.New("http 404: no such page")
|
|
}
|
|
if r.URL == "" {
|
|
r.URL = u
|
|
}
|
|
if r.ContentType == "" {
|
|
r.ContentType = "text/html"
|
|
}
|
|
return &r, nil
|
|
}
|
|
|
|
func TestPage_HonoursCrawlDelay(t *testing.T) {
|
|
// deploy/README says Crawl-delay is honoured. It was parsed into Rules and
|
|
// never read: the only pacing was the fetcher's flat one request per host
|
|
// per second, which cannot express what a site asked for.
|
|
const delay = 120 * time.Millisecond
|
|
f := &timedFetcher{pages: map[string]Response{
|
|
"https://example.org/robots.txt": {Body: []byte(fmt.Sprintf("User-agent: *\nCrawl-delay: %.3f\n", delay.Seconds())), ContentType: "text/plain"},
|
|
"https://example.org/a": {Body: []byte("<html><body>a</body></html>")},
|
|
}}
|
|
c := New(f, Config{UserAgent: "Maven/1.0"})
|
|
if _, err := c.Page(context.Background(), "https://example.org/a"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(f.at) != 2 {
|
|
t.Fatalf("requests = %v; want robots.txt then the page", f.urls)
|
|
}
|
|
if gap := f.at[1].Sub(f.at[0]); gap < delay {
|
|
t.Errorf("the page was fetched %s after robots.txt; the site asked for %s", gap, delay)
|
|
}
|
|
}
|
|
|
|
func TestPage_ACrawlDelayLongerThanTheTurnFailsInsteadOfBlocking(t *testing.T) {
|
|
f := &timedFetcher{pages: map[string]Response{
|
|
"https://example.org/robots.txt": {Body: []byte("User-agent: *\nCrawl-delay: 30\n"), ContentType: "text/plain"},
|
|
"https://example.org/a": {Body: []byte("<html><body>a</body></html>")},
|
|
}}
|
|
c := New(f, Config{UserAgent: "Maven/1.0"})
|
|
ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond)
|
|
defer cancel()
|
|
if _, err := c.Page(ctx, "https://example.org/a"); err == nil {
|
|
t.Fatal("a 30-second Crawl-delay was ignored inside a turn that cannot wait that long")
|
|
}
|
|
if len(f.urls) != 1 {
|
|
t.Errorf("requests = %v; the page must not be fetched before the wait it refused", f.urls)
|
|
}
|
|
}
|
|
|
|
func TestPage_ABrokenRobotsServerIsNotPermissionToCrawl(t *testing.T) {
|
|
// A 404 means unrestricted, per the standard. A 500 does not: the standard
|
|
// asks for the opposite, and "the site is broken, so read it" is the wrong
|
|
// way to resolve an unknown.
|
|
f := &timedFetcher{
|
|
pages: map[string]Response{"https://example.org/a": {Body: []byte("<html><body>a</body></html>")}},
|
|
errs: map[string]error{"https://example.org/robots.txt": &StatusError{Code: 503}},
|
|
}
|
|
c := New(f, Config{UserAgent: "Maven/1.0"})
|
|
if _, err := c.Page(context.Background(), "https://example.org/a"); !errors.Is(err, ErrFetchStatus) {
|
|
t.Fatalf("Page over a 503 robots.txt = %v; want a refusal", err)
|
|
}
|
|
if len(f.urls) != 1 {
|
|
t.Errorf("requests = %v; the page was read anyway", f.urls)
|
|
}
|
|
}
|
|
|
|
func TestPage_AFetcherRefusalIsReportedAsItself(t *testing.T) {
|
|
// The old check matched three substrings of webfetch's message from a
|
|
// package that cannot import webfetch. The sentinel is mapped by the
|
|
// adapter that owns both (cmd/mavend/crawls.go).
|
|
f := &timedFetcher{errs: map[string]error{
|
|
"https://example.org/robots.txt": fmt.Errorf("%w: host is not allowed", ErrFetchRefused),
|
|
}}
|
|
c := New(f, Config{UserAgent: "Maven/1.0"})
|
|
if _, err := c.Page(context.Background(), "https://example.org/a"); !errors.Is(err, ErrFetchRefused) {
|
|
t.Fatalf("Page = %v; want the fetcher's own refusal, not a robots verdict", err)
|
|
}
|
|
}
|
|
|
|
func TestParseRobots_MostSpecificAgentWinsRegardlessOfOrder(t *testing.T) {
|
|
body := "User-agent: maven\nDisallow: /private\n\nUser-agent: mav\nDisallow: /\n"
|
|
r := ParseRobots(body, "maven/1.0")
|
|
if !r.Allowed("/public") {
|
|
t.Error("the shorter agent group won by file order; the longer prefix is the more specific match")
|
|
}
|
|
if r.Allowed("/private") {
|
|
t.Error("the group that names us was not applied")
|
|
}
|
|
}
|