webfetch checks the status before it reads the body (V-581)

A non-2xx reply was read in full first and only then rejected. Two costs followed. A 500 with a large error page pulled up to MaxBytes off the wire for nothing. An error page over the cap returned ErrTooLarge, which names the size and hides the status the server actually sent. The status is a typed error now. webfetch.StatusError carries the code and unwraps to ErrStatus, so errors.Is keeps working and errors.As reads the number. crawl.StatusError is the same shape on the other side of the seam, and cmd/mavend/crawls.go carries the code across. That removes the string grep in crawl.isServerError, which decided whether a failed robots.txt blocks a crawl by looking for " 50" in an error message it did not own. A reworded error would have turned a 503 robots.txt into permission to crawl. It reads the code now. Two comments corrected. webfetch.HostMatches said the crawler calls it and nothing outside the package does. rss.PlainText said the crawler's extractor goes through it and crawl/extract.go has its own pass. The rss poller parses the feed straight off the byte slice instead of copying a document that can run to a megabyte through a string. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-06 03:12:59 +04:00
parent 316fb197a8
commit 04584fb2da
7 changed files with 71 additions and 17 deletions
@@ -57,6 +57,16 @@ var (
 	ErrFetchStatus = errors.New("crawl: the server answered with an error status")
 )

+// StatusError is ErrFetchStatus with the code the server actually sent. The
+// adapter builds it; isServerError reads Code rather than the message, so a
+// reworded error can no longer turn a 503 robots.txt into permission to crawl.
+type StatusError struct{ Code int }
+
+func (e *StatusError) Error() string {
+	return fmt.Sprintf("crawl: the server answered with status %d", e.Code)
+}
+func (e *StatusError) Unwrap() error { return ErrFetchStatus }
+
 // Fetcher is the guarded HTTP door (internal/webfetch adapted by the daemon). An
 // interface so this package constructs no http.Client of its own and can be
 // tested without a network.
@@ -233,16 +243,15 @@ func (c *Crawler) markFetched(host string) {
 	c.mu.Unlock()
 }

-// isServerError — a 5xx rather than any other non-2xx. The adapter formats the
-// status into the message, which is the only place it survives.
+// isServerError — a 5xx rather than any other non-2xx. A status the adapter
+// could not recover reads as 0 and is not a server error, which keeps the
+// standard's "404 means allow" as the default for an unknown.
 func isServerError(err error) bool {
-	s := err.Error()
-	for _, code := range []string{" 50", " 51", " 52", " 53"} {
-		if strings.Contains(s, code) {
-			return true
-		}
+	var se *StatusError
+	if !errors.As(err, &se) {
+		return false
 	}
-	return false
+	return se.Code >= 500 && se.Code <= 599
 }

 // Hash is the dedup key for a crawl result: the sha256 of the extracted text,
@@ -79,7 +79,7 @@ func TestPage_ABrokenRobotsServerIsNotPermissionToCrawl(t *testing.T) {
 	// way to resolve an unknown.
 	f := &timedFetcher{
 		pages: map[string]Response{"https://example.org/a": {Body: []byte("<html><body>a</body></html>")}},
-		errs:  map[string]error{"https://example.org/robots.txt": fmt.Errorf("%w: 503", ErrFetchStatus)},
+		errs:  map[string]error{"https://example.org/robots.txt": &StatusError{Code: 503}},
 	}
 	c := New(f, Config{UserAgent: "Maven/1.0"})
 	if _, err := c.Page(context.Background(), "https://example.org/a"); !errors.Is(err, ErrFetchStatus) {