webfetch checks the status before it reads the body (V-581)

A non-2xx reply was read in full first and only then rejected. Two costs
followed. A 500 with a large error page pulled up to MaxBytes off the wire for
nothing. An error page over the cap returned ErrTooLarge, which names the size
and hides the status the server actually sent.

The status is a typed error now. webfetch.StatusError carries the code and
unwraps to ErrStatus, so errors.Is keeps working and errors.As reads the number.
crawl.StatusError is the same shape on the other side of the seam, and
cmd/mavend/crawls.go carries the code across.

That removes the string grep in crawl.isServerError, which decided whether a
failed robots.txt blocks a crawl by looking for " 50" in an error message it did
not own. A reworded error would have turned a 503 robots.txt into permission to
crawl. It reads the code now.

Two comments corrected. webfetch.HostMatches said the crawler calls it and
nothing outside the package does. rss.PlainText said the crawler's extractor
goes through it and crawl/extract.go has its own pass.

The rss poller parses the feed straight off the byte slice instead of copying a
document that can run to a megabyte through a string.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

internal/webfetch/webfetch_test.go

@@ -1,6 +1,7 @@
 package webfetch
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"io"
@@ -212,6 +213,26 @@ func TestNon2xxIsAnError(t *testing.T) {
 	}
 }
 
+func TestNon2xxCarriesTheCodeAndBeatsTheSizeCap(t *testing.T) {
+	// A big error page used to be read in full and reported as ErrTooLarge,
+	// which names the size and hides the 503. The status is checked first now,
+	// and the code survives for a caller that has to tell 5xx from 404.
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusServiceUnavailable)
+		w.Write(bytes.Repeat([]byte("x"), 5000))
+	}))
+	defer srv.Close()
+	f := testFetcher(t, Config{MaxBytes: 100})
+	_, err := f.Get(context.Background(), srv.URL)
+	if !errors.Is(err, ErrStatus) || errors.Is(err, ErrTooLarge) {
+		t.Fatalf("error = %v, want ErrStatus and not ErrTooLarge", err)
+	}
+	var se *StatusError
+	if !errors.As(err, &se) || se.Code != http.StatusServiceUnavailable {
+		t.Fatalf("error = %v, want a StatusError carrying 503", err)
+	}
+}
+
 func TestUserAgentIsSent(t *testing.T) {
 	got := make(chan string, 1)
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {