Drop host rate-limit entries that can no longer delay anything (V-641)

webfetch.Fetcher.last held one entry per distinct host the crawler ever
dialed, never pruned. Bounded in practice by how many hosts get crawled, but
crawl.on_demand is true in deploy, so the host set is whatever he names out
loud.

An entry older than HostInterval cannot delay a request — waitTurn would let
the next one straight through — so it is dropped. The sweep runs on write and
only once the map passes 64 entries, below which walking it costs more than
the entries do.

Rate limiting is unchanged: a host dialed inside the interval is kept, which
the test asserts, because pruning one would hand out a free turn.

internal/webfetch/webfetch_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"errors"
+	"fmt"
 	"io"
 	"net"
 	"net/http"
@@ -318,3 +319,37 @@ func TestPostObeysDenylist(t *testing.T) {
 		t.Fatalf("error = %v, want ErrBlocked", err)
 	}
 }
+
+// f.last used to hold one entry per host ever dialed, for the life of the
+// process. A host whose last dial is older than HostInterval cannot delay
+// anything, so it is dropped once the map is worth walking.
+func TestHostRateMapIsPruned(t *testing.T) {
+	f := New(Config{HostInterval: time.Minute, AllowPrivate: true})
+	stale := time.Now().Add(-time.Hour)
+	for i := 0; i < pruneHostsAbove*2; i++ {
+		f.last[fmt.Sprintf("h%d.example", i)] = stale
+	}
+
+	// One real turn is what triggers the sweep.
+	if err := f.waitTurn(context.Background(), "fresh.example"); err != nil {
+		t.Fatal(err)
+	}
+	if len(f.last) != 1 {
+		t.Fatalf("len(f.last) = %d after the sweep, want 1 (only the host just dialed)", len(f.last))
+	}
+	if _, ok := f.last["fresh.example"]; !ok {
+		t.Fatal("the host just dialed was pruned, so its own rate limit is lost")
+	}
+
+	// A host inside the interval is kept: pruning must not hand out a free turn.
+	f.last["recent.example"] = time.Now()
+	for i := 0; i < pruneHostsAbove*2; i++ {
+		f.last[fmt.Sprintf("g%d.example", i)] = stale
+	}
+	if err := f.waitTurn(context.Background(), "other.example"); err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := f.last["recent.example"]; !ok {
+		t.Fatal("a host dialed inside HostInterval was pruned")
+	}
+}