Drop host rate-limit entries that can no longer delay anything (V-641)

webfetch.Fetcher.last held one entry per distinct host the crawler ever
dialed, never pruned. Bounded in practice by how many hosts get crawled, but
crawl.on_demand is true in deploy, so the host set is whatever he names out
loud.

An entry older than HostInterval cannot delay a request — waitTurn would let
the next one straight through — so it is dropped. The sweep runs on write and
only once the map passes 64 entries, below which walking it costs more than
the entries do.

Rate limiting is unchanged: a host dialed inside the interval is kept, which
the test asserts, because pruning one would hand out a free turn.

internal/webfetch/webfetch.go

@@ -295,6 +295,27 @@ func (f *Fetcher) checkURL(u *url.URL) error {
 	return nil
 }
 
+// pruneHostsAbove is when pruneLocked bothers to walk the map. Below it the
+// walk costs more than the entries do, and `crawl.on_demand` means the host set
+// is whatever he names out loud, so it grows slowly.
+const pruneHostsAbove = 64
+
+// pruneLocked drops hosts whose last dial is further back than HostInterval.
+// Such an entry cannot delay anything — waitTurn would let the next request
+// through immediately — so keeping it only holds memory for the life of the
+// process. Caller holds f.mu.
+func (f *Fetcher) pruneLocked(now time.Time) {
+	if len(f.last) <= pruneHostsAbove {
+		return
+	}
+	cutoff := now.Add(-f.cfg.HostInterval)
+	for h, at := range f.last {
+		if at.Before(cutoff) {
+			delete(f.last, h)
+		}
+	}
+}
+
 // waitTurn blocks until this host's rate-limit interval has elapsed. It holds
 // no lock while sleeping, so two hosts never wait on each other.
 func (f *Fetcher) waitTurn(ctx context.Context, host string) error {
@@ -304,6 +325,7 @@ func (f *Fetcher) waitTurn(ctx context.Context, host string) error {
 		earliest := f.last[host].Add(f.cfg.HostInterval)
 		if !now.Before(earliest) {
 			f.last[host] = now
+			f.pruneLocked(now)
 			f.mu.Unlock()
 			return nil
 		}

internal/webfetch/webfetch_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"errors"
+	"fmt"
 	"io"
 	"net"
 	"net/http"
@@ -318,3 +319,37 @@ func TestPostObeysDenylist(t *testing.T) {
 		t.Fatalf("error = %v, want ErrBlocked", err)
 	}
 }
+
+// f.last used to hold one entry per host ever dialed, for the life of the
+// process. A host whose last dial is older than HostInterval cannot delay
+// anything, so it is dropped once the map is worth walking.
+func TestHostRateMapIsPruned(t *testing.T) {
+	f := New(Config{HostInterval: time.Minute, AllowPrivate: true})
+	stale := time.Now().Add(-time.Hour)
+	for i := 0; i < pruneHostsAbove*2; i++ {
+		f.last[fmt.Sprintf("h%d.example", i)] = stale
+	}
+
+	// One real turn is what triggers the sweep.
+	if err := f.waitTurn(context.Background(), "fresh.example"); err != nil {
+		t.Fatal(err)
+	}
+	if len(f.last) != 1 {
+		t.Fatalf("len(f.last) = %d after the sweep, want 1 (only the host just dialed)", len(f.last))
+	}
+	if _, ok := f.last["fresh.example"]; !ok {
+		t.Fatal("the host just dialed was pruned, so its own rate limit is lost")
+	}
+
+	// A host inside the interval is kept: pruning must not hand out a free turn.
+	f.last["recent.example"] = time.Now()
+	for i := 0; i < pruneHostsAbove*2; i++ {
+		f.last[fmt.Sprintf("g%d.example", i)] = stale
+	}
+	if err := f.waitTurn(context.Background(), "other.example"); err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := f.last["recent.example"]; !ok {
+		t.Fatal("a host dialed inside HostInterval was pruned")
+	}
+}