diff --git a/internal/webfetch/webfetch.go b/internal/webfetch/webfetch.go index aea5284..1ebedde 100644 --- a/internal/webfetch/webfetch.go +++ b/internal/webfetch/webfetch.go @@ -295,6 +295,27 @@ func (f *Fetcher) checkURL(u *url.URL) error { return nil } +// pruneHostsAbove is when pruneLocked bothers to walk the map. Below it the +// walk costs more than the entries do, and `crawl.on_demand` means the host set +// is whatever he names out loud, so it grows slowly. +const pruneHostsAbove = 64 + +// pruneLocked drops hosts whose last dial is further back than HostInterval. +// Such an entry cannot delay anything — waitTurn would let the next request +// through immediately — so keeping it only holds memory for the life of the +// process. Caller holds f.mu. +func (f *Fetcher) pruneLocked(now time.Time) { + if len(f.last) <= pruneHostsAbove { + return + } + cutoff := now.Add(-f.cfg.HostInterval) + for h, at := range f.last { + if at.Before(cutoff) { + delete(f.last, h) + } + } +} + // waitTurn blocks until this host's rate-limit interval has elapsed. It holds // no lock while sleeping, so two hosts never wait on each other. func (f *Fetcher) waitTurn(ctx context.Context, host string) error { @@ -304,6 +325,7 @@ func (f *Fetcher) waitTurn(ctx context.Context, host string) error { earliest := f.last[host].Add(f.cfg.HostInterval) if !now.Before(earliest) { f.last[host] = now + f.pruneLocked(now) f.mu.Unlock() return nil } diff --git a/internal/webfetch/webfetch_test.go b/internal/webfetch/webfetch_test.go index 3eb8d3a..2ec2df2 100644 --- a/internal/webfetch/webfetch_test.go +++ b/internal/webfetch/webfetch_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "errors" + "fmt" "io" "net" "net/http" @@ -318,3 +319,37 @@ func TestPostObeysDenylist(t *testing.T) { t.Fatalf("error = %v, want ErrBlocked", err) } } + +// f.last used to hold one entry per host ever dialed, for the life of the +// process. A host whose last dial is older than HostInterval cannot delay +// anything, so it is dropped once the map is worth walking. +func TestHostRateMapIsPruned(t *testing.T) { + f := New(Config{HostInterval: time.Minute, AllowPrivate: true}) + stale := time.Now().Add(-time.Hour) + for i := 0; i < pruneHostsAbove*2; i++ { + f.last[fmt.Sprintf("h%d.example", i)] = stale + } + + // One real turn is what triggers the sweep. + if err := f.waitTurn(context.Background(), "fresh.example"); err != nil { + t.Fatal(err) + } + if len(f.last) != 1 { + t.Fatalf("len(f.last) = %d after the sweep, want 1 (only the host just dialed)", len(f.last)) + } + if _, ok := f.last["fresh.example"]; !ok { + t.Fatal("the host just dialed was pruned, so its own rate limit is lost") + } + + // A host inside the interval is kept: pruning must not hand out a free turn. + f.last["recent.example"] = time.Now() + for i := 0; i < pruneHostsAbove*2; i++ { + f.last[fmt.Sprintf("g%d.example", i)] = stale + } + if err := f.waitTurn(context.Background(), "other.example"); err != nil { + t.Fatal(err) + } + if _, ok := f.last["recent.example"]; !ok { + t.Fatal("a host dialed inside HostInterval was pruned") + } +}