Drop host rate-limit entries that can no longer delay anything (V-641)

webfetch.Fetcher.last held one entry per distinct host the crawler ever
dialed, never pruned. Bounded in practice by how many hosts get crawled, but
crawl.on_demand is true in deploy, so the host set is whatever he names out
loud.

An entry older than HostInterval cannot delay a request — waitTurn would let
the next one straight through — so it is dropped. The sweep runs on write and
only once the map passes 64 entries, below which walking it costs more than
the entries do.

Rate limiting is unchanged: a host dialed inside the interval is kept, which
the test asserts, because pruning one would hand out a free turn.

CLAUDE.md

@@ -101,9 +101,15 @@ protocol; the config in `deploy/mavend.json` (with `${VAR}` env expansion from g
 Count against compose, not against the table. Four of the nine daemons are absent, and each
 absence has a different reason.
 
-`mavmaild` is commented out in compose, with the reason written beside it: it needs a mail
-account and this box has none. `mavcaldav` appears nowhere at all, and unlike the other
-three that is an oversight rather than a decision (V-644).
+`mavmaild` and `mavcaldav` are commented out in compose, each with the reason written
+beside it: the first needs a mail account, the second a CalDAV account, and this box has
+neither. `mavcaldav` used to appear nowhere at all, which was an oversight; it became a
+recorded decision on 07-08-2026 (V-644). Two things ride on that absence and the block
+names them. Agenda questions route to `IntentQuery` at stage 0 (V-498) and the `calendar`
+query source then reads a table nobody writes. And `loop.State.CalendarBusy` is fed by the
+same facts, so the gate's "do not nag mid-meeting" is permanently false. Its password is
+read from a file (`-pass-file`, and `-render-pass-file` for the render collection), never
+taken as a flag value, which is the rule `mavpoll` and `mavmaild` follow too.
 
 **`mavwaked` and `mavenclient` are absent by decision, not oversight** (Vikunja #463,
 `docs/plans/17-where-the-voice-loop-runs.md`).

cmd/mavcaldav/main.go

@@ -50,10 +50,10 @@ func run(args []string) error {
 	socket := fs.String("socket", "", "core IPC socket path (required)")
 	url := fs.String("url", "", "CalDAV calendar URL, e.g. http://localhost:5232/kami/personal (required)")
 	user := fs.String("user", "", "CalDAV basic-auth username (required)")
-	pass := fs.String("pass", "", "CalDAV basic-auth password (required)")
+	passFile := fs.String("pass-file", "", "file holding the CalDAV basic-auth password (required — never passed as a flag value)")
 	renderURL := fs.String("render-url", "", "CalDAV collection maven publishes her own reminders to; empty disables rendering")
 	renderUser := fs.String("render-user", "", "basic-auth username for -render-url (defaults to -user)")
-	renderPass := fs.String("render-pass", "", "basic-auth password for -render-url (defaults to -pass)")
+	renderPassFile := fs.String("render-pass-file", "", "file holding the password for -render-url (defaults to -pass-file)")
 	renderDur := fs.Duration("render-duration", calendar.DefaultReminderDuration, "how long a rendered reminder occupies")
 	interval := fs.Duration("interval", 5*time.Minute, "poll cadence")
 	timeout := fs.Duration("timeout", 10*time.Second, "per-request HTTP timeout")
@@ -63,13 +63,22 @@ func run(args []string) error {
 	if *socket == "" {
 		return fmt.Errorf("-socket is required")
 	}
-	if *url == "" || *user == "" || *pass == "" {
-		return fmt.Errorf("-url, -user, -pass are required")
+	if *url == "" || *user == "" || *passFile == "" {
+		return fmt.Errorf("-url, -user, -pass-file are required")
 	}
 	if err := checkRenderTarget([]string{*url}, *renderURL); err != nil {
 		return err
 	}
 
+	// The password is read from a file, never taken as a flag value: an argv
+	// secret is visible in `ps` to every user on the box and lands in the compose
+	// file and the shell history. Same rule mavmaild and mavpoll follow. Read
+	// once at start, so a rotated password means a restart.
+	pass, err := readSecret(*passFile)
+	if err != nil {
+		return err
+	}
+
 	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
 	defer stop()
 
@@ -85,17 +94,20 @@ func run(args []string) error {
 		http: hc,
 		url:  strings.TrimRight(*url, "/"),
 		user: *user,
-		pass: *pass,
+		pass: pass,
 	}
 
 	var rend *renderer
 	if *renderURL != "" {
-		ru, rp := *renderUser, *renderPass
+		ru, rp := *renderUser, pass
 		if ru == "" {
 			ru = *user
 		}
-		if rp == "" {
-			rp = *pass
+		if *renderPassFile != "" {
+			rp, err = readSecret(*renderPassFile)
+			if err != nil {
+				return err
+			}
 		}
 		rend = newRenderer(core, hc, *renderURL, ru, rp, *renderDur)
 		log.Printf("mavcaldav: rendering reminders to %s", *renderURL)
@@ -131,6 +143,21 @@ func run(args []string) error {
 // It takes the whole read set, not one URL. The guarantee in the package
 // comment is about every calendar maven reads, and a second read target added
 // later must not quietly fall outside the check.
+// readSecret reads one credential from a file and refuses an empty one. An
+// empty file is a deployment mistake, not a password, and CalDAV basic auth
+// would send it and get a 401 every poll.
+func readSecret(path string) (string, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return "", fmt.Errorf("read password file: %w", err)
+	}
+	secret := strings.TrimSpace(string(raw))
+	if secret == "" {
+		return "", fmt.Errorf("password file %s is empty", path)
+	}
+	return secret, nil
+}
+
 func checkRenderTarget(readURLs []string, renderURL string) error {
 	if renderURL == "" {
 		return nil

cmd/mavcaldav/main_test.go

@@ -5,12 +5,38 @@ import (
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"os"
+	"path/filepath"
 	"testing"
 	"time"
 
 	"github.com/kami/maven/internal/ipc"
 )
 
+// The password comes from a file so it never reaches argv. An empty or missing
+// file must fail at start rather than authenticate as "" against his calendar.
+func TestReadSecret(t *testing.T) {
+	dir := t.TempDir()
+	good := filepath.Join(dir, "ok")
+	if err := os.WriteFile(good, []byte("  hunter2\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if got, err := readSecret(good); err != nil || got != "hunter2" {
+		t.Fatalf("readSecret(good) = %q, %v; want \"hunter2\", nil", got, err)
+	}
+
+	empty := filepath.Join(dir, "empty")
+	if err := os.WriteFile(empty, []byte("\n \n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := readSecret(empty); err == nil {
+		t.Fatal("readSecret(empty) = nil error, want refusal")
+	}
+	if _, err := readSecret(filepath.Join(dir, "absent")); err == nil {
+		t.Fatal("readSecret(absent) = nil error, want refusal")
+	}
+}
+
 type fakeCore struct {
 	ipc.UnimplementedCoreAPI
 	facts    map[string]ipc.Fact // composite key "key|source" → Fact

docker-compose.yml

@@ -157,6 +157,44 @@ services:
   #     - maildata:/var/lib/mavmaild
   #     - ./deploy/imap.password:/run/secrets/imap.password:ro
 
+  # The calendar reader (Vikunja #644) is OFF and commented out: it needs a
+  # CalDAV account, and there is none on this box. It was built, listed in
+  # `make build`, and deployed nowhere, which is the worst of the three states —
+  # this block records the decision instead.
+  #
+  # What its absence costs, so the cost is visible from here:
+  #   - Agenda questions route correctly and answer from nothing. Stage 0 sends
+  #     "что у меня сегодня" to IntentQuery (V-498) and the `calendar` query
+  #     source reads facts(kind=env, source=caldav:*) that nobody writes.
+  #   - The nudge gate loses a suppressor. loop.State.CalendarBusy is fed by
+  #     those same facts, so "do not nag mid-meeting" is permanently false.
+  #
+  # Core never sees the CalDAV password: the reader polls the collection itself
+  # and hands core one fact per event over WriteFact. Nothing here can create a
+  # reminder, so a misread event cannot fire.
+  #
+  # The password is read from a FILE, so it never appears in `ps`, in this file,
+  # or in shell history — the same rule mavpoll and mavmaild follow.
+  #
+  # To enable: write the password to deploy/caldav.password (0600, gitignored),
+  # point -url at the collection, and uncomment this service. No mavend.json
+  # block is needed — events arrive over IPC as facts. -render-url is optional
+  # and OFF here: it publishes Maven's own reminders back as events, and it must
+  # not name the collection -url reads, or the poller reads its own writes back
+  # in (checkRenderTarget refuses that). It takes -render-pass-file, and falls
+  # back to this password when that is not given.
+  # mavcaldav:
+  #   <<: *image
+  #   command: ["mavcaldav", "-socket", "/run/maven/mavend.sock",
+  #             "-url", "http://localhost:5232/kami/personal",
+  #             "-user", "kami",
+  #             "-pass-file", "/run/secrets/caldav.password",
+  #             "-interval", "5m"]
+  #   depends_on: [mavend]
+  #   volumes:
+  #     - sockets:/run/maven
+  #     - ./deploy/caldav.password:/run/secrets/caldav.password:ro
+
 volumes:
   dbdata:
   sockets:

internal/rss/poller.go

@@ -89,8 +89,8 @@ type Poller struct {
 	ranker  Ranker
 	cfg     Config
 	nextDue map[string]time.Time
-	seen    map[string]map[string]bool // feed → item ID, for items with no date
-	polled  map[string]bool            // feed → polled at least once in THIS process
+	seen    map[string]*seenIDs // feed → item IDs, for items with no date
+	polled  map[string]bool     // feed → polled at least once in THIS process
 }
 
 // NewPoller wires a poller. Returns nil when there is nothing to poll — a
@@ -122,7 +122,7 @@ func NewPoller(feeds []FeedConfig, fetch Fetcher, notes Notes, marks Marks, embe
 		feeds: valid, fetch: fetch, notes: notes, marks: marks,
 		embed: embed, ranker: ranker, cfg: cfg,
 		nextDue: map[string]time.Time{},
-		seen:    map[string]map[string]bool{},
+		seen:    map[string]*seenIDs{},
 		polled:  map[string]bool{},
 	}
 }
@@ -283,6 +283,38 @@ func (p *Poller) mark(ctx context.Context, feed string, now time.Time) (time.Tim
 	return at, true
 }
 
+// maxSeenPerFeed bounds the undated-item set. It has to stay comfortably above
+// any one feed's front page, or an item still listed there would fall out of the
+// set and be written a second time. A few hundred entries covers the largest
+// page anyone publishes, and the set only has to span one poll window plus the
+// resync guard, not all of history.
+const maxSeenPerFeed = 512
+
+// seenIDs is a bounded insertion-ordered set. The map answers the lookup, the
+// slice remembers what to drop first, so an undated feed cannot grow the poller
+// for as long as mavend runs.
+type seenIDs struct {
+	ids   map[string]bool
+	order []string
+}
+
+// add records id and reports whether it was new.
+func (s *seenIDs) add(id string) bool {
+	if s.ids == nil {
+		s.ids = make(map[string]bool, maxSeenPerFeed)
+	}
+	if s.ids[id] {
+		return false
+	}
+	s.ids[id] = true
+	s.order = append(s.order, id)
+	if len(s.order) > maxSeenPerFeed {
+		delete(s.ids, s.order[0])
+		s.order = s.order[1:]
+	}
+	return true
+}
+
 // fresh — two dedup rules, because feeds are inconsistent about dates. A dated
 // item must be newer than the mark; an undated one is kept once per process by
 // ID.
@@ -308,12 +340,11 @@ func (p *Poller) fresh(f FeedConfig, it Item, mark, now time.Time, resync bool)
 		id = it.Title
 	}
 	if p.seen[f.Name] == nil {
-		p.seen[f.Name] = map[string]bool{}
+		p.seen[f.Name] = &seenIDs{}
 	}
-	if p.seen[f.Name][id] {
+	if !p.seen[f.Name].add(id) {
 		return false
 	}
-	p.seen[f.Name][id] = true
 	return !resync
 }
 

internal/rss/poller_test.go

@@ -3,6 +3,7 @@ package rss
 import (
 	"context"
 	"errors"
+	"fmt"
 	"strings"
 	"testing"
 	"time"
@@ -208,3 +209,27 @@ func TestNoFeedsMeansNoPoller(t *testing.T) {
 		t.Fatal("a feed with no name or url is not a configuration")
 	}
 }
+
+// An undated feed used to grow p.seen for as long as mavend ran. The set is
+// bounded now, and the bound must not cost the dedupe an item still on the
+// front page — only ids far older than any page fall out.
+func TestSeenIDsBounded(t *testing.T) {
+	var s seenIDs
+	for i := 0; i < maxSeenPerFeed*3; i++ {
+		if !s.add(fmt.Sprintf("item-%d", i)) {
+			t.Fatalf("item-%d read as already seen", i)
+		}
+		if len(s.ids) > maxSeenPerFeed || len(s.order) > maxSeenPerFeed {
+			t.Fatalf("after %d inserts: ids=%d order=%d, cap is %d",
+				i+1, len(s.ids), len(s.order), maxSeenPerFeed)
+		}
+	}
+	// The newest insert is still deduped; the oldest was evicted.
+	last := fmt.Sprintf("item-%d", maxSeenPerFeed*3-1)
+	if s.add(last) {
+		t.Fatalf("%s read as new, so the most recent id was dropped", last)
+	}
+	if !s.add("item-0") {
+		t.Fatal("item-0 survived, so nothing was evicted")
+	}
+}

internal/webfetch/webfetch.go

@@ -295,6 +295,27 @@ func (f *Fetcher) checkURL(u *url.URL) error {
 	return nil
 }
 
+// pruneHostsAbove is when pruneLocked bothers to walk the map. Below it the
+// walk costs more than the entries do, and `crawl.on_demand` means the host set
+// is whatever he names out loud, so it grows slowly.
+const pruneHostsAbove = 64
+
+// pruneLocked drops hosts whose last dial is further back than HostInterval.
+// Such an entry cannot delay anything — waitTurn would let the next request
+// through immediately — so keeping it only holds memory for the life of the
+// process. Caller holds f.mu.
+func (f *Fetcher) pruneLocked(now time.Time) {
+	if len(f.last) <= pruneHostsAbove {
+		return
+	}
+	cutoff := now.Add(-f.cfg.HostInterval)
+	for h, at := range f.last {
+		if at.Before(cutoff) {
+			delete(f.last, h)
+		}
+	}
+}
+
 // waitTurn blocks until this host's rate-limit interval has elapsed. It holds
 // no lock while sleeping, so two hosts never wait on each other.
 func (f *Fetcher) waitTurn(ctx context.Context, host string) error {
@@ -304,6 +325,7 @@ func (f *Fetcher) waitTurn(ctx context.Context, host string) error {
 		earliest := f.last[host].Add(f.cfg.HostInterval)
 		if !now.Before(earliest) {
 			f.last[host] = now
+			f.pruneLocked(now)
 			f.mu.Unlock()
 			return nil
 		}

internal/webfetch/webfetch_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"errors"
+	"fmt"
 	"io"
 	"net"
 	"net/http"
@@ -318,3 +319,37 @@ func TestPostObeysDenylist(t *testing.T) {
 		t.Fatalf("error = %v, want ErrBlocked", err)
 	}
 }
+
+// f.last used to hold one entry per host ever dialed, for the life of the
+// process. A host whose last dial is older than HostInterval cannot delay
+// anything, so it is dropped once the map is worth walking.
+func TestHostRateMapIsPruned(t *testing.T) {
+	f := New(Config{HostInterval: time.Minute, AllowPrivate: true})
+	stale := time.Now().Add(-time.Hour)
+	for i := 0; i < pruneHostsAbove*2; i++ {
+		f.last[fmt.Sprintf("h%d.example", i)] = stale
+	}
+
+	// One real turn is what triggers the sweep.
+	if err := f.waitTurn(context.Background(), "fresh.example"); err != nil {
+		t.Fatal(err)
+	}
+	if len(f.last) != 1 {
+		t.Fatalf("len(f.last) = %d after the sweep, want 1 (only the host just dialed)", len(f.last))
+	}
+	if _, ok := f.last["fresh.example"]; !ok {
+		t.Fatal("the host just dialed was pruned, so its own rate limit is lost")
+	}
+
+	// A host inside the interval is kept: pruning must not hand out a free turn.
+	f.last["recent.example"] = time.Now()
+	for i := 0; i < pruneHostsAbove*2; i++ {
+		f.last[fmt.Sprintf("g%d.example", i)] = stale
+	}
+	if err := f.waitTurn(context.Background(), "other.example"); err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := f.last["recent.example"]; !ok {
+		t.Fatal("a host dialed inside HostInterval was pruned")
+	}
+}