store: keep what she read out of what he said

Nothing at read time told a feed item or a crawled page apart from his own
notes. QueryNotes ranked every note by cosine and the notes answer handed the
nearest five to the phraser, so "что я говорил про переезд" could be answered
out of a stranger's web page, prefixed with "вот что я нашла: ". Recall now
excludes the read sources, rss: and crawl:, and the list is one place.

The feed answer needed a different read as a result, and it needed one anyway:
it scanned the last 200 notes of any source, so a busy day of voice notes pushed
the newest headline out of the window and she said "в лентах пока ничего
нового" while the poller was working fine. RecentNotesFromSource asks for feed
notes by source, so the window holds 200 of them.

Found in review of #66 and #67.

cmd/mavend/actions_query.go

@@ -206,7 +206,7 @@ func (h *reactiveHandler) queryHabits(ctx context.Context, t *queryTurn) (string
 	return profile.FormatOverallRU(), true
 }
 
-// feedNoteWindow — how many recent notes are scanned for feed items, and
+// feedNoteWindow — how many recent FEED notes are scanned, and
 // feedReadOut — how many headlines she actually reads back. She summarises the
 // top of the pile, she does not recite a river.
 const (
@@ -231,16 +231,16 @@ func (h *reactiveHandler) queryFeeds(ctx context.Context, t *queryTurn) (string,
 		// news bulletin.
 		return "я пока не читаю ленты — они не настроены.", true
 	}
-	notes, err := h.api.RecentNotes(ctx, feedNoteWindow)
+	// By source, not the last 200 notes of any kind: a busy day of voice notes
+	// used to push the newest headline out of the window, and she answered "в
+	// лентах пока ничего нового" while the poller was working fine.
+	notes, err := h.api.RecentNotesFromSource(ctx, rss.SourcePrefix, feedNoteWindow)
 	if err != nil {
 		log.Printf("voice: feeds: recent notes: %v", err)
 		return "не получилось посмотреть ленты.", true
 	}
 	var picked []string
 	for _, n := range notes {
-		if !strings.HasPrefix(n.Source, rss.SourcePrefix) {
-			continue
-		}
 		if !router.CategoryMatches(rss.NoteCategory(n.Text), q.Category) {
 			continue
 		}

cmd/mavend/feeds_test.go

@@ -88,12 +88,12 @@ func TestQueryFeedsByCategory(t *testing.T) {
 // answered by the model inventing a bulletin.
 func TestQueryFeedsOffAndEmptyDiffer(t *testing.T) {
 	off := buildFeedHandler(t, false)
-	reply, ok := askFeeds(t, off, "что нового?")
+	reply, ok := askFeeds(t, off, "что нового в лентах?")
 	if !ok || !strings.Contains(reply, "не настроены") {
 		t.Fatalf("feeds off: reply = %q, ok = %v", reply, ok)
 	}
 	on := buildFeedHandler(t, true)
-	reply, ok = askFeeds(t, on, "что нового?")
+	reply, ok = askFeeds(t, on, "что нового в лентах?")
 	if !ok || !strings.Contains(reply, "ничего нового") {
 		t.Fatalf("feeds on but empty: reply = %q, ok = %v", reply, ok)
 	}
@@ -104,6 +104,25 @@ func TestQueryFeedsPassesOnANonFeedQuestion(t *testing.T) {
 	if reply, ok := askFeeds(t, h, "напомни полить цветы"); ok {
 		t.Fatalf("claimed an unrelated question with %q", reply)
 	}
+	// The bare greeting is not a request for headlines. It used to be answered
+	// with a configuration status.
+	if reply, ok := askFeeds(t, h, "что нового?"); ok {
+		t.Fatalf("claimed a greeting with %q", reply)
+	}
+}
+
+// A busy day of his own notes must not push the newest headline out of the
+// window the feed answer scans.
+func TestQueryFeedsIsNotCrowdedOutByHisOwnNotes(t *testing.T) {
+	notes := []ipc.Note{{Text: "Релиз ядра [технологии]", Source: "rss:habr"}}
+	for i := 0; i < feedNoteWindow+10; i++ {
+		notes = append(notes, ipc.Note{Text: "мысль вслух", Source: "tap:voice"})
+	}
+	h := buildFeedHandler(t, true, notes...)
+	reply, ok := askFeeds(t, h, "что нового в лентах?")
+	if !ok || !strings.Contains(reply, "ядра") {
+		t.Fatalf("reply = %q, ok = %v; the headline fell out of the window", reply, ok)
+	}
 }
 
 // The mark is what stops a restart from re-noting yesterday's headlines, so the

deploy/README.md

@@ -65,7 +65,13 @@ What it does and does not do:
   response at 2 MiB and redirects at 3, and makes at most one request per host
   per second. See `internal/webfetch`;
 - how far each feed was read is stored as a config fact `rss:latest:<name>`, so
-  a restart does not re-note yesterday's headlines.
+  a restart does not re-note yesterday's headlines. A feed whose items carry no
+  dates gets the same mark, and the first poll after a restart takes those items
+  as already read rather than writing them all again;
+- `max_items` paces, it does not drop: a burst larger than the cap arrives over
+  the following polls, oldest first;
+- feed notes are **not** part of recall. "что я говорил про X" searches what he
+  said; headlines are read back only by asking about the feeds.
 
 ### Reading a page (`crawl`, also off by default)
 
@@ -95,7 +101,10 @@ switched:
   refusal she says out loud. `Crawl-delay` is honoured;
 - same guarded fetcher as the feeds: allowlist/denylist, no private addresses,
   size cap, redirect cap, timeout, one request per host per second;
-- dedup state is the config fact `crawl:hash:<name>`.
+- dedup state is the config fact `crawl:hash:<name>`;
+- like feed notes, watch notes are kept out of recall (`store.ReadSourcePrefixes`).
+  Text from someone else's page is not something he said, so it must not come
+  back as an answer to a question about him. Watch notes are visible on `/dash`.
 
 ## Not yet verified / host-dependent
 

internal/ipc/api.go

@@ -454,6 +454,10 @@ type outcomesReq struct {
 type nReq struct {
 	N int `json:"n"`
 }
+type sourceNReq struct {
+	Prefix string `json:"prefix"`
+	N      int    `json:"n"`
+}
 type calendarEventsReq struct {
 	From time.Time `json:"from"`
 	To   time.Time `json:"to"`
@@ -606,6 +610,11 @@ type CoreAPI interface {
 	WriteNote(ctx context.Context, ts time.Time, text string, embedding []float32, source string) (int64, error)
 	QueryNotes(ctx context.Context, embedding []float32, k int) ([]Note, error)
 	RecentNotes(ctx context.Context, n int) ([]Note, error)
+	// RecentNotesFromSource — the newest n notes whose source starts with
+	// prefix. Notes Maven read rather than heard (rss:, crawl:) are excluded
+	// from recall, so this is the only way to reach them, and it keeps the feed
+	// answer from being crowded out of a fixed window by his own notes.
+	RecentNotesFromSource(ctx context.Context, prefix string, n int) ([]Note, error)
 
 	// ProposeTool drafts an inert 'proposed' tool scaffold (maven-callable);
 	// returns whether a new proposal was written. EnableTool fills cmd +

internal/ipc/client.go

@@ -66,6 +66,7 @@ var readOnlyMethods = map[Method]bool{
 	MethodRecentNudges:         true,
 	MethodQueryNotes:           true,
 	MethodRecentNotes:          true,
+	MethodRecentNotesFromSource: true,
 	MethodLookupTool:           true,
 	MethodListTools:            true,
 	MethodListProposedRoutines: true,
@@ -375,6 +376,14 @@ func (c *Client) RecentNotes(ctx context.Context, n int) ([]Note, error) {
 	return out, nil
 }
 
+func (c *Client) RecentNotesFromSource(ctx context.Context, prefix string, n int) ([]Note, error) {
+	var out []Note
+	if err := c.call(ctx, MethodRecentNotesFromSource, sourceNReq{Prefix: prefix, N: n}, &out); err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 func (c *Client) ProposeTool(ctx context.Context, name, utterance, scope string, ts time.Time) (bool, error) {
 	var r proposeToolResp
 	if err := c.call(ctx, MethodProposeTool, proposeToolReq{Name: name, Scope: scope, Utterance: utterance, Ts: ts}, &r); err != nil {

internal/ipc/server.go

@@ -161,6 +161,18 @@ func (a *storeAPI) QueryNotes(ctx context.Context, embedding []float32, k int) (
 	return out, nil
 }
 
+func (a *storeAPI) RecentNotesFromSource(ctx context.Context, prefix string, n int) ([]Note, error) {
+	ns, err := a.s.RecentNotesFromSource(ctx, prefix, n)
+	if err != nil {
+		return nil, mapErr(err)
+	}
+	out := make([]Note, len(ns))
+	for i, note := range ns {
+		out[i] = toNote(note)
+	}
+	return out, nil
+}
+
 func (a *storeAPI) RecentNotes(ctx context.Context, n int) ([]Note, error) {
 	ns, err := a.s.RecentNotes(ctx, n)
 	if err != nil {
@@ -803,6 +815,16 @@ var methodTable = map[Method]handlerFunc{
 		}
 		return out, nil
 	}),
+	MethodRecentNotesFromSource: withParams(func(ctx context.Context, api CoreAPI, p sourceNReq) ([]Note, error) {
+		out, err := api.RecentNotesFromSource(ctx, p.Prefix, p.N)
+		if err != nil {
+			return nil, err
+		}
+		if out == nil {
+			out = []Note{}
+		}
+		return out, nil
+	}),
 	MethodRecentNotes: withParams(func(ctx context.Context, api CoreAPI, p nReq) ([]Note, error) {
 		out, err := api.RecentNotes(ctx, p.N)
 		if err != nil {

internal/ipc/unimplemented.go

@@ -74,6 +74,10 @@ func (UnimplementedCoreAPI) WriteNote(ctx context.Context, ts time.Time, text st
 func (UnimplementedCoreAPI) QueryNotes(ctx context.Context, embedding []float32, k int) ([]Note, error) {
 	return nil, ErrNotImplemented
 }
+func (UnimplementedCoreAPI) RecentNotesFromSource(ctx context.Context, prefix string, n int) ([]Note, error) {
+	return nil, ErrUnknownMethod
+}
+
 func (UnimplementedCoreAPI) RecentNotes(ctx context.Context, n int) ([]Note, error) {
 	return nil, ErrNotImplemented
 }

internal/ipc/wire.go

@@ -30,6 +30,7 @@ const (
 	MethodWriteNote              Method = "write_note"
 	MethodQueryNotes             Method = "query_notes"
 	MethodRecentNotes            Method = "recent_notes"
+	MethodRecentNotesFromSource  Method = "recent_notes_source"
 	MethodProposeTool            Method = "propose_tool"
 	MethodEnableTool             Method = "enable_tool"
 	MethodDisableTool            Method = "disable_tool"

internal/store/notes.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"math"
 	"sort"
+	"strings"
 	"time"
 )
 
@@ -42,7 +43,8 @@ func (s *Store) WriteNote(ctx context.Context, ts time.Time, text string, embedd
 // or an ANN index only when note count or latency actually bites — at personal
 // scale (hundreds–thousands) a full scan is sub-millisecond.
 func (s *Store) QueryNotes(ctx context.Context, embedding []float32, k int) ([]Note, error) {
-	rows, err := s.db.QueryContext(ctx, `SELECT id, ts, text, embedding, source FROM notes`)
+	rows, err := s.db.QueryContext(ctx,
+		`SELECT id, ts, text, embedding, source FROM notes WHERE `+notHisWordsSQL)
 	if err != nil {
 		return nil, fmt.Errorf("query notes: %w", err)
 	}
@@ -76,6 +78,61 @@ func (s *Store) QueryNotes(ctx context.Context, embedding []float32, k int) ([]N
 	return out, nil
 }
 
+// ReadSourcePrefixes — note sources that are text Maven READ somewhere, not
+// text he said or wrote: RSS items (internal/rss) and crawled pages
+// (internal/crawl).
+//
+// They are notes because that is where fetched text lands, and they are kept out
+// of recall because recall answers questions about HIM. "что я говорил про
+// переезд" must not be answered out of a stranger's web page that happened to
+// land near in the vector space, and the fallback line for that is "вот что я
+// нашла: " followed by the stranger's words. Ask for them by source instead —
+// that is what RecentNotesFromSource is for.
+var ReadSourcePrefixes = []string{"rss:", "crawl:"}
+
+// notHisWordsSQL — the WHERE clause that drops the read sources. Built from the
+// list above so adding a source is one line.
+var notHisWordsSQL = buildNotHisWordsSQL()
+
+func buildNotHisWordsSQL() string {
+	var b strings.Builder
+	b.WriteString("(source IS NULL OR (")
+	for i, p := range ReadSourcePrefixes {
+		if i > 0 {
+			b.WriteString(" AND ")
+		}
+		fmt.Fprintf(&b, "source NOT LIKE '%s%%'", p)
+	}
+	b.WriteString("))")
+	return b.String()
+}
+
+// RecentNotesFromSource returns the newest n notes whose source starts with
+// prefix, newest first. The answer path for feeds and watches uses it: scanning
+// the last 200 notes of ANY source meant a busy day of voice notes pushed the
+// newest headline out of the window, and she said "в лентах пока ничего нового"
+// while the poller was working fine.
+func (s *Store) RecentNotesFromSource(ctx context.Context, prefix string, n int) ([]Note, error) {
+	rows, err := s.db.QueryContext(ctx,
+		`SELECT id, ts, text, source FROM notes WHERE source LIKE ? ORDER BY ts DESC LIMIT ?`,
+		prefix+"%", n)
+	if err != nil {
+		return nil, fmt.Errorf("recent notes by source: %w", err)
+	}
+	defer rows.Close()
+	var out []Note
+	for rows.Next() {
+		var nt Note
+		var tsMilli int64
+		if err := rows.Scan(&nt.ID, &tsMilli, &nt.Text, &nt.Source); err != nil {
+			return nil, err
+		}
+		nt.Ts = time.UnixMilli(tsMilli).UTC()
+		out = append(out, nt)
+	}
+	return out, rows.Err()
+}
+
 // RecentNotes returns the newest n notes, newest first — a browse view (no
 // embedding math; Score stays 0). This is the read surface for /dash: notes
 // captured by voice are otherwise only reachable through semantic query.

internal/store/notes_test.go

@@ -36,3 +36,43 @@ func TestQueryNotesRanksByCosine(t *testing.T) {
 		t.Errorf("scores not descending: %.3f then %.3f", got[0].Score, got[1].Score)
 	}
 }
+
+// Recall answers questions about HIM. A feed item and a crawled page are text
+// Maven read somewhere, and letting them into the nearest-neighbour pool means
+// "что я говорил про переезд" can be answered with a stranger's sentence, under
+// the line "вот что я нашла: ".
+func TestQueryNotesLeavesOutWhatSheOnlyRead(t *testing.T) {
+	ctx := context.Background()
+	st := newTestStore(t)
+	now := time.Now()
+	// The read sources sit exactly on the query vector; his own note is further
+	// away, so ranking alone would put them first.
+	for _, n := range []struct{ text, source string }{
+		{"переезд в новую квартиру описан тут", "rss:habr"},
+		{"страница про переезд", "crawl:changelog"},
+	} {
+		if _, err := st.WriteNote(ctx, now, n.text, []float32{1, 0, 0}, n.source); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if _, err := st.WriteNote(ctx, now, "переезд в субботу", []float32{0.8, 0.6, 0}, "tap:voice"); err != nil {
+		t.Fatal(err)
+	}
+
+	got, err := st.QueryNotes(ctx, []float32{1, 0, 0}, 5)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 1 || got[0].Source != "tap:voice" {
+		t.Fatalf("recall returned %+v; want only what he said himself", got)
+	}
+
+	// They are still reachable, by source.
+	feed, err := st.RecentNotesFromSource(ctx, "rss:", 10)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(feed) != 1 || feed[0].Source != "rss:habr" {
+		t.Fatalf("RecentNotesFromSource(rss:) = %+v; want the one feed note", feed)
+	}
+}