store: keep what she read out of what he said

Nothing at read time told a feed item or a crawled page apart from his own
notes. QueryNotes ranked every note by cosine and the notes answer handed the
nearest five to the phraser, so "что я говорил про переезд" could be answered
out of a stranger's web page, prefixed with "вот что я нашла: ". Recall now
excludes the read sources, rss: and crawl:, and the list is one place.

The feed answer needed a different read as a result, and it needed one anyway:
it scanned the last 200 notes of any source, so a busy day of voice notes pushed
the newest headline out of the window and she said "в лентах пока ничего
нового" while the poller was working fine. RecentNotesFromSource asks for feed
notes by source, so the window holds 200 of them.

Found in review of #66 and #67.

internal/ipc/api.go

@@ -454,6 +454,10 @@ type outcomesReq struct {
 type nReq struct {
 	N int `json:"n"`
 }
+type sourceNReq struct {
+	Prefix string `json:"prefix"`
+	N      int    `json:"n"`
+}
 type calendarEventsReq struct {
 	From time.Time `json:"from"`
 	To   time.Time `json:"to"`
@@ -606,6 +610,11 @@ type CoreAPI interface {
 	WriteNote(ctx context.Context, ts time.Time, text string, embedding []float32, source string) (int64, error)
 	QueryNotes(ctx context.Context, embedding []float32, k int) ([]Note, error)
 	RecentNotes(ctx context.Context, n int) ([]Note, error)
+	// RecentNotesFromSource — the newest n notes whose source starts with
+	// prefix. Notes Maven read rather than heard (rss:, crawl:) are excluded
+	// from recall, so this is the only way to reach them, and it keeps the feed
+	// answer from being crowded out of a fixed window by his own notes.
+	RecentNotesFromSource(ctx context.Context, prefix string, n int) ([]Note, error)
 
 	// ProposeTool drafts an inert 'proposed' tool scaffold (maven-callable);
 	// returns whether a new proposal was written. EnableTool fills cmd +

internal/ipc/client.go

@@ -66,6 +66,7 @@ var readOnlyMethods = map[Method]bool{
 	MethodRecentNudges:         true,
 	MethodQueryNotes:           true,
 	MethodRecentNotes:          true,
+	MethodRecentNotesFromSource: true,
 	MethodLookupTool:           true,
 	MethodListTools:            true,
 	MethodListProposedRoutines: true,
@@ -375,6 +376,14 @@ func (c *Client) RecentNotes(ctx context.Context, n int) ([]Note, error) {
 	return out, nil
 }
 
+func (c *Client) RecentNotesFromSource(ctx context.Context, prefix string, n int) ([]Note, error) {
+	var out []Note
+	if err := c.call(ctx, MethodRecentNotesFromSource, sourceNReq{Prefix: prefix, N: n}, &out); err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 func (c *Client) ProposeTool(ctx context.Context, name, utterance, scope string, ts time.Time) (bool, error) {
 	var r proposeToolResp
 	if err := c.call(ctx, MethodProposeTool, proposeToolReq{Name: name, Scope: scope, Utterance: utterance, Ts: ts}, &r); err != nil {

internal/ipc/server.go

@@ -161,6 +161,18 @@ func (a *storeAPI) QueryNotes(ctx context.Context, embedding []float32, k int) (
 	return out, nil
 }
 
+func (a *storeAPI) RecentNotesFromSource(ctx context.Context, prefix string, n int) ([]Note, error) {
+	ns, err := a.s.RecentNotesFromSource(ctx, prefix, n)
+	if err != nil {
+		return nil, mapErr(err)
+	}
+	out := make([]Note, len(ns))
+	for i, note := range ns {
+		out[i] = toNote(note)
+	}
+	return out, nil
+}
+
 func (a *storeAPI) RecentNotes(ctx context.Context, n int) ([]Note, error) {
 	ns, err := a.s.RecentNotes(ctx, n)
 	if err != nil {
@@ -803,6 +815,16 @@ var methodTable = map[Method]handlerFunc{
 		}
 		return out, nil
 	}),
+	MethodRecentNotesFromSource: withParams(func(ctx context.Context, api CoreAPI, p sourceNReq) ([]Note, error) {
+		out, err := api.RecentNotesFromSource(ctx, p.Prefix, p.N)
+		if err != nil {
+			return nil, err
+		}
+		if out == nil {
+			out = []Note{}
+		}
+		return out, nil
+	}),
 	MethodRecentNotes: withParams(func(ctx context.Context, api CoreAPI, p nReq) ([]Note, error) {
 		out, err := api.RecentNotes(ctx, p.N)
 		if err != nil {

internal/ipc/unimplemented.go

@@ -74,6 +74,10 @@ func (UnimplementedCoreAPI) WriteNote(ctx context.Context, ts time.Time, text st
 func (UnimplementedCoreAPI) QueryNotes(ctx context.Context, embedding []float32, k int) ([]Note, error) {
 	return nil, ErrNotImplemented
 }
+func (UnimplementedCoreAPI) RecentNotesFromSource(ctx context.Context, prefix string, n int) ([]Note, error) {
+	return nil, ErrUnknownMethod
+}
+
 func (UnimplementedCoreAPI) RecentNotes(ctx context.Context, n int) ([]Note, error) {
 	return nil, ErrNotImplemented
 }

internal/ipc/wire.go

@@ -30,6 +30,7 @@ const (
 	MethodWriteNote              Method = "write_note"
 	MethodQueryNotes             Method = "query_notes"
 	MethodRecentNotes            Method = "recent_notes"
+	MethodRecentNotesFromSource  Method = "recent_notes_source"
 	MethodProposeTool            Method = "propose_tool"
 	MethodEnableTool             Method = "enable_tool"
 	MethodDisableTool            Method = "disable_tool"

internal/store/notes.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"math"
 	"sort"
+	"strings"
 	"time"
 )
 
@@ -42,7 +43,8 @@ func (s *Store) WriteNote(ctx context.Context, ts time.Time, text string, embedd
 // or an ANN index only when note count or latency actually bites — at personal
 // scale (hundreds–thousands) a full scan is sub-millisecond.
 func (s *Store) QueryNotes(ctx context.Context, embedding []float32, k int) ([]Note, error) {
-	rows, err := s.db.QueryContext(ctx, `SELECT id, ts, text, embedding, source FROM notes`)
+	rows, err := s.db.QueryContext(ctx,
+		`SELECT id, ts, text, embedding, source FROM notes WHERE `+notHisWordsSQL)
 	if err != nil {
 		return nil, fmt.Errorf("query notes: %w", err)
 	}
@@ -76,6 +78,61 @@ func (s *Store) QueryNotes(ctx context.Context, embedding []float32, k int) ([]N
 	return out, nil
 }
 
+// ReadSourcePrefixes — note sources that are text Maven READ somewhere, not
+// text he said or wrote: RSS items (internal/rss) and crawled pages
+// (internal/crawl).
+//
+// They are notes because that is where fetched text lands, and they are kept out
+// of recall because recall answers questions about HIM. "что я говорил про
+// переезд" must not be answered out of a stranger's web page that happened to
+// land near in the vector space, and the fallback line for that is "вот что я
+// нашла: " followed by the stranger's words. Ask for them by source instead —
+// that is what RecentNotesFromSource is for.
+var ReadSourcePrefixes = []string{"rss:", "crawl:"}
+
+// notHisWordsSQL — the WHERE clause that drops the read sources. Built from the
+// list above so adding a source is one line.
+var notHisWordsSQL = buildNotHisWordsSQL()
+
+func buildNotHisWordsSQL() string {
+	var b strings.Builder
+	b.WriteString("(source IS NULL OR (")
+	for i, p := range ReadSourcePrefixes {
+		if i > 0 {
+			b.WriteString(" AND ")
+		}
+		fmt.Fprintf(&b, "source NOT LIKE '%s%%'", p)
+	}
+	b.WriteString("))")
+	return b.String()
+}
+
+// RecentNotesFromSource returns the newest n notes whose source starts with
+// prefix, newest first. The answer path for feeds and watches uses it: scanning
+// the last 200 notes of ANY source meant a busy day of voice notes pushed the
+// newest headline out of the window, and she said "в лентах пока ничего нового"
+// while the poller was working fine.
+func (s *Store) RecentNotesFromSource(ctx context.Context, prefix string, n int) ([]Note, error) {
+	rows, err := s.db.QueryContext(ctx,
+		`SELECT id, ts, text, source FROM notes WHERE source LIKE ? ORDER BY ts DESC LIMIT ?`,
+		prefix+"%", n)
+	if err != nil {
+		return nil, fmt.Errorf("recent notes by source: %w", err)
+	}
+	defer rows.Close()
+	var out []Note
+	for rows.Next() {
+		var nt Note
+		var tsMilli int64
+		if err := rows.Scan(&nt.ID, &tsMilli, &nt.Text, &nt.Source); err != nil {
+			return nil, err
+		}
+		nt.Ts = time.UnixMilli(tsMilli).UTC()
+		out = append(out, nt)
+	}
+	return out, rows.Err()
+}
+
 // RecentNotes returns the newest n notes, newest first — a browse view (no
 // embedding math; Score stays 0). This is the read surface for /dash: notes
 // captured by voice are otherwise only reachable through semantic query.

internal/store/notes_test.go

@@ -36,3 +36,43 @@ func TestQueryNotesRanksByCosine(t *testing.T) {
 		t.Errorf("scores not descending: %.3f then %.3f", got[0].Score, got[1].Score)
 	}
 }
+
+// Recall answers questions about HIM. A feed item and a crawled page are text
+// Maven read somewhere, and letting them into the nearest-neighbour pool means
+// "что я говорил про переезд" can be answered with a stranger's sentence, under
+// the line "вот что я нашла: ".
+func TestQueryNotesLeavesOutWhatSheOnlyRead(t *testing.T) {
+	ctx := context.Background()
+	st := newTestStore(t)
+	now := time.Now()
+	// The read sources sit exactly on the query vector; his own note is further
+	// away, so ranking alone would put them first.
+	for _, n := range []struct{ text, source string }{
+		{"переезд в новую квартиру описан тут", "rss:habr"},
+		{"страница про переезд", "crawl:changelog"},
+	} {
+		if _, err := st.WriteNote(ctx, now, n.text, []float32{1, 0, 0}, n.source); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if _, err := st.WriteNote(ctx, now, "переезд в субботу", []float32{0.8, 0.6, 0}, "tap:voice"); err != nil {
+		t.Fatal(err)
+	}
+
+	got, err := st.QueryNotes(ctx, []float32{1, 0, 0}, 5)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 1 || got[0].Source != "tap:voice" {
+		t.Fatalf("recall returned %+v; want only what he said himself", got)
+	}
+
+	// They are still reachable, by source.
+	feed, err := st.RecentNotesFromSource(ctx, "rss:", 10)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(feed) != 1 || feed[0].Source != "rss:habr" {
+		t.Fatalf("RecentNotesFromSource(rss:) = %+v; want the one feed note", feed)
+	}
+}