From 57161fb762d640a107ead8e20f18525087fcbea1 Mon Sep 17 00:00:00 2001 From: kami Date: Sat, 1 Aug 2026 14:24:40 +0400 Subject: [PATCH] store: keep what she read out of what he said MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing at read time told a feed item or a crawled page apart from his own notes. QueryNotes ranked every note by cosine and the notes answer handed the nearest five to the phraser, so "что я говорил про переезд" could be answered out of a stranger's web page, prefixed with "вот что я нашла: ". Recall now excludes the read sources, rss: and crawl:, and the list is one place. The feed answer needed a different read as a result, and it needed one anyway: it scanned the last 200 notes of any source, so a busy day of voice notes pushed the newest headline out of the window and she said "в лентах пока ничего нового" while the poller was working fine. RecentNotesFromSource asks for feed notes by source, so the window holds 200 of them. Found in review of #66 and #67. --- cmd/mavend/actions_query.go | 10 +++--- cmd/mavend/feeds_test.go | 23 ++++++++++++-- deploy/README.md | 13 ++++++-- internal/ipc/api.go | 9 ++++++ internal/ipc/client.go | 9 ++++++ internal/ipc/server.go | 22 +++++++++++++ internal/ipc/unimplemented.go | 4 +++ internal/ipc/wire.go | 1 + internal/store/notes.go | 59 ++++++++++++++++++++++++++++++++++- internal/store/notes_test.go | 40 ++++++++++++++++++++++++ 10 files changed, 180 insertions(+), 10 deletions(-) diff --git a/cmd/mavend/actions_query.go b/cmd/mavend/actions_query.go index 13b33be..6918283 100644 --- a/cmd/mavend/actions_query.go +++ b/cmd/mavend/actions_query.go @@ -206,7 +206,7 @@ func (h *reactiveHandler) queryHabits(ctx context.Context, t *queryTurn) (string return profile.FormatOverallRU(), true } -// feedNoteWindow — how many recent notes are scanned for feed items, and +// feedNoteWindow — how many recent FEED notes are scanned, and // feedReadOut — how many headlines she actually reads back. She summarises the // top of the pile, she does not recite a river. const ( @@ -231,16 +231,16 @@ func (h *reactiveHandler) queryFeeds(ctx context.Context, t *queryTurn) (string, // news bulletin. return "я пока не читаю ленты — они не настроены.", true } - notes, err := h.api.RecentNotes(ctx, feedNoteWindow) + // By source, not the last 200 notes of any kind: a busy day of voice notes + // used to push the newest headline out of the window, and she answered "в + // лентах пока ничего нового" while the poller was working fine. + notes, err := h.api.RecentNotesFromSource(ctx, rss.SourcePrefix, feedNoteWindow) if err != nil { log.Printf("voice: feeds: recent notes: %v", err) return "не получилось посмотреть ленты.", true } var picked []string for _, n := range notes { - if !strings.HasPrefix(n.Source, rss.SourcePrefix) { - continue - } if !router.CategoryMatches(rss.NoteCategory(n.Text), q.Category) { continue } diff --git a/cmd/mavend/feeds_test.go b/cmd/mavend/feeds_test.go index 50783b0..0759c50 100644 --- a/cmd/mavend/feeds_test.go +++ b/cmd/mavend/feeds_test.go @@ -88,12 +88,12 @@ func TestQueryFeedsByCategory(t *testing.T) { // answered by the model inventing a bulletin. func TestQueryFeedsOffAndEmptyDiffer(t *testing.T) { off := buildFeedHandler(t, false) - reply, ok := askFeeds(t, off, "что нового?") + reply, ok := askFeeds(t, off, "что нового в лентах?") if !ok || !strings.Contains(reply, "не настроены") { t.Fatalf("feeds off: reply = %q, ok = %v", reply, ok) } on := buildFeedHandler(t, true) - reply, ok = askFeeds(t, on, "что нового?") + reply, ok = askFeeds(t, on, "что нового в лентах?") if !ok || !strings.Contains(reply, "ничего нового") { t.Fatalf("feeds on but empty: reply = %q, ok = %v", reply, ok) } @@ -104,6 +104,25 @@ func TestQueryFeedsPassesOnANonFeedQuestion(t *testing.T) { if reply, ok := askFeeds(t, h, "напомни полить цветы"); ok { t.Fatalf("claimed an unrelated question with %q", reply) } + // The bare greeting is not a request for headlines. It used to be answered + // with a configuration status. + if reply, ok := askFeeds(t, h, "что нового?"); ok { + t.Fatalf("claimed a greeting with %q", reply) + } +} + +// A busy day of his own notes must not push the newest headline out of the +// window the feed answer scans. +func TestQueryFeedsIsNotCrowdedOutByHisOwnNotes(t *testing.T) { + notes := []ipc.Note{{Text: "Релиз ядра [технологии]", Source: "rss:habr"}} + for i := 0; i < feedNoteWindow+10; i++ { + notes = append(notes, ipc.Note{Text: "мысль вслух", Source: "tap:voice"}) + } + h := buildFeedHandler(t, true, notes...) + reply, ok := askFeeds(t, h, "что нового в лентах?") + if !ok || !strings.Contains(reply, "ядра") { + t.Fatalf("reply = %q, ok = %v; the headline fell out of the window", reply, ok) + } } // The mark is what stops a restart from re-noting yesterday's headlines, so the diff --git a/deploy/README.md b/deploy/README.md index d57942f..a1d2cb1 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -65,7 +65,13 @@ What it does and does not do: response at 2 MiB and redirects at 3, and makes at most one request per host per second. See `internal/webfetch`; - how far each feed was read is stored as a config fact `rss:latest:`, so - a restart does not re-note yesterday's headlines. + a restart does not re-note yesterday's headlines. A feed whose items carry no + dates gets the same mark, and the first poll after a restart takes those items + as already read rather than writing them all again; +- `max_items` paces, it does not drop: a burst larger than the cap arrives over + the following polls, oldest first; +- feed notes are **not** part of recall. "что я говорил про X" searches what he + said; headlines are read back only by asking about the feeds. ### Reading a page (`crawl`, also off by default) @@ -95,7 +101,10 @@ switched: refusal she says out loud. `Crawl-delay` is honoured; - same guarded fetcher as the feeds: allowlist/denylist, no private addresses, size cap, redirect cap, timeout, one request per host per second; -- dedup state is the config fact `crawl:hash:`. +- dedup state is the config fact `crawl:hash:`; +- like feed notes, watch notes are kept out of recall (`store.ReadSourcePrefixes`). + Text from someone else's page is not something he said, so it must not come + back as an answer to a question about him. Watch notes are visible on `/dash`. ## Not yet verified / host-dependent diff --git a/internal/ipc/api.go b/internal/ipc/api.go index 711c2aa..b6a276d 100644 --- a/internal/ipc/api.go +++ b/internal/ipc/api.go @@ -454,6 +454,10 @@ type outcomesReq struct { type nReq struct { N int `json:"n"` } +type sourceNReq struct { + Prefix string `json:"prefix"` + N int `json:"n"` +} type calendarEventsReq struct { From time.Time `json:"from"` To time.Time `json:"to"` @@ -606,6 +610,11 @@ type CoreAPI interface { WriteNote(ctx context.Context, ts time.Time, text string, embedding []float32, source string) (int64, error) QueryNotes(ctx context.Context, embedding []float32, k int) ([]Note, error) RecentNotes(ctx context.Context, n int) ([]Note, error) + // RecentNotesFromSource — the newest n notes whose source starts with + // prefix. Notes Maven read rather than heard (rss:, crawl:) are excluded + // from recall, so this is the only way to reach them, and it keeps the feed + // answer from being crowded out of a fixed window by his own notes. + RecentNotesFromSource(ctx context.Context, prefix string, n int) ([]Note, error) // ProposeTool drafts an inert 'proposed' tool scaffold (maven-callable); // returns whether a new proposal was written. EnableTool fills cmd + diff --git a/internal/ipc/client.go b/internal/ipc/client.go index 1009e58..e5c2617 100644 --- a/internal/ipc/client.go +++ b/internal/ipc/client.go @@ -66,6 +66,7 @@ var readOnlyMethods = map[Method]bool{ MethodRecentNudges: true, MethodQueryNotes: true, MethodRecentNotes: true, + MethodRecentNotesFromSource: true, MethodLookupTool: true, MethodListTools: true, MethodListProposedRoutines: true, @@ -375,6 +376,14 @@ func (c *Client) RecentNotes(ctx context.Context, n int) ([]Note, error) { return out, nil } +func (c *Client) RecentNotesFromSource(ctx context.Context, prefix string, n int) ([]Note, error) { + var out []Note + if err := c.call(ctx, MethodRecentNotesFromSource, sourceNReq{Prefix: prefix, N: n}, &out); err != nil { + return nil, err + } + return out, nil +} + func (c *Client) ProposeTool(ctx context.Context, name, utterance, scope string, ts time.Time) (bool, error) { var r proposeToolResp if err := c.call(ctx, MethodProposeTool, proposeToolReq{Name: name, Scope: scope, Utterance: utterance, Ts: ts}, &r); err != nil { diff --git a/internal/ipc/server.go b/internal/ipc/server.go index 772012d..42e34d9 100644 --- a/internal/ipc/server.go +++ b/internal/ipc/server.go @@ -161,6 +161,18 @@ func (a *storeAPI) QueryNotes(ctx context.Context, embedding []float32, k int) ( return out, nil } +func (a *storeAPI) RecentNotesFromSource(ctx context.Context, prefix string, n int) ([]Note, error) { + ns, err := a.s.RecentNotesFromSource(ctx, prefix, n) + if err != nil { + return nil, mapErr(err) + } + out := make([]Note, len(ns)) + for i, note := range ns { + out[i] = toNote(note) + } + return out, nil +} + func (a *storeAPI) RecentNotes(ctx context.Context, n int) ([]Note, error) { ns, err := a.s.RecentNotes(ctx, n) if err != nil { @@ -803,6 +815,16 @@ var methodTable = map[Method]handlerFunc{ } return out, nil }), + MethodRecentNotesFromSource: withParams(func(ctx context.Context, api CoreAPI, p sourceNReq) ([]Note, error) { + out, err := api.RecentNotesFromSource(ctx, p.Prefix, p.N) + if err != nil { + return nil, err + } + if out == nil { + out = []Note{} + } + return out, nil + }), MethodRecentNotes: withParams(func(ctx context.Context, api CoreAPI, p nReq) ([]Note, error) { out, err := api.RecentNotes(ctx, p.N) if err != nil { diff --git a/internal/ipc/unimplemented.go b/internal/ipc/unimplemented.go index 20e1b9f..b29d7ec 100644 --- a/internal/ipc/unimplemented.go +++ b/internal/ipc/unimplemented.go @@ -74,6 +74,10 @@ func (UnimplementedCoreAPI) WriteNote(ctx context.Context, ts time.Time, text st func (UnimplementedCoreAPI) QueryNotes(ctx context.Context, embedding []float32, k int) ([]Note, error) { return nil, ErrNotImplemented } +func (UnimplementedCoreAPI) RecentNotesFromSource(ctx context.Context, prefix string, n int) ([]Note, error) { + return nil, ErrUnknownMethod +} + func (UnimplementedCoreAPI) RecentNotes(ctx context.Context, n int) ([]Note, error) { return nil, ErrNotImplemented } diff --git a/internal/ipc/wire.go b/internal/ipc/wire.go index d9eea80..e63dfb0 100644 --- a/internal/ipc/wire.go +++ b/internal/ipc/wire.go @@ -30,6 +30,7 @@ const ( MethodWriteNote Method = "write_note" MethodQueryNotes Method = "query_notes" MethodRecentNotes Method = "recent_notes" + MethodRecentNotesFromSource Method = "recent_notes_source" MethodProposeTool Method = "propose_tool" MethodEnableTool Method = "enable_tool" MethodDisableTool Method = "disable_tool" diff --git a/internal/store/notes.go b/internal/store/notes.go index b2b4179..114b82e 100644 --- a/internal/store/notes.go +++ b/internal/store/notes.go @@ -6,6 +6,7 @@ import ( "fmt" "math" "sort" + "strings" "time" ) @@ -42,7 +43,8 @@ func (s *Store) WriteNote(ctx context.Context, ts time.Time, text string, embedd // or an ANN index only when note count or latency actually bites — at personal // scale (hundreds–thousands) a full scan is sub-millisecond. func (s *Store) QueryNotes(ctx context.Context, embedding []float32, k int) ([]Note, error) { - rows, err := s.db.QueryContext(ctx, `SELECT id, ts, text, embedding, source FROM notes`) + rows, err := s.db.QueryContext(ctx, + `SELECT id, ts, text, embedding, source FROM notes WHERE `+notHisWordsSQL) if err != nil { return nil, fmt.Errorf("query notes: %w", err) } @@ -76,6 +78,61 @@ func (s *Store) QueryNotes(ctx context.Context, embedding []float32, k int) ([]N return out, nil } +// ReadSourcePrefixes — note sources that are text Maven READ somewhere, not +// text he said or wrote: RSS items (internal/rss) and crawled pages +// (internal/crawl). +// +// They are notes because that is where fetched text lands, and they are kept out +// of recall because recall answers questions about HIM. "что я говорил про +// переезд" must not be answered out of a stranger's web page that happened to +// land near in the vector space, and the fallback line for that is "вот что я +// нашла: " followed by the stranger's words. Ask for them by source instead — +// that is what RecentNotesFromSource is for. +var ReadSourcePrefixes = []string{"rss:", "crawl:"} + +// notHisWordsSQL — the WHERE clause that drops the read sources. Built from the +// list above so adding a source is one line. +var notHisWordsSQL = buildNotHisWordsSQL() + +func buildNotHisWordsSQL() string { + var b strings.Builder + b.WriteString("(source IS NULL OR (") + for i, p := range ReadSourcePrefixes { + if i > 0 { + b.WriteString(" AND ") + } + fmt.Fprintf(&b, "source NOT LIKE '%s%%'", p) + } + b.WriteString("))") + return b.String() +} + +// RecentNotesFromSource returns the newest n notes whose source starts with +// prefix, newest first. The answer path for feeds and watches uses it: scanning +// the last 200 notes of ANY source meant a busy day of voice notes pushed the +// newest headline out of the window, and she said "в лентах пока ничего нового" +// while the poller was working fine. +func (s *Store) RecentNotesFromSource(ctx context.Context, prefix string, n int) ([]Note, error) { + rows, err := s.db.QueryContext(ctx, + `SELECT id, ts, text, source FROM notes WHERE source LIKE ? ORDER BY ts DESC LIMIT ?`, + prefix+"%", n) + if err != nil { + return nil, fmt.Errorf("recent notes by source: %w", err) + } + defer rows.Close() + var out []Note + for rows.Next() { + var nt Note + var tsMilli int64 + if err := rows.Scan(&nt.ID, &tsMilli, &nt.Text, &nt.Source); err != nil { + return nil, err + } + nt.Ts = time.UnixMilli(tsMilli).UTC() + out = append(out, nt) + } + return out, rows.Err() +} + // RecentNotes returns the newest n notes, newest first — a browse view (no // embedding math; Score stays 0). This is the read surface for /dash: notes // captured by voice are otherwise only reachable through semantic query. diff --git a/internal/store/notes_test.go b/internal/store/notes_test.go index 068eb40..df7efa8 100644 --- a/internal/store/notes_test.go +++ b/internal/store/notes_test.go @@ -36,3 +36,43 @@ func TestQueryNotesRanksByCosine(t *testing.T) { t.Errorf("scores not descending: %.3f then %.3f", got[0].Score, got[1].Score) } } + +// Recall answers questions about HIM. A feed item and a crawled page are text +// Maven read somewhere, and letting them into the nearest-neighbour pool means +// "что я говорил про переезд" can be answered with a stranger's sentence, under +// the line "вот что я нашла: ". +func TestQueryNotesLeavesOutWhatSheOnlyRead(t *testing.T) { + ctx := context.Background() + st := newTestStore(t) + now := time.Now() + // The read sources sit exactly on the query vector; his own note is further + // away, so ranking alone would put them first. + for _, n := range []struct{ text, source string }{ + {"переезд в новую квартиру описан тут", "rss:habr"}, + {"страница про переезд", "crawl:changelog"}, + } { + if _, err := st.WriteNote(ctx, now, n.text, []float32{1, 0, 0}, n.source); err != nil { + t.Fatal(err) + } + } + if _, err := st.WriteNote(ctx, now, "переезд в субботу", []float32{0.8, 0.6, 0}, "tap:voice"); err != nil { + t.Fatal(err) + } + + got, err := st.QueryNotes(ctx, []float32{1, 0, 0}, 5) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].Source != "tap:voice" { + t.Fatalf("recall returned %+v; want only what he said himself", got) + } + + // They are still reachable, by source. + feed, err := st.RecentNotesFromSource(ctx, "rss:", 10) + if err != nil { + t.Fatal(err) + } + if len(feed) != 1 || feed[0].Source != "rss:habr" { + t.Fatalf("RecentNotesFromSource(rss:) = %+v; want the one feed note", feed) + } +}