diff --git a/cmd/mavend/actions_query.go b/cmd/mavend/actions_query.go index 13b33be..6918283 100644 --- a/cmd/mavend/actions_query.go +++ b/cmd/mavend/actions_query.go @@ -206,7 +206,7 @@ func (h *reactiveHandler) queryHabits(ctx context.Context, t *queryTurn) (string return profile.FormatOverallRU(), true } -// feedNoteWindow — how many recent notes are scanned for feed items, and +// feedNoteWindow — how many recent FEED notes are scanned, and // feedReadOut — how many headlines she actually reads back. She summarises the // top of the pile, she does not recite a river. const ( @@ -231,16 +231,16 @@ func (h *reactiveHandler) queryFeeds(ctx context.Context, t *queryTurn) (string, // news bulletin. return "я пока не читаю ленты — они не настроены.", true } - notes, err := h.api.RecentNotes(ctx, feedNoteWindow) + // By source, not the last 200 notes of any kind: a busy day of voice notes + // used to push the newest headline out of the window, and she answered "в + // лентах пока ничего нового" while the poller was working fine. + notes, err := h.api.RecentNotesFromSource(ctx, rss.SourcePrefix, feedNoteWindow) if err != nil { log.Printf("voice: feeds: recent notes: %v", err) return "не получилось посмотреть ленты.", true } var picked []string for _, n := range notes { - if !strings.HasPrefix(n.Source, rss.SourcePrefix) { - continue - } if !router.CategoryMatches(rss.NoteCategory(n.Text), q.Category) { continue } diff --git a/cmd/mavend/feeds_test.go b/cmd/mavend/feeds_test.go index 50783b0..0759c50 100644 --- a/cmd/mavend/feeds_test.go +++ b/cmd/mavend/feeds_test.go @@ -88,12 +88,12 @@ func TestQueryFeedsByCategory(t *testing.T) { // answered by the model inventing a bulletin. func TestQueryFeedsOffAndEmptyDiffer(t *testing.T) { off := buildFeedHandler(t, false) - reply, ok := askFeeds(t, off, "что нового?") + reply, ok := askFeeds(t, off, "что нового в лентах?") if !ok || !strings.Contains(reply, "не настроены") { t.Fatalf("feeds off: reply = %q, ok = %v", reply, ok) } on := buildFeedHandler(t, true) - reply, ok = askFeeds(t, on, "что нового?") + reply, ok = askFeeds(t, on, "что нового в лентах?") if !ok || !strings.Contains(reply, "ничего нового") { t.Fatalf("feeds on but empty: reply = %q, ok = %v", reply, ok) } @@ -104,6 +104,25 @@ func TestQueryFeedsPassesOnANonFeedQuestion(t *testing.T) { if reply, ok := askFeeds(t, h, "напомни полить цветы"); ok { t.Fatalf("claimed an unrelated question with %q", reply) } + // The bare greeting is not a request for headlines. It used to be answered + // with a configuration status. + if reply, ok := askFeeds(t, h, "что нового?"); ok { + t.Fatalf("claimed a greeting with %q", reply) + } +} + +// A busy day of his own notes must not push the newest headline out of the +// window the feed answer scans. +func TestQueryFeedsIsNotCrowdedOutByHisOwnNotes(t *testing.T) { + notes := []ipc.Note{{Text: "Релиз ядра [технологии]", Source: "rss:habr"}} + for i := 0; i < feedNoteWindow+10; i++ { + notes = append(notes, ipc.Note{Text: "мысль вслух", Source: "tap:voice"}) + } + h := buildFeedHandler(t, true, notes...) + reply, ok := askFeeds(t, h, "что нового в лентах?") + if !ok || !strings.Contains(reply, "ядра") { + t.Fatalf("reply = %q, ok = %v; the headline fell out of the window", reply, ok) + } } // The mark is what stops a restart from re-noting yesterday's headlines, so the diff --git a/deploy/README.md b/deploy/README.md index d57942f..a1d2cb1 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -65,7 +65,13 @@ What it does and does not do: response at 2 MiB and redirects at 3, and makes at most one request per host per second. See `internal/webfetch`; - how far each feed was read is stored as a config fact `rss:latest:`, so - a restart does not re-note yesterday's headlines. + a restart does not re-note yesterday's headlines. A feed whose items carry no + dates gets the same mark, and the first poll after a restart takes those items + as already read rather than writing them all again; +- `max_items` paces, it does not drop: a burst larger than the cap arrives over + the following polls, oldest first; +- feed notes are **not** part of recall. "что я говорил про X" searches what he + said; headlines are read back only by asking about the feeds. ### Reading a page (`crawl`, also off by default) @@ -95,7 +101,10 @@ switched: refusal she says out loud. `Crawl-delay` is honoured; - same guarded fetcher as the feeds: allowlist/denylist, no private addresses, size cap, redirect cap, timeout, one request per host per second; -- dedup state is the config fact `crawl:hash:`. +- dedup state is the config fact `crawl:hash:`; +- like feed notes, watch notes are kept out of recall (`store.ReadSourcePrefixes`). + Text from someone else's page is not something he said, so it must not come + back as an answer to a question about him. Watch notes are visible on `/dash`. ## Not yet verified / host-dependent diff --git a/internal/ipc/api.go b/internal/ipc/api.go index 711c2aa..b6a276d 100644 --- a/internal/ipc/api.go +++ b/internal/ipc/api.go @@ -454,6 +454,10 @@ type outcomesReq struct { type nReq struct { N int `json:"n"` } +type sourceNReq struct { + Prefix string `json:"prefix"` + N int `json:"n"` +} type calendarEventsReq struct { From time.Time `json:"from"` To time.Time `json:"to"` @@ -606,6 +610,11 @@ type CoreAPI interface { WriteNote(ctx context.Context, ts time.Time, text string, embedding []float32, source string) (int64, error) QueryNotes(ctx context.Context, embedding []float32, k int) ([]Note, error) RecentNotes(ctx context.Context, n int) ([]Note, error) + // RecentNotesFromSource — the newest n notes whose source starts with + // prefix. Notes Maven read rather than heard (rss:, crawl:) are excluded + // from recall, so this is the only way to reach them, and it keeps the feed + // answer from being crowded out of a fixed window by his own notes. + RecentNotesFromSource(ctx context.Context, prefix string, n int) ([]Note, error) // ProposeTool drafts an inert 'proposed' tool scaffold (maven-callable); // returns whether a new proposal was written. EnableTool fills cmd + diff --git a/internal/ipc/client.go b/internal/ipc/client.go index 1009e58..e5c2617 100644 --- a/internal/ipc/client.go +++ b/internal/ipc/client.go @@ -66,6 +66,7 @@ var readOnlyMethods = map[Method]bool{ MethodRecentNudges: true, MethodQueryNotes: true, MethodRecentNotes: true, + MethodRecentNotesFromSource: true, MethodLookupTool: true, MethodListTools: true, MethodListProposedRoutines: true, @@ -375,6 +376,14 @@ func (c *Client) RecentNotes(ctx context.Context, n int) ([]Note, error) { return out, nil } +func (c *Client) RecentNotesFromSource(ctx context.Context, prefix string, n int) ([]Note, error) { + var out []Note + if err := c.call(ctx, MethodRecentNotesFromSource, sourceNReq{Prefix: prefix, N: n}, &out); err != nil { + return nil, err + } + return out, nil +} + func (c *Client) ProposeTool(ctx context.Context, name, utterance, scope string, ts time.Time) (bool, error) { var r proposeToolResp if err := c.call(ctx, MethodProposeTool, proposeToolReq{Name: name, Scope: scope, Utterance: utterance, Ts: ts}, &r); err != nil { diff --git a/internal/ipc/server.go b/internal/ipc/server.go index 772012d..42e34d9 100644 --- a/internal/ipc/server.go +++ b/internal/ipc/server.go @@ -161,6 +161,18 @@ func (a *storeAPI) QueryNotes(ctx context.Context, embedding []float32, k int) ( return out, nil } +func (a *storeAPI) RecentNotesFromSource(ctx context.Context, prefix string, n int) ([]Note, error) { + ns, err := a.s.RecentNotesFromSource(ctx, prefix, n) + if err != nil { + return nil, mapErr(err) + } + out := make([]Note, len(ns)) + for i, note := range ns { + out[i] = toNote(note) + } + return out, nil +} + func (a *storeAPI) RecentNotes(ctx context.Context, n int) ([]Note, error) { ns, err := a.s.RecentNotes(ctx, n) if err != nil { @@ -803,6 +815,16 @@ var methodTable = map[Method]handlerFunc{ } return out, nil }), + MethodRecentNotesFromSource: withParams(func(ctx context.Context, api CoreAPI, p sourceNReq) ([]Note, error) { + out, err := api.RecentNotesFromSource(ctx, p.Prefix, p.N) + if err != nil { + return nil, err + } + if out == nil { + out = []Note{} + } + return out, nil + }), MethodRecentNotes: withParams(func(ctx context.Context, api CoreAPI, p nReq) ([]Note, error) { out, err := api.RecentNotes(ctx, p.N) if err != nil { diff --git a/internal/ipc/unimplemented.go b/internal/ipc/unimplemented.go index 20e1b9f..b29d7ec 100644 --- a/internal/ipc/unimplemented.go +++ b/internal/ipc/unimplemented.go @@ -74,6 +74,10 @@ func (UnimplementedCoreAPI) WriteNote(ctx context.Context, ts time.Time, text st func (UnimplementedCoreAPI) QueryNotes(ctx context.Context, embedding []float32, k int) ([]Note, error) { return nil, ErrNotImplemented } +func (UnimplementedCoreAPI) RecentNotesFromSource(ctx context.Context, prefix string, n int) ([]Note, error) { + return nil, ErrUnknownMethod +} + func (UnimplementedCoreAPI) RecentNotes(ctx context.Context, n int) ([]Note, error) { return nil, ErrNotImplemented } diff --git a/internal/ipc/wire.go b/internal/ipc/wire.go index d9eea80..e63dfb0 100644 --- a/internal/ipc/wire.go +++ b/internal/ipc/wire.go @@ -30,6 +30,7 @@ const ( MethodWriteNote Method = "write_note" MethodQueryNotes Method = "query_notes" MethodRecentNotes Method = "recent_notes" + MethodRecentNotesFromSource Method = "recent_notes_source" MethodProposeTool Method = "propose_tool" MethodEnableTool Method = "enable_tool" MethodDisableTool Method = "disable_tool" diff --git a/internal/store/notes.go b/internal/store/notes.go index b2b4179..114b82e 100644 --- a/internal/store/notes.go +++ b/internal/store/notes.go @@ -6,6 +6,7 @@ import ( "fmt" "math" "sort" + "strings" "time" ) @@ -42,7 +43,8 @@ func (s *Store) WriteNote(ctx context.Context, ts time.Time, text string, embedd // or an ANN index only when note count or latency actually bites — at personal // scale (hundreds–thousands) a full scan is sub-millisecond. func (s *Store) QueryNotes(ctx context.Context, embedding []float32, k int) ([]Note, error) { - rows, err := s.db.QueryContext(ctx, `SELECT id, ts, text, embedding, source FROM notes`) + rows, err := s.db.QueryContext(ctx, + `SELECT id, ts, text, embedding, source FROM notes WHERE `+notHisWordsSQL) if err != nil { return nil, fmt.Errorf("query notes: %w", err) } @@ -76,6 +78,61 @@ func (s *Store) QueryNotes(ctx context.Context, embedding []float32, k int) ([]N return out, nil } +// ReadSourcePrefixes — note sources that are text Maven READ somewhere, not +// text he said or wrote: RSS items (internal/rss) and crawled pages +// (internal/crawl). +// +// They are notes because that is where fetched text lands, and they are kept out +// of recall because recall answers questions about HIM. "что я говорил про +// переезд" must not be answered out of a stranger's web page that happened to +// land near in the vector space, and the fallback line for that is "вот что я +// нашла: " followed by the stranger's words. Ask for them by source instead — +// that is what RecentNotesFromSource is for. +var ReadSourcePrefixes = []string{"rss:", "crawl:"} + +// notHisWordsSQL — the WHERE clause that drops the read sources. Built from the +// list above so adding a source is one line. +var notHisWordsSQL = buildNotHisWordsSQL() + +func buildNotHisWordsSQL() string { + var b strings.Builder + b.WriteString("(source IS NULL OR (") + for i, p := range ReadSourcePrefixes { + if i > 0 { + b.WriteString(" AND ") + } + fmt.Fprintf(&b, "source NOT LIKE '%s%%'", p) + } + b.WriteString("))") + return b.String() +} + +// RecentNotesFromSource returns the newest n notes whose source starts with +// prefix, newest first. The answer path for feeds and watches uses it: scanning +// the last 200 notes of ANY source meant a busy day of voice notes pushed the +// newest headline out of the window, and she said "в лентах пока ничего нового" +// while the poller was working fine. +func (s *Store) RecentNotesFromSource(ctx context.Context, prefix string, n int) ([]Note, error) { + rows, err := s.db.QueryContext(ctx, + `SELECT id, ts, text, source FROM notes WHERE source LIKE ? ORDER BY ts DESC LIMIT ?`, + prefix+"%", n) + if err != nil { + return nil, fmt.Errorf("recent notes by source: %w", err) + } + defer rows.Close() + var out []Note + for rows.Next() { + var nt Note + var tsMilli int64 + if err := rows.Scan(&nt.ID, &tsMilli, &nt.Text, &nt.Source); err != nil { + return nil, err + } + nt.Ts = time.UnixMilli(tsMilli).UTC() + out = append(out, nt) + } + return out, rows.Err() +} + // RecentNotes returns the newest n notes, newest first — a browse view (no // embedding math; Score stays 0). This is the read surface for /dash: notes // captured by voice are otherwise only reachable through semantic query. diff --git a/internal/store/notes_test.go b/internal/store/notes_test.go index 068eb40..df7efa8 100644 --- a/internal/store/notes_test.go +++ b/internal/store/notes_test.go @@ -36,3 +36,43 @@ func TestQueryNotesRanksByCosine(t *testing.T) { t.Errorf("scores not descending: %.3f then %.3f", got[0].Score, got[1].Score) } } + +// Recall answers questions about HIM. A feed item and a crawled page are text +// Maven read somewhere, and letting them into the nearest-neighbour pool means +// "что я говорил про переезд" can be answered with a stranger's sentence, under +// the line "вот что я нашла: ". +func TestQueryNotesLeavesOutWhatSheOnlyRead(t *testing.T) { + ctx := context.Background() + st := newTestStore(t) + now := time.Now() + // The read sources sit exactly on the query vector; his own note is further + // away, so ranking alone would put them first. + for _, n := range []struct{ text, source string }{ + {"переезд в новую квартиру описан тут", "rss:habr"}, + {"страница про переезд", "crawl:changelog"}, + } { + if _, err := st.WriteNote(ctx, now, n.text, []float32{1, 0, 0}, n.source); err != nil { + t.Fatal(err) + } + } + if _, err := st.WriteNote(ctx, now, "переезд в субботу", []float32{0.8, 0.6, 0}, "tap:voice"); err != nil { + t.Fatal(err) + } + + got, err := st.QueryNotes(ctx, []float32{1, 0, 0}, 5) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].Source != "tap:voice" { + t.Fatalf("recall returned %+v; want only what he said himself", got) + } + + // They are still reachable, by source. + feed, err := st.RecentNotesFromSource(ctx, "rss:", 10) + if err != nil { + t.Fatal(err) + } + if len(feed) != 1 || feed[0].Source != "rss:habr" { + t.Fatalf("RecentNotesFromSource(rss:) = %+v; want the one feed note", feed) + } +}