store: keep what she read out of what he said
Nothing at read time told a feed item or a crawled page apart from his own notes. QueryNotes ranked every note by cosine and the notes answer handed the nearest five to the phraser, so "что я говорил про переезд" could be answered out of a stranger's web page, prefixed with "вот что я нашла: ". Recall now excludes the read sources, rss: and crawl:, and the list is one place. The feed answer needed a different read as a result, and it needed one anyway: it scanned the last 200 notes of any source, so a busy day of voice notes pushed the newest headline out of the window and she said "в лентах пока ничего нового" while the poller was working fine. RecentNotesFromSource asks for feed notes by source, so the window holds 200 of them. Found in review of #66 and #67.
This commit is contained in:
+58
-1
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -42,7 +43,8 @@ func (s *Store) WriteNote(ctx context.Context, ts time.Time, text string, embedd
|
||||
// or an ANN index only when note count or latency actually bites — at personal
|
||||
// scale (hundreds–thousands) a full scan is sub-millisecond.
|
||||
func (s *Store) QueryNotes(ctx context.Context, embedding []float32, k int) ([]Note, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT id, ts, text, embedding, source FROM notes`)
|
||||
rows, err := s.db.QueryContext(ctx,
|
||||
`SELECT id, ts, text, embedding, source FROM notes WHERE `+notHisWordsSQL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query notes: %w", err)
|
||||
}
|
||||
@@ -76,6 +78,61 @@ func (s *Store) QueryNotes(ctx context.Context, embedding []float32, k int) ([]N
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ReadSourcePrefixes — note sources that are text Maven READ somewhere, not
|
||||
// text he said or wrote: RSS items (internal/rss) and crawled pages
|
||||
// (internal/crawl).
|
||||
//
|
||||
// They are notes because that is where fetched text lands, and they are kept out
|
||||
// of recall because recall answers questions about HIM. "что я говорил про
|
||||
// переезд" must not be answered out of a stranger's web page that happened to
|
||||
// land near in the vector space, and the fallback line for that is "вот что я
|
||||
// нашла: " followed by the stranger's words. Ask for them by source instead —
|
||||
// that is what RecentNotesFromSource is for.
|
||||
var ReadSourcePrefixes = []string{"rss:", "crawl:"}
|
||||
|
||||
// notHisWordsSQL — the WHERE clause that drops the read sources. Built from the
|
||||
// list above so adding a source is one line.
|
||||
var notHisWordsSQL = buildNotHisWordsSQL()
|
||||
|
||||
func buildNotHisWordsSQL() string {
|
||||
var b strings.Builder
|
||||
b.WriteString("(source IS NULL OR (")
|
||||
for i, p := range ReadSourcePrefixes {
|
||||
if i > 0 {
|
||||
b.WriteString(" AND ")
|
||||
}
|
||||
fmt.Fprintf(&b, "source NOT LIKE '%s%%'", p)
|
||||
}
|
||||
b.WriteString("))")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// RecentNotesFromSource returns the newest n notes whose source starts with
|
||||
// prefix, newest first. The answer path for feeds and watches uses it: scanning
|
||||
// the last 200 notes of ANY source meant a busy day of voice notes pushed the
|
||||
// newest headline out of the window, and she said "в лентах пока ничего нового"
|
||||
// while the poller was working fine.
|
||||
func (s *Store) RecentNotesFromSource(ctx context.Context, prefix string, n int) ([]Note, error) {
|
||||
rows, err := s.db.QueryContext(ctx,
|
||||
`SELECT id, ts, text, source FROM notes WHERE source LIKE ? ORDER BY ts DESC LIMIT ?`,
|
||||
prefix+"%", n)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("recent notes by source: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Note
|
||||
for rows.Next() {
|
||||
var nt Note
|
||||
var tsMilli int64
|
||||
if err := rows.Scan(&nt.ID, &tsMilli, &nt.Text, &nt.Source); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nt.Ts = time.UnixMilli(tsMilli).UTC()
|
||||
out = append(out, nt)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// RecentNotes returns the newest n notes, newest first — a browse view (no
|
||||
// embedding math; Score stays 0). This is the read surface for /dash: notes
|
||||
// captured by voice are otherwise only reachable through semantic query.
|
||||
|
||||
Reference in New Issue
Block a user