package crawl import ( "context" "fmt" "log" "strings" "time" ) // Scheduled crawls: a page is re-read on an interval, and when its TEXT changed // the new text is written as a note. Nothing is dispatched — same rule as the // feed poller (Vikunja #258). A page that announced its own change would be a // nag, and "the docs page changed" is not worth interrupting anyone for. // // Dedup is by content hash, so a page that re-renders identically writes nothing // and a rotating ad slot does not count as news. // WatchConfig — one page to keep an eye on. type WatchConfig struct { Name string // note source is "crawl:" URL string // http(s), guarded by the fetcher Interval time.Duration // 0 ⇒ Watcher's default } // Notes is core's note-writing half (same shape as ipc.CoreAPI's method). type Notes interface { WriteNote(ctx context.Context, ts time.Time, text string, embedding []float32, source string) (int64, error) } // Hashes remembers the last text hash per watch, durably, so a restart does not // re-note an unchanged page. The daemon backs this with config facts // ("crawl:hash:"). type Hashes interface { LastHash(ctx context.Context, name string) (string, error) SetHash(ctx context.Context, name, hash string) error } // Embedder embeds a note on its way into the store. nil ⇒ no vector. type Embedder interface { Embed(ctx context.Context, text string) ([]float32, error) } // DefaultWatchInterval — pages change slowly, and every check is a request in // someone's log. const DefaultWatchInterval = 6 * time.Hour // Watcher re-reads watched pages on their interval. type Watcher struct { c *Crawler watches []WatchConfig notes Notes hashes Hashes embed Embedder interval time.Duration nextDue map[string]time.Time } // NewWatcher wires the scheduled half, or returns nil when there is nothing to // watch. Callers check for nil: no watches, no goroutine, no request. func NewWatcher(c *Crawler, watches []WatchConfig, notes Notes, hashes Hashes, embed Embedder, defaultInterval time.Duration) *Watcher { if c == nil || notes == nil { return nil } var valid []WatchConfig for _, w := range watches { if strings.TrimSpace(w.Name) == "" || strings.TrimSpace(w.URL) == "" { log.Printf("crawl: skipping a watch with no name or no url") continue } valid = append(valid, w) } if len(valid) == 0 { return nil } if defaultInterval <= 0 { defaultInterval = DefaultWatchInterval } return &Watcher{ c: c, watches: valid, notes: notes, hashes: hashes, embed: embed, interval: defaultInterval, nextDue: map[string]time.Time{}, } } // Watches returns the configured watches. func (w *Watcher) Watches() []WatchConfig { return w.watches } // CheckDue re-reads every watch whose interval elapsed and returns how many // notes were written. Errors are logged per watch, never returned: one dead page // must not stop the others. func (w *Watcher) CheckDue(ctx context.Context, now time.Time) int { written := 0 for _, watch := range w.watches { if due, ok := w.nextDue[watch.Name]; ok && now.Before(due) { continue } interval := watch.Interval if interval <= 0 { interval = w.interval } w.nextDue[watch.Name] = now.Add(interval) changed, err := w.Check(ctx, watch, now) if err != nil { log.Printf("crawl: watch %s: %v", watch.Name, err) continue } if changed { log.Printf("crawl: watch %s: page changed, noted", watch.Name) written++ } } return written } // Check re-reads one watch now and reports whether it wrote a note. func (w *Watcher) Check(ctx context.Context, watch WatchConfig, now time.Time) (bool, error) { page, err := w.c.Page(ctx, watch.URL) if err != nil { return false, err } // Title included: a page whose headline changed has changed. h := Hash(page.Title + "\n" + page.Text) if w.hashes != nil { prev, err := w.hashes.LastHash(ctx, watch.Name) if err != nil { log.Printf("crawl: watch %s: read hash: %v", watch.Name, err) } if prev == h { return false, nil } } text := NoteText(watch, page) var vec []float32 if w.embed != nil { v, err := w.embed.Embed(ctx, text) if err != nil { log.Printf("crawl: watch %s: embed: %v", watch.Name, err) } else { vec = v } } if _, err := w.notes.WriteNote(ctx, now, text, vec, SourceFor(watch.Name)); err != nil { return false, fmt.Errorf("write note: %w", err) } if w.hashes != nil { if err := w.hashes.SetHash(ctx, watch.Name, h); err != nil { log.Printf("crawl: watch %s: save hash: %v", watch.Name, err) } } return true, nil } // SourceFor is the note source for a watch, and SourcePrefix is what the answer // path matches to recognise one. func SourceFor(name string) string { return SourcePrefix + name } // SourcePrefix — provenance for anything read off the network on a schedule. const SourcePrefix = "crawl:" // noteRunes — how much of a watched page goes into a note. Shorter than what the // on-demand path reads: a note is a record of a change, not an archive. const noteRunes = 800 // NoteText renders a watched page as a note body. func NoteText(watch WatchConfig, page Page) string { var b strings.Builder if page.Title != "" { b.WriteString(page.Title) } else { b.WriteString(watch.Name) } b.WriteString("\n") b.WriteString(TrimRunes(page.Text, noteRunes)) b.WriteString("\n") b.WriteString(watch.URL) return b.String() }