Maven/internal/crawl/watch.go

package crawl

import (
	"context"
	"fmt"
	"log"
	"strings"
	"time"
)

// Scheduled crawls: a page is re-read on an interval, and when its TEXT changed
// the new text is written as a note. Nothing is dispatched — same rule as the
// feed poller (Vikunja #258). A page that announced its own change would be a
// nag, and "the docs page changed" is not worth interrupting anyone for.
//
// Dedup is by content hash, so a page that re-renders identically writes nothing
// and a rotating ad slot does not count as news.

// WatchConfig — one page to keep an eye on.
type WatchConfig struct {
	Name     string        // note source is "crawl:<Name>"
	URL      string        // http(s), guarded by the fetcher
	Interval time.Duration // 0 ⇒ Watcher's default
}

// Notes is core's note-writing half (same shape as ipc.CoreAPI's method).
type Notes interface {
	WriteNote(ctx context.Context, ts time.Time, text string, embedding []float32, source string) (int64, error)
}

// Hashes remembers the last text hash per watch, durably, so a restart does not
// re-note an unchanged page. The daemon backs this with config facts
// ("crawl:hash:<name>").
type Hashes interface {
	LastHash(ctx context.Context, name string) (string, error)
	SetHash(ctx context.Context, name, hash string) error
}

// Embedder embeds a note on its way into the store. nil ⇒ no vector.
type Embedder interface {
	Embed(ctx context.Context, text string) ([]float32, error)
}

// DefaultWatchInterval — pages change slowly, and every check is a request in
// someone's log.
const DefaultWatchInterval = 6 * time.Hour

// Watcher re-reads watched pages on their interval.
type Watcher struct {
	c        *Crawler
	watches  []WatchConfig
	notes    Notes
	hashes   Hashes
	embed    Embedder
	interval time.Duration
	nextDue  map[string]time.Time
}

// NewWatcher wires the scheduled half, or returns nil when there is nothing to
// watch. Callers check for nil: no watches, no goroutine, no request.
func NewWatcher(c *Crawler, watches []WatchConfig, notes Notes, hashes Hashes, embed Embedder, defaultInterval time.Duration) *Watcher {
	if c == nil || notes == nil {
		return nil
	}
	var valid []WatchConfig
	for _, w := range watches {
		if strings.TrimSpace(w.Name) == "" || strings.TrimSpace(w.URL) == "" {
			log.Printf("crawl: skipping a watch with no name or no url")
			continue
		}
		valid = append(valid, w)
	}
	if len(valid) == 0 {
		return nil
	}
	if defaultInterval <= 0 {
		defaultInterval = DefaultWatchInterval
	}
	return &Watcher{
		c: c, watches: valid, notes: notes, hashes: hashes, embed: embed,
		interval: defaultInterval, nextDue: map[string]time.Time{},
	}
}

// Watches returns the configured watches.
func (w *Watcher) Watches() []WatchConfig { return w.watches }

// CheckDue re-reads every watch whose interval elapsed and returns how many
// notes were written. Errors are logged per watch, never returned: one dead page
// must not stop the others.
func (w *Watcher) CheckDue(ctx context.Context, now time.Time) int {
	written := 0
	for _, watch := range w.watches {
		if due, ok := w.nextDue[watch.Name]; ok && now.Before(due) {
			continue
		}
		interval := watch.Interval
		if interval <= 0 {
			interval = w.interval
		}
		w.nextDue[watch.Name] = now.Add(interval)
		changed, err := w.Check(ctx, watch, now)
		if err != nil {
			log.Printf("crawl: watch %s: %v", watch.Name, err)
			continue
		}
		if changed {
			log.Printf("crawl: watch %s: page changed, noted", watch.Name)
			written++
		}
	}
	return written
}

// Check re-reads one watch now and reports whether it wrote a note.
func (w *Watcher) Check(ctx context.Context, watch WatchConfig, now time.Time) (bool, error) {
	page, err := w.c.Page(ctx, watch.URL)
	if err != nil {
		return false, err
	}
	// Title included: a page whose headline changed has changed.
	h := Hash(page.Title + "\n" + page.Text)
	if w.hashes != nil {
		prev, err := w.hashes.LastHash(ctx, watch.Name)
		if err != nil {
			log.Printf("crawl: watch %s: read hash: %v", watch.Name, err)
		}
		if prev == h {
			return false, nil
		}
	}
	text := NoteText(watch, page)
	var vec []float32
	if w.embed != nil {
		v, err := w.embed.Embed(ctx, text)
		if err != nil {
			log.Printf("crawl: watch %s: embed: %v", watch.Name, err)
		} else {
			vec = v
		}
	}
	if _, err := w.notes.WriteNote(ctx, now, text, vec, SourceFor(watch.Name)); err != nil {
		return false, fmt.Errorf("write note: %w", err)
	}
	if w.hashes != nil {
		if err := w.hashes.SetHash(ctx, watch.Name, h); err != nil {
			log.Printf("crawl: watch %s: save hash: %v", watch.Name, err)
		}
	}
	return true, nil
}

// SourceFor is the note source for a watch, and SourcePrefix is what the answer
// path matches to recognise one.
func SourceFor(name string) string { return SourcePrefix + name }

// SourcePrefix — provenance for anything read off the network on a schedule.
const SourcePrefix = "crawl:"

// noteRunes — how much of a watched page goes into a note. Shorter than what the
// on-demand path reads: a note is a record of a change, not an archive.
const noteRunes = 800

// NoteText renders a watched page as a note body.
func NoteText(watch WatchConfig, page Page) string {
	var b strings.Builder
	if page.Title != "" {
		b.WriteString(page.Title)
	} else {
		b.WriteString(watch.Name)
	}
	b.WriteString("\n")
	b.WriteString(TrimRunes(page.Text, noteRunes))
	b.WriteString("\n")
	b.WriteString(watch.URL)
	return b.String()
}