package crawl import ( "context" "strings" "testing" "time" ) type note struct { text string source string } type fakeNotes struct{ notes []note } func (n *fakeNotes) WriteNote(_ context.Context, _ time.Time, text string, _ []float32, source string) (int64, error) { n.notes = append(n.notes, note{text, source}) return int64(len(n.notes)), nil } type fakeHashes struct{ m map[string]string } func newHashes() *fakeHashes { return &fakeHashes{m: map[string]string{}} } func (f *fakeHashes) LastHash(_ context.Context, name string) (string, error) { return f.m[name], nil } func (f *fakeHashes) SetHash(_ context.Context, name, h string) error { f.m[name] = h; return nil } var t0 = time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC) func TestWatchNotesAChangedPage(t *testing.T) { f := &fakeFetcher{pages: map[string]Response{ "https://example.org/docs": {Body: []byte(htmlPage)}, }} notes := &fakeNotes{} hashes := newHashes() w := NewWatcher(newTestCrawler(f), []WatchConfig{{Name: "docs", URL: "https://example.org/docs"}}, notes, hashes, nil, time.Hour) if w == nil { t.Fatal("NewWatcher returned nil for a configured watch") } if n := w.CheckDue(context.Background(), t0); n != 1 { t.Fatalf("first check wrote %d notes, want 1", n) } if notes.notes[0].source != "crawl:docs" { t.Errorf("source = %q, want crawl:docs", notes.notes[0].source) } if !strings.Contains(notes.notes[0].text, "https://example.org/docs") { t.Errorf("note does not carry the url: %q", notes.notes[0].text) } // Unchanged page, interval elapsed: nothing written. if n := w.CheckDue(context.Background(), t0.Add(2*time.Hour)); n != 0 { t.Fatalf("an unchanged page wrote %d notes", n) } // Changed page: one note. f.pages["https://example.org/docs"] = Response{Body: []byte(strings.Replace(htmlPage, "синее", "серое", 1))} if n := w.CheckDue(context.Background(), t0.Add(4*time.Hour)); n != 1 { t.Fatalf("a changed page wrote %d notes, want 1", n) } } func TestWatchIntervalIsRespected(t *testing.T) { f := &fakeFetcher{pages: map[string]Response{"https://example.org/d": {Body: []byte(htmlPage)}}} w := NewWatcher(newTestCrawler(f), []WatchConfig{{Name: "d", URL: "https://example.org/d", Interval: time.Hour}}, &fakeNotes{}, newHashes(), nil, 0) w.CheckDue(context.Background(), t0) before := len(f.calls) w.CheckDue(context.Background(), t0.Add(time.Minute)) if len(f.calls) != before { t.Fatal("the page was re-read inside its interval") } } // The hash is durable so a restart does not re-note an unchanged page. func TestWatchHashSurvivesRestart(t *testing.T) { f := &fakeFetcher{pages: map[string]Response{"https://example.org/d": {Body: []byte(htmlPage)}}} hashes := newHashes() watches := []WatchConfig{{Name: "d", URL: "https://example.org/d"}} NewWatcher(newTestCrawler(f), watches, &fakeNotes{}, hashes, nil, time.Hour).CheckDue(context.Background(), t0) notes2 := &fakeNotes{} NewWatcher(newTestCrawler(f), watches, notes2, hashes, nil, time.Hour).CheckDue(context.Background(), t0.Add(time.Hour)) if len(notes2.notes) != 0 { t.Fatalf("a fresh watcher re-noted an unchanged page: %q", notes2.notes[0].text) } } func TestWatchDeadPageDoesNotStopTheOthers(t *testing.T) { f := &fakeFetcher{pages: map[string]Response{"https://example.org/live": {Body: []byte(htmlPage)}}} notes := &fakeNotes{} w := NewWatcher(newTestCrawler(f), []WatchConfig{ {Name: "dead", URL: "https://example.org/gone"}, {Name: "live", URL: "https://example.org/live"}, }, notes, newHashes(), nil, time.Hour) if n := w.CheckDue(context.Background(), t0); n != 1 { t.Fatalf("wrote %d notes, want 1 (the live page)", n) } if notes.notes[0].source != "crawl:live" { t.Fatalf("source = %q", notes.notes[0].source) } } func TestNoWatchesMeansNoWatcher(t *testing.T) { c := newTestCrawler(&fakeFetcher{}) if NewWatcher(c, nil, &fakeNotes{}, nil, nil, 0) != nil { t.Fatal("no watches must mean no watcher") } if NewWatcher(nil, []WatchConfig{{Name: "a", URL: "u"}}, &fakeNotes{}, nil, nil, 0) != nil { t.Fatal("no crawler must mean no watcher") } if NewWatcher(c, []WatchConfig{{Name: "", URL: ""}}, &fakeNotes{}, nil, nil, 0) != nil { t.Fatal("a watch with no name or url is not a configuration") } }