package rss import ( "context" "fmt" "strings" "testing" "time" ) // undatedFeed — a feed whose items carry no pubDate. Plenty of real ones do not. func undatedFeed(titles ...string) string { var b strings.Builder b.WriteString(`u`) for _, t := range titles { fmt.Fprintf(&b, `%shttps://example.org/%s%s`, t, t, t) } b.WriteString(``) return b.String() } // datedFeed — newest first, one hour apart, the standard shape. func datedFeed(base time.Time, n int) string { var b strings.Builder b.WriteString(`d`) for i := 0; i < n; i++ { ts := base.Add(-time.Duration(i) * time.Hour) fmt.Fprintf(&b, `item-%dhttps://example.org/%dg%d%s`, i, i, i, ts.Format(time.RFC1123Z)) } b.WriteString(``) return b.String() } func TestPoll_UndatedFeedIsNotReNotedAfterARestart(t *testing.T) { // The seen-IDs map dies with the process, so before the durable mark was // consulted every boot re-noted the whole front page, stamped now, on top of // the recent-notes window. A crash loop made that a flood. feed := FeedConfig{Name: "u", URL: "https://example.org/rss"} fetch := &fakeFetch{body: undatedFeed("a", "b", "c")} marks := newMarks() notes1 := &fakeNotes{} p1 := NewPoller([]FeedConfig{feed}, fetch, notes1, marks, nil, nil, Config{}) p1.PollDue(context.Background(), now) if len(notes1.notes) != 3 { t.Fatalf("first boot wrote %d notes, want 3", len(notes1.notes)) } if marks.m["u"].IsZero() { t.Fatal("an undated feed left no mark, so the next process cannot tell it has been read") } // Restart. Same process-lifetime dedup map, gone. notes2 := &fakeNotes{} p2 := NewPoller([]FeedConfig{feed}, fetch, notes2, marks, nil, nil, Config{}) p2.PollDue(context.Background(), now.Add(time.Hour)) if len(notes2.notes) != 0 { t.Fatalf("a restart re-noted %d undated items: %v", len(notes2.notes), notes2.notes) } // And the same process still notices something genuinely new. fetch.body = undatedFeed("a", "b", "c", "d") p2.PollDue(context.Background(), now.Add(2*time.Hour)) if len(notes2.notes) != 1 { t.Fatalf("a new undated item after the resync wrote %d notes, want 1", len(notes2.notes)) } } func TestPoll_BurstLargerThanMaxItemsIsPacedNotDropped(t *testing.T) { // max_items reads as pacing in the config doc. Marking the newest item // written put everything below the cap behind the mark, permanently. feed := FeedConfig{Name: "d", URL: "https://example.org/rss"} fetch := &fakeFetch{body: datedFeed(now.Add(-time.Minute), 12)} notes := &fakeNotes{} marks := newMarks() p := NewPoller([]FeedConfig{feed}, fetch, notes, marks, nil, nil, Config{MaxItems: 5, MaxAge: 48 * time.Hour}) at := now for i := 0; i < 3; i++ { if _, err := p.PollFeed(context.Background(), feed, at); err != nil { t.Fatal(err) } at = at.Add(time.Hour) } seen := map[string]bool{} for _, n := range notes.notes { title := strings.SplitN(n.text, "\n", 2)[0] if seen[title] { t.Errorf("item %q was noted twice", title) } seen[title] = true } if len(seen) != 12 { t.Errorf("after three polls of a 12-item burst she has %d of them; the rest were dropped for good", len(seen)) } } func TestNoteHeadlineAndCategory(t *testing.T) { text := NoteText(FeedConfig{Category: "технологии"}, Item{ Title: "Новая уязвимость", Summary: "Патч вышел", Link: "https://example.org/a", }) if got := NoteHeadline(text); got != "Новая уязвимость" { t.Errorf("NoteHeadline = %q; she reads the brackets out loud", got) } if got := NoteCategory(text); got != "технологии" { t.Errorf("NoteCategory = %q, want технологии", got) } // A feed's own leading tag stays part of the title. if got := NoteHeadline("[перевод] Что-то"); got != "[перевод] Что-то" { t.Errorf("NoteHeadline stripped the feed's own tag: %q", got) } if got := NoteCategory("Без категории"); got != "" { t.Errorf("NoteCategory on an untagged note = %q, want empty", got) } }