Files
Maven/internal/store/reminders_delivery_test.go
T
claude 35c6ff5a71 Make delivery and integration failures explicit
Persist reminder presentations and retry state, atomically complete collapsed deliveries, fall back across away reaches, and block permanent failures visibly (V-715, V-678). Fail closed when enabled integrations lack credentials and keep remote arms explicitly dark (V-691). Give mavweb one sanitized, request-correlated error contract (V-689). Owner explicitly requested direct commits to master.
2026-08-13 02:50:59 +04:00

418 lines
14 KiB
Go

package store
import (
"context"
"errors"
"path/filepath"
"sync"
"testing"
"time"
)
func TestReminderPhraseAndRetrySurviveRestart(t *testing.T) {
ctx := context.Background()
path := filepath.Join(t.TempDir(), "reminders.db")
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
s, err := Open(ctx, path)
if err != nil {
t.Fatal(err)
}
id, err := s.CreateReminder(ctx, now.Add(-time.Minute), `{"text":"позвонить маме"}`, "")
if err != nil {
t.Fatal(err)
}
due, err := s.DueReminders(ctx, now)
if err != nil || len(due) != 1 {
t.Fatalf("initial due reminders = %d, err=%v", len(due), err)
}
if err := s.CacheReminderPhrase(ctx, due, "reminder:one", "Позвони маме.", "Позвони маме", "warm"); err != nil {
t.Fatalf("cache phrase: %v", err)
}
if err := s.ScheduleReminderRetry(ctx, due, now); err != nil {
t.Fatalf("schedule retry: %v", err)
}
if err := s.Close(); err != nil {
t.Fatal(err)
}
s, err = Open(ctx, path)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = s.Close() })
if got, err := s.DueReminders(ctx, now.Add(30*time.Second)); err != nil || len(got) != 0 {
t.Fatalf("during retry wait: got %d due, err=%v", len(got), err)
}
due, err = s.DueReminders(ctx, now.Add(ReminderRetryBase))
if err != nil || len(due) != 1 {
t.Fatalf("at retry time: got %d due, err=%v", len(due), err)
}
r := due[0]
if r.ID != id || r.DeliveryGroup != "reminder:one" || r.PhraseBody != "Позвони маме." ||
r.PhraseSummary != "Позвони маме" || r.PhraseMood != "warm" {
t.Fatalf("persisted delivery presentation = %+v", r)
}
if r.DeliveryAttempts != 1 {
t.Fatalf("attempts = %d, want 1", r.DeliveryAttempts)
}
if want := now.Add(ReminderRetryBase); !r.NextAttemptTs.Equal(want) {
t.Fatalf("next attempt = %s, want %s", r.NextAttemptTs, want)
}
}
func TestReminderRetryDelayIsExponentialAndBounded(t *testing.T) {
wants := []time.Duration{
time.Minute, 2 * time.Minute, 4 * time.Minute, 8 * time.Minute,
16 * time.Minute, 32 * time.Minute, time.Hour, time.Hour, time.Hour,
}
for i, want := range wants {
if got := ReminderRetryDelay(i + 1); got != want {
t.Errorf("attempt %d delay = %s, want %s", i+1, got, want)
}
}
}
func TestCollapsedReminderRetryStateMovesTogether(t *testing.T) {
s := newTestStore(t)
ctx := context.Background()
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
for _, text := range []string{"полить цветы", "позвонить врачу"} {
if _, err := s.CreateReminder(ctx, now.Add(-time.Minute), text, ""); err != nil {
t.Fatal(err)
}
}
originals, err := s.DueReminders(ctx, now)
if err != nil || len(originals) != 2 {
t.Fatalf("due originals = %d, err=%v", len(originals), err)
}
if err := s.CacheReminderPhrase(ctx, originals, "reminder:bundle", "У тебя два напоминания.", "Два напоминания", "neutral"); err != nil {
t.Fatal(err)
}
if err := s.ScheduleReminderRetry(ctx, originals, now); err != nil {
t.Fatal(err)
}
rows, err := s.ListReminders(ctx, 10)
if err != nil || len(rows) != 2 {
t.Fatalf("list = %d, err=%v", len(rows), err)
}
for _, r := range rows {
if r.DeliveryGroup != "reminder:bundle" || r.PhraseBody != "У тебя два напоминания." || r.DeliveryAttempts != 1 {
t.Errorf("bundle original did not move with group: %+v", r)
}
if !r.NextAttemptTs.Equal(now.Add(time.Minute)) {
t.Errorf("next attempt = %s, want %s", r.NextAttemptTs, now.Add(time.Minute))
}
}
}
func TestRecurringReminderClearsDeliveryStateForNextOccurrence(t *testing.T) {
s := newTestStore(t)
ctx := context.Background()
loc := time.FixedZone("MSK", 3*60*60)
fire := time.Date(2026, 8, 13, 9, 0, 0, 0, loc)
id, err := s.CreateReminder(ctx, fire, `{"text":"стендап"}`, "0 9 * * *")
if err != nil {
t.Fatal(err)
}
due, err := s.DueReminders(ctx, fire)
if err != nil || len(due) != 1 {
t.Fatalf("due = %d, err=%v", len(due), err)
}
if err := s.CacheReminderPhrase(ctx, due, "reminder:occurrence", "Пора на стендап.", "Стендап", "neutral"); err != nil {
t.Fatal(err)
}
if err := s.ScheduleReminderRetry(ctx, due, fire); err != nil {
t.Fatal(err)
}
if err := s.rescheduleReminderIn(ctx, id, fire.Add(time.Minute), loc); err != nil {
t.Fatalf("reschedule: %v", err)
}
rows, err := s.ListReminders(ctx, 1)
if err != nil || len(rows) != 1 {
t.Fatalf("list = %d, err=%v", len(rows), err)
}
r := rows[0]
if r.Status != ReminderPending || r.DeliveryGroup != "" || r.PhraseBody != "" ||
r.PhraseSummary != "" || r.PhraseMood != "" || r.DeliveryAttempts != 0 || !r.NextAttemptTs.IsZero() {
t.Fatalf("next occurrence retained old delivery state: %+v", r)
}
want := time.Date(2026, 8, 14, 9, 0, 0, 0, loc)
if !r.NextFireTs.Equal(want) {
t.Fatalf("next fire = %s, want %s", r.NextFireTs.In(loc), want)
}
}
func TestReminderTerminalTransitionHasExactlyOneWinner(t *testing.T) {
s := newTestStore(t)
ctx := context.Background()
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
for iteration := 0; iteration < 40; iteration++ {
id, err := s.CreateReminder(ctx, now, "race", "")
if err != nil {
t.Fatal(err)
}
start := make(chan struct{})
var wg sync.WaitGroup
errs := make([]error, 2)
statuses := []string{ReminderFired, ReminderCancelled}
for i := range statuses {
wg.Add(1)
go func(i int) {
defer wg.Done()
<-start
errs[i] = s.MarkReminder(ctx, id, statuses[i])
}(i)
}
close(start)
wg.Wait()
successes := 0
losers := 0
winner := ""
for i, err := range errs {
switch {
case err == nil:
successes++
winner = statuses[i]
case errors.Is(err, ErrReminderState):
losers++
default:
t.Fatalf("iteration %d transition %s: %v", iteration, statuses[i], err)
}
}
if successes != 1 || losers != 1 {
t.Fatalf("iteration %d: successes=%d losers=%d errors=%v", iteration, successes, losers, errs)
}
var stored string
if err := s.db.QueryRowContext(ctx, `SELECT status FROM reminders WHERE id = ?`, id).Scan(&stored); err != nil {
t.Fatal(err)
}
if stored != winner {
t.Fatalf("iteration %d: stored %q, successful transition %q", iteration, stored, winner)
}
}
}
func TestAmbiguousCollapsedDeliveryIsNotAutomaticallyRepeated(t *testing.T) {
s := newTestStore(t)
ctx := context.Background()
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
for _, text := range []string{"полить цветы", "позвонить врачу"} {
if _, err := s.CreateReminder(ctx, now.Add(-time.Minute), text, ""); err != nil {
t.Fatal(err)
}
}
originals, err := s.DueReminders(ctx, now)
if err != nil || len(originals) != 2 {
t.Fatalf("due originals = %d, err=%v", len(originals), err)
}
const group = "reminder:ambiguous-bundle"
if err := s.CacheReminderPhrase(ctx, originals, group, "У тебя два напоминания.", "Два напоминания", "neutral"); err != nil {
t.Fatal(err)
}
for i := range originals {
originals[i].DeliveryGroup = group
}
// Simulate a process dying after Begin and before it can record whether the
// external sink accepted the bundle. Synthetic reminder zero is never put
// in the outbox: a real original is the human-readable representative and
// the group is the occurrence identity shared by both originals.
attemptID, err := s.BeginDeliveryAttempt(ctx, "reminder", "", originals[0].ID, group, "telegram", "hash", now)
if err != nil {
t.Fatal(err)
}
if _, err := s.ReconcileStaleDeliveryAttempts(ctx, now.Add(time.Minute)); err != nil {
t.Fatal(err)
}
if due, err := s.DueReminders(ctx, now.Add(24*time.Hour)); err != nil || len(due) != 0 {
t.Fatalf("ambiguous delivered bundle was eligible for automatic repeat: due=%+v err=%v", due, err)
}
attempts, err := s.ListDeliveryAttempts(ctx, "", 10)
if err != nil || len(attempts) != 1 {
t.Fatalf("attempts=%+v err=%v", attempts, err)
}
if attempts[0].ID != attemptID || attempts[0].Status != DeliveryUnknown || attempts[0].DeliveryGroup != group {
t.Fatalf("ambiguous occurrence lost its durable identity: %+v", attempts[0])
}
if err := s.ScheduleReminderRetry(ctx, originals, now.Add(time.Minute)); err != nil {
t.Fatal(err)
}
rows, err := s.ListReminders(ctx, 10)
if err != nil || len(rows) != 2 {
t.Fatalf("rows=%+v err=%v", rows, err)
}
for _, r := range rows {
if r.DeliveryAttempts != 0 || !r.NextAttemptTs.IsZero() {
t.Fatalf("ambiguous occurrence was scheduled for automatic retry: %+v", r)
}
}
}
func TestDefiniteFailedDeliveryRemainsRetryEligible(t *testing.T) {
s := newTestStore(t)
ctx := context.Background()
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
if _, err := s.CreateReminder(ctx, now.Add(-time.Minute), "позвонить врачу", ""); err != nil {
t.Fatal(err)
}
originals, err := s.DueReminders(ctx, now)
if err != nil || len(originals) != 1 {
t.Fatalf("due originals = %d, err=%v", len(originals), err)
}
const group = "reminder:definite-failure"
if err := s.CacheReminderPhrase(ctx, originals, group, "Позвони врачу.", "Позвони врачу", "neutral"); err != nil {
t.Fatal(err)
}
id, err := s.BeginDeliveryAttempt(ctx, "reminder", "", originals[0].ID, group, "telegram", "hash", now)
if err != nil {
t.Fatal(err)
}
if err := s.CompleteDeliveryAttempt(ctx, id, DeliveryFailed, now); err != nil {
t.Fatal(err)
}
if due, err := s.DueReminders(ctx, now); err != nil || len(due) != 1 {
t.Fatalf("definite failure was not retry eligible: due=%+v err=%v", due, err)
}
}
func TestCollapsedCompletionRollsBackAsOneTransaction(t *testing.T) {
s := newTestStore(t)
ctx := context.Background()
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
for _, text := range []string{"first", "fail-second"} {
if _, err := s.CreateReminder(ctx, now.Add(-time.Minute), text, ""); err != nil {
t.Fatal(err)
}
}
originals, err := s.DueReminders(ctx, now)
if err != nil || len(originals) != 2 {
t.Fatalf("due originals = %d, err=%v", len(originals), err)
}
if _, err := s.db.ExecContext(ctx, `
CREATE TRIGGER fail_second_reminder_completion
BEFORE UPDATE OF status ON reminders
WHEN OLD.payload = 'fail-second'
BEGIN
SELECT RAISE(ABORT, 'injected second completion failure');
END`); err != nil {
t.Fatal(err)
}
if err := s.CompleteReminderDelivery(ctx, originals, now); err == nil {
t.Fatal("injected second-row failure did not fail the bundle completion")
}
rows, err := s.ListReminders(ctx, 10)
if err != nil || len(rows) != 2 {
t.Fatalf("rows=%+v err=%v", rows, err)
}
for _, r := range rows {
if r.Status != ReminderPending {
t.Fatalf("bundle completion partially committed reminder %d as %q", r.ID, r.Status)
}
}
}
func TestSuccessfulAttemptAndCollapsedCompletionCommitAtomically(t *testing.T) {
s := newTestStore(t)
ctx := context.Background()
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
for _, text := range []string{"first", "fail-second"} {
if _, err := s.CreateReminder(ctx, now.Add(-time.Minute), text, ""); err != nil {
t.Fatal(err)
}
}
originals, err := s.DueReminders(ctx, now)
if err != nil || len(originals) != 2 {
t.Fatalf("due originals = %d, err=%v", len(originals), err)
}
const group = "reminder:atomic-success"
if err := s.CacheReminderPhrase(ctx, originals, group, "Two reminders.", "Two reminders", "neutral"); err != nil {
t.Fatal(err)
}
for i := range originals {
originals[i].DeliveryGroup = group
}
attemptID, err := s.BeginDeliveryAttempt(ctx, "reminder", "", originals[0].ID, group, "telegram", "hash", now)
if err != nil {
t.Fatal(err)
}
if _, err := s.db.ExecContext(ctx, `
CREATE TRIGGER fail_atomic_second_completion
BEFORE UPDATE OF status ON reminders
WHEN OLD.payload = 'fail-second'
BEGIN
SELECT RAISE(ABORT, 'injected second completion failure');
END`); err != nil {
t.Fatal(err)
}
if err := s.CompleteSuccessfulReminderAttempt(ctx, attemptID, originals, now); err == nil {
t.Fatal("injected reminder completion failure did not fail the local commit")
}
attempts, err := s.ListDeliveryAttempts(ctx, "", 10)
if err != nil || len(attempts) != 1 || attempts[0].Status != DeliveryPending {
t.Fatalf("successful outbox marker committed without reminder state: attempts=%+v err=%v", attempts, err)
}
rows, err := s.ListReminders(ctx, 10)
if err != nil || len(rows) != 2 {
t.Fatalf("rows=%+v err=%v", rows, err)
}
for _, r := range rows {
if r.Status != ReminderPending {
t.Fatalf("atomic rollback partially completed reminder %d as %q", r.ID, r.Status)
}
}
if _, err := s.ReconcileStaleDeliveryAttempts(ctx, now.Add(time.Minute)); err != nil {
t.Fatal(err)
}
if due, err := s.DueReminders(ctx, now.Add(24*time.Hour)); err != nil || len(due) != 0 {
t.Fatalf("restart auto-replayed ambiguous accepted send: due=%+v err=%v", due, err)
}
}
func TestPermanentReminderFailureBlocksUntilDeliberatelyUnblocked(t *testing.T) {
s := newTestStore(t)
ctx := context.Background()
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
id, err := s.CreateReminder(ctx, now.Add(-time.Minute), "позвонить врачу", "")
if err != nil {
t.Fatal(err)
}
originals, err := s.DueReminders(ctx, now)
if err != nil || len(originals) != 1 {
t.Fatalf("due originals=%+v err=%v", originals, err)
}
const group = "reminder:revoked-token"
if err := s.CacheReminderPhrase(ctx, originals, group, "Позвони врачу.", "Позвони врачу", "neutral"); err != nil {
t.Fatal(err)
}
originals[0].DeliveryGroup = group
if err := s.BlockReminderDelivery(ctx, originals, now, "ntfy credentials rejected"); err != nil {
t.Fatal(err)
}
if err := s.ScheduleReminderRetry(ctx, originals, now); err != nil {
t.Fatal(err)
}
if due, err := s.DueReminders(ctx, now.Add(24*time.Hour)); err != nil || len(due) != 0 {
t.Fatalf("blocked reminder retried automatically: due=%+v err=%v", due, err)
}
rows, err := s.ListReminders(ctx, 1)
if err != nil || len(rows) != 1 {
t.Fatalf("rows=%+v err=%v", rows, err)
}
if rows[0].DeliveryBlockedTs.IsZero() || rows[0].DeliveryBlockedError != "ntfy credentials rejected" ||
rows[0].DeliveryAttempts != 0 || !rows[0].NextAttemptTs.IsZero() {
t.Fatalf("blocked state is not visible/durable: %+v", rows[0])
}
if err := s.UnblockReminderDelivery(ctx, id); err != nil {
t.Fatal(err)
}
due, err := s.DueReminders(ctx, now)
if err != nil || len(due) != 1 || due[0].ID != id || !due[0].DeliveryBlockedTs.IsZero() || due[0].DeliveryBlockedError != "" {
t.Fatalf("deliberate unblock did not restore eligibility: due=%+v err=%v", due, err)
}
}