35c6ff5a71
Persist reminder presentations and retry state, atomically complete collapsed deliveries, fall back across away reaches, and block permanent failures visibly (V-715, V-678). Fail closed when enabled integrations lack credentials and keep remote arms explicitly dark (V-691). Give mavweb one sanitized, request-correlated error contract (V-689). Owner explicitly requested direct commits to master.
418 lines
14 KiB
Go
418 lines
14 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"path/filepath"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestReminderPhraseAndRetrySurviveRestart(t *testing.T) {
|
|
ctx := context.Background()
|
|
path := filepath.Join(t.TempDir(), "reminders.db")
|
|
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
|
|
|
|
s, err := Open(ctx, path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
id, err := s.CreateReminder(ctx, now.Add(-time.Minute), `{"text":"позвонить маме"}`, "")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
due, err := s.DueReminders(ctx, now)
|
|
if err != nil || len(due) != 1 {
|
|
t.Fatalf("initial due reminders = %d, err=%v", len(due), err)
|
|
}
|
|
if err := s.CacheReminderPhrase(ctx, due, "reminder:one", "Позвони маме.", "Позвони маме", "warm"); err != nil {
|
|
t.Fatalf("cache phrase: %v", err)
|
|
}
|
|
if err := s.ScheduleReminderRetry(ctx, due, now); err != nil {
|
|
t.Fatalf("schedule retry: %v", err)
|
|
}
|
|
if err := s.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
s, err = Open(ctx, path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = s.Close() })
|
|
if got, err := s.DueReminders(ctx, now.Add(30*time.Second)); err != nil || len(got) != 0 {
|
|
t.Fatalf("during retry wait: got %d due, err=%v", len(got), err)
|
|
}
|
|
due, err = s.DueReminders(ctx, now.Add(ReminderRetryBase))
|
|
if err != nil || len(due) != 1 {
|
|
t.Fatalf("at retry time: got %d due, err=%v", len(due), err)
|
|
}
|
|
r := due[0]
|
|
if r.ID != id || r.DeliveryGroup != "reminder:one" || r.PhraseBody != "Позвони маме." ||
|
|
r.PhraseSummary != "Позвони маме" || r.PhraseMood != "warm" {
|
|
t.Fatalf("persisted delivery presentation = %+v", r)
|
|
}
|
|
if r.DeliveryAttempts != 1 {
|
|
t.Fatalf("attempts = %d, want 1", r.DeliveryAttempts)
|
|
}
|
|
if want := now.Add(ReminderRetryBase); !r.NextAttemptTs.Equal(want) {
|
|
t.Fatalf("next attempt = %s, want %s", r.NextAttemptTs, want)
|
|
}
|
|
}
|
|
|
|
func TestReminderRetryDelayIsExponentialAndBounded(t *testing.T) {
|
|
wants := []time.Duration{
|
|
time.Minute, 2 * time.Minute, 4 * time.Minute, 8 * time.Minute,
|
|
16 * time.Minute, 32 * time.Minute, time.Hour, time.Hour, time.Hour,
|
|
}
|
|
for i, want := range wants {
|
|
if got := ReminderRetryDelay(i + 1); got != want {
|
|
t.Errorf("attempt %d delay = %s, want %s", i+1, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestCollapsedReminderRetryStateMovesTogether(t *testing.T) {
|
|
s := newTestStore(t)
|
|
ctx := context.Background()
|
|
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
|
|
for _, text := range []string{"полить цветы", "позвонить врачу"} {
|
|
if _, err := s.CreateReminder(ctx, now.Add(-time.Minute), text, ""); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
originals, err := s.DueReminders(ctx, now)
|
|
if err != nil || len(originals) != 2 {
|
|
t.Fatalf("due originals = %d, err=%v", len(originals), err)
|
|
}
|
|
if err := s.CacheReminderPhrase(ctx, originals, "reminder:bundle", "У тебя два напоминания.", "Два напоминания", "neutral"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.ScheduleReminderRetry(ctx, originals, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
rows, err := s.ListReminders(ctx, 10)
|
|
if err != nil || len(rows) != 2 {
|
|
t.Fatalf("list = %d, err=%v", len(rows), err)
|
|
}
|
|
for _, r := range rows {
|
|
if r.DeliveryGroup != "reminder:bundle" || r.PhraseBody != "У тебя два напоминания." || r.DeliveryAttempts != 1 {
|
|
t.Errorf("bundle original did not move with group: %+v", r)
|
|
}
|
|
if !r.NextAttemptTs.Equal(now.Add(time.Minute)) {
|
|
t.Errorf("next attempt = %s, want %s", r.NextAttemptTs, now.Add(time.Minute))
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRecurringReminderClearsDeliveryStateForNextOccurrence(t *testing.T) {
|
|
s := newTestStore(t)
|
|
ctx := context.Background()
|
|
loc := time.FixedZone("MSK", 3*60*60)
|
|
fire := time.Date(2026, 8, 13, 9, 0, 0, 0, loc)
|
|
id, err := s.CreateReminder(ctx, fire, `{"text":"стендап"}`, "0 9 * * *")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
due, err := s.DueReminders(ctx, fire)
|
|
if err != nil || len(due) != 1 {
|
|
t.Fatalf("due = %d, err=%v", len(due), err)
|
|
}
|
|
if err := s.CacheReminderPhrase(ctx, due, "reminder:occurrence", "Пора на стендап.", "Стендап", "neutral"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.ScheduleReminderRetry(ctx, due, fire); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.rescheduleReminderIn(ctx, id, fire.Add(time.Minute), loc); err != nil {
|
|
t.Fatalf("reschedule: %v", err)
|
|
}
|
|
|
|
rows, err := s.ListReminders(ctx, 1)
|
|
if err != nil || len(rows) != 1 {
|
|
t.Fatalf("list = %d, err=%v", len(rows), err)
|
|
}
|
|
r := rows[0]
|
|
if r.Status != ReminderPending || r.DeliveryGroup != "" || r.PhraseBody != "" ||
|
|
r.PhraseSummary != "" || r.PhraseMood != "" || r.DeliveryAttempts != 0 || !r.NextAttemptTs.IsZero() {
|
|
t.Fatalf("next occurrence retained old delivery state: %+v", r)
|
|
}
|
|
want := time.Date(2026, 8, 14, 9, 0, 0, 0, loc)
|
|
if !r.NextFireTs.Equal(want) {
|
|
t.Fatalf("next fire = %s, want %s", r.NextFireTs.In(loc), want)
|
|
}
|
|
}
|
|
|
|
func TestReminderTerminalTransitionHasExactlyOneWinner(t *testing.T) {
|
|
s := newTestStore(t)
|
|
ctx := context.Background()
|
|
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
|
|
|
|
for iteration := 0; iteration < 40; iteration++ {
|
|
id, err := s.CreateReminder(ctx, now, "race", "")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
start := make(chan struct{})
|
|
var wg sync.WaitGroup
|
|
errs := make([]error, 2)
|
|
statuses := []string{ReminderFired, ReminderCancelled}
|
|
for i := range statuses {
|
|
wg.Add(1)
|
|
go func(i int) {
|
|
defer wg.Done()
|
|
<-start
|
|
errs[i] = s.MarkReminder(ctx, id, statuses[i])
|
|
}(i)
|
|
}
|
|
close(start)
|
|
wg.Wait()
|
|
|
|
successes := 0
|
|
losers := 0
|
|
winner := ""
|
|
for i, err := range errs {
|
|
switch {
|
|
case err == nil:
|
|
successes++
|
|
winner = statuses[i]
|
|
case errors.Is(err, ErrReminderState):
|
|
losers++
|
|
default:
|
|
t.Fatalf("iteration %d transition %s: %v", iteration, statuses[i], err)
|
|
}
|
|
}
|
|
if successes != 1 || losers != 1 {
|
|
t.Fatalf("iteration %d: successes=%d losers=%d errors=%v", iteration, successes, losers, errs)
|
|
}
|
|
var stored string
|
|
if err := s.db.QueryRowContext(ctx, `SELECT status FROM reminders WHERE id = ?`, id).Scan(&stored); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if stored != winner {
|
|
t.Fatalf("iteration %d: stored %q, successful transition %q", iteration, stored, winner)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAmbiguousCollapsedDeliveryIsNotAutomaticallyRepeated(t *testing.T) {
|
|
s := newTestStore(t)
|
|
ctx := context.Background()
|
|
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
|
|
for _, text := range []string{"полить цветы", "позвонить врачу"} {
|
|
if _, err := s.CreateReminder(ctx, now.Add(-time.Minute), text, ""); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
originals, err := s.DueReminders(ctx, now)
|
|
if err != nil || len(originals) != 2 {
|
|
t.Fatalf("due originals = %d, err=%v", len(originals), err)
|
|
}
|
|
const group = "reminder:ambiguous-bundle"
|
|
if err := s.CacheReminderPhrase(ctx, originals, group, "У тебя два напоминания.", "Два напоминания", "neutral"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for i := range originals {
|
|
originals[i].DeliveryGroup = group
|
|
}
|
|
|
|
// Simulate a process dying after Begin and before it can record whether the
|
|
// external sink accepted the bundle. Synthetic reminder zero is never put
|
|
// in the outbox: a real original is the human-readable representative and
|
|
// the group is the occurrence identity shared by both originals.
|
|
attemptID, err := s.BeginDeliveryAttempt(ctx, "reminder", "", originals[0].ID, group, "telegram", "hash", now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := s.ReconcileStaleDeliveryAttempts(ctx, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if due, err := s.DueReminders(ctx, now.Add(24*time.Hour)); err != nil || len(due) != 0 {
|
|
t.Fatalf("ambiguous delivered bundle was eligible for automatic repeat: due=%+v err=%v", due, err)
|
|
}
|
|
attempts, err := s.ListDeliveryAttempts(ctx, "", 10)
|
|
if err != nil || len(attempts) != 1 {
|
|
t.Fatalf("attempts=%+v err=%v", attempts, err)
|
|
}
|
|
if attempts[0].ID != attemptID || attempts[0].Status != DeliveryUnknown || attempts[0].DeliveryGroup != group {
|
|
t.Fatalf("ambiguous occurrence lost its durable identity: %+v", attempts[0])
|
|
}
|
|
if err := s.ScheduleReminderRetry(ctx, originals, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
rows, err := s.ListReminders(ctx, 10)
|
|
if err != nil || len(rows) != 2 {
|
|
t.Fatalf("rows=%+v err=%v", rows, err)
|
|
}
|
|
for _, r := range rows {
|
|
if r.DeliveryAttempts != 0 || !r.NextAttemptTs.IsZero() {
|
|
t.Fatalf("ambiguous occurrence was scheduled for automatic retry: %+v", r)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestDefiniteFailedDeliveryRemainsRetryEligible(t *testing.T) {
|
|
s := newTestStore(t)
|
|
ctx := context.Background()
|
|
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
|
|
if _, err := s.CreateReminder(ctx, now.Add(-time.Minute), "позвонить врачу", ""); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
originals, err := s.DueReminders(ctx, now)
|
|
if err != nil || len(originals) != 1 {
|
|
t.Fatalf("due originals = %d, err=%v", len(originals), err)
|
|
}
|
|
const group = "reminder:definite-failure"
|
|
if err := s.CacheReminderPhrase(ctx, originals, group, "Позвони врачу.", "Позвони врачу", "neutral"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
id, err := s.BeginDeliveryAttempt(ctx, "reminder", "", originals[0].ID, group, "telegram", "hash", now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.CompleteDeliveryAttempt(ctx, id, DeliveryFailed, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if due, err := s.DueReminders(ctx, now); err != nil || len(due) != 1 {
|
|
t.Fatalf("definite failure was not retry eligible: due=%+v err=%v", due, err)
|
|
}
|
|
}
|
|
|
|
func TestCollapsedCompletionRollsBackAsOneTransaction(t *testing.T) {
|
|
s := newTestStore(t)
|
|
ctx := context.Background()
|
|
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
|
|
for _, text := range []string{"first", "fail-second"} {
|
|
if _, err := s.CreateReminder(ctx, now.Add(-time.Minute), text, ""); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
originals, err := s.DueReminders(ctx, now)
|
|
if err != nil || len(originals) != 2 {
|
|
t.Fatalf("due originals = %d, err=%v", len(originals), err)
|
|
}
|
|
if _, err := s.db.ExecContext(ctx, `
|
|
CREATE TRIGGER fail_second_reminder_completion
|
|
BEFORE UPDATE OF status ON reminders
|
|
WHEN OLD.payload = 'fail-second'
|
|
BEGIN
|
|
SELECT RAISE(ABORT, 'injected second completion failure');
|
|
END`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.CompleteReminderDelivery(ctx, originals, now); err == nil {
|
|
t.Fatal("injected second-row failure did not fail the bundle completion")
|
|
}
|
|
rows, err := s.ListReminders(ctx, 10)
|
|
if err != nil || len(rows) != 2 {
|
|
t.Fatalf("rows=%+v err=%v", rows, err)
|
|
}
|
|
for _, r := range rows {
|
|
if r.Status != ReminderPending {
|
|
t.Fatalf("bundle completion partially committed reminder %d as %q", r.ID, r.Status)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSuccessfulAttemptAndCollapsedCompletionCommitAtomically(t *testing.T) {
|
|
s := newTestStore(t)
|
|
ctx := context.Background()
|
|
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
|
|
for _, text := range []string{"first", "fail-second"} {
|
|
if _, err := s.CreateReminder(ctx, now.Add(-time.Minute), text, ""); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
originals, err := s.DueReminders(ctx, now)
|
|
if err != nil || len(originals) != 2 {
|
|
t.Fatalf("due originals = %d, err=%v", len(originals), err)
|
|
}
|
|
const group = "reminder:atomic-success"
|
|
if err := s.CacheReminderPhrase(ctx, originals, group, "Two reminders.", "Two reminders", "neutral"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for i := range originals {
|
|
originals[i].DeliveryGroup = group
|
|
}
|
|
attemptID, err := s.BeginDeliveryAttempt(ctx, "reminder", "", originals[0].ID, group, "telegram", "hash", now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := s.db.ExecContext(ctx, `
|
|
CREATE TRIGGER fail_atomic_second_completion
|
|
BEFORE UPDATE OF status ON reminders
|
|
WHEN OLD.payload = 'fail-second'
|
|
BEGIN
|
|
SELECT RAISE(ABORT, 'injected second completion failure');
|
|
END`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.CompleteSuccessfulReminderAttempt(ctx, attemptID, originals, now); err == nil {
|
|
t.Fatal("injected reminder completion failure did not fail the local commit")
|
|
}
|
|
attempts, err := s.ListDeliveryAttempts(ctx, "", 10)
|
|
if err != nil || len(attempts) != 1 || attempts[0].Status != DeliveryPending {
|
|
t.Fatalf("successful outbox marker committed without reminder state: attempts=%+v err=%v", attempts, err)
|
|
}
|
|
rows, err := s.ListReminders(ctx, 10)
|
|
if err != nil || len(rows) != 2 {
|
|
t.Fatalf("rows=%+v err=%v", rows, err)
|
|
}
|
|
for _, r := range rows {
|
|
if r.Status != ReminderPending {
|
|
t.Fatalf("atomic rollback partially completed reminder %d as %q", r.ID, r.Status)
|
|
}
|
|
}
|
|
if _, err := s.ReconcileStaleDeliveryAttempts(ctx, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if due, err := s.DueReminders(ctx, now.Add(24*time.Hour)); err != nil || len(due) != 0 {
|
|
t.Fatalf("restart auto-replayed ambiguous accepted send: due=%+v err=%v", due, err)
|
|
}
|
|
}
|
|
|
|
func TestPermanentReminderFailureBlocksUntilDeliberatelyUnblocked(t *testing.T) {
|
|
s := newTestStore(t)
|
|
ctx := context.Background()
|
|
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
|
|
id, err := s.CreateReminder(ctx, now.Add(-time.Minute), "позвонить врачу", "")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
originals, err := s.DueReminders(ctx, now)
|
|
if err != nil || len(originals) != 1 {
|
|
t.Fatalf("due originals=%+v err=%v", originals, err)
|
|
}
|
|
const group = "reminder:revoked-token"
|
|
if err := s.CacheReminderPhrase(ctx, originals, group, "Позвони врачу.", "Позвони врачу", "neutral"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
originals[0].DeliveryGroup = group
|
|
if err := s.BlockReminderDelivery(ctx, originals, now, "ntfy credentials rejected"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.ScheduleReminderRetry(ctx, originals, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if due, err := s.DueReminders(ctx, now.Add(24*time.Hour)); err != nil || len(due) != 0 {
|
|
t.Fatalf("blocked reminder retried automatically: due=%+v err=%v", due, err)
|
|
}
|
|
rows, err := s.ListReminders(ctx, 1)
|
|
if err != nil || len(rows) != 1 {
|
|
t.Fatalf("rows=%+v err=%v", rows, err)
|
|
}
|
|
if rows[0].DeliveryBlockedTs.IsZero() || rows[0].DeliveryBlockedError != "ntfy credentials rejected" ||
|
|
rows[0].DeliveryAttempts != 0 || !rows[0].NextAttemptTs.IsZero() {
|
|
t.Fatalf("blocked state is not visible/durable: %+v", rows[0])
|
|
}
|
|
if err := s.UnblockReminderDelivery(ctx, id); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
due, err := s.DueReminders(ctx, now)
|
|
if err != nil || len(due) != 1 || due[0].ID != id || !due[0].DeliveryBlockedTs.IsZero() || due[0].DeliveryBlockedError != "" {
|
|
t.Fatalf("deliberate unblock did not restore eligibility: due=%+v err=%v", due, err)
|
|
}
|
|
}
|