Make delivery and integration failures explicit
Persist reminder presentations and retry state, atomically complete collapsed deliveries, fall back across away reaches, and block permanent failures visibly (V-715, V-678). Fail closed when enabled integrations lack credentials and keep remote arms explicitly dark (V-691). Give mavweb one sanitized, request-correlated error contract (V-689). Owner explicitly requested direct commits to master.
This commit is contained in:
+18
-15
@@ -28,14 +28,16 @@ const (
|
||||
// send happens, so a crash between "sent externally" and "recorded" leaves a
|
||||
// trace instead of silence. kind is "nudge" or "reminder"; rule is set for
|
||||
// nudges, reminderID for reminders (the other left at its zero value).
|
||||
// deliveryGroup is the exact persisted reminder occurrence or collapsed
|
||||
// bundle; it is empty for nudges and legacy reminder attempts.
|
||||
// bodyHash is an opaque caller-computed key (e.g. sha256 of channel+body) —
|
||||
// stored for post-crash operator triage, not enforced as a uniqueness
|
||||
// constraint (a rule/reminder legitimately re-sends across ticks).
|
||||
func (s *Store) BeginDeliveryAttempt(ctx context.Context, kind, rule string, reminderID int64, channel, bodyHash string, now time.Time) (int64, error) {
|
||||
func (s *Store) BeginDeliveryAttempt(ctx context.Context, kind, rule string, reminderID int64, deliveryGroup, channel, bodyHash string, now time.Time) (int64, error) {
|
||||
res, err := s.db.ExecContext(ctx,
|
||||
`INSERT INTO delivery_attempts (kind, rule, reminder_id, channel, body_hash, status, created_ts)
|
||||
VALUES (?, ?, ?, ?, ?, 'pending', ?)`,
|
||||
kind, rule, reminderID, channel, bodyHash, now.UnixMilli())
|
||||
`INSERT INTO delivery_attempts (kind, rule, reminder_id, delivery_group, channel, body_hash, status, created_ts)
|
||||
VALUES (?, ?, ?, ?, ?, ?, 'pending', ?)`,
|
||||
kind, rule, reminderID, deliveryGroup, channel, bodyHash, now.UnixMilli())
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("begin delivery attempt: %w", err)
|
||||
}
|
||||
@@ -90,15 +92,16 @@ func (s *Store) ReconcileStaleDeliveryAttempts(ctx context.Context, now time.Tim
|
||||
|
||||
// DeliveryAttempt — one row of the outbox, as a reader sees it.
|
||||
type DeliveryAttempt struct {
|
||||
ID int64
|
||||
Kind string // nudge|reminder
|
||||
Rule string // set for nudges
|
||||
ReminderID int64 // set for reminders
|
||||
Channel string
|
||||
Status string // one of the Delivery* constants
|
||||
Created time.Time
|
||||
Completed time.Time // zero while pending
|
||||
HasComplete bool
|
||||
ID int64
|
||||
Kind string // nudge|reminder
|
||||
Rule string // set for nudges
|
||||
ReminderID int64 // set for reminders
|
||||
DeliveryGroup string // exact reminder occurrence/bundle; empty for nudges and legacy rows
|
||||
Channel string
|
||||
Status string // one of the Delivery* constants
|
||||
Created time.Time
|
||||
Completed time.Time // zero while pending
|
||||
HasComplete bool
|
||||
}
|
||||
|
||||
// ListDeliveryAttempts returns recent attempts, newest first. An empty status
|
||||
@@ -117,7 +120,7 @@ func (s *Store) ListDeliveryAttempts(ctx context.Context, status string, limit i
|
||||
if limit <= 0 {
|
||||
limit = 50
|
||||
}
|
||||
q := `SELECT id, kind, rule, reminder_id, channel, status, created_ts, completed_ts
|
||||
q := `SELECT id, kind, rule, reminder_id, delivery_group, channel, status, created_ts, completed_ts
|
||||
FROM delivery_attempts`
|
||||
args := []any{}
|
||||
if status != "" {
|
||||
@@ -138,7 +141,7 @@ func (s *Store) ListDeliveryAttempts(ctx context.Context, status string, limit i
|
||||
var a DeliveryAttempt
|
||||
var created int64
|
||||
var completed *int64
|
||||
if err := rows.Scan(&a.ID, &a.Kind, &a.Rule, &a.ReminderID, &a.Channel, &a.Status, &created, &completed); err != nil {
|
||||
if err := rows.Scan(&a.ID, &a.Kind, &a.Rule, &a.ReminderID, &a.DeliveryGroup, &a.Channel, &a.Status, &created, &completed); err != nil {
|
||||
return nil, fmt.Errorf("list delivery attempts: scan: %w", err)
|
||||
}
|
||||
a.Created = time.UnixMilli(created)
|
||||
|
||||
@@ -15,7 +15,7 @@ func TestDroppedDeliveryAttemptRoundTrips(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
now := time.Now()
|
||||
|
||||
id, err := s.BeginDeliveryAttempt(ctx, "nudge", "water", 0, "drop", "abc123", now)
|
||||
id, err := s.BeginDeliveryAttempt(ctx, "nudge", "water", 0, "", "drop", "abc123", now)
|
||||
if err != nil {
|
||||
t.Fatalf("BeginDeliveryAttempt: %v", err)
|
||||
}
|
||||
@@ -40,21 +40,21 @@ func TestListDeliveryAttempts(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
base := time.Date(2026, 8, 4, 9, 0, 0, 0, time.UTC)
|
||||
|
||||
sent, err := s.BeginDeliveryAttempt(ctx, "nudge", "water", 0, "telegram", "h1", base)
|
||||
sent, err := s.BeginDeliveryAttempt(ctx, "nudge", "water", 0, "", "telegram", "h1", base)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.CompleteDeliveryAttempt(ctx, sent, DeliverySent, base.Add(time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dropped, err := s.BeginDeliveryAttempt(ctx, "nudge", "care", 0, "telegram", "h2", base.Add(time.Minute))
|
||||
dropped, err := s.BeginDeliveryAttempt(ctx, "nudge", "care", 0, "", "telegram", "h2", base.Add(time.Minute))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.CompleteDeliveryAttempt(ctx, dropped, DeliveryDropped, base.Add(time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.BeginDeliveryAttempt(ctx, "reminder", "", 7, "voice", "h3", base.Add(2*time.Minute)); err != nil {
|
||||
if _, err := s.BeginDeliveryAttempt(ctx, "reminder", "", 7, "reminder:test", "voice", "h3", base.Add(2*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
|
||||
@@ -351,6 +351,29 @@ ALTER TABLE reminders ADD COLUMN next_fire_ts INTEGER;`, // #2
|
||||
source TEXT NOT NULL DEFAULT '',
|
||||
encoder_id TEXT NOT NULL DEFAULT ''
|
||||
);`,
|
||||
// #25 — durable reminder delivery state (V-651). A reminder can remain
|
||||
// pending for a long time when every reachable transport is unhealthy. The
|
||||
// phrased text belongs to that delivery occurrence, so retaining it here
|
||||
// avoids spending the resident model again on every retry and across daemon
|
||||
// restarts. next_attempt_ts makes transport failures wait on a bounded
|
||||
// exponential backoff instead of retrying at the tick rate. delivery_group
|
||||
// keeps the originals of a collapsed catch-up bundle together; a reminder
|
||||
// that becomes due later must not make the old bundle get re-phrased.
|
||||
`ALTER TABLE reminders ADD COLUMN delivery_group TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE reminders ADD COLUMN phrase_body TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE reminders ADD COLUMN phrase_summary TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE reminders ADD COLUMN phrase_mood TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE reminders ADD COLUMN delivery_attempts INTEGER NOT NULL DEFAULT 0 CHECK (delivery_attempts >= 0);
|
||||
ALTER TABLE reminders ADD COLUMN next_attempt_ts INTEGER;
|
||||
ALTER TABLE delivery_attempts ADD COLUMN delivery_group TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE reminders ADD COLUMN delivery_blocked_ts INTEGER;
|
||||
ALTER TABLE reminders ADD COLUMN delivery_blocked_error TEXT NOT NULL DEFAULT '';
|
||||
CREATE INDEX IF NOT EXISTS idx_reminders_due
|
||||
ON reminders (next_fire_ts, next_attempt_ts)
|
||||
WHERE status = 'pending';
|
||||
CREATE INDEX IF NOT EXISTS idx_delivery_attempts_reminder_group
|
||||
ON delivery_attempts (delivery_group, status)
|
||||
WHERE kind = 'reminder' AND delivery_group <> '';`,
|
||||
}
|
||||
|
||||
// migrate applies every migration with a number greater than the DB's current
|
||||
|
||||
+453
-23
@@ -22,6 +22,20 @@ type Reminder struct {
|
||||
Status string // pending | fired | cancelled
|
||||
Cron string // cron expression, empty for one-shot
|
||||
|
||||
// DeliveryGroup and Phrase* are the durable presentation for this exact
|
||||
// occurrence. A collapsed catch-up bundle stores the same group and phrase
|
||||
// on every original reminder, so a retry (including after restart) says the
|
||||
// same thing without asking the model again. Rescheduling a recurring
|
||||
// reminder clears them for its next occurrence.
|
||||
DeliveryGroup string
|
||||
PhraseBody string
|
||||
PhraseSummary string
|
||||
PhraseMood string
|
||||
DeliveryAttempts int
|
||||
NextAttemptTs time.Time
|
||||
DeliveryBlockedTs time.Time
|
||||
DeliveryBlockedError string
|
||||
|
||||
// Collapsed — set only on a synthetic digest reminder (ID=0): the original
|
||||
// due reminders it stands in for. Not persisted. The dispatcher completes
|
||||
// (marks fired / reschedules) each of these after the digest delivers.
|
||||
@@ -62,18 +76,35 @@ const (
|
||||
ReminderPending = "pending"
|
||||
ReminderFired = "fired"
|
||||
ReminderCancelled = "cancelled"
|
||||
|
||||
// ReminderRetryBase and ReminderRetryMax bound the retry cadence. Attempts
|
||||
// continue indefinitely because a reminder must not disappear during a
|
||||
// long transport outage; only the delay stops growing.
|
||||
ReminderRetryBase = time.Minute
|
||||
ReminderRetryMax = time.Hour
|
||||
)
|
||||
|
||||
var (
|
||||
ErrReminderNotFound = errors.New("store: reminder not found")
|
||||
ErrReminderState = errors.New("store: reminder not in a mutable state")
|
||||
ErrReminderPhrase = errors.New("store: reminder delivery phrase invalid")
|
||||
)
|
||||
|
||||
const reminderColumns = `id, created_ts, fire_ts, next_fire_ts, payload, status, cron,
|
||||
delivery_group, phrase_body, phrase_summary, phrase_mood, delivery_attempts, next_attempt_ts,
|
||||
delivery_blocked_ts, delivery_blocked_error`
|
||||
|
||||
func scanReminder(sc scanner) (Reminder, error) {
|
||||
var r Reminder
|
||||
var created, fire, nextFire int64
|
||||
var cron *string
|
||||
if err := sc.Scan(&r.ID, &created, &fire, &nextFire, &r.Payload, &r.Status, &cron); err != nil {
|
||||
var nextAttempt *int64
|
||||
var blocked *int64
|
||||
if err := sc.Scan(
|
||||
&r.ID, &created, &fire, &nextFire, &r.Payload, &r.Status, &cron,
|
||||
&r.DeliveryGroup, &r.PhraseBody, &r.PhraseSummary, &r.PhraseMood,
|
||||
&r.DeliveryAttempts, &nextAttempt, &blocked, &r.DeliveryBlockedError,
|
||||
); err != nil {
|
||||
return Reminder{}, err
|
||||
}
|
||||
r.CreatedTs = time.UnixMilli(created).UTC()
|
||||
@@ -82,6 +113,12 @@ func scanReminder(sc scanner) (Reminder, error) {
|
||||
if cron != nil {
|
||||
r.Cron = *cron
|
||||
}
|
||||
if nextAttempt != nil {
|
||||
r.NextAttemptTs = time.UnixMilli(*nextAttempt).UTC()
|
||||
}
|
||||
if blocked != nil {
|
||||
r.DeliveryBlockedTs = time.UnixMilli(*blocked).UTC()
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
@@ -111,11 +148,20 @@ func (s *Store) CreateReminder(ctx context.Context, fire time.Time, payload, cro
|
||||
// DueReminders returns pending reminders with next_fire_ts <= now, oldest first.
|
||||
// This is the predicate input from the loop side: `next_fire_ts <= now AND status='pending'`.
|
||||
func (s *Store) DueReminders(ctx context.Context, now time.Time) ([]Reminder, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT id, created_ts, fire_ts, next_fire_ts, payload, status, cron
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT `+reminderColumns+`
|
||||
FROM reminders
|
||||
WHERE status = 'pending' AND next_fire_ts <= ?
|
||||
ORDER BY next_fire_ts ASC`, now.UnixMilli())
|
||||
AND (next_attempt_ts IS NULL OR next_attempt_ts <= ?)
|
||||
AND delivery_blocked_ts IS NULL
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM delivery_attempts AS attempt
|
||||
WHERE attempt.kind = 'reminder'
|
||||
AND attempt.delivery_group = reminders.delivery_group
|
||||
AND reminders.delivery_group <> ''
|
||||
AND attempt.status IN ('pending', 'sent', 'unknown')
|
||||
)
|
||||
ORDER BY next_fire_ts ASC, id ASC`, now.UnixMilli(), now.UnixMilli())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("due reminders: %w", err)
|
||||
}
|
||||
@@ -140,8 +186,7 @@ func (s *Store) DueReminders(ctx context.Context, now time.Time) ([]Reminder, er
|
||||
// for next year stayed on it. Bounding by fire time drops what is out of range
|
||||
// instead of what is old.
|
||||
func (s *Store) PendingReminders(ctx context.Context, from, to time.Time) ([]Reminder, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT id, created_ts, fire_ts, next_fire_ts, payload, status, cron
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT `+reminderColumns+`
|
||||
FROM reminders
|
||||
WHERE status = ? AND next_fire_ts >= ? AND next_fire_ts < ?
|
||||
ORDER BY next_fire_ts ASC, id ASC`,
|
||||
@@ -164,29 +209,42 @@ func (s *Store) PendingReminders(ctx context.Context, from, to time.Time) ([]Rem
|
||||
// MarkReminder sets a reminder's status. Only valid transitions: pending→fired,
|
||||
// pending→cancelled. Anything else is a programming error.
|
||||
func (s *Store) MarkReminder(ctx context.Context, id int64, status string) error {
|
||||
if status != "fired" && status != "cancelled" {
|
||||
if status != ReminderFired && status != ReminderCancelled {
|
||||
return fmt.Errorf("%w: %s", ErrReminderState, status)
|
||||
}
|
||||
// pending → fired|cancelled only.
|
||||
// The old read-then-write transition allowed two callers to both observe
|
||||
// pending and both report success. Keeping the source state in the UPDATE
|
||||
// predicate makes pending → fired|cancelled one atomic contest (V-678).
|
||||
res, err := s.db.ExecContext(ctx,
|
||||
"UPDATE reminders SET status = ? WHERE id = ? AND status = ?",
|
||||
status, id, ReminderPending)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
n, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return fmt.Errorf("mark reminder: rows affected: %w", err)
|
||||
}
|
||||
if n == 1 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Preserve the public distinction between a missing id and a completed
|
||||
// state without weakening the atomic transition above.
|
||||
var current string
|
||||
err := s.db.QueryRowContext(ctx, "SELECT status FROM reminders WHERE id = ?", id).Scan(¤t)
|
||||
err = s.db.QueryRowContext(ctx, "SELECT status FROM reminders WHERE id = ?", id).Scan(¤t)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return ErrReminderNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if current != "pending" {
|
||||
return fmt.Errorf("%w: currently %s", ErrReminderState, current)
|
||||
}
|
||||
_, err = s.db.ExecContext(ctx, "UPDATE reminders SET status = ? WHERE id = ?", status, id)
|
||||
return err
|
||||
return fmt.Errorf("%w: currently %s", ErrReminderState, current)
|
||||
}
|
||||
|
||||
// ListReminders returns the n most recent reminders, newest first.
|
||||
func (s *Store) ListReminders(ctx context.Context, n int) ([]Reminder, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT id, created_ts, fire_ts, next_fire_ts, payload, status, cron
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT `+reminderColumns+`
|
||||
FROM reminders ORDER BY created_ts DESC, id DESC LIMIT ?`, n)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list reminders: %w", err)
|
||||
@@ -203,6 +261,352 @@ func (s *Store) ListReminders(ctx context.Context, n int) ([]Reminder, error) {
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// HasDeliveryPhrase reports whether this occurrence already has a durable
|
||||
// presentation. Summary may intentionally be empty (the delivery boundary has
|
||||
// a generic privacy-preserving fallback), so Body is the readiness marker.
|
||||
func (r Reminder) HasDeliveryPhrase() bool {
|
||||
return r.DeliveryGroup != "" && r.PhraseBody != ""
|
||||
}
|
||||
|
||||
// ReminderRetryDelay returns the delay after attempt (one-based). It grows
|
||||
// exponentially from one minute and stays at one hour; retries do not stop.
|
||||
func ReminderRetryDelay(attempt int) time.Duration {
|
||||
if attempt <= 1 {
|
||||
return ReminderRetryBase
|
||||
}
|
||||
delay := ReminderRetryBase
|
||||
for i := 1; i < attempt && delay < ReminderRetryMax; i++ {
|
||||
if delay >= ReminderRetryMax/2 {
|
||||
return ReminderRetryMax
|
||||
}
|
||||
delay *= 2
|
||||
}
|
||||
if delay > ReminderRetryMax {
|
||||
return ReminderRetryMax
|
||||
}
|
||||
return delay
|
||||
}
|
||||
|
||||
// CacheReminderPhrase stores one presentation on every original represented
|
||||
// by a reminder delivery. For a collapsed bundle, originals contains every
|
||||
// row in Reminder.Collapsed and group is shared across all of them.
|
||||
//
|
||||
// The occurrence timestamp and empty-body predicates keep this from attaching
|
||||
// an old phrase to a recurring reminder's next occurrence or overwriting a
|
||||
// phrase another delivery already claimed. The transaction prevents a partial
|
||||
// bundle cache: after a crash either every original can reconstruct the bundle
|
||||
// or none can.
|
||||
func (s *Store) CacheReminderPhrase(
|
||||
ctx context.Context,
|
||||
originals []Reminder,
|
||||
group, body, summary, mood string,
|
||||
) error {
|
||||
if len(originals) == 0 || group == "" || body == "" {
|
||||
return ErrReminderPhrase
|
||||
}
|
||||
if mood == "" {
|
||||
mood = "neutral"
|
||||
}
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cache reminder phrase: begin: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
seen := make(map[int64]struct{}, len(originals))
|
||||
for _, r := range originals {
|
||||
if r.ID <= 0 || r.NextFireTs.IsZero() {
|
||||
return ErrReminderPhrase
|
||||
}
|
||||
if _, ok := seen[r.ID]; ok {
|
||||
return ErrReminderPhrase
|
||||
}
|
||||
seen[r.ID] = struct{}{}
|
||||
res, err := tx.ExecContext(ctx, `
|
||||
UPDATE reminders
|
||||
SET delivery_group = ?, phrase_body = ?, phrase_summary = ?, phrase_mood = ?
|
||||
WHERE id = ? AND status = ? AND next_fire_ts = ? AND phrase_body = ''`,
|
||||
group, body, summary, mood, r.ID, ReminderPending, r.NextFireTs.UnixMilli())
|
||||
if err != nil {
|
||||
return fmt.Errorf("cache reminder phrase %d: %w", r.ID, err)
|
||||
}
|
||||
n, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cache reminder phrase %d: rows affected: %w", r.ID, err)
|
||||
}
|
||||
if n != 1 {
|
||||
return fmt.Errorf("%w: reminder %d is no longer an unphrased pending occurrence", ErrReminderState, r.ID)
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("cache reminder phrase: commit: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ScheduleReminderRetry moves every still-pending original into a retry wait.
|
||||
// Terminal rows are skipped: cancellation winning while a send was in flight
|
||||
// must not be resurrected. A recurring row whose occurrence changed is skipped
|
||||
// for the same reason. All remaining originals get their own persisted attempt
|
||||
// count; normally a collapsed bundle keeps those counts in lockstep.
|
||||
func (s *Store) ScheduleReminderRetry(ctx context.Context, originals []Reminder, now time.Time) error {
|
||||
if len(originals) == 0 {
|
||||
return ErrReminderNotFound
|
||||
}
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("schedule reminder retry: begin: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
seen := make(map[int64]struct{}, len(originals))
|
||||
for _, expected := range originals {
|
||||
if expected.ID <= 0 || expected.NextFireTs.IsZero() {
|
||||
return ErrReminderNotFound
|
||||
}
|
||||
if _, ok := seen[expected.ID]; ok {
|
||||
continue
|
||||
}
|
||||
seen[expected.ID] = struct{}{}
|
||||
|
||||
var status string
|
||||
var nextFire int64
|
||||
var attempts int
|
||||
var blocked *int64
|
||||
err := tx.QueryRowContext(ctx,
|
||||
`SELECT status, next_fire_ts, delivery_attempts, delivery_blocked_ts FROM reminders WHERE id = ?`,
|
||||
expected.ID).Scan(&status, &nextFire, &attempts, &blocked)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return ErrReminderNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("schedule reminder retry %d: read: %w", expected.ID, err)
|
||||
}
|
||||
if status != ReminderPending || nextFire != expected.NextFireTs.UnixMilli() || blocked != nil {
|
||||
continue
|
||||
}
|
||||
if expected.DeliveryGroup != "" {
|
||||
var ambiguous int
|
||||
if err := tx.QueryRowContext(ctx, `
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM delivery_attempts
|
||||
WHERE kind = 'reminder' AND delivery_group = ?
|
||||
AND status IN ('pending', 'sent', 'unknown')
|
||||
)`, expected.DeliveryGroup).Scan(&ambiguous); err != nil {
|
||||
return fmt.Errorf("schedule reminder retry %d: inspect outbox: %w", expected.ID, err)
|
||||
}
|
||||
if ambiguous != 0 {
|
||||
// The sink may have accepted this presentation. Leave the reminder
|
||||
// pending but ineligible for automatic retry; operator resolution
|
||||
// must not be replaced with a guessed duplicate.
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
attempts++
|
||||
nextAttempt := now.Add(ReminderRetryDelay(attempts)).UnixMilli()
|
||||
res, err := tx.ExecContext(ctx, `
|
||||
UPDATE reminders
|
||||
SET delivery_attempts = ?, next_attempt_ts = ?
|
||||
WHERE id = ? AND status = ? AND next_fire_ts = ? AND delivery_attempts = ?`,
|
||||
attempts, nextAttempt, expected.ID, ReminderPending, nextFire, attempts-1)
|
||||
if err != nil {
|
||||
return fmt.Errorf("schedule reminder retry %d: %w", expected.ID, err)
|
||||
}
|
||||
n, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return fmt.Errorf("schedule reminder retry %d: rows affected: %w", expected.ID, err)
|
||||
}
|
||||
if n != 1 {
|
||||
return fmt.Errorf("%w: reminder %d changed concurrently", ErrReminderState, expected.ID)
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("schedule reminder retry: commit: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// BlockReminderDelivery records a permanent transport/configuration refusal on
|
||||
// every original represented by one presentation. Blocked rows stay pending and
|
||||
// visible, but never retry automatically. UnblockReminderDelivery is the
|
||||
// deliberate recovery path after credentials or policy are repaired.
|
||||
func (s *Store) BlockReminderDelivery(ctx context.Context, originals []Reminder, now time.Time, reason string) error {
|
||||
if len(originals) == 0 || strings.TrimSpace(reason) == "" {
|
||||
return ErrReminderState
|
||||
}
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("block reminder delivery: begin: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
seen := make(map[int64]struct{}, len(originals))
|
||||
for _, expected := range originals {
|
||||
if expected.ID <= 0 || expected.NextFireTs.IsZero() {
|
||||
return ErrReminderNotFound
|
||||
}
|
||||
if _, ok := seen[expected.ID]; ok {
|
||||
continue
|
||||
}
|
||||
seen[expected.ID] = struct{}{}
|
||||
res, err := tx.ExecContext(ctx, `
|
||||
UPDATE reminders
|
||||
SET delivery_blocked_ts = ?, delivery_blocked_error = ?, next_attempt_ts = NULL
|
||||
WHERE id = ? AND status = ? AND next_fire_ts = ?`,
|
||||
now.UnixMilli(), reason, expected.ID, ReminderPending, expected.NextFireTs.UnixMilli())
|
||||
if err != nil {
|
||||
return fmt.Errorf("block reminder %d: %w", expected.ID, err)
|
||||
}
|
||||
if err := requireOneReminderRow(res, expected.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("block reminder delivery: commit: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) UnblockReminderDelivery(ctx context.Context, id int64) error {
|
||||
res, err := s.db.ExecContext(ctx, `
|
||||
UPDATE reminders
|
||||
SET delivery_blocked_ts = NULL, delivery_blocked_error = '', next_attempt_ts = NULL
|
||||
WHERE id = ? AND status = ? AND delivery_blocked_ts IS NOT NULL`, id, ReminderPending)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unblock reminder %d: %w", id, err)
|
||||
}
|
||||
return requireOneReminderRow(res, id)
|
||||
}
|
||||
|
||||
// CompleteReminderDelivery applies the bookkeeping for every original covered
|
||||
// by one successful external send in a single transaction. One-shot reminders
|
||||
// become fired; recurring reminders advance to their next occurrence and shed
|
||||
// the old presentation/retry state. The all-or-nothing boundary prevents a
|
||||
// collapsed digest from becoming half-fired and then being repeated.
|
||||
func (s *Store) CompleteReminderDelivery(ctx context.Context, originals []Reminder, now time.Time) error {
|
||||
return s.completeReminderDeliveryIn(ctx, 0, originals, now, time.Local)
|
||||
}
|
||||
|
||||
// CompleteSuccessfulReminderAttempt atomically closes a definitely successful
|
||||
// external attempt and advances every reminder occurrence it represented. If
|
||||
// this transaction cannot commit, the attempt remains pending; startup then
|
||||
// reconciles it to unknown, and the occurrence is held from automatic replay.
|
||||
func (s *Store) CompleteSuccessfulReminderAttempt(ctx context.Context, attemptID int64, originals []Reminder, now time.Time) error {
|
||||
if attemptID <= 0 {
|
||||
return fmt.Errorf("complete reminder delivery: invalid attempt id %d", attemptID)
|
||||
}
|
||||
return s.completeReminderDeliveryIn(ctx, attemptID, originals, now, time.Local)
|
||||
}
|
||||
|
||||
func (s *Store) completeReminderDeliveryIn(ctx context.Context, attemptID int64, originals []Reminder, now time.Time, loc *time.Location) error {
|
||||
if len(originals) == 0 {
|
||||
return ErrReminderNotFound
|
||||
}
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("complete reminder delivery: begin: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if attemptID != 0 {
|
||||
res, err := tx.ExecContext(ctx,
|
||||
`UPDATE delivery_attempts SET status = ?, completed_ts = ? WHERE id = ? AND status = ?`,
|
||||
DeliverySent, now.UnixMilli(), attemptID, DeliveryPending)
|
||||
if err != nil {
|
||||
return fmt.Errorf("complete reminder delivery attempt %d: %w", attemptID, err)
|
||||
}
|
||||
n, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return fmt.Errorf("complete reminder delivery attempt %d: rows affected: %w", attemptID, err)
|
||||
}
|
||||
if n != 1 {
|
||||
return fmt.Errorf("complete reminder delivery attempt %d: not pending", attemptID)
|
||||
}
|
||||
}
|
||||
|
||||
seen := make(map[int64]struct{}, len(originals))
|
||||
for _, expected := range originals {
|
||||
if expected.ID <= 0 || expected.NextFireTs.IsZero() {
|
||||
return ErrReminderNotFound
|
||||
}
|
||||
if _, ok := seen[expected.ID]; ok {
|
||||
return fmt.Errorf("%w: duplicate reminder %d in one delivery", ErrReminderState, expected.ID)
|
||||
}
|
||||
seen[expected.ID] = struct{}{}
|
||||
|
||||
row := tx.QueryRowContext(ctx, `SELECT `+reminderColumns+` FROM reminders WHERE id = ?`, expected.ID)
|
||||
current, err := scanReminder(row)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return ErrReminderNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("complete reminder %d: read: %w", expected.ID, err)
|
||||
}
|
||||
if current.Status != ReminderPending || !current.NextFireTs.Equal(expected.NextFireTs) {
|
||||
return fmt.Errorf("%w: reminder %d occurrence changed", ErrReminderState, expected.ID)
|
||||
}
|
||||
if expected.DeliveryGroup != "" && current.DeliveryGroup != expected.DeliveryGroup {
|
||||
return fmt.Errorf("%w: reminder %d delivery group changed", ErrReminderState, expected.ID)
|
||||
}
|
||||
|
||||
if current.Cron == "" {
|
||||
if err := completeOneShotReminderTx(ctx, tx, current); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
next, err := nextReminderOccurrence(current, now, loc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if next.IsZero() {
|
||||
if err := completeOneShotReminderTx(ctx, tx, current); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
res, err := tx.ExecContext(ctx, `
|
||||
UPDATE reminders
|
||||
SET next_fire_ts = ?, delivery_group = '', phrase_body = '',
|
||||
phrase_summary = '', phrase_mood = '', delivery_attempts = 0,
|
||||
next_attempt_ts = NULL, delivery_blocked_ts = NULL,
|
||||
delivery_blocked_error = ''
|
||||
WHERE id = ? AND status = ? AND next_fire_ts = ?`,
|
||||
next.UnixMilli(), current.ID, ReminderPending, current.NextFireTs.UnixMilli())
|
||||
if err != nil {
|
||||
return fmt.Errorf("complete recurring reminder %d: %w", current.ID, err)
|
||||
}
|
||||
if err := requireOneReminderRow(res, current.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("complete reminder delivery: commit: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func completeOneShotReminderTx(ctx context.Context, tx *sql.Tx, r Reminder) error {
|
||||
res, err := tx.ExecContext(ctx,
|
||||
`UPDATE reminders SET status = ? WHERE id = ? AND status = ? AND next_fire_ts = ?`,
|
||||
ReminderFired, r.ID, ReminderPending, r.NextFireTs.UnixMilli())
|
||||
if err != nil {
|
||||
return fmt.Errorf("complete reminder %d: %w", r.ID, err)
|
||||
}
|
||||
return requireOneReminderRow(res, r.ID)
|
||||
}
|
||||
|
||||
func requireOneReminderRow(res sql.Result, id int64) error {
|
||||
n, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return fmt.Errorf("complete reminder %d: rows affected: %w", id, err)
|
||||
}
|
||||
if n != 1 {
|
||||
return fmt.Errorf("%w: reminder %d changed concurrently", ErrReminderState, id)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RescheduleReminder computes the next fire time for a recurring reminder and
|
||||
// updates next_fire_ts. Returns ErrReminderState if the reminder is not
|
||||
// recurring or not pending. If the schedule yields no further fire time at all,
|
||||
@@ -229,8 +633,7 @@ func (s *Store) RescheduleReminder(ctx context.Context, id int64, now time.Time)
|
||||
}
|
||||
|
||||
func (s *Store) rescheduleReminderIn(ctx context.Context, id int64, now time.Time, loc *time.Location) error {
|
||||
row := s.db.QueryRowContext(ctx, `
|
||||
SELECT id, created_ts, fire_ts, next_fire_ts, payload, status, cron
|
||||
row := s.db.QueryRowContext(ctx, `SELECT `+reminderColumns+`
|
||||
FROM reminders WHERE id = ?`, id)
|
||||
r, err := scanReminder(row)
|
||||
if err != nil {
|
||||
@@ -243,9 +646,38 @@ func (s *Store) rescheduleReminderIn(ctx context.Context, id int64, now time.Tim
|
||||
return fmt.Errorf("%w: currently %s", ErrReminderState, r.Status)
|
||||
}
|
||||
|
||||
next, err := nextReminderOccurrence(r, now, loc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if next.IsZero() {
|
||||
return s.MarkReminder(ctx, id, ReminderFired)
|
||||
}
|
||||
res, err := s.db.ExecContext(ctx, `
|
||||
UPDATE reminders
|
||||
SET next_fire_ts = ?, delivery_group = '', phrase_body = '',
|
||||
phrase_summary = '', phrase_mood = '', delivery_attempts = 0,
|
||||
next_attempt_ts = NULL, delivery_blocked_ts = NULL,
|
||||
delivery_blocked_error = ''
|
||||
WHERE id = ? AND status = ? AND next_fire_ts = ?`,
|
||||
next.UnixMilli(), id, ReminderPending, r.NextFireTs.UnixMilli())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
n, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return fmt.Errorf("reschedule reminder: rows affected: %w", err)
|
||||
}
|
||||
if n != 1 {
|
||||
return fmt.Errorf("%w: reminder %d changed concurrently", ErrReminderState, id)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func nextReminderOccurrence(r Reminder, now time.Time, loc *time.Location) (time.Time, error) {
|
||||
sched, err := cron.ParseStandard(r.Cron)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse cron %q: %w", r.Cron, err)
|
||||
return time.Time{}, fmt.Errorf("parse cron %q: %w", r.Cron, err)
|
||||
}
|
||||
// Next is strictly after the time it is given, so the last fire cannot be
|
||||
// returned again and no fudge minute is needed. The bound stops a schedule
|
||||
@@ -255,9 +687,7 @@ func (s *Store) rescheduleReminderIn(ctx context.Context, id int64, now time.Tim
|
||||
next = sched.Next(next)
|
||||
}
|
||||
if next.IsZero() || !next.After(now) {
|
||||
_, err = s.db.ExecContext(ctx, "UPDATE reminders SET status = 'fired' WHERE id = ?", id)
|
||||
return err
|
||||
return time.Time{}, nil
|
||||
}
|
||||
_, err = s.db.ExecContext(ctx, "UPDATE reminders SET next_fire_ts = ? WHERE id = ?", next.UnixMilli(), id)
|
||||
return err
|
||||
return next, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,417 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestReminderPhraseAndRetrySurviveRestart(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
path := filepath.Join(t.TempDir(), "reminders.db")
|
||||
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
|
||||
|
||||
s, err := Open(ctx, path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id, err := s.CreateReminder(ctx, now.Add(-time.Minute), `{"text":"позвонить маме"}`, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
due, err := s.DueReminders(ctx, now)
|
||||
if err != nil || len(due) != 1 {
|
||||
t.Fatalf("initial due reminders = %d, err=%v", len(due), err)
|
||||
}
|
||||
if err := s.CacheReminderPhrase(ctx, due, "reminder:one", "Позвони маме.", "Позвони маме", "warm"); err != nil {
|
||||
t.Fatalf("cache phrase: %v", err)
|
||||
}
|
||||
if err := s.ScheduleReminderRetry(ctx, due, now); err != nil {
|
||||
t.Fatalf("schedule retry: %v", err)
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
s, err = Open(ctx, path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
if got, err := s.DueReminders(ctx, now.Add(30*time.Second)); err != nil || len(got) != 0 {
|
||||
t.Fatalf("during retry wait: got %d due, err=%v", len(got), err)
|
||||
}
|
||||
due, err = s.DueReminders(ctx, now.Add(ReminderRetryBase))
|
||||
if err != nil || len(due) != 1 {
|
||||
t.Fatalf("at retry time: got %d due, err=%v", len(due), err)
|
||||
}
|
||||
r := due[0]
|
||||
if r.ID != id || r.DeliveryGroup != "reminder:one" || r.PhraseBody != "Позвони маме." ||
|
||||
r.PhraseSummary != "Позвони маме" || r.PhraseMood != "warm" {
|
||||
t.Fatalf("persisted delivery presentation = %+v", r)
|
||||
}
|
||||
if r.DeliveryAttempts != 1 {
|
||||
t.Fatalf("attempts = %d, want 1", r.DeliveryAttempts)
|
||||
}
|
||||
if want := now.Add(ReminderRetryBase); !r.NextAttemptTs.Equal(want) {
|
||||
t.Fatalf("next attempt = %s, want %s", r.NextAttemptTs, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReminderRetryDelayIsExponentialAndBounded(t *testing.T) {
|
||||
wants := []time.Duration{
|
||||
time.Minute, 2 * time.Minute, 4 * time.Minute, 8 * time.Minute,
|
||||
16 * time.Minute, 32 * time.Minute, time.Hour, time.Hour, time.Hour,
|
||||
}
|
||||
for i, want := range wants {
|
||||
if got := ReminderRetryDelay(i + 1); got != want {
|
||||
t.Errorf("attempt %d delay = %s, want %s", i+1, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollapsedReminderRetryStateMovesTogether(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
|
||||
for _, text := range []string{"полить цветы", "позвонить врачу"} {
|
||||
if _, err := s.CreateReminder(ctx, now.Add(-time.Minute), text, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
originals, err := s.DueReminders(ctx, now)
|
||||
if err != nil || len(originals) != 2 {
|
||||
t.Fatalf("due originals = %d, err=%v", len(originals), err)
|
||||
}
|
||||
if err := s.CacheReminderPhrase(ctx, originals, "reminder:bundle", "У тебя два напоминания.", "Два напоминания", "neutral"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.ScheduleReminderRetry(ctx, originals, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
rows, err := s.ListReminders(ctx, 10)
|
||||
if err != nil || len(rows) != 2 {
|
||||
t.Fatalf("list = %d, err=%v", len(rows), err)
|
||||
}
|
||||
for _, r := range rows {
|
||||
if r.DeliveryGroup != "reminder:bundle" || r.PhraseBody != "У тебя два напоминания." || r.DeliveryAttempts != 1 {
|
||||
t.Errorf("bundle original did not move with group: %+v", r)
|
||||
}
|
||||
if !r.NextAttemptTs.Equal(now.Add(time.Minute)) {
|
||||
t.Errorf("next attempt = %s, want %s", r.NextAttemptTs, now.Add(time.Minute))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecurringReminderClearsDeliveryStateForNextOccurrence(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
ctx := context.Background()
|
||||
loc := time.FixedZone("MSK", 3*60*60)
|
||||
fire := time.Date(2026, 8, 13, 9, 0, 0, 0, loc)
|
||||
id, err := s.CreateReminder(ctx, fire, `{"text":"стендап"}`, "0 9 * * *")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
due, err := s.DueReminders(ctx, fire)
|
||||
if err != nil || len(due) != 1 {
|
||||
t.Fatalf("due = %d, err=%v", len(due), err)
|
||||
}
|
||||
if err := s.CacheReminderPhrase(ctx, due, "reminder:occurrence", "Пора на стендап.", "Стендап", "neutral"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.ScheduleReminderRetry(ctx, due, fire); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.rescheduleReminderIn(ctx, id, fire.Add(time.Minute), loc); err != nil {
|
||||
t.Fatalf("reschedule: %v", err)
|
||||
}
|
||||
|
||||
rows, err := s.ListReminders(ctx, 1)
|
||||
if err != nil || len(rows) != 1 {
|
||||
t.Fatalf("list = %d, err=%v", len(rows), err)
|
||||
}
|
||||
r := rows[0]
|
||||
if r.Status != ReminderPending || r.DeliveryGroup != "" || r.PhraseBody != "" ||
|
||||
r.PhraseSummary != "" || r.PhraseMood != "" || r.DeliveryAttempts != 0 || !r.NextAttemptTs.IsZero() {
|
||||
t.Fatalf("next occurrence retained old delivery state: %+v", r)
|
||||
}
|
||||
want := time.Date(2026, 8, 14, 9, 0, 0, 0, loc)
|
||||
if !r.NextFireTs.Equal(want) {
|
||||
t.Fatalf("next fire = %s, want %s", r.NextFireTs.In(loc), want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReminderTerminalTransitionHasExactlyOneWinner(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
|
||||
|
||||
for iteration := 0; iteration < 40; iteration++ {
|
||||
id, err := s.CreateReminder(ctx, now, "race", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
start := make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
errs := make([]error, 2)
|
||||
statuses := []string{ReminderFired, ReminderCancelled}
|
||||
for i := range statuses {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
<-start
|
||||
errs[i] = s.MarkReminder(ctx, id, statuses[i])
|
||||
}(i)
|
||||
}
|
||||
close(start)
|
||||
wg.Wait()
|
||||
|
||||
successes := 0
|
||||
losers := 0
|
||||
winner := ""
|
||||
for i, err := range errs {
|
||||
switch {
|
||||
case err == nil:
|
||||
successes++
|
||||
winner = statuses[i]
|
||||
case errors.Is(err, ErrReminderState):
|
||||
losers++
|
||||
default:
|
||||
t.Fatalf("iteration %d transition %s: %v", iteration, statuses[i], err)
|
||||
}
|
||||
}
|
||||
if successes != 1 || losers != 1 {
|
||||
t.Fatalf("iteration %d: successes=%d losers=%d errors=%v", iteration, successes, losers, errs)
|
||||
}
|
||||
var stored string
|
||||
if err := s.db.QueryRowContext(ctx, `SELECT status FROM reminders WHERE id = ?`, id).Scan(&stored); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stored != winner {
|
||||
t.Fatalf("iteration %d: stored %q, successful transition %q", iteration, stored, winner)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAmbiguousCollapsedDeliveryIsNotAutomaticallyRepeated(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
|
||||
for _, text := range []string{"полить цветы", "позвонить врачу"} {
|
||||
if _, err := s.CreateReminder(ctx, now.Add(-time.Minute), text, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
originals, err := s.DueReminders(ctx, now)
|
||||
if err != nil || len(originals) != 2 {
|
||||
t.Fatalf("due originals = %d, err=%v", len(originals), err)
|
||||
}
|
||||
const group = "reminder:ambiguous-bundle"
|
||||
if err := s.CacheReminderPhrase(ctx, originals, group, "У тебя два напоминания.", "Два напоминания", "neutral"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := range originals {
|
||||
originals[i].DeliveryGroup = group
|
||||
}
|
||||
|
||||
// Simulate a process dying after Begin and before it can record whether the
|
||||
// external sink accepted the bundle. Synthetic reminder zero is never put
|
||||
// in the outbox: a real original is the human-readable representative and
|
||||
// the group is the occurrence identity shared by both originals.
|
||||
attemptID, err := s.BeginDeliveryAttempt(ctx, "reminder", "", originals[0].ID, group, "telegram", "hash", now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.ReconcileStaleDeliveryAttempts(ctx, now.Add(time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if due, err := s.DueReminders(ctx, now.Add(24*time.Hour)); err != nil || len(due) != 0 {
|
||||
t.Fatalf("ambiguous delivered bundle was eligible for automatic repeat: due=%+v err=%v", due, err)
|
||||
}
|
||||
attempts, err := s.ListDeliveryAttempts(ctx, "", 10)
|
||||
if err != nil || len(attempts) != 1 {
|
||||
t.Fatalf("attempts=%+v err=%v", attempts, err)
|
||||
}
|
||||
if attempts[0].ID != attemptID || attempts[0].Status != DeliveryUnknown || attempts[0].DeliveryGroup != group {
|
||||
t.Fatalf("ambiguous occurrence lost its durable identity: %+v", attempts[0])
|
||||
}
|
||||
if err := s.ScheduleReminderRetry(ctx, originals, now.Add(time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rows, err := s.ListReminders(ctx, 10)
|
||||
if err != nil || len(rows) != 2 {
|
||||
t.Fatalf("rows=%+v err=%v", rows, err)
|
||||
}
|
||||
for _, r := range rows {
|
||||
if r.DeliveryAttempts != 0 || !r.NextAttemptTs.IsZero() {
|
||||
t.Fatalf("ambiguous occurrence was scheduled for automatic retry: %+v", r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefiniteFailedDeliveryRemainsRetryEligible(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
|
||||
if _, err := s.CreateReminder(ctx, now.Add(-time.Minute), "позвонить врачу", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
originals, err := s.DueReminders(ctx, now)
|
||||
if err != nil || len(originals) != 1 {
|
||||
t.Fatalf("due originals = %d, err=%v", len(originals), err)
|
||||
}
|
||||
const group = "reminder:definite-failure"
|
||||
if err := s.CacheReminderPhrase(ctx, originals, group, "Позвони врачу.", "Позвони врачу", "neutral"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id, err := s.BeginDeliveryAttempt(ctx, "reminder", "", originals[0].ID, group, "telegram", "hash", now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.CompleteDeliveryAttempt(ctx, id, DeliveryFailed, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if due, err := s.DueReminders(ctx, now); err != nil || len(due) != 1 {
|
||||
t.Fatalf("definite failure was not retry eligible: due=%+v err=%v", due, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollapsedCompletionRollsBackAsOneTransaction(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
|
||||
for _, text := range []string{"first", "fail-second"} {
|
||||
if _, err := s.CreateReminder(ctx, now.Add(-time.Minute), text, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
originals, err := s.DueReminders(ctx, now)
|
||||
if err != nil || len(originals) != 2 {
|
||||
t.Fatalf("due originals = %d, err=%v", len(originals), err)
|
||||
}
|
||||
if _, err := s.db.ExecContext(ctx, `
|
||||
CREATE TRIGGER fail_second_reminder_completion
|
||||
BEFORE UPDATE OF status ON reminders
|
||||
WHEN OLD.payload = 'fail-second'
|
||||
BEGIN
|
||||
SELECT RAISE(ABORT, 'injected second completion failure');
|
||||
END`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.CompleteReminderDelivery(ctx, originals, now); err == nil {
|
||||
t.Fatal("injected second-row failure did not fail the bundle completion")
|
||||
}
|
||||
rows, err := s.ListReminders(ctx, 10)
|
||||
if err != nil || len(rows) != 2 {
|
||||
t.Fatalf("rows=%+v err=%v", rows, err)
|
||||
}
|
||||
for _, r := range rows {
|
||||
if r.Status != ReminderPending {
|
||||
t.Fatalf("bundle completion partially committed reminder %d as %q", r.ID, r.Status)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSuccessfulAttemptAndCollapsedCompletionCommitAtomically(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
|
||||
for _, text := range []string{"first", "fail-second"} {
|
||||
if _, err := s.CreateReminder(ctx, now.Add(-time.Minute), text, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
originals, err := s.DueReminders(ctx, now)
|
||||
if err != nil || len(originals) != 2 {
|
||||
t.Fatalf("due originals = %d, err=%v", len(originals), err)
|
||||
}
|
||||
const group = "reminder:atomic-success"
|
||||
if err := s.CacheReminderPhrase(ctx, originals, group, "Two reminders.", "Two reminders", "neutral"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := range originals {
|
||||
originals[i].DeliveryGroup = group
|
||||
}
|
||||
attemptID, err := s.BeginDeliveryAttempt(ctx, "reminder", "", originals[0].ID, group, "telegram", "hash", now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.db.ExecContext(ctx, `
|
||||
CREATE TRIGGER fail_atomic_second_completion
|
||||
BEFORE UPDATE OF status ON reminders
|
||||
WHEN OLD.payload = 'fail-second'
|
||||
BEGIN
|
||||
SELECT RAISE(ABORT, 'injected second completion failure');
|
||||
END`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.CompleteSuccessfulReminderAttempt(ctx, attemptID, originals, now); err == nil {
|
||||
t.Fatal("injected reminder completion failure did not fail the local commit")
|
||||
}
|
||||
attempts, err := s.ListDeliveryAttempts(ctx, "", 10)
|
||||
if err != nil || len(attempts) != 1 || attempts[0].Status != DeliveryPending {
|
||||
t.Fatalf("successful outbox marker committed without reminder state: attempts=%+v err=%v", attempts, err)
|
||||
}
|
||||
rows, err := s.ListReminders(ctx, 10)
|
||||
if err != nil || len(rows) != 2 {
|
||||
t.Fatalf("rows=%+v err=%v", rows, err)
|
||||
}
|
||||
for _, r := range rows {
|
||||
if r.Status != ReminderPending {
|
||||
t.Fatalf("atomic rollback partially completed reminder %d as %q", r.ID, r.Status)
|
||||
}
|
||||
}
|
||||
if _, err := s.ReconcileStaleDeliveryAttempts(ctx, now.Add(time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if due, err := s.DueReminders(ctx, now.Add(24*time.Hour)); err != nil || len(due) != 0 {
|
||||
t.Fatalf("restart auto-replayed ambiguous accepted send: due=%+v err=%v", due, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPermanentReminderFailureBlocksUntilDeliberatelyUnblocked(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
|
||||
id, err := s.CreateReminder(ctx, now.Add(-time.Minute), "позвонить врачу", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
originals, err := s.DueReminders(ctx, now)
|
||||
if err != nil || len(originals) != 1 {
|
||||
t.Fatalf("due originals=%+v err=%v", originals, err)
|
||||
}
|
||||
const group = "reminder:revoked-token"
|
||||
if err := s.CacheReminderPhrase(ctx, originals, group, "Позвони врачу.", "Позвони врачу", "neutral"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
originals[0].DeliveryGroup = group
|
||||
if err := s.BlockReminderDelivery(ctx, originals, now, "ntfy credentials rejected"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.ScheduleReminderRetry(ctx, originals, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if due, err := s.DueReminders(ctx, now.Add(24*time.Hour)); err != nil || len(due) != 0 {
|
||||
t.Fatalf("blocked reminder retried automatically: due=%+v err=%v", due, err)
|
||||
}
|
||||
rows, err := s.ListReminders(ctx, 1)
|
||||
if err != nil || len(rows) != 1 {
|
||||
t.Fatalf("rows=%+v err=%v", rows, err)
|
||||
}
|
||||
if rows[0].DeliveryBlockedTs.IsZero() || rows[0].DeliveryBlockedError != "ntfy credentials rejected" ||
|
||||
rows[0].DeliveryAttempts != 0 || !rows[0].NextAttemptTs.IsZero() {
|
||||
t.Fatalf("blocked state is not visible/durable: %+v", rows[0])
|
||||
}
|
||||
if err := s.UnblockReminderDelivery(ctx, id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
due, err := s.DueReminders(ctx, now)
|
||||
if err != nil || len(due) != 1 || due[0].ID != id || !due[0].DeliveryBlockedTs.IsZero() || due[0].DeliveryBlockedError != "" {
|
||||
t.Fatalf("deliberate unblock did not restore eligibility: due=%+v err=%v", due, err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user