Make delivery and integration failures explicit

Persist reminder presentations and retry state, atomically complete collapsed deliveries, fall back across away reaches, and block permanent failures visibly (V-715, V-678). Fail closed when enabled integrations lack credentials and keep remote arms explicitly dark (V-691). Give mavweb one sanitized, request-correlated error contract (V-689). Owner explicitly requested direct commits to master.
This commit is contained in:
2026-08-13 02:50:59 +04:00
parent da9114b623
commit 35c6ff5a71
67 changed files with 3174 additions and 477 deletions
+18 -15
View File
@@ -28,14 +28,16 @@ const (
// send happens, so a crash between "sent externally" and "recorded" leaves a
// trace instead of silence. kind is "nudge" or "reminder"; rule is set for
// nudges, reminderID for reminders (the other left at its zero value).
// deliveryGroup is the exact persisted reminder occurrence or collapsed
// bundle; it is empty for nudges and legacy reminder attempts.
// bodyHash is an opaque caller-computed key (e.g. sha256 of channel+body) —
// stored for post-crash operator triage, not enforced as a uniqueness
// constraint (a rule/reminder legitimately re-sends across ticks).
func (s *Store) BeginDeliveryAttempt(ctx context.Context, kind, rule string, reminderID int64, channel, bodyHash string, now time.Time) (int64, error) {
func (s *Store) BeginDeliveryAttempt(ctx context.Context, kind, rule string, reminderID int64, deliveryGroup, channel, bodyHash string, now time.Time) (int64, error) {
res, err := s.db.ExecContext(ctx,
`INSERT INTO delivery_attempts (kind, rule, reminder_id, channel, body_hash, status, created_ts)
VALUES (?, ?, ?, ?, ?, 'pending', ?)`,
kind, rule, reminderID, channel, bodyHash, now.UnixMilli())
`INSERT INTO delivery_attempts (kind, rule, reminder_id, delivery_group, channel, body_hash, status, created_ts)
VALUES (?, ?, ?, ?, ?, ?, 'pending', ?)`,
kind, rule, reminderID, deliveryGroup, channel, bodyHash, now.UnixMilli())
if err != nil {
return 0, fmt.Errorf("begin delivery attempt: %w", err)
}
@@ -90,15 +92,16 @@ func (s *Store) ReconcileStaleDeliveryAttempts(ctx context.Context, now time.Tim
// DeliveryAttempt — one row of the outbox, as a reader sees it.
type DeliveryAttempt struct {
ID int64
Kind string // nudge|reminder
Rule string // set for nudges
ReminderID int64 // set for reminders
Channel string
Status string // one of the Delivery* constants
Created time.Time
Completed time.Time // zero while pending
HasComplete bool
ID int64
Kind string // nudge|reminder
Rule string // set for nudges
ReminderID int64 // set for reminders
DeliveryGroup string // exact reminder occurrence/bundle; empty for nudges and legacy rows
Channel string
Status string // one of the Delivery* constants
Created time.Time
Completed time.Time // zero while pending
HasComplete bool
}
// ListDeliveryAttempts returns recent attempts, newest first. An empty status
@@ -117,7 +120,7 @@ func (s *Store) ListDeliveryAttempts(ctx context.Context, status string, limit i
if limit <= 0 {
limit = 50
}
q := `SELECT id, kind, rule, reminder_id, channel, status, created_ts, completed_ts
q := `SELECT id, kind, rule, reminder_id, delivery_group, channel, status, created_ts, completed_ts
FROM delivery_attempts`
args := []any{}
if status != "" {
@@ -138,7 +141,7 @@ func (s *Store) ListDeliveryAttempts(ctx context.Context, status string, limit i
var a DeliveryAttempt
var created int64
var completed *int64
if err := rows.Scan(&a.ID, &a.Kind, &a.Rule, &a.ReminderID, &a.Channel, &a.Status, &created, &completed); err != nil {
if err := rows.Scan(&a.ID, &a.Kind, &a.Rule, &a.ReminderID, &a.DeliveryGroup, &a.Channel, &a.Status, &created, &completed); err != nil {
return nil, fmt.Errorf("list delivery attempts: scan: %w", err)
}
a.Created = time.UnixMilli(created)
+4 -4
View File
@@ -15,7 +15,7 @@ func TestDroppedDeliveryAttemptRoundTrips(t *testing.T) {
ctx := context.Background()
now := time.Now()
id, err := s.BeginDeliveryAttempt(ctx, "nudge", "water", 0, "drop", "abc123", now)
id, err := s.BeginDeliveryAttempt(ctx, "nudge", "water", 0, "", "drop", "abc123", now)
if err != nil {
t.Fatalf("BeginDeliveryAttempt: %v", err)
}
@@ -40,21 +40,21 @@ func TestListDeliveryAttempts(t *testing.T) {
s := newTestStore(t)
base := time.Date(2026, 8, 4, 9, 0, 0, 0, time.UTC)
sent, err := s.BeginDeliveryAttempt(ctx, "nudge", "water", 0, "telegram", "h1", base)
sent, err := s.BeginDeliveryAttempt(ctx, "nudge", "water", 0, "", "telegram", "h1", base)
if err != nil {
t.Fatal(err)
}
if err := s.CompleteDeliveryAttempt(ctx, sent, DeliverySent, base.Add(time.Second)); err != nil {
t.Fatal(err)
}
dropped, err := s.BeginDeliveryAttempt(ctx, "nudge", "care", 0, "telegram", "h2", base.Add(time.Minute))
dropped, err := s.BeginDeliveryAttempt(ctx, "nudge", "care", 0, "", "telegram", "h2", base.Add(time.Minute))
if err != nil {
t.Fatal(err)
}
if err := s.CompleteDeliveryAttempt(ctx, dropped, DeliveryDropped, base.Add(time.Minute)); err != nil {
t.Fatal(err)
}
if _, err := s.BeginDeliveryAttempt(ctx, "reminder", "", 7, "voice", "h3", base.Add(2*time.Minute)); err != nil {
if _, err := s.BeginDeliveryAttempt(ctx, "reminder", "", 7, "reminder:test", "voice", "h3", base.Add(2*time.Minute)); err != nil {
t.Fatal(err)
}
+23
View File
@@ -351,6 +351,29 @@ ALTER TABLE reminders ADD COLUMN next_fire_ts INTEGER;`, // #2
source TEXT NOT NULL DEFAULT '',
encoder_id TEXT NOT NULL DEFAULT ''
);`,
// #25 — durable reminder delivery state (V-651). A reminder can remain
// pending for a long time when every reachable transport is unhealthy. The
// phrased text belongs to that delivery occurrence, so retaining it here
// avoids spending the resident model again on every retry and across daemon
// restarts. next_attempt_ts makes transport failures wait on a bounded
// exponential backoff instead of retrying at the tick rate. delivery_group
// keeps the originals of a collapsed catch-up bundle together; a reminder
// that becomes due later must not make the old bundle get re-phrased.
`ALTER TABLE reminders ADD COLUMN delivery_group TEXT NOT NULL DEFAULT '';
ALTER TABLE reminders ADD COLUMN phrase_body TEXT NOT NULL DEFAULT '';
ALTER TABLE reminders ADD COLUMN phrase_summary TEXT NOT NULL DEFAULT '';
ALTER TABLE reminders ADD COLUMN phrase_mood TEXT NOT NULL DEFAULT '';
ALTER TABLE reminders ADD COLUMN delivery_attempts INTEGER NOT NULL DEFAULT 0 CHECK (delivery_attempts >= 0);
ALTER TABLE reminders ADD COLUMN next_attempt_ts INTEGER;
ALTER TABLE delivery_attempts ADD COLUMN delivery_group TEXT NOT NULL DEFAULT '';
ALTER TABLE reminders ADD COLUMN delivery_blocked_ts INTEGER;
ALTER TABLE reminders ADD COLUMN delivery_blocked_error TEXT NOT NULL DEFAULT '';
CREATE INDEX IF NOT EXISTS idx_reminders_due
ON reminders (next_fire_ts, next_attempt_ts)
WHERE status = 'pending';
CREATE INDEX IF NOT EXISTS idx_delivery_attempts_reminder_group
ON delivery_attempts (delivery_group, status)
WHERE kind = 'reminder' AND delivery_group <> '';`,
}
// migrate applies every migration with a number greater than the DB's current
+453 -23
View File
@@ -22,6 +22,20 @@ type Reminder struct {
Status string // pending | fired | cancelled
Cron string // cron expression, empty for one-shot
// DeliveryGroup and Phrase* are the durable presentation for this exact
// occurrence. A collapsed catch-up bundle stores the same group and phrase
// on every original reminder, so a retry (including after restart) says the
// same thing without asking the model again. Rescheduling a recurring
// reminder clears them for its next occurrence.
DeliveryGroup string
PhraseBody string
PhraseSummary string
PhraseMood string
DeliveryAttempts int
NextAttemptTs time.Time
DeliveryBlockedTs time.Time
DeliveryBlockedError string
// Collapsed — set only on a synthetic digest reminder (ID=0): the original
// due reminders it stands in for. Not persisted. The dispatcher completes
// (marks fired / reschedules) each of these after the digest delivers.
@@ -62,18 +76,35 @@ const (
ReminderPending = "pending"
ReminderFired = "fired"
ReminderCancelled = "cancelled"
// ReminderRetryBase and ReminderRetryMax bound the retry cadence. Attempts
// continue indefinitely because a reminder must not disappear during a
// long transport outage; only the delay stops growing.
ReminderRetryBase = time.Minute
ReminderRetryMax = time.Hour
)
var (
ErrReminderNotFound = errors.New("store: reminder not found")
ErrReminderState = errors.New("store: reminder not in a mutable state")
ErrReminderPhrase = errors.New("store: reminder delivery phrase invalid")
)
const reminderColumns = `id, created_ts, fire_ts, next_fire_ts, payload, status, cron,
delivery_group, phrase_body, phrase_summary, phrase_mood, delivery_attempts, next_attempt_ts,
delivery_blocked_ts, delivery_blocked_error`
func scanReminder(sc scanner) (Reminder, error) {
var r Reminder
var created, fire, nextFire int64
var cron *string
if err := sc.Scan(&r.ID, &created, &fire, &nextFire, &r.Payload, &r.Status, &cron); err != nil {
var nextAttempt *int64
var blocked *int64
if err := sc.Scan(
&r.ID, &created, &fire, &nextFire, &r.Payload, &r.Status, &cron,
&r.DeliveryGroup, &r.PhraseBody, &r.PhraseSummary, &r.PhraseMood,
&r.DeliveryAttempts, &nextAttempt, &blocked, &r.DeliveryBlockedError,
); err != nil {
return Reminder{}, err
}
r.CreatedTs = time.UnixMilli(created).UTC()
@@ -82,6 +113,12 @@ func scanReminder(sc scanner) (Reminder, error) {
if cron != nil {
r.Cron = *cron
}
if nextAttempt != nil {
r.NextAttemptTs = time.UnixMilli(*nextAttempt).UTC()
}
if blocked != nil {
r.DeliveryBlockedTs = time.UnixMilli(*blocked).UTC()
}
return r, nil
}
@@ -111,11 +148,20 @@ func (s *Store) CreateReminder(ctx context.Context, fire time.Time, payload, cro
// DueReminders returns pending reminders with next_fire_ts <= now, oldest first.
// This is the predicate input from the loop side: `next_fire_ts <= now AND status='pending'`.
func (s *Store) DueReminders(ctx context.Context, now time.Time) ([]Reminder, error) {
rows, err := s.db.QueryContext(ctx, `
SELECT id, created_ts, fire_ts, next_fire_ts, payload, status, cron
rows, err := s.db.QueryContext(ctx, `SELECT `+reminderColumns+`
FROM reminders
WHERE status = 'pending' AND next_fire_ts <= ?
ORDER BY next_fire_ts ASC`, now.UnixMilli())
AND (next_attempt_ts IS NULL OR next_attempt_ts <= ?)
AND delivery_blocked_ts IS NULL
AND NOT EXISTS (
SELECT 1
FROM delivery_attempts AS attempt
WHERE attempt.kind = 'reminder'
AND attempt.delivery_group = reminders.delivery_group
AND reminders.delivery_group <> ''
AND attempt.status IN ('pending', 'sent', 'unknown')
)
ORDER BY next_fire_ts ASC, id ASC`, now.UnixMilli(), now.UnixMilli())
if err != nil {
return nil, fmt.Errorf("due reminders: %w", err)
}
@@ -140,8 +186,7 @@ func (s *Store) DueReminders(ctx context.Context, now time.Time) ([]Reminder, er
// for next year stayed on it. Bounding by fire time drops what is out of range
// instead of what is old.
func (s *Store) PendingReminders(ctx context.Context, from, to time.Time) ([]Reminder, error) {
rows, err := s.db.QueryContext(ctx, `
SELECT id, created_ts, fire_ts, next_fire_ts, payload, status, cron
rows, err := s.db.QueryContext(ctx, `SELECT `+reminderColumns+`
FROM reminders
WHERE status = ? AND next_fire_ts >= ? AND next_fire_ts < ?
ORDER BY next_fire_ts ASC, id ASC`,
@@ -164,29 +209,42 @@ func (s *Store) PendingReminders(ctx context.Context, from, to time.Time) ([]Rem
// MarkReminder sets a reminder's status. Only valid transitions: pending→fired,
// pending→cancelled. Anything else is a programming error.
func (s *Store) MarkReminder(ctx context.Context, id int64, status string) error {
if status != "fired" && status != "cancelled" {
if status != ReminderFired && status != ReminderCancelled {
return fmt.Errorf("%w: %s", ErrReminderState, status)
}
// pending → fired|cancelled only.
// The old read-then-write transition allowed two callers to both observe
// pending and both report success. Keeping the source state in the UPDATE
// predicate makes pending → fired|cancelled one atomic contest (V-678).
res, err := s.db.ExecContext(ctx,
"UPDATE reminders SET status = ? WHERE id = ? AND status = ?",
status, id, ReminderPending)
if err != nil {
return err
}
n, err := res.RowsAffected()
if err != nil {
return fmt.Errorf("mark reminder: rows affected: %w", err)
}
if n == 1 {
return nil
}
// Preserve the public distinction between a missing id and a completed
// state without weakening the atomic transition above.
var current string
err := s.db.QueryRowContext(ctx, "SELECT status FROM reminders WHERE id = ?", id).Scan(&current)
err = s.db.QueryRowContext(ctx, "SELECT status FROM reminders WHERE id = ?", id).Scan(&current)
if errors.Is(err, sql.ErrNoRows) {
return ErrReminderNotFound
}
if err != nil {
return err
}
if current != "pending" {
return fmt.Errorf("%w: currently %s", ErrReminderState, current)
}
_, err = s.db.ExecContext(ctx, "UPDATE reminders SET status = ? WHERE id = ?", status, id)
return err
return fmt.Errorf("%w: currently %s", ErrReminderState, current)
}
// ListReminders returns the n most recent reminders, newest first.
func (s *Store) ListReminders(ctx context.Context, n int) ([]Reminder, error) {
rows, err := s.db.QueryContext(ctx, `
SELECT id, created_ts, fire_ts, next_fire_ts, payload, status, cron
rows, err := s.db.QueryContext(ctx, `SELECT `+reminderColumns+`
FROM reminders ORDER BY created_ts DESC, id DESC LIMIT ?`, n)
if err != nil {
return nil, fmt.Errorf("list reminders: %w", err)
@@ -203,6 +261,352 @@ func (s *Store) ListReminders(ctx context.Context, n int) ([]Reminder, error) {
return out, rows.Err()
}
// HasDeliveryPhrase reports whether this occurrence already has a durable
// presentation. Summary may intentionally be empty (the delivery boundary has
// a generic privacy-preserving fallback), so Body is the readiness marker.
func (r Reminder) HasDeliveryPhrase() bool {
return r.DeliveryGroup != "" && r.PhraseBody != ""
}
// ReminderRetryDelay returns the delay after attempt (one-based). It grows
// exponentially from one minute and stays at one hour; retries do not stop.
func ReminderRetryDelay(attempt int) time.Duration {
if attempt <= 1 {
return ReminderRetryBase
}
delay := ReminderRetryBase
for i := 1; i < attempt && delay < ReminderRetryMax; i++ {
if delay >= ReminderRetryMax/2 {
return ReminderRetryMax
}
delay *= 2
}
if delay > ReminderRetryMax {
return ReminderRetryMax
}
return delay
}
// CacheReminderPhrase stores one presentation on every original represented
// by a reminder delivery. For a collapsed bundle, originals contains every
// row in Reminder.Collapsed and group is shared across all of them.
//
// The occurrence timestamp and empty-body predicates keep this from attaching
// an old phrase to a recurring reminder's next occurrence or overwriting a
// phrase another delivery already claimed. The transaction prevents a partial
// bundle cache: after a crash either every original can reconstruct the bundle
// or none can.
func (s *Store) CacheReminderPhrase(
ctx context.Context,
originals []Reminder,
group, body, summary, mood string,
) error {
if len(originals) == 0 || group == "" || body == "" {
return ErrReminderPhrase
}
if mood == "" {
mood = "neutral"
}
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return fmt.Errorf("cache reminder phrase: begin: %w", err)
}
defer tx.Rollback()
seen := make(map[int64]struct{}, len(originals))
for _, r := range originals {
if r.ID <= 0 || r.NextFireTs.IsZero() {
return ErrReminderPhrase
}
if _, ok := seen[r.ID]; ok {
return ErrReminderPhrase
}
seen[r.ID] = struct{}{}
res, err := tx.ExecContext(ctx, `
UPDATE reminders
SET delivery_group = ?, phrase_body = ?, phrase_summary = ?, phrase_mood = ?
WHERE id = ? AND status = ? AND next_fire_ts = ? AND phrase_body = ''`,
group, body, summary, mood, r.ID, ReminderPending, r.NextFireTs.UnixMilli())
if err != nil {
return fmt.Errorf("cache reminder phrase %d: %w", r.ID, err)
}
n, err := res.RowsAffected()
if err != nil {
return fmt.Errorf("cache reminder phrase %d: rows affected: %w", r.ID, err)
}
if n != 1 {
return fmt.Errorf("%w: reminder %d is no longer an unphrased pending occurrence", ErrReminderState, r.ID)
}
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("cache reminder phrase: commit: %w", err)
}
return nil
}
// ScheduleReminderRetry moves every still-pending original into a retry wait.
// Terminal rows are skipped: cancellation winning while a send was in flight
// must not be resurrected. A recurring row whose occurrence changed is skipped
// for the same reason. All remaining originals get their own persisted attempt
// count; normally a collapsed bundle keeps those counts in lockstep.
func (s *Store) ScheduleReminderRetry(ctx context.Context, originals []Reminder, now time.Time) error {
if len(originals) == 0 {
return ErrReminderNotFound
}
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return fmt.Errorf("schedule reminder retry: begin: %w", err)
}
defer tx.Rollback()
seen := make(map[int64]struct{}, len(originals))
for _, expected := range originals {
if expected.ID <= 0 || expected.NextFireTs.IsZero() {
return ErrReminderNotFound
}
if _, ok := seen[expected.ID]; ok {
continue
}
seen[expected.ID] = struct{}{}
var status string
var nextFire int64
var attempts int
var blocked *int64
err := tx.QueryRowContext(ctx,
`SELECT status, next_fire_ts, delivery_attempts, delivery_blocked_ts FROM reminders WHERE id = ?`,
expected.ID).Scan(&status, &nextFire, &attempts, &blocked)
if errors.Is(err, sql.ErrNoRows) {
return ErrReminderNotFound
}
if err != nil {
return fmt.Errorf("schedule reminder retry %d: read: %w", expected.ID, err)
}
if status != ReminderPending || nextFire != expected.NextFireTs.UnixMilli() || blocked != nil {
continue
}
if expected.DeliveryGroup != "" {
var ambiguous int
if err := tx.QueryRowContext(ctx, `
SELECT EXISTS (
SELECT 1 FROM delivery_attempts
WHERE kind = 'reminder' AND delivery_group = ?
AND status IN ('pending', 'sent', 'unknown')
)`, expected.DeliveryGroup).Scan(&ambiguous); err != nil {
return fmt.Errorf("schedule reminder retry %d: inspect outbox: %w", expected.ID, err)
}
if ambiguous != 0 {
// The sink may have accepted this presentation. Leave the reminder
// pending but ineligible for automatic retry; operator resolution
// must not be replaced with a guessed duplicate.
continue
}
}
attempts++
nextAttempt := now.Add(ReminderRetryDelay(attempts)).UnixMilli()
res, err := tx.ExecContext(ctx, `
UPDATE reminders
SET delivery_attempts = ?, next_attempt_ts = ?
WHERE id = ? AND status = ? AND next_fire_ts = ? AND delivery_attempts = ?`,
attempts, nextAttempt, expected.ID, ReminderPending, nextFire, attempts-1)
if err != nil {
return fmt.Errorf("schedule reminder retry %d: %w", expected.ID, err)
}
n, err := res.RowsAffected()
if err != nil {
return fmt.Errorf("schedule reminder retry %d: rows affected: %w", expected.ID, err)
}
if n != 1 {
return fmt.Errorf("%w: reminder %d changed concurrently", ErrReminderState, expected.ID)
}
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("schedule reminder retry: commit: %w", err)
}
return nil
}
// BlockReminderDelivery records a permanent transport/configuration refusal on
// every original represented by one presentation. Blocked rows stay pending and
// visible, but never retry automatically. UnblockReminderDelivery is the
// deliberate recovery path after credentials or policy are repaired.
func (s *Store) BlockReminderDelivery(ctx context.Context, originals []Reminder, now time.Time, reason string) error {
if len(originals) == 0 || strings.TrimSpace(reason) == "" {
return ErrReminderState
}
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return fmt.Errorf("block reminder delivery: begin: %w", err)
}
defer tx.Rollback()
seen := make(map[int64]struct{}, len(originals))
for _, expected := range originals {
if expected.ID <= 0 || expected.NextFireTs.IsZero() {
return ErrReminderNotFound
}
if _, ok := seen[expected.ID]; ok {
continue
}
seen[expected.ID] = struct{}{}
res, err := tx.ExecContext(ctx, `
UPDATE reminders
SET delivery_blocked_ts = ?, delivery_blocked_error = ?, next_attempt_ts = NULL
WHERE id = ? AND status = ? AND next_fire_ts = ?`,
now.UnixMilli(), reason, expected.ID, ReminderPending, expected.NextFireTs.UnixMilli())
if err != nil {
return fmt.Errorf("block reminder %d: %w", expected.ID, err)
}
if err := requireOneReminderRow(res, expected.ID); err != nil {
return err
}
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("block reminder delivery: commit: %w", err)
}
return nil
}
func (s *Store) UnblockReminderDelivery(ctx context.Context, id int64) error {
res, err := s.db.ExecContext(ctx, `
UPDATE reminders
SET delivery_blocked_ts = NULL, delivery_blocked_error = '', next_attempt_ts = NULL
WHERE id = ? AND status = ? AND delivery_blocked_ts IS NOT NULL`, id, ReminderPending)
if err != nil {
return fmt.Errorf("unblock reminder %d: %w", id, err)
}
return requireOneReminderRow(res, id)
}
// CompleteReminderDelivery applies the bookkeeping for every original covered
// by one successful external send in a single transaction. One-shot reminders
// become fired; recurring reminders advance to their next occurrence and shed
// the old presentation/retry state. The all-or-nothing boundary prevents a
// collapsed digest from becoming half-fired and then being repeated.
func (s *Store) CompleteReminderDelivery(ctx context.Context, originals []Reminder, now time.Time) error {
return s.completeReminderDeliveryIn(ctx, 0, originals, now, time.Local)
}
// CompleteSuccessfulReminderAttempt atomically closes a definitely successful
// external attempt and advances every reminder occurrence it represented. If
// this transaction cannot commit, the attempt remains pending; startup then
// reconciles it to unknown, and the occurrence is held from automatic replay.
func (s *Store) CompleteSuccessfulReminderAttempt(ctx context.Context, attemptID int64, originals []Reminder, now time.Time) error {
if attemptID <= 0 {
return fmt.Errorf("complete reminder delivery: invalid attempt id %d", attemptID)
}
return s.completeReminderDeliveryIn(ctx, attemptID, originals, now, time.Local)
}
func (s *Store) completeReminderDeliveryIn(ctx context.Context, attemptID int64, originals []Reminder, now time.Time, loc *time.Location) error {
if len(originals) == 0 {
return ErrReminderNotFound
}
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return fmt.Errorf("complete reminder delivery: begin: %w", err)
}
defer tx.Rollback()
if attemptID != 0 {
res, err := tx.ExecContext(ctx,
`UPDATE delivery_attempts SET status = ?, completed_ts = ? WHERE id = ? AND status = ?`,
DeliverySent, now.UnixMilli(), attemptID, DeliveryPending)
if err != nil {
return fmt.Errorf("complete reminder delivery attempt %d: %w", attemptID, err)
}
n, err := res.RowsAffected()
if err != nil {
return fmt.Errorf("complete reminder delivery attempt %d: rows affected: %w", attemptID, err)
}
if n != 1 {
return fmt.Errorf("complete reminder delivery attempt %d: not pending", attemptID)
}
}
seen := make(map[int64]struct{}, len(originals))
for _, expected := range originals {
if expected.ID <= 0 || expected.NextFireTs.IsZero() {
return ErrReminderNotFound
}
if _, ok := seen[expected.ID]; ok {
return fmt.Errorf("%w: duplicate reminder %d in one delivery", ErrReminderState, expected.ID)
}
seen[expected.ID] = struct{}{}
row := tx.QueryRowContext(ctx, `SELECT `+reminderColumns+` FROM reminders WHERE id = ?`, expected.ID)
current, err := scanReminder(row)
if errors.Is(err, sql.ErrNoRows) {
return ErrReminderNotFound
}
if err != nil {
return fmt.Errorf("complete reminder %d: read: %w", expected.ID, err)
}
if current.Status != ReminderPending || !current.NextFireTs.Equal(expected.NextFireTs) {
return fmt.Errorf("%w: reminder %d occurrence changed", ErrReminderState, expected.ID)
}
if expected.DeliveryGroup != "" && current.DeliveryGroup != expected.DeliveryGroup {
return fmt.Errorf("%w: reminder %d delivery group changed", ErrReminderState, expected.ID)
}
if current.Cron == "" {
if err := completeOneShotReminderTx(ctx, tx, current); err != nil {
return err
}
continue
}
next, err := nextReminderOccurrence(current, now, loc)
if err != nil {
return err
}
if next.IsZero() {
if err := completeOneShotReminderTx(ctx, tx, current); err != nil {
return err
}
continue
}
res, err := tx.ExecContext(ctx, `
UPDATE reminders
SET next_fire_ts = ?, delivery_group = '', phrase_body = '',
phrase_summary = '', phrase_mood = '', delivery_attempts = 0,
next_attempt_ts = NULL, delivery_blocked_ts = NULL,
delivery_blocked_error = ''
WHERE id = ? AND status = ? AND next_fire_ts = ?`,
next.UnixMilli(), current.ID, ReminderPending, current.NextFireTs.UnixMilli())
if err != nil {
return fmt.Errorf("complete recurring reminder %d: %w", current.ID, err)
}
if err := requireOneReminderRow(res, current.ID); err != nil {
return err
}
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("complete reminder delivery: commit: %w", err)
}
return nil
}
func completeOneShotReminderTx(ctx context.Context, tx *sql.Tx, r Reminder) error {
res, err := tx.ExecContext(ctx,
`UPDATE reminders SET status = ? WHERE id = ? AND status = ? AND next_fire_ts = ?`,
ReminderFired, r.ID, ReminderPending, r.NextFireTs.UnixMilli())
if err != nil {
return fmt.Errorf("complete reminder %d: %w", r.ID, err)
}
return requireOneReminderRow(res, r.ID)
}
func requireOneReminderRow(res sql.Result, id int64) error {
n, err := res.RowsAffected()
if err != nil {
return fmt.Errorf("complete reminder %d: rows affected: %w", id, err)
}
if n != 1 {
return fmt.Errorf("%w: reminder %d changed concurrently", ErrReminderState, id)
}
return nil
}
// RescheduleReminder computes the next fire time for a recurring reminder and
// updates next_fire_ts. Returns ErrReminderState if the reminder is not
// recurring or not pending. If the schedule yields no further fire time at all,
@@ -229,8 +633,7 @@ func (s *Store) RescheduleReminder(ctx context.Context, id int64, now time.Time)
}
func (s *Store) rescheduleReminderIn(ctx context.Context, id int64, now time.Time, loc *time.Location) error {
row := s.db.QueryRowContext(ctx, `
SELECT id, created_ts, fire_ts, next_fire_ts, payload, status, cron
row := s.db.QueryRowContext(ctx, `SELECT `+reminderColumns+`
FROM reminders WHERE id = ?`, id)
r, err := scanReminder(row)
if err != nil {
@@ -243,9 +646,38 @@ func (s *Store) rescheduleReminderIn(ctx context.Context, id int64, now time.Tim
return fmt.Errorf("%w: currently %s", ErrReminderState, r.Status)
}
next, err := nextReminderOccurrence(r, now, loc)
if err != nil {
return err
}
if next.IsZero() {
return s.MarkReminder(ctx, id, ReminderFired)
}
res, err := s.db.ExecContext(ctx, `
UPDATE reminders
SET next_fire_ts = ?, delivery_group = '', phrase_body = '',
phrase_summary = '', phrase_mood = '', delivery_attempts = 0,
next_attempt_ts = NULL, delivery_blocked_ts = NULL,
delivery_blocked_error = ''
WHERE id = ? AND status = ? AND next_fire_ts = ?`,
next.UnixMilli(), id, ReminderPending, r.NextFireTs.UnixMilli())
if err != nil {
return err
}
n, err := res.RowsAffected()
if err != nil {
return fmt.Errorf("reschedule reminder: rows affected: %w", err)
}
if n != 1 {
return fmt.Errorf("%w: reminder %d changed concurrently", ErrReminderState, id)
}
return nil
}
func nextReminderOccurrence(r Reminder, now time.Time, loc *time.Location) (time.Time, error) {
sched, err := cron.ParseStandard(r.Cron)
if err != nil {
return fmt.Errorf("parse cron %q: %w", r.Cron, err)
return time.Time{}, fmt.Errorf("parse cron %q: %w", r.Cron, err)
}
// Next is strictly after the time it is given, so the last fire cannot be
// returned again and no fudge minute is needed. The bound stops a schedule
@@ -255,9 +687,7 @@ func (s *Store) rescheduleReminderIn(ctx context.Context, id int64, now time.Tim
next = sched.Next(next)
}
if next.IsZero() || !next.After(now) {
_, err = s.db.ExecContext(ctx, "UPDATE reminders SET status = 'fired' WHERE id = ?", id)
return err
return time.Time{}, nil
}
_, err = s.db.ExecContext(ctx, "UPDATE reminders SET next_fire_ts = ? WHERE id = ?", next.UnixMilli(), id)
return err
return next, nil
}
+417
View File
@@ -0,0 +1,417 @@
package store
import (
"context"
"errors"
"path/filepath"
"sync"
"testing"
"time"
)
func TestReminderPhraseAndRetrySurviveRestart(t *testing.T) {
ctx := context.Background()
path := filepath.Join(t.TempDir(), "reminders.db")
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
s, err := Open(ctx, path)
if err != nil {
t.Fatal(err)
}
id, err := s.CreateReminder(ctx, now.Add(-time.Minute), `{"text":"позвонить маме"}`, "")
if err != nil {
t.Fatal(err)
}
due, err := s.DueReminders(ctx, now)
if err != nil || len(due) != 1 {
t.Fatalf("initial due reminders = %d, err=%v", len(due), err)
}
if err := s.CacheReminderPhrase(ctx, due, "reminder:one", "Позвони маме.", "Позвони маме", "warm"); err != nil {
t.Fatalf("cache phrase: %v", err)
}
if err := s.ScheduleReminderRetry(ctx, due, now); err != nil {
t.Fatalf("schedule retry: %v", err)
}
if err := s.Close(); err != nil {
t.Fatal(err)
}
s, err = Open(ctx, path)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = s.Close() })
if got, err := s.DueReminders(ctx, now.Add(30*time.Second)); err != nil || len(got) != 0 {
t.Fatalf("during retry wait: got %d due, err=%v", len(got), err)
}
due, err = s.DueReminders(ctx, now.Add(ReminderRetryBase))
if err != nil || len(due) != 1 {
t.Fatalf("at retry time: got %d due, err=%v", len(due), err)
}
r := due[0]
if r.ID != id || r.DeliveryGroup != "reminder:one" || r.PhraseBody != "Позвони маме." ||
r.PhraseSummary != "Позвони маме" || r.PhraseMood != "warm" {
t.Fatalf("persisted delivery presentation = %+v", r)
}
if r.DeliveryAttempts != 1 {
t.Fatalf("attempts = %d, want 1", r.DeliveryAttempts)
}
if want := now.Add(ReminderRetryBase); !r.NextAttemptTs.Equal(want) {
t.Fatalf("next attempt = %s, want %s", r.NextAttemptTs, want)
}
}
func TestReminderRetryDelayIsExponentialAndBounded(t *testing.T) {
wants := []time.Duration{
time.Minute, 2 * time.Minute, 4 * time.Minute, 8 * time.Minute,
16 * time.Minute, 32 * time.Minute, time.Hour, time.Hour, time.Hour,
}
for i, want := range wants {
if got := ReminderRetryDelay(i + 1); got != want {
t.Errorf("attempt %d delay = %s, want %s", i+1, got, want)
}
}
}
func TestCollapsedReminderRetryStateMovesTogether(t *testing.T) {
s := newTestStore(t)
ctx := context.Background()
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
for _, text := range []string{"полить цветы", "позвонить врачу"} {
if _, err := s.CreateReminder(ctx, now.Add(-time.Minute), text, ""); err != nil {
t.Fatal(err)
}
}
originals, err := s.DueReminders(ctx, now)
if err != nil || len(originals) != 2 {
t.Fatalf("due originals = %d, err=%v", len(originals), err)
}
if err := s.CacheReminderPhrase(ctx, originals, "reminder:bundle", "У тебя два напоминания.", "Два напоминания", "neutral"); err != nil {
t.Fatal(err)
}
if err := s.ScheduleReminderRetry(ctx, originals, now); err != nil {
t.Fatal(err)
}
rows, err := s.ListReminders(ctx, 10)
if err != nil || len(rows) != 2 {
t.Fatalf("list = %d, err=%v", len(rows), err)
}
for _, r := range rows {
if r.DeliveryGroup != "reminder:bundle" || r.PhraseBody != "У тебя два напоминания." || r.DeliveryAttempts != 1 {
t.Errorf("bundle original did not move with group: %+v", r)
}
if !r.NextAttemptTs.Equal(now.Add(time.Minute)) {
t.Errorf("next attempt = %s, want %s", r.NextAttemptTs, now.Add(time.Minute))
}
}
}
func TestRecurringReminderClearsDeliveryStateForNextOccurrence(t *testing.T) {
s := newTestStore(t)
ctx := context.Background()
loc := time.FixedZone("MSK", 3*60*60)
fire := time.Date(2026, 8, 13, 9, 0, 0, 0, loc)
id, err := s.CreateReminder(ctx, fire, `{"text":"стендап"}`, "0 9 * * *")
if err != nil {
t.Fatal(err)
}
due, err := s.DueReminders(ctx, fire)
if err != nil || len(due) != 1 {
t.Fatalf("due = %d, err=%v", len(due), err)
}
if err := s.CacheReminderPhrase(ctx, due, "reminder:occurrence", "Пора на стендап.", "Стендап", "neutral"); err != nil {
t.Fatal(err)
}
if err := s.ScheduleReminderRetry(ctx, due, fire); err != nil {
t.Fatal(err)
}
if err := s.rescheduleReminderIn(ctx, id, fire.Add(time.Minute), loc); err != nil {
t.Fatalf("reschedule: %v", err)
}
rows, err := s.ListReminders(ctx, 1)
if err != nil || len(rows) != 1 {
t.Fatalf("list = %d, err=%v", len(rows), err)
}
r := rows[0]
if r.Status != ReminderPending || r.DeliveryGroup != "" || r.PhraseBody != "" ||
r.PhraseSummary != "" || r.PhraseMood != "" || r.DeliveryAttempts != 0 || !r.NextAttemptTs.IsZero() {
t.Fatalf("next occurrence retained old delivery state: %+v", r)
}
want := time.Date(2026, 8, 14, 9, 0, 0, 0, loc)
if !r.NextFireTs.Equal(want) {
t.Fatalf("next fire = %s, want %s", r.NextFireTs.In(loc), want)
}
}
func TestReminderTerminalTransitionHasExactlyOneWinner(t *testing.T) {
s := newTestStore(t)
ctx := context.Background()
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
for iteration := 0; iteration < 40; iteration++ {
id, err := s.CreateReminder(ctx, now, "race", "")
if err != nil {
t.Fatal(err)
}
start := make(chan struct{})
var wg sync.WaitGroup
errs := make([]error, 2)
statuses := []string{ReminderFired, ReminderCancelled}
for i := range statuses {
wg.Add(1)
go func(i int) {
defer wg.Done()
<-start
errs[i] = s.MarkReminder(ctx, id, statuses[i])
}(i)
}
close(start)
wg.Wait()
successes := 0
losers := 0
winner := ""
for i, err := range errs {
switch {
case err == nil:
successes++
winner = statuses[i]
case errors.Is(err, ErrReminderState):
losers++
default:
t.Fatalf("iteration %d transition %s: %v", iteration, statuses[i], err)
}
}
if successes != 1 || losers != 1 {
t.Fatalf("iteration %d: successes=%d losers=%d errors=%v", iteration, successes, losers, errs)
}
var stored string
if err := s.db.QueryRowContext(ctx, `SELECT status FROM reminders WHERE id = ?`, id).Scan(&stored); err != nil {
t.Fatal(err)
}
if stored != winner {
t.Fatalf("iteration %d: stored %q, successful transition %q", iteration, stored, winner)
}
}
}
func TestAmbiguousCollapsedDeliveryIsNotAutomaticallyRepeated(t *testing.T) {
s := newTestStore(t)
ctx := context.Background()
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
for _, text := range []string{"полить цветы", "позвонить врачу"} {
if _, err := s.CreateReminder(ctx, now.Add(-time.Minute), text, ""); err != nil {
t.Fatal(err)
}
}
originals, err := s.DueReminders(ctx, now)
if err != nil || len(originals) != 2 {
t.Fatalf("due originals = %d, err=%v", len(originals), err)
}
const group = "reminder:ambiguous-bundle"
if err := s.CacheReminderPhrase(ctx, originals, group, "У тебя два напоминания.", "Два напоминания", "neutral"); err != nil {
t.Fatal(err)
}
for i := range originals {
originals[i].DeliveryGroup = group
}
// Simulate a process dying after Begin and before it can record whether the
// external sink accepted the bundle. Synthetic reminder zero is never put
// in the outbox: a real original is the human-readable representative and
// the group is the occurrence identity shared by both originals.
attemptID, err := s.BeginDeliveryAttempt(ctx, "reminder", "", originals[0].ID, group, "telegram", "hash", now)
if err != nil {
t.Fatal(err)
}
if _, err := s.ReconcileStaleDeliveryAttempts(ctx, now.Add(time.Minute)); err != nil {
t.Fatal(err)
}
if due, err := s.DueReminders(ctx, now.Add(24*time.Hour)); err != nil || len(due) != 0 {
t.Fatalf("ambiguous delivered bundle was eligible for automatic repeat: due=%+v err=%v", due, err)
}
attempts, err := s.ListDeliveryAttempts(ctx, "", 10)
if err != nil || len(attempts) != 1 {
t.Fatalf("attempts=%+v err=%v", attempts, err)
}
if attempts[0].ID != attemptID || attempts[0].Status != DeliveryUnknown || attempts[0].DeliveryGroup != group {
t.Fatalf("ambiguous occurrence lost its durable identity: %+v", attempts[0])
}
if err := s.ScheduleReminderRetry(ctx, originals, now.Add(time.Minute)); err != nil {
t.Fatal(err)
}
rows, err := s.ListReminders(ctx, 10)
if err != nil || len(rows) != 2 {
t.Fatalf("rows=%+v err=%v", rows, err)
}
for _, r := range rows {
if r.DeliveryAttempts != 0 || !r.NextAttemptTs.IsZero() {
t.Fatalf("ambiguous occurrence was scheduled for automatic retry: %+v", r)
}
}
}
func TestDefiniteFailedDeliveryRemainsRetryEligible(t *testing.T) {
s := newTestStore(t)
ctx := context.Background()
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
if _, err := s.CreateReminder(ctx, now.Add(-time.Minute), "позвонить врачу", ""); err != nil {
t.Fatal(err)
}
originals, err := s.DueReminders(ctx, now)
if err != nil || len(originals) != 1 {
t.Fatalf("due originals = %d, err=%v", len(originals), err)
}
const group = "reminder:definite-failure"
if err := s.CacheReminderPhrase(ctx, originals, group, "Позвони врачу.", "Позвони врачу", "neutral"); err != nil {
t.Fatal(err)
}
id, err := s.BeginDeliveryAttempt(ctx, "reminder", "", originals[0].ID, group, "telegram", "hash", now)
if err != nil {
t.Fatal(err)
}
if err := s.CompleteDeliveryAttempt(ctx, id, DeliveryFailed, now); err != nil {
t.Fatal(err)
}
if due, err := s.DueReminders(ctx, now); err != nil || len(due) != 1 {
t.Fatalf("definite failure was not retry eligible: due=%+v err=%v", due, err)
}
}
func TestCollapsedCompletionRollsBackAsOneTransaction(t *testing.T) {
s := newTestStore(t)
ctx := context.Background()
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
for _, text := range []string{"first", "fail-second"} {
if _, err := s.CreateReminder(ctx, now.Add(-time.Minute), text, ""); err != nil {
t.Fatal(err)
}
}
originals, err := s.DueReminders(ctx, now)
if err != nil || len(originals) != 2 {
t.Fatalf("due originals = %d, err=%v", len(originals), err)
}
if _, err := s.db.ExecContext(ctx, `
CREATE TRIGGER fail_second_reminder_completion
BEFORE UPDATE OF status ON reminders
WHEN OLD.payload = 'fail-second'
BEGIN
SELECT RAISE(ABORT, 'injected second completion failure');
END`); err != nil {
t.Fatal(err)
}
if err := s.CompleteReminderDelivery(ctx, originals, now); err == nil {
t.Fatal("injected second-row failure did not fail the bundle completion")
}
rows, err := s.ListReminders(ctx, 10)
if err != nil || len(rows) != 2 {
t.Fatalf("rows=%+v err=%v", rows, err)
}
for _, r := range rows {
if r.Status != ReminderPending {
t.Fatalf("bundle completion partially committed reminder %d as %q", r.ID, r.Status)
}
}
}
func TestSuccessfulAttemptAndCollapsedCompletionCommitAtomically(t *testing.T) {
s := newTestStore(t)
ctx := context.Background()
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
for _, text := range []string{"first", "fail-second"} {
if _, err := s.CreateReminder(ctx, now.Add(-time.Minute), text, ""); err != nil {
t.Fatal(err)
}
}
originals, err := s.DueReminders(ctx, now)
if err != nil || len(originals) != 2 {
t.Fatalf("due originals = %d, err=%v", len(originals), err)
}
const group = "reminder:atomic-success"
if err := s.CacheReminderPhrase(ctx, originals, group, "Two reminders.", "Two reminders", "neutral"); err != nil {
t.Fatal(err)
}
for i := range originals {
originals[i].DeliveryGroup = group
}
attemptID, err := s.BeginDeliveryAttempt(ctx, "reminder", "", originals[0].ID, group, "telegram", "hash", now)
if err != nil {
t.Fatal(err)
}
if _, err := s.db.ExecContext(ctx, `
CREATE TRIGGER fail_atomic_second_completion
BEFORE UPDATE OF status ON reminders
WHEN OLD.payload = 'fail-second'
BEGIN
SELECT RAISE(ABORT, 'injected second completion failure');
END`); err != nil {
t.Fatal(err)
}
if err := s.CompleteSuccessfulReminderAttempt(ctx, attemptID, originals, now); err == nil {
t.Fatal("injected reminder completion failure did not fail the local commit")
}
attempts, err := s.ListDeliveryAttempts(ctx, "", 10)
if err != nil || len(attempts) != 1 || attempts[0].Status != DeliveryPending {
t.Fatalf("successful outbox marker committed without reminder state: attempts=%+v err=%v", attempts, err)
}
rows, err := s.ListReminders(ctx, 10)
if err != nil || len(rows) != 2 {
t.Fatalf("rows=%+v err=%v", rows, err)
}
for _, r := range rows {
if r.Status != ReminderPending {
t.Fatalf("atomic rollback partially completed reminder %d as %q", r.ID, r.Status)
}
}
if _, err := s.ReconcileStaleDeliveryAttempts(ctx, now.Add(time.Minute)); err != nil {
t.Fatal(err)
}
if due, err := s.DueReminders(ctx, now.Add(24*time.Hour)); err != nil || len(due) != 0 {
t.Fatalf("restart auto-replayed ambiguous accepted send: due=%+v err=%v", due, err)
}
}
func TestPermanentReminderFailureBlocksUntilDeliberatelyUnblocked(t *testing.T) {
s := newTestStore(t)
ctx := context.Background()
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
id, err := s.CreateReminder(ctx, now.Add(-time.Minute), "позвонить врачу", "")
if err != nil {
t.Fatal(err)
}
originals, err := s.DueReminders(ctx, now)
if err != nil || len(originals) != 1 {
t.Fatalf("due originals=%+v err=%v", originals, err)
}
const group = "reminder:revoked-token"
if err := s.CacheReminderPhrase(ctx, originals, group, "Позвони врачу.", "Позвони врачу", "neutral"); err != nil {
t.Fatal(err)
}
originals[0].DeliveryGroup = group
if err := s.BlockReminderDelivery(ctx, originals, now, "ntfy credentials rejected"); err != nil {
t.Fatal(err)
}
if err := s.ScheduleReminderRetry(ctx, originals, now); err != nil {
t.Fatal(err)
}
if due, err := s.DueReminders(ctx, now.Add(24*time.Hour)); err != nil || len(due) != 0 {
t.Fatalf("blocked reminder retried automatically: due=%+v err=%v", due, err)
}
rows, err := s.ListReminders(ctx, 1)
if err != nil || len(rows) != 1 {
t.Fatalf("rows=%+v err=%v", rows, err)
}
if rows[0].DeliveryBlockedTs.IsZero() || rows[0].DeliveryBlockedError != "ntfy credentials rejected" ||
rows[0].DeliveryAttempts != 0 || !rows[0].NextAttemptTs.IsZero() {
t.Fatalf("blocked state is not visible/durable: %+v", rows[0])
}
if err := s.UnblockReminderDelivery(ctx, id); err != nil {
t.Fatal(err)
}
due, err := s.DueReminders(ctx, now)
if err != nil || len(due) != 1 || due[0].ID != id || !due[0].DeliveryBlockedTs.IsZero() || due[0].DeliveryBlockedError != "" {
t.Fatalf("deliberate unblock did not restore eligibility: due=%+v err=%v", due, err)
}
}