Make delivery and integration failures explicit

Persist reminder presentations and retry state, atomically complete collapsed deliveries, fall back across away reaches, and block permanent failures visibly (V-715, V-678). Fail closed when enabled integrations lack credentials and keep remote arms explicitly dark (V-691). Give mavweb one sanitized, request-correlated error contract (V-689). Owner explicitly requested direct commits to master.
This commit is contained in:
2026-08-13 02:50:59 +04:00
parent da9114b623
commit 35c6ff5a71
67 changed files with 3174 additions and 477 deletions
+453 -23
View File
@@ -22,6 +22,20 @@ type Reminder struct {
Status string // pending | fired | cancelled
Cron string // cron expression, empty for one-shot
// DeliveryGroup and Phrase* are the durable presentation for this exact
// occurrence. A collapsed catch-up bundle stores the same group and phrase
// on every original reminder, so a retry (including after restart) says the
// same thing without asking the model again. Rescheduling a recurring
// reminder clears them for its next occurrence.
DeliveryGroup string
PhraseBody string
PhraseSummary string
PhraseMood string
DeliveryAttempts int
NextAttemptTs time.Time
DeliveryBlockedTs time.Time
DeliveryBlockedError string
// Collapsed — set only on a synthetic digest reminder (ID=0): the original
// due reminders it stands in for. Not persisted. The dispatcher completes
// (marks fired / reschedules) each of these after the digest delivers.
@@ -62,18 +76,35 @@ const (
ReminderPending = "pending"
ReminderFired = "fired"
ReminderCancelled = "cancelled"
// ReminderRetryBase and ReminderRetryMax bound the retry cadence. Attempts
// continue indefinitely because a reminder must not disappear during a
// long transport outage; only the delay stops growing.
ReminderRetryBase = time.Minute
ReminderRetryMax = time.Hour
)
var (
ErrReminderNotFound = errors.New("store: reminder not found")
ErrReminderState = errors.New("store: reminder not in a mutable state")
ErrReminderPhrase = errors.New("store: reminder delivery phrase invalid")
)
const reminderColumns = `id, created_ts, fire_ts, next_fire_ts, payload, status, cron,
delivery_group, phrase_body, phrase_summary, phrase_mood, delivery_attempts, next_attempt_ts,
delivery_blocked_ts, delivery_blocked_error`
func scanReminder(sc scanner) (Reminder, error) {
var r Reminder
var created, fire, nextFire int64
var cron *string
if err := sc.Scan(&r.ID, &created, &fire, &nextFire, &r.Payload, &r.Status, &cron); err != nil {
var nextAttempt *int64
var blocked *int64
if err := sc.Scan(
&r.ID, &created, &fire, &nextFire, &r.Payload, &r.Status, &cron,
&r.DeliveryGroup, &r.PhraseBody, &r.PhraseSummary, &r.PhraseMood,
&r.DeliveryAttempts, &nextAttempt, &blocked, &r.DeliveryBlockedError,
); err != nil {
return Reminder{}, err
}
r.CreatedTs = time.UnixMilli(created).UTC()
@@ -82,6 +113,12 @@ func scanReminder(sc scanner) (Reminder, error) {
if cron != nil {
r.Cron = *cron
}
if nextAttempt != nil {
r.NextAttemptTs = time.UnixMilli(*nextAttempt).UTC()
}
if blocked != nil {
r.DeliveryBlockedTs = time.UnixMilli(*blocked).UTC()
}
return r, nil
}
@@ -111,11 +148,20 @@ func (s *Store) CreateReminder(ctx context.Context, fire time.Time, payload, cro
// DueReminders returns pending reminders with next_fire_ts <= now, oldest first.
// This is the predicate input from the loop side: `next_fire_ts <= now AND status='pending'`.
func (s *Store) DueReminders(ctx context.Context, now time.Time) ([]Reminder, error) {
rows, err := s.db.QueryContext(ctx, `
SELECT id, created_ts, fire_ts, next_fire_ts, payload, status, cron
rows, err := s.db.QueryContext(ctx, `SELECT `+reminderColumns+`
FROM reminders
WHERE status = 'pending' AND next_fire_ts <= ?
ORDER BY next_fire_ts ASC`, now.UnixMilli())
AND (next_attempt_ts IS NULL OR next_attempt_ts <= ?)
AND delivery_blocked_ts IS NULL
AND NOT EXISTS (
SELECT 1
FROM delivery_attempts AS attempt
WHERE attempt.kind = 'reminder'
AND attempt.delivery_group = reminders.delivery_group
AND reminders.delivery_group <> ''
AND attempt.status IN ('pending', 'sent', 'unknown')
)
ORDER BY next_fire_ts ASC, id ASC`, now.UnixMilli(), now.UnixMilli())
if err != nil {
return nil, fmt.Errorf("due reminders: %w", err)
}
@@ -140,8 +186,7 @@ func (s *Store) DueReminders(ctx context.Context, now time.Time) ([]Reminder, er
// for next year stayed on it. Bounding by fire time drops what is out of range
// instead of what is old.
func (s *Store) PendingReminders(ctx context.Context, from, to time.Time) ([]Reminder, error) {
rows, err := s.db.QueryContext(ctx, `
SELECT id, created_ts, fire_ts, next_fire_ts, payload, status, cron
rows, err := s.db.QueryContext(ctx, `SELECT `+reminderColumns+`
FROM reminders
WHERE status = ? AND next_fire_ts >= ? AND next_fire_ts < ?
ORDER BY next_fire_ts ASC, id ASC`,
@@ -164,29 +209,42 @@ func (s *Store) PendingReminders(ctx context.Context, from, to time.Time) ([]Rem
// MarkReminder sets a reminder's status. Only valid transitions: pending→fired,
// pending→cancelled. Anything else is a programming error.
func (s *Store) MarkReminder(ctx context.Context, id int64, status string) error {
if status != "fired" && status != "cancelled" {
if status != ReminderFired && status != ReminderCancelled {
return fmt.Errorf("%w: %s", ErrReminderState, status)
}
// pending → fired|cancelled only.
// The old read-then-write transition allowed two callers to both observe
// pending and both report success. Keeping the source state in the UPDATE
// predicate makes pending → fired|cancelled one atomic contest (V-678).
res, err := s.db.ExecContext(ctx,
"UPDATE reminders SET status = ? WHERE id = ? AND status = ?",
status, id, ReminderPending)
if err != nil {
return err
}
n, err := res.RowsAffected()
if err != nil {
return fmt.Errorf("mark reminder: rows affected: %w", err)
}
if n == 1 {
return nil
}
// Preserve the public distinction between a missing id and a completed
// state without weakening the atomic transition above.
var current string
err := s.db.QueryRowContext(ctx, "SELECT status FROM reminders WHERE id = ?", id).Scan(&current)
err = s.db.QueryRowContext(ctx, "SELECT status FROM reminders WHERE id = ?", id).Scan(&current)
if errors.Is(err, sql.ErrNoRows) {
return ErrReminderNotFound
}
if err != nil {
return err
}
if current != "pending" {
return fmt.Errorf("%w: currently %s", ErrReminderState, current)
}
_, err = s.db.ExecContext(ctx, "UPDATE reminders SET status = ? WHERE id = ?", status, id)
return err
return fmt.Errorf("%w: currently %s", ErrReminderState, current)
}
// ListReminders returns the n most recent reminders, newest first.
func (s *Store) ListReminders(ctx context.Context, n int) ([]Reminder, error) {
rows, err := s.db.QueryContext(ctx, `
SELECT id, created_ts, fire_ts, next_fire_ts, payload, status, cron
rows, err := s.db.QueryContext(ctx, `SELECT `+reminderColumns+`
FROM reminders ORDER BY created_ts DESC, id DESC LIMIT ?`, n)
if err != nil {
return nil, fmt.Errorf("list reminders: %w", err)
@@ -203,6 +261,352 @@ func (s *Store) ListReminders(ctx context.Context, n int) ([]Reminder, error) {
return out, rows.Err()
}
// HasDeliveryPhrase reports whether this occurrence already has a durable
// presentation. Summary may intentionally be empty (the delivery boundary has
// a generic privacy-preserving fallback), so Body is the readiness marker.
func (r Reminder) HasDeliveryPhrase() bool {
return r.DeliveryGroup != "" && r.PhraseBody != ""
}
// ReminderRetryDelay returns the delay after attempt (one-based). It grows
// exponentially from one minute and stays at one hour; retries do not stop.
func ReminderRetryDelay(attempt int) time.Duration {
if attempt <= 1 {
return ReminderRetryBase
}
delay := ReminderRetryBase
for i := 1; i < attempt && delay < ReminderRetryMax; i++ {
if delay >= ReminderRetryMax/2 {
return ReminderRetryMax
}
delay *= 2
}
if delay > ReminderRetryMax {
return ReminderRetryMax
}
return delay
}
// CacheReminderPhrase stores one presentation on every original represented
// by a reminder delivery. For a collapsed bundle, originals contains every
// row in Reminder.Collapsed and group is shared across all of them.
//
// The occurrence timestamp and empty-body predicates keep this from attaching
// an old phrase to a recurring reminder's next occurrence or overwriting a
// phrase another delivery already claimed. The transaction prevents a partial
// bundle cache: after a crash either every original can reconstruct the bundle
// or none can.
func (s *Store) CacheReminderPhrase(
ctx context.Context,
originals []Reminder,
group, body, summary, mood string,
) error {
if len(originals) == 0 || group == "" || body == "" {
return ErrReminderPhrase
}
if mood == "" {
mood = "neutral"
}
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return fmt.Errorf("cache reminder phrase: begin: %w", err)
}
defer tx.Rollback()
seen := make(map[int64]struct{}, len(originals))
for _, r := range originals {
if r.ID <= 0 || r.NextFireTs.IsZero() {
return ErrReminderPhrase
}
if _, ok := seen[r.ID]; ok {
return ErrReminderPhrase
}
seen[r.ID] = struct{}{}
res, err := tx.ExecContext(ctx, `
UPDATE reminders
SET delivery_group = ?, phrase_body = ?, phrase_summary = ?, phrase_mood = ?
WHERE id = ? AND status = ? AND next_fire_ts = ? AND phrase_body = ''`,
group, body, summary, mood, r.ID, ReminderPending, r.NextFireTs.UnixMilli())
if err != nil {
return fmt.Errorf("cache reminder phrase %d: %w", r.ID, err)
}
n, err := res.RowsAffected()
if err != nil {
return fmt.Errorf("cache reminder phrase %d: rows affected: %w", r.ID, err)
}
if n != 1 {
return fmt.Errorf("%w: reminder %d is no longer an unphrased pending occurrence", ErrReminderState, r.ID)
}
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("cache reminder phrase: commit: %w", err)
}
return nil
}
// ScheduleReminderRetry moves every still-pending original into a retry wait.
// Terminal rows are skipped: cancellation winning while a send was in flight
// must not be resurrected. A recurring row whose occurrence changed is skipped
// for the same reason. All remaining originals get their own persisted attempt
// count; normally a collapsed bundle keeps those counts in lockstep.
func (s *Store) ScheduleReminderRetry(ctx context.Context, originals []Reminder, now time.Time) error {
if len(originals) == 0 {
return ErrReminderNotFound
}
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return fmt.Errorf("schedule reminder retry: begin: %w", err)
}
defer tx.Rollback()
seen := make(map[int64]struct{}, len(originals))
for _, expected := range originals {
if expected.ID <= 0 || expected.NextFireTs.IsZero() {
return ErrReminderNotFound
}
if _, ok := seen[expected.ID]; ok {
continue
}
seen[expected.ID] = struct{}{}
var status string
var nextFire int64
var attempts int
var blocked *int64
err := tx.QueryRowContext(ctx,
`SELECT status, next_fire_ts, delivery_attempts, delivery_blocked_ts FROM reminders WHERE id = ?`,
expected.ID).Scan(&status, &nextFire, &attempts, &blocked)
if errors.Is(err, sql.ErrNoRows) {
return ErrReminderNotFound
}
if err != nil {
return fmt.Errorf("schedule reminder retry %d: read: %w", expected.ID, err)
}
if status != ReminderPending || nextFire != expected.NextFireTs.UnixMilli() || blocked != nil {
continue
}
if expected.DeliveryGroup != "" {
var ambiguous int
if err := tx.QueryRowContext(ctx, `
SELECT EXISTS (
SELECT 1 FROM delivery_attempts
WHERE kind = 'reminder' AND delivery_group = ?
AND status IN ('pending', 'sent', 'unknown')
)`, expected.DeliveryGroup).Scan(&ambiguous); err != nil {
return fmt.Errorf("schedule reminder retry %d: inspect outbox: %w", expected.ID, err)
}
if ambiguous != 0 {
// The sink may have accepted this presentation. Leave the reminder
// pending but ineligible for automatic retry; operator resolution
// must not be replaced with a guessed duplicate.
continue
}
}
attempts++
nextAttempt := now.Add(ReminderRetryDelay(attempts)).UnixMilli()
res, err := tx.ExecContext(ctx, `
UPDATE reminders
SET delivery_attempts = ?, next_attempt_ts = ?
WHERE id = ? AND status = ? AND next_fire_ts = ? AND delivery_attempts = ?`,
attempts, nextAttempt, expected.ID, ReminderPending, nextFire, attempts-1)
if err != nil {
return fmt.Errorf("schedule reminder retry %d: %w", expected.ID, err)
}
n, err := res.RowsAffected()
if err != nil {
return fmt.Errorf("schedule reminder retry %d: rows affected: %w", expected.ID, err)
}
if n != 1 {
return fmt.Errorf("%w: reminder %d changed concurrently", ErrReminderState, expected.ID)
}
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("schedule reminder retry: commit: %w", err)
}
return nil
}
// BlockReminderDelivery records a permanent transport/configuration refusal on
// every original represented by one presentation. Blocked rows stay pending and
// visible, but never retry automatically. UnblockReminderDelivery is the
// deliberate recovery path after credentials or policy are repaired.
func (s *Store) BlockReminderDelivery(ctx context.Context, originals []Reminder, now time.Time, reason string) error {
if len(originals) == 0 || strings.TrimSpace(reason) == "" {
return ErrReminderState
}
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return fmt.Errorf("block reminder delivery: begin: %w", err)
}
defer tx.Rollback()
seen := make(map[int64]struct{}, len(originals))
for _, expected := range originals {
if expected.ID <= 0 || expected.NextFireTs.IsZero() {
return ErrReminderNotFound
}
if _, ok := seen[expected.ID]; ok {
continue
}
seen[expected.ID] = struct{}{}
res, err := tx.ExecContext(ctx, `
UPDATE reminders
SET delivery_blocked_ts = ?, delivery_blocked_error = ?, next_attempt_ts = NULL
WHERE id = ? AND status = ? AND next_fire_ts = ?`,
now.UnixMilli(), reason, expected.ID, ReminderPending, expected.NextFireTs.UnixMilli())
if err != nil {
return fmt.Errorf("block reminder %d: %w", expected.ID, err)
}
if err := requireOneReminderRow(res, expected.ID); err != nil {
return err
}
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("block reminder delivery: commit: %w", err)
}
return nil
}
func (s *Store) UnblockReminderDelivery(ctx context.Context, id int64) error {
res, err := s.db.ExecContext(ctx, `
UPDATE reminders
SET delivery_blocked_ts = NULL, delivery_blocked_error = '', next_attempt_ts = NULL
WHERE id = ? AND status = ? AND delivery_blocked_ts IS NOT NULL`, id, ReminderPending)
if err != nil {
return fmt.Errorf("unblock reminder %d: %w", id, err)
}
return requireOneReminderRow(res, id)
}
// CompleteReminderDelivery applies the bookkeeping for every original covered
// by one successful external send in a single transaction. One-shot reminders
// become fired; recurring reminders advance to their next occurrence and shed
// the old presentation/retry state. The all-or-nothing boundary prevents a
// collapsed digest from becoming half-fired and then being repeated.
func (s *Store) CompleteReminderDelivery(ctx context.Context, originals []Reminder, now time.Time) error {
return s.completeReminderDeliveryIn(ctx, 0, originals, now, time.Local)
}
// CompleteSuccessfulReminderAttempt atomically closes a definitely successful
// external attempt and advances every reminder occurrence it represented. If
// this transaction cannot commit, the attempt remains pending; startup then
// reconciles it to unknown, and the occurrence is held from automatic replay.
func (s *Store) CompleteSuccessfulReminderAttempt(ctx context.Context, attemptID int64, originals []Reminder, now time.Time) error {
if attemptID <= 0 {
return fmt.Errorf("complete reminder delivery: invalid attempt id %d", attemptID)
}
return s.completeReminderDeliveryIn(ctx, attemptID, originals, now, time.Local)
}
func (s *Store) completeReminderDeliveryIn(ctx context.Context, attemptID int64, originals []Reminder, now time.Time, loc *time.Location) error {
if len(originals) == 0 {
return ErrReminderNotFound
}
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return fmt.Errorf("complete reminder delivery: begin: %w", err)
}
defer tx.Rollback()
if attemptID != 0 {
res, err := tx.ExecContext(ctx,
`UPDATE delivery_attempts SET status = ?, completed_ts = ? WHERE id = ? AND status = ?`,
DeliverySent, now.UnixMilli(), attemptID, DeliveryPending)
if err != nil {
return fmt.Errorf("complete reminder delivery attempt %d: %w", attemptID, err)
}
n, err := res.RowsAffected()
if err != nil {
return fmt.Errorf("complete reminder delivery attempt %d: rows affected: %w", attemptID, err)
}
if n != 1 {
return fmt.Errorf("complete reminder delivery attempt %d: not pending", attemptID)
}
}
seen := make(map[int64]struct{}, len(originals))
for _, expected := range originals {
if expected.ID <= 0 || expected.NextFireTs.IsZero() {
return ErrReminderNotFound
}
if _, ok := seen[expected.ID]; ok {
return fmt.Errorf("%w: duplicate reminder %d in one delivery", ErrReminderState, expected.ID)
}
seen[expected.ID] = struct{}{}
row := tx.QueryRowContext(ctx, `SELECT `+reminderColumns+` FROM reminders WHERE id = ?`, expected.ID)
current, err := scanReminder(row)
if errors.Is(err, sql.ErrNoRows) {
return ErrReminderNotFound
}
if err != nil {
return fmt.Errorf("complete reminder %d: read: %w", expected.ID, err)
}
if current.Status != ReminderPending || !current.NextFireTs.Equal(expected.NextFireTs) {
return fmt.Errorf("%w: reminder %d occurrence changed", ErrReminderState, expected.ID)
}
if expected.DeliveryGroup != "" && current.DeliveryGroup != expected.DeliveryGroup {
return fmt.Errorf("%w: reminder %d delivery group changed", ErrReminderState, expected.ID)
}
if current.Cron == "" {
if err := completeOneShotReminderTx(ctx, tx, current); err != nil {
return err
}
continue
}
next, err := nextReminderOccurrence(current, now, loc)
if err != nil {
return err
}
if next.IsZero() {
if err := completeOneShotReminderTx(ctx, tx, current); err != nil {
return err
}
continue
}
res, err := tx.ExecContext(ctx, `
UPDATE reminders
SET next_fire_ts = ?, delivery_group = '', phrase_body = '',
phrase_summary = '', phrase_mood = '', delivery_attempts = 0,
next_attempt_ts = NULL, delivery_blocked_ts = NULL,
delivery_blocked_error = ''
WHERE id = ? AND status = ? AND next_fire_ts = ?`,
next.UnixMilli(), current.ID, ReminderPending, current.NextFireTs.UnixMilli())
if err != nil {
return fmt.Errorf("complete recurring reminder %d: %w", current.ID, err)
}
if err := requireOneReminderRow(res, current.ID); err != nil {
return err
}
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("complete reminder delivery: commit: %w", err)
}
return nil
}
func completeOneShotReminderTx(ctx context.Context, tx *sql.Tx, r Reminder) error {
res, err := tx.ExecContext(ctx,
`UPDATE reminders SET status = ? WHERE id = ? AND status = ? AND next_fire_ts = ?`,
ReminderFired, r.ID, ReminderPending, r.NextFireTs.UnixMilli())
if err != nil {
return fmt.Errorf("complete reminder %d: %w", r.ID, err)
}
return requireOneReminderRow(res, r.ID)
}
func requireOneReminderRow(res sql.Result, id int64) error {
n, err := res.RowsAffected()
if err != nil {
return fmt.Errorf("complete reminder %d: rows affected: %w", id, err)
}
if n != 1 {
return fmt.Errorf("%w: reminder %d changed concurrently", ErrReminderState, id)
}
return nil
}
// RescheduleReminder computes the next fire time for a recurring reminder and
// updates next_fire_ts. Returns ErrReminderState if the reminder is not
// recurring or not pending. If the schedule yields no further fire time at all,
@@ -229,8 +633,7 @@ func (s *Store) RescheduleReminder(ctx context.Context, id int64, now time.Time)
}
func (s *Store) rescheduleReminderIn(ctx context.Context, id int64, now time.Time, loc *time.Location) error {
row := s.db.QueryRowContext(ctx, `
SELECT id, created_ts, fire_ts, next_fire_ts, payload, status, cron
row := s.db.QueryRowContext(ctx, `SELECT `+reminderColumns+`
FROM reminders WHERE id = ?`, id)
r, err := scanReminder(row)
if err != nil {
@@ -243,9 +646,38 @@ func (s *Store) rescheduleReminderIn(ctx context.Context, id int64, now time.Tim
return fmt.Errorf("%w: currently %s", ErrReminderState, r.Status)
}
next, err := nextReminderOccurrence(r, now, loc)
if err != nil {
return err
}
if next.IsZero() {
return s.MarkReminder(ctx, id, ReminderFired)
}
res, err := s.db.ExecContext(ctx, `
UPDATE reminders
SET next_fire_ts = ?, delivery_group = '', phrase_body = '',
phrase_summary = '', phrase_mood = '', delivery_attempts = 0,
next_attempt_ts = NULL, delivery_blocked_ts = NULL,
delivery_blocked_error = ''
WHERE id = ? AND status = ? AND next_fire_ts = ?`,
next.UnixMilli(), id, ReminderPending, r.NextFireTs.UnixMilli())
if err != nil {
return err
}
n, err := res.RowsAffected()
if err != nil {
return fmt.Errorf("reschedule reminder: rows affected: %w", err)
}
if n != 1 {
return fmt.Errorf("%w: reminder %d changed concurrently", ErrReminderState, id)
}
return nil
}
func nextReminderOccurrence(r Reminder, now time.Time, loc *time.Location) (time.Time, error) {
sched, err := cron.ParseStandard(r.Cron)
if err != nil {
return fmt.Errorf("parse cron %q: %w", r.Cron, err)
return time.Time{}, fmt.Errorf("parse cron %q: %w", r.Cron, err)
}
// Next is strictly after the time it is given, so the last fire cannot be
// returned again and no fudge minute is needed. The bound stops a schedule
@@ -255,9 +687,7 @@ func (s *Store) rescheduleReminderIn(ctx context.Context, id int64, now time.Tim
next = sched.Next(next)
}
if next.IsZero() || !next.After(now) {
_, err = s.db.ExecContext(ctx, "UPDATE reminders SET status = 'fired' WHERE id = ?", id)
return err
return time.Time{}, nil
}
_, err = s.db.ExecContext(ctx, "UPDATE reminders SET next_fire_ts = ? WHERE id = ?", next.UnixMilli(), id)
return err
return next, nil
}