Make delivery and integration failures explicit
Persist reminder presentations and retry state, atomically complete collapsed deliveries, fall back across away reaches, and block permanent failures visibly (V-715, V-678). Fail closed when enabled integrations lack credentials and keep remote arms explicitly dark (V-691). Give mavweb one sanitized, request-correlated error contract (V-689). Owner explicitly requested direct commits to master.
This commit is contained in:
+125
-11
@@ -10,11 +10,13 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -226,18 +228,11 @@ func (t *tickLoop) tick(ctx context.Context, now time.Time) {
|
||||
// detectPatterns below for how idempotence and dismissal are respected.
|
||||
t.detectPatterns(ctx, now, state)
|
||||
|
||||
// reminders: gate-bypassing class. fired once, marked after a successful
|
||||
// delivery. a failed send leaves the reminder pending — the next tick
|
||||
// re-gathers and re-attempts.
|
||||
// reminders: gate-bypassing class. The presentation and retry clock live on
|
||||
// the reminder occurrence, so a transport outage neither spends the model
|
||||
// every tick nor changes what the reminder says after a restart.
|
||||
for _, d := range loop.RemindDecisions(state, due) {
|
||||
pr, err := t.phraser.PhraseReminder(ctx, d)
|
||||
if err != nil {
|
||||
log.Printf("tick: phrase reminder %d: %v", d.Reminder.ID, err)
|
||||
continue
|
||||
}
|
||||
if _, err := t.dispatcher.DispatchReminder(ctx, pr, now); err != nil {
|
||||
log.Printf("tick: dispatch reminder %d: %v", d.Reminder.ID, err)
|
||||
}
|
||||
t.deliverReminder(ctx, d, now)
|
||||
}
|
||||
|
||||
// sev4-away repeats: re-send un-acked telegram nudges per repeatInterval.
|
||||
@@ -263,6 +258,125 @@ func (t *tickLoop) tick(ctx context.Context, now time.Time) {
|
||||
}
|
||||
}
|
||||
|
||||
// deliverReminder advances one due reminder (or collapsed bundle) through the
|
||||
// durable delivery state. A phrase is cached before the first external send;
|
||||
// every definite failure advances the persisted bounded backoff.
|
||||
func (t *tickLoop) deliverReminder(ctx context.Context, d loop.ReminderDecision, now time.Time) {
|
||||
originals := reminderOriginals(d.Reminder)
|
||||
pr, cached := cachedReminderPhrase(d, originals)
|
||||
if !cached {
|
||||
var err error
|
||||
pr, err = t.phraser.PhraseReminder(ctx, d)
|
||||
if err == nil && pr.Body == "" {
|
||||
err = errors.New("phraser returned an empty reminder body")
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("tick: phrase reminder %d: %v", d.Reminder.ID, err)
|
||||
t.scheduleReminderRetry(ctx, originals, now)
|
||||
return
|
||||
}
|
||||
if pr.Mood == "" {
|
||||
pr.Mood = "neutral"
|
||||
}
|
||||
group := reminderDeliveryGroup(originals)
|
||||
if err := t.store.CacheReminderPhrase(
|
||||
ctx, originals, group, pr.Body, pr.Summary, pr.Mood,
|
||||
); err != nil {
|
||||
// A cancellation or another completion can win while phrasing. Do
|
||||
// not send a presentation that no longer owns every original.
|
||||
log.Printf("tick: cache reminder %d phrase: %v", d.Reminder.ID, err)
|
||||
return
|
||||
}
|
||||
// The store now owns the phrase, but this tick's value predates that
|
||||
// write. Stamp the exact persisted occurrence identity onto the value
|
||||
// handed to the dispatcher so its outbox row can suppress an ambiguous
|
||||
// crash for both a real reminder and a synthetic collapsed bundle.
|
||||
for i := range originals {
|
||||
originals[i].DeliveryGroup = group
|
||||
originals[i].PhraseBody = pr.Body
|
||||
originals[i].PhraseSummary = pr.Summary
|
||||
originals[i].PhraseMood = pr.Mood
|
||||
}
|
||||
if d.Reminder.ID == 0 {
|
||||
d.Reminder.Collapsed = originals
|
||||
} else {
|
||||
d.Reminder = originals[0]
|
||||
}
|
||||
}
|
||||
// A phraser is not allowed to substitute the reminder decision. In
|
||||
// particular, the durable group stamped above must reach the outbox.
|
||||
pr.Decision = d
|
||||
|
||||
if _, err := t.dispatcher.DispatchReminder(ctx, pr, now); err != nil {
|
||||
log.Printf("tick: dispatch reminder %d: %v", d.Reminder.ID, err)
|
||||
t.scheduleReminderRetry(ctx, originals, now)
|
||||
}
|
||||
}
|
||||
|
||||
func (t *tickLoop) scheduleReminderRetry(ctx context.Context, originals []store.Reminder, now time.Time) {
|
||||
if err := t.store.ScheduleReminderRetry(ctx, originals, now); err != nil {
|
||||
log.Printf("tick: schedule reminder retry: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// reminderOriginals converts the synthetic ID=0 bundle back to real store
|
||||
// rows. Keeping this in one helper makes it impossible to accidentally persist
|
||||
// retry state against reminder zero.
|
||||
func reminderOriginals(r store.Reminder) []store.Reminder {
|
||||
if r.ID == 0 {
|
||||
return append([]store.Reminder(nil), r.Collapsed...)
|
||||
}
|
||||
return []store.Reminder{r}
|
||||
}
|
||||
|
||||
// cachedReminderPhrase reconstructs a PhrasedReminder only when every original
|
||||
// agrees on one persisted group and presentation. That agreement is what lets
|
||||
// a collapsed bundle survive a restart without being re-phrased.
|
||||
func cachedReminderPhrase(d loop.ReminderDecision, originals []store.Reminder) (delivery.PhrasedReminder, bool) {
|
||||
if len(originals) == 0 || !originals[0].HasDeliveryPhrase() {
|
||||
return delivery.PhrasedReminder{}, false
|
||||
}
|
||||
first := originals[0]
|
||||
for _, r := range originals[1:] {
|
||||
if !r.HasDeliveryPhrase() ||
|
||||
r.DeliveryGroup != first.DeliveryGroup ||
|
||||
r.PhraseBody != first.PhraseBody ||
|
||||
r.PhraseSummary != first.PhraseSummary ||
|
||||
r.PhraseMood != first.PhraseMood {
|
||||
return delivery.PhrasedReminder{}, false
|
||||
}
|
||||
}
|
||||
mood := first.PhraseMood
|
||||
if mood == "" {
|
||||
mood = "neutral"
|
||||
}
|
||||
return delivery.PhrasedReminder{
|
||||
Decision: d,
|
||||
Body: first.PhraseBody,
|
||||
Summary: first.PhraseSummary,
|
||||
Mood: mood,
|
||||
}, true
|
||||
}
|
||||
|
||||
// reminderDeliveryGroup deterministically names one occurrence or collapsed
|
||||
// set. The next-fire instant is part of the identity so a recurring reminder's
|
||||
// later occurrence can never inherit the previous occurrence's phrase.
|
||||
func reminderDeliveryGroup(originals []store.Reminder) string {
|
||||
ordered := append([]store.Reminder(nil), originals...)
|
||||
sort.Slice(ordered, func(i, j int) bool {
|
||||
if ordered[i].ID == ordered[j].ID {
|
||||
return ordered[i].NextFireTs.Before(ordered[j].NextFireTs)
|
||||
}
|
||||
return ordered[i].ID < ordered[j].ID
|
||||
})
|
||||
h := sha256.New()
|
||||
for _, r := range ordered {
|
||||
_, _ = fmt.Fprintf(h, "%d:%d;", r.ID, r.NextFireTs.UnixMilli())
|
||||
}
|
||||
sum := h.Sum(nil)
|
||||
return fmt.Sprintf("reminder:%x", sum[:12])
|
||||
}
|
||||
|
||||
// savePresence writes back the bucket GatherState just resolved.
|
||||
//
|
||||
// It lives here and not in GatherState because that method holds a read-only
|
||||
|
||||
Reference in New Issue
Block a user