kami/Maven
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Make delivery and integration failures explicit

Browse Source 
Persist reminder presentations and retry state, atomically complete collapsed deliveries, fall back across away reaches, and block permanent failures visibly (V-715, V-678). Fail closed when enabled integrations lack credentials and keep remote arms explicitly dark (V-691). Give mavweb one sanitized, request-correlated error contract (V-689). Owner explicitly requested direct commits to master.
This commit is contained in:
claude
2026-08-13 02:50:59 +04:00
parent da9114b623
commit 35c6ff5a71
67 changed files with 3174 additions and 477 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/delivery/dispatcher.go
+133 -56
View File
@@ -22,13 +22,25 @@ type NudgeRecorder interface {
RecordNudge(ctx context.Context, rule, channel, message string, ts time.Time) (int64, error)
}
// ReminderCompleter — the seam the store implements. For one-shot reminders:
// pending → fired after successful delivery. For recurring reminders (with
// cron): reschedule after successful delivery. A failed send does NOT mark or
// reschedule it (it stays pending; the next tick re-delivers).
// ReminderCompleter — the seam the store implements. Every original represented
// by one external delivery is completed in one transaction. That matters for a
// collapsed catch-up bundle: partially firing the originals would make the
// next tick repeat a presentation that the user already received.
type ReminderCompleter interface {
MarkReminder(ctx context.Context, id int64, status string) error
RescheduleReminder(ctx context.Context, id int64, now time.Time) error
CompleteReminderDelivery(ctx context.Context, originals []store.Reminder, now time.Time) error
}
// DurableReminderCompleter closes the successful outbox attempt and advances
// every reminder occurrence in one local transaction. The external send and
// local commit cannot be one transaction, but the local half must be: a crash
// between `attempt=sent` and `reminder=fired` otherwise strands the reminder in
// a permanently suppressed state.
type DurableReminderCompleter interface {
CompleteSuccessfulReminderAttempt(ctx context.Context, attemptID int64, originals []store.Reminder, now time.Time) error
}
type ReminderBlocker interface {
BlockReminderDelivery(ctx context.Context, originals []store.Reminder, now time.Time, reason string) error
}
// Outbox — the durable delivery ledger. Begin is recorded BEFORE the external
@@ -39,7 +51,7 @@ type ReminderCompleter interface {
// disabled (existing send/record behavior, unchanged — test scenarios that
// don't care about crash recovery).
type Outbox interface {
BeginDeliveryAttempt(ctx context.Context, kind, rule string, reminderID int64, channel, bodyHash string, now time.Time) (int64, error)
BeginDeliveryAttempt(ctx context.Context, kind, rule string, reminderID int64, deliveryGroup, channel, bodyHash string, now time.Time) (int64, error)
CompleteDeliveryAttempt(ctx context.Context, id int64, status string, now time.Time) error
}
@@ -57,11 +69,11 @@ func bodyHash(channel Channel, body string) string {
// on one attempt shouldn't block a nudge/reminder actually reaching the user
// — but it does mean this attempt can't be reconciled after a crash, so it's
// logged. Returns 0 (no-op id) when unrecorded.
func (d *Dispatcher) beginOutbox(ctx context.Context, kind, rule string, reminderID int64, channel Channel, body string, now time.Time) int64 {
func (d *Dispatcher) beginOutbox(ctx context.Context, kind, rule string, reminderID int64, deliveryGroup string, channel Channel, body string, now time.Time) int64 {
if d.cfg.Outbox == nil {
return 0
}
id, err := d.cfg.Outbox.BeginDeliveryAttempt(ctx, kind, rule, reminderID, string(channel), bodyHash(channel, body), now)
id, err := d.cfg.Outbox.BeginDeliveryAttempt(ctx, kind, rule, reminderID, deliveryGroup, string(channel), bodyHash(channel, body), now)
if err != nil {
log.Printf("dispatcher: outbox begin failed (send proceeds untracked): %v", err)
return 0
@@ -69,16 +81,37 @@ func (d *Dispatcher) beginOutbox(ctx context.Context, kind, rule string, reminde
return id
}
// beginReminderOutbox is stricter than the nudge helper above. A reminder may
// be retried indefinitely, so sending it without the durable attempt row would
// reopen an unobservable duplicate window after a crash. A configured but
// unhealthy outbox therefore blocks this transport attempt; a deliberately nil
// outbox still supports small isolated test/development wiring.
func (d *Dispatcher) beginReminderOutbox(ctx context.Context, reminderID int64, deliveryGroup string, channel Channel, body string, now time.Time) (int64, error) {
if d.cfg.Outbox == nil {
return 0, nil
}
id, err := d.cfg.Outbox.BeginDeliveryAttempt(
ctx, "reminder", "", reminderID, deliveryGroup,
string(channel), bodyHash(channel, body), now,
)
if err != nil {
return 0, fmt.Errorf("begin reminder delivery attempt: %w", err)
}
return id, nil
}
// completeOutbox records the sink's outcome for a prior beginOutbox call.
// id==0 means either tracking is disabled or the begin failed — nothing to
// complete either way.
func (d *Dispatcher) completeOutbox(ctx context.Context, id int64, status string, now time.Time) {
func (d *Dispatcher) completeOutbox(ctx context.Context, id int64, status string, now time.Time) error {
if id == 0 || d.cfg.Outbox == nil {
return
return nil
}
if err := d.cfg.Outbox.CompleteDeliveryAttempt(ctx, id, status, now); err != nil {
log.Printf("dispatcher: outbox complete failed: %v", err)
return err
}
return nil
}
// PhrasedNudge — the phraser module's output for a nudge. the phraser (the
@@ -155,7 +188,7 @@ func (d *Dispatcher) DispatchNudge(ctx context.Context, pn PhrasedNudge, now tim
// the same afterwards. no nudges row: that table feeds the
// ignored_rate signal, and a nudge nobody could see must not
// count as ignored.
id := d.beginOutbox(ctx, "nudge", c.Rule.Name, 0, ch, pn.Summary, now)
id := d.beginOutbox(ctx, "nudge", c.Rule.Name, 0, "", ch, pn.Summary, now)
d.completeOutbox(ctx, id, store.DeliveryDropped, now)
log.Printf("dispatcher: dropped %s (sev%d, presence=%s) — routing table suppressed it",
c.Rule.Name, c.Severity, c.State.Presence)
@@ -176,7 +209,7 @@ func (d *Dispatcher) DispatchNudge(ctx context.Context, pn PhrasedNudge, now tim
if sink == nil {
continue
}
attemptID := d.beginOutbox(ctx, "nudge", c.Rule.Name, 0, ch, messageForChannel(s), now)
attemptID := d.beginOutbox(ctx, "nudge", c.Rule.Name, 0, "", ch, messageForChannel(s), now)
if err := safeSend(ctx, sink, s); err != nil {
if errors.Is(err, ErrSinkPanicked) {
// one broken sink must not eat the other channels for this
@@ -226,14 +259,17 @@ func (d *Dispatcher) DispatchNudge(ctx context.Context, pn PhrasedNudge, now tim
}
// DispatchReminder — routes a phrased reminder. reminders bypass the gate and
// fire once (pending → fired after successful delivery). voice when present,
// ntfy when away. no repeat (reminders fire once). marks the reminder fired
// only if at least one channel succeeded — a failed send leaves it pending
// for the next tick to re-deliver.
// fire once (pending → fired after successful delivery). Voice is preferred
// when present; if it has no live session, delivery falls back to the ordered
// away alternatives. Away delivery tries ntfy, then telegram, and stops after
// the first success. A failed or unwired alternative falls through to the next
// one. If every selected alternative fails, the reminder stays pending and an
// error is returned for the tick's retry path.
func (d *Dispatcher) DispatchReminder(ctx context.Context, pr PhrasedReminder, now time.Time) ([]Dispatch, error) {
rd := pr.Decision
channels := ChannelsForReminder(rd.State.Presence)
var out []Dispatch
var failures []error
allPermanent := true
for i := 0; i < len(channels); i++ {
ch := channels[i]
s := Sendable{
@@ -245,62 +281,103 @@ func (d *Dispatcher) DispatchReminder(ctx context.Context, pr PhrasedReminder, n
Ts: now,
}
s = minimalForAway(s)
sink := d.sinkFor(ch)
if sink == nil {
reminderID, deliveryGroup := reminderDeliveryIdentity(rd.Reminder)
attemptID, err := d.beginReminderOutbox(ctx, reminderID, deliveryGroup, ch, messageForChannel(s), now)
if err != nil {
allPermanent = false
failures = append(failures, err)
log.Printf("dispatcher: reminder %d delivery via %s withheld: %v", rd.Reminder.ID, ch, err)
continue
}
sink := d.sinkFor(ch)
if sink == nil {
d.completeOutbox(ctx, attemptID, store.DeliveryFailed, now)
err := fmt.Errorf("%s sink is not configured", ch)
failures = append(failures, err)
log.Printf("dispatcher: reminder %d delivery via %s failed: %v", rd.Reminder.ID, ch, err)
if ch == ChannelVoice {
channels = ChannelsForReminder(store.Away)
failures = nil
allPermanent = true
i = -1
}
continue
}
attemptID := d.beginOutbox(ctx, "reminder", "", rd.Reminder.ID, ch, messageForChannel(s), now)
if err := safeSend(ctx, sink, s); err != nil {
if errors.Is(err, ErrSinkPanicked) {
d.completeOutbox(ctx, attemptID, store.DeliveryFailed, now)
continue
d.completeOutbox(ctx, attemptID, store.DeliveryFailed, now)
failures = append(failures, fmt.Errorf("send %s: %w", ch, err))
if !errors.Is(err, ErrPermanent) {
allPermanent = false
}
if errors.Is(err, ErrVoiceNoSession) {
// presence guess was wrong — reroute reminder to the away
// channel (ntfy). voice is the only present channel, so nothing
// has been sent yet.
d.completeOutbox(ctx, attemptID, store.DeliveryFailed, now)
// Presence was stale. Voice is the only present alternative, so
// nothing has been sent and it is safe to start the away chain.
log.Printf("dispatcher: no live voice session for reminder %d, rerouting to away channels", rd.Reminder.ID)
channels = ChannelsForReminder(store.Away)
failures = nil
allPermanent = true
i = -1
continue
}
d.completeOutbox(ctx, attemptID, store.DeliveryFailed, now)
return out, fmt.Errorf("send %s: %w", ch, err)
log.Printf("dispatcher: reminder %d delivery via %s failed: %v", rd.Reminder.ID, ch, err)
continue
}
d.completeOutbox(ctx, attemptID, store.DeliverySent, now)
out = append(out, Dispatch{Sendable: s})
}
if d.cfg.Reminders != nil && len(out) > 0 {
// ID=0 is a synthetic digest reminder; it's not in the DB. Complete
// the collapsed originals it stands in for instead — only now, after
// a successful send, so a failed digest leaves them all pending.
out := []Dispatch{{Sendable: s}}
originals := []store.Reminder{rd.Reminder}
if rd.Reminder.ID == 0 {
for _, orig := range rd.Reminder.Collapsed {
if err := d.completeReminder(ctx, orig, now); err != nil {
return out, err
}
originals = rd.Reminder.Collapsed
}
if durable, ok := d.cfg.Reminders.(DurableReminderCompleter); ok && attemptID != 0 {
if err := durable.CompleteSuccessfulReminderAttempt(ctx, attemptID, originals, now); err != nil {
return out, fmt.Errorf("commit successful reminder delivery: %w", err)
}
return out, nil
}
if err := d.completeOutbox(ctx, attemptID, store.DeliverySent, now); err != nil {
// The external sink accepted the reminder, but its durable outcome is
// ambiguous. Do not complete the reminder row: startup reconciliation
// will mark the attempt unknown and DueReminders will hold the exact
// occurrence for operator resolution rather than sending a duplicate.
return out, fmt.Errorf("record successful reminder delivery: %w", err)
}
if d.cfg.Reminders != nil {
if err := d.cfg.Reminders.CompleteReminderDelivery(ctx, originals, now); err != nil {
return out, fmt.Errorf("complete reminder delivery: %w", err)
}
}
return out, nil
}
if len(failures) == 0 {
failures = append(failures, errors.New("no delivery alternatives selected"))
allPermanent = false
}
joined := errors.Join(failures...)
if allPermanent && d.cfg.Reminders != nil {
originals := []store.Reminder{rd.Reminder}
if rd.Reminder.ID == 0 {
originals = rd.Reminder.Collapsed
}
if blocker, ok := d.cfg.Reminders.(ReminderBlocker); ok {
if err := blocker.BlockReminderDelivery(ctx, originals, now, joined.Error()); err != nil {
return nil, fmt.Errorf("block permanently undeliverable reminder %d: %w", rd.Reminder.ID, err)
}
} else if err := d.completeReminder(ctx, rd.Reminder, now); err != nil {
return out, err
}
}
return out, nil
return nil, fmt.Errorf("deliver reminder %d: %w", rd.Reminder.ID, joined)
}
// completeReminder — post-delivery bookkeeping for one reminder: recurring
// (cron set) reschedules, one-shot marks fired.
func (d *Dispatcher) completeReminder(ctx context.Context, r store.Reminder, now time.Time) error {
if r.Cron != "" {
if err := d.cfg.Reminders.RescheduleReminder(ctx, r.ID, now); err != nil {
return fmt.Errorf("reschedule reminder %d: %w", r.ID, err)
}
return nil
// reminderDeliveryIdentity gives the outbox both a human-readable real row id
// and the exact occurrence key used for crash suppression. A collapsed digest
// has synthetic ID zero, so its first original is the representative; the
// shared delivery group still identifies every original atomically.
func reminderDeliveryIdentity(r store.Reminder) (int64, string) {
if r.ID != 0 {
return r.ID, r.DeliveryGroup
}
if err := d.cfg.Reminders.MarkReminder(ctx, r.ID, "fired"); err != nil {
return fmt.Errorf("mark reminder %d fired: %w", r.ID, err)
if len(r.Collapsed) == 0 {
return 0, ""
}
return nil
return r.Collapsed[0].ID, r.Collapsed[0].DeliveryGroup
}
// RepeatUnacked — the daemon calls this each tick to re-send un-acked sev4
@@ -340,7 +417,7 @@ func (d *Dispatcher) RepeatUnacked(ctx context.Context, keys []string, now time.
Ts: now,
}
s = minimalForAway(s)
attemptID := d.beginOutbox(ctx, "nudge", key, 0, ChannelTelegram, messageForChannel(s), now)
attemptID := d.beginOutbox(ctx, "nudge", key, 0, "", ChannelTelegram, messageForChannel(s), now)
if err := safeSend(ctx, d.cfg.Telegram, s); err != nil {
d.completeOutbox(ctx, attemptID, store.DeliveryFailed, now)
if errors.Is(err, ErrSinkPanicked) {