Make delivery and integration failures explicit

Persist reminder presentations and retry state, atomically complete collapsed deliveries, fall back across away reaches, and block permanent failures visibly (V-715, V-678). Fail closed when enabled integrations lack credentials and keep remote arms explicitly dark (V-691). Give mavweb one sanitized, request-correlated error contract (V-689). Owner explicitly requested direct commits to master.
2026-08-13 02:50:59 +04:00
parent da9114b623
commit 35c6ff5a71
67 changed files with 3174 additions and 477 deletions
@@ -3,6 +3,7 @@ package delivery
 import (
 	"context"
 	"errors"
+	"fmt"
 	"testing"
 	"time"

@@ -61,9 +62,22 @@ type fakeReminderCompleter struct {
 		status string
 	}
 	rescheduled []int64
+	blocked     []int64
+	blockReason string
 	err         error
 }

+func (f *fakeReminderCompleter) BlockReminderDelivery(_ context.Context, originals []store.Reminder, _ time.Time, reason string) error {
+	if f.err != nil {
+		return f.err
+	}
+	for _, r := range originals {
+		f.blocked = append(f.blocked, r.ID)
+	}
+	f.blockReason = reason
+	return nil
+}
+
 func (f *fakeReminderCompleter) MarkReminder(_ context.Context, id int64, status string) error {
 	if f.err != nil {
 		return f.err
@@ -83,6 +97,23 @@ func (f *fakeReminderCompleter) RescheduleReminder(_ context.Context, id int64,
 	return nil
 }

+func (f *fakeReminderCompleter) CompleteReminderDelivery(_ context.Context, originals []store.Reminder, _ time.Time) error {
+	if f.err != nil {
+		return f.err
+	}
+	for _, r := range originals {
+		if r.Cron != "" {
+			f.rescheduled = append(f.rescheduled, r.ID)
+			continue
+		}
+		f.marked = append(f.marked, struct {
+			id     int64
+			status string
+		}{r.ID, store.ReminderFired})
+	}
+	return nil
+}
+
 type fakeAck struct {
 	acked    map[string]bool
 	lastSent map[string]time.Time
@@ -152,10 +183,10 @@ func TestChannelsForReminderPresentVoice(t *testing.T) {
 	}
 }

-func TestChannelsForReminderAwayNtfy(t *testing.T) {
+func TestChannelsForReminderAwayAlternatives(t *testing.T) {
 	got := ChannelsForReminder(store.Away)
-	if len(got) != 1 || got[0] != ChannelNtfy {
-		t.Fatalf("reminder away: want [ntfy], got %v", got)
+	if len(got) != 2 || got[0] != ChannelNtfy || got[1] != ChannelTelegram {
+		t.Fatalf("reminder away: want [ntfy telegram], got %v", got)
 	}
 }

@@ -353,8 +384,9 @@ func TestDispatchReminderPresentVoice(t *testing.T) {

 func TestDispatchReminderAwayNtfy(t *testing.T) {
 	ntfy := &fakeSink{}
+	telegram := &fakeSink{}
 	rc := &fakeReminderCompleter{}
-	d := NewDispatcher(Config{Ntfy: ntfy, Reminders: rc})
+	d := NewDispatcher(Config{Ntfy: ntfy, Telegram: telegram, Reminders: rc})

 	rd := loop.ReminderDecision{
 		Reminder: store.Reminder{ID: 7, Status: "pending"},
@@ -369,6 +401,9 @@ func TestDispatchReminderAwayNtfy(t *testing.T) {
 	if len(out) != 1 || out[0].Sendable.Channel != ChannelNtfy {
 		t.Fatalf("want 1 ntfy, got %+v", out)
 	}
+	if len(telegram.sends) != 0 {
+		t.Fatalf("ntfy succeeded; telegram must not receive a duplicate, got %d sends", len(telegram.sends))
+	}
 	// away channel gets summary, not body
 	if ntfy.sends[0].Summary != "wake up" {
 		t.Fatalf("ntfy summary: want 'wake up', got %q", ntfy.sends[0].Summary)
@@ -378,6 +413,203 @@ func TestDispatchReminderAwayNtfy(t *testing.T) {
 	}
 }

+func TestDispatchReminderAwayNtfyFailureFallsBackToTelegram(t *testing.T) {
+	ntfy := &fakeSink{err: errors.New("ntfy unavailable")}
+	telegram := &fakeSink{}
+	rc := &fakeReminderCompleter{}
+	ob := &fakeOutbox{}
+	d := NewDispatcher(Config{Ntfy: ntfy, Telegram: telegram, Reminders: rc, Outbox: ob})
+
+	out, err := d.DispatchReminder(context.Background(), PhrasedReminder{
+		Decision: loop.ReminderDecision{
+			Reminder: store.Reminder{ID: 8, Status: "pending"},
+			State:    loop.State{Now: refNow(), Presence: store.Away},
+		},
+		Body: "full medication detail", Summary: "take medication",
+	}, refNow())
+	if err != nil {
+		t.Fatalf("dispatch: %v", err)
+	}
+	if len(out) != 1 || out[0].Sendable.Channel != ChannelTelegram || len(telegram.sends) != 1 {
+		t.Fatalf("want one telegram fallback, got out=%+v sends=%d", out, len(telegram.sends))
+	}
+	if got := messageForChannel(telegram.sends[0]); got != "take medication" {
+		t.Fatalf("telegram fallback must retain the minimal away body, got %q", got)
+	}
+	if len(rc.marked) != 1 || rc.marked[0].id != 8 || rc.marked[0].status != "fired" {
+		t.Fatalf("successful fallback must complete the reminder, got %+v", rc.marked)
+	}
+	if len(ob.attempts) != 2 || ob.attempts[0].channel != "ntfy" || ob.attempts[0].status != store.DeliveryFailed ||
+		ob.attempts[1].channel != "telegram" || ob.attempts[1].status != store.DeliverySent {
+		t.Fatalf("want ntfy failed then telegram sent attempts, got %+v", ob.attempts)
+	}
+}
+
+func TestDispatchReminderAwayNilNtfyFallsBackVisibly(t *testing.T) {
+	telegram := &fakeSink{}
+	rc := &fakeReminderCompleter{}
+	ob := &fakeOutbox{}
+	d := NewDispatcher(Config{Telegram: telegram, Reminders: rc, Outbox: ob})
+
+	out, err := d.DispatchReminder(context.Background(), PhrasedReminder{
+		Decision: loop.ReminderDecision{
+			Reminder: store.Reminder{ID: 9, Status: "pending"},
+			State:    loop.State{Now: refNow(), Presence: store.Away},
+		},
+		Body: "full detail", Summary: "short reminder",
+	}, refNow())
+	if err != nil {
+		t.Fatalf("dispatch: %v", err)
+	}
+	if len(out) != 1 || out[0].Sendable.Channel != ChannelTelegram || len(telegram.sends) != 1 {
+		t.Fatalf("want one telegram fallback, got out=%+v sends=%d", out, len(telegram.sends))
+	}
+	if len(ob.attempts) != 2 || ob.attempts[0].channel != "ntfy" || ob.attempts[0].status != store.DeliveryFailed ||
+		ob.attempts[1].channel != "telegram" || ob.attempts[1].status != store.DeliverySent {
+		t.Fatalf("nil ntfy must leave a failed row before telegram succeeds, got %+v", ob.attempts)
+	}
+	if len(rc.marked) != 1 || rc.marked[0].id != 9 {
+		t.Fatalf("successful fallback must complete the reminder, got %+v", rc.marked)
+	}
+}
+
+func TestDispatchReminderAwayAllAlternativesFailStaysPending(t *testing.T) {
+	ntfy := &fakeSink{err: errors.New("ntfy unavailable")}
+	telegram := &fakeSink{err: errors.New("telegram unavailable")}
+	rc := &fakeReminderCompleter{}
+	ob := &fakeOutbox{}
+	d := NewDispatcher(Config{Ntfy: ntfy, Telegram: telegram, Reminders: rc, Outbox: ob})
+
+	out, err := d.DispatchReminder(context.Background(), PhrasedReminder{
+		Decision: loop.ReminderDecision{
+			Reminder: store.Reminder{ID: 10, Status: "pending"},
+			State:    loop.State{Now: refNow(), Presence: store.Away},
+		},
+		Body: "full detail", Summary: "short reminder",
+	}, refNow())
+	if err == nil {
+		t.Fatal("all alternatives failed: want an error")
+	}
+	if len(out) != 0 || len(rc.marked) != 0 || len(rc.rescheduled) != 0 {
+		t.Fatalf("failed reminder must stay pending, got out=%+v marked=%+v rescheduled=%+v", out, rc.marked, rc.rescheduled)
+	}
+	if len(ob.attempts) != 2 || ob.attempts[0].status != store.DeliveryFailed || ob.attempts[1].status != store.DeliveryFailed {
+		t.Fatalf("want two failed attempts, got %+v", ob.attempts)
+	}
+}
+
+func TestDispatchReminderAllPermanentAlternativesBlockGroup(t *testing.T) {
+	permanent := fmt.Errorf("%w: credentials rejected", ErrPermanent)
+	ntfy := &fakeSink{err: permanent}
+	telegram := &fakeSink{err: permanent}
+	rc := &fakeReminderCompleter{}
+	d := NewDispatcher(Config{Ntfy: ntfy, Telegram: telegram, Reminders: rc})
+	now := refNow()
+	originals := []store.Reminder{
+		{ID: 21, Status: store.ReminderPending, NextFireTs: now, DeliveryGroup: "reminder:permanent"},
+		{ID: 22, Status: store.ReminderPending, NextFireTs: now, DeliveryGroup: "reminder:permanent"},
+	}
+
+	_, err := d.DispatchReminder(context.Background(), PhrasedReminder{
+		Decision: loop.ReminderDecision{
+			Reminder: store.Reminder{ID: 0, Status: store.ReminderPending, Collapsed: originals},
+			State:    loop.State{Now: now, Presence: store.Away},
+		}, Body: "details", Summary: "two reminders",
+	}, now)
+	if err == nil || !errors.Is(err, ErrPermanent) {
+		t.Fatalf("permanent alternatives error = %v", err)
+	}
+	if len(ntfy.sends) != 0 || len(telegram.sends) != 0 {
+		// fakeSink records only successes, so this also asserts neither was
+		// mistaken for a successful delivery.
+		t.Fatalf("permanent failure produced successful sends: ntfy=%d telegram=%d", len(ntfy.sends), len(telegram.sends))
+	}
+	if len(rc.blocked) != 2 || rc.blocked[0] != 21 || rc.blocked[1] != 22 || rc.blockReason == "" {
+		t.Fatalf("permanent bundle was not durably blocked: ids=%v reason=%q", rc.blocked, rc.blockReason)
+	}
+}
+
+func TestDispatchReminderWithUnrecordableOutboxDoesNotSend(t *testing.T) {
+	telegram := &fakeSink{}
+	rc := &fakeReminderCompleter{}
+	ob := &fakeOutbox{beginErr: errors.New("database unavailable")}
+	d := NewDispatcher(Config{Telegram: telegram, Reminders: rc, Outbox: ob})
+
+	out, err := d.DispatchReminder(context.Background(), PhrasedReminder{
+		Decision: loop.ReminderDecision{
+			Reminder: store.Reminder{ID: 11, Status: "pending", DeliveryGroup: "reminder:11"},
+			State:    loop.State{Now: refNow(), Presence: store.Away},
+		},
+		Body: "full detail", Summary: "short reminder",
+	}, refNow())
+	if err == nil {
+		t.Fatal("unrecordable reminder attempt must be withheld")
+	}
+	if len(out) != 0 || len(telegram.sends) != 0 || len(rc.marked) != 0 {
+		t.Fatalf("unrecordable reminder escaped durable boundary: out=%+v sends=%d completed=%+v", out, len(telegram.sends), rc.marked)
+	}
+}
+
+func TestDispatchReminderOutboxCompletionAmbiguityDoesNotCompleteOccurrence(t *testing.T) {
+	telegram := &fakeSink{}
+	rc := &fakeReminderCompleter{}
+	ob := &fakeOutbox{completeErr: errors.New("database unavailable after send")}
+	d := NewDispatcher(Config{Telegram: telegram, Reminders: rc, Outbox: ob})
+
+	out, err := d.DispatchReminder(context.Background(), PhrasedReminder{
+		Decision: loop.ReminderDecision{
+			Reminder: store.Reminder{ID: 12, Status: store.ReminderPending, DeliveryGroup: "reminder:12"},
+			State:    loop.State{Now: refNow(), Presence: store.Away},
+		},
+		Body: "full detail", Summary: "short reminder",
+	}, refNow())
+	if err == nil {
+		t.Fatal("ambiguous outbox completion must surface an error")
+	}
+	if len(out) != 1 || out[0].Sendable.Channel != ChannelTelegram || len(telegram.sends) != 1 {
+		t.Fatalf("the accepted external send must be reported once: out=%+v sends=%d", out, len(telegram.sends))
+	}
+	if len(rc.marked) != 0 || len(rc.rescheduled) != 0 {
+		t.Fatalf("ambiguous accepted send completed the occurrence: marked=%+v rescheduled=%+v", rc.marked, rc.rescheduled)
+	}
+	if len(ob.attempts) != 2 || ob.attempts[1].status != store.DeliveryPending {
+		t.Fatalf("accepted send should remain pending for startup reconciliation: %+v", ob.attempts)
+	}
+}
+
+func TestDispatchReminderCollapsedOutboxUsesRealIDAndSharedGroup(t *testing.T) {
+	telegram := &fakeSink{}
+	rc := &fakeReminderCompleter{}
+	ob := &fakeOutbox{}
+	d := NewDispatcher(Config{Telegram: telegram, Reminders: rc, Outbox: ob})
+	now := refNow()
+	originals := []store.Reminder{
+		{ID: 81, Status: store.ReminderPending, NextFireTs: now, DeliveryGroup: "reminder:bundle"},
+		{ID: 82, Status: store.ReminderPending, NextFireTs: now, DeliveryGroup: "reminder:bundle"},
+	}
+
+	if _, err := d.DispatchReminder(context.Background(), PhrasedReminder{
+		Decision: loop.ReminderDecision{
+			Reminder: store.Reminder{ID: 0, Status: store.ReminderPending, Collapsed: originals},
+			State:    loop.State{Now: now, Presence: store.Away},
+		},
+		Body: "two reminders", Summary: "two reminders",
+	}, now); err != nil {
+		t.Fatalf("dispatch: %v", err)
+	}
+	if len(ob.attempts) != 2 {
+		t.Fatalf("attempts=%d, want nil-ntfy plus telegram", len(ob.attempts))
+	}
+	for _, a := range ob.attempts {
+		if a.reminderID != 81 || a.deliveryGroup != "reminder:bundle" {
+			t.Fatalf("synthetic ID escaped into outbox: %+v", a)
+		}
+	}
+	if len(rc.marked) != 2 || rc.marked[0].id != 81 || rc.marked[1].id != 82 {
+		t.Fatalf("collapsed completion did not cover originals: %+v", rc.marked)
+	}
+}
+
 func TestDispatchReminderFailedSendNotMarkedFired(t *testing.T) {
 	// a failed send must not mark the reminder fired — it stays pending for
 	// the next tick to re-deliver. same instinct as "record after success."
@@ -486,11 +718,12 @@ func TestDispatchNudgeVoiceNoSessionSev2Drops(t *testing.T) {
 }

 func TestDispatchReminderVoiceNoSessionRoutesNtfy(t *testing.T) {
-	// present reminder → [voice]. voice has no session → away = ntfy.
+	// present reminder → [voice]. voice has no session → away starts at ntfy.
 	voice := &fakeSink{err: ErrVoiceNoSession}
 	ntfy := &fakeSink{}
+	telegram := &fakeSink{}
 	rc := &fakeReminderCompleter{}
-	d := NewDispatcher(Config{Voice: voice, Ntfy: ntfy, Reminders: rc})
+	d := NewDispatcher(Config{Voice: voice, Ntfy: ntfy, Telegram: telegram, Reminders: rc})

 	rd := loop.ReminderDecision{
 		Reminder: store.Reminder{ID: 99, Status: "pending"},
@@ -505,6 +738,9 @@ func TestDispatchReminderVoiceNoSessionRoutesNtfy(t *testing.T) {
 	if len(ntfy.sends) != 1 || len(out) != 1 || out[0].Sendable.Channel != ChannelNtfy {
 		t.Fatalf("reminder voice-no-session: want 1 ntfy, got ntfy=%d out=%+v", len(ntfy.sends), out)
 	}
+	if len(telegram.sends) != 0 {
+		t.Fatalf("successful ntfy fallback must stop before telegram, got %d sends", len(telegram.sends))
+	}
 	if len(rc.marked) != 1 || rc.marked[0].status != "fired" {
 		t.Fatalf("rerouted reminder must be marked fired: %+v", rc.marked)
 	}
@@ -641,6 +877,7 @@ func TestDispatchRecurringReminderReschedules(t *testing.T) {
 type outboxAttempt struct {
 	kind, rule      string
 	reminderID      int64
+	deliveryGroup   string
 	channel, hash   string
 	status          string
 	begunAt, doneAt time.Time
@@ -659,13 +896,13 @@ type fakeOutbox struct {
 	completeErr error
 }

-func (f *fakeOutbox) BeginDeliveryAttempt(_ context.Context, kind, rule string, reminderID int64, channel, hash string, now time.Time) (int64, error) {
+func (f *fakeOutbox) BeginDeliveryAttempt(_ context.Context, kind, rule string, reminderID int64, deliveryGroup, channel, hash string, now time.Time) (int64, error) {
 	if f.beginErr != nil {
 		return 0, f.beginErr
 	}
 	f.nextID++
 	f.attempts = append(f.attempts, &outboxAttempt{
-		kind: kind, rule: rule, reminderID: reminderID, channel: channel, hash: hash,
+		kind: kind, rule: rule, reminderID: reminderID, deliveryGroup: deliveryGroup, channel: channel, hash: hash,
 		status: "pending", begunAt: now,
 	})
 	return f.nextID, nil
@@ -985,8 +1222,8 @@ func TestPanicInReminderSinkResolvesTheAttempt(t *testing.T) {
 		},
 		Body: "звонок", Summary: "звонок",
 	}, refNow())
-	if err != nil {
-		t.Fatalf("a panicking sink must not fail the dispatch: %v", err)
+	if err == nil {
+		t.Fatal("a panicking sole reminder sink must report delivery failure")
 	}
 	if len(out) != 0 {
 		t.Fatalf("nothing was delivered, want no dispatches, got %+v", out)