Make delivery and integration failures explicit
Persist reminder presentations and retry state, atomically complete collapsed deliveries, fall back across away reaches, and block permanent failures visibly (V-715, V-678). Fail closed when enabled integrations lack credentials and keep remote arms explicitly dark (V-691). Give mavweb one sanitized, request-correlated error contract (V-689). Owner explicitly requested direct commits to master.
This commit is contained in:
+21
-9
@@ -559,7 +559,7 @@ func run(args []string) error {
|
||||
func personaFacts(cfg *config.Config) persona.Facts {
|
||||
f := persona.Facts{
|
||||
// Telegram lives outside the voice block, so it counts either way.
|
||||
Telegram: cfg.Telegram != nil && cfg.Telegram.BotToken != "" && cfg.Telegram.ChatID != "",
|
||||
Telegram: cfg.Telegram != nil && !cfg.Telegram.Disabled && cfg.Telegram.BotToken != "" && cfg.Telegram.ChatID != "",
|
||||
}
|
||||
if cfg.Voice == nil {
|
||||
return f
|
||||
@@ -678,16 +678,12 @@ func wireGatherer(st *store.Store, cfg *config.Config, rules []loop.Rule) *loop.
|
||||
// reconciled to "unknown" here, before the tick loop resumes sending, so
|
||||
// nothing auto-resends into that ambiguity.
|
||||
func wireDispatcher(st *store.Store, cfg *config.Config, voiceW *voiceWiring) (*delivery.Dispatcher, error) {
|
||||
var ntfy delivery.Sink
|
||||
if cfg.Ntfy != nil {
|
||||
s, err := ntfysink.New(*cfg.Ntfy)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("wire ntfy sink: %w", err)
|
||||
}
|
||||
ntfy = s
|
||||
ntfy, err := wireNtfySink(cfg.Ntfy)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var telegram delivery.Sink
|
||||
if cfg.Telegram != nil {
|
||||
if cfg.Telegram != nil && !cfg.Telegram.Disabled {
|
||||
s, err := telegramsink.New(*cfg.Telegram)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("wire telegram sink: %w", err)
|
||||
@@ -712,6 +708,22 @@ func wireDispatcher(st *store.Store, cfg *config.Config, voiceW *voiceWiring) (*
|
||||
}), nil
|
||||
}
|
||||
|
||||
// wireNtfySink keeps an optional reach optional without ever turning a missing
|
||||
// secret into anonymous publishing. A block is live unless it says disabled;
|
||||
// therefore an expanded-empty token in a live block fails startup instead of
|
||||
// spending days in a permanent 403 retry loop. Disabled is an explicit
|
||||
// operator choice and lets another away reach take over.
|
||||
func wireNtfySink(cfg *ntfysink.Config) (delivery.Sink, error) {
|
||||
if cfg == nil || cfg.Disabled {
|
||||
return nil, nil
|
||||
}
|
||||
sink, err := ntfysink.New(*cfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("wire ntfy sink: %w", err)
|
||||
}
|
||||
return sink, nil
|
||||
}
|
||||
|
||||
// wireTickLoop reads the loop's three intervals and its schedules out of the
|
||||
// config, so the two boot paths cannot disagree about them.
|
||||
func wireTickLoop(st *store.Store, gatherer *loop.Gatherer, dispatcher *delivery.Dispatcher, phr phraser.Phraser, rules []loop.Rule, cfg *config.Config) *tickLoop {
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/kami/maven/internal/delivery/ntfysink"
|
||||
)
|
||||
|
||||
func TestWireNtfySinkRejectsMissingCredentialWhenEnabled(t *testing.T) {
|
||||
_, err := wireNtfySink(&ntfysink.Config{
|
||||
BaseURL: "https://ntfy.example", Topic: "maven",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expanded-empty credential did not fail an enabled reach")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWireNtfySinkLeavesExplicitlyDisabledReachDark(t *testing.T) {
|
||||
sink, err := wireNtfySink(&ntfysink.Config{
|
||||
Disabled: true, BaseURL: "https://ntfy.example", Topic: "maven",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("wireNtfySink: %v", err)
|
||||
}
|
||||
if sink != nil {
|
||||
t.Fatal("disabled reach built a live sink")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWireNtfySinkRejectsMalformedEnabledConfig(t *testing.T) {
|
||||
_, err := wireNtfySink(&ntfysink.Config{Token: "token", Topic: "maven"})
|
||||
if err == nil {
|
||||
t.Fatal("malformed enabled config did not fail wiring")
|
||||
}
|
||||
}
|
||||
@@ -25,7 +25,7 @@ import (
|
||||
// already failed the boot in wireDispatcher for the same config, so a second
|
||||
// hard failure would only lose that message.
|
||||
func wireTelegramIntake(ctx context.Context, wg *sync.WaitGroup, api ipc.CoreAPI, cfg *config.Config) {
|
||||
if cfg == nil || cfg.Telegram == nil || !cfg.Telegram.Intake || api == nil {
|
||||
if cfg == nil || cfg.Telegram == nil || cfg.Telegram.Disabled || !cfg.Telegram.Intake || api == nil {
|
||||
return
|
||||
}
|
||||
sink, err := telegramsink.New(*cfg.Telegram)
|
||||
|
||||
+125
-11
@@ -10,11 +10,13 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -226,18 +228,11 @@ func (t *tickLoop) tick(ctx context.Context, now time.Time) {
|
||||
// detectPatterns below for how idempotence and dismissal are respected.
|
||||
t.detectPatterns(ctx, now, state)
|
||||
|
||||
// reminders: gate-bypassing class. fired once, marked after a successful
|
||||
// delivery. a failed send leaves the reminder pending — the next tick
|
||||
// re-gathers and re-attempts.
|
||||
// reminders: gate-bypassing class. The presentation and retry clock live on
|
||||
// the reminder occurrence, so a transport outage neither spends the model
|
||||
// every tick nor changes what the reminder says after a restart.
|
||||
for _, d := range loop.RemindDecisions(state, due) {
|
||||
pr, err := t.phraser.PhraseReminder(ctx, d)
|
||||
if err != nil {
|
||||
log.Printf("tick: phrase reminder %d: %v", d.Reminder.ID, err)
|
||||
continue
|
||||
}
|
||||
if _, err := t.dispatcher.DispatchReminder(ctx, pr, now); err != nil {
|
||||
log.Printf("tick: dispatch reminder %d: %v", d.Reminder.ID, err)
|
||||
}
|
||||
t.deliverReminder(ctx, d, now)
|
||||
}
|
||||
|
||||
// sev4-away repeats: re-send un-acked telegram nudges per repeatInterval.
|
||||
@@ -263,6 +258,125 @@ func (t *tickLoop) tick(ctx context.Context, now time.Time) {
|
||||
}
|
||||
}
|
||||
|
||||
// deliverReminder advances one due reminder (or collapsed bundle) through the
|
||||
// durable delivery state. A phrase is cached before the first external send;
|
||||
// every definite failure advances the persisted bounded backoff.
|
||||
func (t *tickLoop) deliverReminder(ctx context.Context, d loop.ReminderDecision, now time.Time) {
|
||||
originals := reminderOriginals(d.Reminder)
|
||||
pr, cached := cachedReminderPhrase(d, originals)
|
||||
if !cached {
|
||||
var err error
|
||||
pr, err = t.phraser.PhraseReminder(ctx, d)
|
||||
if err == nil && pr.Body == "" {
|
||||
err = errors.New("phraser returned an empty reminder body")
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("tick: phrase reminder %d: %v", d.Reminder.ID, err)
|
||||
t.scheduleReminderRetry(ctx, originals, now)
|
||||
return
|
||||
}
|
||||
if pr.Mood == "" {
|
||||
pr.Mood = "neutral"
|
||||
}
|
||||
group := reminderDeliveryGroup(originals)
|
||||
if err := t.store.CacheReminderPhrase(
|
||||
ctx, originals, group, pr.Body, pr.Summary, pr.Mood,
|
||||
); err != nil {
|
||||
// A cancellation or another completion can win while phrasing. Do
|
||||
// not send a presentation that no longer owns every original.
|
||||
log.Printf("tick: cache reminder %d phrase: %v", d.Reminder.ID, err)
|
||||
return
|
||||
}
|
||||
// The store now owns the phrase, but this tick's value predates that
|
||||
// write. Stamp the exact persisted occurrence identity onto the value
|
||||
// handed to the dispatcher so its outbox row can suppress an ambiguous
|
||||
// crash for both a real reminder and a synthetic collapsed bundle.
|
||||
for i := range originals {
|
||||
originals[i].DeliveryGroup = group
|
||||
originals[i].PhraseBody = pr.Body
|
||||
originals[i].PhraseSummary = pr.Summary
|
||||
originals[i].PhraseMood = pr.Mood
|
||||
}
|
||||
if d.Reminder.ID == 0 {
|
||||
d.Reminder.Collapsed = originals
|
||||
} else {
|
||||
d.Reminder = originals[0]
|
||||
}
|
||||
}
|
||||
// A phraser is not allowed to substitute the reminder decision. In
|
||||
// particular, the durable group stamped above must reach the outbox.
|
||||
pr.Decision = d
|
||||
|
||||
if _, err := t.dispatcher.DispatchReminder(ctx, pr, now); err != nil {
|
||||
log.Printf("tick: dispatch reminder %d: %v", d.Reminder.ID, err)
|
||||
t.scheduleReminderRetry(ctx, originals, now)
|
||||
}
|
||||
}
|
||||
|
||||
func (t *tickLoop) scheduleReminderRetry(ctx context.Context, originals []store.Reminder, now time.Time) {
|
||||
if err := t.store.ScheduleReminderRetry(ctx, originals, now); err != nil {
|
||||
log.Printf("tick: schedule reminder retry: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// reminderOriginals converts the synthetic ID=0 bundle back to real store
|
||||
// rows. Keeping this in one helper makes it impossible to accidentally persist
|
||||
// retry state against reminder zero.
|
||||
func reminderOriginals(r store.Reminder) []store.Reminder {
|
||||
if r.ID == 0 {
|
||||
return append([]store.Reminder(nil), r.Collapsed...)
|
||||
}
|
||||
return []store.Reminder{r}
|
||||
}
|
||||
|
||||
// cachedReminderPhrase reconstructs a PhrasedReminder only when every original
|
||||
// agrees on one persisted group and presentation. That agreement is what lets
|
||||
// a collapsed bundle survive a restart without being re-phrased.
|
||||
func cachedReminderPhrase(d loop.ReminderDecision, originals []store.Reminder) (delivery.PhrasedReminder, bool) {
|
||||
if len(originals) == 0 || !originals[0].HasDeliveryPhrase() {
|
||||
return delivery.PhrasedReminder{}, false
|
||||
}
|
||||
first := originals[0]
|
||||
for _, r := range originals[1:] {
|
||||
if !r.HasDeliveryPhrase() ||
|
||||
r.DeliveryGroup != first.DeliveryGroup ||
|
||||
r.PhraseBody != first.PhraseBody ||
|
||||
r.PhraseSummary != first.PhraseSummary ||
|
||||
r.PhraseMood != first.PhraseMood {
|
||||
return delivery.PhrasedReminder{}, false
|
||||
}
|
||||
}
|
||||
mood := first.PhraseMood
|
||||
if mood == "" {
|
||||
mood = "neutral"
|
||||
}
|
||||
return delivery.PhrasedReminder{
|
||||
Decision: d,
|
||||
Body: first.PhraseBody,
|
||||
Summary: first.PhraseSummary,
|
||||
Mood: mood,
|
||||
}, true
|
||||
}
|
||||
|
||||
// reminderDeliveryGroup deterministically names one occurrence or collapsed
|
||||
// set. The next-fire instant is part of the identity so a recurring reminder's
|
||||
// later occurrence can never inherit the previous occurrence's phrase.
|
||||
func reminderDeliveryGroup(originals []store.Reminder) string {
|
||||
ordered := append([]store.Reminder(nil), originals...)
|
||||
sort.Slice(ordered, func(i, j int) bool {
|
||||
if ordered[i].ID == ordered[j].ID {
|
||||
return ordered[i].NextFireTs.Before(ordered[j].NextFireTs)
|
||||
}
|
||||
return ordered[i].ID < ordered[j].ID
|
||||
})
|
||||
h := sha256.New()
|
||||
for _, r := range ordered {
|
||||
_, _ = fmt.Fprintf(h, "%d:%d;", r.ID, r.NextFireTs.UnixMilli())
|
||||
}
|
||||
sum := h.Sum(nil)
|
||||
return fmt.Sprintf("reminder:%x", sum[:12])
|
||||
}
|
||||
|
||||
// savePresence writes back the bucket GatherState just resolved.
|
||||
//
|
||||
// It lives here and not in GatherState because that method holds a read-only
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/delivery"
|
||||
"github.com/kami/maven/internal/loop"
|
||||
"github.com/kami/maven/internal/phraser"
|
||||
"github.com/kami/maven/internal/store"
|
||||
)
|
||||
|
||||
type reminderCountingPhraser struct {
|
||||
phraser.Phraser
|
||||
calls int
|
||||
body string
|
||||
summary string
|
||||
mood string
|
||||
}
|
||||
|
||||
func (p *reminderCountingPhraser) PhraseReminder(_ context.Context, d loop.ReminderDecision) (delivery.PhrasedReminder, error) {
|
||||
p.calls++
|
||||
return delivery.PhrasedReminder{
|
||||
Decision: d,
|
||||
Body: p.body,
|
||||
Summary: p.summary,
|
||||
Mood: p.mood,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type reminderFailSink struct {
|
||||
sends int
|
||||
}
|
||||
|
||||
func (s *reminderFailSink) Send(_ context.Context, _ delivery.Sendable) error {
|
||||
s.sends++
|
||||
return errors.New("transport unavailable")
|
||||
}
|
||||
|
||||
func newReminderDeliveryLoop(t *testing.T, st *store.Store, sink delivery.Sink, p phraser.Phraser) *tickLoop {
|
||||
t.Helper()
|
||||
rules := loop.DefaultRules()
|
||||
return newTickLoop(
|
||||
st,
|
||||
loop.NewGatherer(st, rules),
|
||||
delivery.NewDispatcher(delivery.Config{
|
||||
Voice: sink, Ntfy: sink, Telegram: sink,
|
||||
Nudges: st, Reminders: st, Outbox: st,
|
||||
}),
|
||||
p,
|
||||
rules,
|
||||
time.Second,
|
||||
5*time.Minute,
|
||||
0,
|
||||
nil, nil, nil, nil,
|
||||
)
|
||||
}
|
||||
|
||||
func TestTickReminderRetryUsesPersistedPhraseAfterRestart(t *testing.T) {
|
||||
st := newTestStore(t)
|
||||
ctx := context.Background()
|
||||
now := refNow()
|
||||
if _, err := st.CreateReminder(ctx, now.Add(-time.Minute), `{"text":"позвонить маме"}`, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
fail := &reminderFailSink{}
|
||||
firstPhraser := &reminderCountingPhraser{
|
||||
Phraser: phraser.NewStub(), body: "Не забудь позвонить маме.",
|
||||
summary: "Позвонить маме", mood: "warm",
|
||||
}
|
||||
tl := newReminderDeliveryLoop(t, st, fail, firstPhraser)
|
||||
tl.tick(ctx, now)
|
||||
if firstPhraser.calls != 1 {
|
||||
t.Fatalf("first tick phrased %d times, want 1", firstPhraser.calls)
|
||||
}
|
||||
rows, err := st.ListReminders(ctx, 1)
|
||||
if err != nil || len(rows) != 1 {
|
||||
t.Fatalf("list = %d, err=%v", len(rows), err)
|
||||
}
|
||||
if !rows[0].HasDeliveryPhrase() || rows[0].DeliveryAttempts != 1 {
|
||||
t.Fatalf("failed delivery state was not persisted: %+v", rows[0])
|
||||
}
|
||||
if want := now.Add(store.ReminderRetryBase); !rows[0].NextAttemptTs.Equal(want) {
|
||||
t.Fatalf("next attempt = %s, want %s", rows[0].NextAttemptTs, want)
|
||||
}
|
||||
|
||||
// A normal tick inside the wait does no transport work and no model work.
|
||||
sendsAfterFirst := fail.sends
|
||||
tl.tick(ctx, now.Add(30*time.Second))
|
||||
if firstPhraser.calls != 1 || fail.sends != sendsAfterFirst {
|
||||
t.Fatalf("retry wait did work: phrase calls=%d, sends=%d (was %d)", firstPhraser.calls, fail.sends, sendsAfterFirst)
|
||||
}
|
||||
|
||||
// Constructing a new loop is the daemon-restart boundary. Its phraser would
|
||||
// say something different if called; the stored phrase must win instead.
|
||||
success := &fakeSink{}
|
||||
afterRestart := &reminderCountingPhraser{
|
||||
Phraser: phraser.NewStub(), body: "WRONG NEW PHRASE", summary: "WRONG", mood: "neutral",
|
||||
}
|
||||
restarted := newReminderDeliveryLoop(t, st, success, afterRestart)
|
||||
restarted.tick(ctx, now.Add(store.ReminderRetryBase))
|
||||
if afterRestart.calls != 0 {
|
||||
t.Fatalf("restart re-phrased the reminder %d times", afterRestart.calls)
|
||||
}
|
||||
if len(success.sends) != 1 {
|
||||
t.Fatalf("retry sends = %d, want 1", len(success.sends))
|
||||
}
|
||||
if got := success.sends[0].Body; got != "Позвонить маме" {
|
||||
t.Fatalf("away retry body = %q, want persisted summary", got)
|
||||
}
|
||||
rows, err = st.ListReminders(ctx, 1)
|
||||
if err != nil || rows[0].Status != store.ReminderFired {
|
||||
t.Fatalf("successful retry did not fire reminder: rows=%+v err=%v", rows, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTickCollapsedReminderRetriesOnePhraseAndCompletesOriginals(t *testing.T) {
|
||||
st := newTestStore(t)
|
||||
ctx := context.Background()
|
||||
now := refNow()
|
||||
for _, text := range []string{"полить цветы", "записаться к врачу"} {
|
||||
if _, err := st.CreateReminder(ctx, now.Add(-time.Minute), text, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
fail := &reminderFailSink{}
|
||||
firstPhraser := &reminderCountingPhraser{
|
||||
Phraser: phraser.NewStub(), body: "У тебя два напоминания.",
|
||||
summary: "Два напоминания", mood: "neutral",
|
||||
}
|
||||
newReminderDeliveryLoop(t, st, fail, firstPhraser).tick(ctx, now)
|
||||
if firstPhraser.calls != 1 {
|
||||
t.Fatalf("collapsed bundle phrased %d times, want 1", firstPhraser.calls)
|
||||
}
|
||||
rows, err := st.ListReminders(ctx, 10)
|
||||
if err != nil || len(rows) != 2 {
|
||||
t.Fatalf("list = %d, err=%v", len(rows), err)
|
||||
}
|
||||
for _, r := range rows {
|
||||
if r.DeliveryGroup == "" || r.DeliveryGroup != rows[0].DeliveryGroup ||
|
||||
r.PhraseBody != "У тебя два напоминания." || r.DeliveryAttempts != 1 {
|
||||
t.Fatalf("collapsed original lost shared state: %+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
success := &fakeSink{}
|
||||
afterRestart := &reminderCountingPhraser{
|
||||
Phraser: phraser.NewStub(), body: "WRONG", summary: "WRONG", mood: "neutral",
|
||||
}
|
||||
newReminderDeliveryLoop(t, st, success, afterRestart).tick(ctx, now.Add(store.ReminderRetryBase))
|
||||
if afterRestart.calls != 0 {
|
||||
t.Fatalf("collapsed retry re-phrased %d times", afterRestart.calls)
|
||||
}
|
||||
if len(success.sends) != 1 || success.sends[0].ReminderID != 0 {
|
||||
t.Fatalf("collapsed retry sends = %+v, want one synthetic delivery", success.sends)
|
||||
}
|
||||
rows, err = st.ListReminders(ctx, 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range rows {
|
||||
if r.Status != store.ReminderFired {
|
||||
t.Fatalf("collapsed original %d status = %q, want fired", r.ID, r.Status)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -377,19 +377,19 @@ func wireVoice(cfg *config.Config, coreAPI ipc.CoreAPI, phr phraser.Phraser, mem
|
||||
// degraded mode, so the seam is nil and the cascade routes with the classifier.
|
||||
func modelSeam(cfg *config.Config, resident *llm.Client) (router.Completer, *llm.Pair) {
|
||||
if resident == nil {
|
||||
if cfg.Workstation != nil {
|
||||
if cfg.Workstation != nil && !cfg.Workstation.ModelDisabled {
|
||||
log.Printf("voice: a workstation is configured but there is no resident model to floor it with — ignoring the block")
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
if cfg.Workstation == nil {
|
||||
if cfg.Workstation == nil || cfg.Workstation.ModelDisabled {
|
||||
return resident, nil
|
||||
}
|
||||
ws := cfg.Workstation
|
||||
remote := llm.New(ws.URL, time.Duration(ws.Timeout))
|
||||
remote.SetToken(ws.Token)
|
||||
if ws.Token == "" {
|
||||
log.Printf("voice: no workstation.token — mavgpud refuses an unauthenticated request, so this reads as a card that is always busy")
|
||||
log.Printf("voice: unauthenticated workstation model endpoint is loopback-only")
|
||||
}
|
||||
pair := llm.NewPair(
|
||||
remote,
|
||||
@@ -434,7 +434,7 @@ func sttSeam(cfg *config.Config, floor stt.Transcriber) (stt.Transcriber, *stt.P
|
||||
)
|
||||
pair.Start(context.Background())
|
||||
if s.Token == "" {
|
||||
log.Print("voice: the workstation transcriber has no token, so anything on the LAN can post audio to it")
|
||||
log.Print("voice: unauthenticated workstation transcriber endpoint is loopback-only")
|
||||
}
|
||||
log.Printf("voice: workstation transcriber at %s, probed every %s, mavsttd as the floor",
|
||||
s.URL, time.Duration(s.Probe))
|
||||
|
||||
Reference in New Issue
Block a user