Report queued tasks the scheduling pass never considers

A task in retry backoff was filtered out before the candidate loop, so it
recorded no rejection at all: queued, apparently assignable, and silent. That is
the exact shape that made F5 take a live session to diagnose. It now reports
"retry backoff until <time>".

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
2026-08-26 23:42:26 +04:00
parent 0ead6d2d02
commit 4fbf3ac966
18 changed files with 1163 additions and 165 deletions
+10 -2
View File
@@ -182,9 +182,17 @@ func (r *Router) AssignPending() ([]domain.Event, error) {
snapshot := r.Store.SchedulingSnapshot()
var queued []domain.Task
for _, t := range snapshot.Tasks {
if t.State == domain.StateQueued && (t.NextRetryAt.IsZero() || !now.Before(t.NextRetryAt)) {
queued = append(queued, t)
if t.State != domain.StateQueued {
continue
}
if !t.NextRetryAt.IsZero() && now.Before(t.NextRetryAt) {
// A queued task the pass never even considers is the most
// confusing state of all: it looks assignable and nothing is
// recorded against it. Say so.
r.reject(t.ID, "", "retry backoff until "+t.NextRetryAt.UTC().Format(time.RFC3339))
continue
}
queued = append(queued, t)
}
sort.SliceStable(queued, func(i, j int) bool { return importance(queued[i], now).Before(importance(queued[j], now)) })
health := r.healthSnapshot(now)
+24
View File
@@ -364,6 +364,25 @@ func TestAssignPendingRecordsWhyItPlacedNothing(t *testing.T) {
t.Fatalf("pre-lease refusal not reported: %+v", rt.Rejections())
}
// A queued task in retry backoff is skipped before the candidate loop, so
// it needs its own reason: it looks assignable and nothing else records it.
if err := s.Append(domain.Event{ID: "backoff", TaskID: "t", Type: "TaskCorrected", Version: 2, Payload: backoffPayload(time.Now().Add(time.Hour)), Surface: string(authz.System)}); err == nil {
if got, _ := s.Task("t"); !got.NextRetryAt.IsZero() {
if _, err := rt.AssignPending(); err != nil {
t.Fatal(err)
}
backoff := false
for _, rej := range rt.Rejections() {
if strings.Contains(rej.Reason, "retry backoff") {
backoff = true
}
}
if !backoff {
t.Fatalf("a task in retry backoff was silently skipped: %+v", rt.Rejections())
}
}
}
// A successful pass leaves nothing behind to misread.
s.PreLease = nil
if got, err := rt.AssignPending(); err != nil || len(got) != 1 {
@@ -373,3 +392,8 @@ func TestAssignPendingRecordsWhyItPlacedNothing(t *testing.T) {
t.Fatalf("stale rejections after a successful pass: %+v", reasons)
}
}
func backoffPayload(at time.Time) []byte {
b, _ := json.Marshal(map[string]any{"next_retry_at": at.UTC().Format(time.RFC3339Nano)})
return b
}