Report queued tasks the scheduling pass never considers
A task in retry backoff was filtered out before the candidate loop, so it recorded no rejection at all: queued, apparently assignable, and silent. That is the exact shape that made F5 take a live session to diagnose. It now reports "retry backoff until <time>". Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -182,9 +182,17 @@ func (r *Router) AssignPending() ([]domain.Event, error) {
|
||||
snapshot := r.Store.SchedulingSnapshot()
|
||||
var queued []domain.Task
|
||||
for _, t := range snapshot.Tasks {
|
||||
if t.State == domain.StateQueued && (t.NextRetryAt.IsZero() || !now.Before(t.NextRetryAt)) {
|
||||
queued = append(queued, t)
|
||||
if t.State != domain.StateQueued {
|
||||
continue
|
||||
}
|
||||
if !t.NextRetryAt.IsZero() && now.Before(t.NextRetryAt) {
|
||||
// A queued task the pass never even considers is the most
|
||||
// confusing state of all: it looks assignable and nothing is
|
||||
// recorded against it. Say so.
|
||||
r.reject(t.ID, "", "retry backoff until "+t.NextRetryAt.UTC().Format(time.RFC3339))
|
||||
continue
|
||||
}
|
||||
queued = append(queued, t)
|
||||
}
|
||||
sort.SliceStable(queued, func(i, j int) bool { return importance(queued[i], now).Before(importance(queued[j], now)) })
|
||||
health := r.healthSnapshot(now)
|
||||
|
||||
@@ -364,6 +364,25 @@ func TestAssignPendingRecordsWhyItPlacedNothing(t *testing.T) {
|
||||
t.Fatalf("pre-lease refusal not reported: %+v", rt.Rejections())
|
||||
}
|
||||
|
||||
// A queued task in retry backoff is skipped before the candidate loop, so
|
||||
// it needs its own reason: it looks assignable and nothing else records it.
|
||||
if err := s.Append(domain.Event{ID: "backoff", TaskID: "t", Type: "TaskCorrected", Version: 2, Payload: backoffPayload(time.Now().Add(time.Hour)), Surface: string(authz.System)}); err == nil {
|
||||
if got, _ := s.Task("t"); !got.NextRetryAt.IsZero() {
|
||||
if _, err := rt.AssignPending(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
backoff := false
|
||||
for _, rej := range rt.Rejections() {
|
||||
if strings.Contains(rej.Reason, "retry backoff") {
|
||||
backoff = true
|
||||
}
|
||||
}
|
||||
if !backoff {
|
||||
t.Fatalf("a task in retry backoff was silently skipped: %+v", rt.Rejections())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A successful pass leaves nothing behind to misread.
|
||||
s.PreLease = nil
|
||||
if got, err := rt.AssignPending(); err != nil || len(got) != 1 {
|
||||
@@ -373,3 +392,8 @@ func TestAssignPendingRecordsWhyItPlacedNothing(t *testing.T) {
|
||||
t.Fatalf("stale rejections after a successful pass: %+v", reasons)
|
||||
}
|
||||
}
|
||||
|
||||
func backoffPayload(at time.Time) []byte {
|
||||
b, _ := json.Marshal(map[string]any{"next_retry_at": at.UTC().Format(time.RFC3339Nano)})
|
||||
return b
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user