Admit a first lease when a quota limit has no receipt history
Burn-in run 2 ingested its task and then sat queued forever. Every herdr in the live config declares quota_limit_5h and quota_limit_weekly, the event log holds zero QuotaReported events, and QuotaSince reported an empty window as unknown. QuotaAvailability fails closed on unknown, so no harness could ever be leased, and the only producer of a receipt is a completed lease. The event log is Orchestra's whole accounting source, so a window holding no receipts is observable zero consumption. QuotaSince now reports known for an empty window and for a harness that has never reported. A receipt that declares its own consumption unknown still fails closed. The refusal also lied about its cause. federatedAvailability collapsed a base gate refusal into the federation health string, so router health said "stale heartbeat or unhealthy local backend" while the heartbeat was one second old. Availability gates now name themselves through an optional ReasonedAvailability contract: quota refusals say whether usage is unknown or the window is exhausted and by how much, and worker refusals distinguish an unregistered worker, a never-probed backend, a stale heartbeat, a stale health check, and an unreachable backend. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -413,21 +413,38 @@ func (r *Registry) Heartbeat(id string, health ...WorkerHealth) error {
|
||||
// Available refreshes TTL state and reports whether a registered worker owns
|
||||
// this harness id. Router admission uses it so a reachable TCP bridge alone
|
||||
// can never make an offline worker eligible for a lease.
|
||||
func (r *Registry) Available(id string) bool {
|
||||
func (r *Registry) Available(id string) bool { return r.Unavailable(id) == "" }
|
||||
|
||||
// Unavailable refreshes TTL state and returns "" when a registered worker owns
|
||||
// this harness id and may be leased, otherwise the specific reason. A single
|
||||
// collapsed reason once reported "stale heartbeat" for a worker whose
|
||||
// heartbeat was one second old, so each condition names itself.
|
||||
func (r *Registry) Unavailable(id string) string {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.init()
|
||||
w, ok := r.workers[id]
|
||||
if !ok {
|
||||
return false
|
||||
return "worker unavailable: no worker registered for this harness"
|
||||
}
|
||||
// A heartbeat merely proves the worker process can reach the coordinator.
|
||||
// Lease admission additionally requires a fresh probe of the worker's
|
||||
// local execution backend; otherwise a partitioned/down backend still
|
||||
// attracts work.
|
||||
w.Online = time.Since(w.LastSeen) <= r.TTL && w.Health.HerdrStatus == "reachable" && !w.Health.CheckedAt.IsZero() && time.Since(w.Health.CheckedAt) <= r.TTL
|
||||
reason := ""
|
||||
switch {
|
||||
case time.Since(w.LastSeen) > r.TTL:
|
||||
reason = "worker unavailable: stale heartbeat"
|
||||
case w.Health.CheckedAt.IsZero():
|
||||
reason = "worker unavailable: backend health never reported"
|
||||
case time.Since(w.Health.CheckedAt) > r.TTL:
|
||||
reason = "worker unavailable: stale backend health check"
|
||||
case w.Health.HerdrStatus != "reachable":
|
||||
reason = "worker unavailable: backend " + w.Health.HerdrStatus
|
||||
}
|
||||
w.Online = reason == ""
|
||||
r.workers[id] = w
|
||||
return w.Online
|
||||
return reason
|
||||
}
|
||||
|
||||
// Supports reports whether an online worker explicitly declared the project.
|
||||
|
||||
@@ -268,3 +268,40 @@ func TestResolvedCommandsArePrunedButPendingOnesSurvive(t *testing.T) {
|
||||
t.Fatalf("pending command was pruned: %#v ok=%v", c, ok)
|
||||
}
|
||||
}
|
||||
|
||||
// Each unavailability condition must name itself. One collapsed reason once
|
||||
// reported a stale heartbeat for a worker whose heartbeat was a second old.
|
||||
func TestUnavailableNamesTheFailingCondition(t *testing.T) {
|
||||
r := &Registry{TTL: time.Minute}
|
||||
if got, want := r.Unavailable("nobody"), "worker unavailable: no worker registered for this harness"; got != want {
|
||||
t.Fatalf("unregistered reason=%q, want %q", got, want)
|
||||
}
|
||||
if err := r.Register(Worker{ID: "w", Token: "t"}, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, want := r.Unavailable("w"), "worker unavailable: backend health never reported"; got != want {
|
||||
t.Fatalf("unprobed reason=%q, want %q", got, want)
|
||||
}
|
||||
if err := r.Heartbeat("w", WorkerHealth{HerdrStatus: "unreachable", CheckedAt: time.Now().UTC()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, want := r.Unavailable("w"), "worker unavailable: backend unreachable"; got != want {
|
||||
t.Fatalf("unhealthy-backend reason=%q, want %q", got, want)
|
||||
}
|
||||
if err := r.Heartbeat("w", WorkerHealth{HerdrStatus: "reachable", CheckedAt: time.Now().UTC()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := r.Unavailable("w"); got != "" {
|
||||
t.Fatalf("fresh reachable worker reason=%q, want admitted", got)
|
||||
}
|
||||
stale := &Registry{TTL: time.Nanosecond}
|
||||
if err := stale.Register(Worker{ID: "w", Token: "t"}, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := stale.Heartbeat("w", WorkerHealth{HerdrStatus: "reachable", CheckedAt: time.Now().UTC()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, want := stale.Unavailable("w"), "worker unavailable: stale heartbeat"; got != want {
|
||||
t.Fatalf("stale reason=%q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user