Admit a first lease when a quota limit has no receipt history
Burn-in run 2 ingested its task and then sat queued forever. Every herdr in the live config declares quota_limit_5h and quota_limit_weekly, the event log holds zero QuotaReported events, and QuotaSince reported an empty window as unknown. QuotaAvailability fails closed on unknown, so no harness could ever be leased, and the only producer of a receipt is a completed lease. The event log is Orchestra's whole accounting source, so a window holding no receipts is observable zero consumption. QuotaSince now reports known for an empty window and for a harness that has never reported. A receipt that declares its own consumption unknown still fails closed. The refusal also lied about its cause. federatedAvailability collapsed a base gate refusal into the federation health string, so router health said "stale heartbeat or unhealthy local backend" while the heartbeat was one second old. Availability gates now name themselves through an optional ReasonedAvailability contract: quota refusals say whether usage is unknown or the window is exhausted and by how much, and worker refusals distinguish an unregistered worker, a never-probed backend, a stale heartbeat, a stale health check, and an unreachable backend. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
+10
-2
@@ -910,16 +910,24 @@ func (s *Store) SchedulingSnapshot() SchedulingSnapshot {
|
||||
// QuotaSince answers a rolling-window usage query from the per-harness index
|
||||
// instead of walking events.jsonl. known is false when the interval has no
|
||||
// native receipt or any receipt explicitly reports unknown usage.
|
||||
// QuotaSince sums the receipts at or after `since`. The event log is
|
||||
// Orchestra's complete accounting source, so a window holding no receipts is
|
||||
// observable zero consumption, not missing data: it reports known. `known` is
|
||||
// false only when a receipt inside the window said its own consumption was
|
||||
// unknown. Reporting an empty window as unknown deadlocked admission — a
|
||||
// harness with a configured quota limit and no receipt history could never be
|
||||
// leased, and the only producer of a receipt is a completed lease. Found on
|
||||
// burn-in run 2, 2026-08-26.
|
||||
func (s *Store) QuotaSince(harness string, since time.Time) (consumed float64, known bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
index, ok := s.quota[harness]
|
||||
if !ok {
|
||||
return 0, false
|
||||
return 0, true
|
||||
}
|
||||
start := sort.Search(len(index.records), func(i int) bool { return !index.records[i].At.Before(since) })
|
||||
if start == len(index.records) {
|
||||
return 0, false
|
||||
return 0, true
|
||||
}
|
||||
return index.prefix[len(index.records)] - index.prefix[start], index.unknownPrefix[len(index.records)] == index.unknownPrefix[start]
|
||||
}
|
||||
|
||||
@@ -637,3 +637,29 @@ func TestReleaseTransactionSurvivesReLeaseUntilMatchingPickup(t *testing.T) {
|
||||
t.Fatalf("pickup not bound to transaction/epoch: %+v", task)
|
||||
}
|
||||
}
|
||||
|
||||
// An empty window is observable zero consumption, not missing data. Reporting
|
||||
// it as unknown made the router refuse every harness that had a configured
|
||||
// quota limit and no receipt history, which no first lease could ever produce.
|
||||
func TestQuotaSinceReportsEmptyWindowAsKnownZero(t *testing.T) {
|
||||
s, err := Open(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if used, known := s.QuotaSince("fresh", now.Add(-fiveHours)); used != 0 || !known {
|
||||
t.Fatalf("never-reported harness quota=(%v,%v), want (0,true)", used, known)
|
||||
}
|
||||
payload, _ := json.Marshal(map[string]any{"harness_id": "h1", "consumed": 7.0, "known": true})
|
||||
if err := s.Append(domain.Event{ID: domain.NewID(), Type: "QuotaReported", TaskID: "quota", Version: 1, At: now.Add(-24 * time.Hour), Payload: payload, Surface: string(authz.System)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if used, known := s.QuotaSince("h1", now.Add(-fiveHours)); used != 0 || !known {
|
||||
t.Fatalf("receipts only outside the window quota=(%v,%v), want (0,true)", used, known)
|
||||
}
|
||||
if used, known := s.QuotaSince("h1", now.Add(-48*time.Hour)); used != 7 || !known {
|
||||
t.Fatalf("receipts inside the window quota=(%v,%v), want (7,true)", used, known)
|
||||
}
|
||||
}
|
||||
|
||||
const fiveHours = 5 * time.Hour
|
||||
|
||||
Reference in New Issue
Block a user