Admit a first lease when a quota limit has no receipt history

Burn-in run 2 ingested its task and then sat queued forever. Every herdr in
the live config declares quota_limit_5h and quota_limit_weekly, the event log
holds zero QuotaReported events, and QuotaSince reported an empty window as
unknown. QuotaAvailability fails closed on unknown, so no harness could ever
be leased, and the only producer of a receipt is a completed lease.

The event log is Orchestra's whole accounting source, so a window holding no
receipts is observable zero consumption. QuotaSince now reports known for an
empty window and for a harness that has never reported. A receipt that
declares its own consumption unknown still fails closed.

The refusal also lied about its cause. federatedAvailability collapsed a base
gate refusal into the federation health string, so router health said "stale
heartbeat or unhealthy local backend" while the heartbeat was one second old.
Availability gates now name themselves through an optional
ReasonedAvailability contract: quota refusals say whether usage is unknown or
the window is exhausted and by how much, and worker refusals distinguish an
unregistered worker, a never-probed backend, a stale heartbeat, a stale
health check, and an unreachable backend.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
2026-08-27 00:28:57 +04:00
parent c833e0eb62
commit 0d67af9976
7 changed files with 277 additions and 19 deletions
+63 -6
View File
@@ -3,6 +3,7 @@ package router
import (
"encoding/json"
"fmt"
"errors"
"orchestra/internal/authz"
"orchestra/internal/domain"
@@ -21,6 +22,15 @@ type Availability interface{ Available(h registry.Herdr) bool }
type ProjectAvailability interface {
Supports(h registry.Herdr, project string) bool
}
// ReasonedAvailability is an optional contract that names why a herdr was
// refused. Without it every refusal collapses into one string, which told an
// operator "stale heartbeat" while the heartbeat was one second old and the
// real refusal came from the quota gate. Found on burn-in run 2, 2026-08-26.
type ReasonedAvailability interface {
// Unavailable returns "" when the herdr may be leased, otherwise the
// specific reason.
Unavailable(h registry.Herdr) string
}
type AlwaysAvailable struct{}
func (AlwaysAvailable) Available(registry.Herdr) bool { return true }
@@ -85,6 +95,53 @@ func (q QuotaAvailability) Available(h registry.Herdr) bool {
return true
}
func (q QuotaAvailability) Unavailable(h registry.Herdr) string {
if q.Store == nil {
return "quota unavailable: no receipt store"
}
limits, bounded := q.Limits[h.ID]
if !bounded || (limits.FiveHour <= 0 && limits.Weekly <= 0) {
return ""
}
now := time.Now()
if q.Now != nil {
now = q.Now()
}
windows := []struct {
name string
limit float64
since time.Time
}{
{"5h", limits.FiveHour, now.Add(-fiveHourWindow)},
{"weekly", limits.Weekly, now.Add(-weeklyWindow)},
}
for _, w := range windows {
if w.limit <= 0 {
continue
}
used, known := q.sumSince(h.ID, w.since)
if !known {
return "quota unavailable: " + w.name + " usage unknown"
}
if cap := w.limit * quotaConservativeFraction; used >= cap {
return fmt.Sprintf("quota exhausted: %s window %.0f/%.0f conservative limit (of %.0f)", w.name, used, cap, w.limit)
}
}
return ""
}
// unavailableReason prefers a specific reason when the availability supports
// one, and never lets a bare Available disagree with it.
func unavailableReason(a Availability, h registry.Herdr) string {
if ra, ok := a.(ReasonedAvailability); ok {
return ra.Unavailable(h)
}
if a.Available(h) {
return ""
}
return "worker unavailable"
}
type RetryPolicy struct {
MaxAttempts int
Backoff time.Duration
@@ -197,7 +254,7 @@ func (r *Router) AssignPending() ([]domain.Event, error) {
sort.SliceStable(queued, func(i, j int) bool { return importance(queued[i], now).Before(importance(queued[j], now)) })
health := r.healthSnapshot(now)
candidates := make(map[string][]registry.Herdr)
availability := make(map[string]bool)
availability := make(map[string]string)
availabilityKnown := make(map[string]bool)
projectSupport := make(map[string]bool)
projectSupportKnown := make(map[string]bool)
@@ -245,13 +302,13 @@ func (r *Router) AssignPending() ([]domain.Event, error) {
r.reject(t.ID, h.ID, "capability mismatch")
continue
}
available, checked := availability[h.ID], availabilityKnown[h.ID]
reason, checked := availability[h.ID], availabilityKnown[h.ID]
if !checked {
available = r.Availability.Available(h)
availability[h.ID], availabilityKnown[h.ID] = available, true
reason = unavailableReason(r.Availability, h)
availability[h.ID], availabilityKnown[h.ID] = reason, true
}
if !available {
r.reject(t.ID, h.ID, "worker unavailable: stale heartbeat or unhealthy local backend")
if reason != "" {
r.reject(t.ID, h.ID, reason)
continue
}
if occupiedCount(activeLeases[h.ID], h.Concurrency) {