Admit a first lease when a quota limit has no receipt history
Burn-in run 2 ingested its task and then sat queued forever. Every herdr in the live config declares quota_limit_5h and quota_limit_weekly, the event log holds zero QuotaReported events, and QuotaSince reported an empty window as unknown. QuotaAvailability fails closed on unknown, so no harness could ever be leased, and the only producer of a receipt is a completed lease. The event log is Orchestra's whole accounting source, so a window holding no receipts is observable zero consumption. QuotaSince now reports known for an empty window and for a harness that has never reported. A receipt that declares its own consumption unknown still fails closed. The refusal also lied about its cause. federatedAvailability collapsed a base gate refusal into the federation health string, so router health said "stale heartbeat or unhealthy local backend" while the heartbeat was one second old. Availability gates now name themselves through an optional ReasonedAvailability contract: quota refusals say whether usage is unknown or the window is exhausted and by how much, and worker refusals distinguish an unregistered worker, a never-probed backend, a stale heartbeat, a stale health check, and an unreachable backend. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -3,6 +3,7 @@ package router
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"errors"
|
||||
"orchestra/internal/authz"
|
||||
"orchestra/internal/domain"
|
||||
@@ -21,6 +22,15 @@ type Availability interface{ Available(h registry.Herdr) bool }
|
||||
type ProjectAvailability interface {
|
||||
Supports(h registry.Herdr, project string) bool
|
||||
}
|
||||
// ReasonedAvailability is an optional contract that names why a herdr was
|
||||
// refused. Without it every refusal collapses into one string, which told an
|
||||
// operator "stale heartbeat" while the heartbeat was one second old and the
|
||||
// real refusal came from the quota gate. Found on burn-in run 2, 2026-08-26.
|
||||
type ReasonedAvailability interface {
|
||||
// Unavailable returns "" when the herdr may be leased, otherwise the
|
||||
// specific reason.
|
||||
Unavailable(h registry.Herdr) string
|
||||
}
|
||||
type AlwaysAvailable struct{}
|
||||
|
||||
func (AlwaysAvailable) Available(registry.Herdr) bool { return true }
|
||||
@@ -85,6 +95,53 @@ func (q QuotaAvailability) Available(h registry.Herdr) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (q QuotaAvailability) Unavailable(h registry.Herdr) string {
|
||||
if q.Store == nil {
|
||||
return "quota unavailable: no receipt store"
|
||||
}
|
||||
limits, bounded := q.Limits[h.ID]
|
||||
if !bounded || (limits.FiveHour <= 0 && limits.Weekly <= 0) {
|
||||
return ""
|
||||
}
|
||||
now := time.Now()
|
||||
if q.Now != nil {
|
||||
now = q.Now()
|
||||
}
|
||||
windows := []struct {
|
||||
name string
|
||||
limit float64
|
||||
since time.Time
|
||||
}{
|
||||
{"5h", limits.FiveHour, now.Add(-fiveHourWindow)},
|
||||
{"weekly", limits.Weekly, now.Add(-weeklyWindow)},
|
||||
}
|
||||
for _, w := range windows {
|
||||
if w.limit <= 0 {
|
||||
continue
|
||||
}
|
||||
used, known := q.sumSince(h.ID, w.since)
|
||||
if !known {
|
||||
return "quota unavailable: " + w.name + " usage unknown"
|
||||
}
|
||||
if cap := w.limit * quotaConservativeFraction; used >= cap {
|
||||
return fmt.Sprintf("quota exhausted: %s window %.0f/%.0f conservative limit (of %.0f)", w.name, used, cap, w.limit)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// unavailableReason prefers a specific reason when the availability supports
|
||||
// one, and never lets a bare Available disagree with it.
|
||||
func unavailableReason(a Availability, h registry.Herdr) string {
|
||||
if ra, ok := a.(ReasonedAvailability); ok {
|
||||
return ra.Unavailable(h)
|
||||
}
|
||||
if a.Available(h) {
|
||||
return ""
|
||||
}
|
||||
return "worker unavailable"
|
||||
}
|
||||
|
||||
type RetryPolicy struct {
|
||||
MaxAttempts int
|
||||
Backoff time.Duration
|
||||
@@ -197,7 +254,7 @@ func (r *Router) AssignPending() ([]domain.Event, error) {
|
||||
sort.SliceStable(queued, func(i, j int) bool { return importance(queued[i], now).Before(importance(queued[j], now)) })
|
||||
health := r.healthSnapshot(now)
|
||||
candidates := make(map[string][]registry.Herdr)
|
||||
availability := make(map[string]bool)
|
||||
availability := make(map[string]string)
|
||||
availabilityKnown := make(map[string]bool)
|
||||
projectSupport := make(map[string]bool)
|
||||
projectSupportKnown := make(map[string]bool)
|
||||
@@ -245,13 +302,13 @@ func (r *Router) AssignPending() ([]domain.Event, error) {
|
||||
r.reject(t.ID, h.ID, "capability mismatch")
|
||||
continue
|
||||
}
|
||||
available, checked := availability[h.ID], availabilityKnown[h.ID]
|
||||
reason, checked := availability[h.ID], availabilityKnown[h.ID]
|
||||
if !checked {
|
||||
available = r.Availability.Available(h)
|
||||
availability[h.ID], availabilityKnown[h.ID] = available, true
|
||||
reason = unavailableReason(r.Availability, h)
|
||||
availability[h.ID], availabilityKnown[h.ID] = reason, true
|
||||
}
|
||||
if !available {
|
||||
r.reject(t.ID, h.ID, "worker unavailable: stale heartbeat or unhealthy local backend")
|
||||
if reason != "" {
|
||||
r.reject(t.ID, h.ID, reason)
|
||||
continue
|
||||
}
|
||||
if occupiedCount(activeLeases[h.ID], h.Concurrency) {
|
||||
|
||||
Reference in New Issue
Block a user