kami/orchestra
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Admit a first lease when a quota limit has no receipt history

Browse Source 
Burn-in run 2 ingested its task and then sat queued forever. Every herdr in
the live config declares quota_limit_5h and quota_limit_weekly, the event log
holds zero QuotaReported events, and QuotaSince reported an empty window as
unknown. QuotaAvailability fails closed on unknown, so no harness could ever
be leased, and the only producer of a receipt is a completed lease.

The event log is Orchestra's whole accounting source, so a window holding no
receipts is observable zero consumption. QuotaSince now reports known for an
empty window and for a harness that has never reported. A receipt that
declares its own consumption unknown still fails closed.

The refusal also lied about its cause. federatedAvailability collapsed a base
gate refusal into the federation health string, so router health said "stale
heartbeat or unhealthy local backend" while the heartbeat was one second old.
Availability gates now name themselves through an optional
ReasonedAvailability contract: quota refusals say whether usage is unknown or
the window is exhausted and by how much, and worker refusals distinguish an
unregistered worker, a never-probed backend, a stale heartbeat, a stale
health check, and an unreachable backend.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
kami
2026-08-27 00:28:57 +04:00
parent c833e0eb62
commit 0d67af9976
7 changed files with 277 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/router/router.go
+63 -6
View File
@@ -3,6 +3,7 @@ package router
import (
"encoding/json"
"fmt"
"errors"
"orchestra/internal/authz"
"orchestra/internal/domain"
@@ -21,6 +22,15 @@ type Availability interface{ Available(h registry.Herdr) bool }
type ProjectAvailability interface {
Supports(h registry.Herdr, project string) bool
}
// ReasonedAvailability is an optional contract that names why a herdr was
// refused. Without it every refusal collapses into one string, which told an
// operator "stale heartbeat" while the heartbeat was one second old and the
// real refusal came from the quota gate. Found on burn-in run 2, 2026-08-26.
type ReasonedAvailability interface {
// Unavailable returns "" when the herdr may be leased, otherwise the
// specific reason.
Unavailable(h registry.Herdr) string
}
type AlwaysAvailable struct{}
func (AlwaysAvailable) Available(registry.Herdr) bool { return true }
@@ -85,6 +95,53 @@ func (q QuotaAvailability) Available(h registry.Herdr) bool {
return true
}
func (q QuotaAvailability) Unavailable(h registry.Herdr) string {
if q.Store == nil {
return "quota unavailable: no receipt store"
}
limits, bounded := q.Limits[h.ID]
if !bounded || (limits.FiveHour <= 0 && limits.Weekly <= 0) {
return ""
}
now := time.Now()
if q.Now != nil {
now = q.Now()
}
windows := []struct {
name string
limit float64
since time.Time
}{
{"5h", limits.FiveHour, now.Add(-fiveHourWindow)},
{"weekly", limits.Weekly, now.Add(-weeklyWindow)},
}
for _, w := range windows {
if w.limit <= 0 {
continue
}
used, known := q.sumSince(h.ID, w.since)
if !known {
return "quota unavailable: " + w.name + " usage unknown"
}
if cap := w.limit * quotaConservativeFraction; used >= cap {
return fmt.Sprintf("quota exhausted: %s window %.0f/%.0f conservative limit (of %.0f)", w.name, used, cap, w.limit)
}
}
return ""
}
// unavailableReason prefers a specific reason when the availability supports
// one, and never lets a bare Available disagree with it.
func unavailableReason(a Availability, h registry.Herdr) string {
if ra, ok := a.(ReasonedAvailability); ok {
return ra.Unavailable(h)
}
if a.Available(h) {
return ""
}
return "worker unavailable"
}
type RetryPolicy struct {
MaxAttempts int
Backoff time.Duration
@@ -197,7 +254,7 @@ func (r *Router) AssignPending() ([]domain.Event, error) {
sort.SliceStable(queued, func(i, j int) bool { return importance(queued[i], now).Before(importance(queued[j], now)) })
health := r.healthSnapshot(now)
candidates := make(map[string][]registry.Herdr)
availability := make(map[string]bool)
availability := make(map[string]string)
availabilityKnown := make(map[string]bool)
projectSupport := make(map[string]bool)
projectSupportKnown := make(map[string]bool)
@@ -245,13 +302,13 @@ func (r *Router) AssignPending() ([]domain.Event, error) {
r.reject(t.ID, h.ID, "capability mismatch")
continue
}
available, checked := availability[h.ID], availabilityKnown[h.ID]
reason, checked := availability[h.ID], availabilityKnown[h.ID]
if !checked {
available = r.Availability.Available(h)
availability[h.ID], availabilityKnown[h.ID] = available, true
reason = unavailableReason(r.Availability, h)
availability[h.ID], availabilityKnown[h.ID] = reason, true
}
if !available {
r.reject(t.ID, h.ID, "worker unavailable: stale heartbeat or unhealthy local backend")
if reason != "" {
r.reject(t.ID, h.ID, reason)
continue
}
if occupiedCount(activeLeases[h.ID], h.Concurrency) {
internal/router/router_test.go
+96 -3
View File
@@ -297,9 +297,102 @@ func TestQuotaWindowsAreIndependent(t *testing.T) {
if both.Available(registry.Herdr{ID: "h1"}) {
t.Fatal("5h window should be exhausted at 90/100 (>=80%) regardless of weekly headroom")
}
unknown := QuotaAvailability{Store: s, Limits: map[string]QuotaWindowLimits{"unknown": {FiveHour: 100}}, Now: func() time.Time { return now }}
if unknown.Available(registry.Herdr{ID: "unknown"}) {
t.Fatal("bounded harness without a native usage receipt must fail closed")
// A bounded harness that has never reported is at zero consumption, not
// at unknown consumption: the event log is the whole accounting source.
// Failing closed here deadlocked the first lease, because only a completed
// lease can produce the receipt the gate demanded.
fresh := QuotaAvailability{Store: s, Limits: map[string]QuotaWindowLimits{"unreported": {FiveHour: 100}}, Now: func() time.Time { return now }}
if !fresh.Available(registry.Herdr{ID: "unreported"}) {
t.Fatal("bounded harness with no receipt history must still admit a first lease")
}
// A receipt that declares its own consumption unknown still fails closed.
p, _ := json.Marshal(map[string]any{"harness_id": "opaque", "consumed": 0, "known": false})
if err := s.Append(domain.Event{ID: domain.NewID(), Type: "QuotaReported", TaskID: "quota", Version: 1, Surface: string(authz.System), Payload: p, At: now}); err != nil {
t.Fatal(err)
}
opaque := QuotaAvailability{Store: s, Limits: map[string]QuotaWindowLimits{"opaque": {FiveHour: 100}}, Now: func() time.Time { return now }}
if opaque.Available(registry.Herdr{ID: "opaque"}) {
t.Fatal("a receipt reporting unknown consumption must fail closed")
}
if got, want := opaque.Unavailable(registry.Herdr{ID: "opaque"}), "quota unavailable: 5h usage unknown"; got != want {
t.Fatalf("unknown-usage reason=%q, want %q", got, want)
}
if got := both.Unavailable(registry.Herdr{ID: "h1"}); !strings.HasPrefix(got, "quota exhausted: 5h window ") {
t.Fatalf("exhausted reason=%q, want a 5h quota-exhausted reason", got)
}
}
// The router must not restate a quota refusal as worker health. An operator
// reading "stale heartbeat" against a one-second-old heartbeat looks at the
// wrong half of the system.
func TestQuotaRefusalKeepsItsOwnRejectionReason(t *testing.T) {
s, err := store.Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
r, err := registry.New(registry.Config{
Projects: []registry.Project{{ID: "p", MachineAffinity: []string{"m"}}},
Machines: []registry.Machine{{ID: "m", Address: "unused"}},
Herdrs: []registry.Herdr{{ID: "h1", MachineID: "m", Concurrency: 1}},
})
if err != nil {
t.Fatal(err)
}
now := time.Now().UTC()
p, _ := json.Marshal(map[string]any{"harness_id": "h1", "consumed": 90.0})
if err := s.Append(domain.Event{ID: domain.NewID(), Type: "QuotaReported", TaskID: "quota", Version: 1, Surface: string(authz.System), Payload: p, At: now}); err != nil {
t.Fatal(err)
}
b, _ := json.Marshal(map[string]any{"source": "test", "external_id": "t", "project": "p"})
if err := s.Append(domain.Event{ID: "t", TaskID: "t", Type: "TaskCreated", Version: 1, Payload: b, Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
rt := Router{Store: s, Registry: r, Reachability: reachable{},
Availability: QuotaAvailability{Store: s, Limits: map[string]QuotaWindowLimits{"h1": {FiveHour: 100}}, Now: func() time.Time { return now }}}
if got, err := rt.AssignPending(); err != nil || len(got) != 0 {
t.Fatalf("assigned %d events, err=%v; want none", len(got), err)
}
found := false
for _, rej := range rt.Rejections() {
if rej.HerdrID != "h1" {
continue
}
found = true
if !strings.HasPrefix(rej.Reason, "quota exhausted: ") {
t.Fatalf("rejection reason=%q, want a quota-exhausted reason", rej.Reason)
}
}
if !found {
t.Fatal("no rejection recorded for h1")
}
}
// A bounded harness with no receipt history must be leasable, end to end
// through the router. This is the burn-in run 2 blocker: every herdr in the
// live config declares a quota limit, nothing had ever reported a receipt, and
// so no task could ever be assigned autonomously.
func TestFirstLeaseSucceedsWithQuotaConfiguredAndNoReceipts(t *testing.T) {
s, err := store.Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
r, err := registry.New(registry.Config{
Projects: []registry.Project{{ID: "p", MachineAffinity: []string{"m"}}},
Machines: []registry.Machine{{ID: "m", Address: "unused"}},
Herdrs: []registry.Herdr{{ID: "h1", MachineID: "m", Concurrency: 1}},
})
if err != nil {
t.Fatal(err)
}
b, _ := json.Marshal(map[string]any{"source": "test", "external_id": "t", "project": "p"})
if err := s.Append(domain.Event{ID: "t", TaskID: "t", Type: "TaskCreated", Version: 1, Payload: b, Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
rt := Router{Store: s, Registry: r, Reachability: reachable{},
Availability: QuotaAvailability{Store: s, Limits: map[string]QuotaWindowLimits{"h1": {FiveHour: 50, Weekly: 500}}}}
got, err := rt.AssignPending()
if err != nil || len(got) != 1 {
t.Fatalf("assigned %d events, err=%v; want exactly 1: %+v", len(got), err, rt.Rejections())
}
}