kami/orchestra
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Complete autonomous recovery controls

Browse Source
This commit is contained in:
kami
2026-07-30 14:57:25 +04:00
parent 8174400b1a
commit e8fadfc998
18 changed files with 364 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/router/router.go
+21 -38
View File
@@ -53,8 +53,9 @@ type QuotaAvailability struct {
Now func() time.Time
}
func (q QuotaAvailability) sumSince(harnessID string, since time.Time) float64 {
func (q QuotaAvailability) sumSince(harnessID string, since time.Time) (float64, bool) {
var consumed float64
known := false
for _, e := range q.Store.Events(0) {
if e.Type != "QuotaReported" || e.At.Before(since) {
continue
@@ -62,12 +63,17 @@ func (q QuotaAvailability) sumSince(harnessID string, since time.Time) float64 {
var p struct {
HarnessID string `json:"harness_id"`
Consumed float64 `json:"consumed"`
Known *bool `json:"known"`
}
if json.Unmarshal(e.Payload, &p) == nil && p.HarnessID == harnessID && p.Consumed >= 0 {
if p.Known != nil && !*p.Known {
return 0, false
}
known = true
consumed += p.Consumed
}
}
return consumed
return consumed, known
}
func (q QuotaAvailability) Available(h registry.Herdr) bool {
@@ -82,11 +88,17 @@ func (q QuotaAvailability) Available(h registry.Herdr) bool {
if q.Now != nil {
now = q.Now()
}
if limits.FiveHour > 0 && q.sumSince(h.ID, now.Add(-fiveHourWindow)) >= limits.FiveHour*quotaConservativeFraction {
return false
if limits.FiveHour > 0 {
used, known := q.sumSince(h.ID, now.Add(-fiveHourWindow))
if !known || used >= limits.FiveHour*quotaConservativeFraction {
return false
}
}
if limits.Weekly > 0 && q.sumSince(h.ID, now.Add(-weeklyWindow)) >= limits.Weekly*quotaConservativeFraction {
return false
if limits.Weekly > 0 {
used, known := q.sumSince(h.ID, now.Add(-weeklyWindow))
if !known || used >= limits.Weekly*quotaConservativeFraction {
return false
}
}
return true
}
@@ -103,8 +115,6 @@ type Router struct {
Timeout time.Duration
Retry RetryPolicy
Now func() time.Time
backoff map[string]time.Time
attempts map[string]int
OnLease func(domain.Event) error
}
@@ -115,12 +125,6 @@ func (r *Router) init() {
if r.Now == nil {
r.Now = time.Now
}
if r.backoff == nil {
r.backoff = map[string]time.Time{}
}
if r.attempts == nil {
r.attempts = map[string]int{}
}
}
// HandleEvent evaluates the sink after creation and after a lease is freed.
@@ -129,23 +133,6 @@ func (r *Router) HandleEvent(e domain.Event) ([]domain.Event, error) {
if e.Type != "TaskCreated" && e.Type != "TaskReleased" {
return nil, nil
}
if e.Type == "TaskReleased" {
// Rotation *is* TaskReleased (spec §5.3: "rotation = intra-task
// lease transfer") — a task healthy enough to rotate repeatedly
// must not be killed by the retry limit meant for genuine failures
// (expiry, crash). Only a release without a valid handoff_ref
// (expiry/crash) counts against MaxAttempts.
var p struct {
HandoffRef string `json:"handoff_ref"`
}
isRotation := json.Unmarshal(e.Payload, &p) == nil && p.HandoffRef != ""
if !isRotation {
r.attempts[e.TaskID]++
if r.Retry.Backoff > 0 {
r.backoff[e.TaskID] = r.Now().Add(r.Retry.Backoff)
}
}
}
return r.AssignPending()
}
@@ -156,14 +143,14 @@ func (r *Router) AssignPending() ([]domain.Event, error) {
}
var queued []domain.Task
for _, t := range r.Store.Tasks() {
if t.State == domain.StateQueued && !r.Now().Before(r.backoff[t.ID]) {
if t.State == domain.StateQueued && (t.NextRetryAt.IsZero() || !r.Now().Before(t.NextRetryAt)) {
queued = append(queued, t)
}
}
sort.SliceStable(queued, func(i, j int) bool { return importance(queued[i], r.Now()).Before(importance(queued[j], r.Now())) })
var out []domain.Event
for _, t := range queued {
if r.Retry.MaxAttempts > 0 && r.attempts[t.ID] >= r.Retry.MaxAttempts {
if r.Retry.MaxAttempts > 0 && t.Attempt >= r.Retry.MaxAttempts {
e, err := r.fail(t)
if err != nil {
return out, err
@@ -187,10 +174,6 @@ func (r *Router) AssignPending() ([]domain.Event, error) {
if err != nil {
continue
}
// attempts is the failure counter checked against MaxAttempts
// above; it advances only on a non-rotation TaskReleased (see
// HandleEvent), not here, so a task that leases and rotates
// repeatedly is not double-counted toward the retry limit.
out = append(out, e)
if r.OnLease != nil {
if err := r.OnLease(e); err != nil {
@@ -234,7 +217,7 @@ func importance(t domain.Task, now time.Time) time.Time {
return now.Add(-time.Duration(t.InherentPriority) * time.Hour)
}
func (r *Router) fail(t domain.Task) (domain.Event, error) {
b, _ := json.Marshal(map[string]any{"reason": "retry_limit", "attempts": r.attempts[t.ID]})
b, _ := json.Marshal(map[string]any{"reason": "retry_limit", "attempts": t.Attempt, "failure_class": t.FailureClass})
e := domain.Event{ID: domain.NewID(), Type: "TaskFailed", TaskID: t.ID, Version: t.Version + 1, Payload: b, Surface: string(authz.System)}
return e, r.Store.Append(e)
}
internal/router/router_test.go
+7 -1
View File
@@ -184,7 +184,9 @@ func TestQuotaWindowsAreIndependent(t *testing.T) {
}
// Case 2: only a 5h limit configured. The same 6h-old receipt is outside
// the 5h window and must not count.
// the 5h window and must not count; a fresh zero receipt proves the native
// usage source is known for this window.
report(now, 0)
fiveHourOnly := QuotaAvailability{Store: s, Limits: map[string]QuotaWindowLimits{"h1": {FiveHour: 100}}, Now: func() time.Time { return now }}
if !fiveHourOnly.Available(registry.Herdr{ID: "h1"}) {
t.Fatal("receipt outside the 5h window incorrectly counted against it")
@@ -198,4 +200,8 @@ func TestQuotaWindowsAreIndependent(t *testing.T) {
if both.Available(registry.Herdr{ID: "h1"}) {
t.Fatal("5h window should be exhausted at 90/100 (>=80%) regardless of weekly headroom")
}
unknown := QuotaAvailability{Store: s, Limits: map[string]QuotaWindowLimits{"unknown": {FiveHour: 100}}, Now: func() time.Time { return now }}
if unknown.Available(registry.Herdr{ID: "unknown"}) {
t.Fatal("bounded harness without a native usage receipt must fail closed")
}
}