Complete autonomous recovery controls

This commit is contained in:
kami
2026-07-30 14:57:25 +04:00
parent 8174400b1a
commit e8fadfc998
18 changed files with 364 additions and 77 deletions
+14 -2
View File
@@ -151,6 +151,14 @@ type Task struct {
LastHarness string `json:"last_harness_id,omitempty"`
PaneState string `json:"pane_state,omitempty"` // open, closed, unreachable, unknown
LastSession SessionEvidence `json:"last_session,omitempty"`
// Recovery state is part of the durable projection, never process-local
// router memory. This makes retry and operator diagnostics survive a
// coordinator restart.
Attempt int `json:"attempt,omitempty"`
NextRetryAt time.Time `json:"next_retry_at,omitempty"`
FailureClass string `json:"failure_class,omitempty"`
LifecyclePhase string `json:"lifecycle_phase,omitempty"`
LastError string `json:"last_error,omitempty"`
}
type Event struct {
@@ -190,7 +198,7 @@ func ValidateEvent(e Event) error {
if e.SchemaVersion >= 2 && strings.TrimSpace(e.Surface) == "" {
return fmt.Errorf("%w: surface required", ErrInvalid)
}
allowed := map[string]bool{"TaskCreated": true, "TaskLeased": true, "TaskLeaseRenewed": true, "TaskReleased": true, "TaskPickupValidated": true, "TaskCompleted": true, "TaskFailed": true, "TaskBlocked": true, "TaskNeedsAttention": true, "ApprovalRequested": true, "ApprovalGranted": true, "ApprovalDenied": true, "TaskAmended": true, "TaskCorrected": true, "QuotaReported": true, "StandupAdvisory": true}
allowed := map[string]bool{"TaskCreated": true, "TaskLeased": true, "TaskLeaseRenewed": true, "TaskReleased": true, "TaskLaunchAcknowledged": true, "TaskPickupValidated": true, "TaskCompleted": true, "TaskFailed": true, "TaskBlocked": true, "TaskNeedsAttention": true, "ApprovalRequested": true, "ApprovalGranted": true, "ApprovalDenied": true, "TaskAmended": true, "TaskCorrected": true, "QuotaReported": true, "StandupAdvisory": true}
if !allowed[e.Type] {
return fmt.Errorf("%w: unknown type %q", ErrInvalid, e.Type)
}
@@ -206,7 +214,7 @@ func ValidateEvent(e Event) error {
}
if e.SchemaVersion >= 3 {
switch e.Type {
case "TaskLeased", "TaskLeaseRenewed", "TaskPickupValidated":
case "TaskLeased", "TaskLeaseRenewed", "TaskLaunchAcknowledged", "TaskPickupValidated":
if v, ok := p["lease_epoch"].(string); !ok || strings.TrimSpace(v) == "" {
return fmt.Errorf("%w: lease_epoch required", ErrInvalid)
}
@@ -313,6 +321,10 @@ func ValidatePayload(typ string, p map[string]any) error {
if err := requiredString("reason"); err != nil {
return err
}
case "TaskLaunchAcknowledged":
if err := requiredString("harness_id"); err != nil {
return err
}
case "TaskBlocked", "TaskNeedsAttention":
if err := requiredString("blocker"); err != nil {
return err
+1 -1
View File
@@ -10,7 +10,7 @@ import (
// of these on adversarial input is a defect regardless of whether real
// producers happen to send well-formed payloads.
var eventTypesUnderTest = []string{
"TaskCreated", "TaskLeased", "TaskReleased", "TaskCompleted", "TaskFailed",
"TaskCreated", "TaskLeased", "TaskReleased", "TaskLaunchAcknowledged", "TaskCompleted", "TaskFailed",
"TaskBlocked", "TaskNeedsAttention", "ApprovalRequested", "ApprovalGranted", "ApprovalDenied",
"TaskAmended", "QuotaReported", "StandupAdvisory",
}
+14
View File
@@ -138,6 +138,20 @@ func (c Client) Renew(ctx context.Context, taskID, epoch string, expectedVersion
}
return err
}
func (c Client) Start(ctx context.Context, taskID, epoch string, expectedVersion int, evidence domain.SessionEvidence) error {
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/start", map[string]any{"task_id": taskID, "lease_epoch": epoch, "expected_version": expectedVersion, "session_evidence": evidence})
if resp != nil {
resp.Body.Close()
}
return err
}
func (c Client) NackStart(ctx context.Context, taskID, epoch string, expectedVersion int, failureClass, detail string, evidence domain.SessionEvidence) error {
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/nack", map[string]any{"task_id": taskID, "lease_epoch": epoch, "expected_version": expectedVersion, "failure_class": failureClass, "last_error": detail, "session_evidence": evidence})
if resp != nil {
resp.Body.Close()
}
return err
}
func (c Client) Artifact(ctx context.Context, ref string) ([]byte, error) {
resp, err := c.request(ctx, http.MethodGet, "/v1/artifacts/"+url.PathEscape(ref), nil)
if err != nil {
+25
View File
@@ -2,8 +2,10 @@ package federation
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"orchestra/internal/domain"
"testing"
)
@@ -49,3 +51,26 @@ func TestClientRegistersPollsAndReadsArtifactAsWorker(t *testing.T) {
}
}
}
func TestClientReportsTypedLaunchAckAndNack(t *testing.T) {
seen := map[string]map[string]any{}
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var body map[string]any
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
t.Fatal(err)
}
seen[r.URL.Path] = body
w.WriteHeader(http.StatusNoContent)
}))
defer s.Close()
c := Client{BaseURL: s.URL, WorkerID: "h1", Token: "worker"}
if err := c.Start(context.Background(), "task", "epoch", 7, domain.SessionEvidence{PaneID: "p"}); err != nil {
t.Fatal(err)
}
if err := c.NackStart(context.Background(), "task", "epoch", 8, "invalid_handoff", "anchor mismatch", domain.SessionEvidence{PaneID: "p"}); err != nil {
t.Fatal(err)
}
if seen["/v1/federation/workers/h1/start"]["expected_version"] != float64(7) || seen["/v1/federation/workers/h1/nack"]["failure_class"] != "invalid_handoff" {
t.Fatalf("launch reports=%#v", seen)
}
}
@@ -180,6 +180,10 @@ func TestCrossMachineLeaseAnchorAndQuotaArePerHost(t *testing.T) {
}
}
quotaReport("homesrv-h1", 95)
// A zero native delta is still a receipt: it proves workpc's independent
// usage source is available, rather than treating missing quota data as
// harmless headroom.
quotaReport("workpc-h1", 0)
limits := map[string]router.QuotaWindowLimits{
"homesrv-h1": {Weekly: 100},
"workpc-h1": {Weekly: 100},
+11
View File
@@ -61,6 +61,17 @@ type QuotaReceipt struct {
At time.Time `json:"at"`
}
// QuotaWindows publishes the two routing windows from the same additive
// per-lease receipts used by the scheduler.
type QuotaWindows struct {
FiveHour map[string]float64 `json:"five_hour"`
Weekly map[string]float64 `json:"weekly"`
}
func ProjectQuotaWindows(events []domain.Event, now time.Time) QuotaWindows {
return QuotaWindows{FiveHour: AggregateQuota(events, now.Add(-5*time.Hour), now), Weekly: AggregateQuota(events, now.Add(-7*24*time.Hour), now)}
}
// AggregateQuota sums native receipts in the requested window. It does not
// de-duplicate rotations or use a cumulative session total.
func AggregateQuota(events []domain.Event, from, to time.Time) map[string]float64 {
+9 -5
View File
@@ -752,17 +752,21 @@ func (c *Coordinator) rotate(ctx context.Context, hard float64) {
}
ref, err := a.Release(ctx, session)
if err != nil {
// A release failure is operational state, not a silent retry. Keep
// the fenced lease and pane for recovery while durably exposing the
// failed phase to the worker and operator.
_ = c.block(task, "rotation release: "+err.Error())
continue
}
if ref == "" {
_ = c.block(task, "rotation release: empty handoff reference")
continue
}
anchorSHA, err := herdr.HeadSHA(session.Worktree)
if err != nil {
// Cannot certify the anchor: do not release with an invalid
// TaskReleased payload (it would fail validation and strand
// the lease/session). Leave the lease intact for the next
// tick or TTL expiry to reclaim.
// Cannot certify the anchor. Record the fault while retaining the
// owner; an unseen bare continue used to leave this state opaque.
_ = c.block(task, "rotation anchor: "+err.Error())
continue
}
b, _ := json.Marshal(map[string]any{"handoff_ref": ref, "reason": reason, "anchor_sha": anchorSHA, "harness_id": task.Lease.HarnessID, "lease_epoch": task.Lease.Epoch, "expected_version": task.Version})
@@ -1020,7 +1024,7 @@ func (c *Coordinator) rememberSession(taskID string, s herdr.Session) error {
}
func (c *Coordinator) block(t domain.Task, reason string) error {
p := map[string]any{"blocker": reason, "block_reason": string(domain.InferBlockReason(reason)), "pane_state": "unknown", "session_evidence": domain.SessionEvidence{PaneState: "unknown", Source: "coordinator", CheckedAt: time.Now().UTC()}}
p := map[string]any{"blocker": reason, "block_reason": string(domain.InferBlockReason(reason)), "lifecycle_phase": "needs_attention", "last_error": reason, "pane_state": "unknown", "session_evidence": domain.SessionEvidence{PaneState: "unknown", Source: "coordinator", CheckedAt: time.Now().UTC()}}
if s, ok := c.Session(t.ID); ok {
p["pane_id"] = s.PaneID
p["harness_id"] = s.HerdrID
+12
View File
@@ -29,6 +29,11 @@ type Project struct {
Repo string `json:"repo,omitempty"`
WorktreeRoot string `json:"worktree_root,omitempty"`
QualityGate string `json:"quality_gate,omitempty"`
// SafeOperations is an audited, deliberately small allow-list for work
// inside this project's task worktree. It documents what workers may
// perform without an operator grant; network, secrets, destructive Git,
// and paths outside the worktree are never represented here.
SafeOperations []string `json:"safe_operations,omitempty"`
}
type Machine struct {
ID string `json:"id"`
@@ -131,6 +136,13 @@ func New(c Config) (Registry, error) {
if len(p.MachineAffinity) == 0 {
return Registry{}, fmt.Errorf("project %q: %w", p.ID, ErrNoAffinity)
}
for _, op := range p.SafeOperations {
switch op {
case "read", "edit", "test", "git":
default:
return Registry{}, fmt.Errorf("project %q: unsafe operation %q is not policy-configurable", p.ID, op)
}
}
r.projects[p.ID] = p
}
for _, m := range c.Machines {
+10
View File
@@ -34,3 +34,13 @@ func TestNewRejectsBrokenReferences(t *testing.T) {
t.Fatalf("err=%v", err)
}
}
func TestProjectSafeOperationsAreNarrowAndAudited(t *testing.T) {
_, err := New(Config{Projects: []Project{{ID: "p", MachineAffinity: []string{"m"}, SafeOperations: []string{"read", "network"}}}, Machines: []Machine{{ID: "m", Address: "m:1"}}})
if err == nil {
t.Fatal("network operation was accepted into no-grant policy")
}
if _, err := New(Config{Projects: []Project{{ID: "p", MachineAffinity: []string{"m"}, SafeOperations: []string{"read", "edit", "test", "git"}}}, Machines: []Machine{{ID: "m", Address: "m:1"}}}); err != nil {
t.Fatalf("safe policy rejected: %v", err)
}
}
+21 -38
View File
@@ -53,8 +53,9 @@ type QuotaAvailability struct {
Now func() time.Time
}
func (q QuotaAvailability) sumSince(harnessID string, since time.Time) float64 {
func (q QuotaAvailability) sumSince(harnessID string, since time.Time) (float64, bool) {
var consumed float64
known := false
for _, e := range q.Store.Events(0) {
if e.Type != "QuotaReported" || e.At.Before(since) {
continue
@@ -62,12 +63,17 @@ func (q QuotaAvailability) sumSince(harnessID string, since time.Time) float64 {
var p struct {
HarnessID string `json:"harness_id"`
Consumed float64 `json:"consumed"`
Known *bool `json:"known"`
}
if json.Unmarshal(e.Payload, &p) == nil && p.HarnessID == harnessID && p.Consumed >= 0 {
if p.Known != nil && !*p.Known {
return 0, false
}
known = true
consumed += p.Consumed
}
}
return consumed
return consumed, known
}
func (q QuotaAvailability) Available(h registry.Herdr) bool {
@@ -82,11 +88,17 @@ func (q QuotaAvailability) Available(h registry.Herdr) bool {
if q.Now != nil {
now = q.Now()
}
if limits.FiveHour > 0 && q.sumSince(h.ID, now.Add(-fiveHourWindow)) >= limits.FiveHour*quotaConservativeFraction {
return false
if limits.FiveHour > 0 {
used, known := q.sumSince(h.ID, now.Add(-fiveHourWindow))
if !known || used >= limits.FiveHour*quotaConservativeFraction {
return false
}
}
if limits.Weekly > 0 && q.sumSince(h.ID, now.Add(-weeklyWindow)) >= limits.Weekly*quotaConservativeFraction {
return false
if limits.Weekly > 0 {
used, known := q.sumSince(h.ID, now.Add(-weeklyWindow))
if !known || used >= limits.Weekly*quotaConservativeFraction {
return false
}
}
return true
}
@@ -103,8 +115,6 @@ type Router struct {
Timeout time.Duration
Retry RetryPolicy
Now func() time.Time
backoff map[string]time.Time
attempts map[string]int
OnLease func(domain.Event) error
}
@@ -115,12 +125,6 @@ func (r *Router) init() {
if r.Now == nil {
r.Now = time.Now
}
if r.backoff == nil {
r.backoff = map[string]time.Time{}
}
if r.attempts == nil {
r.attempts = map[string]int{}
}
}
// HandleEvent evaluates the sink after creation and after a lease is freed.
@@ -129,23 +133,6 @@ func (r *Router) HandleEvent(e domain.Event) ([]domain.Event, error) {
if e.Type != "TaskCreated" && e.Type != "TaskReleased" {
return nil, nil
}
if e.Type == "TaskReleased" {
// Rotation *is* TaskReleased (spec §5.3: "rotation = intra-task
// lease transfer") — a task healthy enough to rotate repeatedly
// must not be killed by the retry limit meant for genuine failures
// (expiry, crash). Only a release without a valid handoff_ref
// (expiry/crash) counts against MaxAttempts.
var p struct {
HandoffRef string `json:"handoff_ref"`
}
isRotation := json.Unmarshal(e.Payload, &p) == nil && p.HandoffRef != ""
if !isRotation {
r.attempts[e.TaskID]++
if r.Retry.Backoff > 0 {
r.backoff[e.TaskID] = r.Now().Add(r.Retry.Backoff)
}
}
}
return r.AssignPending()
}
@@ -156,14 +143,14 @@ func (r *Router) AssignPending() ([]domain.Event, error) {
}
var queued []domain.Task
for _, t := range r.Store.Tasks() {
if t.State == domain.StateQueued && !r.Now().Before(r.backoff[t.ID]) {
if t.State == domain.StateQueued && (t.NextRetryAt.IsZero() || !r.Now().Before(t.NextRetryAt)) {
queued = append(queued, t)
}
}
sort.SliceStable(queued, func(i, j int) bool { return importance(queued[i], r.Now()).Before(importance(queued[j], r.Now())) })
var out []domain.Event
for _, t := range queued {
if r.Retry.MaxAttempts > 0 && r.attempts[t.ID] >= r.Retry.MaxAttempts {
if r.Retry.MaxAttempts > 0 && t.Attempt >= r.Retry.MaxAttempts {
e, err := r.fail(t)
if err != nil {
return out, err
@@ -187,10 +174,6 @@ func (r *Router) AssignPending() ([]domain.Event, error) {
if err != nil {
continue
}
// attempts is the failure counter checked against MaxAttempts
// above; it advances only on a non-rotation TaskReleased (see
// HandleEvent), not here, so a task that leases and rotates
// repeatedly is not double-counted toward the retry limit.
out = append(out, e)
if r.OnLease != nil {
if err := r.OnLease(e); err != nil {
@@ -234,7 +217,7 @@ func importance(t domain.Task, now time.Time) time.Time {
return now.Add(-time.Duration(t.InherentPriority) * time.Hour)
}
func (r *Router) fail(t domain.Task) (domain.Event, error) {
b, _ := json.Marshal(map[string]any{"reason": "retry_limit", "attempts": r.attempts[t.ID]})
b, _ := json.Marshal(map[string]any{"reason": "retry_limit", "attempts": t.Attempt, "failure_class": t.FailureClass})
e := domain.Event{ID: domain.NewID(), Type: "TaskFailed", TaskID: t.ID, Version: t.Version + 1, Payload: b, Surface: string(authz.System)}
return e, r.Store.Append(e)
}
+7 -1
View File
@@ -184,7 +184,9 @@ func TestQuotaWindowsAreIndependent(t *testing.T) {
}
// Case 2: only a 5h limit configured. The same 6h-old receipt is outside
// the 5h window and must not count.
// the 5h window and must not count; a fresh zero receipt proves the native
// usage source is known for this window.
report(now, 0)
fiveHourOnly := QuotaAvailability{Store: s, Limits: map[string]QuotaWindowLimits{"h1": {FiveHour: 100}}, Now: func() time.Time { return now }}
if !fiveHourOnly.Available(registry.Herdr{ID: "h1"}) {
t.Fatal("receipt outside the 5h window incorrectly counted against it")
@@ -198,4 +200,8 @@ func TestQuotaWindowsAreIndependent(t *testing.T) {
if both.Available(registry.Herdr{ID: "h1"}) {
t.Fatal("5h window should be exhausted at 90/100 (>=80%) regardless of weekly headroom")
}
unknown := QuotaAvailability{Store: s, Limits: map[string]QuotaWindowLimits{"unknown": {FiveHour: 100}}, Now: func() time.Time { return now }}
if unknown.Available(registry.Herdr{ID: "unknown"}) {
t.Fatal("bounded harness without a native usage receipt must fail closed")
}
}
+40 -1
View File
@@ -131,6 +131,8 @@ func (s *Store) apply(e domain.Event) error {
s.external[t.Source+"\x00"+t.ExternalID] = t.ID
case "TaskLeased":
t.State = domain.StateLeased
t.LifecyclePhase = "lease_issued"
t.LastError = ""
epoch, _ := p["lease_epoch"].(string)
if epoch == "" {
// A pre-fencing event cannot safely be renewed by an old worker.
@@ -145,13 +147,30 @@ func (s *Store) apply(e domain.Event) error {
epoch = t.Lease.Epoch
}
t.Lease = &domain.Lease{HarnessID: p["harness_id"].(string), Epoch: epoch, Until: time.Unix(0, int64(p["until_ns"].(float64)))}
case "TaskLaunchAcknowledged":
t.LifecyclePhase = "started"
case "TaskReleased":
t.State = domain.StateQueued
t.LifecyclePhase = "reclaimed"
t.Lease = nil
t.HandoffRef, _ = p["handoff_ref"].(string)
t.ReleaseTransaction, _ = p["transaction_id"].(string)
t.ReleaseAnchor, _ = p["anchor_sha"].(string)
t.PickupTransaction, t.PickupLeaseVersion = "", 0
if t.HandoffRef == "" {
// A handoff-less release is a reclaim. Persist the retry decision
// here so expiry, pane exit, and a worker NACK all use the same
// crash-safe transition instead of router-local counters.
t.Attempt++
t.FailureClass, _ = p["failure_class"].(string)
if t.FailureClass == "" {
t.FailureClass, _ = p["reason"].(string)
}
t.NextRetryAt = e.At.Add(retryBackoff(t.Attempt))
} else {
t.NextRetryAt = time.Time{}
t.FailureClass = ""
}
case "TaskPickupValidated":
t.PickupTransaction, _ = p["transaction_id"].(string)
if v, ok := p["lease_version"].(float64); ok {
@@ -221,6 +240,12 @@ func (s *Store) apply(e domain.Event) error {
}
}
}
if phase, ok := p["lifecycle_phase"].(string); ok && phase != "" {
t.LifecyclePhase = phase
}
if last, ok := p["last_error"].(string); ok {
t.LastError = last
}
// Terminal and release events may carry an owner-produced snapshot from
// immediately before a worker/coordinator drops its live session mapping.
// Preserve it independently of the current task state so historical task
@@ -244,6 +269,20 @@ func (s *Store) apply(e domain.Event) error {
s.tasks[e.TaskID] = t
return nil
}
func retryBackoff(attempt int) time.Duration {
if attempt < 1 {
attempt = 1
}
backoff := time.Minute
for i := 1; i < attempt && backoff < 30*time.Minute; i++ {
backoff *= 2
}
if backoff > 30*time.Minute {
return 30 * time.Minute
}
return backoff
}
func (s *Store) Append(e domain.Event) error {
s.mu.Lock()
defer s.mu.Unlock()
@@ -381,7 +420,7 @@ func (s *Store) validateTransition(e domain.Event, t domain.Task, exists bool, p
return nil
}
switch e.Type {
case "TaskLeaseRenewed", "TaskReleased", "TaskPickupValidated", "TaskCompleted", "TaskBlocked", "TaskNeedsAttention", "TaskFailed":
case "TaskLeaseRenewed", "TaskLaunchAcknowledged", "TaskReleased", "TaskPickupValidated", "TaskCompleted", "TaskBlocked", "TaskNeedsAttention", "TaskFailed":
owner, _ := p["harness_id"].(string)
epoch, _ := p["lease_epoch"].(string)
// Expiry is the one coordinator-owned relinquish path. It still binds
+32
View File
@@ -159,6 +159,38 @@ func TestNeedsAttentionRetainsFencedLeaseForLateCompletion(t *testing.T) {
}
}
func TestReclaimPersistsAttemptAndBackoffAcrossReopen(t *testing.T) {
dir := t.TempDir()
s, err := Open(dir)
if err != nil {
t.Fatal(err)
}
if err := s.Append(domain.Event{ID: "create", Type: "TaskCreated", TaskID: "t", Version: 1, Payload: []byte(`{"source":"s","external_id":"retry","project":"p"}`), Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
if _, err := s.Lease("t", "worker", time.Hour); err != nil {
t.Fatal(err)
}
task, _ := s.Task("t")
at := time.Date(2026, 7, 30, 12, 0, 0, 0, time.UTC)
p, _ := json.Marshal(map[string]any{"reason": "lease_expired", "failure_class": "worker_lost", "harness_id": task.Lease.HarnessID, "lease_epoch": task.Lease.Epoch, "expected_version": task.Version})
if err := s.Append(domain.Event{ID: "reclaim", Type: "TaskReleased", TaskID: "t", Version: task.Version + 1, At: at, Payload: p, Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
got, _ := s.Task("t")
if got.Attempt != 1 || got.FailureClass != "worker_lost" || !got.NextRetryAt.Equal(at.Add(time.Minute)) {
t.Fatalf("reclaim projection=%+v", got)
}
reopened, err := Open(dir)
if err != nil {
t.Fatal(err)
}
got, _ = reopened.Task("t")
if got.Attempt != 1 || !got.NextRetryAt.Equal(at.Add(time.Minute)) {
t.Fatalf("reopen lost retry state: %+v", got)
}
}
func TestOpenRebuildsOnlyFromLogAndIgnoresCorruptSnapshot(t *testing.T) {
dir := t.TempDir()
s, err := Open(dir)