kami/orchestra
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Preserve leases needing recovery

Browse Source
This commit is contained in:
kami
2026-07-30 14:37:34 +04:00
parent f6ee0e3060
commit 8174400b1a
12 changed files with 106 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/orchestra-worker/main.go
+10 -1
View File
@@ -814,6 +814,15 @@ func (w *worker) once(ctx context.Context) error {
w.leases[e.TaskID] = l
}
}
if e.Type == "TaskNeedsAttention" {
// The diagnostic event increments the aggregate version but leaves
// ownership intact. Keep our locally persisted expected version in
// sync so a late, otherwise valid completion is not self-staled.
if l, ok := w.leases[e.TaskID]; ok {
l.Version = e.Version
w.leases[e.TaskID] = l
}
}
if e.Type == "TaskCompleted" {
delete(w.leases, e.TaskID)
if session, active := w.sessions[e.TaskID]; active {
@@ -932,7 +941,7 @@ func (w *worker) reconcileLeases(ctx context.Context) error {
active := make(map[string]lease)
for _, task := range tasks {
w.tasks[task.ID] = task
if task.State == domain.StateLeased && task.Lease != nil && task.Lease.HarnessID == w.harnessID {
if (task.State == domain.StateLeased || task.State == domain.StateNeedsAttention) && task.Lease != nil && task.Lease.HarnessID == w.harnessID {
active[task.ID] = lease{Epoch: task.Lease.Epoch, HandoffRef: task.HandoffRef, TransactionID: task.ReleaseTransaction, AnchorSHA: task.ReleaseAnchor, Version: task.Version, Until: task.Lease.Until}
}
}
cmd/orchestra/main.go
+5 -5
View File
@@ -799,7 +799,7 @@ func main() {
}
var e domain.Event
var err error
actionTypes := map[string]string{"lease": "TaskLeased", "release": "TaskReleased", "complete": "TaskCompleted", "block": "TaskBlocked"}
actionTypes := map[string]string{"lease": "TaskLeased", "release": "TaskReleased", "complete": "TaskCompleted", "block": "TaskBlocked", "attention": "TaskNeedsAttention"}
if typ, known := actionTypes[action]; known {
if err := authz.AuthorizeEvent(surface(r), typ); err != nil {
http.Error(w, err.Error(), http.StatusForbidden)
@@ -820,13 +820,13 @@ func main() {
p.TTLSeconds = 1800
}
e, err = s.Lease(taskID, p.HarnessID, time.Duration(p.TTLSeconds)*time.Second)
case "release", "complete", "block":
case "release", "complete", "block", "attention":
t, ok := s.Task(taskID)
if !ok {
http.Error(w, "task not found", 404)
return
}
types := map[string]string{"release": "TaskReleased", "complete": "TaskCompleted", "block": "TaskBlocked"}
types := map[string]string{"release": "TaskReleased", "complete": "TaskCompleted", "block": "TaskBlocked", "attention": "TaskNeedsAttention"}
var p map[string]any
if r.Body == nil || json.NewDecoder(r.Body).Decode(&p) != nil || p == nil {
http.Error(w, "invalid lifecycle payload", http.StatusBadRequest)
@@ -836,7 +836,7 @@ func main() {
http.Error(w, "reason or handoff_ref required", http.StatusBadRequest)
return
}
if action == "block" && p["blocker"] == nil {
if (action == "block" || action == "attention") && p["blocker"] == nil {
http.Error(w, "blocker required", http.StatusBadRequest)
return
}
@@ -1133,7 +1133,7 @@ func main() {
http.Error(w, "task not found", 404)
return
}
ownedLease := t.State == domain.StateLeased && t.Lease != nil && t.Lease.HarnessID == parts[3] && t.Lease.Epoch == b.LeaseEpoch
ownedLease := (t.State == domain.StateLeased || t.State == domain.StateNeedsAttention) && t.Lease != nil && t.Lease.HarnessID == parts[3] && t.Lease.Epoch == b.LeaseEpoch
// A response can be lost after the append/fsync. Retrying the exact
// release transaction is therefore a successful no-op, never a second
// TaskReleased event and never a reason to discard the predecessor.