kami/orchestra
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Harden lease lifecycle durability

Browse Source
This commit is contained in:
kami
2026-07-30 14:34:29 +04:00
parent 1ff0af2e69
commit f6ee0e3060
40 changed files with 2108 additions and 590 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/orchestra/main.go
+27 -33
View File
@@ -142,12 +142,14 @@ func (h harnessCompletion) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
var p struct {
TaskID string `json:"task_id"`
WorkerID string `json:"worker_id"`
LeaseEpoch string `json:"lease_epoch"`
Harness string `json:"harness"`
TranscriptPath string `json:"transcript_path"`
Report string `json:"report"`
}
if json.NewDecoder(r.Body).Decode(&p) != nil || p.TaskID == "" || p.Report == "" || p.TranscriptPath == "" {
http.Error(w, "task_id, transcript_path, and report are required", http.StatusBadRequest)
if json.NewDecoder(r.Body).Decode(&p) != nil || p.TaskID == "" || p.WorkerID == "" || p.LeaseEpoch == "" || p.Report == "" || p.TranscriptPath == "" {
http.Error(w, "task_id, worker_id, lease_epoch, transcript_path, and report are required", http.StatusBadRequest)
return
}
t, ok := h.store.Task(p.TaskID)
@@ -155,6 +157,10 @@ func (h harnessCompletion) ServeHTTP(w http.ResponseWriter, r *http.Request) {
http.Error(w, "task not found", http.StatusNotFound)
return
}
if t.State != domain.StateLeased || t.Lease == nil || t.Lease.HarnessID != p.WorkerID || t.Lease.Epoch != p.LeaseEpoch {
http.Error(w, "lease not owned", http.StatusConflict)
return
}
var usage herdr.Usage
var err error
switch p.Harness {
@@ -177,7 +183,7 @@ func (h harnessCompletion) ServeHTTP(w http.ResponseWriter, r *http.Request) {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
payload, _ := json.Marshal(map[string]any{"report_ref": ref, "receipt": map[string]any{
payload, _ := json.Marshal(map[string]any{"report_ref": ref, "harness_id": p.WorkerID, "lease_epoch": p.LeaseEpoch, "expected_version": t.Version, "receipt": map[string]any{
"input_tokens": usage.Input, "cache_read_tokens": usage.CacheRead,
"cache_write_tokens": usage.CacheWrite, "output_tokens": usage.Output, "numerator": usage.Numerator(),
}})
@@ -381,17 +387,10 @@ func main() {
return err
}}.Handler())
mux.Handle("/", webui.Handler())
workers.OnOffline = func(w federation.Worker) {
for _, t := range s.Tasks() {
if t.State == domain.StateLeased && t.Lease != nil && t.Lease.HarnessID == w.ID {
p, _ := json.Marshal(map[string]any{"reason": "worker_offline", "harness_id": w.ID})
e := domain.Event{ID: id(), Type: "TaskReleased", TaskID: t.ID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
if err := s.Append(e); err == nil && rt != nil {
_, _ = rt.HandleEvent(e)
}
}
}
}
// A missed heartbeat is not relinquishment. Releasing here used to lease
// the same task to a successor while the old pane was still running. The
// authoritative lease timer performs the only automatic reassignment.
workers.OnOffline = func(w federation.Worker) { log.Printf("worker %s offline; retaining leases until expiry", w.ID) }
go func() {
ticker := time.NewTicker(15 * time.Second)
defer ticker.Stop()
@@ -560,13 +559,12 @@ func main() {
// same session-file assumption as CLIAdapter.Occupancy — rather than
// trusting a self-reported number.
harnessToken := os.Getenv("ORCHESTRA_HARNESS_TOKEN")
mux.Handle("/v1/harness/complete", harnessCompletion{store: s, token: harnessToken, route: func(e domain.Event) error {
if rt == nil {
return nil
}
_, err := rt.HandleEvent(e)
return err
}})
// The unaffiliated harness hook has no durable worker identity or fencing
// epoch, so it cannot safely mutate a leased task. Completion is accepted
// only through the authenticated federation worker endpoint below.
mux.HandleFunc("/v1/harness/complete", func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "legacy harness completion endpoint retired; use worker completion", http.StatusGone)
})
// /v1/harness/turn is the unified turn-decision endpoint (AUDIT.md Phase
// 2 items 1-2): the Face-B stop hook posts here on every ordinary turn
// boundary (report marker absent — /v1/harness/complete covers task
@@ -1062,7 +1060,7 @@ func main() {
w.WriteHeader(http.StatusNoContent)
})
mux.HandleFunc("/v1/federation/workers/", func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || (!strings.HasSuffix(r.URL.Path, "/heartbeat") && !strings.HasSuffix(r.URL.Path, "/renew") && !strings.HasSuffix(r.URL.Path, "/handoff") && !strings.HasSuffix(r.URL.Path, "/complete") && !strings.HasSuffix(r.URL.Path, "/captures")) {
if r.Method != http.MethodPost || (!strings.HasSuffix(r.URL.Path, "/heartbeat") && !strings.HasSuffix(r.URL.Path, "/renew") && !strings.HasSuffix(r.URL.Path, "/handoff") && !strings.HasSuffix(r.URL.Path, "/pickup") && !strings.HasSuffix(r.URL.Path, "/complete") && !strings.HasSuffix(r.URL.Path, "/captures")) {
http.Error(w, "not found", 404)
return
}
@@ -1119,6 +1117,7 @@ func main() {
AnchorSHA string `json:"anchor_sha"`
TransactionID string `json:"transaction_id"`
LeaseVersion int `json:"lease_version"`
LeaseEpoch string `json:"lease_epoch"`
ResultSHA string `json:"result_sha"`
Branch string `json:"branch"`
Remote string `json:"remote"`
@@ -1134,7 +1133,7 @@ func main() {
http.Error(w, "task not found", 404)
return
}
ownedLease := t.State == domain.StateLeased && t.Lease != nil && t.Lease.HarnessID == parts[3]
ownedLease := t.State == domain.StateLeased && t.Lease != nil && t.Lease.HarnessID == parts[3] && t.Lease.Epoch == b.LeaseEpoch
// A response can be lost after the append/fsync. Retrying the exact
// release transaction is therefore a successful no-op, never a second
// TaskReleased event and never a reason to discard the predecessor.
@@ -1142,12 +1141,7 @@ func main() {
w.WriteHeader(http.StatusNoContent)
return
}
// A prompt timeout can block the coordinator after herdr already
// accepted the request. If that same authenticated worker later reports
// a durable completion, reconcile it rather than preserving a known
// false blocked state. No other blocked task is admitted here.
recoverableBlocked := strings.HasSuffix(r.URL.Path, "/complete") && t.State == domain.StateBlocked && t.LastHarness == parts[3]
if !ownedLease && !recoverableBlocked {
if !ownedLease {
http.Error(w, "lease not owned", 409)
return
}
@@ -1160,7 +1154,7 @@ func main() {
if ttl == 0 {
ttl = int((30 * time.Minute).Seconds())
}
e, err := s.RenewLease(b.TaskID, parts[3], b.ExpectedVersion, time.Duration(ttl)*time.Second)
e, err := s.RenewLease(b.TaskID, parts[3], b.LeaseEpoch, b.ExpectedVersion, time.Duration(ttl)*time.Second)
if err != nil {
http.Error(w, err.Error(), http.StatusConflict)
return
@@ -1181,7 +1175,7 @@ func main() {
http.Error(w, "lease version conflict", http.StatusConflict)
return
}
p, _ := json.Marshal(map[string]any{"transaction_id": b.TransactionID, "handoff_ref": b.HandoffRef, "anchor_sha": b.AnchorSHA, "harness_id": parts[3], "lease_version": b.LeaseVersion, "expected_version": t.Version, "session_evidence": b.SessionEvidence})
p, _ := json.Marshal(map[string]any{"transaction_id": b.TransactionID, "handoff_ref": b.HandoffRef, "anchor_sha": b.AnchorSHA, "harness_id": parts[3], "lease_epoch": b.LeaseEpoch, "lease_version": b.LeaseVersion, "expected_version": t.Version, "session_evidence": b.SessionEvidence})
e := domain.Event{ID: id(), Type: "TaskPickupValidated", TaskID: b.TaskID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
if err := s.Append(e); err != nil {
http.Error(w, err.Error(), http.StatusConflict)
@@ -1206,7 +1200,7 @@ func main() {
if _, ok := b.Receipt["consumed"]; !ok {
b.Receipt["consumed"] = 0
}
p, _ := json.Marshal(map[string]any{"report_ref": b.HandoffRef, "receipt": b.Receipt, "result_sha": b.ResultSHA, "branch": b.Branch, "remote": b.Remote, "session_evidence": b.SessionEvidence})
p, _ := json.Marshal(map[string]any{"report_ref": b.HandoffRef, "receipt": b.Receipt, "result_sha": b.ResultSHA, "branch": b.Branch, "remote": b.Remote, "harness_id": parts[3], "lease_epoch": b.LeaseEpoch, "expected_version": t.Version, "session_evidence": b.SessionEvidence})
e := domain.Event{ID: id(), Type: "TaskCompleted", TaskID: b.TaskID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
if err := s.Append(e); err != nil {
http.Error(w, err.Error(), 409)
@@ -1233,7 +1227,7 @@ func main() {
http.Error(w, "release transaction and current lease version required", http.StatusConflict)
return
}
p, _ := json.Marshal(map[string]any{"handoff_ref": b.HandoffRef, "harness_id": parts[3], "anchor_sha": b.AnchorSHA, "transaction_id": b.TransactionID, "expected_version": t.Version, "session_evidence": b.SessionEvidence})
p, _ := json.Marshal(map[string]any{"handoff_ref": b.HandoffRef, "harness_id": parts[3], "lease_epoch": b.LeaseEpoch, "anchor_sha": b.AnchorSHA, "transaction_id": b.TransactionID, "expected_version": t.Version, "session_evidence": b.SessionEvidence})
e := domain.Event{ID: id(), Type: "TaskReleased", TaskID: b.TaskID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
if err := s.Append(e); err != nil {
http.Error(w, err.Error(), 409)
cmd/orchestra/main_test.go
+16 -1
View File
@@ -41,6 +41,20 @@ func TestFederatedReachabilityDefersRemoteHerdrToWorkerHeartbeat(t *testing.T) {
}
}
func TestCoordinatorOwnsOnlyLocalHerdrInFederationMode(t *testing.T) {
local := registry.Herdr{ID: "homesrv-opencode", MachineID: "homesrv"}
remote := registry.Herdr{ID: "workpc-opencode", MachineID: "workpc"}
if !coordinatorOwnsHerdr(local, "homesrv") {
t.Fatal("coordinator does not own its local herdr")
}
if coordinatorOwnsHerdr(remote, "homesrv") {
t.Fatal("coordinator claimed a worker-owned remote herdr")
}
if !coordinatorOwnsHerdr(remote, "") {
t.Fatal("single-machine mode should retain legacy local ownership")
}
}
func TestHarnessCompletionBuildsReceiptAndQuotaFromTranscript(t *testing.T) {
s, err := store.Open(t.TempDir())
if err != nil {
@@ -56,7 +70,8 @@ func TestHarnessCompletionBuildsReceiptAndQuotaFromTranscript(t *testing.T) {
if err := os.WriteFile(transcript, []byte(`{"message":{"usage":{"input_tokens":100,"cache_read_input_tokens":20,"cache_creation_input_tokens":5,"output_tokens":7}}}`+"\n"), 0600); err != nil {
t.Fatal(err)
}
body, _ := json.Marshal(map[string]string{"task_id": "done", "transcript_path": transcript, "report": "# done"})
task, _ := s.Task("done")
body, _ := json.Marshal(map[string]string{"task_id": "done", "worker_id": "local-claude", "lease_epoch": task.Lease.Epoch, "transcript_path": transcript, "report": "# done"})
req := httptest.NewRequest(http.MethodPost, "/v1/harness/complete", bytes.NewReader(body))
res := httptest.NewRecorder()
harnessCompletion{store: s, token: "secret"}.ServeHTTP(res, req)