kami/orchestra
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: make worker handoff rotation durable

Browse Source
This commit is contained in:
kami
2026-07-30 01:30:59 +04:00
parent ce02c60106
commit 1ff0af2e69
16 changed files with 1488 additions and 1566 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/orchestra/main.go
+132 -28
View File
@@ -2,7 +2,6 @@ package main
import (
"context"
"crypto/subtle"
"encoding/json"
"errors"
"fmt"
@@ -12,6 +11,7 @@ import (
"net/http"
"orchestra/internal/admin"
"orchestra/internal/authz"
"orchestra/internal/buildinfo"
"orchestra/internal/delivery"
"orchestra/internal/domain"
"orchestra/internal/federation"
@@ -83,6 +83,14 @@ func remoteHerdrAddresses(rr registry.Registry, localMachine string) map[string]
return remote
}
// coordinatorOwnsHerdr identifies the only herdr sockets the coordinator may
// probe or adapt. In federation mode a remote pane belongs to its worker;
// reaching into that machine would turn a worker-owned health signal back
// into a misleading coordinator TCP result.
func coordinatorOwnsHerdr(h registry.Herdr, localMachine string) bool {
return localMachine == "" || h.MachineID == localMachine
}
func (a federatedAvailability) Available(h registry.Herdr) bool {
if a.base != nil && !a.base.Available(h) {
return false
@@ -93,6 +101,16 @@ func (a federatedAvailability) Available(h registry.Herdr) bool {
return a.workers.Available(h.ID)
}
func (a federatedAvailability) Supports(h registry.Herdr, project string) bool {
// Locally-owned herdrs keep their static registry/project affinity. A
// remote worker must additionally prove it has a local checkout for the
// project before the router can offer it a lease.
if a.localMachine == "" || h.MachineID == a.localMachine {
return true
}
return a.workers.Supports(h.ID, project)
}
func validateLocalMachine(rr registry.Registry, localMachine string) error {
machines := rr.Machines()
if len(machines) <= 1 {
@@ -231,6 +249,10 @@ func main() {
if repo, root := os.Getenv("ORCHESTRA_REPO"), os.Getenv("ORCHESTRA_WORKTREE_ROOT"); repo != "" && root != "" {
adapters := map[string]herdr.Adapter{}
for _, h := range rr.Herdrs() {
if !coordinatorOwnsHerdr(h, localMachine) {
log.Printf("herdr %s is worker-owned on %s; coordinator probe skipped", h.ID, h.MachineID)
continue
}
address := herdrAddress(rr, h)
if address == "" {
continue
@@ -244,6 +266,7 @@ func main() {
log.Printf("herdr %s unavailable: %v", h.ID, err)
continue
}
log.Printf("herdr %s reachable at %s (protocol %s, harness %s)", h.ID, address, protocol, h.Harness)
switch h.Harness {
case "claude":
adapters[h.ID] = herdr.Claude(client, 200000, s)
@@ -267,7 +290,7 @@ func main() {
}
coordinator = &orchestrator.Coordinator{Store: s, StatePath: filepath.Join(dir, "runtime-sessions.json"), Worktrees: worktrees, Adapters: orchestrator.AdapterFactory{Herdrs: adapters}, LocalHerdr: func(id string) bool {
h, ok := rr.Herdr(id)
return ok && (localMachine == "" || h.MachineID == localMachine)
return ok && coordinatorOwnsHerdr(h, localMachine)
}}
rt.OnLease = func(e domain.Event) error {
// In federated mode the coordinator must never inspect a remote
@@ -301,17 +324,14 @@ func main() {
}
mux := http.NewServeMux()
// B18: the UI is a full control plane — it can create tasks, release or
// complete them, and inject approval keystrokes into live panes. Refuse
// to serve it unauthenticated rather than silently exposing that on
// whatever interface the listener binds to.
webToken := os.Getenv("ORCHESTRA_WEB_TOKEN")
if webToken == "" {
log.Fatal("ORCHESTRA_WEB_TOKEN must be set: it gates the web UI's task, lifecycle and approval controls")
// complete them, and inject approval keystrokes into live panes. It has
// one explicit operator identity and is never enabled by a missing env var.
webCredentials := authz.WebCredentials{Username: os.Getenv("ORCHESTRA_WEB_USERNAME"), PasswordHash: os.Getenv("ORCHESTRA_WEB_PASSWORD_HASH")}
if err := webCredentials.Validate(); err != nil {
log.Fatalf("web login configuration: %v", err)
}
sessions := &authz.Sessions{}
// A browser cannot put a Bearer token on a document load, so it trades
// the token once for an HttpOnly cookie. Same credential, presentable
// form; no new authority is created here.
// A browser login exchanges verified credentials for an HttpOnly cookie.
mux.HandleFunc("/v1/ui/session", func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodDelete {
// Expire the browser credential even if it is already absent or stale.
@@ -329,15 +349,13 @@ func main() {
return
}
var body struct {
Token string `json:"token"`
Username string `json:"username"`
Password string `json:"password"`
}
_ = json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&body)
supplied := body.Token
if supplied == "" {
supplied = strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
}
if subtle.ConstantTimeCompare([]byte(supplied), []byte(webToken)) != 1 {
http.Error(w, "invalid token", http.StatusUnauthorized)
decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&body); err != nil || !webCredentials.Authenticate(body.Username, body.Password) {
http.Error(w, "invalid credentials", http.StatusUnauthorized)
return
}
v, err := sessions.Issue()
@@ -414,6 +432,17 @@ func main() {
http.Error(w, "invalid json", 400)
return
}
// Make the selected project's deterministic gate part of the immutable
// task contract before any worker can create TASK.md. A caller may
// override it only when it has deliberately supplied a task-specific
// gate; routing never asks a harness to choose one.
if _, set := p["quality_gate"]; !set {
if projectID, _ := p["project"].(string); projectID != "" {
if project, ok := rr.Project(projectID); ok && project.QualityGate != "" {
p["quality_gate"] = project.QualityGate
}
}
}
b, _ := json.Marshal(p)
e := domain.Event{ID: id(), Type: "TaskCreated", TaskID: id(), Version: 1, Payload: b, Surface: string(surface(r))}
if err := s.Append(e); err != nil {
@@ -665,7 +694,7 @@ func main() {
}
json.NewEncoder(w).Encode(out)
})
adminServer := &admin.Server{Store: s, RouterReady: rt != nil, Providers: providerHealth, Probes: map[string]admin.ProbeFunc{
adminServer := &admin.Server{Store: s, RouterReady: rt != nil, Build: buildinfo.Current(), Providers: providerHealth, Probes: map[string]admin.ProbeFunc{
"router": func() (bool, string) { return rt != nil, "configured router" },
"gitea": func() (bool, string) {
configured := os.Getenv("ORCHESTRA_GITEA_URL") != "" || os.Getenv("ORCHESTRA_GITEA_CONFIG") != ""
@@ -1033,7 +1062,7 @@ func main() {
w.WriteHeader(http.StatusNoContent)
})
mux.HandleFunc("/v1/federation/workers/", func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || (!strings.HasSuffix(r.URL.Path, "/heartbeat") && !strings.HasSuffix(r.URL.Path, "/handoff") && !strings.HasSuffix(r.URL.Path, "/complete") && !strings.HasSuffix(r.URL.Path, "/captures")) {
if r.Method != http.MethodPost || (!strings.HasSuffix(r.URL.Path, "/heartbeat") && !strings.HasSuffix(r.URL.Path, "/renew") && !strings.HasSuffix(r.URL.Path, "/handoff") && !strings.HasSuffix(r.URL.Path, "/complete") && !strings.HasSuffix(r.URL.Path, "/captures")) {
http.Error(w, "not found", 404)
return
}
@@ -1083,10 +1112,18 @@ func main() {
return
}
var b struct {
TaskID string `json:"task_id"`
TTLSeconds int `json:"ttl_seconds"`
HandoffRef string `json:"handoff_ref"`
AnchorSHA string `json:"anchor_sha"`
TaskID string `json:"task_id"`
TTLSeconds int `json:"ttl_seconds"`
ExpectedVersion int `json:"expected_version"`
HandoffRef string `json:"handoff_ref"`
AnchorSHA string `json:"anchor_sha"`
TransactionID string `json:"transaction_id"`
LeaseVersion int `json:"lease_version"`
ResultSHA string `json:"result_sha"`
Branch string `json:"branch"`
Remote string `json:"remote"`
Receipt map[string]any `json:"receipt"`
SessionEvidence domain.SessionEvidence `json:"session_evidence"`
}
if json.NewDecoder(r.Body).Decode(&b) != nil || b.TaskID == "" {
http.Error(w, "invalid lease body", 400)
@@ -1098,6 +1135,13 @@ func main() {
return
}
ownedLease := t.State == domain.StateLeased && t.Lease != nil && t.Lease.HarnessID == parts[3]
// A response can be lost after the append/fsync. Retrying the exact
// release transaction is therefore a successful no-op, never a second
// TaskReleased event and never a reason to discard the predecessor.
if strings.HasSuffix(r.URL.Path, "/handoff") && t.State == domain.StateQueued && b.TransactionID != "" && b.TransactionID == t.ReleaseTransaction && b.HandoffRef == t.HandoffRef && b.AnchorSHA == t.ReleaseAnchor {
w.WriteHeader(http.StatusNoContent)
return
}
// A prompt timeout can block the coordinator after herdr already
// accepted the request. If that same authenticated worker later reports
// a durable completion, reconcile it rather than preserving a known
@@ -1107,17 +1151,73 @@ func main() {
http.Error(w, "lease not owned", 409)
return
}
if strings.HasSuffix(r.URL.Path, "/complete") && b.ExpectedVersion != t.Version {
http.Error(w, "lease version conflict", http.StatusConflict)
return
}
if strings.HasSuffix(r.URL.Path, "/renew") {
ttl := b.TTLSeconds
if ttl == 0 {
ttl = int((30 * time.Minute).Seconds())
}
e, err := s.RenewLease(b.TaskID, parts[3], b.ExpectedVersion, time.Duration(ttl)*time.Second)
if err != nil {
http.Error(w, err.Error(), http.StatusConflict)
return
}
json.NewEncoder(w).Encode(e)
return
}
if strings.HasSuffix(r.URL.Path, "/pickup") {
if !ownedLease || b.TransactionID == "" || b.TransactionID != t.ReleaseTransaction || b.HandoffRef != t.HandoffRef || b.AnchorSHA != t.ReleaseAnchor {
http.Error(w, "pickup does not match active release", http.StatusConflict)
return
}
if t.PickupTransaction == b.TransactionID && t.PickupLeaseVersion == b.LeaseVersion {
w.WriteHeader(http.StatusNoContent)
return
}
if b.LeaseVersion != t.Version {
http.Error(w, "lease version conflict", http.StatusConflict)
return
}
p, _ := json.Marshal(map[string]any{"transaction_id": b.TransactionID, "handoff_ref": b.HandoffRef, "anchor_sha": b.AnchorSHA, "harness_id": parts[3], "lease_version": b.LeaseVersion, "expected_version": t.Version, "session_evidence": b.SessionEvidence})
e := domain.Event{ID: id(), Type: "TaskPickupValidated", TaskID: b.TaskID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
if err := s.Append(e); err != nil {
http.Error(w, err.Error(), http.StatusConflict)
return
}
json.NewEncoder(w).Encode(e)
return
}
if strings.HasSuffix(r.URL.Path, "/complete") {
if b.HandoffRef == "" {
http.Error(w, "report_ref required", 400)
return
}
p, _ := json.Marshal(map[string]any{"report_ref": b.HandoffRef, "receipt": map[string]any{"harness_id": parts[3], "consumed": 0}})
if len(b.ResultSHA) != 40 || b.Branch == "" || b.Remote == "" {
http.Error(w, "verified result_sha, branch, and remote required", 400)
return
}
if b.Receipt == nil {
b.Receipt = map[string]any{}
}
b.Receipt["harness_id"] = parts[3]
if _, ok := b.Receipt["consumed"]; !ok {
b.Receipt["consumed"] = 0
}
p, _ := json.Marshal(map[string]any{"report_ref": b.HandoffRef, "receipt": b.Receipt, "result_sha": b.ResultSHA, "branch": b.Branch, "remote": b.Remote, "session_evidence": b.SessionEvidence})
e := domain.Event{ID: id(), Type: "TaskCompleted", TaskID: b.TaskID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
if err := s.Append(e); err != nil {
http.Error(w, err.Error(), 409)
return
}
if consumed, ok := b.Receipt["consumed"].(float64); ok && consumed > 0 {
qp, _ := json.Marshal(map[string]any{"harness_id": parts[3], "consumed": consumed})
if err := s.Append(domain.Event{ID: id(), Type: "QuotaReported", TaskID: "system", Payload: qp, Surface: string(authz.System)}); err != nil {
log.Printf("federated quota report %s: %v", b.TaskID, err)
}
}
json.NewEncoder(w).Encode(e)
return
}
@@ -1129,7 +1229,11 @@ func main() {
http.Error(w, "anchor_sha required", 400)
return
}
p, _ := json.Marshal(map[string]any{"handoff_ref": b.HandoffRef, "harness_id": parts[3], "anchor_sha": b.AnchorSHA})
if b.TransactionID == "" || b.ExpectedVersion != t.Version {
http.Error(w, "release transaction and current lease version required", http.StatusConflict)
return
}
p, _ := json.Marshal(map[string]any{"handoff_ref": b.HandoffRef, "harness_id": parts[3], "anchor_sha": b.AnchorSHA, "transaction_id": b.TransactionID, "expected_version": t.Version, "session_evidence": b.SessionEvidence})
e := domain.Event{ID: id(), Type: "TaskReleased", TaskID: b.TaskID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
if err := s.Append(e); err != nil {
http.Error(w, err.Error(), 409)
@@ -1257,7 +1361,7 @@ func main() {
}
log.Println("orchestra listening on :" + port)
tokens := map[authz.Surface]string{
authz.TUI: os.Getenv("ORCHESTRA_TUI_TOKEN"), authz.Web: os.Getenv("ORCHESTRA_WEB_TOKEN"),
authz.TUI: os.Getenv("ORCHESTRA_TUI_TOKEN"),
authz.MCP: os.Getenv("ORCHESTRA_MCP_TOKEN"), authz.Maven: os.Getenv("ORCHESTRA_MAVEN_TOKEN"),
// S12: this is the credential callers present *to* Orchestra on the
// ntfy surface. ORCHESTRA_NTFY_TOKEN is a different secret entirely —