kami/orchestra
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Harden lease lifecycle durability

Browse Source
This commit is contained in:
kami
2026-07-30 14:34:29 +04:00
parent 1ff0af2e69
commit f6ee0e3060
40 changed files with 2108 additions and 590 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/federation/client.go
+8 -8
View File
@@ -131,8 +131,8 @@ func (c Client) Heartbeat(ctx context.Context, health WorkerHealth) error {
}
return err
}
func (c Client) Renew(ctx context.Context, taskID string, expectedVersion, ttlSeconds int) error {
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/renew", map[string]any{"task_id": taskID, "expected_version": expectedVersion, "ttl_seconds": ttlSeconds})
func (c Client) Renew(ctx context.Context, taskID, epoch string, expectedVersion, ttlSeconds int) error {
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/renew", map[string]any{"task_id": taskID, "lease_epoch": epoch, "expected_version": expectedVersion, "ttl_seconds": ttlSeconds})
if resp != nil {
resp.Body.Close()
}
@@ -176,22 +176,22 @@ func (c Client) PutArtifact(ctx context.Context, b []byte) (string, error) {
}
return out.Ref, nil
}
func (c Client) Release(ctx context.Context, taskID, ref, anchor, transactionID string, expectedVersion int, evidence domain.SessionEvidence) error {
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/handoff", map[string]any{"task_id": taskID, "handoff_ref": ref, "anchor_sha": anchor, "transaction_id": transactionID, "expected_version": expectedVersion, "session_evidence": evidence})
func (c Client) Release(ctx context.Context, taskID, ref, anchor, transactionID, epoch string, expectedVersion int, evidence domain.SessionEvidence) error {
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/handoff", map[string]any{"task_id": taskID, "handoff_ref": ref, "anchor_sha": anchor, "transaction_id": transactionID, "lease_epoch": epoch, "expected_version": expectedVersion, "session_evidence": evidence})
if resp != nil {
resp.Body.Close()
}
return err
}
func (c Client) Pickup(ctx context.Context, taskID, ref, anchor, transactionID string, leaseVersion int, evidence domain.SessionEvidence) error {
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/pickup", map[string]any{"task_id": taskID, "handoff_ref": ref, "anchor_sha": anchor, "transaction_id": transactionID, "lease_version": leaseVersion, "session_evidence": evidence})
func (c Client) Pickup(ctx context.Context, taskID, ref, anchor, transactionID, epoch string, leaseVersion int, evidence domain.SessionEvidence) error {
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/pickup", map[string]any{"task_id": taskID, "handoff_ref": ref, "anchor_sha": anchor, "transaction_id": transactionID, "lease_epoch": epoch, "lease_version": leaseVersion, "session_evidence": evidence})
if resp != nil {
resp.Body.Close()
}
return err
}
func (c Client) Complete(ctx context.Context, taskID, reportRef, resultSHA, branch, remote string, expectedVersion int, receipt map[string]any, evidence domain.SessionEvidence) error {
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/complete", map[string]any{"task_id": taskID, "handoff_ref": reportRef, "result_sha": resultSHA, "branch": branch, "remote": remote, "expected_version": expectedVersion, "receipt": receipt, "session_evidence": evidence})
func (c Client) Complete(ctx context.Context, taskID, reportRef, resultSHA, branch, remote, epoch string, expectedVersion int, receipt map[string]any, evidence domain.SessionEvidence) error {
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/complete", map[string]any{"task_id": taskID, "handoff_ref": reportRef, "result_sha": resultSHA, "branch": branch, "remote": remote, "lease_epoch": epoch, "expected_version": expectedVersion, "receipt": receipt, "session_evidence": evidence})
if resp != nil {
resp.Body.Close()
}
internal/federation/federation.go
+61 -15
View File
@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"orchestra/internal/buildinfo"
"os"
"path/filepath"
"sync"
@@ -15,13 +16,15 @@ var ErrUnknownWorker = errors.New("unknown worker")
var ErrUnauthorized = errors.New("worker authentication failed")
type Worker struct {
ID string `json:"id"`
Address string `json:"address"`
Capacity int `json:"capacity"`
LastSeen time.Time `json:"last_seen"`
Online bool `json:"online"`
Health WorkerHealth `json:"health"`
Token string `json:"-"`
ID string `json:"id"`
Address string `json:"address"`
Capacity int `json:"capacity"`
SupportedProjects []string `json:"supported_projects"`
Build buildinfo.Info `json:"build"`
LastSeen time.Time `json:"last_seen"`
Online bool `json:"online"`
Health WorkerHealth `json:"health"`
Token string `json:"-"`
}
// WorkerHealth is reported by the worker that owns the local herdr socket.
@@ -86,10 +89,12 @@ type persistedState struct {
// is mode 0600, and retaining this binding prevents an arbitrary process from
// registering a recovered worker ID and executing its pending approval.
type persistedWorker struct {
ID string `json:"id"`
Address string `json:"address"`
Capacity int `json:"capacity"`
Token string `json:"token"`
ID string `json:"id"`
Address string `json:"address"`
Capacity int `json:"capacity"`
SupportedProjects []string `json:"supported_projects"`
Build buildinfo.Info `json:"build"`
Token string `json:"token"`
}
// Load restores durable capture/command state. Call this before accepting
@@ -123,7 +128,7 @@ func (r *Registry) Load() error {
if id == "" || w.ID != id || w.Token == "" {
return fmt.Errorf("invalid federation worker %q", id)
}
r.workers[id] = Worker{ID: w.ID, Address: w.Address, Capacity: w.Capacity, Token: w.Token}
r.workers[id] = Worker{ID: w.ID, Address: w.Address, Capacity: w.Capacity, SupportedProjects: w.SupportedProjects, Build: w.Build, Token: w.Token}
}
return nil
}
@@ -135,7 +140,7 @@ func (r *Registry) persistLocked() error {
}
workers := make(map[string]persistedWorker, len(r.workers))
for id, w := range r.workers {
workers[id] = persistedWorker{ID: w.ID, Address: w.Address, Capacity: w.Capacity, Token: w.Token}
workers[id] = persistedWorker{ID: w.ID, Address: w.Address, Capacity: w.Capacity, SupportedProjects: w.SupportedProjects, Build: w.Build, Token: w.Token}
}
b, err := json.Marshal(persistedState{Captures: r.captures, Commands: r.commands, Workers: workers})
if err != nil {
@@ -145,12 +150,31 @@ func (r *Registry) persistLocked() error {
return err
}
tmp := r.StatePath + ".tmp"
if err := os.WriteFile(tmp, b, 0600); err != nil {
f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
if err != nil {
return err
}
if _, err = f.Write(b); err == nil {
err = f.Sync()
}
if closeErr := f.Close(); err == nil {
err = closeErr
}
if err != nil {
_ = os.Remove(tmp)
return err
}
if err := os.Rename(tmp, r.StatePath); err != nil {
return err
}
dir, err := os.Open(filepath.Dir(r.StatePath))
if err != nil {
return err
}
defer dir.Close()
if err := dir.Sync(); err != nil {
return err
}
return os.Chmod(r.StatePath, 0600)
}
@@ -396,10 +420,32 @@ func (r *Registry) Available(id string) bool {
if !ok {
return false
}
w.Online = time.Since(w.LastSeen) <= r.TTL
// A heartbeat merely proves the worker process can reach the coordinator.
// Lease admission additionally requires a fresh probe of the worker's
// local herdr; otherwise a partitioned/down herdr still attracts work.
w.Online = time.Since(w.LastSeen) <= r.TTL && w.Health.HerdrStatus == "reachable" && !w.Health.CheckedAt.IsZero() && time.Since(w.Health.CheckedAt) <= r.TTL
r.workers[id] = w
return w.Online
}
// Supports reports whether an online worker explicitly declared the project.
// An omitted declaration is deliberately not treated as a wildcard: workers
// must never receive a project for which they have no local checkout.
func (r *Registry) Supports(id, project string) bool {
r.mu.Lock()
defer r.mu.Unlock()
r.init()
w, ok := r.workers[id]
if !ok || time.Since(w.LastSeen) > r.TTL {
return false
}
for _, candidate := range w.SupportedProjects {
if candidate == project {
return true
}
}
return false
}
func (r *Registry) Snapshot() []Worker {
r.mu.Lock()
defer r.mu.Unlock()
internal/federation/federation_test.go
+43
View File
@@ -29,6 +29,49 @@ func TestCursorIsMonotonicAndAuthenticationIsRequired(t *testing.T) {
}
}
func TestSupportedProjectsPersistAndGateAvailability(t *testing.T) {
path := filepath.Join(t.TempDir(), "state.json")
r := &Registry{StatePath: path}
if err := r.Register(Worker{ID: "w", Token: "t", SupportedProjects: []string{"test-e2e"}}, ""); err != nil {
t.Fatal(err)
}
if !r.Supports("w", "test-e2e") || r.Supports("w", "correx") {
t.Fatalf("unexpected project support")
}
restarted := &Registry{StatePath: path}
if err := restarted.Load(); err != nil {
t.Fatal(err)
}
if err := restarted.Register(Worker{ID: "w", Token: "t", SupportedProjects: []string{"test-e2e"}}, ""); err != nil {
t.Fatal(err)
}
if !restarted.Supports("w", "test-e2e") || restarted.Supports("w", "correx") {
t.Fatalf("project support did not survive restart")
}
}
func TestAvailableRequiresFreshReachableLocalHerdrHealth(t *testing.T) {
r := &Registry{}
if err := r.Register(Worker{ID: "w", Token: "t"}, ""); err != nil {
t.Fatal(err)
}
if r.Available("w") {
t.Fatal("registration without local herdr probe admitted a worker")
}
if err := r.Heartbeat("w", WorkerHealth{HerdrStatus: "reachable", CheckedAt: time.Now().UTC()}); err != nil {
t.Fatal(err)
}
if !r.Available("w") {
t.Fatal("fresh reachable local herdr was not admitted")
}
if err := r.Heartbeat("w", WorkerHealth{HerdrStatus: "unreachable", CheckedAt: time.Now().UTC()}); err != nil {
t.Fatal(err)
}
if r.Available("w") {
t.Fatal("unreachable local herdr was admitted")
}
}
func TestPendingApprovalSurvivesRegistryRestart(t *testing.T) {
path := filepath.Join(t.TempDir(), "federation-state.json")
r := &Registry{StatePath: path}