kami/orchestra
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Harden worker federation and operator UI

Browse Source
This commit is contained in:
kami
2026-07-29 13:30:55 +04:00
parent 95a96d87a5
commit 1ca9d64e89
35 changed files with 1195 additions and 581 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/orchestrator/orchestrator.go
+26 -24
View File
@@ -170,11 +170,15 @@ type Coordinator struct {
Worktrees Worktrees
Adapters Adapters
StatePath string
mu sync.Mutex
sessions map[string]herdr.Session
loaded bool
healthMu sync.RWMutex
health MonitorHealth
// LocalHerdr, when set, is the coordinator's machine-ownership boundary.
// A coordinator must never operate a pane or checkout owned by another
// machine; federation workers own those operations locally.
LocalHerdr func(string) bool
mu sync.Mutex
sessions map[string]herdr.Session
loaded bool
healthMu sync.RWMutex
health MonitorHealth
// Hard is the occupancy threshold Monitor's periodic rotate() runs
// against, mirrored here so TurnDecision (the synchronous, per-turn
// counterpart driven by the Face-B stop hook) evaluates the same
@@ -300,6 +304,9 @@ func (c *Coordinator) adapterFor(taskID string, session herdr.Session) (herdr.Ad
if id == "" {
id = session.Harness
}
if c.LocalHerdr != nil && !c.LocalHerdr(id) {
return nil, fmt.Errorf("session %s is owned by non-local herdr %s", taskID, id)
}
return c.Adapters.Adapter(id)
}
@@ -875,31 +882,20 @@ func (c *Coordinator) Start(ctx context.Context, e domain.Event) error {
if err := json.Unmarshal(e.Payload, &p); err != nil || p.HarnessID == "" {
return fmt.Errorf("orchestrator: invalid lease")
}
if c.LocalHerdr != nil && !c.LocalHerdr(p.HarnessID) {
return c.block(t, "remote herdr must be operated by its federation worker")
}
a, err := c.Adapters.Adapter(p.HarnessID)
if err != nil {
return c.block(t, "adapter: "+err.Error())
}
var w string
if creator, ok := a.(herdr.WorktreeCreator); ok {
planner, planned := c.Worktrees.(WorktreeSpec)
if !planned {
return c.block(t, "worktree: repository specification unavailable")
}
repo, root, valid := planner.Spec(t)
if !valid {
return c.block(t, "worktree: repository and root required")
}
w, err = creator.CreateWorktree(ctx, repo, root, t.ID)
} else {
w, err = c.Worktrees.Create(ctx, t)
}
// Worktrees, including immutable TASK.md, are coordinator-local state.
// A remote herdr must be driven by its federation worker instead of being
// asked to create an opaque checkout that this coordinator cannot validate.
w, err := c.Worktrees.Create(ctx, t)
if err != nil {
return c.block(t, "worktree: "+err.Error())
}
// Best-effort: TASK.md only exists for worktrees this process can read
// locally (the GitWorktrees path). A herdr-hosted worktree on a remote
// machine (WorktreeCreator path) is the same cross-host gap named in
// AUDIT.md's federation-fork section — not solved here.
taskFileSHA, _ := continuity.TaskFileHash(w)
prompt := taskLaunchPrompt(t)
var s herdr.Session
@@ -983,7 +979,13 @@ func (c *Coordinator) rememberSession(taskID string, s herdr.Session) error {
}
func (c *Coordinator) block(t domain.Task, reason string) error {
b, _ := json.Marshal(map[string]string{"blocker": reason})
p := map[string]string{"blocker": reason, "pane_state": "unknown"}
if s, ok := c.Session(t.ID); ok {
p["pane_id"] = s.PaneID
p["harness_id"] = s.HerdrID
p["pane_state"] = "open"
}
b, _ := json.Marshal(p)
return c.Store.Append(domain.Event{ID: domain.NewID(), Type: "TaskBlocked", TaskID: t.ID, Version: t.Version + 1, Payload: b, Surface: string(authz.System)})
}
internal/orchestrator/rotation_test.go
+120
View File
@@ -21,9 +21,17 @@ type fakeAdapter struct {
boundary bool
ref string
releases int
leases int
approval struct {
called bool
grant bool
session herdr.Session
capture string
}
}
func (a *fakeAdapter) Lease(_ context.Context, _ string, worktree string) (herdr.Session, error) {
a.leases++
return herdr.Session{Harness: "h1", PaneID: "pane-1", Worktree: worktree}, nil
}
func (a *fakeAdapter) Bootstrap(context.Context, herdr.Session, string) error { return nil }
@@ -36,6 +44,13 @@ func (a *fakeAdapter) Occupancy(herdr.Session) (float64, error) { return a.occu
func (a *fakeAdapter) AtTurnBoundary(context.Context, herdr.Session) (bool, error) {
return a.boundary, nil
}
func (a *fakeAdapter) RespondApproval(_ context.Context, s herdr.Session, grant bool, capture string) error {
a.approval.called = true
a.approval.grant = grant
a.approval.session = s
a.approval.capture = capture
return nil
}
type worktrees struct{ path string }
@@ -45,6 +60,12 @@ type adapters struct{ a herdr.Adapter }
func (a adapters) Adapter(string) (herdr.Adapter, error) { return a.a, nil }
type promptFailureAdapter struct{ fakeAdapter }
func (a *promptFailureAdapter) LeasePrompt(_ context.Context, _ string, worktree, _ string) (herdr.Session, error) {
return herdr.Session{Harness: "h1", PaneID: "pane-created-before-timeout", Worktree: worktree}, errors.New("prompt delivery uncertain")
}
func run(t *testing.T, dir string, args ...string) {
t.Helper()
cmd := exec.Command("git", append([]string{"-C", dir}, args...)...)
@@ -53,6 +74,105 @@ func run(t *testing.T, dir string, args ...string) {
}
}
func TestPromptFailureRetainsLivePaneForBlockedTaskAcrossRestart(t *testing.T) {
s, err := store.Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
if err := s.Append(domain.Event{ID: domain.NewID(), Type: "TaskCreated", TaskID: "blocked-live-pane", Surface: string(authz.System), Payload: mustJSON(map[string]any{
"source": "qa", "external_id": "prompt-timeout", "project": "p",
})}); err != nil {
t.Fatal(err)
}
task, ok := s.Task("blocked-live-pane")
if !ok {
t.Fatal("created task missing")
}
lease, err := s.Lease(task.ID, "h1", time.Minute)
if err != nil {
t.Fatal(err)
}
statePath := t.TempDir() + "/sessions.json"
a := &promptFailureAdapter{}
c := &orchestrator.Coordinator{Store: s, Worktrees: worktrees{path: t.TempDir()}, Adapters: adapters{a}, StatePath: statePath}
if err := c.Start(context.Background(), lease); err != nil {
t.Fatal(err)
}
if got, ok := s.Task(task.ID); !ok || got.State != domain.StateBlocked {
t.Fatalf("task state = %+v, want blocked", got)
}
if session, ok := c.Session(task.ID); !ok || session.PaneID != "pane-created-before-timeout" || session.HerdrID != "h1" {
t.Fatalf("retained session = %+v, present=%v", session, ok)
}
// A fresh coordinator must retain the mapping for a blocked task rather
// than treating it as an orphan after restart.
restarted := &orchestrator.Coordinator{Store: s, Worktrees: worktrees{path: t.TempDir()}, Adapters: adapters{a}, StatePath: statePath}
if err := restarted.Reconcile(context.Background()); err != nil {
t.Fatal(err)
}
if session, ok := restarted.Session(task.ID); !ok || session.PaneID != "pane-created-before-timeout" {
t.Fatalf("restarted session = %+v, present=%v", session, ok)
}
}
func TestRespondApprovalUsesOwningSessionAndPreservesCaptureBinding(t *testing.T) {
s, err := store.Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
if err := s.Append(domain.Event{ID: domain.NewID(), Type: "TaskCreated", TaskID: "approval-task", Surface: string(authz.System), Payload: mustJSON(map[string]any{
"source": "qa", "external_id": "approval", "project": "p",
})}); err != nil {
t.Fatal(err)
}
lease, err := s.Lease("approval-task", "herdr-1", time.Minute)
if err != nil {
t.Fatal(err)
}
a := &fakeAdapter{}
c := &orchestrator.Coordinator{Store: s, Worktrees: worktrees{path: t.TempDir()}, Adapters: adapters{a}, StatePath: t.TempDir() + "/sessions.json"}
if err := c.Start(context.Background(), lease); err != nil {
t.Fatal(err)
}
const capture = "Approval required\n$ go test ./...\n[y/n]"
if err := c.RespondApproval(context.Background(), "approval-task", true, capture); err != nil {
t.Fatal(err)
}
if !a.approval.called || !a.approval.grant || a.approval.capture != capture {
t.Fatalf("approval invocation = %#v", a.approval)
}
if a.approval.session.HerdrID != "herdr-1" || a.approval.session.PaneID == "" {
t.Fatalf("approval used wrong session: %#v", a.approval.session)
}
}
func TestCoordinatorRefusesRemoteHerdrOperations(t *testing.T) {
s, err := store.Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
if err := s.Append(domain.Event{ID: domain.NewID(), Type: "TaskCreated", TaskID: "remote", Surface: string(authz.System), Payload: mustJSON(map[string]any{
"source": "qa", "external_id": "remote", "project": "p",
})}); err != nil {
t.Fatal(err)
}
lease, err := s.Lease("remote", "remote", time.Minute)
if err != nil {
t.Fatal(err)
}
a := &fakeAdapter{}
c := &orchestrator.Coordinator{Store: s, Worktrees: worktrees{path: t.TempDir()}, Adapters: adapters{a}, LocalHerdr: func(id string) bool { return id == "local" }}
if err := c.Start(context.Background(), lease); err != nil {
t.Fatal(err)
}
if a.leases != 0 {
t.Fatal("remote adapter was started by coordinator")
}
if task, ok := s.Task("remote"); !ok || task.State != domain.StateBlocked {
t.Fatalf("remote task state = %#v, present=%v; want blocked", task, ok)
}
}
// TestRotationEmitsValidReleaseWithAnchorSHA guards the highest-priority spec
// defect noted in progress.md: automated rotation must emit a TaskReleased
// event that satisfies domain.ValidatePayload (handoff_ref + anchor_sha), not