kami/orchestra
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: make worker handoff rotation durable

Browse Source
This commit is contained in:
kami
2026-07-30 01:30:59 +04:00
parent ce02c60106
commit 1ff0af2e69
16 changed files with 1488 additions and 1566 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/orchestrator/orchestrator.go
+14 -22
View File
@@ -774,18 +774,21 @@ func (c *Coordinator) TurnDecision(ctx context.Context, taskID string) (string,
if existingReason := handoffReason(session.Worktree); existingReason == "manual" || existingReason == "milestone" || existingReason == "thrash" {
return c.finishRelease(ctx, taskID, task, session, a, existingReason)
}
if trigger, deadEnds := checkActivityTriggers(ctx, a, session, c.Thrash); trigger != "" {
c.requestReasonedHandoff(ctx, taskID, session, a, trigger, deadEnds)
return TurnPrepareHandoff, nil
d := (RotationStateMachine{Soft: c.soft(), Hard: c.Hard, Thrash: c.Thrash}).Evaluate(ctx, a, session)
if d.Degraded != nil {
return "", fmt.Errorf("orchestrator: rotation: %w", d.Degraded)
}
occupancy, err := a.Occupancy(session)
if err != nil {
return "", fmt.Errorf("orchestrator: occupancy: %w", err)
}
if occupancy < c.soft() {
if d.Action == TurnContinue {
return TurnContinue, nil
}
if occupancy < c.Hard {
if d.Action == TurnRefuse {
return TurnRefuse, nil
}
if d.Reason == "milestone" || d.Reason == "thrash" {
c.requestReasonedHandoff(ctx, taskID, session, a, d.Reason, d.DeadEnds)
return TurnPrepareHandoff, nil
}
if d.Action == TurnPrepareHandoff {
// Soft threshold (§5.3): advisory only. Ask the agent to start
// preparing a handoff well before Hard forces one, but don't block
// the turn on a boundary check — the agent is free to keep working.
@@ -805,18 +808,6 @@ func (c *Coordinator) TurnDecision(ctx context.Context, taskID string) (string,
}
return TurnPrepareHandoff, nil
}
if boundary, ok := a.(herdr.TurnBoundary); ok {
atBoundary, boundaryErr := boundary.AtTurnBoundary(ctx, session)
if boundaryErr != nil {
c.recordTurnBoundaryDegraded()
return TurnRefuse, nil
}
if !atBoundary {
return TurnRefuse, nil
}
} else {
c.recordTurnBoundaryDegraded()
}
if requester, ok := a.(herdr.HandoffRequester); ok {
if _, statErr := os.Stat(filepath.Join(session.Worktree, herdr.HandoffReportFile)); statErr != nil {
if !session.HandoffRequested {
@@ -979,11 +970,12 @@ func (c *Coordinator) rememberSession(taskID string, s herdr.Session) error {
}
func (c *Coordinator) block(t domain.Task, reason string) error {
p := map[string]string{"blocker": reason, "pane_state": "unknown"}
p := map[string]any{"blocker": reason, "block_reason": string(domain.InferBlockReason(reason)), "pane_state": "unknown", "session_evidence": domain.SessionEvidence{PaneState: "unknown", Source: "coordinator", CheckedAt: time.Now().UTC()}}
if s, ok := c.Session(t.ID); ok {
p["pane_id"] = s.PaneID
p["harness_id"] = s.HerdrID
p["pane_state"] = "open"
p["session_evidence"] = domain.SessionEvidence{PaneID: s.PaneID, HarnessID: s.HerdrID, PaneState: "open", Source: "coordinator", CheckedAt: time.Now().UTC()}
}
b, _ := json.Marshal(p)
return c.Store.Append(domain.Event{ID: domain.NewID(), Type: "TaskBlocked", TaskID: t.ID, Version: t.Version + 1, Payload: b, Surface: string(authz.System)})
internal/orchestrator/rotation_state.go
+82
View File
@@ -0,0 +1,82 @@
package orchestrator
import (
"context"
"fmt"
"orchestra/internal/continuity"
"orchestra/internal/herdr"
)
// RotationStateMachine is the shared, side-effect-free rotation policy used
// by both the coordinator's synchronous turn path and federation workers.
// Callers persist request/release side effects themselves, but must never
// replace an unavailable occupancy reading with zero.
type RotationStateMachine struct {
Soft float64
Hard float64
Thrash herdr.ThrashConfig
}
type RotationDecision struct {
Action string // continue, prepare_handoff, rotate_now
Reason string
DeadEnds []continuity.DeadEnd
// ActivityDegraded is advisory (threshold rotation still has a real usage
// source); it is surfaced so a missing milestone/thrash feed cannot be a
// silent no-op.
ActivityDegraded error
Degraded error
}
func (m RotationStateMachine) Evaluate(ctx context.Context, a herdr.Adapter, s herdr.Session) RotationDecision {
soft := m.Soft
if soft <= 0 {
soft = defaultSoft
}
if m.Hard <= 0 || m.Hard <= soft {
return RotationDecision{Degraded: fmt.Errorf("invalid rotation thresholds soft=%v hard=%v", soft, m.Hard)}
}
if reader, ok := a.(herdr.ActivityReader); ok {
calls, err := reader.Activity(ctx, s)
if err == nil {
if thrash, deadEnds := herdr.DetectThrash(calls, m.Thrash); thrash {
return RotationDecision{Action: TurnPrepareHandoff, Reason: "thrash", DeadEnds: deadEnds}
}
if herdr.DetectMilestone(calls) {
return RotationDecision{Action: TurnPrepareHandoff, Reason: "milestone"}
}
} else {
return m.evaluateOccupancy(ctx, a, s, fmt.Errorf("activity unknown: %w", err))
}
}
return m.evaluateOccupancy(ctx, a, s, nil)
}
func (m RotationStateMachine) evaluateOccupancy(ctx context.Context, a herdr.Adapter, s herdr.Session, activityErr error) RotationDecision {
soft := m.Soft
if soft <= 0 {
soft = defaultSoft
}
occupancy, err := a.Occupancy(s)
if err != nil {
return RotationDecision{ActivityDegraded: activityErr, Degraded: fmt.Errorf("occupancy unknown: %w", err)}
}
if occupancy < soft {
return RotationDecision{Action: TurnContinue, ActivityDegraded: activityErr}
}
if occupancy < m.Hard {
return RotationDecision{Action: TurnPrepareHandoff, Reason: "threshold", ActivityDegraded: activityErr}
}
boundary, ok := a.(herdr.TurnBoundary)
if !ok {
return RotationDecision{ActivityDegraded: activityErr, Degraded: fmt.Errorf("turn boundary unknown at hard threshold"), Action: TurnRefuse, Reason: "threshold"}
}
atBoundary, err := boundary.AtTurnBoundary(ctx, s)
if err != nil {
return RotationDecision{ActivityDegraded: activityErr, Degraded: fmt.Errorf("turn boundary unknown: %w", err), Action: TurnRefuse, Reason: "threshold"}
}
if !atBoundary {
return RotationDecision{Action: TurnRefuse, Reason: "threshold", ActivityDegraded: activityErr}
}
return RotationDecision{Action: TurnRotateNow, Reason: "threshold", ActivityDegraded: activityErr}
}
internal/orchestrator/rotation_test.go
+30 -12
View File
@@ -708,6 +708,7 @@ type activityAdapter struct {
activityErr error
reasonAsked []string
deadEndsSeen []continuity.DeadEnd
observed chan struct{}
}
func (a *activityAdapter) Activity(context.Context, herdr.Session) ([]herdr.ToolCall, error) {
@@ -717,6 +718,12 @@ func (a *activityAdapter) Activity(context.Context, herdr.Session) ([]herdr.Tool
func (a *activityAdapter) RequestHandoffReason(_ context.Context, _ herdr.Session, reason string, deadEnds []continuity.DeadEnd) error {
a.reasonAsked = append(a.reasonAsked, reason)
a.deadEndsSeen = deadEnds
if a.observed != nil {
select {
case a.observed <- struct{}{}:
default:
}
}
return nil
}
@@ -764,7 +771,7 @@ func TestActivityTriggersRequestReasonedHandoffWithoutReleasing(t *testing.T) {
}
t.Run("thrash requests a reasoned handoff and does not release, via TurnDecision", func(t *testing.T) {
a := &activityAdapter{fakeAdapter: fakeAdapter{occupancy: 0, boundary: true}, calls: thrashCalls}
a := &activityAdapter{fakeAdapter: fakeAdapter{occupancy: 0, boundary: true}, calls: thrashCalls, observed: make(chan struct{}, 1)}
c, st, task := newCoordinator(a)
decision, err := c.TurnDecision(context.Background(), task.ID)
if err != nil {
@@ -862,13 +869,15 @@ func TestActivityTriggersRequestReasonedHandoffWithoutReleasing(t *testing.T) {
c, st, task := newCoordinator(a)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go c.Monitor(ctx, .8, time.Millisecond)
done := make(chan error, 1)
go func() { done <- c.Monitor(ctx, .8, time.Millisecond) }()
deadline := time.Now().Add(300 * time.Millisecond)
for time.Now().Before(deadline) && len(a.reasonAsked) == 0 {
time.Sleep(time.Millisecond)
select {
case <-a.observed:
case <-time.After(300 * time.Millisecond):
}
cancel()
<-done
if len(a.reasonAsked) == 0 || a.reasonAsked[0] != "thrash" {
t.Fatalf("reasonAsked=%v want a thrash request from rotate()", a.reasonAsked)
}
@@ -959,10 +968,17 @@ func (w specWorktrees) Spec(domain.Task) (string, string, bool) { re
type conventionsAdapter struct {
fakeAdapter
notifications int
observed chan struct{}
}
func (a *conventionsAdapter) NotifyConventionsChanged(context.Context, herdr.Session) error {
a.notifications++
if a.observed != nil {
select {
case a.observed <- struct{}{}:
default:
}
}
return nil
}
@@ -991,7 +1007,7 @@ func TestConventionsDriftNotifiesActiveSession(t *testing.T) {
t.Fatal(err)
}
task := s.Tasks()[0]
a := &conventionsAdapter{fakeAdapter: fakeAdapter{occupancy: 0}}
a := &conventionsAdapter{fakeAdapter: fakeAdapter{occupancy: 0}, observed: make(chan struct{}, 1)}
c := &orchestrator.Coordinator{Store: s, Worktrees: specWorktrees{wtPath: worktree, repoPath: repo}, Adapters: adapters{a}, StatePath: t.TempDir() + "/sessions.json"}
leaseEvt, err := s.Lease(task.ID, "h1", time.Minute)
@@ -1003,8 +1019,8 @@ func TestConventionsDriftNotifiesActiveSession(t *testing.T) {
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go c.Monitor(ctx, .8, time.Millisecond)
done := make(chan error, 1)
go func() { done <- c.Monitor(ctx, .8, time.Millisecond) }()
time.Sleep(50 * time.Millisecond)
if a.notifications != 0 {
@@ -1015,10 +1031,12 @@ func TestConventionsDriftNotifiesActiveSession(t *testing.T) {
t.Fatal(err)
}
deadline := time.Now().Add(time.Second)
for time.Now().Before(deadline) && a.notifications == 0 {
time.Sleep(time.Millisecond)
select {
case <-a.observed:
case <-time.After(time.Second):
}
cancel()
<-done
if a.notifications == 0 {
t.Fatal("session was never notified of the conventions-doc update")
}