7f12c7fc37
The v3 stack, previously an uncommitted working tree, plus this session's two units and the burn-in instrument. This commit is the burn-in build identity: coordinator and worker must both report this revision before a task is created. Workflow (earlier sessions, uncommitted until now): human decision events and reduction, source cursors and reconcile-before-launch, turn-boundary reconciliation, internal/agentctx as the single renderer, ace-fca phases with sealed artifacts, the trajectory gate, bounded grilling, independent review, task pr enforcement, and human review reflection. Capability restrictions at the agent boundary: an authz.Agent surface at GatedWrite may ask and may not act. It also fixes two bugs the unit exposed -- gated surfaces could not reach the two endpoints written for them, and RequestHumanDecision would block an unowned task while rejecting a question from the session that did own it. Turn-boundary reconcile-failure escalation: a streak of consecutive failures asks the session to hand off, fenced on the lease epoch, with reconcile_failure as a real handoff reason. The worker was dropping the coordinator's verdict on the floor; it now acts on it. Burn-in: herdr.WriteLaunchContext dumps the exact agentctx.Build result to <worktree>/.orchestra/launch.md at every launch, local and federated. BURNIN.md is the runbook. deploy/build.sh stamps both binaries from one commit. go build, go vet and go test ./... pass, 20 packages. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
244 lines
7.7 KiB
Go
244 lines
7.7 KiB
Go
package orchestrator_test
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"orchestra/internal/authz"
|
|
"orchestra/internal/domain"
|
|
"orchestra/internal/herdr"
|
|
"orchestra/internal/orchestrator"
|
|
"orchestra/internal/store"
|
|
)
|
|
|
|
// notifyingAdapter records what a live agent was told mid-lease.
|
|
type notifyingAdapter struct {
|
|
fakeAdapter
|
|
notices []string
|
|
err error
|
|
}
|
|
|
|
func (a *notifyingAdapter) NotifyDecisions(_ context.Context, _ herdr.Session, text string) error {
|
|
if a.err != nil {
|
|
return a.err
|
|
}
|
|
a.notices = append(a.notices, text)
|
|
return nil
|
|
}
|
|
|
|
func leasedCoordinator(t *testing.T, a herdr.Adapter, repo string) (*orchestrator.Coordinator, *store.Store, domain.Task) {
|
|
t.Helper()
|
|
s, err := store.Open(t.TempDir())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.Append(domain.Event{ID: domain.NewID(), Type: "TaskCreated", TaskID: "t1", Surface: string(authz.System), Payload: mustJSON(map[string]any{
|
|
"source": "gitea", "external_id": "381", "project": "p",
|
|
})}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
task := s.Tasks()[0]
|
|
c := &orchestrator.Coordinator{Store: s, Worktrees: worktrees{path: repo}, Adapters: adapters{a}, StatePath: t.TempDir() + "/sessions.json", Hard: .8}
|
|
leaseEvt, err := s.Lease(task.ID, "h1", time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := c.Start(context.Background(), leaseEvt); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return c, s, task
|
|
}
|
|
|
|
func recordDecision(t *testing.T, s *store.Store, taskID, id, value string) {
|
|
t.Helper()
|
|
task, ok := s.Task(taskID)
|
|
if !ok {
|
|
t.Fatal("task missing")
|
|
}
|
|
if err := s.Append(domain.Event{
|
|
ID: domain.NewID(), Type: domain.EventHumanDecisionRecorded, TaskID: taskID,
|
|
Version: task.Version + 1, Surface: string(authz.System),
|
|
Payload: mustJSON(map[string]any{
|
|
"decision_id": id, "kind": "correction", "subject": "strategy", "value": value,
|
|
"source": map[string]any{"provider": "gitea", "external_id": "c-" + id},
|
|
}),
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func gitRepo(t *testing.T) string {
|
|
t.Helper()
|
|
repo := t.TempDir()
|
|
run(t, repo, "init")
|
|
run(t, repo, "config", "user.email", "t@t")
|
|
run(t, repo, "config", "user.name", "t")
|
|
run(t, repo, "commit", "--allow-empty", "-m", "init")
|
|
return repo
|
|
}
|
|
|
|
// A correction written while the lease is live reaches the agent at the next
|
|
// verified turn boundary, without preempting anything.
|
|
func TestDecisionDeliveredAtTurnBoundary(t *testing.T) {
|
|
a := ¬ifyingAdapter{fakeAdapter: fakeAdapter{occupancy: .5}}
|
|
c, s, task := leasedCoordinator(t, a, gitRepo(t))
|
|
|
|
reconciled := 0
|
|
c.ReconcileHumanInput = func(_ context.Context, taskID string) error {
|
|
reconciled++
|
|
if reconciled == 1 {
|
|
recordDecision(t, s, taskID, "d1", "no, use b")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
verdict, err := c.TurnDecision(context.Background(), task.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if verdict != orchestrator.TurnContinue {
|
|
t.Fatalf("verdict = %q, want continue", verdict)
|
|
}
|
|
if reconciled != 1 {
|
|
t.Fatalf("reconciled %d times, want 1", reconciled)
|
|
}
|
|
if len(a.notices) != 1 || !strings.Contains(a.notices[0], "no, use b") {
|
|
t.Fatalf("notices = %v", a.notices)
|
|
}
|
|
if !strings.Contains(a.notices[0], "outrank") {
|
|
t.Fatal("notice does not state that the decision outranks the current plan")
|
|
}
|
|
|
|
// Same decision at the next boundary is not re-sent.
|
|
if _, err := c.TurnDecision(context.Background(), task.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(a.notices) != 1 {
|
|
t.Fatalf("decision re-delivered: %v", a.notices)
|
|
}
|
|
|
|
// A second, newer decision is delivered on its own.
|
|
recordDecision(t, s, task.ID, "d2", "and keep the old flag")
|
|
if _, err := c.TurnDecision(context.Background(), task.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(a.notices) != 2 || !strings.Contains(a.notices[1], "and keep the old flag") {
|
|
t.Fatalf("notices = %v", a.notices)
|
|
}
|
|
if strings.Contains(a.notices[1], "no, use b") {
|
|
t.Fatal("already delivered decision repeated")
|
|
}
|
|
}
|
|
|
|
// Decisions carried by the launch instruction are not re-announced as news.
|
|
func TestDecisionsFromLaunchAreNotRedelivered(t *testing.T) {
|
|
repo := gitRepo(t)
|
|
s, err := store.Open(t.TempDir())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.Append(domain.Event{ID: domain.NewID(), Type: "TaskCreated", TaskID: "t1", Surface: string(authz.System), Payload: mustJSON(map[string]any{
|
|
"source": "gitea", "external_id": "381", "project": "p",
|
|
})}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
task := s.Tasks()[0]
|
|
recordDecision(t, s, task.ID, "d1", "no, use b")
|
|
|
|
a := ¬ifyingAdapter{fakeAdapter: fakeAdapter{occupancy: .5}}
|
|
c := &orchestrator.Coordinator{Store: s, Worktrees: worktrees{path: repo}, Adapters: adapters{a}, StatePath: t.TempDir() + "/sessions.json", Hard: .8}
|
|
leaseEvt, err := s.Lease(task.ID, "h1", time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := c.Start(context.Background(), leaseEvt); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
c.ReconcileHumanInput = func(context.Context, string) error { return nil }
|
|
if _, err := c.TurnDecision(context.Background(), task.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(a.notices) != 0 {
|
|
t.Fatalf("launch-carried decision re-delivered: %v", a.notices)
|
|
}
|
|
}
|
|
|
|
// Rotation wins over delivery: the successor gets the decision through the
|
|
// pre-lease gate, so nothing is sent to an agent that is about to hand off.
|
|
func TestRotationSkipsDelivery(t *testing.T) {
|
|
a := ¬ifyingAdapter{fakeAdapter: fakeAdapter{occupancy: .95, boundary: true}}
|
|
c, s, task := leasedCoordinator(t, a, gitRepo(t))
|
|
ref, err := s.PutArtifact([]byte("handoff"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
a.ref = ref
|
|
c.ReconcileHumanInput = func(_ context.Context, taskID string) error {
|
|
recordDecision(t, s, taskID, "d1", "no, use b")
|
|
return nil
|
|
}
|
|
verdict, err := c.TurnDecision(context.Background(), task.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if verdict != orchestrator.TurnRotateNow {
|
|
t.Fatalf("verdict = %q, want rotate_now", verdict)
|
|
}
|
|
if len(a.notices) != 0 {
|
|
t.Fatalf("delivered to a rotating session: %v", a.notices)
|
|
}
|
|
// The release must still succeed against the version the decision bumped.
|
|
got, _ := s.Task(task.ID)
|
|
if got.State != domain.StateQueued {
|
|
t.Fatalf("state = %s, want queued after release", got.State)
|
|
}
|
|
if got.HandoffRef != ref {
|
|
t.Fatalf("handoff ref = %q", got.HandoffRef)
|
|
}
|
|
}
|
|
|
|
// A reconciliation failure at a turn boundary is recorded and does not block
|
|
// the turn. Ownership is where reconciliation fails closed.
|
|
func TestReconcileFailureAtBoundaryIsRecordedNotFatal(t *testing.T) {
|
|
a := ¬ifyingAdapter{fakeAdapter: fakeAdapter{occupancy: .5}}
|
|
c, _, task := leasedCoordinator(t, a, gitRepo(t))
|
|
c.ReconcileHumanInput = func(context.Context, string) error {
|
|
return context.DeadlineExceeded
|
|
}
|
|
verdict, err := c.TurnDecision(context.Background(), task.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if verdict != orchestrator.TurnContinue {
|
|
t.Fatalf("verdict = %q, want continue", verdict)
|
|
}
|
|
h := c.MonitorHealth().Sessions[task.ID]
|
|
if !strings.Contains(h.LastError, "reconcile human input") {
|
|
t.Fatalf("failure not observable: %+v", h)
|
|
}
|
|
}
|
|
|
|
// Delivery failure must not mark the decision as delivered.
|
|
func TestDeliveryFailureRetriesNextBoundary(t *testing.T) {
|
|
a := ¬ifyingAdapter{fakeAdapter: fakeAdapter{occupancy: .5}, err: context.DeadlineExceeded}
|
|
c, s, task := leasedCoordinator(t, a, gitRepo(t))
|
|
c.ReconcileHumanInput = func(context.Context, string) error { return nil }
|
|
recordDecision(t, s, task.ID, "d1", "no, use b")
|
|
if _, err := c.TurnDecision(context.Background(), task.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h := c.MonitorHealth().Sessions[task.ID]
|
|
if !strings.Contains(h.LastError, "deliver decisions") {
|
|
t.Fatalf("failure not observable: %+v", h)
|
|
}
|
|
a.err = nil
|
|
if _, err := c.TurnDecision(context.Background(), task.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(a.notices) != 1 || !strings.Contains(a.notices[0], "no, use b") {
|
|
t.Fatalf("notices = %v", a.notices)
|
|
}
|
|
}
|