v3 workflow: intent, phases, review, submission, enforcement, burn-in

The v3 stack, previously an uncommitted working tree, plus this session's two
units and the burn-in instrument. This commit is the burn-in build identity:
coordinator and worker must both report this revision before a task is created.

Workflow (earlier sessions, uncommitted until now): human decision events and
reduction, source cursors and reconcile-before-launch, turn-boundary
reconciliation, internal/agentctx as the single renderer, ace-fca phases with
sealed artifacts, the trajectory gate, bounded grilling, independent review,
task pr enforcement, and human review reflection.

Capability restrictions at the agent boundary: an authz.Agent surface at
GatedWrite may ask and may not act. It also fixes two bugs the unit exposed --
gated surfaces could not reach the two endpoints written for them, and
RequestHumanDecision would block an unowned task while rejecting a question
from the session that did own it.

Turn-boundary reconcile-failure escalation: a streak of consecutive failures
asks the session to hand off, fenced on the lease epoch, with reconcile_failure
as a real handoff reason. The worker was dropping the coordinator's verdict on
the floor; it now acts on it.

Burn-in: herdr.WriteLaunchContext dumps the exact agentctx.Build result to
<worktree>/.orchestra/launch.md at every launch, local and federated. BURNIN.md
is the runbook. deploy/build.sh stamps both binaries from one commit.

go build, go vet and go test ./... pass, 20 packages.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
2026-08-26 18:31:20 +04:00
parent 97a9c65302
commit 7f12c7fc37
78 changed files with 16417 additions and 352 deletions
+243
View File
@@ -0,0 +1,243 @@
package orchestrator_test
import (
"context"
"strings"
"testing"
"time"
"orchestra/internal/authz"
"orchestra/internal/domain"
"orchestra/internal/herdr"
"orchestra/internal/orchestrator"
"orchestra/internal/store"
)
// notifyingAdapter records what a live agent was told mid-lease.
type notifyingAdapter struct {
fakeAdapter
notices []string
err error
}
func (a *notifyingAdapter) NotifyDecisions(_ context.Context, _ herdr.Session, text string) error {
if a.err != nil {
return a.err
}
a.notices = append(a.notices, text)
return nil
}
func leasedCoordinator(t *testing.T, a herdr.Adapter, repo string) (*orchestrator.Coordinator, *store.Store, domain.Task) {
t.Helper()
s, err := store.Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
if err := s.Append(domain.Event{ID: domain.NewID(), Type: "TaskCreated", TaskID: "t1", Surface: string(authz.System), Payload: mustJSON(map[string]any{
"source": "gitea", "external_id": "381", "project": "p",
})}); err != nil {
t.Fatal(err)
}
task := s.Tasks()[0]
c := &orchestrator.Coordinator{Store: s, Worktrees: worktrees{path: repo}, Adapters: adapters{a}, StatePath: t.TempDir() + "/sessions.json", Hard: .8}
leaseEvt, err := s.Lease(task.ID, "h1", time.Minute)
if err != nil {
t.Fatal(err)
}
if err := c.Start(context.Background(), leaseEvt); err != nil {
t.Fatal(err)
}
return c, s, task
}
func recordDecision(t *testing.T, s *store.Store, taskID, id, value string) {
t.Helper()
task, ok := s.Task(taskID)
if !ok {
t.Fatal("task missing")
}
if err := s.Append(domain.Event{
ID: domain.NewID(), Type: domain.EventHumanDecisionRecorded, TaskID: taskID,
Version: task.Version + 1, Surface: string(authz.System),
Payload: mustJSON(map[string]any{
"decision_id": id, "kind": "correction", "subject": "strategy", "value": value,
"source": map[string]any{"provider": "gitea", "external_id": "c-" + id},
}),
}); err != nil {
t.Fatal(err)
}
}
func gitRepo(t *testing.T) string {
t.Helper()
repo := t.TempDir()
run(t, repo, "init")
run(t, repo, "config", "user.email", "t@t")
run(t, repo, "config", "user.name", "t")
run(t, repo, "commit", "--allow-empty", "-m", "init")
return repo
}
// A correction written while the lease is live reaches the agent at the next
// verified turn boundary, without preempting anything.
func TestDecisionDeliveredAtTurnBoundary(t *testing.T) {
a := &notifyingAdapter{fakeAdapter: fakeAdapter{occupancy: .5}}
c, s, task := leasedCoordinator(t, a, gitRepo(t))
reconciled := 0
c.ReconcileHumanInput = func(_ context.Context, taskID string) error {
reconciled++
if reconciled == 1 {
recordDecision(t, s, taskID, "d1", "no, use b")
}
return nil
}
verdict, err := c.TurnDecision(context.Background(), task.ID)
if err != nil {
t.Fatal(err)
}
if verdict != orchestrator.TurnContinue {
t.Fatalf("verdict = %q, want continue", verdict)
}
if reconciled != 1 {
t.Fatalf("reconciled %d times, want 1", reconciled)
}
if len(a.notices) != 1 || !strings.Contains(a.notices[0], "no, use b") {
t.Fatalf("notices = %v", a.notices)
}
if !strings.Contains(a.notices[0], "outrank") {
t.Fatal("notice does not state that the decision outranks the current plan")
}
// Same decision at the next boundary is not re-sent.
if _, err := c.TurnDecision(context.Background(), task.ID); err != nil {
t.Fatal(err)
}
if len(a.notices) != 1 {
t.Fatalf("decision re-delivered: %v", a.notices)
}
// A second, newer decision is delivered on its own.
recordDecision(t, s, task.ID, "d2", "and keep the old flag")
if _, err := c.TurnDecision(context.Background(), task.ID); err != nil {
t.Fatal(err)
}
if len(a.notices) != 2 || !strings.Contains(a.notices[1], "and keep the old flag") {
t.Fatalf("notices = %v", a.notices)
}
if strings.Contains(a.notices[1], "no, use b") {
t.Fatal("already delivered decision repeated")
}
}
// Decisions carried by the launch instruction are not re-announced as news.
func TestDecisionsFromLaunchAreNotRedelivered(t *testing.T) {
repo := gitRepo(t)
s, err := store.Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
if err := s.Append(domain.Event{ID: domain.NewID(), Type: "TaskCreated", TaskID: "t1", Surface: string(authz.System), Payload: mustJSON(map[string]any{
"source": "gitea", "external_id": "381", "project": "p",
})}); err != nil {
t.Fatal(err)
}
task := s.Tasks()[0]
recordDecision(t, s, task.ID, "d1", "no, use b")
a := &notifyingAdapter{fakeAdapter: fakeAdapter{occupancy: .5}}
c := &orchestrator.Coordinator{Store: s, Worktrees: worktrees{path: repo}, Adapters: adapters{a}, StatePath: t.TempDir() + "/sessions.json", Hard: .8}
leaseEvt, err := s.Lease(task.ID, "h1", time.Minute)
if err != nil {
t.Fatal(err)
}
if err := c.Start(context.Background(), leaseEvt); err != nil {
t.Fatal(err)
}
c.ReconcileHumanInput = func(context.Context, string) error { return nil }
if _, err := c.TurnDecision(context.Background(), task.ID); err != nil {
t.Fatal(err)
}
if len(a.notices) != 0 {
t.Fatalf("launch-carried decision re-delivered: %v", a.notices)
}
}
// Rotation wins over delivery: the successor gets the decision through the
// pre-lease gate, so nothing is sent to an agent that is about to hand off.
func TestRotationSkipsDelivery(t *testing.T) {
a := &notifyingAdapter{fakeAdapter: fakeAdapter{occupancy: .95, boundary: true}}
c, s, task := leasedCoordinator(t, a, gitRepo(t))
ref, err := s.PutArtifact([]byte("handoff"))
if err != nil {
t.Fatal(err)
}
a.ref = ref
c.ReconcileHumanInput = func(_ context.Context, taskID string) error {
recordDecision(t, s, taskID, "d1", "no, use b")
return nil
}
verdict, err := c.TurnDecision(context.Background(), task.ID)
if err != nil {
t.Fatal(err)
}
if verdict != orchestrator.TurnRotateNow {
t.Fatalf("verdict = %q, want rotate_now", verdict)
}
if len(a.notices) != 0 {
t.Fatalf("delivered to a rotating session: %v", a.notices)
}
// The release must still succeed against the version the decision bumped.
got, _ := s.Task(task.ID)
if got.State != domain.StateQueued {
t.Fatalf("state = %s, want queued after release", got.State)
}
if got.HandoffRef != ref {
t.Fatalf("handoff ref = %q", got.HandoffRef)
}
}
// A reconciliation failure at a turn boundary is recorded and does not block
// the turn. Ownership is where reconciliation fails closed.
func TestReconcileFailureAtBoundaryIsRecordedNotFatal(t *testing.T) {
a := &notifyingAdapter{fakeAdapter: fakeAdapter{occupancy: .5}}
c, _, task := leasedCoordinator(t, a, gitRepo(t))
c.ReconcileHumanInput = func(context.Context, string) error {
return context.DeadlineExceeded
}
verdict, err := c.TurnDecision(context.Background(), task.ID)
if err != nil {
t.Fatal(err)
}
if verdict != orchestrator.TurnContinue {
t.Fatalf("verdict = %q, want continue", verdict)
}
h := c.MonitorHealth().Sessions[task.ID]
if !strings.Contains(h.LastError, "reconcile human input") {
t.Fatalf("failure not observable: %+v", h)
}
}
// Delivery failure must not mark the decision as delivered.
func TestDeliveryFailureRetriesNextBoundary(t *testing.T) {
a := &notifyingAdapter{fakeAdapter: fakeAdapter{occupancy: .5}, err: context.DeadlineExceeded}
c, s, task := leasedCoordinator(t, a, gitRepo(t))
c.ReconcileHumanInput = func(context.Context, string) error { return nil }
recordDecision(t, s, task.ID, "d1", "no, use b")
if _, err := c.TurnDecision(context.Background(), task.ID); err != nil {
t.Fatal(err)
}
h := c.MonitorHealth().Sessions[task.ID]
if !strings.Contains(h.LastError, "deliver decisions") {
t.Fatalf("failure not observable: %+v", h)
}
a.err = nil
if _, err := c.TurnDecision(context.Background(), task.ID); err != nil {
t.Fatal(err)
}
if len(a.notices) != 1 || !strings.Contains(a.notices[0], "no, use b") {
t.Fatalf("notices = %v", a.notices)
}
}
@@ -1,20 +0,0 @@
package orchestrator
import (
"orchestra/internal/domain"
"strings"
"testing"
)
func TestTaskLaunchPromptIncludesRemoteTaskInstructions(t *testing.T) {
prompt := taskLaunchPrompt(domain.Task{
ID: "task-1",
Title: "Create marker",
Description: "Create E2E_RESULT.md containing ok.",
})
for _, want := range []string{"task-1", "Create marker", "Create E2E_RESULT.md containing ok."} {
if !strings.Contains(prompt, want) {
t.Fatalf("launch prompt missing %q: %s", want, prompt)
}
}
}
+313 -34
View File
@@ -8,11 +8,14 @@ import (
"encoding/json"
"errors"
"fmt"
"orchestra/internal/agentctx"
"orchestra/internal/authz"
"orchestra/internal/continuity"
"orchestra/internal/domain"
"orchestra/internal/herdr"
"orchestra/internal/operations"
"orchestra/internal/store"
"orchestra/internal/workphase"
"os"
"os/exec"
"path/filepath"
@@ -192,9 +195,69 @@ type Coordinator struct {
// defaultSoft), so existing callers that never set this field keep
// working unchanged.
Soft float64
// Reconcile imports newer human input for one task. Store.PreLease covers
// the moment ownership begins; this field covers the other half, a
// correction written while a lease is already live. It runs only at a
// verified turn boundary, so nothing preempts a running tool call.
ReconcileHumanInput func(ctx context.Context, taskID string) error
// Thrash tunes DetectThrash's three circuit breakers (§5.3). Zero-value
// fields fall back to herdr's own defaults, so leaving this unset works.
Thrash herdr.ThrashConfig
// ReconcileFailureHandoff is how many *consecutive* failed turn-boundary
// reconciles escalate to prepare_handoff. One failure is transient and
// continuing is right; a streak means Orchestra can no longer promise
// that the newest human input outranks this session's intent, so the
// honest move is to hand the task to a successor whose Store.PreLease
// reconcile fails closed while the source is down. Zero means the package
// default (defaultReconcileFailureHandoff).
ReconcileFailureHandoff int
// reconcileFailures is the streak per task, fenced on the lease epoch so
// a successor never inherits its predecessor's count and no release path
// needs a cleanup hook. Guarded by healthMu.
reconcileFailures map[string]reconcileStreak
}
type reconcileStreak struct {
Epoch string
N int
}
// defaultReconcileFailureHandoff is used whenever ReconcileFailureHandoff is
// unset. Three consecutive verified boundaries is long enough to ride out a
// restart or a brief network fault, short enough that a stuck source does not
// let a session run indefinitely on intent Orchestra cannot refresh.
const defaultReconcileFailureHandoff = 3
func (c *Coordinator) reconcileFailureThreshold() int {
if c.ReconcileFailureHandoff > 0 {
return c.ReconcileFailureHandoff
}
return defaultReconcileFailureHandoff
}
// noteReconcileResult records one turn boundary's reconcile outcome and reports
// whether this session has reached the escalation threshold. Success resets the
// streak, so two failures followed by a success escalate nothing.
func (c *Coordinator) noteReconcileResult(taskID, epoch string, err error) bool {
c.healthMu.Lock()
defer c.healthMu.Unlock()
if c.reconcileFailures == nil {
c.reconcileFailures = map[string]reconcileStreak{}
}
if err == nil {
delete(c.reconcileFailures, taskID)
return false
}
streak := c.reconcileFailures[taskID]
if streak.Epoch != epoch {
// A different owner: count this session's failures, not the previous
// lease's.
streak = reconcileStreak{Epoch: epoch}
}
streak.N++
c.reconcileFailures[taskID] = streak
c.recordSessionErrorLocked(taskID, fmt.Sprintf("reconcile human input (%d consecutive): %v", streak.N, err))
return streak.N >= c.reconcileFailureThreshold()
}
// defaultSoft is used whenever Coordinator.Soft is unset (zero value).
@@ -340,6 +403,67 @@ func (c *Coordinator) recordTurnBoundaryDegraded() {
c.health.TurnBoundaryDegraded++
}
// recordSessionError keeps a non-fatal failure observable instead of letting
// a bare continue hide it, which is this codebase's recurring bug shape.
func (c *Coordinator) recordSessionError(taskID, msg string) {
c.healthMu.Lock()
defer c.healthMu.Unlock()
c.recordSessionErrorLocked(taskID, msg)
}
// recordSessionErrorLocked is recordSessionError for callers already holding
// healthMu.
func (c *Coordinator) recordSessionErrorLocked(taskID, msg string) {
if c.health.Sessions == nil {
c.health.Sessions = map[string]SessionHealth{}
}
h := c.health.Sessions[taskID]
h.LastError = msg
h.UpdatedAt = time.Now().UTC()
c.health.Sessions[taskID] = h
}
// deliverDecisions sends the human decisions this session has not been shown
// yet. It runs only at a verified turn boundary, and only when the turn
// verdict is continue, so it never interrupts a running tool call and never
// competes with a rotation that is about to hand the task to a successor.
func (c *Coordinator) deliverDecisions(ctx context.Context, taskID string, session herdr.Session, a herdr.Adapter) {
notifier, ok := a.(herdr.DecisionNotifier)
if !ok {
return
}
intent, err := c.Store.EffectiveIntent(taskID)
if err != nil {
c.recordSessionError(taskID, "effective intent: "+err.Error())
return
}
seen := make(map[string]bool, len(session.DeliveredDecisions))
for _, id := range session.DeliveredDecisions {
seen[id] = true
}
var fresh []domain.HumanDecision
for _, d := range intent.Decisions {
if !seen[d.ID] {
fresh = append(fresh, d)
}
}
if len(fresh) == 0 {
return
}
if err := notifier.NotifyDecisions(ctx, session, agentctx.DecisionNotice(fresh)); err != nil {
// Not recorded as delivered, so the next boundary retries.
c.recordSessionError(taskID, "deliver decisions: "+err.Error())
return
}
for _, d := range fresh {
session.DeliveredDecisions = append(session.DeliveredDecisions, d.ID)
}
c.mu.Lock()
c.sessions[taskID] = session
_ = c.saveSessionsLocked()
c.mu.Unlock()
}
func waitingForApproval(status string) bool {
s := strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(status, "-", "_"), " ", "_"))
return s == "waiting_for_approval" || s == "awaiting_approval" || s == "approval_required"
@@ -681,6 +805,18 @@ func handoffReason(worktree string) string {
return h.Meta.Reason
}
// reasonReconcileFailure is the handoff reason for a session released because
// human input could not be reconciled at repeated verified turn boundaries.
const reasonReconcileFailure = "reconcile_failure"
// bypassReason reports whether a handoff already carrying this reason is
// itself the boundary signal, so occupancy and the turn-boundary probe are
// skipped and the session is released immediately. reconcile_failure joins the
// list because Orchestra, not the context window, asked for that handoff.
func bypassReason(r string) bool {
return r == "manual" || r == "milestone" || r == "thrash" || r == reasonReconcileFailure
}
func (c *Coordinator) rotate(ctx context.Context, hard float64) {
c.loadSessions()
c.mu.Lock()
@@ -705,7 +841,7 @@ func (c *Coordinator) rotate(ctx context.Context, hard float64) {
// question has already been answered, so skip occupancy and the
// turn-boundary probe and go straight to release.
existingReason := handoffReason(session.Worktree)
bypass := existingReason == "manual" || existingReason == "milestone" || existingReason == "thrash"
bypass := bypassReason(existingReason)
if bypass {
reason = existingReason
} else {
@@ -805,6 +941,67 @@ func (c *Coordinator) rotate(ctx context.Context, hard float64) {
}
}
// RemoteTurn is the federated half of a turn boundary. A worker owns the pane,
// so it evaluates rotation locally and reports the verdict it reached; the
// coordinator owns authority, so it reconciles human input here and answers
// with the decisions that session has not been shown yet.
//
// The split is deliberate. Duplicating the rotation state machine in the
// worker would give two answers to "should this session stop"; asking the
// coordinator to probe a remote pane would give it a checkout it cannot
// validate. Neither half is authoritative about the other's state.
//
// Decisions are returned only when the verdict is continue, matching the local
// path: a rotating session's successor picks them up at re-lease.
func (c *Coordinator) RemoteTurn(ctx context.Context, taskID, epoch, verdict string, delivered []string) (string, []domain.HumanDecision, error) {
if c.Store == nil {
return "", nil, fmt.Errorf("orchestrator: dependencies required")
}
t, ok := c.Store.Task(taskID)
if !ok {
return "", nil, domain.ErrNotFound
}
if t.State != domain.StateLeased && t.State != domain.StateNeedsAttention {
return "", nil, fmt.Errorf("orchestrator: task %q not leased", taskID)
}
// Fenced like every other worker-driven call: a worker whose lease was
// reassigned must not be handed the current session's decisions.
if t.Lease == nil || epoch == "" || t.Lease.Epoch != epoch {
return "", nil, domain.ErrConflict
}
escalate := false
if c.ReconcileHumanInput != nil {
// Same contract as the local boundary: one failure is observable, not
// fatal, because refusing would freeze a live remote session without
// making its current intent any less stale. A streak escalates, on the
// same threshold the local path uses.
escalate = c.noteReconcileResult(taskID, epoch, c.ReconcileHumanInput(ctx, taskID))
}
if verdict != TurnContinue {
// The worker already wants to stop. Answering with a second reason
// would manufacture a rotation trigger nothing needs.
return verdict, nil, nil
}
if escalate {
return TurnPrepareHandoff, nil, nil
}
intent, err := c.Store.EffectiveIntent(taskID)
if err != nil {
return "", nil, err
}
seen := make(map[string]bool, len(delivered))
for _, id := range delivered {
seen[id] = true
}
var fresh []domain.HumanDecision
for _, d := range intent.Decisions {
if !seen[d.ID] {
fresh = append(fresh, d)
}
}
return verdict, fresh, nil
}
// Turn decision verdicts (spec §5.3, AUDIT.md Phase 2 items 1-2). These are
// the only valid results of TurnDecision and the only values the
// POST /v1/harness/turn endpoint may return.
@@ -839,11 +1036,26 @@ func (c *Coordinator) TurnDecision(ctx context.Context, taskID string) (string,
if err != nil {
return "", fmt.Errorf("orchestrator: adapter: %w", err)
}
// A turn boundary is the one point where the agent is verifiably between
// actions, so it is where newer human input is imported for a live lease.
// The task is re-read afterwards because a recorded decision bumps its
// version, and finishRelease below writes against that version.
escalate := false
if c.ReconcileHumanInput != nil {
// One failure is recorded and the turn continues: blocking would not
// remove stale intent from the running agent, and a source outage
// would freeze every live session. A streak is different, and acts on
// the continue path below.
escalate = c.noteReconcileResult(taskID, task.Lease.Epoch, c.ReconcileHumanInput(ctx, taskID))
if fresh, ok := c.Store.Task(taskID); ok {
task = fresh
}
}
// Agent-initiated ROTATE, and the two orchestrator-detected triggers
// (§5.3: manual / milestone / thrash): a handoff already written with one
// of these reasons is itself the boundary signal — skip occupancy and the
// turn-boundary probe and release immediately.
if existingReason := handoffReason(session.Worktree); existingReason == "manual" || existingReason == "milestone" || existingReason == "thrash" {
if existingReason := handoffReason(session.Worktree); bypassReason(existingReason) {
return c.finishRelease(ctx, taskID, task, session, a, existingReason)
}
d := (RotationStateMachine{Soft: c.soft(), Hard: c.Hard, Thrash: c.Thrash}).Evaluate(ctx, a, session)
@@ -851,6 +1063,18 @@ func (c *Coordinator) TurnDecision(ctx context.Context, taskID string) (string,
return "", fmt.Errorf("orchestrator: rotation: %w", d.Degraded)
}
if d.Action == TurnContinue {
if escalate {
// Rotation has no reason of its own, so this is the one place the
// reconcile streak can act. Ask for a handoff; release runs through
// the ordinary bypass path once the agent writes it, and the
// successor's Store.PreLease reconcile fails closed while the
// source is still down.
c.requestReasonedHandoff(ctx, taskID, session, a, reasonReconcileFailure, nil)
return TurnPrepareHandoff, nil
}
// Only on continue. A rotating session's successor picks the decision
// up through Store.PreLease when it acquires the lease.
c.deliverDecisions(ctx, taskID, session, a)
return TurnContinue, nil
}
if d.Action == TurnRefuse {
@@ -963,7 +1187,66 @@ func (c *Coordinator) Start(ctx context.Context, e domain.Event) error {
return c.block(t, "worktree: "+err.Error())
}
taskFileSHA, _ := continuity.TaskFileHash(w)
prompt := taskLaunchPrompt(t)
// One renderer. The launch instruction is built by agentctx so a decision
// the human recorded while this task was queued is visible to the agent
// from its first turn, above anything it will later read as continuity.
intent, err := c.Store.EffectiveIntent(t.ID)
if err != nil {
return c.block(t, "effective intent: "+err.Error())
}
git := agentctx.GitState{Worktree: w, Branch: "orchestra/" + t.ID}
if sha, shaErr := herdr.HeadSHA(w); shaErr == nil {
git.HeadSHA = sha
}
// §6.2 pickup validation happens before the agent exists, not after: the
// handoff is part of the launch instruction now, so it must be trusted
// before it is rendered. A failure blocks the task without ever starting
// a session (this is the gap AUDIT.md's B6 named as unreached).
var handoff *continuity.Handoff
if p.HandoffRef != "" {
h, loadErr := continuity.Load(p.HandoffRef, c.Store)
if loadErr != nil {
return c.block(t, "handoff: "+loadErr.Error())
}
if err := continuity.ValidatePickup(w, h, taskFileSHA); err != nil {
return c.block(t, "pickup: "+err.Error())
}
handoff = &h
}
in := agentctx.Input{
Task: t, Intent: intent, Handoff: handoff, Git: git,
Phase: t.WorkPhase, RepoRules: agentctx.DiscoverRepoRules(w),
DecisionRequest: t.DecisionRequest,
}
if t.ResearchRef != "" {
r, refErr := c.research(t.ResearchRef)
if refErr != nil {
return c.block(t, "research artifact: "+refErr.Error())
}
in.Research = r
}
if t.PlanRef != "" {
pl, refErr := c.plan(t.PlanRef)
if refErr != nil {
return c.block(t, "plan artifact: "+refErr.Error())
}
in.Plan = pl
}
if t.Review != nil {
r, refErr := operations.TaskReview(c.Store, t)
if refErr != nil {
return c.block(t, "review artifact: "+refErr.Error())
}
in.Review = r
}
built, err := agentctx.Build(in)
if err != nil {
return c.block(t, "context: "+err.Error())
}
prompt := built.System + "\n\n" + built.Task
if writeErr := herdr.WriteLaunchContext(w, prompt); writeErr != nil {
c.recordSessionError(t.ID, "launch context: "+writeErr.Error())
}
var s herdr.Session
if promptLeaser, ok := a.(herdr.PromptLeaser); ok {
s, err = promptLeaser.LeasePrompt(ctx, t.ID, w, prompt)
@@ -982,28 +1265,13 @@ func (c *Coordinator) Start(ctx context.Context, e domain.Event) error {
}
return c.block(t, "lease: "+err.Error())
}
if p.HandoffRef != "" {
// §6.2 pickup validation: never bootstrap a successor onto a handoff
// whose anchor/dirty-file/TASK.md hashes don't match what's actually
// in the worktree. A failure here blocks the task rather than
// silently trusting an unvalidated ref (this is the gap AUDIT.md's
// B6 named as unreached from the live path).
h, err := continuity.Load(p.HandoffRef, c.Store)
if err != nil {
_ = a.Kill(ctx, s)
return c.block(t, "handoff: "+err.Error())
}
if err := continuity.ValidatePickup(w, h, taskFileSHA); err != nil {
_ = a.Kill(ctx, s)
return c.block(t, "pickup: "+err.Error())
}
if err = a.Bootstrap(ctx, s, p.HandoffRef); err != nil {
_ = a.Kill(ctx, s)
return c.block(t, "bootstrap: "+err.Error())
}
}
s.HerdrID = p.HarnessID
s.TaskFileSHA = taskFileSHA
// The launch instruction carried these, so the first turn boundary must
// not re-deliver them as news.
for _, d := range intent.Decisions {
s.DeliveredDecisions = append(s.DeliveredDecisions, d.ID)
}
// Best-effort, same caveat as taskFileSHA above: only meaningful for a
// worktree this process can read locally. Snapshots the shared-docs
// state this session starts trusting; checkConventions notices drift
@@ -1021,19 +1289,30 @@ func (c *Coordinator) Start(ctx context.Context, e domain.Event) error {
return err
}
func taskLaunchPrompt(t domain.Task) string {
var b strings.Builder
fmt.Fprintf(&b, "Begin Orchestra task %s.\n", t.ID)
if t.Title != "" {
fmt.Fprintf(&b, "Title: %s\n", t.Title)
// research and plan read a sealed phase artifact. A stored ref that will not
// decode is a blocked task, not a silently empty context section.
func (c *Coordinator) research(ref string) (*workphase.Research, error) {
b, err := c.Store.Artifact(ref)
if err != nil {
return nil, err
}
if t.Description != "" {
fmt.Fprintf(&b, "Instructions:\n%s\n", t.Description)
} else {
b.WriteString("Inspect the repository, understand the task context, and proceed with the requested work.\n")
r, err := workphase.DecodeResearch(b)
if err != nil {
return nil, err
}
b.WriteString("This is the authoritative task instruction. Work only within this task's worktree. Do not edit TASK.md if it exists.")
return b.String()
return &r, nil
}
func (c *Coordinator) plan(ref string) (*workphase.Plan, error) {
b, err := c.Store.Artifact(ref)
if err != nil {
return nil, err
}
p, err := workphase.DecodePlan(b)
if err != nil {
return nil, err
}
return &p, nil
}
func (c *Coordinator) rememberSession(taskID string, s herdr.Session) error {
@@ -0,0 +1,252 @@
package orchestrator_test
import (
"context"
"encoding/json"
"errors"
"os"
"strings"
"testing"
"orchestra/internal/continuity"
"orchestra/internal/domain"
"orchestra/internal/herdr"
"orchestra/internal/orchestrator"
)
// reasoningAdapter records the rotation reasons Orchestra asked a handoff for.
type reasoningAdapter struct {
fakeAdapter
reasons []string
}
func (a *reasoningAdapter) RequestHandoffReason(_ context.Context, _ herdr.Session, reason string, _ []continuity.DeadEnd) error {
a.reasons = append(a.reasons, reason)
return nil
}
var down = errors.New("gitea unreachable")
// Repeated failure at a verified boundary means Orchestra can no longer uphold
// "the newest human input outranks the agent's current intent". The first two
// turns continue, because one outage should not stop work. The third hands the
// task to a successor, whose pre-lease reconcile fails closed while the source
// is still down.
func TestReconcileFailureStreakEscalatesToHandoff(t *testing.T) {
a := &reasoningAdapter{fakeAdapter: fakeAdapter{occupancy: .5}}
c, _, task := leasedCoordinator(t, a, gitRepo(t))
c.ReconcileFailureHandoff = 3
c.ReconcileHumanInput = func(context.Context, string) error { return down }
for turn := 1; turn <= 2; turn++ {
verdict, err := c.TurnDecision(context.Background(), task.ID)
if err != nil {
t.Fatal(err)
}
if verdict != orchestrator.TurnContinue {
t.Fatalf("turn %d verdict = %q, want continue", turn, verdict)
}
if len(a.reasons) != 0 {
t.Fatalf("turn %d asked for a handoff: %v", turn, a.reasons)
}
}
verdict, err := c.TurnDecision(context.Background(), task.ID)
if err != nil {
t.Fatal(err)
}
if verdict != orchestrator.TurnPrepareHandoff {
t.Fatalf("verdict = %q, want prepare_handoff", verdict)
}
if len(a.reasons) != 1 || a.reasons[0] != "reconcile_failure" {
t.Fatalf("reasons = %v", a.reasons)
}
// The count is observable, not just acted on.
if h := c.MonitorHealth().Sessions[task.ID]; !strings.Contains(h.LastError, "3 consecutive") {
t.Fatalf("streak not observable: %+v", h)
}
}
// One success clears the streak. Two failures then a success then a failure is
// one failure, not three.
func TestReconcileSuccessResetsTheStreak(t *testing.T) {
a := &reasoningAdapter{fakeAdapter: fakeAdapter{occupancy: .5}}
c, _, task := leasedCoordinator(t, a, gitRepo(t))
c.ReconcileFailureHandoff = 3
failing := true
c.ReconcileHumanInput = func(context.Context, string) error {
if failing {
return down
}
return nil
}
turn := func() string {
t.Helper()
v, err := c.TurnDecision(context.Background(), task.ID)
if err != nil {
t.Fatal(err)
}
return v
}
turn()
turn()
failing = false
turn()
failing = true
if v := turn(); v != orchestrator.TurnContinue {
t.Fatalf("verdict = %q, want continue after the streak reset", v)
}
if len(a.reasons) != 0 {
t.Fatalf("escalated on a reset streak: %v", a.reasons)
}
}
// A rotation that already wants to stop keeps its own reason. Orchestra must
// not manufacture a second trigger for a session that is already handing off.
func TestReconcileStreakDoesNotOverrideAnExistingRotation(t *testing.T) {
repo := gitRepo(t)
a := &reasoningAdapter{fakeAdapter: fakeAdapter{occupancy: .95, boundary: true}}
c, s, task := leasedCoordinator(t, a, repo)
ref, err := s.PutArtifact([]byte("handoff"))
if err != nil {
t.Fatal(err)
}
a.ref = ref
c.ReconcileFailureHandoff = 1
c.ReconcileHumanInput = func(context.Context, string) error { return down }
verdict, err := c.TurnDecision(context.Background(), task.ID)
if err != nil {
t.Fatal(err)
}
if verdict != orchestrator.TurnRotateNow {
t.Fatalf("verdict = %q, want rotate_now", verdict)
}
if len(a.reasons) != 0 {
t.Fatalf("manufactured a reason for a rotating session: %v", a.reasons)
}
}
// The escape path has to complete. Once the agent writes the handoff with this
// reason, the ordinary bypass releases the task, exactly as it does for
// manual, milestone and thrash.
func TestReconcileFailureHandoffReleasesTheTask(t *testing.T) {
repo := gitRepo(t)
a := &reasoningAdapter{fakeAdapter: fakeAdapter{occupancy: 0, boundary: false}}
c, s, task := leasedCoordinator(t, a, repo)
ref, err := s.PutArtifact([]byte("handoff"))
if err != nil {
t.Fatal(err)
}
a.ref = ref
head, err := herdr.HeadSHA(repo)
if err != nil {
t.Fatal(err)
}
b, err := json.Marshal(map[string]any{
"meta": map[string]any{"id": "h2", "reason": "reconcile_failure", "rotation_index": 0},
"anchor": map[string]any{"git_sha": head, "branch": "orchestra/t1"},
"action": "re-read the task intent before continuing", "command": "go test ./...",
})
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(repo+"/"+herdr.HandoffFile, b, 0o644); err != nil {
t.Fatal(err)
}
// The reason must survive handoff validation, or the successor cannot read
// the artifact this release produced.
if _, err := continuity.Decode(b); err != nil {
t.Fatalf("handoff rejected: %v", err)
}
c.ReconcileHumanInput = func(context.Context, string) error { return down }
verdict, err := c.TurnDecision(context.Background(), task.ID)
if err != nil {
t.Fatal(err)
}
if verdict != orchestrator.TurnRotateNow {
t.Fatalf("verdict = %q, want rotate_now", verdict)
}
if got, _ := s.Task(task.ID); got.State != domain.StateQueued {
t.Fatalf("state = %s, want queued", got.State)
}
}
// No human source means nothing to fail, so nothing ever escalates.
func TestNoHumanSourceNeverEscalates(t *testing.T) {
a := &reasoningAdapter{fakeAdapter: fakeAdapter{occupancy: .5}}
c, _, task := leasedCoordinator(t, a, gitRepo(t))
c.ReconcileFailureHandoff = 1
for turn := 0; turn < 5; turn++ {
verdict, err := c.TurnDecision(context.Background(), task.ID)
if err != nil {
t.Fatal(err)
}
if verdict != orchestrator.TurnContinue {
t.Fatalf("verdict = %q, want continue", verdict)
}
}
if len(a.reasons) != 0 {
t.Fatalf("escalated with no reconciler configured: %v", a.reasons)
}
}
// Delivering a decision is not reconciling one. A pane that cannot be written
// to must not spend the reconcile budget.
func TestDeliveryFailureIsNotAReconcileFailure(t *testing.T) {
a := &notifyingAdapter{fakeAdapter: fakeAdapter{occupancy: .5}, err: context.DeadlineExceeded}
c, s, task := leasedCoordinator(t, a, gitRepo(t))
c.ReconcileFailureHandoff = 2
c.ReconcileHumanInput = func(context.Context, string) error { return nil }
recordDecision(t, s, task.ID, "d1", "no, use b")
for turn := 0; turn < 3; turn++ {
verdict, err := c.TurnDecision(context.Background(), task.ID)
if err != nil {
t.Fatal(err)
}
if verdict != orchestrator.TurnContinue {
t.Fatalf("verdict = %q, want continue", verdict)
}
}
}
// The federated half uses the same threshold, so a worker-owned session and a
// local one behave identically.
func TestRemoteTurnEscalatesOnTheSameThreshold(t *testing.T) {
c, _, task := remoteLeased(t)
c.ReconcileFailureHandoff = 3
c.ReconcileHumanInput = func(context.Context, string) error { return down }
for turn := 1; turn <= 2; turn++ {
verdict, _, err := c.RemoteTurn(context.Background(), task.ID, task.Lease.Epoch, orchestrator.TurnContinue, nil)
if err != nil {
t.Fatal(err)
}
if verdict != orchestrator.TurnContinue {
t.Fatalf("turn %d verdict = %q, want continue", turn, verdict)
}
}
verdict, decisions, err := c.RemoteTurn(context.Background(), task.ID, task.Lease.Epoch, orchestrator.TurnContinue, nil)
if err != nil {
t.Fatal(err)
}
if verdict != orchestrator.TurnPrepareHandoff {
t.Fatalf("verdict = %q, want prepare_handoff", verdict)
}
if len(decisions) != 0 {
t.Fatalf("decisions returned to a rotating session: %+v", decisions)
}
}
// A worker that already reported a stop keeps its own verdict.
func TestRemoteTurnKeepsTheWorkersVerdict(t *testing.T) {
c, _, task := remoteLeased(t)
c.ReconcileFailureHandoff = 1
c.ReconcileHumanInput = func(context.Context, string) error { return down }
verdict, _, err := c.RemoteTurn(context.Background(), task.ID, task.Lease.Epoch, orchestrator.TurnRotateNow, nil)
if err != nil {
t.Fatal(err)
}
if verdict != orchestrator.TurnRotateNow {
t.Fatalf("verdict = %q, want the worker's own rotate_now", verdict)
}
}
+120
View File
@@ -0,0 +1,120 @@
package orchestrator_test
import (
"context"
"errors"
"strings"
"testing"
"time"
"orchestra/internal/authz"
"orchestra/internal/domain"
"orchestra/internal/orchestrator"
"orchestra/internal/store"
)
func remoteLeased(t *testing.T) (*orchestrator.Coordinator, *store.Store, domain.Task) {
t.Helper()
s, err := store.Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
if err := s.Append(domain.Event{ID: domain.NewID(), Type: "TaskCreated", TaskID: "t1", Surface: string(authz.System), Payload: mustJSON(map[string]any{
"source": "gitea", "external_id": "381", "project": "p",
})}); err != nil {
t.Fatal(err)
}
task := s.Tasks()[0]
if _, err := s.Lease(task.ID, "workpc-opencode", time.Minute); err != nil {
t.Fatal(err)
}
// No Worktrees and no Adapters: the coordinator never touches a remote pane.
c := &orchestrator.Coordinator{Store: s, StatePath: t.TempDir() + "/sessions.json"}
got, _ := s.Task(task.ID)
return c, s, got
}
// A worker at a verified boundary reconciles through the coordinator and gets
// the decisions its session has not seen.
func TestRemoteTurnReconcilesAndReturnsUndeliveredDecisions(t *testing.T) {
c, s, task := remoteLeased(t)
reconciled := 0
c.ReconcileHumanInput = func(_ context.Context, taskID string) error {
reconciled++
if reconciled == 1 {
recordDecision(t, s, taskID, "d1", "no, use b")
}
return nil
}
verdict, decisions, err := c.RemoteTurn(context.Background(), task.ID, task.Lease.Epoch, orchestrator.TurnContinue, nil)
if err != nil {
t.Fatal(err)
}
if verdict != orchestrator.TurnContinue {
t.Fatalf("verdict = %q", verdict)
}
if len(decisions) != 1 || decisions[0].Value != "no, use b" {
t.Fatalf("decisions = %+v", decisions)
}
// Delivered once. The worker reports what it has shown, so the same
// decision is not returned twice.
_, again, err := c.RemoteTurn(context.Background(), task.ID, task.Lease.Epoch, orchestrator.TurnContinue, []string{decisions[0].ID})
if err != nil {
t.Fatal(err)
}
if len(again) != 0 {
t.Fatalf("decision returned twice: %+v", again)
}
}
// Rotating sessions get no decisions: the successor picks them up at re-lease.
func TestRemoteTurnWithholdsDecisionsWhenRotating(t *testing.T) {
c, s, task := remoteLeased(t)
once := 0
c.ReconcileHumanInput = func(_ context.Context, taskID string) error {
once++
if once == 1 {
recordDecision(t, s, taskID, "d1", "no, use b")
}
return nil
}
for _, verdict := range []string{orchestrator.TurnRotateNow, orchestrator.TurnPrepareHandoff, orchestrator.TurnRefuse} {
got, decisions, err := c.RemoteTurn(context.Background(), task.ID, task.Lease.Epoch, verdict, nil)
if err != nil {
t.Fatal(err)
}
if got != verdict || len(decisions) != 0 {
t.Fatalf("verdict %q returned %+v", verdict, decisions)
}
}
}
// Fenced like every other worker-driven call.
func TestRemoteTurnRefusesStaleEpoch(t *testing.T) {
c, _, task := remoteLeased(t)
if _, _, err := c.RemoteTurn(context.Background(), task.ID, "stale", orchestrator.TurnContinue, nil); !errors.Is(err, domain.ErrConflict) {
t.Fatalf("want ErrConflict, got %v", err)
}
if _, _, err := c.RemoteTurn(context.Background(), "missing", "e", orchestrator.TurnContinue, nil); !errors.Is(err, domain.ErrNotFound) {
t.Fatalf("want ErrNotFound, got %v", err)
}
}
// Same contract as the local boundary: a source failure is observable and the
// session keeps running.
func TestRemoteTurnReconcileFailureIsObservableNotFatal(t *testing.T) {
c, _, task := remoteLeased(t)
c.ReconcileHumanInput = func(context.Context, string) error { return context.DeadlineExceeded }
verdict, _, err := c.RemoteTurn(context.Background(), task.ID, task.Lease.Epoch, orchestrator.TurnContinue, nil)
if err != nil {
t.Fatal(err)
}
if verdict != orchestrator.TurnContinue {
t.Fatalf("verdict = %q", verdict)
}
h := c.MonitorHealth().Sessions[task.ID]
if !strings.Contains(h.LastError, "reconcile human input") {
t.Fatalf("failure not observable: %+v", h)
}
}
-2
View File
@@ -34,7 +34,6 @@ func (a *fakeAdapter) Lease(_ context.Context, _ string, worktree string) (herdr
a.leases++
return herdr.Session{Harness: "h1", PaneID: "pane-1", Worktree: worktree}, nil
}
func (a *fakeAdapter) Bootstrap(context.Context, herdr.Session, string) error { return nil }
func (a *fakeAdapter) Release(context.Context, herdr.Session) (string, error) {
a.releases++
return a.ref, nil
@@ -580,7 +579,6 @@ type noBoundaryAdapter struct {
func (a *noBoundaryAdapter) Lease(_ context.Context, _ string, worktree string) (herdr.Session, error) {
return herdr.Session{Harness: "h1", PaneID: "pane-1", Worktree: worktree}, nil
}
func (a *noBoundaryAdapter) Bootstrap(context.Context, herdr.Session, string) error { return nil }
func (a *noBoundaryAdapter) Release(context.Context, herdr.Session) (string, error) {
a.releases++
return a.ref, nil