kami/orchestra
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Let Orchestra establish plan progress instead of the implementer asserting it

Browse Source 
A detailed plan that nothing enforces is a document. This makes the phases
executable: the implementer may write exactly one status, and every other
status is a conclusion Orchestra reaches by running the plan's own commands.

    agent may request:  ready_for_verification
    agent may not assert: verified, awaiting_manual_verification, failed, skipped

The worker resolves commands from the coordinator, never from the request, so a
request cannot smuggle in a command the planner did not write. They run as argv
through exec with Dir set to the worktree, which is the quality gate's existing
envelope and not a weaker one. There is no shell, so a pipe is a literal
argument.

Project policy decides executable reach. registry.Project.Verification matches
argv positionally, and an absent policy refuses everything: a plan command is
agent-authored, so inheriting the operator-authored gate's reach by default
would be the wrong direction to fail in. A refused command is refused before
anything runs, and the refusal names the project and the command so the planner
learns its real reach.

Two bindings make the record mean something later. PlanRef, so progress earned
under plan A cannot survive into plan B. AtSHA, so "verified" does not outlive
the code that made it true: a record whose commit has moved is retained as
provenance and rendered as stale, never as a claim about the current tree.
Both are the same failure this codebase already fixed for reviews, which bind
to the commit they examined.

Manual steps hold a phase at awaiting_manual_verification. The sign-off is an
ordinary human decision whose subject carries the plan ref and the phase id, so
a later "looks good" on an unrelated thread cannot satisfy a gate nobody was
discussing.

A plan sealed before plan.md declares no executable unit, and says so: the
implement context states that phase progress is unavailable and the work
continues under the old semantics. Inventing phases it never had would be worse
than admitting it has none.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CVbaKucEYBjMqVeUgJUsc1
This commit is contained in:
kami
2026-08-28 11:59:39 +04:00
parent 57c028f94f
commit a221502356
15 changed files with 1282 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/orchestra-worker/main.go
+138
View File
@@ -1776,6 +1776,13 @@ func (w *worker) federatedTurn(ctx context.Context, id string, a herdr.Adapter,
w.rotateForPhase(ctx, id, a, s)
return
}
// A plan phase asks to be verified before the work phase asks to move.
// Checked first because verifying the last phase is usually what makes a
// session ready to leave implement at all, and this session keeps running
// either way: verification is progress within a phase, not a change of one.
if w.requestPlanVerification(ctx, id, s) {
return
}
// The agent asks for a phase change here, at a boundary it has reached
// (F21). Orchestra decides, and an accepted change ends this session.
if w.requestPhase(ctx, id, s) {
@@ -2048,3 +2055,134 @@ func (w *worker) paneProgress(ctx context.Context, adapter herdr.CLIAdapter, s h
}
return adapter.PaneCapture(ctx, s, "recent")
}
// planProgressFile is the implementer's bounded verification request. It
// carries one phase and one status, and the only status it may carry is a
// request: an agent that could write "verified" would be marking its own work
// done, which is the whole thing this machinery exists to prevent.
const planProgressFile = "plan-progress.json"
type planProgressRequest struct {
Phase string `json:"phase"`
Status string `json:"status"`
}
// requestPlanVerification runs one phase's automated checks and posts the
// results. It reports whether anything was done, so the caller does not treat
// an absent request as a failure.
//
// The commands come from the accepted plan, which the coordinator resolves and
// authorises. This worker never reads a command out of the request: the
// request names a phase, and nothing else about it is trusted.
func (w *worker) requestPlanVerification(ctx context.Context, id string, s herdr.Session) bool {
path := filepath.Join(s.Worktree, ".orchestra", planProgressFile)
b, err := os.ReadFile(path)
if err != nil {
return false
}
var req planProgressRequest
if err := json.Unmarshal(b, &req); err != nil {
w.answerRefusedProgress(ctx, id, s, path, fmt.Sprintf(".orchestra/%s is not valid JSON: %v", planProgressFile, err))
return false
}
if req.Status != domain.PlanPhaseRequestStatus {
// Naming this refusal precisely matters: an agent that wrote
// "verified" is trying to establish a fact, and it has to learn that
// establishing facts is not its job rather than retry the same file.
w.answerRefusedProgress(ctx, id, s, path, fmt.Sprintf("status %q is not writable by you. The only status you may request is %q; Orchestra decides the rest by running the plan's own commands.", req.Status, domain.PlanPhaseRequestStatus))
return false
}
l := w.leases[id]
commands, err := w.api.PlanPhaseCommands(ctx, id, req.Phase, l.Epoch)
if err != nil {
// A refusal is an answer and reaches the agent. A transport failure is
// not, and the request survives for the next boundary.
var status *federation.StatusError
if errors.As(err, &status) && status.Code == http.StatusConflict {
w.answerRefusedProgress(ctx, id, s, path, status.Body)
return false
}
w.recordError(fmt.Errorf("plan verification %s: %w", id, err))
return false
}
head, err := git(ctx, s.Worktree, "rev-parse", "HEAD")
if err != nil {
w.recordError(fmt.Errorf("plan verification %s: head: %s: %w", id, head, err))
return false
}
runs := make([]federation.VerificationRun, 0, len(commands))
for _, argv := range commands {
// No shell. The quality gate's envelope is exec in the worktree, and a
// plan command is agent-authored, so it gets that envelope and never a
// weaker one. A pipe here is a literal argument.
cmd := exec.CommandContext(ctx, argv[0], argv[1:]...)
cmd.Dir = s.Worktree
out, runErr := cmd.CombinedOutput()
code := 0
if runErr != nil {
code = 1
var exit *exec.ExitError
if errors.As(runErr, &exit) {
code = exit.ExitCode()
}
}
runs = append(runs, federation.VerificationRun{Command: argv, ExitCode: code, Output: tail(string(out), review.MaxGateOutputBytes)})
}
status, err := w.api.RecordPlanPhase(ctx, id, req.Phase, strings.TrimSpace(string(head)), l.Epoch, runs)
if err != nil {
w.recordError(fmt.Errorf("plan verification %s: %w", id, err))
return false
}
// Durable before the request is removed. Removing first would lose the
// request if the response were lost, and the agent would wait on an answer
// that already happened.
if err := os.Remove(path); err != nil {
w.recordError(fmt.Errorf("plan verification %s: %w", id, err))
}
w.tellProgressOutcome(ctx, id, s, req.Phase, status, runs)
return true
}
// tellProgressOutcome delivers what Orchestra established. A phase that did
// not verify is the case that matters: without this the agent sees a request
// disappear and has to guess whether it worked.
func (w *worker) tellProgressOutcome(ctx context.Context, id string, s herdr.Session, phase, status string, runs []federation.VerificationRun) {
var b strings.Builder
switch status {
case string(domain.PlanPhaseVerified):
fmt.Fprintf(&b, "Orchestra verified %s. Every automated check passed. Move to the next phase of the plan.", phase)
case string(domain.PlanPhaseAwaitingManual):
fmt.Fprintf(&b, "Orchestra ran %s's automated checks and they passed. The phase is waiting for the human to confirm its manual steps, which you cannot do. Continue with the next phase; the sign-off arrives on its own.", phase)
default:
fmt.Fprintf(&b, "Orchestra ran %s's checks and it is not verified. Fix what failed, then write .orchestra/%s again.\n", phase, planProgressFile)
for _, r := range runs {
if r.ExitCode == 0 {
continue
}
fmt.Fprintf(&b, "\n%s exited %d:\n%s\n", strings.Join(r.Command, " "), r.ExitCode, r.Output)
}
}
if err := w.sendPrompt(ctx, s, b.String()); err != nil {
w.recordError(fmt.Errorf("deliver plan verification %s: %w", id, err))
}
log.Printf("plan phase %s of %s: %s", phase, id, status)
}
// answerRefusedProgress tells the implementer why its request was refused and
// drops the file so a corrected one can be written. Recording a refusal only
// in worker health leaves a live session rewriting the same rejected file at
// every boundary, which is the silent-loop shape this codebase keeps producing
// (F39, F42, and the completion tail).
func (w *worker) answerRefusedProgress(ctx context.Context, id string, s herdr.Session, path, reason string) {
w.recordError(fmt.Errorf("plan verification %s refused: %s", id, reason))
text := "Orchestra refused your verification request: " + reason +
"\n\nWrite a corrected .orchestra/" + planProgressFile + ", or keep working. Do not repeat the refused request."
if err := w.sendPrompt(ctx, s, text); err != nil {
w.recordError(fmt.Errorf("deliver plan verification refusal %s: %w", id, err))
return
}
if err := os.Remove(path); err != nil {
w.recordError(fmt.Errorf("plan verification %s: %w", id, err))
}
log.Printf("plan verification %s refused: %s", id, reason)
}
cmd/orchestra-worker/phase_test.go
+98
View File
@@ -359,3 +359,101 @@ func TestClaudeHarnessReachesTheTurnBoundary(t *testing.T) {
}
_ = backend
}
// An agent may request verification. It may never assert one: writing
// "verified" is claiming its own work is done, which is what the whole
// machinery exists to prevent. The refusal has to reach the pane, or the
// session rewrites the same rejected file at every boundary.
func TestPlanVerificationRefusesAnyStatusButARequest(t *testing.T) {
for _, status := range []string{"verified", "awaiting_manual_verification", "failed", "skipped", ""} {
w, backend, wt, done := phaseWorker(t, func(rw http.ResponseWriter, r *http.Request) {
t.Errorf("a refused request reached the coordinator at %s", r.URL.Path)
rw.WriteHeader(http.StatusNotFound)
})
path := filepath.Join(wt, ".orchestra", planProgressFile)
if err := os.WriteFile(path, []byte(`{"phase":"phase-1","status":"`+status+`"}`), 0o644); err != nil {
t.Fatal(err)
}
if w.requestPlanVerification(context.Background(), "task", w.sessions["task"]) {
t.Fatalf("status %q was accepted", status)
}
if len(backend.prompts) == 0 {
t.Fatalf("status %q was refused with nothing delivered to the pane", status)
}
if !strings.Contains(backend.prompts[0], "ready_for_verification") {
t.Fatalf("the refusal does not name the only writable status: %s", backend.prompts[0])
}
if _, err := os.Stat(path); !os.IsNotExist(err) {
t.Fatalf("status %q left the refused request in place", status)
}
done()
}
}
// An absent request is not a failure and must not reach the pane.
func TestNoPlanVerificationRequestIsSilent(t *testing.T) {
w, backend, _, done := phaseWorker(t, func(rw http.ResponseWriter, r *http.Request) {
t.Errorf("an absent request reached the coordinator at %s", r.URL.Path)
})
defer done()
if w.requestPlanVerification(context.Background(), "task", w.sessions["task"]) {
t.Fatal("an absent request reported work done")
}
if len(backend.prompts) != 0 {
t.Fatalf("an absent request spoke to the pane: %v", backend.prompts)
}
}
// The commands come from the accepted plan, resolved and authorised by the
// coordinator. The worker must never take one out of the agent's request.
func TestPlanVerificationRunsThePlansCommandsAndReportsExitCodes(t *testing.T) {
var reported map[string]any
w, backend, wt, done := phaseWorker(t, func(rw http.ResponseWriter, r *http.Request) {
switch {
case strings.HasSuffix(r.URL.Path, "/plan-phase"):
json.NewEncoder(rw).Encode(map[string]any{"commands": [][]string{{"true"}, {"false"}}})
case strings.HasSuffix(r.URL.Path, "/plan-phase-result"):
json.NewDecoder(r.Body).Decode(&reported)
json.NewEncoder(rw).Encode(map[string]any{"status": "in_progress"})
default:
t.Errorf("unexpected %s", r.URL.Path)
rw.WriteHeader(http.StatusNotFound)
}
})
defer done()
// A git worktree, so the verification can anchor to a real commit.
for _, args := range [][]string{{"init"}, {"config", "user.email", "t@example.com"}, {"config", "user.name", "t"}, {"commit", "--allow-empty", "-m", "base"}} {
if out, err := git(context.Background(), wt, args...); err != nil {
t.Fatalf("git %v: %s: %v", args, out, err)
}
}
path := filepath.Join(wt, ".orchestra", planProgressFile)
if err := os.WriteFile(path, []byte(`{"phase":"phase-1","status":"ready_for_verification","commands":[["rm","-rf","/"]]}`), 0o644); err != nil {
t.Fatal(err)
}
if !w.requestPlanVerification(context.Background(), "task", w.sessions["task"]) {
t.Fatal("a valid request did nothing")
}
runs, _ := reported["runs"].([]any)
if len(runs) != 2 {
t.Fatalf("reported %d runs, want the plan's 2: %v", len(runs), reported)
}
first, _ := runs[0].(map[string]any)
second, _ := runs[1].(map[string]any)
if first["exit_code"].(float64) != 0 || second["exit_code"].(float64) == 0 {
t.Fatalf("exit codes were not reported faithfully: %v", runs)
}
// The command list in the request is ignored entirely.
cmd, _ := first["command"].([]any)
if len(cmd) != 1 || cmd[0].(string) != "true" {
t.Fatalf("the worker ran something other than the plan's command: %v", cmd)
}
if sha, _ := reported["at_sha"].(string); len(sha) != 40 {
t.Fatalf("the verification was not anchored to a commit: %q", sha)
}
// A phase that did not verify has to say why, or the agent sees its
// request vanish and guesses.
if len(backend.prompts) == 0 || !strings.Contains(backend.prompts[0], "not verified") {
t.Fatalf("the outcome was not delivered: %v", backend.prompts)
}
}