Let Orchestra establish plan progress instead of the implementer asserting it

A detailed plan that nothing enforces is a document. This makes the phases
executable: the implementer may write exactly one status, and every other
status is a conclusion Orchestra reaches by running the plan's own commands.

    agent may request:  ready_for_verification
    agent may not assert: verified, awaiting_manual_verification, failed, skipped

The worker resolves commands from the coordinator, never from the request, so a
request cannot smuggle in a command the planner did not write. They run as argv
through exec with Dir set to the worktree, which is the quality gate's existing
envelope and not a weaker one. There is no shell, so a pipe is a literal
argument.

Project policy decides executable reach. registry.Project.Verification matches
argv positionally, and an absent policy refuses everything: a plan command is
agent-authored, so inheriting the operator-authored gate's reach by default
would be the wrong direction to fail in. A refused command is refused before
anything runs, and the refusal names the project and the command so the planner
learns its real reach.

Two bindings make the record mean something later. PlanRef, so progress earned
under plan A cannot survive into plan B. AtSHA, so "verified" does not outlive
the code that made it true: a record whose commit has moved is retained as
provenance and rendered as stale, never as a claim about the current tree.
Both are the same failure this codebase already fixed for reviews, which bind
to the commit they examined.

Manual steps hold a phase at awaiting_manual_verification. The sign-off is an
ordinary human decision whose subject carries the plan ref and the phase id, so
a later "looks good" on an unrelated thread cannot satisfy a gate nobody was
discussing.

A plan sealed before plan.md declares no executable unit, and says so: the
implement context states that phase progress is unavailable and the work
continues under the old semantics. Inventing phases it never had would be worse
than admitting it has none.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CVbaKucEYBjMqVeUgJUsc1
This commit is contained in:
2026-08-28 11:59:39 +04:00
parent 57c028f94f
commit a221502356
15 changed files with 1282 additions and 22 deletions
+98
View File
@@ -359,3 +359,101 @@ func TestClaudeHarnessReachesTheTurnBoundary(t *testing.T) {
}
_ = backend
}
// An agent may request verification. It may never assert one: writing
// "verified" is claiming its own work is done, which is what the whole
// machinery exists to prevent. The refusal has to reach the pane, or the
// session rewrites the same rejected file at every boundary.
func TestPlanVerificationRefusesAnyStatusButARequest(t *testing.T) {
for _, status := range []string{"verified", "awaiting_manual_verification", "failed", "skipped", ""} {
w, backend, wt, done := phaseWorker(t, func(rw http.ResponseWriter, r *http.Request) {
t.Errorf("a refused request reached the coordinator at %s", r.URL.Path)
rw.WriteHeader(http.StatusNotFound)
})
path := filepath.Join(wt, ".orchestra", planProgressFile)
if err := os.WriteFile(path, []byte(`{"phase":"phase-1","status":"`+status+`"}`), 0o644); err != nil {
t.Fatal(err)
}
if w.requestPlanVerification(context.Background(), "task", w.sessions["task"]) {
t.Fatalf("status %q was accepted", status)
}
if len(backend.prompts) == 0 {
t.Fatalf("status %q was refused with nothing delivered to the pane", status)
}
if !strings.Contains(backend.prompts[0], "ready_for_verification") {
t.Fatalf("the refusal does not name the only writable status: %s", backend.prompts[0])
}
if _, err := os.Stat(path); !os.IsNotExist(err) {
t.Fatalf("status %q left the refused request in place", status)
}
done()
}
}
// An absent request is not a failure and must not reach the pane.
func TestNoPlanVerificationRequestIsSilent(t *testing.T) {
w, backend, _, done := phaseWorker(t, func(rw http.ResponseWriter, r *http.Request) {
t.Errorf("an absent request reached the coordinator at %s", r.URL.Path)
})
defer done()
if w.requestPlanVerification(context.Background(), "task", w.sessions["task"]) {
t.Fatal("an absent request reported work done")
}
if len(backend.prompts) != 0 {
t.Fatalf("an absent request spoke to the pane: %v", backend.prompts)
}
}
// The commands come from the accepted plan, resolved and authorised by the
// coordinator. The worker must never take one out of the agent's request.
func TestPlanVerificationRunsThePlansCommandsAndReportsExitCodes(t *testing.T) {
var reported map[string]any
w, backend, wt, done := phaseWorker(t, func(rw http.ResponseWriter, r *http.Request) {
switch {
case strings.HasSuffix(r.URL.Path, "/plan-phase"):
json.NewEncoder(rw).Encode(map[string]any{"commands": [][]string{{"true"}, {"false"}}})
case strings.HasSuffix(r.URL.Path, "/plan-phase-result"):
json.NewDecoder(r.Body).Decode(&reported)
json.NewEncoder(rw).Encode(map[string]any{"status": "in_progress"})
default:
t.Errorf("unexpected %s", r.URL.Path)
rw.WriteHeader(http.StatusNotFound)
}
})
defer done()
// A git worktree, so the verification can anchor to a real commit.
for _, args := range [][]string{{"init"}, {"config", "user.email", "t@example.com"}, {"config", "user.name", "t"}, {"commit", "--allow-empty", "-m", "base"}} {
if out, err := git(context.Background(), wt, args...); err != nil {
t.Fatalf("git %v: %s: %v", args, out, err)
}
}
path := filepath.Join(wt, ".orchestra", planProgressFile)
if err := os.WriteFile(path, []byte(`{"phase":"phase-1","status":"ready_for_verification","commands":[["rm","-rf","/"]]}`), 0o644); err != nil {
t.Fatal(err)
}
if !w.requestPlanVerification(context.Background(), "task", w.sessions["task"]) {
t.Fatal("a valid request did nothing")
}
runs, _ := reported["runs"].([]any)
if len(runs) != 2 {
t.Fatalf("reported %d runs, want the plan's 2: %v", len(runs), reported)
}
first, _ := runs[0].(map[string]any)
second, _ := runs[1].(map[string]any)
if first["exit_code"].(float64) != 0 || second["exit_code"].(float64) == 0 {
t.Fatalf("exit codes were not reported faithfully: %v", runs)
}
// The command list in the request is ignored entirely.
cmd, _ := first["command"].([]any)
if len(cmd) != 1 || cmd[0].(string) != "true" {
t.Fatalf("the worker ran something other than the plan's command: %v", cmd)
}
if sha, _ := reported["at_sha"].(string); len(sha) != 40 {
t.Fatalf("the verification was not anchored to a commit: %q", sha)
}
// A phase that did not verify has to say why, or the agent sees its
// request vanish and guesses.
if len(backend.prompts) == 0 || !strings.Contains(backend.prompts[0], "not verified") {
t.Fatalf("the outcome was not delivered: %v", backend.prompts)
}
}