dcd9af4806
F42, live on run 5's completion tail. The review agent wrote .orchestra/done,
the worker recognised it, confirmed the agent idle, and then failed the result
commit on every attempt:
stage result: The following paths are ignored by one of your .gitignore
files: .orchestra/done
.orchestra carries a .gitignore of "*" (internal/herdr/adapter.go:132), so the
marker is ignored, and git refuses an add whose pathspec names an ignored path.
The exclusion now names the directory. Reproduced against git 2.55.0 in a
scratch repo both ways, and the regression test uses the same constant the
worker passes to git.
The failure retried every five seconds for 22 minutes with the task stuck in
review and nothing observable outside the journal, because each identical error
overwrote the single last_error slot. That is F18, still open.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CVbaKucEYBjMqVeUgJUsc1
1105 lines
43 KiB
Go
1105 lines
43 KiB
Go
package main
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"orchestra/internal/domain"
|
|
"orchestra/internal/federation"
|
|
"orchestra/internal/herdr"
|
|
"orchestra/internal/orchestrator"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestWorkerReregistersAfterCoordinatorForgetsIt(t *testing.T) {
|
|
registered := false
|
|
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodPost || r.URL.Path != "/v1/federation/workers" {
|
|
t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
|
|
}
|
|
registered = true
|
|
w.WriteHeader(http.StatusCreated)
|
|
}))
|
|
defer s.Close()
|
|
w := &worker{api: federation.Client{BaseURL: s.URL, WorkerID: "h", Token: "t"}, registration: federation.Worker{ID: "h", Capacity: 1}}
|
|
if !w.reRegisterAfterCoordinatorRestart(context.Background(), errors.New("federation: 401 Unauthorized: unknown worker")) {
|
|
t.Fatal("worker did not identify a coordinator restart")
|
|
}
|
|
if !registered {
|
|
t.Fatal("worker did not register")
|
|
}
|
|
}
|
|
|
|
func TestWorkerRefusesCorruptDurableState(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "state.json")
|
|
if err := os.WriteFile(path, []byte(`{"cursor":`), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
w := &worker{statePath: path}
|
|
if err := w.load(); err == nil {
|
|
t.Fatal("corrupt worker state was accepted")
|
|
}
|
|
}
|
|
|
|
func TestClassifyLaunchErrorPreservesUncertainLivePane(t *testing.T) {
|
|
if got := classifyLaunchError(errors.New("prompt response lost"), true); got != "launch_uncertain" {
|
|
t.Fatalf("live pane class=%q", got)
|
|
}
|
|
if got := classifyLaunchError(errors.New("pickup anchor mismatch"), false); got != "invalid_handoff" {
|
|
t.Fatalf("bad handoff class=%q", got)
|
|
}
|
|
if got := classifyLaunchError(errors.New("temporary herdr outage"), false); got != "launch_transient" {
|
|
t.Fatalf("transient class=%q", got)
|
|
}
|
|
}
|
|
|
|
func TestInitialReplayDoesNotResurrectReleasedLease(t *testing.T) {
|
|
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
switch r.URL.Path {
|
|
case "/v1/federation/events":
|
|
_, _ = w.Write([]byte(`{"cursor":0,"events":[{"seq":1,"id":"c","type":"TaskCreated","task_id":"t","version":1,"payload":{"source":"s","external_id":"x","project":"p"},"surface":"system"},{"seq":2,"id":"l","type":"TaskLeased","task_id":"t","version":2,"payload":{"harness_id":"h"},"surface":"system"},{"seq":3,"id":"r","type":"TaskReleased","task_id":"t","version":3,"payload":{"reason":"expired"},"surface":"system"}]}`))
|
|
case "/v1/federation/events/ack":
|
|
w.WriteHeader(http.StatusNoContent)
|
|
default:
|
|
t.Fatalf("unexpected %s", r.URL.Path)
|
|
}
|
|
}))
|
|
defer s.Close()
|
|
w := &worker{api: federation.Client{BaseURL: s.URL, WorkerID: "h", Token: "t"}, harnessID: "h", tasks: map[string]domain.Task{}, sessions: map[string]herdr.Session{}, leases: map[string]lease{}, statePath: t.TempDir() + "/state.json", hard: .75}
|
|
if err := w.once(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(w.leases) != 0 || len(w.sessions) != 0 {
|
|
t.Fatalf("replayed lease survived: leases=%v sessions=%v", w.leases, w.sessions)
|
|
}
|
|
if w.cursor != 3 {
|
|
t.Fatalf("cursor=%d want 3", w.cursor)
|
|
}
|
|
}
|
|
|
|
func TestWorkerHydratesLeaseMissingTaskCache(t *testing.T) {
|
|
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
switch r.URL.Path {
|
|
case "/v1/federation/events":
|
|
_, _ = w.Write([]byte(`{"cursor":2,"events":[{"seq":2,"id":"l","type":"TaskLeased","task_id":"t","version":2,"payload":{"harness_id":"h"},"surface":"system"}]}`))
|
|
case "/v1/tasks":
|
|
_, _ = w.Write([]byte(`[{"id":"t","source":"s","external_id":"x","project":"p","state":"leased"}]`))
|
|
case "/v1/federation/events/ack":
|
|
w.WriteHeader(http.StatusNoContent)
|
|
default:
|
|
t.Fatalf("unexpected %s", r.URL.Path)
|
|
}
|
|
}))
|
|
defer s.Close()
|
|
w := &worker{api: federation.Client{BaseURL: s.URL, WorkerID: "h", Token: "t"}, harnessID: "h", cursor: 1, tasks: map[string]domain.Task{}, sessions: map[string]herdr.Session{"t": {}}, leases: map[string]lease{}, statePath: t.TempDir() + "/state.json", hard: .75}
|
|
if err := w.once(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, ok := w.tasks["t"]; !ok || w.cursor != 2 {
|
|
t.Fatalf("task was not hydrated: tasks=%v cursor=%d", w.tasks, w.cursor)
|
|
}
|
|
}
|
|
|
|
func TestWorkerReconcilesLeaseAfterEmptyEventReplay(t *testing.T) {
|
|
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
switch r.URL.Path {
|
|
case "/v1/federation/events":
|
|
_, _ = w.Write([]byte(`{"cursor":42,"events":[]}`))
|
|
case "/v1/tasks":
|
|
_, _ = w.Write([]byte(`[{"id":"t","source":"s","external_id":"x","project":"p","state":"leased","lease":{"harness_id":"h"}}]`))
|
|
case "/v1/federation/events/ack":
|
|
w.WriteHeader(http.StatusNoContent)
|
|
default:
|
|
t.Fatalf("unexpected %s", r.URL.Path)
|
|
}
|
|
}))
|
|
defer s.Close()
|
|
w := &worker{api: federation.Client{BaseURL: s.URL, WorkerID: "h", Token: "t"}, harnessID: "h", cursor: 42, tasks: map[string]domain.Task{}, sessions: map[string]herdr.Session{"t": {}}, leases: map[string]lease{}, statePath: t.TempDir() + "/state.json", hard: .75}
|
|
if err := w.once(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, ok := w.leases["t"]; !ok {
|
|
t.Fatal("empty event replay did not recover current lease")
|
|
}
|
|
}
|
|
|
|
func TestWorkerStartsRouterIssuedLeaseInLocalGitWorktree(t *testing.T) {
|
|
remote := filepath.Join(t.TempDir(), "remote.git")
|
|
if out, err := exec.Command("git", "init", "--bare", remote).CombinedOutput(); err != nil {
|
|
t.Fatalf("remote: %v %s", err, out)
|
|
}
|
|
seed := t.TempDir()
|
|
for _, a := range [][]string{{"init", seed}, {"-C", seed, "config", "user.email", "t@t"}, {"-C", seed, "config", "user.name", "t"}, {"-C", seed, "commit", "--allow-empty", "-m", "init"}, {"-C", seed, "remote", "add", "origin", remote}, {"-C", seed, "push", "-u", "origin", "HEAD:master"}} {
|
|
if out, err := exec.Command("git", a...).CombinedOutput(); err != nil {
|
|
t.Fatalf("git %v: %v %s", a, err, out)
|
|
}
|
|
}
|
|
repo := filepath.Join(t.TempDir(), "repo")
|
|
if out, err := exec.Command("git", "clone", remote, repo).CombinedOutput(); err != nil {
|
|
t.Fatal(string(out))
|
|
}
|
|
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer ln.Close()
|
|
var promptMu sync.Mutex
|
|
var prompts []string
|
|
go func() {
|
|
for {
|
|
c, e := ln.Accept()
|
|
if e != nil {
|
|
return
|
|
}
|
|
go func() {
|
|
defer c.Close()
|
|
var r herdr.Request
|
|
if json.NewDecoder(bufio.NewReader(c)).Decode(&r) != nil {
|
|
return
|
|
}
|
|
var result string
|
|
switch r.Method {
|
|
case "worktree.create":
|
|
result = `{"path":"` + filepath.Join(filepath.Dir(repo), "worktrees", "task") + `","root_pane":{"pane_id":"p1"}}`
|
|
case "agent.start":
|
|
result = `{}`
|
|
case "pane.get":
|
|
result = `{"pane":{"agent":"opencode","agent_status":"idle"}}`
|
|
case "pane.read":
|
|
result = `{"read":{"text":""}}`
|
|
case "agent.prompt":
|
|
if m, ok := r.Params.(map[string]any); ok {
|
|
text, _ := m["text"].(string)
|
|
promptMu.Lock()
|
|
prompts = append(prompts, text)
|
|
promptMu.Unlock()
|
|
}
|
|
result = `{}`
|
|
default:
|
|
result = `{}`
|
|
}
|
|
_ = json.NewEncoder(c).Encode(herdr.Response{ID: r.ID, Result: json.RawMessage(result)})
|
|
}()
|
|
}
|
|
}()
|
|
root := filepath.Join(filepath.Dir(repo), "worktrees")
|
|
// The worker renders its own launch instruction from the coordinator's
|
|
// reduced authority, so the fake API must serve it.
|
|
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if strings.HasSuffix(r.URL.Path, "/intent") {
|
|
json.NewEncoder(w).Encode(domain.EffectiveIntent{
|
|
Task: domain.Task{ID: "task", Title: "test", Description: "do work"},
|
|
Decisions: []domain.HumanDecision{{
|
|
ID: "d1", TaskID: "task", Kind: domain.HumanDecisionCorrection,
|
|
Subject: "strategy", Value: "no, use b",
|
|
}},
|
|
})
|
|
return
|
|
}
|
|
w.WriteHeader(200)
|
|
}))
|
|
defer api.Close()
|
|
w := &worker{api: federation.Client{BaseURL: api.URL, WorkerID: "h", Token: "t"}, herdr: &herdr.Client{Path: ln.Addr().String()}, repo: repo, root: root, remote: "origin", harness: "opencode", tasks: map[string]domain.Task{}, sessions: map[string]herdr.Session{}, leases: map[string]lease{}, statePath: filepath.Join(t.TempDir(), "state.json"), hard: .75}
|
|
// New() is needed for its pane map; override the address for the fake.
|
|
w.herdr = herdr.New(ln.Addr().String())
|
|
task := domain.Task{ID: "task", Source: "s", ExternalID: "x", Project: "p", Title: "test", Description: "do work"}
|
|
if err := w.start(context.Background(), task, ""); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, ok := w.sessions[task.ID]; !ok {
|
|
t.Fatal("session not persisted")
|
|
}
|
|
if _, err := os.Stat(filepath.Join(root, "task", "TASK.md")); err != nil {
|
|
t.Fatalf("TASK.md: %v", err)
|
|
}
|
|
// The deployed worker path renders through agentctx, so a decision the
|
|
// human recorded before this session existed reaches the agent.
|
|
promptMu.Lock()
|
|
launched := strings.Join(prompts, "\n")
|
|
promptMu.Unlock()
|
|
for _, want := range []string{"## Current human decisions", "no, use b", "Authority order"} {
|
|
if !strings.Contains(launched, want) {
|
|
t.Fatalf("worker launch instruction missing %q:\n%s", want, launched)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestFinalizeRunsGateCommitsPushesAndVerifiesRemote(t *testing.T) {
|
|
remote := filepath.Join(t.TempDir(), "remote.git")
|
|
if out, err := exec.Command("git", "init", "--bare", remote).CombinedOutput(); err != nil {
|
|
t.Fatalf("remote: %v %s", err, out)
|
|
}
|
|
seed := t.TempDir()
|
|
for _, a := range [][]string{{"init", seed}, {"-C", seed, "config", "user.email", "t@t"}, {"-C", seed, "config", "user.name", "t"}, {"-C", seed, "commit", "--allow-empty", "-m", "init"}, {"-C", seed, "remote", "add", "origin", remote}, {"-C", seed, "push", "-u", "origin", "HEAD:master"}} {
|
|
if out, err := exec.Command("git", a...).CombinedOutput(); err != nil {
|
|
t.Fatalf("git %v: %v %s", a, err, out)
|
|
}
|
|
}
|
|
repo := filepath.Join(t.TempDir(), "repo")
|
|
if out, err := exec.Command("git", "clone", remote, repo).CombinedOutput(); err != nil {
|
|
t.Fatalf("clone: %v %s", err, out)
|
|
}
|
|
for _, a := range [][]string{{"-C", repo, "config", "user.email", "t@t"}, {"-C", repo, "config", "user.name", "t"}} {
|
|
if out, err := exec.Command("git", a...).CombinedOutput(); err != nil {
|
|
t.Fatalf("git %v: %v %s", a, err, out)
|
|
}
|
|
}
|
|
root := filepath.Join(t.TempDir(), "worktrees")
|
|
task := domain.Task{ID: "task", Source: "s", ExternalID: "delivery", Project: "p", QualityGate: "test -f result.txt"}
|
|
wt, err := (orchestrator.GitWorktrees{Repo: repo, Root: root}).Create(context.Background(), task)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(wt, "result.txt"), []byte("delivered"), 0644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.MkdirAll(filepath.Join(wt, ".orchestra"), 0755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(wt, ".orchestra", "done"), nil, 0644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
w := &worker{harnessID: "worker", harness: "opencode", repo: repo, root: root, remote: "origin", tasks: map[string]domain.Task{task.ID: task}}
|
|
evidence, err := w.finalize(context.Background(), task.ID, herdr.Session{PaneID: "pane", Worktree: wt, TaskFileSHA: taskHash(task)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if evidence.ResultSHA == evidence.BaseSHA || evidence.Branch != "orchestra/task" || evidence.QualityGate != task.QualityGate {
|
|
t.Fatalf("unexpected evidence: %+v", evidence)
|
|
}
|
|
out, err := exec.Command("git", "-C", repo, "ls-remote", "origin", "refs/heads/orchestra/task").CombinedOutput()
|
|
if err != nil || !strings.HasPrefix(string(out), evidence.ResultSHA+"\t") {
|
|
t.Fatalf("remote result=%q err=%v want %s", out, err, evidence.ResultSHA)
|
|
}
|
|
}
|
|
|
|
func TestWorkerApprovalCommandIsRevisionBoundAndAcknowledged(t *testing.T) {
|
|
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer ln.Close()
|
|
sent := make(chan string, 1)
|
|
go func() {
|
|
for {
|
|
c, err := ln.Accept()
|
|
if err != nil {
|
|
return
|
|
}
|
|
go func() {
|
|
defer c.Close()
|
|
var request herdr.Request
|
|
if json.NewDecoder(c).Decode(&request) != nil {
|
|
return
|
|
}
|
|
result := `{}`
|
|
switch request.Method {
|
|
case "pane.read":
|
|
result = `{"read":{"text":"Permission required\n$ git status\nProceed? [y/n]"}}`
|
|
case "pane.send_text":
|
|
var p struct {
|
|
Text string `json:"text"`
|
|
}
|
|
_ = json.Unmarshal(mustJSON(request.Params), &p)
|
|
sent <- p.Text
|
|
}
|
|
_ = json.NewEncoder(c).Encode(herdr.Response{ID: request.ID, Result: json.RawMessage(result)})
|
|
}()
|
|
}
|
|
}()
|
|
resolved := make(chan map[string]string, 1)
|
|
api := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
|
|
switch {
|
|
case r.Method == http.MethodGet && r.URL.Path == "/v1/federation/commands":
|
|
_ = json.NewEncoder(rw).Encode([]federation.Command{{ID: "c1", TaskID: "task", Kind: "grant_approval", PaneID: "pane", CaptureRevision: 7}})
|
|
case r.Method == http.MethodPost && r.URL.Path == "/v1/federation/workers/h/captures":
|
|
_ = json.NewEncoder(rw).Encode(federation.Capture{TaskID: "task", PaneID: "pane", Revision: 7})
|
|
case r.Method == http.MethodPost && r.URL.Path == "/v1/federation/commands/c1":
|
|
var body map[string]string
|
|
_ = json.NewDecoder(r.Body).Decode(&body)
|
|
resolved <- body
|
|
rw.WriteHeader(http.StatusNoContent)
|
|
default:
|
|
t.Errorf("unexpected %s %s", r.Method, r.URL.Path)
|
|
rw.WriteHeader(http.StatusNotFound)
|
|
}
|
|
}))
|
|
defer api.Close()
|
|
w := &worker{api: federation.Client{BaseURL: api.URL, WorkerID: "h", Token: "t"}, herdr: herdr.New(ln.Addr().String()), harness: "opencode", sessions: map[string]herdr.Session{"task": {PaneID: "pane"}}}
|
|
w.runCommands(context.Background())
|
|
select {
|
|
case got := <-sent:
|
|
if got != "y\n" {
|
|
t.Fatalf("approval input=%q", got)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("worker did not send approval")
|
|
}
|
|
select {
|
|
case got := <-resolved:
|
|
if got["status"] != "acknowledged" {
|
|
t.Fatalf("resolution=%v", got)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("worker did not resolve command")
|
|
}
|
|
}
|
|
|
|
func TestApprovalResponseOpenCodeAllowOnce(t *testing.T) {
|
|
text := "Permission required\nAllow once Allow always Reject\n⇆ select enter confirm"
|
|
if got, ok := approvalResponse(text, "grant_approval"); !ok || !reflect.DeepEqual(got.Keys, []string{"ENTER"}) || got.Text != "" {
|
|
t.Fatalf("grant response = %+v, %v", got, ok)
|
|
}
|
|
if got, ok := approvalResponse(text, "deny_approval"); ok || got.Text != "" || len(got.Keys) != 0 {
|
|
t.Fatalf("deny response = %+v, %v; reject must not guess selector navigation", got, ok)
|
|
}
|
|
}
|
|
|
|
func mustJSON(v any) []byte { b, _ := json.Marshal(v); return b }
|
|
|
|
type recordingBackend struct {
|
|
status string
|
|
capture string
|
|
progress string
|
|
calls []string
|
|
prompts []string
|
|
}
|
|
|
|
// PaneProgress makes this stub the kind of backend that can separate harness
|
|
// output from input, which is what the worker must prefer over a raw capture.
|
|
func (b *recordingBackend) PaneProgress(context.Context, herdr.Session) (string, error) {
|
|
return b.progress, nil
|
|
}
|
|
|
|
func (b *recordingBackend) Kind() string { return "recording" }
|
|
func (b *recordingBackend) Check(context.Context) error { return nil }
|
|
func (b *recordingBackend) Worktree(_ context.Context, _, path, _ string) (string, error) {
|
|
return path, nil
|
|
}
|
|
func (b *recordingBackend) StartAgent(_ context.Context, _, path, _, harness, _ string) (herdr.Session, error) {
|
|
return herdr.Session{PaneID: "pane", Worktree: path, Harness: harness}, nil
|
|
}
|
|
func (b *recordingBackend) Prompt(_ context.Context, _, text string, _ time.Duration) error {
|
|
b.prompts = append(b.prompts, text)
|
|
return nil
|
|
}
|
|
func (b *recordingBackend) Kill(context.Context, herdr.Session) error { return nil }
|
|
func (b *recordingBackend) AgentStatus(context.Context, herdr.Session) (string, error) {
|
|
if b.status == "" {
|
|
return "idle", nil
|
|
}
|
|
return b.status, nil
|
|
}
|
|
func (b *recordingBackend) PaneCapture(context.Context, herdr.Session, string) (string, error) {
|
|
return b.capture, nil
|
|
}
|
|
|
|
func TestRenewLeasesRequiresProgress(t *testing.T) {
|
|
renewals := 0
|
|
api := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
|
|
renewals++
|
|
rw.Write([]byte(`{}`))
|
|
}))
|
|
defer api.Close()
|
|
backend := &recordingBackend{status: "idle", progress: "same screen"}
|
|
w := &worker{
|
|
api: federation.Client{BaseURL: api.URL, WorkerID: "h", Token: "t"},
|
|
backend: backend,
|
|
harness: "claude",
|
|
sessions: map[string]herdr.Session{"task": {PaneID: "pane"}},
|
|
leases: map[string]lease{"task": {Epoch: "e", Version: 1, Until: time.Now(), ProgressSHA: domain.Hash([]byte("same screen"))}},
|
|
quarantined: map[string]bool{},
|
|
statePath: filepath.Join(t.TempDir(), "state.json"),
|
|
}
|
|
w.renewLeases(context.Background())
|
|
if renewals != 0 {
|
|
t.Fatalf("idle pane with an unchanged capture renewed its lease %d times", renewals)
|
|
}
|
|
backend.status = "busy"
|
|
w.renewLeases(context.Background())
|
|
if renewals != 1 {
|
|
t.Fatalf("busy agent renewals=%d, want 1", renewals)
|
|
}
|
|
backend.status, backend.progress = "idle", "new output"
|
|
l := w.leases["task"]
|
|
l.Until = time.Now()
|
|
w.leases["task"] = l
|
|
w.renewLeases(context.Background())
|
|
if renewals != 2 {
|
|
t.Fatalf("changed capture renewals=%d, want 2", renewals)
|
|
}
|
|
}
|
|
func (b *recordingBackend) SendText(_ context.Context, _ herdr.Session, text string) error {
|
|
b.calls = append(b.calls, "text:"+text)
|
|
return nil
|
|
}
|
|
func (b *recordingBackend) SendKeys(_ context.Context, _ herdr.Session, keys []string) error {
|
|
b.calls = append(b.calls, "keys:"+strings.Join(keys, ","))
|
|
return nil
|
|
}
|
|
func (b *recordingBackend) ReleaseAgent(context.Context, herdr.Session, string) error { return nil }
|
|
|
|
func TestClaudeContextHookRolloverSendsClearThenHandoff(t *testing.T) {
|
|
worktree := t.TempDir()
|
|
handoff := filepath.Join(worktree, "HANDOFF.md")
|
|
if err := os.WriteFile(handoff, []byte("old handoff"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
oldSHA, err := fileSHA256(handoff)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(handoff, []byte("new durable handoff"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
backend := &recordingBackend{}
|
|
session := herdr.Session{PaneID: "pane", Worktree: worktree, Harness: "claude", SessionFile: "exhausted.jsonl", ContextHandoffSHA: oldSHA}
|
|
w := &worker{
|
|
backend: backend,
|
|
harness: "claude",
|
|
sessions: map[string]herdr.Session{"task": session},
|
|
tasks: map[string]domain.Task{},
|
|
leases: map[string]lease{},
|
|
releases: map[string]releaseTransaction{},
|
|
quarantined: map[string]bool{},
|
|
statePath: filepath.Join(t.TempDir(), "state.json"),
|
|
}
|
|
if err := w.advanceClaudeContextReset(context.Background(), "task", session); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := []string{"text:/clear", "keys:ENTER", "text:@HANDOFF.md", "keys:ENTER"}
|
|
if !reflect.DeepEqual(backend.calls, want) {
|
|
t.Fatalf("rollover calls=%v want %v", backend.calls, want)
|
|
}
|
|
got := w.sessions["task"]
|
|
newSHA, _ := fileSHA256(handoff)
|
|
if got.ContextHandoffSHA != newSHA || got.ContextResetSHA != "" || got.ContextResetPhase != "" || got.SessionFile != "" {
|
|
t.Fatalf("rollover state was not finalized: %+v", got)
|
|
}
|
|
if err := w.advanceClaudeContextReset(context.Background(), "task", got); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !reflect.DeepEqual(backend.calls, want) {
|
|
t.Fatalf("unchanged handoff retriggered rollover: %v", backend.calls)
|
|
}
|
|
}
|
|
|
|
func TestClaudeContextHookRolloverWaitsForIdle(t *testing.T) {
|
|
worktree := t.TempDir()
|
|
if err := os.WriteFile(filepath.Join(worktree, "HANDOFF.md"), []byte("handoff"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
backend := &recordingBackend{status: "busy"}
|
|
session := herdr.Session{PaneID: "pane", Worktree: worktree, Harness: "claude"}
|
|
w := &worker{backend: backend, harness: "claude", sessions: map[string]herdr.Session{"task": session}, statePath: filepath.Join(t.TempDir(), "state.json")}
|
|
if err := w.advanceClaudeContextReset(context.Background(), "task", session); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(backend.calls) != 0 {
|
|
t.Fatalf("busy Claude received rollover input: %v", backend.calls)
|
|
}
|
|
}
|
|
|
|
func TestHarnessSpecsFallsBackToSingleHarnessEnvironment(t *testing.T) {
|
|
t.Setenv("ORCHESTRA_WORKER_HERDR_ID", "workpc-opencode")
|
|
t.Setenv("ORCHESTRA_WORKER_HARNESS", "opencode")
|
|
t.Setenv("ORCHESTRA_WORKER_TOKEN", "secret")
|
|
t.Setenv("ORCHESTRA_WORKER_HERDR", "127.0.0.1:9247")
|
|
specs := harnessSpecs()
|
|
if len(specs) != 1 {
|
|
t.Fatalf("want one legacy spec, got %d", len(specs))
|
|
}
|
|
if specs[0].ID != "workpc-opencode" || specs[0].Harness != "opencode" || specs[0].Token != "secret" || specs[0].Herdr != "127.0.0.1:9247" {
|
|
t.Fatalf("legacy environment was not carried through: %+v", specs[0])
|
|
}
|
|
}
|
|
|
|
func TestHarnessSpecsReadsMultipleHarnessesAndPerIdentityTokens(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "harnesses.json")
|
|
if err := os.WriteFile(path, []byte(`[
|
|
{"id":"workpc-claude","harness":"claude","backend":"tmux","tmux_socket":"orchestra","command":"/usr/bin/true"},
|
|
{"id":"workpc-opencode","harness":"opencode","backend":"herdr","herdr":"127.0.0.1:9247","token":"inline"}
|
|
]`), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Setenv("ORCHESTRA_WORKER_HARNESS_CONFIG_FILE", path)
|
|
t.Setenv("ORCHESTRA_WORKER_TOKEN_WORKPC_CLAUDE", "from-env")
|
|
specs := harnessSpecs()
|
|
if len(specs) != 2 {
|
|
t.Fatalf("want two specs, got %d", len(specs))
|
|
}
|
|
if specs[0].Token != "from-env" {
|
|
t.Fatalf("per-identity token env was not consulted: %q", specs[0].Token)
|
|
}
|
|
if specs[1].Token != "inline" {
|
|
t.Fatalf("config-file token was overridden: %q", specs[1].Token)
|
|
}
|
|
// Distinct backends in one process is the point of the multi-harness form.
|
|
if got := backendFor(specs[0]).Kind(); got != "tmux" {
|
|
t.Fatalf("first harness backend = %q, want tmux", got)
|
|
}
|
|
if got := backendFor(specs[1]).Kind(); got != "herdr" {
|
|
t.Fatalf("second harness backend = %q, want herdr", got)
|
|
}
|
|
}
|
|
|
|
func TestTokenEnvKeySanitizesHarnessID(t *testing.T) {
|
|
if got := tokenEnvKey("workpc-claude"); got != "ORCHESTRA_WORKER_TOKEN_WORKPC_CLAUDE" {
|
|
t.Fatalf("tokenEnvKey = %q", got)
|
|
}
|
|
if got := tokenEnvKey("box.1-opencode"); got != "ORCHESTRA_WORKER_TOKEN_BOX_1_OPENCODE" {
|
|
t.Fatalf("tokenEnvKey = %q", got)
|
|
}
|
|
}
|
|
|
|
func TestStatePathIsolatesEveryHarnessIdentity(t *testing.T) {
|
|
claude := harnessSpec{ID: "workpc-claude"}
|
|
opencode := harnessSpec{ID: "workpc-opencode"}
|
|
// The single-harness form must keep the historical path so an upgraded
|
|
// deployment recovers its sessions rather than orphaning their panes.
|
|
if got := statePathFor(opencode, "/wt", "", true); got != "/wt/.orchestra-worker-state.json" {
|
|
t.Fatalf("single-harness state path changed: %q", got)
|
|
}
|
|
a := statePathFor(claude, "/wt", "", false)
|
|
b := statePathFor(opencode, "/wt", "", false)
|
|
if a == b {
|
|
t.Fatalf("two harnesses share one state file: %q", a)
|
|
}
|
|
if got := statePathFor(claude, "/wt", "/var/lib/orchestra-worker", false); got != "/var/lib/orchestra-worker/state-workpc-claude.json" {
|
|
t.Fatalf("state dir ignored: %q", got)
|
|
}
|
|
explicit := harnessSpec{ID: "workpc-claude", State: "/srv/claude.json"}
|
|
if got := statePathFor(explicit, "/wt", "/var/lib/orchestra-worker", false); got != "/srv/claude.json" {
|
|
t.Fatalf("explicit per-entry state path ignored: %q", got)
|
|
}
|
|
}
|
|
|
|
// boundaryAdapter reports a verified turn boundary without a live pane.
|
|
type boundaryAdapter struct {
|
|
at bool
|
|
err error
|
|
}
|
|
|
|
func (b boundaryAdapter) Lease(context.Context, string, string) (herdr.Session, error) {
|
|
return herdr.Session{}, nil
|
|
}
|
|
func (b boundaryAdapter) Release(context.Context, herdr.Session) (string, error) { return "", nil }
|
|
func (b boundaryAdapter) Kill(context.Context, herdr.Session) error { return nil }
|
|
func (b boundaryAdapter) Occupancy(herdr.Session) (float64, error) { return 0, nil }
|
|
func (b boundaryAdapter) AtTurnBoundary(context.Context, herdr.Session) (bool, error) {
|
|
return b.at, b.err
|
|
}
|
|
|
|
// The federated half of the authority model: at a verified boundary the worker
|
|
// asks the coordinator, delivers the correction into its own pane once, and
|
|
// records it so the next boundary stays quiet.
|
|
func TestFederatedTurnDeliversCorrectionOnce(t *testing.T) {
|
|
prompts := make(chan string, 4)
|
|
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer ln.Close()
|
|
go func() {
|
|
for {
|
|
c, e := ln.Accept()
|
|
if e != nil {
|
|
return
|
|
}
|
|
go func() {
|
|
defer c.Close()
|
|
var request herdr.Request
|
|
if json.NewDecoder(c).Decode(&request) != nil {
|
|
return
|
|
}
|
|
result := `{}`
|
|
switch request.Method {
|
|
case "pane.get":
|
|
result = `{"pane":{"agent":"opencode","agent_status":"idle"}}`
|
|
case "pane.read":
|
|
result = `{"read":{"text":""}}`
|
|
case "agent.prompt":
|
|
if m, ok := request.Params.(map[string]any); ok {
|
|
text, _ := m["text"].(string)
|
|
prompts <- text
|
|
}
|
|
}
|
|
_ = json.NewEncoder(c).Encode(herdr.Response{ID: request.ID, Result: json.RawMessage(result)})
|
|
}()
|
|
}
|
|
}()
|
|
|
|
var turnCalls int
|
|
var lastDelivered []string
|
|
fail := false
|
|
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if !strings.HasSuffix(r.URL.Path, "/federation/turn") {
|
|
w.WriteHeader(200)
|
|
return
|
|
}
|
|
if fail {
|
|
http.Error(w, "coordinator down", 503)
|
|
return
|
|
}
|
|
turnCalls++
|
|
var body struct {
|
|
Delivered []string `json:"delivered_decisions"`
|
|
}
|
|
_ = json.NewDecoder(r.Body).Decode(&body)
|
|
lastDelivered = body.Delivered
|
|
out := federation.TurnDecision{Verdict: "continue"}
|
|
if len(body.Delivered) == 0 {
|
|
out.Decisions = []domain.HumanDecision{{
|
|
ID: "d1", Kind: domain.HumanDecisionCorrection, Subject: "strategy", Value: "no, use b",
|
|
}}
|
|
}
|
|
json.NewEncoder(w).Encode(out)
|
|
}))
|
|
defer api.Close()
|
|
|
|
w := &worker{
|
|
api: federation.Client{BaseURL: api.URL, WorkerID: "h", Token: "t"},
|
|
herdr: herdr.New(ln.Addr().String()), harness: "opencode",
|
|
sessions: map[string]herdr.Session{"task": {PaneID: "pane", Harness: "opencode"}},
|
|
leases: map[string]lease{"task": {Epoch: "e1", Version: 2}},
|
|
statePath: t.TempDir() + "/state.json",
|
|
}
|
|
|
|
// Not at a boundary: nothing is asked and nothing is delivered.
|
|
w.federatedTurn(context.Background(), "task", boundaryAdapter{at: false}, "continue")
|
|
if turnCalls != 0 {
|
|
t.Fatal("asked the coordinator without a verified boundary")
|
|
}
|
|
|
|
w.federatedTurn(context.Background(), "task", boundaryAdapter{at: true}, "continue")
|
|
select {
|
|
case got := <-prompts:
|
|
if !strings.Contains(got, "no, use b") || !strings.Contains(got, "outrank") {
|
|
t.Fatalf("delivered prompt = %q", got)
|
|
}
|
|
default:
|
|
t.Fatal("correction was not delivered to the pane")
|
|
}
|
|
if ids := w.sessions["task"].DeliveredDecisions; len(ids) != 1 || ids[0] != "d1" {
|
|
t.Fatalf("delivered ids = %v", ids)
|
|
}
|
|
|
|
// Second boundary: the worker reports what it has shown, so nothing repeats.
|
|
w.federatedTurn(context.Background(), "task", boundaryAdapter{at: true}, "continue")
|
|
if len(lastDelivered) != 1 || lastDelivered[0] != "d1" {
|
|
t.Fatalf("worker did not report delivered ids: %v", lastDelivered)
|
|
}
|
|
select {
|
|
case got := <-prompts:
|
|
t.Fatalf("correction delivered twice: %q", got)
|
|
default:
|
|
}
|
|
|
|
// Coordinator unreachable: observable, and the session keeps running.
|
|
fail = true
|
|
w.lastError = ""
|
|
w.federatedTurn(context.Background(), "task", boundaryAdapter{at: true}, "continue")
|
|
if !strings.Contains(w.lastError, "federated turn") {
|
|
t.Fatalf("transport failure not observable: %q", w.lastError)
|
|
}
|
|
}
|
|
|
|
// A worker must act on the coordinator's verdict, not only on its decisions.
|
|
// When the coordinator can no longer reconcile human input for this task, it
|
|
// answers prepare_handoff, and the worker asks its own agent to hand off. The
|
|
// release loop then takes over on the next tick, because the handoff report is
|
|
// what it watches for.
|
|
func TestFederatedTurnActsOnPrepareHandoffVerdict(t *testing.T) {
|
|
api := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path != "/v1/federation/turn" {
|
|
rw.WriteHeader(http.StatusNotFound)
|
|
return
|
|
}
|
|
json.NewEncoder(rw).Encode(federation.TurnDecision{Verdict: orchestrator.TurnPrepareHandoff})
|
|
}))
|
|
defer api.Close()
|
|
|
|
backend := &recordingBackend{}
|
|
session := herdr.Session{PaneID: "pane", Worktree: t.TempDir(), Harness: "opencode"}
|
|
w := &worker{
|
|
api: federation.Client{BaseURL: api.URL, WorkerID: "h", Token: "t"},
|
|
backend: backend,
|
|
harness: "opencode",
|
|
sessions: map[string]herdr.Session{"task": session},
|
|
leases: map[string]lease{"task": {Epoch: "e1", Version: 2}},
|
|
tasks: map[string]domain.Task{},
|
|
statePath: filepath.Join(t.TempDir(), "state.json"),
|
|
}
|
|
a := herdr.CLIAdapter{Backend: backend, Harness: "opencode"}
|
|
w.federatedTurn(context.Background(), "task", a, orchestrator.TurnContinue)
|
|
|
|
got := w.sessions["task"]
|
|
if !got.HandoffRequested || got.HandoffReason != "reconcile_failure" {
|
|
t.Fatalf("session did not record the requested handoff: %+v", got)
|
|
}
|
|
if len(backend.prompts) != 1 || !strings.Contains(backend.prompts[0], "reconcile_failure") {
|
|
t.Fatalf("agent was not asked to hand off: %v", backend.prompts)
|
|
}
|
|
|
|
// Idempotent: a second boundary must not re-prompt a session that is
|
|
// already preparing its handoff.
|
|
w.federatedTurn(context.Background(), "task", a, orchestrator.TurnContinue)
|
|
if len(backend.prompts) != 1 {
|
|
t.Fatalf("handoff re-requested: %v", backend.prompts)
|
|
}
|
|
}
|
|
|
|
// launchBackend is a recordingBackend that also declares a launch transport
|
|
// and a confirmation outcome, which is the seam F15 added.
|
|
type launchBackend struct {
|
|
recordingBackend
|
|
transport herdr.LaunchTransport
|
|
confirmed bool
|
|
killed int
|
|
}
|
|
|
|
func (b *launchBackend) LaunchTransport(string) herdr.LaunchTransport { return b.transport }
|
|
func (b *launchBackend) ConfirmInput(context.Context, herdr.Session, string) (string, error) {
|
|
if !b.confirmed {
|
|
return "", fmt.Errorf("%w: editor still holds the prompt", herdr.ErrPromptNotSubmitted)
|
|
}
|
|
return "input editor cleared, agent busy", nil
|
|
}
|
|
func (b *launchBackend) Kill(context.Context, herdr.Session) error { b.killed++; return nil }
|
|
|
|
// startFixture builds the git remote, local clone and coordinator stub that
|
|
// worker.start needs, and returns a worker wired to the given backend.
|
|
func startFixture(t *testing.T, backend herdr.Backend, harness string) (*worker, domain.Task, string) {
|
|
t.Helper()
|
|
remote := filepath.Join(t.TempDir(), "remote.git")
|
|
if out, err := exec.Command("git", "init", "--bare", remote).CombinedOutput(); err != nil {
|
|
t.Fatalf("remote: %v %s", err, out)
|
|
}
|
|
seed := t.TempDir()
|
|
for _, a := range [][]string{{"init", seed}, {"-C", seed, "config", "user.email", "t@t"}, {"-C", seed, "config", "user.name", "t"}, {"-C", seed, "commit", "--allow-empty", "-m", "init"}, {"-C", seed, "remote", "add", "origin", remote}, {"-C", seed, "push", "-u", "origin", "HEAD:master"}} {
|
|
if out, err := exec.Command("git", a...).CombinedOutput(); err != nil {
|
|
t.Fatalf("git %v: %v %s", a, err, out)
|
|
}
|
|
}
|
|
repo := filepath.Join(t.TempDir(), "repo")
|
|
if out, err := exec.Command("git", "clone", remote, repo).CombinedOutput(); err != nil {
|
|
t.Fatal(string(out))
|
|
}
|
|
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if strings.HasSuffix(r.URL.Path, "/intent") {
|
|
json.NewEncoder(w).Encode(domain.EffectiveIntent{Task: domain.Task{ID: "task", Title: "test", Description: "do work"}})
|
|
return
|
|
}
|
|
w.WriteHeader(200)
|
|
}))
|
|
t.Cleanup(api.Close)
|
|
root := filepath.Join(filepath.Dir(repo), "worktrees")
|
|
w := &worker{
|
|
api: federation.Client{BaseURL: api.URL, WorkerID: "h", Token: "t"}, harnessID: "h",
|
|
backend: backend, repo: repo, root: root, remote: "origin", harness: harness,
|
|
tasks: map[string]domain.Task{}, sessions: map[string]herdr.Session{}, leases: map[string]lease{},
|
|
releases: map[string]releaseTransaction{}, quarantined: map[string]bool{},
|
|
statePath: filepath.Join(t.TempDir(), "state.json"), hard: .75,
|
|
}
|
|
return w, domain.Task{ID: "task", Source: "s", ExternalID: "x", Project: "p", Title: "test", Description: "do work"}, root
|
|
}
|
|
|
|
// The claude/tmux launch submits one line, while the agent still receives the
|
|
// exact bytes agentctx rendered, through the file the line points at.
|
|
func TestClaudeLaunchSubmitsAFileReferenceAndConfirmsIt(t *testing.T) {
|
|
backend := &launchBackend{transport: herdr.LaunchFileRef, confirmed: true}
|
|
w, task, root := startFixture(t, backend, "claude")
|
|
if err := w.start(context.Background(), task, ""); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(backend.prompts) != 1 || backend.prompts[0] != herdr.LaunchReference {
|
|
t.Fatalf("submitted %q, want the one-line launch reference", backend.prompts)
|
|
}
|
|
b, err := os.ReadFile(filepath.Join(root, "task", herdr.LaunchContextFile))
|
|
if err != nil {
|
|
t.Fatalf("launch context: %v", err)
|
|
}
|
|
for _, want := range []string{"Authority order", "## Goal", "do work"} {
|
|
if !strings.Contains(string(b), want) {
|
|
t.Fatalf("launch context missing %q", want)
|
|
}
|
|
}
|
|
if _, ok := w.sessions[task.ID]; !ok {
|
|
t.Fatal("confirmed launch did not keep its session")
|
|
}
|
|
}
|
|
|
|
// Prompt returning nil is not an acknowledgement. An unconfirmed launch must
|
|
// reclaim the pane, drop the session so the retry can start clean, and
|
|
// classify as prompt_not_submitted so the lease is released rather than held.
|
|
func TestUnconfirmedLaunchFailsAndReclaimsThePane(t *testing.T) {
|
|
backend := &launchBackend{transport: herdr.LaunchFileRef, confirmed: false}
|
|
w, task, _ := startFixture(t, backend, "claude")
|
|
err := w.start(context.Background(), task, "")
|
|
if !errors.Is(err, herdr.ErrPromptNotSubmitted) {
|
|
t.Fatalf("start err=%v, want ErrPromptNotSubmitted", err)
|
|
}
|
|
if _, ok := w.sessions[task.ID]; ok {
|
|
t.Fatal("a launch that never happened must not leave a session behind")
|
|
}
|
|
if backend.killed != 1 {
|
|
t.Fatalf("killed %d panes, want 1", backend.killed)
|
|
}
|
|
if got := classifyLaunchError(err, false); got != "prompt_not_submitted" {
|
|
t.Fatalf("class=%q, want prompt_not_submitted", got)
|
|
}
|
|
// Even with a session still recorded, positive evidence outranks the
|
|
// uncertain classification that would otherwise hold the lease forever.
|
|
if got := classifyLaunchError(err, true); got != "prompt_not_submitted" {
|
|
t.Fatalf("class with session=%q, want prompt_not_submitted", got)
|
|
}
|
|
}
|
|
|
|
// A backend that cannot confirm keeps the inline launch it was verified on.
|
|
func TestInlineTransportSubmitsTheWholeInstruction(t *testing.T) {
|
|
backend := &launchBackend{transport: herdr.LaunchInline, confirmed: true}
|
|
w, task, _ := startFixture(t, backend, "opencode")
|
|
if err := w.start(context.Background(), task, ""); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(backend.prompts) != 1 || !strings.Contains(backend.prompts[0], "Authority order") {
|
|
t.Fatalf("submitted %q, want the whole instruction", backend.prompts)
|
|
}
|
|
}
|
|
|
|
// Found live during burn-in run 3: reconcileLeases rebuilt each lease from the
|
|
// coordinator's view every tick, which wiped ProgressSHA and made the renewal
|
|
// progress check renew unconditionally on its no-baseline branch. The fix in
|
|
// isolation is worthless if the live call path resets its input.
|
|
func TestReconcileLeasesKeepsWorkerLocalObservations(t *testing.T) {
|
|
until := time.Now().Add(30 * time.Minute)
|
|
epoch := "epoch-1"
|
|
api := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
|
|
rw.Write(mustJSON([]domain.Task{{
|
|
ID: "task", State: domain.StateLeased, Version: 12,
|
|
Lease: &domain.Lease{HarnessID: "h", Epoch: epoch, Until: until},
|
|
}}))
|
|
}))
|
|
defer api.Close()
|
|
w := &worker{
|
|
api: federation.Client{BaseURL: api.URL, WorkerID: "h", Token: "t"},
|
|
harnessID: "h",
|
|
tasks: map[string]domain.Task{},
|
|
leases: map[string]lease{"task": {Epoch: "epoch-1", Version: 11, ProgressSHA: "sha-1", UsageBaseline: 0.25, PickupAcknowledged: true}},
|
|
statePath: filepath.Join(t.TempDir(), "state.json"),
|
|
}
|
|
if err := w.reconcileLeases(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got := w.leases["task"]
|
|
if got.ProgressSHA != "sha-1" || got.UsageBaseline != 0.25 || !got.PickupAcknowledged {
|
|
t.Fatalf("reconcile dropped worker-local observations: %+v", got)
|
|
}
|
|
if got.Version != 12 {
|
|
t.Fatalf("version=%d, want the coordinator's 12", got.Version)
|
|
}
|
|
|
|
// A different epoch is a different lease. Nothing observed under the old
|
|
// one may carry into it.
|
|
epoch = "epoch-2"
|
|
if err := w.reconcileLeases(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got = w.leases["task"]
|
|
if got.ProgressSHA != "" || got.UsageBaseline != 0 || got.PickupAcknowledged {
|
|
t.Fatalf("observations survived a new epoch: %+v", got)
|
|
}
|
|
}
|
|
|
|
// The live shape of the run-3 stall: someone typed into the pane, the capture
|
|
// changed, and nothing the agent did changed at all. Renewal must refuse.
|
|
func TestRenewLeasesIgnoresPaneInput(t *testing.T) {
|
|
renewals := 0
|
|
api := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
|
|
renewals++
|
|
rw.Write([]byte(`{}`))
|
|
}))
|
|
defer api.Close()
|
|
backend := &recordingBackend{status: "idle", capture: "screen\n\u276f ", progress: "screen"}
|
|
w := &worker{
|
|
api: federation.Client{BaseURL: api.URL, WorkerID: "h", Token: "t"},
|
|
backend: backend,
|
|
harness: "claude",
|
|
sessions: map[string]herdr.Session{"task": {PaneID: "pane"}},
|
|
leases: map[string]lease{"task": {Epoch: "e", Version: 1, Until: time.Now(), ProgressSHA: domain.Hash([]byte("screen"))}},
|
|
quarantined: map[string]bool{},
|
|
statePath: filepath.Join(t.TempDir(), "state.json"),
|
|
}
|
|
// Only the input line changes. The agent has done nothing.
|
|
backend.capture = "screen\n\u276f go ahead and implement it"
|
|
w.renewLeases(context.Background())
|
|
if renewals != 0 {
|
|
t.Fatalf("pane input renewed the lease %d times", renewals)
|
|
}
|
|
// Harness output moves the digest, and only then does the lease renew.
|
|
backend.progress = "screen\nedited the script"
|
|
w.renewLeases(context.Background())
|
|
if renewals != 1 {
|
|
t.Fatalf("renewals=%d, want 1 after real progress", renewals)
|
|
}
|
|
}
|
|
|
|
// TestBlockedTaskWithUnpushedReleaseFreesTheSessionSlot guards F30. A release
|
|
// transaction that never reached anchor_pushed has no artifact for a successor
|
|
// to pick up, so keeping its session mapping protects nothing. Live, one stuck
|
|
// at "prepared" pinned workpc-claude's only capacity slot: health() kept
|
|
// reporting ActiveTask and the harness never leased again, with no log line.
|
|
func TestBlockedTaskWithUnpushedReleaseFreesTheSessionSlot(t *testing.T) {
|
|
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
switch r.URL.Path {
|
|
case "/v1/federation/events":
|
|
_, _ = w.Write([]byte(`{"cursor":0,"events":[{"seq":1,"id":"b","type":"TaskBlocked","task_id":"t","version":2,"payload":{"blocker":"parked"},"surface":"web"}]}`))
|
|
case "/v1/federation/events/ack":
|
|
w.WriteHeader(http.StatusNoContent)
|
|
default:
|
|
// A worker with a live backend also polls captures and controls.
|
|
// Neither is what this test asserts on.
|
|
w.WriteHeader(http.StatusNoContent)
|
|
}
|
|
}))
|
|
defer s.Close()
|
|
w := &worker{
|
|
api: federation.Client{BaseURL: s.URL, WorkerID: "h", Token: "t"},
|
|
harnessID: "h",
|
|
backend: deadTmuxBackend(t),
|
|
tasks: map[string]domain.Task{},
|
|
sessions: map[string]herdr.Session{"t": {PaneID: "pane"}},
|
|
leases: map[string]lease{"t": {Epoch: "e", Version: 1}},
|
|
releases: map[string]releaseTransaction{"t": {ID: "tx", Phase: "prepared"}},
|
|
quarantined: map[string]bool{},
|
|
statePath: t.TempDir() + "/state.json",
|
|
hard: .75,
|
|
}
|
|
if err := w.once(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(w.sessions) != 0 {
|
|
t.Fatalf("session still pinned: %v", w.sessions)
|
|
}
|
|
if len(w.releases) != 0 {
|
|
t.Fatalf("unrecoverable transaction retained: %v", w.releases)
|
|
}
|
|
if got := w.health(context.Background()).ActiveTask; got != "" {
|
|
t.Fatalf("ActiveTask=%q, worker still advertises itself as busy", got)
|
|
}
|
|
}
|
|
|
|
// The opposite branch must not regress: once an anchor exists, a successor can
|
|
// still pick it up, so the mapping stays until TaskPickupValidated.
|
|
func TestBlockedTaskWithPushedAnchorKeepsItsSession(t *testing.T) {
|
|
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
switch r.URL.Path {
|
|
case "/v1/federation/events":
|
|
_, _ = w.Write([]byte(`{"cursor":0,"events":[{"seq":1,"id":"b","type":"TaskBlocked","task_id":"t","version":2,"payload":{"blocker":"parked"},"surface":"web"}]}`))
|
|
case "/v1/federation/events/ack":
|
|
w.WriteHeader(http.StatusNoContent)
|
|
default:
|
|
t.Fatalf("unexpected %s", r.URL.Path)
|
|
}
|
|
}))
|
|
defer s.Close()
|
|
w := &worker{
|
|
api: federation.Client{BaseURL: s.URL, WorkerID: "h", Token: "t"},
|
|
harnessID: "h",
|
|
tasks: map[string]domain.Task{},
|
|
sessions: map[string]herdr.Session{"t": {PaneID: "pane"}},
|
|
leases: map[string]lease{"t": {Epoch: "e", Version: 1}},
|
|
releases: map[string]releaseTransaction{"t": {ID: "tx", Phase: "anchor_pushed", Ref: "sha256:abc"}},
|
|
quarantined: map[string]bool{},
|
|
statePath: t.TempDir() + "/state.json",
|
|
hard: .75,
|
|
}
|
|
if err := w.once(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(w.sessions) != 1 || len(w.releases) != 1 {
|
|
t.Fatalf("recoverable handoff dropped: sessions=%v releases=%v", w.sessions, w.releases)
|
|
}
|
|
}
|
|
|
|
// deadTmuxBackend stands in for a runtime whose server is gone. tmux answers
|
|
// "no server running", which TmuxBackend.Kill reports as an already-dead
|
|
// session rather than an error.
|
|
func deadTmuxBackend(t *testing.T) *herdr.TmuxBackend {
|
|
t.Helper()
|
|
stub := filepath.Join(t.TempDir(), "tmux")
|
|
if err := os.WriteFile(stub, []byte("#!/bin/sh\necho 'no server running' >&2\nexit 1\n"), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return &herdr.TmuxBackend{Socket: "gone", Binary: stub}
|
|
}
|
|
|
|
// F42, live on run 5: the review agent wrote .orchestra/done, and the result
|
|
// commit's staging step named that file in an exclude pathspec. Git refuses the
|
|
// whole add when a pathspec names an ignored path, so completion failed every
|
|
// five seconds and the task never left review. The marker directory ignores
|
|
// itself, so the exclusion has to name the directory.
|
|
func TestStageExcludeSurvivesTheIgnoredMarker(t *testing.T) {
|
|
repo := t.TempDir()
|
|
run := func(args ...string) (string, error) {
|
|
cmd := exec.Command("git", args...)
|
|
cmd.Dir = repo
|
|
out, err := cmd.CombinedOutput()
|
|
return string(out), err
|
|
}
|
|
for _, args := range [][]string{
|
|
{"init", "-q", "."},
|
|
{"config", "user.email", "t@example.invalid"},
|
|
{"config", "user.name", "t"},
|
|
} {
|
|
if out, err := run(args...); err != nil {
|
|
t.Fatalf("git %v: %s: %v", args, out, err)
|
|
}
|
|
}
|
|
if err := os.WriteFile(filepath.Join(repo, "a.txt"), []byte("one\n"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if out, err := run("add", "a.txt"); err != nil {
|
|
t.Fatalf("%s: %v", out, err)
|
|
}
|
|
if out, err := run("commit", "-qm", "init"); err != nil {
|
|
t.Fatalf("%s: %v", out, err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(repo, "a.txt"), []byte("one\ntwo\n"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
control := filepath.Join(repo, ".orchestra")
|
|
if err := os.MkdirAll(control, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Exactly what the adapter writes, and the marker the agent writes.
|
|
if err := os.WriteFile(filepath.Join(control, ".gitignore"), []byte("*\n"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(control, "done"), nil, 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if out, err := run("add", "-A", "--", ".", stageExclude); err != nil {
|
|
t.Fatalf("staging refused the ignored marker: %s: %v", out, err)
|
|
}
|
|
staged, err := run("diff", "--cached", "--name-only")
|
|
if err != nil {
|
|
t.Fatalf("%s: %v", staged, err)
|
|
}
|
|
if strings.TrimSpace(staged) != "a.txt" {
|
|
t.Fatalf("staged %q, want only a.txt", strings.TrimSpace(staged))
|
|
}
|
|
}
|