Harden worker federation and operator UI

This commit is contained in:
2026-07-29 13:30:55 +04:00
parent 95a96d87a5
commit 1ca9d64e89
35 changed files with 1195 additions and 581 deletions
+28
View File
@@ -141,6 +141,19 @@ func (s *Sessions) Valid(v string) bool {
return true
}
// Revoke removes one browser session. It is deliberately idempotent so a
// logout request remains safe after expiry or after a cookie was cleared by
// the browser.
func (s *Sessions) Revoke(v string) {
if v == "" {
return
}
sum := sha256.Sum256([]byte(v))
s.mu.Lock()
defer s.mu.Unlock()
delete(s.ids, hex.EncodeToString(sum[:]))
}
// HTTP enforces the same policy at the bus boundary. Authentication is
// optional for local development; when a token is supplied, control surfaces
// must present it as a Bearer token.
@@ -153,6 +166,21 @@ func HTTP(tokens map[Surface]string, next http.Handler) http.Handler {
// still has to present the token directly.
func HTTPWithSessions(tokens map[Surface]string, sessions *Sessions, next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Federation has per-worker credentials, not one shared surface token.
// Let only its registration request and requests that name a worker
// reach their handlers; those handlers authenticate the admission token
// or worker token respectively. Without this exception, an authenticated
// worker is incorrectly treated as the default Web surface.
worker := r.Header.Get("X-Orchestra-Worker") != ""
federationRegistration := r.Method == http.MethodPost && r.URL.Path == "/v1/federation/workers"
workerPath := strings.HasPrefix(r.URL.Path, "/v1/federation/") ||
(r.Method == http.MethodGet && r.URL.Path == "/v1/tasks") ||
(r.Method == http.MethodPost && r.URL.Path == "/v1/artifacts") ||
(r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/v1/artifacts/"))
if federationRegistration || (worker && workerPath) {
next.ServeHTTP(w, r)
return
}
s := ParseSurface(r.Header.Get("X-Orchestra-Surface"))
if s == "" {
s = Web
+50
View File
@@ -96,6 +96,41 @@ func TestWebSessionCookieGatesControlPathsOnly(t *testing.T) {
}
}
func TestFederationRequestsUseTheirOwnCredentials(t *testing.T) {
tokens := map[Surface]string{Web: "web-secret"}
h := HTTPWithSessions(tokens, nil, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNoContent)
}))
for _, tc := range []struct {
name string
method string
path string
worker string
want int
}{
{name: "registration reaches admission handler", method: http.MethodPost, path: "/v1/federation/workers", want: http.StatusNoContent},
{name: "worker request reaches worker handler", method: http.MethodGet, path: "/v1/federation/events", worker: "workpc-opencode", want: http.StatusNoContent},
{name: "worker task reconciliation reaches worker handler", method: http.MethodGet, path: "/v1/tasks", worker: "workpc-opencode", want: http.StatusNoContent},
{name: "worker artifact read reaches worker handler", method: http.MethodGet, path: "/v1/artifacts/ref", worker: "workpc-opencode", want: http.StatusNoContent},
{name: "worker artifact upload reaches worker handler", method: http.MethodPost, path: "/v1/artifacts", worker: "workpc-opencode", want: http.StatusNoContent},
{name: "unnamed worker request remains web gated", method: http.MethodGet, path: "/v1/federation/events", want: http.StatusUnauthorized},
{name: "worker list remains web gated", method: http.MethodGet, path: "/v1/federation/workers", want: http.StatusUnauthorized},
} {
t.Run(tc.name, func(t *testing.T) {
r := httptest.NewRequest(tc.method, tc.path, nil)
if tc.worker != "" {
r.Header.Set("X-Orchestra-Worker", tc.worker)
}
w := httptest.NewRecorder()
h.ServeHTTP(w, r)
if w.Code != tc.want {
t.Fatalf("status = %d, want %d", w.Code, tc.want)
}
})
}
}
func TestSessionExpires(t *testing.T) {
s := &Sessions{TTL: time.Millisecond}
v, err := s.Issue()
@@ -110,3 +145,18 @@ func TestSessionExpires(t *testing.T) {
t.Fatal("empty session accepted")
}
}
func TestSessionRevoke(t *testing.T) {
s := &Sessions{}
v, err := s.Issue()
if err != nil {
t.Fatal(err)
}
if !s.Valid(v) {
t.Fatal("fresh session must be valid")
}
s.Revoke(v)
if s.Valid(v) {
t.Fatal("revoked session must not be valid")
}
}
+10
View File
@@ -66,6 +66,13 @@ type Task struct {
Version int `json:"version"`
Title string `json:"title,omitempty"`
Description string `json:"description,omitempty"`
// Block evidence is projected from TaskBlocked so terminal records remain
// diagnosable after the live coordinator mapping is gone.
Blocker string `json:"blocker,omitempty"`
BlockedAt time.Time `json:"blocked_at,omitempty"`
LastPaneID string `json:"last_pane_id,omitempty"`
LastHarness string `json:"last_harness_id,omitempty"`
PaneState string `json:"pane_state,omitempty"` // open, closed, unreachable, unknown
}
type Event struct {
@@ -189,6 +196,9 @@ func ValidatePayload(typ string, p map[string]any) error {
return err
}
}
if v, ok := p["pane_state"]; ok && v != "open" && v != "closed" && v != "unreachable" && v != "unknown" {
return fmt.Errorf("%w: pane_state invalid", ErrInvalid)
}
case "TaskAmended":
if len(p) == 0 {
return fmt.Errorf("%w: amendment cannot be empty", ErrInvalid)
+4 -2
View File
@@ -124,8 +124,8 @@ func (c Client) Ack(ctx context.Context, cursor uint64) error {
}
return err
}
func (c Client) Heartbeat(ctx context.Context) error {
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/heartbeat", nil)
func (c Client) Heartbeat(ctx context.Context, health WorkerHealth) error {
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/heartbeat", health)
if resp != nil {
resp.Body.Close()
}
@@ -146,6 +146,8 @@ func (c Client) PutArtifact(ctx context.Context, b []byte) (string, error) {
return "", err
}
req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("X-Orchestra-Worker", c.WorkerID)
req.Header.Set("Authorization", "Bearer "+c.Token)
h := c.HTTP
if h == nil {
h = http.DefaultClient
+130 -10
View File
@@ -2,8 +2,11 @@ package federation
import (
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"sync"
"time"
)
@@ -12,12 +15,26 @@ var ErrUnknownWorker = errors.New("unknown worker")
var ErrUnauthorized = errors.New("worker authentication failed")
type Worker struct {
ID string `json:"id"`
Address string `json:"address"`
Capacity int `json:"capacity"`
LastSeen time.Time `json:"last_seen"`
Online bool `json:"online"`
Token string `json:"-"`
ID string `json:"id"`
Address string `json:"address"`
Capacity int `json:"capacity"`
LastSeen time.Time `json:"last_seen"`
Online bool `json:"online"`
Health WorkerHealth `json:"health"`
Token string `json:"-"`
}
// WorkerHealth is reported by the worker that owns the local herdr socket.
// It intentionally does not reuse coordinator TCP-probe state: a remote
// socket is meaningful only from the machine where the worker and checkout
// live.
type WorkerHealth struct {
HerdrStatus string `json:"herdr_status"` // reachable, unreachable, or unknown
CheckedAt time.Time `json:"checked_at,omitempty"`
ActiveTask string `json:"active_task_id,omitempty"`
ActivePane string `json:"active_pane_id,omitempty"`
LastError string `json:"last_error,omitempty"`
ErrorAt time.Time `json:"error_at,omitempty"`
}
// Capture is published by a worker that owns the pane. The coordinator never
@@ -53,6 +70,88 @@ type Registry struct {
cursors map[string]uint64
captures map[string]Capture // worker/task
commands map[string][]Command
// StatePath preserves worker-owned pane captures and pending approval
// commands across coordinator restarts. A worker must still re-register to
// be online before it can read or act on recovered state.
StatePath string
}
type persistedState struct {
Captures map[string]Capture `json:"captures"`
Commands map[string][]Command `json:"commands"`
Workers map[string]persistedWorker `json:"workers"`
}
// persistedWorker deliberately includes the per-worker token. The state file
// is mode 0600, and retaining this binding prevents an arbitrary process from
// registering a recovered worker ID and executing its pending approval.
type persistedWorker struct {
ID string `json:"id"`
Address string `json:"address"`
Capacity int `json:"capacity"`
Token string `json:"token"`
}
// Load restores durable capture/command state. Call this before accepting
// federation requests; an unreadable state file is unsafe because it could
// otherwise make a pending approval silently disappear.
func (r *Registry) Load() error {
r.mu.Lock()
defer r.mu.Unlock()
r.init()
if r.StatePath == "" {
return nil
}
b, err := os.ReadFile(r.StatePath)
if errors.Is(err, os.ErrNotExist) {
return nil
}
if err != nil {
return err
}
var state persistedState
if err := json.Unmarshal(b, &state); err != nil {
return fmt.Errorf("invalid federation state: %w", err)
}
if state.Captures != nil {
r.captures = state.Captures
}
if state.Commands != nil {
r.commands = state.Commands
}
for id, w := range state.Workers {
if id == "" || w.ID != id || w.Token == "" {
return fmt.Errorf("invalid federation worker %q", id)
}
r.workers[id] = Worker{ID: w.ID, Address: w.Address, Capacity: w.Capacity, Token: w.Token}
}
return nil
}
// persistLocked atomically replaces the state file. Callers hold r.mu.
func (r *Registry) persistLocked() error {
if r.StatePath == "" {
return nil
}
workers := make(map[string]persistedWorker, len(r.workers))
for id, w := range r.workers {
workers[id] = persistedWorker{ID: w.ID, Address: w.Address, Capacity: w.Capacity, Token: w.Token}
}
b, err := json.Marshal(persistedState{Captures: r.captures, Commands: r.commands, Workers: workers})
if err != nil {
return err
}
if err := os.MkdirAll(filepath.Dir(r.StatePath), 0755); err != nil {
return err
}
tmp := r.StatePath + ".tmp"
if err := os.WriteFile(tmp, b, 0600); err != nil {
return err
}
if err := os.Rename(tmp, r.StatePath); err != nil {
return err
}
return os.Chmod(r.StatePath, 0600)
}
func (r *Registry) init() {
@@ -94,6 +193,9 @@ func (r *Registry) PutCapture(worker string, c Capture) (Capture, error) {
}
c.At = time.Now().UTC()
r.captures[k] = c
if err := r.persistLocked(); err != nil {
return Capture{}, fmt.Errorf("persist capture: %w", err)
}
return c, nil
}
func (r *Registry) Capture(worker, task string) (Capture, bool) {
@@ -118,6 +220,9 @@ func (r *Registry) Queue(worker string, c Command) (Command, error) {
c.Status = "pending"
r.commands[worker] = append(r.commands[worker], c)
r.pruneCommands(worker)
if err := r.persistLocked(); err != nil {
return Command{}, fmt.Errorf("persist command: %w", err)
}
return c, nil
}
@@ -129,7 +234,7 @@ const CommandRetention = 30 * time.Minute
// pruneCommands drops resolved commands past CommandRetention. B21: this list
// was append-only, so resolved commands accumulated for the process lifetime
// and every worker poll rescanned the entire history. Callers hold r.mu.
func (r *Registry) pruneCommands(worker string) {
func (r *Registry) pruneCommands(worker string) bool {
cutoff := time.Now().UTC().Add(-CommandRetention)
in := r.commands[worker]
out := in[:0]
@@ -139,10 +244,12 @@ func (r *Registry) pruneCommands(worker string) {
}
}
if len(out) == 0 {
changed := len(in) != 0
delete(r.commands, worker)
return
return changed
}
r.commands[worker] = out
return len(out) != len(in)
}
func (r *Registry) Commands(worker string) ([]Command, error) {
r.mu.Lock()
@@ -151,7 +258,11 @@ func (r *Registry) Commands(worker string) ([]Command, error) {
if _, ok := r.workers[worker]; !ok {
return nil, ErrUnknownWorker
}
r.pruneCommands(worker)
if r.pruneCommands(worker) {
if err := r.persistLocked(); err != nil {
return nil, fmt.Errorf("persist pruned commands: %w", err)
}
}
var out []Command
for _, c := range r.commands[worker] {
if c.Status == "pending" {
@@ -182,6 +293,9 @@ func (r *Registry) CompleteCommand(worker, id, status, message string) error {
}
r.commands[worker][i].Status = status
r.commands[worker][i].Error = message
if err := r.persistLocked(); err != nil {
return fmt.Errorf("persist command resolution: %w", err)
}
return nil
}
}
@@ -214,6 +328,9 @@ func (r *Registry) Register(w Worker, admitToken string) error {
if _, ok := r.cursors[w.ID]; !ok {
r.cursors[w.ID] = 0
}
if err := r.persistLocked(); err != nil {
return fmt.Errorf("persist worker registration: %w", err)
}
return nil
}
func (r *Registry) Authenticate(id, token string) error {
@@ -251,7 +368,7 @@ func (r *Registry) Ack(id string, cursor uint64) error {
r.cursors[id] = cursor
return nil
}
func (r *Registry) Heartbeat(id string) error {
func (r *Registry) Heartbeat(id string, health ...WorkerHealth) error {
r.mu.Lock()
defer r.mu.Unlock()
r.init()
@@ -261,6 +378,9 @@ func (r *Registry) Heartbeat(id string) error {
}
w.LastSeen = time.Now().UTC()
w.Online = true
if len(health) > 0 {
w.Health = health[0]
}
r.workers[id] = w
return nil
}
+87
View File
@@ -1,6 +1,8 @@
package federation
import (
"os"
"path/filepath"
"testing"
"time"
)
@@ -27,6 +29,68 @@ func TestCursorIsMonotonicAndAuthenticationIsRequired(t *testing.T) {
}
}
func TestPendingApprovalSurvivesRegistryRestart(t *testing.T) {
path := filepath.Join(t.TempDir(), "federation-state.json")
r := &Registry{StatePath: path}
if err := r.Load(); err != nil {
t.Fatal(err)
}
if err := r.Register(Worker{ID: "w", Token: "t"}, ""); err != nil {
t.Fatal(err)
}
capture, err := r.PutCapture("w", Capture{TaskID: "task", PaneID: "pane", Text: "Allow command?"})
if err != nil {
t.Fatal(err)
}
queued, err := r.Queue("w", Command{TaskID: "task", Kind: "grant_approval", PaneID: "pane", CaptureRevision: capture.Revision})
if err != nil {
t.Fatal(err)
}
info, err := os.Stat(path)
if err != nil {
t.Fatal(err)
}
if info.Mode().Perm() != 0600 {
t.Fatalf("federation state permissions = %o, want 0600", info.Mode().Perm())
}
restarted := &Registry{StatePath: path}
if err := restarted.Load(); err != nil {
t.Fatal(err)
}
if err := restarted.Register(Worker{ID: "w", Token: "intruder"}, ""); err != ErrUnauthorized {
t.Fatalf("recovered worker identity was hijackable: %v", err)
}
// A restart does not mark the worker online; it must prove its retained
// identity by registering again before recovered controls become available.
if err := restarted.Register(Worker{ID: "w", Token: "t"}, ""); err != nil {
t.Fatal(err)
}
gotCapture, ok := restarted.Capture("w", "task")
if !ok || gotCapture.Revision != capture.Revision || gotCapture.Text != capture.Text {
t.Fatalf("capture after restart = %#v, present=%v", gotCapture, ok)
}
commands, err := restarted.Commands("w")
if err != nil || len(commands) != 1 || commands[0].ID != queued.ID {
t.Fatalf("commands after restart = %#v, err=%v", commands, err)
}
if err := restarted.CompleteCommand("w", queued.ID, "acknowledged", ""); err != nil {
t.Fatal(err)
}
again := &Registry{StatePath: path}
if err := again.Load(); err != nil {
t.Fatal(err)
}
if err := again.Register(Worker{ID: "w", Token: "t"}, ""); err != nil {
t.Fatal(err)
}
commands, err = again.Commands("w")
if err != nil || len(commands) != 0 {
t.Fatalf("resolved command recovered as pending: %#v, err=%v", commands, err)
}
}
func TestRegisterRequiresAdmitTokenAndOwnToken(t *testing.T) {
r := &Registry{AdmitToken: "admit-secret"}
if err := r.Register(Worker{ID: "workpc", Token: "secret"}, "wrong"); err != ErrUnauthorized {
@@ -76,6 +140,29 @@ func TestOfflineHookRunsOnceOnTransition(t *testing.T) {
}
}
func TestHeartbeatProjectsWorkerOwnedHealth(t *testing.T) {
r := &Registry{}
if err := r.Register(Worker{ID: "workpc-opencode", Token: "secret"}, ""); err != nil {
t.Fatal(err)
}
checked := time.Now().UTC().Round(0)
errAt := checked.Add(-time.Minute)
if err := r.Heartbeat("workpc-opencode", WorkerHealth{
HerdrStatus: "unreachable", CheckedAt: checked, ActiveTask: "task-1", ActivePane: "pane-1",
LastError: "local herdr: connection refused", ErrorAt: errAt,
}); err != nil {
t.Fatal(err)
}
workers := r.Snapshot()
if len(workers) != 1 {
t.Fatalf("workers=%#v", workers)
}
h := workers[0].Health
if h.HerdrStatus != "unreachable" || h.ActiveTask != "task-1" || h.ActivePane != "pane-1" || h.LastError == "" || !h.CheckedAt.Equal(checked) || !h.ErrorAt.Equal(errAt) {
t.Fatalf("health=%#v", h)
}
}
func TestCaptureRevisionAndCommandQueue(t *testing.T) {
r := &Registry{}
if err := r.Register(Worker{ID: "w", Token: "t"}, ""); err != nil {
+30 -18
View File
@@ -142,7 +142,7 @@ func ClaudeActivity(path string) ([]ToolCall, error) {
// 2. Shell commands: Codex's actual tool surface is a single freeform
// `custom_tool_call` named "exec" whose `input` is a JS snippet calling
// `tools.exec_command({cmd:"...", ...})` — not a flat arguments object.
// codexExecCommand best-effort-extracts the first embedded cmd string.
// codexExecCommands extracts every embedded cmd string in source order.
// Success is read from the paired `custom_tool_call_output`'s text
// blocks: a failed script's output observably starts with "Script
// error:" on this machine's real transcripts (both a JS syntax error and
@@ -173,7 +173,7 @@ func CodexActivity(path string) ([]ToolCall, error) {
Payload payload `json:"payload"`
}
pending := map[string]ToolCall{}
pending := map[string][]ToolCall{}
var calls []ToolCall
s := bufio.NewScanner(f)
s.Buffer(make([]byte, 1<<20), 10<<20)
@@ -188,13 +188,20 @@ func CodexActivity(path string) ([]ToolCall, error) {
calls = append(calls, ToolCall{Name: "apply_patch", Kind: "file", Key: path, Success: e.Payload.Success})
}
case e.Type == "response_item" && e.Payload.Type == "custom_tool_call":
kind, key := "", ""
if cmd := codexExecCommand(e.Payload.Input); cmd != "" {
kind, key = "command", cmd
var pendingCalls []ToolCall
if e.Payload.Name == "exec" {
for _, cmd := range codexExecCommands(e.Payload.Input) {
pendingCalls = append(pendingCalls, ToolCall{Name: e.Payload.Name, Kind: "command", Key: cmd, IsTest: isTestCommand("command", cmd)})
}
}
pending[e.Payload.CallID] = ToolCall{Name: e.Payload.Name, Kind: kind, Key: key, IsTest: isTestCommand(kind, key)}
// Retain a resolved call without an extractable command as activity:
// it is useful for ordering, but deliberately carries no key.
if len(pendingCalls) == 0 {
pendingCalls = []ToolCall{{Name: e.Payload.Name}}
}
pending[e.Payload.CallID] = pendingCalls
case e.Type == "response_item" && e.Payload.Type == "custom_tool_call_output":
if tc, ok := pending[e.Payload.CallID]; ok {
if pendingCalls, ok := pending[e.Payload.CallID]; ok {
failed := false
for _, o := range e.Payload.Output {
if strings.HasPrefix(strings.TrimSpace(o.Text), "Script error:") {
@@ -202,8 +209,10 @@ func CodexActivity(path string) ([]ToolCall, error) {
break
}
}
tc.Success = !failed
calls = append(calls, tc)
for _, tc := range pendingCalls {
tc.Success = !failed
calls = append(calls, tc)
}
delete(pending, e.Payload.CallID)
}
}
@@ -211,18 +220,21 @@ func CodexActivity(path string) ([]ToolCall, error) {
return calls, s.Err()
}
// codexExecCmdRe extracts the first `cmd:"..."` argument out of an "exec"
// custom-tool-call's JS-scripted input. Only the first embedded command in a
// multi-call script is captured — a documented limitation, not an oversight.
// codexExecCmdRe extracts `cmd:"..."` arguments out of an "exec"
// custom-tool-call's JS-scripted input. A single script can invoke several
// commands; their source order is the observable execution order.
var codexExecCmdRe = regexp.MustCompile(`cmd\s*:\s*"((?:[^"\\]|\\.)*)"`)
func codexExecCommand(input string) string {
m := codexExecCmdRe.FindStringSubmatch(input)
if m == nil {
return ""
func codexExecCommands(input string) []string {
matches := codexExecCmdRe.FindAllStringSubmatch(input, -1)
commands := make([]string, 0, len(matches))
for _, m := range matches {
cmd := strings.TrimSpace(strings.NewReplacer(`\"`, `"`, `\n`, "\n", `\t`, "\t", `\\`, `\`).Replace(m[1]))
if cmd != "" {
commands = append(commands, cmd)
}
}
cmd := strings.NewReplacer(`\"`, `"`, `\n`, "\n", `\t`, "\t", `\\`, `\`).Replace(m[1])
return strings.TrimSpace(cmd)
return commands
}
// OpenCodeActivity has no verified source. OpenCodeUsage already only reads
+20
View File
@@ -178,6 +178,26 @@ func TestCodexActivityMarksScriptErrorAsFailure(t *testing.T) {
}
}
func TestCodexActivityPreservesAllCommandsInAScript(t *testing.T) {
path := writeJSONL(t, []string{
codexExecCallLine(t, "c1", `const test = await tools.exec_command({cmd:"go test ./..."}); const commit = await tools.exec_command({cmd:"git commit -am done"}); text(test.output); text(commit.output)`),
codexExecOutputLine(t, "c1", "Script completed"),
})
calls, err := CodexActivity(path)
if err != nil {
t.Fatal(err)
}
if len(calls) != 2 {
t.Fatalf("calls=%+v, want both commands", calls)
}
if calls[0].Key != "go test ./..." || calls[1].Key != "git commit -am done" || !calls[0].Success || !calls[1].Success {
t.Fatalf("calls=%+v, want successful commands in source order", calls)
}
if !DetectMilestone(calls) {
t.Fatalf("want the later successful git commit to be a milestone")
}
}
func TestDetectThrashConsecutiveTestFailures(t *testing.T) {
calls := []ToolCall{
{Name: "Bash", Kind: "command", Key: "go test ./...", Success: false, IsTest: true},
-15
View File
@@ -42,10 +42,6 @@ type PromptLeaser interface {
LeasePrompt(context.Context, string, string, string) (Session, error)
}
type WorktreeCreator interface {
CreateWorktree(context.Context, string, string, string) (string, error)
}
// TurnBoundary is optional so older herdr deployments remain usable. A true
// result means the current harness turn has ended and handoff is safe.
type TurnBoundary interface {
@@ -99,17 +95,6 @@ const HandoffFile = ".orchestra-handoff.json"
// seals the resulting canonical JSON.
const HandoffReportFile = ".orchestra-handoff-report.md"
func (a CLIAdapter) CreateWorktree(ctx context.Context, repo, root, taskID string) (string, error) {
path, err := a.Client.Worktree(ctx, repo, filepath.Join(root, taskID), "orchestra/"+taskID)
if err != nil {
return "", err
}
if path == "" {
return "", fmt.Errorf("adapter: herdr returned empty worktree path")
}
return path, nil
}
func (a CLIAdapter) Lease(ctx context.Context, task, worktree string) (Session, error) {
return a.LeasePrompt(ctx, task, worktree, defaultTaskPrompt(task))
}
+5 -3
View File
@@ -193,14 +193,16 @@ const (
// after 2+ minutes of polling, with no error surfaced anywhere. A
// legitimate attach has been observed taking "well over a minute", so
// this window is deliberately longer than bootRetryWindow.
agentAttachWindow = 90 * time.Second
agentAttachPoll = 2 * time.Second
claudeTrustObserveWindow = 15 * time.Second
claudeTrustClearWindow = 15 * time.Second
claudeTrustPoll = 500 * time.Millisecond
)
var (
agentAttachWindow = 90 * time.Second
agentAttachPoll = 2 * time.Second
)
type paneStatus struct {
Agent string `json:"agent"`
AgentStatus string `json:"agent_status"`
+92
View File
@@ -7,6 +7,7 @@ import (
"net"
"reflect"
"regexp"
"strings"
"testing"
"time"
)
@@ -82,6 +83,97 @@ func TestStartAgentPassesEmptyHarnessArgs(t *testing.T) {
}
}
func TestStartAgentAttachesTwoSameHarnessSessionsWithDistinctNames(t *testing.T) {
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = ln.Close() })
starts := make(chan Request, 2)
go func() {
for i := 0; i < 4; i++ {
conn, err := ln.Accept()
if err != nil {
return
}
var req Request
if json.NewDecoder(bufio.NewReader(conn)).Decode(&req) == nil {
switch req.Method {
case "agent.start":
starts <- req
_ = json.NewEncoder(conn).Encode(Response{ID: req.ID, Result: json.RawMessage(`{}`)})
case "pane.get":
_ = json.NewEncoder(conn).Encode(Response{ID: req.ID, Result: json.RawMessage(`{"type":"pane_info","pane":{"agent":"opencode","agent_status":"idle"}}`)})
}
}
_ = conn.Close()
}
}()
c := &Client{Path: ln.Addr().String(), panes: map[string]string{"/one": "w1:p1", "/two": "w2:p1"}, dial: func() (net.Conn, error) {
return net.Dial("tcp", ln.Addr().String())
}}
first, err := c.StartAgent(context.Background(), "", "/one", "", "opencode", "first-task")
if err != nil {
t.Fatal(err)
}
second, err := c.StartAgent(context.Background(), "", "/two", "", "opencode", "second-task")
if err != nil {
t.Fatal(err)
}
if first.AgentName == second.AgentName || first.AgentName == "" || second.AgentName == "" {
t.Fatalf("agent names must be distinct and persisted: %+v / %+v", first, second)
}
for _, want := range []string{first.AgentName, second.AgentName} {
req := <-starts
params, _ := json.Marshal(req.Params)
var got struct {
Name string `json:"name"`
}
_ = json.Unmarshal(params, &got)
if got.Name != want {
t.Fatalf("agent.start name = %q, want %q", got.Name, want)
}
}
}
func TestStartAgentRejectsSuccessWithoutAttachment(t *testing.T) {
oldWindow, oldPoll := agentAttachWindow, agentAttachPoll
agentAttachWindow, agentAttachPoll = 25*time.Millisecond, time.Millisecond
t.Cleanup(func() { agentAttachWindow, agentAttachPoll = oldWindow, oldPoll })
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = ln.Close() })
go func() {
for {
conn, err := ln.Accept()
if err != nil {
return
}
go func() {
defer conn.Close()
var req Request
if json.NewDecoder(bufio.NewReader(conn)).Decode(&req) != nil {
return
}
result := json.RawMessage(`{"type":"pane_info","pane":{"agent_status":"unknown"}}`)
if req.Method == "agent.start" {
result = json.RawMessage(`{}`)
}
_ = json.NewEncoder(conn).Encode(Response{ID: req.ID, Result: result})
}()
}
}()
c := &Client{Path: ln.Addr().String(), panes: map[string]string{"/worktree": "w1:p1"}, dial: func() (net.Conn, error) {
return net.Dial("tcp", ln.Addr().String())
}}
_, err = c.StartAgent(context.Background(), "", "/worktree", "", "opencode", "silent-noop")
if err == nil || !strings.Contains(err.Error(), "no agent attached") {
t.Fatalf("StartAgent error = %v, want explicit missing attachment", err)
}
}
func TestAgentNameIsBoundedAndValid(t *testing.T) {
got := agentName("OpenCode", "TASK With spaces / and symbols !!! 0123456789")
if len(got) > 32 || !regexp.MustCompile(`^[a-z0-9_-]+$`).MatchString(got) {
+26 -24
View File
@@ -170,11 +170,15 @@ type Coordinator struct {
Worktrees Worktrees
Adapters Adapters
StatePath string
mu sync.Mutex
sessions map[string]herdr.Session
loaded bool
healthMu sync.RWMutex
health MonitorHealth
// LocalHerdr, when set, is the coordinator's machine-ownership boundary.
// A coordinator must never operate a pane or checkout owned by another
// machine; federation workers own those operations locally.
LocalHerdr func(string) bool
mu sync.Mutex
sessions map[string]herdr.Session
loaded bool
healthMu sync.RWMutex
health MonitorHealth
// Hard is the occupancy threshold Monitor's periodic rotate() runs
// against, mirrored here so TurnDecision (the synchronous, per-turn
// counterpart driven by the Face-B stop hook) evaluates the same
@@ -300,6 +304,9 @@ func (c *Coordinator) adapterFor(taskID string, session herdr.Session) (herdr.Ad
if id == "" {
id = session.Harness
}
if c.LocalHerdr != nil && !c.LocalHerdr(id) {
return nil, fmt.Errorf("session %s is owned by non-local herdr %s", taskID, id)
}
return c.Adapters.Adapter(id)
}
@@ -875,31 +882,20 @@ func (c *Coordinator) Start(ctx context.Context, e domain.Event) error {
if err := json.Unmarshal(e.Payload, &p); err != nil || p.HarnessID == "" {
return fmt.Errorf("orchestrator: invalid lease")
}
if c.LocalHerdr != nil && !c.LocalHerdr(p.HarnessID) {
return c.block(t, "remote herdr must be operated by its federation worker")
}
a, err := c.Adapters.Adapter(p.HarnessID)
if err != nil {
return c.block(t, "adapter: "+err.Error())
}
var w string
if creator, ok := a.(herdr.WorktreeCreator); ok {
planner, planned := c.Worktrees.(WorktreeSpec)
if !planned {
return c.block(t, "worktree: repository specification unavailable")
}
repo, root, valid := planner.Spec(t)
if !valid {
return c.block(t, "worktree: repository and root required")
}
w, err = creator.CreateWorktree(ctx, repo, root, t.ID)
} else {
w, err = c.Worktrees.Create(ctx, t)
}
// Worktrees, including immutable TASK.md, are coordinator-local state.
// A remote herdr must be driven by its federation worker instead of being
// asked to create an opaque checkout that this coordinator cannot validate.
w, err := c.Worktrees.Create(ctx, t)
if err != nil {
return c.block(t, "worktree: "+err.Error())
}
// Best-effort: TASK.md only exists for worktrees this process can read
// locally (the GitWorktrees path). A herdr-hosted worktree on a remote
// machine (WorktreeCreator path) is the same cross-host gap named in
// AUDIT.md's federation-fork section — not solved here.
taskFileSHA, _ := continuity.TaskFileHash(w)
prompt := taskLaunchPrompt(t)
var s herdr.Session
@@ -983,7 +979,13 @@ func (c *Coordinator) rememberSession(taskID string, s herdr.Session) error {
}
func (c *Coordinator) block(t domain.Task, reason string) error {
b, _ := json.Marshal(map[string]string{"blocker": reason})
p := map[string]string{"blocker": reason, "pane_state": "unknown"}
if s, ok := c.Session(t.ID); ok {
p["pane_id"] = s.PaneID
p["harness_id"] = s.HerdrID
p["pane_state"] = "open"
}
b, _ := json.Marshal(p)
return c.Store.Append(domain.Event{ID: domain.NewID(), Type: "TaskBlocked", TaskID: t.ID, Version: t.Version + 1, Payload: b, Surface: string(authz.System)})
}
+120
View File
@@ -21,9 +21,17 @@ type fakeAdapter struct {
boundary bool
ref string
releases int
leases int
approval struct {
called bool
grant bool
session herdr.Session
capture string
}
}
func (a *fakeAdapter) Lease(_ context.Context, _ string, worktree string) (herdr.Session, error) {
a.leases++
return herdr.Session{Harness: "h1", PaneID: "pane-1", Worktree: worktree}, nil
}
func (a *fakeAdapter) Bootstrap(context.Context, herdr.Session, string) error { return nil }
@@ -36,6 +44,13 @@ func (a *fakeAdapter) Occupancy(herdr.Session) (float64, error) { return a.occu
func (a *fakeAdapter) AtTurnBoundary(context.Context, herdr.Session) (bool, error) {
return a.boundary, nil
}
func (a *fakeAdapter) RespondApproval(_ context.Context, s herdr.Session, grant bool, capture string) error {
a.approval.called = true
a.approval.grant = grant
a.approval.session = s
a.approval.capture = capture
return nil
}
type worktrees struct{ path string }
@@ -45,6 +60,12 @@ type adapters struct{ a herdr.Adapter }
func (a adapters) Adapter(string) (herdr.Adapter, error) { return a.a, nil }
type promptFailureAdapter struct{ fakeAdapter }
func (a *promptFailureAdapter) LeasePrompt(_ context.Context, _ string, worktree, _ string) (herdr.Session, error) {
return herdr.Session{Harness: "h1", PaneID: "pane-created-before-timeout", Worktree: worktree}, errors.New("prompt delivery uncertain")
}
func run(t *testing.T, dir string, args ...string) {
t.Helper()
cmd := exec.Command("git", append([]string{"-C", dir}, args...)...)
@@ -53,6 +74,105 @@ func run(t *testing.T, dir string, args ...string) {
}
}
func TestPromptFailureRetainsLivePaneForBlockedTaskAcrossRestart(t *testing.T) {
s, err := store.Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
if err := s.Append(domain.Event{ID: domain.NewID(), Type: "TaskCreated", TaskID: "blocked-live-pane", Surface: string(authz.System), Payload: mustJSON(map[string]any{
"source": "qa", "external_id": "prompt-timeout", "project": "p",
})}); err != nil {
t.Fatal(err)
}
task, ok := s.Task("blocked-live-pane")
if !ok {
t.Fatal("created task missing")
}
lease, err := s.Lease(task.ID, "h1", time.Minute)
if err != nil {
t.Fatal(err)
}
statePath := t.TempDir() + "/sessions.json"
a := &promptFailureAdapter{}
c := &orchestrator.Coordinator{Store: s, Worktrees: worktrees{path: t.TempDir()}, Adapters: adapters{a}, StatePath: statePath}
if err := c.Start(context.Background(), lease); err != nil {
t.Fatal(err)
}
if got, ok := s.Task(task.ID); !ok || got.State != domain.StateBlocked {
t.Fatalf("task state = %+v, want blocked", got)
}
if session, ok := c.Session(task.ID); !ok || session.PaneID != "pane-created-before-timeout" || session.HerdrID != "h1" {
t.Fatalf("retained session = %+v, present=%v", session, ok)
}
// A fresh coordinator must retain the mapping for a blocked task rather
// than treating it as an orphan after restart.
restarted := &orchestrator.Coordinator{Store: s, Worktrees: worktrees{path: t.TempDir()}, Adapters: adapters{a}, StatePath: statePath}
if err := restarted.Reconcile(context.Background()); err != nil {
t.Fatal(err)
}
if session, ok := restarted.Session(task.ID); !ok || session.PaneID != "pane-created-before-timeout" {
t.Fatalf("restarted session = %+v, present=%v", session, ok)
}
}
func TestRespondApprovalUsesOwningSessionAndPreservesCaptureBinding(t *testing.T) {
s, err := store.Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
if err := s.Append(domain.Event{ID: domain.NewID(), Type: "TaskCreated", TaskID: "approval-task", Surface: string(authz.System), Payload: mustJSON(map[string]any{
"source": "qa", "external_id": "approval", "project": "p",
})}); err != nil {
t.Fatal(err)
}
lease, err := s.Lease("approval-task", "herdr-1", time.Minute)
if err != nil {
t.Fatal(err)
}
a := &fakeAdapter{}
c := &orchestrator.Coordinator{Store: s, Worktrees: worktrees{path: t.TempDir()}, Adapters: adapters{a}, StatePath: t.TempDir() + "/sessions.json"}
if err := c.Start(context.Background(), lease); err != nil {
t.Fatal(err)
}
const capture = "Approval required\n$ go test ./...\n[y/n]"
if err := c.RespondApproval(context.Background(), "approval-task", true, capture); err != nil {
t.Fatal(err)
}
if !a.approval.called || !a.approval.grant || a.approval.capture != capture {
t.Fatalf("approval invocation = %#v", a.approval)
}
if a.approval.session.HerdrID != "herdr-1" || a.approval.session.PaneID == "" {
t.Fatalf("approval used wrong session: %#v", a.approval.session)
}
}
func TestCoordinatorRefusesRemoteHerdrOperations(t *testing.T) {
s, err := store.Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
if err := s.Append(domain.Event{ID: domain.NewID(), Type: "TaskCreated", TaskID: "remote", Surface: string(authz.System), Payload: mustJSON(map[string]any{
"source": "qa", "external_id": "remote", "project": "p",
})}); err != nil {
t.Fatal(err)
}
lease, err := s.Lease("remote", "remote", time.Minute)
if err != nil {
t.Fatal(err)
}
a := &fakeAdapter{}
c := &orchestrator.Coordinator{Store: s, Worktrees: worktrees{path: t.TempDir()}, Adapters: adapters{a}, LocalHerdr: func(id string) bool { return id == "local" }}
if err := c.Start(context.Background(), lease); err != nil {
t.Fatal(err)
}
if a.leases != 0 {
t.Fatal("remote adapter was started by coordinator")
}
if task, ok := s.Task("remote"); !ok || task.State != domain.StateBlocked {
t.Fatalf("remote task state = %#v, present=%v; want blocked", task, ok)
}
}
// TestRotationEmitsValidReleaseWithAnchorSHA guards the highest-priority spec
// defect noted in progress.md: automated rotation must emit a TaskReleased
// event that satisfies domain.ValidatePayload (handoff_ref + anchor_sha), not
+8
View File
@@ -174,6 +174,14 @@ func putID[T any](m map[string]T, id, kind string) error {
func (r Registry) Project(id string) (Project, bool) { p, ok := r.projects[id]; return p, ok }
func (r Registry) Machine(id string) (Machine, bool) { m, ok := r.machines[id]; return m, ok }
func (r Registry) Herdr(id string) (Herdr, bool) { h, ok := r.herdrs[id]; return h, ok }
func (r Registry) Machines() []Machine {
out := make([]Machine, 0, len(r.machines))
for _, m := range r.machines {
out = append(out, m)
}
sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
return out
}
func (r Registry) Herdrs() []Herdr {
out := make([]Herdr, 0, len(r.herdrs))
for _, h := range r.herdrs {
+8
View File
@@ -155,6 +155,14 @@ func (s *Store) apply(e domain.Event) error {
case "TaskBlocked":
t.State = domain.StateBlocked
t.Lease = nil
t.Blocker, _ = p["blocker"].(string)
t.BlockedAt = e.At
t.LastPaneID, _ = p["pane_id"].(string)
t.LastHarness, _ = p["harness_id"].(string)
t.PaneState, _ = p["pane_state"].(string)
if t.PaneState == "" {
t.PaneState = "unknown"
}
case "TaskAmended":
if v, ok := p["title"].(string); ok {
t.Title = v
+17 -2
View File
@@ -136,7 +136,9 @@ func (s Server) detail(ctx context.Context, id string) (TaskDetail, error) {
if !ok {
return TaskDetail{}, domain.ErrNotFound
}
d := TaskDetail{Task: t, Actions: actions(t)}
// Keep collection fields as JSON arrays for browser clients, including
// older task records that genuinely have no retained events.
d := TaskDetail{Task: t, Events: []domain.Event{}, Actions: actions(t)}
for _, e := range s.Store.Events(0) {
if e.TaskID != id {
continue
@@ -165,9 +167,14 @@ func (s Server) detail(ctx context.Context, id string) (TaskDetail, error) {
session.Blocker = "capture unavailable: " + err.Error()
}
d.Session = session
} else if t.State == domain.StateBlocked {
// A blocked task has no active lease by definition, but must retain its
// last observed pane evidence instead of rendering an unexplained void.
d.Session = &Session{PaneID: t.LastPaneID, HarnessID: t.LastHarness, AgentStatus: t.PaneState, Blocker: t.Blocker}
}
return d, nil
}
// captureRevision identifies *what the operator saw*, not when they saw it.
// B20: this was UnixNano, so it changed on every read and said nothing about
// whether the pane had changed. A content hash changes if and only if the
@@ -206,7 +213,15 @@ func actions(t domain.Task) []Action {
return []Action{{ID: "handoff", Enabled: active, Reason: "requires a live leased session"}, {ID: "release", Enabled: active, Needs: []string{"reason or handoff_ref"}}, {ID: "block", Enabled: active, Needs: []string{"blocker"}}, {ID: "complete", Enabled: active, Needs: []string{"report_ref", "receipt"}}}
}
func (s Server) Overview(ctx context.Context) Overview {
out := Overview{Tasks: s.Store.Tasks(), UpdatedAt: time.Now().UTC()}
// JSON null is not an empty collection to browser clients. In particular,
// the shell renders the number of active sessions before any page-level
// loading state, so a nil Sessions slice made an otherwise healthy empty
// worker pool crash the entire SPA on `sessions.length`.
tasks := s.Store.Tasks()
if tasks == nil {
tasks = []domain.Task{}
}
out := Overview{Tasks: tasks, Workers: []federation.Worker{}, Sessions: []Session{}, UpdatedAt: time.Now().UTC()}
if s.Workers != nil {
out.Workers = s.Workers.Snapshot()
}
File diff suppressed because one or more lines are too long
@@ -1 +0,0 @@
:root{font:16px system-ui;color:#e7edf3;background:#111827}body{max-width:1200px;margin:auto;padding:1.5rem}a{color:#8dd5ff}header{display:flex;gap:2rem;align-items:center}.board{display:grid;grid-template-columns:repeat(5,1fr);gap:1rem}.board section{background:#1f2937;border-radius:8px;padding:.7rem;min-height:12rem}.card{display:block;color:inherit;background:#374151;padding:.6rem;margin:.5rem 0;border-radius:5px;text-decoration:none}.card small{display:block;color:#b9c3d0}input,textarea,button{padding:.55rem;margin:.25rem}textarea{min-height:5rem}.create{display:grid;max-width:38rem;margin-top:2rem}pre{background:#030712;padding:1rem;overflow:auto;white-space:pre-wrap}.approval{border:2px solid #fbbf24;background:#422006;padding:1rem;border-radius:8px}table{border-collapse:collapse}td,th{padding:.5rem;border:1px solid #4b5563}@media(max-width:800px){.board{grid-template-columns:1fr 1fr}}
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -1 +0,0 @@
:root{font:16px system-ui;color:#e7edf3;background:#111827}body{max-width:1200px;margin:auto;padding:1.5rem}a{color:#8dd5ff}header{display:flex;gap:2rem;align-items:center}.board{display:grid;grid-template-columns:repeat(5,1fr);gap:1rem}.board section{background:#1f2937;border-radius:8px;padding:.7rem;min-height:12rem}.card{display:block;color:inherit;background:#374151;padding:.6rem;margin:.5rem 0;border-radius:5px;text-decoration:none}.card small{display:block;color:#b9c3d0}input,textarea,button{padding:.55rem;margin:.25rem}textarea{min-height:5rem}.create{display:grid;max-width:38rem;margin-top:2rem}pre{background:#030712;padding:1rem;overflow:auto;white-space:pre-wrap}.approval{border:2px solid #fbbf24;background:#422006;padding:1rem;border-radius:8px}table{border-collapse:collapse}td,th{padding:.5rem;border:1px solid #4b5563}@media(max-width:800px){.board{grid-template-columns:1fr 1fr}}.actions{display:grid;gap:.75rem;max-width:42rem}.actions form{display:flex;flex-wrap:wrap;align-items:center}.actions textarea{flex:1;min-width:16rem}.approval-backdrop{position:fixed;top:0;right:0;bottom:0;left:0;background:#000a;display:grid;place-items:center;padding:1rem;z-index:10}.approval{max-width:50rem;max-height:90vh;overflow:auto}details textarea{width:100%}
File diff suppressed because one or more lines are too long
+2 -2
View File
@@ -1,3 +1,3 @@
<script type="module" crossorigin src="/assets/index-hnZ7xNV9.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-hcvlnkHy.css">
<script type="module" crossorigin src="/assets/index-Dqz7-YV3.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-BzSA27i7.css">
<div id="root"></div>