Harden worker federation and operator UI
This commit is contained in:
@@ -141,6 +141,19 @@ func (s *Sessions) Valid(v string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// Revoke removes one browser session. It is deliberately idempotent so a
|
||||
// logout request remains safe after expiry or after a cookie was cleared by
|
||||
// the browser.
|
||||
func (s *Sessions) Revoke(v string) {
|
||||
if v == "" {
|
||||
return
|
||||
}
|
||||
sum := sha256.Sum256([]byte(v))
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
delete(s.ids, hex.EncodeToString(sum[:]))
|
||||
}
|
||||
|
||||
// HTTP enforces the same policy at the bus boundary. Authentication is
|
||||
// optional for local development; when a token is supplied, control surfaces
|
||||
// must present it as a Bearer token.
|
||||
@@ -153,6 +166,21 @@ func HTTP(tokens map[Surface]string, next http.Handler) http.Handler {
|
||||
// still has to present the token directly.
|
||||
func HTTPWithSessions(tokens map[Surface]string, sessions *Sessions, next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Federation has per-worker credentials, not one shared surface token.
|
||||
// Let only its registration request and requests that name a worker
|
||||
// reach their handlers; those handlers authenticate the admission token
|
||||
// or worker token respectively. Without this exception, an authenticated
|
||||
// worker is incorrectly treated as the default Web surface.
|
||||
worker := r.Header.Get("X-Orchestra-Worker") != ""
|
||||
federationRegistration := r.Method == http.MethodPost && r.URL.Path == "/v1/federation/workers"
|
||||
workerPath := strings.HasPrefix(r.URL.Path, "/v1/federation/") ||
|
||||
(r.Method == http.MethodGet && r.URL.Path == "/v1/tasks") ||
|
||||
(r.Method == http.MethodPost && r.URL.Path == "/v1/artifacts") ||
|
||||
(r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/v1/artifacts/"))
|
||||
if federationRegistration || (worker && workerPath) {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
s := ParseSurface(r.Header.Get("X-Orchestra-Surface"))
|
||||
if s == "" {
|
||||
s = Web
|
||||
|
||||
@@ -96,6 +96,41 @@ func TestWebSessionCookieGatesControlPathsOnly(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFederationRequestsUseTheirOwnCredentials(t *testing.T) {
|
||||
tokens := map[Surface]string{Web: "web-secret"}
|
||||
h := HTTPWithSessions(tokens, nil, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
method string
|
||||
path string
|
||||
worker string
|
||||
want int
|
||||
}{
|
||||
{name: "registration reaches admission handler", method: http.MethodPost, path: "/v1/federation/workers", want: http.StatusNoContent},
|
||||
{name: "worker request reaches worker handler", method: http.MethodGet, path: "/v1/federation/events", worker: "workpc-opencode", want: http.StatusNoContent},
|
||||
{name: "worker task reconciliation reaches worker handler", method: http.MethodGet, path: "/v1/tasks", worker: "workpc-opencode", want: http.StatusNoContent},
|
||||
{name: "worker artifact read reaches worker handler", method: http.MethodGet, path: "/v1/artifacts/ref", worker: "workpc-opencode", want: http.StatusNoContent},
|
||||
{name: "worker artifact upload reaches worker handler", method: http.MethodPost, path: "/v1/artifacts", worker: "workpc-opencode", want: http.StatusNoContent},
|
||||
{name: "unnamed worker request remains web gated", method: http.MethodGet, path: "/v1/federation/events", want: http.StatusUnauthorized},
|
||||
{name: "worker list remains web gated", method: http.MethodGet, path: "/v1/federation/workers", want: http.StatusUnauthorized},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
r := httptest.NewRequest(tc.method, tc.path, nil)
|
||||
if tc.worker != "" {
|
||||
r.Header.Set("X-Orchestra-Worker", tc.worker)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, r)
|
||||
if w.Code != tc.want {
|
||||
t.Fatalf("status = %d, want %d", w.Code, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionExpires(t *testing.T) {
|
||||
s := &Sessions{TTL: time.Millisecond}
|
||||
v, err := s.Issue()
|
||||
@@ -110,3 +145,18 @@ func TestSessionExpires(t *testing.T) {
|
||||
t.Fatal("empty session accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionRevoke(t *testing.T) {
|
||||
s := &Sessions{}
|
||||
v, err := s.Issue()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !s.Valid(v) {
|
||||
t.Fatal("fresh session must be valid")
|
||||
}
|
||||
s.Revoke(v)
|
||||
if s.Valid(v) {
|
||||
t.Fatal("revoked session must not be valid")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -66,6 +66,13 @@ type Task struct {
|
||||
Version int `json:"version"`
|
||||
Title string `json:"title,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
// Block evidence is projected from TaskBlocked so terminal records remain
|
||||
// diagnosable after the live coordinator mapping is gone.
|
||||
Blocker string `json:"blocker,omitempty"`
|
||||
BlockedAt time.Time `json:"blocked_at,omitempty"`
|
||||
LastPaneID string `json:"last_pane_id,omitempty"`
|
||||
LastHarness string `json:"last_harness_id,omitempty"`
|
||||
PaneState string `json:"pane_state,omitempty"` // open, closed, unreachable, unknown
|
||||
}
|
||||
|
||||
type Event struct {
|
||||
@@ -189,6 +196,9 @@ func ValidatePayload(typ string, p map[string]any) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if v, ok := p["pane_state"]; ok && v != "open" && v != "closed" && v != "unreachable" && v != "unknown" {
|
||||
return fmt.Errorf("%w: pane_state invalid", ErrInvalid)
|
||||
}
|
||||
case "TaskAmended":
|
||||
if len(p) == 0 {
|
||||
return fmt.Errorf("%w: amendment cannot be empty", ErrInvalid)
|
||||
|
||||
@@ -124,8 +124,8 @@ func (c Client) Ack(ctx context.Context, cursor uint64) error {
|
||||
}
|
||||
return err
|
||||
}
|
||||
func (c Client) Heartbeat(ctx context.Context) error {
|
||||
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/heartbeat", nil)
|
||||
func (c Client) Heartbeat(ctx context.Context, health WorkerHealth) error {
|
||||
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/heartbeat", health)
|
||||
if resp != nil {
|
||||
resp.Body.Close()
|
||||
}
|
||||
@@ -146,6 +146,8 @@ func (c Client) PutArtifact(ctx context.Context, b []byte) (string, error) {
|
||||
return "", err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/octet-stream")
|
||||
req.Header.Set("X-Orchestra-Worker", c.WorkerID)
|
||||
req.Header.Set("Authorization", "Bearer "+c.Token)
|
||||
h := c.HTTP
|
||||
if h == nil {
|
||||
h = http.DefaultClient
|
||||
|
||||
@@ -2,8 +2,11 @@ package federation
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
@@ -12,12 +15,26 @@ var ErrUnknownWorker = errors.New("unknown worker")
|
||||
var ErrUnauthorized = errors.New("worker authentication failed")
|
||||
|
||||
type Worker struct {
|
||||
ID string `json:"id"`
|
||||
Address string `json:"address"`
|
||||
Capacity int `json:"capacity"`
|
||||
LastSeen time.Time `json:"last_seen"`
|
||||
Online bool `json:"online"`
|
||||
Token string `json:"-"`
|
||||
ID string `json:"id"`
|
||||
Address string `json:"address"`
|
||||
Capacity int `json:"capacity"`
|
||||
LastSeen time.Time `json:"last_seen"`
|
||||
Online bool `json:"online"`
|
||||
Health WorkerHealth `json:"health"`
|
||||
Token string `json:"-"`
|
||||
}
|
||||
|
||||
// WorkerHealth is reported by the worker that owns the local herdr socket.
|
||||
// It intentionally does not reuse coordinator TCP-probe state: a remote
|
||||
// socket is meaningful only from the machine where the worker and checkout
|
||||
// live.
|
||||
type WorkerHealth struct {
|
||||
HerdrStatus string `json:"herdr_status"` // reachable, unreachable, or unknown
|
||||
CheckedAt time.Time `json:"checked_at,omitempty"`
|
||||
ActiveTask string `json:"active_task_id,omitempty"`
|
||||
ActivePane string `json:"active_pane_id,omitempty"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
ErrorAt time.Time `json:"error_at,omitempty"`
|
||||
}
|
||||
|
||||
// Capture is published by a worker that owns the pane. The coordinator never
|
||||
@@ -53,6 +70,88 @@ type Registry struct {
|
||||
cursors map[string]uint64
|
||||
captures map[string]Capture // worker/task
|
||||
commands map[string][]Command
|
||||
// StatePath preserves worker-owned pane captures and pending approval
|
||||
// commands across coordinator restarts. A worker must still re-register to
|
||||
// be online before it can read or act on recovered state.
|
||||
StatePath string
|
||||
}
|
||||
|
||||
type persistedState struct {
|
||||
Captures map[string]Capture `json:"captures"`
|
||||
Commands map[string][]Command `json:"commands"`
|
||||
Workers map[string]persistedWorker `json:"workers"`
|
||||
}
|
||||
|
||||
// persistedWorker deliberately includes the per-worker token. The state file
|
||||
// is mode 0600, and retaining this binding prevents an arbitrary process from
|
||||
// registering a recovered worker ID and executing its pending approval.
|
||||
type persistedWorker struct {
|
||||
ID string `json:"id"`
|
||||
Address string `json:"address"`
|
||||
Capacity int `json:"capacity"`
|
||||
Token string `json:"token"`
|
||||
}
|
||||
|
||||
// Load restores durable capture/command state. Call this before accepting
|
||||
// federation requests; an unreadable state file is unsafe because it could
|
||||
// otherwise make a pending approval silently disappear.
|
||||
func (r *Registry) Load() error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.init()
|
||||
if r.StatePath == "" {
|
||||
return nil
|
||||
}
|
||||
b, err := os.ReadFile(r.StatePath)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var state persistedState
|
||||
if err := json.Unmarshal(b, &state); err != nil {
|
||||
return fmt.Errorf("invalid federation state: %w", err)
|
||||
}
|
||||
if state.Captures != nil {
|
||||
r.captures = state.Captures
|
||||
}
|
||||
if state.Commands != nil {
|
||||
r.commands = state.Commands
|
||||
}
|
||||
for id, w := range state.Workers {
|
||||
if id == "" || w.ID != id || w.Token == "" {
|
||||
return fmt.Errorf("invalid federation worker %q", id)
|
||||
}
|
||||
r.workers[id] = Worker{ID: w.ID, Address: w.Address, Capacity: w.Capacity, Token: w.Token}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// persistLocked atomically replaces the state file. Callers hold r.mu.
|
||||
func (r *Registry) persistLocked() error {
|
||||
if r.StatePath == "" {
|
||||
return nil
|
||||
}
|
||||
workers := make(map[string]persistedWorker, len(r.workers))
|
||||
for id, w := range r.workers {
|
||||
workers[id] = persistedWorker{ID: w.ID, Address: w.Address, Capacity: w.Capacity, Token: w.Token}
|
||||
}
|
||||
b, err := json.Marshal(persistedState{Captures: r.captures, Commands: r.commands, Workers: workers})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(r.StatePath), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := r.StatePath + ".tmp"
|
||||
if err := os.WriteFile(tmp, b, 0600); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(tmp, r.StatePath); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Chmod(r.StatePath, 0600)
|
||||
}
|
||||
|
||||
func (r *Registry) init() {
|
||||
@@ -94,6 +193,9 @@ func (r *Registry) PutCapture(worker string, c Capture) (Capture, error) {
|
||||
}
|
||||
c.At = time.Now().UTC()
|
||||
r.captures[k] = c
|
||||
if err := r.persistLocked(); err != nil {
|
||||
return Capture{}, fmt.Errorf("persist capture: %w", err)
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
func (r *Registry) Capture(worker, task string) (Capture, bool) {
|
||||
@@ -118,6 +220,9 @@ func (r *Registry) Queue(worker string, c Command) (Command, error) {
|
||||
c.Status = "pending"
|
||||
r.commands[worker] = append(r.commands[worker], c)
|
||||
r.pruneCommands(worker)
|
||||
if err := r.persistLocked(); err != nil {
|
||||
return Command{}, fmt.Errorf("persist command: %w", err)
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
@@ -129,7 +234,7 @@ const CommandRetention = 30 * time.Minute
|
||||
// pruneCommands drops resolved commands past CommandRetention. B21: this list
|
||||
// was append-only, so resolved commands accumulated for the process lifetime
|
||||
// and every worker poll rescanned the entire history. Callers hold r.mu.
|
||||
func (r *Registry) pruneCommands(worker string) {
|
||||
func (r *Registry) pruneCommands(worker string) bool {
|
||||
cutoff := time.Now().UTC().Add(-CommandRetention)
|
||||
in := r.commands[worker]
|
||||
out := in[:0]
|
||||
@@ -139,10 +244,12 @@ func (r *Registry) pruneCommands(worker string) {
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
changed := len(in) != 0
|
||||
delete(r.commands, worker)
|
||||
return
|
||||
return changed
|
||||
}
|
||||
r.commands[worker] = out
|
||||
return len(out) != len(in)
|
||||
}
|
||||
func (r *Registry) Commands(worker string) ([]Command, error) {
|
||||
r.mu.Lock()
|
||||
@@ -151,7 +258,11 @@ func (r *Registry) Commands(worker string) ([]Command, error) {
|
||||
if _, ok := r.workers[worker]; !ok {
|
||||
return nil, ErrUnknownWorker
|
||||
}
|
||||
r.pruneCommands(worker)
|
||||
if r.pruneCommands(worker) {
|
||||
if err := r.persistLocked(); err != nil {
|
||||
return nil, fmt.Errorf("persist pruned commands: %w", err)
|
||||
}
|
||||
}
|
||||
var out []Command
|
||||
for _, c := range r.commands[worker] {
|
||||
if c.Status == "pending" {
|
||||
@@ -182,6 +293,9 @@ func (r *Registry) CompleteCommand(worker, id, status, message string) error {
|
||||
}
|
||||
r.commands[worker][i].Status = status
|
||||
r.commands[worker][i].Error = message
|
||||
if err := r.persistLocked(); err != nil {
|
||||
return fmt.Errorf("persist command resolution: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -214,6 +328,9 @@ func (r *Registry) Register(w Worker, admitToken string) error {
|
||||
if _, ok := r.cursors[w.ID]; !ok {
|
||||
r.cursors[w.ID] = 0
|
||||
}
|
||||
if err := r.persistLocked(); err != nil {
|
||||
return fmt.Errorf("persist worker registration: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (r *Registry) Authenticate(id, token string) error {
|
||||
@@ -251,7 +368,7 @@ func (r *Registry) Ack(id string, cursor uint64) error {
|
||||
r.cursors[id] = cursor
|
||||
return nil
|
||||
}
|
||||
func (r *Registry) Heartbeat(id string) error {
|
||||
func (r *Registry) Heartbeat(id string, health ...WorkerHealth) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.init()
|
||||
@@ -261,6 +378,9 @@ func (r *Registry) Heartbeat(id string) error {
|
||||
}
|
||||
w.LastSeen = time.Now().UTC()
|
||||
w.Online = true
|
||||
if len(health) > 0 {
|
||||
w.Health = health[0]
|
||||
}
|
||||
r.workers[id] = w
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package federation
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -27,6 +29,68 @@ func TestCursorIsMonotonicAndAuthenticationIsRequired(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPendingApprovalSurvivesRegistryRestart(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "federation-state.json")
|
||||
r := &Registry{StatePath: path}
|
||||
if err := r.Load(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.Register(Worker{ID: "w", Token: "t"}, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
capture, err := r.PutCapture("w", Capture{TaskID: "task", PaneID: "pane", Text: "Allow command?"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
queued, err := r.Queue("w", Command{TaskID: "task", Kind: "grant_approval", PaneID: "pane", CaptureRevision: capture.Revision})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.Mode().Perm() != 0600 {
|
||||
t.Fatalf("federation state permissions = %o, want 0600", info.Mode().Perm())
|
||||
}
|
||||
|
||||
restarted := &Registry{StatePath: path}
|
||||
if err := restarted.Load(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := restarted.Register(Worker{ID: "w", Token: "intruder"}, ""); err != ErrUnauthorized {
|
||||
t.Fatalf("recovered worker identity was hijackable: %v", err)
|
||||
}
|
||||
// A restart does not mark the worker online; it must prove its retained
|
||||
// identity by registering again before recovered controls become available.
|
||||
if err := restarted.Register(Worker{ID: "w", Token: "t"}, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gotCapture, ok := restarted.Capture("w", "task")
|
||||
if !ok || gotCapture.Revision != capture.Revision || gotCapture.Text != capture.Text {
|
||||
t.Fatalf("capture after restart = %#v, present=%v", gotCapture, ok)
|
||||
}
|
||||
commands, err := restarted.Commands("w")
|
||||
if err != nil || len(commands) != 1 || commands[0].ID != queued.ID {
|
||||
t.Fatalf("commands after restart = %#v, err=%v", commands, err)
|
||||
}
|
||||
if err := restarted.CompleteCommand("w", queued.ID, "acknowledged", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
again := &Registry{StatePath: path}
|
||||
if err := again.Load(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := again.Register(Worker{ID: "w", Token: "t"}, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
commands, err = again.Commands("w")
|
||||
if err != nil || len(commands) != 0 {
|
||||
t.Fatalf("resolved command recovered as pending: %#v, err=%v", commands, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisterRequiresAdmitTokenAndOwnToken(t *testing.T) {
|
||||
r := &Registry{AdmitToken: "admit-secret"}
|
||||
if err := r.Register(Worker{ID: "workpc", Token: "secret"}, "wrong"); err != ErrUnauthorized {
|
||||
@@ -76,6 +140,29 @@ func TestOfflineHookRunsOnceOnTransition(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeartbeatProjectsWorkerOwnedHealth(t *testing.T) {
|
||||
r := &Registry{}
|
||||
if err := r.Register(Worker{ID: "workpc-opencode", Token: "secret"}, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
checked := time.Now().UTC().Round(0)
|
||||
errAt := checked.Add(-time.Minute)
|
||||
if err := r.Heartbeat("workpc-opencode", WorkerHealth{
|
||||
HerdrStatus: "unreachable", CheckedAt: checked, ActiveTask: "task-1", ActivePane: "pane-1",
|
||||
LastError: "local herdr: connection refused", ErrorAt: errAt,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
workers := r.Snapshot()
|
||||
if len(workers) != 1 {
|
||||
t.Fatalf("workers=%#v", workers)
|
||||
}
|
||||
h := workers[0].Health
|
||||
if h.HerdrStatus != "unreachable" || h.ActiveTask != "task-1" || h.ActivePane != "pane-1" || h.LastError == "" || !h.CheckedAt.Equal(checked) || !h.ErrorAt.Equal(errAt) {
|
||||
t.Fatalf("health=%#v", h)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCaptureRevisionAndCommandQueue(t *testing.T) {
|
||||
r := &Registry{}
|
||||
if err := r.Register(Worker{ID: "w", Token: "t"}, ""); err != nil {
|
||||
|
||||
+30
-18
@@ -142,7 +142,7 @@ func ClaudeActivity(path string) ([]ToolCall, error) {
|
||||
// 2. Shell commands: Codex's actual tool surface is a single freeform
|
||||
// `custom_tool_call` named "exec" whose `input` is a JS snippet calling
|
||||
// `tools.exec_command({cmd:"...", ...})` — not a flat arguments object.
|
||||
// codexExecCommand best-effort-extracts the first embedded cmd string.
|
||||
// codexExecCommands extracts every embedded cmd string in source order.
|
||||
// Success is read from the paired `custom_tool_call_output`'s text
|
||||
// blocks: a failed script's output observably starts with "Script
|
||||
// error:" on this machine's real transcripts (both a JS syntax error and
|
||||
@@ -173,7 +173,7 @@ func CodexActivity(path string) ([]ToolCall, error) {
|
||||
Payload payload `json:"payload"`
|
||||
}
|
||||
|
||||
pending := map[string]ToolCall{}
|
||||
pending := map[string][]ToolCall{}
|
||||
var calls []ToolCall
|
||||
s := bufio.NewScanner(f)
|
||||
s.Buffer(make([]byte, 1<<20), 10<<20)
|
||||
@@ -188,13 +188,20 @@ func CodexActivity(path string) ([]ToolCall, error) {
|
||||
calls = append(calls, ToolCall{Name: "apply_patch", Kind: "file", Key: path, Success: e.Payload.Success})
|
||||
}
|
||||
case e.Type == "response_item" && e.Payload.Type == "custom_tool_call":
|
||||
kind, key := "", ""
|
||||
if cmd := codexExecCommand(e.Payload.Input); cmd != "" {
|
||||
kind, key = "command", cmd
|
||||
var pendingCalls []ToolCall
|
||||
if e.Payload.Name == "exec" {
|
||||
for _, cmd := range codexExecCommands(e.Payload.Input) {
|
||||
pendingCalls = append(pendingCalls, ToolCall{Name: e.Payload.Name, Kind: "command", Key: cmd, IsTest: isTestCommand("command", cmd)})
|
||||
}
|
||||
}
|
||||
pending[e.Payload.CallID] = ToolCall{Name: e.Payload.Name, Kind: kind, Key: key, IsTest: isTestCommand(kind, key)}
|
||||
// Retain a resolved call without an extractable command as activity:
|
||||
// it is useful for ordering, but deliberately carries no key.
|
||||
if len(pendingCalls) == 0 {
|
||||
pendingCalls = []ToolCall{{Name: e.Payload.Name}}
|
||||
}
|
||||
pending[e.Payload.CallID] = pendingCalls
|
||||
case e.Type == "response_item" && e.Payload.Type == "custom_tool_call_output":
|
||||
if tc, ok := pending[e.Payload.CallID]; ok {
|
||||
if pendingCalls, ok := pending[e.Payload.CallID]; ok {
|
||||
failed := false
|
||||
for _, o := range e.Payload.Output {
|
||||
if strings.HasPrefix(strings.TrimSpace(o.Text), "Script error:") {
|
||||
@@ -202,8 +209,10 @@ func CodexActivity(path string) ([]ToolCall, error) {
|
||||
break
|
||||
}
|
||||
}
|
||||
tc.Success = !failed
|
||||
calls = append(calls, tc)
|
||||
for _, tc := range pendingCalls {
|
||||
tc.Success = !failed
|
||||
calls = append(calls, tc)
|
||||
}
|
||||
delete(pending, e.Payload.CallID)
|
||||
}
|
||||
}
|
||||
@@ -211,18 +220,21 @@ func CodexActivity(path string) ([]ToolCall, error) {
|
||||
return calls, s.Err()
|
||||
}
|
||||
|
||||
// codexExecCmdRe extracts the first `cmd:"..."` argument out of an "exec"
|
||||
// custom-tool-call's JS-scripted input. Only the first embedded command in a
|
||||
// multi-call script is captured — a documented limitation, not an oversight.
|
||||
// codexExecCmdRe extracts `cmd:"..."` arguments out of an "exec"
|
||||
// custom-tool-call's JS-scripted input. A single script can invoke several
|
||||
// commands; their source order is the observable execution order.
|
||||
var codexExecCmdRe = regexp.MustCompile(`cmd\s*:\s*"((?:[^"\\]|\\.)*)"`)
|
||||
|
||||
func codexExecCommand(input string) string {
|
||||
m := codexExecCmdRe.FindStringSubmatch(input)
|
||||
if m == nil {
|
||||
return ""
|
||||
func codexExecCommands(input string) []string {
|
||||
matches := codexExecCmdRe.FindAllStringSubmatch(input, -1)
|
||||
commands := make([]string, 0, len(matches))
|
||||
for _, m := range matches {
|
||||
cmd := strings.TrimSpace(strings.NewReplacer(`\"`, `"`, `\n`, "\n", `\t`, "\t", `\\`, `\`).Replace(m[1]))
|
||||
if cmd != "" {
|
||||
commands = append(commands, cmd)
|
||||
}
|
||||
}
|
||||
cmd := strings.NewReplacer(`\"`, `"`, `\n`, "\n", `\t`, "\t", `\\`, `\`).Replace(m[1])
|
||||
return strings.TrimSpace(cmd)
|
||||
return commands
|
||||
}
|
||||
|
||||
// OpenCodeActivity has no verified source. OpenCodeUsage already only reads
|
||||
|
||||
@@ -178,6 +178,26 @@ func TestCodexActivityMarksScriptErrorAsFailure(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCodexActivityPreservesAllCommandsInAScript(t *testing.T) {
|
||||
path := writeJSONL(t, []string{
|
||||
codexExecCallLine(t, "c1", `const test = await tools.exec_command({cmd:"go test ./..."}); const commit = await tools.exec_command({cmd:"git commit -am done"}); text(test.output); text(commit.output)`),
|
||||
codexExecOutputLine(t, "c1", "Script completed"),
|
||||
})
|
||||
calls, err := CodexActivity(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(calls) != 2 {
|
||||
t.Fatalf("calls=%+v, want both commands", calls)
|
||||
}
|
||||
if calls[0].Key != "go test ./..." || calls[1].Key != "git commit -am done" || !calls[0].Success || !calls[1].Success {
|
||||
t.Fatalf("calls=%+v, want successful commands in source order", calls)
|
||||
}
|
||||
if !DetectMilestone(calls) {
|
||||
t.Fatalf("want the later successful git commit to be a milestone")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectThrashConsecutiveTestFailures(t *testing.T) {
|
||||
calls := []ToolCall{
|
||||
{Name: "Bash", Kind: "command", Key: "go test ./...", Success: false, IsTest: true},
|
||||
|
||||
@@ -42,10 +42,6 @@ type PromptLeaser interface {
|
||||
LeasePrompt(context.Context, string, string, string) (Session, error)
|
||||
}
|
||||
|
||||
type WorktreeCreator interface {
|
||||
CreateWorktree(context.Context, string, string, string) (string, error)
|
||||
}
|
||||
|
||||
// TurnBoundary is optional so older herdr deployments remain usable. A true
|
||||
// result means the current harness turn has ended and handoff is safe.
|
||||
type TurnBoundary interface {
|
||||
@@ -99,17 +95,6 @@ const HandoffFile = ".orchestra-handoff.json"
|
||||
// seals the resulting canonical JSON.
|
||||
const HandoffReportFile = ".orchestra-handoff-report.md"
|
||||
|
||||
func (a CLIAdapter) CreateWorktree(ctx context.Context, repo, root, taskID string) (string, error) {
|
||||
path, err := a.Client.Worktree(ctx, repo, filepath.Join(root, taskID), "orchestra/"+taskID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if path == "" {
|
||||
return "", fmt.Errorf("adapter: herdr returned empty worktree path")
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
|
||||
func (a CLIAdapter) Lease(ctx context.Context, task, worktree string) (Session, error) {
|
||||
return a.LeasePrompt(ctx, task, worktree, defaultTaskPrompt(task))
|
||||
}
|
||||
|
||||
@@ -193,14 +193,16 @@ const (
|
||||
// after 2+ minutes of polling, with no error surfaced anywhere. A
|
||||
// legitimate attach has been observed taking "well over a minute", so
|
||||
// this window is deliberately longer than bootRetryWindow.
|
||||
agentAttachWindow = 90 * time.Second
|
||||
agentAttachPoll = 2 * time.Second
|
||||
|
||||
claudeTrustObserveWindow = 15 * time.Second
|
||||
claudeTrustClearWindow = 15 * time.Second
|
||||
claudeTrustPoll = 500 * time.Millisecond
|
||||
)
|
||||
|
||||
var (
|
||||
agentAttachWindow = 90 * time.Second
|
||||
agentAttachPoll = 2 * time.Second
|
||||
)
|
||||
|
||||
type paneStatus struct {
|
||||
Agent string `json:"agent"`
|
||||
AgentStatus string `json:"agent_status"`
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"net"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -82,6 +83,97 @@ func TestStartAgentPassesEmptyHarnessArgs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartAgentAttachesTwoSameHarnessSessionsWithDistinctNames(t *testing.T) {
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = ln.Close() })
|
||||
starts := make(chan Request, 2)
|
||||
go func() {
|
||||
for i := 0; i < 4; i++ {
|
||||
conn, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var req Request
|
||||
if json.NewDecoder(bufio.NewReader(conn)).Decode(&req) == nil {
|
||||
switch req.Method {
|
||||
case "agent.start":
|
||||
starts <- req
|
||||
_ = json.NewEncoder(conn).Encode(Response{ID: req.ID, Result: json.RawMessage(`{}`)})
|
||||
case "pane.get":
|
||||
_ = json.NewEncoder(conn).Encode(Response{ID: req.ID, Result: json.RawMessage(`{"type":"pane_info","pane":{"agent":"opencode","agent_status":"idle"}}`)})
|
||||
}
|
||||
}
|
||||
_ = conn.Close()
|
||||
}
|
||||
}()
|
||||
c := &Client{Path: ln.Addr().String(), panes: map[string]string{"/one": "w1:p1", "/two": "w2:p1"}, dial: func() (net.Conn, error) {
|
||||
return net.Dial("tcp", ln.Addr().String())
|
||||
}}
|
||||
first, err := c.StartAgent(context.Background(), "", "/one", "", "opencode", "first-task")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := c.StartAgent(context.Background(), "", "/two", "", "opencode", "second-task")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first.AgentName == second.AgentName || first.AgentName == "" || second.AgentName == "" {
|
||||
t.Fatalf("agent names must be distinct and persisted: %+v / %+v", first, second)
|
||||
}
|
||||
for _, want := range []string{first.AgentName, second.AgentName} {
|
||||
req := <-starts
|
||||
params, _ := json.Marshal(req.Params)
|
||||
var got struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
_ = json.Unmarshal(params, &got)
|
||||
if got.Name != want {
|
||||
t.Fatalf("agent.start name = %q, want %q", got.Name, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartAgentRejectsSuccessWithoutAttachment(t *testing.T) {
|
||||
oldWindow, oldPoll := agentAttachWindow, agentAttachPoll
|
||||
agentAttachWindow, agentAttachPoll = 25*time.Millisecond, time.Millisecond
|
||||
t.Cleanup(func() { agentAttachWindow, agentAttachPoll = oldWindow, oldPoll })
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = ln.Close() })
|
||||
go func() {
|
||||
for {
|
||||
conn, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
defer conn.Close()
|
||||
var req Request
|
||||
if json.NewDecoder(bufio.NewReader(conn)).Decode(&req) != nil {
|
||||
return
|
||||
}
|
||||
result := json.RawMessage(`{"type":"pane_info","pane":{"agent_status":"unknown"}}`)
|
||||
if req.Method == "agent.start" {
|
||||
result = json.RawMessage(`{}`)
|
||||
}
|
||||
_ = json.NewEncoder(conn).Encode(Response{ID: req.ID, Result: result})
|
||||
}()
|
||||
}
|
||||
}()
|
||||
c := &Client{Path: ln.Addr().String(), panes: map[string]string{"/worktree": "w1:p1"}, dial: func() (net.Conn, error) {
|
||||
return net.Dial("tcp", ln.Addr().String())
|
||||
}}
|
||||
_, err = c.StartAgent(context.Background(), "", "/worktree", "", "opencode", "silent-noop")
|
||||
if err == nil || !strings.Contains(err.Error(), "no agent attached") {
|
||||
t.Fatalf("StartAgent error = %v, want explicit missing attachment", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentNameIsBoundedAndValid(t *testing.T) {
|
||||
got := agentName("OpenCode", "TASK With spaces / and symbols !!! 0123456789")
|
||||
if len(got) > 32 || !regexp.MustCompile(`^[a-z0-9_-]+$`).MatchString(got) {
|
||||
|
||||
@@ -170,11 +170,15 @@ type Coordinator struct {
|
||||
Worktrees Worktrees
|
||||
Adapters Adapters
|
||||
StatePath string
|
||||
mu sync.Mutex
|
||||
sessions map[string]herdr.Session
|
||||
loaded bool
|
||||
healthMu sync.RWMutex
|
||||
health MonitorHealth
|
||||
// LocalHerdr, when set, is the coordinator's machine-ownership boundary.
|
||||
// A coordinator must never operate a pane or checkout owned by another
|
||||
// machine; federation workers own those operations locally.
|
||||
LocalHerdr func(string) bool
|
||||
mu sync.Mutex
|
||||
sessions map[string]herdr.Session
|
||||
loaded bool
|
||||
healthMu sync.RWMutex
|
||||
health MonitorHealth
|
||||
// Hard is the occupancy threshold Monitor's periodic rotate() runs
|
||||
// against, mirrored here so TurnDecision (the synchronous, per-turn
|
||||
// counterpart driven by the Face-B stop hook) evaluates the same
|
||||
@@ -300,6 +304,9 @@ func (c *Coordinator) adapterFor(taskID string, session herdr.Session) (herdr.Ad
|
||||
if id == "" {
|
||||
id = session.Harness
|
||||
}
|
||||
if c.LocalHerdr != nil && !c.LocalHerdr(id) {
|
||||
return nil, fmt.Errorf("session %s is owned by non-local herdr %s", taskID, id)
|
||||
}
|
||||
return c.Adapters.Adapter(id)
|
||||
}
|
||||
|
||||
@@ -875,31 +882,20 @@ func (c *Coordinator) Start(ctx context.Context, e domain.Event) error {
|
||||
if err := json.Unmarshal(e.Payload, &p); err != nil || p.HarnessID == "" {
|
||||
return fmt.Errorf("orchestrator: invalid lease")
|
||||
}
|
||||
if c.LocalHerdr != nil && !c.LocalHerdr(p.HarnessID) {
|
||||
return c.block(t, "remote herdr must be operated by its federation worker")
|
||||
}
|
||||
a, err := c.Adapters.Adapter(p.HarnessID)
|
||||
if err != nil {
|
||||
return c.block(t, "adapter: "+err.Error())
|
||||
}
|
||||
var w string
|
||||
if creator, ok := a.(herdr.WorktreeCreator); ok {
|
||||
planner, planned := c.Worktrees.(WorktreeSpec)
|
||||
if !planned {
|
||||
return c.block(t, "worktree: repository specification unavailable")
|
||||
}
|
||||
repo, root, valid := planner.Spec(t)
|
||||
if !valid {
|
||||
return c.block(t, "worktree: repository and root required")
|
||||
}
|
||||
w, err = creator.CreateWorktree(ctx, repo, root, t.ID)
|
||||
} else {
|
||||
w, err = c.Worktrees.Create(ctx, t)
|
||||
}
|
||||
// Worktrees, including immutable TASK.md, are coordinator-local state.
|
||||
// A remote herdr must be driven by its federation worker instead of being
|
||||
// asked to create an opaque checkout that this coordinator cannot validate.
|
||||
w, err := c.Worktrees.Create(ctx, t)
|
||||
if err != nil {
|
||||
return c.block(t, "worktree: "+err.Error())
|
||||
}
|
||||
// Best-effort: TASK.md only exists for worktrees this process can read
|
||||
// locally (the GitWorktrees path). A herdr-hosted worktree on a remote
|
||||
// machine (WorktreeCreator path) is the same cross-host gap named in
|
||||
// AUDIT.md's federation-fork section — not solved here.
|
||||
taskFileSHA, _ := continuity.TaskFileHash(w)
|
||||
prompt := taskLaunchPrompt(t)
|
||||
var s herdr.Session
|
||||
@@ -983,7 +979,13 @@ func (c *Coordinator) rememberSession(taskID string, s herdr.Session) error {
|
||||
}
|
||||
|
||||
func (c *Coordinator) block(t domain.Task, reason string) error {
|
||||
b, _ := json.Marshal(map[string]string{"blocker": reason})
|
||||
p := map[string]string{"blocker": reason, "pane_state": "unknown"}
|
||||
if s, ok := c.Session(t.ID); ok {
|
||||
p["pane_id"] = s.PaneID
|
||||
p["harness_id"] = s.HerdrID
|
||||
p["pane_state"] = "open"
|
||||
}
|
||||
b, _ := json.Marshal(p)
|
||||
return c.Store.Append(domain.Event{ID: domain.NewID(), Type: "TaskBlocked", TaskID: t.ID, Version: t.Version + 1, Payload: b, Surface: string(authz.System)})
|
||||
}
|
||||
|
||||
|
||||
@@ -21,9 +21,17 @@ type fakeAdapter struct {
|
||||
boundary bool
|
||||
ref string
|
||||
releases int
|
||||
leases int
|
||||
approval struct {
|
||||
called bool
|
||||
grant bool
|
||||
session herdr.Session
|
||||
capture string
|
||||
}
|
||||
}
|
||||
|
||||
func (a *fakeAdapter) Lease(_ context.Context, _ string, worktree string) (herdr.Session, error) {
|
||||
a.leases++
|
||||
return herdr.Session{Harness: "h1", PaneID: "pane-1", Worktree: worktree}, nil
|
||||
}
|
||||
func (a *fakeAdapter) Bootstrap(context.Context, herdr.Session, string) error { return nil }
|
||||
@@ -36,6 +44,13 @@ func (a *fakeAdapter) Occupancy(herdr.Session) (float64, error) { return a.occu
|
||||
func (a *fakeAdapter) AtTurnBoundary(context.Context, herdr.Session) (bool, error) {
|
||||
return a.boundary, nil
|
||||
}
|
||||
func (a *fakeAdapter) RespondApproval(_ context.Context, s herdr.Session, grant bool, capture string) error {
|
||||
a.approval.called = true
|
||||
a.approval.grant = grant
|
||||
a.approval.session = s
|
||||
a.approval.capture = capture
|
||||
return nil
|
||||
}
|
||||
|
||||
type worktrees struct{ path string }
|
||||
|
||||
@@ -45,6 +60,12 @@ type adapters struct{ a herdr.Adapter }
|
||||
|
||||
func (a adapters) Adapter(string) (herdr.Adapter, error) { return a.a, nil }
|
||||
|
||||
type promptFailureAdapter struct{ fakeAdapter }
|
||||
|
||||
func (a *promptFailureAdapter) LeasePrompt(_ context.Context, _ string, worktree, _ string) (herdr.Session, error) {
|
||||
return herdr.Session{Harness: "h1", PaneID: "pane-created-before-timeout", Worktree: worktree}, errors.New("prompt delivery uncertain")
|
||||
}
|
||||
|
||||
func run(t *testing.T, dir string, args ...string) {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", append([]string{"-C", dir}, args...)...)
|
||||
@@ -53,6 +74,105 @@ func run(t *testing.T, dir string, args ...string) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptFailureRetainsLivePaneForBlockedTaskAcrossRestart(t *testing.T) {
|
||||
s, err := store.Open(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Append(domain.Event{ID: domain.NewID(), Type: "TaskCreated", TaskID: "blocked-live-pane", Surface: string(authz.System), Payload: mustJSON(map[string]any{
|
||||
"source": "qa", "external_id": "prompt-timeout", "project": "p",
|
||||
})}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task, ok := s.Task("blocked-live-pane")
|
||||
if !ok {
|
||||
t.Fatal("created task missing")
|
||||
}
|
||||
lease, err := s.Lease(task.ID, "h1", time.Minute)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
statePath := t.TempDir() + "/sessions.json"
|
||||
a := &promptFailureAdapter{}
|
||||
c := &orchestrator.Coordinator{Store: s, Worktrees: worktrees{path: t.TempDir()}, Adapters: adapters{a}, StatePath: statePath}
|
||||
if err := c.Start(context.Background(), lease); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, ok := s.Task(task.ID); !ok || got.State != domain.StateBlocked {
|
||||
t.Fatalf("task state = %+v, want blocked", got)
|
||||
}
|
||||
if session, ok := c.Session(task.ID); !ok || session.PaneID != "pane-created-before-timeout" || session.HerdrID != "h1" {
|
||||
t.Fatalf("retained session = %+v, present=%v", session, ok)
|
||||
}
|
||||
// A fresh coordinator must retain the mapping for a blocked task rather
|
||||
// than treating it as an orphan after restart.
|
||||
restarted := &orchestrator.Coordinator{Store: s, Worktrees: worktrees{path: t.TempDir()}, Adapters: adapters{a}, StatePath: statePath}
|
||||
if err := restarted.Reconcile(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if session, ok := restarted.Session(task.ID); !ok || session.PaneID != "pane-created-before-timeout" {
|
||||
t.Fatalf("restarted session = %+v, present=%v", session, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRespondApprovalUsesOwningSessionAndPreservesCaptureBinding(t *testing.T) {
|
||||
s, err := store.Open(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Append(domain.Event{ID: domain.NewID(), Type: "TaskCreated", TaskID: "approval-task", Surface: string(authz.System), Payload: mustJSON(map[string]any{
|
||||
"source": "qa", "external_id": "approval", "project": "p",
|
||||
})}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lease, err := s.Lease("approval-task", "herdr-1", time.Minute)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := &fakeAdapter{}
|
||||
c := &orchestrator.Coordinator{Store: s, Worktrees: worktrees{path: t.TempDir()}, Adapters: adapters{a}, StatePath: t.TempDir() + "/sessions.json"}
|
||||
if err := c.Start(context.Background(), lease); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const capture = "Approval required\n$ go test ./...\n[y/n]"
|
||||
if err := c.RespondApproval(context.Background(), "approval-task", true, capture); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !a.approval.called || !a.approval.grant || a.approval.capture != capture {
|
||||
t.Fatalf("approval invocation = %#v", a.approval)
|
||||
}
|
||||
if a.approval.session.HerdrID != "herdr-1" || a.approval.session.PaneID == "" {
|
||||
t.Fatalf("approval used wrong session: %#v", a.approval.session)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoordinatorRefusesRemoteHerdrOperations(t *testing.T) {
|
||||
s, err := store.Open(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Append(domain.Event{ID: domain.NewID(), Type: "TaskCreated", TaskID: "remote", Surface: string(authz.System), Payload: mustJSON(map[string]any{
|
||||
"source": "qa", "external_id": "remote", "project": "p",
|
||||
})}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lease, err := s.Lease("remote", "remote", time.Minute)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := &fakeAdapter{}
|
||||
c := &orchestrator.Coordinator{Store: s, Worktrees: worktrees{path: t.TempDir()}, Adapters: adapters{a}, LocalHerdr: func(id string) bool { return id == "local" }}
|
||||
if err := c.Start(context.Background(), lease); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a.leases != 0 {
|
||||
t.Fatal("remote adapter was started by coordinator")
|
||||
}
|
||||
if task, ok := s.Task("remote"); !ok || task.State != domain.StateBlocked {
|
||||
t.Fatalf("remote task state = %#v, present=%v; want blocked", task, ok)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRotationEmitsValidReleaseWithAnchorSHA guards the highest-priority spec
|
||||
// defect noted in progress.md: automated rotation must emit a TaskReleased
|
||||
// event that satisfies domain.ValidatePayload (handoff_ref + anchor_sha), not
|
||||
|
||||
@@ -174,6 +174,14 @@ func putID[T any](m map[string]T, id, kind string) error {
|
||||
func (r Registry) Project(id string) (Project, bool) { p, ok := r.projects[id]; return p, ok }
|
||||
func (r Registry) Machine(id string) (Machine, bool) { m, ok := r.machines[id]; return m, ok }
|
||||
func (r Registry) Herdr(id string) (Herdr, bool) { h, ok := r.herdrs[id]; return h, ok }
|
||||
func (r Registry) Machines() []Machine {
|
||||
out := make([]Machine, 0, len(r.machines))
|
||||
for _, m := range r.machines {
|
||||
out = append(out, m)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
|
||||
return out
|
||||
}
|
||||
func (r Registry) Herdrs() []Herdr {
|
||||
out := make([]Herdr, 0, len(r.herdrs))
|
||||
for _, h := range r.herdrs {
|
||||
|
||||
@@ -155,6 +155,14 @@ func (s *Store) apply(e domain.Event) error {
|
||||
case "TaskBlocked":
|
||||
t.State = domain.StateBlocked
|
||||
t.Lease = nil
|
||||
t.Blocker, _ = p["blocker"].(string)
|
||||
t.BlockedAt = e.At
|
||||
t.LastPaneID, _ = p["pane_id"].(string)
|
||||
t.LastHarness, _ = p["harness_id"].(string)
|
||||
t.PaneState, _ = p["pane_state"].(string)
|
||||
if t.PaneState == "" {
|
||||
t.PaneState = "unknown"
|
||||
}
|
||||
case "TaskAmended":
|
||||
if v, ok := p["title"].(string); ok {
|
||||
t.Title = v
|
||||
|
||||
+17
-2
@@ -136,7 +136,9 @@ func (s Server) detail(ctx context.Context, id string) (TaskDetail, error) {
|
||||
if !ok {
|
||||
return TaskDetail{}, domain.ErrNotFound
|
||||
}
|
||||
d := TaskDetail{Task: t, Actions: actions(t)}
|
||||
// Keep collection fields as JSON arrays for browser clients, including
|
||||
// older task records that genuinely have no retained events.
|
||||
d := TaskDetail{Task: t, Events: []domain.Event{}, Actions: actions(t)}
|
||||
for _, e := range s.Store.Events(0) {
|
||||
if e.TaskID != id {
|
||||
continue
|
||||
@@ -165,9 +167,14 @@ func (s Server) detail(ctx context.Context, id string) (TaskDetail, error) {
|
||||
session.Blocker = "capture unavailable: " + err.Error()
|
||||
}
|
||||
d.Session = session
|
||||
} else if t.State == domain.StateBlocked {
|
||||
// A blocked task has no active lease by definition, but must retain its
|
||||
// last observed pane evidence instead of rendering an unexplained void.
|
||||
d.Session = &Session{PaneID: t.LastPaneID, HarnessID: t.LastHarness, AgentStatus: t.PaneState, Blocker: t.Blocker}
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// captureRevision identifies *what the operator saw*, not when they saw it.
|
||||
// B20: this was UnixNano, so it changed on every read and said nothing about
|
||||
// whether the pane had changed. A content hash changes if and only if the
|
||||
@@ -206,7 +213,15 @@ func actions(t domain.Task) []Action {
|
||||
return []Action{{ID: "handoff", Enabled: active, Reason: "requires a live leased session"}, {ID: "release", Enabled: active, Needs: []string{"reason or handoff_ref"}}, {ID: "block", Enabled: active, Needs: []string{"blocker"}}, {ID: "complete", Enabled: active, Needs: []string{"report_ref", "receipt"}}}
|
||||
}
|
||||
func (s Server) Overview(ctx context.Context) Overview {
|
||||
out := Overview{Tasks: s.Store.Tasks(), UpdatedAt: time.Now().UTC()}
|
||||
// JSON null is not an empty collection to browser clients. In particular,
|
||||
// the shell renders the number of active sessions before any page-level
|
||||
// loading state, so a nil Sessions slice made an otherwise healthy empty
|
||||
// worker pool crash the entire SPA on `sessions.length`.
|
||||
tasks := s.Store.Tasks()
|
||||
if tasks == nil {
|
||||
tasks = []domain.Task{}
|
||||
}
|
||||
out := Overview{Tasks: tasks, Workers: []federation.Worker{}, Sessions: []Session{}, UpdatedAt: time.Now().UTC()}
|
||||
if s.Workers != nil {
|
||||
out.Workers = s.Workers.Snapshot()
|
||||
}
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -1 +0,0 @@
|
||||
:root{font:16px system-ui;color:#e7edf3;background:#111827}body{max-width:1200px;margin:auto;padding:1.5rem}a{color:#8dd5ff}header{display:flex;gap:2rem;align-items:center}.board{display:grid;grid-template-columns:repeat(5,1fr);gap:1rem}.board section{background:#1f2937;border-radius:8px;padding:.7rem;min-height:12rem}.card{display:block;color:inherit;background:#374151;padding:.6rem;margin:.5rem 0;border-radius:5px;text-decoration:none}.card small{display:block;color:#b9c3d0}input,textarea,button{padding:.55rem;margin:.25rem}textarea{min-height:5rem}.create{display:grid;max-width:38rem;margin-top:2rem}pre{background:#030712;padding:1rem;overflow:auto;white-space:pre-wrap}.approval{border:2px solid #fbbf24;background:#422006;padding:1rem;border-radius:8px}table{border-collapse:collapse}td,th{padding:.5rem;border:1px solid #4b5563}@media(max-width:800px){.board{grid-template-columns:1fr 1fr}}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -1 +0,0 @@
|
||||
:root{font:16px system-ui;color:#e7edf3;background:#111827}body{max-width:1200px;margin:auto;padding:1.5rem}a{color:#8dd5ff}header{display:flex;gap:2rem;align-items:center}.board{display:grid;grid-template-columns:repeat(5,1fr);gap:1rem}.board section{background:#1f2937;border-radius:8px;padding:.7rem;min-height:12rem}.card{display:block;color:inherit;background:#374151;padding:.6rem;margin:.5rem 0;border-radius:5px;text-decoration:none}.card small{display:block;color:#b9c3d0}input,textarea,button{padding:.55rem;margin:.25rem}textarea{min-height:5rem}.create{display:grid;max-width:38rem;margin-top:2rem}pre{background:#030712;padding:1rem;overflow:auto;white-space:pre-wrap}.approval{border:2px solid #fbbf24;background:#422006;padding:1rem;border-radius:8px}table{border-collapse:collapse}td,th{padding:.5rem;border:1px solid #4b5563}@media(max-width:800px){.board{grid-template-columns:1fr 1fr}}.actions{display:grid;gap:.75rem;max-width:42rem}.actions form{display:flex;flex-wrap:wrap;align-items:center}.actions textarea{flex:1;min-width:16rem}.approval-backdrop{position:fixed;top:0;right:0;bottom:0;left:0;background:#000a;display:grid;place-items:center;padding:1rem;z-index:10}.approval{max-width:50rem;max-height:90vh;overflow:auto}details textarea{width:100%}
|
||||
File diff suppressed because one or more lines are too long
@@ -1,3 +1,3 @@
|
||||
<script type="module" crossorigin src="/assets/index-hnZ7xNV9.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/assets/index-hcvlnkHy.css">
|
||||
<script type="module" crossorigin src="/assets/index-Dqz7-YV3.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/assets/index-BzSA27i7.css">
|
||||
<div id="root"></div>
|
||||
|
||||
Reference in New Issue
Block a user