Harden worker federation and operator UI

This commit is contained in:
2026-07-29 13:30:55 +04:00
parent 95a96d87a5
commit 1ca9d64e89
35 changed files with 1195 additions and 581 deletions
+57 -8
View File
@@ -34,7 +34,43 @@ type worker struct {
statePath string
hard float64
registration federation.Worker
lastError string
lastErrorAt time.Time
}
func (w *worker) recordError(err error) {
if err == nil {
return
}
w.lastError = err.Error()
w.lastErrorAt = time.Now().UTC()
}
func (w *worker) health(ctx context.Context) federation.WorkerHealth {
h := federation.WorkerHealth{HerdrStatus: "unknown"}
for taskID, session := range w.sessions {
// Workers currently advertise capacity one. Pick deterministically so a
// recovered legacy state with more sessions remains intelligible.
if h.ActiveTask == "" || taskID < h.ActiveTask {
h.ActiveTask, h.ActivePane = taskID, session.PaneID
}
}
if w.herdr != nil {
checkCtx, cancel := context.WithTimeout(ctx, 3*time.Second)
err := w.herdr.CheckProtocol(checkCtx, "17")
cancel()
h.CheckedAt = time.Now().UTC()
if err == nil {
h.HerdrStatus = "reachable"
} else {
h.HerdrStatus = "unreachable"
w.recordError(fmt.Errorf("local herdr: %w", err))
}
}
h.LastError, h.ErrorAt = w.lastError, w.lastErrorAt
return h
}
type lease struct {
HandoffRef string `json:"handoff_ref,omitempty"`
}
@@ -248,22 +284,29 @@ func (w *worker) publishCaptures(ctx context.Context) {
}
}
func approvalResponse(text, kind string) (string, bool) {
type approvalInput struct {
Text string
Keys []string
}
func approvalResponse(text, kind string) (approvalInput, bool) {
low := strings.ToLower(text)
// Never invent a keystroke. y/n prompts label both decisions directly.
if strings.Contains(low, "[y/n]") || strings.Contains(low, "(y/n)") {
if kind == "grant_approval" {
return "y\n", true
return approvalInput{Text: "y\n"}, true
}
return "n\n", true
return approvalInput{Text: "n\n"}, true
}
// OpenCode's explicit selector states "Allow once Allow always Reject"
// and "enter confirm". Enter is consequently a bounded one-time grant;
// and "enter confirm". Send a real ENTER key, not a newline through
// pane.send_text: OpenCode's selector does not treat the latter as input.
// Enter is consequently a bounded one-time grant;
// rejection would require unobservable selector navigation, so refuse it.
if kind == "grant_approval" && strings.Contains(low, "allow once") && strings.Contains(low, "allow always") && strings.Contains(low, "reject") && strings.Contains(low, "enter confirm") {
return "\n", true
return approvalInput{Keys: []string{"ENTER"}}, true
}
return "", false
return approvalInput{}, false
}
func (w *worker) runCommands(ctx context.Context) {
commands, err := w.api.Commands(ctx)
@@ -296,7 +339,11 @@ func (w *worker) runCommands(ctx context.Context) {
_ = w.api.ResolveCommand(ctx, command.ID, "rejected", "prompt does not expose an executable approval control")
continue
}
if err := w.herdr.Call(ctx, "pane.send_text", map[string]any{"pane_id": session.PaneID, "text": input}, nil); err != nil {
method, params := "pane.send_text", map[string]any{"pane_id": session.PaneID, "text": input.Text}
if len(input.Keys) > 0 {
method, params = "pane.send_keys", map[string]any{"pane_id": session.PaneID, "keys": input.Keys}
}
if err := w.herdr.Call(ctx, method, params, nil); err != nil {
_ = w.api.ResolveCommand(ctx, command.ID, "rejected", "herdr did not acknowledge input: "+err.Error())
continue
}
@@ -450,13 +497,15 @@ func main() {
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
for {
if err := w.api.Heartbeat(ctx); err != nil {
if err := w.api.Heartbeat(ctx, w.health(ctx)); err != nil {
w.recordError(fmt.Errorf("heartbeat: %w", err))
log.Printf("heartbeat: %v", err)
if w.reRegisterAfterCoordinatorRestart(ctx, err) {
continue
}
}
if err := w.once(ctx); err != nil {
w.recordError(fmt.Errorf("poll: %w", err))
log.Printf("poll: %v", err)
w.reRegisterAfterCoordinatorRestart(ctx, err)
}
+5 -4
View File
@@ -14,6 +14,7 @@ import (
"os"
"os/exec"
"path/filepath"
"reflect"
"testing"
"time"
)
@@ -248,11 +249,11 @@ func TestWorkerApprovalCommandIsRevisionBoundAndAcknowledged(t *testing.T) {
func TestApprovalResponseOpenCodeAllowOnce(t *testing.T) {
text := "Permission required\nAllow once Allow always Reject\n⇆ select enter confirm"
if got, ok := approvalResponse(text, "grant_approval"); !ok || got != "\n" {
t.Fatalf("grant response = %q, %v", got, ok)
if got, ok := approvalResponse(text, "grant_approval"); !ok || !reflect.DeepEqual(got.Keys, []string{"ENTER"}) || got.Text != "" {
t.Fatalf("grant response = %+v, %v", got, ok)
}
if got, ok := approvalResponse(text, "deny_approval"); ok || got != "" {
t.Fatalf("deny response = %q, %v; reject must not guess selector navigation", got, ok)
if got, ok := approvalResponse(text, "deny_approval"); ok || got.Text != "" || len(got.Keys) != 0 {
t.Fatalf("deny response = %+v, %v; reject must not guess selector navigation", got, ok)
}
}
+145 -91
View File
@@ -5,6 +5,7 @@ import (
"crypto/subtle"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"net"
@@ -92,6 +93,95 @@ func (a federatedAvailability) Available(h registry.Herdr) bool {
return a.workers.Available(h.ID)
}
func validateLocalMachine(rr registry.Registry, localMachine string) error {
machines := rr.Machines()
if len(machines) <= 1 {
return nil
}
if localMachine == "" {
return fmt.Errorf("ORCHESTRA_MACHINE_ID is required for a multi-machine registry; refusing unsafe remote-herdr coordination")
}
if _, ok := rr.Machine(localMachine); !ok {
return fmt.Errorf("ORCHESTRA_MACHINE_ID %q is not in the registry", localMachine)
}
return nil
}
type harnessCompletion struct {
store *store.Store
route func(domain.Event) error
token string
}
func (h harnessCompletion) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
if h.token != "" && r.Header.Get("Authorization") != "Bearer "+h.token {
http.Error(w, "unauthorized", http.StatusUnauthorized)
return
}
var p struct {
TaskID string `json:"task_id"`
Harness string `json:"harness"`
TranscriptPath string `json:"transcript_path"`
Report string `json:"report"`
}
if json.NewDecoder(r.Body).Decode(&p) != nil || p.TaskID == "" || p.Report == "" || p.TranscriptPath == "" {
http.Error(w, "task_id, transcript_path, and report are required", http.StatusBadRequest)
return
}
t, ok := h.store.Task(p.TaskID)
if !ok {
http.Error(w, "task not found", http.StatusNotFound)
return
}
var usage herdr.Usage
var err error
switch p.Harness {
case "codex":
usage, err = herdr.CodexUsage(p.TranscriptPath)
case "opencode":
usage, err = herdr.OpenCodeUsage(p.TranscriptPath)
case "", "claude":
usage, err = herdr.ClaudeUsage(p.TranscriptPath)
default:
http.Error(w, "unknown harness: "+p.Harness, http.StatusBadRequest)
return
}
if err != nil {
http.Error(w, "reading transcript: "+err.Error(), http.StatusBadRequest)
return
}
ref, err := h.store.PutArtifact([]byte(p.Report))
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
payload, _ := json.Marshal(map[string]any{"report_ref": ref, "receipt": map[string]any{
"input_tokens": usage.Input, "cache_read_tokens": usage.CacheRead,
"cache_write_tokens": usage.CacheWrite, "output_tokens": usage.Output, "numerator": usage.Numerator(),
}})
e := domain.Event{ID: id(), Type: "TaskCompleted", TaskID: p.TaskID, Version: t.Version + 1, Payload: payload, Surface: string(authz.System)}
if err := h.store.Append(e); err != nil {
http.Error(w, err.Error(), http.StatusConflict)
return
}
if h.route != nil {
if err := h.route(e); err != nil {
log.Printf("route task: %v", err)
}
}
if t.Lease != nil && t.Lease.HarnessID != "" {
qp, _ := json.Marshal(map[string]any{"harness_id": t.Lease.HarnessID, "consumed": float64(usage.Numerator())})
if err := h.store.Append(domain.Event{ID: id(), Type: "QuotaReported", TaskID: "system", Payload: qp, Surface: string(authz.System)}); err != nil {
log.Printf("quota report: %v", err)
}
}
json.NewEncoder(w).Encode(e)
}
func main() {
dir := os.Getenv("ORCHESTRA_DATA")
if dir == "" {
@@ -105,11 +195,17 @@ func main() {
var rt *router.Router
var coordinator *orchestrator.Coordinator
localMachine := os.Getenv("ORCHESTRA_MACHINE_ID")
workers := &federation.Registry{AdmitToken: os.Getenv("ORCHESTRA_FEDERATION_ADMIT_TOKEN")}
workers := &federation.Registry{AdmitToken: os.Getenv("ORCHESTRA_FEDERATION_ADMIT_TOKEN"), StatePath: filepath.Join(dir, "federation-state.json")}
if err := workers.Load(); err != nil {
log.Fatalf("load federation state: %v", err)
}
if config := os.Getenv("ORCHESTRA_CONFIG"); config != "" {
if rr, err = registry.Load(config); err != nil {
log.Fatalf("load orchestra config: %v", err)
}
if err := validateLocalMachine(rr, localMachine); err != nil {
log.Fatal(err)
}
reachability := registry.Reachability(registry.TCPReachability{})
if localMachine != "" {
reachability = federatedReachability{base: reachability, remote: remoteHerdrAddresses(rr, localMachine)}
@@ -169,7 +265,10 @@ func main() {
Projects: projectRepos,
Default: orchestrator.GitWorktrees{Root: root, Repo: repo},
}
coordinator = &orchestrator.Coordinator{Store: s, StatePath: filepath.Join(dir, "runtime-sessions.json"), Worktrees: worktrees, Adapters: orchestrator.AdapterFactory{Herdrs: adapters}}
coordinator = &orchestrator.Coordinator{Store: s, StatePath: filepath.Join(dir, "runtime-sessions.json"), Worktrees: worktrees, Adapters: orchestrator.AdapterFactory{Herdrs: adapters}, LocalHerdr: func(id string) bool {
h, ok := rr.Herdr(id)
return ok && (localMachine == "" || h.MachineID == localMachine)
}}
rt.OnLease = func(e domain.Event) error {
// In federated mode the coordinator must never inspect a remote
// checkout. Its worker consumes the router-issued lease event and
@@ -214,6 +313,17 @@ func main() {
// the token once for an HttpOnly cookie. Same credential, presentable
// form; no new authority is created here.
mux.HandleFunc("/v1/ui/session", func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodDelete {
// Expire the browser credential even if it is already absent or stale.
// The client never has access to the HttpOnly value, so this is the
// only reliable way for an operator to end a browser session.
if cookie, err := r.Cookie(authz.SessionCookie); err == nil {
sessions.Revoke(cookie.Value)
}
http.SetCookie(w, &http.Cookie{Name: authz.SessionCookie, Value: "", Path: "/", MaxAge: -1, HttpOnly: true, SameSite: http.SameSiteStrictMode, Secure: os.Getenv("ORCHESTRA_UI_INSECURE_COOKIE") == ""})
w.WriteHeader(http.StatusNoContent)
return
}
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
@@ -284,6 +394,13 @@ func main() {
return v
}
mux.HandleFunc("/v1/tasks", func(w http.ResponseWriter, r *http.Request) {
if wid := r.Header.Get("X-Orchestra-Worker"); wid != "" {
token := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
if err := workers.Authenticate(wid, token); err != nil {
http.Error(w, err.Error(), http.StatusUnauthorized)
return
}
}
if r.Method == "GET" {
json.NewEncoder(w).Encode(s.Tasks())
return
@@ -353,6 +470,13 @@ func main() {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
if wid := r.Header.Get("X-Orchestra-Worker"); wid != "" {
token := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
if err := workers.Authenticate(wid, token); err != nil {
http.Error(w, err.Error(), http.StatusUnauthorized)
return
}
}
// Reports and handoffs are content-addressed evidence. Keep uploads
// bounded because event payloads only carry their resulting hash.
r.Body = http.MaxBytesReader(w, r.Body, 4<<20)
@@ -407,94 +531,13 @@ func main() {
// same session-file assumption as CLIAdapter.Occupancy — rather than
// trusting a self-reported number.
harnessToken := os.Getenv("ORCHESTRA_HARNESS_TOKEN")
mux.HandleFunc("/v1/harness/complete", func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
mux.Handle("/v1/harness/complete", harnessCompletion{store: s, token: harnessToken, route: func(e domain.Event) error {
if rt == nil {
return nil
}
if harnessToken != "" && r.Header.Get("Authorization") != "Bearer "+harnessToken {
http.Error(w, "unauthorized", http.StatusUnauthorized)
return
}
var p struct {
TaskID string `json:"task_id"`
Harness string `json:"harness"`
TranscriptPath string `json:"transcript_path"`
Report string `json:"report"`
}
if json.NewDecoder(r.Body).Decode(&p) != nil || p.TaskID == "" || p.Report == "" || p.TranscriptPath == "" {
http.Error(w, "task_id, transcript_path, and report are required", http.StatusBadRequest)
return
}
t, ok := s.Task(p.TaskID)
if !ok {
http.Error(w, "task not found", http.StatusNotFound)
return
}
// Harness-specific session-state readers (AUDIT.md "Codex/opencode
// completion producers"). Same local-filesystem assumption B3 already
// made for Claude: the caller supplies the path to its own session
// state (transcript / rollout / message file), never a herdr pane id.
var usage herdr.Usage
var err error
switch p.Harness {
case "codex":
usage, err = herdr.CodexUsage(p.TranscriptPath)
case "opencode":
usage, err = herdr.OpenCodeUsage(p.TranscriptPath)
case "", "claude":
usage, err = herdr.ClaudeUsage(p.TranscriptPath)
default:
http.Error(w, "unknown harness: "+p.Harness, http.StatusBadRequest)
return
}
if err != nil {
http.Error(w, "reading transcript: "+err.Error(), http.StatusBadRequest)
return
}
ref, err := s.PutArtifact([]byte(p.Report))
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
payload, _ := json.Marshal(map[string]any{
"report_ref": ref,
"receipt": map[string]any{
"input_tokens": usage.Input,
"cache_read_tokens": usage.CacheRead,
"cache_write_tokens": usage.CacheWrite,
"output_tokens": usage.Output,
"numerator": usage.Numerator(),
},
})
e := domain.Event{ID: id(), Type: "TaskCompleted", TaskID: p.TaskID, Version: t.Version + 1, Payload: payload, Surface: string(authz.System)}
if err := s.Append(e); err != nil {
http.Error(w, err.Error(), http.StatusConflict)
return
}
if rt != nil {
if _, routeErr := rt.HandleEvent(e); routeErr != nil {
log.Printf("route task: %v", routeErr)
}
}
// B7 (AUDIT.md): QuotaReported has no other producer, so router's
// 5h/weekly availability filter and the brief's quota_consumed are
// permanently zero without this. Fed by the same per-harness usage
// read as the receipt above (spec §7.2 — "same per-harness session
// state as §5.2.1"); harness_id comes from the lease this completion
// closes out, before it's released.
if t.Lease != nil && t.Lease.HarnessID != "" {
qp, _ := json.Marshal(map[string]any{
"harness_id": t.Lease.HarnessID,
"consumed": float64(usage.Numerator()),
})
qe := domain.Event{ID: id(), Type: "QuotaReported", TaskID: "system", Payload: qp, Surface: string(authz.System)}
if err := s.Append(qe); err != nil {
log.Printf("quota report: %v", err)
}
}
json.NewEncoder(w).Encode(e)
})
_, err := rt.HandleEvent(e)
return err
}})
// /v1/harness/turn is the unified turn-decision endpoint (AUDIT.md Phase
// 2 items 1-2): the Face-B stop hook posts here on every ordinary turn
// boundary (report marker absent — /v1/harness/complete covers task
@@ -1009,7 +1052,12 @@ func main() {
return
}
if strings.HasSuffix(r.URL.Path, "/heartbeat") {
if err := workers.Heartbeat(parts[3]); err != nil {
var health federation.WorkerHealth
if err := json.NewDecoder(r.Body).Decode(&health); err != nil && !errors.Is(err, io.EOF) {
http.Error(w, "invalid worker health", 400)
return
}
if err := workers.Heartbeat(parts[3], health); err != nil {
http.Error(w, err.Error(), 404)
return
}
@@ -1049,7 +1097,13 @@ func main() {
http.Error(w, "task not found", 404)
return
}
if t.State != domain.StateLeased || t.Lease == nil || t.Lease.HarnessID != parts[3] {
ownedLease := t.State == domain.StateLeased && t.Lease != nil && t.Lease.HarnessID == parts[3]
// A prompt timeout can block the coordinator after herdr already
// accepted the request. If that same authenticated worker later reports
// a durable completion, reconcile it rather than preserving a known
// false blocked state. No other blocked task is admitted here.
recoverableBlocked := strings.HasSuffix(r.URL.Path, "/complete") && t.State == domain.StateBlocked && t.LastHarness == parts[3]
if !ownedLease && !recoverableBlocked {
http.Error(w, "lease not owned", 409)
return
}
+97
View File
@@ -1,12 +1,19 @@
package main
import (
"bytes"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"time"
"orchestra/internal/authz"
"orchestra/internal/domain"
"orchestra/internal/registry"
"orchestra/internal/store"
)
type unreachable struct{}
@@ -33,3 +40,93 @@ func TestFederatedReachabilityDefersRemoteHerdrToWorkerHeartbeat(t *testing.T) {
t.Fatal("local herdr should still require its TCP probe")
}
}
func TestHarnessCompletionBuildsReceiptAndQuotaFromTranscript(t *testing.T) {
s, err := store.Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
if err := s.Append(domain.Event{ID: domain.NewID(), Type: "TaskCreated", TaskID: "done", Surface: string(authz.System), Payload: []byte(`{"source":"qa","external_id":"done","project":"p"}`)}); err != nil {
t.Fatal(err)
}
if _, err := s.Lease("done", "local-claude", time.Minute); err != nil {
t.Fatal(err)
}
transcript := filepath.Join(t.TempDir(), "transcript.jsonl")
if err := os.WriteFile(transcript, []byte(`{"message":{"usage":{"input_tokens":100,"cache_read_input_tokens":20,"cache_creation_input_tokens":5,"output_tokens":7}}}`+"\n"), 0600); err != nil {
t.Fatal(err)
}
body, _ := json.Marshal(map[string]string{"task_id": "done", "transcript_path": transcript, "report": "# done"})
req := httptest.NewRequest(http.MethodPost, "/v1/harness/complete", bytes.NewReader(body))
res := httptest.NewRecorder()
harnessCompletion{store: s, token: "secret"}.ServeHTTP(res, req)
if res.Code != http.StatusUnauthorized {
t.Fatalf("missing token status = %d, want 401", res.Code)
}
req.Header.Set("Authorization", "Bearer secret")
req = httptest.NewRequest(http.MethodPost, "/v1/harness/complete", bytes.NewReader(body))
req.Header.Set("Authorization", "Bearer secret")
res = httptest.NewRecorder()
harnessCompletion{store: s, token: "secret"}.ServeHTTP(res, req)
if res.Code != http.StatusOK {
t.Fatalf("completion status = %d: %s", res.Code, res.Body.String())
}
task, ok := s.Task("done")
if !ok || task.State != domain.StateCompleted {
t.Fatalf("task after completion = %#v, present=%v", task, ok)
}
var completed struct {
Receipt struct {
Input int `json:"input_tokens"`
CacheRead int `json:"cache_read_tokens"`
CacheWrite int `json:"cache_write_tokens"`
Output int `json:"output_tokens"`
} `json:"receipt"`
}
for _, e := range s.Events(0) {
if e.Type == "TaskCompleted" {
if err := json.Unmarshal(e.Payload, &completed); err != nil {
t.Fatal(err)
}
}
}
if completed.Receipt.Input != 100 || completed.Receipt.CacheRead != 20 || completed.Receipt.CacheWrite != 5 || completed.Receipt.Output != 7 {
t.Fatalf("receipt = %#v", completed.Receipt)
}
var quota struct {
Harness string `json:"harness_id"`
Consumed float64 `json:"consumed"`
}
found := false
for _, e := range s.Events(0) {
if e.Type == "QuotaReported" {
_ = json.Unmarshal(e.Payload, &quota)
found = true
}
}
if !found || quota.Harness != "local-claude" || quota.Consumed != 125 {
t.Fatalf("quota report = %#v, found=%v", quota, found)
}
}
func TestMultiMachineRegistryRequiresKnownLocalMachine(t *testing.T) {
path := filepath.Join(t.TempDir(), "config.json")
if err := os.WriteFile(path, []byte(`{
"machines":[{"id":"homesrv","address":"192.168.1.104:9145"},{"id":"workpc","address":"192.168.1.105:9145"}]
}`), 0o600); err != nil {
t.Fatal(err)
}
r, err := registry.Load(path)
if err != nil {
t.Fatal(err)
}
if err := validateLocalMachine(r, ""); err == nil {
t.Fatal("missing local machine accepted")
}
if err := validateLocalMachine(r, "missing"); err == nil {
t.Fatal("unknown local machine accepted")
}
if err := validateLocalMachine(r, "homesrv"); err != nil {
t.Fatalf("known local machine rejected: %v", err)
}
}