Harden worker federation and operator UI

cmd/orchestra-worker/main.go

@@ -34,7 +34,43 @@ type worker struct {
 	statePath                              string
 	hard                                   float64
 	registration                           federation.Worker
+	lastError                              string
+	lastErrorAt                            time.Time
 }
+
+func (w *worker) recordError(err error) {
+	if err == nil {
+		return
+	}
+	w.lastError = err.Error()
+	w.lastErrorAt = time.Now().UTC()
+}
+
+func (w *worker) health(ctx context.Context) federation.WorkerHealth {
+	h := federation.WorkerHealth{HerdrStatus: "unknown"}
+	for taskID, session := range w.sessions {
+		// Workers currently advertise capacity one. Pick deterministically so a
+		// recovered legacy state with more sessions remains intelligible.
+		if h.ActiveTask == "" || taskID < h.ActiveTask {
+			h.ActiveTask, h.ActivePane = taskID, session.PaneID
+		}
+	}
+	if w.herdr != nil {
+		checkCtx, cancel := context.WithTimeout(ctx, 3*time.Second)
+		err := w.herdr.CheckProtocol(checkCtx, "17")
+		cancel()
+		h.CheckedAt = time.Now().UTC()
+		if err == nil {
+			h.HerdrStatus = "reachable"
+		} else {
+			h.HerdrStatus = "unreachable"
+			w.recordError(fmt.Errorf("local herdr: %w", err))
+		}
+	}
+	h.LastError, h.ErrorAt = w.lastError, w.lastErrorAt
+	return h
+}
+
 type lease struct {
 	HandoffRef string `json:"handoff_ref,omitempty"`
 }
@@ -248,22 +284,29 @@ func (w *worker) publishCaptures(ctx context.Context) {
 	}
 }
 
-func approvalResponse(text, kind string) (string, bool) {
+type approvalInput struct {
+	Text string
+	Keys []string
+}
+
+func approvalResponse(text, kind string) (approvalInput, bool) {
 	low := strings.ToLower(text)
 	// Never invent a keystroke. y/n prompts label both decisions directly.
 	if strings.Contains(low, "[y/n]") || strings.Contains(low, "(y/n)") {
 		if kind == "grant_approval" {
-			return "y\n", true
+			return approvalInput{Text: "y\n"}, true
 		}
-		return "n\n", true
+		return approvalInput{Text: "n\n"}, true
 	}
 	// OpenCode's explicit selector states "Allow once  Allow always  Reject"
-	// and "enter confirm". Enter is consequently a bounded one-time grant;
+	// and "enter confirm". Send a real ENTER key, not a newline through
+	// pane.send_text: OpenCode's selector does not treat the latter as input.
+	// Enter is consequently a bounded one-time grant;
 	// rejection would require unobservable selector navigation, so refuse it.
 	if kind == "grant_approval" && strings.Contains(low, "allow once") && strings.Contains(low, "allow always") && strings.Contains(low, "reject") && strings.Contains(low, "enter confirm") {
-		return "\n", true
+		return approvalInput{Keys: []string{"ENTER"}}, true
 	}
-	return "", false
+	return approvalInput{}, false
 }
 func (w *worker) runCommands(ctx context.Context) {
 	commands, err := w.api.Commands(ctx)
@@ -296,7 +339,11 @@ func (w *worker) runCommands(ctx context.Context) {
 			_ = w.api.ResolveCommand(ctx, command.ID, "rejected", "prompt does not expose an executable approval control")
 			continue
 		}
-		if err := w.herdr.Call(ctx, "pane.send_text", map[string]any{"pane_id": session.PaneID, "text": input}, nil); err != nil {
+		method, params := "pane.send_text", map[string]any{"pane_id": session.PaneID, "text": input.Text}
+		if len(input.Keys) > 0 {
+			method, params = "pane.send_keys", map[string]any{"pane_id": session.PaneID, "keys": input.Keys}
+		}
+		if err := w.herdr.Call(ctx, method, params, nil); err != nil {
 			_ = w.api.ResolveCommand(ctx, command.ID, "rejected", "herdr did not acknowledge input: "+err.Error())
 			continue
 		}
@@ -450,13 +497,15 @@ func main() {
 	ticker := time.NewTicker(5 * time.Second)
 	defer ticker.Stop()
 	for {
-		if err := w.api.Heartbeat(ctx); err != nil {
+		if err := w.api.Heartbeat(ctx, w.health(ctx)); err != nil {
+			w.recordError(fmt.Errorf("heartbeat: %w", err))
 			log.Printf("heartbeat: %v", err)
 			if w.reRegisterAfterCoordinatorRestart(ctx, err) {
 				continue
 			}
 		}
 		if err := w.once(ctx); err != nil {
+			w.recordError(fmt.Errorf("poll: %w", err))
 			log.Printf("poll: %v", err)
 			w.reRegisterAfterCoordinatorRestart(ctx, err)
 		}

cmd/orchestra-worker/main_test.go

@@ -14,6 +14,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"reflect"
 	"testing"
 	"time"
 )
@@ -248,11 +249,11 @@ func TestWorkerApprovalCommandIsRevisionBoundAndAcknowledged(t *testing.T) {
 
 func TestApprovalResponseOpenCodeAllowOnce(t *testing.T) {
 	text := "Permission required\nAllow once  Allow always  Reject\n⇆ select  enter confirm"
-	if got, ok := approvalResponse(text, "grant_approval"); !ok || got != "\n" {
-		t.Fatalf("grant response = %q, %v", got, ok)
+	if got, ok := approvalResponse(text, "grant_approval"); !ok || !reflect.DeepEqual(got.Keys, []string{"ENTER"}) || got.Text != "" {
+		t.Fatalf("grant response = %+v, %v", got, ok)
 	}
-	if got, ok := approvalResponse(text, "deny_approval"); ok || got != "" {
-		t.Fatalf("deny response = %q, %v; reject must not guess selector navigation", got, ok)
+	if got, ok := approvalResponse(text, "deny_approval"); ok || got.Text != "" || len(got.Keys) != 0 {
+		t.Fatalf("deny response = %+v, %v; reject must not guess selector navigation", got, ok)
 	}
 }