// orchestra-worker consumes router-issued leases for one local herdr. Homesrv // remains the scheduler and CAS authority; this process owns only local Git // and pane operations. package main import ( "context" "encoding/json" "fmt" "log" "orchestra/internal/continuity" "orchestra/internal/domain" "orchestra/internal/federation" "orchestra/internal/herdr" "orchestra/internal/orchestrator" "os" "os/exec" "os/signal" "path/filepath" "strconv" "strings" "syscall" "time" ) type worker struct { api federation.Client herdr *herdr.Client harnessID, harness, repo, root, remote string cursor uint64 tasks map[string]domain.Task sessions map[string]herdr.Session leases map[string]lease statePath string hard float64 registration federation.Worker lastError string lastErrorAt time.Time } func (w *worker) recordError(err error) { if err == nil { return } w.lastError = err.Error() w.lastErrorAt = time.Now().UTC() } func (w *worker) health(ctx context.Context) federation.WorkerHealth { h := federation.WorkerHealth{HerdrStatus: "unknown"} for taskID, session := range w.sessions { // Workers currently advertise capacity one. Pick deterministically so a // recovered legacy state with more sessions remains intelligible. if h.ActiveTask == "" || taskID < h.ActiveTask { h.ActiveTask, h.ActivePane = taskID, session.PaneID } } if w.herdr != nil { checkCtx, cancel := context.WithTimeout(ctx, 3*time.Second) err := w.herdr.CheckProtocol(checkCtx, "17") cancel() h.CheckedAt = time.Now().UTC() if err == nil { h.HerdrStatus = "reachable" } else { h.HerdrStatus = "unreachable" w.recordError(fmt.Errorf("local herdr: %w", err)) } } h.LastError, h.ErrorAt = w.lastError, w.lastErrorAt return h } type lease struct { HandoffRef string `json:"handoff_ref,omitempty"` } type workerState struct { Cursor uint64 `json:"cursor"` Sessions map[string]herdr.Session `json:"sessions"` Tasks map[string]domain.Task `json:"tasks"` Leases map[string]lease `json:"leases"` } func (w *worker) load() { b, e := os.ReadFile(w.statePath) if e == nil { var s workerState if json.Unmarshal(b, &s) == nil { w.cursor = s.Cursor w.sessions = s.Sessions w.tasks = s.Tasks w.leases = s.Leases } } if w.sessions == nil { w.sessions = map[string]herdr.Session{} } if w.tasks == nil { w.tasks = map[string]domain.Task{} } if w.leases == nil { w.leases = map[string]lease{} } } func (w *worker) save() error { b, e := json.Marshal(workerState{Cursor: w.cursor, Sessions: w.sessions, Tasks: w.tasks, Leases: w.leases}) if e != nil { return e } if e := os.MkdirAll(filepath.Dir(w.statePath), 0700); e != nil { return e } return os.WriteFile(w.statePath, b, 0600) } type artifactCAS struct{ api federation.Client } func (c artifactCAS) PutArtifact(b []byte) (string, error) { return c.api.PutArtifact(context.Background(), b) } func (c artifactCAS) Artifact(ref string) ([]byte, error) { return c.api.Artifact(context.Background(), ref) } func created(e domain.Event) (domain.Task, bool) { if e.Type != "TaskCreated" { return domain.Task{}, false } var p struct { Source string `json:"source"` ExternalID string `json:"external_id"` Project string `json:"project"` Capability []string `json:"capability"` Title string `json:"title"` Description string `json:"description"` } if json.Unmarshal(e.Payload, &p) != nil || p.Source == "" || p.ExternalID == "" || p.Project == "" { return domain.Task{}, false } return domain.Task{ID: e.TaskID, Source: p.Source, ExternalID: p.ExternalID, Project: p.Project, Capability: p.Capability, Title: p.Title, Description: p.Description}, true } func (w *worker) syncBase(ctx context.Context) error { if out, err := exec.CommandContext(ctx, "git", "-C", w.repo, "fetch", w.remote, "--prune").CombinedOutput(); err != nil { return fmt.Errorf("fetch base checkout: %s: %w", out, err) } branch, err := exec.CommandContext(ctx, "git", "-C", w.repo, "symbolic-ref", "--quiet", "--short", "HEAD").Output() if err != nil { return fmt.Errorf("identify base branch: %w", err) } branchName := strings.TrimSpace(string(branch)) if out, err := exec.CommandContext(ctx, "git", "-C", w.repo, "merge", "--ff-only", w.remote+"/"+branchName).CombinedOutput(); err != nil { return fmt.Errorf("fast-forward base checkout: %s: %w", out, err) } return nil } func (w *worker) start(ctx context.Context, t domain.Task, ref string) error { var wt string var h continuity.Handoff var err error // Synchronize the local base before any worktree operation. A worker never // treats a coordinator-side path as truth; the Git remote is the only // cross-machine transport. if err := w.syncBase(ctx); err != nil { return err } if ref != "" { b, err := w.api.Artifact(ctx, ref) if err != nil { return err } h, err = continuity.Decode(b) if err != nil { return err } if out, err := exec.CommandContext(ctx, "git", "-C", w.repo, "fetch", w.remote, "--prune").CombinedOutput(); err != nil { return fmt.Errorf("fetch pickup anchor: %s: %w", out, err) } wt = filepath.Join(w.root, t.ID) if _, err := os.Stat(wt); os.IsNotExist(err) { if out, err := exec.CommandContext(ctx, "git", "-C", w.repo, "worktree", "add", "-b", "orchestra/"+t.ID, wt, h.Anchor.GitSHA).CombinedOutput(); err != nil { return fmt.Errorf("create pickup worktree: %s: %w", out, err) } } if err = continuity.ValidatePickup(wt, h, taskHash(t)); err != nil { return err } } else { wt, err = (orchestrator.GitWorktrees{Repo: w.repo, Root: w.root}).Create(ctx, t) if err != nil { return err } } if _, err = w.herdr.Worktree(ctx, w.repo, wt, "orchestra/"+t.ID); err != nil { return err } s, err := w.herdr.StartAgent(ctx, wt, wt, "orchestra/"+t.ID, w.harness, t.ID) if err != nil { return err } p := "Begin Orchestra task " + t.ID + ".\nTitle: " + t.Title + "\nInstructions:\n" + t.Description + "\nWork only in this worktree. Do not edit TASK.md." if ref != "" { p += "\nA validated handoff exists. Read TASK.md and inspect local git history before continuing." } w.sessions[t.ID] = s if err := w.save(); err != nil { return err } // A prompt response can be lost after herdr accepted it. Persist the // session first so the worker can reconcile/release it after restart. return w.herdr.Prompt(ctx, s.PaneID, p, 0) } func taskHash(t domain.Task) string { b := continuity.RenderTaskFile(t); return domain.Hash(b) } func (w *worker) releaseReady(ctx context.Context) { for id, s := range w.sessions { if report, err := os.ReadFile(filepath.Join(s.Worktree, ".orchestra-report.md")); err == nil && len(report) > 0 { if ref, err := w.api.PutArtifact(ctx, report); err == nil { if err = w.api.Complete(ctx, id, ref); err == nil { delete(w.sessions, id) delete(w.leases, id) _ = w.save() continue } log.Printf("complete %s: %v", id, err) } else { log.Printf("upload completion %s: %v", id, err) } } if _, err := os.Stat(filepath.Join(s.Worktree, herdr.HandoffReportFile)); err != nil { if !s.HandoffRequested { a := herdr.CLIAdapter{Client: w.herdr, Harness: w.harness} if occ, occErr := a.Occupancy(s); occErr == nil && occ >= w.hard { if boundary, boundaryErr := a.AtTurnBoundary(ctx, s); boundaryErr == nil && boundary { if err := a.RequestHandoff(ctx, s); err == nil { s.HandoffRequested, s.HandoffReason = true, "threshold" w.sessions[id] = s _ = w.save() } } } } continue } a := herdr.CLIAdapter{Client: w.herdr, Harness: w.harness, CAS: artifactCAS{w.api}, Remote: w.remote} ref, err := a.Release(ctx, s) if err != nil { log.Printf("release %s: %v", id, err) continue } sha, err := herdr.HeadSHA(s.Worktree) if err != nil { log.Printf("release %s anchor: %v", id, err) continue } if err = w.api.Release(ctx, id, ref, sha); err != nil { log.Printf("publish release %s: %v", id, err) continue } // release_agent only removes herdr's binding. Closing the released pane // after the handoff is durable prevents the next StartAgent from // inheriting the predecessor's still-running terminal process. if err := a.Kill(ctx, s); err != nil { log.Printf("close released pane %s: %v", id, err) } delete(w.sessions, id) _ = w.save() } } // publishCaptures makes remote panes observable without allowing the // coordinator to touch their unix herdr socket. func (w *worker) publishCaptures(ctx context.Context) { for taskID, session := range w.sessions { text, err := (herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}).PaneCapture(ctx, session, "recent") if err != nil { continue } if _, err := w.api.PublishCapture(ctx, federation.Capture{TaskID: taskID, PaneID: session.PaneID, Text: text}); err != nil { log.Printf("publish capture %s: %v", taskID, err) } } } type approvalInput struct { Text string Keys []string } func approvalResponse(text, kind string) (approvalInput, bool) { low := strings.ToLower(text) // Never invent a keystroke. y/n prompts label both decisions directly. if strings.Contains(low, "[y/n]") || strings.Contains(low, "(y/n)") { if kind == "grant_approval" { return approvalInput{Text: "y\n"}, true } return approvalInput{Text: "n\n"}, true } // OpenCode's explicit selector states "Allow once Allow always Reject" // and "enter confirm". Send a real ENTER key, not a newline through // pane.send_text: OpenCode's selector does not treat the latter as input. // Enter is consequently a bounded one-time grant; // rejection would require unobservable selector navigation, so refuse it. if kind == "grant_approval" && strings.Contains(low, "allow once") && strings.Contains(low, "allow always") && strings.Contains(low, "reject") && strings.Contains(low, "enter confirm") { return approvalInput{Keys: []string{"ENTER"}}, true } return approvalInput{}, false } func (w *worker) runCommands(ctx context.Context) { commands, err := w.api.Commands(ctx) if err != nil { log.Printf("poll controls: %v", err) return } for _, command := range commands { session, ok := w.sessions[command.TaskID] if !ok || session.PaneID != command.PaneID { _ = w.api.ResolveCommand(ctx, command.ID, "stale", "session or pane changed") continue } text, err := (herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}).PaneCapture(ctx, session, "recent") if err != nil { _ = w.api.ResolveCommand(ctx, command.ID, "rejected", "capture unavailable: "+err.Error()) continue } capture, err := w.api.PublishCapture(ctx, federation.Capture{TaskID: command.TaskID, PaneID: session.PaneID, Text: text}) if err != nil { _ = w.api.ResolveCommand(ctx, command.ID, "rejected", "cannot publish capture: "+err.Error()) continue } if capture.Revision != command.CaptureRevision { _ = w.api.ResolveCommand(ctx, command.ID, "stale", "capture revision changed") continue } input, ok := approvalResponse(text, command.Kind) if !ok { _ = w.api.ResolveCommand(ctx, command.ID, "rejected", "prompt does not expose an executable approval control") continue } method, params := "pane.send_text", map[string]any{"pane_id": session.PaneID, "text": input.Text} if len(input.Keys) > 0 { method, params = "pane.send_keys", map[string]any{"pane_id": session.PaneID, "keys": input.Keys} } if err := w.herdr.Call(ctx, method, params, nil); err != nil { _ = w.api.ResolveCommand(ctx, command.ID, "rejected", "herdr did not acknowledge input: "+err.Error()) continue } if err := w.api.ResolveCommand(ctx, command.ID, "acknowledged", ""); err != nil { log.Printf("ack command %s: %v", command.ID, err) } } } func (w *worker) once(ctx context.Context) error { es, _, err := w.api.Events(ctx, w.cursor) if err != nil { return err } for _, e := range es { if t, ok := created(e); ok { w.tasks[t.ID] = t } if e.Type == "TaskLeased" { var p struct { HarnessID string `json:"harness_id"` HandoffRef string `json:"handoff_ref"` } if json.Unmarshal(e.Payload, &p) == nil && p.HarnessID == w.harnessID { w.leases[e.TaskID] = lease{HandoffRef: p.HandoffRef} } } if e.Type == "TaskReleased" || e.Type == "TaskCompleted" || e.Type == "TaskFailed" || e.Type == "TaskBlocked" { delete(w.leases, e.TaskID) delete(w.sessions, e.TaskID) } if e.Seq > w.cursor { w.cursor = e.Seq } } // A coordinator restart can restore its task snapshot without retaining // the in-memory event tail. In that state a worker with a persisted cursor // receives an empty page even though a lease is currently assigned to it. // Reconcile the authoritative projection before treating an empty page as // "nothing to do"; otherwise the lease remains invisible until expiry. if len(es) == 0 { if err := w.reconcileLeases(ctx); err != nil { return err } } // State is only a cache. If a lease survived but its TaskCreated event is // older than the worker's cursor (or the event has been compacted), hydrate // the authoritative task projection before deciding whether to start. for taskID := range w.leases { if _, ok := w.tasks[taskID]; !ok { tasks, err := w.api.Tasks(ctx) if err != nil { return fmt.Errorf("hydrate leased task %s: %w", taskID, err) } for _, task := range tasks { w.tasks[task.ID] = task } break } } // Project the whole batch before launching. This prevents a new worker // from resurrecting every historical lease during its initial replay. for taskID, l := range w.leases { if _, started := w.sessions[taskID]; started { continue } if t, ok := w.tasks[taskID]; ok { if err := w.start(ctx, t, l.HandoffRef); err != nil { log.Printf("lease %s: %v", t.ID, err) } } } // Unit/replay-only workers intentionally have no herdr connection. A // production worker always does, and only then participates in the live // capture/control protocol. if w.herdr != nil { w.publishCaptures(ctx) w.runCommands(ctx) } w.releaseReady(ctx) if err := w.save(); err != nil { return err } return w.api.Ack(ctx, w.cursor) } func (w *worker) reconcileLeases(ctx context.Context) error { tasks, err := w.api.Tasks(ctx) if err != nil { return fmt.Errorf("reconcile leased tasks: %w", err) } active := make(map[string]lease) for _, task := range tasks { w.tasks[task.ID] = task if task.State == domain.StateLeased && task.Lease != nil && task.Lease.HarnessID == w.harnessID { active[task.ID] = lease{HandoffRef: task.HandoffRef} } } for taskID := range w.leases { if _, ok := active[taskID]; !ok { delete(w.leases, taskID) } } for taskID, l := range active { w.leases[taskID] = l } return nil } // reRegisterAfterCoordinatorRestart restores the coordinator's in-memory // worker registry. A worker must survive a server restart without operator // intervention; its persisted cursor and sessions remain valid. func (w *worker) reRegisterAfterCoordinatorRestart(ctx context.Context, cause error) bool { if cause == nil || !strings.Contains(cause.Error(), "401 Unauthorized: unknown worker") { return false } if err := w.api.Register(ctx, w.registration); err != nil { log.Printf("re-register: %v", err) return false } log.Printf("re-registered after coordinator restart") return true } func required(k string) string { v := os.Getenv(k) if v == "" { log.Fatalf("%s required", k) } return v } func main() { id, token := required("ORCHESTRA_WORKER_ID"), required("ORCHESTRA_WORKER_TOKEN") hard := .75 if v, err := strconv.ParseFloat(os.Getenv("ORCHESTRA_OCCUPANCY_HARD"), 64); err == nil && v > 0 && v < 1 { hard = v } w := &worker{api: federation.Client{BaseURL: required("ORCHESTRA_URL"), WorkerID: id, Token: token, AdmitToken: os.Getenv("ORCHESTRA_FEDERATION_ADMIT_TOKEN")}, harnessID: required("ORCHESTRA_WORKER_HERDR_ID"), harness: required("ORCHESTRA_WORKER_HARNESS"), repo: required("ORCHESTRA_REPO"), root: required("ORCHESTRA_WORKTREE_ROOT"), remote: required("ORCHESTRA_GIT_REMOTE"), tasks: map[string]domain.Task{}, sessions: map[string]herdr.Session{}, leases: map[string]lease{}, statePath: os.Getenv("ORCHESTRA_WORKER_STATE"), hard: hard, registration: federation.Worker{ID: id, Address: os.Getenv("ORCHESTRA_WORKER_ADDRESS"), Capacity: 1}} if w.statePath == "" { w.statePath = filepath.Join(w.root, ".orchestra-worker-state.json") } w.load() w.herdr = herdr.New(required("ORCHESTRA_WORKER_HERDR")) if id != w.harnessID { log.Fatal("ORCHESTRA_WORKER_ID must equal ORCHESTRA_WORKER_HERDR_ID so leases and offline recovery have one owner") } if err := w.api.Register(context.Background(), w.registration); err != nil { log.Fatal(err) } ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() ticker := time.NewTicker(5 * time.Second) defer ticker.Stop() for { if err := w.api.Heartbeat(ctx, w.health(ctx)); err != nil { w.recordError(fmt.Errorf("heartbeat: %w", err)) log.Printf("heartbeat: %v", err) if w.reRegisterAfterCoordinatorRestart(ctx, err) { continue } } if err := w.once(ctx); err != nil { w.recordError(fmt.Errorf("poll: %w", err)) log.Printf("poll: %v", err) w.reRegisterAfterCoordinatorRestart(ctx, err) } select { case <-ctx.Done(): return case <-ticker.C: } } }