diff --git a/.gitignore b/.gitignore index 73e5dd6..7a096e2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,12 @@ # Private, composed deployment configuration. Install into /etc/orchestra/ # only after all environment-specific values have been filled in. .orchestra-config/ -clients/ + +# Built binaries (never commit — a stale committed binary is a deployment- +# confusion hazard). `clients/` is deliberately NOT ignored: it holds the +# still-deployed Design A herdr bridge, and deployed code must be tracked. /orchestra +/orchestra-worker # Web UI build inputs/outputs. node_modules in particular ships vendored Go # packages (e.g. flatted/golang), so leaving it merely untracked is not diff --git a/AGENTS.md b/AGENTS.md index fb53ea8..438087c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,10 +14,10 @@ continuity,federation,delivery,authz,operations,admin}` + `cmd/orchestra/main.go This repo has a documented history of code that *looks* wired but isn't — packages with tests that pass in isolation while the live call path silently no-ops (bare `continue` on error, discarded return values). See `AUDIT.md` -for the full audit and `progress.md` for a running log. **Before trusting a -claim in progress.md that something "works" or "is fixed," check the actual -call site** — the file is written by past sessions of this same assistant and -has previously overstated completion. +for the full audit; it is also the running log (there is no separate log +file). **Before trusting a claim in `AUDIT.md` that something "works" +or "is fixed," check the actual call site** — the file is written by past +sessions of this same assistant and has previously overstated completion. The single most reliable way to verify herdr-adapter code is right: don't read `internal/herdr/adapter.go` and assume the method names are real. Ping @@ -80,20 +80,26 @@ the live herdr instance and check. unstick an already-orphaned pane; that needs a manual kill/restart once the release path is trustworthy. -## The federation fork — read before touching anything cross-machine +## Federation — Design B is the live design (as of 2026-07-30) -Two incompatible designs coexist. **Design A** ("drive the remote socket", -currently deployed via `clients/herdr-bridge.go`) has homesrv call -`worktree.create`/`agent.start` etc. directly on workpc's herdr over TCP as -if it were local — meaning anchor validation (`git rev-parse HEAD`) run by -the coordinator executes on the *wrong machine* relative to the actual -checkout. **Design B** ("workers pull tasks", `/v1/federation/*`) is fully -built server-side but has zero clients — no worker binary exists. Decision -(AUDIT.md, 2026-07-27): keep Design A through Phase 5, commit to Design B in -Phase 6, with two guardrails landed immediately (refuse to rotate/cleanup a -lease held by a non-local herdr rather than validate against the wrong -checkout). Don't build on top of Design A's cross-machine calls without -reading that section first. +**Design B** ("workers pull tasks", `/v1/federation/*`) is the live design and +has a real client: `cmd/orchestra-worker/main.go` (~1,131 lines, with tests in +`cmd/orchestra-worker/main_test.go`) is the deployed worker — the workpc +OpenCode worker runs it. Build new cross-machine work on Design B. + +The **Design A guardrail has landed**: `Coordinator.adapterFor` +(`internal/orchestrator/orchestrator.go`, the `LocalHerdr` check) refuses to +resolve an adapter for a session owned by a non-local herdr, returning +`session %s is owned by non-local herdr %s` instead of validating a git anchor +(`git rev-parse HEAD`) against the wrong machine's checkout. Rotation/cleanup +therefore no longer act on remote leases. + +**Design A is retained, not dead.** `clients/herdr-bridge.go` ("drive the +remote socket": homesrv calling `worktree.create`/`agent.start` directly on +workpc's herdr over TCP as if it were local) is kept through Phase 5 per the +standing AUDIT.md decision of 2026-07-27; deletion of Design A and `clients/` +is deferred to the Phase 6 cutover. Do not delete it early, and do not add new +cross-machine call paths to it. ## Working conventions diff --git a/AUDIT.md b/AUDIT.md index 5e367ef..7b040b1 100644 --- a/AUDIT.md +++ b/AUDIT.md @@ -22,6 +22,25 @@ QA matrix has passed for all three harnesses. restarted at 2026-07-30 16:07 +04, has emitted no federation failures since, and its configured Unix-socket herdr answered `ping` with protocol `17`. +## Federation design status (updated 2026-07-30) + +Design B ("workers pull tasks", `/v1/federation/*`) is the **live** design. It +is no longer clientless: `cmd/orchestra-worker/main.go` (~1,131 lines, tests in +`cmd/orchestra-worker/main_test.go`) is the deployed worker, and the workpc +OpenCode worker runs it. Any earlier statement here or in `CLAUDE.md` that +Design B "has zero clients — no worker binary exists" is obsolete. + +The Design A guardrail from the 2026-07-27 decision has landed: +`Coordinator.adapterFor` (`internal/orchestrator/orchestrator.go`) refuses to +resolve an adapter for a session owned by a non-local herdr, so rotation and +cleanup can no longer validate a git anchor against the wrong machine's +checkout. + +Design A itself is **retained through Phase 5** per that same decision. +`clients/herdr-bridge.go` is now tracked in git (deployed code must be in +version control); deleting Design A and `clients/` is deferred to the Phase 6 +cutover. + ## Remaining release blockers - **Only OpenCode capacity is ready.** Workpc's `workpc-opencode` worker has diff --git a/CLAUDE.md b/CLAUDE.md index ad4f807..167209f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -14,10 +14,10 @@ continuity,federation,delivery,authz,operations,admin}` + `cmd/orchestra/main.go This repo has a documented history of code that *looks* wired but isn't — packages with tests that pass in isolation while the live call path silently no-ops (bare `continue` on error, discarded return values). See `AUDIT.md` -for the full audit and `progress.md` for a running log. **Before trusting a -claim in progress.md that something "works" or "is fixed," check the actual -call site** — the file is written by past sessions of this same assistant and -has previously overstated completion. +for the full audit; it is also the running log (there is no separate log +file). **Before trusting a claim in `AUDIT.md` that something "works" +or "is fixed," check the actual call site** — the file is written by past +sessions of this same assistant and has previously overstated completion. The single most reliable way to verify herdr-adapter code is right: don't read `internal/herdr/adapter.go` and assume the method names are real. Ping @@ -93,20 +93,26 @@ the live herdr instance and check. unstick an already-orphaned pane; that needs a manual kill/restart once the release path is trustworthy. -## The federation fork — read before touching anything cross-machine +## Federation — Design B is the live design (as of 2026-07-30) -Two incompatible designs coexist. **Design A** ("drive the remote socket", -currently deployed via `clients/herdr-bridge.go`) has homesrv call -`worktree.create`/`agent.start` etc. directly on workpc's herdr over TCP as -if it were local — meaning anchor validation (`git rev-parse HEAD`) run by -the coordinator executes on the *wrong machine* relative to the actual -checkout. **Design B** ("workers pull tasks", `/v1/federation/*`) is fully -built server-side but has zero clients — no worker binary exists. Decision -(AUDIT.md, 2026-07-27): keep Design A through Phase 5, commit to Design B in -Phase 6, with two guardrails landed immediately (refuse to rotate/cleanup a -lease held by a non-local herdr rather than validate against the wrong -checkout). Don't build on top of Design A's cross-machine calls without -reading that section first. +**Design B** ("workers pull tasks", `/v1/federation/*`) is the live design and +has a real client: `cmd/orchestra-worker/main.go` (~1,131 lines, with tests in +`cmd/orchestra-worker/main_test.go`) is the deployed worker — the workpc +OpenCode worker runs it. Build new cross-machine work on Design B. + +The **Design A guardrail has landed**: `Coordinator.adapterFor` +(`internal/orchestrator/orchestrator.go`, the `LocalHerdr` check) refuses to +resolve an adapter for a session owned by a non-local herdr, returning +`session %s is owned by non-local herdr %s` instead of validating a git anchor +(`git rev-parse HEAD`) against the wrong machine's checkout. Rotation/cleanup +therefore no longer act on remote leases. + +**Design A is retained, not dead.** `clients/herdr-bridge.go` ("drive the +remote socket": homesrv calling `worktree.create`/`agent.start` directly on +workpc's herdr over TCP as if it were local) is kept through Phase 5 per the +standing AUDIT.md decision of 2026-07-27; deletion of Design A and `clients/` +is deferred to the Phase 6 cutover. Do not delete it early, and do not add new +cross-machine call paths to it. ## Working conventions diff --git a/clients/README.md b/clients/README.md new file mode 100644 index 0000000..80ed090 --- /dev/null +++ b/clients/README.md @@ -0,0 +1,32 @@ +# workpc herdr TCP bridges + +These files are intentionally local and gitignored. The workpc herdr exposes +one local Unix socket; Orchestra reaches it over the LAN through three bound +TCP bridges: + +| machine | listen | herdr socket | +|---|---|---| +| workpc | 192.168.1.105:9245 | ~/.config/herdr/herdr.sock | + +The bridge is a byte-preserving proxy for the newline-delimited JSON protocol. +It does not interpret or authenticate requests; restrict the ports with the +LAN firewall/WireGuard policy. + +## Not a federation worker + +`herdr-bridge` deliberately does **not** own a checkout. It cannot write +`TASK.md`, inspect Git `HEAD`/branch/dirty hashes, make a scratch commit, or +publish a canonical continuity handoff. Treat it as Design A compatibility +only, not the cross-machine protocol described by Orchestra's spec. + +`cmd/orchestra-worker` is the worker-side client. It registers and +heartbeats as the configured herdr id, consumes only router-issued +`TaskLeased` events for that id, creates/validates the local checkout, drives +the local herdr, and seals/pushes a canonical handoff locally before calling +the federation handoff endpoint. It deliberately has no task-claim operation: +homesrv's router is the only scheduler. + +Install `deploy/orchestra-worker.service` on the worker host and configure +the federation variables in `deploy/orchestra.env.example`. Set +`ORCHESTRA_MACHINE_ID` on homesrv; that prevents its coordinator from ever +opening, validating, rotating, or cleaning up a workpc checkout. diff --git a/clients/herdr-bridge.go b/clients/herdr-bridge.go new file mode 100644 index 0000000..5c90c23 --- /dev/null +++ b/clients/herdr-bridge.go @@ -0,0 +1,42 @@ +package main + +import ( + "flag" + "io" + "log" + "net" + "os" +) + +func main() { + listen := flag.String("listen", "192.168.1.105:9245", "TCP address to expose") + socket := flag.String("socket", os.Getenv("HOME")+"/.config/herdr/herdr.sock", "herdr Unix socket") + flag.Parse() + ln, err := net.Listen("tcp", *listen) + if err != nil { + log.Fatal(err) + } + log.Printf("herdr bridge listening on %s -> unix:%s", *listen, *socket) + for { + c, err := ln.Accept() + if err != nil { + log.Printf("accept: %v", err) + continue + } + go proxy(c, *socket) + } +} + +func proxy(c net.Conn, socket string) { + defer c.Close() + u, err := net.Dial("unix", socket) + if err != nil { + log.Printf("dial herdr: %v", err) + return + } + defer u.Close() + done := make(chan struct{}, 2) + go func() { _, _ = io.Copy(u, c); _ = u.Close(); done <- struct{}{} }() + go func() { _, _ = io.Copy(c, u); _ = c.Close(); done <- struct{}{} }() + <-done +} diff --git a/clients/herdr-bridge.service b/clients/herdr-bridge.service new file mode 100644 index 0000000..759d9b4 --- /dev/null +++ b/clients/herdr-bridge.service @@ -0,0 +1,11 @@ +[Unit] +Description=Herdr LAN bridge for Orchestra +After=network-online.target herdr.service + +[Service] +ExecStart=/usr/local/bin/herdr-bridge -listen 192.168.1.105:9245 -socket %h/.config/herdr/herdr.sock +Restart=on-failure +NoNewPrivileges=true + +[Install] +WantedBy=default.target diff --git a/cmd/orchestra/main.go b/cmd/orchestra/main.go index 843bc81..237ab32 100644 --- a/cmd/orchestra/main.go +++ b/cmd/orchestra/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "crypto/subtle" "encoding/json" "errors" "fmt" @@ -136,7 +137,7 @@ func (h harnessCompletion) ServeHTTP(w http.ResponseWriter, r *http.Request) { http.Error(w, "method not allowed", http.StatusMethodNotAllowed) return } - if h.token != "" && r.Header.Get("Authorization") != "Bearer "+h.token { + if h.token != "" && subtle.ConstantTimeCompare([]byte(r.Header.Get("Authorization")), []byte("Bearer "+h.token)) != 1 { http.Error(w, "unauthorized", http.StatusUnauthorized) return } @@ -567,8 +568,9 @@ func main() { }) // /v1/harness/turn is the unified turn-decision endpoint (AUDIT.md Phase // 2 items 1-2): the Face-B stop hook posts here on every ordinary turn - // boundary (report marker absent — /v1/harness/complete covers task - // completion separately) and gets back exactly one of continue / + // boundary (task completion is not handled here — it goes through the + // authenticated worker completion endpoint, since the retired + // /v1/harness/complete had no fencing epoch) and gets back continue / // prepare_handoff / rotate_now / refuse, per spec §5.3. This replaces // what would otherwise be separate ad-hoc marker-file conventions per // decision. @@ -577,7 +579,7 @@ func main() { http.Error(w, "method not allowed", http.StatusMethodNotAllowed) return } - if harnessToken != "" && r.Header.Get("Authorization") != "Bearer "+harnessToken { + if harnessToken != "" && subtle.ConstantTimeCompare([]byte(r.Header.Get("Authorization")), []byte("Bearer "+harnessToken)) != 1 { http.Error(w, "unauthorized", http.StatusUnauthorized) return } diff --git a/deploy/hooks/orchestra-codex-poll.sh b/deploy/hooks/orchestra-codex-poll.sh index 39512dc..c213cdb 100755 --- a/deploy/hooks/orchestra-codex-poll.sh +++ b/deploy/hooks/orchestra-codex-poll.sh @@ -3,8 +3,8 @@ # # Unlike Claude Code, Codex has no native Stop hook — nothing calls out at a # turn boundary. This script is meant to run as a background loop inside the -# pane alongside the codex process (AUDIT.md Phase 2 item 4 / progress.md's -# "Not done: Codex/opencode Stop-hook-equivalent scripts"), polling the same +# pane alongside the codex process (AUDIT.md Phase 2 item 4, "Not done: +# Codex/opencode Stop-hook-equivalent scripts"), polling the same # two endpoints the Claude Stop hook (orchestra-stop.sh) calls on every turn: # - completion: if the agent has written .orchestra-report.md, POST # /v1/harness/complete with harness=codex and the newest active rollout diff --git a/internal/orchestrator/orchestrator.go b/internal/orchestrator/orchestrator.go index 615b7a8..6bb6156 100644 --- a/internal/orchestrator/orchestrator.go +++ b/internal/orchestrator/orchestrator.go @@ -287,6 +287,11 @@ type SessionHealth struct { // silently treated as "not time to rotate yet" by a bare continue. Occupancy float64 `json:"occupancy,omitempty"` OccupancyError string `json:"occupancy_error,omitempty"` + // Observed distinguishes a health entry this coordinator actually read + // from a live adapter this pass from one it could not reach (remote-owned + // session, unresolvable adapter) or has merely seeded at lease time. A + // consumer must not read Status as current unless Observed is true. + Observed bool `json:"observed"` } // adapterFor resolves the herdr adapter for a session. Session.HerdrID (the @@ -356,9 +361,25 @@ func (c *Coordinator) refreshSessionHealth(ctx context.Context) { for taskID, session := range sessions { a, err := c.adapterFor(taskID, session) if err != nil { + // Adapter resolution failing is the normal case for a session + // owned by a remote worker's herdr, and the abnormal case for a + // misregistered local one. Either way the previous entry — often + // the Status:"running" rememberSession writes at lease time — must + // not be left standing as if it were freshly observed, or + // /v1/tasks//health reports a dead pane as running forever. + // Record the resolution failure so it is observable, per the + // contract SessionHealth documents. + c.healthMu.Lock() + prev := c.health.Sessions[taskID] + prev.LastError = err.Error() + prev.UpdatedAt = time.Now().UTC() + prev.Observed = false + c.health.Sessions[taskID] = prev + c.healthMu.Unlock() continue } var h SessionHealth + h.Observed = true h.UpdatedAt = time.Now().UTC() if occ, occErr := a.Occupancy(session); occErr != nil { h.OccupancyError = occErr.Error() @@ -993,6 +1014,8 @@ func (c *Coordinator) Start(ctx context.Context, e domain.Event) error { if c.health.Sessions == nil { c.health.Sessions = map[string]SessionHealth{} } + // Seeded at lease time, not observed from the harness: Observed stays + // false until refreshSessionHealth reads a live adapter. c.health.Sessions[t.ID] = SessionHealth{Status: "running", UpdatedAt: time.Now().UTC()} c.healthMu.Unlock() return err diff --git a/internal/orchestrator/rotation_test.go b/internal/orchestrator/rotation_test.go index 7483768..e3c873c 100644 --- a/internal/orchestrator/rotation_test.go +++ b/internal/orchestrator/rotation_test.go @@ -174,7 +174,7 @@ func TestCoordinatorRefusesRemoteHerdrOperations(t *testing.T) { } // TestRotationEmitsValidReleaseWithAnchorSHA guards the highest-priority spec -// defect noted in progress.md: automated rotation must emit a TaskReleased +// defect noted in AUDIT.md: automated rotation must emit a TaskReleased // event that satisfies domain.ValidatePayload (handoff_ref + anchor_sha), not // a payload missing anchor_sha that silently fails to append. func TestRotationEmitsValidReleaseWithAnchorSHA(t *testing.T) { @@ -1041,3 +1041,55 @@ func TestConventionsDriftNotifiesActiveSession(t *testing.T) { t.Fatal("session was never notified of the conventions-doc update") } } + +// TestUnresolvableAdapterRecordsObservableSessionHealth pins the fix for the +// recurring silent-continue pattern this repo keeps regrowing (CLAUDE.md, +// AUDIT.md P1 "Observability"): refreshSessionHealth used to `continue` on an +// adapter-resolution failure, discarding the reason entirely. The operator +// endpoint /v1/tasks//health then 404s with no way to tell "no such task" +// from "this coordinator cannot see the harness that owns it", and any health +// already seeded at lease time is left standing as if freshly observed. +func TestUnresolvableAdapterRecordsObservableSessionHealth(t *testing.T) { + s, err := store.Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if err := s.Append(domain.Event{ID: domain.NewID(), Type: "TaskCreated", TaskID: "handover", Surface: string(authz.System), Payload: mustJSON(map[string]any{ + "source": "qa", "external_id": "handover", "project": "p", + })}); err != nil { + t.Fatal(err) + } + lease, err := s.Lease("handover", "local", time.Minute) + if err != nil { + t.Fatal(err) + } + statePath := t.TempDir() + "/sessions.json" + a := &fakeAdapter{} + owner := &orchestrator.Coordinator{Store: s, Worktrees: worktrees{path: t.TempDir()}, Adapters: adapters{a}, StatePath: statePath, + LocalHerdr: func(id string) bool { return id == "local" }} + if err := owner.Start(context.Background(), lease); err != nil { + t.Fatal(err) + } + if h, ok := owner.MonitorHealth().Sessions["handover"]; !ok || h.Observed { + t.Fatalf("lease-time seeded health = %+v, present=%v; want present and Observed=false", h, ok) + } + + // Same durable session mapping, but this coordinator no longer owns that + // herdr — the live federated case, and the case where a registry change + // leaves a persisted HerdrID unresolvable. + foreign := &orchestrator.Coordinator{Store: s, Worktrees: worktrees{path: t.TempDir()}, Adapters: adapters{a}, StatePath: statePath, + LocalHerdr: func(id string) bool { return false }} + if err := foreign.Reconcile(context.Background()); err != nil { + t.Fatal(err) + } + h, ok := foreign.MonitorHealth().Sessions["handover"] + if !ok { + t.Fatal("unresolvable session recorded no health at all; the resolution failure was swallowed") + } + if h.Observed { + t.Fatalf("health = %+v; want Observed=false for a session this coordinator cannot read", h) + } + if h.LastError == "" { + t.Fatalf("health = %+v; want the adapter-resolution error recorded in LastError", h) + } +} diff --git a/orchestra-worker b/orchestra-worker deleted file mode 100755 index 3193a7e..0000000 Binary files a/orchestra-worker and /dev/null differ diff --git a/web/go.mod b/web/go.mod new file mode 100644 index 0000000..5988413 --- /dev/null +++ b/web/go.mod @@ -0,0 +1,17 @@ +// This directory is the web UI (Vite/TypeScript); it contains no Go code of +// our own. This file exists solely to make web/ a separate Go module so the +// parent module's package graph stops at the directory boundary. +// +// Without it, `go list ./...` / `go build ./...` / `go test ./...` walk into +// web/node_modules and compile Go packages vendored inside npm dependencies +// (e.g. flatted/golang/pkg/flatted). Any npm dep shipping non-compiling Go +// would then break the entire Orchestra build. A build tag cannot fix this: +// the offending package is already in the module's package list before any +// tag is evaluated, and we do not control third-party sources anyway. +// +// Nothing imports this module and nothing should. Do not add Go code here. +// The web UI is built by web/Dockerfile (npm) and vite emits into +// ../internal/webui/assets for the Go embed; neither is affected by this file. +module orchestra/web + +go 1.22