Files
orchestra/cmd/orchestra/main.go
T
kami 95454afa72 Close B19-B21 and S12-S13, and fix the flaky router test
All five defects filed while implementing B18, plus the router flake that
predated them. None of this has run on the deployed instance: the service
is stopped and /usr/local/bin/orchestra predates every change here.

B20 is the one that could silently defeat approvals. The capture revision
was UnixNano, so it changed on every read and said nothing about whether
the pane had changed; it is now an FNV-1a hash of the pane text, changing
iff the text does. The worse half was precedence: capture() preferred the
coordinator over a published worker capture, handing Queue a timestamp the
owning worker's staleness check could never match, so every federated
approval resolved "stale" and the keystroke never happened. Worker captures
now win — their existence means a registered worker owns that pane — and
capturePane follows the same precedence via Capture.Source rather than
guessing.

B19 was filed as "federated approvals emit no event", which overstated it:
the resolution half already existed, and correctly fires only on an
acknowledged worker report. The missing half was the request. Server.action
now appends ApprovalRequested at queue time, subject_ref set to the command
ID the later resolution carries. If that append fails the queued command is
resolved "rejected" — a keystroke that left no audit trail must not run.

B21 bounds the command list: resolved commands prune after 30 minutes on
both Queue and Commands, pending ones never at any age, since dropping one
would discard an operator decision. The persistence half stays open and is
recorded as such — captures and commands are still in-memory only.

S12 splits ORCHESTRA_NTFY_TOKEN, which was both the secret handed to the
ntfy server and a valid inbound credential for the ntfy surface; the latter
is now ORCHESTRA_NTFY_SURFACE_TOKEN. Breaking: a deployment relying on the
old dual use has no inbound gate until it sets the new variable. S13
deletes the dead auth() copy of the authorization policy.

The router flake was in the test, not in assignment. Store.Tasks() ranges a
map, and the assertion indexed two separate Tasks() calls, failing whenever
the orderings disagreed; instrumenting it showed a valid TaskLeased and a
genuinely leased task on every "failing" run. It now snapshots once and
asserts that exactly one task is leased, and passes at -count=60.

AUDIT.md records what is still not done: the deployed env and binary, the
live re-verification B13-B17 has always lacked, and two operational faults
found in the journal that block it — all six herdrs are refusing
connections, and ntfy delivery is failing 403 on every send.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01535A3Y8RtkAi8wYuWhtkEd
2026-07-29 01:33:59 +04:00

1216 lines
41 KiB
Go

package main
import (
"context"
"crypto/subtle"
"encoding/json"
"errors"
"io"
"log"
"net"
"net/http"
"orchestra/internal/admin"
"orchestra/internal/authz"
"orchestra/internal/delivery"
"orchestra/internal/domain"
"orchestra/internal/federation"
"orchestra/internal/herdr"
"orchestra/internal/operations"
"orchestra/internal/orchestrator"
"orchestra/internal/provider"
"orchestra/internal/registry"
"orchestra/internal/router"
"orchestra/internal/store"
"orchestra/internal/ui"
"orchestra/internal/webui"
"os"
"path/filepath"
"strconv"
"strings"
"time"
)
func id() string { return domain.NewID() }
const defaultHerdrPort = "9245"
func herdrAddress(rr registry.Registry, h registry.Herdr) string {
if h.Address != "" {
return h.Address
}
m, ok := rr.Machine(h.MachineID)
if !ok {
return ""
}
host, _, err := net.SplitHostPort(m.Address)
if err != nil {
return m.Address
}
return net.JoinHostPort(host, defaultHerdrPort)
}
type federatedAvailability struct {
base router.Availability
workers *federation.Registry
localMachine string
}
// federatedReachability keeps the legacy TCP probe for local herdrs, while
// avoiding a coordinator-side probe of a remote worker's herdr socket. A
// remote harness is reachable precisely when its worker is registered (as
// enforced by federatedAvailability); probing its raw herdr endpoint here
// would reintroduce the cross-machine Design A dependency.
type federatedReachability struct {
base registry.Reachability
remote map[string]bool
}
func (r federatedReachability) Reachable(address string, timeout time.Duration) bool {
if r.remote[address] {
return true
}
return r.base.Reachable(address, timeout)
}
func remoteHerdrAddresses(rr registry.Registry, localMachine string) map[string]bool {
remote := map[string]bool{}
for _, h := range rr.Herdrs() {
if h.MachineID != localMachine {
remote[herdrAddress(rr, h)] = true
}
}
return remote
}
func (a federatedAvailability) Available(h registry.Herdr) bool {
if a.base != nil && !a.base.Available(h) {
return false
}
if a.localMachine == "" || h.MachineID == a.localMachine {
return true
}
return a.workers.Available(h.ID)
}
func main() {
dir := os.Getenv("ORCHESTRA_DATA")
if dir == "" {
dir = "./data"
}
s, err := store.Open(dir)
if err != nil {
log.Fatal(err)
}
var rr registry.Registry
var rt *router.Router
var coordinator *orchestrator.Coordinator
localMachine := os.Getenv("ORCHESTRA_MACHINE_ID")
workers := &federation.Registry{AdmitToken: os.Getenv("ORCHESTRA_FEDERATION_ADMIT_TOKEN")}
if config := os.Getenv("ORCHESTRA_CONFIG"); config != "" {
if rr, err = registry.Load(config); err != nil {
log.Fatalf("load orchestra config: %v", err)
}
reachability := registry.Reachability(registry.TCPReachability{})
if localMachine != "" {
reachability = federatedReachability{base: reachability, remote: remoteHerdrAddresses(rr, localMachine)}
}
rt = &router.Router{Store: s, Registry: rr, Reachability: reachability, Timeout: time.Second, Retry: router.RetryPolicy{MaxAttempts: 3, Backoff: time.Minute}}
limits := map[string]router.QuotaWindowLimits{}
for _, h := range rr.Herdrs() {
w := router.QuotaWindowLimits{FiveHour: h.QuotaLimit5h, Weekly: h.QuotaLimitWeekly}
if w.Weekly <= 0 && h.QuotaLimit > 0 {
// Back-compat: the old single-window field meant weekly.
w.Weekly = h.QuotaLimit
}
if w.FiveHour > 0 || w.Weekly > 0 {
limits[h.ID] = w
}
}
if len(limits) > 0 {
rt.Availability = router.QuotaAvailability{Store: s, Limits: limits}
}
if localMachine != "" {
rt.Availability = federatedAvailability{base: rt.Availability, workers: workers, localMachine: localMachine}
}
if repo, root := os.Getenv("ORCHESTRA_REPO"), os.Getenv("ORCHESTRA_WORKTREE_ROOT"); repo != "" && root != "" {
adapters := map[string]herdr.Adapter{}
for _, h := range rr.Herdrs() {
address := herdrAddress(rr, h)
if address == "" {
continue
}
client := herdr.New(address)
protocol := h.Protocol
if protocol == "" {
protocol = os.Getenv("ORCHESTRA_HERDR_PROTOCOL")
}
if err := client.CheckProtocol(context.Background(), protocol); err != nil {
log.Printf("herdr %s unavailable: %v", h.ID, err)
continue
}
switch h.Harness {
case "claude":
adapters[h.ID] = herdr.Claude(client, 200000, s)
case "opencode":
adapters[h.ID] = herdr.OpenCode(client, 200000, s)
case "codex", "":
adapters[h.ID] = herdr.Codex(client, 200000, s)
default:
log.Printf("herdr %s has unsupported harness %q", h.ID, h.Harness)
}
}
projectRepos := map[string]orchestrator.ProjectRepo{}
for _, p := range rr.Projects() {
if p.Repo != "" && p.WorktreeRoot != "" {
projectRepos[p.ID] = orchestrator.ProjectRepo{Repo: p.Repo, WorktreeRoot: p.WorktreeRoot}
}
}
worktrees := orchestrator.PerProjectGitWorktrees{
Projects: projectRepos,
Default: orchestrator.GitWorktrees{Root: root, Repo: repo},
}
coordinator = &orchestrator.Coordinator{Store: s, StatePath: filepath.Join(dir, "runtime-sessions.json"), Worktrees: worktrees, Adapters: orchestrator.AdapterFactory{Herdrs: adapters}}
rt.OnLease = func(e domain.Event) error {
// In federated mode the coordinator must never inspect a remote
// checkout. Its worker consumes the router-issued lease event and
// performs all Git/herdr operations on that machine (§2.1).
if localMachine != "" {
var p struct {
HarnessID string `json:"harness_id"`
}
if json.Unmarshal(e.Payload, &p) == nil {
if h, ok := rr.Herdr(p.HarnessID); ok && h.MachineID != localMachine {
return nil
}
}
}
return coordinator.Start(context.Background(), e)
}
hard := 0.75
if v, parseErr := strconv.ParseFloat(os.Getenv("ORCHESTRA_OCCUPANCY_HARD"), 64); parseErr == nil && v > 0 && v < 1 {
hard = v
}
if v, parseErr := strconv.ParseFloat(os.Getenv("ORCHESTRA_OCCUPANCY_SOFT"), 64); parseErr == nil && v > 0 && v < 1 {
coordinator.Soft = v
}
go func() {
if monitorErr := coordinator.Monitor(context.Background(), hard, 30*time.Second); monitorErr != nil {
log.Printf("orchestrator monitor: %v", monitorErr)
}
}()
}
}
mux := http.NewServeMux()
// B18: the UI is a full control plane — it can create tasks, release or
// complete them, and inject approval keystrokes into live panes. Refuse
// to serve it unauthenticated rather than silently exposing that on
// whatever interface the listener binds to.
webToken := os.Getenv("ORCHESTRA_WEB_TOKEN")
if webToken == "" {
log.Fatal("ORCHESTRA_WEB_TOKEN must be set: it gates the web UI's task, lifecycle and approval controls")
}
sessions := &authz.Sessions{}
// A browser cannot put a Bearer token on a document load, so it trades
// the token once for an HttpOnly cookie. Same credential, presentable
// form; no new authority is created here.
mux.HandleFunc("/v1/ui/session", func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
var body struct {
Token string `json:"token"`
}
_ = json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&body)
supplied := body.Token
if supplied == "" {
supplied = strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
}
if subtle.ConstantTimeCompare([]byte(supplied), []byte(webToken)) != 1 {
http.Error(w, "invalid token", http.StatusUnauthorized)
return
}
v, err := sessions.Issue()
if err != nil {
http.Error(w, "session unavailable", http.StatusInternalServerError)
return
}
http.SetCookie(w, &http.Cookie{
Name: authz.SessionCookie, Value: v, Path: "/",
HttpOnly: true, SameSite: http.SameSiteStrictMode,
Secure: os.Getenv("ORCHESTRA_UI_INSECURE_COOKIE") == "",
MaxAge: int((12 * time.Hour).Seconds()),
})
w.WriteHeader(http.StatusNoContent)
})
// Browser-specific endpoints intentionally present a joined read model;
// raw lifecycle endpoints below remain stable for workers and harnesses.
mux.Handle("/v1/ui/", ui.Server{Store: s, Workers: workers, Coordinator: coordinator, Route: func(e domain.Event) error {
if rt == nil {
return nil
}
_, err := rt.HandleEvent(e)
return err
}}.Handler())
mux.Handle("/", webui.Handler())
workers.OnOffline = func(w federation.Worker) {
for _, t := range s.Tasks() {
if t.State == domain.StateLeased && t.Lease != nil && t.Lease.HarnessID == w.ID {
p, _ := json.Marshal(map[string]any{"reason": "worker_offline", "harness_id": w.ID})
e := domain.Event{ID: id(), Type: "TaskReleased", TaskID: t.ID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
if err := s.Append(e); err == nil && rt != nil {
_, _ = rt.HandleEvent(e)
}
}
}
}
go func() {
ticker := time.NewTicker(15 * time.Second)
defer ticker.Stop()
for range ticker.C {
_ = workers.Snapshot()
}
}()
providerHealth := map[string]*provider.Supervisor{}
surface := func(r *http.Request) authz.Surface {
v := authz.ParseSurface(r.Header.Get("X-Orchestra-Surface"))
if v == "" || v == authz.System {
// System means "the plane itself, in-process" and is always
// FullControl with no token gate (AUDIT.md B8) — it must never
// be constructible from an HTTP request, or any LAN caller
// declaring this header gets unauthenticated full control.
return authz.Web
}
return v
}
mux.HandleFunc("/v1/tasks", func(w http.ResponseWriter, r *http.Request) {
if r.Method == "GET" {
json.NewEncoder(w).Encode(s.Tasks())
return
}
if r.Method != "POST" {
http.Error(w, "method not allowed", 405)
return
}
var p map[string]any
if json.NewDecoder(r.Body).Decode(&p) != nil {
http.Error(w, "invalid json", 400)
return
}
b, _ := json.Marshal(p)
e := domain.Event{ID: id(), Type: "TaskCreated", TaskID: id(), Version: 1, Payload: b, Surface: string(surface(r))}
if err := s.Append(e); err != nil {
if errors.Is(err, domain.ErrDuplicate) {
// (source, external_id) was already ingested. Nothing was
// appended; report the existing task idempotently rather
// than fabricating a 201 with an unrelated event (S6).
source, _ := p["source"].(string)
externalID, _ := p["external_id"].(string)
if t, ok := s.TaskBySource(source, externalID); ok {
w.WriteHeader(200)
json.NewEncoder(w).Encode(t)
return
}
}
http.Error(w, err.Error(), 400)
return
}
if rt != nil {
if _, err := rt.HandleEvent(e); err != nil {
log.Printf("route task: %v", err)
}
}
w.WriteHeader(201)
json.NewEncoder(w).Encode(e)
})
mux.HandleFunc("/v1/events", func(w http.ResponseWriter, r *http.Request) {
var n uint64
if x, err := strconv.ParseUint(r.URL.Query().Get("since"), 10, 64); err == nil {
n = x
}
json.NewEncoder(w).Encode(s.Events(n))
})
mux.HandleFunc("/v1/handoffs", func(w http.ResponseWriter, r *http.Request) {
out := make([]domain.Event, 0)
for _, e := range s.Events(0) {
if e.Type == "TaskReleased" {
out = append(out, e)
}
}
json.NewEncoder(w).Encode(out)
})
mux.HandleFunc("/v1/quotas", func(w http.ResponseWriter, r *http.Request) {
out := make([]domain.Event, 0)
for _, e := range s.Events(0) {
if e.Type == "QuotaReported" {
out = append(out, e)
}
}
json.NewEncoder(w).Encode(out)
})
mux.HandleFunc("/v1/artifacts", func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
// Reports and handoffs are content-addressed evidence. Keep uploads
// bounded because event payloads only carry their resulting hash.
r.Body = http.MaxBytesReader(w, r.Body, 4<<20)
b, err := io.ReadAll(r.Body)
if err != nil {
http.Error(w, "artifact too large or unreadable", http.StatusRequestEntityTooLarge)
return
}
if len(b) == 0 {
http.Error(w, "artifact is empty", http.StatusBadRequest)
return
}
ref, err := s.PutArtifact(b)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusCreated)
json.NewEncoder(w).Encode(map[string]string{"ref": ref})
})
// Artifacts are normally write-only to public surfaces. A federation
// worker may read a handoff only after authenticating as the worker that
// will validate it against its own checkout (§2.1, §6.2).
mux.HandleFunc("/v1/artifacts/", func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
wid := r.Header.Get("X-Orchestra-Worker")
token := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
if err := workers.Authenticate(wid, token); err != nil {
http.Error(w, err.Error(), http.StatusUnauthorized)
return
}
ref := strings.TrimPrefix(r.URL.Path, "/v1/artifacts/")
if len(ref) != 64 {
http.Error(w, "artifact ref required", http.StatusBadRequest)
return
}
b, err := s.Artifact(ref)
if err != nil {
http.Error(w, "artifact not found", http.StatusNotFound)
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write(b)
})
// /v1/harness/complete is the automatic TaskCompleted producer (AUDIT.md
// B3): a harness-side hook posts here when the agent has declared the
// task done (see deploy/hooks/orchestra-stop.sh), not on every turn
// boundary. It reads the transcript locally to build an honest receipt —
// same session-file assumption as CLIAdapter.Occupancy — rather than
// trusting a self-reported number.
harnessToken := os.Getenv("ORCHESTRA_HARNESS_TOKEN")
mux.HandleFunc("/v1/harness/complete", func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
if harnessToken != "" && r.Header.Get("Authorization") != "Bearer "+harnessToken {
http.Error(w, "unauthorized", http.StatusUnauthorized)
return
}
var p struct {
TaskID string `json:"task_id"`
Harness string `json:"harness"`
TranscriptPath string `json:"transcript_path"`
Report string `json:"report"`
}
if json.NewDecoder(r.Body).Decode(&p) != nil || p.TaskID == "" || p.Report == "" || p.TranscriptPath == "" {
http.Error(w, "task_id, transcript_path, and report are required", http.StatusBadRequest)
return
}
t, ok := s.Task(p.TaskID)
if !ok {
http.Error(w, "task not found", http.StatusNotFound)
return
}
// Harness-specific session-state readers (AUDIT.md "Codex/opencode
// completion producers"). Same local-filesystem assumption B3 already
// made for Claude: the caller supplies the path to its own session
// state (transcript / rollout / message file), never a herdr pane id.
var usage herdr.Usage
var err error
switch p.Harness {
case "codex":
usage, err = herdr.CodexUsage(p.TranscriptPath)
case "opencode":
usage, err = herdr.OpenCodeUsage(p.TranscriptPath)
case "", "claude":
usage, err = herdr.ClaudeUsage(p.TranscriptPath)
default:
http.Error(w, "unknown harness: "+p.Harness, http.StatusBadRequest)
return
}
if err != nil {
http.Error(w, "reading transcript: "+err.Error(), http.StatusBadRequest)
return
}
ref, err := s.PutArtifact([]byte(p.Report))
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
payload, _ := json.Marshal(map[string]any{
"report_ref": ref,
"receipt": map[string]any{
"input_tokens": usage.Input,
"cache_read_tokens": usage.CacheRead,
"cache_write_tokens": usage.CacheWrite,
"output_tokens": usage.Output,
"numerator": usage.Numerator(),
},
})
e := domain.Event{ID: id(), Type: "TaskCompleted", TaskID: p.TaskID, Version: t.Version + 1, Payload: payload, Surface: string(authz.System)}
if err := s.Append(e); err != nil {
http.Error(w, err.Error(), http.StatusConflict)
return
}
if rt != nil {
if _, routeErr := rt.HandleEvent(e); routeErr != nil {
log.Printf("route task: %v", routeErr)
}
}
// B7 (AUDIT.md): QuotaReported has no other producer, so router's
// 5h/weekly availability filter and the brief's quota_consumed are
// permanently zero without this. Fed by the same per-harness usage
// read as the receipt above (spec §7.2 — "same per-harness session
// state as §5.2.1"); harness_id comes from the lease this completion
// closes out, before it's released.
if t.Lease != nil && t.Lease.HarnessID != "" {
qp, _ := json.Marshal(map[string]any{
"harness_id": t.Lease.HarnessID,
"consumed": float64(usage.Numerator()),
})
qe := domain.Event{ID: id(), Type: "QuotaReported", TaskID: "system", Payload: qp, Surface: string(authz.System)}
if err := s.Append(qe); err != nil {
log.Printf("quota report: %v", err)
}
}
json.NewEncoder(w).Encode(e)
})
// /v1/harness/turn is the unified turn-decision endpoint (AUDIT.md Phase
// 2 items 1-2): the Face-B stop hook posts here on every ordinary turn
// boundary (report marker absent — /v1/harness/complete covers task
// completion separately) and gets back exactly one of continue /
// prepare_handoff / rotate_now / refuse, per spec §5.3. This replaces
// what would otherwise be separate ad-hoc marker-file conventions per
// decision.
mux.HandleFunc("/v1/harness/turn", func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
if harnessToken != "" && r.Header.Get("Authorization") != "Bearer "+harnessToken {
http.Error(w, "unauthorized", http.StatusUnauthorized)
return
}
if coordinator == nil {
http.Error(w, "coordinator not configured", http.StatusServiceUnavailable)
return
}
var p struct {
TaskID string `json:"task_id"`
}
if json.NewDecoder(r.Body).Decode(&p) != nil || p.TaskID == "" {
http.Error(w, "task_id is required", http.StatusBadRequest)
return
}
decision, err := coordinator.TurnDecision(r.Context(), p.TaskID)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
json.NewEncoder(w).Encode(map[string]string{"decision": decision})
})
mux.HandleFunc("/v1/brief", func(w http.ResponseWriter, r *http.Request) {
to := time.Now().UTC()
from := to.Add(-12 * time.Hour)
if v, parseErr := time.Parse(time.RFC3339, r.URL.Query().Get("from")); parseErr == nil {
from = v
}
if v, parseErr := time.Parse(time.RFC3339, r.URL.Query().Get("to")); parseErr == nil {
to = v
}
git := map[string]operations.GitSync{}
for _, p := range rr.Projects() {
if p.Repo == "" {
continue
}
git[p.ID] = operations.GitState(p.Repo)
}
if len(git) == 0 {
if repo := os.Getenv("ORCHESTRA_REPO"); repo != "" {
git["default"] = operations.GitState(repo)
}
}
json.NewEncoder(w).Encode(operations.BuildBrief(s.Events(0), from, to, git))
})
standup := func() (domain.Event, error) {
return operations.GenerateStandupAdvisory(s, time.Now().UTC())
}
mux.HandleFunc("/v1/standup", func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodGet {
json.NewEncoder(w).Encode(operations.StandupItems(s.Tasks()))
return
}
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", 405)
return
}
if err := authz.AuthorizeEvent(surface(r), "StandupAdvisory"); err != nil {
http.Error(w, err.Error(), http.StatusForbidden)
return
}
e, err := standup()
if err != nil {
http.Error(w, err.Error(), 400)
return
}
json.NewEncoder(w).Encode(e)
})
mux.HandleFunc("/v1/standup/approve", func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", 405)
return
}
if err := authz.AuthorizeEvent(surface(r), "ApprovalGranted"); err != nil {
http.Error(w, err.Error(), 403)
return
}
var p struct {
AdvisoryID string `json:"advisory_id"`
}
if json.NewDecoder(r.Body).Decode(&p) != nil || p.AdvisoryID == "" {
http.Error(w, "advisory_id required", 400)
return
}
b, _ := json.Marshal(map[string]any{"subject_ref": p.AdvisoryID})
e := domain.Event{ID: id(), Type: "ApprovalGranted", TaskID: "system", Version: 0, Payload: b, Surface: string(surface(r))}
if err := s.Append(e); err != nil {
http.Error(w, err.Error(), 400)
return
}
json.NewEncoder(w).Encode(e)
})
mux.HandleFunc("/v1/standup/apply", func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", 405)
return
}
if err := authz.AuthorizeEvent(surface(r), "TaskAmended"); err != nil {
http.Error(w, err.Error(), 403)
return
}
var p struct {
AdvisoryID string `json:"advisory_id"`
}
if json.NewDecoder(r.Body).Decode(&p) != nil || p.AdvisoryID == "" {
http.Error(w, "advisory_id required", 400)
return
}
out, err := operations.ApplyAdvisory(s, p.AdvisoryID)
if err != nil {
http.Error(w, err.Error(), 400)
return
}
json.NewEncoder(w).Encode(out)
})
adminServer := &admin.Server{Store: s, RouterReady: rt != nil, Providers: providerHealth, Probes: map[string]admin.ProbeFunc{
"router": func() (bool, string) { return rt != nil, "configured router" },
"gitea": func() (bool, string) {
configured := os.Getenv("ORCHESTRA_GITEA_URL") != "" || os.Getenv("ORCHESTRA_GITEA_CONFIG") != ""
if !configured {
return true, "configured provider"
}
for name := range providerHealth {
if name == "gitea" || strings.HasPrefix(name, "gitea:") {
return true, "configured provider"
}
}
return false, "configured provider"
},
"jsonl": func() (bool, string) {
return os.Getenv("ORCHESTRA_JSONL") == "" || providerHealth["jsonl"] != nil, "configured provider"
},
}}
mux.HandleFunc("/metrics", adminServer.Metrics)
mux.HandleFunc("/v1/events/subscribe", adminServer.Subscribe)
mux.HandleFunc("/v1/admin/diagnostics", adminServer.Diagnostics)
mux.HandleFunc("/v1/tasks/", func(w http.ResponseWriter, r *http.Request) {
parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/")
if r.Method == http.MethodGet && coordinator != nil && len(parts) == 4 {
taskID, view := parts[2], parts[3]
if view == "health" {
if h, ok := coordinator.MonitorHealth().Sessions[taskID]; ok {
json.NewEncoder(w).Encode(h)
return
}
http.Error(w, "session health not found", 404)
return
}
if view == "capture" {
body, err := coordinator.Capture(r.Context(), taskID, r.URL.Query().Get("source"))
if err != nil {
http.Error(w, err.Error(), 404)
return
}
json.NewEncoder(w).Encode(map[string]string{"task_id": taskID, "source": r.URL.Query().Get("source"), "text": body})
return
}
}
if len(parts) < 4 || len(parts) > 5 || r.Method != "POST" {
http.Error(w, "not found", http.StatusNotFound)
return
}
taskID, action := parts[2], parts[3]
if action == "approval" {
if err := authz.AuthorizeEvent(surface(r), map[bool]string{true: "ApprovalRequested", false: "ApprovalGranted"}[len(parts) == 4]); err != nil && len(parts) == 4 {
http.Error(w, err.Error(), http.StatusForbidden)
return
}
if len(parts) == 5 {
typ := "ApprovalGranted"
if parts[4] == "deny" {
typ = "ApprovalDenied"
}
if err := authz.AuthorizeEvent(surface(r), typ); err != nil {
http.Error(w, err.Error(), http.StatusForbidden)
return
}
if parts[4] != "grant" && parts[4] != "deny" {
http.Error(w, "unknown approval action", 404)
return
}
t, ok := s.Task(taskID)
if !ok {
http.Error(w, "task not found", 404)
return
}
by := r.Header.Get("X-Orchestra-Actor")
if by == "" {
by = "surface"
}
b, _ := json.Marshal(map[string]any{"subject_ref": taskID, "by": by})
e := domain.Event{ID: id(), Type: typ, TaskID: taskID, Version: t.Version + 1, Payload: b, Surface: string(surface(r))}
if err := s.Append(e); err != nil {
http.Error(w, err.Error(), 409)
return
}
json.NewEncoder(w).Encode(e)
return
}
var p struct {
Options []any `json:"options"`
}
if json.NewDecoder(r.Body).Decode(&p) != nil || len(p.Options) == 0 {
http.Error(w, "options required", 400)
return
}
if _, ok := s.Task(taskID); !ok {
http.Error(w, "task not found", 404)
return
}
b, _ := json.Marshal(map[string]any{"subject_ref": taskID, "options": p.Options})
t, _ := s.Task(taskID)
e := domain.Event{ID: id(), Type: "ApprovalRequested", TaskID: taskID, Version: t.Version + 1, Payload: b, Surface: string(surface(r))}
if err := s.Append(e); err != nil {
http.Error(w, err.Error(), 400)
return
}
json.NewEncoder(w).Encode(e)
return
}
var e domain.Event
var err error
actionTypes := map[string]string{"lease": "TaskLeased", "release": "TaskReleased", "complete": "TaskCompleted", "block": "TaskBlocked"}
if typ, known := actionTypes[action]; known {
if err := authz.AuthorizeEvent(surface(r), typ); err != nil {
http.Error(w, err.Error(), http.StatusForbidden)
return
}
}
switch action {
case "lease":
var p struct {
HarnessID string `json:"harness_id"`
TTLSeconds int `json:"ttl_seconds"`
}
if json.NewDecoder(r.Body).Decode(&p) != nil || p.HarnessID == "" {
http.Error(w, "harness_id required", 400)
return
}
if p.TTLSeconds <= 0 {
p.TTLSeconds = 1800
}
e, err = s.Lease(taskID, p.HarnessID, time.Duration(p.TTLSeconds)*time.Second)
case "release", "complete", "block":
t, ok := s.Task(taskID)
if !ok {
http.Error(w, "task not found", 404)
return
}
types := map[string]string{"release": "TaskReleased", "complete": "TaskCompleted", "block": "TaskBlocked"}
var p map[string]any
if r.Body == nil || json.NewDecoder(r.Body).Decode(&p) != nil || p == nil {
http.Error(w, "invalid lifecycle payload", http.StatusBadRequest)
return
}
if action == "release" && p["reason"] == nil && p["handoff_ref"] == nil {
http.Error(w, "reason or handoff_ref required", http.StatusBadRequest)
return
}
if action == "block" && p["blocker"] == nil {
http.Error(w, "blocker required", http.StatusBadRequest)
return
}
if action == "complete" && p["report_ref"] == nil {
http.Error(w, "report_ref required", http.StatusBadRequest)
return
}
ePayload, _ := json.Marshal(p)
e = domain.Event{ID: id(), Type: types[action], TaskID: taskID, Version: t.Version + 1, Payload: ePayload, Surface: string(surface(r))}
err = s.Append(e)
default:
http.Error(w, "unknown action", 404)
return
}
if err != nil {
http.Error(w, err.Error(), 409)
return
}
if rt != nil {
if _, routeErr := rt.HandleEvent(e); routeErr != nil {
log.Printf("route task: %v", routeErr)
}
}
json.NewEncoder(w).Encode(e)
})
if rt != nil {
go func() {
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for range ticker.C {
// When a coordinator is running, its own Monitor loop (every
// 30s) already calls Store.ExpireLeases and, critically,
// kills the herdr session for whatever it expires (S9: two
// independent expiry loops raced on the same reclaim, and
// since this one runs every second vs. the coordinator's
// 30s, it almost always won — meaning the coordinator's
// ExpireLeases call saw nothing left to expire and its
// session-kill path never ran, silently orphaning panes
// past their lease TTL). Calling ExpireLeases here too would
// just resurrect that race, so leave reclaim to the
// coordinator and only keep retrying pending assignment.
if coordinator != nil {
if _, err := rt.AssignPending(); err != nil {
log.Printf("route expired task: %v", err)
}
continue
}
if _, err := s.ExpireLeases(time.Now()); err != nil {
log.Printf("expire leases: %v", err)
} else if _, err := rt.AssignPending(); err != nil {
log.Printf("route expired task: %v", err)
}
}
}()
}
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("ok\n")) })
mux.HandleFunc("/readyz", adminServer.Readiness)
mux.HandleFunc("/v1/providers/health", func(w http.ResponseWriter, r *http.Request) {
out := map[string]provider.Health{}
for name, sup := range providerHealth {
out[name] = sup.Health()
}
json.NewEncoder(w).Encode(out)
})
mux.HandleFunc("/v1/orchestrator/health", func(w http.ResponseWriter, r *http.Request) {
if coordinator == nil {
http.Error(w, "orchestrator unavailable", http.StatusServiceUnavailable)
return
}
json.NewEncoder(w).Encode(coordinator.MonitorHealth())
})
mux.HandleFunc("/v1/federation/workers", func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodGet {
json.NewEncoder(w).Encode(workers.Snapshot())
return
}
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", 405)
return
}
var registration struct {
federation.Worker
Token string `json:"token"`
}
if json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10)).Decode(&registration) != nil {
http.Error(w, "invalid worker", 400)
return
}
worker := registration.Worker
worker.Token = registration.Token
if worker.Token == "" {
http.Error(w, "token required", 400)
return
}
admitToken := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
if err := workers.Register(worker, admitToken); err != nil {
status := 400
if err == federation.ErrUnauthorized {
status = 401
}
http.Error(w, err.Error(), status)
return
}
if rt != nil {
if _, err := rt.AssignPending(); err != nil {
log.Printf("route after worker registration: %v", err)
}
}
w.WriteHeader(http.StatusCreated)
json.NewEncoder(w).Encode(worker)
})
workerAuth := func(r *http.Request) (string, error) {
wid := r.Header.Get("X-Orchestra-Worker")
tok := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
if err := workers.Authenticate(wid, tok); err != nil {
return "", err
}
return wid, nil
}
mux.HandleFunc("/v1/federation/commands", func(w http.ResponseWriter, r *http.Request) {
wid, err := workerAuth(r)
if err != nil {
http.Error(w, err.Error(), http.StatusUnauthorized)
return
}
if r.Method != http.MethodGet {
http.Error(w, "method not allowed", 405)
return
}
out, err := workers.Commands(wid)
if err != nil {
http.Error(w, err.Error(), 404)
return
}
json.NewEncoder(w).Encode(out)
})
mux.HandleFunc("/v1/federation/commands/", func(w http.ResponseWriter, r *http.Request) {
wid, err := workerAuth(r)
if err != nil {
http.Error(w, err.Error(), http.StatusUnauthorized)
return
}
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", 405)
return
}
var body struct {
Status string `json:"status"`
Message string `json:"message"`
}
if json.NewDecoder(r.Body).Decode(&body) != nil || (body.Status != "acknowledged" && body.Status != "stale" && body.Status != "rejected") {
http.Error(w, "invalid command result", 400)
return
}
commandID := strings.TrimPrefix(r.URL.Path, "/v1/federation/commands/")
command, ok := workers.Command(wid, commandID)
if !ok {
http.Error(w, "command not found", 404)
return
}
if err := workers.CompleteCommand(wid, commandID, body.Status, body.Message); err != nil {
http.Error(w, err.Error(), 409)
return
}
// Audit lifecycle evidence only after the worker reports the herdr
// input was acknowledged; a queued browser click is never an approval.
if body.Status == "acknowledged" {
if t, ok := s.Task(command.TaskID); ok {
typ := "ApprovalGranted"
if command.Kind == "deny_approval" {
typ = "ApprovalDenied"
}
payload, _ := json.Marshal(map[string]any{"subject_ref": command.ID, "pane_id": command.PaneID, "capture_revision": command.CaptureRevision})
e := domain.Event{ID: id(), Type: typ, TaskID: command.TaskID, Version: t.Version + 1, Payload: payload, Surface: string(authz.System)}
if err := s.Append(e); err != nil {
log.Printf("record approval %s: %v", command.ID, err)
}
}
}
w.WriteHeader(http.StatusNoContent)
})
mux.HandleFunc("/v1/federation/events", func(w http.ResponseWriter, r *http.Request) {
wid, err := workerAuth(r)
if err != nil {
http.Error(w, err.Error(), http.StatusUnauthorized)
return
}
if r.Method != http.MethodGet {
http.Error(w, "method not allowed", 405)
return
}
cursor, _ := workers.Cursor(wid)
if q := r.URL.Query().Get("since"); q != "" {
cursor, _ = strconv.ParseUint(q, 10, 64)
}
events := s.Events(cursor)
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]any{"cursor": cursor, "events": events})
})
mux.HandleFunc("/v1/federation/events/ack", func(w http.ResponseWriter, r *http.Request) {
wid, err := workerAuth(r)
if err != nil {
http.Error(w, err.Error(), 401)
return
}
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", 405)
return
}
var body struct {
Cursor uint64 `json:"cursor"`
}
if json.NewDecoder(r.Body).Decode(&body) != nil {
http.Error(w, "invalid cursor", 400)
return
}
if err := workers.Ack(wid, body.Cursor); err != nil {
http.Error(w, err.Error(), 409)
return
}
w.WriteHeader(http.StatusNoContent)
})
mux.HandleFunc("/v1/federation/workers/", func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || (!strings.HasSuffix(r.URL.Path, "/heartbeat") && !strings.HasSuffix(r.URL.Path, "/handoff") && !strings.HasSuffix(r.URL.Path, "/complete") && !strings.HasSuffix(r.URL.Path, "/captures")) {
http.Error(w, "not found", 404)
return
}
parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/")
if len(parts) != 5 {
http.Error(w, "not found", 404)
return
}
wid, err := workerAuth(r)
if err != nil {
http.Error(w, err.Error(), 401)
return
}
if wid != parts[3] {
http.Error(w, "worker identity mismatch", http.StatusForbidden)
return
}
if strings.HasSuffix(r.URL.Path, "/heartbeat") {
if err := workers.Heartbeat(parts[3]); err != nil {
http.Error(w, err.Error(), 404)
return
}
w.WriteHeader(http.StatusNoContent)
return
}
if strings.HasSuffix(r.URL.Path, "/captures") {
var c federation.Capture
if json.NewDecoder(r.Body).Decode(&c) != nil {
http.Error(w, "invalid capture", 400)
return
}
if t, ok := s.Task(c.TaskID); !ok || t.Lease == nil || t.Lease.HarnessID != parts[3] {
http.Error(w, "lease not owned", 409)
return
}
out, err := workers.PutCapture(parts[3], c)
if err != nil {
http.Error(w, err.Error(), 400)
return
}
json.NewEncoder(w).Encode(out)
return
}
var b struct {
TaskID string `json:"task_id"`
TTLSeconds int `json:"ttl_seconds"`
HandoffRef string `json:"handoff_ref"`
AnchorSHA string `json:"anchor_sha"`
}
if json.NewDecoder(r.Body).Decode(&b) != nil || b.TaskID == "" {
http.Error(w, "invalid lease body", 400)
return
}
t, ok := s.Task(b.TaskID)
if !ok {
http.Error(w, "task not found", 404)
return
}
if t.State != domain.StateLeased || t.Lease == nil || t.Lease.HarnessID != parts[3] {
http.Error(w, "lease not owned", 409)
return
}
if strings.HasSuffix(r.URL.Path, "/complete") {
if b.HandoffRef == "" {
http.Error(w, "report_ref required", 400)
return
}
p, _ := json.Marshal(map[string]any{"report_ref": b.HandoffRef, "receipt": map[string]any{"harness_id": parts[3], "consumed": 0}})
e := domain.Event{ID: id(), Type: "TaskCompleted", TaskID: b.TaskID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
if err := s.Append(e); err != nil {
http.Error(w, err.Error(), 409)
return
}
json.NewEncoder(w).Encode(e)
return
}
if b.HandoffRef == "" {
http.Error(w, "handoff_ref required", 400)
return
}
if len(b.AnchorSHA) != 40 {
http.Error(w, "anchor_sha required", 400)
return
}
p, _ := json.Marshal(map[string]any{"handoff_ref": b.HandoffRef, "harness_id": parts[3], "anchor_sha": b.AnchorSHA})
e := domain.Event{ID: id(), Type: "TaskReleased", TaskID: b.TaskID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
if err := s.Append(e); err != nil {
http.Error(w, err.Error(), 409)
return
}
if rt != nil {
_, _ = rt.HandleEvent(e)
}
json.NewEncoder(w).Encode(e)
})
var giteaSources []provider.GiteaSourceConfig
if path := os.Getenv("ORCHESTRA_GITEA_CONFIG"); path != "" {
cfgs, err := provider.LoadGiteaConfigs(path)
if err != nil {
log.Fatalf("load gitea config: %v", err)
}
giteaSources = cfgs
} else if base := os.Getenv("ORCHESTRA_GITEA_URL"); base != "" {
// Legacy single-repo configuration: project defaults to the repo
// name, matching the historical (pre-multi-source) behavior.
giteaSources = []provider.GiteaSourceConfig{{
Project: os.Getenv("ORCHESTRA_GITEA_REPO"), BaseURL: base,
Owner: os.Getenv("ORCHESTRA_GITEA_OWNER"), Repo: os.Getenv("ORCHESTRA_GITEA_REPO"),
Token: os.Getenv("ORCHESTRA_GITEA_TOKEN"), WebhookSecret: os.Getenv("ORCHESTRA_GITEA_WEBHOOK_SECRET"),
}}
}
if len(giteaSources) > 0 {
reflectors := map[string]provider.Gitea{}
for _, c := range giteaSources {
g := provider.Gitea{BaseURL: c.BaseURL, Token: c.Token, WebhookSecret: c.WebhookSecret, Owner: c.Owner, Repo: c.Repo, Project: c.Project}
reflectors[g.SourceName()] = g
}
reflecting := provider.ReflectingSink{Sink: s, Tasks: s, Reflector: provider.MultiGitea{Sources: reflectors}}
for _, c := range giteaSources {
g := provider.Gitea{BaseURL: c.BaseURL, Token: c.Token, WebhookSecret: c.WebhookSecret, Owner: c.Owner, Repo: c.Repo, Project: c.Project}
name := "gitea:" + c.Project
webhookPath := "/v1/providers/gitea/webhook/" + c.Project
if len(giteaSources) == 1 && os.Getenv("ORCHESTRA_GITEA_CONFIG") == "" {
// Preserve the legacy unprefixed webhook path when running
// the single-source (env-var) configuration, so existing
// Gitea webhook configs don't need to be re-pointed.
webhookPath = "/v1/providers/gitea/webhook"
name = "gitea"
}
mux.Handle(webhookPath, g.WebhookHandler(reflecting))
sup := &provider.Supervisor{Name: name, Run: func(ctx context.Context) error {
pollCtx, cancel := context.WithTimeout(ctx, time.Minute)
defer cancel()
_, err := g.Poll(pollCtx, reflecting)
if err == nil {
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(time.Minute):
}
}
return err
}}
sup.Start(context.Background())
providerHealth[name] = sup
}
}
if path := os.Getenv("ORCHESTRA_JSONL"); path != "" {
sup := &provider.Supervisor{Name: "jsonl", Run: func(ctx context.Context) error {
return (provider.JSONLWatcher{Path: path, Interval: time.Second, Provider: provider.JSONL{Source: "jsonl"}}).Run(ctx, s)
}}
sup.Start(context.Background())
providerHealth["jsonl"] = sup
}
var senders []delivery.Sender
if token, chat := os.Getenv("ORCHESTRA_TELEGRAM_BOT_TOKEN"), os.Getenv("ORCHESTRA_TELEGRAM_CHAT_ID"); token != "" && chat != "" {
senders = append(senders, delivery.Telegram{Token: token, ChatID: chat})
}
if topic := os.Getenv("ORCHESTRA_NTFY_TOPIC"); topic != "" {
senders = append(senders, delivery.Ntfy{Topic: topic, Token: os.Getenv("ORCHESTRA_NTFY_TOKEN"), URL: os.Getenv("ORCHESTRA_NTFY_URL")})
}
if len(senders) > 0 {
cursorPath := filepath.Join(dir, "delivery-cursor")
var startCursor uint64
if b, err := os.ReadFile(cursorPath); err == nil {
if v, err := strconv.ParseUint(strings.TrimSpace(string(b)), 10, 64); err == nil {
startCursor = v
}
}
go func() {
fanout := &delivery.Fanout{
Senders: senders,
Cursor: startCursor,
SaveCursor: func(cursor uint64) {
if err := os.WriteFile(cursorPath, []byte(strconv.FormatUint(cursor, 10)), 0o644); err != nil {
log.Printf("delivery cursor persist: %v", err)
}
},
}
err := fanout.Run(context.Background(), s.Events)
if err != nil {
log.Printf("delivery fanout: %v", err)
}
}()
}
go func() {
lastDay := ""
t := time.NewTicker(time.Minute)
defer t.Stop()
for now := range t.C {
utc := now.UTC()
day := utc.Format("2006-01-02")
if utc.Hour() == 3 && day != lastDay {
if _, err := standup(); err != nil {
log.Printf("standup advisory: %v", err)
} else {
lastDay = day
}
}
}
}()
if rt != nil {
if _, err := rt.AssignPending(); err != nil {
log.Printf("startup task assignment: %v", err)
}
}
port := os.Getenv("ORCHESTRA_PORT")
if port == "" {
port = "9145"
}
log.Println("orchestra listening on :" + port)
tokens := map[authz.Surface]string{
authz.TUI: os.Getenv("ORCHESTRA_TUI_TOKEN"), authz.Web: os.Getenv("ORCHESTRA_WEB_TOKEN"),
authz.MCP: os.Getenv("ORCHESTRA_MCP_TOKEN"), authz.Maven: os.Getenv("ORCHESTRA_MAVEN_TOKEN"),
// S12: this is the credential callers present *to* Orchestra on the
// ntfy surface. ORCHESTRA_NTFY_TOKEN is a different secret entirely —
// it is handed out to the third-party ntfy server (see the sender
// above) and must never be accepted as an inbound credential.
authz.Telegram: os.Getenv("ORCHESTRA_TELEGRAM_TOKEN"), authz.Ntfy: os.Getenv("ORCHESTRA_NTFY_SURFACE_TOKEN"),
}
log.Fatal(http.ListenAndServe(":"+port, authz.HTTPWithSessions(tokens, sessions, mux)))
}