Files
orchestra/cmd/orchestra/main.go
T
kami 4af9880b86 Start the submission reflection loop after something can fill its map
The loop that reads merged pull requests was guarded by len(pullRequests) > 0
at a point 600 lines before the Gitea wiring that writes to that map. The
length was always zero, so the goroutine never started and a merged pull
request could never complete its task. Live on the first submission this
deployment made: PR #8 took a trusted comment and nothing moved.

Moving the block below the wiring fixes the ordering and avoids the race that
reading the map inside the tick would have introduced.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CVbaKucEYBjMqVeUgJUsc1
2026-08-28 10:53:47 +04:00

1850 lines
67 KiB
Go

package main
import (
"context"
"crypto/subtle"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"net"
"net/http"
"orchestra/internal/admin"
"orchestra/internal/authn"
"orchestra/internal/authz"
"orchestra/internal/buildinfo"
"orchestra/internal/delivery"
"orchestra/internal/domain"
"orchestra/internal/federation"
"orchestra/internal/herdr"
"orchestra/internal/human"
"orchestra/internal/operations"
"orchestra/internal/orchestrator"
"orchestra/internal/provider"
"orchestra/internal/registry"
"orchestra/internal/review"
"orchestra/internal/router"
"orchestra/internal/store"
"orchestra/internal/ui"
"orchestra/internal/webui"
"os"
"path/filepath"
"strconv"
"strings"
"time"
)
func id() string { return domain.NewID() }
const defaultHerdrPort = "9245"
func herdrAddress(rr registry.Registry, h registry.Herdr) string {
if h.Backend == "tmux" {
return rr.Endpoint(h)
}
if h.Address != "" {
return h.Address
}
m, ok := rr.Machine(h.MachineID)
if !ok {
return ""
}
host, _, err := net.SplitHostPort(m.Address)
if err != nil {
return m.Address
}
return net.JoinHostPort(host, defaultHerdrPort)
}
type federatedAvailability struct {
base router.Availability
workers *federation.Registry
localMachine string
}
// federatedReachability keeps the legacy TCP probe for coordinator-owned
// herdrs, while avoiding a coordinator-side probe of a worker-owned backend.
// A worker-owned harness is reachable precisely when its worker is registered (as
// enforced by federatedAvailability); probing its raw herdr endpoint here
// would reintroduce the cross-machine Design A dependency.
type federatedReachability struct {
base registry.Reachability
remote map[string]bool
}
func (r federatedReachability) Reachable(address string, timeout time.Duration) bool {
if r.remote[address] {
return true
}
return r.base.Reachable(address, timeout)
}
func remoteHerdrAddresses(rr registry.Registry, localMachine string) map[string]bool {
remote := map[string]bool{}
for _, h := range rr.Herdrs() {
if !coordinatorOwnsHerdr(h, localMachine) {
remote[herdrAddress(rr, h)] = true
}
}
return remote
}
// coordinatorOwnsHerdr identifies the only herdr sockets the coordinator may
// probe or adapt. In federation mode a remote pane belongs to its worker;
// reaching into that machine would turn a worker-owned health signal back
// into a misleading coordinator TCP result.
func coordinatorOwnsHerdr(h registry.Herdr, localMachine string) bool {
if h.Backend == "tmux" {
return false
}
return localMachine == "" || h.MachineID == localMachine
}
func (a federatedAvailability) Available(h registry.Herdr) bool {
return a.Unavailable(h) == ""
}
// Unavailable keeps the base gate's own reason instead of restating every
// refusal as worker health. A quota refusal reported as a stale heartbeat sent
// burn-in run 2 looking at a worker that was one second fresh.
func (a federatedAvailability) Unavailable(h registry.Herdr) string {
if a.base != nil {
if reason := unavailableReason(a.base, h); reason != "" {
return reason
}
}
if coordinatorOwnsHerdr(h, a.localMachine) {
return ""
}
return a.workers.Unavailable(h.ID)
}
// unavailableReason mirrors router.unavailableReason for the wrapped base gate.
func unavailableReason(a router.Availability, h registry.Herdr) string {
if ra, ok := a.(router.ReasonedAvailability); ok {
return ra.Unavailable(h)
}
if a.Available(h) {
return ""
}
return "worker unavailable"
}
func (a federatedAvailability) Supports(h registry.Herdr, project string) bool {
// Locally-owned herdrs keep their static registry/project affinity. A
// remote worker must additionally prove it has a local checkout for the
// project before the router can offer it a lease.
if coordinatorOwnsHerdr(h, a.localMachine) {
return true
}
return a.workers.Supports(h.ID, project)
}
func validateLocalMachine(rr registry.Registry, localMachine string) error {
machines := rr.Machines()
requiresWorkerOwnership := false
for _, h := range rr.Herdrs() {
if h.Backend == "tmux" {
requiresWorkerOwnership = true
break
}
}
if len(machines) <= 1 && !requiresWorkerOwnership {
return nil
}
if localMachine == "" {
return fmt.Errorf("ORCHESTRA_MACHINE_ID is required when the registry is multi-machine or has worker-owned backends")
}
if _, ok := rr.Machine(localMachine); !ok {
return fmt.Errorf("ORCHESTRA_MACHINE_ID %q is not in the registry", localMachine)
}
return nil
}
func main() {
dir := os.Getenv("ORCHESTRA_DATA")
if dir == "" {
dir = "./data"
}
s, err := store.Open(dir)
if err != nil {
log.Fatal(err)
}
users, err := authn.Open(authn.Path(dir))
if err != nil {
log.Fatal(err)
}
defer users.Close()
userCount, err := users.Count()
if err != nil {
log.Fatalf("read operator database: %v", err)
}
legacyUsername := os.Getenv("ORCHESTRA_WEB_USERNAME")
legacyPasswordHash := os.Getenv("ORCHESTRA_WEB_PASSWORD_HASH")
if userCount == 0 {
if legacyUsername != "" || legacyPasswordHash != "" {
if legacyUsername == "" || legacyPasswordHash == "" {
log.Fatal("operator database is empty and the legacy web credential is incomplete: both ORCHESTRA_WEB_USERNAME and ORCHESTRA_WEB_PASSWORD_HASH are required for one-time migration")
}
imported, importErr := users.ImportBcrypt(legacyUsername, legacyPasswordHash)
if importErr != nil {
log.Fatalf("migrate legacy web credential: %v", importErr)
}
if imported {
userCount = 1
log.Printf("migrated web operator %q into %s; remove ORCHESTRA_WEB_USERNAME and ORCHESTRA_WEB_PASSWORD_HASH from the deployment environment", legacyUsername, authn.Path(dir))
}
}
if userCount == 0 {
log.Fatalf("operator database is empty: stop Orchestra and run orchestra-user set -data %s -username NAME", dir)
}
} else if legacyUsername != "" || legacyPasswordHash != "" {
log.Printf("operator database already contains %d account(s); legacy ORCHESTRA_WEB_USERNAME and ORCHESTRA_WEB_PASSWORD_HASH are ignored and should be removed", userCount)
}
var rr registry.Registry
var rt *router.Router
var coordinator *orchestrator.Coordinator
// submissionPublisher is nil until a forge is configured. Submission then
// returns the verified plan instead of performing it, which keeps `task
// pr` the only path without inventing a fake success.
var submissionPublisher func(operations.SubmissionPlan) operations.Publisher
// Worktree root per project, so a submission can find the checkout that
// holds the commit it is publishing.
projectRoots := map[string]string{}
// Pull-request readers by source name, for reflecting submitted work.
pullRequests := map[string]human.PullRequestSource{}
localMachine := os.Getenv("ORCHESTRA_MACHINE_ID")
workers := &federation.Registry{AdmitToken: os.Getenv("ORCHESTRA_FEDERATION_ADMIT_TOKEN"), StatePath: filepath.Join(dir, "federation-state.json")}
if err := workers.Load(); err != nil {
log.Fatalf("load federation state: %v", err)
}
if config := os.Getenv("ORCHESTRA_CONFIG"); config != "" {
if rr, err = registry.Load(config); err != nil {
log.Fatalf("load orchestra config: %v", err)
}
if err := validateLocalMachine(rr, localMachine); err != nil {
log.Fatal(err)
}
reachability := registry.Reachability(registry.TCPReachability{})
if localMachine != "" {
reachability = federatedReachability{base: reachability, remote: remoteHerdrAddresses(rr, localMachine)}
}
rt = &router.Router{Store: s, Registry: rr, Reachability: reachability, Timeout: time.Second, Retry: router.RetryPolicy{MaxAttempts: 3, Backoff: time.Minute}}
limits := map[string]router.QuotaWindowLimits{}
for _, h := range rr.Herdrs() {
w := router.QuotaWindowLimits{FiveHour: h.QuotaLimit5h, Weekly: h.QuotaLimitWeekly}
if w.Weekly <= 0 && h.QuotaLimit > 0 {
// Back-compat: the old single-window field meant weekly.
w.Weekly = h.QuotaLimit
}
if w.FiveHour > 0 || w.Weekly > 0 {
limits[h.ID] = w
}
}
if len(limits) > 0 {
rt.Availability = router.QuotaAvailability{Store: s, Limits: limits}
}
if localMachine != "" {
rt.Availability = federatedAvailability{base: rt.Availability, workers: workers, localMachine: localMachine}
}
if repo, root := os.Getenv("ORCHESTRA_REPO"), os.Getenv("ORCHESTRA_WORKTREE_ROOT"); repo != "" && root != "" {
adapters := map[string]herdr.Adapter{}
for _, h := range rr.Herdrs() {
if !coordinatorOwnsHerdr(h, localMachine) {
log.Printf("herdr %s is worker-owned on %s; coordinator probe skipped", h.ID, h.MachineID)
continue
}
address := herdrAddress(rr, h)
if address == "" {
continue
}
client := herdr.New(address)
protocol := h.Protocol
if protocol == "" {
protocol = os.Getenv("ORCHESTRA_HERDR_PROTOCOL")
}
if err := client.CheckProtocol(context.Background(), protocol); err != nil {
log.Printf("herdr %s unavailable: %v", h.ID, err)
continue
}
log.Printf("herdr %s reachable at %s (protocol %s, harness %s)", h.ID, address, protocol, h.Harness)
switch h.Harness {
case "claude":
adapters[h.ID] = herdr.Claude(client, 200000, s)
case "opencode":
adapters[h.ID] = herdr.OpenCode(client, 200000, s)
case "codex", "":
adapters[h.ID] = herdr.Codex(client, 200000, s)
default:
log.Printf("herdr %s has unsupported harness %q", h.ID, h.Harness)
}
}
projectRepos := map[string]orchestrator.ProjectRepo{}
for _, p := range rr.Projects() {
if p.Repo != "" && p.WorktreeRoot != "" {
projectRepos[p.ID] = orchestrator.ProjectRepo{Repo: p.Repo, WorktreeRoot: p.WorktreeRoot}
projectRoots[p.ID] = p.WorktreeRoot
}
}
worktrees := orchestrator.PerProjectGitWorktrees{
Projects: projectRepos,
Default: orchestrator.GitWorktrees{Root: root, Repo: repo},
}
coordinator = &orchestrator.Coordinator{Store: s, StatePath: filepath.Join(dir, "runtime-sessions.json"), Worktrees: worktrees, Adapters: orchestrator.AdapterFactory{Herdrs: adapters}, LocalHerdr: func(id string) bool {
h, ok := rr.Herdr(id)
return ok && coordinatorOwnsHerdr(h, localMachine)
}}
rt.OnLease = func(e domain.Event) error {
// In federated mode the coordinator must never inspect a worker-owned
// checkout. Its worker consumes the router-issued lease event and
// performs all Git/backend operations on that machine (§2.1).
if localMachine != "" {
var p struct {
HarnessID string `json:"harness_id"`
}
if json.Unmarshal(e.Payload, &p) == nil {
if h, ok := rr.Herdr(p.HarnessID); ok && !coordinatorOwnsHerdr(h, localMachine) {
return nil
}
}
}
return coordinator.Start(context.Background(), e)
}
hard := 0.75
if v, parseErr := strconv.ParseFloat(os.Getenv("ORCHESTRA_OCCUPANCY_HARD"), 64); parseErr == nil && v > 0 && v < 1 {
hard = v
}
if v, parseErr := strconv.ParseFloat(os.Getenv("ORCHESTRA_OCCUPANCY_SOFT"), 64); parseErr == nil && v > 0 && v < 1 {
coordinator.Soft = v
}
go func() {
if monitorErr := coordinator.Monitor(context.Background(), hard, 30*time.Second); monitorErr != nil {
log.Printf("orchestrator monitor: %v", monitorErr)
}
}()
}
}
mux := http.NewServeMux()
// B18: the UI is a full control plane — it can create tasks, release or
// complete them, and inject approval keystrokes into live panes. Browser
// sessions are backed by operator accounts in the embedded auth database;
// no missing environment variable can open this surface.
sessions := &authz.Sessions{}
browserAuth := authn.HTTP{Users: users, Sessions: sessions, SecureCookie: os.Getenv("ORCHESTRA_UI_INSECURE_COOKIE") == ""}
mux.HandleFunc(authz.SessionPath, browserAuth.Session)
mux.HandleFunc("/v1/ui/account", browserAuth.Account)
// Browser-specific endpoints intentionally present a joined read model;
// raw lifecycle endpoints below remain stable for workers and harnesses.
mux.Handle("/v1/ui/", ui.Server{Store: s, Workers: workers, Coordinator: coordinator, Route: func(e domain.Event) error {
if rt == nil {
return nil
}
_, err := rt.HandleEvent(e)
return err
}}.Handler())
mux.Handle("/", webui.Handler())
// A missed heartbeat is not relinquishment. Releasing here used to lease
// the same task to a successor while the old pane was still running. The
// authoritative lease timer performs the only automatic reassignment.
workers.OnOffline = func(w federation.Worker) { log.Printf("worker %s offline; retaining leases until expiry", w.ID) }
go func() {
ticker := time.NewTicker(15 * time.Second)
defer ticker.Stop()
for range ticker.C {
_ = workers.Snapshot()
}
}()
providerHealth := map[string]*provider.Supervisor{}
surface := func(r *http.Request) authz.Surface {
v := authz.ParseSurface(r.Header.Get("X-Orchestra-Surface"))
if v == "" || v == authz.System {
// System means "the plane itself, in-process" and is always
// FullControl with no token gate (AUDIT.md B8) — it must never
// be constructible from an HTTP request, or any LAN caller
// declaring this header gets unauthenticated full control.
return authz.Web
}
return v
}
mux.HandleFunc("/v1/tasks", func(w http.ResponseWriter, r *http.Request) {
if wid := r.Header.Get("X-Orchestra-Worker"); wid != "" {
token := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
if err := workers.Authenticate(wid, token); err != nil {
http.Error(w, err.Error(), http.StatusUnauthorized)
return
}
}
if r.Method == "GET" {
json.NewEncoder(w).Encode(s.Tasks())
return
}
if r.Method != "POST" {
http.Error(w, "method not allowed", 405)
return
}
var p map[string]any
if json.NewDecoder(r.Body).Decode(&p) != nil {
http.Error(w, "invalid json", 400)
return
}
// Make the selected project's deterministic gate part of the immutable
// task contract before any worker can create TASK.md. A caller may
// override it only when it has deliberately supplied a task-specific
// gate; routing never asks a harness to choose one.
if _, set := p["quality_gate"]; !set {
if projectID, _ := p["project"].(string); projectID != "" {
if project, ok := rr.Project(projectID); ok && project.QualityGate != "" {
p["quality_gate"] = project.QualityGate
}
}
}
b, _ := json.Marshal(p)
e := domain.Event{ID: id(), Type: "TaskCreated", TaskID: id(), Version: 1, Payload: b, Surface: string(surface(r))}
if err := s.Append(e); err != nil {
if errors.Is(err, domain.ErrDuplicate) {
// (source, external_id) was already ingested. Nothing was
// appended; report the existing task idempotently rather
// than fabricating a 201 with an unrelated event (S6).
source, _ := p["source"].(string)
externalID, _ := p["external_id"].(string)
if t, ok := s.TaskBySource(source, externalID); ok {
w.WriteHeader(200)
json.NewEncoder(w).Encode(t)
return
}
}
http.Error(w, err.Error(), 400)
return
}
if rt != nil {
if _, err := rt.HandleEvent(e); err != nil {
log.Printf("route task: %v", err)
}
}
w.WriteHeader(201)
json.NewEncoder(w).Encode(e)
})
mux.HandleFunc("/v1/events", func(w http.ResponseWriter, r *http.Request) {
var n uint64
if x, err := strconv.ParseUint(r.URL.Query().Get("since"), 10, 64); err == nil {
n = x
}
json.NewEncoder(w).Encode(s.Events(n))
})
mux.HandleFunc("/v1/handoffs", func(w http.ResponseWriter, r *http.Request) {
out := make([]domain.Event, 0)
for _, e := range s.Events(0) {
if e.Type == "TaskReleased" {
out = append(out, e)
}
}
json.NewEncoder(w).Encode(out)
})
mux.HandleFunc("/v1/quotas", func(w http.ResponseWriter, r *http.Request) {
out := make([]domain.Event, 0)
for _, e := range s.Events(0) {
if e.Type == "QuotaReported" {
out = append(out, e)
}
}
json.NewEncoder(w).Encode(out)
})
mux.HandleFunc("/v1/artifacts", func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
if wid := r.Header.Get("X-Orchestra-Worker"); wid != "" {
token := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
if err := workers.Authenticate(wid, token); err != nil {
http.Error(w, err.Error(), http.StatusUnauthorized)
return
}
}
// Reports and handoffs are content-addressed evidence. Keep uploads
// bounded because event payloads only carry their resulting hash.
r.Body = http.MaxBytesReader(w, r.Body, 4<<20)
b, err := io.ReadAll(r.Body)
if err != nil {
http.Error(w, "artifact too large or unreadable", http.StatusRequestEntityTooLarge)
return
}
if len(b) == 0 {
http.Error(w, "artifact is empty", http.StatusBadRequest)
return
}
ref, err := s.PutArtifact(b)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusCreated)
json.NewEncoder(w).Encode(map[string]string{"ref": ref})
})
// Artifacts are normally write-only to public surfaces. A federation
// worker may read a handoff only after authenticating as the worker that
// will validate it against its own checkout (§2.1, §6.2).
mux.HandleFunc("/v1/artifacts/", func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
wid := r.Header.Get("X-Orchestra-Worker")
token := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
if err := workers.Authenticate(wid, token); err != nil {
http.Error(w, err.Error(), http.StatusUnauthorized)
return
}
ref := strings.TrimPrefix(r.URL.Path, "/v1/artifacts/")
if len(ref) != 64 {
http.Error(w, "artifact ref required", http.StatusBadRequest)
return
}
b, err := s.Artifact(ref)
if err != nil {
http.Error(w, "artifact not found", http.StatusNotFound)
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write(b)
})
harnessToken := os.Getenv("ORCHESTRA_HARNESS_TOKEN")
// /v1/harness/turn is the unified turn-decision endpoint (AUDIT.md Phase
// 2 items 1-2): a harness-side caller posts here on an ordinary turn
// boundary and gets back continue / prepare_handoff / rotate_now / refuse,
// per spec §5.3, instead of separate ad-hoc marker-file conventions per
// decision.
//
// Completion does NOT go through this endpoint, and there is no longer a
// /v1/harness/complete: an unaffiliated harness hook has no durable worker
// identity or fencing epoch, so it cannot safely mutate a leased task.
// orchestra-worker owns completion — it watches for the .orchestra/done
// marker, confirms the agent is no longer busy, and posts through the
// authenticated federation endpoints with both lease epoch and version.
mux.HandleFunc("/v1/harness/turn", func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
if harnessToken != "" && subtle.ConstantTimeCompare([]byte(r.Header.Get("Authorization")), []byte("Bearer "+harnessToken)) != 1 {
http.Error(w, "unauthorized", http.StatusUnauthorized)
return
}
if coordinator == nil {
http.Error(w, "coordinator not configured", http.StatusServiceUnavailable)
return
}
var p struct {
TaskID string `json:"task_id"`
}
if json.NewDecoder(r.Body).Decode(&p) != nil || p.TaskID == "" {
http.Error(w, "task_id is required", http.StatusBadRequest)
return
}
decision, err := coordinator.TurnDecision(r.Context(), p.TaskID)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
json.NewEncoder(w).Encode(map[string]string{"decision": decision})
})
mux.HandleFunc("/v1/brief", func(w http.ResponseWriter, r *http.Request) {
to := time.Now().UTC()
from := to.Add(-12 * time.Hour)
if v, parseErr := time.Parse(time.RFC3339, r.URL.Query().Get("from")); parseErr == nil {
from = v
}
if v, parseErr := time.Parse(time.RFC3339, r.URL.Query().Get("to")); parseErr == nil {
to = v
}
git := map[string]operations.GitSync{}
for _, p := range rr.Projects() {
if p.Repo == "" {
continue
}
git[p.ID] = operations.GitState(p.Repo)
}
if len(git) == 0 {
if repo := os.Getenv("ORCHESTRA_REPO"); repo != "" {
git["default"] = operations.GitState(repo)
}
}
json.NewEncoder(w).Encode(operations.BuildBrief(s.Events(0), from, to, git))
})
standup := func() (domain.Event, error) {
return operations.GenerateStandupAdvisory(s, time.Now().UTC())
}
mux.HandleFunc("/v1/standup", func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodGet {
json.NewEncoder(w).Encode(operations.StandupItems(s.Tasks()))
return
}
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", 405)
return
}
if err := authz.AuthorizeEvent(surface(r), "StandupAdvisory"); err != nil {
http.Error(w, err.Error(), http.StatusForbidden)
return
}
e, err := standup()
if err != nil {
http.Error(w, err.Error(), 400)
return
}
json.NewEncoder(w).Encode(e)
})
mux.HandleFunc("/v1/standup/approve", func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", 405)
return
}
if err := authz.AuthorizeEvent(surface(r), "ApprovalGranted"); err != nil {
http.Error(w, err.Error(), 403)
return
}
var p struct {
AdvisoryID string `json:"advisory_id"`
}
if json.NewDecoder(r.Body).Decode(&p) != nil || p.AdvisoryID == "" {
http.Error(w, "advisory_id required", 400)
return
}
b, _ := json.Marshal(map[string]any{"subject_ref": p.AdvisoryID})
e := domain.Event{ID: id(), Type: "ApprovalGranted", TaskID: "system", Version: 0, Payload: b, Surface: string(surface(r))}
if err := s.Append(e); err != nil {
http.Error(w, err.Error(), 400)
return
}
json.NewEncoder(w).Encode(e)
})
mux.HandleFunc("/v1/standup/apply", func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", 405)
return
}
if err := authz.AuthorizeEvent(surface(r), "TaskAmended"); err != nil {
http.Error(w, err.Error(), 403)
return
}
var p struct {
AdvisoryID string `json:"advisory_id"`
}
if json.NewDecoder(r.Body).Decode(&p) != nil || p.AdvisoryID == "" {
http.Error(w, "advisory_id required", 400)
return
}
out, err := operations.ApplyAdvisory(s, p.AdvisoryID)
if err != nil {
http.Error(w, err.Error(), 400)
return
}
json.NewEncoder(w).Encode(out)
})
adminServer := &admin.Server{Store: s, RouterReady: rt != nil, Build: buildinfo.Current(), Providers: providerHealth, Probes: map[string]admin.ProbeFunc{
"router": func() (bool, string) { return rt != nil, "configured router" },
"gitea": func() (bool, string) {
configured := os.Getenv("ORCHESTRA_GITEA_URL") != "" || os.Getenv("ORCHESTRA_GITEA_CONFIG") != ""
if !configured {
return true, "configured provider"
}
for name := range providerHealth {
if name == "gitea" || strings.HasPrefix(name, "gitea:") {
return true, "configured provider"
}
}
return false, "configured provider"
},
"jsonl": func() (bool, string) {
return os.Getenv("ORCHESTRA_JSONL") == "" || providerHealth["jsonl"] != nil, "configured provider"
},
}}
mux.HandleFunc("/metrics", adminServer.Metrics)
mux.HandleFunc("/v1/events/subscribe", adminServer.Subscribe)
mux.HandleFunc("/v1/admin/diagnostics", adminServer.Diagnostics)
mux.HandleFunc("/v1/tasks/", func(w http.ResponseWriter, r *http.Request) {
parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/")
if r.Method == http.MethodGet && len(parts) == 4 && parts[3] == "intent" {
// The reduced authority for one task. Federation workers read this
// and render their own launch instruction with agentctx, so there
// is one renderer in the codebase rather than one per machine.
intent, err := s.EffectiveIntent(parts[2])
if err != nil {
http.Error(w, err.Error(), 404)
return
}
json.NewEncoder(w).Encode(intent)
return
}
if r.Method == http.MethodGet && coordinator != nil && len(parts) == 4 {
taskID, view := parts[2], parts[3]
if view == "health" {
if h, ok := coordinator.MonitorHealth().Sessions[taskID]; ok {
json.NewEncoder(w).Encode(h)
return
}
http.Error(w, "session health not found", 404)
return
}
if view == "capture" {
body, err := coordinator.Capture(r.Context(), taskID, r.URL.Query().Get("source"))
if err != nil {
http.Error(w, err.Error(), 404)
return
}
json.NewEncoder(w).Encode(map[string]string{"task_id": taskID, "source": r.URL.Query().Get("source"), "text": body})
return
}
}
if len(parts) < 4 || len(parts) > 5 || r.Method != "POST" {
http.Error(w, "not found", http.StatusNotFound)
return
}
taskID, action := parts[2], parts[3]
if (action == "decision-request" || action == "deferred") && len(parts) == 4 {
// An agent surface may state a bounded question or a deferred
// finding. Neither moves the lifecycle: Orchestra decides what a
// question does to the task.
if err := authz.AuthorizeEvent(surface(r), "ApprovalRequested"); err != nil {
http.Error(w, err.Error(), http.StatusForbidden)
return
}
t, ok := s.Task(taskID)
if !ok {
http.Error(w, "task not found", 404)
return
}
project, _ := rr.Project(t.Project)
r.Body = http.MaxBytesReader(w, r.Body, 64<<10)
if action == "deferred" {
var f domain.DeferredFinding
if json.NewDecoder(r.Body).Decode(&f) != nil {
http.Error(w, "invalid deferred finding", http.StatusBadRequest)
return
}
e, err := operations.RecordDeferredFinding(s, taskID, f)
if err != nil {
http.Error(w, err.Error(), 409)
return
}
json.NewEncoder(w).Encode(e)
return
}
var req domain.DecisionRequest
if json.NewDecoder(r.Body).Decode(&req) != nil {
http.Error(w, "invalid decision request", http.StatusBadRequest)
return
}
e, err := operations.RequestHumanDecision(s, project, taskID, req)
if errors.Is(err, operations.ErrDecisionBudgetSpent) {
w.WriteHeader(http.StatusAccepted)
json.NewEncoder(w).Encode(map[string]string{"status": "operator_required", "detail": err.Error()})
return
}
if err != nil {
http.Error(w, err.Error(), 409)
return
}
w.WriteHeader(http.StatusAccepted)
json.NewEncoder(w).Encode(e)
return
}
if action == "submission" && len(parts) == 4 {
// The only path from a reviewed implementation to the human. It
// verifies first and returns the plan, or performs the submission
// when the caller supplies a publisher-backed request.
if err := authz.AuthorizeEvent(surface(r), domain.EventTaskSubmitted); err != nil {
http.Error(w, err.Error(), http.StatusForbidden)
return
}
t, ok := s.Task(taskID)
if !ok {
http.Error(w, "task not found", 404)
return
}
project, ok := rr.Project(t.Project)
if !ok {
http.Error(w, "unknown project "+t.Project, 409)
return
}
r.Body = http.MaxBytesReader(w, r.Body, 1<<20)
var body struct {
HeadSHA string `json:"head_sha"`
Gate domain.GateResult `json:"gate"`
Notes operations.Notes `json:"notes"`
DryRun bool `json:"dry_run"`
}
if json.NewDecoder(r.Body).Decode(&body) != nil || body.HeadSHA == "" {
http.Error(w, "head_sha and gate are required", http.StatusBadRequest)
return
}
check := domain.CheckSubmission(t, body.HeadSHA, body.Gate)
check.Reasons = append(check.Reasons, t.RequirePhaseArtifacts(project.Phases())...)
check.Eligible = len(check.Reasons) == 0
if body.DryRun || !check.Eligible {
code := http.StatusOK
if !check.Eligible {
code = http.StatusConflict
}
w.WriteHeader(code)
json.NewEncoder(w).Encode(check)
return
}
plan, err := operations.PrepareSubmission(s, project, taskID, body.HeadSHA, body.Gate, body.Notes)
if err != nil {
http.Error(w, err.Error(), http.StatusConflict)
return
}
if submissionPublisher == nil {
// No forge configured: hand back the verified plan so an
// operator can perform the submission themselves.
w.WriteHeader(http.StatusAccepted)
json.NewEncoder(w).Encode(map[string]any{"status": "no_publisher", "plan": plan})
return
}
e, err := operations.ExecuteSubmission(r.Context(), s, plan, submissionPublisher(plan), nil)
if err != nil {
http.Error(w, err.Error(), http.StatusConflict)
return
}
json.NewEncoder(w).Encode(e)
return
}
if action == "review" && len(parts) == 4 {
// Entering review and sealing a review are both Orchestra's, not
// the reviewing session's. The session only supplies findings.
if err := authz.AuthorizeEvent(surface(r), domain.EventReviewRecorded); err != nil {
http.Error(w, err.Error(), http.StatusForbidden)
return
}
t, ok := s.Task(taskID)
if !ok {
http.Error(w, "task not found", 404)
return
}
project, ok := rr.Project(t.Project)
if !ok {
http.Error(w, "unknown project "+t.Project, 409)
return
}
r.Body = http.MaxBytesReader(w, r.Body, int64(review.MaxDiffBytes)+(1<<20))
var body struct {
Evidence *review.Evidence `json:"evidence"`
Result *review.Result `json:"result"`
}
if json.NewDecoder(r.Body).Decode(&body) != nil || (body.Evidence == nil) == (body.Result == nil) {
http.Error(w, "send either evidence (to enter review) or result (to seal one)", http.StatusBadRequest)
return
}
var e domain.Event
var err error
if body.Evidence != nil {
e, err = operations.EnterReview(s, project, taskID, *body.Evidence)
} else {
e, err = operations.RecordReview(s, project, taskID, *body.Result)
}
if errors.Is(err, operations.ErrReviewNotEligible) || errors.Is(err, domain.ErrInvalid) {
http.Error(w, err.Error(), http.StatusConflict)
return
}
if err != nil {
http.Error(w, err.Error(), 409)
return
}
json.NewEncoder(w).Encode(e)
return
}
if action == "phase" && len(parts) == 4 {
// Orchestra owns the phase. An agent asks for a change through the
// approval surface; this endpoint is how the decision is applied.
if err := authz.AuthorizeEvent(surface(r), domain.EventWorkPhaseChanged); err != nil {
http.Error(w, err.Error(), http.StatusForbidden)
return
}
t, ok := s.Task(taskID)
if !ok {
http.Error(w, "task not found", 404)
return
}
project, ok := rr.Project(t.Project)
if !ok {
http.Error(w, "unknown project "+t.Project, 409)
return
}
// The body, when present, is the artifact the finished phase
// produced. Bounded like every other artifact upload.
r.Body = http.MaxBytesReader(w, r.Body, 1<<20)
artifact, readErr := io.ReadAll(r.Body)
if readErr != nil {
http.Error(w, "artifact unreadable", http.StatusRequestEntityTooLarge)
return
}
e, err := operations.AdvanceWorkPhase(s, project, taskID, artifact)
if errors.Is(err, operations.ErrTrajectoryGate) {
// Not a failure: the task is blocked on the human, and the
// packet is on the block event the notification surfaces read.
w.WriteHeader(http.StatusAccepted)
json.NewEncoder(w).Encode(map[string]string{"status": "trajectory_gate", "detail": err.Error()})
return
}
if err != nil {
http.Error(w, err.Error(), 409)
return
}
json.NewEncoder(w).Encode(e)
return
}
if action == "approval" {
if err := authz.AuthorizeEvent(surface(r), map[bool]string{true: "ApprovalRequested", false: "ApprovalGranted"}[len(parts) == 4]); err != nil && len(parts) == 4 {
http.Error(w, err.Error(), http.StatusForbidden)
return
}
if len(parts) == 5 {
typ := "ApprovalGranted"
if parts[4] == "deny" {
typ = "ApprovalDenied"
}
if err := authz.AuthorizeEvent(surface(r), typ); err != nil {
http.Error(w, err.Error(), http.StatusForbidden)
return
}
if parts[4] != "grant" && parts[4] != "deny" {
http.Error(w, "unknown approval action", 404)
return
}
t, ok := s.Task(taskID)
if !ok {
http.Error(w, "task not found", 404)
return
}
by := r.Header.Get("X-Orchestra-Actor")
if by == "" {
by = "surface"
}
b, _ := json.Marshal(map[string]any{"subject_ref": taskID, "by": by})
e := domain.Event{ID: id(), Type: typ, TaskID: taskID, Version: t.Version + 1, Payload: b, Surface: string(surface(r))}
if err := s.Append(e); err != nil {
http.Error(w, err.Error(), 409)
return
}
json.NewEncoder(w).Encode(e)
return
}
var p struct {
Options []any `json:"options"`
}
if json.NewDecoder(r.Body).Decode(&p) != nil || len(p.Options) == 0 {
http.Error(w, "options required", 400)
return
}
if _, ok := s.Task(taskID); !ok {
http.Error(w, "task not found", 404)
return
}
b, _ := json.Marshal(map[string]any{"subject_ref": taskID, "options": p.Options})
t, _ := s.Task(taskID)
e := domain.Event{ID: id(), Type: "ApprovalRequested", TaskID: taskID, Version: t.Version + 1, Payload: b, Surface: string(surface(r))}
if err := s.Append(e); err != nil {
http.Error(w, err.Error(), 400)
return
}
json.NewEncoder(w).Encode(e)
return
}
var e domain.Event
var err error
actionTypes := map[string]string{"lease": "TaskLeased", "release": "TaskReleased", "complete": "TaskCompleted", "block": "TaskBlocked", "attention": "TaskNeedsAttention", "retry": "TaskCorrected"}
if typ, known := actionTypes[action]; known {
if err := authz.AuthorizeEvent(surface(r), typ); err != nil {
http.Error(w, err.Error(), http.StatusForbidden)
return
}
}
switch action {
case "retry":
// Narrow on purpose: RetryTask restores a retry budget on a
// terminal task. There is no generic correction endpoint, because
// one would be arbitrary task mutation over HTTP.
var p struct {
OperationID string `json:"operation_id"`
}
if r.Body == nil || json.NewDecoder(r.Body).Decode(&p) != nil || p.OperationID == "" {
http.Error(w, "operation_id required", http.StatusBadRequest)
return
}
e, err = operations.RetryTask(s, surface(r), taskID, p.OperationID)
switch {
case errors.Is(err, domain.ErrNotFound):
http.Error(w, err.Error(), http.StatusNotFound)
return
case errors.Is(err, operations.ErrNotRetryable), errors.Is(err, domain.ErrInvalid):
http.Error(w, err.Error(), http.StatusConflict)
return
}
case "lease":
var p struct {
HarnessID string `json:"harness_id"`
TTLSeconds int `json:"ttl_seconds"`
}
if json.NewDecoder(r.Body).Decode(&p) != nil || p.HarnessID == "" {
http.Error(w, "harness_id required", 400)
return
}
if p.TTLSeconds <= 0 {
p.TTLSeconds = int(domain.LeaseTTL.Seconds())
}
e, err = s.Lease(taskID, p.HarnessID, time.Duration(p.TTLSeconds)*time.Second)
case "release", "complete", "block", "attention":
t, ok := s.Task(taskID)
if !ok {
http.Error(w, "task not found", 404)
return
}
types := map[string]string{"release": "TaskReleased", "complete": "TaskCompleted", "block": "TaskBlocked", "attention": "TaskNeedsAttention"}
var p map[string]any
if r.Body == nil || json.NewDecoder(r.Body).Decode(&p) != nil || p == nil {
http.Error(w, "invalid lifecycle payload", http.StatusBadRequest)
return
}
if action == "release" && p["reason"] == nil && p["handoff_ref"] == nil {
http.Error(w, "reason or handoff_ref required", http.StatusBadRequest)
return
}
if (action == "block" || action == "attention") && p["blocker"] == nil {
http.Error(w, "blocker required", http.StatusBadRequest)
return
}
if action == "complete" && p["report_ref"] == nil {
http.Error(w, "report_ref required", http.StatusBadRequest)
return
}
ePayload, _ := json.Marshal(p)
e = domain.Event{ID: id(), Type: types[action], TaskID: taskID, Version: t.Version + 1, Payload: ePayload, Surface: string(surface(r))}
err = s.Append(e)
default:
http.Error(w, "unknown action", 404)
return
}
if err != nil {
http.Error(w, err.Error(), 409)
return
}
if rt != nil {
if _, routeErr := rt.HandleEvent(e); routeErr != nil {
log.Printf("route task: %v", routeErr)
}
}
json.NewEncoder(w).Encode(e)
})
if rt != nil {
go func() {
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for range ticker.C {
// When a coordinator is running, its own Monitor loop (every
// 30s) already calls Store.ExpireLeases and, critically,
// kills the herdr session for whatever it expires (S9: two
// independent expiry loops raced on the same reclaim, and
// since this one runs every second vs. the coordinator's
// 30s, it almost always won — meaning the coordinator's
// ExpireLeases call saw nothing left to expire and its
// session-kill path never ran, silently orphaning panes
// past their lease TTL). Calling ExpireLeases here too would
// just resurrect that race, so leave reclaim to the
// coordinator and only keep retrying pending assignment.
// A blocked task is invisible to the router, so answered
// blockers are returned to the queue here, immediately
// upstream of assignment.
if _, err := operations.ResumeAnsweredBlockers(s); err != nil {
log.Printf("resume answered blockers: %v", err)
}
if coordinator != nil {
if _, err := rt.AssignPending(); err != nil {
log.Printf("route expired task: %v", err)
}
continue
}
if _, err := s.ExpireLeases(time.Now()); err != nil {
log.Printf("expire leases: %v", err)
} else if _, err := rt.AssignPending(); err != nil {
log.Printf("route expired task: %v", err)
}
}
}()
}
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("ok\n")) })
mux.HandleFunc("/readyz", adminServer.Readiness)
mux.HandleFunc("/v1/providers/health", func(w http.ResponseWriter, r *http.Request) {
out := map[string]provider.Health{}
for name, sup := range providerHealth {
out[name] = sup.Health()
}
json.NewEncoder(w).Encode(out)
})
mux.HandleFunc("/v1/router/health", func(w http.ResponseWriter, r *http.Request) {
// Why the last scheduling pass placed nothing. A silent eligibility
// gate is indistinguishable from an empty queue, which cost a burn-in
// run to diagnose by hand.
if rt == nil {
http.Error(w, "router not configured", http.StatusServiceUnavailable)
return
}
json.NewEncoder(w).Encode(map[string]any{"rejections": rt.Rejections()})
})
mux.HandleFunc("/v1/orchestrator/health", func(w http.ResponseWriter, r *http.Request) {
if coordinator == nil {
http.Error(w, "orchestrator unavailable", http.StatusServiceUnavailable)
return
}
json.NewEncoder(w).Encode(coordinator.MonitorHealth())
})
mux.HandleFunc("/v1/federation/workers", func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodGet {
json.NewEncoder(w).Encode(workers.Snapshot())
return
}
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", 405)
return
}
var registration struct {
federation.Worker
Token string `json:"token"`
}
if json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10)).Decode(&registration) != nil {
http.Error(w, "invalid worker", 400)
return
}
worker := registration.Worker
worker.Token = registration.Token
if worker.Token == "" {
http.Error(w, "token required", 400)
return
}
admitToken := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
if err := workers.Register(worker, admitToken); err != nil {
status := 400
if err == federation.ErrUnauthorized {
status = 401
}
http.Error(w, err.Error(), status)
return
}
if rt != nil {
if _, err := rt.AssignPending(); err != nil {
log.Printf("route after worker registration: %v", err)
}
}
w.WriteHeader(http.StatusCreated)
json.NewEncoder(w).Encode(worker)
})
workerAuth := func(r *http.Request) (string, error) {
wid := r.Header.Get("X-Orchestra-Worker")
tok := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
if err := workers.Authenticate(wid, tok); err != nil {
return "", err
}
return wid, nil
}
mux.HandleFunc("/v1/federation/turn", func(w http.ResponseWriter, r *http.Request) {
if _, err := workerAuth(r); err != nil {
http.Error(w, err.Error(), http.StatusUnauthorized)
return
}
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", 405)
return
}
if coordinator == nil {
http.Error(w, "coordinator not configured", http.StatusServiceUnavailable)
return
}
var body struct {
TaskID string `json:"task_id"`
LeaseEpoch string `json:"lease_epoch"`
Verdict string `json:"verdict"`
Delivered []string `json:"delivered_decisions"`
}
if json.NewDecoder(r.Body).Decode(&body) != nil || body.TaskID == "" || body.LeaseEpoch == "" {
http.Error(w, "task_id and lease_epoch are required", http.StatusBadRequest)
return
}
switch body.Verdict {
case orchestrator.TurnContinue, orchestrator.TurnPrepareHandoff, orchestrator.TurnRotateNow, orchestrator.TurnRefuse:
default:
http.Error(w, "unknown verdict "+body.Verdict, http.StatusBadRequest)
return
}
verdict, decisions, err := coordinator.RemoteTurn(r.Context(), body.TaskID, body.LeaseEpoch, body.Verdict, body.Delivered)
if err != nil {
http.Error(w, err.Error(), http.StatusConflict)
return
}
json.NewEncoder(w).Encode(federation.TurnDecision{Verdict: verdict, Decisions: decisions})
})
// F21. The worker's report of an agent's bounded phase-change request.
// The agent asks by writing a file; Orchestra decides here, through the
// same AdvanceWorkPhase the operator surface uses.
mux.HandleFunc("/v1/federation/phase", func(w http.ResponseWriter, r *http.Request) {
if _, err := workerAuth(r); err != nil {
http.Error(w, err.Error(), http.StatusUnauthorized)
return
}
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", 405)
return
}
var body struct {
TaskID string `json:"task_id"`
LeaseEpoch string `json:"lease_epoch"`
OperationID string `json:"operation_id"`
From domain.WorkPhase `json:"from"`
To domain.WorkPhase `json:"to"`
Artifact []byte `json:"artifact"`
}
if json.NewDecoder(r.Body).Decode(&body) != nil || body.TaskID == "" || body.LeaseEpoch == "" {
http.Error(w, "task_id and lease_epoch are required", http.StatusBadRequest)
return
}
t, ok := s.Task(body.TaskID)
if !ok {
http.Error(w, "task not found", 404)
return
}
project, ok := rr.Project(t.Project)
if !ok {
http.Error(w, "unknown project "+t.Project, 409)
return
}
e, err := operations.RequestWorkPhase(s, project, body.TaskID, body.LeaseEpoch, body.OperationID, body.From, body.To, body.Artifact)
if errors.Is(err, operations.ErrTrajectoryGate) {
// The human is being asked. Not a failure, and not a phase change
// the worker should rotate on yet.
w.WriteHeader(http.StatusAccepted)
json.NewEncoder(w).Encode(map[string]string{"status": "trajectory_gate", "detail": err.Error()})
return
}
if err != nil {
http.Error(w, err.Error(), http.StatusConflict)
return
}
var p struct {
Phase domain.WorkPhase `json:"phase"`
}
_ = json.Unmarshal(e.Payload, &p)
json.NewEncoder(w).Encode(map[string]string{"phase": string(p.Phase)})
})
mux.HandleFunc("/v1/federation/commands", func(w http.ResponseWriter, r *http.Request) {
wid, err := workerAuth(r)
if err != nil {
http.Error(w, err.Error(), http.StatusUnauthorized)
return
}
if r.Method != http.MethodGet {
http.Error(w, "method not allowed", 405)
return
}
out, err := workers.Commands(wid)
if err != nil {
http.Error(w, err.Error(), 404)
return
}
json.NewEncoder(w).Encode(out)
})
mux.HandleFunc("/v1/federation/commands/", func(w http.ResponseWriter, r *http.Request) {
wid, err := workerAuth(r)
if err != nil {
http.Error(w, err.Error(), http.StatusUnauthorized)
return
}
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", 405)
return
}
var body struct {
Status string `json:"status"`
Message string `json:"message"`
}
if json.NewDecoder(r.Body).Decode(&body) != nil || (body.Status != "acknowledged" && body.Status != "stale" && body.Status != "rejected") {
http.Error(w, "invalid command result", 400)
return
}
commandID := strings.TrimPrefix(r.URL.Path, "/v1/federation/commands/")
command, ok := workers.Command(wid, commandID)
if !ok {
http.Error(w, "command not found", 404)
return
}
if err := workers.CompleteCommand(wid, commandID, body.Status, body.Message); err != nil {
http.Error(w, err.Error(), 409)
return
}
// Audit lifecycle evidence only after the worker reports the herdr
// input was acknowledged; a queued browser click is never an approval.
if body.Status == "acknowledged" {
if t, ok := s.Task(command.TaskID); ok {
typ := "ApprovalGranted"
if command.Kind == "deny_approval" {
typ = "ApprovalDenied"
}
payload, _ := json.Marshal(map[string]any{"subject_ref": command.ID, "pane_id": command.PaneID, "capture_revision": command.CaptureRevision})
e := domain.Event{ID: id(), Type: typ, TaskID: command.TaskID, Version: t.Version + 1, Payload: payload, Surface: string(authz.System)}
if err := s.Append(e); err != nil {
log.Printf("record approval %s: %v", command.ID, err)
}
}
}
w.WriteHeader(http.StatusNoContent)
})
mux.HandleFunc("/v1/federation/events", func(w http.ResponseWriter, r *http.Request) {
wid, err := workerAuth(r)
if err != nil {
http.Error(w, err.Error(), http.StatusUnauthorized)
return
}
if r.Method != http.MethodGet {
http.Error(w, "method not allowed", 405)
return
}
cursor, _ := workers.Cursor(wid)
if q := r.URL.Query().Get("since"); q != "" {
cursor, _ = strconv.ParseUint(q, 10, 64)
}
events := s.Events(cursor)
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]any{"cursor": cursor, "events": events})
})
mux.HandleFunc("/v1/federation/events/ack", func(w http.ResponseWriter, r *http.Request) {
wid, err := workerAuth(r)
if err != nil {
http.Error(w, err.Error(), 401)
return
}
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", 405)
return
}
var body struct {
Cursor uint64 `json:"cursor"`
}
if json.NewDecoder(r.Body).Decode(&body) != nil {
http.Error(w, "invalid cursor", 400)
return
}
if err := workers.Ack(wid, body.Cursor); err != nil {
http.Error(w, err.Error(), 409)
return
}
w.WriteHeader(http.StatusNoContent)
})
mux.HandleFunc("/v1/federation/workers/", func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || (!strings.HasSuffix(r.URL.Path, "/heartbeat") && !strings.HasSuffix(r.URL.Path, "/renew") && !strings.HasSuffix(r.URL.Path, "/start") && !strings.HasSuffix(r.URL.Path, "/nack") && !strings.HasSuffix(r.URL.Path, "/handoff") && !strings.HasSuffix(r.URL.Path, "/pickup") && !strings.HasSuffix(r.URL.Path, "/complete") && !strings.HasSuffix(r.URL.Path, "/submit") && !strings.HasSuffix(r.URL.Path, "/captures")) {
http.Error(w, "not found", 404)
return
}
parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/")
if len(parts) != 5 {
http.Error(w, "not found", 404)
return
}
wid, err := workerAuth(r)
if err != nil {
http.Error(w, err.Error(), 401)
return
}
if wid != parts[3] {
http.Error(w, "worker identity mismatch", http.StatusForbidden)
return
}
if strings.HasSuffix(r.URL.Path, "/heartbeat") {
var health federation.WorkerHealth
if err := json.NewDecoder(r.Body).Decode(&health); err != nil && !errors.Is(err, io.EOF) {
http.Error(w, "invalid worker health", 400)
return
}
if err := workers.Heartbeat(parts[3], health); err != nil {
http.Error(w, err.Error(), 404)
return
}
w.WriteHeader(http.StatusNoContent)
return
}
if strings.HasSuffix(r.URL.Path, "/captures") {
var c federation.Capture
if json.NewDecoder(r.Body).Decode(&c) != nil {
http.Error(w, "invalid capture", 400)
return
}
if t, ok := s.Task(c.TaskID); !ok || t.Lease == nil || t.Lease.HarnessID != parts[3] {
http.Error(w, "lease not owned", 409)
return
}
out, err := workers.PutCapture(parts[3], c)
if err != nil {
http.Error(w, err.Error(), 400)
return
}
json.NewEncoder(w).Encode(out)
return
}
var b struct {
TaskID string `json:"task_id"`
TTLSeconds int `json:"ttl_seconds"`
ExpectedVersion int `json:"expected_version"`
HandoffRef string `json:"handoff_ref"`
AnchorSHA string `json:"anchor_sha"`
TransactionID string `json:"transaction_id"`
LeaseVersion int `json:"lease_version"`
LeaseEpoch string `json:"lease_epoch"`
ResultSHA string `json:"result_sha"`
Branch string `json:"branch"`
Remote string `json:"remote"`
Receipt map[string]any `json:"receipt"`
FailureClass string `json:"failure_class"`
LastError string `json:"last_error"`
SessionEvidence domain.SessionEvidence `json:"session_evidence"`
Review review.Result `json:"review"`
Gate domain.GateResult `json:"gate"`
}
if json.NewDecoder(r.Body).Decode(&b) != nil || b.TaskID == "" {
http.Error(w, "invalid lease body", 400)
return
}
t, ok := s.Task(b.TaskID)
if !ok {
http.Error(w, "task not found", 404)
return
}
ownedLease := (t.State == domain.StateLeased || t.State == domain.StateNeedsAttention) && t.Lease != nil && t.Lease.HarnessID == parts[3] && t.Lease.Epoch == b.LeaseEpoch
// A response can be lost after the append/fsync. Retrying the exact
// release transaction is therefore a successful no-op, never a second
// TaskReleased event and never a reason to discard the predecessor.
if strings.HasSuffix(r.URL.Path, "/handoff") && t.State == domain.StateQueued && b.TransactionID != "" && b.TransactionID == t.ReleaseTransaction && b.HandoffRef == t.HandoffRef && b.AnchorSHA == t.ReleaseAnchor {
w.WriteHeader(http.StatusNoContent)
return
}
if !ownedLease {
http.Error(w, "lease not owned", 409)
return
}
if (strings.HasSuffix(r.URL.Path, "/complete") || strings.HasSuffix(r.URL.Path, "/submit")) && b.ExpectedVersion != t.Version {
http.Error(w, "lease version conflict", http.StatusConflict)
return
}
if strings.HasSuffix(r.URL.Path, "/submit") {
// Seal the review, then submit the commit it examined. Both events
// are Orchestra's; the worker supplies verified evidence and the
// coordinator decides what it means.
project, ok := rr.Project(t.Project)
if !ok {
http.Error(w, "unknown project "+t.Project, 409)
return
}
if len(b.ResultSHA) != 40 {
http.Error(w, "verified result_sha required", 400)
return
}
if !t.Submitted(b.ResultSHA) && (t.Review == nil || t.Review.ResultSHA != b.ResultSHA) {
b.Review.ResultSHA = b.ResultSHA
if _, err := operations.RecordReview(s, project, b.TaskID, b.Review); err != nil {
http.Error(w, err.Error(), http.StatusConflict)
return
}
if !b.Review.Accepted() {
// The review sent the work back to implementation. That is
// an outcome, not a failure, and there is nothing to submit.
json.NewEncoder(w).Encode(map[string]any{"status": federation.SubmitChangesRequested, "blocking": len(b.Review.Blocking())})
return
}
}
if submissionPublisher == nil {
json.NewEncoder(w).Encode(map[string]string{"status": federation.SubmitNoPublisher})
return
}
plan, err := operations.PrepareSubmission(s, project, b.TaskID, b.ResultSHA, b.Gate, operations.Notes{})
if err != nil {
http.Error(w, err.Error(), http.StatusConflict)
return
}
if b.Remote != "" {
plan.Remote = b.Remote
}
e, err := operations.ExecuteSubmission(r.Context(), s, plan, submissionPublisher(plan), nil)
if err != nil {
http.Error(w, err.Error(), http.StatusConflict)
return
}
json.NewEncoder(w).Encode(map[string]any{"status": federation.SubmitSubmitted, "event": e})
return
}
if strings.HasSuffix(r.URL.Path, "/start") {
// A lost response after append is an idempotent start ACK, not a
// reason to strand the running pane behind a stale version.
if t.LifecyclePhase == "started" && t.Lease != nil && t.Lease.HarnessID == parts[3] && t.Lease.Epoch == b.LeaseEpoch {
w.WriteHeader(http.StatusNoContent)
return
}
if b.ExpectedVersion != t.Version {
http.Error(w, "lease version conflict", http.StatusConflict)
return
}
p, _ := json.Marshal(map[string]any{"harness_id": parts[3], "lease_epoch": b.LeaseEpoch, "expected_version": t.Version, "lifecycle_phase": "started", "session_evidence": b.SessionEvidence})
e := domain.Event{ID: id(), Type: "TaskLaunchAcknowledged", TaskID: b.TaskID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
if err := s.Append(e); err != nil {
http.Error(w, err.Error(), http.StatusConflict)
return
}
json.NewEncoder(w).Encode(e)
return
}
if strings.HasSuffix(r.URL.Path, "/nack") {
if b.ExpectedVersion != t.Version || b.FailureClass == "" || b.LastError == "" {
http.Error(w, "current lease version, failure_class, and last_error required", http.StatusConflict)
return
}
var typ string
var p []byte
switch b.FailureClass {
case "invalid_handoff":
typ = "TaskBlocked"
p, _ = json.Marshal(map[string]any{"blocker": b.LastError, "block_reason": string(domain.BlockReasonHandoffValidation), "harness_id": parts[3], "lease_epoch": b.LeaseEpoch, "expected_version": t.Version, "lifecycle_phase": "launch_nacked", "last_error": b.LastError, "session_evidence": b.SessionEvidence})
case "launch_uncertain":
typ = "TaskNeedsAttention"
p, _ = json.Marshal(map[string]any{"blocker": b.LastError, "block_reason": string(domain.BlockReasonLeaseFailure), "harness_id": parts[3], "lease_epoch": b.LeaseEpoch, "expected_version": t.Version, "lifecycle_phase": "launch_uncertain", "last_error": b.LastError, "session_evidence": b.SessionEvidence})
default:
typ = "TaskReleased"
p, _ = json.Marshal(map[string]any{"reason": "launch_failed", "failure_class": b.FailureClass, "harness_id": parts[3], "lease_epoch": b.LeaseEpoch, "expected_version": t.Version, "lifecycle_phase": "launch_nacked", "last_error": b.LastError, "session_evidence": b.SessionEvidence})
}
e := domain.Event{ID: id(), Type: typ, TaskID: b.TaskID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
if err := s.Append(e); err != nil {
http.Error(w, err.Error(), http.StatusConflict)
return
}
if typ == "TaskReleased" && rt != nil {
_, _ = rt.HandleEvent(e)
}
json.NewEncoder(w).Encode(e)
return
}
if strings.HasSuffix(r.URL.Path, "/renew") {
ttl := b.TTLSeconds
if ttl == 0 {
ttl = int(domain.LeaseTTL.Seconds())
}
e, err := s.RenewLease(b.TaskID, parts[3], b.LeaseEpoch, b.ExpectedVersion, time.Duration(ttl)*time.Second)
if err != nil {
http.Error(w, err.Error(), http.StatusConflict)
return
}
json.NewEncoder(w).Encode(e)
return
}
if strings.HasSuffix(r.URL.Path, "/pickup") {
if !ownedLease || b.TransactionID == "" || b.TransactionID != t.ReleaseTransaction || b.HandoffRef != t.HandoffRef || b.AnchorSHA != t.ReleaseAnchor {
http.Error(w, "pickup does not match active release", http.StatusConflict)
return
}
if t.PickupTransaction == b.TransactionID && t.PickupLeaseVersion == b.LeaseVersion {
w.WriteHeader(http.StatusNoContent)
return
}
if b.LeaseVersion != t.Version {
http.Error(w, "lease version conflict", http.StatusConflict)
return
}
p, _ := json.Marshal(map[string]any{"transaction_id": b.TransactionID, "handoff_ref": b.HandoffRef, "anchor_sha": b.AnchorSHA, "harness_id": parts[3], "lease_epoch": b.LeaseEpoch, "lease_version": b.LeaseVersion, "expected_version": t.Version, "session_evidence": b.SessionEvidence})
e := domain.Event{ID: id(), Type: "TaskPickupValidated", TaskID: b.TaskID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
if err := s.Append(e); err != nil {
http.Error(w, err.Error(), http.StatusConflict)
return
}
json.NewEncoder(w).Encode(e)
return
}
if strings.HasSuffix(r.URL.Path, "/complete") {
if b.HandoffRef == "" {
http.Error(w, "report_ref required", 400)
return
}
if len(b.ResultSHA) != 40 || b.Branch == "" || b.Remote == "" {
http.Error(w, "verified result_sha, branch, and remote required", 400)
return
}
if b.Receipt == nil {
b.Receipt = map[string]any{}
}
b.Receipt["harness_id"] = parts[3]
if _, ok := b.Receipt["consumed"]; !ok {
b.Receipt["consumed"] = 0
}
p, _ := json.Marshal(map[string]any{"report_ref": b.HandoffRef, "receipt": b.Receipt, "result_sha": b.ResultSHA, "branch": b.Branch, "remote": b.Remote, "harness_id": parts[3], "lease_epoch": b.LeaseEpoch, "expected_version": t.Version, "session_evidence": b.SessionEvidence})
e := domain.Event{ID: id(), Type: "TaskCompleted", TaskID: b.TaskID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
if err := s.Append(e); err != nil {
http.Error(w, err.Error(), 409)
return
}
if consumed, ok := b.Receipt["consumed"].(float64); ok && consumed > 0 {
qp, _ := json.Marshal(map[string]any{"harness_id": parts[3], "consumed": consumed})
if err := s.Append(domain.Event{ID: id(), Type: "QuotaReported", TaskID: "system", Payload: qp, Surface: string(authz.System)}); err != nil {
log.Printf("federated quota report %s: %v", b.TaskID, err)
}
}
json.NewEncoder(w).Encode(e)
return
}
if b.HandoffRef == "" {
http.Error(w, "handoff_ref required", 400)
return
}
if len(b.AnchorSHA) != 40 {
http.Error(w, "anchor_sha required", 400)
return
}
if b.TransactionID == "" || b.ExpectedVersion != t.Version {
http.Error(w, "release transaction and current lease version required", http.StatusConflict)
return
}
p, _ := json.Marshal(map[string]any{"handoff_ref": b.HandoffRef, "harness_id": parts[3], "lease_epoch": b.LeaseEpoch, "anchor_sha": b.AnchorSHA, "transaction_id": b.TransactionID, "expected_version": t.Version, "session_evidence": b.SessionEvidence})
e := domain.Event{ID: id(), Type: "TaskReleased", TaskID: b.TaskID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
if err := s.Append(e); err != nil {
http.Error(w, err.Error(), 409)
return
}
if rt != nil {
_, _ = rt.HandleEvent(e)
}
json.NewEncoder(w).Encode(e)
})
var giteaSources []provider.GiteaSourceConfig
if path := os.Getenv("ORCHESTRA_GITEA_CONFIG"); path != "" {
cfgs, err := provider.LoadGiteaConfigs(path)
if err != nil {
log.Fatalf("load gitea config: %v", err)
}
giteaSources = cfgs
} else if base := os.Getenv("ORCHESTRA_GITEA_URL"); base != "" {
// Legacy single-repo configuration: project defaults to the repo
// name, matching the historical (pre-multi-source) behavior.
giteaSources = []provider.GiteaSourceConfig{{
Project: os.Getenv("ORCHESTRA_GITEA_REPO"), BaseURL: base,
Owner: os.Getenv("ORCHESTRA_GITEA_OWNER"), Repo: os.Getenv("ORCHESTRA_GITEA_REPO"),
Token: os.Getenv("ORCHESTRA_GITEA_TOKEN"), WebhookSecret: os.Getenv("ORCHESTRA_GITEA_WEBHOOK_SECRET"),
}}
}
humanSources := map[string]human.Source{}
if len(giteaSources) > 0 {
reflectors := map[string]provider.Gitea{}
for _, c := range giteaSources {
g := provider.Gitea{BaseURL: c.BaseURL, Token: c.Token, WebhookSecret: c.WebhookSecret, Owner: c.Owner, Repo: c.Repo, Project: c.Project}
reflectors[g.SourceName()] = g
}
reflecting := provider.ReflectingSink{Sink: s, Tasks: s, Reflector: provider.MultiGitea{Sources: reflectors}}
for _, c := range giteaSources {
g := provider.Gitea{BaseURL: c.BaseURL, Token: c.Token, WebhookSecret: c.WebhookSecret, Owner: c.Owner, Repo: c.Repo, Project: c.Project}
name := "gitea:" + c.Project
webhookPath := "/v1/providers/gitea/webhook/" + c.Project
if len(giteaSources) == 1 && os.Getenv("ORCHESTRA_GITEA_CONFIG") == "" {
// Preserve the legacy unprefixed webhook path when running
// the single-source (env-var) configuration, so existing
// Gitea webhook configs don't need to be re-pointed.
webhookPath = "/v1/providers/gitea/webhook"
name = "gitea"
}
mux.Handle(webhookPath, g.WebhookHandler(reflecting))
sup := &provider.Supervisor{Name: name, Run: func(ctx context.Context) error {
pollCtx, cancel := context.WithTimeout(ctx, time.Minute)
defer cancel()
_, err := g.Poll(pollCtx, reflecting)
if err == nil {
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(time.Minute):
}
}
return err
}}
sup.Start(context.Background())
providerHealth[name] = sup
humanSources[g.SourceName()] = provider.GiteaComments{Gitea: g}
if publisher := (provider.GiteaPublisher{Gitea: g, Base: os.Getenv("ORCHESTRA_PR_BASE")}); publisher.BaseURL != "" {
root := projectRoots[c.Project]
submissionPublisher = func(plan operations.SubmissionPlan) operations.Publisher {
p := publisher
p.Root = filepath.Join(root, plan.TaskID)
return p
}
pullRequests[g.SourceName()] = publisher
}
}
}
// Started here, after the forge wiring above fills pullRequests. Guarding
// on the map before anything writes to it made this loop dead code: the
// length was always zero, so a merged pull request never completed its
// task, and moving the read into the tick would race the startup writes.
if len(pullRequests) > 0 {
// Submitted work is reconciled on its own loop, not behind
// Store.PreLease: an in-review task cannot be leased, so a pre-lease
// hook could never see the feedback that should make it leasable.
trust := human.Trust{
Accepted: splitList(os.Getenv("ORCHESTRA_REVIEW_ACTORS")),
Ignored: splitList(os.Getenv("ORCHESTRA_REVIEW_IGNORE_ACTORS")),
}
go func() {
ticker := time.NewTicker(time.Minute)
defer ticker.Stop()
for range ticker.C {
for _, t := range s.Tasks() {
if t.Submission == nil || (t.State != domain.StateInReview && t.State != domain.StateQueued) {
continue
}
source, ok := pullRequests[t.Submission.PR.Provider]
if !ok {
continue
}
project, ok := rr.Project(t.Project)
if !ok {
continue
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
state, err := source.PullRequest(ctx, t)
cancel()
if err != nil {
// Observable, and the task stays exactly where it was.
log.Printf("reflect submission %s: %v", t.ID, err)
continue
}
if _, err := operations.ReflectSubmission(s, project, t.ID, state, trust); err != nil {
log.Printf("reflect submission %s: %v", t.ID, err)
}
}
}
}()
}
// Reconciliation runs immediately before every lease, which is where
// ownership of a task begins. A configured source that cannot be read
// refuses the lease rather than letting a successor resume from an older
// intent. Set ORCHESTRA_HUMAN_RECONCILE=off to disable it for an
// operator who needs to run while a source is down.
if len(humanSources) > 0 && !strings.EqualFold(os.Getenv("ORCHESTRA_HUMAN_RECONCILE"), "off") {
reconciler := &human.Reconciler{Store: s, Sources: humanSources, Timeout: 30 * time.Second}
s.PreLease = func(taskID string) error {
return reconciler.Reconcile(context.Background(), taskID)
}
if coordinator != nil {
// The second reconciliation point: a verified turn boundary on a
// lease that is already running. Nothing here preempts the agent.
coordinator.ReconcileHumanInput = reconciler.Reconcile
// After this many consecutive failures at that boundary, the
// session is asked to hand off rather than keep running on intent
// Orchestra can no longer refresh.
if v, parseErr := strconv.Atoi(os.Getenv("ORCHESTRA_RECONCILE_FAILURE_HANDOFF")); parseErr == nil && v > 0 {
coordinator.ReconcileFailureHandoff = v
}
}
}
if path := os.Getenv("ORCHESTRA_JSONL"); path != "" {
sup := &provider.Supervisor{Name: "jsonl", Run: func(ctx context.Context) error {
return (provider.JSONLWatcher{Path: path, Interval: time.Second, Provider: provider.JSONL{Source: "jsonl"}}).Run(ctx, s)
}}
sup.Start(context.Background())
providerHealth["jsonl"] = sup
}
var senders []delivery.Sender
if token, chat := os.Getenv("ORCHESTRA_TELEGRAM_BOT_TOKEN"), os.Getenv("ORCHESTRA_TELEGRAM_CHAT_ID"); token != "" && chat != "" {
senders = append(senders, delivery.Telegram{Token: token, ChatID: chat})
}
if topic := os.Getenv("ORCHESTRA_NTFY_TOPIC"); topic != "" {
senders = append(senders, delivery.Ntfy{Topic: topic, Token: os.Getenv("ORCHESTRA_NTFY_TOKEN"), URL: os.Getenv("ORCHESTRA_NTFY_URL")})
}
if len(senders) > 0 {
cursorPath := filepath.Join(dir, "delivery-cursor")
var startCursor uint64
if b, err := os.ReadFile(cursorPath); err == nil {
if v, err := strconv.ParseUint(strings.TrimSpace(string(b)), 10, 64); err == nil {
startCursor = v
}
}
go func() {
fanout := &delivery.Fanout{
Senders: senders,
Cursor: startCursor,
SaveCursor: func(cursor uint64) {
if err := os.WriteFile(cursorPath, []byte(strconv.FormatUint(cursor, 10)), 0o644); err != nil {
log.Printf("delivery cursor persist: %v", err)
}
},
}
err := fanout.Run(context.Background(), s.Events)
if err != nil {
log.Printf("delivery fanout: %v", err)
}
}()
}
go func() {
lastDay := ""
t := time.NewTicker(time.Minute)
defer t.Stop()
for now := range t.C {
utc := now.UTC()
day := utc.Format("2006-01-02")
if utc.Hour() == 3 && day != lastDay {
if _, err := standup(); err != nil {
log.Printf("standup advisory: %v", err)
} else {
lastDay = day
}
}
}
}()
if rt != nil {
if _, err := rt.AssignPending(); err != nil {
log.Printf("startup task assignment: %v", err)
}
}
port := os.Getenv("ORCHESTRA_PORT")
if port == "" {
port = "9145"
}
// Deployed identity, without a credential. A burn-in run pairs a
// coordinator and a worker, and matching revisions must be evidence rather
// than assumption. /v1/admin/diagnostics carries the same object behind
// the operator login.
b := buildinfo.Current()
log.Printf("orchestra revision %s built %s dirty %s", b.Revision, b.Time, b.Dirty)
log.Println("orchestra listening on :" + port)
tokens := map[authz.Surface]string{
authz.TUI: os.Getenv("ORCHESTRA_TUI_TOKEN"),
authz.MCP: os.Getenv("ORCHESTRA_MCP_TOKEN"), authz.Maven: os.Getenv("ORCHESTRA_MAVEN_TOKEN"),
// The credential an in-pane coding session presents. It buys the two
// request endpoints and read access — never a lifecycle mutation. The
// forge, Vikunja and operator tokens must never reach an agent pane;
// this one is the only Orchestra credential an agent may hold.
authz.Agent: os.Getenv("ORCHESTRA_AGENT_TOKEN"),
// S12: this is the credential callers present *to* Orchestra on the
// ntfy surface. ORCHESTRA_NTFY_TOKEN is a different secret entirely —
// it is handed out to the third-party ntfy server (see the sender
// above) and must never be accepted as an inbound credential.
authz.Telegram: os.Getenv("ORCHESTRA_TELEGRAM_TOKEN"), authz.Ntfy: os.Getenv("ORCHESTRA_NTFY_SURFACE_TOKEN"),
}
// A full-control surface with no credential is an open control plane, not a
// disabled one, because an unset token makes the middleware skip its check.
// Refuse to start rather than log it and serve.
if err := authz.RequireCredentials(tokens); err != nil {
log.Fatalf("refusing to serve: %v", err)
}
log.Fatal(http.ListenAndServe(":"+port, authz.HTTPWithSessions(tokens, sessions, mux)))
}
// splitList reads a comma-separated env list, ignoring blanks.
func splitList(v string) []string {
var out []string
for _, item := range strings.Split(v, ",") {
if s := strings.TrimSpace(item); s != "" {
out = append(out, s)
}
}
return out
}