4af9880b86
The loop that reads merged pull requests was guarded by len(pullRequests) > 0 at a point 600 lines before the Gitea wiring that writes to that map. The length was always zero, so the goroutine never started and a merged pull request could never complete its task. Live on the first submission this deployment made: PR #8 took a trusted comment and nothing moved. Moving the block below the wiring fixes the ordering and avoids the race that reading the map inside the tick would have introduced. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CVbaKucEYBjMqVeUgJUsc1
1850 lines
67 KiB
Go
1850 lines
67 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"crypto/subtle"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"net"
|
|
"net/http"
|
|
"orchestra/internal/admin"
|
|
"orchestra/internal/authn"
|
|
"orchestra/internal/authz"
|
|
"orchestra/internal/buildinfo"
|
|
"orchestra/internal/delivery"
|
|
"orchestra/internal/domain"
|
|
"orchestra/internal/federation"
|
|
"orchestra/internal/herdr"
|
|
"orchestra/internal/human"
|
|
"orchestra/internal/operations"
|
|
"orchestra/internal/orchestrator"
|
|
"orchestra/internal/provider"
|
|
"orchestra/internal/registry"
|
|
"orchestra/internal/review"
|
|
"orchestra/internal/router"
|
|
"orchestra/internal/store"
|
|
"orchestra/internal/ui"
|
|
"orchestra/internal/webui"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
func id() string { return domain.NewID() }
|
|
|
|
const defaultHerdrPort = "9245"
|
|
|
|
func herdrAddress(rr registry.Registry, h registry.Herdr) string {
|
|
if h.Backend == "tmux" {
|
|
return rr.Endpoint(h)
|
|
}
|
|
if h.Address != "" {
|
|
return h.Address
|
|
}
|
|
m, ok := rr.Machine(h.MachineID)
|
|
if !ok {
|
|
return ""
|
|
}
|
|
host, _, err := net.SplitHostPort(m.Address)
|
|
if err != nil {
|
|
return m.Address
|
|
}
|
|
return net.JoinHostPort(host, defaultHerdrPort)
|
|
}
|
|
|
|
type federatedAvailability struct {
|
|
base router.Availability
|
|
workers *federation.Registry
|
|
localMachine string
|
|
}
|
|
|
|
// federatedReachability keeps the legacy TCP probe for coordinator-owned
|
|
// herdrs, while avoiding a coordinator-side probe of a worker-owned backend.
|
|
// A worker-owned harness is reachable precisely when its worker is registered (as
|
|
// enforced by federatedAvailability); probing its raw herdr endpoint here
|
|
// would reintroduce the cross-machine Design A dependency.
|
|
type federatedReachability struct {
|
|
base registry.Reachability
|
|
remote map[string]bool
|
|
}
|
|
|
|
func (r federatedReachability) Reachable(address string, timeout time.Duration) bool {
|
|
if r.remote[address] {
|
|
return true
|
|
}
|
|
return r.base.Reachable(address, timeout)
|
|
}
|
|
|
|
func remoteHerdrAddresses(rr registry.Registry, localMachine string) map[string]bool {
|
|
remote := map[string]bool{}
|
|
for _, h := range rr.Herdrs() {
|
|
if !coordinatorOwnsHerdr(h, localMachine) {
|
|
remote[herdrAddress(rr, h)] = true
|
|
}
|
|
}
|
|
return remote
|
|
}
|
|
|
|
// coordinatorOwnsHerdr identifies the only herdr sockets the coordinator may
|
|
// probe or adapt. In federation mode a remote pane belongs to its worker;
|
|
// reaching into that machine would turn a worker-owned health signal back
|
|
// into a misleading coordinator TCP result.
|
|
func coordinatorOwnsHerdr(h registry.Herdr, localMachine string) bool {
|
|
if h.Backend == "tmux" {
|
|
return false
|
|
}
|
|
return localMachine == "" || h.MachineID == localMachine
|
|
}
|
|
|
|
func (a federatedAvailability) Available(h registry.Herdr) bool {
|
|
return a.Unavailable(h) == ""
|
|
}
|
|
|
|
// Unavailable keeps the base gate's own reason instead of restating every
|
|
// refusal as worker health. A quota refusal reported as a stale heartbeat sent
|
|
// burn-in run 2 looking at a worker that was one second fresh.
|
|
func (a federatedAvailability) Unavailable(h registry.Herdr) string {
|
|
if a.base != nil {
|
|
if reason := unavailableReason(a.base, h); reason != "" {
|
|
return reason
|
|
}
|
|
}
|
|
if coordinatorOwnsHerdr(h, a.localMachine) {
|
|
return ""
|
|
}
|
|
return a.workers.Unavailable(h.ID)
|
|
}
|
|
|
|
// unavailableReason mirrors router.unavailableReason for the wrapped base gate.
|
|
func unavailableReason(a router.Availability, h registry.Herdr) string {
|
|
if ra, ok := a.(router.ReasonedAvailability); ok {
|
|
return ra.Unavailable(h)
|
|
}
|
|
if a.Available(h) {
|
|
return ""
|
|
}
|
|
return "worker unavailable"
|
|
}
|
|
|
|
func (a federatedAvailability) Supports(h registry.Herdr, project string) bool {
|
|
// Locally-owned herdrs keep their static registry/project affinity. A
|
|
// remote worker must additionally prove it has a local checkout for the
|
|
// project before the router can offer it a lease.
|
|
if coordinatorOwnsHerdr(h, a.localMachine) {
|
|
return true
|
|
}
|
|
return a.workers.Supports(h.ID, project)
|
|
}
|
|
|
|
func validateLocalMachine(rr registry.Registry, localMachine string) error {
|
|
machines := rr.Machines()
|
|
requiresWorkerOwnership := false
|
|
for _, h := range rr.Herdrs() {
|
|
if h.Backend == "tmux" {
|
|
requiresWorkerOwnership = true
|
|
break
|
|
}
|
|
}
|
|
if len(machines) <= 1 && !requiresWorkerOwnership {
|
|
return nil
|
|
}
|
|
if localMachine == "" {
|
|
return fmt.Errorf("ORCHESTRA_MACHINE_ID is required when the registry is multi-machine or has worker-owned backends")
|
|
}
|
|
if _, ok := rr.Machine(localMachine); !ok {
|
|
return fmt.Errorf("ORCHESTRA_MACHINE_ID %q is not in the registry", localMachine)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func main() {
|
|
dir := os.Getenv("ORCHESTRA_DATA")
|
|
if dir == "" {
|
|
dir = "./data"
|
|
}
|
|
s, err := store.Open(dir)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
users, err := authn.Open(authn.Path(dir))
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
defer users.Close()
|
|
userCount, err := users.Count()
|
|
if err != nil {
|
|
log.Fatalf("read operator database: %v", err)
|
|
}
|
|
legacyUsername := os.Getenv("ORCHESTRA_WEB_USERNAME")
|
|
legacyPasswordHash := os.Getenv("ORCHESTRA_WEB_PASSWORD_HASH")
|
|
if userCount == 0 {
|
|
if legacyUsername != "" || legacyPasswordHash != "" {
|
|
if legacyUsername == "" || legacyPasswordHash == "" {
|
|
log.Fatal("operator database is empty and the legacy web credential is incomplete: both ORCHESTRA_WEB_USERNAME and ORCHESTRA_WEB_PASSWORD_HASH are required for one-time migration")
|
|
}
|
|
imported, importErr := users.ImportBcrypt(legacyUsername, legacyPasswordHash)
|
|
if importErr != nil {
|
|
log.Fatalf("migrate legacy web credential: %v", importErr)
|
|
}
|
|
if imported {
|
|
userCount = 1
|
|
log.Printf("migrated web operator %q into %s; remove ORCHESTRA_WEB_USERNAME and ORCHESTRA_WEB_PASSWORD_HASH from the deployment environment", legacyUsername, authn.Path(dir))
|
|
}
|
|
}
|
|
if userCount == 0 {
|
|
log.Fatalf("operator database is empty: stop Orchestra and run orchestra-user set -data %s -username NAME", dir)
|
|
}
|
|
} else if legacyUsername != "" || legacyPasswordHash != "" {
|
|
log.Printf("operator database already contains %d account(s); legacy ORCHESTRA_WEB_USERNAME and ORCHESTRA_WEB_PASSWORD_HASH are ignored and should be removed", userCount)
|
|
}
|
|
var rr registry.Registry
|
|
var rt *router.Router
|
|
var coordinator *orchestrator.Coordinator
|
|
// submissionPublisher is nil until a forge is configured. Submission then
|
|
// returns the verified plan instead of performing it, which keeps `task
|
|
// pr` the only path without inventing a fake success.
|
|
var submissionPublisher func(operations.SubmissionPlan) operations.Publisher
|
|
// Worktree root per project, so a submission can find the checkout that
|
|
// holds the commit it is publishing.
|
|
projectRoots := map[string]string{}
|
|
// Pull-request readers by source name, for reflecting submitted work.
|
|
pullRequests := map[string]human.PullRequestSource{}
|
|
localMachine := os.Getenv("ORCHESTRA_MACHINE_ID")
|
|
workers := &federation.Registry{AdmitToken: os.Getenv("ORCHESTRA_FEDERATION_ADMIT_TOKEN"), StatePath: filepath.Join(dir, "federation-state.json")}
|
|
if err := workers.Load(); err != nil {
|
|
log.Fatalf("load federation state: %v", err)
|
|
}
|
|
if config := os.Getenv("ORCHESTRA_CONFIG"); config != "" {
|
|
if rr, err = registry.Load(config); err != nil {
|
|
log.Fatalf("load orchestra config: %v", err)
|
|
}
|
|
if err := validateLocalMachine(rr, localMachine); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
reachability := registry.Reachability(registry.TCPReachability{})
|
|
if localMachine != "" {
|
|
reachability = federatedReachability{base: reachability, remote: remoteHerdrAddresses(rr, localMachine)}
|
|
}
|
|
rt = &router.Router{Store: s, Registry: rr, Reachability: reachability, Timeout: time.Second, Retry: router.RetryPolicy{MaxAttempts: 3, Backoff: time.Minute}}
|
|
limits := map[string]router.QuotaWindowLimits{}
|
|
for _, h := range rr.Herdrs() {
|
|
w := router.QuotaWindowLimits{FiveHour: h.QuotaLimit5h, Weekly: h.QuotaLimitWeekly}
|
|
if w.Weekly <= 0 && h.QuotaLimit > 0 {
|
|
// Back-compat: the old single-window field meant weekly.
|
|
w.Weekly = h.QuotaLimit
|
|
}
|
|
if w.FiveHour > 0 || w.Weekly > 0 {
|
|
limits[h.ID] = w
|
|
}
|
|
}
|
|
if len(limits) > 0 {
|
|
rt.Availability = router.QuotaAvailability{Store: s, Limits: limits}
|
|
}
|
|
if localMachine != "" {
|
|
rt.Availability = federatedAvailability{base: rt.Availability, workers: workers, localMachine: localMachine}
|
|
}
|
|
if repo, root := os.Getenv("ORCHESTRA_REPO"), os.Getenv("ORCHESTRA_WORKTREE_ROOT"); repo != "" && root != "" {
|
|
adapters := map[string]herdr.Adapter{}
|
|
for _, h := range rr.Herdrs() {
|
|
if !coordinatorOwnsHerdr(h, localMachine) {
|
|
log.Printf("herdr %s is worker-owned on %s; coordinator probe skipped", h.ID, h.MachineID)
|
|
continue
|
|
}
|
|
address := herdrAddress(rr, h)
|
|
if address == "" {
|
|
continue
|
|
}
|
|
client := herdr.New(address)
|
|
protocol := h.Protocol
|
|
if protocol == "" {
|
|
protocol = os.Getenv("ORCHESTRA_HERDR_PROTOCOL")
|
|
}
|
|
if err := client.CheckProtocol(context.Background(), protocol); err != nil {
|
|
log.Printf("herdr %s unavailable: %v", h.ID, err)
|
|
continue
|
|
}
|
|
log.Printf("herdr %s reachable at %s (protocol %s, harness %s)", h.ID, address, protocol, h.Harness)
|
|
switch h.Harness {
|
|
case "claude":
|
|
adapters[h.ID] = herdr.Claude(client, 200000, s)
|
|
case "opencode":
|
|
adapters[h.ID] = herdr.OpenCode(client, 200000, s)
|
|
case "codex", "":
|
|
adapters[h.ID] = herdr.Codex(client, 200000, s)
|
|
default:
|
|
log.Printf("herdr %s has unsupported harness %q", h.ID, h.Harness)
|
|
}
|
|
}
|
|
projectRepos := map[string]orchestrator.ProjectRepo{}
|
|
for _, p := range rr.Projects() {
|
|
if p.Repo != "" && p.WorktreeRoot != "" {
|
|
projectRepos[p.ID] = orchestrator.ProjectRepo{Repo: p.Repo, WorktreeRoot: p.WorktreeRoot}
|
|
projectRoots[p.ID] = p.WorktreeRoot
|
|
}
|
|
}
|
|
worktrees := orchestrator.PerProjectGitWorktrees{
|
|
Projects: projectRepos,
|
|
Default: orchestrator.GitWorktrees{Root: root, Repo: repo},
|
|
}
|
|
coordinator = &orchestrator.Coordinator{Store: s, StatePath: filepath.Join(dir, "runtime-sessions.json"), Worktrees: worktrees, Adapters: orchestrator.AdapterFactory{Herdrs: adapters}, LocalHerdr: func(id string) bool {
|
|
h, ok := rr.Herdr(id)
|
|
return ok && coordinatorOwnsHerdr(h, localMachine)
|
|
}}
|
|
rt.OnLease = func(e domain.Event) error {
|
|
// In federated mode the coordinator must never inspect a worker-owned
|
|
// checkout. Its worker consumes the router-issued lease event and
|
|
// performs all Git/backend operations on that machine (§2.1).
|
|
if localMachine != "" {
|
|
var p struct {
|
|
HarnessID string `json:"harness_id"`
|
|
}
|
|
if json.Unmarshal(e.Payload, &p) == nil {
|
|
if h, ok := rr.Herdr(p.HarnessID); ok && !coordinatorOwnsHerdr(h, localMachine) {
|
|
return nil
|
|
}
|
|
}
|
|
}
|
|
return coordinator.Start(context.Background(), e)
|
|
}
|
|
hard := 0.75
|
|
if v, parseErr := strconv.ParseFloat(os.Getenv("ORCHESTRA_OCCUPANCY_HARD"), 64); parseErr == nil && v > 0 && v < 1 {
|
|
hard = v
|
|
}
|
|
if v, parseErr := strconv.ParseFloat(os.Getenv("ORCHESTRA_OCCUPANCY_SOFT"), 64); parseErr == nil && v > 0 && v < 1 {
|
|
coordinator.Soft = v
|
|
}
|
|
go func() {
|
|
if monitorErr := coordinator.Monitor(context.Background(), hard, 30*time.Second); monitorErr != nil {
|
|
log.Printf("orchestrator monitor: %v", monitorErr)
|
|
}
|
|
}()
|
|
}
|
|
}
|
|
mux := http.NewServeMux()
|
|
// B18: the UI is a full control plane — it can create tasks, release or
|
|
// complete them, and inject approval keystrokes into live panes. Browser
|
|
// sessions are backed by operator accounts in the embedded auth database;
|
|
// no missing environment variable can open this surface.
|
|
sessions := &authz.Sessions{}
|
|
browserAuth := authn.HTTP{Users: users, Sessions: sessions, SecureCookie: os.Getenv("ORCHESTRA_UI_INSECURE_COOKIE") == ""}
|
|
mux.HandleFunc(authz.SessionPath, browserAuth.Session)
|
|
mux.HandleFunc("/v1/ui/account", browserAuth.Account)
|
|
// Browser-specific endpoints intentionally present a joined read model;
|
|
// raw lifecycle endpoints below remain stable for workers and harnesses.
|
|
mux.Handle("/v1/ui/", ui.Server{Store: s, Workers: workers, Coordinator: coordinator, Route: func(e domain.Event) error {
|
|
if rt == nil {
|
|
return nil
|
|
}
|
|
_, err := rt.HandleEvent(e)
|
|
return err
|
|
}}.Handler())
|
|
mux.Handle("/", webui.Handler())
|
|
// A missed heartbeat is not relinquishment. Releasing here used to lease
|
|
// the same task to a successor while the old pane was still running. The
|
|
// authoritative lease timer performs the only automatic reassignment.
|
|
workers.OnOffline = func(w federation.Worker) { log.Printf("worker %s offline; retaining leases until expiry", w.ID) }
|
|
go func() {
|
|
ticker := time.NewTicker(15 * time.Second)
|
|
defer ticker.Stop()
|
|
for range ticker.C {
|
|
_ = workers.Snapshot()
|
|
}
|
|
}()
|
|
providerHealth := map[string]*provider.Supervisor{}
|
|
surface := func(r *http.Request) authz.Surface {
|
|
v := authz.ParseSurface(r.Header.Get("X-Orchestra-Surface"))
|
|
if v == "" || v == authz.System {
|
|
// System means "the plane itself, in-process" and is always
|
|
// FullControl with no token gate (AUDIT.md B8) — it must never
|
|
// be constructible from an HTTP request, or any LAN caller
|
|
// declaring this header gets unauthenticated full control.
|
|
return authz.Web
|
|
}
|
|
return v
|
|
}
|
|
mux.HandleFunc("/v1/tasks", func(w http.ResponseWriter, r *http.Request) {
|
|
if wid := r.Header.Get("X-Orchestra-Worker"); wid != "" {
|
|
token := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
|
|
if err := workers.Authenticate(wid, token); err != nil {
|
|
http.Error(w, err.Error(), http.StatusUnauthorized)
|
|
return
|
|
}
|
|
}
|
|
if r.Method == "GET" {
|
|
json.NewEncoder(w).Encode(s.Tasks())
|
|
return
|
|
}
|
|
if r.Method != "POST" {
|
|
http.Error(w, "method not allowed", 405)
|
|
return
|
|
}
|
|
var p map[string]any
|
|
if json.NewDecoder(r.Body).Decode(&p) != nil {
|
|
http.Error(w, "invalid json", 400)
|
|
return
|
|
}
|
|
// Make the selected project's deterministic gate part of the immutable
|
|
// task contract before any worker can create TASK.md. A caller may
|
|
// override it only when it has deliberately supplied a task-specific
|
|
// gate; routing never asks a harness to choose one.
|
|
if _, set := p["quality_gate"]; !set {
|
|
if projectID, _ := p["project"].(string); projectID != "" {
|
|
if project, ok := rr.Project(projectID); ok && project.QualityGate != "" {
|
|
p["quality_gate"] = project.QualityGate
|
|
}
|
|
}
|
|
}
|
|
b, _ := json.Marshal(p)
|
|
e := domain.Event{ID: id(), Type: "TaskCreated", TaskID: id(), Version: 1, Payload: b, Surface: string(surface(r))}
|
|
if err := s.Append(e); err != nil {
|
|
if errors.Is(err, domain.ErrDuplicate) {
|
|
// (source, external_id) was already ingested. Nothing was
|
|
// appended; report the existing task idempotently rather
|
|
// than fabricating a 201 with an unrelated event (S6).
|
|
source, _ := p["source"].(string)
|
|
externalID, _ := p["external_id"].(string)
|
|
if t, ok := s.TaskBySource(source, externalID); ok {
|
|
w.WriteHeader(200)
|
|
json.NewEncoder(w).Encode(t)
|
|
return
|
|
}
|
|
}
|
|
http.Error(w, err.Error(), 400)
|
|
return
|
|
}
|
|
if rt != nil {
|
|
if _, err := rt.HandleEvent(e); err != nil {
|
|
log.Printf("route task: %v", err)
|
|
}
|
|
}
|
|
w.WriteHeader(201)
|
|
json.NewEncoder(w).Encode(e)
|
|
})
|
|
mux.HandleFunc("/v1/events", func(w http.ResponseWriter, r *http.Request) {
|
|
var n uint64
|
|
if x, err := strconv.ParseUint(r.URL.Query().Get("since"), 10, 64); err == nil {
|
|
n = x
|
|
}
|
|
json.NewEncoder(w).Encode(s.Events(n))
|
|
})
|
|
mux.HandleFunc("/v1/handoffs", func(w http.ResponseWriter, r *http.Request) {
|
|
out := make([]domain.Event, 0)
|
|
for _, e := range s.Events(0) {
|
|
if e.Type == "TaskReleased" {
|
|
out = append(out, e)
|
|
}
|
|
}
|
|
json.NewEncoder(w).Encode(out)
|
|
})
|
|
mux.HandleFunc("/v1/quotas", func(w http.ResponseWriter, r *http.Request) {
|
|
out := make([]domain.Event, 0)
|
|
for _, e := range s.Events(0) {
|
|
if e.Type == "QuotaReported" {
|
|
out = append(out, e)
|
|
}
|
|
}
|
|
json.NewEncoder(w).Encode(out)
|
|
})
|
|
mux.HandleFunc("/v1/artifacts", func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodPost {
|
|
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
|
return
|
|
}
|
|
if wid := r.Header.Get("X-Orchestra-Worker"); wid != "" {
|
|
token := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
|
|
if err := workers.Authenticate(wid, token); err != nil {
|
|
http.Error(w, err.Error(), http.StatusUnauthorized)
|
|
return
|
|
}
|
|
}
|
|
// Reports and handoffs are content-addressed evidence. Keep uploads
|
|
// bounded because event payloads only carry their resulting hash.
|
|
r.Body = http.MaxBytesReader(w, r.Body, 4<<20)
|
|
b, err := io.ReadAll(r.Body)
|
|
if err != nil {
|
|
http.Error(w, "artifact too large or unreadable", http.StatusRequestEntityTooLarge)
|
|
return
|
|
}
|
|
if len(b) == 0 {
|
|
http.Error(w, "artifact is empty", http.StatusBadRequest)
|
|
return
|
|
}
|
|
ref, err := s.PutArtifact(b)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
w.WriteHeader(http.StatusCreated)
|
|
json.NewEncoder(w).Encode(map[string]string{"ref": ref})
|
|
})
|
|
// Artifacts are normally write-only to public surfaces. A federation
|
|
// worker may read a handoff only after authenticating as the worker that
|
|
// will validate it against its own checkout (§2.1, §6.2).
|
|
mux.HandleFunc("/v1/artifacts/", func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodGet {
|
|
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
|
return
|
|
}
|
|
wid := r.Header.Get("X-Orchestra-Worker")
|
|
token := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
|
|
if err := workers.Authenticate(wid, token); err != nil {
|
|
http.Error(w, err.Error(), http.StatusUnauthorized)
|
|
return
|
|
}
|
|
ref := strings.TrimPrefix(r.URL.Path, "/v1/artifacts/")
|
|
if len(ref) != 64 {
|
|
http.Error(w, "artifact ref required", http.StatusBadRequest)
|
|
return
|
|
}
|
|
b, err := s.Artifact(ref)
|
|
if err != nil {
|
|
http.Error(w, "artifact not found", http.StatusNotFound)
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = w.Write(b)
|
|
})
|
|
harnessToken := os.Getenv("ORCHESTRA_HARNESS_TOKEN")
|
|
// /v1/harness/turn is the unified turn-decision endpoint (AUDIT.md Phase
|
|
// 2 items 1-2): a harness-side caller posts here on an ordinary turn
|
|
// boundary and gets back continue / prepare_handoff / rotate_now / refuse,
|
|
// per spec §5.3, instead of separate ad-hoc marker-file conventions per
|
|
// decision.
|
|
//
|
|
// Completion does NOT go through this endpoint, and there is no longer a
|
|
// /v1/harness/complete: an unaffiliated harness hook has no durable worker
|
|
// identity or fencing epoch, so it cannot safely mutate a leased task.
|
|
// orchestra-worker owns completion — it watches for the .orchestra/done
|
|
// marker, confirms the agent is no longer busy, and posts through the
|
|
// authenticated federation endpoints with both lease epoch and version.
|
|
mux.HandleFunc("/v1/harness/turn", func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodPost {
|
|
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
|
return
|
|
}
|
|
if harnessToken != "" && subtle.ConstantTimeCompare([]byte(r.Header.Get("Authorization")), []byte("Bearer "+harnessToken)) != 1 {
|
|
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
|
return
|
|
}
|
|
if coordinator == nil {
|
|
http.Error(w, "coordinator not configured", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
var p struct {
|
|
TaskID string `json:"task_id"`
|
|
}
|
|
if json.NewDecoder(r.Body).Decode(&p) != nil || p.TaskID == "" {
|
|
http.Error(w, "task_id is required", http.StatusBadRequest)
|
|
return
|
|
}
|
|
decision, err := coordinator.TurnDecision(r.Context(), p.TaskID)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
|
return
|
|
}
|
|
json.NewEncoder(w).Encode(map[string]string{"decision": decision})
|
|
})
|
|
mux.HandleFunc("/v1/brief", func(w http.ResponseWriter, r *http.Request) {
|
|
to := time.Now().UTC()
|
|
from := to.Add(-12 * time.Hour)
|
|
if v, parseErr := time.Parse(time.RFC3339, r.URL.Query().Get("from")); parseErr == nil {
|
|
from = v
|
|
}
|
|
if v, parseErr := time.Parse(time.RFC3339, r.URL.Query().Get("to")); parseErr == nil {
|
|
to = v
|
|
}
|
|
git := map[string]operations.GitSync{}
|
|
for _, p := range rr.Projects() {
|
|
if p.Repo == "" {
|
|
continue
|
|
}
|
|
git[p.ID] = operations.GitState(p.Repo)
|
|
}
|
|
if len(git) == 0 {
|
|
if repo := os.Getenv("ORCHESTRA_REPO"); repo != "" {
|
|
git["default"] = operations.GitState(repo)
|
|
}
|
|
}
|
|
json.NewEncoder(w).Encode(operations.BuildBrief(s.Events(0), from, to, git))
|
|
})
|
|
standup := func() (domain.Event, error) {
|
|
return operations.GenerateStandupAdvisory(s, time.Now().UTC())
|
|
}
|
|
mux.HandleFunc("/v1/standup", func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method == http.MethodGet {
|
|
json.NewEncoder(w).Encode(operations.StandupItems(s.Tasks()))
|
|
return
|
|
}
|
|
if r.Method != http.MethodPost {
|
|
http.Error(w, "method not allowed", 405)
|
|
return
|
|
}
|
|
if err := authz.AuthorizeEvent(surface(r), "StandupAdvisory"); err != nil {
|
|
http.Error(w, err.Error(), http.StatusForbidden)
|
|
return
|
|
}
|
|
e, err := standup()
|
|
if err != nil {
|
|
http.Error(w, err.Error(), 400)
|
|
return
|
|
}
|
|
json.NewEncoder(w).Encode(e)
|
|
})
|
|
mux.HandleFunc("/v1/standup/approve", func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodPost {
|
|
http.Error(w, "method not allowed", 405)
|
|
return
|
|
}
|
|
if err := authz.AuthorizeEvent(surface(r), "ApprovalGranted"); err != nil {
|
|
http.Error(w, err.Error(), 403)
|
|
return
|
|
}
|
|
var p struct {
|
|
AdvisoryID string `json:"advisory_id"`
|
|
}
|
|
if json.NewDecoder(r.Body).Decode(&p) != nil || p.AdvisoryID == "" {
|
|
http.Error(w, "advisory_id required", 400)
|
|
return
|
|
}
|
|
b, _ := json.Marshal(map[string]any{"subject_ref": p.AdvisoryID})
|
|
e := domain.Event{ID: id(), Type: "ApprovalGranted", TaskID: "system", Version: 0, Payload: b, Surface: string(surface(r))}
|
|
if err := s.Append(e); err != nil {
|
|
http.Error(w, err.Error(), 400)
|
|
return
|
|
}
|
|
json.NewEncoder(w).Encode(e)
|
|
})
|
|
mux.HandleFunc("/v1/standup/apply", func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodPost {
|
|
http.Error(w, "method not allowed", 405)
|
|
return
|
|
}
|
|
if err := authz.AuthorizeEvent(surface(r), "TaskAmended"); err != nil {
|
|
http.Error(w, err.Error(), 403)
|
|
return
|
|
}
|
|
var p struct {
|
|
AdvisoryID string `json:"advisory_id"`
|
|
}
|
|
if json.NewDecoder(r.Body).Decode(&p) != nil || p.AdvisoryID == "" {
|
|
http.Error(w, "advisory_id required", 400)
|
|
return
|
|
}
|
|
out, err := operations.ApplyAdvisory(s, p.AdvisoryID)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), 400)
|
|
return
|
|
}
|
|
json.NewEncoder(w).Encode(out)
|
|
})
|
|
adminServer := &admin.Server{Store: s, RouterReady: rt != nil, Build: buildinfo.Current(), Providers: providerHealth, Probes: map[string]admin.ProbeFunc{
|
|
"router": func() (bool, string) { return rt != nil, "configured router" },
|
|
"gitea": func() (bool, string) {
|
|
configured := os.Getenv("ORCHESTRA_GITEA_URL") != "" || os.Getenv("ORCHESTRA_GITEA_CONFIG") != ""
|
|
if !configured {
|
|
return true, "configured provider"
|
|
}
|
|
for name := range providerHealth {
|
|
if name == "gitea" || strings.HasPrefix(name, "gitea:") {
|
|
return true, "configured provider"
|
|
}
|
|
}
|
|
return false, "configured provider"
|
|
},
|
|
"jsonl": func() (bool, string) {
|
|
return os.Getenv("ORCHESTRA_JSONL") == "" || providerHealth["jsonl"] != nil, "configured provider"
|
|
},
|
|
}}
|
|
mux.HandleFunc("/metrics", adminServer.Metrics)
|
|
mux.HandleFunc("/v1/events/subscribe", adminServer.Subscribe)
|
|
mux.HandleFunc("/v1/admin/diagnostics", adminServer.Diagnostics)
|
|
mux.HandleFunc("/v1/tasks/", func(w http.ResponseWriter, r *http.Request) {
|
|
parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/")
|
|
if r.Method == http.MethodGet && len(parts) == 4 && parts[3] == "intent" {
|
|
// The reduced authority for one task. Federation workers read this
|
|
// and render their own launch instruction with agentctx, so there
|
|
// is one renderer in the codebase rather than one per machine.
|
|
intent, err := s.EffectiveIntent(parts[2])
|
|
if err != nil {
|
|
http.Error(w, err.Error(), 404)
|
|
return
|
|
}
|
|
json.NewEncoder(w).Encode(intent)
|
|
return
|
|
}
|
|
if r.Method == http.MethodGet && coordinator != nil && len(parts) == 4 {
|
|
taskID, view := parts[2], parts[3]
|
|
if view == "health" {
|
|
if h, ok := coordinator.MonitorHealth().Sessions[taskID]; ok {
|
|
json.NewEncoder(w).Encode(h)
|
|
return
|
|
}
|
|
http.Error(w, "session health not found", 404)
|
|
return
|
|
}
|
|
if view == "capture" {
|
|
body, err := coordinator.Capture(r.Context(), taskID, r.URL.Query().Get("source"))
|
|
if err != nil {
|
|
http.Error(w, err.Error(), 404)
|
|
return
|
|
}
|
|
json.NewEncoder(w).Encode(map[string]string{"task_id": taskID, "source": r.URL.Query().Get("source"), "text": body})
|
|
return
|
|
}
|
|
}
|
|
if len(parts) < 4 || len(parts) > 5 || r.Method != "POST" {
|
|
http.Error(w, "not found", http.StatusNotFound)
|
|
return
|
|
}
|
|
taskID, action := parts[2], parts[3]
|
|
if (action == "decision-request" || action == "deferred") && len(parts) == 4 {
|
|
// An agent surface may state a bounded question or a deferred
|
|
// finding. Neither moves the lifecycle: Orchestra decides what a
|
|
// question does to the task.
|
|
if err := authz.AuthorizeEvent(surface(r), "ApprovalRequested"); err != nil {
|
|
http.Error(w, err.Error(), http.StatusForbidden)
|
|
return
|
|
}
|
|
t, ok := s.Task(taskID)
|
|
if !ok {
|
|
http.Error(w, "task not found", 404)
|
|
return
|
|
}
|
|
project, _ := rr.Project(t.Project)
|
|
r.Body = http.MaxBytesReader(w, r.Body, 64<<10)
|
|
if action == "deferred" {
|
|
var f domain.DeferredFinding
|
|
if json.NewDecoder(r.Body).Decode(&f) != nil {
|
|
http.Error(w, "invalid deferred finding", http.StatusBadRequest)
|
|
return
|
|
}
|
|
e, err := operations.RecordDeferredFinding(s, taskID, f)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), 409)
|
|
return
|
|
}
|
|
json.NewEncoder(w).Encode(e)
|
|
return
|
|
}
|
|
var req domain.DecisionRequest
|
|
if json.NewDecoder(r.Body).Decode(&req) != nil {
|
|
http.Error(w, "invalid decision request", http.StatusBadRequest)
|
|
return
|
|
}
|
|
e, err := operations.RequestHumanDecision(s, project, taskID, req)
|
|
if errors.Is(err, operations.ErrDecisionBudgetSpent) {
|
|
w.WriteHeader(http.StatusAccepted)
|
|
json.NewEncoder(w).Encode(map[string]string{"status": "operator_required", "detail": err.Error()})
|
|
return
|
|
}
|
|
if err != nil {
|
|
http.Error(w, err.Error(), 409)
|
|
return
|
|
}
|
|
w.WriteHeader(http.StatusAccepted)
|
|
json.NewEncoder(w).Encode(e)
|
|
return
|
|
}
|
|
if action == "submission" && len(parts) == 4 {
|
|
// The only path from a reviewed implementation to the human. It
|
|
// verifies first and returns the plan, or performs the submission
|
|
// when the caller supplies a publisher-backed request.
|
|
if err := authz.AuthorizeEvent(surface(r), domain.EventTaskSubmitted); err != nil {
|
|
http.Error(w, err.Error(), http.StatusForbidden)
|
|
return
|
|
}
|
|
t, ok := s.Task(taskID)
|
|
if !ok {
|
|
http.Error(w, "task not found", 404)
|
|
return
|
|
}
|
|
project, ok := rr.Project(t.Project)
|
|
if !ok {
|
|
http.Error(w, "unknown project "+t.Project, 409)
|
|
return
|
|
}
|
|
r.Body = http.MaxBytesReader(w, r.Body, 1<<20)
|
|
var body struct {
|
|
HeadSHA string `json:"head_sha"`
|
|
Gate domain.GateResult `json:"gate"`
|
|
Notes operations.Notes `json:"notes"`
|
|
DryRun bool `json:"dry_run"`
|
|
}
|
|
if json.NewDecoder(r.Body).Decode(&body) != nil || body.HeadSHA == "" {
|
|
http.Error(w, "head_sha and gate are required", http.StatusBadRequest)
|
|
return
|
|
}
|
|
check := domain.CheckSubmission(t, body.HeadSHA, body.Gate)
|
|
check.Reasons = append(check.Reasons, t.RequirePhaseArtifacts(project.Phases())...)
|
|
check.Eligible = len(check.Reasons) == 0
|
|
if body.DryRun || !check.Eligible {
|
|
code := http.StatusOK
|
|
if !check.Eligible {
|
|
code = http.StatusConflict
|
|
}
|
|
w.WriteHeader(code)
|
|
json.NewEncoder(w).Encode(check)
|
|
return
|
|
}
|
|
plan, err := operations.PrepareSubmission(s, project, taskID, body.HeadSHA, body.Gate, body.Notes)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusConflict)
|
|
return
|
|
}
|
|
if submissionPublisher == nil {
|
|
// No forge configured: hand back the verified plan so an
|
|
// operator can perform the submission themselves.
|
|
w.WriteHeader(http.StatusAccepted)
|
|
json.NewEncoder(w).Encode(map[string]any{"status": "no_publisher", "plan": plan})
|
|
return
|
|
}
|
|
e, err := operations.ExecuteSubmission(r.Context(), s, plan, submissionPublisher(plan), nil)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusConflict)
|
|
return
|
|
}
|
|
json.NewEncoder(w).Encode(e)
|
|
return
|
|
}
|
|
if action == "review" && len(parts) == 4 {
|
|
// Entering review and sealing a review are both Orchestra's, not
|
|
// the reviewing session's. The session only supplies findings.
|
|
if err := authz.AuthorizeEvent(surface(r), domain.EventReviewRecorded); err != nil {
|
|
http.Error(w, err.Error(), http.StatusForbidden)
|
|
return
|
|
}
|
|
t, ok := s.Task(taskID)
|
|
if !ok {
|
|
http.Error(w, "task not found", 404)
|
|
return
|
|
}
|
|
project, ok := rr.Project(t.Project)
|
|
if !ok {
|
|
http.Error(w, "unknown project "+t.Project, 409)
|
|
return
|
|
}
|
|
r.Body = http.MaxBytesReader(w, r.Body, int64(review.MaxDiffBytes)+(1<<20))
|
|
var body struct {
|
|
Evidence *review.Evidence `json:"evidence"`
|
|
Result *review.Result `json:"result"`
|
|
}
|
|
if json.NewDecoder(r.Body).Decode(&body) != nil || (body.Evidence == nil) == (body.Result == nil) {
|
|
http.Error(w, "send either evidence (to enter review) or result (to seal one)", http.StatusBadRequest)
|
|
return
|
|
}
|
|
var e domain.Event
|
|
var err error
|
|
if body.Evidence != nil {
|
|
e, err = operations.EnterReview(s, project, taskID, *body.Evidence)
|
|
} else {
|
|
e, err = operations.RecordReview(s, project, taskID, *body.Result)
|
|
}
|
|
if errors.Is(err, operations.ErrReviewNotEligible) || errors.Is(err, domain.ErrInvalid) {
|
|
http.Error(w, err.Error(), http.StatusConflict)
|
|
return
|
|
}
|
|
if err != nil {
|
|
http.Error(w, err.Error(), 409)
|
|
return
|
|
}
|
|
json.NewEncoder(w).Encode(e)
|
|
return
|
|
}
|
|
if action == "phase" && len(parts) == 4 {
|
|
// Orchestra owns the phase. An agent asks for a change through the
|
|
// approval surface; this endpoint is how the decision is applied.
|
|
if err := authz.AuthorizeEvent(surface(r), domain.EventWorkPhaseChanged); err != nil {
|
|
http.Error(w, err.Error(), http.StatusForbidden)
|
|
return
|
|
}
|
|
t, ok := s.Task(taskID)
|
|
if !ok {
|
|
http.Error(w, "task not found", 404)
|
|
return
|
|
}
|
|
project, ok := rr.Project(t.Project)
|
|
if !ok {
|
|
http.Error(w, "unknown project "+t.Project, 409)
|
|
return
|
|
}
|
|
// The body, when present, is the artifact the finished phase
|
|
// produced. Bounded like every other artifact upload.
|
|
r.Body = http.MaxBytesReader(w, r.Body, 1<<20)
|
|
artifact, readErr := io.ReadAll(r.Body)
|
|
if readErr != nil {
|
|
http.Error(w, "artifact unreadable", http.StatusRequestEntityTooLarge)
|
|
return
|
|
}
|
|
e, err := operations.AdvanceWorkPhase(s, project, taskID, artifact)
|
|
if errors.Is(err, operations.ErrTrajectoryGate) {
|
|
// Not a failure: the task is blocked on the human, and the
|
|
// packet is on the block event the notification surfaces read.
|
|
w.WriteHeader(http.StatusAccepted)
|
|
json.NewEncoder(w).Encode(map[string]string{"status": "trajectory_gate", "detail": err.Error()})
|
|
return
|
|
}
|
|
if err != nil {
|
|
http.Error(w, err.Error(), 409)
|
|
return
|
|
}
|
|
json.NewEncoder(w).Encode(e)
|
|
return
|
|
}
|
|
if action == "approval" {
|
|
if err := authz.AuthorizeEvent(surface(r), map[bool]string{true: "ApprovalRequested", false: "ApprovalGranted"}[len(parts) == 4]); err != nil && len(parts) == 4 {
|
|
http.Error(w, err.Error(), http.StatusForbidden)
|
|
return
|
|
}
|
|
if len(parts) == 5 {
|
|
typ := "ApprovalGranted"
|
|
if parts[4] == "deny" {
|
|
typ = "ApprovalDenied"
|
|
}
|
|
if err := authz.AuthorizeEvent(surface(r), typ); err != nil {
|
|
http.Error(w, err.Error(), http.StatusForbidden)
|
|
return
|
|
}
|
|
if parts[4] != "grant" && parts[4] != "deny" {
|
|
http.Error(w, "unknown approval action", 404)
|
|
return
|
|
}
|
|
t, ok := s.Task(taskID)
|
|
if !ok {
|
|
http.Error(w, "task not found", 404)
|
|
return
|
|
}
|
|
by := r.Header.Get("X-Orchestra-Actor")
|
|
if by == "" {
|
|
by = "surface"
|
|
}
|
|
b, _ := json.Marshal(map[string]any{"subject_ref": taskID, "by": by})
|
|
e := domain.Event{ID: id(), Type: typ, TaskID: taskID, Version: t.Version + 1, Payload: b, Surface: string(surface(r))}
|
|
if err := s.Append(e); err != nil {
|
|
http.Error(w, err.Error(), 409)
|
|
return
|
|
}
|
|
json.NewEncoder(w).Encode(e)
|
|
return
|
|
}
|
|
var p struct {
|
|
Options []any `json:"options"`
|
|
}
|
|
if json.NewDecoder(r.Body).Decode(&p) != nil || len(p.Options) == 0 {
|
|
http.Error(w, "options required", 400)
|
|
return
|
|
}
|
|
if _, ok := s.Task(taskID); !ok {
|
|
http.Error(w, "task not found", 404)
|
|
return
|
|
}
|
|
b, _ := json.Marshal(map[string]any{"subject_ref": taskID, "options": p.Options})
|
|
t, _ := s.Task(taskID)
|
|
e := domain.Event{ID: id(), Type: "ApprovalRequested", TaskID: taskID, Version: t.Version + 1, Payload: b, Surface: string(surface(r))}
|
|
if err := s.Append(e); err != nil {
|
|
http.Error(w, err.Error(), 400)
|
|
return
|
|
}
|
|
json.NewEncoder(w).Encode(e)
|
|
return
|
|
}
|
|
var e domain.Event
|
|
var err error
|
|
actionTypes := map[string]string{"lease": "TaskLeased", "release": "TaskReleased", "complete": "TaskCompleted", "block": "TaskBlocked", "attention": "TaskNeedsAttention", "retry": "TaskCorrected"}
|
|
if typ, known := actionTypes[action]; known {
|
|
if err := authz.AuthorizeEvent(surface(r), typ); err != nil {
|
|
http.Error(w, err.Error(), http.StatusForbidden)
|
|
return
|
|
}
|
|
}
|
|
switch action {
|
|
case "retry":
|
|
// Narrow on purpose: RetryTask restores a retry budget on a
|
|
// terminal task. There is no generic correction endpoint, because
|
|
// one would be arbitrary task mutation over HTTP.
|
|
var p struct {
|
|
OperationID string `json:"operation_id"`
|
|
}
|
|
if r.Body == nil || json.NewDecoder(r.Body).Decode(&p) != nil || p.OperationID == "" {
|
|
http.Error(w, "operation_id required", http.StatusBadRequest)
|
|
return
|
|
}
|
|
e, err = operations.RetryTask(s, surface(r), taskID, p.OperationID)
|
|
switch {
|
|
case errors.Is(err, domain.ErrNotFound):
|
|
http.Error(w, err.Error(), http.StatusNotFound)
|
|
return
|
|
case errors.Is(err, operations.ErrNotRetryable), errors.Is(err, domain.ErrInvalid):
|
|
http.Error(w, err.Error(), http.StatusConflict)
|
|
return
|
|
}
|
|
case "lease":
|
|
var p struct {
|
|
HarnessID string `json:"harness_id"`
|
|
TTLSeconds int `json:"ttl_seconds"`
|
|
}
|
|
if json.NewDecoder(r.Body).Decode(&p) != nil || p.HarnessID == "" {
|
|
http.Error(w, "harness_id required", 400)
|
|
return
|
|
}
|
|
if p.TTLSeconds <= 0 {
|
|
p.TTLSeconds = int(domain.LeaseTTL.Seconds())
|
|
}
|
|
e, err = s.Lease(taskID, p.HarnessID, time.Duration(p.TTLSeconds)*time.Second)
|
|
case "release", "complete", "block", "attention":
|
|
t, ok := s.Task(taskID)
|
|
if !ok {
|
|
http.Error(w, "task not found", 404)
|
|
return
|
|
}
|
|
types := map[string]string{"release": "TaskReleased", "complete": "TaskCompleted", "block": "TaskBlocked", "attention": "TaskNeedsAttention"}
|
|
var p map[string]any
|
|
if r.Body == nil || json.NewDecoder(r.Body).Decode(&p) != nil || p == nil {
|
|
http.Error(w, "invalid lifecycle payload", http.StatusBadRequest)
|
|
return
|
|
}
|
|
if action == "release" && p["reason"] == nil && p["handoff_ref"] == nil {
|
|
http.Error(w, "reason or handoff_ref required", http.StatusBadRequest)
|
|
return
|
|
}
|
|
if (action == "block" || action == "attention") && p["blocker"] == nil {
|
|
http.Error(w, "blocker required", http.StatusBadRequest)
|
|
return
|
|
}
|
|
if action == "complete" && p["report_ref"] == nil {
|
|
http.Error(w, "report_ref required", http.StatusBadRequest)
|
|
return
|
|
}
|
|
ePayload, _ := json.Marshal(p)
|
|
e = domain.Event{ID: id(), Type: types[action], TaskID: taskID, Version: t.Version + 1, Payload: ePayload, Surface: string(surface(r))}
|
|
err = s.Append(e)
|
|
default:
|
|
http.Error(w, "unknown action", 404)
|
|
return
|
|
}
|
|
if err != nil {
|
|
http.Error(w, err.Error(), 409)
|
|
return
|
|
}
|
|
if rt != nil {
|
|
if _, routeErr := rt.HandleEvent(e); routeErr != nil {
|
|
log.Printf("route task: %v", routeErr)
|
|
}
|
|
}
|
|
json.NewEncoder(w).Encode(e)
|
|
})
|
|
if rt != nil {
|
|
go func() {
|
|
ticker := time.NewTicker(time.Second)
|
|
defer ticker.Stop()
|
|
for range ticker.C {
|
|
// When a coordinator is running, its own Monitor loop (every
|
|
// 30s) already calls Store.ExpireLeases and, critically,
|
|
// kills the herdr session for whatever it expires (S9: two
|
|
// independent expiry loops raced on the same reclaim, and
|
|
// since this one runs every second vs. the coordinator's
|
|
// 30s, it almost always won — meaning the coordinator's
|
|
// ExpireLeases call saw nothing left to expire and its
|
|
// session-kill path never ran, silently orphaning panes
|
|
// past their lease TTL). Calling ExpireLeases here too would
|
|
// just resurrect that race, so leave reclaim to the
|
|
// coordinator and only keep retrying pending assignment.
|
|
// A blocked task is invisible to the router, so answered
|
|
// blockers are returned to the queue here, immediately
|
|
// upstream of assignment.
|
|
if _, err := operations.ResumeAnsweredBlockers(s); err != nil {
|
|
log.Printf("resume answered blockers: %v", err)
|
|
}
|
|
if coordinator != nil {
|
|
if _, err := rt.AssignPending(); err != nil {
|
|
log.Printf("route expired task: %v", err)
|
|
}
|
|
continue
|
|
}
|
|
if _, err := s.ExpireLeases(time.Now()); err != nil {
|
|
log.Printf("expire leases: %v", err)
|
|
} else if _, err := rt.AssignPending(); err != nil {
|
|
log.Printf("route expired task: %v", err)
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("ok\n")) })
|
|
mux.HandleFunc("/readyz", adminServer.Readiness)
|
|
mux.HandleFunc("/v1/providers/health", func(w http.ResponseWriter, r *http.Request) {
|
|
out := map[string]provider.Health{}
|
|
for name, sup := range providerHealth {
|
|
out[name] = sup.Health()
|
|
}
|
|
json.NewEncoder(w).Encode(out)
|
|
})
|
|
mux.HandleFunc("/v1/router/health", func(w http.ResponseWriter, r *http.Request) {
|
|
// Why the last scheduling pass placed nothing. A silent eligibility
|
|
// gate is indistinguishable from an empty queue, which cost a burn-in
|
|
// run to diagnose by hand.
|
|
if rt == nil {
|
|
http.Error(w, "router not configured", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
json.NewEncoder(w).Encode(map[string]any{"rejections": rt.Rejections()})
|
|
})
|
|
mux.HandleFunc("/v1/orchestrator/health", func(w http.ResponseWriter, r *http.Request) {
|
|
if coordinator == nil {
|
|
http.Error(w, "orchestrator unavailable", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
json.NewEncoder(w).Encode(coordinator.MonitorHealth())
|
|
})
|
|
mux.HandleFunc("/v1/federation/workers", func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method == http.MethodGet {
|
|
json.NewEncoder(w).Encode(workers.Snapshot())
|
|
return
|
|
}
|
|
if r.Method != http.MethodPost {
|
|
http.Error(w, "method not allowed", 405)
|
|
return
|
|
}
|
|
var registration struct {
|
|
federation.Worker
|
|
Token string `json:"token"`
|
|
}
|
|
if json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10)).Decode(®istration) != nil {
|
|
http.Error(w, "invalid worker", 400)
|
|
return
|
|
}
|
|
worker := registration.Worker
|
|
worker.Token = registration.Token
|
|
if worker.Token == "" {
|
|
http.Error(w, "token required", 400)
|
|
return
|
|
}
|
|
admitToken := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
|
|
if err := workers.Register(worker, admitToken); err != nil {
|
|
status := 400
|
|
if err == federation.ErrUnauthorized {
|
|
status = 401
|
|
}
|
|
http.Error(w, err.Error(), status)
|
|
return
|
|
}
|
|
if rt != nil {
|
|
if _, err := rt.AssignPending(); err != nil {
|
|
log.Printf("route after worker registration: %v", err)
|
|
}
|
|
}
|
|
w.WriteHeader(http.StatusCreated)
|
|
json.NewEncoder(w).Encode(worker)
|
|
})
|
|
workerAuth := func(r *http.Request) (string, error) {
|
|
wid := r.Header.Get("X-Orchestra-Worker")
|
|
tok := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
|
|
if err := workers.Authenticate(wid, tok); err != nil {
|
|
return "", err
|
|
}
|
|
return wid, nil
|
|
}
|
|
mux.HandleFunc("/v1/federation/turn", func(w http.ResponseWriter, r *http.Request) {
|
|
if _, err := workerAuth(r); err != nil {
|
|
http.Error(w, err.Error(), http.StatusUnauthorized)
|
|
return
|
|
}
|
|
if r.Method != http.MethodPost {
|
|
http.Error(w, "method not allowed", 405)
|
|
return
|
|
}
|
|
if coordinator == nil {
|
|
http.Error(w, "coordinator not configured", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
var body struct {
|
|
TaskID string `json:"task_id"`
|
|
LeaseEpoch string `json:"lease_epoch"`
|
|
Verdict string `json:"verdict"`
|
|
Delivered []string `json:"delivered_decisions"`
|
|
}
|
|
if json.NewDecoder(r.Body).Decode(&body) != nil || body.TaskID == "" || body.LeaseEpoch == "" {
|
|
http.Error(w, "task_id and lease_epoch are required", http.StatusBadRequest)
|
|
return
|
|
}
|
|
switch body.Verdict {
|
|
case orchestrator.TurnContinue, orchestrator.TurnPrepareHandoff, orchestrator.TurnRotateNow, orchestrator.TurnRefuse:
|
|
default:
|
|
http.Error(w, "unknown verdict "+body.Verdict, http.StatusBadRequest)
|
|
return
|
|
}
|
|
verdict, decisions, err := coordinator.RemoteTurn(r.Context(), body.TaskID, body.LeaseEpoch, body.Verdict, body.Delivered)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusConflict)
|
|
return
|
|
}
|
|
json.NewEncoder(w).Encode(federation.TurnDecision{Verdict: verdict, Decisions: decisions})
|
|
})
|
|
// F21. The worker's report of an agent's bounded phase-change request.
|
|
// The agent asks by writing a file; Orchestra decides here, through the
|
|
// same AdvanceWorkPhase the operator surface uses.
|
|
mux.HandleFunc("/v1/federation/phase", func(w http.ResponseWriter, r *http.Request) {
|
|
if _, err := workerAuth(r); err != nil {
|
|
http.Error(w, err.Error(), http.StatusUnauthorized)
|
|
return
|
|
}
|
|
if r.Method != http.MethodPost {
|
|
http.Error(w, "method not allowed", 405)
|
|
return
|
|
}
|
|
var body struct {
|
|
TaskID string `json:"task_id"`
|
|
LeaseEpoch string `json:"lease_epoch"`
|
|
OperationID string `json:"operation_id"`
|
|
From domain.WorkPhase `json:"from"`
|
|
To domain.WorkPhase `json:"to"`
|
|
Artifact []byte `json:"artifact"`
|
|
}
|
|
if json.NewDecoder(r.Body).Decode(&body) != nil || body.TaskID == "" || body.LeaseEpoch == "" {
|
|
http.Error(w, "task_id and lease_epoch are required", http.StatusBadRequest)
|
|
return
|
|
}
|
|
t, ok := s.Task(body.TaskID)
|
|
if !ok {
|
|
http.Error(w, "task not found", 404)
|
|
return
|
|
}
|
|
project, ok := rr.Project(t.Project)
|
|
if !ok {
|
|
http.Error(w, "unknown project "+t.Project, 409)
|
|
return
|
|
}
|
|
e, err := operations.RequestWorkPhase(s, project, body.TaskID, body.LeaseEpoch, body.OperationID, body.From, body.To, body.Artifact)
|
|
if errors.Is(err, operations.ErrTrajectoryGate) {
|
|
// The human is being asked. Not a failure, and not a phase change
|
|
// the worker should rotate on yet.
|
|
w.WriteHeader(http.StatusAccepted)
|
|
json.NewEncoder(w).Encode(map[string]string{"status": "trajectory_gate", "detail": err.Error()})
|
|
return
|
|
}
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusConflict)
|
|
return
|
|
}
|
|
var p struct {
|
|
Phase domain.WorkPhase `json:"phase"`
|
|
}
|
|
_ = json.Unmarshal(e.Payload, &p)
|
|
json.NewEncoder(w).Encode(map[string]string{"phase": string(p.Phase)})
|
|
})
|
|
mux.HandleFunc("/v1/federation/commands", func(w http.ResponseWriter, r *http.Request) {
|
|
wid, err := workerAuth(r)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusUnauthorized)
|
|
return
|
|
}
|
|
if r.Method != http.MethodGet {
|
|
http.Error(w, "method not allowed", 405)
|
|
return
|
|
}
|
|
out, err := workers.Commands(wid)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), 404)
|
|
return
|
|
}
|
|
json.NewEncoder(w).Encode(out)
|
|
})
|
|
mux.HandleFunc("/v1/federation/commands/", func(w http.ResponseWriter, r *http.Request) {
|
|
wid, err := workerAuth(r)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusUnauthorized)
|
|
return
|
|
}
|
|
if r.Method != http.MethodPost {
|
|
http.Error(w, "method not allowed", 405)
|
|
return
|
|
}
|
|
var body struct {
|
|
Status string `json:"status"`
|
|
Message string `json:"message"`
|
|
}
|
|
if json.NewDecoder(r.Body).Decode(&body) != nil || (body.Status != "acknowledged" && body.Status != "stale" && body.Status != "rejected") {
|
|
http.Error(w, "invalid command result", 400)
|
|
return
|
|
}
|
|
commandID := strings.TrimPrefix(r.URL.Path, "/v1/federation/commands/")
|
|
command, ok := workers.Command(wid, commandID)
|
|
if !ok {
|
|
http.Error(w, "command not found", 404)
|
|
return
|
|
}
|
|
if err := workers.CompleteCommand(wid, commandID, body.Status, body.Message); err != nil {
|
|
http.Error(w, err.Error(), 409)
|
|
return
|
|
}
|
|
// Audit lifecycle evidence only after the worker reports the herdr
|
|
// input was acknowledged; a queued browser click is never an approval.
|
|
if body.Status == "acknowledged" {
|
|
if t, ok := s.Task(command.TaskID); ok {
|
|
typ := "ApprovalGranted"
|
|
if command.Kind == "deny_approval" {
|
|
typ = "ApprovalDenied"
|
|
}
|
|
payload, _ := json.Marshal(map[string]any{"subject_ref": command.ID, "pane_id": command.PaneID, "capture_revision": command.CaptureRevision})
|
|
e := domain.Event{ID: id(), Type: typ, TaskID: command.TaskID, Version: t.Version + 1, Payload: payload, Surface: string(authz.System)}
|
|
if err := s.Append(e); err != nil {
|
|
log.Printf("record approval %s: %v", command.ID, err)
|
|
}
|
|
}
|
|
}
|
|
w.WriteHeader(http.StatusNoContent)
|
|
})
|
|
mux.HandleFunc("/v1/federation/events", func(w http.ResponseWriter, r *http.Request) {
|
|
wid, err := workerAuth(r)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusUnauthorized)
|
|
return
|
|
}
|
|
if r.Method != http.MethodGet {
|
|
http.Error(w, "method not allowed", 405)
|
|
return
|
|
}
|
|
cursor, _ := workers.Cursor(wid)
|
|
if q := r.URL.Query().Get("since"); q != "" {
|
|
cursor, _ = strconv.ParseUint(q, 10, 64)
|
|
}
|
|
events := s.Events(cursor)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
json.NewEncoder(w).Encode(map[string]any{"cursor": cursor, "events": events})
|
|
})
|
|
mux.HandleFunc("/v1/federation/events/ack", func(w http.ResponseWriter, r *http.Request) {
|
|
wid, err := workerAuth(r)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), 401)
|
|
return
|
|
}
|
|
if r.Method != http.MethodPost {
|
|
http.Error(w, "method not allowed", 405)
|
|
return
|
|
}
|
|
var body struct {
|
|
Cursor uint64 `json:"cursor"`
|
|
}
|
|
if json.NewDecoder(r.Body).Decode(&body) != nil {
|
|
http.Error(w, "invalid cursor", 400)
|
|
return
|
|
}
|
|
if err := workers.Ack(wid, body.Cursor); err != nil {
|
|
http.Error(w, err.Error(), 409)
|
|
return
|
|
}
|
|
w.WriteHeader(http.StatusNoContent)
|
|
})
|
|
mux.HandleFunc("/v1/federation/workers/", func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodPost || (!strings.HasSuffix(r.URL.Path, "/heartbeat") && !strings.HasSuffix(r.URL.Path, "/renew") && !strings.HasSuffix(r.URL.Path, "/start") && !strings.HasSuffix(r.URL.Path, "/nack") && !strings.HasSuffix(r.URL.Path, "/handoff") && !strings.HasSuffix(r.URL.Path, "/pickup") && !strings.HasSuffix(r.URL.Path, "/complete") && !strings.HasSuffix(r.URL.Path, "/submit") && !strings.HasSuffix(r.URL.Path, "/captures")) {
|
|
http.Error(w, "not found", 404)
|
|
return
|
|
}
|
|
parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/")
|
|
if len(parts) != 5 {
|
|
http.Error(w, "not found", 404)
|
|
return
|
|
}
|
|
wid, err := workerAuth(r)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), 401)
|
|
return
|
|
}
|
|
if wid != parts[3] {
|
|
http.Error(w, "worker identity mismatch", http.StatusForbidden)
|
|
return
|
|
}
|
|
if strings.HasSuffix(r.URL.Path, "/heartbeat") {
|
|
var health federation.WorkerHealth
|
|
if err := json.NewDecoder(r.Body).Decode(&health); err != nil && !errors.Is(err, io.EOF) {
|
|
http.Error(w, "invalid worker health", 400)
|
|
return
|
|
}
|
|
if err := workers.Heartbeat(parts[3], health); err != nil {
|
|
http.Error(w, err.Error(), 404)
|
|
return
|
|
}
|
|
w.WriteHeader(http.StatusNoContent)
|
|
return
|
|
}
|
|
if strings.HasSuffix(r.URL.Path, "/captures") {
|
|
var c federation.Capture
|
|
if json.NewDecoder(r.Body).Decode(&c) != nil {
|
|
http.Error(w, "invalid capture", 400)
|
|
return
|
|
}
|
|
if t, ok := s.Task(c.TaskID); !ok || t.Lease == nil || t.Lease.HarnessID != parts[3] {
|
|
http.Error(w, "lease not owned", 409)
|
|
return
|
|
}
|
|
out, err := workers.PutCapture(parts[3], c)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), 400)
|
|
return
|
|
}
|
|
json.NewEncoder(w).Encode(out)
|
|
return
|
|
}
|
|
var b struct {
|
|
TaskID string `json:"task_id"`
|
|
TTLSeconds int `json:"ttl_seconds"`
|
|
ExpectedVersion int `json:"expected_version"`
|
|
HandoffRef string `json:"handoff_ref"`
|
|
AnchorSHA string `json:"anchor_sha"`
|
|
TransactionID string `json:"transaction_id"`
|
|
LeaseVersion int `json:"lease_version"`
|
|
LeaseEpoch string `json:"lease_epoch"`
|
|
ResultSHA string `json:"result_sha"`
|
|
Branch string `json:"branch"`
|
|
Remote string `json:"remote"`
|
|
Receipt map[string]any `json:"receipt"`
|
|
FailureClass string `json:"failure_class"`
|
|
LastError string `json:"last_error"`
|
|
SessionEvidence domain.SessionEvidence `json:"session_evidence"`
|
|
Review review.Result `json:"review"`
|
|
Gate domain.GateResult `json:"gate"`
|
|
}
|
|
if json.NewDecoder(r.Body).Decode(&b) != nil || b.TaskID == "" {
|
|
http.Error(w, "invalid lease body", 400)
|
|
return
|
|
}
|
|
t, ok := s.Task(b.TaskID)
|
|
if !ok {
|
|
http.Error(w, "task not found", 404)
|
|
return
|
|
}
|
|
ownedLease := (t.State == domain.StateLeased || t.State == domain.StateNeedsAttention) && t.Lease != nil && t.Lease.HarnessID == parts[3] && t.Lease.Epoch == b.LeaseEpoch
|
|
// A response can be lost after the append/fsync. Retrying the exact
|
|
// release transaction is therefore a successful no-op, never a second
|
|
// TaskReleased event and never a reason to discard the predecessor.
|
|
if strings.HasSuffix(r.URL.Path, "/handoff") && t.State == domain.StateQueued && b.TransactionID != "" && b.TransactionID == t.ReleaseTransaction && b.HandoffRef == t.HandoffRef && b.AnchorSHA == t.ReleaseAnchor {
|
|
w.WriteHeader(http.StatusNoContent)
|
|
return
|
|
}
|
|
if !ownedLease {
|
|
http.Error(w, "lease not owned", 409)
|
|
return
|
|
}
|
|
if (strings.HasSuffix(r.URL.Path, "/complete") || strings.HasSuffix(r.URL.Path, "/submit")) && b.ExpectedVersion != t.Version {
|
|
http.Error(w, "lease version conflict", http.StatusConflict)
|
|
return
|
|
}
|
|
if strings.HasSuffix(r.URL.Path, "/submit") {
|
|
// Seal the review, then submit the commit it examined. Both events
|
|
// are Orchestra's; the worker supplies verified evidence and the
|
|
// coordinator decides what it means.
|
|
project, ok := rr.Project(t.Project)
|
|
if !ok {
|
|
http.Error(w, "unknown project "+t.Project, 409)
|
|
return
|
|
}
|
|
if len(b.ResultSHA) != 40 {
|
|
http.Error(w, "verified result_sha required", 400)
|
|
return
|
|
}
|
|
if !t.Submitted(b.ResultSHA) && (t.Review == nil || t.Review.ResultSHA != b.ResultSHA) {
|
|
b.Review.ResultSHA = b.ResultSHA
|
|
if _, err := operations.RecordReview(s, project, b.TaskID, b.Review); err != nil {
|
|
http.Error(w, err.Error(), http.StatusConflict)
|
|
return
|
|
}
|
|
if !b.Review.Accepted() {
|
|
// The review sent the work back to implementation. That is
|
|
// an outcome, not a failure, and there is nothing to submit.
|
|
json.NewEncoder(w).Encode(map[string]any{"status": federation.SubmitChangesRequested, "blocking": len(b.Review.Blocking())})
|
|
return
|
|
}
|
|
}
|
|
if submissionPublisher == nil {
|
|
json.NewEncoder(w).Encode(map[string]string{"status": federation.SubmitNoPublisher})
|
|
return
|
|
}
|
|
plan, err := operations.PrepareSubmission(s, project, b.TaskID, b.ResultSHA, b.Gate, operations.Notes{})
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusConflict)
|
|
return
|
|
}
|
|
if b.Remote != "" {
|
|
plan.Remote = b.Remote
|
|
}
|
|
e, err := operations.ExecuteSubmission(r.Context(), s, plan, submissionPublisher(plan), nil)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusConflict)
|
|
return
|
|
}
|
|
json.NewEncoder(w).Encode(map[string]any{"status": federation.SubmitSubmitted, "event": e})
|
|
return
|
|
}
|
|
if strings.HasSuffix(r.URL.Path, "/start") {
|
|
// A lost response after append is an idempotent start ACK, not a
|
|
// reason to strand the running pane behind a stale version.
|
|
if t.LifecyclePhase == "started" && t.Lease != nil && t.Lease.HarnessID == parts[3] && t.Lease.Epoch == b.LeaseEpoch {
|
|
w.WriteHeader(http.StatusNoContent)
|
|
return
|
|
}
|
|
if b.ExpectedVersion != t.Version {
|
|
http.Error(w, "lease version conflict", http.StatusConflict)
|
|
return
|
|
}
|
|
p, _ := json.Marshal(map[string]any{"harness_id": parts[3], "lease_epoch": b.LeaseEpoch, "expected_version": t.Version, "lifecycle_phase": "started", "session_evidence": b.SessionEvidence})
|
|
e := domain.Event{ID: id(), Type: "TaskLaunchAcknowledged", TaskID: b.TaskID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
|
|
if err := s.Append(e); err != nil {
|
|
http.Error(w, err.Error(), http.StatusConflict)
|
|
return
|
|
}
|
|
json.NewEncoder(w).Encode(e)
|
|
return
|
|
}
|
|
if strings.HasSuffix(r.URL.Path, "/nack") {
|
|
if b.ExpectedVersion != t.Version || b.FailureClass == "" || b.LastError == "" {
|
|
http.Error(w, "current lease version, failure_class, and last_error required", http.StatusConflict)
|
|
return
|
|
}
|
|
var typ string
|
|
var p []byte
|
|
switch b.FailureClass {
|
|
case "invalid_handoff":
|
|
typ = "TaskBlocked"
|
|
p, _ = json.Marshal(map[string]any{"blocker": b.LastError, "block_reason": string(domain.BlockReasonHandoffValidation), "harness_id": parts[3], "lease_epoch": b.LeaseEpoch, "expected_version": t.Version, "lifecycle_phase": "launch_nacked", "last_error": b.LastError, "session_evidence": b.SessionEvidence})
|
|
case "launch_uncertain":
|
|
typ = "TaskNeedsAttention"
|
|
p, _ = json.Marshal(map[string]any{"blocker": b.LastError, "block_reason": string(domain.BlockReasonLeaseFailure), "harness_id": parts[3], "lease_epoch": b.LeaseEpoch, "expected_version": t.Version, "lifecycle_phase": "launch_uncertain", "last_error": b.LastError, "session_evidence": b.SessionEvidence})
|
|
default:
|
|
typ = "TaskReleased"
|
|
p, _ = json.Marshal(map[string]any{"reason": "launch_failed", "failure_class": b.FailureClass, "harness_id": parts[3], "lease_epoch": b.LeaseEpoch, "expected_version": t.Version, "lifecycle_phase": "launch_nacked", "last_error": b.LastError, "session_evidence": b.SessionEvidence})
|
|
}
|
|
e := domain.Event{ID: id(), Type: typ, TaskID: b.TaskID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
|
|
if err := s.Append(e); err != nil {
|
|
http.Error(w, err.Error(), http.StatusConflict)
|
|
return
|
|
}
|
|
if typ == "TaskReleased" && rt != nil {
|
|
_, _ = rt.HandleEvent(e)
|
|
}
|
|
json.NewEncoder(w).Encode(e)
|
|
return
|
|
}
|
|
if strings.HasSuffix(r.URL.Path, "/renew") {
|
|
ttl := b.TTLSeconds
|
|
if ttl == 0 {
|
|
ttl = int(domain.LeaseTTL.Seconds())
|
|
}
|
|
e, err := s.RenewLease(b.TaskID, parts[3], b.LeaseEpoch, b.ExpectedVersion, time.Duration(ttl)*time.Second)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusConflict)
|
|
return
|
|
}
|
|
json.NewEncoder(w).Encode(e)
|
|
return
|
|
}
|
|
if strings.HasSuffix(r.URL.Path, "/pickup") {
|
|
if !ownedLease || b.TransactionID == "" || b.TransactionID != t.ReleaseTransaction || b.HandoffRef != t.HandoffRef || b.AnchorSHA != t.ReleaseAnchor {
|
|
http.Error(w, "pickup does not match active release", http.StatusConflict)
|
|
return
|
|
}
|
|
if t.PickupTransaction == b.TransactionID && t.PickupLeaseVersion == b.LeaseVersion {
|
|
w.WriteHeader(http.StatusNoContent)
|
|
return
|
|
}
|
|
if b.LeaseVersion != t.Version {
|
|
http.Error(w, "lease version conflict", http.StatusConflict)
|
|
return
|
|
}
|
|
p, _ := json.Marshal(map[string]any{"transaction_id": b.TransactionID, "handoff_ref": b.HandoffRef, "anchor_sha": b.AnchorSHA, "harness_id": parts[3], "lease_epoch": b.LeaseEpoch, "lease_version": b.LeaseVersion, "expected_version": t.Version, "session_evidence": b.SessionEvidence})
|
|
e := domain.Event{ID: id(), Type: "TaskPickupValidated", TaskID: b.TaskID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
|
|
if err := s.Append(e); err != nil {
|
|
http.Error(w, err.Error(), http.StatusConflict)
|
|
return
|
|
}
|
|
json.NewEncoder(w).Encode(e)
|
|
return
|
|
}
|
|
if strings.HasSuffix(r.URL.Path, "/complete") {
|
|
if b.HandoffRef == "" {
|
|
http.Error(w, "report_ref required", 400)
|
|
return
|
|
}
|
|
if len(b.ResultSHA) != 40 || b.Branch == "" || b.Remote == "" {
|
|
http.Error(w, "verified result_sha, branch, and remote required", 400)
|
|
return
|
|
}
|
|
if b.Receipt == nil {
|
|
b.Receipt = map[string]any{}
|
|
}
|
|
b.Receipt["harness_id"] = parts[3]
|
|
if _, ok := b.Receipt["consumed"]; !ok {
|
|
b.Receipt["consumed"] = 0
|
|
}
|
|
p, _ := json.Marshal(map[string]any{"report_ref": b.HandoffRef, "receipt": b.Receipt, "result_sha": b.ResultSHA, "branch": b.Branch, "remote": b.Remote, "harness_id": parts[3], "lease_epoch": b.LeaseEpoch, "expected_version": t.Version, "session_evidence": b.SessionEvidence})
|
|
e := domain.Event{ID: id(), Type: "TaskCompleted", TaskID: b.TaskID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
|
|
if err := s.Append(e); err != nil {
|
|
http.Error(w, err.Error(), 409)
|
|
return
|
|
}
|
|
if consumed, ok := b.Receipt["consumed"].(float64); ok && consumed > 0 {
|
|
qp, _ := json.Marshal(map[string]any{"harness_id": parts[3], "consumed": consumed})
|
|
if err := s.Append(domain.Event{ID: id(), Type: "QuotaReported", TaskID: "system", Payload: qp, Surface: string(authz.System)}); err != nil {
|
|
log.Printf("federated quota report %s: %v", b.TaskID, err)
|
|
}
|
|
}
|
|
json.NewEncoder(w).Encode(e)
|
|
return
|
|
}
|
|
if b.HandoffRef == "" {
|
|
http.Error(w, "handoff_ref required", 400)
|
|
return
|
|
}
|
|
if len(b.AnchorSHA) != 40 {
|
|
http.Error(w, "anchor_sha required", 400)
|
|
return
|
|
}
|
|
if b.TransactionID == "" || b.ExpectedVersion != t.Version {
|
|
http.Error(w, "release transaction and current lease version required", http.StatusConflict)
|
|
return
|
|
}
|
|
p, _ := json.Marshal(map[string]any{"handoff_ref": b.HandoffRef, "harness_id": parts[3], "lease_epoch": b.LeaseEpoch, "anchor_sha": b.AnchorSHA, "transaction_id": b.TransactionID, "expected_version": t.Version, "session_evidence": b.SessionEvidence})
|
|
e := domain.Event{ID: id(), Type: "TaskReleased", TaskID: b.TaskID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
|
|
if err := s.Append(e); err != nil {
|
|
http.Error(w, err.Error(), 409)
|
|
return
|
|
}
|
|
if rt != nil {
|
|
_, _ = rt.HandleEvent(e)
|
|
}
|
|
json.NewEncoder(w).Encode(e)
|
|
})
|
|
var giteaSources []provider.GiteaSourceConfig
|
|
if path := os.Getenv("ORCHESTRA_GITEA_CONFIG"); path != "" {
|
|
cfgs, err := provider.LoadGiteaConfigs(path)
|
|
if err != nil {
|
|
log.Fatalf("load gitea config: %v", err)
|
|
}
|
|
giteaSources = cfgs
|
|
} else if base := os.Getenv("ORCHESTRA_GITEA_URL"); base != "" {
|
|
// Legacy single-repo configuration: project defaults to the repo
|
|
// name, matching the historical (pre-multi-source) behavior.
|
|
giteaSources = []provider.GiteaSourceConfig{{
|
|
Project: os.Getenv("ORCHESTRA_GITEA_REPO"), BaseURL: base,
|
|
Owner: os.Getenv("ORCHESTRA_GITEA_OWNER"), Repo: os.Getenv("ORCHESTRA_GITEA_REPO"),
|
|
Token: os.Getenv("ORCHESTRA_GITEA_TOKEN"), WebhookSecret: os.Getenv("ORCHESTRA_GITEA_WEBHOOK_SECRET"),
|
|
}}
|
|
}
|
|
humanSources := map[string]human.Source{}
|
|
if len(giteaSources) > 0 {
|
|
reflectors := map[string]provider.Gitea{}
|
|
for _, c := range giteaSources {
|
|
g := provider.Gitea{BaseURL: c.BaseURL, Token: c.Token, WebhookSecret: c.WebhookSecret, Owner: c.Owner, Repo: c.Repo, Project: c.Project}
|
|
reflectors[g.SourceName()] = g
|
|
}
|
|
reflecting := provider.ReflectingSink{Sink: s, Tasks: s, Reflector: provider.MultiGitea{Sources: reflectors}}
|
|
for _, c := range giteaSources {
|
|
g := provider.Gitea{BaseURL: c.BaseURL, Token: c.Token, WebhookSecret: c.WebhookSecret, Owner: c.Owner, Repo: c.Repo, Project: c.Project}
|
|
name := "gitea:" + c.Project
|
|
webhookPath := "/v1/providers/gitea/webhook/" + c.Project
|
|
if len(giteaSources) == 1 && os.Getenv("ORCHESTRA_GITEA_CONFIG") == "" {
|
|
// Preserve the legacy unprefixed webhook path when running
|
|
// the single-source (env-var) configuration, so existing
|
|
// Gitea webhook configs don't need to be re-pointed.
|
|
webhookPath = "/v1/providers/gitea/webhook"
|
|
name = "gitea"
|
|
}
|
|
mux.Handle(webhookPath, g.WebhookHandler(reflecting))
|
|
sup := &provider.Supervisor{Name: name, Run: func(ctx context.Context) error {
|
|
pollCtx, cancel := context.WithTimeout(ctx, time.Minute)
|
|
defer cancel()
|
|
_, err := g.Poll(pollCtx, reflecting)
|
|
if err == nil {
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-time.After(time.Minute):
|
|
}
|
|
}
|
|
return err
|
|
}}
|
|
sup.Start(context.Background())
|
|
providerHealth[name] = sup
|
|
humanSources[g.SourceName()] = provider.GiteaComments{Gitea: g}
|
|
if publisher := (provider.GiteaPublisher{Gitea: g, Base: os.Getenv("ORCHESTRA_PR_BASE")}); publisher.BaseURL != "" {
|
|
root := projectRoots[c.Project]
|
|
submissionPublisher = func(plan operations.SubmissionPlan) operations.Publisher {
|
|
p := publisher
|
|
p.Root = filepath.Join(root, plan.TaskID)
|
|
return p
|
|
}
|
|
pullRequests[g.SourceName()] = publisher
|
|
}
|
|
}
|
|
}
|
|
// Started here, after the forge wiring above fills pullRequests. Guarding
|
|
// on the map before anything writes to it made this loop dead code: the
|
|
// length was always zero, so a merged pull request never completed its
|
|
// task, and moving the read into the tick would race the startup writes.
|
|
if len(pullRequests) > 0 {
|
|
// Submitted work is reconciled on its own loop, not behind
|
|
// Store.PreLease: an in-review task cannot be leased, so a pre-lease
|
|
// hook could never see the feedback that should make it leasable.
|
|
trust := human.Trust{
|
|
Accepted: splitList(os.Getenv("ORCHESTRA_REVIEW_ACTORS")),
|
|
Ignored: splitList(os.Getenv("ORCHESTRA_REVIEW_IGNORE_ACTORS")),
|
|
}
|
|
go func() {
|
|
ticker := time.NewTicker(time.Minute)
|
|
defer ticker.Stop()
|
|
for range ticker.C {
|
|
for _, t := range s.Tasks() {
|
|
if t.Submission == nil || (t.State != domain.StateInReview && t.State != domain.StateQueued) {
|
|
continue
|
|
}
|
|
source, ok := pullRequests[t.Submission.PR.Provider]
|
|
if !ok {
|
|
continue
|
|
}
|
|
project, ok := rr.Project(t.Project)
|
|
if !ok {
|
|
continue
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
state, err := source.PullRequest(ctx, t)
|
|
cancel()
|
|
if err != nil {
|
|
// Observable, and the task stays exactly where it was.
|
|
log.Printf("reflect submission %s: %v", t.ID, err)
|
|
continue
|
|
}
|
|
if _, err := operations.ReflectSubmission(s, project, t.ID, state, trust); err != nil {
|
|
log.Printf("reflect submission %s: %v", t.ID, err)
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
// Reconciliation runs immediately before every lease, which is where
|
|
// ownership of a task begins. A configured source that cannot be read
|
|
// refuses the lease rather than letting a successor resume from an older
|
|
// intent. Set ORCHESTRA_HUMAN_RECONCILE=off to disable it for an
|
|
// operator who needs to run while a source is down.
|
|
if len(humanSources) > 0 && !strings.EqualFold(os.Getenv("ORCHESTRA_HUMAN_RECONCILE"), "off") {
|
|
reconciler := &human.Reconciler{Store: s, Sources: humanSources, Timeout: 30 * time.Second}
|
|
s.PreLease = func(taskID string) error {
|
|
return reconciler.Reconcile(context.Background(), taskID)
|
|
}
|
|
if coordinator != nil {
|
|
// The second reconciliation point: a verified turn boundary on a
|
|
// lease that is already running. Nothing here preempts the agent.
|
|
coordinator.ReconcileHumanInput = reconciler.Reconcile
|
|
// After this many consecutive failures at that boundary, the
|
|
// session is asked to hand off rather than keep running on intent
|
|
// Orchestra can no longer refresh.
|
|
if v, parseErr := strconv.Atoi(os.Getenv("ORCHESTRA_RECONCILE_FAILURE_HANDOFF")); parseErr == nil && v > 0 {
|
|
coordinator.ReconcileFailureHandoff = v
|
|
}
|
|
}
|
|
}
|
|
if path := os.Getenv("ORCHESTRA_JSONL"); path != "" {
|
|
sup := &provider.Supervisor{Name: "jsonl", Run: func(ctx context.Context) error {
|
|
return (provider.JSONLWatcher{Path: path, Interval: time.Second, Provider: provider.JSONL{Source: "jsonl"}}).Run(ctx, s)
|
|
}}
|
|
sup.Start(context.Background())
|
|
providerHealth["jsonl"] = sup
|
|
}
|
|
var senders []delivery.Sender
|
|
if token, chat := os.Getenv("ORCHESTRA_TELEGRAM_BOT_TOKEN"), os.Getenv("ORCHESTRA_TELEGRAM_CHAT_ID"); token != "" && chat != "" {
|
|
senders = append(senders, delivery.Telegram{Token: token, ChatID: chat})
|
|
}
|
|
if topic := os.Getenv("ORCHESTRA_NTFY_TOPIC"); topic != "" {
|
|
senders = append(senders, delivery.Ntfy{Topic: topic, Token: os.Getenv("ORCHESTRA_NTFY_TOKEN"), URL: os.Getenv("ORCHESTRA_NTFY_URL")})
|
|
}
|
|
if len(senders) > 0 {
|
|
cursorPath := filepath.Join(dir, "delivery-cursor")
|
|
var startCursor uint64
|
|
if b, err := os.ReadFile(cursorPath); err == nil {
|
|
if v, err := strconv.ParseUint(strings.TrimSpace(string(b)), 10, 64); err == nil {
|
|
startCursor = v
|
|
}
|
|
}
|
|
go func() {
|
|
fanout := &delivery.Fanout{
|
|
Senders: senders,
|
|
Cursor: startCursor,
|
|
SaveCursor: func(cursor uint64) {
|
|
if err := os.WriteFile(cursorPath, []byte(strconv.FormatUint(cursor, 10)), 0o644); err != nil {
|
|
log.Printf("delivery cursor persist: %v", err)
|
|
}
|
|
},
|
|
}
|
|
err := fanout.Run(context.Background(), s.Events)
|
|
if err != nil {
|
|
log.Printf("delivery fanout: %v", err)
|
|
}
|
|
}()
|
|
}
|
|
go func() {
|
|
lastDay := ""
|
|
t := time.NewTicker(time.Minute)
|
|
defer t.Stop()
|
|
for now := range t.C {
|
|
utc := now.UTC()
|
|
day := utc.Format("2006-01-02")
|
|
if utc.Hour() == 3 && day != lastDay {
|
|
if _, err := standup(); err != nil {
|
|
log.Printf("standup advisory: %v", err)
|
|
} else {
|
|
lastDay = day
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
if rt != nil {
|
|
if _, err := rt.AssignPending(); err != nil {
|
|
log.Printf("startup task assignment: %v", err)
|
|
}
|
|
}
|
|
port := os.Getenv("ORCHESTRA_PORT")
|
|
if port == "" {
|
|
port = "9145"
|
|
}
|
|
// Deployed identity, without a credential. A burn-in run pairs a
|
|
// coordinator and a worker, and matching revisions must be evidence rather
|
|
// than assumption. /v1/admin/diagnostics carries the same object behind
|
|
// the operator login.
|
|
b := buildinfo.Current()
|
|
log.Printf("orchestra revision %s built %s dirty %s", b.Revision, b.Time, b.Dirty)
|
|
log.Println("orchestra listening on :" + port)
|
|
tokens := map[authz.Surface]string{
|
|
authz.TUI: os.Getenv("ORCHESTRA_TUI_TOKEN"),
|
|
authz.MCP: os.Getenv("ORCHESTRA_MCP_TOKEN"), authz.Maven: os.Getenv("ORCHESTRA_MAVEN_TOKEN"),
|
|
// The credential an in-pane coding session presents. It buys the two
|
|
// request endpoints and read access — never a lifecycle mutation. The
|
|
// forge, Vikunja and operator tokens must never reach an agent pane;
|
|
// this one is the only Orchestra credential an agent may hold.
|
|
authz.Agent: os.Getenv("ORCHESTRA_AGENT_TOKEN"),
|
|
// S12: this is the credential callers present *to* Orchestra on the
|
|
// ntfy surface. ORCHESTRA_NTFY_TOKEN is a different secret entirely —
|
|
// it is handed out to the third-party ntfy server (see the sender
|
|
// above) and must never be accepted as an inbound credential.
|
|
authz.Telegram: os.Getenv("ORCHESTRA_TELEGRAM_TOKEN"), authz.Ntfy: os.Getenv("ORCHESTRA_NTFY_SURFACE_TOKEN"),
|
|
}
|
|
// A full-control surface with no credential is an open control plane, not a
|
|
// disabled one, because an unset token makes the middleware skip its check.
|
|
// Refuse to start rather than log it and serve.
|
|
if err := authz.RequireCredentials(tokens); err != nil {
|
|
log.Fatalf("refusing to serve: %v", err)
|
|
}
|
|
log.Fatal(http.ListenAndServe(":"+port, authz.HTTPWithSessions(tokens, sessions, mux)))
|
|
}
|
|
|
|
// splitList reads a comma-separated env list, ignoring blanks.
|
|
func splitList(v string) []string {
|
|
var out []string
|
|
for _, item := range strings.Split(v, ",") {
|
|
if s := strings.TrimSpace(item); s != "" {
|
|
out = append(out, s)
|
|
}
|
|
}
|
|
return out
|
|
}
|