kami/orchestra
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: make worker handoff rotation durable

Browse Source
This commit is contained in:
kami
2026-07-30 01:30:59 +04:00
parent ce02c60106
commit 1ff0af2e69
16 changed files with 1488 additions and 1566 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AUDIT.md
+82 -1337
View File
File diff suppressed because it is too large Load Diff
cmd/orchestra-worker/main.go
+540 -69
View File
@@ -6,8 +6,10 @@ package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"orchestra/internal/buildinfo"
"orchestra/internal/continuity"
"orchestra/internal/domain"
"orchestra/internal/federation"
@@ -17,6 +19,7 @@ import (
"os/exec"
"os/signal"
"path/filepath"
"sort"
"strconv"
"strings"
"syscall"
@@ -27,15 +30,19 @@ type worker struct {
api federation.Client
herdr *herdr.Client
harnessID, harness, repo, root, remote string
projects map[string]projectConfig
cursor uint64
tasks map[string]domain.Task
sessions map[string]herdr.Session
leases map[string]lease
releases map[string]releaseTransaction
statePath string
hard float64
registration federation.Worker
lastError string
lastErrorAt time.Time
soft float64
window int64
}
func (w *worker) recordError(err error) {
@@ -72,13 +79,49 @@ func (w *worker) health(ctx context.Context) federation.WorkerHealth {
}
type lease struct {
HandoffRef string `json:"handoff_ref,omitempty"`
HandoffRef string `json:"handoff_ref,omitempty"`
TransactionID string `json:"transaction_id,omitempty"`
AnchorSHA string `json:"anchor_sha,omitempty"`
PickupAcknowledged bool `json:"pickup_acknowledged,omitempty"`
Version int `json:"version"`
Until time.Time `json:"until"`
}
type releaseTransaction struct {
ID string `json:"id"`
LeaseVersion int `json:"lease_version"`
Ref string `json:"handoff_ref,omitempty"`
AnchorSHA string `json:"anchor_sha,omitempty"`
Phase string `json:"phase"` // prepared, anchor_pushed, event_committed, pickup_validated, predecessor_retired
AgentReleased bool `json:"agent_released,omitempty"`
LastError string `json:"last_error,omitempty"`
UpdatedAt time.Time `json:"updated_at"`
}
type projectConfig struct {
Repo string `json:"repo"`
Root string `json:"worktree_root"`
Remote string `json:"remote"`
QualityGate string `json:"quality_gate,omitempty"`
}
type completionEvidence struct {
TaskID string `json:"task_id"`
Project string `json:"project"`
Worker string `json:"worker"`
Harness string `json:"harness"`
PaneID string `json:"pane_id"`
BaseSHA string `json:"base_sha"`
ResultSHA string `json:"result_sha"`
Branch string `json:"branch"`
Remote string `json:"remote"`
QualityGate string `json:"quality_gate,omitempty"`
GateExit int `json:"gate_exit"`
CompletedAt time.Time `json:"completed_at"`
}
type workerState struct {
Cursor uint64 `json:"cursor"`
Sessions map[string]herdr.Session `json:"sessions"`
Tasks map[string]domain.Task `json:"tasks"`
Leases map[string]lease `json:"leases"`
Cursor uint64 `json:"cursor"`
Sessions map[string]herdr.Session `json:"sessions"`
Tasks map[string]domain.Task `json:"tasks"`
Leases map[string]lease `json:"leases"`
Releases map[string]releaseTransaction `json:"releases"`
}
func (w *worker) load() {
@@ -90,6 +133,7 @@ func (w *worker) load() {
w.sessions = s.Sessions
w.tasks = s.Tasks
w.leases = s.Leases
w.releases = s.Releases
}
}
if w.sessions == nil {
@@ -101,9 +145,12 @@ func (w *worker) load() {
if w.leases == nil {
w.leases = map[string]lease{}
}
if w.releases == nil {
w.releases = map[string]releaseTransaction{}
}
}
func (w *worker) save() error {
b, e := json.Marshal(workerState{Cursor: w.cursor, Sessions: w.sessions, Tasks: w.tasks, Leases: w.leases})
b, e := json.Marshal(workerState{Cursor: w.cursor, Sessions: w.sessions, Tasks: w.tasks, Leases: w.leases, Releases: w.releases})
if e != nil {
return e
}
@@ -133,23 +180,35 @@ func created(e domain.Event) (domain.Task, bool) {
Capability []string `json:"capability"`
Title string `json:"title"`
Description string `json:"description"`
Acceptance []string `json:"acceptance"`
QualityGate string `json:"quality_gate"`
}
if json.Unmarshal(e.Payload, &p) != nil || p.Source == "" || p.ExternalID == "" || p.Project == "" {
return domain.Task{}, false
}
return domain.Task{ID: e.TaskID, Source: p.Source, ExternalID: p.ExternalID, Project: p.Project, Capability: p.Capability, Title: p.Title, Description: p.Description}, true
return domain.Task{ID: e.TaskID, Source: p.Source, ExternalID: p.ExternalID, Project: p.Project, Capability: p.Capability, Title: p.Title, Description: p.Description, Acceptance: p.Acceptance, QualityGate: p.QualityGate}, true
}
func (w *worker) syncBase(ctx context.Context) error {
if out, err := exec.CommandContext(ctx, "git", "-C", w.repo, "fetch", w.remote, "--prune").CombinedOutput(); err != nil {
func (w *worker) project(t domain.Task) (projectConfig, error) {
if w.projects != nil {
if p, ok := w.projects[t.Project]; ok && p.Repo != "" && p.Root != "" && p.Remote != "" {
return p, nil
}
return projectConfig{}, fmt.Errorf("project %q is not configured on worker", t.Project)
}
return projectConfig{Repo: w.repo, Root: w.root, Remote: w.remote}, nil
}
func (w *worker) syncBase(ctx context.Context, p projectConfig) error {
if out, err := exec.CommandContext(ctx, "git", "-C", p.Repo, "fetch", p.Remote, "--prune").CombinedOutput(); err != nil {
return fmt.Errorf("fetch base checkout: %s: %w", out, err)
}
branch, err := exec.CommandContext(ctx, "git", "-C", w.repo, "symbolic-ref", "--quiet", "--short", "HEAD").Output()
branch, err := exec.CommandContext(ctx, "git", "-C", p.Repo, "symbolic-ref", "--quiet", "--short", "HEAD").Output()
if err != nil {
return fmt.Errorf("identify base branch: %w", err)
}
branchName := strings.TrimSpace(string(branch))
if out, err := exec.CommandContext(ctx, "git", "-C", w.repo, "merge", "--ff-only", w.remote+"/"+branchName).CombinedOutput(); err != nil {
if out, err := exec.CommandContext(ctx, "git", "-C", p.Repo, "merge", "--ff-only", p.Remote+"/"+branchName).CombinedOutput(); err != nil {
return fmt.Errorf("fast-forward base checkout: %s: %w", out, err)
}
return nil
@@ -159,10 +218,14 @@ func (w *worker) start(ctx context.Context, t domain.Task, ref string) error {
var wt string
var h continuity.Handoff
var err error
p, err := w.project(t)
if err != nil {
return err
}
// Synchronize the local base before any worktree operation. A worker never
// treats a coordinator-side path as truth; the Git remote is the only
// cross-machine transport.
if err := w.syncBase(ctx); err != nil {
if err := w.syncBase(ctx, p); err != nil {
return err
}
if ref != "" {
@@ -174,12 +237,12 @@ func (w *worker) start(ctx context.Context, t domain.Task, ref string) error {
if err != nil {
return err
}
if out, err := exec.CommandContext(ctx, "git", "-C", w.repo, "fetch", w.remote, "--prune").CombinedOutput(); err != nil {
if out, err := exec.CommandContext(ctx, "git", "-C", p.Repo, "fetch", p.Remote, "--prune").CombinedOutput(); err != nil {
return fmt.Errorf("fetch pickup anchor: %s: %w", out, err)
}
wt = filepath.Join(w.root, t.ID)
wt = filepath.Join(p.Root, t.ID)
if _, err := os.Stat(wt); os.IsNotExist(err) {
if out, err := exec.CommandContext(ctx, "git", "-C", w.repo, "worktree", "add", "-b", "orchestra/"+t.ID, wt, h.Anchor.GitSHA).CombinedOutput(); err != nil {
if out, err := exec.CommandContext(ctx, "git", "-C", p.Repo, "worktree", "add", "-b", "orchestra/"+t.ID, wt, h.Anchor.GitSHA).CombinedOutput(); err != nil {
return fmt.Errorf("create pickup worktree: %s: %w", out, err)
}
}
@@ -187,21 +250,22 @@ func (w *worker) start(ctx context.Context, t domain.Task, ref string) error {
return err
}
} else {
wt, err = (orchestrator.GitWorktrees{Repo: w.repo, Root: w.root}).Create(ctx, t)
wt, err = (orchestrator.GitWorktrees{Repo: p.Repo, Root: p.Root}).Create(ctx, t)
if err != nil {
return err
}
}
if _, err = w.herdr.Worktree(ctx, w.repo, wt, "orchestra/"+t.ID); err != nil {
if _, err = w.herdr.Worktree(ctx, p.Repo, wt, "orchestra/"+t.ID); err != nil {
return err
}
s, err := w.herdr.StartAgent(ctx, wt, wt, "orchestra/"+t.ID, w.harness, t.ID)
if err != nil {
return err
}
p := "Begin Orchestra task " + t.ID + ".\nTitle: " + t.Title + "\nInstructions:\n" + t.Description + "\nWork only in this worktree. Do not edit TASK.md."
s.TaskFileSHA = taskHash(t)
prompt := "Read TASK.md at the worktree root and execute it."
if ref != "" {
p += "\nA validated handoff exists. Read TASK.md and inspect local git history before continuing."
prompt += " A validated handoff exists; inspect local Git history and the recorded checkpoint before continuing."
}
w.sessions[t.ID] = s
if err := w.save(); err != nil {
@@ -209,65 +273,362 @@ func (w *worker) start(ctx context.Context, t domain.Task, ref string) error {
}
// A prompt response can be lost after herdr accepted it. Persist the
// session first so the worker can reconcile/release it after restart.
return w.herdr.Prompt(ctx, s.PaneID, p, 0)
if err := w.herdr.Prompt(ctx, s.PaneID, prompt, 0); err != nil {
return err
}
if ref != "" {
return w.ackPickup(ctx, t.ID, s)
}
return nil
}
func taskHash(t domain.Task) string { b := continuity.RenderTaskFile(t); return domain.Hash(b) }
func (w *worker) releaseReady(ctx context.Context) {
for id, s := range w.sessions {
if report, err := os.ReadFile(filepath.Join(s.Worktree, ".orchestra-report.md")); err == nil && len(report) > 0 {
if ref, err := w.api.PutArtifact(ctx, report); err == nil {
if err = w.api.Complete(ctx, id, ref); err == nil {
delete(w.sessions, id)
delete(w.leases, id)
_ = w.save()
continue
}
if l := w.leases[id]; l.HandoffRef != "" && !l.PickupAcknowledged {
if err := w.ackPickup(ctx, id, s); err != nil {
w.recordError(err)
continue
}
}
if _, err := os.Stat(filepath.Join(s.Worktree, ".orchestra", "done")); err == nil {
evidence, err := w.finalize(ctx, id, s)
if err != nil {
w.recordError(fmt.Errorf("complete %s: %w", id, err))
log.Printf("complete %s: %v", id, err)
} else {
continue
}
report, _ := json.Marshal(evidence)
ref, err := w.api.PutArtifact(ctx, report)
if err != nil {
w.recordError(fmt.Errorf("upload completion %s: %w", id, err))
log.Printf("upload completion %s: %v", id, err)
continue
}
}
if _, err := os.Stat(filepath.Join(s.Worktree, herdr.HandoffReportFile)); err != nil {
if !s.HandoffRequested {
a := herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}
if occ, occErr := a.Occupancy(s); occErr == nil && occ >= w.hard {
if boundary, boundaryErr := a.AtTurnBoundary(ctx, s); boundaryErr == nil && boundary {
if err := a.RequestHandoff(ctx, s); err == nil {
s.HandoffRequested, s.HandoffReason = true, "threshold"
w.sessions[id] = s
_ = w.save()
}
}
}
if err = w.api.Complete(ctx, id, ref, evidence.ResultSHA, evidence.Branch, evidence.Remote, w.leases[id].Version, w.usageReceipt(s), w.sessionEvidence(ctx, id, s)); err != nil {
w.recordError(fmt.Errorf("complete %s: %w", id, err))
log.Printf("complete %s: %v", id, err)
continue
}
// Completion is durable before closing the exact pane. If close
// fails, retain the session mapping for a later explicit cleanup.
a := herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}
if err := a.Kill(ctx, s); err != nil {
w.recordError(fmt.Errorf("close completed pane %s: %w", id, err))
log.Printf("close completed pane %s: %v", id, err)
continue
}
_ = os.Remove(filepath.Join(s.Worktree, ".orchestra", "done"))
_ = os.Remove(filepath.Join(s.Worktree, ".orchestra"))
delete(w.sessions, id)
delete(w.leases, id)
_ = w.save()
continue
}
a := herdr.CLIAdapter{Client: w.herdr, Harness: w.harness, CAS: artifactCAS{w.api}, Remote: w.remote}
ref, err := a.Release(ctx, s)
if err != nil {
log.Printf("release %s: %v", id, err)
if _, err := os.Stat(filepath.Join(s.Worktree, herdr.HandoffReportFile)); err == nil || w.releases[id].ID != "" {
w.advanceRelease(ctx, id, s)
continue
}
sha, err := herdr.HeadSHA(s.Worktree)
if err != nil {
log.Printf("release %s anchor: %v", id, err)
continue
}
if err = w.api.Release(ctx, id, ref, sha); err != nil {
log.Printf("publish release %s: %v", id, err)
continue
}
// release_agent only removes herdr's binding. Closing the released pane
// after the handoff is durable prevents the next StartAgent from
// inheriting the predecessor's still-running terminal process.
if err := a.Kill(ctx, s); err != nil {
log.Printf("close released pane %s: %v", id, err)
}
delete(w.sessions, id)
w.rotationTick(ctx, id, s)
}
}
func (w *worker) adapter(s herdr.Session, remote string) herdr.CLIAdapter {
a := herdr.CLIAdapter{Client: w.herdr, Harness: w.harness, Window: w.window, CAS: artifactCAS{w.api}, Remote: remote}
switch w.harness {
case "claude":
a.Usage = herdr.ClaudeUsage
case "codex":
a.Usage = herdr.CodexUsage
case "opencode":
a.Usage = herdr.OpenCodeUsage
}
return a
}
// rotationTick is the checkout-owner state machine. Occupancy, tool activity,
// and pane status are all read from the persisted harness session identity;
// any unknown source is recorded and never treated as zero usage.
func (w *worker) rotationTick(ctx context.Context, id string, s herdr.Session) {
t, ok := w.tasks[id]
if !ok {
w.recordError(fmt.Errorf("rotation %s: task cache missing", id))
return
}
p, err := w.project(t)
if err != nil {
w.recordError(err)
return
}
a := w.adapter(s, p.Remote)
resolved, err := a.ResolveSessionIdentity(s)
if err != nil {
w.recordError(fmt.Errorf("rotation %s occupancy degraded: %w", id, err))
return
}
if resolved != s {
w.sessions[id] = resolved
s = resolved
_ = w.save()
}
d := (orchestrator.RotationStateMachine{Soft: w.soft, Hard: w.hard}).Evaluate(ctx, a, s)
if d.ActivityDegraded != nil {
w.recordError(fmt.Errorf("rotation %s activity degraded: %w", id, d.ActivityDegraded))
}
if d.Degraded != nil {
w.recordError(fmt.Errorf("rotation %s degraded: %w", id, d.Degraded))
if d.Action == orchestrator.TurnContinue || d.Action == "" {
return
}
}
if d.Action == orchestrator.TurnContinue || d.Action == orchestrator.TurnRefuse || s.HandoffRequested {
return
}
if d.Reason == "milestone" || d.Reason == "thrash" {
if err := a.RequestHandoffReason(ctx, s, d.Reason, d.DeadEnds); err != nil {
w.recordError(fmt.Errorf("rotation %s %s prompt: %w", id, d.Reason, err))
return
}
} else if err := a.RequestHandoff(ctx, s); err != nil {
w.recordError(fmt.Errorf("rotation %s threshold prompt: %w", id, err))
return
}
s.HandoffRequested, s.HandoffReason = true, d.Reason
w.sessions[id] = s
_ = w.save()
}
func (w *worker) advanceRelease(ctx context.Context, id string, s herdr.Session) {
t, ok := w.tasks[id]
if !ok {
w.recordError(fmt.Errorf("release %s: task cache missing", id))
return
}
p, err := w.project(t)
if err != nil {
w.recordError(fmt.Errorf("release %s: %w", id, err))
return
}
if w.releases == nil {
w.releases = map[string]releaseTransaction{}
}
tx := w.releases[id]
if tx.ID == "" {
l, ok := w.leases[id]
if !ok {
w.recordError(fmt.Errorf("release %s: lease missing", id))
return
}
tx = releaseTransaction{ID: domain.NewID(), LeaseVersion: l.Version, Phase: "prepared", UpdatedAt: time.Now().UTC()}
w.releases[id] = tx
_ = w.save()
}
a := w.adapter(s, p.Remote)
if tx.Phase == "prepared" {
prepared, err := a.PrepareRelease(ctx, s)
if err != nil {
tx.LastError, tx.UpdatedAt = err.Error(), time.Now().UTC()
w.releases[id] = tx
_ = w.save()
w.recordError(fmt.Errorf("release %s prepare: %w", id, err))
return
}
tx.Ref, tx.AnchorSHA, tx.Phase, tx.LastError, tx.UpdatedAt = prepared.Ref, prepared.AnchorSHA, "anchor_pushed", "", time.Now().UTC()
w.releases[id] = tx
_ = w.save()
}
if tx.Phase == "anchor_pushed" {
if err := w.api.Release(ctx, id, tx.Ref, tx.AnchorSHA, tx.ID, tx.LeaseVersion, w.sessionEvidence(ctx, id, s)); err != nil {
tx.LastError, tx.UpdatedAt = err.Error(), time.Now().UTC()
w.releases[id] = tx
_ = w.save()
w.recordError(fmt.Errorf("release %s commit: %w", id, err))
return
}
tx.Phase, tx.LastError, tx.UpdatedAt = "event_committed", "", time.Now().UTC()
w.releases[id] = tx
_ = w.save()
}
if tx.Phase == "event_committed" && !tx.AgentReleased {
if err := a.ReleaseAgent(ctx, s); err != nil {
tx.LastError, tx.UpdatedAt = err.Error(), time.Now().UTC()
w.releases[id] = tx
_ = w.save()
w.recordError(fmt.Errorf("release %s release agent: %w", id, err))
return
}
tx.AgentReleased, tx.LastError, tx.UpdatedAt = true, "", time.Now().UTC()
w.releases[id] = tx
_ = w.save()
}
if tx.Phase == "pickup_validated" {
if err := a.Kill(ctx, s); err != nil {
tx.LastError, tx.UpdatedAt = err.Error(), time.Now().UTC()
w.releases[id] = tx
_ = w.save()
w.recordError(fmt.Errorf("release %s retire predecessor: %w", id, err))
return
}
_ = os.Remove(filepath.Join(s.Worktree, herdr.HandoffReportFile))
tx.Phase, tx.UpdatedAt = "predecessor_retired", time.Now().UTC()
w.releases[id] = tx
_ = w.save()
delete(w.sessions, id)
delete(w.releases, id)
_ = w.save()
}
}
// ackPickup retries the successor acknowledgement from persisted lease state.
// It is safe after a lost response: the coordinator recognizes the exact
// transaction/lease epoch as an idempotent pickup.
func (w *worker) ackPickup(ctx context.Context, id string, s herdr.Session) error {
l, ok := w.leases[id]
if !ok || l.HandoffRef == "" || l.TransactionID == "" || l.AnchorSHA == "" {
return fmt.Errorf("pickup %s: lease is missing its release transaction", id)
}
if l.PickupAcknowledged {
return nil
}
if err := w.api.Pickup(ctx, id, l.HandoffRef, l.AnchorSHA, l.TransactionID, l.Version, w.sessionEvidence(ctx, id, s)); err != nil {
return fmt.Errorf("pickup %s acknowledgement: %w", id, err)
}
l.PickupAcknowledged = true
l.Version++ // TaskPickupValidated increments the task version.
w.leases[id] = l
return w.save()
}
func (w *worker) sessionEvidence(ctx context.Context, taskID string, s herdr.Session) domain.SessionEvidence {
e := domain.SessionEvidence{PaneID: s.PaneID, HarnessID: w.harnessID, PaneState: "open", Source: "worker", CheckedAt: time.Now().UTC()}
text, err := (herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}).PaneCapture(ctx, s, "recent")
if err != nil {
e.PaneState = "unreachable"
return e
}
if capture, err := w.api.PublishCapture(ctx, federation.Capture{TaskID: taskID, PaneID: s.PaneID, Text: text}); err == nil {
e.CapturedAt = capture.At
}
return e
}
func (w *worker) usageReceipt(s herdr.Session) map[string]any {
if s.SessionFile == "" && !(w.harness == "opencode" && s.SessionID != "") {
return map[string]any{"harness_id": w.harnessID, "consumed": 0}
}
var usage herdr.Usage
var err error
switch w.harness {
case "claude":
usage, err = herdr.ClaudeUsage(s.SessionFile)
case "codex":
usage, err = herdr.CodexUsage(s.SessionFile)
case "opencode":
usage, err = herdr.OpenCodeSessionUsage(s.SessionID)
}
if err != nil {
return map[string]any{"harness_id": w.harnessID, "consumed": 0, "error": err.Error()}
}
return map[string]any{"harness_id": w.harnessID, "input_tokens": usage.Input, "cache_read_tokens": usage.CacheRead, "cache_write_tokens": usage.CacheWrite, "output_tokens": usage.Output, "consumed": usage.Numerator()}
}
func git(ctx context.Context, dir string, args ...string) ([]byte, error) {
return exec.CommandContext(ctx, "git", append([]string{"-C", dir}, args...)...).CombinedOutput()
}
// finalize performs only mechanical delivery work. It never asks the harness
// to narrate Git state, gates, or a report; those are generated from the
// worker-owned checkout and then verified against the configured remote.
func (w *worker) finalize(ctx context.Context, id string, s herdr.Session) (completionEvidence, error) {
t, ok := w.tasks[id]
if !ok {
return completionEvidence{}, fmt.Errorf("task cache missing")
}
p, err := w.project(t)
if err != nil {
return completionEvidence{}, err
}
if s.TaskFileSHA != "" {
if err := continuity.VerifyTaskFile(s.Worktree, s.TaskFileSHA); err != nil {
return completionEvidence{}, fmt.Errorf("verify immutable TASK.md: %w", err)
}
}
base, err := git(ctx, s.Worktree, "rev-parse", "HEAD")
if err != nil {
return completionEvidence{}, fmt.Errorf("base sha: %s: %w", base, err)
}
e := completionEvidence{TaskID: id, Project: t.Project, Worker: w.harnessID, Harness: w.harness, PaneID: s.PaneID, BaseSHA: strings.TrimSpace(string(base)), Remote: p.Remote, QualityGate: t.QualityGate, CompletedAt: time.Now().UTC()}
gateCommand := t.QualityGate
if gateCommand == "" {
gateCommand = p.QualityGate
}
e.QualityGate = gateCommand
if gateCommand != "" {
gate := exec.CommandContext(ctx, "sh", "-c", gateCommand)
gate.Dir = s.Worktree
if out, err := gate.CombinedOutput(); err != nil {
e.GateExit = 1
return completionEvidence{}, fmt.Errorf("quality gate %q: %s: %w", gateCommand, out, err)
}
}
if _, err := git(ctx, s.Worktree, "diff", "--quiet", "--", "TASK.md"); err != nil {
return completionEvidence{}, errors.New("TASK.md was modified")
}
if out, err := git(ctx, s.Worktree, "add", "-A", "--", ".", ":!.orchestra/done"); err != nil {
return completionEvidence{}, fmt.Errorf("stage result: %s: %w", out, err)
}
if _, err := git(ctx, s.Worktree, "diff", "--cached", "--quiet"); err != nil {
if out, err := git(ctx, s.Worktree, "commit", "-m", "orchestra: complete "+id); err != nil {
return completionEvidence{}, fmt.Errorf("commit result: %s: %w", out, err)
}
}
branch, err := git(ctx, s.Worktree, "branch", "--show-current")
if err != nil || strings.TrimSpace(string(branch)) == "" {
return completionEvidence{}, fmt.Errorf("result branch: %s: %w", branch, err)
}
e.Branch = strings.TrimSpace(string(branch))
sha, err := git(ctx, s.Worktree, "rev-parse", "HEAD")
if err != nil {
return completionEvidence{}, fmt.Errorf("result sha: %s: %w", sha, err)
}
e.ResultSHA = strings.TrimSpace(string(sha))
if out, err := git(ctx, s.Worktree, "push", p.Remote, "HEAD:refs/heads/"+e.Branch); err != nil {
return completionEvidence{}, fmt.Errorf("push result: %s: %w", out, err)
}
remote, err := git(ctx, s.Worktree, "ls-remote", p.Remote, "refs/heads/"+e.Branch)
if err != nil || !strings.HasPrefix(string(remote), e.ResultSHA+"\t") {
return completionEvidence{}, fmt.Errorf("verify pushed sha: got %q: %w", strings.TrimSpace(string(remote)), err)
}
return e, nil
}
func (w *worker) renewLeases(ctx context.Context) {
if w.herdr == nil {
return
}
now := time.Now()
for taskID, l := range w.leases {
s, ok := w.sessions[taskID]
if !ok || l.Version == 0 || l.Until.After(now.Add(10*time.Minute)) {
continue
}
if _, err := (herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}).PaneCapture(ctx, s, "recent"); err != nil {
w.recordError(fmt.Errorf("validate lease %s: %w", taskID, err))
continue
}
if err := w.api.Renew(ctx, taskID, l.Version, int((30 * time.Minute).Seconds())); err != nil {
w.recordError(fmt.Errorf("renew lease %s: %w", taskID, err))
log.Printf("renew lease %s: %v", taskID, err)
} else {
// RenewLease appends one event. Retain that epoch locally until its
// replay arrives so a release transaction uses the same version.
l.Version++
l.Until = now.Add(30 * time.Minute)
w.leases[taskID] = l
_ = w.save()
}
}
}
// publishCaptures makes remote panes observable without allowing the
@@ -364,16 +725,79 @@ func (w *worker) once(ctx context.Context) error {
}
if e.Type == "TaskLeased" {
var p struct {
HarnessID string `json:"harness_id"`
HandoffRef string `json:"handoff_ref"`
HarnessID string `json:"harness_id"`
HandoffRef string `json:"handoff_ref"`
TransactionID string `json:"transaction_id"`
AnchorSHA string `json:"anchor_sha"`
}
if json.Unmarshal(e.Payload, &p) == nil && p.HarnessID == w.harnessID {
w.leases[e.TaskID] = lease{HandoffRef: p.HandoffRef}
var until struct {
UntilNS int64 `json:"until_ns"`
}
_ = json.Unmarshal(e.Payload, &until)
w.leases[e.TaskID] = lease{HandoffRef: p.HandoffRef, TransactionID: p.TransactionID, AnchorSHA: p.AnchorSHA, Version: e.Version, Until: time.Unix(0, until.UntilNS)}
}
}
if e.Type == "TaskReleased" || e.Type == "TaskCompleted" || e.Type == "TaskFailed" || e.Type == "TaskBlocked" {
if e.Type == "TaskLeaseRenewed" {
var p struct {
HarnessID string `json:"harness_id"`
UntilNS int64 `json:"until_ns"`
}
if json.Unmarshal(e.Payload, &p) == nil && p.HarnessID == w.harnessID {
l := w.leases[e.TaskID]
l.Version, l.Until = e.Version, time.Unix(0, p.UntilNS)
w.leases[e.TaskID] = l
}
}
if e.Type == "TaskCompleted" {
delete(w.leases, e.TaskID)
delete(w.sessions, e.TaskID)
if session, active := w.sessions[e.TaskID]; active {
if w.herdr == nil {
delete(w.sessions, e.TaskID)
} else if err := (herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}).Kill(ctx, session); err != nil {
w.recordError(fmt.Errorf("close completed pane %s: %w", e.TaskID, err))
continue
} else {
_ = os.Remove(filepath.Join(session.Worktree, ".orchestra", "done"))
_ = os.Remove(filepath.Join(session.Worktree, ".orchestra"))
delete(w.sessions, e.TaskID)
}
}
}
if e.Type == "TaskPickupValidated" {
var p struct {
TransactionID string `json:"transaction_id"`
}
if json.Unmarshal(e.Payload, &p) == nil {
if tx := w.releases[e.TaskID]; tx.ID != "" && tx.ID == p.TransactionID {
tx.Phase, tx.UpdatedAt = "pickup_validated", time.Now().UTC()
w.releases[e.TaskID] = tx
}
}
if l, ok := w.leases[e.TaskID]; ok {
l.Version = e.Version
w.leases[e.TaskID] = l
}
}
if e.Type == "TaskReleased" || e.Type == "TaskFailed" || e.Type == "TaskBlocked" {
if e.Type == "TaskReleased" {
var p struct {
TransactionID string `json:"transaction_id"`
}
if json.Unmarshal(e.Payload, &p) == nil {
if tx := w.releases[e.TaskID]; tx.ID != "" && tx.ID == p.TransactionID && tx.Phase == "anchor_pushed" {
tx.Phase, tx.LastError, tx.UpdatedAt = "event_committed", "", time.Now().UTC()
w.releases[e.TaskID] = tx
}
}
}
delete(w.leases, e.TaskID)
// A releasing predecessor remains intentionally recoverable until
// TaskPickupValidated for its transaction. Do not erase its pane
// mapping merely because our own release event was replayed.
if _, releasing := w.releases[e.TaskID]; !releasing {
delete(w.sessions, e.TaskID)
}
}
if e.Seq > w.cursor {
w.cursor = e.Seq
@@ -422,6 +846,7 @@ func (w *worker) once(ctx context.Context) error {
if w.herdr != nil {
w.publishCaptures(ctx)
w.runCommands(ctx)
w.renewLeases(ctx)
}
w.releaseReady(ctx)
if err := w.save(); err != nil {
@@ -439,7 +864,7 @@ func (w *worker) reconcileLeases(ctx context.Context) error {
for _, task := range tasks {
w.tasks[task.ID] = task
if task.State == domain.StateLeased && task.Lease != nil && task.Lease.HarnessID == w.harnessID {
active[task.ID] = lease{HandoffRef: task.HandoffRef}
active[task.ID] = lease{HandoffRef: task.HandoffRef, TransactionID: task.ReleaseTransaction, AnchorSHA: task.ReleaseAnchor, Version: task.Version, Until: task.Lease.Until}
}
}
for taskID := range w.leases {
@@ -480,7 +905,53 @@ func main() {
if v, err := strconv.ParseFloat(os.Getenv("ORCHESTRA_OCCUPANCY_HARD"), 64); err == nil && v > 0 && v < 1 {
hard = v
}
w := &worker{api: federation.Client{BaseURL: required("ORCHESTRA_URL"), WorkerID: id, Token: token, AdmitToken: os.Getenv("ORCHESTRA_FEDERATION_ADMIT_TOKEN")}, harnessID: required("ORCHESTRA_WORKER_HERDR_ID"), harness: required("ORCHESTRA_WORKER_HARNESS"), repo: required("ORCHESTRA_REPO"), root: required("ORCHESTRA_WORKTREE_ROOT"), remote: required("ORCHESTRA_GIT_REMOTE"), tasks: map[string]domain.Task{}, sessions: map[string]herdr.Session{}, leases: map[string]lease{}, statePath: os.Getenv("ORCHESTRA_WORKER_STATE"), hard: hard, registration: federation.Worker{ID: id, Address: os.Getenv("ORCHESTRA_WORKER_ADDRESS"), Capacity: 1}}
projects := map[string]projectConfig{}
if path := os.Getenv("ORCHESTRA_WORKER_PROJECT_CONFIG_FILE"); path != "" {
b, err := os.ReadFile(path)
if err != nil {
log.Fatalf("read ORCHESTRA_WORKER_PROJECT_CONFIG_FILE: %v", err)
}
if err := json.Unmarshal(b, &projects); err != nil {
log.Fatalf("parse ORCHESTRA_WORKER_PROJECT_CONFIG_FILE: %v", err)
}
} else {
for _, project := range strings.Split(os.Getenv("ORCHESTRA_WORKER_PROJECTS"), ",") {
if project = strings.TrimSpace(project); project != "" {
projects[project] = projectConfig{Repo: required("ORCHESTRA_REPO"), Root: required("ORCHESTRA_WORKTREE_ROOT"), Remote: required("ORCHESTRA_GIT_REMOTE")}
}
}
}
if len(projects) == 0 {
// Existing deployments may be upgraded before their protected systemd
// environment is amended. Stay observable and fail closed in that
// interval: an empty declaration makes this worker ineligible for all
// new leases instead of turning a configuration rollout into a crash
// loop or treating its legacy global checkout as every project.
log.Printf("no ORCHESTRA_WORKER_PROJECT_CONFIG_FILE/ORCHESTRA_WORKER_PROJECTS; registering with no supported projects")
}
for project, config := range projects {
if config.Repo == "" || config.Root == "" || config.Remote == "" {
log.Fatalf("project %q requires repo, worktree_root, and remote", project)
}
}
supported := make([]string, 0, len(projects))
for project := range projects {
supported = append(supported, project)
}
sort.Strings(supported)
repo, root, remote := required("ORCHESTRA_REPO"), required("ORCHESTRA_WORKTREE_ROOT"), required("ORCHESTRA_GIT_REMOTE")
if len(supported) > 0 {
first := projects[supported[0]]
repo, root, remote = first.Repo, first.Root, first.Remote
}
soft, window := .55, int64(200000)
if v, err := strconv.ParseFloat(os.Getenv("ORCHESTRA_OCCUPANCY_SOFT"), 64); err == nil && v > 0 && v < hard {
soft = v
}
if v, err := strconv.ParseInt(os.Getenv("ORCHESTRA_CONTEXT_WINDOW"), 10, 64); err == nil && v > 0 {
window = v
}
w := &worker{api: federation.Client{BaseURL: required("ORCHESTRA_URL"), WorkerID: id, Token: token, AdmitToken: os.Getenv("ORCHESTRA_FEDERATION_ADMIT_TOKEN")}, harnessID: required("ORCHESTRA_WORKER_HERDR_ID"), harness: required("ORCHESTRA_WORKER_HARNESS"), repo: repo, root: root, remote: remote, projects: projects, tasks: map[string]domain.Task{}, sessions: map[string]herdr.Session{}, leases: map[string]lease{}, releases: map[string]releaseTransaction{}, statePath: os.Getenv("ORCHESTRA_WORKER_STATE"), hard: hard, soft: soft, window: window, registration: federation.Worker{ID: id, Address: os.Getenv("ORCHESTRA_WORKER_ADDRESS"), Capacity: 1, SupportedProjects: supported, Build: buildinfo.Current()}}
if w.statePath == "" {
w.statePath = filepath.Join(w.root, ".orchestra-worker-state.json")
}
cmd/orchestra/main.go
+132 -28
View File
@@ -2,7 +2,6 @@ package main
import (
"context"
"crypto/subtle"
"encoding/json"
"errors"
"fmt"
@@ -12,6 +11,7 @@ import (
"net/http"
"orchestra/internal/admin"
"orchestra/internal/authz"
"orchestra/internal/buildinfo"
"orchestra/internal/delivery"
"orchestra/internal/domain"
"orchestra/internal/federation"
@@ -83,6 +83,14 @@ func remoteHerdrAddresses(rr registry.Registry, localMachine string) map[string]
return remote
}
// coordinatorOwnsHerdr identifies the only herdr sockets the coordinator may
// probe or adapt. In federation mode a remote pane belongs to its worker;
// reaching into that machine would turn a worker-owned health signal back
// into a misleading coordinator TCP result.
func coordinatorOwnsHerdr(h registry.Herdr, localMachine string) bool {
return localMachine == "" || h.MachineID == localMachine
}
func (a federatedAvailability) Available(h registry.Herdr) bool {
if a.base != nil && !a.base.Available(h) {
return false
@@ -93,6 +101,16 @@ func (a federatedAvailability) Available(h registry.Herdr) bool {
return a.workers.Available(h.ID)
}
func (a federatedAvailability) Supports(h registry.Herdr, project string) bool {
// Locally-owned herdrs keep their static registry/project affinity. A
// remote worker must additionally prove it has a local checkout for the
// project before the router can offer it a lease.
if a.localMachine == "" || h.MachineID == a.localMachine {
return true
}
return a.workers.Supports(h.ID, project)
}
func validateLocalMachine(rr registry.Registry, localMachine string) error {
machines := rr.Machines()
if len(machines) <= 1 {
@@ -231,6 +249,10 @@ func main() {
if repo, root := os.Getenv("ORCHESTRA_REPO"), os.Getenv("ORCHESTRA_WORKTREE_ROOT"); repo != "" && root != "" {
adapters := map[string]herdr.Adapter{}
for _, h := range rr.Herdrs() {
if !coordinatorOwnsHerdr(h, localMachine) {
log.Printf("herdr %s is worker-owned on %s; coordinator probe skipped", h.ID, h.MachineID)
continue
}
address := herdrAddress(rr, h)
if address == "" {
continue
@@ -244,6 +266,7 @@ func main() {
log.Printf("herdr %s unavailable: %v", h.ID, err)
continue
}
log.Printf("herdr %s reachable at %s (protocol %s, harness %s)", h.ID, address, protocol, h.Harness)
switch h.Harness {
case "claude":
adapters[h.ID] = herdr.Claude(client, 200000, s)
@@ -267,7 +290,7 @@ func main() {
}
coordinator = &orchestrator.Coordinator{Store: s, StatePath: filepath.Join(dir, "runtime-sessions.json"), Worktrees: worktrees, Adapters: orchestrator.AdapterFactory{Herdrs: adapters}, LocalHerdr: func(id string) bool {
h, ok := rr.Herdr(id)
return ok && (localMachine == "" || h.MachineID == localMachine)
return ok && coordinatorOwnsHerdr(h, localMachine)
}}
rt.OnLease = func(e domain.Event) error {
// In federated mode the coordinator must never inspect a remote
@@ -301,17 +324,14 @@ func main() {
}
mux := http.NewServeMux()
// B18: the UI is a full control plane — it can create tasks, release or
// complete them, and inject approval keystrokes into live panes. Refuse
// to serve it unauthenticated rather than silently exposing that on
// whatever interface the listener binds to.
webToken := os.Getenv("ORCHESTRA_WEB_TOKEN")
if webToken == "" {
log.Fatal("ORCHESTRA_WEB_TOKEN must be set: it gates the web UI's task, lifecycle and approval controls")
// complete them, and inject approval keystrokes into live panes. It has
// one explicit operator identity and is never enabled by a missing env var.
webCredentials := authz.WebCredentials{Username: os.Getenv("ORCHESTRA_WEB_USERNAME"), PasswordHash: os.Getenv("ORCHESTRA_WEB_PASSWORD_HASH")}
if err := webCredentials.Validate(); err != nil {
log.Fatalf("web login configuration: %v", err)
}
sessions := &authz.Sessions{}
// A browser cannot put a Bearer token on a document load, so it trades
// the token once for an HttpOnly cookie. Same credential, presentable
// form; no new authority is created here.
// A browser login exchanges verified credentials for an HttpOnly cookie.
mux.HandleFunc("/v1/ui/session", func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodDelete {
// Expire the browser credential even if it is already absent or stale.
@@ -329,15 +349,13 @@ func main() {
return
}
var body struct {
Token string `json:"token"`
Username string `json:"username"`
Password string `json:"password"`
}
_ = json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&body)
supplied := body.Token
if supplied == "" {
supplied = strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
}
if subtle.ConstantTimeCompare([]byte(supplied), []byte(webToken)) != 1 {
http.Error(w, "invalid token", http.StatusUnauthorized)
decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&body); err != nil || !webCredentials.Authenticate(body.Username, body.Password) {
http.Error(w, "invalid credentials", http.StatusUnauthorized)
return
}
v, err := sessions.Issue()
@@ -414,6 +432,17 @@ func main() {
http.Error(w, "invalid json", 400)
return
}
// Make the selected project's deterministic gate part of the immutable
// task contract before any worker can create TASK.md. A caller may
// override it only when it has deliberately supplied a task-specific
// gate; routing never asks a harness to choose one.
if _, set := p["quality_gate"]; !set {
if projectID, _ := p["project"].(string); projectID != "" {
if project, ok := rr.Project(projectID); ok && project.QualityGate != "" {
p["quality_gate"] = project.QualityGate
}
}
}
b, _ := json.Marshal(p)
e := domain.Event{ID: id(), Type: "TaskCreated", TaskID: id(), Version: 1, Payload: b, Surface: string(surface(r))}
if err := s.Append(e); err != nil {
@@ -665,7 +694,7 @@ func main() {
}
json.NewEncoder(w).Encode(out)
})
adminServer := &admin.Server{Store: s, RouterReady: rt != nil, Providers: providerHealth, Probes: map[string]admin.ProbeFunc{
adminServer := &admin.Server{Store: s, RouterReady: rt != nil, Build: buildinfo.Current(), Providers: providerHealth, Probes: map[string]admin.ProbeFunc{
"router": func() (bool, string) { return rt != nil, "configured router" },
"gitea": func() (bool, string) {
configured := os.Getenv("ORCHESTRA_GITEA_URL") != "" || os.Getenv("ORCHESTRA_GITEA_CONFIG") != ""
@@ -1033,7 +1062,7 @@ func main() {
w.WriteHeader(http.StatusNoContent)
})
mux.HandleFunc("/v1/federation/workers/", func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || (!strings.HasSuffix(r.URL.Path, "/heartbeat") && !strings.HasSuffix(r.URL.Path, "/handoff") && !strings.HasSuffix(r.URL.Path, "/complete") && !strings.HasSuffix(r.URL.Path, "/captures")) {
if r.Method != http.MethodPost || (!strings.HasSuffix(r.URL.Path, "/heartbeat") && !strings.HasSuffix(r.URL.Path, "/renew") && !strings.HasSuffix(r.URL.Path, "/handoff") && !strings.HasSuffix(r.URL.Path, "/complete") && !strings.HasSuffix(r.URL.Path, "/captures")) {
http.Error(w, "not found", 404)
return
}
@@ -1083,10 +1112,18 @@ func main() {
return
}
var b struct {
TaskID string `json:"task_id"`
TTLSeconds int `json:"ttl_seconds"`
HandoffRef string `json:"handoff_ref"`
AnchorSHA string `json:"anchor_sha"`
TaskID string `json:"task_id"`
TTLSeconds int `json:"ttl_seconds"`
ExpectedVersion int `json:"expected_version"`
HandoffRef string `json:"handoff_ref"`
AnchorSHA string `json:"anchor_sha"`
TransactionID string `json:"transaction_id"`
LeaseVersion int `json:"lease_version"`
ResultSHA string `json:"result_sha"`
Branch string `json:"branch"`
Remote string `json:"remote"`
Receipt map[string]any `json:"receipt"`
SessionEvidence domain.SessionEvidence `json:"session_evidence"`
}
if json.NewDecoder(r.Body).Decode(&b) != nil || b.TaskID == "" {
http.Error(w, "invalid lease body", 400)
@@ -1098,6 +1135,13 @@ func main() {
return
}
ownedLease := t.State == domain.StateLeased && t.Lease != nil && t.Lease.HarnessID == parts[3]
// A response can be lost after the append/fsync. Retrying the exact
// release transaction is therefore a successful no-op, never a second
// TaskReleased event and never a reason to discard the predecessor.
if strings.HasSuffix(r.URL.Path, "/handoff") && t.State == domain.StateQueued && b.TransactionID != "" && b.TransactionID == t.ReleaseTransaction && b.HandoffRef == t.HandoffRef && b.AnchorSHA == t.ReleaseAnchor {
w.WriteHeader(http.StatusNoContent)
return
}
// A prompt timeout can block the coordinator after herdr already
// accepted the request. If that same authenticated worker later reports
// a durable completion, reconcile it rather than preserving a known
@@ -1107,17 +1151,73 @@ func main() {
http.Error(w, "lease not owned", 409)
return
}
if strings.HasSuffix(r.URL.Path, "/complete") && b.ExpectedVersion != t.Version {
http.Error(w, "lease version conflict", http.StatusConflict)
return
}
if strings.HasSuffix(r.URL.Path, "/renew") {
ttl := b.TTLSeconds
if ttl == 0 {
ttl = int((30 * time.Minute).Seconds())
}
e, err := s.RenewLease(b.TaskID, parts[3], b.ExpectedVersion, time.Duration(ttl)*time.Second)
if err != nil {
http.Error(w, err.Error(), http.StatusConflict)
return
}
json.NewEncoder(w).Encode(e)
return
}
if strings.HasSuffix(r.URL.Path, "/pickup") {
if !ownedLease || b.TransactionID == "" || b.TransactionID != t.ReleaseTransaction || b.HandoffRef != t.HandoffRef || b.AnchorSHA != t.ReleaseAnchor {
http.Error(w, "pickup does not match active release", http.StatusConflict)
return
}
if t.PickupTransaction == b.TransactionID && t.PickupLeaseVersion == b.LeaseVersion {
w.WriteHeader(http.StatusNoContent)
return
}
if b.LeaseVersion != t.Version {
http.Error(w, "lease version conflict", http.StatusConflict)
return
}
p, _ := json.Marshal(map[string]any{"transaction_id": b.TransactionID, "handoff_ref": b.HandoffRef, "anchor_sha": b.AnchorSHA, "harness_id": parts[3], "lease_version": b.LeaseVersion, "expected_version": t.Version, "session_evidence": b.SessionEvidence})
e := domain.Event{ID: id(), Type: "TaskPickupValidated", TaskID: b.TaskID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
if err := s.Append(e); err != nil {
http.Error(w, err.Error(), http.StatusConflict)
return
}
json.NewEncoder(w).Encode(e)
return
}
if strings.HasSuffix(r.URL.Path, "/complete") {
if b.HandoffRef == "" {
http.Error(w, "report_ref required", 400)
return
}
p, _ := json.Marshal(map[string]any{"report_ref": b.HandoffRef, "receipt": map[string]any{"harness_id": parts[3], "consumed": 0}})
if len(b.ResultSHA) != 40 || b.Branch == "" || b.Remote == "" {
http.Error(w, "verified result_sha, branch, and remote required", 400)
return
}
if b.Receipt == nil {
b.Receipt = map[string]any{}
}
b.Receipt["harness_id"] = parts[3]
if _, ok := b.Receipt["consumed"]; !ok {
b.Receipt["consumed"] = 0
}
p, _ := json.Marshal(map[string]any{"report_ref": b.HandoffRef, "receipt": b.Receipt, "result_sha": b.ResultSHA, "branch": b.Branch, "remote": b.Remote, "session_evidence": b.SessionEvidence})
e := domain.Event{ID: id(), Type: "TaskCompleted", TaskID: b.TaskID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
if err := s.Append(e); err != nil {
http.Error(w, err.Error(), 409)
return
}
if consumed, ok := b.Receipt["consumed"].(float64); ok && consumed > 0 {
qp, _ := json.Marshal(map[string]any{"harness_id": parts[3], "consumed": consumed})
if err := s.Append(domain.Event{ID: id(), Type: "QuotaReported", TaskID: "system", Payload: qp, Surface: string(authz.System)}); err != nil {
log.Printf("federated quota report %s: %v", b.TaskID, err)
}
}
json.NewEncoder(w).Encode(e)
return
}
@@ -1129,7 +1229,11 @@ func main() {
http.Error(w, "anchor_sha required", 400)
return
}
p, _ := json.Marshal(map[string]any{"handoff_ref": b.HandoffRef, "harness_id": parts[3], "anchor_sha": b.AnchorSHA})
if b.TransactionID == "" || b.ExpectedVersion != t.Version {
http.Error(w, "release transaction and current lease version required", http.StatusConflict)
return
}
p, _ := json.Marshal(map[string]any{"handoff_ref": b.HandoffRef, "harness_id": parts[3], "anchor_sha": b.AnchorSHA, "transaction_id": b.TransactionID, "expected_version": t.Version, "session_evidence": b.SessionEvidence})
e := domain.Event{ID: id(), Type: "TaskReleased", TaskID: b.TaskID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
if err := s.Append(e); err != nil {
http.Error(w, err.Error(), 409)
@@ -1257,7 +1361,7 @@ func main() {
}
log.Println("orchestra listening on :" + port)
tokens := map[authz.Surface]string{
authz.TUI: os.Getenv("ORCHESTRA_TUI_TOKEN"), authz.Web: os.Getenv("ORCHESTRA_WEB_TOKEN"),
authz.TUI: os.Getenv("ORCHESTRA_TUI_TOKEN"),
authz.MCP: os.Getenv("ORCHESTRA_MCP_TOKEN"), authz.Maven: os.Getenv("ORCHESTRA_MAVEN_TOKEN"),
// S12: this is the credential callers present *to* Orchestra on the
// ntfy surface. ORCHESTRA_NTFY_TOKEN is a different secret entirely —
deploy/orchestra.env.example
+14 -4
View File
@@ -36,6 +36,14 @@ ORCHESTRA_WORKTREE_ROOT=/var/lib/orchestra/worktrees
# Hard rotation occupancy threshold (0 < x < 1). Default 0.75 if unset/invalid.
ORCHESTRA_OCCUPANCY_HARD=0.75
# Advisory handoff threshold and the harness context window used to turn
# per-session token counts into occupancy. Both values are worker-local.
ORCHESTRA_OCCUPANCY_SOFT=0.55
ORCHESTRA_CONTEXT_WINDOW=200000
# OpenCode stores per-session token counters in SQLite. This optional override
# must point at the worker-local database; the worker persists the resolved
# session ID for each lease, never "the latest" session.
#ORCHESTRA_OPENCODE_DB=/home/orchestra/.local/share/opencode/opencode.db
# --- Providers ---
# Local JSONL task ingestion (baseline adapter).
@@ -74,10 +82,12 @@ ORCHESTRA_OCCUPANCY_HARD=0.75
# --- Bus authorization tokens (bearer auth per surface; a surface with no
# token set has no auth requirement — set these once you have real clients) ---
#ORCHESTRA_TUI_TOKEN=
# Required: the service refuses to start without it. It gates the web UI's
# task, lifecycle and approval controls, and it is also the token for any
# /v1/ caller that does not declare a surface (they default to Web).
ORCHESTRA_WEB_TOKEN=
# Required: the service refuses to start without both. The browser UI's
# task, lifecycle and approval controls are session-gated; it no longer
# accepts a shared Web bearer token. Generate the bcrypt hash with:
# go run ./cmd/orchestra-password
ORCHESTRA_WEB_USERNAME=operator
ORCHESTRA_WEB_PASSWORD_HASH=
# Set when the UI is served over plain HTTP, so the session cookie can be
# sent without Secure. Leave unset behind TLS.
#ORCHESTRA_UI_INSECURE_COOKIE=1
internal/continuity/continuity.go
+39 -11
View File
@@ -40,7 +40,17 @@ func RenderTaskFile(t domain.Task) []byte {
if strings.TrimSpace(t.Description) != "" {
fmt.Fprintf(&b, "\n## Instructions\n\n%s\n", t.Description)
}
b.WriteString("\nThis file is immutable for the lifetime of the task (§6.2) — its hash is\ncarried in every handoff and re-verified on every pickup. Do not edit it.\n")
if len(t.Acceptance) > 0 {
b.WriteString("\n## Acceptance criteria\n")
for _, criterion := range t.Acceptance {
fmt.Fprintf(&b, "\n- %s", criterion)
}
b.WriteByte('\n')
}
if t.QualityGate != "" {
fmt.Fprintf(&b, "\n## Quality gate\n\n%s\n", t.QualityGate)
}
b.WriteString("\n## Completion\n\nRun the configured quality gate. When the task is ready for the worker to verify and deliver, create `.orchestra/done`. Do not write a prose completion report.\n\nThis file is immutable for the lifetime of the task (§6.2) — its hash is\ncarried in every handoff and re-verified on every pickup. Do not edit it.\n")
return []byte(b.String())
}
@@ -80,8 +90,9 @@ func ConventionsHash(root string) (string, error) {
}
type Dirty struct {
Path string `json:"path"`
SHA256 string `json:"sha256"`
Path string `json:"path"`
SHA256 string `json:"sha256"`
Deleted bool `json:"deleted,omitempty"`
}
type Completed struct {
What string `json:"what"`
@@ -147,7 +158,7 @@ func (h Handoff) Validate() error {
}
}
for _, d := range h.Anchor.Dirty {
if filepath.IsAbs(d.Path) || d.Path == "" || len(d.SHA256) != 64 {
if filepath.IsAbs(d.Path) || d.Path == "" || (!d.Deleted && len(d.SHA256) != 64) {
return errors.New("invalid dirty anchor")
}
}
@@ -199,6 +210,12 @@ func ValidatePickup(root string, h Handoff, taskFileSHA string) error {
return errors.New("handoff anchor HEAD mismatch")
}
for _, d := range h.Anchor.Dirty {
if d.Deleted {
if _, e := os.Stat(filepath.Join(root, d.Path)); !errors.Is(e, os.ErrNotExist) {
return fmt.Errorf("handoff deleted file restored: %s", d.Path)
}
continue
}
b, e := os.ReadFile(filepath.Join(root, d.Path))
if e != nil {
return e
@@ -257,7 +274,12 @@ func Load(ref string, cas CAS) (Handoff, error) {
return Decode(b)
}
// ScratchCommit records WIP atomically on a dedicated branch before rotation.
// ScratchCommit records every piece of repository work except Orchestra's
// ephemeral protocol markers. In particular, git add -A is intentional: it
// includes already-staged changes, deletions, renames, and untracked files.
// TASK.md is checked before touching the index; it is an immutable input, not
// deliverable work. The report/done markers remain local so a successor never
// mistakes an old protocol signal for a new one.
func ScratchCommit(root, branch, message string) error {
if branch == "" || strings.ContainsAny(branch, " \t\n") {
return errors.New("invalid scratch branch")
@@ -279,14 +301,20 @@ func ScratchCommit(root, branch, message string) error {
return err
}
}
if err := exec.Command("git", "-C", root, "add", "-A").Run(); err != nil {
// A harness may have staged a protocol marker itself. Remove it from the
// index before staging the real checkpoint; this does not alter its working
// tree contents and makes the exclusion apply to staged state too.
for _, marker := range []string{".orchestra", ".orchestra-handoff.json", ".orchestra-handoff-report.md"} {
if err := exec.Command("git", "-C", root, "reset", "-q", "HEAD", "--", marker).Run(); err != nil {
return err
}
}
if err := exec.Command("git", "-C", root, "add", "-A", "--", ".", ":(exclude)TASK.md", ":(exclude).orchestra", ":(exclude).orchestra-handoff.json", ":(exclude).orchestra-handoff-report.md").Run(); err != nil {
return err
}
full, err := exec.Command("git", "-C", root, "status", "--porcelain").Output()
if err != nil {
return err
}
if len(full) == 0 {
// Only staged non-protocol work is committed. Remaining marker files are
// expected and must not suppress a clean committed-anchor checkpoint.
if exec.Command("git", "-C", root, "diff", "--cached", "--quiet").Run() == nil {
return nil // nothing to snapshot; branch already reflects the worktree
}
return exec.Command("git", "-C", root, "commit", "-m", message).Run()
internal/continuity/continuity_test.go
+48
View File
@@ -12,6 +12,54 @@ import (
"orchestra/internal/store"
)
func TestScratchCommitCapturesAllGitStatesExceptProtocolMarkers(t *testing.T) {
repo := t.TempDir()
run := func(args ...string) {
t.Helper()
if out, err := exec.Command("git", append([]string{"-C", repo}, args...)...).CombinedOutput(); err != nil {
t.Fatalf("git %v: %v: %s", args, err, out)
}
}
run("init")
run("config", "user.email", "t@t")
run("config", "user.name", "t")
for _, name := range []string{"TASK.md", "deleted.txt", "renamed.txt", "staged.txt"} {
if err := os.WriteFile(filepath.Join(repo, name), []byte(name), 0644); err != nil {
t.Fatal(err)
}
}
run("add", "-A")
run("commit", "-m", "base")
if err := os.WriteFile(filepath.Join(repo, "staged.txt"), []byte("staged change"), 0644); err != nil {
t.Fatal(err)
}
run("add", "staged.txt")
if err := os.Remove(filepath.Join(repo, "deleted.txt")); err != nil {
t.Fatal(err)
}
run("mv", "renamed.txt", "renamed-new.txt")
if err := os.WriteFile(filepath.Join(repo, "untracked.txt"), []byte("new"), 0644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(repo, ".orchestra-handoff-report.md"), []byte("protocol"), 0644); err != nil {
t.Fatal(err)
}
if err := ScratchCommit(repo, "orchestra/scratch/test", "checkpoint"); err != nil {
t.Fatal(err)
}
for _, want := range []string{"staged.txt", "renamed-new.txt", "untracked.txt"} {
if err := exec.Command("git", "-C", repo, "cat-file", "-e", "HEAD:"+want).Run(); err != nil {
t.Fatalf("checkpoint omitted %s: %v", want, err)
}
}
if err := exec.Command("git", "-C", repo, "cat-file", "-e", "HEAD:deleted.txt").Run(); err == nil {
t.Fatal("checkpoint retained deleted file")
}
if err := exec.Command("git", "-C", repo, "cat-file", "-e", "HEAD:.orchestra-handoff-report.md").Run(); err == nil {
t.Fatal("checkpoint committed protocol marker")
}
}
func TestHandoffCASAndPickup(t *testing.T) {
root := t.TempDir()
run := func(a ...string) {
internal/domain/domain.go
+121 -10
View File
@@ -38,11 +38,69 @@ const (
StateBlocked TaskState = "blocked"
)
// BlockReason is the machine-readable diagnosis for a TaskBlocked event.
// Blocker remains the operator-facing detail; this field lets projections
// group attention without repeatedly parsing prose at read time.
type BlockReason string
const (
BlockReasonLeaseFailure BlockReason = "lease_failure"
BlockReasonWorkerOffline BlockReason = "worker_offline"
BlockReasonLeaseExpired BlockReason = "lease_expired"
BlockReasonApproval BlockReason = "approval"
BlockReasonHandoffValidation BlockReason = "handoff_validation"
BlockReasonOperator BlockReason = "operator_block"
BlockReasonSystem BlockReason = "system_error"
BlockReasonUnknown BlockReason = "unknown"
)
func (r BlockReason) Valid() bool {
switch r {
case BlockReasonLeaseFailure, BlockReasonWorkerOffline, BlockReasonLeaseExpired,
BlockReasonApproval, BlockReasonHandoffValidation, BlockReasonOperator,
BlockReasonSystem, BlockReasonUnknown:
return true
}
return false
}
// InferBlockReason supplies a stable category for older events which only
// recorded a prose blocker. New producers should send block_reason directly.
func InferBlockReason(blocker string) BlockReason {
v := strings.ToLower(blocker)
switch {
case strings.Contains(v, "handoff"):
return BlockReasonHandoffValidation
case strings.Contains(v, "approval") || strings.Contains(v, "permission"):
return BlockReasonApproval
case strings.Contains(v, "expired") && strings.Contains(v, "lease"):
return BlockReasonLeaseExpired
case strings.Contains(v, "worker") && (strings.Contains(v, "offline") || strings.Contains(v, "unreachable")):
return BlockReasonWorkerOffline
case strings.Contains(v, "lease") || strings.Contains(v, "agent.start") || strings.Contains(v, "pane"):
return BlockReasonLeaseFailure
default:
return BlockReasonSystem
}
}
type Estimate struct {
Value float64 `json:"value"`
Who string `json:"who"`
Confidence float64 `json:"confidence"`
}
// SessionEvidence is captured by the machine that owns a pane immediately
// before it drops its mapping. It is deliberately observation-only: it never
// claims that a pane is still live after the worker has closed it.
type SessionEvidence struct {
PaneID string `json:"pane_id,omitempty"`
HarnessID string `json:"harness_id,omitempty"`
PaneState string `json:"pane_state,omitempty"`
Source string `json:"source,omitempty"`
CapturedAt time.Time `json:"captured_at,omitempty"`
CheckedAt time.Time `json:"checked_at,omitempty"`
}
type Lease struct {
HarnessID string `json:"harness_id"`
Until time.Time `json:"until"`
@@ -62,17 +120,27 @@ type Task struct {
// HandoffRef survives the queued interval between TaskReleased and the
// next router-owned TaskLeased event; it is the only artifact the worker
// may use for local pickup validation.
HandoffRef string `json:"handoff_ref,omitempty"`
Version int `json:"version"`
Title string `json:"title,omitempty"`
Description string `json:"description,omitempty"`
HandoffRef string `json:"handoff_ref,omitempty"`
// ReleaseTransaction and ReleaseAnchor bind successor pickup to the exact
// durable predecessor checkpoint. They survive queueing and re-lease.
ReleaseTransaction string `json:"release_transaction,omitempty"`
ReleaseAnchor string `json:"release_anchor,omitempty"`
PickupTransaction string `json:"pickup_transaction,omitempty"`
PickupLeaseVersion int `json:"pickup_lease_version,omitempty"`
Version int `json:"version"`
Title string `json:"title,omitempty"`
Description string `json:"description,omitempty"`
Acceptance []string `json:"acceptance,omitempty"`
QualityGate string `json:"quality_gate,omitempty"`
// Block evidence is projected from TaskBlocked so terminal records remain
// diagnosable after the live coordinator mapping is gone.
Blocker string `json:"blocker,omitempty"`
BlockedAt time.Time `json:"blocked_at,omitempty"`
LastPaneID string `json:"last_pane_id,omitempty"`
LastHarness string `json:"last_harness_id,omitempty"`
PaneState string `json:"pane_state,omitempty"` // open, closed, unreachable, unknown
Blocker string `json:"blocker,omitempty"`
BlockReason BlockReason `json:"block_reason,omitempty"`
BlockedAt time.Time `json:"blocked_at,omitempty"`
LastPaneID string `json:"last_pane_id,omitempty"`
LastHarness string `json:"last_harness_id,omitempty"`
PaneState string `json:"pane_state,omitempty"` // open, closed, unreachable, unknown
LastSession SessionEvidence `json:"last_session,omitempty"`
}
type Event struct {
@@ -112,7 +180,7 @@ func ValidateEvent(e Event) error {
if e.SchemaVersion >= 2 && strings.TrimSpace(e.Surface) == "" {
return fmt.Errorf("%w: surface required", ErrInvalid)
}
allowed := map[string]bool{"TaskCreated": true, "TaskLeased": true, "TaskReleased": true, "TaskCompleted": true, "TaskFailed": true, "TaskBlocked": true, "ApprovalRequested": true, "ApprovalGranted": true, "ApprovalDenied": true, "TaskAmended": true, "TaskCorrected": true, "QuotaReported": true, "StandupAdvisory": true}
allowed := map[string]bool{"TaskCreated": true, "TaskLeased": true, "TaskLeaseRenewed": true, "TaskReleased": true, "TaskPickupValidated": true, "TaskCompleted": true, "TaskFailed": true, "TaskBlocked": true, "ApprovalRequested": true, "ApprovalGranted": true, "ApprovalDenied": true, "TaskAmended": true, "TaskCorrected": true, "QuotaReported": true, "StandupAdvisory": true}
if !allowed[e.Type] {
return fmt.Errorf("%w: unknown type %q", ErrInvalid, e.Type)
}
@@ -160,10 +228,36 @@ func ValidatePayload(typ string, p map[string]any) error {
if v, ok := p["expected_version"].(float64); !ok || v < 0 || v != float64(int(v)) {
return fmt.Errorf("%w: expected_version invalid", ErrInvalid)
}
case "TaskLeaseRenewed":
if err := requiredString("harness_id"); err != nil {
return err
}
until, ok := p["until_ns"].(float64)
if !ok || until <= float64(time.Now().UnixNano()) {
return fmt.Errorf("%w: until_ns required", ErrInvalid)
}
if v, ok := p["expected_version"].(float64); !ok || v < 0 || v != float64(int(v)) {
return fmt.Errorf("%w: expected_version invalid", ErrInvalid)
}
case "TaskReleased":
if err := requiredString("handoff_ref"); err != nil && p["reason"] == nil {
return err
}
case "TaskPickupValidated":
for _, key := range []string{"transaction_id", "handoff_ref", "anchor_sha", "harness_id"} {
if err := requiredString(key); err != nil {
return err
}
}
if err := requiredHash(p, "handoff_ref"); err != nil {
return err
}
if v, ok := p["anchor_sha"].(string); !ok || len(v) != 40 {
return fmt.Errorf("%w: anchor_sha invalid", ErrInvalid)
}
if v, ok := p["lease_version"].(float64); !ok || v < 1 || v != float64(int(v)) {
return fmt.Errorf("%w: lease_version invalid", ErrInvalid)
}
if _, ok := p["handoff_ref"]; ok {
if err := requiredHash(p, "handoff_ref"); err != nil {
return err
@@ -183,6 +277,17 @@ func ValidatePayload(typ string, p map[string]any) error {
if receipt, ok := p["receipt"].(map[string]any); !ok || len(receipt) == 0 {
return fmt.Errorf("%w: receipt required", ErrInvalid)
}
if v, ok := p["result_sha"]; ok {
if s, ok := v.(string); !ok || len(s) != 40 {
return fmt.Errorf("%w: result_sha invalid", ErrInvalid)
}
if err := requiredString("branch"); err != nil {
return err
}
if err := requiredString("remote"); err != nil {
return err
}
}
case "TaskFailed":
if err := requiredString("reason"); err != nil {
return err
@@ -191,6 +296,12 @@ func ValidatePayload(typ string, p map[string]any) error {
if err := requiredString("blocker"); err != nil {
return err
}
if v, ok := p["block_reason"]; ok {
s, ok := v.(string)
if !ok || !BlockReason(s).Valid() {
return fmt.Errorf("%w: block_reason invalid", ErrInvalid)
}
}
if _, ok := p["handoff_ref"]; ok {
if err := requiredHash(p, "handoff_ref"); err != nil {
return err
internal/federation/client.go
+19 -5
View File
@@ -24,7 +24,7 @@ type Client struct {
}
func (c Client) Register(ctx context.Context, w Worker) error {
b, err := json.Marshal(map[string]any{"id": w.ID, "address": w.Address, "capacity": w.Capacity, "token": c.Token})
b, err := json.Marshal(map[string]any{"id": w.ID, "address": w.Address, "capacity": w.Capacity, "supported_projects": w.SupportedProjects, "build": w.Build, "token": c.Token})
if err != nil {
return err
}
@@ -131,6 +131,13 @@ func (c Client) Heartbeat(ctx context.Context, health WorkerHealth) error {
}
return err
}
func (c Client) Renew(ctx context.Context, taskID string, expectedVersion, ttlSeconds int) error {
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/renew", map[string]any{"task_id": taskID, "expected_version": expectedVersion, "ttl_seconds": ttlSeconds})
if resp != nil {
resp.Body.Close()
}
return err
}
func (c Client) Artifact(ctx context.Context, ref string) ([]byte, error) {
resp, err := c.request(ctx, http.MethodGet, "/v1/artifacts/"+url.PathEscape(ref), nil)
if err != nil {
@@ -169,15 +176,22 @@ func (c Client) PutArtifact(ctx context.Context, b []byte) (string, error) {
}
return out.Ref, nil
}
func (c Client) Release(ctx context.Context, taskID, ref, anchor string) error {
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/handoff", map[string]string{"task_id": taskID, "handoff_ref": ref, "anchor_sha": anchor})
func (c Client) Release(ctx context.Context, taskID, ref, anchor, transactionID string, expectedVersion int, evidence domain.SessionEvidence) error {
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/handoff", map[string]any{"task_id": taskID, "handoff_ref": ref, "anchor_sha": anchor, "transaction_id": transactionID, "expected_version": expectedVersion, "session_evidence": evidence})
if resp != nil {
resp.Body.Close()
}
return err
}
func (c Client) Complete(ctx context.Context, taskID, reportRef string) error {
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/complete", map[string]string{"task_id": taskID, "handoff_ref": reportRef})
func (c Client) Pickup(ctx context.Context, taskID, ref, anchor, transactionID string, leaseVersion int, evidence domain.SessionEvidence) error {
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/pickup", map[string]any{"task_id": taskID, "handoff_ref": ref, "anchor_sha": anchor, "transaction_id": transactionID, "lease_version": leaseVersion, "session_evidence": evidence})
if resp != nil {
resp.Body.Close()
}
return err
}
func (c Client) Complete(ctx context.Context, taskID, reportRef, resultSHA, branch, remote string, expectedVersion int, receipt map[string]any, evidence domain.SessionEvidence) error {
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/complete", map[string]any{"task_id": taskID, "handoff_ref": reportRef, "result_sha": resultSHA, "branch": branch, "remote": remote, "expected_version": expectedVersion, "receipt": receipt, "session_evidence": evidence})
if resp != nil {
resp.Body.Close()
}
internal/herdr/adapter.go
+130 -68
View File
@@ -248,80 +248,99 @@ func (a CLIAdapter) NotifyConventionsChanged(ctx context.Context, s Session) err
// handoff is refused rather than guessed at: the caller (Coordinator.rotate)
// leaves the lease intact and retries next tick, giving the agent time to
// finish writing it.
func (a CLIAdapter) Release(ctx context.Context, s Session) (string, error) {
// PreparedRelease is the durable, coordinator-independent half of a release.
// The worker persists it before publishing TaskReleased so a lost HTTP reply
// never requires reconstructing (or deleting) the agent's report.
type PreparedRelease struct {
Ref string
AnchorSHA string
}
// PrepareRelease seals an immutable Git checkpoint and uploads its canonical
// handoff, but deliberately leaves both the pane claim and report in place.
// The caller controls the retryable transaction around coordinator acceptance.
func (a CLIAdapter) PrepareRelease(ctx context.Context, s Session) (PreparedRelease, error) {
if a.CAS == nil {
return "", fmt.Errorf("adapter: CAS store required to upload handoff")
return PreparedRelease{}, fmt.Errorf("adapter: CAS store required to upload handoff")
}
// A federation worker always supplies the immutable task hash and remote.
// The empty-hash case is retained solely for old in-process adapter users;
// it is not reachable from the worker release path.
if s.TaskFileSHA != "" {
if a.Remote == "" {
return PreparedRelease{}, fmt.Errorf("adapter: project remote required for checkpoint")
}
if err := continuity.VerifyTaskFile(s.Worktree, s.TaskFileSHA); err != nil {
return PreparedRelease{}, fmt.Errorf("adapter: verify immutable TASK.md: %w", err)
}
}
path := filepath.Join(s.Worktree, HandoffReportFile)
b, err := os.ReadFile(path)
if err != nil {
return "", fmt.Errorf("adapter: semantic handoff report not written yet (%s): %w", path, err)
return PreparedRelease{}, fmt.Errorf("adapter: semantic handoff report not written yet (%s): %w", path, err)
}
if strings.TrimSpace(string(b)) == "" {
return "", fmt.Errorf("adapter: semantic handoff report is empty")
return PreparedRelease{}, fmt.Errorf("adapter: semantic handoff report is empty")
}
h, err := canonicalHandoff(s, string(b), a.lastObservedCommand(s))
if err != nil {
return "", err
return PreparedRelease{}, err
}
sha, err := HeadSHA(s.Worktree)
// Always checkpoint and push, including already-committed clean work. Git
// is the cross-machine transport, so merely observing a local clean HEAD is
// not a sufficient anchor.
branch := "orchestra/scratch/" + h.Meta.ID
if err := continuity.ScratchCommit(s.Worktree, branch, "orchestra: pre-release WIP snapshot ("+h.Meta.ID+")"); err != nil {
return PreparedRelease{}, fmt.Errorf("adapter: scratch commit: %w", err)
}
anchor, err := HeadSHA(s.Worktree)
if err != nil {
return "", fmt.Errorf("adapter: read worktree HEAD: %w", err)
return PreparedRelease{}, fmt.Errorf("adapter: read checkpoint HEAD: %w", err)
}
_ = sha
for _, d := range h.Anchor.Dirty {
if hex.EncodeToString(sha256sum(filepath.Join(s.Worktree, d.Path))) != d.SHA256 {
return "", fmt.Errorf("adapter: handoff dirty file changed since it was written: %s", d.Path)
if a.Remote != "" {
if err := continuity.ScratchPush(s.Worktree, branch, a.Remote); err != nil {
return PreparedRelease{}, fmt.Errorf("adapter: push checkpoint: %w", err)
}
out, err := exec.CommandContext(ctx, "git", "-C", s.Worktree, "ls-remote", a.Remote, "refs/heads/"+branch).Output()
if err != nil || !strings.HasPrefix(string(out), anchor+"\t") {
return PreparedRelease{}, fmt.Errorf("adapter: verify pushed anchor: got %q: %w", strings.TrimSpace(string(out)), err)
}
}
// The semantic report is transferred in the CAS handoff, not in the
// scratch checkout. Keeping it in the scratch commit makes a successor
// mistake the predecessor's report for a newly requested handoff and can
// cause an immediate release/pickup loop.
dirty := h.Anchor.Dirty[:0]
for _, d := range h.Anchor.Dirty {
if filepath.Clean(d.Path) != HandoffReportFile {
dirty = append(dirty, d)
}
}
h.Anchor.Dirty = dirty
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
return "", fmt.Errorf("adapter: remove transferred semantic report: %w", err)
}
// Atomically commit whatever the handoff described as dirty onto a
// per-task scratch branch (§6.2 step 3) *before* uploading, so the
// successor's pickup validation collapses to a single HEAD compare
// instead of re-hashing every dirty file individually.
if len(h.Anchor.Dirty) > 0 {
branch := "orchestra/scratch/" + h.Meta.ID
if err := continuity.ScratchCommit(s.Worktree, branch, "orchestra: pre-release WIP snapshot ("+h.Meta.ID+")"); err != nil {
return "", fmt.Errorf("adapter: scratch commit: %w", err)
}
if a.Remote != "" {
if err := continuity.ScratchPush(s.Worktree, branch, a.Remote); err != nil {
return "", fmt.Errorf("adapter: push scratch branch: %w", err)
}
}
newSHA, err := HeadSHA(s.Worktree)
if err != nil {
return "", fmt.Errorf("adapter: read scratch HEAD: %w", err)
}
h.Anchor.GitSHA = newSHA
h.Anchor.Branch = branch
h.Anchor.Dirty = nil
}
h.Anchor.GitSHA, h.Anchor.Branch, h.Anchor.Dirty = anchor, branch, nil
ref, err := continuity.Save(h, a.CAS)
if err != nil {
return "", fmt.Errorf("adapter: upload handoff: %w", err)
return PreparedRelease{}, fmt.Errorf("adapter: upload handoff: %w", err)
}
return PreparedRelease{Ref: ref, AnchorSHA: anchor}, nil
}
// ReleaseAgent drops only herdr's harness binding. It does not close the pane:
// a predecessor stays recoverable until the successor has validated pickup.
func (a CLIAdapter) ReleaseAgent(ctx context.Context, s Session) error {
if err := a.Client.Call(ctx, "pane.release_agent", map[string]any{
"pane_id": s.PaneID,
"source": "herdr:" + a.Harness,
"agent": agentForSession(s, a.Harness),
}, nil); err != nil {
return "", fmt.Errorf("adapter: pane.release_agent: %w", err)
return fmt.Errorf("adapter: pane.release_agent: %w", err)
}
return ref, nil
return nil
}
// Release is retained for the coordinator's legacy local path. Federation
// workers use PrepareRelease and ReleaseAgent as separate durable phases.
func (a CLIAdapter) Release(ctx context.Context, s Session) (string, error) {
p, err := a.PrepareRelease(ctx, s)
if err != nil {
return "", err
}
if err := a.ReleaseAgent(ctx, s); err != nil {
return "", err
}
if err := os.Remove(filepath.Join(s.Worktree, HandoffReportFile)); err != nil && !os.IsNotExist(err) {
return "", fmt.Errorf("adapter: remove transferred semantic report: %w", err)
}
return p.Ref, nil
}
// canonicalHandoff keeps Git-derived protocol facts on the worker that owns
@@ -483,16 +502,27 @@ func handoffID(s Session) string {
}
func dirtyFiles(root string) ([]continuity.Dirty, error) {
out, err := exec.Command("git", "-C", root, "status", "--porcelain=v1", "-z").Output()
if err != nil {
return nil, err
}
paths := map[string]bool{}
for _, args := range [][]string{{"diff", "--name-only", "-z"}, {"ls-files", "--others", "--exclude-standard", "-z"}} {
out, err := exec.Command("git", append([]string{"-C", root}, args...)...).Output()
if err != nil {
return nil, err
deleted := map[string]bool{}
parts := strings.Split(string(out), "\x00")
for i := 0; i < len(parts); i++ {
record := parts[i]
if len(record) < 4 {
continue
}
for _, path := range strings.Split(string(out), "\x00") {
if path != "" && path != HandoffFile {
paths[path] = true
}
status, path := record[:2], record[3:]
if path == HandoffFile || path == HandoffReportFile || path == ".orchestra/done" || strings.HasPrefix(path, ".orchestra/") {
continue
}
paths[path] = true
deleted[path] = strings.Contains(status, "D")
// A rename/copy record has the original path as the next NUL item.
if status[0] == 'R' || status[0] == 'C' || status[1] == 'R' || status[1] == 'C' {
i++
}
}
keys := make([]string, 0, len(paths))
@@ -502,11 +532,15 @@ func dirtyFiles(root string) ([]continuity.Dirty, error) {
sort.Strings(keys)
dirty := make([]continuity.Dirty, 0, len(keys))
for _, path := range keys {
sum := sha256sum(filepath.Join(root, path))
if len(sum) == 0 {
return nil, fmt.Errorf("adapter: hash dirty file %s", path)
d := continuity.Dirty{Path: path, Deleted: deleted[path]}
if !d.Deleted {
sum := sha256sum(filepath.Join(root, path))
if len(sum) == 0 {
return nil, fmt.Errorf("adapter: hash dirty file %s", path)
}
d.SHA256 = hex.EncodeToString(sum)
}
dirty = append(dirty, continuity.Dirty{Path: path, SHA256: hex.EncodeToString(sum)})
dirty = append(dirty, d)
}
return dirty, nil
}
@@ -641,6 +675,13 @@ var _ = json.RawMessage{}
// callers (Coordinator.rotate, refreshSessionHealth) surface it instead of
// mistaking "we don't know" for "occupancy is zero".
func (a CLIAdapter) Occupancy(s Session) (float64, error) {
if a.Harness == "opencode" {
u, err := OpenCodeSessionUsage(s.SessionID)
if err != nil {
return 0, err
}
return Fraction(u, a.Window), nil
}
if a.Usage == nil {
return 0, fmt.Errorf("adapter: usage reader required")
}
@@ -659,6 +700,32 @@ func (a CLIAdapter) Occupancy(s Session) (float64, error) {
return Fraction(u, a.Window), nil
}
// ResolveSessionIdentity discovers and returns the harness-native session
// identity. Callers persist the returned Session before relying on occupancy,
// so restart recovery keeps observing the same harness session.
func (a CLIAdapter) ResolveSessionIdentity(s Session) (Session, error) {
if a.Harness == "opencode" {
if s.SessionID != "" {
return s, nil
}
id, err := OpenCodeSessionID(s.Worktree)
if err != nil {
return s, err
}
s.SessionID = id
return s, nil
}
if s.SessionFile != "" {
return s, nil
}
path, err := a.resolveSessionFile(s)
if err != nil {
return s, err
}
s.SessionFile = path
return s, nil
}
func (a CLIAdapter) resolveSessionFile(s Session) (string, error) {
switch a.Harness {
case "claude":
@@ -667,12 +734,7 @@ func (a CLIAdapter) resolveSessionFile(s Session) (string, error) {
_, path, err := CodexActiveUsage("")
return path, err
default:
// opencode's session-file resolution needs the running session id,
// which is only available via the SSE/status API (OpenCodeStatus),
// not derivable from the worktree alone. Per AUDIT.md Phase 1, wiring
// this needs verification against a live opencode instance before it
// can drive rotation — refuse loudly rather than guess a path.
return "", fmt.Errorf("adapter: harness %q has no session-file resolver; verify against a live session first (AUDIT.md Phase 1)", a.Harness)
return "", fmt.Errorf("adapter: harness %q has no session-file resolver", a.Harness)
}
}
internal/herdr/herdr.go
+4
View File
@@ -152,6 +152,10 @@ type Session struct {
// CLIAdapter.Occupancy), since the file may not exist yet immediately
// after lease.
SessionFile string `json:"session_file,omitempty"`
// SessionID is the harness-native identity when its usage is stored in a
// database rather than a transcript. OpenCode's SQLite session ID is kept
// here so rotation never guesses "the newest session" after a restart.
SessionID string `json:"session_id,omitempty"`
// TaskFileSHA is the sha256 of the worktree's TASK.md at the time this
// session's lease was created — the immutable-spec hash continuity's
// pickup validation compares against on the next rotation (§6.2).
internal/herdr/occupancy.go
+56
View File
@@ -10,6 +10,7 @@ import (
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"time"
)
@@ -30,6 +31,7 @@ func Fraction(u Usage, w int64) float64 {
}
return f
}
// ClaudeSessionFile resolves the transcript file for a Claude Code session
// running against worktree, by newest-mtime under Claude Code's encoded
// project directory (~/.claude/projects/<abs-worktree-path-with-/-replaced-
@@ -202,6 +204,60 @@ func OpenCodeUsage(p string) (Usage, error) {
return Usage{x.Tokens.Input, x.Tokens.Cache.Read, x.Tokens.Cache.Write, x.Tokens.Output}, e
}
// OpenCodeSessionID resolves the exact OpenCode session associated with a
// checkout. OpenCode stores usage in its SQLite session table, not in a pane
// transcript. Selecting by directory and persisting the returned id prevents
// a multi-pane worker from attributing another task's newest session to this
// lease.
func OpenCodeSessionID(worktree string) (string, error) {
db := os.Getenv("ORCHESTRA_OPENCODE_DB")
if db == "" {
db = filepath.Join(os.Getenv("HOME"), ".local", "share", "opencode", "opencode.db")
}
abs, err := filepath.Abs(worktree)
if err != nil {
return "", err
}
out, err := exec.Command("sqlite3", "-readonly", "-noheader", db, "select id from session where directory = "+sqliteQuote(abs)+" order by time_updated desc limit 1;").Output()
if err != nil {
return "", fmt.Errorf("opencode session lookup: %w", err)
}
id := strings.TrimSpace(string(out))
if id == "" {
return "", fmt.Errorf("opencode session lookup: no session for %s", abs)
}
return id, nil
}
// OpenCodeSessionUsage reads the token counters for one persisted session.
func OpenCodeSessionUsage(sessionID string) (Usage, error) {
if sessionID == "" {
return Usage{}, fmt.Errorf("opencode session id required")
}
db := os.Getenv("ORCHESTRA_OPENCODE_DB")
if db == "" {
db = filepath.Join(os.Getenv("HOME"), ".local", "share", "opencode", "opencode.db")
}
out, err := exec.Command("sqlite3", "-readonly", "-noheader", "-separator", "|", db, "select tokens_input,tokens_cache_read,tokens_cache_write,tokens_output from session where id = "+sqliteQuote(sessionID)+";").Output()
if err != nil {
return Usage{}, fmt.Errorf("opencode usage lookup: %w", err)
}
parts := strings.Split(strings.TrimSpace(string(out)), "|")
if len(parts) != 4 {
return Usage{}, fmt.Errorf("opencode usage lookup: unknown session %q", sessionID)
}
values := [4]int64{}
for i, part := range parts {
values[i], err = strconv.ParseInt(part, 10, 64)
if err != nil {
return Usage{}, fmt.Errorf("opencode usage lookup: %w", err)
}
}
return Usage{Input: values[0], CacheRead: values[1], CacheWrite: values[2], Output: values[3]}, nil
}
func sqliteQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", "''") + "'" }
// OpenCodeStatus probes the server fast path. Callers can use the returned
// status and fall back to OpenCodeUsage when the SSE/server is unavailable.
func OpenCodeStatus(ctx context.Context, baseURL, sessionID string) (string, error) {
internal/orchestrator/orchestrator.go
+14 -22
View File
@@ -774,18 +774,21 @@ func (c *Coordinator) TurnDecision(ctx context.Context, taskID string) (string,
if existingReason := handoffReason(session.Worktree); existingReason == "manual" || existingReason == "milestone" || existingReason == "thrash" {
return c.finishRelease(ctx, taskID, task, session, a, existingReason)
}
if trigger, deadEnds := checkActivityTriggers(ctx, a, session, c.Thrash); trigger != "" {
c.requestReasonedHandoff(ctx, taskID, session, a, trigger, deadEnds)
return TurnPrepareHandoff, nil
d := (RotationStateMachine{Soft: c.soft(), Hard: c.Hard, Thrash: c.Thrash}).Evaluate(ctx, a, session)
if d.Degraded != nil {
return "", fmt.Errorf("orchestrator: rotation: %w", d.Degraded)
}
occupancy, err := a.Occupancy(session)
if err != nil {
return "", fmt.Errorf("orchestrator: occupancy: %w", err)
}
if occupancy < c.soft() {
if d.Action == TurnContinue {
return TurnContinue, nil
}
if occupancy < c.Hard {
if d.Action == TurnRefuse {
return TurnRefuse, nil
}
if d.Reason == "milestone" || d.Reason == "thrash" {
c.requestReasonedHandoff(ctx, taskID, session, a, d.Reason, d.DeadEnds)
return TurnPrepareHandoff, nil
}
if d.Action == TurnPrepareHandoff {
// Soft threshold (§5.3): advisory only. Ask the agent to start
// preparing a handoff well before Hard forces one, but don't block
// the turn on a boundary check — the agent is free to keep working.
@@ -805,18 +808,6 @@ func (c *Coordinator) TurnDecision(ctx context.Context, taskID string) (string,
}
return TurnPrepareHandoff, nil
}
if boundary, ok := a.(herdr.TurnBoundary); ok {
atBoundary, boundaryErr := boundary.AtTurnBoundary(ctx, session)
if boundaryErr != nil {
c.recordTurnBoundaryDegraded()
return TurnRefuse, nil
}
if !atBoundary {
return TurnRefuse, nil
}
} else {
c.recordTurnBoundaryDegraded()
}
if requester, ok := a.(herdr.HandoffRequester); ok {
if _, statErr := os.Stat(filepath.Join(session.Worktree, herdr.HandoffReportFile)); statErr != nil {
if !session.HandoffRequested {
@@ -979,11 +970,12 @@ func (c *Coordinator) rememberSession(taskID string, s herdr.Session) error {
}
func (c *Coordinator) block(t domain.Task, reason string) error {
p := map[string]string{"blocker": reason, "pane_state": "unknown"}
p := map[string]any{"blocker": reason, "block_reason": string(domain.InferBlockReason(reason)), "pane_state": "unknown", "session_evidence": domain.SessionEvidence{PaneState: "unknown", Source: "coordinator", CheckedAt: time.Now().UTC()}}
if s, ok := c.Session(t.ID); ok {
p["pane_id"] = s.PaneID
p["harness_id"] = s.HerdrID
p["pane_state"] = "open"
p["session_evidence"] = domain.SessionEvidence{PaneID: s.PaneID, HarnessID: s.HerdrID, PaneState: "open", Source: "coordinator", CheckedAt: time.Now().UTC()}
}
b, _ := json.Marshal(p)
return c.Store.Append(domain.Event{ID: domain.NewID(), Type: "TaskBlocked", TaskID: t.ID, Version: t.Version + 1, Payload: b, Surface: string(authz.System)})
internal/orchestrator/rotation_state.go
+82
View File
@@ -0,0 +1,82 @@
package orchestrator
import (
"context"
"fmt"
"orchestra/internal/continuity"
"orchestra/internal/herdr"
)
// RotationStateMachine is the shared, side-effect-free rotation policy used
// by both the coordinator's synchronous turn path and federation workers.
// Callers persist request/release side effects themselves, but must never
// replace an unavailable occupancy reading with zero.
type RotationStateMachine struct {
Soft float64
Hard float64
Thrash herdr.ThrashConfig
}
type RotationDecision struct {
Action string // continue, prepare_handoff, rotate_now
Reason string
DeadEnds []continuity.DeadEnd
// ActivityDegraded is advisory (threshold rotation still has a real usage
// source); it is surfaced so a missing milestone/thrash feed cannot be a
// silent no-op.
ActivityDegraded error
Degraded error
}
func (m RotationStateMachine) Evaluate(ctx context.Context, a herdr.Adapter, s herdr.Session) RotationDecision {
soft := m.Soft
if soft <= 0 {
soft = defaultSoft
}
if m.Hard <= 0 || m.Hard <= soft {
return RotationDecision{Degraded: fmt.Errorf("invalid rotation thresholds soft=%v hard=%v", soft, m.Hard)}
}
if reader, ok := a.(herdr.ActivityReader); ok {
calls, err := reader.Activity(ctx, s)
if err == nil {
if thrash, deadEnds := herdr.DetectThrash(calls, m.Thrash); thrash {
return RotationDecision{Action: TurnPrepareHandoff, Reason: "thrash", DeadEnds: deadEnds}
}
if herdr.DetectMilestone(calls) {
return RotationDecision{Action: TurnPrepareHandoff, Reason: "milestone"}
}
} else {
return m.evaluateOccupancy(ctx, a, s, fmt.Errorf("activity unknown: %w", err))
}
}
return m.evaluateOccupancy(ctx, a, s, nil)
}
func (m RotationStateMachine) evaluateOccupancy(ctx context.Context, a herdr.Adapter, s herdr.Session, activityErr error) RotationDecision {
soft := m.Soft
if soft <= 0 {
soft = defaultSoft
}
occupancy, err := a.Occupancy(s)
if err != nil {
return RotationDecision{ActivityDegraded: activityErr, Degraded: fmt.Errorf("occupancy unknown: %w", err)}
}
if occupancy < soft {
return RotationDecision{Action: TurnContinue, ActivityDegraded: activityErr}
}
if occupancy < m.Hard {
return RotationDecision{Action: TurnPrepareHandoff, Reason: "threshold", ActivityDegraded: activityErr}
}
boundary, ok := a.(herdr.TurnBoundary)
if !ok {
return RotationDecision{ActivityDegraded: activityErr, Degraded: fmt.Errorf("turn boundary unknown at hard threshold"), Action: TurnRefuse, Reason: "threshold"}
}
atBoundary, err := boundary.AtTurnBoundary(ctx, s)
if err != nil {
return RotationDecision{ActivityDegraded: activityErr, Degraded: fmt.Errorf("turn boundary unknown: %w", err), Action: TurnRefuse, Reason: "threshold"}
}
if !atBoundary {
return RotationDecision{Action: TurnRefuse, Reason: "threshold", ActivityDegraded: activityErr}
}
return RotationDecision{Action: TurnRotateNow, Reason: "threshold", ActivityDegraded: activityErr}
}
internal/orchestrator/rotation_test.go
+30 -12
View File
@@ -708,6 +708,7 @@ type activityAdapter struct {
activityErr error
reasonAsked []string
deadEndsSeen []continuity.DeadEnd
observed chan struct{}
}
func (a *activityAdapter) Activity(context.Context, herdr.Session) ([]herdr.ToolCall, error) {
@@ -717,6 +718,12 @@ func (a *activityAdapter) Activity(context.Context, herdr.Session) ([]herdr.Tool
func (a *activityAdapter) RequestHandoffReason(_ context.Context, _ herdr.Session, reason string, deadEnds []continuity.DeadEnd) error {
a.reasonAsked = append(a.reasonAsked, reason)
a.deadEndsSeen = deadEnds
if a.observed != nil {
select {
case a.observed <- struct{}{}:
default:
}
}
return nil
}
@@ -764,7 +771,7 @@ func TestActivityTriggersRequestReasonedHandoffWithoutReleasing(t *testing.T) {
}
t.Run("thrash requests a reasoned handoff and does not release, via TurnDecision", func(t *testing.T) {
a := &activityAdapter{fakeAdapter: fakeAdapter{occupancy: 0, boundary: true}, calls: thrashCalls}
a := &activityAdapter{fakeAdapter: fakeAdapter{occupancy: 0, boundary: true}, calls: thrashCalls, observed: make(chan struct{}, 1)}
c, st, task := newCoordinator(a)
decision, err := c.TurnDecision(context.Background(), task.ID)
if err != nil {
@@ -862,13 +869,15 @@ func TestActivityTriggersRequestReasonedHandoffWithoutReleasing(t *testing.T) {
c, st, task := newCoordinator(a)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go c.Monitor(ctx, .8, time.Millisecond)
done := make(chan error, 1)
go func() { done <- c.Monitor(ctx, .8, time.Millisecond) }()
deadline := time.Now().Add(300 * time.Millisecond)
for time.Now().Before(deadline) && len(a.reasonAsked) == 0 {
time.Sleep(time.Millisecond)
select {
case <-a.observed:
case <-time.After(300 * time.Millisecond):
}
cancel()
<-done
if len(a.reasonAsked) == 0 || a.reasonAsked[0] != "thrash" {
t.Fatalf("reasonAsked=%v want a thrash request from rotate()", a.reasonAsked)
}
@@ -959,10 +968,17 @@ func (w specWorktrees) Spec(domain.Task) (string, string, bool) { re
type conventionsAdapter struct {
fakeAdapter
notifications int
observed chan struct{}
}
func (a *conventionsAdapter) NotifyConventionsChanged(context.Context, herdr.Session) error {
a.notifications++
if a.observed != nil {
select {
case a.observed <- struct{}{}:
default:
}
}
return nil
}
@@ -991,7 +1007,7 @@ func TestConventionsDriftNotifiesActiveSession(t *testing.T) {
t.Fatal(err)
}
task := s.Tasks()[0]
a := &conventionsAdapter{fakeAdapter: fakeAdapter{occupancy: 0}}
a := &conventionsAdapter{fakeAdapter: fakeAdapter{occupancy: 0}, observed: make(chan struct{}, 1)}
c := &orchestrator.Coordinator{Store: s, Worktrees: specWorktrees{wtPath: worktree, repoPath: repo}, Adapters: adapters{a}, StatePath: t.TempDir() + "/sessions.json"}
leaseEvt, err := s.Lease(task.ID, "h1", time.Minute)
@@ -1003,8 +1019,8 @@ func TestConventionsDriftNotifiesActiveSession(t *testing.T) {
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go c.Monitor(ctx, .8, time.Millisecond)
done := make(chan error, 1)
go func() { done <- c.Monitor(ctx, .8, time.Millisecond) }()
time.Sleep(50 * time.Millisecond)
if a.notifications != 0 {
@@ -1015,10 +1031,12 @@ func TestConventionsDriftNotifiesActiveSession(t *testing.T) {
t.Fatal(err)
}
deadline := time.Now().Add(time.Second)
for time.Now().Before(deadline) && a.notifications == 0 {
time.Sleep(time.Millisecond)
select {
case <-a.observed:
case <-time.After(time.Second):
}
cancel()
<-done
if a.notifications == 0 {
t.Fatal("session was never notified of the conventions-doc update")
}
internal/store/store.go
+64
View File
@@ -138,14 +138,34 @@ func (s *Store) apply(e domain.Event) error {
if v, ok := p["description"].(string); ok {
t.Description = v
}
if v, ok := p["acceptance"].([]any); ok {
for _, item := range v {
if text, ok := item.(string); ok {
t.Acceptance = append(t.Acceptance, text)
}
}
}
if v, ok := p["quality_gate"].(string); ok {
t.QualityGate = v
}
s.external[t.Source+"\x00"+t.ExternalID] = t.ID
case "TaskLeased":
t.State = domain.StateLeased
t.Lease = &domain.Lease{HarnessID: p["harness_id"].(string), Until: time.Unix(0, int64(p["until_ns"].(float64)))}
case "TaskLeaseRenewed":
t.Lease = &domain.Lease{HarnessID: p["harness_id"].(string), Until: time.Unix(0, int64(p["until_ns"].(float64)))}
case "TaskReleased":
t.State = domain.StateQueued
t.Lease = nil
t.HandoffRef, _ = p["handoff_ref"].(string)
t.ReleaseTransaction, _ = p["transaction_id"].(string)
t.ReleaseAnchor, _ = p["anchor_sha"].(string)
t.PickupTransaction, t.PickupLeaseVersion = "", 0
case "TaskPickupValidated":
t.PickupTransaction, _ = p["transaction_id"].(string)
if v, ok := p["lease_version"].(float64); ok {
t.PickupLeaseVersion = int(v)
}
case "TaskCompleted":
t.State = domain.StateCompleted
t.Lease = nil
@@ -156,6 +176,10 @@ func (s *Store) apply(e domain.Event) error {
t.State = domain.StateBlocked
t.Lease = nil
t.Blocker, _ = p["blocker"].(string)
t.BlockReason = domain.InferBlockReason(t.Blocker)
if v, ok := p["block_reason"].(string); ok && domain.BlockReason(v).Valid() {
t.BlockReason = domain.BlockReason(v)
}
t.BlockedAt = e.At
t.LastPaneID, _ = p["pane_id"].(string)
t.LastHarness, _ = p["harness_id"].(string)
@@ -200,6 +224,25 @@ func (s *Store) apply(e domain.Event) error {
}
}
}
// Terminal and release events may carry an owner-produced snapshot from
// immediately before a worker/coordinator drops its live session mapping.
// Preserve it independently of the current task state so historical task
// pages never have to imply a pane is live just because its ID is known.
if raw, ok := p["session_evidence"].(map[string]any); ok {
var evidence domain.SessionEvidence
if b, err := json.Marshal(raw); err == nil && json.Unmarshal(b, &evidence) == nil {
t.LastSession = evidence
if evidence.PaneID != "" {
t.LastPaneID = evidence.PaneID
}
if evidence.HarnessID != "" {
t.LastHarness = evidence.HarnessID
}
if evidence.PaneState != "" {
t.PaneState = evidence.PaneState
}
}
}
t.Version = e.Version
s.tasks[e.TaskID] = t
return nil
@@ -410,12 +453,33 @@ func (s *Store) Lease(id, harness string, ttl time.Duration) (domain.Event, erro
payload := map[string]any{"harness_id": harness, "ttl": ttl.Seconds(), "until_ns": time.Now().Add(ttl).UnixNano(), "expected_version": t.Version}
if t.HandoffRef != "" {
payload["handoff_ref"] = t.HandoffRef
payload["transaction_id"] = t.ReleaseTransaction
payload["anchor_sha"] = t.ReleaseAnchor
}
p, _ := json.Marshal(payload)
e := domain.Event{ID: domain.NewID(), Type: "TaskLeased", TaskID: id, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
return e, s.Append(e)
}
// RenewLease atomically extends the current owner's lease. The observed task
// version is part of the request so an old worker can never renew a lease
// after release/reassignment.
func (s *Store) RenewLease(id, harness string, expectedVersion int, ttl time.Duration) (domain.Event, error) {
if ttl <= 0 {
return domain.Event{}, fmt.Errorf("%w: ttl must be positive", domain.ErrInvalid)
}
t, ok := s.Task(id)
if !ok {
return domain.Event{}, domain.ErrNotFound
}
if t.State != domain.StateLeased || t.Lease == nil || t.Lease.HarnessID != harness || t.Version != expectedVersion {
return domain.Event{}, domain.ErrConflict
}
p, _ := json.Marshal(map[string]any{"harness_id": harness, "until_ns": time.Now().Add(ttl).UnixNano(), "expected_version": expectedVersion})
e := domain.Event{ID: domain.NewID(), Type: "TaskLeaseRenewed", TaskID: id, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
return e, s.Append(e)
}
func (s *Store) ExpireLeases(now time.Time) ([]domain.Event, error) {
var out []domain.Event
for _, t := range s.Tasks() {
internal/store/store_test.go
+113
View File
@@ -5,6 +5,7 @@ import (
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -51,6 +52,80 @@ func TestLeaseCarriesReleasedHandoffRef(t *testing.T) {
}
}
func TestRenewLeaseRequiresCurrentOwnerAndVersion(t *testing.T) {
s, err := Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
if err := s.Append(domain.Event{ID: "create", Type: "TaskCreated", TaskID: "t", Version: 1, Payload: []byte(`{"source":"s","external_id":"renew","project":"p"}`), Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
if _, err := s.Lease("t", "worker-a", time.Minute); err != nil {
t.Fatal(err)
}
before, _ := s.Task("t")
if _, err := s.RenewLease("t", "worker-b", before.Version, time.Hour); !errors.Is(err, domain.ErrConflict) {
t.Fatalf("other worker renewal = %v, want conflict", err)
}
if _, err := s.RenewLease("t", "worker-a", before.Version-1, time.Hour); !errors.Is(err, domain.ErrConflict) {
t.Fatalf("stale renewal = %v, want conflict", err)
}
e, err := s.RenewLease("t", "worker-a", before.Version, time.Hour)
if err != nil {
t.Fatal(err)
}
after, _ := s.Task("t")
if e.Type != "TaskLeaseRenewed" || after.Version != before.Version+1 || after.Lease == nil || !after.Lease.Until.After(before.Lease.Until) {
t.Fatalf("renewal was not projected: before=%+v after=%+v event=%+v", before, after, e)
}
if _, err := s.RenewLease("t", "worker-a", before.Version, time.Hour); !errors.Is(err, domain.ErrConflict) {
t.Fatalf("replayed renewal = %v, want conflict", err)
}
}
func TestBlockedTaskProjectsStructuredDiagnosisAndLegacyFallback(t *testing.T) {
s, err := Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
if err := s.Append(domain.Event{ID: "create", Type: "TaskCreated", TaskID: "t", Version: 1, Payload: []byte(`{"source":"s","external_id":"blocked","project":"p"}`), Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
payload := []byte(`{"blocker":"worker is offline","block_reason":"worker_offline","pane_id":"p1","harness_id":"w1","pane_state":"unreachable"}`)
if err := s.Append(domain.Event{ID: "blocked", Type: "TaskBlocked", TaskID: "t", Version: 2, Payload: payload, Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
task, _ := s.Task("t")
if task.BlockReason != domain.BlockReasonWorkerOffline || task.LastPaneID != "p1" || task.PaneState != "unreachable" {
t.Fatalf("blocked diagnosis was not projected: %+v", task)
}
if got := domain.InferBlockReason("handoff validation failed"); got != domain.BlockReasonHandoffValidation {
t.Fatalf("legacy fallback=%q", got)
}
}
func TestTerminalSessionEvidenceSurvivesSessionCleanup(t *testing.T) {
s, err := Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
if err := s.Append(domain.Event{ID: "create", Type: "TaskCreated", TaskID: "t", Version: 1, Payload: []byte(`{"source":"s","external_id":"evidence","project":"p"}`), Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
ref, err := s.PutArtifact([]byte("report"))
if err != nil {
t.Fatal(err)
}
payload := []byte(`{"report_ref":"` + ref + `","receipt":{"source":"worker"},"session_evidence":{"pane_id":"w:p1","harness_id":"worker-1","pane_state":"open","source":"worker","captured_at":"2026-07-29T12:00:00Z","checked_at":"2026-07-29T12:00:01Z"}}`)
if err := s.Append(domain.Event{ID: "complete", Type: "TaskCompleted", TaskID: "t", Version: 2, Payload: payload, Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
task, _ := s.Task("t")
if task.LastSession.PaneID != "w:p1" || task.LastSession.Source != "worker" || task.LastSession.CapturedAt.IsZero() || task.LastHarness != "worker-1" {
t.Fatalf("terminal session evidence was lost: %+v", task)
}
}
func TestAppendReplayAndDeduplicate(t *testing.T) {
dir := t.TempDir()
s, err := Open(dir)
@@ -314,3 +389,41 @@ func TestExpectedVersionIsCheckedForEveryWriter(t *testing.T) {
t.Fatalf("expected CAS conflict, got %v", err)
}
}
func TestReleaseTransactionSurvivesReLeaseUntilMatchingPickup(t *testing.T) {
s, err := Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
if err := s.Append(created("create")); err != nil {
t.Fatal(err)
}
leased, err := s.Lease("task-1", "predecessor", time.Minute)
if err != nil {
t.Fatal(err)
}
ref, err := s.PutArtifact([]byte("handoff"))
if err != nil {
t.Fatal(err)
}
anchor := strings.Repeat("a", 40)
p, _ := json.Marshal(map[string]any{"handoff_ref": ref, "anchor_sha": anchor, "transaction_id": "tx-1", "expected_version": leased.Version})
if err := s.Append(domain.Event{ID: "release", Type: "TaskReleased", TaskID: "task-1", Version: leased.Version + 1, Payload: p, Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
if _, err := s.Lease("task-1", "successor", time.Minute); err != nil {
t.Fatal(err)
}
task, _ := s.Task("task-1")
if task.ReleaseTransaction != "tx-1" || task.ReleaseAnchor != anchor || task.HandoffRef != ref {
t.Fatalf("re-lease lost transaction: %+v", task)
}
p, _ = json.Marshal(map[string]any{"transaction_id": "tx-1", "handoff_ref": ref, "anchor_sha": anchor, "harness_id": "successor", "lease_version": task.Version, "expected_version": task.Version})
if err := s.Append(domain.Event{ID: "pickup", Type: "TaskPickupValidated", TaskID: "task-1", Version: task.Version + 1, Payload: p, Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
task, _ = s.Task("task-1")
if task.PickupTransaction != "tx-1" || task.PickupLeaseVersion != 4 {
t.Fatalf("pickup not bound to transaction/epoch: %+v", task)
}
}