fix: make worker handoff rotation durable

2026-07-30 01:30:59 +04:00
parent ce02c60106
commit 1ff0af2e69
16 changed files with 1488 additions and 1566 deletions
@@ -6,8 +6,10 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"log"
+	"orchestra/internal/buildinfo"
 	"orchestra/internal/continuity"
 	"orchestra/internal/domain"
 	"orchestra/internal/federation"
@@ -17,6 +19,7 @@ import (
 	"os/exec"
 	"os/signal"
 	"path/filepath"
+	"sort"
 	"strconv"
 	"strings"
 	"syscall"
@@ -27,15 +30,19 @@ type worker struct {
 	api                                    federation.Client
 	herdr                                  *herdr.Client
 	harnessID, harness, repo, root, remote string
+	projects                               map[string]projectConfig
 	cursor                                 uint64
 	tasks                                  map[string]domain.Task
 	sessions                               map[string]herdr.Session
 	leases                                 map[string]lease
+	releases                               map[string]releaseTransaction
 	statePath                              string
 	hard                                   float64
 	registration                           federation.Worker
 	lastError                              string
 	lastErrorAt                            time.Time
+	soft                                   float64
+	window                                 int64
 }

 func (w *worker) recordError(err error) {
@@ -72,13 +79,49 @@ func (w *worker) health(ctx context.Context) federation.WorkerHealth {
 }

 type lease struct {
-	HandoffRef string `json:"handoff_ref,omitempty"`
+	HandoffRef         string    `json:"handoff_ref,omitempty"`
+	TransactionID      string    `json:"transaction_id,omitempty"`
+	AnchorSHA          string    `json:"anchor_sha,omitempty"`
+	PickupAcknowledged bool      `json:"pickup_acknowledged,omitempty"`
+	Version            int       `json:"version"`
+	Until              time.Time `json:"until"`
+}
+type releaseTransaction struct {
+	ID            string    `json:"id"`
+	LeaseVersion  int       `json:"lease_version"`
+	Ref           string    `json:"handoff_ref,omitempty"`
+	AnchorSHA     string    `json:"anchor_sha,omitempty"`
+	Phase         string    `json:"phase"` // prepared, anchor_pushed, event_committed, pickup_validated, predecessor_retired
+	AgentReleased bool      `json:"agent_released,omitempty"`
+	LastError     string    `json:"last_error,omitempty"`
+	UpdatedAt     time.Time `json:"updated_at"`
+}
+type projectConfig struct {
+	Repo        string `json:"repo"`
+	Root        string `json:"worktree_root"`
+	Remote      string `json:"remote"`
+	QualityGate string `json:"quality_gate,omitempty"`
+}
+type completionEvidence struct {
+	TaskID      string    `json:"task_id"`
+	Project     string    `json:"project"`
+	Worker      string    `json:"worker"`
+	Harness     string    `json:"harness"`
+	PaneID      string    `json:"pane_id"`
+	BaseSHA     string    `json:"base_sha"`
+	ResultSHA   string    `json:"result_sha"`
+	Branch      string    `json:"branch"`
+	Remote      string    `json:"remote"`
+	QualityGate string    `json:"quality_gate,omitempty"`
+	GateExit    int       `json:"gate_exit"`
+	CompletedAt time.Time `json:"completed_at"`
 }
 type workerState struct {
-	Cursor   uint64                   `json:"cursor"`
-	Sessions map[string]herdr.Session `json:"sessions"`
-	Tasks    map[string]domain.Task   `json:"tasks"`
-	Leases   map[string]lease         `json:"leases"`
+	Cursor   uint64                        `json:"cursor"`
+	Sessions map[string]herdr.Session      `json:"sessions"`
+	Tasks    map[string]domain.Task        `json:"tasks"`
+	Leases   map[string]lease              `json:"leases"`
+	Releases map[string]releaseTransaction `json:"releases"`
 }

 func (w *worker) load() {
@@ -90,6 +133,7 @@ func (w *worker) load() {
 			w.sessions = s.Sessions
 			w.tasks = s.Tasks
 			w.leases = s.Leases
+			w.releases = s.Releases
 		}
 	}
 	if w.sessions == nil {
@@ -101,9 +145,12 @@ func (w *worker) load() {
 	if w.leases == nil {
 		w.leases = map[string]lease{}
 	}
+	if w.releases == nil {
+		w.releases = map[string]releaseTransaction{}
+	}
 }
 func (w *worker) save() error {
-	b, e := json.Marshal(workerState{Cursor: w.cursor, Sessions: w.sessions, Tasks: w.tasks, Leases: w.leases})
+	b, e := json.Marshal(workerState{Cursor: w.cursor, Sessions: w.sessions, Tasks: w.tasks, Leases: w.leases, Releases: w.releases})
 	if e != nil {
 		return e
 	}
@@ -133,23 +180,35 @@ func created(e domain.Event) (domain.Task, bool) {
 		Capability  []string `json:"capability"`
 		Title       string   `json:"title"`
 		Description string   `json:"description"`
+		Acceptance  []string `json:"acceptance"`
+		QualityGate string   `json:"quality_gate"`
 	}
 	if json.Unmarshal(e.Payload, &p) != nil || p.Source == "" || p.ExternalID == "" || p.Project == "" {
 		return domain.Task{}, false
 	}
-	return domain.Task{ID: e.TaskID, Source: p.Source, ExternalID: p.ExternalID, Project: p.Project, Capability: p.Capability, Title: p.Title, Description: p.Description}, true
+	return domain.Task{ID: e.TaskID, Source: p.Source, ExternalID: p.ExternalID, Project: p.Project, Capability: p.Capability, Title: p.Title, Description: p.Description, Acceptance: p.Acceptance, QualityGate: p.QualityGate}, true
 }

-func (w *worker) syncBase(ctx context.Context) error {
-	if out, err := exec.CommandContext(ctx, "git", "-C", w.repo, "fetch", w.remote, "--prune").CombinedOutput(); err != nil {
+func (w *worker) project(t domain.Task) (projectConfig, error) {
+	if w.projects != nil {
+		if p, ok := w.projects[t.Project]; ok && p.Repo != "" && p.Root != "" && p.Remote != "" {
+			return p, nil
+		}
+		return projectConfig{}, fmt.Errorf("project %q is not configured on worker", t.Project)
+	}
+	return projectConfig{Repo: w.repo, Root: w.root, Remote: w.remote}, nil
+}
+
+func (w *worker) syncBase(ctx context.Context, p projectConfig) error {
+	if out, err := exec.CommandContext(ctx, "git", "-C", p.Repo, "fetch", p.Remote, "--prune").CombinedOutput(); err != nil {
 		return fmt.Errorf("fetch base checkout: %s: %w", out, err)
 	}
-	branch, err := exec.CommandContext(ctx, "git", "-C", w.repo, "symbolic-ref", "--quiet", "--short", "HEAD").Output()
+	branch, err := exec.CommandContext(ctx, "git", "-C", p.Repo, "symbolic-ref", "--quiet", "--short", "HEAD").Output()
 	if err != nil {
 		return fmt.Errorf("identify base branch: %w", err)
 	}
 	branchName := strings.TrimSpace(string(branch))
-	if out, err := exec.CommandContext(ctx, "git", "-C", w.repo, "merge", "--ff-only", w.remote+"/"+branchName).CombinedOutput(); err != nil {
+	if out, err := exec.CommandContext(ctx, "git", "-C", p.Repo, "merge", "--ff-only", p.Remote+"/"+branchName).CombinedOutput(); err != nil {
 		return fmt.Errorf("fast-forward base checkout: %s: %w", out, err)
 	}
 	return nil
@@ -159,10 +218,14 @@ func (w *worker) start(ctx context.Context, t domain.Task, ref string) error {
 	var wt string
 	var h continuity.Handoff
 	var err error
+	p, err := w.project(t)
+	if err != nil {
+		return err
+	}
 	// Synchronize the local base before any worktree operation. A worker never
 	// treats a coordinator-side path as truth; the Git remote is the only
 	// cross-machine transport.
-	if err := w.syncBase(ctx); err != nil {
+	if err := w.syncBase(ctx, p); err != nil {
 		return err
 	}
 	if ref != "" {
@@ -174,12 +237,12 @@ func (w *worker) start(ctx context.Context, t domain.Task, ref string) error {
 		if err != nil {
 			return err
 		}
-		if out, err := exec.CommandContext(ctx, "git", "-C", w.repo, "fetch", w.remote, "--prune").CombinedOutput(); err != nil {
+		if out, err := exec.CommandContext(ctx, "git", "-C", p.Repo, "fetch", p.Remote, "--prune").CombinedOutput(); err != nil {
 			return fmt.Errorf("fetch pickup anchor: %s: %w", out, err)
 		}
-		wt = filepath.Join(w.root, t.ID)
+		wt = filepath.Join(p.Root, t.ID)
 		if _, err := os.Stat(wt); os.IsNotExist(err) {
-			if out, err := exec.CommandContext(ctx, "git", "-C", w.repo, "worktree", "add", "-b", "orchestra/"+t.ID, wt, h.Anchor.GitSHA).CombinedOutput(); err != nil {
+			if out, err := exec.CommandContext(ctx, "git", "-C", p.Repo, "worktree", "add", "-b", "orchestra/"+t.ID, wt, h.Anchor.GitSHA).CombinedOutput(); err != nil {
 				return fmt.Errorf("create pickup worktree: %s: %w", out, err)
 			}
 		}
@@ -187,21 +250,22 @@ func (w *worker) start(ctx context.Context, t domain.Task, ref string) error {
 			return err
 		}
 	} else {
-		wt, err = (orchestrator.GitWorktrees{Repo: w.repo, Root: w.root}).Create(ctx, t)
+		wt, err = (orchestrator.GitWorktrees{Repo: p.Repo, Root: p.Root}).Create(ctx, t)
 		if err != nil {
 			return err
 		}
 	}
-	if _, err = w.herdr.Worktree(ctx, w.repo, wt, "orchestra/"+t.ID); err != nil {
+	if _, err = w.herdr.Worktree(ctx, p.Repo, wt, "orchestra/"+t.ID); err != nil {
 		return err
 	}
 	s, err := w.herdr.StartAgent(ctx, wt, wt, "orchestra/"+t.ID, w.harness, t.ID)
 	if err != nil {
 		return err
 	}
-	p := "Begin Orchestra task " + t.ID + ".\nTitle: " + t.Title + "\nInstructions:\n" + t.Description + "\nWork only in this worktree. Do not edit TASK.md."
+	s.TaskFileSHA = taskHash(t)
+	prompt := "Read TASK.md at the worktree root and execute it."
 	if ref != "" {
-		p += "\nA validated handoff exists. Read TASK.md and inspect local git history before continuing."
+		prompt += " A validated handoff exists; inspect local Git history and the recorded checkpoint before continuing."
 	}
 	w.sessions[t.ID] = s
 	if err := w.save(); err != nil {
@@ -209,65 +273,362 @@ func (w *worker) start(ctx context.Context, t domain.Task, ref string) error {
 	}
 	// A prompt response can be lost after herdr accepted it. Persist the
 	// session first so the worker can reconcile/release it after restart.
-	return w.herdr.Prompt(ctx, s.PaneID, p, 0)
+	if err := w.herdr.Prompt(ctx, s.PaneID, prompt, 0); err != nil {
+		return err
+	}
+	if ref != "" {
+		return w.ackPickup(ctx, t.ID, s)
+	}
+	return nil
 }

 func taskHash(t domain.Task) string { b := continuity.RenderTaskFile(t); return domain.Hash(b) }

 func (w *worker) releaseReady(ctx context.Context) {
 	for id, s := range w.sessions {
-		if report, err := os.ReadFile(filepath.Join(s.Worktree, ".orchestra-report.md")); err == nil && len(report) > 0 {
-			if ref, err := w.api.PutArtifact(ctx, report); err == nil {
-				if err = w.api.Complete(ctx, id, ref); err == nil {
-					delete(w.sessions, id)
-					delete(w.leases, id)
-					_ = w.save()
-					continue
-				}
+		if l := w.leases[id]; l.HandoffRef != "" && !l.PickupAcknowledged {
+			if err := w.ackPickup(ctx, id, s); err != nil {
+				w.recordError(err)
+				continue
+			}
+		}
+		if _, err := os.Stat(filepath.Join(s.Worktree, ".orchestra", "done")); err == nil {
+			evidence, err := w.finalize(ctx, id, s)
+			if err != nil {
+				w.recordError(fmt.Errorf("complete %s: %w", id, err))
 				log.Printf("complete %s: %v", id, err)
-			} else {
+				continue
+			}
+			report, _ := json.Marshal(evidence)
+			ref, err := w.api.PutArtifact(ctx, report)
+			if err != nil {
+				w.recordError(fmt.Errorf("upload completion %s: %w", id, err))
 				log.Printf("upload completion %s: %v", id, err)
+				continue
 			}
-		}
-		if _, err := os.Stat(filepath.Join(s.Worktree, herdr.HandoffReportFile)); err != nil {
-			if !s.HandoffRequested {
-				a := herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}
-				if occ, occErr := a.Occupancy(s); occErr == nil && occ >= w.hard {
-					if boundary, boundaryErr := a.AtTurnBoundary(ctx, s); boundaryErr == nil && boundary {
-						if err := a.RequestHandoff(ctx, s); err == nil {
-							s.HandoffRequested, s.HandoffReason = true, "threshold"
-							w.sessions[id] = s
-							_ = w.save()
-						}
-					}
-				}
+			if err = w.api.Complete(ctx, id, ref, evidence.ResultSHA, evidence.Branch, evidence.Remote, w.leases[id].Version, w.usageReceipt(s), w.sessionEvidence(ctx, id, s)); err != nil {
+				w.recordError(fmt.Errorf("complete %s: %w", id, err))
+				log.Printf("complete %s: %v", id, err)
+				continue
 			}
+			// Completion is durable before closing the exact pane. If close
+			// fails, retain the session mapping for a later explicit cleanup.
+			a := herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}
+			if err := a.Kill(ctx, s); err != nil {
+				w.recordError(fmt.Errorf("close completed pane %s: %w", id, err))
+				log.Printf("close completed pane %s: %v", id, err)
+				continue
+			}
+			_ = os.Remove(filepath.Join(s.Worktree, ".orchestra", "done"))
+			_ = os.Remove(filepath.Join(s.Worktree, ".orchestra"))
+			delete(w.sessions, id)
+			delete(w.leases, id)
+			_ = w.save()
 			continue
 		}
-		a := herdr.CLIAdapter{Client: w.herdr, Harness: w.harness, CAS: artifactCAS{w.api}, Remote: w.remote}
-		ref, err := a.Release(ctx, s)
-		if err != nil {
-			log.Printf("release %s: %v", id, err)
+		if _, err := os.Stat(filepath.Join(s.Worktree, herdr.HandoffReportFile)); err == nil || w.releases[id].ID != "" {
+			w.advanceRelease(ctx, id, s)
 			continue
 		}
-		sha, err := herdr.HeadSHA(s.Worktree)
-		if err != nil {
-			log.Printf("release %s anchor: %v", id, err)
-			continue
-		}
-		if err = w.api.Release(ctx, id, ref, sha); err != nil {
-			log.Printf("publish release %s: %v", id, err)
-			continue
-		}
-		// release_agent only removes herdr's binding. Closing the released pane
-		// after the handoff is durable prevents the next StartAgent from
-		// inheriting the predecessor's still-running terminal process.
-		if err := a.Kill(ctx, s); err != nil {
-			log.Printf("close released pane %s: %v", id, err)
-		}
-		delete(w.sessions, id)
+		w.rotationTick(ctx, id, s)
+	}
+}
+
+func (w *worker) adapter(s herdr.Session, remote string) herdr.CLIAdapter {
+	a := herdr.CLIAdapter{Client: w.herdr, Harness: w.harness, Window: w.window, CAS: artifactCAS{w.api}, Remote: remote}
+	switch w.harness {
+	case "claude":
+		a.Usage = herdr.ClaudeUsage
+	case "codex":
+		a.Usage = herdr.CodexUsage
+	case "opencode":
+		a.Usage = herdr.OpenCodeUsage
+	}
+	return a
+}
+
+// rotationTick is the checkout-owner state machine. Occupancy, tool activity,
+// and pane status are all read from the persisted harness session identity;
+// any unknown source is recorded and never treated as zero usage.
+func (w *worker) rotationTick(ctx context.Context, id string, s herdr.Session) {
+	t, ok := w.tasks[id]
+	if !ok {
+		w.recordError(fmt.Errorf("rotation %s: task cache missing", id))
+		return
+	}
+	p, err := w.project(t)
+	if err != nil {
+		w.recordError(err)
+		return
+	}
+	a := w.adapter(s, p.Remote)
+	resolved, err := a.ResolveSessionIdentity(s)
+	if err != nil {
+		w.recordError(fmt.Errorf("rotation %s occupancy degraded: %w", id, err))
+		return
+	}
+	if resolved != s {
+		w.sessions[id] = resolved
+		s = resolved
 		_ = w.save()
 	}
+	d := (orchestrator.RotationStateMachine{Soft: w.soft, Hard: w.hard}).Evaluate(ctx, a, s)
+	if d.ActivityDegraded != nil {
+		w.recordError(fmt.Errorf("rotation %s activity degraded: %w", id, d.ActivityDegraded))
+	}
+	if d.Degraded != nil {
+		w.recordError(fmt.Errorf("rotation %s degraded: %w", id, d.Degraded))
+		if d.Action == orchestrator.TurnContinue || d.Action == "" {
+			return
+		}
+	}
+	if d.Action == orchestrator.TurnContinue || d.Action == orchestrator.TurnRefuse || s.HandoffRequested {
+		return
+	}
+	if d.Reason == "milestone" || d.Reason == "thrash" {
+		if err := a.RequestHandoffReason(ctx, s, d.Reason, d.DeadEnds); err != nil {
+			w.recordError(fmt.Errorf("rotation %s %s prompt: %w", id, d.Reason, err))
+			return
+		}
+	} else if err := a.RequestHandoff(ctx, s); err != nil {
+		w.recordError(fmt.Errorf("rotation %s threshold prompt: %w", id, err))
+		return
+	}
+	s.HandoffRequested, s.HandoffReason = true, d.Reason
+	w.sessions[id] = s
+	_ = w.save()
+}
+
+func (w *worker) advanceRelease(ctx context.Context, id string, s herdr.Session) {
+	t, ok := w.tasks[id]
+	if !ok {
+		w.recordError(fmt.Errorf("release %s: task cache missing", id))
+		return
+	}
+	p, err := w.project(t)
+	if err != nil {
+		w.recordError(fmt.Errorf("release %s: %w", id, err))
+		return
+	}
+	if w.releases == nil {
+		w.releases = map[string]releaseTransaction{}
+	}
+	tx := w.releases[id]
+	if tx.ID == "" {
+		l, ok := w.leases[id]
+		if !ok {
+			w.recordError(fmt.Errorf("release %s: lease missing", id))
+			return
+		}
+		tx = releaseTransaction{ID: domain.NewID(), LeaseVersion: l.Version, Phase: "prepared", UpdatedAt: time.Now().UTC()}
+		w.releases[id] = tx
+		_ = w.save()
+	}
+	a := w.adapter(s, p.Remote)
+	if tx.Phase == "prepared" {
+		prepared, err := a.PrepareRelease(ctx, s)
+		if err != nil {
+			tx.LastError, tx.UpdatedAt = err.Error(), time.Now().UTC()
+			w.releases[id] = tx
+			_ = w.save()
+			w.recordError(fmt.Errorf("release %s prepare: %w", id, err))
+			return
+		}
+		tx.Ref, tx.AnchorSHA, tx.Phase, tx.LastError, tx.UpdatedAt = prepared.Ref, prepared.AnchorSHA, "anchor_pushed", "", time.Now().UTC()
+		w.releases[id] = tx
+		_ = w.save()
+	}
+	if tx.Phase == "anchor_pushed" {
+		if err := w.api.Release(ctx, id, tx.Ref, tx.AnchorSHA, tx.ID, tx.LeaseVersion, w.sessionEvidence(ctx, id, s)); err != nil {
+			tx.LastError, tx.UpdatedAt = err.Error(), time.Now().UTC()
+			w.releases[id] = tx
+			_ = w.save()
+			w.recordError(fmt.Errorf("release %s commit: %w", id, err))
+			return
+		}
+		tx.Phase, tx.LastError, tx.UpdatedAt = "event_committed", "", time.Now().UTC()
+		w.releases[id] = tx
+		_ = w.save()
+	}
+	if tx.Phase == "event_committed" && !tx.AgentReleased {
+		if err := a.ReleaseAgent(ctx, s); err != nil {
+			tx.LastError, tx.UpdatedAt = err.Error(), time.Now().UTC()
+			w.releases[id] = tx
+			_ = w.save()
+			w.recordError(fmt.Errorf("release %s release agent: %w", id, err))
+			return
+		}
+		tx.AgentReleased, tx.LastError, tx.UpdatedAt = true, "", time.Now().UTC()
+		w.releases[id] = tx
+		_ = w.save()
+	}
+	if tx.Phase == "pickup_validated" {
+		if err := a.Kill(ctx, s); err != nil {
+			tx.LastError, tx.UpdatedAt = err.Error(), time.Now().UTC()
+			w.releases[id] = tx
+			_ = w.save()
+			w.recordError(fmt.Errorf("release %s retire predecessor: %w", id, err))
+			return
+		}
+		_ = os.Remove(filepath.Join(s.Worktree, herdr.HandoffReportFile))
+		tx.Phase, tx.UpdatedAt = "predecessor_retired", time.Now().UTC()
+		w.releases[id] = tx
+		_ = w.save()
+		delete(w.sessions, id)
+		delete(w.releases, id)
+		_ = w.save()
+	}
+}
+
+// ackPickup retries the successor acknowledgement from persisted lease state.
+// It is safe after a lost response: the coordinator recognizes the exact
+// transaction/lease epoch as an idempotent pickup.
+func (w *worker) ackPickup(ctx context.Context, id string, s herdr.Session) error {
+	l, ok := w.leases[id]
+	if !ok || l.HandoffRef == "" || l.TransactionID == "" || l.AnchorSHA == "" {
+		return fmt.Errorf("pickup %s: lease is missing its release transaction", id)
+	}
+	if l.PickupAcknowledged {
+		return nil
+	}
+	if err := w.api.Pickup(ctx, id, l.HandoffRef, l.AnchorSHA, l.TransactionID, l.Version, w.sessionEvidence(ctx, id, s)); err != nil {
+		return fmt.Errorf("pickup %s acknowledgement: %w", id, err)
+	}
+	l.PickupAcknowledged = true
+	l.Version++ // TaskPickupValidated increments the task version.
+	w.leases[id] = l
+	return w.save()
+}
+
+func (w *worker) sessionEvidence(ctx context.Context, taskID string, s herdr.Session) domain.SessionEvidence {
+	e := domain.SessionEvidence{PaneID: s.PaneID, HarnessID: w.harnessID, PaneState: "open", Source: "worker", CheckedAt: time.Now().UTC()}
+	text, err := (herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}).PaneCapture(ctx, s, "recent")
+	if err != nil {
+		e.PaneState = "unreachable"
+		return e
+	}
+	if capture, err := w.api.PublishCapture(ctx, federation.Capture{TaskID: taskID, PaneID: s.PaneID, Text: text}); err == nil {
+		e.CapturedAt = capture.At
+	}
+	return e
+}
+
+func (w *worker) usageReceipt(s herdr.Session) map[string]any {
+	if s.SessionFile == "" && !(w.harness == "opencode" && s.SessionID != "") {
+		return map[string]any{"harness_id": w.harnessID, "consumed": 0}
+	}
+	var usage herdr.Usage
+	var err error
+	switch w.harness {
+	case "claude":
+		usage, err = herdr.ClaudeUsage(s.SessionFile)
+	case "codex":
+		usage, err = herdr.CodexUsage(s.SessionFile)
+	case "opencode":
+		usage, err = herdr.OpenCodeSessionUsage(s.SessionID)
+	}
+	if err != nil {
+		return map[string]any{"harness_id": w.harnessID, "consumed": 0, "error": err.Error()}
+	}
+	return map[string]any{"harness_id": w.harnessID, "input_tokens": usage.Input, "cache_read_tokens": usage.CacheRead, "cache_write_tokens": usage.CacheWrite, "output_tokens": usage.Output, "consumed": usage.Numerator()}
+}
+
+func git(ctx context.Context, dir string, args ...string) ([]byte, error) {
+	return exec.CommandContext(ctx, "git", append([]string{"-C", dir}, args...)...).CombinedOutput()
+}
+
+// finalize performs only mechanical delivery work. It never asks the harness
+// to narrate Git state, gates, or a report; those are generated from the
+// worker-owned checkout and then verified against the configured remote.
+func (w *worker) finalize(ctx context.Context, id string, s herdr.Session) (completionEvidence, error) {
+	t, ok := w.tasks[id]
+	if !ok {
+		return completionEvidence{}, fmt.Errorf("task cache missing")
+	}
+	p, err := w.project(t)
+	if err != nil {
+		return completionEvidence{}, err
+	}
+	if s.TaskFileSHA != "" {
+		if err := continuity.VerifyTaskFile(s.Worktree, s.TaskFileSHA); err != nil {
+			return completionEvidence{}, fmt.Errorf("verify immutable TASK.md: %w", err)
+		}
+	}
+	base, err := git(ctx, s.Worktree, "rev-parse", "HEAD")
+	if err != nil {
+		return completionEvidence{}, fmt.Errorf("base sha: %s: %w", base, err)
+	}
+	e := completionEvidence{TaskID: id, Project: t.Project, Worker: w.harnessID, Harness: w.harness, PaneID: s.PaneID, BaseSHA: strings.TrimSpace(string(base)), Remote: p.Remote, QualityGate: t.QualityGate, CompletedAt: time.Now().UTC()}
+	gateCommand := t.QualityGate
+	if gateCommand == "" {
+		gateCommand = p.QualityGate
+	}
+	e.QualityGate = gateCommand
+	if gateCommand != "" {
+		gate := exec.CommandContext(ctx, "sh", "-c", gateCommand)
+		gate.Dir = s.Worktree
+		if out, err := gate.CombinedOutput(); err != nil {
+			e.GateExit = 1
+			return completionEvidence{}, fmt.Errorf("quality gate %q: %s: %w", gateCommand, out, err)
+		}
+	}
+	if _, err := git(ctx, s.Worktree, "diff", "--quiet", "--", "TASK.md"); err != nil {
+		return completionEvidence{}, errors.New("TASK.md was modified")
+	}
+	if out, err := git(ctx, s.Worktree, "add", "-A", "--", ".", ":!.orchestra/done"); err != nil {
+		return completionEvidence{}, fmt.Errorf("stage result: %s: %w", out, err)
+	}
+	if _, err := git(ctx, s.Worktree, "diff", "--cached", "--quiet"); err != nil {
+		if out, err := git(ctx, s.Worktree, "commit", "-m", "orchestra: complete "+id); err != nil {
+			return completionEvidence{}, fmt.Errorf("commit result: %s: %w", out, err)
+		}
+	}
+	branch, err := git(ctx, s.Worktree, "branch", "--show-current")
+	if err != nil || strings.TrimSpace(string(branch)) == "" {
+		return completionEvidence{}, fmt.Errorf("result branch: %s: %w", branch, err)
+	}
+	e.Branch = strings.TrimSpace(string(branch))
+	sha, err := git(ctx, s.Worktree, "rev-parse", "HEAD")
+	if err != nil {
+		return completionEvidence{}, fmt.Errorf("result sha: %s: %w", sha, err)
+	}
+	e.ResultSHA = strings.TrimSpace(string(sha))
+	if out, err := git(ctx, s.Worktree, "push", p.Remote, "HEAD:refs/heads/"+e.Branch); err != nil {
+		return completionEvidence{}, fmt.Errorf("push result: %s: %w", out, err)
+	}
+	remote, err := git(ctx, s.Worktree, "ls-remote", p.Remote, "refs/heads/"+e.Branch)
+	if err != nil || !strings.HasPrefix(string(remote), e.ResultSHA+"\t") {
+		return completionEvidence{}, fmt.Errorf("verify pushed sha: got %q: %w", strings.TrimSpace(string(remote)), err)
+	}
+	return e, nil
+}
+
+func (w *worker) renewLeases(ctx context.Context) {
+	if w.herdr == nil {
+		return
+	}
+	now := time.Now()
+	for taskID, l := range w.leases {
+		s, ok := w.sessions[taskID]
+		if !ok || l.Version == 0 || l.Until.After(now.Add(10*time.Minute)) {
+			continue
+		}
+		if _, err := (herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}).PaneCapture(ctx, s, "recent"); err != nil {
+			w.recordError(fmt.Errorf("validate lease %s: %w", taskID, err))
+			continue
+		}
+		if err := w.api.Renew(ctx, taskID, l.Version, int((30 * time.Minute).Seconds())); err != nil {
+			w.recordError(fmt.Errorf("renew lease %s: %w", taskID, err))
+			log.Printf("renew lease %s: %v", taskID, err)
+		} else {
+			// RenewLease appends one event. Retain that epoch locally until its
+			// replay arrives so a release transaction uses the same version.
+			l.Version++
+			l.Until = now.Add(30 * time.Minute)
+			w.leases[taskID] = l
+			_ = w.save()
+		}
+	}
 }

 // publishCaptures makes remote panes observable without allowing the
@@ -364,16 +725,79 @@ func (w *worker) once(ctx context.Context) error {
 		}
 		if e.Type == "TaskLeased" {
 			var p struct {
-				HarnessID  string `json:"harness_id"`
-				HandoffRef string `json:"handoff_ref"`
+				HarnessID     string `json:"harness_id"`
+				HandoffRef    string `json:"handoff_ref"`
+				TransactionID string `json:"transaction_id"`
+				AnchorSHA     string `json:"anchor_sha"`
 			}
 			if json.Unmarshal(e.Payload, &p) == nil && p.HarnessID == w.harnessID {
-				w.leases[e.TaskID] = lease{HandoffRef: p.HandoffRef}
+				var until struct {
+					UntilNS int64 `json:"until_ns"`
+				}
+				_ = json.Unmarshal(e.Payload, &until)
+				w.leases[e.TaskID] = lease{HandoffRef: p.HandoffRef, TransactionID: p.TransactionID, AnchorSHA: p.AnchorSHA, Version: e.Version, Until: time.Unix(0, until.UntilNS)}
 			}
 		}
-		if e.Type == "TaskReleased" || e.Type == "TaskCompleted" || e.Type == "TaskFailed" || e.Type == "TaskBlocked" {
+		if e.Type == "TaskLeaseRenewed" {
+			var p struct {
+				HarnessID string `json:"harness_id"`
+				UntilNS   int64  `json:"until_ns"`
+			}
+			if json.Unmarshal(e.Payload, &p) == nil && p.HarnessID == w.harnessID {
+				l := w.leases[e.TaskID]
+				l.Version, l.Until = e.Version, time.Unix(0, p.UntilNS)
+				w.leases[e.TaskID] = l
+			}
+		}
+		if e.Type == "TaskCompleted" {
 			delete(w.leases, e.TaskID)
-			delete(w.sessions, e.TaskID)
+			if session, active := w.sessions[e.TaskID]; active {
+				if w.herdr == nil {
+					delete(w.sessions, e.TaskID)
+				} else if err := (herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}).Kill(ctx, session); err != nil {
+					w.recordError(fmt.Errorf("close completed pane %s: %w", e.TaskID, err))
+					continue
+				} else {
+					_ = os.Remove(filepath.Join(session.Worktree, ".orchestra", "done"))
+					_ = os.Remove(filepath.Join(session.Worktree, ".orchestra"))
+					delete(w.sessions, e.TaskID)
+				}
+			}
+		}
+		if e.Type == "TaskPickupValidated" {
+			var p struct {
+				TransactionID string `json:"transaction_id"`
+			}
+			if json.Unmarshal(e.Payload, &p) == nil {
+				if tx := w.releases[e.TaskID]; tx.ID != "" && tx.ID == p.TransactionID {
+					tx.Phase, tx.UpdatedAt = "pickup_validated", time.Now().UTC()
+					w.releases[e.TaskID] = tx
+				}
+			}
+			if l, ok := w.leases[e.TaskID]; ok {
+				l.Version = e.Version
+				w.leases[e.TaskID] = l
+			}
+		}
+		if e.Type == "TaskReleased" || e.Type == "TaskFailed" || e.Type == "TaskBlocked" {
+			if e.Type == "TaskReleased" {
+				var p struct {
+					TransactionID string `json:"transaction_id"`
+				}
+				if json.Unmarshal(e.Payload, &p) == nil {
+					if tx := w.releases[e.TaskID]; tx.ID != "" && tx.ID == p.TransactionID && tx.Phase == "anchor_pushed" {
+						tx.Phase, tx.LastError, tx.UpdatedAt = "event_committed", "", time.Now().UTC()
+						w.releases[e.TaskID] = tx
+					}
+				}
+			}
+			delete(w.leases, e.TaskID)
+			// A releasing predecessor remains intentionally recoverable until
+			// TaskPickupValidated for its transaction. Do not erase its pane
+			// mapping merely because our own release event was replayed.
+			if _, releasing := w.releases[e.TaskID]; !releasing {
+				delete(w.sessions, e.TaskID)
+			}
 		}
 		if e.Seq > w.cursor {
 			w.cursor = e.Seq
@@ -422,6 +846,7 @@ func (w *worker) once(ctx context.Context) error {
 	if w.herdr != nil {
 		w.publishCaptures(ctx)
 		w.runCommands(ctx)
+		w.renewLeases(ctx)
 	}
 	w.releaseReady(ctx)
 	if err := w.save(); err != nil {
@@ -439,7 +864,7 @@ func (w *worker) reconcileLeases(ctx context.Context) error {
 	for _, task := range tasks {
 		w.tasks[task.ID] = task
 		if task.State == domain.StateLeased && task.Lease != nil && task.Lease.HarnessID == w.harnessID {
-			active[task.ID] = lease{HandoffRef: task.HandoffRef}
+			active[task.ID] = lease{HandoffRef: task.HandoffRef, TransactionID: task.ReleaseTransaction, AnchorSHA: task.ReleaseAnchor, Version: task.Version, Until: task.Lease.Until}
 		}
 	}
 	for taskID := range w.leases {
@@ -480,7 +905,53 @@ func main() {
 	if v, err := strconv.ParseFloat(os.Getenv("ORCHESTRA_OCCUPANCY_HARD"), 64); err == nil && v > 0 && v < 1 {
 		hard = v
 	}
-	w := &worker{api: federation.Client{BaseURL: required("ORCHESTRA_URL"), WorkerID: id, Token: token, AdmitToken: os.Getenv("ORCHESTRA_FEDERATION_ADMIT_TOKEN")}, harnessID: required("ORCHESTRA_WORKER_HERDR_ID"), harness: required("ORCHESTRA_WORKER_HARNESS"), repo: required("ORCHESTRA_REPO"), root: required("ORCHESTRA_WORKTREE_ROOT"), remote: required("ORCHESTRA_GIT_REMOTE"), tasks: map[string]domain.Task{}, sessions: map[string]herdr.Session{}, leases: map[string]lease{}, statePath: os.Getenv("ORCHESTRA_WORKER_STATE"), hard: hard, registration: federation.Worker{ID: id, Address: os.Getenv("ORCHESTRA_WORKER_ADDRESS"), Capacity: 1}}
+	projects := map[string]projectConfig{}
+	if path := os.Getenv("ORCHESTRA_WORKER_PROJECT_CONFIG_FILE"); path != "" {
+		b, err := os.ReadFile(path)
+		if err != nil {
+			log.Fatalf("read ORCHESTRA_WORKER_PROJECT_CONFIG_FILE: %v", err)
+		}
+		if err := json.Unmarshal(b, &projects); err != nil {
+			log.Fatalf("parse ORCHESTRA_WORKER_PROJECT_CONFIG_FILE: %v", err)
+		}
+	} else {
+		for _, project := range strings.Split(os.Getenv("ORCHESTRA_WORKER_PROJECTS"), ",") {
+			if project = strings.TrimSpace(project); project != "" {
+				projects[project] = projectConfig{Repo: required("ORCHESTRA_REPO"), Root: required("ORCHESTRA_WORKTREE_ROOT"), Remote: required("ORCHESTRA_GIT_REMOTE")}
+			}
+		}
+	}
+	if len(projects) == 0 {
+		// Existing deployments may be upgraded before their protected systemd
+		// environment is amended. Stay observable and fail closed in that
+		// interval: an empty declaration makes this worker ineligible for all
+		// new leases instead of turning a configuration rollout into a crash
+		// loop or treating its legacy global checkout as every project.
+		log.Printf("no ORCHESTRA_WORKER_PROJECT_CONFIG_FILE/ORCHESTRA_WORKER_PROJECTS; registering with no supported projects")
+	}
+	for project, config := range projects {
+		if config.Repo == "" || config.Root == "" || config.Remote == "" {
+			log.Fatalf("project %q requires repo, worktree_root, and remote", project)
+		}
+	}
+	supported := make([]string, 0, len(projects))
+	for project := range projects {
+		supported = append(supported, project)
+	}
+	sort.Strings(supported)
+	repo, root, remote := required("ORCHESTRA_REPO"), required("ORCHESTRA_WORKTREE_ROOT"), required("ORCHESTRA_GIT_REMOTE")
+	if len(supported) > 0 {
+		first := projects[supported[0]]
+		repo, root, remote = first.Repo, first.Root, first.Remote
+	}
+	soft, window := .55, int64(200000)
+	if v, err := strconv.ParseFloat(os.Getenv("ORCHESTRA_OCCUPANCY_SOFT"), 64); err == nil && v > 0 && v < hard {
+		soft = v
+	}
+	if v, err := strconv.ParseInt(os.Getenv("ORCHESTRA_CONTEXT_WINDOW"), 10, 64); err == nil && v > 0 {
+		window = v
+	}
+	w := &worker{api: federation.Client{BaseURL: required("ORCHESTRA_URL"), WorkerID: id, Token: token, AdmitToken: os.Getenv("ORCHESTRA_FEDERATION_ADMIT_TOKEN")}, harnessID: required("ORCHESTRA_WORKER_HERDR_ID"), harness: required("ORCHESTRA_WORKER_HARNESS"), repo: repo, root: root, remote: remote, projects: projects, tasks: map[string]domain.Task{}, sessions: map[string]herdr.Session{}, leases: map[string]lease{}, releases: map[string]releaseTransaction{}, statePath: os.Getenv("ORCHESTRA_WORKER_STATE"), hard: hard, soft: soft, window: window, registration: federation.Worker{ID: id, Address: os.Getenv("ORCHESTRA_WORKER_ADDRESS"), Capacity: 1, SupportedProjects: supported, Build: buildinfo.Current()}}
 	if w.statePath == "" {
 		w.statePath = filepath.Join(w.root, ".orchestra-worker-state.json")
 	}
@@ -2,7 +2,6 @@ package main

 import (
 	"context"
-	"crypto/subtle"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -12,6 +11,7 @@ import (
 	"net/http"
 	"orchestra/internal/admin"
 	"orchestra/internal/authz"
+	"orchestra/internal/buildinfo"
 	"orchestra/internal/delivery"
 	"orchestra/internal/domain"
 	"orchestra/internal/federation"
@@ -83,6 +83,14 @@ func remoteHerdrAddresses(rr registry.Registry, localMachine string) map[string]
 	return remote
 }

+// coordinatorOwnsHerdr identifies the only herdr sockets the coordinator may
+// probe or adapt. In federation mode a remote pane belongs to its worker;
+// reaching into that machine would turn a worker-owned health signal back
+// into a misleading coordinator TCP result.
+func coordinatorOwnsHerdr(h registry.Herdr, localMachine string) bool {
+	return localMachine == "" || h.MachineID == localMachine
+}
+
 func (a federatedAvailability) Available(h registry.Herdr) bool {
 	if a.base != nil && !a.base.Available(h) {
 		return false
@@ -93,6 +101,16 @@ func (a federatedAvailability) Available(h registry.Herdr) bool {
 	return a.workers.Available(h.ID)
 }

+func (a federatedAvailability) Supports(h registry.Herdr, project string) bool {
+	// Locally-owned herdrs keep their static registry/project affinity. A
+	// remote worker must additionally prove it has a local checkout for the
+	// project before the router can offer it a lease.
+	if a.localMachine == "" || h.MachineID == a.localMachine {
+		return true
+	}
+	return a.workers.Supports(h.ID, project)
+}
+
 func validateLocalMachine(rr registry.Registry, localMachine string) error {
 	machines := rr.Machines()
 	if len(machines) <= 1 {
@@ -231,6 +249,10 @@ func main() {
 		if repo, root := os.Getenv("ORCHESTRA_REPO"), os.Getenv("ORCHESTRA_WORKTREE_ROOT"); repo != "" && root != "" {
 			adapters := map[string]herdr.Adapter{}
 			for _, h := range rr.Herdrs() {
+				if !coordinatorOwnsHerdr(h, localMachine) {
+					log.Printf("herdr %s is worker-owned on %s; coordinator probe skipped", h.ID, h.MachineID)
+					continue
+				}
 				address := herdrAddress(rr, h)
 				if address == "" {
 					continue
@@ -244,6 +266,7 @@ func main() {
 					log.Printf("herdr %s unavailable: %v", h.ID, err)
 					continue
 				}
+				log.Printf("herdr %s reachable at %s (protocol %s, harness %s)", h.ID, address, protocol, h.Harness)
 				switch h.Harness {
 				case "claude":
 					adapters[h.ID] = herdr.Claude(client, 200000, s)
@@ -267,7 +290,7 @@ func main() {
 			}
 			coordinator = &orchestrator.Coordinator{Store: s, StatePath: filepath.Join(dir, "runtime-sessions.json"), Worktrees: worktrees, Adapters: orchestrator.AdapterFactory{Herdrs: adapters}, LocalHerdr: func(id string) bool {
 				h, ok := rr.Herdr(id)
-				return ok && (localMachine == "" || h.MachineID == localMachine)
+				return ok && coordinatorOwnsHerdr(h, localMachine)
 			}}
 			rt.OnLease = func(e domain.Event) error {
 				// In federated mode the coordinator must never inspect a remote
@@ -301,17 +324,14 @@ func main() {
 	}
 	mux := http.NewServeMux()
 	// B18: the UI is a full control plane — it can create tasks, release or
-	// complete them, and inject approval keystrokes into live panes. Refuse
-	// to serve it unauthenticated rather than silently exposing that on
-	// whatever interface the listener binds to.
-	webToken := os.Getenv("ORCHESTRA_WEB_TOKEN")
-	if webToken == "" {
-		log.Fatal("ORCHESTRA_WEB_TOKEN must be set: it gates the web UI's task, lifecycle and approval controls")
+	// complete them, and inject approval keystrokes into live panes. It has
+	// one explicit operator identity and is never enabled by a missing env var.
+	webCredentials := authz.WebCredentials{Username: os.Getenv("ORCHESTRA_WEB_USERNAME"), PasswordHash: os.Getenv("ORCHESTRA_WEB_PASSWORD_HASH")}
+	if err := webCredentials.Validate(); err != nil {
+		log.Fatalf("web login configuration: %v", err)
 	}
 	sessions := &authz.Sessions{}
-	// A browser cannot put a Bearer token on a document load, so it trades
-	// the token once for an HttpOnly cookie. Same credential, presentable
-	// form; no new authority is created here.
+	// A browser login exchanges verified credentials for an HttpOnly cookie.
 	mux.HandleFunc("/v1/ui/session", func(w http.ResponseWriter, r *http.Request) {
 		if r.Method == http.MethodDelete {
 			// Expire the browser credential even if it is already absent or stale.
@@ -329,15 +349,13 @@ func main() {
 			return
 		}
 		var body struct {
-			Token string `json:"token"`
+			Username string `json:"username"`
+			Password string `json:"password"`
 		}
-		_ = json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&body)
-		supplied := body.Token
-		if supplied == "" {
-			supplied = strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
-		}
-		if subtle.ConstantTimeCompare([]byte(supplied), []byte(webToken)) != 1 {
-			http.Error(w, "invalid token", http.StatusUnauthorized)
+		decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096))
+		decoder.DisallowUnknownFields()
+		if err := decoder.Decode(&body); err != nil || !webCredentials.Authenticate(body.Username, body.Password) {
+			http.Error(w, "invalid credentials", http.StatusUnauthorized)
 			return
 		}
 		v, err := sessions.Issue()
@@ -414,6 +432,17 @@ func main() {
 			http.Error(w, "invalid json", 400)
 			return
 		}
+		// Make the selected project's deterministic gate part of the immutable
+		// task contract before any worker can create TASK.md. A caller may
+		// override it only when it has deliberately supplied a task-specific
+		// gate; routing never asks a harness to choose one.
+		if _, set := p["quality_gate"]; !set {
+			if projectID, _ := p["project"].(string); projectID != "" {
+				if project, ok := rr.Project(projectID); ok && project.QualityGate != "" {
+					p["quality_gate"] = project.QualityGate
+				}
+			}
+		}
 		b, _ := json.Marshal(p)
 		e := domain.Event{ID: id(), Type: "TaskCreated", TaskID: id(), Version: 1, Payload: b, Surface: string(surface(r))}
 		if err := s.Append(e); err != nil {
@@ -665,7 +694,7 @@ func main() {
 		}
 		json.NewEncoder(w).Encode(out)
 	})
-	adminServer := &admin.Server{Store: s, RouterReady: rt != nil, Providers: providerHealth, Probes: map[string]admin.ProbeFunc{
+	adminServer := &admin.Server{Store: s, RouterReady: rt != nil, Build: buildinfo.Current(), Providers: providerHealth, Probes: map[string]admin.ProbeFunc{
 		"router": func() (bool, string) { return rt != nil, "configured router" },
 		"gitea": func() (bool, string) {
 			configured := os.Getenv("ORCHESTRA_GITEA_URL") != "" || os.Getenv("ORCHESTRA_GITEA_CONFIG") != ""
@@ -1033,7 +1062,7 @@ func main() {
 		w.WriteHeader(http.StatusNoContent)
 	})
 	mux.HandleFunc("/v1/federation/workers/", func(w http.ResponseWriter, r *http.Request) {
-		if r.Method != http.MethodPost || (!strings.HasSuffix(r.URL.Path, "/heartbeat") && !strings.HasSuffix(r.URL.Path, "/handoff") && !strings.HasSuffix(r.URL.Path, "/complete") && !strings.HasSuffix(r.URL.Path, "/captures")) {
+		if r.Method != http.MethodPost || (!strings.HasSuffix(r.URL.Path, "/heartbeat") && !strings.HasSuffix(r.URL.Path, "/renew") && !strings.HasSuffix(r.URL.Path, "/handoff") && !strings.HasSuffix(r.URL.Path, "/complete") && !strings.HasSuffix(r.URL.Path, "/captures")) {
 			http.Error(w, "not found", 404)
 			return
 		}
@@ -1083,10 +1112,18 @@ func main() {
 			return
 		}
 		var b struct {
-			TaskID     string `json:"task_id"`
-			TTLSeconds int    `json:"ttl_seconds"`
-			HandoffRef string `json:"handoff_ref"`
-			AnchorSHA  string `json:"anchor_sha"`
+			TaskID          string                 `json:"task_id"`
+			TTLSeconds      int                    `json:"ttl_seconds"`
+			ExpectedVersion int                    `json:"expected_version"`
+			HandoffRef      string                 `json:"handoff_ref"`
+			AnchorSHA       string                 `json:"anchor_sha"`
+			TransactionID   string                 `json:"transaction_id"`
+			LeaseVersion    int                    `json:"lease_version"`
+			ResultSHA       string                 `json:"result_sha"`
+			Branch          string                 `json:"branch"`
+			Remote          string                 `json:"remote"`
+			Receipt         map[string]any         `json:"receipt"`
+			SessionEvidence domain.SessionEvidence `json:"session_evidence"`
 		}
 		if json.NewDecoder(r.Body).Decode(&b) != nil || b.TaskID == "" {
 			http.Error(w, "invalid lease body", 400)
@@ -1098,6 +1135,13 @@ func main() {
 			return
 		}
 		ownedLease := t.State == domain.StateLeased && t.Lease != nil && t.Lease.HarnessID == parts[3]
+		// A response can be lost after the append/fsync. Retrying the exact
+		// release transaction is therefore a successful no-op, never a second
+		// TaskReleased event and never a reason to discard the predecessor.
+		if strings.HasSuffix(r.URL.Path, "/handoff") && t.State == domain.StateQueued && b.TransactionID != "" && b.TransactionID == t.ReleaseTransaction && b.HandoffRef == t.HandoffRef && b.AnchorSHA == t.ReleaseAnchor {
+			w.WriteHeader(http.StatusNoContent)
+			return
+		}
 		// A prompt timeout can block the coordinator after herdr already
 		// accepted the request. If that same authenticated worker later reports
 		// a durable completion, reconcile it rather than preserving a known
@@ -1107,17 +1151,73 @@ func main() {
 			http.Error(w, "lease not owned", 409)
 			return
 		}
+		if strings.HasSuffix(r.URL.Path, "/complete") && b.ExpectedVersion != t.Version {
+			http.Error(w, "lease version conflict", http.StatusConflict)
+			return
+		}
+		if strings.HasSuffix(r.URL.Path, "/renew") {
+			ttl := b.TTLSeconds
+			if ttl == 0 {
+				ttl = int((30 * time.Minute).Seconds())
+			}
+			e, err := s.RenewLease(b.TaskID, parts[3], b.ExpectedVersion, time.Duration(ttl)*time.Second)
+			if err != nil {
+				http.Error(w, err.Error(), http.StatusConflict)
+				return
+			}
+			json.NewEncoder(w).Encode(e)
+			return
+		}
+		if strings.HasSuffix(r.URL.Path, "/pickup") {
+			if !ownedLease || b.TransactionID == "" || b.TransactionID != t.ReleaseTransaction || b.HandoffRef != t.HandoffRef || b.AnchorSHA != t.ReleaseAnchor {
+				http.Error(w, "pickup does not match active release", http.StatusConflict)
+				return
+			}
+			if t.PickupTransaction == b.TransactionID && t.PickupLeaseVersion == b.LeaseVersion {
+				w.WriteHeader(http.StatusNoContent)
+				return
+			}
+			if b.LeaseVersion != t.Version {
+				http.Error(w, "lease version conflict", http.StatusConflict)
+				return
+			}
+			p, _ := json.Marshal(map[string]any{"transaction_id": b.TransactionID, "handoff_ref": b.HandoffRef, "anchor_sha": b.AnchorSHA, "harness_id": parts[3], "lease_version": b.LeaseVersion, "expected_version": t.Version, "session_evidence": b.SessionEvidence})
+			e := domain.Event{ID: id(), Type: "TaskPickupValidated", TaskID: b.TaskID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
+			if err := s.Append(e); err != nil {
+				http.Error(w, err.Error(), http.StatusConflict)
+				return
+			}
+			json.NewEncoder(w).Encode(e)
+			return
+		}
 		if strings.HasSuffix(r.URL.Path, "/complete") {
 			if b.HandoffRef == "" {
 				http.Error(w, "report_ref required", 400)
 				return
 			}
-			p, _ := json.Marshal(map[string]any{"report_ref": b.HandoffRef, "receipt": map[string]any{"harness_id": parts[3], "consumed": 0}})
+			if len(b.ResultSHA) != 40 || b.Branch == "" || b.Remote == "" {
+				http.Error(w, "verified result_sha, branch, and remote required", 400)
+				return
+			}
+			if b.Receipt == nil {
+				b.Receipt = map[string]any{}
+			}
+			b.Receipt["harness_id"] = parts[3]
+			if _, ok := b.Receipt["consumed"]; !ok {
+				b.Receipt["consumed"] = 0
+			}
+			p, _ := json.Marshal(map[string]any{"report_ref": b.HandoffRef, "receipt": b.Receipt, "result_sha": b.ResultSHA, "branch": b.Branch, "remote": b.Remote, "session_evidence": b.SessionEvidence})
 			e := domain.Event{ID: id(), Type: "TaskCompleted", TaskID: b.TaskID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
 			if err := s.Append(e); err != nil {
 				http.Error(w, err.Error(), 409)
 				return
 			}
+			if consumed, ok := b.Receipt["consumed"].(float64); ok && consumed > 0 {
+				qp, _ := json.Marshal(map[string]any{"harness_id": parts[3], "consumed": consumed})
+				if err := s.Append(domain.Event{ID: id(), Type: "QuotaReported", TaskID: "system", Payload: qp, Surface: string(authz.System)}); err != nil {
+					log.Printf("federated quota report %s: %v", b.TaskID, err)
+				}
+			}
 			json.NewEncoder(w).Encode(e)
 			return
 		}
@@ -1129,7 +1229,11 @@ func main() {
 			http.Error(w, "anchor_sha required", 400)
 			return
 		}
-		p, _ := json.Marshal(map[string]any{"handoff_ref": b.HandoffRef, "harness_id": parts[3], "anchor_sha": b.AnchorSHA})
+		if b.TransactionID == "" || b.ExpectedVersion != t.Version {
+			http.Error(w, "release transaction and current lease version required", http.StatusConflict)
+			return
+		}
+		p, _ := json.Marshal(map[string]any{"handoff_ref": b.HandoffRef, "harness_id": parts[3], "anchor_sha": b.AnchorSHA, "transaction_id": b.TransactionID, "expected_version": t.Version, "session_evidence": b.SessionEvidence})
 		e := domain.Event{ID: id(), Type: "TaskReleased", TaskID: b.TaskID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
 		if err := s.Append(e); err != nil {
 			http.Error(w, err.Error(), 409)
@@ -1257,7 +1361,7 @@ func main() {
 	}
 	log.Println("orchestra listening on :" + port)
 	tokens := map[authz.Surface]string{
-		authz.TUI: os.Getenv("ORCHESTRA_TUI_TOKEN"), authz.Web: os.Getenv("ORCHESTRA_WEB_TOKEN"),
+		authz.TUI: os.Getenv("ORCHESTRA_TUI_TOKEN"),
 		authz.MCP: os.Getenv("ORCHESTRA_MCP_TOKEN"), authz.Maven: os.Getenv("ORCHESTRA_MAVEN_TOKEN"),
 		// S12: this is the credential callers present *to* Orchestra on the
 		// ntfy surface. ORCHESTRA_NTFY_TOKEN is a different secret entirely —