Harden lease lifecycle durability

cmd/orchestra-password/main.go

@@ -0,0 +1,27 @@
+// orchestra-password prints a bcrypt hash suitable for ORCHESTRA_WEB_PASSWORD_HASH.
+package main
+
+import (
+	"fmt"
+	"os"
+	"syscall"
+
+	"golang.org/x/crypto/bcrypt"
+	"golang.org/x/term"
+)
+
+func main() {
+	fmt.Fprint(os.Stderr, "Password: ")
+	password, err := term.ReadPassword(int(syscall.Stdin))
+	fmt.Fprintln(os.Stderr)
+	if err != nil || len(password) == 0 {
+		fmt.Fprintln(os.Stderr, "password is required")
+		os.Exit(1)
+	}
+	hash, err := bcrypt.GenerateFromPassword(password, bcrypt.DefaultCost)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "hash password:", err)
+		os.Exit(1)
+	}
+	fmt.Println(string(hash))
+}

cmd/orchestra-worker/main.go

@@ -36,6 +36,7 @@ type worker struct {
 	sessions                               map[string]herdr.Session
 	leases                                 map[string]lease
 	releases                               map[string]releaseTransaction
+	quarantined                            map[string]bool
 	statePath                              string
 	hard                                   float64
 	registration                           federation.Worker
@@ -79,6 +80,7 @@ func (w *worker) health(ctx context.Context) federation.WorkerHealth {
 }
 
 type lease struct {
+	Epoch              string    `json:"epoch"`
 	HandoffRef         string    `json:"handoff_ref,omitempty"`
 	TransactionID      string    `json:"transaction_id,omitempty"`
 	AnchorSHA          string    `json:"anchor_sha,omitempty"`
@@ -117,24 +119,29 @@ type completionEvidence struct {
 	CompletedAt time.Time `json:"completed_at"`
 }
 type workerState struct {
-	Cursor   uint64                        `json:"cursor"`
-	Sessions map[string]herdr.Session      `json:"sessions"`
-	Tasks    map[string]domain.Task        `json:"tasks"`
-	Leases   map[string]lease              `json:"leases"`
-	Releases map[string]releaseTransaction `json:"releases"`
+	Cursor      uint64                        `json:"cursor"`
+	Sessions    map[string]herdr.Session      `json:"sessions"`
+	Tasks       map[string]domain.Task        `json:"tasks"`
+	Leases      map[string]lease              `json:"leases"`
+	Releases    map[string]releaseTransaction `json:"releases"`
+	Quarantined map[string]bool               `json:"quarantined,omitempty"`
 }
 
-func (w *worker) load() {
+func (w *worker) load() error {
 	b, e := os.ReadFile(w.statePath)
 	if e == nil {
 		var s workerState
-		if json.Unmarshal(b, &s) == nil {
-			w.cursor = s.Cursor
-			w.sessions = s.Sessions
-			w.tasks = s.Tasks
-			w.leases = s.Leases
-			w.releases = s.Releases
+		if err := json.Unmarshal(b, &s); err != nil {
+			return fmt.Errorf("corrupt worker state %s: %w", w.statePath, err)
 		}
+		w.cursor = s.Cursor
+		w.sessions = s.Sessions
+		w.tasks = s.Tasks
+		w.leases = s.Leases
+		w.releases = s.Releases
+		w.quarantined = s.Quarantined
+	} else if !errors.Is(e, os.ErrNotExist) {
+		return fmt.Errorf("read worker state %s: %w", w.statePath, e)
 	}
 	if w.sessions == nil {
 		w.sessions = map[string]herdr.Session{}
@@ -148,16 +155,68 @@ func (w *worker) load() {
 	if w.releases == nil {
 		w.releases = map[string]releaseTransaction{}
 	}
+	if w.quarantined == nil {
+		w.quarantined = map[string]bool{}
+	}
+	return nil
 }
 func (w *worker) save() error {
-	b, e := json.Marshal(workerState{Cursor: w.cursor, Sessions: w.sessions, Tasks: w.tasks, Leases: w.leases, Releases: w.releases})
+	b, e := json.Marshal(workerState{Cursor: w.cursor, Sessions: w.sessions, Tasks: w.tasks, Leases: w.leases, Releases: w.releases, Quarantined: w.quarantined})
 	if e != nil {
 		return e
 	}
 	if e := os.MkdirAll(filepath.Dir(w.statePath), 0700); e != nil {
 		return e
 	}
-	return os.WriteFile(w.statePath, b, 0600)
+	tmp := w.statePath + ".tmp"
+	f, e := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
+	if e != nil {
+		return e
+	}
+	if _, e = f.Write(b); e == nil {
+		e = f.Sync()
+	}
+	if closeErr := f.Close(); e == nil {
+		e = closeErr
+	}
+	if e != nil {
+		_ = os.Remove(tmp)
+		return e
+	}
+	if e = os.Rename(tmp, w.statePath); e != nil {
+		return e
+	}
+	dir, e := os.Open(filepath.Dir(w.statePath))
+	if e != nil {
+		return e
+	}
+	defer dir.Close()
+	return dir.Sync()
+}
+
+// quarantine stops a pane before its lost lease mapping can be forgotten.
+// A failed close remains durable and is retried; it is never treated as a
+// harmless cleanup error while the old harness could still be working.
+func (w *worker) quarantine(ctx context.Context, taskID string, s herdr.Session) {
+	if w.herdr == nil {
+		w.quarantined[taskID] = true
+		return
+	}
+	if err := (herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}).Kill(ctx, s); err != nil {
+		w.quarantined[taskID] = true
+		w.recordError(fmt.Errorf("quarantine %s: %w", taskID, err))
+		return
+	}
+	delete(w.sessions, taskID)
+	delete(w.quarantined, taskID)
+}
+
+func (w *worker) retryQuarantines(ctx context.Context) {
+	for taskID := range w.quarantined {
+		if s, ok := w.sessions[taskID]; ok {
+			w.quarantine(ctx, taskID, s)
+		}
+	}
 }
 
 type artifactCAS struct{ api federation.Client }
@@ -286,6 +345,9 @@ func taskHash(t domain.Task) string { b := continuity.RenderTaskFile(t); return
 
 func (w *worker) releaseReady(ctx context.Context) {
 	for id, s := range w.sessions {
+		if w.quarantined[id] {
+			continue
+		}
 		if l := w.leases[id]; l.HandoffRef != "" && !l.PickupAcknowledged {
 			if err := w.ackPickup(ctx, id, s); err != nil {
 				w.recordError(err)
@@ -306,7 +368,7 @@ func (w *worker) releaseReady(ctx context.Context) {
 				log.Printf("upload completion %s: %v", id, err)
 				continue
 			}
-			if err = w.api.Complete(ctx, id, ref, evidence.ResultSHA, evidence.Branch, evidence.Remote, w.leases[id].Version, w.usageReceipt(s), w.sessionEvidence(ctx, id, s)); err != nil {
+			if err = w.api.Complete(ctx, id, ref, evidence.ResultSHA, evidence.Branch, evidence.Remote, w.leases[id].Epoch, w.leases[id].Version, w.usageReceipt(s), w.sessionEvidence(ctx, id, s)); err != nil {
 				w.recordError(fmt.Errorf("complete %s: %w", id, err))
 				log.Printf("complete %s: %v", id, err)
 				continue
@@ -439,7 +501,8 @@ func (w *worker) advanceRelease(ctx context.Context, id string, s herdr.Session)
 		_ = w.save()
 	}
 	if tx.Phase == "anchor_pushed" {
-		if err := w.api.Release(ctx, id, tx.Ref, tx.AnchorSHA, tx.ID, tx.LeaseVersion, w.sessionEvidence(ctx, id, s)); err != nil {
+		l := w.leases[id]
+		if err := w.api.Release(ctx, id, tx.Ref, tx.AnchorSHA, tx.ID, l.Epoch, tx.LeaseVersion, w.sessionEvidence(ctx, id, s)); err != nil {
 			tx.LastError, tx.UpdatedAt = err.Error(), time.Now().UTC()
 			w.releases[id] = tx
 			_ = w.save()
@@ -491,7 +554,7 @@ func (w *worker) ackPickup(ctx context.Context, id string, s herdr.Session) erro
 	if l.PickupAcknowledged {
 		return nil
 	}
-	if err := w.api.Pickup(ctx, id, l.HandoffRef, l.AnchorSHA, l.TransactionID, l.Version, w.sessionEvidence(ctx, id, s)); err != nil {
+	if err := w.api.Pickup(ctx, id, l.HandoffRef, l.AnchorSHA, l.TransactionID, l.Epoch, l.Version, w.sessionEvidence(ctx, id, s)); err != nil {
 		return fmt.Errorf("pickup %s acknowledgement: %w", id, err)
 	}
 	l.PickupAcknowledged = true
@@ -617,7 +680,7 @@ func (w *worker) renewLeases(ctx context.Context) {
 			w.recordError(fmt.Errorf("validate lease %s: %w", taskID, err))
 			continue
 		}
-		if err := w.api.Renew(ctx, taskID, l.Version, int((30 * time.Minute).Seconds())); err != nil {
+		if err := w.api.Renew(ctx, taskID, l.Epoch, l.Version, int((30 * time.Minute).Seconds())); err != nil {
 			w.recordError(fmt.Errorf("renew lease %s: %w", taskID, err))
 			log.Printf("renew lease %s: %v", taskID, err)
 		} else {
@@ -726,6 +789,7 @@ func (w *worker) once(ctx context.Context) error {
 		if e.Type == "TaskLeased" {
 			var p struct {
 				HarnessID     string `json:"harness_id"`
+				Epoch         string `json:"lease_epoch"`
 				HandoffRef    string `json:"handoff_ref"`
 				TransactionID string `json:"transaction_id"`
 				AnchorSHA     string `json:"anchor_sha"`
@@ -735,17 +799,18 @@ func (w *worker) once(ctx context.Context) error {
 					UntilNS int64 `json:"until_ns"`
 				}
 				_ = json.Unmarshal(e.Payload, &until)
-				w.leases[e.TaskID] = lease{HandoffRef: p.HandoffRef, TransactionID: p.TransactionID, AnchorSHA: p.AnchorSHA, Version: e.Version, Until: time.Unix(0, until.UntilNS)}
+				w.leases[e.TaskID] = lease{Epoch: p.Epoch, HandoffRef: p.HandoffRef, TransactionID: p.TransactionID, AnchorSHA: p.AnchorSHA, Version: e.Version, Until: time.Unix(0, until.UntilNS)}
 			}
 		}
 		if e.Type == "TaskLeaseRenewed" {
 			var p struct {
 				HarnessID string `json:"harness_id"`
+				Epoch     string `json:"lease_epoch"`
 				UntilNS   int64  `json:"until_ns"`
 			}
 			if json.Unmarshal(e.Payload, &p) == nil && p.HarnessID == w.harnessID {
 				l := w.leases[e.TaskID]
-				l.Version, l.Until = e.Version, time.Unix(0, p.UntilNS)
+				l.Version, l.Epoch, l.Until = e.Version, p.Epoch, time.Unix(0, p.UntilNS)
 				w.leases[e.TaskID] = l
 			}
 		}
@@ -755,6 +820,7 @@ func (w *worker) once(ctx context.Context) error {
 				if w.herdr == nil {
 					delete(w.sessions, e.TaskID)
 				} else if err := (herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}).Kill(ctx, session); err != nil {
+					w.quarantined[e.TaskID] = true
 					w.recordError(fmt.Errorf("close completed pane %s: %w", e.TaskID, err))
 					continue
 				} else {
@@ -796,7 +862,9 @@ func (w *worker) once(ctx context.Context) error {
 			// TaskPickupValidated for its transaction. Do not erase its pane
 			// mapping merely because our own release event was replayed.
 			if _, releasing := w.releases[e.TaskID]; !releasing {
-				delete(w.sessions, e.TaskID)
+				if session, active := w.sessions[e.TaskID]; active {
+					w.quarantine(ctx, e.TaskID, session)
+				}
 			}
 		}
 		if e.Seq > w.cursor {
@@ -848,6 +916,7 @@ func (w *worker) once(ctx context.Context) error {
 		w.runCommands(ctx)
 		w.renewLeases(ctx)
 	}
+	w.retryQuarantines(ctx)
 	w.releaseReady(ctx)
 	if err := w.save(); err != nil {
 		return err
@@ -864,7 +933,7 @@ func (w *worker) reconcileLeases(ctx context.Context) error {
 	for _, task := range tasks {
 		w.tasks[task.ID] = task
 		if task.State == domain.StateLeased && task.Lease != nil && task.Lease.HarnessID == w.harnessID {
-			active[task.ID] = lease{HandoffRef: task.HandoffRef, TransactionID: task.ReleaseTransaction, AnchorSHA: task.ReleaseAnchor, Version: task.Version, Until: task.Lease.Until}
+			active[task.ID] = lease{Epoch: task.Lease.Epoch, HandoffRef: task.HandoffRef, TransactionID: task.ReleaseTransaction, AnchorSHA: task.ReleaseAnchor, Version: task.Version, Until: task.Lease.Until}
 		}
 	}
 	for taskID := range w.leases {
@@ -951,11 +1020,13 @@ func main() {
 	if v, err := strconv.ParseInt(os.Getenv("ORCHESTRA_CONTEXT_WINDOW"), 10, 64); err == nil && v > 0 {
 		window = v
 	}
-	w := &worker{api: federation.Client{BaseURL: required("ORCHESTRA_URL"), WorkerID: id, Token: token, AdmitToken: os.Getenv("ORCHESTRA_FEDERATION_ADMIT_TOKEN")}, harnessID: required("ORCHESTRA_WORKER_HERDR_ID"), harness: required("ORCHESTRA_WORKER_HARNESS"), repo: repo, root: root, remote: remote, projects: projects, tasks: map[string]domain.Task{}, sessions: map[string]herdr.Session{}, leases: map[string]lease{}, releases: map[string]releaseTransaction{}, statePath: os.Getenv("ORCHESTRA_WORKER_STATE"), hard: hard, soft: soft, window: window, registration: federation.Worker{ID: id, Address: os.Getenv("ORCHESTRA_WORKER_ADDRESS"), Capacity: 1, SupportedProjects: supported, Build: buildinfo.Current()}}
+	w := &worker{api: federation.Client{BaseURL: required("ORCHESTRA_URL"), WorkerID: id, Token: token, AdmitToken: os.Getenv("ORCHESTRA_FEDERATION_ADMIT_TOKEN")}, harnessID: required("ORCHESTRA_WORKER_HERDR_ID"), harness: required("ORCHESTRA_WORKER_HARNESS"), repo: repo, root: root, remote: remote, projects: projects, tasks: map[string]domain.Task{}, sessions: map[string]herdr.Session{}, leases: map[string]lease{}, releases: map[string]releaseTransaction{}, quarantined: map[string]bool{}, statePath: os.Getenv("ORCHESTRA_WORKER_STATE"), hard: hard, soft: soft, window: window, registration: federation.Worker{ID: id, Address: os.Getenv("ORCHESTRA_WORKER_ADDRESS"), Capacity: 1, SupportedProjects: supported, Build: buildinfo.Current()}}
 	if w.statePath == "" {
 		w.statePath = filepath.Join(w.root, ".orchestra-worker-state.json")
 	}
-	w.load()
+	if err := w.load(); err != nil {
+		log.Fatal(err)
+	}
 	w.herdr = herdr.New(required("ORCHESTRA_WORKER_HERDR"))
 	if id != w.harnessID {
 		log.Fatal("ORCHESTRA_WORKER_ID must equal ORCHESTRA_WORKER_HERDR_ID so leases and offline recovery have one owner")

cmd/orchestra-worker/main_test.go

@@ -11,10 +11,12 @@ import (
 	"orchestra/internal/domain"
 	"orchestra/internal/federation"
 	"orchestra/internal/herdr"
+	"orchestra/internal/orchestrator"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"reflect"
+	"strings"
 	"testing"
 	"time"
 )
@@ -38,6 +40,17 @@ func TestWorkerReregistersAfterCoordinatorForgetsIt(t *testing.T) {
 	}
 }
 
+func TestWorkerRefusesCorruptDurableState(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "state.json")
+	if err := os.WriteFile(path, []byte(`{"cursor":`), 0600); err != nil {
+		t.Fatal(err)
+	}
+	w := &worker{statePath: path}
+	if err := w.load(); err == nil {
+		t.Fatal("corrupt worker state was accepted")
+	}
+}
+
 func TestInitialReplayDoesNotResurrectReleasedLease(t *testing.T) {
 	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		switch r.URL.Path {
@@ -175,6 +188,55 @@ func TestWorkerStartsRouterIssuedLeaseInLocalGitWorktree(t *testing.T) {
 	}
 }
 
+func TestFinalizeRunsGateCommitsPushesAndVerifiesRemote(t *testing.T) {
+	remote := filepath.Join(t.TempDir(), "remote.git")
+	if out, err := exec.Command("git", "init", "--bare", remote).CombinedOutput(); err != nil {
+		t.Fatalf("remote: %v %s", err, out)
+	}
+	seed := t.TempDir()
+	for _, a := range [][]string{{"init", seed}, {"-C", seed, "config", "user.email", "t@t"}, {"-C", seed, "config", "user.name", "t"}, {"-C", seed, "commit", "--allow-empty", "-m", "init"}, {"-C", seed, "remote", "add", "origin", remote}, {"-C", seed, "push", "-u", "origin", "HEAD:master"}} {
+		if out, err := exec.Command("git", a...).CombinedOutput(); err != nil {
+			t.Fatalf("git %v: %v %s", a, err, out)
+		}
+	}
+	repo := filepath.Join(t.TempDir(), "repo")
+	if out, err := exec.Command("git", "clone", remote, repo).CombinedOutput(); err != nil {
+		t.Fatalf("clone: %v %s", err, out)
+	}
+	for _, a := range [][]string{{"-C", repo, "config", "user.email", "t@t"}, {"-C", repo, "config", "user.name", "t"}} {
+		if out, err := exec.Command("git", a...).CombinedOutput(); err != nil {
+			t.Fatalf("git %v: %v %s", a, err, out)
+		}
+	}
+	root := filepath.Join(t.TempDir(), "worktrees")
+	task := domain.Task{ID: "task", Source: "s", ExternalID: "delivery", Project: "p", QualityGate: "test -f result.txt"}
+	wt, err := (orchestrator.GitWorktrees{Repo: repo, Root: root}).Create(context.Background(), task)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(wt, "result.txt"), []byte("delivered"), 0644); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.MkdirAll(filepath.Join(wt, ".orchestra"), 0755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(wt, ".orchestra", "done"), nil, 0644); err != nil {
+		t.Fatal(err)
+	}
+	w := &worker{harnessID: "worker", harness: "opencode", repo: repo, root: root, remote: "origin", tasks: map[string]domain.Task{task.ID: task}}
+	evidence, err := w.finalize(context.Background(), task.ID, herdr.Session{PaneID: "pane", Worktree: wt, TaskFileSHA: taskHash(task)})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if evidence.ResultSHA == evidence.BaseSHA || evidence.Branch != "orchestra/task" || evidence.QualityGate != task.QualityGate {
+		t.Fatalf("unexpected evidence: %+v", evidence)
+	}
+	out, err := exec.Command("git", "-C", repo, "ls-remote", "origin", "refs/heads/orchestra/task").CombinedOutput()
+	if err != nil || !strings.HasPrefix(string(out), evidence.ResultSHA+"\t") {
+		t.Fatalf("remote result=%q err=%v want %s", out, err, evidence.ResultSHA)
+	}
+}
+
 func TestWorkerApprovalCommandIsRevisionBoundAndAcknowledged(t *testing.T) {
 	ln, err := net.Listen("tcp", "127.0.0.1:0")
 	if err != nil {

cmd/orchestra/main.go

@@ -142,12 +142,14 @@ func (h harnessCompletion) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	}
 	var p struct {
 		TaskID         string `json:"task_id"`
+		WorkerID       string `json:"worker_id"`
+		LeaseEpoch     string `json:"lease_epoch"`
 		Harness        string `json:"harness"`
 		TranscriptPath string `json:"transcript_path"`
 		Report         string `json:"report"`
 	}
-	if json.NewDecoder(r.Body).Decode(&p) != nil || p.TaskID == "" || p.Report == "" || p.TranscriptPath == "" {
-		http.Error(w, "task_id, transcript_path, and report are required", http.StatusBadRequest)
+	if json.NewDecoder(r.Body).Decode(&p) != nil || p.TaskID == "" || p.WorkerID == "" || p.LeaseEpoch == "" || p.Report == "" || p.TranscriptPath == "" {
+		http.Error(w, "task_id, worker_id, lease_epoch, transcript_path, and report are required", http.StatusBadRequest)
 		return
 	}
 	t, ok := h.store.Task(p.TaskID)
@@ -155,6 +157,10 @@ func (h harnessCompletion) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "task not found", http.StatusNotFound)
 		return
 	}
+	if t.State != domain.StateLeased || t.Lease == nil || t.Lease.HarnessID != p.WorkerID || t.Lease.Epoch != p.LeaseEpoch {
+		http.Error(w, "lease not owned", http.StatusConflict)
+		return
+	}
 	var usage herdr.Usage
 	var err error
 	switch p.Harness {
@@ -177,7 +183,7 @@ func (h harnessCompletion) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
-	payload, _ := json.Marshal(map[string]any{"report_ref": ref, "receipt": map[string]any{
+	payload, _ := json.Marshal(map[string]any{"report_ref": ref, "harness_id": p.WorkerID, "lease_epoch": p.LeaseEpoch, "expected_version": t.Version, "receipt": map[string]any{
 		"input_tokens": usage.Input, "cache_read_tokens": usage.CacheRead,
 		"cache_write_tokens": usage.CacheWrite, "output_tokens": usage.Output, "numerator": usage.Numerator(),
 	}})
@@ -381,17 +387,10 @@ func main() {
 		return err
 	}}.Handler())
 	mux.Handle("/", webui.Handler())
-	workers.OnOffline = func(w federation.Worker) {
-		for _, t := range s.Tasks() {
-			if t.State == domain.StateLeased && t.Lease != nil && t.Lease.HarnessID == w.ID {
-				p, _ := json.Marshal(map[string]any{"reason": "worker_offline", "harness_id": w.ID})
-				e := domain.Event{ID: id(), Type: "TaskReleased", TaskID: t.ID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
-				if err := s.Append(e); err == nil && rt != nil {
-					_, _ = rt.HandleEvent(e)
-				}
-			}
-		}
-	}
+	// A missed heartbeat is not relinquishment. Releasing here used to lease
+	// the same task to a successor while the old pane was still running. The
+	// authoritative lease timer performs the only automatic reassignment.
+	workers.OnOffline = func(w federation.Worker) { log.Printf("worker %s offline; retaining leases until expiry", w.ID) }
 	go func() {
 		ticker := time.NewTicker(15 * time.Second)
 		defer ticker.Stop()
@@ -560,13 +559,12 @@ func main() {
 	// same session-file assumption as CLIAdapter.Occupancy — rather than
 	// trusting a self-reported number.
 	harnessToken := os.Getenv("ORCHESTRA_HARNESS_TOKEN")
-	mux.Handle("/v1/harness/complete", harnessCompletion{store: s, token: harnessToken, route: func(e domain.Event) error {
-		if rt == nil {
-			return nil
-		}
-		_, err := rt.HandleEvent(e)
-		return err
-	}})
+	// The unaffiliated harness hook has no durable worker identity or fencing
+	// epoch, so it cannot safely mutate a leased task. Completion is accepted
+	// only through the authenticated federation worker endpoint below.
+	mux.HandleFunc("/v1/harness/complete", func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "legacy harness completion endpoint retired; use worker completion", http.StatusGone)
+	})
 	// /v1/harness/turn is the unified turn-decision endpoint (AUDIT.md Phase
 	// 2 items 1-2): the Face-B stop hook posts here on every ordinary turn
 	// boundary (report marker absent — /v1/harness/complete covers task
@@ -1062,7 +1060,7 @@ func main() {
 		w.WriteHeader(http.StatusNoContent)
 	})
 	mux.HandleFunc("/v1/federation/workers/", func(w http.ResponseWriter, r *http.Request) {
-		if r.Method != http.MethodPost || (!strings.HasSuffix(r.URL.Path, "/heartbeat") && !strings.HasSuffix(r.URL.Path, "/renew") && !strings.HasSuffix(r.URL.Path, "/handoff") && !strings.HasSuffix(r.URL.Path, "/complete") && !strings.HasSuffix(r.URL.Path, "/captures")) {
+		if r.Method != http.MethodPost || (!strings.HasSuffix(r.URL.Path, "/heartbeat") && !strings.HasSuffix(r.URL.Path, "/renew") && !strings.HasSuffix(r.URL.Path, "/handoff") && !strings.HasSuffix(r.URL.Path, "/pickup") && !strings.HasSuffix(r.URL.Path, "/complete") && !strings.HasSuffix(r.URL.Path, "/captures")) {
 			http.Error(w, "not found", 404)
 			return
 		}
@@ -1119,6 +1117,7 @@ func main() {
 			AnchorSHA       string                 `json:"anchor_sha"`
 			TransactionID   string                 `json:"transaction_id"`
 			LeaseVersion    int                    `json:"lease_version"`
+			LeaseEpoch      string                 `json:"lease_epoch"`
 			ResultSHA       string                 `json:"result_sha"`
 			Branch          string                 `json:"branch"`
 			Remote          string                 `json:"remote"`
@@ -1134,7 +1133,7 @@ func main() {
 			http.Error(w, "task not found", 404)
 			return
 		}
-		ownedLease := t.State == domain.StateLeased && t.Lease != nil && t.Lease.HarnessID == parts[3]
+		ownedLease := t.State == domain.StateLeased && t.Lease != nil && t.Lease.HarnessID == parts[3] && t.Lease.Epoch == b.LeaseEpoch
 		// A response can be lost after the append/fsync. Retrying the exact
 		// release transaction is therefore a successful no-op, never a second
 		// TaskReleased event and never a reason to discard the predecessor.
@@ -1142,12 +1141,7 @@ func main() {
 			w.WriteHeader(http.StatusNoContent)
 			return
 		}
-		// A prompt timeout can block the coordinator after herdr already
-		// accepted the request. If that same authenticated worker later reports
-		// a durable completion, reconcile it rather than preserving a known
-		// false blocked state. No other blocked task is admitted here.
-		recoverableBlocked := strings.HasSuffix(r.URL.Path, "/complete") && t.State == domain.StateBlocked && t.LastHarness == parts[3]
-		if !ownedLease && !recoverableBlocked {
+		if !ownedLease {
 			http.Error(w, "lease not owned", 409)
 			return
 		}
@@ -1160,7 +1154,7 @@ func main() {
 			if ttl == 0 {
 				ttl = int((30 * time.Minute).Seconds())
 			}
-			e, err := s.RenewLease(b.TaskID, parts[3], b.ExpectedVersion, time.Duration(ttl)*time.Second)
+			e, err := s.RenewLease(b.TaskID, parts[3], b.LeaseEpoch, b.ExpectedVersion, time.Duration(ttl)*time.Second)
 			if err != nil {
 				http.Error(w, err.Error(), http.StatusConflict)
 				return
@@ -1181,7 +1175,7 @@ func main() {
 				http.Error(w, "lease version conflict", http.StatusConflict)
 				return
 			}
-			p, _ := json.Marshal(map[string]any{"transaction_id": b.TransactionID, "handoff_ref": b.HandoffRef, "anchor_sha": b.AnchorSHA, "harness_id": parts[3], "lease_version": b.LeaseVersion, "expected_version": t.Version, "session_evidence": b.SessionEvidence})
+			p, _ := json.Marshal(map[string]any{"transaction_id": b.TransactionID, "handoff_ref": b.HandoffRef, "anchor_sha": b.AnchorSHA, "harness_id": parts[3], "lease_epoch": b.LeaseEpoch, "lease_version": b.LeaseVersion, "expected_version": t.Version, "session_evidence": b.SessionEvidence})
 			e := domain.Event{ID: id(), Type: "TaskPickupValidated", TaskID: b.TaskID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
 			if err := s.Append(e); err != nil {
 				http.Error(w, err.Error(), http.StatusConflict)
@@ -1206,7 +1200,7 @@ func main() {
 			if _, ok := b.Receipt["consumed"]; !ok {
 				b.Receipt["consumed"] = 0
 			}
-			p, _ := json.Marshal(map[string]any{"report_ref": b.HandoffRef, "receipt": b.Receipt, "result_sha": b.ResultSHA, "branch": b.Branch, "remote": b.Remote, "session_evidence": b.SessionEvidence})
+			p, _ := json.Marshal(map[string]any{"report_ref": b.HandoffRef, "receipt": b.Receipt, "result_sha": b.ResultSHA, "branch": b.Branch, "remote": b.Remote, "harness_id": parts[3], "lease_epoch": b.LeaseEpoch, "expected_version": t.Version, "session_evidence": b.SessionEvidence})
 			e := domain.Event{ID: id(), Type: "TaskCompleted", TaskID: b.TaskID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
 			if err := s.Append(e); err != nil {
 				http.Error(w, err.Error(), 409)
@@ -1233,7 +1227,7 @@ func main() {
 			http.Error(w, "release transaction and current lease version required", http.StatusConflict)
 			return
 		}
-		p, _ := json.Marshal(map[string]any{"handoff_ref": b.HandoffRef, "harness_id": parts[3], "anchor_sha": b.AnchorSHA, "transaction_id": b.TransactionID, "expected_version": t.Version, "session_evidence": b.SessionEvidence})
+		p, _ := json.Marshal(map[string]any{"handoff_ref": b.HandoffRef, "harness_id": parts[3], "lease_epoch": b.LeaseEpoch, "anchor_sha": b.AnchorSHA, "transaction_id": b.TransactionID, "expected_version": t.Version, "session_evidence": b.SessionEvidence})
 		e := domain.Event{ID: id(), Type: "TaskReleased", TaskID: b.TaskID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
 		if err := s.Append(e); err != nil {
 			http.Error(w, err.Error(), 409)

cmd/orchestra/main_test.go

@@ -41,6 +41,20 @@ func TestFederatedReachabilityDefersRemoteHerdrToWorkerHeartbeat(t *testing.T) {
 	}
 }
 
+func TestCoordinatorOwnsOnlyLocalHerdrInFederationMode(t *testing.T) {
+	local := registry.Herdr{ID: "homesrv-opencode", MachineID: "homesrv"}
+	remote := registry.Herdr{ID: "workpc-opencode", MachineID: "workpc"}
+	if !coordinatorOwnsHerdr(local, "homesrv") {
+		t.Fatal("coordinator does not own its local herdr")
+	}
+	if coordinatorOwnsHerdr(remote, "homesrv") {
+		t.Fatal("coordinator claimed a worker-owned remote herdr")
+	}
+	if !coordinatorOwnsHerdr(remote, "") {
+		t.Fatal("single-machine mode should retain legacy local ownership")
+	}
+}
+
 func TestHarnessCompletionBuildsReceiptAndQuotaFromTranscript(t *testing.T) {
 	s, err := store.Open(t.TempDir())
 	if err != nil {
@@ -56,7 +70,8 @@ func TestHarnessCompletionBuildsReceiptAndQuotaFromTranscript(t *testing.T) {
 	if err := os.WriteFile(transcript, []byte(`{"message":{"usage":{"input_tokens":100,"cache_read_input_tokens":20,"cache_creation_input_tokens":5,"output_tokens":7}}}`+"\n"), 0600); err != nil {
 		t.Fatal(err)
 	}
-	body, _ := json.Marshal(map[string]string{"task_id": "done", "transcript_path": transcript, "report": "# done"})
+	task, _ := s.Task("done")
+	body, _ := json.Marshal(map[string]string{"task_id": "done", "worker_id": "local-claude", "lease_epoch": task.Lease.Epoch, "transcript_path": transcript, "report": "# done"})
 	req := httptest.NewRequest(http.MethodPost, "/v1/harness/complete", bytes.NewReader(body))
 	res := httptest.NewRecorder()
 	harnessCompletion{store: s, token: "secret"}.ServeHTTP(res, req)