Bound the wait for a handoff nobody answers

F62. The rotation is agent-driven: the worker asks, and the agent must write
its handoff. When the agent never does, renewals stopped on the ordinary
progress gate, the lease expired, and the task lost an attempt with nothing on
record saying a handoff had ever been requested. Run 16 showed only "agent
status idle and pane unchanged", 34 times.

The request is now stamped, and the wait around it is bounded. While Orchestra
is explicitly waiting the lease renews, because a quiet pane is the answer the
agent was told to give. The request is re-sent once after four minutes, with
the reason it was first asked with. At ten minutes the worker nacks with
failure class handoff_unanswered, and the coordinator releases the task naming
that cause instead of letting the lease die as generic idleness.

The class is known to DebtClassForFailureClass, so a harness that ignores
handoff requests accumulates as its own debt item rather than hiding inside
lease_expired.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CVbaKucEYBjMqVeUgJUsc1

cmd/orchestra-worker/handoff_test.go

@@ -0,0 +1,110 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"orchestra/internal/domain"
+	"orchestra/internal/federation"
+	"orchestra/internal/herdr"
+)
+
+// F62. Run 16: the agent was asked to hand off, never wrote HANDOFF.md,
+// renewals stopped, and the lease died as ordinary idleness. Waiting is now
+// bounded: re-ask once, then give the task up with a class that says why.
+func TestUnansweredHandoffIsRetriedThenGivenUp(t *testing.T) {
+	var nack map[string]any
+	w, backend, _, done := phaseWorker(t, func(rw http.ResponseWriter, r *http.Request) {
+		if strings.HasSuffix(r.URL.Path, "/nack") {
+			_ = json.NewDecoder(r.Body).Decode(&nack)
+		}
+		rw.Write([]byte(`{}`))
+	})
+	defer done()
+	ctx := context.Background()
+	requested := func(ago time.Duration) herdr.Session {
+		s := w.sessions["task"]
+		s.HandoffRequested, s.HandoffReason = true, "phase_changed"
+		s.HandoffRequestedAt = time.Now().UTC().Add(-ago)
+		w.sessions["task"] = s
+		return s
+	}
+
+	// Still inside the answering window: nothing said, nothing given up.
+	if s, gaveUp := w.watchHandoff(ctx, "task", requested(time.Minute)); gaveUp || s.HandoffRetried {
+		t.Fatalf("gave up while still waiting: gaveUp=%v session=%+v", gaveUp, s)
+	}
+	if len(backend.prompts) != 0 {
+		t.Fatalf("re-asked too early: %q", backend.prompts)
+	}
+
+	// Past the retry point: asked again, exactly once.
+	s, gaveUp := w.watchHandoff(ctx, "task", requested(handoffRetryAfter+time.Minute))
+	if gaveUp || !s.HandoffRetried || len(backend.prompts) != 1 {
+		t.Fatalf("retry: gaveUp=%v retried=%v prompts=%q", gaveUp, s.HandoffRetried, backend.prompts)
+	}
+	if _, gaveUp = w.watchHandoff(ctx, "task", s); gaveUp || len(backend.prompts) != 1 {
+		t.Fatalf("re-asked every tick: %q", backend.prompts)
+	}
+
+	// Past the bound: a causal reclaim, and no lease left to renew.
+	s = requested(handoffAnswerTimeout + time.Second)
+	s.HandoffRetried = true
+	w.sessions["task"] = s
+	if _, gaveUp = w.watchHandoff(ctx, "task", s); !gaveUp {
+		t.Fatal("an unanswered handoff waited forever")
+	}
+	if nack["failure_class"] != "handoff_unanswered" {
+		t.Fatalf("nack = %+v", nack)
+	}
+	if detail, _ := nack["last_error"].(string); !strings.Contains(detail, "phase_changed") {
+		t.Fatalf("the reclaim does not name the request: %q", detail)
+	}
+	if _, held := w.leases["task"]; held {
+		t.Fatal("the given-up task kept its lease")
+	}
+}
+
+// The lease must survive the wait it was asked to make: an idle pane is the
+// answer Orchestra requested, not evidence of an agent that stopped working.
+func TestWaitingForAHandoffKeepsTheLease(t *testing.T) {
+	renewals := 0
+	api := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+		renewals++
+		rw.Write([]byte(`{}`))
+	}))
+	defer api.Close()
+	backend := &recordingBackend{status: "idle", progress: "same screen"}
+	w := &worker{
+		api:         federation.Client{BaseURL: api.URL, WorkerID: "h", Token: "t"},
+		backend:     backend,
+		harness:     "claude",
+		sessions:    map[string]herdr.Session{"task": {PaneID: "pane", HandoffRequested: true, HandoffRequestedAt: time.Now().UTC()}},
+		leases:      map[string]lease{"task": {Epoch: "e", Version: 1, Until: time.Now(), ProgressSHA: domain.Hash([]byte("same screen"))}},
+		quarantined: map[string]bool{},
+		statePath:   filepath.Join(t.TempDir(), "state.json"),
+	}
+	w.renewLeases(context.Background())
+	if renewals != 1 {
+		t.Fatalf("a lease waiting on a requested handoff renewed %d times, want 1", renewals)
+	}
+
+	// Past the bound the exemption stops: watchHandoff has given the task up
+	// by then, and nothing keeps an unanswered request alive.
+	s := w.sessions["task"]
+	s.HandoffRequestedAt = time.Now().UTC().Add(-handoffAnswerTimeout - time.Second)
+	w.sessions["task"] = s
+	l := w.leases["task"]
+	l.Until = time.Now()
+	w.leases["task"] = l
+	w.renewLeases(context.Background())
+	if renewals != 1 {
+		t.Fatalf("the exemption outlived its bound: renewals=%d", renewals)
+	}
+}

cmd/orchestra-worker/main.go

@@ -174,6 +174,7 @@ func releaseBackoff(attempts int) time.Duration {
 	}
 	return d
 }
+
 type projectConfig struct {
 	Repo           string   `json:"repo"`
 	Root           string   `json:"worktree_root"`
@@ -673,6 +674,13 @@ func (w *worker) releaseReady(ctx context.Context) {
 			w.advanceRelease(ctx, id, s)
 			continue
 		}
+		if s.HandoffRequested {
+			next, gaveUp := w.watchHandoff(ctx, id, s)
+			if gaveUp {
+				continue
+			}
+			s = next
+		}
 		w.rotationTick(ctx, id, s)
 	}
 }
@@ -772,7 +780,7 @@ func (w *worker) rotationTick(ctx context.Context, id string, s herdr.Session) {
 		w.recordError(fmt.Errorf("rotation %s threshold prompt: %w", id, err))
 		return
 	}
-	s.HandoffRequested, s.HandoffReason = true, d.Reason
+	s.HandoffRequested, s.HandoffReason, s.HandoffRequestedAt = true, d.Reason, time.Now().UTC()
 	w.sessions[id] = s
 	_ = w.save()
 }
@@ -1166,6 +1174,70 @@ func (w *worker) submit(ctx context.Context, id string, s herdr.Session, e compl
 // the directory also holds for a worktree that has no inner .gitignore.
 const stageExclude = ":!.orchestra"
 
+// F62: a requested handoff nobody answers was invisible. Renewals stopped,
+// the lease expired, and the task lost an attempt with nothing on record
+// saying a handoff had ever been asked for — worker health showed only "agent
+// status idle and pane unchanged", 34 times in run 16.
+const (
+	handoffRetryAfter    = 4 * time.Minute
+	handoffAnswerTimeout = 10 * time.Minute
+)
+
+// watchHandoff bounds the wait for an agent's handoff answer: re-send the
+// request once, then give the task up with a class that names the cause. It
+// returns the session to keep using and whether the task was given up.
+func (w *worker) watchHandoff(ctx context.Context, id string, s herdr.Session) (herdr.Session, bool) {
+	if s.HandoffRequestedAt.IsZero() {
+		// A session persisted before the stamp existed, or requested by a path
+		// that does not set it. Start the clock now rather than time out a
+		// request retroactively.
+		s.HandoffRequestedAt = time.Now().UTC()
+		w.sessions[id] = s
+		_ = w.save()
+		return s, false
+	}
+	waited := time.Since(s.HandoffRequestedAt)
+	if waited < handoffRetryAfter {
+		return s, false
+	}
+	if waited < handoffAnswerTimeout {
+		if s.HandoffRetried || w.executionBackend() == nil {
+			return s, false
+		}
+		a := herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness}
+		var err error
+		if s.HandoffReason != "" {
+			err = a.RequestHandoffReason(ctx, s, s.HandoffReason, nil)
+		} else {
+			err = a.RequestHandoff(ctx, s)
+		}
+		if err != nil {
+			w.recordError(fmt.Errorf("handoff %s re-request: %w", id, err))
+			return s, false
+		}
+		s.HandoffRetried = true
+		w.sessions[id] = s
+		_ = w.save()
+		w.recordError(fmt.Errorf("handoff %s (%s) unanswered for %s: request re-sent", id, s.HandoffReason, waited.Round(time.Second)))
+		return s, false
+	}
+	l, ok := w.leases[id]
+	if !ok {
+		return s, false
+	}
+	detail := fmt.Sprintf("handoff requested (%s) and unanswered for %s", s.HandoffReason, waited.Round(time.Second))
+	if err := w.api.NackStart(ctx, id, l.Epoch, l.Version, "handoff_unanswered", detail, w.sessionEvidence(ctx, id, s)); err != nil {
+		w.recordError(fmt.Errorf("handoff timeout %s: %w", id, err))
+		return s, false
+	}
+	// The coordinator answers with TaskReleased; its replay quarantines the
+	// pane. Drop the lease here so nothing renews it in the meantime.
+	delete(w.leases, id)
+	_ = w.save()
+	w.recordError(errors.New(detail))
+	return s, true
+}
+
 func (w *worker) renewLeases(ctx context.Context) {
 	if w.executionBackend() == nil {
 		return
@@ -1199,6 +1271,11 @@ func (w *worker) renewLeases(ctx context.Context) {
 		case l.ProgressSHA == "":
 			// First renewal has no baseline to compare against. Record one and
 			// allow this renewal; the next one must show real movement.
+		case s.HandoffRequested && !s.HandoffRequestedAt.IsZero() && time.Since(s.HandoffRequestedAt) < handoffAnswerTimeout:
+			// Orchestra told this agent to stop and write its handoff. A quiet
+			// pane is the answer it was asked for, so the lease is held while
+			// the wait is explicitly bounded (F62). Past the bound the case
+			// stops matching and watchHandoff has already given the task up.
 		default:
 			w.recordError(fmt.Errorf("lease %s not renewed: agent status %s and pane unchanged since the last renewal", taskID, status))
 			continue
@@ -1932,7 +2009,7 @@ func (w *worker) federatedTurn(ctx context.Context, id string, a herdr.Adapter,
 			w.recordError(fmt.Errorf("reconcile failure handoff %s: %w", id, err))
 			return
 		}
-		s.HandoffRequested, s.HandoffReason = true, "reconcile_failure"
+		s.HandoffRequested, s.HandoffReason, s.HandoffRequestedAt = true, "reconcile_failure", time.Now().UTC()
 		w.sessions[id] = s
 		_ = w.save()
 		return
@@ -2009,7 +2086,7 @@ func (w *worker) rotateForPhase(ctx context.Context, id string, a herdr.Adapter,
 		w.recordError(fmt.Errorf("phase rotation %s: %w", id, err))
 		return
 	}
-	s.HandoffRequested, s.HandoffReason = true, "phase_changed"
+	s.HandoffRequested, s.HandoffReason, s.HandoffRequestedAt = true, "phase_changed", time.Now().UTC()
 	w.sessions[id] = s
 	_ = w.save()
 	log.Printf("phase changed for %s: session rotating", id)

cmd/orchestra/main.go

@@ -1589,6 +1589,12 @@ func main() {
 			case "invalid_handoff":
 				typ = "TaskBlocked"
 				p, _ = json.Marshal(map[string]any{"blocker": b.LastError, "block_reason": string(domain.BlockReasonHandoffValidation), "harness_id": parts[3], "lease_epoch": b.LeaseEpoch, "expected_version": t.Version, "lifecycle_phase": "launch_nacked", "last_error": b.LastError, "session_evidence": b.SessionEvidence})
+			case "handoff_unanswered":
+				// F62: not a launch failure. The agent was asked to hand off
+				// and never did, so the reclaim says exactly that instead of
+				// arriving as an ordinary idle expiry.
+				typ = "TaskReleased"
+				p, _ = json.Marshal(map[string]any{"reason": "handoff_unanswered", "failure_class": b.FailureClass, "harness_id": parts[3], "lease_epoch": b.LeaseEpoch, "expected_version": t.Version, "lifecycle_phase": "handoff_unanswered", "last_error": b.LastError, "session_evidence": b.SessionEvidence})
 			case "launch_uncertain":
 				typ = "TaskNeedsAttention"
 				p, _ = json.Marshal(map[string]any{"blocker": b.LastError, "block_reason": string(domain.BlockReasonLeaseFailure), "harness_id": parts[3], "lease_epoch": b.LeaseEpoch, "expected_version": t.Version, "lifecycle_phase": "launch_uncertain", "last_error": b.LastError, "session_evidence": b.SessionEvidence})