Complete autonomous recovery controls

2026-07-30 14:57:25 +04:00
parent 8174400b1a
commit e8fadfc998
18 changed files with 364 additions and 77 deletions
@@ -87,6 +87,7 @@ type lease struct {
 	PickupAcknowledged bool      `json:"pickup_acknowledged,omitempty"`
 	Version            int       `json:"version"`
 	Until              time.Time `json:"until"`
+	UsageBaseline      float64   `json:"usage_baseline,omitempty"`
 }
 type releaseTransaction struct {
 	ID            string    `json:"id"`
@@ -99,10 +100,11 @@ type releaseTransaction struct {
 	UpdatedAt     time.Time `json:"updated_at"`
 }
 type projectConfig struct {
-	Repo        string `json:"repo"`
-	Root        string `json:"worktree_root"`
-	Remote      string `json:"remote"`
-	QualityGate string `json:"quality_gate,omitempty"`
+	Repo           string   `json:"repo"`
+	Root           string   `json:"worktree_root"`
+	Remote         string   `json:"remote"`
+	QualityGate    string   `json:"quality_gate,omitempty"`
+	SafeOperations []string `json:"safe_operations,omitempty"`
 }
 type completionEvidence struct {
 	TaskID      string    `json:"task_id"`
@@ -323,6 +325,9 @@ func (w *worker) start(ctx context.Context, t domain.Task, ref string) error {
 	}
 	s.TaskFileSHA = taskHash(t)
 	prompt := "Read TASK.md at the worktree root and execute it."
+	if len(p.SafeOperations) > 0 {
+		prompt += " This project's audited no-grant policy permits only worktree-local " + strings.Join(p.SafeOperations, ", ") + ". Network, secrets, destructive actions, and paths outside this worktree still require an explicit operator approval."
+	}
 	if ref != "" {
 		prompt += " A validated handoff exists; inspect local Git history and the recorded checkpoint before continuing."
 	}
@@ -335,12 +340,35 @@ func (w *worker) start(ctx context.Context, t domain.Task, ref string) error {
 	if err := w.herdr.Prompt(ctx, s.PaneID, prompt, 0); err != nil {
 		return err
 	}
+	if l, ok := w.leases[t.ID]; ok {
+		if err := w.api.Start(ctx, t.ID, l.Epoch, l.Version, w.sessionEvidence(ctx, t.ID, s)); err != nil {
+			return fmt.Errorf("ack start: %w", err)
+		}
+		l.Version++
+		w.leases[t.ID] = l
+		if err := w.save(); err != nil {
+			return err
+		}
+	}
 	if ref != "" {
 		return w.ackPickup(ctx, t.ID, s)
 	}
 	return nil
 }

+func classifyLaunchError(err error, sessionStarted bool) string {
+	if sessionStarted {
+		// A prompt response can be lost after herdr accepted it. Never reclaim
+		// that pane just because its acknowledgement was uncertain.
+		return "launch_uncertain"
+	}
+	text := strings.ToLower(err.Error())
+	if strings.Contains(text, "handoff") || strings.Contains(text, "pickup") || strings.Contains(text, "task.md") {
+		return "invalid_handoff"
+	}
+	return "launch_transient"
+}
+
 func taskHash(t domain.Task) string { b := continuity.RenderTaskFile(t); return domain.Hash(b) }

 func (w *worker) releaseReady(ctx context.Context) {
@@ -355,6 +383,17 @@ func (w *worker) releaseReady(ctx context.Context) {
 			}
 		}
 		if _, err := os.Stat(filepath.Join(s.Worktree, ".orchestra", "done")); err == nil {
+			// A done marker is an intent, not enough on its own: do not race a
+			// still-running native harness into committing half-written work.
+			status, statusErr := (herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}).AgentStatus(ctx, s)
+			if statusErr != nil {
+				w.recordError(fmt.Errorf("completion identity %s: %w", id, statusErr))
+				continue
+			}
+			if herdr.IsBusy(status) {
+				w.recordError(fmt.Errorf("completion %s deferred: agent status %s", id, status))
+				continue
+			}
 			evidence, err := w.finalize(ctx, id, s)
 			if err != nil {
 				w.recordError(fmt.Errorf("complete %s: %w", id, err))
@@ -368,7 +407,7 @@ func (w *worker) releaseReady(ctx context.Context) {
 				log.Printf("upload completion %s: %v", id, err)
 				continue
 			}
-			if err = w.api.Complete(ctx, id, ref, evidence.ResultSHA, evidence.Branch, evidence.Remote, w.leases[id].Epoch, w.leases[id].Version, w.usageReceipt(s), w.sessionEvidence(ctx, id, s)); err != nil {
+			if err = w.api.Complete(ctx, id, ref, evidence.ResultSHA, evidence.Branch, evidence.Remote, w.leases[id].Epoch, w.leases[id].Version, w.usageReceipt(s, w.leases[id]), w.sessionEvidence(ctx, id, s)); err != nil {
 				w.recordError(fmt.Errorf("complete %s: %w", id, err))
 				log.Printf("complete %s: %v", id, err)
 				continue
@@ -576,9 +615,9 @@ func (w *worker) sessionEvidence(ctx context.Context, taskID string, s herdr.Ses
 	return e
 }

-func (w *worker) usageReceipt(s herdr.Session) map[string]any {
+func (w *worker) usageReceipt(s herdr.Session, l lease) map[string]any {
 	if s.SessionFile == "" && !(w.harness == "opencode" && s.SessionID != "") {
-		return map[string]any{"harness_id": w.harnessID, "consumed": 0}
+		return map[string]any{"harness_id": w.harnessID, "consumed": 0, "known": false, "error": "native usage identity missing"}
 	}
 	var usage herdr.Usage
 	var err error
@@ -591,9 +630,13 @@ func (w *worker) usageReceipt(s herdr.Session) map[string]any {
 		usage, err = herdr.OpenCodeSessionUsage(s.SessionID)
 	}
 	if err != nil {
-		return map[string]any{"harness_id": w.harnessID, "consumed": 0, "error": err.Error()}
+		return map[string]any{"harness_id": w.harnessID, "consumed": 0, "known": false, "error": err.Error()}
 	}
-	return map[string]any{"harness_id": w.harnessID, "input_tokens": usage.Input, "cache_read_tokens": usage.CacheRead, "cache_write_tokens": usage.CacheWrite, "output_tokens": usage.Output, "consumed": usage.Numerator()}
+	delta := float64(usage.Numerator()) - l.UsageBaseline
+	if delta < 0 {
+		delta = 0
+	}
+	return map[string]any{"harness_id": w.harnessID, "input_tokens": usage.Input, "cache_read_tokens": usage.CacheRead, "cache_write_tokens": usage.CacheWrite, "consumed": delta, "lease_usage_delta": delta, "known": true}
 }

 func git(ctx context.Context, dir string, args ...string) ([]byte, error) {
@@ -814,6 +857,12 @@ func (w *worker) once(ctx context.Context) error {
 				w.leases[e.TaskID] = l
 			}
 		}
+		if e.Type == "TaskLaunchAcknowledged" {
+			if l, ok := w.leases[e.TaskID]; ok {
+				l.Version = e.Version
+				w.leases[e.TaskID] = l
+			}
+		}
 		if e.Type == "TaskNeedsAttention" {
 			// The diagnostic event increments the aggregate version but leaves
 			// ownership intact. Keep our locally persisted expected version in
@@ -914,6 +963,19 @@ func (w *worker) once(ctx context.Context) error {
 		if t, ok := w.tasks[taskID]; ok {
 			if err := w.start(ctx, t, l.HandoffRef); err != nil {
 				log.Printf("lease %s: %v", t.ID, err)
+				_, started := w.sessions[taskID]
+				class := classifyLaunchError(err, started)
+				var evidence domain.SessionEvidence
+				if session, ok := w.sessions[taskID]; ok {
+					evidence = w.sessionEvidence(ctx, taskID, session)
+				}
+				if nackErr := w.api.NackStart(ctx, taskID, l.Epoch, l.Version, class, err.Error(), evidence); nackErr != nil {
+					w.recordError(fmt.Errorf("nack launch %s: %w", taskID, nackErr))
+					continue
+				}
+				if class != "launch_uncertain" {
+					delete(w.leases, taskID)
+				}
 			}
 		}
 	}
@@ -51,6 +51,18 @@ func TestWorkerRefusesCorruptDurableState(t *testing.T) {
 	}
 }

+func TestClassifyLaunchErrorPreservesUncertainLivePane(t *testing.T) {
+	if got := classifyLaunchError(errors.New("prompt response lost"), true); got != "launch_uncertain" {
+		t.Fatalf("live pane class=%q", got)
+	}
+	if got := classifyLaunchError(errors.New("pickup anchor mismatch"), false); got != "invalid_handoff" {
+		t.Fatalf("bad handoff class=%q", got)
+	}
+	if got := classifyLaunchError(errors.New("temporary herdr outage"), false); got != "launch_transient" {
+		t.Fatalf("transient class=%q", got)
+	}
+}
+
 func TestInitialReplayDoesNotResurrectReleasedLease(t *testing.T) {
 	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		switch r.URL.Path {
@@ -1060,7 +1060,7 @@ func main() {
 		w.WriteHeader(http.StatusNoContent)
 	})
 	mux.HandleFunc("/v1/federation/workers/", func(w http.ResponseWriter, r *http.Request) {
-		if r.Method != http.MethodPost || (!strings.HasSuffix(r.URL.Path, "/heartbeat") && !strings.HasSuffix(r.URL.Path, "/renew") && !strings.HasSuffix(r.URL.Path, "/handoff") && !strings.HasSuffix(r.URL.Path, "/pickup") && !strings.HasSuffix(r.URL.Path, "/complete") && !strings.HasSuffix(r.URL.Path, "/captures")) {
+		if r.Method != http.MethodPost || (!strings.HasSuffix(r.URL.Path, "/heartbeat") && !strings.HasSuffix(r.URL.Path, "/renew") && !strings.HasSuffix(r.URL.Path, "/start") && !strings.HasSuffix(r.URL.Path, "/nack") && !strings.HasSuffix(r.URL.Path, "/handoff") && !strings.HasSuffix(r.URL.Path, "/pickup") && !strings.HasSuffix(r.URL.Path, "/complete") && !strings.HasSuffix(r.URL.Path, "/captures")) {
 			http.Error(w, "not found", 404)
 			return
 		}
@@ -1122,6 +1122,8 @@ func main() {
 			Branch          string                 `json:"branch"`
 			Remote          string                 `json:"remote"`
 			Receipt         map[string]any         `json:"receipt"`
+			FailureClass    string                 `json:"failure_class"`
+			LastError       string                 `json:"last_error"`
 			SessionEvidence domain.SessionEvidence `json:"session_evidence"`
 		}
 		if json.NewDecoder(r.Body).Decode(&b) != nil || b.TaskID == "" {
@@ -1149,6 +1151,55 @@ func main() {
 			http.Error(w, "lease version conflict", http.StatusConflict)
 			return
 		}
+		if strings.HasSuffix(r.URL.Path, "/start") {
+			// A lost response after append is an idempotent start ACK, not a
+			// reason to strand the running pane behind a stale version.
+			if t.LifecyclePhase == "started" && t.Lease != nil && t.Lease.HarnessID == parts[3] && t.Lease.Epoch == b.LeaseEpoch {
+				w.WriteHeader(http.StatusNoContent)
+				return
+			}
+			if b.ExpectedVersion != t.Version {
+				http.Error(w, "lease version conflict", http.StatusConflict)
+				return
+			}
+			p, _ := json.Marshal(map[string]any{"harness_id": parts[3], "lease_epoch": b.LeaseEpoch, "expected_version": t.Version, "lifecycle_phase": "started", "session_evidence": b.SessionEvidence})
+			e := domain.Event{ID: id(), Type: "TaskLaunchAcknowledged", TaskID: b.TaskID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
+			if err := s.Append(e); err != nil {
+				http.Error(w, err.Error(), http.StatusConflict)
+				return
+			}
+			json.NewEncoder(w).Encode(e)
+			return
+		}
+		if strings.HasSuffix(r.URL.Path, "/nack") {
+			if b.ExpectedVersion != t.Version || b.FailureClass == "" || b.LastError == "" {
+				http.Error(w, "current lease version, failure_class, and last_error required", http.StatusConflict)
+				return
+			}
+			var typ string
+			var p []byte
+			switch b.FailureClass {
+			case "invalid_handoff":
+				typ = "TaskBlocked"
+				p, _ = json.Marshal(map[string]any{"blocker": b.LastError, "block_reason": string(domain.BlockReasonHandoffValidation), "harness_id": parts[3], "lease_epoch": b.LeaseEpoch, "expected_version": t.Version, "lifecycle_phase": "launch_nacked", "last_error": b.LastError, "session_evidence": b.SessionEvidence})
+			case "launch_uncertain":
+				typ = "TaskNeedsAttention"
+				p, _ = json.Marshal(map[string]any{"blocker": b.LastError, "block_reason": string(domain.BlockReasonLeaseFailure), "harness_id": parts[3], "lease_epoch": b.LeaseEpoch, "expected_version": t.Version, "lifecycle_phase": "launch_uncertain", "last_error": b.LastError, "session_evidence": b.SessionEvidence})
+			default:
+				typ = "TaskReleased"
+				p, _ = json.Marshal(map[string]any{"reason": "launch_failed", "failure_class": b.FailureClass, "harness_id": parts[3], "lease_epoch": b.LeaseEpoch, "expected_version": t.Version, "lifecycle_phase": "launch_nacked", "last_error": b.LastError, "session_evidence": b.SessionEvidence})
+			}
+			e := domain.Event{ID: id(), Type: typ, TaskID: b.TaskID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
+			if err := s.Append(e); err != nil {
+				http.Error(w, err.Error(), http.StatusConflict)
+				return
+			}
+			if typ == "TaskReleased" && rt != nil {
+				_, _ = rt.HandleEvent(e)
+			}
+			json.NewEncoder(w).Encode(e)
+			return
+		}
 		if strings.HasSuffix(r.URL.Path, "/renew") {
 			ttl := b.TTLSeconds
 			if ttl == 0 {