diff --git a/cmd/orchestra-worker/main.go b/cmd/orchestra-worker/main.go index d8eacbb..edd468a 100644 --- a/cmd/orchestra-worker/main.go +++ b/cmd/orchestra-worker/main.go @@ -997,7 +997,7 @@ func (w *worker) renewLeases(ctx context.Context) { now := time.Now() for taskID, l := range w.leases { s, ok := w.sessions[taskID] - if !ok || l.Version == 0 || l.Until.After(now.Add(10*time.Minute)) { + if !ok || l.Version == 0 || l.Until.After(now.Add(domain.LeaseRenewAt)) { continue } adapter := herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness} @@ -1027,14 +1027,14 @@ func (w *worker) renewLeases(ctx context.Context) { w.recordError(fmt.Errorf("lease %s not renewed: agent status %s and pane unchanged since the last renewal", taskID, status)) continue } - if err := w.api.Renew(ctx, taskID, l.Epoch, l.Version, int((30 * time.Minute).Seconds())); err != nil { + if err := w.api.Renew(ctx, taskID, l.Epoch, l.Version, int(domain.LeaseTTL.Seconds())); err != nil { w.recordError(fmt.Errorf("renew lease %s: %w", taskID, err)) log.Printf("renew lease %s: %v", taskID, err) } else { // RenewLease appends one event. Retain that epoch locally until its // replay arrives so a release transaction uses the same version. l.Version++ - l.Until = now.Add(30 * time.Minute) + l.Until = now.Add(domain.LeaseTTL) l.ProgressSHA = progress w.leases[taskID] = l _ = w.save() diff --git a/cmd/orchestra/main.go b/cmd/orchestra/main.go index dfb73f6..70f8a60 100644 --- a/cmd/orchestra/main.go +++ b/cmd/orchestra/main.go @@ -992,7 +992,7 @@ func main() { return } if p.TTLSeconds <= 0 { - p.TTLSeconds = 1800 + p.TTLSeconds = int(domain.LeaseTTL.Seconds()) } e, err = s.Lease(taskID, p.HarnessID, time.Duration(p.TTLSeconds)*time.Second) case "release", "complete", "block", "attention": @@ -1521,7 +1521,7 @@ func main() { if strings.HasSuffix(r.URL.Path, "/renew") { ttl := b.TTLSeconds if ttl == 0 { - ttl = int((30 * time.Minute).Seconds()) + ttl = int(domain.LeaseTTL.Seconds()) } e, err := s.RenewLease(b.TaskID, parts[3], b.LeaseEpoch, b.ExpectedVersion, time.Duration(ttl)*time.Second) if err != nil { diff --git a/internal/domain/domain.go b/internal/domain/domain.go index d05c19e..a43cfff 100644 --- a/internal/domain/domain.go +++ b/internal/domain/domain.go @@ -26,6 +26,17 @@ var ErrDuplicate = errors.New("duplicate task ingestion") // authorizing Surface; schema 3 adds lease fencing epochs. Older events stay // readable so a deployment can recover its existing log before new writes // are emitted (the store derives a non-renewable legacy epoch on replay). +// LeaseTTL is how long a lease survives without renewal, and LeaseRenewAt is +// how much remaining time makes the worker renew. Both halves read these, so a +// coordinator granting one TTL while a worker assumes another is not possible. +// Short leases are the recovery mechanism for a stalled agent: a stalled pane +// is only reclaimed when its lease runs out, and 30 minutes per window made +// run 5's review stall unbounded in practice. +const ( + LeaseTTL = 5 * time.Minute + LeaseRenewAt = LeaseTTL / 2 +) + const CurrentEventSchema = 3 type TaskState string diff --git a/internal/herdr/tmux.go b/internal/herdr/tmux.go index 7e7d888..03b69de 100644 --- a/internal/herdr/tmux.go +++ b/internal/herdr/tmux.go @@ -374,6 +374,11 @@ func (b *TmuxBackend) LaunchTransport(harness string) LaunchTransport { // what is still unsubmitted; see inputState. var promptLine = regexp.MustCompile(`(?m)^[ \t]*[>❯][ \t]*(.*)$`) +// chromeLine matches the status lines Claude Code redraws on its own schedule: +// the spinner summary with its elapsed timer, and the version notice. Neither +// is the agent writing anything, and both change while a pane sits idle. +var chromeLine = regexp.MustCompile(`^[ \t]*(?:[\x{273B}\x{273D}\x{2733}\x{2722}\x{00B7}*]|current:[ \t])`) + // separatorRow matches the rule Claude Code draws above and below its editor. var separatorRow = regexp.MustCompile(`^[\s─━┄┅┈┉-]+$`) @@ -453,6 +458,31 @@ func (b *TmuxBackend) PaneProgress(ctx context.Context, s Session) (string, erro } kept = append(kept, line) } + // Everything below the editor's lower rule is harness chrome: the model + // name, the rolling usage percentage, the context counter, the update + // banner. F41, live on run 5: the agent produced nothing after 02:12 and + // the 02:46 renewal was granted anyway, because one of those fields ticked + // inside the window. Progress means output the agent wrote, so the tail + // after the last rule is dropped. AgentStatus reads the raw capture, so + // the busy markers that live down there are unaffected. + // ponytail: last rule wins, the editor is always the bottom-most one. + for i := len(kept) - 1; i >= 0; i-- { + if separatorRow.MatchString(kept[i]) { + kept = kept[:i] + break + } + } + // The spinner summary and the version notice render above the editor, so + // the rule cut alone leaves them in. On run 5 the notice was the one that + // moved: "current: 2.1.247 - latest: 2.1.248" gained "Update installed". + for len(kept) > 0 { + last := kept[len(kept)-1] + if strings.TrimSpace(last) == "" || separatorRow.MatchString(last) || chromeLine.MatchString(last) { + kept = kept[:len(kept)-1] + continue + } + break + } return strings.Join(kept, "\n"), nil } diff --git a/internal/herdr/tmux_test.go b/internal/herdr/tmux_test.go index 52d34f1..38e94ae 100644 --- a/internal/herdr/tmux_test.go +++ b/internal/herdr/tmux_test.go @@ -491,3 +491,42 @@ exit 0 t.Fatalf("err=%v, want ErrPromptNotSubmitted", err) } } + +// F41, live on run 5: the review agent produced nothing after 02:12 and the +// 02:46 renewal was granted, because the digest covered the harness footer and +// one of its fields ticked. The layout below is the real pane, with the usage +// percentage and the version banner moved on. +func TestPaneProgressIgnoresHarnessFooter(t *testing.T) { + body := "────\n reviewed the diff, no blocking findings\n────\n❯ \n" + footer := func(pct, banner string) string { + return body + + " [Opus 5] 📁 06G4A4F0TFXKZHJE48N05XN1HG | 7d: " + pct + "\n" + + " cf474986 - Orchestra launch instructions | 57.9…\n" + + " ⏵⏵ auto mode on (shift+tab to cycle) · " + banner + "\n" + } + before := paneTmux(t, footer("51%", "current: 2.1.247 · latest: 2.1.248"), 3) + after := paneTmux(t, footer("52%", "✔ Update installed · Restart to update"), 3) + a, err := before.PaneProgress(context.Background(), Session{PaneID: "s:1.0"}) + if err != nil { + t.Fatal(err) + } + b, err := after.PaneProgress(context.Background(), Session{PaneID: "s:1.0"}) + if err != nil { + t.Fatal(err) + } + if a != b { + t.Fatalf("footer churn changed the progress digest:\n%q\n%q", a, b) + } + if !strings.Contains(a, "reviewed the diff") { + t.Fatalf("progress digest dropped harness output: %q", a) + } + worked := paneTmux(t, "────\n reviewed the diff, wrote .orchestra/done\n────\n❯ \n"+ + " [Opus 5] 📁 06G4A4F0TFXKZHJE48N05XN1HG | 7d: 51%\n", 3) + c, err := worked.PaneProgress(context.Background(), Session{PaneID: "s:1.0"}) + if err != nil { + t.Fatal(err) + } + if c == a { + t.Fatal("real agent output left the progress digest unchanged") + } +} diff --git a/internal/router/router.go b/internal/router/router.go index 4407a81..ae9ee03 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -315,7 +315,7 @@ func (r *Router) AssignPending() ([]domain.Event, error) { r.reject(t.ID, h.ID, "no free concurrency") continue } - e, err := r.Store.Lease(t.ID, h.ID, 30*time.Minute) + e, err := r.Store.Lease(t.ID, h.ID, domain.LeaseTTL) if err != nil { // Includes a pre-lease reconciliation refusal, which is the // one gate that fails closed on purpose.