Files
orchestra/internal/herdr/tmux.go
T
kami 2417a39a1e Stop counting harness chrome as agent progress, and shorten the lease
F41, live on run 5: the review agent produced nothing after 02:12 and the
02:46 renewal was granted anyway. The progress digest covered Claude Code's
status footer, and one of its fields ticked inside the window. Reproduced the
worker's stored progress_sha byte for byte from the live pane, so the branch
taken was progress != ProgressSHA, not IsBusy and not an empty baseline.

PaneProgress now cuts from the editor's lower rule and trims the spinner
summary and version notice above it. AgentStatus still reads the raw capture,
so the busy markers living in the footer are unaffected.

Lease TTL moves to 5 minutes, from domain.LeaseTTL, with renewal at half of
it. Reclaiming a stalled pane happens only at expiry, and 30 minutes per
window made run 5's stall unbounded in practice.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CVbaKucEYBjMqVeUgJUsc1
2026-08-28 03:08:45 +04:00

605 lines
21 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package herdr
import (
"context"
"crypto/sha256"
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"
)
// TmuxBackend runs one Claude Code process per isolated tmux session. It is
// intentionally Claude-only for now: Codex and OpenCode keep using the
// verified herdr protocol until their terminal behavior has been exercised
// against a live installation.
type TmuxBackend struct {
// Socket is a tmux socket name (-L) or an absolute socket path (-S).
// An empty value uses the isolated socket name "orchestra".
Socket string
// Command is the Claude Code executable. An empty value resolves "claude"
// through PATH.
Command string
// Binary is test/packaging override for tmux itself.
Binary string
// LaunchConfirmTimeout and LaunchConfirmPoll bound ConfirmInput. They are
// tunable because how fast a terminal harness visibly reacts is a property
// of the host, not of this code. Zero values mean 10s and 250ms.
LaunchConfirmTimeout time.Duration
LaunchConfirmPoll time.Duration
// Now is a test seam for the confirmation deadline.
Now func() time.Time
}
func (b *TmuxBackend) now() time.Time {
if b.Now != nil {
return b.Now()
}
return time.Now()
}
func NewTmuxBackend(socket, command string) *TmuxBackend {
return &TmuxBackend{Socket: socket, Command: command}
}
func (b *TmuxBackend) Kind() string { return "tmux" }
func (b *TmuxBackend) binary() string {
if b.Binary != "" {
return b.Binary
}
return "tmux"
}
func (b *TmuxBackend) socketArgs() []string {
socket := b.Socket
if socket == "" {
socket = "orchestra"
}
if filepath.IsAbs(socket) {
return []string{"-S", socket}
}
return []string{"-L", socket}
}
func (b *TmuxBackend) command(ctx context.Context, args ...string) ([]byte, error) {
all := append(b.socketArgs(), args...)
out, err := exec.CommandContext(ctx, b.binary(), all...).CombinedOutput()
if err != nil {
return out, fmt.Errorf("tmux %s: %s: %w", strings.Join(args, " "), strings.TrimSpace(string(out)), err)
}
return out, nil
}
func (b *TmuxBackend) Check(ctx context.Context) error {
if _, err := exec.LookPath(b.binary()); err != nil {
return fmt.Errorf("tmux backend: %w", err)
}
// tmux -V does not require a server to exist. An idle backend is healthy
// and will create its isolated server with the first session.
if out, err := exec.CommandContext(ctx, b.binary(), "-V").CombinedOutput(); err != nil {
return fmt.Errorf("tmux backend: %s: %w", strings.TrimSpace(string(out)), err)
}
command := b.Command
if command == "" {
command = "claude"
}
if _, err := exec.LookPath(command); err != nil {
return fmt.Errorf("tmux backend Claude command: %w", err)
}
return nil
}
func (b *TmuxBackend) Worktree(_ context.Context, _ string, path, _ string) (string, error) {
info, err := os.Stat(path)
if err != nil {
return "", fmt.Errorf("tmux backend worktree: %w", err)
}
if !info.IsDir() {
return "", fmt.Errorf("tmux backend worktree %s is not a directory", path)
}
return path, nil
}
func tmuxSessionName(taskID string) string {
id := strings.Trim(invalidAgentName.ReplaceAllString(strings.ToLower(taskID), "-"), "-_")
if id == "" {
id = "session"
}
if len(id) > 36 {
id = strings.TrimRight(id[:36], "-_")
}
sum := sha256.Sum256([]byte(taskID))
return fmt.Sprintf("orchestra-%s-%x", id, sum[:4])
}
func tmuxSession(paneID string) string {
if before, _, ok := strings.Cut(paneID, ":"); ok {
return before
}
return paneID
}
func tmuxTarget(paneID string) string { return "=" + paneID }
func (b *TmuxBackend) hasSession(ctx context.Context, session string) (bool, error) {
out, err := b.command(ctx, "has-session", "-t", "="+session)
if err == nil {
return true, nil
}
message := strings.ToLower(string(out) + " " + err.Error())
// "no current target" is what a *running but empty* server answers: the
// target does not resolve and there is no current session to fall back to.
// It only started appearing once the runtime became its own unit running
// `tmux -D`, which keeps the server alive past its last pane (F32). Before
// that the server exited with its last session and answered "no server
// running", so this branch was never reached. Without it Kill returns an
// error for a pane that is already gone, quarantine never clears, and the
// worker's only session slot stays pinned.
if strings.Contains(message, "can't find session") || strings.Contains(message, "no server running") || strings.Contains(message, "no sessions") || strings.Contains(message, "no current target") || (strings.Contains(message, "error connecting to") && strings.Contains(message, "no such file")) {
return false, nil
}
return false, err
}
func (b *TmuxBackend) StartAgent(ctx context.Context, _, path, _, harness, taskID string) (Session, error) {
if !strings.EqualFold(harness, "claude") {
return Session{}, fmt.Errorf("tmux backend: harness %q is unsupported; only claude is enabled", harness)
}
if _, err := b.Worktree(ctx, "", path, ""); err != nil {
return Session{}, err
}
command := b.Command
if command == "" {
command = "claude"
}
resolved, err := exec.LookPath(command)
if err != nil {
return Session{}, fmt.Errorf("tmux backend Claude command: %w", err)
}
session := tmuxSessionName(taskID)
existing, err := b.hasSession(ctx, session)
if err != nil {
return Session{}, err
}
if !existing {
if _, err := b.command(ctx, "new-session", "-d", "-s", session, "-c", path, resolved); err != nil {
return Session{}, err
}
}
// The exact pane id is resolved below. Deployments may configure tmux
// base-index/base-pane-index, so neither index is assumed to be zero.
paneOut, err := b.command(ctx, "list-panes", "-t", "="+session, "-F", "#{session_name}:#{window_index}.#{pane_index}")
if err != nil {
return Session{}, err
}
paneID := strings.TrimSpace(strings.SplitN(string(paneOut), "\n", 2)[0])
if paneID == "" {
return Session{}, fmt.Errorf("tmux backend: session %s has no pane", session)
}
if existing {
out, err := b.command(ctx, "display-message", "-p", "-t", tmuxTarget(paneID), "#{pane_current_path}")
if err != nil {
return Session{}, err
}
current, currentErr := filepath.Abs(strings.TrimSpace(string(out)))
want, wantErr := filepath.Abs(path)
if currentErr != nil || wantErr != nil || current != want {
return Session{}, fmt.Errorf("tmux backend: existing session %s belongs to %q, not %q", session, current, want)
}
} else {
if _, err := b.command(ctx, "set-window-option", "-t", tmuxTarget(paneID), "remain-on-exit", "on"); err != nil {
return Session{}, err
}
}
s := Session{PaneID: paneID, Worktree: path, Harness: "claude", AgentName: session}
if err := b.confirmClaudeWorkspaceTrust(ctx, s); err != nil {
return Session{}, err
}
status, err := b.AgentStatus(ctx, s)
if err != nil {
return Session{}, err
}
if status == "exited" || status == "dead" {
return Session{}, fmt.Errorf("tmux backend: Claude exited while starting session %s", session)
}
return s, nil
}
func (b *TmuxBackend) confirmClaudeWorkspaceTrust(ctx context.Context, s Session) error {
deadline := time.Now().Add(15 * time.Second)
accepted := false
for {
text, err := b.PaneCapture(ctx, s, "recent")
if err == nil && claudeWorkspaceTrustPrompt(text) {
if !accepted {
if err := b.SendText(ctx, s, "1"); err != nil {
return fmt.Errorf("tmux backend: accept Claude workspace trust: %w", err)
}
if err := b.SendKeys(ctx, s, []string{"Enter"}); err != nil {
return fmt.Errorf("tmux backend: accept Claude workspace trust: %w", err)
}
accepted = true
}
}
// Claude's input prompt is the readiness boundary. A banner or partially
// painted fullscreen UI is not enough: input sent there can be lost.
if err == nil && !claudeWorkspaceTrustPrompt(text) && strings.Contains(text, "") {
return nil
}
if time.Now().After(deadline) {
return fmt.Errorf("tmux backend: Claude input prompt did not become ready in session %s", tmuxSession(s.PaneID))
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(100 * time.Millisecond):
}
}
}
func (b *TmuxBackend) Prompt(ctx context.Context, pane, text string, _ time.Duration) error {
s := Session{PaneID: pane}
status, err := b.AgentStatus(ctx, s)
if err != nil {
return err
}
if status == "blocked" {
return fmt.Errorf("tmux backend: refusing prompt while pane %s shows a permission dialog", pane)
}
if err := b.SendText(ctx, s, text); err != nil {
return err
}
return b.SendKeys(ctx, s, []string{"Enter"})
}
func (b *TmuxBackend) Kill(ctx context.Context, s Session) error {
exists, err := b.hasSession(ctx, tmuxSession(s.PaneID))
if err != nil {
return err
}
if !exists {
return nil
}
_, err = b.command(ctx, "kill-session", "-t", "="+tmuxSession(s.PaneID))
return err
}
func (b *TmuxBackend) paneState(ctx context.Context, s Session) (dead bool, command string, err error) {
out, err := b.command(ctx, "display-message", "-p", "-t", tmuxTarget(s.PaneID), "#{pane_dead}\t#{pane_current_command}")
if err != nil {
return false, "", err
}
parts := strings.SplitN(strings.TrimSpace(string(out)), "\t", 2)
dead = len(parts) > 0 && parts[0] == "1"
if len(parts) == 2 {
command = parts[1]
}
return dead, command, nil
}
func (b *TmuxBackend) AgentStatus(ctx context.Context, s Session) (string, error) {
exists, err := b.hasSession(ctx, tmuxSession(s.PaneID))
if err != nil {
return "", err
}
if !exists {
return "exited", nil
}
dead, command, err := b.paneState(ctx, s)
if err != nil {
return "", err
}
if dead || command == "" {
return "exited", nil
}
text, err := b.PaneCapture(ctx, s, "recent")
if err != nil {
return "", err
}
if permissionPrompt(text) {
return "blocked", nil
}
lower := strings.ToLower(text)
for _, marker := range []string{"esc to interrupt", "ctrl+c to interrupt", "press esc to interrupt"} {
if strings.Contains(lower, marker) {
return "busy", nil
}
}
return "idle", nil
}
func (b *TmuxBackend) PaneCapture(ctx context.Context, s Session, source string) (string, error) {
start := "-200"
if source != "" && source != "recent" {
start = "-1000"
}
out, err := b.command(ctx, "capture-pane", "-p", "-J", "-S", start, "-t", tmuxTarget(s.PaneID))
return string(out), err
}
func (b *TmuxBackend) SendText(ctx context.Context, s Session, text string) error {
_, err := b.command(ctx, "send-keys", "-t", tmuxTarget(s.PaneID), "-l", "--", text)
return err
}
func (b *TmuxBackend) SendKeys(ctx context.Context, s Session, keys []string) error {
if len(keys) == 0 {
return nil
}
for _, key := range keys {
if strings.TrimSpace(key) == "" {
return errors.New("tmux backend: empty key name")
}
}
args := []string{"send-keys", "-t", tmuxTarget(s.PaneID)}
args = append(args, keys...)
_, err := b.command(ctx, args...)
return err
}
// tmux has no separate agent binding to release. Keeping the session alive is
// the tmux equivalent of herdr's split-then-close protocol; the worker kills
// it only after successor pickup has been validated.
func (b *TmuxBackend) ReleaseAgent(ctx context.Context, s Session, _ string) error {
exists, err := b.hasSession(ctx, tmuxSession(s.PaneID))
if err != nil {
return err
}
if !exists {
return fmt.Errorf("tmux backend: session %s is not running", tmuxSession(s.PaneID))
}
return nil
}
var _ Backend = (*TmuxBackend)(nil)
// LaunchTransport keeps the file-reference launch scoped to the harness whose
// paste handling requires it, rather than making it the universal prompt
// format. Every other harness keeps the inline path it was verified on.
func (b *TmuxBackend) LaunchTransport(harness string) LaunchTransport {
if harness == "claude" {
return LaunchFileRef
}
return LaunchInline
}
// promptLine matches a harness input line. A queued or already-accepted
// message renders with the same prefix, so a match alone proves nothing about
// what is still unsubmitted; see inputState.
var promptLine = regexp.MustCompile(`(?m)^[ \t]*[>][ \t]*(.*)$`)
// chromeLine matches the status lines Claude Code redraws on its own schedule:
// the spinner summary with its elapsed timer, and the version notice. Neither
// is the agent writing anything, and both change while a pane sits idle.
var chromeLine = regexp.MustCompile(`^[ \t]*(?:[\x{273B}\x{273D}\x{2733}\x{2722}\x{00B7}*]|current:[ \t])`)
// separatorRow matches the rule Claude Code draws above and below its editor.
var separatorRow = regexp.MustCompile(`^[\s─━┄┅┈┉-]+$`)
// InputState is what the interactive editor holds. Active distinguishes the
// live editor from queued or already-submitted input: both render with the
// same "" prefix, but only the live editor owns the pane cursor (F17,
// found live during burn-in run 3).
type InputState struct {
Text string
Active bool
}
// sameInput compares editor content to what was submitted. Soft wrapping may
// break the text at any column and re-indent the continuation, so the
// comparison ignores whitespace entirely.
func sameInput(a, b string) bool {
strip := func(s string) string { return strings.Join(strings.Fields(s), "") }
return strip(a) == strip(b)
}
// inputState reads the editor that owns the cursor. Scanning every line
// beginning with "" cannot tell an unsubmitted prompt from one the
// harness already queued, which is why this asks tmux where the cursor is.
func (b *TmuxBackend) inputState(ctx context.Context, s Session) (InputState, error) {
out, err := b.command(ctx, "display-message", "-p", "-t", tmuxTarget(s.PaneID), "#{cursor_y}")
if err != nil {
return InputState{}, err
}
row, err := strconv.Atoi(strings.TrimSpace(string(out)))
if err != nil {
return InputState{}, fmt.Errorf("tmux backend: cursor row %q: %w", strings.TrimSpace(string(out)), err)
}
// Screen rows, unjoined: -J merges wrapped rows and would invalidate the
// cursor row index.
raw, err := b.command(ctx, "capture-pane", "-p", "-t", tmuxTarget(s.PaneID))
if err != nil {
return InputState{}, err
}
rows := strings.Split(strings.TrimRight(string(raw), "\n"), "\n")
if row < 0 || row >= len(rows) {
return InputState{}, nil
}
start := -1
for i := row; i >= 0; i-- {
if promptLine.MatchString(rows[i]) {
start = i
break
}
if separatorRow.MatchString(rows[i]) {
break
}
}
if start < 0 {
return InputState{}, nil
}
parts := []string{strings.TrimSpace(promptLine.FindStringSubmatch(rows[start])[1])}
for i := start + 1; i <= row; i++ {
parts = append(parts, strings.TrimSpace(rows[i]))
}
return InputState{Text: strings.TrimSpace(strings.Join(parts, " ")), Active: true}, nil
}
// PaneProgress hashes what the harness produced, not what someone typed at it.
// The capture joins wrapped lines, so an input block is one line beginning
// with the prompt marker and dropping those lines removes it whole. Found live
// during burn-in run 3: unexplained keystrokes in a pane kept a stalled lease
// renewing, because the renewal check hashed the whole capture.
func (b *TmuxBackend) PaneProgress(ctx context.Context, s Session) (string, error) {
text, err := b.PaneCapture(ctx, s, "recent")
if err != nil {
return "", err
}
var kept []string
for _, line := range strings.Split(text, "\n") {
if promptLine.MatchString(line) {
continue
}
kept = append(kept, line)
}
// Everything below the editor's lower rule is harness chrome: the model
// name, the rolling usage percentage, the context counter, the update
// banner. F41, live on run 5: the agent produced nothing after 02:12 and
// the 02:46 renewal was granted anyway, because one of those fields ticked
// inside the window. Progress means output the agent wrote, so the tail
// after the last rule is dropped. AgentStatus reads the raw capture, so
// the busy markers that live down there are unaffected.
// ponytail: last rule wins, the editor is always the bottom-most one.
for i := len(kept) - 1; i >= 0; i-- {
if separatorRow.MatchString(kept[i]) {
kept = kept[:i]
break
}
}
// The spinner summary and the version notice render above the editor, so
// the rule cut alone leaves them in. On run 5 the notice was the one that
// moved: "current: 2.1.247 - latest: 2.1.248" gained "Update installed".
for len(kept) > 0 {
last := kept[len(kept)-1]
if strings.TrimSpace(last) == "" || separatorRow.MatchString(last) || chromeLine.MatchString(last) {
kept = kept[:len(kept)-1]
continue
}
break
}
return strings.Join(kept, "\n"), nil
}
// launchResubmitLimit bounds the intervention independently of the observation
// deadline. A slow TUI must not receive a fortieth Enter after it accepted the
// first: three exact-editor resubmits, then observation only.
const launchResubmitLimit = 3
// launchSettle is how long an empty editor must stay empty before it counts as
// proof of submission. A TUI takes tens of milliseconds to render pasted text,
// and the first poll can run inside that gap. F33: a poll 7ms after the submit
// found nothing in the editor, reported confirmation=editor_cleared, and the
// launch text then sat unsent in the input box for the rest of the lease. The
// two launches that actually worked took ~500ms and a second Enter, so the
// difference between success and this false positive was scheduling luck.
//
// Only the never-observed case waits. Text seen and then gone is real
// evidence, and busy, blocked or queued still confirm immediately.
const launchSettle = time.Second
// ConfirmInput drives the submit to a decision instead of assuming one Enter
// landed. Burn-in run 3 proved the submit is not deterministic: the text
// reached the editor on all three attempts and the following Enter never took
// effect. So this resends Enter while the live editor still holds exactly what
// was submitted, up to launchResubmitLimit times, then observes until the
// deadline.
//
// The returned evidence records how many submits it took, which is the only
// way to tell a harness that needs a second Enter from one that needed none.
func (b *TmuxBackend) ConfirmInput(ctx context.Context, s Session, submitted string) (string, error) {
timeout, poll := b.LaunchConfirmTimeout, b.LaunchConfirmPoll
if timeout <= 0 {
timeout = 10 * time.Second
}
if poll <= 0 {
poll = 250 * time.Millisecond
}
first := b.now()
deadline := first.Add(timeout)
attempts, resubmits := 1, 0
var lastResubmit time.Time
evidence := func(kind string) string {
return fmt.Sprintf("confirmation=%s submit_attempts=%d first_submit_at=%s confirmed_at=%s",
kind, attempts, first.UTC().Format(time.RFC3339Nano), b.now().UTC().Format(time.RFC3339Nano))
}
var last InputState
// Whether the editor was ever observed holding what we submitted. Without
// it, "empty" is indistinguishable from "not rendered yet".
seen := false
for {
state, err := b.inputState(ctx, s)
if err != nil {
return "", err
}
last = state
if state.Active && sameInput(state.Text, submitted) {
seen = true
}
switch {
case state.Active && sameInput(state.Text, submitted):
// Exactly what was submitted still owns the cursor, so the submit
// did not take. Resend Enter, spaced so a TUI that accepted the
// previous one cannot receive another inside its own redraw.
if resubmits < launchResubmitLimit && (lastResubmit.IsZero() || b.now().Sub(lastResubmit) >= 2*poll) {
if err := b.SendKeys(ctx, s, []string{"Enter"}); err != nil {
return "", err
}
resubmits++
attempts++
lastResubmit = b.now()
}
case state.Active && queuedInput(state.Text):
// The harness accepted the instruction and parked it behind the
// current turn. Queued is submitted.
return evidence("queued"), nil
case state.Active && state.Text != "":
// Something else is in the editor, a paste placeholder for
// instance. Not proof of acceptance, so keep observing.
default:
status, statusErr := b.AgentStatus(ctx, s)
if statusErr != nil {
return "", statusErr
}
switch status {
case "exited":
return "", fmt.Errorf("%w: pane %s exited before the harness reacted", ErrPromptNotSubmitted, s.PaneID)
case "busy":
return evidence("busy"), nil
case "blocked":
// A permission dialog is the harness acting on the
// instruction, so submission is proven even though the
// session now needs an operator.
return evidence("blocked"), nil
default:
// An editor that never held the text may simply not have
// rendered it yet. Give it the settle window before calling an
// empty box proof that the harness took the prompt.
if !seen && b.now().Sub(first) < launchSettle {
break
}
return evidence("editor_cleared"), nil
}
}
if !b.now().Before(deadline) {
return "", fmt.Errorf("%w: pane %s still holds %q after %s and %d submits", ErrPromptNotSubmitted, s.PaneID, last.Text, timeout, attempts)
}
select {
case <-ctx.Done():
return "", ctx.Err()
case <-time.After(poll):
}
}
}
// queuedInput recognizes the editor Claude Code shows once it has taken a
// prompt and parked it behind the running turn.
func queuedInput(text string) bool {
return strings.Contains(strings.ToLower(text), "queued message")
}