Files
orchestra/internal/herdr/tmux.go
T
kami 32220d929b Stop an unrendered editor from confirming a launch
ConfirmInput's first poll can run before the TUI renders the pasted text. It
then finds an empty editor, falls through to the idle branch, and reports
confirmation=editor_cleared. The launch text sits unsent for the whole lease
while the coordinator believes the agent is working.

Run 5 died on exactly that: first_submit_at 21:13:19.570, confirmed_at
21:13:19.577. Seven milliseconds. The two launches that worked took ~500ms and
a second Enter, so the difference was scheduling luck.

An empty editor is only proof once it has held the text, or once it has stayed
empty past a settle window. Text seen and then gone still confirms at once, and
so do busy, blocked and queued. Waiting only happens in the never-observed
case, which is the one that cannot be told apart from a slow render.

With this, the failing case reaches the existing resubmit path instead: the
text appears, is recognised as unsubmitted, and Enter is resent.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011xsXyr5J1RACo71YeKG3Pu
2026-08-28 01:23:48 +04:00

575 lines
19 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package herdr
import (
"context"
"crypto/sha256"
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"
)
// TmuxBackend runs one Claude Code process per isolated tmux session. It is
// intentionally Claude-only for now: Codex and OpenCode keep using the
// verified herdr protocol until their terminal behavior has been exercised
// against a live installation.
type TmuxBackend struct {
// Socket is a tmux socket name (-L) or an absolute socket path (-S).
// An empty value uses the isolated socket name "orchestra".
Socket string
// Command is the Claude Code executable. An empty value resolves "claude"
// through PATH.
Command string
// Binary is test/packaging override for tmux itself.
Binary string
// LaunchConfirmTimeout and LaunchConfirmPoll bound ConfirmInput. They are
// tunable because how fast a terminal harness visibly reacts is a property
// of the host, not of this code. Zero values mean 10s and 250ms.
LaunchConfirmTimeout time.Duration
LaunchConfirmPoll time.Duration
// Now is a test seam for the confirmation deadline.
Now func() time.Time
}
func (b *TmuxBackend) now() time.Time {
if b.Now != nil {
return b.Now()
}
return time.Now()
}
func NewTmuxBackend(socket, command string) *TmuxBackend {
return &TmuxBackend{Socket: socket, Command: command}
}
func (b *TmuxBackend) Kind() string { return "tmux" }
func (b *TmuxBackend) binary() string {
if b.Binary != "" {
return b.Binary
}
return "tmux"
}
func (b *TmuxBackend) socketArgs() []string {
socket := b.Socket
if socket == "" {
socket = "orchestra"
}
if filepath.IsAbs(socket) {
return []string{"-S", socket}
}
return []string{"-L", socket}
}
func (b *TmuxBackend) command(ctx context.Context, args ...string) ([]byte, error) {
all := append(b.socketArgs(), args...)
out, err := exec.CommandContext(ctx, b.binary(), all...).CombinedOutput()
if err != nil {
return out, fmt.Errorf("tmux %s: %s: %w", strings.Join(args, " "), strings.TrimSpace(string(out)), err)
}
return out, nil
}
func (b *TmuxBackend) Check(ctx context.Context) error {
if _, err := exec.LookPath(b.binary()); err != nil {
return fmt.Errorf("tmux backend: %w", err)
}
// tmux -V does not require a server to exist. An idle backend is healthy
// and will create its isolated server with the first session.
if out, err := exec.CommandContext(ctx, b.binary(), "-V").CombinedOutput(); err != nil {
return fmt.Errorf("tmux backend: %s: %w", strings.TrimSpace(string(out)), err)
}
command := b.Command
if command == "" {
command = "claude"
}
if _, err := exec.LookPath(command); err != nil {
return fmt.Errorf("tmux backend Claude command: %w", err)
}
return nil
}
func (b *TmuxBackend) Worktree(_ context.Context, _ string, path, _ string) (string, error) {
info, err := os.Stat(path)
if err != nil {
return "", fmt.Errorf("tmux backend worktree: %w", err)
}
if !info.IsDir() {
return "", fmt.Errorf("tmux backend worktree %s is not a directory", path)
}
return path, nil
}
func tmuxSessionName(taskID string) string {
id := strings.Trim(invalidAgentName.ReplaceAllString(strings.ToLower(taskID), "-"), "-_")
if id == "" {
id = "session"
}
if len(id) > 36 {
id = strings.TrimRight(id[:36], "-_")
}
sum := sha256.Sum256([]byte(taskID))
return fmt.Sprintf("orchestra-%s-%x", id, sum[:4])
}
func tmuxSession(paneID string) string {
if before, _, ok := strings.Cut(paneID, ":"); ok {
return before
}
return paneID
}
func tmuxTarget(paneID string) string { return "=" + paneID }
func (b *TmuxBackend) hasSession(ctx context.Context, session string) (bool, error) {
out, err := b.command(ctx, "has-session", "-t", "="+session)
if err == nil {
return true, nil
}
message := strings.ToLower(string(out) + " " + err.Error())
// "no current target" is what a *running but empty* server answers: the
// target does not resolve and there is no current session to fall back to.
// It only started appearing once the runtime became its own unit running
// `tmux -D`, which keeps the server alive past its last pane (F32). Before
// that the server exited with its last session and answered "no server
// running", so this branch was never reached. Without it Kill returns an
// error for a pane that is already gone, quarantine never clears, and the
// worker's only session slot stays pinned.
if strings.Contains(message, "can't find session") || strings.Contains(message, "no server running") || strings.Contains(message, "no sessions") || strings.Contains(message, "no current target") || (strings.Contains(message, "error connecting to") && strings.Contains(message, "no such file")) {
return false, nil
}
return false, err
}
func (b *TmuxBackend) StartAgent(ctx context.Context, _, path, _, harness, taskID string) (Session, error) {
if !strings.EqualFold(harness, "claude") {
return Session{}, fmt.Errorf("tmux backend: harness %q is unsupported; only claude is enabled", harness)
}
if _, err := b.Worktree(ctx, "", path, ""); err != nil {
return Session{}, err
}
command := b.Command
if command == "" {
command = "claude"
}
resolved, err := exec.LookPath(command)
if err != nil {
return Session{}, fmt.Errorf("tmux backend Claude command: %w", err)
}
session := tmuxSessionName(taskID)
existing, err := b.hasSession(ctx, session)
if err != nil {
return Session{}, err
}
if !existing {
if _, err := b.command(ctx, "new-session", "-d", "-s", session, "-c", path, resolved); err != nil {
return Session{}, err
}
}
// The exact pane id is resolved below. Deployments may configure tmux
// base-index/base-pane-index, so neither index is assumed to be zero.
paneOut, err := b.command(ctx, "list-panes", "-t", "="+session, "-F", "#{session_name}:#{window_index}.#{pane_index}")
if err != nil {
return Session{}, err
}
paneID := strings.TrimSpace(strings.SplitN(string(paneOut), "\n", 2)[0])
if paneID == "" {
return Session{}, fmt.Errorf("tmux backend: session %s has no pane", session)
}
if existing {
out, err := b.command(ctx, "display-message", "-p", "-t", tmuxTarget(paneID), "#{pane_current_path}")
if err != nil {
return Session{}, err
}
current, currentErr := filepath.Abs(strings.TrimSpace(string(out)))
want, wantErr := filepath.Abs(path)
if currentErr != nil || wantErr != nil || current != want {
return Session{}, fmt.Errorf("tmux backend: existing session %s belongs to %q, not %q", session, current, want)
}
} else {
if _, err := b.command(ctx, "set-window-option", "-t", tmuxTarget(paneID), "remain-on-exit", "on"); err != nil {
return Session{}, err
}
}
s := Session{PaneID: paneID, Worktree: path, Harness: "claude", AgentName: session}
if err := b.confirmClaudeWorkspaceTrust(ctx, s); err != nil {
return Session{}, err
}
status, err := b.AgentStatus(ctx, s)
if err != nil {
return Session{}, err
}
if status == "exited" || status == "dead" {
return Session{}, fmt.Errorf("tmux backend: Claude exited while starting session %s", session)
}
return s, nil
}
func (b *TmuxBackend) confirmClaudeWorkspaceTrust(ctx context.Context, s Session) error {
deadline := time.Now().Add(15 * time.Second)
accepted := false
for {
text, err := b.PaneCapture(ctx, s, "recent")
if err == nil && claudeWorkspaceTrustPrompt(text) {
if !accepted {
if err := b.SendText(ctx, s, "1"); err != nil {
return fmt.Errorf("tmux backend: accept Claude workspace trust: %w", err)
}
if err := b.SendKeys(ctx, s, []string{"Enter"}); err != nil {
return fmt.Errorf("tmux backend: accept Claude workspace trust: %w", err)
}
accepted = true
}
}
// Claude's input prompt is the readiness boundary. A banner or partially
// painted fullscreen UI is not enough: input sent there can be lost.
if err == nil && !claudeWorkspaceTrustPrompt(text) && strings.Contains(text, "") {
return nil
}
if time.Now().After(deadline) {
return fmt.Errorf("tmux backend: Claude input prompt did not become ready in session %s", tmuxSession(s.PaneID))
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(100 * time.Millisecond):
}
}
}
func (b *TmuxBackend) Prompt(ctx context.Context, pane, text string, _ time.Duration) error {
s := Session{PaneID: pane}
status, err := b.AgentStatus(ctx, s)
if err != nil {
return err
}
if status == "blocked" {
return fmt.Errorf("tmux backend: refusing prompt while pane %s shows a permission dialog", pane)
}
if err := b.SendText(ctx, s, text); err != nil {
return err
}
return b.SendKeys(ctx, s, []string{"Enter"})
}
func (b *TmuxBackend) Kill(ctx context.Context, s Session) error {
exists, err := b.hasSession(ctx, tmuxSession(s.PaneID))
if err != nil {
return err
}
if !exists {
return nil
}
_, err = b.command(ctx, "kill-session", "-t", "="+tmuxSession(s.PaneID))
return err
}
func (b *TmuxBackend) paneState(ctx context.Context, s Session) (dead bool, command string, err error) {
out, err := b.command(ctx, "display-message", "-p", "-t", tmuxTarget(s.PaneID), "#{pane_dead}\t#{pane_current_command}")
if err != nil {
return false, "", err
}
parts := strings.SplitN(strings.TrimSpace(string(out)), "\t", 2)
dead = len(parts) > 0 && parts[0] == "1"
if len(parts) == 2 {
command = parts[1]
}
return dead, command, nil
}
func (b *TmuxBackend) AgentStatus(ctx context.Context, s Session) (string, error) {
exists, err := b.hasSession(ctx, tmuxSession(s.PaneID))
if err != nil {
return "", err
}
if !exists {
return "exited", nil
}
dead, command, err := b.paneState(ctx, s)
if err != nil {
return "", err
}
if dead || command == "" {
return "exited", nil
}
text, err := b.PaneCapture(ctx, s, "recent")
if err != nil {
return "", err
}
if permissionPrompt(text) {
return "blocked", nil
}
lower := strings.ToLower(text)
for _, marker := range []string{"esc to interrupt", "ctrl+c to interrupt", "press esc to interrupt"} {
if strings.Contains(lower, marker) {
return "busy", nil
}
}
return "idle", nil
}
func (b *TmuxBackend) PaneCapture(ctx context.Context, s Session, source string) (string, error) {
start := "-200"
if source != "" && source != "recent" {
start = "-1000"
}
out, err := b.command(ctx, "capture-pane", "-p", "-J", "-S", start, "-t", tmuxTarget(s.PaneID))
return string(out), err
}
func (b *TmuxBackend) SendText(ctx context.Context, s Session, text string) error {
_, err := b.command(ctx, "send-keys", "-t", tmuxTarget(s.PaneID), "-l", "--", text)
return err
}
func (b *TmuxBackend) SendKeys(ctx context.Context, s Session, keys []string) error {
if len(keys) == 0 {
return nil
}
for _, key := range keys {
if strings.TrimSpace(key) == "" {
return errors.New("tmux backend: empty key name")
}
}
args := []string{"send-keys", "-t", tmuxTarget(s.PaneID)}
args = append(args, keys...)
_, err := b.command(ctx, args...)
return err
}
// tmux has no separate agent binding to release. Keeping the session alive is
// the tmux equivalent of herdr's split-then-close protocol; the worker kills
// it only after successor pickup has been validated.
func (b *TmuxBackend) ReleaseAgent(ctx context.Context, s Session, _ string) error {
exists, err := b.hasSession(ctx, tmuxSession(s.PaneID))
if err != nil {
return err
}
if !exists {
return fmt.Errorf("tmux backend: session %s is not running", tmuxSession(s.PaneID))
}
return nil
}
var _ Backend = (*TmuxBackend)(nil)
// LaunchTransport keeps the file-reference launch scoped to the harness whose
// paste handling requires it, rather than making it the universal prompt
// format. Every other harness keeps the inline path it was verified on.
func (b *TmuxBackend) LaunchTransport(harness string) LaunchTransport {
if harness == "claude" {
return LaunchFileRef
}
return LaunchInline
}
// promptLine matches a harness input line. A queued or already-accepted
// message renders with the same prefix, so a match alone proves nothing about
// what is still unsubmitted; see inputState.
var promptLine = regexp.MustCompile(`(?m)^[ \t]*[>][ \t]*(.*)$`)
// separatorRow matches the rule Claude Code draws above and below its editor.
var separatorRow = regexp.MustCompile(`^[\s─━┄┅┈┉-]+$`)
// InputState is what the interactive editor holds. Active distinguishes the
// live editor from queued or already-submitted input: both render with the
// same "" prefix, but only the live editor owns the pane cursor (F17,
// found live during burn-in run 3).
type InputState struct {
Text string
Active bool
}
// sameInput compares editor content to what was submitted. Soft wrapping may
// break the text at any column and re-indent the continuation, so the
// comparison ignores whitespace entirely.
func sameInput(a, b string) bool {
strip := func(s string) string { return strings.Join(strings.Fields(s), "") }
return strip(a) == strip(b)
}
// inputState reads the editor that owns the cursor. Scanning every line
// beginning with "" cannot tell an unsubmitted prompt from one the
// harness already queued, which is why this asks tmux where the cursor is.
func (b *TmuxBackend) inputState(ctx context.Context, s Session) (InputState, error) {
out, err := b.command(ctx, "display-message", "-p", "-t", tmuxTarget(s.PaneID), "#{cursor_y}")
if err != nil {
return InputState{}, err
}
row, err := strconv.Atoi(strings.TrimSpace(string(out)))
if err != nil {
return InputState{}, fmt.Errorf("tmux backend: cursor row %q: %w", strings.TrimSpace(string(out)), err)
}
// Screen rows, unjoined: -J merges wrapped rows and would invalidate the
// cursor row index.
raw, err := b.command(ctx, "capture-pane", "-p", "-t", tmuxTarget(s.PaneID))
if err != nil {
return InputState{}, err
}
rows := strings.Split(strings.TrimRight(string(raw), "\n"), "\n")
if row < 0 || row >= len(rows) {
return InputState{}, nil
}
start := -1
for i := row; i >= 0; i-- {
if promptLine.MatchString(rows[i]) {
start = i
break
}
if separatorRow.MatchString(rows[i]) {
break
}
}
if start < 0 {
return InputState{}, nil
}
parts := []string{strings.TrimSpace(promptLine.FindStringSubmatch(rows[start])[1])}
for i := start + 1; i <= row; i++ {
parts = append(parts, strings.TrimSpace(rows[i]))
}
return InputState{Text: strings.TrimSpace(strings.Join(parts, " ")), Active: true}, nil
}
// PaneProgress hashes what the harness produced, not what someone typed at it.
// The capture joins wrapped lines, so an input block is one line beginning
// with the prompt marker and dropping those lines removes it whole. Found live
// during burn-in run 3: unexplained keystrokes in a pane kept a stalled lease
// renewing, because the renewal check hashed the whole capture.
func (b *TmuxBackend) PaneProgress(ctx context.Context, s Session) (string, error) {
text, err := b.PaneCapture(ctx, s, "recent")
if err != nil {
return "", err
}
var kept []string
for _, line := range strings.Split(text, "\n") {
if promptLine.MatchString(line) {
continue
}
kept = append(kept, line)
}
return strings.Join(kept, "\n"), nil
}
// launchResubmitLimit bounds the intervention independently of the observation
// deadline. A slow TUI must not receive a fortieth Enter after it accepted the
// first: three exact-editor resubmits, then observation only.
const launchResubmitLimit = 3
// launchSettle is how long an empty editor must stay empty before it counts as
// proof of submission. A TUI takes tens of milliseconds to render pasted text,
// and the first poll can run inside that gap. F33: a poll 7ms after the submit
// found nothing in the editor, reported confirmation=editor_cleared, and the
// launch text then sat unsent in the input box for the rest of the lease. The
// two launches that actually worked took ~500ms and a second Enter, so the
// difference between success and this false positive was scheduling luck.
//
// Only the never-observed case waits. Text seen and then gone is real
// evidence, and busy, blocked or queued still confirm immediately.
const launchSettle = time.Second
// ConfirmInput drives the submit to a decision instead of assuming one Enter
// landed. Burn-in run 3 proved the submit is not deterministic: the text
// reached the editor on all three attempts and the following Enter never took
// effect. So this resends Enter while the live editor still holds exactly what
// was submitted, up to launchResubmitLimit times, then observes until the
// deadline.
//
// The returned evidence records how many submits it took, which is the only
// way to tell a harness that needs a second Enter from one that needed none.
func (b *TmuxBackend) ConfirmInput(ctx context.Context, s Session, submitted string) (string, error) {
timeout, poll := b.LaunchConfirmTimeout, b.LaunchConfirmPoll
if timeout <= 0 {
timeout = 10 * time.Second
}
if poll <= 0 {
poll = 250 * time.Millisecond
}
first := b.now()
deadline := first.Add(timeout)
attempts, resubmits := 1, 0
var lastResubmit time.Time
evidence := func(kind string) string {
return fmt.Sprintf("confirmation=%s submit_attempts=%d first_submit_at=%s confirmed_at=%s",
kind, attempts, first.UTC().Format(time.RFC3339Nano), b.now().UTC().Format(time.RFC3339Nano))
}
var last InputState
// Whether the editor was ever observed holding what we submitted. Without
// it, "empty" is indistinguishable from "not rendered yet".
seen := false
for {
state, err := b.inputState(ctx, s)
if err != nil {
return "", err
}
last = state
if state.Active && sameInput(state.Text, submitted) {
seen = true
}
switch {
case state.Active && sameInput(state.Text, submitted):
// Exactly what was submitted still owns the cursor, so the submit
// did not take. Resend Enter, spaced so a TUI that accepted the
// previous one cannot receive another inside its own redraw.
if resubmits < launchResubmitLimit && (lastResubmit.IsZero() || b.now().Sub(lastResubmit) >= 2*poll) {
if err := b.SendKeys(ctx, s, []string{"Enter"}); err != nil {
return "", err
}
resubmits++
attempts++
lastResubmit = b.now()
}
case state.Active && queuedInput(state.Text):
// The harness accepted the instruction and parked it behind the
// current turn. Queued is submitted.
return evidence("queued"), nil
case state.Active && state.Text != "":
// Something else is in the editor, a paste placeholder for
// instance. Not proof of acceptance, so keep observing.
default:
status, statusErr := b.AgentStatus(ctx, s)
if statusErr != nil {
return "", statusErr
}
switch status {
case "exited":
return "", fmt.Errorf("%w: pane %s exited before the harness reacted", ErrPromptNotSubmitted, s.PaneID)
case "busy":
return evidence("busy"), nil
case "blocked":
// A permission dialog is the harness acting on the
// instruction, so submission is proven even though the
// session now needs an operator.
return evidence("blocked"), nil
default:
// An editor that never held the text may simply not have
// rendered it yet. Give it the settle window before calling an
// empty box proof that the harness took the prompt.
if !seen && b.now().Sub(first) < launchSettle {
break
}
return evidence("editor_cleared"), nil
}
}
if !b.now().Before(deadline) {
return "", fmt.Errorf("%w: pane %s still holds %q after %s and %d submits", ErrPromptNotSubmitted, s.PaneID, last.Text, timeout, attempts)
}
select {
case <-ctx.Done():
return "", ctx.Err()
case <-time.After(poll):
}
}
}
// queuedInput recognizes the editor Claude Code shows once it has taken a
// prompt and parked it behind the running turn.
func queuedInput(text string) bool {
return strings.Contains(strings.ToLower(text), "queued message")
}