feat(orchestrator): milestone rotation and thrash detection (S11)
Closes the last two S11 triggers. internal/herdr/activity.go normalizes tool/function calls per harness (ClaudeActivity verified against the existing transcript format, CodexActivity best-effort/unverified, OpenCodeActivity refuses — no confirmed per-tool-call source exists) and implements the three thrash rules plus a narrow milestone check (successful git commit as the last call). CLIAdapter.RequestHandoffReason asks the agent to write a handoff with meta.reason set, same "ask, don't invent" pattern as the existing handoff/ report requests. rotate() and TurnDecision generalize the manual-bypass shortcut to manual/milestone/thrash and request (never directly release) on a detected trigger. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01W1rkJ2hBMybnJctPbcy4tT
This commit is contained in:
kami
@@ -0,0 +1,314 @@
|
||||
package herdr
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"orchestra/internal/continuity"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ToolCall is a harness-agnostic normalization of one tool/function
|
||||
// invocation and its outcome, extracted from a harness's own transcript or
|
||||
// rollout — the source S11 named as missing ("transcript/tool-call
|
||||
// introspection this repo doesn't have a source for"). Kind distinguishes
|
||||
// what Key means: "file" (Key is a repo-relative or absolute path, from a
|
||||
// tool whose input carried file_path) or "command" (Key is a shell command
|
||||
// string). A call whose input matched neither shape has Kind=="" and Key=="" —
|
||||
// still counted for occupancy-adjacent bookkeeping elsewhere, but ignored by
|
||||
// the detectors below, which only reason about file edits and commands.
|
||||
type ToolCall struct {
|
||||
Name string
|
||||
Kind string
|
||||
Key string
|
||||
Success bool
|
||||
IsTest bool
|
||||
}
|
||||
|
||||
// ActivityReader is optional, mirroring every other Face-B capability in
|
||||
// this package (TurnBoundary, HandoffRequester, ...): an adapter without a
|
||||
// tool-call source for its harness simply doesn't implement it, and callers
|
||||
// (Coordinator.rotate, TurnDecision) skip thrash/milestone detection rather
|
||||
// than guessing from an absent signal.
|
||||
type ActivityReader interface {
|
||||
Activity(context.Context, Session) ([]ToolCall, error)
|
||||
}
|
||||
|
||||
// editToolNames are the tool/function names whose input is a full file
|
||||
// rewrite or patch — the ones the "same file edited too many times without
|
||||
// resolving" thrash rule cares about. Read/Grep/Glob-style tools also carry
|
||||
// a file_path in their input (Kind=="file") but are not edits and must not
|
||||
// count toward this rule.
|
||||
var editToolNames = map[string]bool{
|
||||
"Edit": true, "Write": true, "MultiEdit": true, "NotebookEdit": true,
|
||||
// Codex/opencode function names are unverified against a live rollout
|
||||
// (see CodexActivity's doc comment) — these are best-effort guesses at
|
||||
// what an edit-shaped tool call would be named there.
|
||||
"apply_patch": true, "edit_file": true, "patch_file": true,
|
||||
}
|
||||
|
||||
func isEditTool(name string) bool { return editToolNames[name] }
|
||||
|
||||
// testCommandRe matches shell invocations that run a test suite, for the
|
||||
// "N consecutive failed test runs" thrash rule (spec §5.3).
|
||||
var testCommandRe = regexp.MustCompile(`(?i)\b(go test|pytest|py\.test|npm (run )?test|yarn test|pnpm test|cargo test|make test|jest|mvn test|rspec|ctest)\b`)
|
||||
|
||||
// toolCallMeta pulls (kind, key) out of a tool call's JSON input/arguments —
|
||||
// shared by every harness's parser since Claude's tool "input" and Codex's
|
||||
// function "arguments" are both a flat JSON object using the same
|
||||
// conventional field names (file_path, command).
|
||||
func toolCallMeta(input json.RawMessage) (kind, key string) {
|
||||
var m map[string]any
|
||||
if json.Unmarshal(input, &m) != nil {
|
||||
return "", ""
|
||||
}
|
||||
if v, ok := m["file_path"].(string); ok && v != "" {
|
||||
return "file", v
|
||||
}
|
||||
if v, ok := m["command"].(string); ok && v != "" {
|
||||
return "command", strings.TrimSpace(v)
|
||||
}
|
||||
return "", ""
|
||||
}
|
||||
|
||||
// ClaudeActivity parses a Claude Code transcript's assistant tool_use blocks
|
||||
// paired with their matching user tool_result blocks — the same file
|
||||
// ClaudeSessionFile/ClaudeUsage already open, so no new resolution path is
|
||||
// needed. Each tool_use is matched to its result by tool_use_id; an
|
||||
// unresolved tool_use (session still mid-turn) is simply dropped, not
|
||||
// reported as a call.
|
||||
func ClaudeActivity(path string) ([]ToolCall, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
type block struct {
|
||||
Type string `json:"type"`
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Input json.RawMessage `json:"input"`
|
||||
ToolUseID string `json:"tool_use_id"`
|
||||
IsError bool `json:"is_error"`
|
||||
}
|
||||
type entry struct {
|
||||
Message struct {
|
||||
Content []block `json:"content"`
|
||||
} `json:"message"`
|
||||
}
|
||||
|
||||
pending := map[string]ToolCall{}
|
||||
var calls []ToolCall
|
||||
s := bufio.NewScanner(f)
|
||||
s.Buffer(make([]byte, 1<<20), 10<<20)
|
||||
for s.Scan() {
|
||||
var e entry
|
||||
if json.Unmarshal(s.Bytes(), &e) != nil {
|
||||
continue
|
||||
}
|
||||
for _, b := range e.Message.Content {
|
||||
switch b.Type {
|
||||
case "tool_use":
|
||||
kind, key := toolCallMeta(b.Input)
|
||||
pending[b.ID] = ToolCall{Name: b.Name, Kind: kind, Key: key, IsTest: isTestCommand(kind, key)}
|
||||
case "tool_result":
|
||||
if tc, ok := pending[b.ToolUseID]; ok {
|
||||
tc.Success = !b.IsError
|
||||
calls = append(calls, tc)
|
||||
delete(pending, b.ToolUseID)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return calls, s.Err()
|
||||
}
|
||||
|
||||
// CodexActivity is best-effort and unverified against a live Codex rollout —
|
||||
// unlike ClaudeActivity, which reads the same transcript shape ClaudeUsage
|
||||
// already confirmed. Codex's Responses-API-style rollout is expected to log
|
||||
// a "function_call" payload (name, arguments, call_id) followed later by a
|
||||
// "function_call_output" payload (call_id, output), mirroring the
|
||||
// "payload.type" wrapper CodexUsage already reads for "token_count". Success
|
||||
// is inferred heuristically from the output text (no confirmed structured
|
||||
// exit-code field), which is coarser than Claude's explicit is_error flag.
|
||||
// Treat any thrash/milestone signal derived from this as advisory until
|
||||
// checked against a real rollout, same caveat this file's other Codex-facing
|
||||
// code already carries (AUDIT.md Phase 0's own bar: verify before trusting).
|
||||
func CodexActivity(path string) ([]ToolCall, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
type payload struct {
|
||||
Type string `json:"type"`
|
||||
CallID string `json:"call_id"`
|
||||
Name string `json:"name"`
|
||||
Arguments json.RawMessage `json:"arguments"`
|
||||
Output string `json:"output"`
|
||||
}
|
||||
type entry struct {
|
||||
Payload payload `json:"payload"`
|
||||
}
|
||||
|
||||
pending := map[string]ToolCall{}
|
||||
var calls []ToolCall
|
||||
s := bufio.NewScanner(f)
|
||||
s.Buffer(make([]byte, 1<<20), 10<<20)
|
||||
for s.Scan() {
|
||||
var e entry
|
||||
if json.Unmarshal(s.Bytes(), &e) != nil {
|
||||
continue
|
||||
}
|
||||
switch e.Payload.Type {
|
||||
case "function_call":
|
||||
kind, key := toolCallMeta(e.Payload.Arguments)
|
||||
pending[e.Payload.CallID] = ToolCall{Name: e.Payload.Name, Kind: kind, Key: key, IsTest: isTestCommand(kind, key)}
|
||||
case "function_call_output":
|
||||
if tc, ok := pending[e.Payload.CallID]; ok {
|
||||
tc.Success = !strings.Contains(strings.ToLower(e.Payload.Output), "error")
|
||||
calls = append(calls, tc)
|
||||
delete(pending, e.Payload.CallID)
|
||||
}
|
||||
}
|
||||
}
|
||||
return calls, s.Err()
|
||||
}
|
||||
|
||||
// OpenCodeActivity has no verified source. OpenCodeUsage already only reads
|
||||
// aggregate token counts per message file (~/.local/share/opencode/storage/
|
||||
// message/), not per-tool-call records, and nobody has confirmed opencode's
|
||||
// on-disk format exposes tool name/input/outcome at that granularity the way
|
||||
// Claude's transcript and (probably) Codex's rollout do. Refusing loudly here
|
||||
// matches the existing convention in this file (CLIAdapter.resolveSessionFile's
|
||||
// opencode default case) rather than fabricating a parser against an unknown
|
||||
// shape.
|
||||
func OpenCodeActivity(string) ([]ToolCall, error) {
|
||||
return nil, fmt.Errorf("herdr: opencode tool-call activity has no verified source yet; check the message storage format against a live session first")
|
||||
}
|
||||
|
||||
func isTestCommand(kind, key string) bool {
|
||||
return kind == "command" && testCommandRe.MatchString(key)
|
||||
}
|
||||
|
||||
// ThrashConfig tunes DetectThrash's three independent circuit breakers (spec
|
||||
// §5.3: "N failed test runs / same file M times / identical tool calls").
|
||||
// A zero value for any field means "use the package default" — see
|
||||
// withDefaults — so a zero-value ThrashConfig{} is usable as-is.
|
||||
type ThrashConfig struct {
|
||||
MaxConsecutiveTestFailures int
|
||||
MaxSameFileEdits int
|
||||
MaxIdenticalCalls int
|
||||
}
|
||||
|
||||
func (c ThrashConfig) withDefaults() ThrashConfig {
|
||||
if c.MaxConsecutiveTestFailures <= 0 {
|
||||
c.MaxConsecutiveTestFailures = 3
|
||||
}
|
||||
if c.MaxSameFileEdits <= 0 {
|
||||
c.MaxSameFileEdits = 5
|
||||
}
|
||||
if c.MaxIdenticalCalls <= 0 {
|
||||
c.MaxIdenticalCalls = 4
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// DetectThrash evaluates a window of recent tool calls (oldest first, as
|
||||
// returned by an ActivityReader) against the three thrash rules named in
|
||||
// §5.3, returning whether any tripped and a populated continuity.DeadEnd per
|
||||
// rule that did — meant to be handed straight to
|
||||
// CLIAdapter.RequestHandoffReason so the resulting handoff's dead_ends
|
||||
// (§6.1) aren't left for the agent to invent from scratch.
|
||||
func DetectThrash(calls []ToolCall, cfg ThrashConfig) (bool, []continuity.DeadEnd) {
|
||||
cfg = cfg.withDefaults()
|
||||
var deadEnds []continuity.DeadEnd
|
||||
|
||||
// Rule 1: N consecutive failed test runs, most recent first.
|
||||
streak := 0
|
||||
var lastFailedCmd string
|
||||
for i := len(calls) - 1; i >= 0; i-- {
|
||||
c := calls[i]
|
||||
if !c.IsTest {
|
||||
continue
|
||||
}
|
||||
if c.Success {
|
||||
break
|
||||
}
|
||||
streak++
|
||||
lastFailedCmd = c.Key
|
||||
}
|
||||
if streak >= cfg.MaxConsecutiveTestFailures {
|
||||
deadEnds = append(deadEnds, continuity.DeadEnd{
|
||||
Tried: lastFailedCmd,
|
||||
WhyFailed: fmt.Sprintf("failed %d consecutive times", streak),
|
||||
})
|
||||
}
|
||||
|
||||
// Rule 2: the same file edited M times without the loop resolving.
|
||||
editCounts := map[string]int{}
|
||||
for _, c := range calls {
|
||||
if c.Kind != "file" || c.Key == "" || !isEditTool(c.Name) {
|
||||
continue
|
||||
}
|
||||
editCounts[c.Key]++
|
||||
}
|
||||
for path, n := range editCounts {
|
||||
if n >= cfg.MaxSameFileEdits {
|
||||
deadEnds = append(deadEnds, continuity.DeadEnd{
|
||||
Tried: "editing " + path,
|
||||
WhyFailed: fmt.Sprintf("edited %d times without resolving", n),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Rule 3: the identical tool call (same name+kind+key) repeated back to
|
||||
// back, a sign of a stuck retry loop rather than incremental progress.
|
||||
// Deliberately excludes test commands (re-running the same test after a
|
||||
// fix attempt is expected — that's rule 1's job to catch if it keeps
|
||||
// failing) and non-edit file calls like Read (re-reading the same file
|
||||
// across turns is normal and not a sign of being stuck).
|
||||
run := 0
|
||||
var prevName, prevKind, prevKey string
|
||||
for _, c := range calls {
|
||||
relevant := (c.Kind == "command" && !c.IsTest) || (c.Kind == "file" && isEditTool(c.Name))
|
||||
if !relevant || c.Key == "" {
|
||||
run = 0
|
||||
continue
|
||||
}
|
||||
if c.Name == prevName && c.Kind == prevKind && c.Key == prevKey {
|
||||
run++
|
||||
} else {
|
||||
run = 1
|
||||
prevName, prevKind, prevKey = c.Name, c.Kind, c.Key
|
||||
}
|
||||
if run >= cfg.MaxIdenticalCalls {
|
||||
deadEnds = append(deadEnds, continuity.DeadEnd{
|
||||
Tried: fmt.Sprintf("%s(%s)", c.Name, c.Key),
|
||||
WhyFailed: fmt.Sprintf("repeated identically %d times in a row", run),
|
||||
})
|
||||
run = 0 // one dead end per repeated run is enough
|
||||
}
|
||||
}
|
||||
|
||||
return len(deadEnds) > 0, deadEnds
|
||||
}
|
||||
|
||||
// DetectMilestone recognizes the simplest unambiguous "a coherent unit of
|
||||
// work just finished" signal available from tool-call history alone (spec
|
||||
// §5.3): the most recent call was a successful `git commit`. Anything
|
||||
// fuzzier (a passing test suite, a finished subtask) needs a definition of
|
||||
// "coherent unit" this repo has no source for yet, so it is deliberately not
|
||||
// guessed at here.
|
||||
func DetectMilestone(calls []ToolCall) bool {
|
||||
if len(calls) == 0 {
|
||||
return false
|
||||
}
|
||||
last := calls[len(calls)-1]
|
||||
return last.Success && last.Kind == "command" && strings.HasPrefix(last.Key, "git commit")
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
package herdr
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func writeJSONL(t *testing.T, lines []string) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "transcript.jsonl")
|
||||
if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")+"\n"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func claudeToolLine(t *testing.T, toolUseID, name string, input map[string]any) string {
|
||||
t.Helper()
|
||||
b, err := json.Marshal(map[string]any{
|
||||
"message": map[string]any{
|
||||
"content": []map[string]any{
|
||||
{"type": "tool_use", "id": toolUseID, "name": name, "input": input},
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func claudeResultLine(t *testing.T, toolUseID string, isError bool) string {
|
||||
t.Helper()
|
||||
b, err := json.Marshal(map[string]any{
|
||||
"message": map[string]any{
|
||||
"content": []map[string]any{
|
||||
{"type": "tool_result", "tool_use_id": toolUseID, "is_error": isError},
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func TestClaudeActivityPairsToolUseWithResult(t *testing.T) {
|
||||
path := writeJSONL(t, []string{
|
||||
claudeToolLine(t, "t1", "Bash", map[string]any{"command": "go test ./..."}),
|
||||
claudeResultLine(t, "t1", true),
|
||||
claudeToolLine(t, "t2", "Edit", map[string]any{"file_path": "main.go"}),
|
||||
claudeResultLine(t, "t2", false),
|
||||
// an unresolved tool_use (still mid-turn) must be dropped, not reported
|
||||
claudeToolLine(t, "t3", "Bash", map[string]any{"command": "echo hi"}),
|
||||
})
|
||||
calls, err := ClaudeActivity(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(calls) != 2 {
|
||||
t.Fatalf("calls=%+v, want 2 (unresolved tool_use dropped)", calls)
|
||||
}
|
||||
if calls[0].Name != "Bash" || calls[0].Kind != "command" || calls[0].Key != "go test ./..." || calls[0].Success || !calls[0].IsTest {
|
||||
t.Fatalf("calls[0]=%+v", calls[0])
|
||||
}
|
||||
if calls[1].Name != "Edit" || calls[1].Kind != "file" || calls[1].Key != "main.go" || !calls[1].Success {
|
||||
t.Fatalf("calls[1]=%+v", calls[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectThrashConsecutiveTestFailures(t *testing.T) {
|
||||
calls := []ToolCall{
|
||||
{Name: "Bash", Kind: "command", Key: "go test ./...", Success: false, IsTest: true},
|
||||
{Name: "Bash", Kind: "command", Key: "go test ./...", Success: false, IsTest: true},
|
||||
{Name: "Bash", Kind: "command", Key: "go test ./...", Success: false, IsTest: true},
|
||||
}
|
||||
thrash, deadEnds := DetectThrash(calls, ThrashConfig{})
|
||||
if !thrash {
|
||||
t.Fatalf("want thrash on 3 consecutive test failures (default threshold)")
|
||||
}
|
||||
if len(deadEnds) != 1 || deadEnds[0].Tried != "go test ./..." {
|
||||
t.Fatalf("deadEnds=%+v", deadEnds)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectThrashStopsCountingAtASuccess(t *testing.T) {
|
||||
calls := []ToolCall{
|
||||
{Name: "Bash", Kind: "command", Key: "go test ./...", Success: false, IsTest: true},
|
||||
{Name: "Bash", Kind: "command", Key: "go test ./...", Success: false, IsTest: true},
|
||||
{Name: "Bash", Kind: "command", Key: "go test ./...", Success: true, IsTest: true},
|
||||
{Name: "Bash", Kind: "command", Key: "go test ./...", Success: false, IsTest: true},
|
||||
}
|
||||
// Only one failure since the last success — must not thrash on the
|
||||
// default threshold of 3, even though 3 failures exist in the window.
|
||||
if thrash, _ := DetectThrash(calls, ThrashConfig{}); thrash {
|
||||
t.Fatalf("want no thrash: only 1 failure since the last passing run")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectThrashSameFileEditedTooManyTimes(t *testing.T) {
|
||||
var calls []ToolCall
|
||||
for i := 0; i < 5; i++ {
|
||||
calls = append(calls, ToolCall{Name: "Edit", Kind: "file", Key: "main.go", Success: true})
|
||||
}
|
||||
thrash, deadEnds := DetectThrash(calls, ThrashConfig{})
|
||||
if !thrash {
|
||||
t.Fatalf("want thrash on 5 edits to the same file (default threshold)")
|
||||
}
|
||||
found := false
|
||||
for _, d := range deadEnds {
|
||||
if strings.Contains(d.Tried, "main.go") {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("deadEnds=%+v, want an entry naming main.go", deadEnds)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectThrashIgnoresReadsForFileEditCount(t *testing.T) {
|
||||
var calls []ToolCall
|
||||
for i := 0; i < 10; i++ {
|
||||
calls = append(calls, ToolCall{Name: "Read", Kind: "file", Key: "main.go", Success: true})
|
||||
}
|
||||
// Read is not an edit tool; repeated reads of the same file must never
|
||||
// trip the "edited too many times" rule.
|
||||
if thrash, _ := DetectThrash(calls, ThrashConfig{}); thrash {
|
||||
t.Fatalf("want no thrash: Read is not an edit tool")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectThrashIdenticalCallsRepeated(t *testing.T) {
|
||||
var calls []ToolCall
|
||||
for i := 0; i < 4; i++ {
|
||||
calls = append(calls, ToolCall{Name: "Bash", Kind: "command", Key: "npm install lodash", Success: false})
|
||||
}
|
||||
thrash, _ := DetectThrash(calls, ThrashConfig{})
|
||||
if !thrash {
|
||||
t.Fatalf("want thrash on the identical call repeated 4 times (default threshold)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectThrashCustomThresholds(t *testing.T) {
|
||||
calls := []ToolCall{
|
||||
{Name: "Edit", Kind: "file", Key: "main.go", Success: true},
|
||||
{Name: "Edit", Kind: "file", Key: "main.go", Success: true},
|
||||
}
|
||||
if thrash, _ := DetectThrash(calls, ThrashConfig{}); thrash {
|
||||
t.Fatalf("2 edits must not thrash at the default threshold of 5")
|
||||
}
|
||||
if thrash, _ := DetectThrash(calls, ThrashConfig{MaxSameFileEdits: 2}); !thrash {
|
||||
t.Fatalf("2 edits must thrash once the threshold is lowered to 2")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectMilestoneOnSuccessfulCommit(t *testing.T) {
|
||||
calls := []ToolCall{
|
||||
{Name: "Bash", Kind: "command", Key: "go build ./...", Success: true},
|
||||
{Name: "Bash", Kind: "command", Key: "git commit -m fix", Success: true},
|
||||
}
|
||||
if !DetectMilestone(calls) {
|
||||
t.Fatalf("want milestone: last call is a successful git commit")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectMilestoneRequiresSuccess(t *testing.T) {
|
||||
calls := []ToolCall{
|
||||
{Name: "Bash", Kind: "command", Key: "git commit -m fix", Success: false},
|
||||
}
|
||||
if DetectMilestone(calls) {
|
||||
t.Fatalf("want no milestone: the commit failed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectMilestoneRequiresLastCall(t *testing.T) {
|
||||
calls := []ToolCall{
|
||||
{Name: "Bash", Kind: "command", Key: "git commit -m fix", Success: true},
|
||||
{Name: "Bash", Kind: "command", Key: "go test ./...", Success: true},
|
||||
}
|
||||
if DetectMilestone(calls) {
|
||||
t.Fatalf("want no milestone: the commit isn't the most recent call")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenCodeActivityRefusesRatherThanGuess(t *testing.T) {
|
||||
if _, err := OpenCodeActivity("/nonexistent"); err == nil {
|
||||
t.Fatalf("want an error: opencode tool-call activity has no verified source")
|
||||
}
|
||||
}
|
||||
@@ -136,6 +136,60 @@ func (a CLIAdapter) RequestHandoff(ctx context.Context, s Session) error {
|
||||
return a.Client.Prompt(ctx, s.PaneID, handoffPrompt, time.Minute)
|
||||
}
|
||||
|
||||
// ReasonedHandoffRequester is RequestHandoff's counterpart for the two
|
||||
// orchestrator-detected triggers (S11: milestone, thrash) rather than the
|
||||
// occupancy-driven ones. It exists separately from HandoffRequester because
|
||||
// these prompts need to say *why* — naming the detected dead ends for thrash,
|
||||
// or the recognized completion point for milestone — instead of the generic
|
||||
// "context budget reached" framing handoffPrompt uses.
|
||||
type ReasonedHandoffRequester interface {
|
||||
RequestHandoffReason(ctx context.Context, s Session, reason string, deadEnds []continuity.DeadEnd) error
|
||||
}
|
||||
|
||||
func (a CLIAdapter) RequestHandoffReason(ctx context.Context, s Session, reason string, deadEnds []continuity.DeadEnd) error {
|
||||
var sb strings.Builder
|
||||
fmt.Fprintf(&sb, "Orchestra has detected a %q rotation trigger for this task.\n", reason)
|
||||
switch reason {
|
||||
case "thrash":
|
||||
sb.WriteString("Signs of thrashing were detected (repeated failing test runs, repeated edits to the same file, or the same tool call repeated back to back). Stop the current approach rather than trying it again.\n")
|
||||
case "milestone":
|
||||
sb.WriteString("A coherent unit of work looks complete (a successful commit). If the next step is independent of what you just did, this is a good point to hand off.\n")
|
||||
}
|
||||
fmt.Fprintf(&sb, "Before you stop, write a §6.1 handoff to %s at the worktree root with meta.reason=%q", HandoffFile, reason)
|
||||
if len(deadEnds) > 0 {
|
||||
sb.WriteString(" and a dead_ends entry for each of the following:\n")
|
||||
for _, d := range deadEnds {
|
||||
fmt.Fprintf(&sb, "- tried: %q, why_failed: %q\n", d.Tried, d.WhyFailed)
|
||||
}
|
||||
} else {
|
||||
sb.WriteString(".\n")
|
||||
}
|
||||
sb.WriteString("Use the same anchor convention as any other handoff: the real git_sha via 'git rev-parse HEAD', the real branch, and a real sha256 of any uncommitted files — never fabricated. Do not edit TASK.md. Once written, stop normally.")
|
||||
return a.Client.Prompt(ctx, s.PaneID, sb.String(), time.Minute)
|
||||
}
|
||||
|
||||
// Activity resolves the harness's tool-call history the same way Occupancy
|
||||
// resolves its session file (Session.SessionFile if set, otherwise a fresh
|
||||
// per-harness lookup), then dispatches to the harness-specific parser.
|
||||
func (a CLIAdapter) Activity(ctx context.Context, s Session) ([]ToolCall, error) {
|
||||
path := s.SessionFile
|
||||
if path == "" {
|
||||
resolved, err := a.resolveSessionFile(s)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("adapter: resolve session file: %w", err)
|
||||
}
|
||||
path = resolved
|
||||
}
|
||||
switch a.Harness {
|
||||
case "claude":
|
||||
return ClaudeActivity(path)
|
||||
case "codex":
|
||||
return CodexActivity(path)
|
||||
default:
|
||||
return OpenCodeActivity(path)
|
||||
}
|
||||
}
|
||||
|
||||
// conventionsPrompt is §6.3's "orchestra injects a notice to agents whose
|
||||
// current task is adjacent" — staleness is tracked here, not by trusting the
|
||||
// agent's cached view of the shared docs.
|
||||
|
||||
@@ -187,6 +187,9 @@ type Coordinator struct {
|
||||
// defaultSoft), so existing callers that never set this field keep
|
||||
// working unchanged.
|
||||
Soft float64
|
||||
// Thrash tunes DetectThrash's three circuit breakers (§5.3). Zero-value
|
||||
// fields fall back to herdr's own defaults, so leaving this unset works.
|
||||
Thrash herdr.ThrashConfig
|
||||
}
|
||||
|
||||
// defaultSoft is used whenever Coordinator.Soft is unset (zero value).
|
||||
@@ -199,6 +202,59 @@ func (c *Coordinator) soft() float64 {
|
||||
return defaultSoft
|
||||
}
|
||||
|
||||
// checkActivityTriggers is S11's milestone/thrash pair: given an adapter that
|
||||
// implements herdr.ActivityReader, read its tool-call history and evaluate
|
||||
// both detectors. thrash takes priority (a circuit breaker overrides a
|
||||
// coherent-looking commit), same as the caller would want either way since
|
||||
// only one handoff request happens per tick. Returns the reason to request
|
||||
// ("thrash"/"milestone") and its dead ends, or "" if neither fired or the
|
||||
// adapter has no activity source at all — the latter is not degraded-and-
|
||||
// recorded the way TurnBoundary's absence is, since these two triggers are
|
||||
// additive on top of threshold/manual rotation, not a required safety gate.
|
||||
func checkActivityTriggers(ctx context.Context, a herdr.Adapter, session herdr.Session, cfg herdr.ThrashConfig) (reason string, deadEnds []continuity.DeadEnd) {
|
||||
reader, ok := a.(herdr.ActivityReader)
|
||||
if !ok {
|
||||
return "", nil
|
||||
}
|
||||
calls, err := reader.Activity(ctx, session)
|
||||
if err != nil {
|
||||
return "", nil
|
||||
}
|
||||
if thrash, de := herdr.DetectThrash(calls, cfg); thrash {
|
||||
return "thrash", de
|
||||
}
|
||||
if herdr.DetectMilestone(calls) {
|
||||
return "milestone", nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// requestReasonedHandoff is the shared "ask once, remember we asked" wiring
|
||||
// checkActivityTriggers' two callers (rotate, TurnDecision) both need — same
|
||||
// HandoffRequested guard the occupancy-driven HandoffRequester path already
|
||||
// uses, so a repeated thrash/milestone detection on later ticks doesn't
|
||||
// reprompt every time before the agent has finished writing the file.
|
||||
func (c *Coordinator) requestReasonedHandoff(ctx context.Context, taskID string, session herdr.Session, a herdr.Adapter, reason string, deadEnds []continuity.DeadEnd) {
|
||||
if session.HandoffRequested {
|
||||
return
|
||||
}
|
||||
if _, statErr := os.Stat(filepath.Join(session.Worktree, herdr.HandoffFile)); statErr == nil {
|
||||
return
|
||||
}
|
||||
requester, ok := a.(herdr.ReasonedHandoffRequester)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := requester.RequestHandoffReason(ctx, session, reason, deadEnds); err != nil {
|
||||
return
|
||||
}
|
||||
session.HandoffRequested = true
|
||||
c.mu.Lock()
|
||||
c.sessions[taskID] = session
|
||||
_ = c.saveSessionsLocked()
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
type MonitorHealth struct {
|
||||
Running bool `json:"running"`
|
||||
LastRun time.Time `json:"last_run"`
|
||||
@@ -570,14 +626,20 @@ func (c *Coordinator) rotate(ctx context.Context, hard float64) {
|
||||
continue
|
||||
}
|
||||
reason := "threshold"
|
||||
// Agent-initiated ROTATE (§5.3): the agent itself wrote a handoff
|
||||
// with reason=manual — "a coherent unit finished and the next is
|
||||
// independent". That is the boundary signal in its own right; skip
|
||||
// occupancy and the turn-boundary probe and go straight to release.
|
||||
manual := handoffReason(session.Worktree) == "manual"
|
||||
if manual {
|
||||
reason = "manual"
|
||||
// Agent-initiated ROTATE, and the two orchestrator-detected triggers
|
||||
// (§5.3: manual / milestone / thrash) all short-circuit the same way
|
||||
// once a handoff carrying that reason already exists: the boundary
|
||||
// question has already been answered, so skip occupancy and the
|
||||
// turn-boundary probe and go straight to release.
|
||||
existingReason := handoffReason(session.Worktree)
|
||||
bypass := existingReason == "manual" || existingReason == "milestone" || existingReason == "thrash"
|
||||
if bypass {
|
||||
reason = existingReason
|
||||
} else {
|
||||
if trigger, deadEnds := checkActivityTriggers(ctx, a, session, c.Thrash); trigger != "" {
|
||||
c.requestReasonedHandoff(ctx, taskID, session, a, trigger, deadEnds)
|
||||
continue
|
||||
}
|
||||
occupancy, err := a.Occupancy(session)
|
||||
if err != nil || occupancy < c.soft() {
|
||||
continue
|
||||
@@ -694,11 +756,16 @@ func (c *Coordinator) TurnDecision(ctx context.Context, taskID string) (string,
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("orchestrator: adapter: %w", err)
|
||||
}
|
||||
// Agent-initiated ROTATE (§5.3): a handoff already written with
|
||||
// reason=manual is itself the boundary signal — skip occupancy and the
|
||||
// Agent-initiated ROTATE, and the two orchestrator-detected triggers
|
||||
// (§5.3: manual / milestone / thrash): a handoff already written with one
|
||||
// of these reasons is itself the boundary signal — skip occupancy and the
|
||||
// turn-boundary probe and release immediately.
|
||||
if handoffReason(session.Worktree) == "manual" {
|
||||
return c.finishRelease(ctx, taskID, task, session, a, "manual")
|
||||
if existingReason := handoffReason(session.Worktree); existingReason == "manual" || existingReason == "milestone" || existingReason == "thrash" {
|
||||
return c.finishRelease(ctx, taskID, task, session, a, existingReason)
|
||||
}
|
||||
if trigger, deadEnds := checkActivityTriggers(ctx, a, session, c.Thrash); trigger != "" {
|
||||
c.requestReasonedHandoff(ctx, taskID, session, a, trigger, deadEnds)
|
||||
return TurnPrepareHandoff, nil
|
||||
}
|
||||
occupancy, err := a.Occupancy(session)
|
||||
if err != nil {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"orchestra/internal/authz"
|
||||
"orchestra/internal/continuity"
|
||||
"orchestra/internal/domain"
|
||||
"orchestra/internal/herdr"
|
||||
"orchestra/internal/orchestrator"
|
||||
@@ -578,6 +579,189 @@ func (a *handoffRequestingAdapter) RequestHandoff(context.Context, herdr.Session
|
||||
return nil
|
||||
}
|
||||
|
||||
// activityAdapter is fakeAdapter plus S11's milestone/thrash pair: it reports
|
||||
// a fixed tool-call history and records reasoned handoff requests, so tests
|
||||
// can drive checkActivityTriggers without a real herdr transcript.
|
||||
type activityAdapter struct {
|
||||
fakeAdapter
|
||||
calls []herdr.ToolCall
|
||||
activityErr error
|
||||
reasonAsked []string
|
||||
deadEndsSeen []continuity.DeadEnd
|
||||
}
|
||||
|
||||
func (a *activityAdapter) Activity(context.Context, herdr.Session) ([]herdr.ToolCall, error) {
|
||||
return a.calls, a.activityErr
|
||||
}
|
||||
|
||||
func (a *activityAdapter) RequestHandoffReason(_ context.Context, _ herdr.Session, reason string, deadEnds []continuity.DeadEnd) error {
|
||||
a.reasonAsked = append(a.reasonAsked, reason)
|
||||
a.deadEndsSeen = deadEnds
|
||||
return nil
|
||||
}
|
||||
|
||||
// TestActivityTriggersRequestReasonedHandoffWithoutReleasing guards S11's two
|
||||
// orchestrator-detected rotation triggers (milestone, thrash): both rotate()
|
||||
// and TurnDecision must ask for a reasoned handoff — never release — the
|
||||
// first time the trigger fires, entirely independent of occupancy (both
|
||||
// cases here use occupancy=0, far below even the soft threshold).
|
||||
func TestActivityTriggersRequestReasonedHandoffWithoutReleasing(t *testing.T) {
|
||||
repo := t.TempDir()
|
||||
run(t, repo, "init")
|
||||
run(t, repo, "config", "user.email", "t@t")
|
||||
run(t, repo, "config", "user.name", "t")
|
||||
run(t, repo, "commit", "--allow-empty", "-m", "init")
|
||||
|
||||
newCoordinator := func(a *activityAdapter) (*orchestrator.Coordinator, *store.Store, domain.Task) {
|
||||
s, err := store.Open(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Append(domain.Event{ID: domain.NewID(), Type: "TaskCreated", TaskID: "t1", Surface: string(authz.System), Payload: mustJSON(map[string]any{
|
||||
"source": "jsonl", "external_id": "1", "project": "p",
|
||||
})}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := s.Tasks()[0]
|
||||
c := &orchestrator.Coordinator{Store: s, Worktrees: worktrees{path: repo}, Adapters: adapters{a}, StatePath: t.TempDir() + "/sessions.json", Hard: .8}
|
||||
leaseEvt, err := s.Lease(task.ID, "h1", time.Minute)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Start(context.Background(), leaseEvt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return c, s, task
|
||||
}
|
||||
|
||||
thrashCalls := []herdr.ToolCall{
|
||||
{Name: "Bash", Kind: "command", Key: "go test ./...", Success: false, IsTest: true},
|
||||
{Name: "Bash", Kind: "command", Key: "go test ./...", Success: false, IsTest: true},
|
||||
{Name: "Bash", Kind: "command", Key: "go test ./...", Success: false, IsTest: true},
|
||||
}
|
||||
milestoneCalls := []herdr.ToolCall{
|
||||
{Name: "Bash", Kind: "command", Key: "git commit -m done", Success: true},
|
||||
}
|
||||
|
||||
t.Run("thrash requests a reasoned handoff and does not release, via TurnDecision", func(t *testing.T) {
|
||||
a := &activityAdapter{fakeAdapter: fakeAdapter{occupancy: 0, boundary: true}, calls: thrashCalls}
|
||||
c, st, task := newCoordinator(a)
|
||||
decision, err := c.TurnDecision(context.Background(), task.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if decision != orchestrator.TurnPrepareHandoff {
|
||||
t.Fatalf("decision=%q want %q", decision, orchestrator.TurnPrepareHandoff)
|
||||
}
|
||||
if len(a.reasonAsked) != 1 || a.reasonAsked[0] != "thrash" {
|
||||
t.Fatalf("reasonAsked=%v want [thrash]", a.reasonAsked)
|
||||
}
|
||||
if len(a.deadEndsSeen) == 0 {
|
||||
t.Fatal("want populated dead ends for the thrash trigger")
|
||||
}
|
||||
if a.releases != 0 {
|
||||
t.Fatal("Release must not be invoked on a bare thrash detection")
|
||||
}
|
||||
got, ok := st.Task(task.ID)
|
||||
if !ok || got.State != domain.StateLeased {
|
||||
t.Fatalf("task state=%v ok=%v, want still leased", got.State, ok)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("milestone requests a reasoned handoff and does not release, via TurnDecision", func(t *testing.T) {
|
||||
a := &activityAdapter{fakeAdapter: fakeAdapter{occupancy: 0, boundary: true}, calls: milestoneCalls}
|
||||
c, st, task := newCoordinator(a)
|
||||
decision, err := c.TurnDecision(context.Background(), task.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if decision != orchestrator.TurnPrepareHandoff {
|
||||
t.Fatalf("decision=%q want %q", decision, orchestrator.TurnPrepareHandoff)
|
||||
}
|
||||
if len(a.reasonAsked) != 1 || a.reasonAsked[0] != "milestone" {
|
||||
t.Fatalf("reasonAsked=%v want [milestone]", a.reasonAsked)
|
||||
}
|
||||
if a.releases != 0 {
|
||||
t.Fatal("Release must not be invoked on a bare milestone detection")
|
||||
}
|
||||
got, ok := st.Task(task.ID)
|
||||
if !ok || got.State != domain.StateLeased {
|
||||
t.Fatalf("task state=%v ok=%v, want still leased", got.State, ok)
|
||||
}
|
||||
})
|
||||
|
||||
// Once the agent has actually written a thrash/milestone-reasoned
|
||||
// handoff, that reason is itself the boundary signal (same as manual) —
|
||||
// TurnDecision must release immediately, bypassing occupancy/boundary.
|
||||
t.Run("a written thrash handoff bypasses occupancy and releases", func(t *testing.T) {
|
||||
a := &activityAdapter{fakeAdapter: fakeAdapter{occupancy: 0, boundary: false}, calls: thrashCalls}
|
||||
c, st, task := newCoordinator(a)
|
||||
artifactRef, err := st.PutArtifact([]byte("handoff"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.ref = artifactRef
|
||||
head, err := herdr.HeadSHA(repo)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
handoff := map[string]any{
|
||||
"meta": map[string]any{"id": "h3", "reason": "thrash", "rotation_index": 0},
|
||||
"anchor": map[string]any{"git_sha": head, "branch": "orchestra/t1"},
|
||||
"goal": "g", "done_when": []string{"x"}, "action": "prompt", "command": "go test ./...",
|
||||
"dead_ends": []map[string]any{{"tried": "go test ./...", "why_failed": "failed 3 times"}},
|
||||
}
|
||||
b, err := json.Marshal(handoff)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
handoffPath := repo + "/" + herdr.HandoffFile
|
||||
if err := os.WriteFile(handoffPath, b, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { os.Remove(handoffPath) })
|
||||
|
||||
decision, err := c.TurnDecision(context.Background(), task.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if decision != orchestrator.TurnRotateNow {
|
||||
t.Fatalf("decision=%q want %q", decision, orchestrator.TurnRotateNow)
|
||||
}
|
||||
if a.releases == 0 {
|
||||
t.Fatal("Release was never invoked for a written thrash handoff")
|
||||
}
|
||||
got, ok := st.Task(task.ID)
|
||||
if !ok || got.State != domain.StateQueued {
|
||||
t.Fatalf("task state=%v ok=%v, want queued", got.State, ok)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("rotate() also requests a reasoned handoff on thrash, without releasing", func(t *testing.T) {
|
||||
a := &activityAdapter{fakeAdapter: fakeAdapter{occupancy: 0, boundary: true}, calls: thrashCalls}
|
||||
c, st, task := newCoordinator(a)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go c.Monitor(ctx, .8, time.Millisecond)
|
||||
|
||||
deadline := time.Now().Add(300 * time.Millisecond)
|
||||
for time.Now().Before(deadline) && len(a.reasonAsked) == 0 {
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
if len(a.reasonAsked) == 0 || a.reasonAsked[0] != "thrash" {
|
||||
t.Fatalf("reasonAsked=%v want a thrash request from rotate()", a.reasonAsked)
|
||||
}
|
||||
if a.releases != 0 {
|
||||
t.Fatal("rotate() must not release on a bare thrash detection")
|
||||
}
|
||||
got, ok := st.Task(task.ID)
|
||||
if !ok || got.State != domain.StateLeased {
|
||||
t.Fatalf("task state=%v ok=%v, want still leased", got.State, ok)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestRotationRequestsHandoffBeforeReleasing guards Phase 4 item 2 (AUDIT.md):
|
||||
// rotate() must not call Release until the agent has been told to write its
|
||||
// §6.1 handoff and the file actually exists — never invent or skip the ask.
|
||||
|
||||
Reference in New Issue
Block a user