feat(orchestrator): milestone rotation and thrash detection (S11)
Closes the last two S11 triggers. internal/herdr/activity.go normalizes tool/function calls per harness (ClaudeActivity verified against the existing transcript format, CodexActivity best-effort/unverified, OpenCodeActivity refuses — no confirmed per-tool-call source exists) and implements the three thrash rules plus a narrow milestone check (successful git commit as the last call). CLIAdapter.RequestHandoffReason asks the agent to write a handoff with meta.reason set, same "ask, don't invent" pattern as the existing handoff/ report requests. rotate() and TurnDecision generalize the manual-bypass shortcut to manual/milestone/thrash and request (never directly release) on a detected trigger. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01W1rkJ2hBMybnJctPbcy4tT
This commit is contained in:
kami
@@ -0,0 +1,314 @@
|
||||
package herdr
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"orchestra/internal/continuity"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ToolCall is a harness-agnostic normalization of one tool/function
|
||||
// invocation and its outcome, extracted from a harness's own transcript or
|
||||
// rollout — the source S11 named as missing ("transcript/tool-call
|
||||
// introspection this repo doesn't have a source for"). Kind distinguishes
|
||||
// what Key means: "file" (Key is a repo-relative or absolute path, from a
|
||||
// tool whose input carried file_path) or "command" (Key is a shell command
|
||||
// string). A call whose input matched neither shape has Kind=="" and Key=="" —
|
||||
// still counted for occupancy-adjacent bookkeeping elsewhere, but ignored by
|
||||
// the detectors below, which only reason about file edits and commands.
|
||||
type ToolCall struct {
|
||||
Name string
|
||||
Kind string
|
||||
Key string
|
||||
Success bool
|
||||
IsTest bool
|
||||
}
|
||||
|
||||
// ActivityReader is optional, mirroring every other Face-B capability in
|
||||
// this package (TurnBoundary, HandoffRequester, ...): an adapter without a
|
||||
// tool-call source for its harness simply doesn't implement it, and callers
|
||||
// (Coordinator.rotate, TurnDecision) skip thrash/milestone detection rather
|
||||
// than guessing from an absent signal.
|
||||
type ActivityReader interface {
|
||||
Activity(context.Context, Session) ([]ToolCall, error)
|
||||
}
|
||||
|
||||
// editToolNames are the tool/function names whose input is a full file
|
||||
// rewrite or patch — the ones the "same file edited too many times without
|
||||
// resolving" thrash rule cares about. Read/Grep/Glob-style tools also carry
|
||||
// a file_path in their input (Kind=="file") but are not edits and must not
|
||||
// count toward this rule.
|
||||
var editToolNames = map[string]bool{
|
||||
"Edit": true, "Write": true, "MultiEdit": true, "NotebookEdit": true,
|
||||
// Codex/opencode function names are unverified against a live rollout
|
||||
// (see CodexActivity's doc comment) — these are best-effort guesses at
|
||||
// what an edit-shaped tool call would be named there.
|
||||
"apply_patch": true, "edit_file": true, "patch_file": true,
|
||||
}
|
||||
|
||||
func isEditTool(name string) bool { return editToolNames[name] }
|
||||
|
||||
// testCommandRe matches shell invocations that run a test suite, for the
|
||||
// "N consecutive failed test runs" thrash rule (spec §5.3).
|
||||
var testCommandRe = regexp.MustCompile(`(?i)\b(go test|pytest|py\.test|npm (run )?test|yarn test|pnpm test|cargo test|make test|jest|mvn test|rspec|ctest)\b`)
|
||||
|
||||
// toolCallMeta pulls (kind, key) out of a tool call's JSON input/arguments —
|
||||
// shared by every harness's parser since Claude's tool "input" and Codex's
|
||||
// function "arguments" are both a flat JSON object using the same
|
||||
// conventional field names (file_path, command).
|
||||
func toolCallMeta(input json.RawMessage) (kind, key string) {
|
||||
var m map[string]any
|
||||
if json.Unmarshal(input, &m) != nil {
|
||||
return "", ""
|
||||
}
|
||||
if v, ok := m["file_path"].(string); ok && v != "" {
|
||||
return "file", v
|
||||
}
|
||||
if v, ok := m["command"].(string); ok && v != "" {
|
||||
return "command", strings.TrimSpace(v)
|
||||
}
|
||||
return "", ""
|
||||
}
|
||||
|
||||
// ClaudeActivity parses a Claude Code transcript's assistant tool_use blocks
|
||||
// paired with their matching user tool_result blocks — the same file
|
||||
// ClaudeSessionFile/ClaudeUsage already open, so no new resolution path is
|
||||
// needed. Each tool_use is matched to its result by tool_use_id; an
|
||||
// unresolved tool_use (session still mid-turn) is simply dropped, not
|
||||
// reported as a call.
|
||||
func ClaudeActivity(path string) ([]ToolCall, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
type block struct {
|
||||
Type string `json:"type"`
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Input json.RawMessage `json:"input"`
|
||||
ToolUseID string `json:"tool_use_id"`
|
||||
IsError bool `json:"is_error"`
|
||||
}
|
||||
type entry struct {
|
||||
Message struct {
|
||||
Content []block `json:"content"`
|
||||
} `json:"message"`
|
||||
}
|
||||
|
||||
pending := map[string]ToolCall{}
|
||||
var calls []ToolCall
|
||||
s := bufio.NewScanner(f)
|
||||
s.Buffer(make([]byte, 1<<20), 10<<20)
|
||||
for s.Scan() {
|
||||
var e entry
|
||||
if json.Unmarshal(s.Bytes(), &e) != nil {
|
||||
continue
|
||||
}
|
||||
for _, b := range e.Message.Content {
|
||||
switch b.Type {
|
||||
case "tool_use":
|
||||
kind, key := toolCallMeta(b.Input)
|
||||
pending[b.ID] = ToolCall{Name: b.Name, Kind: kind, Key: key, IsTest: isTestCommand(kind, key)}
|
||||
case "tool_result":
|
||||
if tc, ok := pending[b.ToolUseID]; ok {
|
||||
tc.Success = !b.IsError
|
||||
calls = append(calls, tc)
|
||||
delete(pending, b.ToolUseID)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return calls, s.Err()
|
||||
}
|
||||
|
||||
// CodexActivity is best-effort and unverified against a live Codex rollout —
|
||||
// unlike ClaudeActivity, which reads the same transcript shape ClaudeUsage
|
||||
// already confirmed. Codex's Responses-API-style rollout is expected to log
|
||||
// a "function_call" payload (name, arguments, call_id) followed later by a
|
||||
// "function_call_output" payload (call_id, output), mirroring the
|
||||
// "payload.type" wrapper CodexUsage already reads for "token_count". Success
|
||||
// is inferred heuristically from the output text (no confirmed structured
|
||||
// exit-code field), which is coarser than Claude's explicit is_error flag.
|
||||
// Treat any thrash/milestone signal derived from this as advisory until
|
||||
// checked against a real rollout, same caveat this file's other Codex-facing
|
||||
// code already carries (AUDIT.md Phase 0's own bar: verify before trusting).
|
||||
func CodexActivity(path string) ([]ToolCall, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
type payload struct {
|
||||
Type string `json:"type"`
|
||||
CallID string `json:"call_id"`
|
||||
Name string `json:"name"`
|
||||
Arguments json.RawMessage `json:"arguments"`
|
||||
Output string `json:"output"`
|
||||
}
|
||||
type entry struct {
|
||||
Payload payload `json:"payload"`
|
||||
}
|
||||
|
||||
pending := map[string]ToolCall{}
|
||||
var calls []ToolCall
|
||||
s := bufio.NewScanner(f)
|
||||
s.Buffer(make([]byte, 1<<20), 10<<20)
|
||||
for s.Scan() {
|
||||
var e entry
|
||||
if json.Unmarshal(s.Bytes(), &e) != nil {
|
||||
continue
|
||||
}
|
||||
switch e.Payload.Type {
|
||||
case "function_call":
|
||||
kind, key := toolCallMeta(e.Payload.Arguments)
|
||||
pending[e.Payload.CallID] = ToolCall{Name: e.Payload.Name, Kind: kind, Key: key, IsTest: isTestCommand(kind, key)}
|
||||
case "function_call_output":
|
||||
if tc, ok := pending[e.Payload.CallID]; ok {
|
||||
tc.Success = !strings.Contains(strings.ToLower(e.Payload.Output), "error")
|
||||
calls = append(calls, tc)
|
||||
delete(pending, e.Payload.CallID)
|
||||
}
|
||||
}
|
||||
}
|
||||
return calls, s.Err()
|
||||
}
|
||||
|
||||
// OpenCodeActivity has no verified source. OpenCodeUsage already only reads
|
||||
// aggregate token counts per message file (~/.local/share/opencode/storage/
|
||||
// message/), not per-tool-call records, and nobody has confirmed opencode's
|
||||
// on-disk format exposes tool name/input/outcome at that granularity the way
|
||||
// Claude's transcript and (probably) Codex's rollout do. Refusing loudly here
|
||||
// matches the existing convention in this file (CLIAdapter.resolveSessionFile's
|
||||
// opencode default case) rather than fabricating a parser against an unknown
|
||||
// shape.
|
||||
func OpenCodeActivity(string) ([]ToolCall, error) {
|
||||
return nil, fmt.Errorf("herdr: opencode tool-call activity has no verified source yet; check the message storage format against a live session first")
|
||||
}
|
||||
|
||||
func isTestCommand(kind, key string) bool {
|
||||
return kind == "command" && testCommandRe.MatchString(key)
|
||||
}
|
||||
|
||||
// ThrashConfig tunes DetectThrash's three independent circuit breakers (spec
|
||||
// §5.3: "N failed test runs / same file M times / identical tool calls").
|
||||
// A zero value for any field means "use the package default" — see
|
||||
// withDefaults — so a zero-value ThrashConfig{} is usable as-is.
|
||||
type ThrashConfig struct {
|
||||
MaxConsecutiveTestFailures int
|
||||
MaxSameFileEdits int
|
||||
MaxIdenticalCalls int
|
||||
}
|
||||
|
||||
func (c ThrashConfig) withDefaults() ThrashConfig {
|
||||
if c.MaxConsecutiveTestFailures <= 0 {
|
||||
c.MaxConsecutiveTestFailures = 3
|
||||
}
|
||||
if c.MaxSameFileEdits <= 0 {
|
||||
c.MaxSameFileEdits = 5
|
||||
}
|
||||
if c.MaxIdenticalCalls <= 0 {
|
||||
c.MaxIdenticalCalls = 4
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// DetectThrash evaluates a window of recent tool calls (oldest first, as
|
||||
// returned by an ActivityReader) against the three thrash rules named in
|
||||
// §5.3, returning whether any tripped and a populated continuity.DeadEnd per
|
||||
// rule that did — meant to be handed straight to
|
||||
// CLIAdapter.RequestHandoffReason so the resulting handoff's dead_ends
|
||||
// (§6.1) aren't left for the agent to invent from scratch.
|
||||
func DetectThrash(calls []ToolCall, cfg ThrashConfig) (bool, []continuity.DeadEnd) {
|
||||
cfg = cfg.withDefaults()
|
||||
var deadEnds []continuity.DeadEnd
|
||||
|
||||
// Rule 1: N consecutive failed test runs, most recent first.
|
||||
streak := 0
|
||||
var lastFailedCmd string
|
||||
for i := len(calls) - 1; i >= 0; i-- {
|
||||
c := calls[i]
|
||||
if !c.IsTest {
|
||||
continue
|
||||
}
|
||||
if c.Success {
|
||||
break
|
||||
}
|
||||
streak++
|
||||
lastFailedCmd = c.Key
|
||||
}
|
||||
if streak >= cfg.MaxConsecutiveTestFailures {
|
||||
deadEnds = append(deadEnds, continuity.DeadEnd{
|
||||
Tried: lastFailedCmd,
|
||||
WhyFailed: fmt.Sprintf("failed %d consecutive times", streak),
|
||||
})
|
||||
}
|
||||
|
||||
// Rule 2: the same file edited M times without the loop resolving.
|
||||
editCounts := map[string]int{}
|
||||
for _, c := range calls {
|
||||
if c.Kind != "file" || c.Key == "" || !isEditTool(c.Name) {
|
||||
continue
|
||||
}
|
||||
editCounts[c.Key]++
|
||||
}
|
||||
for path, n := range editCounts {
|
||||
if n >= cfg.MaxSameFileEdits {
|
||||
deadEnds = append(deadEnds, continuity.DeadEnd{
|
||||
Tried: "editing " + path,
|
||||
WhyFailed: fmt.Sprintf("edited %d times without resolving", n),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Rule 3: the identical tool call (same name+kind+key) repeated back to
|
||||
// back, a sign of a stuck retry loop rather than incremental progress.
|
||||
// Deliberately excludes test commands (re-running the same test after a
|
||||
// fix attempt is expected — that's rule 1's job to catch if it keeps
|
||||
// failing) and non-edit file calls like Read (re-reading the same file
|
||||
// across turns is normal and not a sign of being stuck).
|
||||
run := 0
|
||||
var prevName, prevKind, prevKey string
|
||||
for _, c := range calls {
|
||||
relevant := (c.Kind == "command" && !c.IsTest) || (c.Kind == "file" && isEditTool(c.Name))
|
||||
if !relevant || c.Key == "" {
|
||||
run = 0
|
||||
continue
|
||||
}
|
||||
if c.Name == prevName && c.Kind == prevKind && c.Key == prevKey {
|
||||
run++
|
||||
} else {
|
||||
run = 1
|
||||
prevName, prevKind, prevKey = c.Name, c.Kind, c.Key
|
||||
}
|
||||
if run >= cfg.MaxIdenticalCalls {
|
||||
deadEnds = append(deadEnds, continuity.DeadEnd{
|
||||
Tried: fmt.Sprintf("%s(%s)", c.Name, c.Key),
|
||||
WhyFailed: fmt.Sprintf("repeated identically %d times in a row", run),
|
||||
})
|
||||
run = 0 // one dead end per repeated run is enough
|
||||
}
|
||||
}
|
||||
|
||||
return len(deadEnds) > 0, deadEnds
|
||||
}
|
||||
|
||||
// DetectMilestone recognizes the simplest unambiguous "a coherent unit of
|
||||
// work just finished" signal available from tool-call history alone (spec
|
||||
// §5.3): the most recent call was a successful `git commit`. Anything
|
||||
// fuzzier (a passing test suite, a finished subtask) needs a definition of
|
||||
// "coherent unit" this repo has no source for yet, so it is deliberately not
|
||||
// guessed at here.
|
||||
func DetectMilestone(calls []ToolCall) bool {
|
||||
if len(calls) == 0 {
|
||||
return false
|
||||
}
|
||||
last := calls[len(calls)-1]
|
||||
return last.Success && last.Kind == "command" && strings.HasPrefix(last.Key, "git commit")
|
||||
}
|
||||
Reference in New Issue
Block a user