320 lines
10 KiB
Go
320 lines
10 KiB
Go
package herdr
|
|
|
|
import (
|
|
"encoding/json"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func writeJSONL(t *testing.T, lines []string) string {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
path := filepath.Join(dir, "transcript.jsonl")
|
|
if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")+"\n"), 0644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return path
|
|
}
|
|
|
|
func claudeToolLine(t *testing.T, toolUseID, name string, input map[string]any) string {
|
|
t.Helper()
|
|
b, err := json.Marshal(map[string]any{
|
|
"message": map[string]any{
|
|
"content": []map[string]any{
|
|
{"type": "tool_use", "id": toolUseID, "name": name, "input": input},
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return string(b)
|
|
}
|
|
|
|
func claudeResultLine(t *testing.T, toolUseID string, isError bool) string {
|
|
t.Helper()
|
|
b, err := json.Marshal(map[string]any{
|
|
"message": map[string]any{
|
|
"content": []map[string]any{
|
|
{"type": "tool_result", "tool_use_id": toolUseID, "is_error": isError},
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return string(b)
|
|
}
|
|
|
|
func TestClaudeActivityPairsToolUseWithResult(t *testing.T) {
|
|
path := writeJSONL(t, []string{
|
|
claudeToolLine(t, "t1", "Bash", map[string]any{"command": "go test ./..."}),
|
|
claudeResultLine(t, "t1", true),
|
|
claudeToolLine(t, "t2", "Edit", map[string]any{"file_path": "main.go"}),
|
|
claudeResultLine(t, "t2", false),
|
|
// an unresolved tool_use (still mid-turn) must be dropped, not reported
|
|
claudeToolLine(t, "t3", "Bash", map[string]any{"command": "echo hi"}),
|
|
})
|
|
calls, err := ClaudeActivity(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(calls) != 2 {
|
|
t.Fatalf("calls=%+v, want 2 (unresolved tool_use dropped)", calls)
|
|
}
|
|
if calls[0].Name != "Bash" || calls[0].Kind != "command" || calls[0].Key != "go test ./..." || calls[0].Success || !calls[0].IsTest {
|
|
t.Fatalf("calls[0]=%+v", calls[0])
|
|
}
|
|
if calls[1].Name != "Edit" || calls[1].Kind != "file" || calls[1].Key != "main.go" || !calls[1].Success {
|
|
t.Fatalf("calls[1]=%+v", calls[1])
|
|
}
|
|
}
|
|
|
|
func codexPatchApplyLine(t *testing.T, callID string, success bool, paths ...string) string {
|
|
t.Helper()
|
|
changes := map[string]any{}
|
|
for _, p := range paths {
|
|
changes[p] = map[string]any{"type": "update"}
|
|
}
|
|
b, err := json.Marshal(map[string]any{
|
|
"type": "event_msg",
|
|
"payload": map[string]any{
|
|
"type": "patch_apply_end",
|
|
"call_id": callID,
|
|
"success": success,
|
|
"changes": changes,
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return string(b)
|
|
}
|
|
|
|
func codexExecCallLine(t *testing.T, callID, input string) string {
|
|
t.Helper()
|
|
b, err := json.Marshal(map[string]any{
|
|
"type": "response_item",
|
|
"payload": map[string]any{
|
|
"type": "custom_tool_call",
|
|
"call_id": callID,
|
|
"name": "exec",
|
|
"input": input,
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return string(b)
|
|
}
|
|
|
|
func codexExecOutputLine(t *testing.T, callID string, texts ...string) string {
|
|
t.Helper()
|
|
var out []map[string]any
|
|
for _, tx := range texts {
|
|
out = append(out, map[string]any{"type": "input_text", "text": tx})
|
|
}
|
|
b, err := json.Marshal(map[string]any{
|
|
"type": "response_item",
|
|
"payload": map[string]any{
|
|
"type": "custom_tool_call_output",
|
|
"call_id": callID,
|
|
"output": out,
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return string(b)
|
|
}
|
|
|
|
// Fixture shape confirmed live against this machine's own ~/.codex/sessions
|
|
// rollout files on 2026-07-28 (see activity.go's CodexActivity doc comment) —
|
|
// not a guess at the schema.
|
|
func TestCodexActivityParsesRealRolloutShape(t *testing.T) {
|
|
path := writeJSONL(t, []string{
|
|
codexExecCallLine(t, "c1", `const r = await tools.exec_command({cmd:"go test ./...","workdir":"/repo"}); text(r.output)`),
|
|
codexExecOutputLine(t, "c1", "Script completed\nWall time 0.1 seconds\nOutput:\n", "FAIL"),
|
|
codexExecCallLine(t, "c2", `const r = await tools.exec_command({cmd:"git commit -am x"}); text(r.output)`),
|
|
codexExecOutputLine(t, "c2", "Script completed\n"),
|
|
codexPatchApplyLine(t, "exec-1", true, "/repo/main.go"),
|
|
codexExecCallLine(t, "c3", `const r = await tools.update_plan({plan:[]}); text(r)`),
|
|
codexExecOutputLine(t, "c3", "Script error:\nSyntaxError: bad"),
|
|
})
|
|
calls, err := CodexActivity(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(calls) != 4 {
|
|
t.Fatalf("calls=%+v, want 4 (update_plan call resolved with empty kind/key, not dropped)", calls)
|
|
}
|
|
if calls[3].Kind != "" || calls[3].Key != "" {
|
|
t.Fatalf("calls[3]=%+v, want empty kind/key for a call with no embedded cmd:", calls[3])
|
|
}
|
|
if calls[0].Kind != "command" || calls[0].Key != "go test ./..." || !calls[0].IsTest || !calls[0].Success {
|
|
t.Fatalf("calls[0]=%+v", calls[0])
|
|
}
|
|
if calls[1].Kind != "command" || calls[1].Key != "git commit -am x" || !calls[1].Success {
|
|
t.Fatalf("calls[1]=%+v", calls[1])
|
|
}
|
|
if calls[2].Kind != "file" || calls[2].Key != "/repo/main.go" || !calls[2].Success {
|
|
t.Fatalf("calls[2]=%+v", calls[2])
|
|
}
|
|
}
|
|
|
|
func TestCodexActivityMarksScriptErrorAsFailure(t *testing.T) {
|
|
path := writeJSONL(t, []string{
|
|
codexExecCallLine(t, "c1", `const r = await tools.exec_command({cmd:"pytest"}); text(r.output)`),
|
|
codexExecOutputLine(t, "c1", "Script error:\napply_patch verification failed"),
|
|
})
|
|
calls, err := CodexActivity(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(calls) != 1 || calls[0].Success {
|
|
t.Fatalf("calls=%+v, want one failed call", calls)
|
|
}
|
|
}
|
|
|
|
func TestCodexActivityPreservesAllCommandsInAScript(t *testing.T) {
|
|
path := writeJSONL(t, []string{
|
|
codexExecCallLine(t, "c1", `const test = await tools.exec_command({cmd:"go test ./..."}); const commit = await tools.exec_command({cmd:"git commit -am done"}); text(test.output); text(commit.output)`),
|
|
codexExecOutputLine(t, "c1", "Script completed"),
|
|
})
|
|
calls, err := CodexActivity(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(calls) != 2 {
|
|
t.Fatalf("calls=%+v, want both commands", calls)
|
|
}
|
|
if calls[0].Key != "go test ./..." || calls[1].Key != "git commit -am done" || !calls[0].Success || !calls[1].Success {
|
|
t.Fatalf("calls=%+v, want successful commands in source order", calls)
|
|
}
|
|
if !DetectMilestone(calls) {
|
|
t.Fatalf("want the later successful git commit to be a milestone")
|
|
}
|
|
}
|
|
|
|
func TestDetectThrashConsecutiveTestFailures(t *testing.T) {
|
|
calls := []ToolCall{
|
|
{Name: "Bash", Kind: "command", Key: "go test ./...", Success: false, IsTest: true},
|
|
{Name: "Bash", Kind: "command", Key: "go test ./...", Success: false, IsTest: true},
|
|
{Name: "Bash", Kind: "command", Key: "go test ./...", Success: false, IsTest: true},
|
|
}
|
|
thrash, deadEnds := DetectThrash(calls, ThrashConfig{})
|
|
if !thrash {
|
|
t.Fatalf("want thrash on 3 consecutive test failures (default threshold)")
|
|
}
|
|
if len(deadEnds) != 1 || deadEnds[0].Tried != "go test ./..." {
|
|
t.Fatalf("deadEnds=%+v", deadEnds)
|
|
}
|
|
}
|
|
|
|
func TestDetectThrashStopsCountingAtASuccess(t *testing.T) {
|
|
calls := []ToolCall{
|
|
{Name: "Bash", Kind: "command", Key: "go test ./...", Success: false, IsTest: true},
|
|
{Name: "Bash", Kind: "command", Key: "go test ./...", Success: false, IsTest: true},
|
|
{Name: "Bash", Kind: "command", Key: "go test ./...", Success: true, IsTest: true},
|
|
{Name: "Bash", Kind: "command", Key: "go test ./...", Success: false, IsTest: true},
|
|
}
|
|
// Only one failure since the last success — must not thrash on the
|
|
// default threshold of 3, even though 3 failures exist in the window.
|
|
if thrash, _ := DetectThrash(calls, ThrashConfig{}); thrash {
|
|
t.Fatalf("want no thrash: only 1 failure since the last passing run")
|
|
}
|
|
}
|
|
|
|
func TestDetectThrashSameFileEditedTooManyTimes(t *testing.T) {
|
|
var calls []ToolCall
|
|
for i := 0; i < 5; i++ {
|
|
calls = append(calls, ToolCall{Name: "Edit", Kind: "file", Key: "main.go", Success: true})
|
|
}
|
|
thrash, deadEnds := DetectThrash(calls, ThrashConfig{})
|
|
if !thrash {
|
|
t.Fatalf("want thrash on 5 edits to the same file (default threshold)")
|
|
}
|
|
found := false
|
|
for _, d := range deadEnds {
|
|
if strings.Contains(d.Tried, "main.go") {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatalf("deadEnds=%+v, want an entry naming main.go", deadEnds)
|
|
}
|
|
}
|
|
|
|
func TestDetectThrashIgnoresReadsForFileEditCount(t *testing.T) {
|
|
var calls []ToolCall
|
|
for i := 0; i < 10; i++ {
|
|
calls = append(calls, ToolCall{Name: "Read", Kind: "file", Key: "main.go", Success: true})
|
|
}
|
|
// Read is not an edit tool; repeated reads of the same file must never
|
|
// trip the "edited too many times" rule.
|
|
if thrash, _ := DetectThrash(calls, ThrashConfig{}); thrash {
|
|
t.Fatalf("want no thrash: Read is not an edit tool")
|
|
}
|
|
}
|
|
|
|
func TestDetectThrashIdenticalCallsRepeated(t *testing.T) {
|
|
var calls []ToolCall
|
|
for i := 0; i < 4; i++ {
|
|
calls = append(calls, ToolCall{Name: "Bash", Kind: "command", Key: "npm install lodash", Success: false})
|
|
}
|
|
thrash, _ := DetectThrash(calls, ThrashConfig{})
|
|
if !thrash {
|
|
t.Fatalf("want thrash on the identical call repeated 4 times (default threshold)")
|
|
}
|
|
}
|
|
|
|
func TestDetectThrashCustomThresholds(t *testing.T) {
|
|
calls := []ToolCall{
|
|
{Name: "Edit", Kind: "file", Key: "main.go", Success: true},
|
|
{Name: "Edit", Kind: "file", Key: "main.go", Success: true},
|
|
}
|
|
if thrash, _ := DetectThrash(calls, ThrashConfig{}); thrash {
|
|
t.Fatalf("2 edits must not thrash at the default threshold of 5")
|
|
}
|
|
if thrash, _ := DetectThrash(calls, ThrashConfig{MaxSameFileEdits: 2}); !thrash {
|
|
t.Fatalf("2 edits must thrash once the threshold is lowered to 2")
|
|
}
|
|
}
|
|
|
|
func TestDetectMilestoneOnSuccessfulCommit(t *testing.T) {
|
|
calls := []ToolCall{
|
|
{Name: "Bash", Kind: "command", Key: "go build ./...", Success: true},
|
|
{Name: "Bash", Kind: "command", Key: "git commit -m fix", Success: true},
|
|
}
|
|
if !DetectMilestone(calls) {
|
|
t.Fatalf("want milestone: last call is a successful git commit")
|
|
}
|
|
}
|
|
|
|
func TestDetectMilestoneRequiresSuccess(t *testing.T) {
|
|
calls := []ToolCall{
|
|
{Name: "Bash", Kind: "command", Key: "git commit -m fix", Success: false},
|
|
}
|
|
if DetectMilestone(calls) {
|
|
t.Fatalf("want no milestone: the commit failed")
|
|
}
|
|
}
|
|
|
|
func TestDetectMilestoneRequiresLastCall(t *testing.T) {
|
|
calls := []ToolCall{
|
|
{Name: "Bash", Kind: "command", Key: "git commit -m fix", Success: true},
|
|
{Name: "Bash", Kind: "command", Key: "go test ./...", Success: true},
|
|
}
|
|
if DetectMilestone(calls) {
|
|
t.Fatalf("want no milestone: the commit isn't the most recent call")
|
|
}
|
|
}
|
|
|
|
func TestOpenCodeActivityRefusesRatherThanGuess(t *testing.T) {
|
|
if _, err := OpenCodeActivity("/nonexistent"); err == nil {
|
|
t.Fatalf("want an error: opencode tool-call activity has no verified source")
|
|
}
|
|
}
|