package herdr import ( "encoding/json" "os" "path/filepath" "strings" "testing" ) func writeJSONL(t *testing.T, lines []string) string { t.Helper() dir := t.TempDir() path := filepath.Join(dir, "transcript.jsonl") if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")+"\n"), 0644); err != nil { t.Fatal(err) } return path } func claudeToolLine(t *testing.T, toolUseID, name string, input map[string]any) string { t.Helper() b, err := json.Marshal(map[string]any{ "message": map[string]any{ "content": []map[string]any{ {"type": "tool_use", "id": toolUseID, "name": name, "input": input}, }, }, }) if err != nil { t.Fatal(err) } return string(b) } func claudeResultLine(t *testing.T, toolUseID string, isError bool) string { t.Helper() b, err := json.Marshal(map[string]any{ "message": map[string]any{ "content": []map[string]any{ {"type": "tool_result", "tool_use_id": toolUseID, "is_error": isError}, }, }, }) if err != nil { t.Fatal(err) } return string(b) } func TestClaudeActivityPairsToolUseWithResult(t *testing.T) { path := writeJSONL(t, []string{ claudeToolLine(t, "t1", "Bash", map[string]any{"command": "go test ./..."}), claudeResultLine(t, "t1", true), claudeToolLine(t, "t2", "Edit", map[string]any{"file_path": "main.go"}), claudeResultLine(t, "t2", false), // an unresolved tool_use (still mid-turn) must be dropped, not reported claudeToolLine(t, "t3", "Bash", map[string]any{"command": "echo hi"}), }) calls, err := ClaudeActivity(path) if err != nil { t.Fatal(err) } if len(calls) != 2 { t.Fatalf("calls=%+v, want 2 (unresolved tool_use dropped)", calls) } if calls[0].Name != "Bash" || calls[0].Kind != "command" || calls[0].Key != "go test ./..." || calls[0].Success || !calls[0].IsTest { t.Fatalf("calls[0]=%+v", calls[0]) } if calls[1].Name != "Edit" || calls[1].Kind != "file" || calls[1].Key != "main.go" || !calls[1].Success { t.Fatalf("calls[1]=%+v", calls[1]) } } func codexPatchApplyLine(t *testing.T, callID string, success bool, paths ...string) string { t.Helper() changes := map[string]any{} for _, p := range paths { changes[p] = map[string]any{"type": "update"} } b, err := json.Marshal(map[string]any{ "type": "event_msg", "payload": map[string]any{ "type": "patch_apply_end", "call_id": callID, "success": success, "changes": changes, }, }) if err != nil { t.Fatal(err) } return string(b) } func codexExecCallLine(t *testing.T, callID, input string) string { t.Helper() b, err := json.Marshal(map[string]any{ "type": "response_item", "payload": map[string]any{ "type": "custom_tool_call", "call_id": callID, "name": "exec", "input": input, }, }) if err != nil { t.Fatal(err) } return string(b) } func codexExecOutputLine(t *testing.T, callID string, texts ...string) string { t.Helper() var out []map[string]any for _, tx := range texts { out = append(out, map[string]any{"type": "input_text", "text": tx}) } b, err := json.Marshal(map[string]any{ "type": "response_item", "payload": map[string]any{ "type": "custom_tool_call_output", "call_id": callID, "output": out, }, }) if err != nil { t.Fatal(err) } return string(b) } // Fixture shape confirmed live against this machine's own ~/.codex/sessions // rollout files on 2026-07-28 (see activity.go's CodexActivity doc comment) — // not a guess at the schema. func TestCodexActivityParsesRealRolloutShape(t *testing.T) { path := writeJSONL(t, []string{ codexExecCallLine(t, "c1", `const r = await tools.exec_command({cmd:"go test ./...","workdir":"/repo"}); text(r.output)`), codexExecOutputLine(t, "c1", "Script completed\nWall time 0.1 seconds\nOutput:\n", "FAIL"), codexExecCallLine(t, "c2", `const r = await tools.exec_command({cmd:"git commit -am x"}); text(r.output)`), codexExecOutputLine(t, "c2", "Script completed\n"), codexPatchApplyLine(t, "exec-1", true, "/repo/main.go"), codexExecCallLine(t, "c3", `const r = await tools.update_plan({plan:[]}); text(r)`), codexExecOutputLine(t, "c3", "Script error:\nSyntaxError: bad"), }) calls, err := CodexActivity(path) if err != nil { t.Fatal(err) } if len(calls) != 4 { t.Fatalf("calls=%+v, want 4 (update_plan call resolved with empty kind/key, not dropped)", calls) } if calls[3].Kind != "" || calls[3].Key != "" { t.Fatalf("calls[3]=%+v, want empty kind/key for a call with no embedded cmd:", calls[3]) } if calls[0].Kind != "command" || calls[0].Key != "go test ./..." || !calls[0].IsTest || !calls[0].Success { t.Fatalf("calls[0]=%+v", calls[0]) } if calls[1].Kind != "command" || calls[1].Key != "git commit -am x" || !calls[1].Success { t.Fatalf("calls[1]=%+v", calls[1]) } if calls[2].Kind != "file" || calls[2].Key != "/repo/main.go" || !calls[2].Success { t.Fatalf("calls[2]=%+v", calls[2]) } } func TestCodexActivityMarksScriptErrorAsFailure(t *testing.T) { path := writeJSONL(t, []string{ codexExecCallLine(t, "c1", `const r = await tools.exec_command({cmd:"pytest"}); text(r.output)`), codexExecOutputLine(t, "c1", "Script error:\napply_patch verification failed"), }) calls, err := CodexActivity(path) if err != nil { t.Fatal(err) } if len(calls) != 1 || calls[0].Success { t.Fatalf("calls=%+v, want one failed call", calls) } } func TestCodexActivityPreservesAllCommandsInAScript(t *testing.T) { path := writeJSONL(t, []string{ codexExecCallLine(t, "c1", `const test = await tools.exec_command({cmd:"go test ./..."}); const commit = await tools.exec_command({cmd:"git commit -am done"}); text(test.output); text(commit.output)`), codexExecOutputLine(t, "c1", "Script completed"), }) calls, err := CodexActivity(path) if err != nil { t.Fatal(err) } if len(calls) != 2 { t.Fatalf("calls=%+v, want both commands", calls) } if calls[0].Key != "go test ./..." || calls[1].Key != "git commit -am done" || !calls[0].Success || !calls[1].Success { t.Fatalf("calls=%+v, want successful commands in source order", calls) } if !DetectMilestone(calls) { t.Fatalf("want the later successful git commit to be a milestone") } } func TestDetectThrashConsecutiveTestFailures(t *testing.T) { calls := []ToolCall{ {Name: "Bash", Kind: "command", Key: "go test ./...", Success: false, IsTest: true}, {Name: "Bash", Kind: "command", Key: "go test ./...", Success: false, IsTest: true}, {Name: "Bash", Kind: "command", Key: "go test ./...", Success: false, IsTest: true}, } thrash, deadEnds := DetectThrash(calls, ThrashConfig{}) if !thrash { t.Fatalf("want thrash on 3 consecutive test failures (default threshold)") } if len(deadEnds) != 1 || deadEnds[0].Tried != "go test ./..." { t.Fatalf("deadEnds=%+v", deadEnds) } } func TestDetectThrashStopsCountingAtASuccess(t *testing.T) { calls := []ToolCall{ {Name: "Bash", Kind: "command", Key: "go test ./...", Success: false, IsTest: true}, {Name: "Bash", Kind: "command", Key: "go test ./...", Success: false, IsTest: true}, {Name: "Bash", Kind: "command", Key: "go test ./...", Success: true, IsTest: true}, {Name: "Bash", Kind: "command", Key: "go test ./...", Success: false, IsTest: true}, } // Only one failure since the last success — must not thrash on the // default threshold of 3, even though 3 failures exist in the window. if thrash, _ := DetectThrash(calls, ThrashConfig{}); thrash { t.Fatalf("want no thrash: only 1 failure since the last passing run") } } func TestDetectThrashSameFileEditedTooManyTimes(t *testing.T) { var calls []ToolCall for i := 0; i < 5; i++ { calls = append(calls, ToolCall{Name: "Edit", Kind: "file", Key: "main.go", Success: true}) } thrash, deadEnds := DetectThrash(calls, ThrashConfig{}) if !thrash { t.Fatalf("want thrash on 5 edits to the same file (default threshold)") } found := false for _, d := range deadEnds { if strings.Contains(d.Tried, "main.go") { found = true } } if !found { t.Fatalf("deadEnds=%+v, want an entry naming main.go", deadEnds) } } func TestDetectThrashIgnoresReadsForFileEditCount(t *testing.T) { var calls []ToolCall for i := 0; i < 10; i++ { calls = append(calls, ToolCall{Name: "Read", Kind: "file", Key: "main.go", Success: true}) } // Read is not an edit tool; repeated reads of the same file must never // trip the "edited too many times" rule. if thrash, _ := DetectThrash(calls, ThrashConfig{}); thrash { t.Fatalf("want no thrash: Read is not an edit tool") } } func TestDetectThrashIdenticalCallsRepeated(t *testing.T) { var calls []ToolCall for i := 0; i < 4; i++ { calls = append(calls, ToolCall{Name: "Bash", Kind: "command", Key: "npm install lodash", Success: false}) } thrash, _ := DetectThrash(calls, ThrashConfig{}) if !thrash { t.Fatalf("want thrash on the identical call repeated 4 times (default threshold)") } } func TestDetectThrashCustomThresholds(t *testing.T) { calls := []ToolCall{ {Name: "Edit", Kind: "file", Key: "main.go", Success: true}, {Name: "Edit", Kind: "file", Key: "main.go", Success: true}, } if thrash, _ := DetectThrash(calls, ThrashConfig{}); thrash { t.Fatalf("2 edits must not thrash at the default threshold of 5") } if thrash, _ := DetectThrash(calls, ThrashConfig{MaxSameFileEdits: 2}); !thrash { t.Fatalf("2 edits must thrash once the threshold is lowered to 2") } } func TestDetectMilestoneOnSuccessfulCommit(t *testing.T) { calls := []ToolCall{ {Name: "Bash", Kind: "command", Key: "go build ./...", Success: true}, {Name: "Bash", Kind: "command", Key: "git commit -m fix", Success: true}, } if !DetectMilestone(calls) { t.Fatalf("want milestone: last call is a successful git commit") } } func TestDetectMilestoneRequiresSuccess(t *testing.T) { calls := []ToolCall{ {Name: "Bash", Kind: "command", Key: "git commit -m fix", Success: false}, } if DetectMilestone(calls) { t.Fatalf("want no milestone: the commit failed") } } func TestDetectMilestoneRequiresLastCall(t *testing.T) { calls := []ToolCall{ {Name: "Bash", Kind: "command", Key: "git commit -m fix", Success: true}, {Name: "Bash", Kind: "command", Key: "go test ./...", Success: true}, } if DetectMilestone(calls) { t.Fatalf("want no milestone: the commit isn't the most recent call") } } func TestOpenCodeActivityRefusesRatherThanGuess(t *testing.T) { if _, err := OpenCodeActivity("/nonexistent"); err == nil { t.Fatalf("want an error: opencode tool-call activity has no verified source") } }