orchestra/internal/herdr/activity_test.go

package herdr

import (
	"encoding/json"
	"os"
	"path/filepath"
	"strings"
	"testing"
)

func writeJSONL(t *testing.T, lines []string) string {
	t.Helper()
	dir := t.TempDir()
	path := filepath.Join(dir, "transcript.jsonl")
	if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")+"\n"), 0644); err != nil {
		t.Fatal(err)
	}
	return path
}

func claudeToolLine(t *testing.T, toolUseID, name string, input map[string]any) string {
	t.Helper()
	b, err := json.Marshal(map[string]any{
		"message": map[string]any{
			"content": []map[string]any{
				{"type": "tool_use", "id": toolUseID, "name": name, "input": input},
			},
		},
	})
	if err != nil {
		t.Fatal(err)
	}
	return string(b)
}

func claudeResultLine(t *testing.T, toolUseID string, isError bool) string {
	t.Helper()
	b, err := json.Marshal(map[string]any{
		"message": map[string]any{
			"content": []map[string]any{
				{"type": "tool_result", "tool_use_id": toolUseID, "is_error": isError},
			},
		},
	})
	if err != nil {
		t.Fatal(err)
	}
	return string(b)
}

func TestClaudeActivityPairsToolUseWithResult(t *testing.T) {
	path := writeJSONL(t, []string{
		claudeToolLine(t, "t1", "Bash", map[string]any{"command": "go test ./..."}),
		claudeResultLine(t, "t1", true),
		claudeToolLine(t, "t2", "Edit", map[string]any{"file_path": "main.go"}),
		claudeResultLine(t, "t2", false),
		// an unresolved tool_use (still mid-turn) must be dropped, not reported
		claudeToolLine(t, "t3", "Bash", map[string]any{"command": "echo hi"}),
	})
	calls, err := ClaudeActivity(path)
	if err != nil {
		t.Fatal(err)
	}
	if len(calls) != 2 {
		t.Fatalf("calls=%+v, want 2 (unresolved tool_use dropped)", calls)
	}
	if calls[0].Name != "Bash" || calls[0].Kind != "command" || calls[0].Key != "go test ./..." || calls[0].Success || !calls[0].IsTest {
		t.Fatalf("calls[0]=%+v", calls[0])
	}
	if calls[1].Name != "Edit" || calls[1].Kind != "file" || calls[1].Key != "main.go" || !calls[1].Success {
		t.Fatalf("calls[1]=%+v", calls[1])
	}
}

func codexPatchApplyLine(t *testing.T, callID string, success bool, paths ...string) string {
	t.Helper()
	changes := map[string]any{}
	for _, p := range paths {
		changes[p] = map[string]any{"type": "update"}
	}
	b, err := json.Marshal(map[string]any{
		"type": "event_msg",
		"payload": map[string]any{
			"type":    "patch_apply_end",
			"call_id": callID,
			"success": success,
			"changes": changes,
		},
	})
	if err != nil {
		t.Fatal(err)
	}
	return string(b)
}

func codexExecCallLine(t *testing.T, callID, input string) string {
	t.Helper()
	b, err := json.Marshal(map[string]any{
		"type": "response_item",
		"payload": map[string]any{
			"type":    "custom_tool_call",
			"call_id": callID,
			"name":    "exec",
			"input":   input,
		},
	})
	if err != nil {
		t.Fatal(err)
	}
	return string(b)
}

func codexExecOutputLine(t *testing.T, callID string, texts ...string) string {
	t.Helper()
	var out []map[string]any
	for _, tx := range texts {
		out = append(out, map[string]any{"type": "input_text", "text": tx})
	}
	b, err := json.Marshal(map[string]any{
		"type": "response_item",
		"payload": map[string]any{
			"type":    "custom_tool_call_output",
			"call_id": callID,
			"output":  out,
		},
	})
	if err != nil {
		t.Fatal(err)
	}
	return string(b)
}

// Fixture shape confirmed live against this machine's own ~/.codex/sessions
// rollout files on 2026-07-28 (see activity.go's CodexActivity doc comment) —
// not a guess at the schema.
func TestCodexActivityParsesRealRolloutShape(t *testing.T) {
	path := writeJSONL(t, []string{
		codexExecCallLine(t, "c1", `const r = await tools.exec_command({cmd:"go test ./...","workdir":"/repo"}); text(r.output)`),
		codexExecOutputLine(t, "c1", "Script completed\nWall time 0.1 seconds\nOutput:\n", "FAIL"),
		codexExecCallLine(t, "c2", `const r = await tools.exec_command({cmd:"git commit -am x"}); text(r.output)`),
		codexExecOutputLine(t, "c2", "Script completed\n"),
		codexPatchApplyLine(t, "exec-1", true, "/repo/main.go"),
		codexExecCallLine(t, "c3", `const r = await tools.update_plan({plan:[]}); text(r)`),
		codexExecOutputLine(t, "c3", "Script error:\nSyntaxError: bad"),
	})
	calls, err := CodexActivity(path)
	if err != nil {
		t.Fatal(err)
	}
	if len(calls) != 4 {
		t.Fatalf("calls=%+v, want 4 (update_plan call resolved with empty kind/key, not dropped)", calls)
	}
	if calls[3].Kind != "" || calls[3].Key != "" {
		t.Fatalf("calls[3]=%+v, want empty kind/key for a call with no embedded cmd:", calls[3])
	}
	if calls[0].Kind != "command" || calls[0].Key != "go test ./..." || !calls[0].IsTest || !calls[0].Success {
		t.Fatalf("calls[0]=%+v", calls[0])
	}
	if calls[1].Kind != "command" || calls[1].Key != "git commit -am x" || !calls[1].Success {
		t.Fatalf("calls[1]=%+v", calls[1])
	}
	if calls[2].Kind != "file" || calls[2].Key != "/repo/main.go" || !calls[2].Success {
		t.Fatalf("calls[2]=%+v", calls[2])
	}
}

func TestCodexActivityMarksScriptErrorAsFailure(t *testing.T) {
	path := writeJSONL(t, []string{
		codexExecCallLine(t, "c1", `const r = await tools.exec_command({cmd:"pytest"}); text(r.output)`),
		codexExecOutputLine(t, "c1", "Script error:\napply_patch verification failed"),
	})
	calls, err := CodexActivity(path)
	if err != nil {
		t.Fatal(err)
	}
	if len(calls) != 1 || calls[0].Success {
		t.Fatalf("calls=%+v, want one failed call", calls)
	}
}

func TestDetectThrashConsecutiveTestFailures(t *testing.T) {
	calls := []ToolCall{
		{Name: "Bash", Kind: "command", Key: "go test ./...", Success: false, IsTest: true},
		{Name: "Bash", Kind: "command", Key: "go test ./...", Success: false, IsTest: true},
		{Name: "Bash", Kind: "command", Key: "go test ./...", Success: false, IsTest: true},
	}
	thrash, deadEnds := DetectThrash(calls, ThrashConfig{})
	if !thrash {
		t.Fatalf("want thrash on 3 consecutive test failures (default threshold)")
	}
	if len(deadEnds) != 1 || deadEnds[0].Tried != "go test ./..." {
		t.Fatalf("deadEnds=%+v", deadEnds)
	}
}

func TestDetectThrashStopsCountingAtASuccess(t *testing.T) {
	calls := []ToolCall{
		{Name: "Bash", Kind: "command", Key: "go test ./...", Success: false, IsTest: true},
		{Name: "Bash", Kind: "command", Key: "go test ./...", Success: false, IsTest: true},
		{Name: "Bash", Kind: "command", Key: "go test ./...", Success: true, IsTest: true},
		{Name: "Bash", Kind: "command", Key: "go test ./...", Success: false, IsTest: true},
	}
	// Only one failure since the last success — must not thrash on the
	// default threshold of 3, even though 3 failures exist in the window.
	if thrash, _ := DetectThrash(calls, ThrashConfig{}); thrash {
		t.Fatalf("want no thrash: only 1 failure since the last passing run")
	}
}

func TestDetectThrashSameFileEditedTooManyTimes(t *testing.T) {
	var calls []ToolCall
	for i := 0; i < 5; i++ {
		calls = append(calls, ToolCall{Name: "Edit", Kind: "file", Key: "main.go", Success: true})
	}
	thrash, deadEnds := DetectThrash(calls, ThrashConfig{})
	if !thrash {
		t.Fatalf("want thrash on 5 edits to the same file (default threshold)")
	}
	found := false
	for _, d := range deadEnds {
		if strings.Contains(d.Tried, "main.go") {
			found = true
		}
	}
	if !found {
		t.Fatalf("deadEnds=%+v, want an entry naming main.go", deadEnds)
	}
}

func TestDetectThrashIgnoresReadsForFileEditCount(t *testing.T) {
	var calls []ToolCall
	for i := 0; i < 10; i++ {
		calls = append(calls, ToolCall{Name: "Read", Kind: "file", Key: "main.go", Success: true})
	}
	// Read is not an edit tool; repeated reads of the same file must never
	// trip the "edited too many times" rule.
	if thrash, _ := DetectThrash(calls, ThrashConfig{}); thrash {
		t.Fatalf("want no thrash: Read is not an edit tool")
	}
}

func TestDetectThrashIdenticalCallsRepeated(t *testing.T) {
	var calls []ToolCall
	for i := 0; i < 4; i++ {
		calls = append(calls, ToolCall{Name: "Bash", Kind: "command", Key: "npm install lodash", Success: false})
	}
	thrash, _ := DetectThrash(calls, ThrashConfig{})
	if !thrash {
		t.Fatalf("want thrash on the identical call repeated 4 times (default threshold)")
	}
}

func TestDetectThrashCustomThresholds(t *testing.T) {
	calls := []ToolCall{
		{Name: "Edit", Kind: "file", Key: "main.go", Success: true},
		{Name: "Edit", Kind: "file", Key: "main.go", Success: true},
	}
	if thrash, _ := DetectThrash(calls, ThrashConfig{}); thrash {
		t.Fatalf("2 edits must not thrash at the default threshold of 5")
	}
	if thrash, _ := DetectThrash(calls, ThrashConfig{MaxSameFileEdits: 2}); !thrash {
		t.Fatalf("2 edits must thrash once the threshold is lowered to 2")
	}
}

func TestDetectMilestoneOnSuccessfulCommit(t *testing.T) {
	calls := []ToolCall{
		{Name: "Bash", Kind: "command", Key: "go build ./...", Success: true},
		{Name: "Bash", Kind: "command", Key: "git commit -m fix", Success: true},
	}
	if !DetectMilestone(calls) {
		t.Fatalf("want milestone: last call is a successful git commit")
	}
}

func TestDetectMilestoneRequiresSuccess(t *testing.T) {
	calls := []ToolCall{
		{Name: "Bash", Kind: "command", Key: "git commit -m fix", Success: false},
	}
	if DetectMilestone(calls) {
		t.Fatalf("want no milestone: the commit failed")
	}
}

func TestDetectMilestoneRequiresLastCall(t *testing.T) {
	calls := []ToolCall{
		{Name: "Bash", Kind: "command", Key: "git commit -m fix", Success: true},
		{Name: "Bash", Kind: "command", Key: "go test ./...", Success: true},
	}
	if DetectMilestone(calls) {
		t.Fatalf("want no milestone: the commit isn't the most recent call")
	}
}

func TestOpenCodeActivityRefusesRatherThanGuess(t *testing.T) {
	if _, err := OpenCodeActivity("/nonexistent"); err == nil {
		t.Fatalf("want an error: opencode tool-call activity has no verified source")
	}
}