orchestra/internal/orchestrator/reconcile_escalation_test.go

package orchestrator_test

import (
	"context"
	"encoding/json"
	"errors"
	"os"
	"strings"
	"testing"

	"orchestra/internal/continuity"
	"orchestra/internal/domain"
	"orchestra/internal/herdr"
	"orchestra/internal/orchestrator"
)

// reasoningAdapter records the rotation reasons Orchestra asked a handoff for.
type reasoningAdapter struct {
	fakeAdapter
	reasons []string
}

func (a *reasoningAdapter) RequestHandoffReason(_ context.Context, _ herdr.Session, reason string, _ []continuity.DeadEnd) error {
	a.reasons = append(a.reasons, reason)
	return nil
}

var down = errors.New("gitea unreachable")

// Repeated failure at a verified boundary means Orchestra can no longer uphold
// "the newest human input outranks the agent's current intent". The first two
// turns continue, because one outage should not stop work. The third hands the
// task to a successor, whose pre-lease reconcile fails closed while the source
// is still down.
func TestReconcileFailureStreakEscalatesToHandoff(t *testing.T) {
	a := &reasoningAdapter{fakeAdapter: fakeAdapter{occupancy: .5}}
	c, _, task := leasedCoordinator(t, a, gitRepo(t))
	c.ReconcileFailureHandoff = 3
	c.ReconcileHumanInput = func(context.Context, string) error { return down }

	for turn := 1; turn <= 2; turn++ {
		verdict, err := c.TurnDecision(context.Background(), task.ID)
		if err != nil {
			t.Fatal(err)
		}
		if verdict != orchestrator.TurnContinue {
			t.Fatalf("turn %d verdict = %q, want continue", turn, verdict)
		}
		if len(a.reasons) != 0 {
			t.Fatalf("turn %d asked for a handoff: %v", turn, a.reasons)
		}
	}
	verdict, err := c.TurnDecision(context.Background(), task.ID)
	if err != nil {
		t.Fatal(err)
	}
	if verdict != orchestrator.TurnPrepareHandoff {
		t.Fatalf("verdict = %q, want prepare_handoff", verdict)
	}
	if len(a.reasons) != 1 || a.reasons[0] != "reconcile_failure" {
		t.Fatalf("reasons = %v", a.reasons)
	}
	// The count is observable, not just acted on.
	if h := c.MonitorHealth().Sessions[task.ID]; !strings.Contains(h.LastError, "3 consecutive") {
		t.Fatalf("streak not observable: %+v", h)
	}
}

// One success clears the streak. Two failures then a success then a failure is
// one failure, not three.
func TestReconcileSuccessResetsTheStreak(t *testing.T) {
	a := &reasoningAdapter{fakeAdapter: fakeAdapter{occupancy: .5}}
	c, _, task := leasedCoordinator(t, a, gitRepo(t))
	c.ReconcileFailureHandoff = 3
	failing := true
	c.ReconcileHumanInput = func(context.Context, string) error {
		if failing {
			return down
		}
		return nil
	}
	turn := func() string {
		t.Helper()
		v, err := c.TurnDecision(context.Background(), task.ID)
		if err != nil {
			t.Fatal(err)
		}
		return v
	}
	turn()
	turn()
	failing = false
	turn()
	failing = true
	if v := turn(); v != orchestrator.TurnContinue {
		t.Fatalf("verdict = %q, want continue after the streak reset", v)
	}
	if len(a.reasons) != 0 {
		t.Fatalf("escalated on a reset streak: %v", a.reasons)
	}
}

// A rotation that already wants to stop keeps its own reason. Orchestra must
// not manufacture a second trigger for a session that is already handing off.
func TestReconcileStreakDoesNotOverrideAnExistingRotation(t *testing.T) {
	repo := gitRepo(t)
	a := &reasoningAdapter{fakeAdapter: fakeAdapter{occupancy: .95, boundary: true}}
	c, s, task := leasedCoordinator(t, a, repo)
	ref, err := s.PutArtifact([]byte("handoff"))
	if err != nil {
		t.Fatal(err)
	}
	a.ref = ref
	c.ReconcileFailureHandoff = 1
	c.ReconcileHumanInput = func(context.Context, string) error { return down }

	verdict, err := c.TurnDecision(context.Background(), task.ID)
	if err != nil {
		t.Fatal(err)
	}
	if verdict != orchestrator.TurnRotateNow {
		t.Fatalf("verdict = %q, want rotate_now", verdict)
	}
	if len(a.reasons) != 0 {
		t.Fatalf("manufactured a reason for a rotating session: %v", a.reasons)
	}
}

// The escape path has to complete. Once the agent writes the handoff with this
// reason, the ordinary bypass releases the task, exactly as it does for
// manual, milestone and thrash.
func TestReconcileFailureHandoffReleasesTheTask(t *testing.T) {
	repo := gitRepo(t)
	a := &reasoningAdapter{fakeAdapter: fakeAdapter{occupancy: 0, boundary: false}}
	c, s, task := leasedCoordinator(t, a, repo)
	ref, err := s.PutArtifact([]byte("handoff"))
	if err != nil {
		t.Fatal(err)
	}
	a.ref = ref
	head, err := herdr.HeadSHA(repo)
	if err != nil {
		t.Fatal(err)
	}
	b, err := json.Marshal(map[string]any{
		"meta":   map[string]any{"id": "h2", "reason": "reconcile_failure", "rotation_index": 0},
		"anchor": map[string]any{"git_sha": head, "branch": "orchestra/t1"},
		"action": "re-read the task intent before continuing", "command": "go test ./...",
	})
	if err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(repo+"/"+herdr.HandoffFile, b, 0o644); err != nil {
		t.Fatal(err)
	}
	// The reason must survive handoff validation, or the successor cannot read
	// the artifact this release produced.
	if _, err := continuity.Decode(b); err != nil {
		t.Fatalf("handoff rejected: %v", err)
	}
	c.ReconcileHumanInput = func(context.Context, string) error { return down }

	verdict, err := c.TurnDecision(context.Background(), task.ID)
	if err != nil {
		t.Fatal(err)
	}
	if verdict != orchestrator.TurnRotateNow {
		t.Fatalf("verdict = %q, want rotate_now", verdict)
	}
	if got, _ := s.Task(task.ID); got.State != domain.StateQueued {
		t.Fatalf("state = %s, want queued", got.State)
	}
}

// No human source means nothing to fail, so nothing ever escalates.
func TestNoHumanSourceNeverEscalates(t *testing.T) {
	a := &reasoningAdapter{fakeAdapter: fakeAdapter{occupancy: .5}}
	c, _, task := leasedCoordinator(t, a, gitRepo(t))
	c.ReconcileFailureHandoff = 1
	for turn := 0; turn < 5; turn++ {
		verdict, err := c.TurnDecision(context.Background(), task.ID)
		if err != nil {
			t.Fatal(err)
		}
		if verdict != orchestrator.TurnContinue {
			t.Fatalf("verdict = %q, want continue", verdict)
		}
	}
	if len(a.reasons) != 0 {
		t.Fatalf("escalated with no reconciler configured: %v", a.reasons)
	}
}

// Delivering a decision is not reconciling one. A pane that cannot be written
// to must not spend the reconcile budget.
func TestDeliveryFailureIsNotAReconcileFailure(t *testing.T) {
	a := &notifyingAdapter{fakeAdapter: fakeAdapter{occupancy: .5}, err: context.DeadlineExceeded}
	c, s, task := leasedCoordinator(t, a, gitRepo(t))
	c.ReconcileFailureHandoff = 2
	c.ReconcileHumanInput = func(context.Context, string) error { return nil }
	recordDecision(t, s, task.ID, "d1", "no, use b")
	for turn := 0; turn < 3; turn++ {
		verdict, err := c.TurnDecision(context.Background(), task.ID)
		if err != nil {
			t.Fatal(err)
		}
		if verdict != orchestrator.TurnContinue {
			t.Fatalf("verdict = %q, want continue", verdict)
		}
	}
}

// The federated half uses the same threshold, so a worker-owned session and a
// local one behave identically.
func TestRemoteTurnEscalatesOnTheSameThreshold(t *testing.T) {
	c, _, task := remoteLeased(t)
	c.ReconcileFailureHandoff = 3
	c.ReconcileHumanInput = func(context.Context, string) error { return down }
	for turn := 1; turn <= 2; turn++ {
		verdict, _, err := c.RemoteTurn(context.Background(), task.ID, task.Lease.Epoch, orchestrator.TurnContinue, nil)
		if err != nil {
			t.Fatal(err)
		}
		if verdict != orchestrator.TurnContinue {
			t.Fatalf("turn %d verdict = %q, want continue", turn, verdict)
		}
	}
	verdict, decisions, err := c.RemoteTurn(context.Background(), task.ID, task.Lease.Epoch, orchestrator.TurnContinue, nil)
	if err != nil {
		t.Fatal(err)
	}
	if verdict != orchestrator.TurnPrepareHandoff {
		t.Fatalf("verdict = %q, want prepare_handoff", verdict)
	}
	if len(decisions) != 0 {
		t.Fatalf("decisions returned to a rotating session: %+v", decisions)
	}
}

// A worker that already reported a stop keeps its own verdict.
func TestRemoteTurnKeepsTheWorkersVerdict(t *testing.T) {
	c, _, task := remoteLeased(t)
	c.ReconcileFailureHandoff = 1
	c.ReconcileHumanInput = func(context.Context, string) error { return down }
	verdict, _, err := c.RemoteTurn(context.Background(), task.ID, task.Lease.Epoch, orchestrator.TurnRotateNow, nil)
	if err != nil {
		t.Fatal(err)
	}
	if verdict != orchestrator.TurnRotateNow {
		t.Fatalf("verdict = %q, want the worker's own rotate_now", verdict)
	}
}