orchestra/internal/integration/reconcile_escalation_test.go

package integration

import (
	"context"
	"errors"
	"testing"
	"time"

	"orchestra/internal/domain"
	"orchestra/internal/human"
	"orchestra/internal/orchestrator"
	"orchestra/internal/router"
)

// The escape path, end to end. A source that stays down does not freeze a live
// session and does not let it run forever on intent Orchestra cannot refresh:
// the session is handed off, and the task then waits for the source rather
// than resuming from an older authority.
func TestReconcileFailureStreakHandsTheTaskToASuccessor(t *testing.T) {
	s, reg, _ := setup(t)
	task := ingest(t, s, "381")
	src := &tracingSource{tr: &trace{}}
	rec := &human.Reconciler{Store: s, Sources: map[string]human.Source{"gitea": src}}
	s.PreLease = func(id string) error { return rec.Reconcile(context.Background(), id) }

	h := &harness{occupancy: .5}
	c := &orchestrator.Coordinator{Store: s, Worktrees: worktrees{}, Adapters: adapters{h}, StatePath: t.TempDir() + "/sessions.json", Hard: .8}
	c.ReconcileFailureHandoff = 3
	c.ReconcileHumanInput = rec.Reconcile
	rt := router.Router{Store: s, Registry: reg, Reachability: alwaysReachable{}, OnLease: func(e domain.Event) error {
		return c.Start(context.Background(), e)
	}}
	if leased, err := rt.AssignPending(); err != nil || len(leased) != 1 {
		t.Fatalf("leased=%d err=%v", len(leased), err)
	}

	// The source goes down while the session is running.
	src.err = errors.New("gitea unreachable")
	for turn := 1; turn <= 2; turn++ {
		verdict, err := c.TurnDecision(context.Background(), task.ID)
		if err != nil {
			t.Fatal(err)
		}
		if verdict != orchestrator.TurnContinue {
			t.Fatalf("turn %d verdict = %q, want continue", turn, verdict)
		}
	}
	verdict, err := c.TurnDecision(context.Background(), task.ID)
	if err != nil {
		t.Fatal(err)
	}
	if verdict != orchestrator.TurnPrepareHandoff {
		t.Fatalf("third verdict = %q, want prepare_handoff", verdict)
	}

	// The agent writes its handoff and the session releases. (The release
	// mechanics are proven in internal/orchestrator; what matters here is what
	// happens to the task afterwards.)
	ref, err := s.PutArtifact([]byte("handoff: next, keep going from the anchor"))
	if err != nil {
		t.Fatal(err)
	}
	leased, _ := s.Task(task.ID)
	if err := s.Append(domain.Event{ID: domain.NewID(), Type: "TaskReleased", TaskID: task.ID, Version: leased.Version + 1, Surface: "system", Payload: mustJSON(map[string]any{
		"handoff_ref":      ref,
		"reason":           "reconcile_failure",
		"anchor_sha":       "0123456789012345678901234567890123456789",
		"harness_id":       leased.Lease.HarnessID,
		"lease_epoch":      leased.Lease.Epoch,
		"expected_version": leased.Version,
	})}); err != nil {
		t.Fatal(err)
	}

	// Fail closed: while the source is still down, no successor starts.
	if got, err := rt.AssignPending(); len(got) != 0 {
		t.Fatalf("a successor was leased with the source still down: %v (err=%v)", got, err)
	}
	if got, _ := s.Task(task.ID); got.State != domain.StateQueued {
		t.Fatalf("state = %s, want queued", got.State)
	}

	// The source recovers, carrying the correction the outage was hiding.
	src.err = nil
	src.inputs = []human.Input{{Provider: "gitea", ExternalID: "918", Author: "kami", Body: "no, use b", At: time.Now().UTC()}}
	src.next = "918"
	if got, err := rt.AssignPending(); err != nil || len(got) != 1 {
		t.Fatalf("successor leased=%d err=%v", len(got), err)
	}
	intent, err := s.EffectiveIntent(task.ID)
	if err != nil {
		t.Fatal(err)
	}
	if len(intent.Decisions) != 1 || intent.Decisions[0].Value != "no, use b" {
		t.Fatalf("successor authority = %+v", intent.Decisions)
	}
	if intent.Task.HandoffRef != ref {
		t.Fatalf("handoff ref = %q, want %q", intent.Task.HandoffRef, ref)
	}
}