orchestra/internal/federation/federation_test.go

package federation

import (
	"os"
	"path/filepath"
	"testing"
	"time"
)

func TestCursorIsMonotonicAndAuthenticationIsRequired(t *testing.T) {
	r := &Registry{}
	if err := r.Register(Worker{ID: "workpc", Token: "secret"}, ""); err != nil {
		t.Fatal(err)
	}
	if err := r.Authenticate("workpc", "wrong"); err != ErrUnauthorized {
		t.Fatalf("got %v", err)
	}
	if err := r.Authenticate("workpc", "secret"); err != nil {
		t.Fatal(err)
	}
	if err := r.Ack("workpc", 7); err != nil {
		t.Fatal(err)
	}
	if err := r.Ack("workpc", 6); err == nil {
		t.Fatal("backwards cursor accepted")
	}
	if got, _ := r.Cursor("workpc"); got != 7 {
		t.Fatalf("cursor = %d", got)
	}
}

func TestSupportedProjectsPersistAndGateAvailability(t *testing.T) {
	path := filepath.Join(t.TempDir(), "state.json")
	r := &Registry{StatePath: path}
	if err := r.Register(Worker{ID: "w", Token: "t", SupportedProjects: []string{"test-e2e"}}, ""); err != nil {
		t.Fatal(err)
	}
	if !r.Supports("w", "test-e2e") || r.Supports("w", "correx") {
		t.Fatalf("unexpected project support")
	}
	restarted := &Registry{StatePath: path}
	if err := restarted.Load(); err != nil {
		t.Fatal(err)
	}
	if err := restarted.Register(Worker{ID: "w", Token: "t", SupportedProjects: []string{"test-e2e"}}, ""); err != nil {
		t.Fatal(err)
	}
	if !restarted.Supports("w", "test-e2e") || restarted.Supports("w", "correx") {
		t.Fatalf("project support did not survive restart")
	}
}

func TestAvailableRequiresFreshReachableLocalHerdrHealth(t *testing.T) {
	r := &Registry{}
	if err := r.Register(Worker{ID: "w", Token: "t"}, ""); err != nil {
		t.Fatal(err)
	}
	if r.Available("w") {
		t.Fatal("registration without local herdr probe admitted a worker")
	}
	if err := r.Heartbeat("w", WorkerHealth{HerdrStatus: "reachable", CheckedAt: time.Now().UTC()}); err != nil {
		t.Fatal(err)
	}
	if !r.Available("w") {
		t.Fatal("fresh reachable local herdr was not admitted")
	}
	if err := r.Heartbeat("w", WorkerHealth{HerdrStatus: "unreachable", CheckedAt: time.Now().UTC()}); err != nil {
		t.Fatal(err)
	}
	if r.Available("w") {
		t.Fatal("unreachable local herdr was admitted")
	}
}

func TestPendingApprovalSurvivesRegistryRestart(t *testing.T) {
	path := filepath.Join(t.TempDir(), "federation-state.json")
	r := &Registry{StatePath: path}
	if err := r.Load(); err != nil {
		t.Fatal(err)
	}
	if err := r.Register(Worker{ID: "w", Token: "t"}, ""); err != nil {
		t.Fatal(err)
	}
	capture, err := r.PutCapture("w", Capture{TaskID: "task", PaneID: "pane", Text: "Allow command?"})
	if err != nil {
		t.Fatal(err)
	}
	queued, err := r.Queue("w", Command{TaskID: "task", Kind: "grant_approval", PaneID: "pane", CaptureRevision: capture.Revision})
	if err != nil {
		t.Fatal(err)
	}
	info, err := os.Stat(path)
	if err != nil {
		t.Fatal(err)
	}
	if info.Mode().Perm() != 0600 {
		t.Fatalf("federation state permissions = %o, want 0600", info.Mode().Perm())
	}

	restarted := &Registry{StatePath: path}
	if err := restarted.Load(); err != nil {
		t.Fatal(err)
	}
	if err := restarted.Register(Worker{ID: "w", Token: "intruder"}, ""); err != ErrUnauthorized {
		t.Fatalf("recovered worker identity was hijackable: %v", err)
	}
	// A restart does not mark the worker online; it must prove its retained
	// identity by registering again before recovered controls become available.
	if err := restarted.Register(Worker{ID: "w", Token: "t"}, ""); err != nil {
		t.Fatal(err)
	}
	gotCapture, ok := restarted.Capture("w", "task")
	if !ok || gotCapture.Revision != capture.Revision || gotCapture.Text != capture.Text {
		t.Fatalf("capture after restart = %#v, present=%v", gotCapture, ok)
	}
	commands, err := restarted.Commands("w")
	if err != nil || len(commands) != 1 || commands[0].ID != queued.ID {
		t.Fatalf("commands after restart = %#v, err=%v", commands, err)
	}
	if err := restarted.CompleteCommand("w", queued.ID, "acknowledged", ""); err != nil {
		t.Fatal(err)
	}

	again := &Registry{StatePath: path}
	if err := again.Load(); err != nil {
		t.Fatal(err)
	}
	if err := again.Register(Worker{ID: "w", Token: "t"}, ""); err != nil {
		t.Fatal(err)
	}
	commands, err = again.Commands("w")
	if err != nil || len(commands) != 0 {
		t.Fatalf("resolved command recovered as pending: %#v, err=%v", commands, err)
	}
}

func TestRegisterRequiresAdmitTokenAndOwnToken(t *testing.T) {
	r := &Registry{AdmitToken: "admit-secret"}
	if err := r.Register(Worker{ID: "workpc", Token: "secret"}, "wrong"); err != ErrUnauthorized {
		t.Fatalf("wrong admit token: got %v", err)
	}
	if err := r.Register(Worker{ID: "workpc", Token: "secret"}, "admit-secret"); err != nil {
		t.Fatal(err)
	}
	// Re-registering the same id with a different token is a hijack
	// attempt (S10), not a legitimate re-registration, and must be refused
	// even with a valid admit token.
	if err := r.Register(Worker{ID: "workpc", Token: "different"}, "admit-secret"); err != ErrUnauthorized {
		t.Fatalf("hijack with different token: got %v", err)
	}
	// The same worker re-registering with its own token (e.g. after a
	// restart) must still succeed.
	if err := r.Register(Worker{ID: "workpc", Token: "secret"}, "admit-secret"); err != nil {
		t.Fatalf("legitimate re-registration: %v", err)
	}
}

func TestOfflineHookRunsOnceOnTransition(t *testing.T) {
	called := make(chan Worker, 1)
	r := &Registry{TTL: time.Millisecond, OnOffline: func(w Worker) { called <- w }}
	if err := r.Register(Worker{ID: "workpc", Token: "secret"}, ""); err != nil {
		t.Fatal(err)
	}
	r.mu.Lock()
	w := r.workers["workpc"]
	w.LastSeen = time.Now().Add(-time.Second)
	r.workers["workpc"] = w
	r.mu.Unlock()
	r.Snapshot()
	select {
	case got := <-called:
		if got.ID != "workpc" {
			t.Fatal(got.ID)
		}
	case <-time.After(time.Second):
		t.Fatal("offline hook not called")
	}
	r.Snapshot()
	select {
	case <-called:
		t.Fatal("offline hook called twice")
	case <-time.After(10 * time.Millisecond):
	}
}

func TestHeartbeatProjectsWorkerOwnedHealth(t *testing.T) {
	r := &Registry{}
	if err := r.Register(Worker{ID: "workpc-opencode", Token: "secret"}, ""); err != nil {
		t.Fatal(err)
	}
	checked := time.Now().UTC().Round(0)
	errAt := checked.Add(-time.Minute)
	if err := r.Heartbeat("workpc-opencode", WorkerHealth{
		HerdrStatus: "unreachable", CheckedAt: checked, ActiveTask: "task-1", ActivePane: "pane-1",
		LastError: "local herdr: connection refused", ErrorAt: errAt,
	}); err != nil {
		t.Fatal(err)
	}
	workers := r.Snapshot()
	if len(workers) != 1 {
		t.Fatalf("workers=%#v", workers)
	}
	h := workers[0].Health
	if h.HerdrStatus != "unreachable" || h.ActiveTask != "task-1" || h.ActivePane != "pane-1" || h.LastError == "" || !h.CheckedAt.Equal(checked) || !h.ErrorAt.Equal(errAt) {
		t.Fatalf("health=%#v", h)
	}
}

func TestCaptureRevisionAndCommandQueue(t *testing.T) {
	r := &Registry{}
	if err := r.Register(Worker{ID: "w", Token: "t"}, ""); err != nil {
		t.Fatal(err)
	}
	c, err := r.PutCapture("w", Capture{TaskID: "task", PaneID: "pane", Text: "Permission required\n$ ls"})
	if err != nil || c.Revision != 1 {
		t.Fatalf("capture=%#v err=%v", c, err)
	}
	again, err := r.PutCapture("w", Capture{TaskID: "task", PaneID: "pane", Text: c.Text})
	if err != nil || again.Revision != 1 {
		t.Fatalf("same capture=%#v err=%v", again, err)
	}
	cmd, err := r.Queue("w", Command{TaskID: "task", Kind: "grant_approval", PaneID: "pane", CaptureRevision: 1})
	if err != nil {
		t.Fatal(err)
	}
	commands, err := r.Commands("w")
	if err != nil || len(commands) != 1 || commands[0].ID != cmd.ID {
		t.Fatalf("commands=%#v err=%v", commands, err)
	}
	if err := r.CompleteCommand("w", cmd.ID, "acknowledged", ""); err != nil {
		t.Fatal(err)
	}
	commands, _ = r.Commands("w")
	if len(commands) != 0 {
		t.Fatalf("pending=%#v", commands)
	}
}

func TestResolvedCommandsArePrunedButPendingOnesSurvive(t *testing.T) {
	r := &Registry{}
	if err := r.Register(Worker{ID: "w", Token: "t"}, ""); err != nil {
		t.Fatal(err)
	}
	old, err := r.Queue("w", Command{TaskID: "task", Kind: "grant_approval", PaneID: "pane", CaptureRevision: 1})
	if err != nil {
		t.Fatal(err)
	}
	if err := r.CompleteCommand("w", old.ID, "acknowledged", ""); err != nil {
		t.Fatal(err)
	}
	pending, err := r.Queue("w", Command{TaskID: "task", Kind: "deny_approval", PaneID: "pane", CaptureRevision: 2})
	if err != nil {
		t.Fatal(err)
	}
	// Age both past the retention window; only the resolved one may go.
	r.mu.Lock()
	for i := range r.commands["w"] {
		r.commands["w"][i].CreatedAt = time.Now().UTC().Add(-2 * CommandRetention)
	}
	r.mu.Unlock()
	if _, err := r.Commands("w"); err != nil {
		t.Fatal(err)
	}
	if _, ok := r.Command("w", old.ID); ok {
		t.Fatal("resolved command past retention was not pruned")
	}
	if c, ok := r.Command("w", pending.ID); !ok || c.Status != "pending" {
		t.Fatalf("pending command was pruned: %#v ok=%v", c, ok)
	}
}

// Each unavailability condition must name itself. One collapsed reason once
// reported a stale heartbeat for a worker whose heartbeat was a second old.
func TestUnavailableNamesTheFailingCondition(t *testing.T) {
	r := &Registry{TTL: time.Minute}
	if got, want := r.Unavailable("nobody"), "worker unavailable: no worker registered for this harness"; got != want {
		t.Fatalf("unregistered reason=%q, want %q", got, want)
	}
	if err := r.Register(Worker{ID: "w", Token: "t"}, ""); err != nil {
		t.Fatal(err)
	}
	if got, want := r.Unavailable("w"), "worker unavailable: backend health never reported"; got != want {
		t.Fatalf("unprobed reason=%q, want %q", got, want)
	}
	if err := r.Heartbeat("w", WorkerHealth{HerdrStatus: "unreachable", CheckedAt: time.Now().UTC()}); err != nil {
		t.Fatal(err)
	}
	if got, want := r.Unavailable("w"), "worker unavailable: backend unreachable"; got != want {
		t.Fatalf("unhealthy-backend reason=%q, want %q", got, want)
	}
	if err := r.Heartbeat("w", WorkerHealth{HerdrStatus: "reachable", CheckedAt: time.Now().UTC()}); err != nil {
		t.Fatal(err)
	}
	if got := r.Unavailable("w"); got != "" {
		t.Fatalf("fresh reachable worker reason=%q, want admitted", got)
	}
	stale := &Registry{TTL: time.Nanosecond}
	if err := stale.Register(Worker{ID: "w", Token: "t"}, ""); err != nil {
		t.Fatal(err)
	}
	if err := stale.Heartbeat("w", WorkerHealth{HerdrStatus: "reachable", CheckedAt: time.Now().UTC()}); err != nil {
		t.Fatal(err)
	}
	if got, want := stale.Unavailable("w"), "worker unavailable: stale heartbeat"; got != want {
		t.Fatalf("stale reason=%q, want %q", got, want)
	}
}