6565b9fce2
A release pushes the anchor first and commits second. When the lease expired in between, the commit could never land: the worker sent the epoch from w.leases, which the expiry replay had already deleted, and the coordinator refused a /handoff without a live owned lease. Run 10 lost a finished task this way, its work sitting in the worktree until retry_limit. The epoch now belongs to the release transaction, so it survives the lease. TaskReleased retains the ending epoch as Task.LastLeaseEpoch, and lateHandoffAccepted lets exactly that owner commit while the task is queued, unleased, and carrying no handoff of its own. A successor that has already re-leased the task holds the lease, so the late handoff loses. Store.Append's version fence settles the race, which is why the late path skips the version check the worker cannot satisfy. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CVbaKucEYBjMqVeUgJUsc1
112 lines
3.9 KiB
Go
112 lines
3.9 KiB
Go
package main
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"orchestra/internal/domain"
|
|
"orchestra/internal/registry"
|
|
)
|
|
|
|
type unreachable struct{}
|
|
|
|
func (unreachable) Reachable(string, time.Duration) bool { return false }
|
|
|
|
func TestFederatedReachabilityDefersRemoteHerdrToWorkerHeartbeat(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "config.json")
|
|
if err := os.WriteFile(path, []byte(`{
|
|
"machines":[{"id":"homesrv","address":"192.168.1.104:9145"},{"id":"workpc","address":"192.168.1.105:9145"}],
|
|
"herdrs":[{"id":"local","machine_id":"homesrv","harness":"opencode"},{"id":"remote","machine_id":"workpc","harness":"opencode"}]
|
|
}`), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r, err := registry.Load(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
check := federatedReachability{base: unreachable{}, remote: remoteHerdrAddresses(r, "homesrv")}
|
|
if check.Reachable("192.168.1.105:9245", time.Second) != true {
|
|
t.Fatal("remote herdr should be admitted for worker heartbeat gating")
|
|
}
|
|
if check.Reachable("192.168.1.104:9245", time.Second) {
|
|
t.Fatal("local herdr should still require its TCP probe")
|
|
}
|
|
}
|
|
|
|
func TestCoordinatorOwnsOnlyLocalHerdrInFederationMode(t *testing.T) {
|
|
local := registry.Herdr{ID: "homesrv-opencode", MachineID: "homesrv"}
|
|
localTmux := registry.Herdr{ID: "homesrv-claude", MachineID: "homesrv", Backend: "tmux", Harness: "claude"}
|
|
remote := registry.Herdr{ID: "workpc-opencode", MachineID: "workpc"}
|
|
if !coordinatorOwnsHerdr(local, "homesrv") {
|
|
t.Fatal("coordinator does not own its local herdr")
|
|
}
|
|
if coordinatorOwnsHerdr(remote, "homesrv") {
|
|
t.Fatal("coordinator claimed a worker-owned remote herdr")
|
|
}
|
|
if coordinatorOwnsHerdr(localTmux, "homesrv") {
|
|
t.Fatal("coordinator claimed a local worker-owned tmux backend")
|
|
}
|
|
if !coordinatorOwnsHerdr(remote, "") {
|
|
t.Fatal("single-machine mode should retain legacy local ownership")
|
|
}
|
|
}
|
|
|
|
func TestMultiMachineRegistryRequiresKnownLocalMachine(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "config.json")
|
|
if err := os.WriteFile(path, []byte(`{
|
|
"machines":[{"id":"homesrv","address":"192.168.1.104:9145"},{"id":"workpc","address":"192.168.1.105:9145"}]
|
|
}`), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r, err := registry.Load(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := validateLocalMachine(r, ""); err == nil {
|
|
t.Fatal("missing local machine accepted")
|
|
}
|
|
if err := validateLocalMachine(r, "missing"); err == nil {
|
|
t.Fatal("unknown local machine accepted")
|
|
}
|
|
if err := validateLocalMachine(r, "homesrv"); err != nil {
|
|
t.Fatalf("known local machine rejected: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestTmuxRegistryRequiresMachineIdentityEvenOnOneMachine(t *testing.T) {
|
|
r, err := registry.New(registry.Config{
|
|
Machines: []registry.Machine{{ID: "homesrv", Address: "homesrv:9145"}},
|
|
Herdrs: []registry.Herdr{{ID: "homesrv-claude", MachineID: "homesrv", Backend: "tmux", Harness: "claude"}},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := validateLocalMachine(r, ""); err == nil {
|
|
t.Fatal("worker-owned tmux backend accepted without machine identity")
|
|
}
|
|
if err := validateLocalMachine(r, "homesrv"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestLateHandoffAcceptedOnlyFromTheExpiredOwner(t *testing.T) {
|
|
expired := domain.Task{State: domain.StateQueued, LastLeaseEpoch: "ep1"}
|
|
if !lateHandoffAccepted(expired, "ep1") {
|
|
t.Fatal("the expired owner must still commit its pushed anchor")
|
|
}
|
|
for name, tk := range map[string]domain.Task{
|
|
"re-leased": {State: domain.StateLeased, Lease: &domain.Lease{Epoch: "ep2"}, LastLeaseEpoch: "ep1"},
|
|
"already handed off": {State: domain.StateQueued, LastLeaseEpoch: "ep1", HandoffRef: "sha256:x"},
|
|
"blocked": {State: domain.StateBlocked, LastLeaseEpoch: "ep1"},
|
|
} {
|
|
if lateHandoffAccepted(tk, "ep1") {
|
|
t.Fatalf("%s must refuse a late handoff", name)
|
|
}
|
|
}
|
|
if lateHandoffAccepted(expired, "ep2") || lateHandoffAccepted(domain.Task{State: domain.StateQueued}, "") {
|
|
t.Fatal("a foreign or empty epoch must refuse")
|
|
}
|
|
}
|