Harden lease lifecycle durability
This commit is contained in:
@@ -14,6 +14,12 @@ import (
|
||||
)
|
||||
|
||||
type Availability interface{ Available(h registry.Herdr) bool }
|
||||
|
||||
// ProjectAvailability is an optional stricter availability contract used by
|
||||
// federated workers, whose local checkout configuration is authoritative.
|
||||
type ProjectAvailability interface {
|
||||
Supports(h registry.Herdr, project string) bool
|
||||
}
|
||||
type AlwaysAvailable struct{}
|
||||
|
||||
func (AlwaysAvailable) Available(registry.Herdr) bool { return true }
|
||||
@@ -170,7 +176,11 @@ func (r *Router) AssignPending() ([]domain.Event, error) {
|
||||
continue
|
||||
}
|
||||
for _, h := range cs {
|
||||
if !matches(t.Capability, h.Capabilities) || !r.Availability.Available(h) || occupied(r.Store, h.ID, h.Concurrency) {
|
||||
projectOK := true
|
||||
if projects, ok := r.Availability.(ProjectAvailability); ok {
|
||||
projectOK = projects.Supports(h, t.Project)
|
||||
}
|
||||
if !projectOK || !matches(t.Capability, h.Capabilities) || !r.Availability.Available(h) || occupied(r.Store, h.ID, h.Concurrency) {
|
||||
continue
|
||||
}
|
||||
e, err := r.Store.Lease(t.ID, h.ID, 30*time.Minute)
|
||||
|
||||
@@ -14,6 +14,13 @@ type reachable struct{}
|
||||
|
||||
func (reachable) Reachable(string, time.Duration) bool { return true }
|
||||
|
||||
type projectAvailability struct{ projects map[string]bool }
|
||||
|
||||
func (p projectAvailability) Available(registry.Herdr) bool { return true }
|
||||
func (p projectAvailability) Supports(h registry.Herdr, project string) bool {
|
||||
return p.projects[h.ID+"/"+project]
|
||||
}
|
||||
|
||||
func TestAssignsByAffinityCapabilityAndConcurrency(t *testing.T) {
|
||||
s, err := store.Open(t.TempDir())
|
||||
if err != nil {
|
||||
@@ -56,6 +63,32 @@ func TestAssignsByAffinityCapabilityAndConcurrency(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssignPendingRequiresWorkerProjectSupport(t *testing.T) {
|
||||
s, err := store.Open(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r, err := registry.New(registry.Config{Projects: []registry.Project{{ID: "p", MachineAffinity: []string{"m"}}}, Machines: []registry.Machine{{ID: "m", Address: "unused"}}, Herdrs: []registry.Herdr{{ID: "h", MachineID: "m", Concurrency: 1}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, _ := json.Marshal(map[string]any{"source": "test", "external_id": "project-check", "project": "p"})
|
||||
if err := s.Append(domain.Event{ID: "create", TaskID: "t", Type: "TaskCreated", Version: 1, Payload: b, Surface: string(authz.System)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rt := Router{Store: s, Registry: r, Reachability: reachable{}, Availability: projectAvailability{projects: map[string]bool{}}}
|
||||
if got, err := rt.AssignPending(); err != nil || len(got) != 0 {
|
||||
t.Fatalf("unsupported project lease = %#v, %v", got, err)
|
||||
}
|
||||
if got, _ := s.Task("t"); got.State != domain.StateQueued {
|
||||
t.Fatalf("unsupported project state=%s", got.State)
|
||||
}
|
||||
rt.Availability = projectAvailability{projects: map[string]bool{"h/p": true}}
|
||||
if got, err := rt.AssignPending(); err != nil || len(got) != 1 {
|
||||
t.Fatalf("supported project lease = %#v, %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRotationDoesNotCountAgainstRetryLimit guards B4: rotation is
|
||||
// TaskReleased carrying a valid handoff_ref (spec §5.3: "rotation =
|
||||
// intra-task lease transfer"), never a failure. A task healthy enough to
|
||||
@@ -103,10 +136,13 @@ func TestRotationDoesNotCountAgainstRetryLimit(t *testing.T) {
|
||||
}
|
||||
continue
|
||||
}
|
||||
rb, _ := json.Marshal(map[string]string{
|
||||
"handoff_ref": handoffRef,
|
||||
"reason": "threshold",
|
||||
"anchor_sha": "0123456789abcdef0123456789abcdef01234567",
|
||||
rb, _ := json.Marshal(map[string]any{
|
||||
"handoff_ref": handoffRef,
|
||||
"reason": "threshold",
|
||||
"anchor_sha": "0123456789abcdef0123456789abcdef01234567",
|
||||
"harness_id": task.Lease.HarnessID,
|
||||
"lease_epoch": task.Lease.Epoch,
|
||||
"expected_version": task.Version,
|
||||
})
|
||||
release := domain.Event{ID: domain.NewID(), Type: "TaskReleased", TaskID: "a", Version: task.Version + 1, Payload: rb, Surface: string(authz.System)}
|
||||
if err := s.Append(release); err != nil {
|
||||
|
||||
Reference in New Issue
Block a user