Harden lease lifecycle durability

This commit is contained in:
kami
2026-07-30 14:34:29 +04:00
parent 1ff0af2e69
commit f6ee0e3060
40 changed files with 2108 additions and 590 deletions
+11 -1
View File
@@ -14,6 +14,12 @@ import (
)
type Availability interface{ Available(h registry.Herdr) bool }
// ProjectAvailability is an optional stricter availability contract used by
// federated workers, whose local checkout configuration is authoritative.
type ProjectAvailability interface {
Supports(h registry.Herdr, project string) bool
}
type AlwaysAvailable struct{}
func (AlwaysAvailable) Available(registry.Herdr) bool { return true }
@@ -170,7 +176,11 @@ func (r *Router) AssignPending() ([]domain.Event, error) {
continue
}
for _, h := range cs {
if !matches(t.Capability, h.Capabilities) || !r.Availability.Available(h) || occupied(r.Store, h.ID, h.Concurrency) {
projectOK := true
if projects, ok := r.Availability.(ProjectAvailability); ok {
projectOK = projects.Supports(h, t.Project)
}
if !projectOK || !matches(t.Capability, h.Capabilities) || !r.Availability.Available(h) || occupied(r.Store, h.ID, h.Concurrency) {
continue
}
e, err := r.Store.Lease(t.ID, h.ID, 30*time.Minute)
+40 -4
View File
@@ -14,6 +14,13 @@ type reachable struct{}
func (reachable) Reachable(string, time.Duration) bool { return true }
type projectAvailability struct{ projects map[string]bool }
func (p projectAvailability) Available(registry.Herdr) bool { return true }
func (p projectAvailability) Supports(h registry.Herdr, project string) bool {
return p.projects[h.ID+"/"+project]
}
func TestAssignsByAffinityCapabilityAndConcurrency(t *testing.T) {
s, err := store.Open(t.TempDir())
if err != nil {
@@ -56,6 +63,32 @@ func TestAssignsByAffinityCapabilityAndConcurrency(t *testing.T) {
}
}
func TestAssignPendingRequiresWorkerProjectSupport(t *testing.T) {
s, err := store.Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
r, err := registry.New(registry.Config{Projects: []registry.Project{{ID: "p", MachineAffinity: []string{"m"}}}, Machines: []registry.Machine{{ID: "m", Address: "unused"}}, Herdrs: []registry.Herdr{{ID: "h", MachineID: "m", Concurrency: 1}}})
if err != nil {
t.Fatal(err)
}
b, _ := json.Marshal(map[string]any{"source": "test", "external_id": "project-check", "project": "p"})
if err := s.Append(domain.Event{ID: "create", TaskID: "t", Type: "TaskCreated", Version: 1, Payload: b, Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
rt := Router{Store: s, Registry: r, Reachability: reachable{}, Availability: projectAvailability{projects: map[string]bool{}}}
if got, err := rt.AssignPending(); err != nil || len(got) != 0 {
t.Fatalf("unsupported project lease = %#v, %v", got, err)
}
if got, _ := s.Task("t"); got.State != domain.StateQueued {
t.Fatalf("unsupported project state=%s", got.State)
}
rt.Availability = projectAvailability{projects: map[string]bool{"h/p": true}}
if got, err := rt.AssignPending(); err != nil || len(got) != 1 {
t.Fatalf("supported project lease = %#v, %v", got, err)
}
}
// TestRotationDoesNotCountAgainstRetryLimit guards B4: rotation is
// TaskReleased carrying a valid handoff_ref (spec §5.3: "rotation =
// intra-task lease transfer"), never a failure. A task healthy enough to
@@ -103,10 +136,13 @@ func TestRotationDoesNotCountAgainstRetryLimit(t *testing.T) {
}
continue
}
rb, _ := json.Marshal(map[string]string{
"handoff_ref": handoffRef,
"reason": "threshold",
"anchor_sha": "0123456789abcdef0123456789abcdef01234567",
rb, _ := json.Marshal(map[string]any{
"handoff_ref": handoffRef,
"reason": "threshold",
"anchor_sha": "0123456789abcdef0123456789abcdef01234567",
"harness_id": task.Lease.HarnessID,
"lease_epoch": task.Lease.Epoch,
"expected_version": task.Version,
})
release := domain.Event{ID: domain.NewID(), Type: "TaskReleased", TaskID: "a", Version: task.Version + 1, Payload: rb, Surface: string(authz.System)}
if err := s.Append(release); err != nil {