Recover worker leases after empty event replay
This commit is contained in:
@@ -260,6 +260,16 @@ func (w *worker) once(ctx context.Context) error {
|
||||
w.cursor = e.Seq
|
||||
}
|
||||
}
|
||||
// A coordinator restart can restore its task snapshot without retaining
|
||||
// the in-memory event tail. In that state a worker with a persisted cursor
|
||||
// receives an empty page even though a lease is currently assigned to it.
|
||||
// Reconcile the authoritative projection before treating an empty page as
|
||||
// "nothing to do"; otherwise the lease remains invisible until expiry.
|
||||
if len(es) == 0 {
|
||||
if err := w.reconcileLeases(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// State is only a cache. If a lease survived but its TaskCreated event is
|
||||
// older than the worker's cursor (or the event has been compacted), hydrate
|
||||
// the authoritative task projection before deciding whether to start.
|
||||
@@ -294,6 +304,29 @@ func (w *worker) once(ctx context.Context) error {
|
||||
return w.api.Ack(ctx, w.cursor)
|
||||
}
|
||||
|
||||
func (w *worker) reconcileLeases(ctx context.Context) error {
|
||||
tasks, err := w.api.Tasks(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reconcile leased tasks: %w", err)
|
||||
}
|
||||
active := make(map[string]lease)
|
||||
for _, task := range tasks {
|
||||
w.tasks[task.ID] = task
|
||||
if task.State == domain.StateLeased && task.Lease != nil && task.Lease.HarnessID == w.harnessID {
|
||||
active[task.ID] = lease{HandoffRef: task.HandoffRef}
|
||||
}
|
||||
}
|
||||
for taskID := range w.leases {
|
||||
if _, ok := active[taskID]; !ok {
|
||||
delete(w.leases, taskID)
|
||||
}
|
||||
}
|
||||
for taskID, l := range active {
|
||||
w.leases[taskID] = l
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// reRegisterAfterCoordinatorRestart restores the coordinator's in-memory
|
||||
// worker registry. A worker must survive a server restart without operator
|
||||
// intervention; its persisted cursor and sessions remain valid.
|
||||
|
||||
Reference in New Issue
Block a user