fix: make worker handoff rotation durable
This commit is contained in:
@@ -0,0 +1,82 @@
|
||||
package orchestrator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"orchestra/internal/continuity"
|
||||
"orchestra/internal/herdr"
|
||||
)
|
||||
|
||||
// RotationStateMachine is the shared, side-effect-free rotation policy used
|
||||
// by both the coordinator's synchronous turn path and federation workers.
|
||||
// Callers persist request/release side effects themselves, but must never
|
||||
// replace an unavailable occupancy reading with zero.
|
||||
type RotationStateMachine struct {
|
||||
Soft float64
|
||||
Hard float64
|
||||
Thrash herdr.ThrashConfig
|
||||
}
|
||||
|
||||
type RotationDecision struct {
|
||||
Action string // continue, prepare_handoff, rotate_now
|
||||
Reason string
|
||||
DeadEnds []continuity.DeadEnd
|
||||
// ActivityDegraded is advisory (threshold rotation still has a real usage
|
||||
// source); it is surfaced so a missing milestone/thrash feed cannot be a
|
||||
// silent no-op.
|
||||
ActivityDegraded error
|
||||
Degraded error
|
||||
}
|
||||
|
||||
func (m RotationStateMachine) Evaluate(ctx context.Context, a herdr.Adapter, s herdr.Session) RotationDecision {
|
||||
soft := m.Soft
|
||||
if soft <= 0 {
|
||||
soft = defaultSoft
|
||||
}
|
||||
if m.Hard <= 0 || m.Hard <= soft {
|
||||
return RotationDecision{Degraded: fmt.Errorf("invalid rotation thresholds soft=%v hard=%v", soft, m.Hard)}
|
||||
}
|
||||
if reader, ok := a.(herdr.ActivityReader); ok {
|
||||
calls, err := reader.Activity(ctx, s)
|
||||
if err == nil {
|
||||
if thrash, deadEnds := herdr.DetectThrash(calls, m.Thrash); thrash {
|
||||
return RotationDecision{Action: TurnPrepareHandoff, Reason: "thrash", DeadEnds: deadEnds}
|
||||
}
|
||||
if herdr.DetectMilestone(calls) {
|
||||
return RotationDecision{Action: TurnPrepareHandoff, Reason: "milestone"}
|
||||
}
|
||||
} else {
|
||||
return m.evaluateOccupancy(ctx, a, s, fmt.Errorf("activity unknown: %w", err))
|
||||
}
|
||||
}
|
||||
return m.evaluateOccupancy(ctx, a, s, nil)
|
||||
}
|
||||
|
||||
func (m RotationStateMachine) evaluateOccupancy(ctx context.Context, a herdr.Adapter, s herdr.Session, activityErr error) RotationDecision {
|
||||
soft := m.Soft
|
||||
if soft <= 0 {
|
||||
soft = defaultSoft
|
||||
}
|
||||
occupancy, err := a.Occupancy(s)
|
||||
if err != nil {
|
||||
return RotationDecision{ActivityDegraded: activityErr, Degraded: fmt.Errorf("occupancy unknown: %w", err)}
|
||||
}
|
||||
if occupancy < soft {
|
||||
return RotationDecision{Action: TurnContinue, ActivityDegraded: activityErr}
|
||||
}
|
||||
if occupancy < m.Hard {
|
||||
return RotationDecision{Action: TurnPrepareHandoff, Reason: "threshold", ActivityDegraded: activityErr}
|
||||
}
|
||||
boundary, ok := a.(herdr.TurnBoundary)
|
||||
if !ok {
|
||||
return RotationDecision{ActivityDegraded: activityErr, Degraded: fmt.Errorf("turn boundary unknown at hard threshold"), Action: TurnRefuse, Reason: "threshold"}
|
||||
}
|
||||
atBoundary, err := boundary.AtTurnBoundary(ctx, s)
|
||||
if err != nil {
|
||||
return RotationDecision{ActivityDegraded: activityErr, Degraded: fmt.Errorf("turn boundary unknown: %w", err), Action: TurnRefuse, Reason: "threshold"}
|
||||
}
|
||||
if !atBoundary {
|
||||
return RotationDecision{Action: TurnRefuse, Reason: "threshold", ActivityDegraded: activityErr}
|
||||
}
|
||||
return RotationDecision{Action: TurnRotateNow, Reason: "threshold", ActivityDegraded: activityErr}
|
||||
}
|
||||
Reference in New Issue
Block a user