Close F7, F5 and F8 before resuming burn-in
F7, security. An unset surface token makes the middleware skip its check, so a full-control surface with no credential is an open control plane rather than a closed one. With ORCHESTRA_TUI_TOKEN unset, any LAN caller could lease, release, complete or block any task by declaring one header, which is how this session's manual leases were issued. authz.RequireCredentials now refuses startup instead of logging. Web is exempt: Sessions makes its login mandatory. F5, lifecycle. router.go's silent `continue` was the first bug, not the predicate behind it. Every eligibility gate now records a router.Rejection with task, herdr and reason, exposed at GET /v1/router/health, reset per pass. No gate was weakened: a direct Store.Lease succeeding proves the lease path, not that eligibility should have selected that worker. F8, correctness. Reconcile iterated every configured source for every task, so a task's external id was looked up in whatever repository each source pointed at. Once two repositories share an issue number, an unrelated human comment becomes an authoritative decision for the wrong task. Reconciliation is now bound to task.Source, the provider:project identity the ingest stamped, and a source that cannot prove it owns the task is skipped. A task with no matching source reconciles to nothing and still launches, because nothing to import is not a failure to read. The integration fixture ingested from "jsonl" while reconciling from "gitea", which is exactly the shape F8 makes impossible; it now ingests from the source it reconciles. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -102,9 +102,48 @@ type Router struct {
|
||||
// A scheduling pass always has a coherent health snapshot; this cache also
|
||||
// prevents bursts of TaskCreated events from repeatedly dialing the same
|
||||
// herdr between passes.
|
||||
HealthTTL time.Duration
|
||||
healthMu sync.Mutex
|
||||
health map[string]healthProbe
|
||||
HealthTTL time.Duration
|
||||
healthMu sync.Mutex
|
||||
health map[string]healthProbe
|
||||
rejectMu sync.Mutex
|
||||
rejections []Rejection
|
||||
}
|
||||
|
||||
// Rejection is why one scheduling pass did not give one task to one herdr.
|
||||
// Every eligibility gate records one, because a silent `continue` is
|
||||
// indistinguishable from "nothing was queued": during burn-in a task sat
|
||||
// queued while every gate checked out by hand, and the router said nothing.
|
||||
type Rejection struct {
|
||||
TaskID string `json:"task_id"`
|
||||
HerdrID string `json:"herdr_id,omitempty"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
// maxRejections bounds the recorded set so a large queue cannot grow it without
|
||||
// limit. The newest pass always fits, because the list is reset per pass.
|
||||
const maxRejections = 200
|
||||
|
||||
func (r *Router) reject(taskID, herdrID, reason string) {
|
||||
r.rejectMu.Lock()
|
||||
defer r.rejectMu.Unlock()
|
||||
if len(r.rejections) >= maxRejections {
|
||||
return
|
||||
}
|
||||
r.rejections = append(r.rejections, Rejection{TaskID: taskID, HerdrID: herdrID, Reason: reason})
|
||||
}
|
||||
|
||||
// Rejections returns why the most recent scheduling pass assigned nothing to
|
||||
// the tasks it could not place.
|
||||
func (r *Router) Rejections() []Rejection {
|
||||
r.rejectMu.Lock()
|
||||
defer r.rejectMu.Unlock()
|
||||
return append([]Rejection(nil), r.rejections...)
|
||||
}
|
||||
|
||||
func (r *Router) resetRejections() {
|
||||
r.rejectMu.Lock()
|
||||
r.rejections = nil
|
||||
r.rejectMu.Unlock()
|
||||
}
|
||||
|
||||
type healthProbe struct {
|
||||
@@ -139,6 +178,7 @@ func (r *Router) AssignPending() ([]domain.Event, error) {
|
||||
return nil, errors.New("router: store required")
|
||||
}
|
||||
now := r.Now()
|
||||
r.resetRejections()
|
||||
snapshot := r.Store.SchedulingSnapshot()
|
||||
var queued []domain.Task
|
||||
for _, t := range snapshot.Tasks {
|
||||
@@ -169,10 +209,16 @@ func (r *Router) AssignPending() ([]domain.Event, error) {
|
||||
var err error
|
||||
cs, err = r.Registry.CandidatesWithHealth(t.Project, health)
|
||||
if err != nil {
|
||||
r.reject(t.ID, "", "no candidate herdr for project "+t.Project+": "+err.Error())
|
||||
continue
|
||||
}
|
||||
candidates[t.Project] = cs
|
||||
}
|
||||
if len(cs) == 0 {
|
||||
r.reject(t.ID, "", "no candidate herdr for project "+t.Project+" (affinity, capability or reachability)")
|
||||
continue
|
||||
}
|
||||
placed := false
|
||||
for _, h := range cs {
|
||||
projectKey := h.ID + "\x00" + t.Project
|
||||
projectOK, checked := projectSupport[projectKey], projectSupportKnown[projectKey]
|
||||
@@ -183,7 +229,12 @@ func (r *Router) AssignPending() ([]domain.Event, error) {
|
||||
}
|
||||
projectSupport[projectKey], projectSupportKnown[projectKey] = projectOK, true
|
||||
}
|
||||
if !projectOK || !matches(t.Capability, h.Capabilities) {
|
||||
if !projectOK {
|
||||
r.reject(t.ID, h.ID, "worker has not declared project "+t.Project)
|
||||
continue
|
||||
}
|
||||
if !matches(t.Capability, h.Capabilities) {
|
||||
r.reject(t.ID, h.ID, "capability mismatch")
|
||||
continue
|
||||
}
|
||||
available, checked := availability[h.ID], availabilityKnown[h.ID]
|
||||
@@ -191,13 +242,22 @@ func (r *Router) AssignPending() ([]domain.Event, error) {
|
||||
available = r.Availability.Available(h)
|
||||
availability[h.ID], availabilityKnown[h.ID] = available, true
|
||||
}
|
||||
if !available || occupiedCount(activeLeases[h.ID], h.Concurrency) {
|
||||
if !available {
|
||||
r.reject(t.ID, h.ID, "worker unavailable: stale heartbeat or unhealthy local backend")
|
||||
continue
|
||||
}
|
||||
if occupiedCount(activeLeases[h.ID], h.Concurrency) {
|
||||
r.reject(t.ID, h.ID, "no free concurrency")
|
||||
continue
|
||||
}
|
||||
e, err := r.Store.Lease(t.ID, h.ID, 30*time.Minute)
|
||||
if err != nil {
|
||||
// Includes a pre-lease reconciliation refusal, which is the
|
||||
// one gate that fails closed on purpose.
|
||||
r.reject(t.ID, h.ID, "lease refused: "+err.Error())
|
||||
continue
|
||||
}
|
||||
placed = true
|
||||
out = append(out, e)
|
||||
activeLeases[h.ID]++
|
||||
if r.OnLease != nil {
|
||||
@@ -207,6 +267,9 @@ func (r *Router) AssignPending() ([]domain.Event, error) {
|
||||
}
|
||||
break
|
||||
}
|
||||
if !placed {
|
||||
r.reject(t.ID, "", "no candidate accepted the task")
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user