Close F7, F5 and F8 before resuming burn-in

F7, security. An unset surface token makes the middleware skip its check, so a full-control surface with no credential is an open control plane rather than a closed one. With ORCHESTRA_TUI_TOKEN unset, any LAN caller could lease, release, complete or block any task by declaring one header, which is how this session's manual leases were issued. authz.RequireCredentials now refuses startup instead of logging. Web is exempt: Sessions makes its login mandatory. F5, lifecycle. router.go's silent `continue` was the first bug, not the predicate behind it. Every eligibility gate now records a router.Rejection with task, herdr and reason, exposed at GET /v1/router/health, reset per pass. No gate was weakened: a direct Store.Lease succeeding proves the lease path, not that eligibility should have selected that worker. F8, correctness. Reconcile iterated every configured source for every task, so a task's external id was looked up in whatever repository each source pointed at. Once two repositories share an issue number, an unrelated human comment becomes an authoritative decision for the wrong task. Reconciliation is now bound to task.Source, the provider:project identity the ingest stamped, and a source that cannot prove it owns the task is skipped. A task with no matching source reconciles to nothing and still launches, because nothing to import is not a failure to read. The integration fixture ingested from "jsonl" while reconciling from "gitea", which is exactly the shape F8 makes impossible; it now ingests from the source it reconciles. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-26 23:36:10 +04:00
parent 7e49348096
commit 0ead6d2d02
8 changed files with 265 additions and 18 deletions
@@ -102,9 +102,48 @@ type Router struct {
 	// A scheduling pass always has a coherent health snapshot; this cache also
 	// prevents bursts of TaskCreated events from repeatedly dialing the same
 	// herdr between passes.
-	HealthTTL time.Duration
-	healthMu  sync.Mutex
-	health    map[string]healthProbe
+	HealthTTL  time.Duration
+	healthMu   sync.Mutex
+	health     map[string]healthProbe
+	rejectMu   sync.Mutex
+	rejections []Rejection
+}
+
+// Rejection is why one scheduling pass did not give one task to one herdr.
+// Every eligibility gate records one, because a silent `continue` is
+// indistinguishable from "nothing was queued": during burn-in a task sat
+// queued while every gate checked out by hand, and the router said nothing.
+type Rejection struct {
+	TaskID  string `json:"task_id"`
+	HerdrID string `json:"herdr_id,omitempty"`
+	Reason  string `json:"reason"`
+}
+
+// maxRejections bounds the recorded set so a large queue cannot grow it without
+// limit. The newest pass always fits, because the list is reset per pass.
+const maxRejections = 200
+
+func (r *Router) reject(taskID, herdrID, reason string) {
+	r.rejectMu.Lock()
+	defer r.rejectMu.Unlock()
+	if len(r.rejections) >= maxRejections {
+		return
+	}
+	r.rejections = append(r.rejections, Rejection{TaskID: taskID, HerdrID: herdrID, Reason: reason})
+}
+
+// Rejections returns why the most recent scheduling pass assigned nothing to
+// the tasks it could not place.
+func (r *Router) Rejections() []Rejection {
+	r.rejectMu.Lock()
+	defer r.rejectMu.Unlock()
+	return append([]Rejection(nil), r.rejections...)
+}
+
+func (r *Router) resetRejections() {
+	r.rejectMu.Lock()
+	r.rejections = nil
+	r.rejectMu.Unlock()
 }

 type healthProbe struct {
@@ -139,6 +178,7 @@ func (r *Router) AssignPending() ([]domain.Event, error) {
 		return nil, errors.New("router: store required")
 	}
 	now := r.Now()
+	r.resetRejections()
 	snapshot := r.Store.SchedulingSnapshot()
 	var queued []domain.Task
 	for _, t := range snapshot.Tasks {
@@ -169,10 +209,16 @@ func (r *Router) AssignPending() ([]domain.Event, error) {
 			var err error
 			cs, err = r.Registry.CandidatesWithHealth(t.Project, health)
 			if err != nil {
+				r.reject(t.ID, "", "no candidate herdr for project "+t.Project+": "+err.Error())
 				continue
 			}
 			candidates[t.Project] = cs
 		}
+		if len(cs) == 0 {
+			r.reject(t.ID, "", "no candidate herdr for project "+t.Project+" (affinity, capability or reachability)")
+			continue
+		}
+		placed := false
 		for _, h := range cs {
 			projectKey := h.ID + "\x00" + t.Project
 			projectOK, checked := projectSupport[projectKey], projectSupportKnown[projectKey]
@@ -183,7 +229,12 @@ func (r *Router) AssignPending() ([]domain.Event, error) {
 				}
 				projectSupport[projectKey], projectSupportKnown[projectKey] = projectOK, true
 			}
-			if !projectOK || !matches(t.Capability, h.Capabilities) {
+			if !projectOK {
+				r.reject(t.ID, h.ID, "worker has not declared project "+t.Project)
+				continue
+			}
+			if !matches(t.Capability, h.Capabilities) {
+				r.reject(t.ID, h.ID, "capability mismatch")
 				continue
 			}
 			available, checked := availability[h.ID], availabilityKnown[h.ID]
@@ -191,13 +242,22 @@ func (r *Router) AssignPending() ([]domain.Event, error) {
 				available = r.Availability.Available(h)
 				availability[h.ID], availabilityKnown[h.ID] = available, true
 			}
-			if !available || occupiedCount(activeLeases[h.ID], h.Concurrency) {
+			if !available {
+				r.reject(t.ID, h.ID, "worker unavailable: stale heartbeat or unhealthy local backend")
+				continue
+			}
+			if occupiedCount(activeLeases[h.ID], h.Concurrency) {
+				r.reject(t.ID, h.ID, "no free concurrency")
 				continue
 			}
 			e, err := r.Store.Lease(t.ID, h.ID, 30*time.Minute)
 			if err != nil {
+				// Includes a pre-lease reconciliation refusal, which is the
+				// one gate that fails closed on purpose.
+				r.reject(t.ID, h.ID, "lease refused: "+err.Error())
 				continue
 			}
+			placed = true
 			out = append(out, e)
 			activeLeases[h.ID]++
 			if r.OnLease != nil {
@@ -207,6 +267,9 @@ func (r *Router) AssignPending() ([]domain.Event, error) {
 			}
 			break
 		}
+		if !placed {
+			r.reject(t.ID, "", "no candidate accepted the task")
+		}
 	}
 	return out, nil
 }
@@ -2,6 +2,9 @@ package router

 import (
 	"encoding/json"
+	"errors"
+	"strings"
+
 	"orchestra/internal/authz"
 	"orchestra/internal/domain"
 	"orchestra/internal/registry"
@@ -299,3 +302,74 @@ func TestQuotaWindowsAreIndependent(t *testing.T) {
 		t.Fatal("bounded harness without a native usage receipt must fail closed")
 	}
 }
+
+// Every eligibility gate must say why. A silent `continue` is
+// indistinguishable from an empty queue: during burn-in a task sat queued
+// while every gate checked out by hand, and the router reported nothing.
+func TestAssignPendingRecordsWhyItPlacedNothing(t *testing.T) {
+	s, err := store.Open(t.TempDir())
+	if err != nil {
+		t.Fatal(err)
+	}
+	r, err := registry.New(registry.Config{
+		Projects: []registry.Project{{ID: "p", MachineAffinity: []string{"m"}}},
+		Machines: []registry.Machine{{ID: "m", Address: "unused"}},
+		Herdrs:   []registry.Herdr{{ID: "h", MachineID: "m", Concurrency: 1}},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	b, _ := json.Marshal(map[string]any{"source": "test", "external_id": "why", "project": "p"})
+	if err := s.Append(domain.Event{ID: "create", TaskID: "t", Type: "TaskCreated", Version: 1, Payload: b, Surface: string(authz.System)}); err != nil {
+		t.Fatal(err)
+	}
+
+	// The live shape that went undiagnosed: a worker that has not declared the
+	// project. Everything else about it looks healthy.
+	rt := Router{Store: s, Registry: r, Reachability: reachable{}, Availability: projectAvailability{projects: map[string]bool{}}}
+	if got, err := rt.AssignPending(); err != nil || len(got) != 0 {
+		t.Fatalf("lease = %#v, %v", got, err)
+	}
+	reasons := rt.Rejections()
+	if len(reasons) == 0 {
+		t.Fatal("the router placed nothing and said nothing")
+	}
+	var named bool
+	for _, rej := range reasons {
+		if rej.TaskID != "t" {
+			t.Fatalf("rejection for the wrong task: %+v", rej)
+		}
+		if rej.HerdrID == "h" && strings.Contains(rej.Reason, "has not declared project") {
+			named = true
+		}
+	}
+	if !named {
+		t.Fatalf("no rejection names the failing gate: %+v", reasons)
+	}
+
+	// A pre-lease refusal is the gate that fails closed on purpose, and it must
+	// also be visible rather than looking like "no candidates".
+	rt.Availability = projectAvailability{projects: map[string]bool{"h/p": true}}
+	s.PreLease = func(string) error { return errors.New("gitea unreachable") }
+	if got, err := rt.AssignPending(); err != nil || len(got) != 0 {
+		t.Fatalf("lease despite pre-lease refusal = %#v, %v", got, err)
+	}
+	found := false
+	for _, rej := range rt.Rejections() {
+		if strings.Contains(rej.Reason, "lease refused") && strings.Contains(rej.Reason, "gitea unreachable") {
+			found = true
+		}
+	}
+	if !found {
+		t.Fatalf("pre-lease refusal not reported: %+v", rt.Rejections())
+	}
+
+	// A successful pass leaves nothing behind to misread.
+	s.PreLease = nil
+	if got, err := rt.AssignPending(); err != nil || len(got) != 1 {
+		t.Fatalf("lease = %#v, %v", got, err)
+	}
+	if reasons := rt.Rejections(); len(reasons) != 0 {
+		t.Fatalf("stale rejections after a successful pass: %+v", reasons)
+	}
+}