Report queued tasks the scheduling pass never considers

A task in retry backoff was filtered out before the candidate loop, so it
recorded no rejection at all: queued, apparently assignable, and silent. That is
the exact shape that made F5 take a live session to diagnose. It now reports
"retry backoff until <time>".

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
2026-08-26 23:42:26 +04:00
parent 0ead6d2d02
commit 4fbf3ac966
18 changed files with 1163 additions and 165 deletions
+46 -44
View File
@@ -12,8 +12,6 @@ import (
"strings"
"sync"
"time"
"golang.org/x/crypto/bcrypt"
)
type Surface string
@@ -120,41 +118,17 @@ func GatedWritePath(p string) bool {
// because it verifies login credentials and exchanges them for a cookie.
const SessionPath = "/v1/ui/session"
// WebCredentials is the single configured browser operator identity. Only a
// bcrypt password hash is accepted; Orchestra has no self-service account
// creation or password-reset surface.
type WebCredentials struct {
Username string
PasswordHash string
}
func (c WebCredentials) Validate() error {
if strings.TrimSpace(c.Username) == "" {
return fmt.Errorf("web username is required")
}
if c.PasswordHash == "" {
return fmt.Errorf("web password hash is required")
}
if _, err := bcrypt.Cost([]byte(c.PasswordHash)); err != nil {
return fmt.Errorf("web password hash must be bcrypt: %w", err)
}
return nil
}
// Authenticate always performs bcrypt, even for an unknown username, so the
// response does not reveal whether the configured username was correct.
func (c WebCredentials) Authenticate(username, password string) bool {
passwordOK := bcrypt.CompareHashAndPassword([]byte(c.PasswordHash), []byte(password)) == nil
usernameOK := subtle.ConstantTimeCompare([]byte(username), []byte(c.Username)) == 1
return passwordOK && usernameOK
}
// Sessions issues and validates those receipts. Values are random and stored
// hashed, so a leaked snapshot of this map does not yield a usable cookie.
type browserSession struct {
Username string
Expires time.Time
}
type Sessions struct {
mu sync.Mutex
TTL time.Duration
ids map[string]time.Time
ids map[string]browserSession
}
func (s *Sessions) ttl() time.Duration {
@@ -164,9 +138,17 @@ func (s *Sessions) ttl() time.Duration {
return 12 * time.Hour
}
// Duration exposes the configured session lifetime for the cookie Max-Age.
func (s *Sessions) Duration() time.Duration { return s.ttl() }
// Issue mints a session value. The caller must have already verified the
// Web-surface token; Issue does not check credentials itself.
func (s *Sessions) Issue() (string, error) {
return s.IssueFor("")
}
// IssueFor mints a session bound to one database-backed operator identity.
func (s *Sessions) IssueFor(username string) (string, error) {
b := make([]byte, 32)
if _, err := rand.Read(b); err != nil {
return "", err
@@ -176,34 +158,41 @@ func (s *Sessions) Issue() (string, error) {
s.mu.Lock()
defer s.mu.Unlock()
if s.ids == nil {
s.ids = map[string]time.Time{}
s.ids = map[string]browserSession{}
}
now := time.Now()
for k, exp := range s.ids {
if now.After(exp) {
for k, session := range s.ids {
if now.After(session.Expires) {
delete(s.ids, k)
}
}
s.ids[hex.EncodeToString(sum[:])] = now.Add(s.ttl())
s.ids[hex.EncodeToString(sum[:])] = browserSession{Username: username, Expires: now.Add(s.ttl())}
return v, nil
}
func (s *Sessions) Valid(v string) bool {
_, ok := s.Username(v)
return ok
}
// Username validates a receipt and returns the operator it belongs to.
func (s *Sessions) Username(v string) (string, bool) {
if v == "" {
return false
return "", false
}
sum := sha256.Sum256([]byte(v))
key := hex.EncodeToString(sum[:])
s.mu.Lock()
defer s.mu.Unlock()
exp, ok := s.ids[hex.EncodeToString(sum[:])]
session, ok := s.ids[key]
if !ok {
return false
return "", false
}
if time.Now().After(exp) {
delete(s.ids, hex.EncodeToString(sum[:]))
return false
if time.Now().After(session.Expires) {
delete(s.ids, key)
return "", false
}
return true
return session.Username, true
}
// Revoke removes one browser session. It is deliberately idempotent so a
@@ -219,6 +208,18 @@ func (s *Sessions) Revoke(v string) {
delete(s.ids, hex.EncodeToString(sum[:]))
}
// RevokeUser ends every browser session for an identity after its username or
// password changes.
func (s *Sessions) RevokeUser(username string) {
s.mu.Lock()
defer s.mu.Unlock()
for key, session := range s.ids {
if strings.EqualFold(session.Username, username) {
delete(s.ids, key)
}
}
}
// HTTP enforces the same policy at the bus boundary. Authentication is
// optional for local development; when a token is supplied, control surfaces
// must present it as a Bearer token.
@@ -272,7 +273,8 @@ func HTTPWithSessions(tokens map[Surface]string, sessions *Sessions, next http.H
if c, err := r.Cookie(SessionCookie); err == nil {
ok = sessions.Valid(c.Value)
}
// The login endpoint authenticates itself, and the SPA shell must
// The session endpoint authenticates login and session lookup itself,
// and the SPA shell must
// load before a browser can present a session. Static assets are not
// secrets; every other /v1/ control path remains session-gated.
if r.URL.Path == SessionPath || (!strings.HasPrefix(r.URL.Path, "/v1/") && (r.Method == http.MethodGet || r.Method == http.MethodHead)) {
+22 -24
View File
@@ -5,8 +5,6 @@ import (
"net/http/httptest"
"testing"
"time"
"golang.org/x/crypto/bcrypt"
)
func TestSurfaceCapabilities(t *testing.T) {
@@ -100,28 +98,6 @@ func TestWebSessionCookieGatesControlPathsOnly(t *testing.T) {
}
}
func TestWebCredentialsAuthenticate(t *testing.T) {
hash, err := bcrypt.GenerateFromPassword([]byte("correct horse battery staple"), bcrypt.MinCost)
if err != nil {
t.Fatal(err)
}
c := WebCredentials{Username: "operator", PasswordHash: string(hash)}
if err := c.Validate(); err != nil {
t.Fatalf("Validate: %v", err)
}
if !c.Authenticate("operator", "correct horse battery staple") {
t.Fatal("correct credentials rejected")
}
for _, attempt := range []struct{ username, password string }{{"operator", "wrong"}, {"other", "correct horse battery staple"}} {
if c.Authenticate(attempt.username, attempt.password) {
t.Fatalf("invalid credentials accepted: %+v", attempt)
}
}
if err := (WebCredentials{Username: "operator", PasswordHash: "not-a-bcrypt-hash"}).Validate(); err == nil {
t.Fatal("invalid bcrypt hash accepted")
}
}
func TestFederationRequestsUseTheirOwnCredentials(t *testing.T) {
tokens := map[Surface]string{Web: "web-secret"}
h := HTTPWithSessions(tokens, nil, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -189,6 +165,28 @@ func TestSessionRevoke(t *testing.T) {
}
}
func TestSessionTracksAndRevokesOperator(t *testing.T) {
s := &Sessions{}
one, err := s.IssueFor("kami")
if err != nil {
t.Fatal(err)
}
two, err := s.IssueFor("other")
if err != nil {
t.Fatal(err)
}
if username, ok := s.Username(one); !ok || username != "kami" {
t.Fatalf("username=%q ok=%v", username, ok)
}
s.RevokeUser("KAMI")
if s.Valid(one) {
t.Fatal("operator session survived credential change")
}
if !s.Valid(two) {
t.Fatal("another operator's session was revoked")
}
}
// The agent boundary: an agent may perform work and request lifecycle changes,
// never perform one. Both halves are proven here — the bus refuses the event
// types Orchestra owns, and the middleware refuses their endpoints — because