kami/orchestra
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Harden lease lifecycle durability

Browse Source
This commit is contained in:
kami
2026-07-30 14:34:29 +04:00
parent 1ff0af2e69
commit f6ee0e3060
40 changed files with 2108 additions and 590 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/admin/admin.go
+3 -1
View File
@@ -8,6 +8,7 @@ import (
"fmt"
"net/http"
"orchestra/internal/authz"
"orchestra/internal/buildinfo"
"orchestra/internal/domain"
"orchestra/internal/provider"
"orchestra/internal/store"
@@ -51,6 +52,7 @@ type ProbeFunc func() (bool, string)
type Server struct {
Store *store.Store
RouterReady bool
Build buildinfo.Info
Probes map[string]ProbeFunc
Providers map[string]*provider.Supervisor
}
@@ -95,7 +97,7 @@ func (s *Server) Diagnostics(w http.ResponseWriter, r *http.Request) {
}
tasks := s.Store.Tasks()
events := s.Store.Events(0)
_ = json.NewEncoder(w).Encode(map[string]any{"tasks": len(tasks), "events": len(events), "last_seq": func() uint64 {
_ = json.NewEncoder(w).Encode(map[string]any{"build": s.Build, "tasks": len(tasks), "events": len(events), "last_seq": func() uint64 {
if len(events) == 0 {
return 0
}
internal/authz/authz.go
+51 -25
View File
@@ -12,6 +12,8 @@ import (
"strings"
"sync"
"time"
"golang.org/x/crypto/bcrypt"
)
type Surface string
@@ -72,18 +74,44 @@ func AuthorizeEvent(s Surface, typ string) error {
return nil
}
// SessionCookie carries a browser's proof of the Web-surface token. A
// top-level document load cannot set an Authorization header, so a
// bearer-only gate forces operators to run the UI with no token at all
// (AUDIT.md B18). The cookie is the browser-presentable equivalent; it is
// never a second credential, only a receipt for the same token.
// SessionCookie carries a browser's proof of a successful Web login. A
// top-level document load cannot set an Authorization header, so browser
// credentials are exchanged once for this HttpOnly receipt.
const SessionCookie = "orchestra_session"
// SessionPath is the one Web-surface endpoint exempt from the token gate,
// because it *is* the token check: it verifies the Web token itself and
// exchanges it for a cookie. Gating it would make login unreachable.
// SessionPath is the one Web-surface endpoint exempt from the session gate,
// because it verifies login credentials and exchanges them for a cookie.
const SessionPath = "/v1/ui/session"
// WebCredentials is the single configured browser operator identity. Only a
// bcrypt password hash is accepted; Orchestra has no self-service account
// creation or password-reset surface.
type WebCredentials struct {
Username string
PasswordHash string
}
func (c WebCredentials) Validate() error {
if strings.TrimSpace(c.Username) == "" {
return fmt.Errorf("web username is required")
}
if c.PasswordHash == "" {
return fmt.Errorf("web password hash is required")
}
if _, err := bcrypt.Cost([]byte(c.PasswordHash)); err != nil {
return fmt.Errorf("web password hash must be bcrypt: %w", err)
}
return nil
}
// Authenticate always performs bcrypt, even for an unknown username, so the
// response does not reveal whether the configured username was correct.
func (c WebCredentials) Authenticate(username, password string) bool {
passwordOK := bcrypt.CompareHashAndPassword([]byte(c.PasswordHash), []byte(password)) == nil
usernameOK := subtle.ConstantTimeCompare([]byte(username), []byte(c.Username)) == 1
return passwordOK && usernameOK
}
// Sessions issues and validates those receipts. Values are random and stored
// hashed, so a leaked snapshot of this map does not yield a usable cookie.
type Sessions struct {
@@ -161,9 +189,9 @@ func HTTP(tokens map[Surface]string, next http.Handler) http.Handler {
return HTTPWithSessions(tokens, nil, next)
}
// HTTPWithSessions additionally accepts a valid session cookie in place of a
// Bearer token, but only for the Web surface — every non-browser surface
// still has to present the token directly.
// HTTPWithSessions accepts a valid browser session cookie only for the Web
// surface. Supplying Sessions makes Web authentication mandatory even when
// the legacy Web bearer-token slot is empty.
func HTTPWithSessions(tokens map[Surface]string, sessions *Sessions, next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Federation has per-worker credentials, not one shared surface token.
@@ -195,26 +223,24 @@ func HTTPWithSessions(tokens map[Surface]string, sessions *Sessions, next http.H
if s == System {
s = Web
}
if expected := tokens[s]; expected != "" && subtle.ConstantTimeCompare([]byte(r.Header.Get("Authorization")), []byte("Bearer "+expected)) != 1 {
if s == Web && sessions != nil {
ok := false
if s == Web && sessions != nil {
if c, err := r.Cookie(SessionCookie); err == nil {
ok = sessions.Valid(c.Value)
}
// The login endpoint authenticates itself, and the SPA shell
// must load before a browser can present anything. Static
// assets are not secrets; every /v1/ control path stays gated.
if r.URL.Path == SessionPath {
ok = true
}
if !strings.HasPrefix(r.URL.Path, "/v1/") && (r.Method == http.MethodGet || r.Method == http.MethodHead) {
ok = true
}
if c, err := r.Cookie(SessionCookie); err == nil {
ok = sessions.Valid(c.Value)
}
// The login endpoint authenticates itself, and the SPA shell must
// load before a browser can present a session. Static assets are not
// secrets; every other /v1/ control path remains session-gated.
if r.URL.Path == SessionPath || (!strings.HasPrefix(r.URL.Path, "/v1/") && (r.Method == http.MethodGet || r.Method == http.MethodHead)) {
ok = true
}
if !ok {
http.Error(w, "unauthorized surface", http.StatusUnauthorized)
return
}
} else if expected := tokens[s]; expected != "" && subtle.ConstantTimeCompare([]byte(r.Header.Get("Authorization")), []byte("Bearer "+expected)) != 1 {
http.Error(w, "unauthorized surface", http.StatusUnauthorized)
return
}
if (s == Telegram || s == Ntfy) && r.Method != http.MethodGet && r.Method != http.MethodHead {
http.Error(w, "notify-only surface", http.StatusForbidden)
internal/authz/authz_test.go
+27 -1
View File
@@ -5,6 +5,8 @@ import (
"net/http/httptest"
"testing"
"time"
"golang.org/x/crypto/bcrypt"
)
func TestSurfaceCapabilities(t *testing.T) {
@@ -51,7 +53,9 @@ func TestSystemSurfaceDowngradedByHTTPMiddleware(t *testing.T) {
// B18: the web UI is a full control plane. A session cookie must be an
// alternative *presentation* of the Web token, never a widening of it.
func TestWebSessionCookieGatesControlPathsOnly(t *testing.T) {
tokens := map[Surface]string{Web: "secret"}
// An empty legacy Web bearer-token slot must not open the browser surface:
// providing Sessions means the caller needs a session cookie.
tokens := map[Surface]string{}
sessions := &Sessions{}
h := HTTPWithSessions(tokens, sessions, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
@@ -96,6 +100,28 @@ func TestWebSessionCookieGatesControlPathsOnly(t *testing.T) {
}
}
func TestWebCredentialsAuthenticate(t *testing.T) {
hash, err := bcrypt.GenerateFromPassword([]byte("correct horse battery staple"), bcrypt.MinCost)
if err != nil {
t.Fatal(err)
}
c := WebCredentials{Username: "operator", PasswordHash: string(hash)}
if err := c.Validate(); err != nil {
t.Fatalf("Validate: %v", err)
}
if !c.Authenticate("operator", "correct horse battery staple") {
t.Fatal("correct credentials rejected")
}
for _, attempt := range []struct{ username, password string }{{"operator", "wrong"}, {"other", "correct horse battery staple"}} {
if c.Authenticate(attempt.username, attempt.password) {
t.Fatalf("invalid credentials accepted: %+v", attempt)
}
}
if err := (WebCredentials{Username: "operator", PasswordHash: "not-a-bcrypt-hash"}).Validate(); err == nil {
t.Fatal("invalid bcrypt hash accepted")
}
}
func TestFederationRequestsUseTheirOwnCredentials(t *testing.T) {
tokens := map[Surface]string{Web: "web-secret"}
h := HTTPWithSessions(tokens, nil, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
internal/buildinfo/buildinfo.go
+19
View File
@@ -0,0 +1,19 @@
// Package buildinfo exposes the provenance injected into Orchestra binaries.
// Build systems should set Revision, Time, and Dirty with -ldflags. Keeping
// the defaults explicit makes development binaries honest rather than
// pretending to be a deployable revision.
package buildinfo
var (
Revision = "devel"
Time = "unknown"
Dirty = "unknown"
)
type Info struct {
Revision string `json:"revision"`
Time string `json:"time"`
Dirty string `json:"dirty"`
}
func Current() Info { return Info{Revision: Revision, Time: Time, Dirty: Dirty} }
internal/domain/domain.go
+24 -8
View File
@@ -22,11 +22,11 @@ var ErrInvalid = errors.New("invalid event")
// task for this content; nothing was appended.
var ErrDuplicate = errors.New("duplicate task ingestion")
// CurrentEventSchema is 2: schema 2 requires every event to declare its
// authorizing Surface (see ValidateEvent), enforced at the store append
// boundary. Schema 1 events already on disk replay unchanged — tolerant
// reader, not upcast (spec open question #2).
const CurrentEventSchema = 2
// CurrentEventSchema is 3: schema 2 requires every event to declare its
// authorizing Surface; schema 3 adds lease fencing epochs. Older events stay
// readable so a deployment can recover its existing log before new writes
// are emitted (the store derives a non-renewable legacy epoch on replay).
const CurrentEventSchema = 3
type TaskState string
@@ -102,8 +102,13 @@ type SessionEvidence struct {
CheckedAt time.Time `json:"checked_at,omitempty"`
}
type Lease struct {
HarnessID string `json:"harness_id"`
Until time.Time `json:"until"`
HarnessID string `json:"harness_id"`
// Epoch is an opaque fencing token minted for every assignment. Versions
// change for ordinary lifecycle events; an epoch changes only when
// ownership changes, so an old pane can never become current again after
// a release/re-lease cycle.
Epoch string `json:"epoch"`
Until time.Time `json:"until"`
}
type Task struct {
ID string `json:"id"`
@@ -191,7 +196,18 @@ func ValidateEvent(e Event) error {
if p == nil {
return fmt.Errorf("%w: payload must be an object", ErrInvalid)
}
return ValidatePayload(e.Type, p)
if err := ValidatePayload(e.Type, p); err != nil {
return err
}
if e.SchemaVersion >= 3 {
switch e.Type {
case "TaskLeased", "TaskLeaseRenewed", "TaskPickupValidated":
if v, ok := p["lease_epoch"].(string); !ok || strings.TrimSpace(v) == "" {
return fmt.Errorf("%w: lease_epoch required", ErrInvalid)
}
}
}
return nil
}
func ValidateCreated(p map[string]any) error {
for _, k := range []string{"source", "external_id", "project"} {
internal/federation/client.go
+8 -8
View File
@@ -131,8 +131,8 @@ func (c Client) Heartbeat(ctx context.Context, health WorkerHealth) error {
}
return err
}
func (c Client) Renew(ctx context.Context, taskID string, expectedVersion, ttlSeconds int) error {
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/renew", map[string]any{"task_id": taskID, "expected_version": expectedVersion, "ttl_seconds": ttlSeconds})
func (c Client) Renew(ctx context.Context, taskID, epoch string, expectedVersion, ttlSeconds int) error {
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/renew", map[string]any{"task_id": taskID, "lease_epoch": epoch, "expected_version": expectedVersion, "ttl_seconds": ttlSeconds})
if resp != nil {
resp.Body.Close()
}
@@ -176,22 +176,22 @@ func (c Client) PutArtifact(ctx context.Context, b []byte) (string, error) {
}
return out.Ref, nil
}
func (c Client) Release(ctx context.Context, taskID, ref, anchor, transactionID string, expectedVersion int, evidence domain.SessionEvidence) error {
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/handoff", map[string]any{"task_id": taskID, "handoff_ref": ref, "anchor_sha": anchor, "transaction_id": transactionID, "expected_version": expectedVersion, "session_evidence": evidence})
func (c Client) Release(ctx context.Context, taskID, ref, anchor, transactionID, epoch string, expectedVersion int, evidence domain.SessionEvidence) error {
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/handoff", map[string]any{"task_id": taskID, "handoff_ref": ref, "anchor_sha": anchor, "transaction_id": transactionID, "lease_epoch": epoch, "expected_version": expectedVersion, "session_evidence": evidence})
if resp != nil {
resp.Body.Close()
}
return err
}
func (c Client) Pickup(ctx context.Context, taskID, ref, anchor, transactionID string, leaseVersion int, evidence domain.SessionEvidence) error {
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/pickup", map[string]any{"task_id": taskID, "handoff_ref": ref, "anchor_sha": anchor, "transaction_id": transactionID, "lease_version": leaseVersion, "session_evidence": evidence})
func (c Client) Pickup(ctx context.Context, taskID, ref, anchor, transactionID, epoch string, leaseVersion int, evidence domain.SessionEvidence) error {
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/pickup", map[string]any{"task_id": taskID, "handoff_ref": ref, "anchor_sha": anchor, "transaction_id": transactionID, "lease_epoch": epoch, "lease_version": leaseVersion, "session_evidence": evidence})
if resp != nil {
resp.Body.Close()
}
return err
}
func (c Client) Complete(ctx context.Context, taskID, reportRef, resultSHA, branch, remote string, expectedVersion int, receipt map[string]any, evidence domain.SessionEvidence) error {
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/complete", map[string]any{"task_id": taskID, "handoff_ref": reportRef, "result_sha": resultSHA, "branch": branch, "remote": remote, "expected_version": expectedVersion, "receipt": receipt, "session_evidence": evidence})
func (c Client) Complete(ctx context.Context, taskID, reportRef, resultSHA, branch, remote, epoch string, expectedVersion int, receipt map[string]any, evidence domain.SessionEvidence) error {
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/complete", map[string]any{"task_id": taskID, "handoff_ref": reportRef, "result_sha": resultSHA, "branch": branch, "remote": remote, "lease_epoch": epoch, "expected_version": expectedVersion, "receipt": receipt, "session_evidence": evidence})
if resp != nil {
resp.Body.Close()
}
internal/federation/federation.go
+61 -15
View File
@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"orchestra/internal/buildinfo"
"os"
"path/filepath"
"sync"
@@ -15,13 +16,15 @@ var ErrUnknownWorker = errors.New("unknown worker")
var ErrUnauthorized = errors.New("worker authentication failed")
type Worker struct {
ID string `json:"id"`
Address string `json:"address"`
Capacity int `json:"capacity"`
LastSeen time.Time `json:"last_seen"`
Online bool `json:"online"`
Health WorkerHealth `json:"health"`
Token string `json:"-"`
ID string `json:"id"`
Address string `json:"address"`
Capacity int `json:"capacity"`
SupportedProjects []string `json:"supported_projects"`
Build buildinfo.Info `json:"build"`
LastSeen time.Time `json:"last_seen"`
Online bool `json:"online"`
Health WorkerHealth `json:"health"`
Token string `json:"-"`
}
// WorkerHealth is reported by the worker that owns the local herdr socket.
@@ -86,10 +89,12 @@ type persistedState struct {
// is mode 0600, and retaining this binding prevents an arbitrary process from
// registering a recovered worker ID and executing its pending approval.
type persistedWorker struct {
ID string `json:"id"`
Address string `json:"address"`
Capacity int `json:"capacity"`
Token string `json:"token"`
ID string `json:"id"`
Address string `json:"address"`
Capacity int `json:"capacity"`
SupportedProjects []string `json:"supported_projects"`
Build buildinfo.Info `json:"build"`
Token string `json:"token"`
}
// Load restores durable capture/command state. Call this before accepting
@@ -123,7 +128,7 @@ func (r *Registry) Load() error {
if id == "" || w.ID != id || w.Token == "" {
return fmt.Errorf("invalid federation worker %q", id)
}
r.workers[id] = Worker{ID: w.ID, Address: w.Address, Capacity: w.Capacity, Token: w.Token}
r.workers[id] = Worker{ID: w.ID, Address: w.Address, Capacity: w.Capacity, SupportedProjects: w.SupportedProjects, Build: w.Build, Token: w.Token}
}
return nil
}
@@ -135,7 +140,7 @@ func (r *Registry) persistLocked() error {
}
workers := make(map[string]persistedWorker, len(r.workers))
for id, w := range r.workers {
workers[id] = persistedWorker{ID: w.ID, Address: w.Address, Capacity: w.Capacity, Token: w.Token}
workers[id] = persistedWorker{ID: w.ID, Address: w.Address, Capacity: w.Capacity, SupportedProjects: w.SupportedProjects, Build: w.Build, Token: w.Token}
}
b, err := json.Marshal(persistedState{Captures: r.captures, Commands: r.commands, Workers: workers})
if err != nil {
@@ -145,12 +150,31 @@ func (r *Registry) persistLocked() error {
return err
}
tmp := r.StatePath + ".tmp"
if err := os.WriteFile(tmp, b, 0600); err != nil {
f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
if err != nil {
return err
}
if _, err = f.Write(b); err == nil {
err = f.Sync()
}
if closeErr := f.Close(); err == nil {
err = closeErr
}
if err != nil {
_ = os.Remove(tmp)
return err
}
if err := os.Rename(tmp, r.StatePath); err != nil {
return err
}
dir, err := os.Open(filepath.Dir(r.StatePath))
if err != nil {
return err
}
defer dir.Close()
if err := dir.Sync(); err != nil {
return err
}
return os.Chmod(r.StatePath, 0600)
}
@@ -396,10 +420,32 @@ func (r *Registry) Available(id string) bool {
if !ok {
return false
}
w.Online = time.Since(w.LastSeen) <= r.TTL
// A heartbeat merely proves the worker process can reach the coordinator.
// Lease admission additionally requires a fresh probe of the worker's
// local herdr; otherwise a partitioned/down herdr still attracts work.
w.Online = time.Since(w.LastSeen) <= r.TTL && w.Health.HerdrStatus == "reachable" && !w.Health.CheckedAt.IsZero() && time.Since(w.Health.CheckedAt) <= r.TTL
r.workers[id] = w
return w.Online
}
// Supports reports whether an online worker explicitly declared the project.
// An omitted declaration is deliberately not treated as a wildcard: workers
// must never receive a project for which they have no local checkout.
func (r *Registry) Supports(id, project string) bool {
r.mu.Lock()
defer r.mu.Unlock()
r.init()
w, ok := r.workers[id]
if !ok || time.Since(w.LastSeen) > r.TTL {
return false
}
for _, candidate := range w.SupportedProjects {
if candidate == project {
return true
}
}
return false
}
func (r *Registry) Snapshot() []Worker {
r.mu.Lock()
defer r.mu.Unlock()
internal/federation/federation_test.go
+43
View File
@@ -29,6 +29,49 @@ func TestCursorIsMonotonicAndAuthenticationIsRequired(t *testing.T) {
}
}
func TestSupportedProjectsPersistAndGateAvailability(t *testing.T) {
path := filepath.Join(t.TempDir(), "state.json")
r := &Registry{StatePath: path}
if err := r.Register(Worker{ID: "w", Token: "t", SupportedProjects: []string{"test-e2e"}}, ""); err != nil {
t.Fatal(err)
}
if !r.Supports("w", "test-e2e") || r.Supports("w", "correx") {
t.Fatalf("unexpected project support")
}
restarted := &Registry{StatePath: path}
if err := restarted.Load(); err != nil {
t.Fatal(err)
}
if err := restarted.Register(Worker{ID: "w", Token: "t", SupportedProjects: []string{"test-e2e"}}, ""); err != nil {
t.Fatal(err)
}
if !restarted.Supports("w", "test-e2e") || restarted.Supports("w", "correx") {
t.Fatalf("project support did not survive restart")
}
}
func TestAvailableRequiresFreshReachableLocalHerdrHealth(t *testing.T) {
r := &Registry{}
if err := r.Register(Worker{ID: "w", Token: "t"}, ""); err != nil {
t.Fatal(err)
}
if r.Available("w") {
t.Fatal("registration without local herdr probe admitted a worker")
}
if err := r.Heartbeat("w", WorkerHealth{HerdrStatus: "reachable", CheckedAt: time.Now().UTC()}); err != nil {
t.Fatal(err)
}
if !r.Available("w") {
t.Fatal("fresh reachable local herdr was not admitted")
}
if err := r.Heartbeat("w", WorkerHealth{HerdrStatus: "unreachable", CheckedAt: time.Now().UTC()}); err != nil {
t.Fatal(err)
}
if r.Available("w") {
t.Fatal("unreachable local herdr was admitted")
}
}
func TestPendingApprovalSurvivesRegistryRestart(t *testing.T) {
path := filepath.Join(t.TempDir(), "federation-state.json")
r := &Registry{StatePath: path}
internal/integration/end_to_end_test.go
+19 -8
View File
@@ -129,9 +129,12 @@ func TestEndToEndIngestRouteLeaseRotateAndComplete(t *testing.T) {
}
got, _ := s.Task(task.ID)
if got.State == domain.StateLeased && h.releases == 1 {
if err := s.Append(domain.Event{ID: domain.NewID(), Type: "TaskReleased", TaskID: task.ID, Version: got.Version + 1, Surface: string(authz.System), Payload: mustJSON(map[string]string{
"handoff_ref": h.ref,
"anchor_sha": "0123456789012345678901234567890123456789",
if err := s.Append(domain.Event{ID: domain.NewID(), Type: "TaskReleased", TaskID: task.ID, Version: got.Version + 1, Surface: string(authz.System), Payload: mustJSON(map[string]any{
"handoff_ref": h.ref,
"anchor_sha": "0123456789012345678901234567890123456789",
"harness_id": got.Lease.HarnessID,
"lease_epoch": got.Lease.Epoch,
"expected_version": got.Version,
})}); err != nil {
t.Fatal(err)
}
@@ -186,9 +189,13 @@ func TestRestartReplayAndReconcileKillsOrphan(t *testing.T) {
}
// A fresh coordinator sees the durable session, then drops it once the lease is gone.
ref, _ := s.PutArtifact([]byte("handoff"))
if err := s.Append(domain.Event{ID: domain.NewID(), Type: "TaskReleased", TaskID: task.ID, Version: 3, Surface: string(authz.System), Payload: mustJSON(map[string]string{
"handoff_ref": ref,
"anchor_sha": "0123456789012345678901234567890123456789",
leased, _ := s.Task(task.ID)
if err := s.Append(domain.Event{ID: domain.NewID(), Type: "TaskReleased", TaskID: task.ID, Version: 3, Surface: string(authz.System), Payload: mustJSON(map[string]any{
"handoff_ref": ref,
"anchor_sha": "0123456789012345678901234567890123456789",
"harness_id": leased.Lease.HarnessID,
"lease_epoch": leased.Lease.Epoch,
"expected_version": leased.Version,
})}); err != nil {
t.Fatal(err)
}
@@ -230,9 +237,13 @@ func TestProviderRetryReflectionQuotaAndVersionConflict(t *testing.T) {
t.Fatal(err)
}
reflector := &fakeReflector{}
leased, _ := s.Task(task.ID)
if err := (provider.ReflectingSink{Sink: s, Tasks: s, Reflector: reflector}).Append(domain.Event{ID: domain.NewID(), Type: "TaskCompleted", TaskID: task.ID, Version: 3, Surface: string(authz.System), Payload: mustJSON(map[string]any{
"report_ref": ref,
"receipt": map[string]any{"harness_id": "h1", "consumed": 1},
"report_ref": ref,
"receipt": map[string]any{"harness_id": "h1", "consumed": 1},
"harness_id": leased.Lease.HarnessID,
"lease_epoch": leased.Lease.Epoch,
"expected_version": leased.Version,
})}); err != nil {
t.Fatal(err)
}
internal/orchestrator/orchestrator.go
+80 -24
View File
@@ -6,6 +6,7 @@ package orchestrator
import (
"context"
"encoding/json"
"errors"
"fmt"
"orchestra/internal/authz"
"orchestra/internal/continuity"
@@ -412,10 +413,29 @@ func (c *Coordinator) saveSessionsLocked() error {
return err
}
tmp := c.StatePath + ".tmp"
if err = os.WriteFile(tmp, b, 0600); err != nil {
f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
if err != nil {
return err
}
return os.Rename(tmp, c.StatePath)
if _, err = f.Write(b); err == nil {
err = f.Sync()
}
if closeErr := f.Close(); err == nil {
err = closeErr
}
if err != nil {
_ = os.Remove(tmp)
return err
}
if err = os.Rename(tmp, c.StatePath); err != nil {
return err
}
dir, err := os.Open(filepath.Dir(c.StatePath))
if err != nil {
return err
}
defer dir.Close()
return dir.Sync()
}
// Reconcile drops mappings whose task lease did not survive restart and kills
@@ -573,7 +593,7 @@ func (c *Coordinator) expire(ctx context.Context) ([]domain.Event, error) {
if a, ae := c.adapterFor(taskID, s); ae == nil {
if p, ok := a.(herdr.PaneExit); ok {
if exited, ee := p.PaneExited(ctx, s); ee == nil && exited {
b, _ := json.Marshal(map[string]string{"reason": "pane_exited", "harness_id": s.Harness})
b, _ := json.Marshal(map[string]any{"reason": "pane_exited", "harness_id": s.Harness, "lease_epoch": t.Lease.Epoch, "expected_version": t.Version})
_ = c.Store.Append(domain.Event{ID: domain.NewID(), Type: "TaskReleased", TaskID: taskID, Version: t.Version + 1, Payload: b, Surface: string(authz.System)})
}
}
@@ -581,24 +601,47 @@ func (c *Coordinator) expire(ctx context.Context) ([]domain.Event, error) {
}
}
c.mu.Unlock()
events, err := c.Store.ExpireLeases(time.Now())
if err != nil {
return events, err
}
for _, e := range events {
c.loadSessions()
c.mu.Lock()
s, ok := c.sessions[e.TaskID]
delete(c.sessions, e.TaskID)
if ok {
if a, ae := c.adapterFor(e.TaskID, s); ae == nil {
_ = a.Kill(ctx, s)
}
var events []domain.Event
var firstErr error
for _, task := range c.Store.Tasks() {
if task.State != domain.StateLeased || task.Lease == nil || task.Lease.Until.After(time.Now()) {
continue
}
_ = c.saveSessionsLocked()
// Stop a local predecessor before making its lease eligible for a
// successor. If this cannot be done, keep both the mapping and the
// lease: safety beats reclaim speed.
c.mu.Lock()
s, local := c.sessions[task.ID]
c.mu.Unlock()
if local {
a, adapterErr := c.adapterFor(task.ID, s)
if adapterErr != nil {
if firstErr == nil {
firstErr = fmt.Errorf("expire %s: resolve old pane: %w", task.ID, adapterErr)
}
continue
}
if killErr := a.Kill(ctx, s); killErr != nil {
if firstErr == nil {
firstErr = fmt.Errorf("expire %s: quarantine old pane: %w", task.ID, killErr)
}
continue
}
c.mu.Lock()
delete(c.sessions, task.ID)
_ = c.saveSessionsLocked()
c.mu.Unlock()
}
e, expireErr := c.Store.ExpireLease(task.ID, time.Now())
if expireErr != nil {
if !errors.Is(expireErr, domain.ErrConflict) && firstErr == nil {
firstErr = expireErr
}
continue
}
events = append(events, e)
}
return events, nil
return events, firstErr
}
// handoffReason reads HandoffFile from the worktree, if present, and returns
@@ -722,13 +765,17 @@ func (c *Coordinator) rotate(ctx context.Context, hard float64) {
// tick or TTL expiry to reclaim.
continue
}
b, _ := json.Marshal(map[string]string{"handoff_ref": ref, "reason": reason, "anchor_sha": anchorSHA})
b, _ := json.Marshal(map[string]any{"handoff_ref": ref, "reason": reason, "anchor_sha": anchorSHA, "harness_id": task.Lease.HarnessID, "lease_epoch": task.Lease.Epoch, "expected_version": task.Version})
e := domain.Event{ID: domain.NewID(), Type: "TaskReleased", TaskID: taskID, Version: task.Version + 1, Payload: b, Surface: string(authz.System)}
if c.Store.Append(e) == nil {
c.mu.Lock()
delete(c.sessions, taskID)
_ = c.saveSessionsLocked()
c.mu.Unlock()
// A release only transfers the lease; this local coordinator owns
// the predecessor pane until it has actually stopped it.
if err := a.Kill(ctx, session); err == nil {
c.mu.Lock()
delete(c.sessions, taskID)
_ = c.saveSessionsLocked()
c.mu.Unlock()
}
}
}
}
@@ -842,11 +889,14 @@ func (c *Coordinator) finishRelease(ctx context.Context, taskID string, task dom
// invalid TaskReleased payload, same as rotate()'s bare continue.
return TurnRefuse, nil
}
b, _ := json.Marshal(map[string]string{"handoff_ref": ref, "reason": reason, "anchor_sha": anchorSHA})
b, _ := json.Marshal(map[string]any{"handoff_ref": ref, "reason": reason, "anchor_sha": anchorSHA, "harness_id": task.Lease.HarnessID, "lease_epoch": task.Lease.Epoch, "expected_version": task.Version})
e := domain.Event{ID: domain.NewID(), Type: "TaskReleased", TaskID: taskID, Version: task.Version + 1, Payload: b, Surface: string(authz.System)}
if err := c.Store.Append(e); err != nil {
return TurnRefuse, nil
}
if err := a.Kill(ctx, session); err != nil {
return TurnRefuse, nil
}
c.mu.Lock()
delete(c.sessions, taskID)
_ = c.saveSessionsLocked()
@@ -978,6 +1028,12 @@ func (c *Coordinator) block(t domain.Task, reason string) error {
p["session_evidence"] = domain.SessionEvidence{PaneID: s.PaneID, HarnessID: s.HerdrID, PaneState: "open", Source: "coordinator", CheckedAt: time.Now().UTC()}
}
b, _ := json.Marshal(p)
if t.Lease != nil {
p["harness_id"] = t.Lease.HarnessID
p["lease_epoch"] = t.Lease.Epoch
p["expected_version"] = t.Version
b, _ = json.Marshal(p)
}
return c.Store.Append(domain.Event{ID: domain.NewID(), Type: "TaskBlocked", TaskID: t.ID, Version: t.Version + 1, Payload: b, Surface: string(authz.System)})
}
internal/orchestrator/worktrees_test.go
+5 -1
View File
@@ -8,6 +8,7 @@ import (
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
)
@@ -73,7 +74,7 @@ func TestGitWorktreesCommitsTaskFile(t *testing.T) {
initRepo(t, repo)
w := orchestrator.GitWorktrees{Root: filepath.Join(base, "wt"), Repo: repo}
task := domain.Task{ID: "t1", Project: "p", Source: "jsonl", ExternalID: "1", Title: "do the thing"}
task := domain.Task{ID: "t1", Project: "p", Source: "jsonl", ExternalID: "1", Title: "do the thing", Acceptance: []string{"tests pass"}, QualityGate: "go test ./..."}
path, err := w.Create(context.Background(), task)
if err != nil {
@@ -88,6 +89,9 @@ func TestGitWorktreesCommitsTaskFile(t *testing.T) {
if string(got) != string(want) {
t.Fatalf("TASK.md content mismatch:\ngot: %s\nwant: %s", got, want)
}
if !strings.Contains(string(got), "## Acceptance criteria") || !strings.Contains(string(got), ".orchestra/done") {
t.Fatalf("TASK.md is missing deterministic completion contract: %s", got)
}
status, err := exec.Command("git", "-C", path, "status", "--porcelain", "--", "TASK.md").Output()
if err != nil {
internal/registry/registry.go
+1
View File
@@ -28,6 +28,7 @@ type Project struct {
// wires (single-repo deployments keep working unchanged).
Repo string `json:"repo,omitempty"`
WorktreeRoot string `json:"worktree_root,omitempty"`
QualityGate string `json:"quality_gate,omitempty"`
}
type Machine struct {
ID string `json:"id"`
internal/router/router.go
+11 -1
View File
@@ -14,6 +14,12 @@ import (
)
type Availability interface{ Available(h registry.Herdr) bool }
// ProjectAvailability is an optional stricter availability contract used by
// federated workers, whose local checkout configuration is authoritative.
type ProjectAvailability interface {
Supports(h registry.Herdr, project string) bool
}
type AlwaysAvailable struct{}
func (AlwaysAvailable) Available(registry.Herdr) bool { return true }
@@ -170,7 +176,11 @@ func (r *Router) AssignPending() ([]domain.Event, error) {
continue
}
for _, h := range cs {
if !matches(t.Capability, h.Capabilities) || !r.Availability.Available(h) || occupied(r.Store, h.ID, h.Concurrency) {
projectOK := true
if projects, ok := r.Availability.(ProjectAvailability); ok {
projectOK = projects.Supports(h, t.Project)
}
if !projectOK || !matches(t.Capability, h.Capabilities) || !r.Availability.Available(h) || occupied(r.Store, h.ID, h.Concurrency) {
continue
}
e, err := r.Store.Lease(t.ID, h.ID, 30*time.Minute)
internal/router/router_test.go
+40 -4
View File
@@ -14,6 +14,13 @@ type reachable struct{}
func (reachable) Reachable(string, time.Duration) bool { return true }
type projectAvailability struct{ projects map[string]bool }
func (p projectAvailability) Available(registry.Herdr) bool { return true }
func (p projectAvailability) Supports(h registry.Herdr, project string) bool {
return p.projects[h.ID+"/"+project]
}
func TestAssignsByAffinityCapabilityAndConcurrency(t *testing.T) {
s, err := store.Open(t.TempDir())
if err != nil {
@@ -56,6 +63,32 @@ func TestAssignsByAffinityCapabilityAndConcurrency(t *testing.T) {
}
}
func TestAssignPendingRequiresWorkerProjectSupport(t *testing.T) {
s, err := store.Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
r, err := registry.New(registry.Config{Projects: []registry.Project{{ID: "p", MachineAffinity: []string{"m"}}}, Machines: []registry.Machine{{ID: "m", Address: "unused"}}, Herdrs: []registry.Herdr{{ID: "h", MachineID: "m", Concurrency: 1}}})
if err != nil {
t.Fatal(err)
}
b, _ := json.Marshal(map[string]any{"source": "test", "external_id": "project-check", "project": "p"})
if err := s.Append(domain.Event{ID: "create", TaskID: "t", Type: "TaskCreated", Version: 1, Payload: b, Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
rt := Router{Store: s, Registry: r, Reachability: reachable{}, Availability: projectAvailability{projects: map[string]bool{}}}
if got, err := rt.AssignPending(); err != nil || len(got) != 0 {
t.Fatalf("unsupported project lease = %#v, %v", got, err)
}
if got, _ := s.Task("t"); got.State != domain.StateQueued {
t.Fatalf("unsupported project state=%s", got.State)
}
rt.Availability = projectAvailability{projects: map[string]bool{"h/p": true}}
if got, err := rt.AssignPending(); err != nil || len(got) != 1 {
t.Fatalf("supported project lease = %#v, %v", got, err)
}
}
// TestRotationDoesNotCountAgainstRetryLimit guards B4: rotation is
// TaskReleased carrying a valid handoff_ref (spec §5.3: "rotation =
// intra-task lease transfer"), never a failure. A task healthy enough to
@@ -103,10 +136,13 @@ func TestRotationDoesNotCountAgainstRetryLimit(t *testing.T) {
}
continue
}
rb, _ := json.Marshal(map[string]string{
"handoff_ref": handoffRef,
"reason": "threshold",
"anchor_sha": "0123456789abcdef0123456789abcdef01234567",
rb, _ := json.Marshal(map[string]any{
"handoff_ref": handoffRef,
"reason": "threshold",
"anchor_sha": "0123456789abcdef0123456789abcdef01234567",
"harness_id": task.Lease.HarnessID,
"lease_epoch": task.Lease.Epoch,
"expected_version": task.Version,
})
release := domain.Event{ID: domain.NewID(), Type: "TaskReleased", TaskID: "a", Version: task.Version + 1, Payload: rb, Surface: string(authz.System)}
if err := s.Append(release); err != nil {
internal/store/store.go
+148 -43
View File
@@ -32,27 +32,10 @@ func Open(dir string) (*Store, error) {
if err := os.MkdirAll(s.cas, 0755); err != nil {
return nil, err
}
var snapshotSeq uint64
if b, readErr := os.ReadFile(s.snapshot); readErr == nil {
var snap struct {
Seq uint64 `json:"seq"`
Tasks []domain.Task `json:"tasks"`
}
if json.Unmarshal(b, &snap) != nil {
return nil, fmt.Errorf("invalid snapshot")
}
for _, t := range snap.Tasks {
s.tasks[t.ID] = t
s.external[t.Source+"\x00"+t.ExternalID] = t.ID
}
snapshotSeq = snap.Seq
// Continue event numbering after the snapshot. Without restoring this
// cursor, the first append after a restart reused sequence 1 and made
// the append-only log unreplayable.
s.seq = snapshotSeq
} else if !errors.Is(readErr, os.ErrNotExist) {
return nil, readErr
}
// A snapshot is a disposable read cache, never recovery authority. Loading
// it before the log let a partially-written snapshot become a different
// history than events.jsonl after a crash. Rebuild every projection from
// the append-only, fsynced log instead.
f, err := os.Open(s.path)
if os.IsNotExist(err) {
return s, nil
@@ -62,16 +45,13 @@ func Open(dir string) (*Store, error) {
}
defer f.Close()
sc := bufio.NewScanner(f)
var expected uint64 = snapshotSeq + 1
var expected uint64 = 1
for sc.Scan() {
var e domain.Event
if err := json.Unmarshal(sc.Bytes(), &e); err == nil {
if err := domain.ValidateEvent(e); err != nil {
return nil, err
}
if e.Seq < expected {
continue
}
if e.Seq != expected {
return nil, fmt.Errorf("event sequence gap: got %d, want %d", e.Seq, expected)
}
@@ -151,9 +131,20 @@ func (s *Store) apply(e domain.Event) error {
s.external[t.Source+"\x00"+t.ExternalID] = t.ID
case "TaskLeased":
t.State = domain.StateLeased
t.Lease = &domain.Lease{HarnessID: p["harness_id"].(string), Until: time.Unix(0, int64(p["until_ns"].(float64)))}
epoch, _ := p["lease_epoch"].(string)
if epoch == "" {
// A pre-fencing event cannot safely be renewed by an old worker.
// Deriving a stable token from the durable event identity makes the
// recovered lease observable but non-renewable until it expires.
epoch = "legacy:" + e.ID
}
t.Lease = &domain.Lease{HarnessID: p["harness_id"].(string), Epoch: epoch, Until: time.Unix(0, int64(p["until_ns"].(float64)))}
case "TaskLeaseRenewed":
t.Lease = &domain.Lease{HarnessID: p["harness_id"].(string), Until: time.Unix(0, int64(p["until_ns"].(float64)))}
epoch, _ := p["lease_epoch"].(string)
if epoch == "" && t.Lease != nil {
epoch = t.Lease.Epoch
}
t.Lease = &domain.Lease{HarnessID: p["harness_id"].(string), Epoch: epoch, Until: time.Unix(0, int64(p["until_ns"].(float64)))}
case "TaskReleased":
t.State = domain.StateQueued
t.Lease = nil
@@ -292,6 +283,9 @@ func (s *Store) Append(e domain.Event) error {
return domain.ErrConflict
}
}
if err := s.validateTransition(e, t, taskExists, contract); err != nil {
return err
}
if e.Type == "TaskLeased" {
var p struct {
ExpectedVersion *int `json:"expected_version"`
@@ -333,9 +327,6 @@ func (s *Store) Append(e domain.Event) error {
}
}
}
if err := s.apply(e); err != nil {
return err
}
f, err := os.OpenFile(s.path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
if err != nil {
return err
@@ -348,10 +339,60 @@ func (s *Store) Append(e domain.Event) error {
if err = f.Sync(); err != nil {
return err
}
// The event is the commit record. Do not expose a projection that cannot
// be recovered from it after a power loss.
if err := s.apply(e); err != nil {
return err
}
s.events = append(s.events, e)
s.seq = e.Seq
if err := s.writeSnapshot(); err != nil {
return err
// Snapshot failure does not roll back a committed event. Open always
// rebuilds from the log, so leaving a stale cache is safe.
_ = s.writeSnapshot()
return nil
}
// validateTransition keeps lifecycle authority at the durable append
// boundary. A task may be completed/failed while queued by an external
// provider, but once a lease exists its owner and fencing epoch are required
// for every lifecycle mutation.
func (s *Store) validateTransition(e domain.Event, t domain.Task, exists bool, p map[string]any) error {
if !exists {
if e.Type != "TaskCreated" && e.Type != "QuotaReported" && e.Type != "StandupAdvisory" && e.Type != "ApprovalGranted" && e.Type != "ApprovalDenied" {
return domain.ErrNotFound
}
return nil
}
if e.Type == "TaskLeased" && t.State != domain.StateQueued {
return domain.ErrConflict
}
if t.State != domain.StateLeased || t.Lease == nil {
return nil
}
switch e.Type {
case "TaskLeaseRenewed", "TaskReleased", "TaskPickupValidated", "TaskCompleted", "TaskBlocked", "TaskFailed":
owner, _ := p["harness_id"].(string)
epoch, _ := p["lease_epoch"].(string)
// Expiry is the one coordinator-owned relinquish path. It still binds
// the exact epoch that was observed when the timer fired.
if e.Type == "TaskReleased" {
if reason, _ := p["reason"].(string); (reason == "lease_expired" || reason == "pane_exited") && owner == t.Lease.HarnessID && epoch == t.Lease.Epoch {
return nil
}
}
if owner != t.Lease.HarnessID || epoch == "" || epoch != t.Lease.Epoch {
return domain.ErrConflict
}
case "TaskCorrected":
// Corrections may repair metadata while a task is leased, but cannot
// smuggle in a lifecycle transition around the current fenced owner.
if _, changesState := p["state"]; changesState {
owner, _ := p["harness_id"].(string)
epoch, _ := p["lease_epoch"].(string)
if owner != t.Lease.HarnessID || epoch == "" || epoch != t.Lease.Epoch {
return domain.ErrConflict
}
}
}
return nil
}
@@ -368,10 +409,28 @@ func (s *Store) writeSnapshot() error {
return err
}
tmp := s.snapshot + ".tmp"
if err = os.WriteFile(tmp, b, 0644); err != nil {
f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
if err != nil {
return err
}
return os.Rename(tmp, s.snapshot)
if _, err = f.Write(b); err == nil {
err = f.Sync()
}
if closeErr := f.Close(); err == nil {
err = closeErr
}
if err != nil {
return err
}
if err = os.Rename(tmp, s.snapshot); err != nil {
return err
}
dir, err := os.Open(filepath.Dir(s.snapshot))
if err != nil {
return err
}
defer dir.Close()
return dir.Sync()
}
func (s *Store) Tasks() []domain.Task {
s.mu.Lock()
@@ -397,9 +456,39 @@ func (s *Store) PutArtifact(b []byte) (string, error) {
h := domain.Hash(b)
p := filepath.Join(s.cas, h)
if _, err := os.Stat(p); errors.Is(err, os.ErrNotExist) {
if err = os.WriteFile(p, b, 0644); err != nil {
tmp := p + ".tmp-" + domain.NewID()
f, openErr := os.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
if openErr != nil {
return "", openErr
}
if _, err = f.Write(b); err == nil {
err = f.Sync()
}
if closeErr := f.Close(); err == nil {
err = closeErr
}
if err != nil {
_ = os.Remove(tmp)
return "", err
}
if err = os.Rename(tmp, p); err != nil && !errors.Is(err, os.ErrExist) {
_ = os.Remove(tmp)
return "", err
}
if !errors.Is(err, os.ErrExist) {
dir, openErr := os.Open(s.cas)
if openErr != nil {
return "", openErr
}
syncErr := dir.Sync()
closeErr := dir.Close()
if syncErr != nil {
return "", syncErr
}
if closeErr != nil {
return "", closeErr
}
}
}
return h, nil
}
@@ -450,7 +539,7 @@ func (s *Store) Lease(id, harness string, ttl time.Duration) (domain.Event, erro
if t.State != domain.StateQueued {
return domain.Event{}, domain.ErrConflict
}
payload := map[string]any{"harness_id": harness, "ttl": ttl.Seconds(), "until_ns": time.Now().Add(ttl).UnixNano(), "expected_version": t.Version}
payload := map[string]any{"harness_id": harness, "lease_epoch": domain.NewID(), "ttl": ttl.Seconds(), "until_ns": time.Now().Add(ttl).UnixNano(), "expected_version": t.Version}
if t.HandoffRef != "" {
payload["handoff_ref"] = t.HandoffRef
payload["transaction_id"] = t.ReleaseTransaction
@@ -464,7 +553,7 @@ func (s *Store) Lease(id, harness string, ttl time.Duration) (domain.Event, erro
// RenewLease atomically extends the current owner's lease. The observed task
// version is part of the request so an old worker can never renew a lease
// after release/reassignment.
func (s *Store) RenewLease(id, harness string, expectedVersion int, ttl time.Duration) (domain.Event, error) {
func (s *Store) RenewLease(id, harness, epoch string, expectedVersion int, ttl time.Duration) (domain.Event, error) {
if ttl <= 0 {
return domain.Event{}, fmt.Errorf("%w: ttl must be positive", domain.ErrInvalid)
}
@@ -472,10 +561,10 @@ func (s *Store) RenewLease(id, harness string, expectedVersion int, ttl time.Dur
if !ok {
return domain.Event{}, domain.ErrNotFound
}
if t.State != domain.StateLeased || t.Lease == nil || t.Lease.HarnessID != harness || t.Version != expectedVersion {
if t.State != domain.StateLeased || t.Lease == nil || t.Lease.HarnessID != harness || t.Lease.Epoch != epoch || t.Version != expectedVersion {
return domain.Event{}, domain.ErrConflict
}
p, _ := json.Marshal(map[string]any{"harness_id": harness, "until_ns": time.Now().Add(ttl).UnixNano(), "expected_version": expectedVersion})
p, _ := json.Marshal(map[string]any{"harness_id": harness, "lease_epoch": epoch, "until_ns": time.Now().Add(ttl).UnixNano(), "expected_version": expectedVersion})
e := domain.Event{ID: domain.NewID(), Type: "TaskLeaseRenewed", TaskID: id, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
return e, s.Append(e)
}
@@ -483,14 +572,30 @@ func (s *Store) RenewLease(id, harness string, expectedVersion int, ttl time.Dur
func (s *Store) ExpireLeases(now time.Time) ([]domain.Event, error) {
var out []domain.Event
for _, t := range s.Tasks() {
if t.State == domain.StateLeased && t.Lease != nil && !t.Lease.Until.After(now) {
p, _ := json.Marshal(map[string]any{"reason": "lease_expired", "harness_id": t.Lease.HarnessID})
e := domain.Event{ID: domain.NewID(), Type: "TaskReleased", TaskID: t.ID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
if err := s.Append(e); err != nil {
if e, err := s.ExpireLease(t.ID, now); err != nil {
if !errors.Is(err, domain.ErrConflict) {
return out, err
}
} else if e.ID != "" {
out = append(out, e)
}
}
return out, nil
}
// ExpireLease releases exactly the observed lease if, and only if, its TTL
// has elapsed. Coordinators use this one-task form to stop their local pane
// before publishing the release event; the batch helper remains for
// deployments without a local coordinator.
func (s *Store) ExpireLease(id string, now time.Time) (domain.Event, error) {
t, ok := s.Task(id)
if !ok {
return domain.Event{}, domain.ErrNotFound
}
if t.State != domain.StateLeased || t.Lease == nil || t.Lease.Until.After(now) {
return domain.Event{}, domain.ErrConflict
}
p, _ := json.Marshal(map[string]any{"reason": "lease_expired", "harness_id": t.Lease.HarnessID, "lease_epoch": t.Lease.Epoch, "expected_version": t.Version})
e := domain.Event{ID: domain.NewID(), Type: "TaskReleased", TaskID: t.ID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
return e, s.Append(e)
}
internal/store/store_test.go
+105 -7
View File
@@ -3,6 +3,7 @@ package store
import (
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
@@ -35,7 +36,7 @@ func TestLeaseCarriesReleasedHandoffRef(t *testing.T) {
t.Fatal(err)
}
task, _ := s.Task("t")
p, _ := json.Marshal(map[string]string{"handoff_ref": ref, "anchor_sha": "0123456789012345678901234567890123456789"})
p, _ := json.Marshal(map[string]any{"handoff_ref": ref, "anchor_sha": "0123456789012345678901234567890123456789", "harness_id": task.Lease.HarnessID, "lease_epoch": task.Lease.Epoch, "expected_version": task.Version})
if err := s.Append(domain.Event{ID: "release", Type: "TaskReleased", TaskID: "t", Version: task.Version + 1, Payload: p, Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
@@ -64,13 +65,13 @@ func TestRenewLeaseRequiresCurrentOwnerAndVersion(t *testing.T) {
t.Fatal(err)
}
before, _ := s.Task("t")
if _, err := s.RenewLease("t", "worker-b", before.Version, time.Hour); !errors.Is(err, domain.ErrConflict) {
if _, err := s.RenewLease("t", "worker-b", before.Lease.Epoch, before.Version, time.Hour); !errors.Is(err, domain.ErrConflict) {
t.Fatalf("other worker renewal = %v, want conflict", err)
}
if _, err := s.RenewLease("t", "worker-a", before.Version-1, time.Hour); !errors.Is(err, domain.ErrConflict) {
if _, err := s.RenewLease("t", "worker-a", before.Lease.Epoch, before.Version-1, time.Hour); !errors.Is(err, domain.ErrConflict) {
t.Fatalf("stale renewal = %v, want conflict", err)
}
e, err := s.RenewLease("t", "worker-a", before.Version, time.Hour)
e, err := s.RenewLease("t", "worker-a", before.Lease.Epoch, before.Version, time.Hour)
if err != nil {
t.Fatal(err)
}
@@ -78,11 +79,107 @@ func TestRenewLeaseRequiresCurrentOwnerAndVersion(t *testing.T) {
if e.Type != "TaskLeaseRenewed" || after.Version != before.Version+1 || after.Lease == nil || !after.Lease.Until.After(before.Lease.Until) {
t.Fatalf("renewal was not projected: before=%+v after=%+v event=%+v", before, after, e)
}
if _, err := s.RenewLease("t", "worker-a", before.Version, time.Hour); !errors.Is(err, domain.ErrConflict) {
if _, err := s.RenewLease("t", "worker-a", before.Lease.Epoch, before.Version, time.Hour); !errors.Is(err, domain.ErrConflict) {
t.Fatalf("replayed renewal = %v, want conflict", err)
}
}
func TestLeaseEpochFencesStaleOwnerLifecycleWrites(t *testing.T) {
s, err := Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
if err := s.Append(domain.Event{ID: "create", Type: "TaskCreated", TaskID: "t", Version: 1, Payload: []byte(`{"source":"s","external_id":"epoch","project":"p"}`), Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
if _, err := s.Lease("t", "worker", time.Minute); err != nil {
t.Fatal(err)
}
first, _ := s.Task("t")
ref, err := s.PutArtifact([]byte("handoff"))
if err != nil {
t.Fatal(err)
}
staleRelease, _ := json.Marshal(map[string]any{"handoff_ref": ref, "anchor_sha": strings.Repeat("a", 40), "harness_id": "worker", "lease_epoch": "stale", "expected_version": first.Version})
if err := s.Append(domain.Event{ID: "stale-release", Type: "TaskReleased", TaskID: "t", Version: first.Version + 1, Payload: staleRelease, Surface: string(authz.System)}); !errors.Is(err, domain.ErrConflict) {
t.Fatalf("stale release = %v, want conflict", err)
}
release, _ := json.Marshal(map[string]any{"handoff_ref": ref, "anchor_sha": strings.Repeat("a", 40), "harness_id": "worker", "lease_epoch": first.Lease.Epoch, "expected_version": first.Version})
if err := s.Append(domain.Event{ID: "release", Type: "TaskReleased", TaskID: "t", Version: first.Version + 1, Payload: release, Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
if _, err := s.Lease("t", "worker", time.Minute); err != nil {
t.Fatal(err)
}
second, _ := s.Task("t")
if second.Lease.Epoch == first.Lease.Epoch {
t.Fatal("re-lease reused fencing epoch")
}
report, err := s.PutArtifact([]byte("report"))
if err != nil {
t.Fatal(err)
}
staleComplete, _ := json.Marshal(map[string]any{"report_ref": report, "receipt": map[string]any{"consumed": 1}, "harness_id": "worker", "lease_epoch": first.Lease.Epoch, "expected_version": second.Version})
if err := s.Append(domain.Event{ID: "stale-complete", Type: "TaskCompleted", TaskID: "t", Version: second.Version + 1, Payload: staleComplete, Surface: string(authz.System)}); !errors.Is(err, domain.ErrConflict) {
t.Fatalf("stale completion = %v, want conflict", err)
}
}
func TestOpenRebuildsOnlyFromLogAndIgnoresCorruptSnapshot(t *testing.T) {
dir := t.TempDir()
s, err := Open(dir)
if err != nil {
t.Fatal(err)
}
if err := s.Append(created("create")); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "snapshot.json"), []byte(`not json`), 0600); err != nil {
t.Fatal(err)
}
restarted, err := Open(dir)
if err != nil {
t.Fatalf("corrupt disposable snapshot prevented log recovery: %v", err)
}
if task, ok := restarted.Task("task-1"); !ok || task.State != domain.StateQueued {
t.Fatalf("log projection = %#v, present=%v", task, ok)
}
}
func TestReplayLegacyLeaseDerivesNonRenewableFence(t *testing.T) {
dir := t.TempDir()
until := time.Now().Add(time.Hour).UnixNano()
events := []domain.Event{
{SchemaVersion: 2, Seq: 1, ID: "create", Type: "TaskCreated", TaskID: "t", Version: 1, Payload: []byte(`{"source":"s","external_id":"legacy","project":"p"}`), Surface: string(authz.System)},
{SchemaVersion: 2, Seq: 2, ID: "lease", Type: "TaskLeased", TaskID: "t", Version: 2, Payload: []byte(`{"harness_id":"worker","until_ns":` + fmt.Sprint(until) + `,"expected_version":1}`), Surface: string(authz.System)},
}
f, err := os.Create(filepath.Join(dir, "events.jsonl"))
if err != nil {
t.Fatal(err)
}
for _, e := range events {
b, _ := json.Marshal(e)
if _, err := f.Write(append(b, '\n')); err != nil {
_ = f.Close()
t.Fatal(err)
}
}
if err := f.Close(); err != nil {
t.Fatal(err)
}
s, err := Open(dir)
if err != nil {
t.Fatal(err)
}
task, _ := s.Task("t")
if task.Lease == nil || task.Lease.Epoch != "legacy:lease" {
t.Fatalf("legacy lease fence = %#v", task.Lease)
}
if _, err := s.RenewLease("t", "worker", "", task.Version, time.Hour); !errors.Is(err, domain.ErrConflict) {
t.Fatalf("legacy lease renewed without derived fence: %v", err)
}
}
func TestBlockedTaskProjectsStructuredDiagnosisAndLegacyFallback(t *testing.T) {
s, err := Open(t.TempDir())
if err != nil {
@@ -407,7 +504,8 @@ func TestReleaseTransactionSurvivesReLeaseUntilMatchingPickup(t *testing.T) {
t.Fatal(err)
}
anchor := strings.Repeat("a", 40)
p, _ := json.Marshal(map[string]any{"handoff_ref": ref, "anchor_sha": anchor, "transaction_id": "tx-1", "expected_version": leased.Version})
first, _ := s.Task("task-1")
p, _ := json.Marshal(map[string]any{"handoff_ref": ref, "anchor_sha": anchor, "transaction_id": "tx-1", "harness_id": first.Lease.HarnessID, "lease_epoch": first.Lease.Epoch, "expected_version": leased.Version})
if err := s.Append(domain.Event{ID: "release", Type: "TaskReleased", TaskID: "task-1", Version: leased.Version + 1, Payload: p, Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
@@ -418,7 +516,7 @@ func TestReleaseTransactionSurvivesReLeaseUntilMatchingPickup(t *testing.T) {
if task.ReleaseTransaction != "tx-1" || task.ReleaseAnchor != anchor || task.HandoffRef != ref {
t.Fatalf("re-lease lost transaction: %+v", task)
}
p, _ = json.Marshal(map[string]any{"transaction_id": "tx-1", "handoff_ref": ref, "anchor_sha": anchor, "harness_id": "successor", "lease_version": task.Version, "expected_version": task.Version})
p, _ = json.Marshal(map[string]any{"transaction_id": "tx-1", "handoff_ref": ref, "anchor_sha": anchor, "harness_id": "successor", "lease_epoch": task.Lease.Epoch, "lease_version": task.Version, "expected_version": task.Version})
if err := s.Append(domain.Event{ID: "pickup", Type: "TaskPickupValidated", TaskID: "task-1", Version: task.Version + 1, Payload: p, Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
internal/ui/ui.go
+5
View File
@@ -422,6 +422,11 @@ func (s Server) action(w http.ResponseWriter, r *http.Request, id, action string
http.Error(w, "unknown action", 404)
return
}
if action == "block" {
// A browser-created block is an explicit operator decision. Preserve
// that fact even if its prose happens to contain a system keyword.
body["block_reason"] = string(domain.BlockReasonOperator)
}
b, _ := json.Marshal(body)
e := domain.Event{ID: domain.NewID(), Type: typ, TaskID: id, Version: t.Version + 1, Payload: b, Surface: string(authz.Web)}
if err := s.Store.Append(e); err != nil {
internal/webui/assets/assets/index-D1Up3m4b.js → internal/webui/assets/assets/index-BXQHTW_a.js
+13 -13
View File
File diff suppressed because one or more lines are too long
internal/webui/assets/assets/index-DUgNKGY0.css → internal/webui/assets/assets/index-DDZzc9-8.css
+1 -1
View File
File diff suppressed because one or more lines are too long
internal/webui/assets/index.html
+2 -2
View File
@@ -1,3 +1,3 @@
<script type="module" crossorigin src="/assets/index-D1Up3m4b.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-DUgNKGY0.css">
<script type="module" crossorigin src="/assets/index-BXQHTW_a.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-DDZzc9-8.css">
<div id="root"></div>