Reconcile docs with reality; fix module graph, token compare, health #1

Open
kami wants to merge 216 commits from webui-and-audit-reconciliation into master
2 changed files with 96 additions and 5 deletions
Showing only changes of commit 03663f413b - Show all commits
+21 -5
View File
@@ -1291,12 +1291,28 @@ func (w *worker) once(ctx context.Context) error {
TransactionID string `json:"transaction_id"`
AnchorSHA string `json:"anchor_sha"`
}
if json.Unmarshal(e.Payload, &p) == nil && p.HarnessID == w.harnessID {
var until struct {
UntilNS int64 `json:"until_ns"`
if json.Unmarshal(e.Payload, &p) == nil {
if p.HarnessID == w.harnessID {
var until struct {
UntilNS int64 `json:"until_ns"`
}
_ = json.Unmarshal(e.Payload, &until)
w.leases[e.TaskID] = lease{Epoch: p.Epoch, HandoffRef: p.HandoffRef, TransactionID: p.TransactionID, AnchorSHA: p.AnchorSHA, Version: e.Version, Until: time.Unix(0, until.UntilNS)}
}
// A release transaction is superseded the moment the task is
// leased under a different transaction, our own re-lease
// included. Its anchor can never commit: the late-handoff path
// fences on the epoch that expired, and the owner has moved on
// twice since. Retrying forever pins the session, ActiveTask
// and the single last_error slot, which is where run 10's task
// spent seven hours.
if tx, releasing := w.releases[e.TaskID]; releasing && tx.ID != p.TransactionID {
delete(w.releases, e.TaskID)
w.recordError(fmt.Errorf("release %s superseded by lease %s: abandoning transaction %s", e.TaskID, p.Epoch, tx.ID))
if session, active := w.sessions[e.TaskID]; active {
w.quarantine(ctx, e.TaskID, session)
}
}
_ = json.Unmarshal(e.Payload, &until)
w.leases[e.TaskID] = lease{Epoch: p.Epoch, HandoffRef: p.HandoffRef, TransactionID: p.TransactionID, AnchorSHA: p.AnchorSHA, Version: e.Version, Until: time.Unix(0, until.UntilNS)}
}
}
if e.Type == "TaskLeaseRenewed" {
+75
View File
@@ -1160,3 +1160,78 @@ func TestExpiredReleaseStillCommitsWithTheTransactionEpoch(t *testing.T) {
t.Fatalf("phase %q, want event_committed", got)
}
}
// Proven live at 19:01:30Z on 2026-08-28: the lease expired while the anchor
// was pushing, a successor took the task, and the coordinator refused the late
// commit. That refusal is correct and permanent, so the worker must stop
// asking. Without this it retried every five seconds forever, holding the pane
// and pinning the single last_error slot.
func TestSupersededReleaseIsAbandonedNotRetriedForever(t *testing.T) {
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/v1/federation/events":
_, _ = w.Write([]byte(`{"cursor":0,"events":[{"seq":1,"id":"l","type":"TaskLeased","task_id":"t","version":9,` +
`"payload":{"harness_id":"other","lease_epoch":"e2"},"surface":"system"}]}`))
default:
w.WriteHeader(http.StatusNoContent)
}
}))
defer s.Close()
w := &worker{
api: federation.Client{BaseURL: s.URL, WorkerID: "h", Token: "t"},
harnessID: "h",
backend: deadTmuxBackend(t),
tasks: map[string]domain.Task{"t": {ID: "t"}},
sessions: map[string]herdr.Session{"t": {PaneID: "pane"}},
leases: map[string]lease{},
releases: map[string]releaseTransaction{"t": {ID: "tx", Phase: "anchor_pushed", Ref: "sha256:abc", AnchorSHA: "abc", LeaseEpoch: "e1"}},
quarantined: map[string]bool{},
statePath: t.TempDir() + "/state.json",
hard: .75,
}
if err := w.once(context.Background()); err != nil {
t.Fatal(err)
}
if len(w.sessions) != 0 {
t.Fatalf("session still pinned: %v", w.sessions)
}
if _, still := w.releases["t"]; still {
t.Fatalf("superseded transaction retained: %v", w.releases)
}
if got := w.health(context.Background()).ActiveTask; got != "" {
t.Fatalf("ActiveTask=%q, worker still advertises the dead release", got)
}
}
// The successor pickup carries the predecessor's own transaction id. That
// lease is the handoff completing, not a supersession, and dropping it there
// would destroy the recoverable predecessor F30 exists to protect.
func TestOwnPickupLeaseKeepsTheReleaseTransaction(t *testing.T) {
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/v1/federation/events":
_, _ = w.Write([]byte(`{"cursor":0,"events":[{"seq":1,"id":"l","type":"TaskLeased","task_id":"t","version":9,` +
`"payload":{"harness_id":"h","lease_epoch":"e2","transaction_id":"tx","handoff_ref":"sha256:abc"},"surface":"system"}]}`))
default:
w.WriteHeader(http.StatusNoContent)
}
}))
defer s.Close()
w := &worker{
api: federation.Client{BaseURL: s.URL, WorkerID: "h", Token: "t"},
harnessID: "h",
tasks: map[string]domain.Task{"t": {ID: "t"}},
sessions: map[string]herdr.Session{"t": {PaneID: "pane"}},
leases: map[string]lease{},
releases: map[string]releaseTransaction{"t": {ID: "tx", Phase: "event_committed", Ref: "sha256:abc", AnchorSHA: "abc", LeaseEpoch: "e1"}},
quarantined: map[string]bool{},
statePath: t.TempDir() + "/state.json",
hard: .75,
}
if err := w.once(context.Background()); err != nil {
t.Fatal(err)
}
if _, ok := w.releases["t"]; !ok {
t.Fatal("pickup lease dropped its own release transaction")
}
}