kami/orchestra
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: make worker handoff rotation durable

Browse Source
This commit is contained in:
kami
2026-07-30 01:30:59 +04:00
parent ce02c60106
commit 1ff0af2e69
16 changed files with 1488 additions and 1566 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/store/store.go
+64
View File
@@ -138,14 +138,34 @@ func (s *Store) apply(e domain.Event) error {
if v, ok := p["description"].(string); ok {
t.Description = v
}
if v, ok := p["acceptance"].([]any); ok {
for _, item := range v {
if text, ok := item.(string); ok {
t.Acceptance = append(t.Acceptance, text)
}
}
}
if v, ok := p["quality_gate"].(string); ok {
t.QualityGate = v
}
s.external[t.Source+"\x00"+t.ExternalID] = t.ID
case "TaskLeased":
t.State = domain.StateLeased
t.Lease = &domain.Lease{HarnessID: p["harness_id"].(string), Until: time.Unix(0, int64(p["until_ns"].(float64)))}
case "TaskLeaseRenewed":
t.Lease = &domain.Lease{HarnessID: p["harness_id"].(string), Until: time.Unix(0, int64(p["until_ns"].(float64)))}
case "TaskReleased":
t.State = domain.StateQueued
t.Lease = nil
t.HandoffRef, _ = p["handoff_ref"].(string)
t.ReleaseTransaction, _ = p["transaction_id"].(string)
t.ReleaseAnchor, _ = p["anchor_sha"].(string)
t.PickupTransaction, t.PickupLeaseVersion = "", 0
case "TaskPickupValidated":
t.PickupTransaction, _ = p["transaction_id"].(string)
if v, ok := p["lease_version"].(float64); ok {
t.PickupLeaseVersion = int(v)
}
case "TaskCompleted":
t.State = domain.StateCompleted
t.Lease = nil
@@ -156,6 +176,10 @@ func (s *Store) apply(e domain.Event) error {
t.State = domain.StateBlocked
t.Lease = nil
t.Blocker, _ = p["blocker"].(string)
t.BlockReason = domain.InferBlockReason(t.Blocker)
if v, ok := p["block_reason"].(string); ok && domain.BlockReason(v).Valid() {
t.BlockReason = domain.BlockReason(v)
}
t.BlockedAt = e.At
t.LastPaneID, _ = p["pane_id"].(string)
t.LastHarness, _ = p["harness_id"].(string)
@@ -200,6 +224,25 @@ func (s *Store) apply(e domain.Event) error {
}
}
}
// Terminal and release events may carry an owner-produced snapshot from
// immediately before a worker/coordinator drops its live session mapping.
// Preserve it independently of the current task state so historical task
// pages never have to imply a pane is live just because its ID is known.
if raw, ok := p["session_evidence"].(map[string]any); ok {
var evidence domain.SessionEvidence
if b, err := json.Marshal(raw); err == nil && json.Unmarshal(b, &evidence) == nil {
t.LastSession = evidence
if evidence.PaneID != "" {
t.LastPaneID = evidence.PaneID
}
if evidence.HarnessID != "" {
t.LastHarness = evidence.HarnessID
}
if evidence.PaneState != "" {
t.PaneState = evidence.PaneState
}
}
}
t.Version = e.Version
s.tasks[e.TaskID] = t
return nil
@@ -410,12 +453,33 @@ func (s *Store) Lease(id, harness string, ttl time.Duration) (domain.Event, erro
payload := map[string]any{"harness_id": harness, "ttl": ttl.Seconds(), "until_ns": time.Now().Add(ttl).UnixNano(), "expected_version": t.Version}
if t.HandoffRef != "" {
payload["handoff_ref"] = t.HandoffRef
payload["transaction_id"] = t.ReleaseTransaction
payload["anchor_sha"] = t.ReleaseAnchor
}
p, _ := json.Marshal(payload)
e := domain.Event{ID: domain.NewID(), Type: "TaskLeased", TaskID: id, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
return e, s.Append(e)
}
// RenewLease atomically extends the current owner's lease. The observed task
// version is part of the request so an old worker can never renew a lease
// after release/reassignment.
func (s *Store) RenewLease(id, harness string, expectedVersion int, ttl time.Duration) (domain.Event, error) {
if ttl <= 0 {
return domain.Event{}, fmt.Errorf("%w: ttl must be positive", domain.ErrInvalid)
}
t, ok := s.Task(id)
if !ok {
return domain.Event{}, domain.ErrNotFound
}
if t.State != domain.StateLeased || t.Lease == nil || t.Lease.HarnessID != harness || t.Version != expectedVersion {
return domain.Event{}, domain.ErrConflict
}
p, _ := json.Marshal(map[string]any{"harness_id": harness, "until_ns": time.Now().Add(ttl).UnixNano(), "expected_version": expectedVersion})
e := domain.Event{ID: domain.NewID(), Type: "TaskLeaseRenewed", TaskID: id, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
return e, s.Append(e)
}
func (s *Store) ExpireLeases(now time.Time) ([]domain.Event, error) {
var out []domain.Event
for _, t := range s.Tasks() {
internal/store/store_test.go
+113
View File
@@ -5,6 +5,7 @@ import (
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -51,6 +52,80 @@ func TestLeaseCarriesReleasedHandoffRef(t *testing.T) {
}
}
func TestRenewLeaseRequiresCurrentOwnerAndVersion(t *testing.T) {
s, err := Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
if err := s.Append(domain.Event{ID: "create", Type: "TaskCreated", TaskID: "t", Version: 1, Payload: []byte(`{"source":"s","external_id":"renew","project":"p"}`), Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
if _, err := s.Lease("t", "worker-a", time.Minute); err != nil {
t.Fatal(err)
}
before, _ := s.Task("t")
if _, err := s.RenewLease("t", "worker-b", before.Version, time.Hour); !errors.Is(err, domain.ErrConflict) {
t.Fatalf("other worker renewal = %v, want conflict", err)
}
if _, err := s.RenewLease("t", "worker-a", before.Version-1, time.Hour); !errors.Is(err, domain.ErrConflict) {
t.Fatalf("stale renewal = %v, want conflict", err)
}
e, err := s.RenewLease("t", "worker-a", before.Version, time.Hour)
if err != nil {
t.Fatal(err)
}
after, _ := s.Task("t")
if e.Type != "TaskLeaseRenewed" || after.Version != before.Version+1 || after.Lease == nil || !after.Lease.Until.After(before.Lease.Until) {
t.Fatalf("renewal was not projected: before=%+v after=%+v event=%+v", before, after, e)
}
if _, err := s.RenewLease("t", "worker-a", before.Version, time.Hour); !errors.Is(err, domain.ErrConflict) {
t.Fatalf("replayed renewal = %v, want conflict", err)
}
}
func TestBlockedTaskProjectsStructuredDiagnosisAndLegacyFallback(t *testing.T) {
s, err := Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
if err := s.Append(domain.Event{ID: "create", Type: "TaskCreated", TaskID: "t", Version: 1, Payload: []byte(`{"source":"s","external_id":"blocked","project":"p"}`), Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
payload := []byte(`{"blocker":"worker is offline","block_reason":"worker_offline","pane_id":"p1","harness_id":"w1","pane_state":"unreachable"}`)
if err := s.Append(domain.Event{ID: "blocked", Type: "TaskBlocked", TaskID: "t", Version: 2, Payload: payload, Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
task, _ := s.Task("t")
if task.BlockReason != domain.BlockReasonWorkerOffline || task.LastPaneID != "p1" || task.PaneState != "unreachable" {
t.Fatalf("blocked diagnosis was not projected: %+v", task)
}
if got := domain.InferBlockReason("handoff validation failed"); got != domain.BlockReasonHandoffValidation {
t.Fatalf("legacy fallback=%q", got)
}
}
func TestTerminalSessionEvidenceSurvivesSessionCleanup(t *testing.T) {
s, err := Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
if err := s.Append(domain.Event{ID: "create", Type: "TaskCreated", TaskID: "t", Version: 1, Payload: []byte(`{"source":"s","external_id":"evidence","project":"p"}`), Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
ref, err := s.PutArtifact([]byte("report"))
if err != nil {
t.Fatal(err)
}
payload := []byte(`{"report_ref":"` + ref + `","receipt":{"source":"worker"},"session_evidence":{"pane_id":"w:p1","harness_id":"worker-1","pane_state":"open","source":"worker","captured_at":"2026-07-29T12:00:00Z","checked_at":"2026-07-29T12:00:01Z"}}`)
if err := s.Append(domain.Event{ID: "complete", Type: "TaskCompleted", TaskID: "t", Version: 2, Payload: payload, Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
task, _ := s.Task("t")
if task.LastSession.PaneID != "w:p1" || task.LastSession.Source != "worker" || task.LastSession.CapturedAt.IsZero() || task.LastHarness != "worker-1" {
t.Fatalf("terminal session evidence was lost: %+v", task)
}
}
func TestAppendReplayAndDeduplicate(t *testing.T) {
dir := t.TempDir()
s, err := Open(dir)
@@ -314,3 +389,41 @@ func TestExpectedVersionIsCheckedForEveryWriter(t *testing.T) {
t.Fatalf("expected CAS conflict, got %v", err)
}
}
func TestReleaseTransactionSurvivesReLeaseUntilMatchingPickup(t *testing.T) {
s, err := Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
if err := s.Append(created("create")); err != nil {
t.Fatal(err)
}
leased, err := s.Lease("task-1", "predecessor", time.Minute)
if err != nil {
t.Fatal(err)
}
ref, err := s.PutArtifact([]byte("handoff"))
if err != nil {
t.Fatal(err)
}
anchor := strings.Repeat("a", 40)
p, _ := json.Marshal(map[string]any{"handoff_ref": ref, "anchor_sha": anchor, "transaction_id": "tx-1", "expected_version": leased.Version})
if err := s.Append(domain.Event{ID: "release", Type: "TaskReleased", TaskID: "task-1", Version: leased.Version + 1, Payload: p, Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
if _, err := s.Lease("task-1", "successor", time.Minute); err != nil {
t.Fatal(err)
}
task, _ := s.Task("task-1")
if task.ReleaseTransaction != "tx-1" || task.ReleaseAnchor != anchor || task.HandoffRef != ref {
t.Fatalf("re-lease lost transaction: %+v", task)
}
p, _ = json.Marshal(map[string]any{"transaction_id": "tx-1", "handoff_ref": ref, "anchor_sha": anchor, "harness_id": "successor", "lease_version": task.Version, "expected_version": task.Version})
if err := s.Append(domain.Event{ID: "pickup", Type: "TaskPickupValidated", TaskID: "task-1", Version: task.Version + 1, Payload: p, Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
task, _ = s.Task("task-1")
if task.PickupTransaction != "tx-1" || task.PickupLeaseVersion != 4 {
t.Fatalf("pickup not bound to transaction/epoch: %+v", task)
}
}