kami/orchestra
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Harden lease lifecycle durability

Browse Source
This commit is contained in:
kami
2026-07-30 14:34:29 +04:00
parent 1ff0af2e69
commit f6ee0e3060
40 changed files with 2108 additions and 590 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/orchestrator/orchestrator.go
+80 -24
View File
@@ -6,6 +6,7 @@ package orchestrator
import (
"context"
"encoding/json"
"errors"
"fmt"
"orchestra/internal/authz"
"orchestra/internal/continuity"
@@ -412,10 +413,29 @@ func (c *Coordinator) saveSessionsLocked() error {
return err
}
tmp := c.StatePath + ".tmp"
if err = os.WriteFile(tmp, b, 0600); err != nil {
f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
if err != nil {
return err
}
return os.Rename(tmp, c.StatePath)
if _, err = f.Write(b); err == nil {
err = f.Sync()
}
if closeErr := f.Close(); err == nil {
err = closeErr
}
if err != nil {
_ = os.Remove(tmp)
return err
}
if err = os.Rename(tmp, c.StatePath); err != nil {
return err
}
dir, err := os.Open(filepath.Dir(c.StatePath))
if err != nil {
return err
}
defer dir.Close()
return dir.Sync()
}
// Reconcile drops mappings whose task lease did not survive restart and kills
@@ -573,7 +593,7 @@ func (c *Coordinator) expire(ctx context.Context) ([]domain.Event, error) {
if a, ae := c.adapterFor(taskID, s); ae == nil {
if p, ok := a.(herdr.PaneExit); ok {
if exited, ee := p.PaneExited(ctx, s); ee == nil && exited {
b, _ := json.Marshal(map[string]string{"reason": "pane_exited", "harness_id": s.Harness})
b, _ := json.Marshal(map[string]any{"reason": "pane_exited", "harness_id": s.Harness, "lease_epoch": t.Lease.Epoch, "expected_version": t.Version})
_ = c.Store.Append(domain.Event{ID: domain.NewID(), Type: "TaskReleased", TaskID: taskID, Version: t.Version + 1, Payload: b, Surface: string(authz.System)})
}
}
@@ -581,24 +601,47 @@ func (c *Coordinator) expire(ctx context.Context) ([]domain.Event, error) {
}
}
c.mu.Unlock()
events, err := c.Store.ExpireLeases(time.Now())
if err != nil {
return events, err
}
for _, e := range events {
c.loadSessions()
c.mu.Lock()
s, ok := c.sessions[e.TaskID]
delete(c.sessions, e.TaskID)
if ok {
if a, ae := c.adapterFor(e.TaskID, s); ae == nil {
_ = a.Kill(ctx, s)
}
var events []domain.Event
var firstErr error
for _, task := range c.Store.Tasks() {
if task.State != domain.StateLeased || task.Lease == nil || task.Lease.Until.After(time.Now()) {
continue
}
_ = c.saveSessionsLocked()
// Stop a local predecessor before making its lease eligible for a
// successor. If this cannot be done, keep both the mapping and the
// lease: safety beats reclaim speed.
c.mu.Lock()
s, local := c.sessions[task.ID]
c.mu.Unlock()
if local {
a, adapterErr := c.adapterFor(task.ID, s)
if adapterErr != nil {
if firstErr == nil {
firstErr = fmt.Errorf("expire %s: resolve old pane: %w", task.ID, adapterErr)
}
continue
}
if killErr := a.Kill(ctx, s); killErr != nil {
if firstErr == nil {
firstErr = fmt.Errorf("expire %s: quarantine old pane: %w", task.ID, killErr)
}
continue
}
c.mu.Lock()
delete(c.sessions, task.ID)
_ = c.saveSessionsLocked()
c.mu.Unlock()
}
e, expireErr := c.Store.ExpireLease(task.ID, time.Now())
if expireErr != nil {
if !errors.Is(expireErr, domain.ErrConflict) && firstErr == nil {
firstErr = expireErr
}
continue
}
events = append(events, e)
}
return events, nil
return events, firstErr
}
// handoffReason reads HandoffFile from the worktree, if present, and returns
@@ -722,13 +765,17 @@ func (c *Coordinator) rotate(ctx context.Context, hard float64) {
// tick or TTL expiry to reclaim.
continue
}
b, _ := json.Marshal(map[string]string{"handoff_ref": ref, "reason": reason, "anchor_sha": anchorSHA})
b, _ := json.Marshal(map[string]any{"handoff_ref": ref, "reason": reason, "anchor_sha": anchorSHA, "harness_id": task.Lease.HarnessID, "lease_epoch": task.Lease.Epoch, "expected_version": task.Version})
e := domain.Event{ID: domain.NewID(), Type: "TaskReleased", TaskID: taskID, Version: task.Version + 1, Payload: b, Surface: string(authz.System)}
if c.Store.Append(e) == nil {
c.mu.Lock()
delete(c.sessions, taskID)
_ = c.saveSessionsLocked()
c.mu.Unlock()
// A release only transfers the lease; this local coordinator owns
// the predecessor pane until it has actually stopped it.
if err := a.Kill(ctx, session); err == nil {
c.mu.Lock()
delete(c.sessions, taskID)
_ = c.saveSessionsLocked()
c.mu.Unlock()
}
}
}
}
@@ -842,11 +889,14 @@ func (c *Coordinator) finishRelease(ctx context.Context, taskID string, task dom
// invalid TaskReleased payload, same as rotate()'s bare continue.
return TurnRefuse, nil
}
b, _ := json.Marshal(map[string]string{"handoff_ref": ref, "reason": reason, "anchor_sha": anchorSHA})
b, _ := json.Marshal(map[string]any{"handoff_ref": ref, "reason": reason, "anchor_sha": anchorSHA, "harness_id": task.Lease.HarnessID, "lease_epoch": task.Lease.Epoch, "expected_version": task.Version})
e := domain.Event{ID: domain.NewID(), Type: "TaskReleased", TaskID: taskID, Version: task.Version + 1, Payload: b, Surface: string(authz.System)}
if err := c.Store.Append(e); err != nil {
return TurnRefuse, nil
}
if err := a.Kill(ctx, session); err != nil {
return TurnRefuse, nil
}
c.mu.Lock()
delete(c.sessions, taskID)
_ = c.saveSessionsLocked()
@@ -978,6 +1028,12 @@ func (c *Coordinator) block(t domain.Task, reason string) error {
p["session_evidence"] = domain.SessionEvidence{PaneID: s.PaneID, HarnessID: s.HerdrID, PaneState: "open", Source: "coordinator", CheckedAt: time.Now().UTC()}
}
b, _ := json.Marshal(p)
if t.Lease != nil {
p["harness_id"] = t.Lease.HarnessID
p["lease_epoch"] = t.Lease.Epoch
p["expected_version"] = t.Version
b, _ = json.Marshal(p)
}
return c.Store.Append(domain.Event{ID: domain.NewID(), Type: "TaskBlocked", TaskID: t.ID, Version: t.Version + 1, Payload: b, Surface: string(authz.System)})
}
internal/orchestrator/worktrees_test.go
+5 -1
View File
@@ -8,6 +8,7 @@ import (
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
)
@@ -73,7 +74,7 @@ func TestGitWorktreesCommitsTaskFile(t *testing.T) {
initRepo(t, repo)
w := orchestrator.GitWorktrees{Root: filepath.Join(base, "wt"), Repo: repo}
task := domain.Task{ID: "t1", Project: "p", Source: "jsonl", ExternalID: "1", Title: "do the thing"}
task := domain.Task{ID: "t1", Project: "p", Source: "jsonl", ExternalID: "1", Title: "do the thing", Acceptance: []string{"tests pass"}, QualityGate: "go test ./..."}
path, err := w.Create(context.Background(), task)
if err != nil {
@@ -88,6 +89,9 @@ func TestGitWorktreesCommitsTaskFile(t *testing.T) {
if string(got) != string(want) {
t.Fatalf("TASK.md content mismatch:\ngot: %s\nwant: %s", got, want)
}
if !strings.Contains(string(got), "## Acceptance criteria") || !strings.Contains(string(got), ".orchestra/done") {
t.Fatalf("TASK.md is missing deterministic completion contract: %s", got)
}
status, err := exec.Command("git", "-C", path, "status", "--porcelain", "--", "TASK.md").Output()
if err != nil {