fix: make worker handoff rotation durable

This commit is contained in:
kami
2026-07-30 01:30:59 +04:00
parent ce02c60106
commit 1ff0af2e69
16 changed files with 1488 additions and 1566 deletions
+39 -11
View File
@@ -40,7 +40,17 @@ func RenderTaskFile(t domain.Task) []byte {
if strings.TrimSpace(t.Description) != "" {
fmt.Fprintf(&b, "\n## Instructions\n\n%s\n", t.Description)
}
b.WriteString("\nThis file is immutable for the lifetime of the task (§6.2) — its hash is\ncarried in every handoff and re-verified on every pickup. Do not edit it.\n")
if len(t.Acceptance) > 0 {
b.WriteString("\n## Acceptance criteria\n")
for _, criterion := range t.Acceptance {
fmt.Fprintf(&b, "\n- %s", criterion)
}
b.WriteByte('\n')
}
if t.QualityGate != "" {
fmt.Fprintf(&b, "\n## Quality gate\n\n%s\n", t.QualityGate)
}
b.WriteString("\n## Completion\n\nRun the configured quality gate. When the task is ready for the worker to verify and deliver, create `.orchestra/done`. Do not write a prose completion report.\n\nThis file is immutable for the lifetime of the task (§6.2) — its hash is\ncarried in every handoff and re-verified on every pickup. Do not edit it.\n")
return []byte(b.String())
}
@@ -80,8 +90,9 @@ func ConventionsHash(root string) (string, error) {
}
type Dirty struct {
Path string `json:"path"`
SHA256 string `json:"sha256"`
Path string `json:"path"`
SHA256 string `json:"sha256"`
Deleted bool `json:"deleted,omitempty"`
}
type Completed struct {
What string `json:"what"`
@@ -147,7 +158,7 @@ func (h Handoff) Validate() error {
}
}
for _, d := range h.Anchor.Dirty {
if filepath.IsAbs(d.Path) || d.Path == "" || len(d.SHA256) != 64 {
if filepath.IsAbs(d.Path) || d.Path == "" || (!d.Deleted && len(d.SHA256) != 64) {
return errors.New("invalid dirty anchor")
}
}
@@ -199,6 +210,12 @@ func ValidatePickup(root string, h Handoff, taskFileSHA string) error {
return errors.New("handoff anchor HEAD mismatch")
}
for _, d := range h.Anchor.Dirty {
if d.Deleted {
if _, e := os.Stat(filepath.Join(root, d.Path)); !errors.Is(e, os.ErrNotExist) {
return fmt.Errorf("handoff deleted file restored: %s", d.Path)
}
continue
}
b, e := os.ReadFile(filepath.Join(root, d.Path))
if e != nil {
return e
@@ -257,7 +274,12 @@ func Load(ref string, cas CAS) (Handoff, error) {
return Decode(b)
}
// ScratchCommit records WIP atomically on a dedicated branch before rotation.
// ScratchCommit records every piece of repository work except Orchestra's
// ephemeral protocol markers. In particular, git add -A is intentional: it
// includes already-staged changes, deletions, renames, and untracked files.
// TASK.md is checked before touching the index; it is an immutable input, not
// deliverable work. The report/done markers remain local so a successor never
// mistakes an old protocol signal for a new one.
func ScratchCommit(root, branch, message string) error {
if branch == "" || strings.ContainsAny(branch, " \t\n") {
return errors.New("invalid scratch branch")
@@ -279,14 +301,20 @@ func ScratchCommit(root, branch, message string) error {
return err
}
}
if err := exec.Command("git", "-C", root, "add", "-A").Run(); err != nil {
// A harness may have staged a protocol marker itself. Remove it from the
// index before staging the real checkpoint; this does not alter its working
// tree contents and makes the exclusion apply to staged state too.
for _, marker := range []string{".orchestra", ".orchestra-handoff.json", ".orchestra-handoff-report.md"} {
if err := exec.Command("git", "-C", root, "reset", "-q", "HEAD", "--", marker).Run(); err != nil {
return err
}
}
if err := exec.Command("git", "-C", root, "add", "-A", "--", ".", ":(exclude)TASK.md", ":(exclude).orchestra", ":(exclude).orchestra-handoff.json", ":(exclude).orchestra-handoff-report.md").Run(); err != nil {
return err
}
full, err := exec.Command("git", "-C", root, "status", "--porcelain").Output()
if err != nil {
return err
}
if len(full) == 0 {
// Only staged non-protocol work is committed. Remaining marker files are
// expected and must not suppress a clean committed-anchor checkpoint.
if exec.Command("git", "-C", root, "diff", "--cached", "--quiet").Run() == nil {
return nil // nothing to snapshot; branch already reflects the worktree
}
return exec.Command("git", "-C", root, "commit", "-m", message).Run()
+48
View File
@@ -12,6 +12,54 @@ import (
"orchestra/internal/store"
)
func TestScratchCommitCapturesAllGitStatesExceptProtocolMarkers(t *testing.T) {
repo := t.TempDir()
run := func(args ...string) {
t.Helper()
if out, err := exec.Command("git", append([]string{"-C", repo}, args...)...).CombinedOutput(); err != nil {
t.Fatalf("git %v: %v: %s", args, err, out)
}
}
run("init")
run("config", "user.email", "t@t")
run("config", "user.name", "t")
for _, name := range []string{"TASK.md", "deleted.txt", "renamed.txt", "staged.txt"} {
if err := os.WriteFile(filepath.Join(repo, name), []byte(name), 0644); err != nil {
t.Fatal(err)
}
}
run("add", "-A")
run("commit", "-m", "base")
if err := os.WriteFile(filepath.Join(repo, "staged.txt"), []byte("staged change"), 0644); err != nil {
t.Fatal(err)
}
run("add", "staged.txt")
if err := os.Remove(filepath.Join(repo, "deleted.txt")); err != nil {
t.Fatal(err)
}
run("mv", "renamed.txt", "renamed-new.txt")
if err := os.WriteFile(filepath.Join(repo, "untracked.txt"), []byte("new"), 0644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(repo, ".orchestra-handoff-report.md"), []byte("protocol"), 0644); err != nil {
t.Fatal(err)
}
if err := ScratchCommit(repo, "orchestra/scratch/test", "checkpoint"); err != nil {
t.Fatal(err)
}
for _, want := range []string{"staged.txt", "renamed-new.txt", "untracked.txt"} {
if err := exec.Command("git", "-C", repo, "cat-file", "-e", "HEAD:"+want).Run(); err != nil {
t.Fatalf("checkpoint omitted %s: %v", want, err)
}
}
if err := exec.Command("git", "-C", repo, "cat-file", "-e", "HEAD:deleted.txt").Run(); err == nil {
t.Fatal("checkpoint retained deleted file")
}
if err := exec.Command("git", "-C", repo, "cat-file", "-e", "HEAD:.orchestra-handoff-report.md").Run(); err == nil {
t.Fatal("checkpoint committed protocol marker")
}
}
func TestHandoffCASAndPickup(t *testing.T) {
root := t.TempDir()
run := func(a ...string) {
+121 -10
View File
@@ -38,11 +38,69 @@ const (
StateBlocked TaskState = "blocked"
)
// BlockReason is the machine-readable diagnosis for a TaskBlocked event.
// Blocker remains the operator-facing detail; this field lets projections
// group attention without repeatedly parsing prose at read time.
type BlockReason string
const (
BlockReasonLeaseFailure BlockReason = "lease_failure"
BlockReasonWorkerOffline BlockReason = "worker_offline"
BlockReasonLeaseExpired BlockReason = "lease_expired"
BlockReasonApproval BlockReason = "approval"
BlockReasonHandoffValidation BlockReason = "handoff_validation"
BlockReasonOperator BlockReason = "operator_block"
BlockReasonSystem BlockReason = "system_error"
BlockReasonUnknown BlockReason = "unknown"
)
func (r BlockReason) Valid() bool {
switch r {
case BlockReasonLeaseFailure, BlockReasonWorkerOffline, BlockReasonLeaseExpired,
BlockReasonApproval, BlockReasonHandoffValidation, BlockReasonOperator,
BlockReasonSystem, BlockReasonUnknown:
return true
}
return false
}
// InferBlockReason supplies a stable category for older events which only
// recorded a prose blocker. New producers should send block_reason directly.
func InferBlockReason(blocker string) BlockReason {
v := strings.ToLower(blocker)
switch {
case strings.Contains(v, "handoff"):
return BlockReasonHandoffValidation
case strings.Contains(v, "approval") || strings.Contains(v, "permission"):
return BlockReasonApproval
case strings.Contains(v, "expired") && strings.Contains(v, "lease"):
return BlockReasonLeaseExpired
case strings.Contains(v, "worker") && (strings.Contains(v, "offline") || strings.Contains(v, "unreachable")):
return BlockReasonWorkerOffline
case strings.Contains(v, "lease") || strings.Contains(v, "agent.start") || strings.Contains(v, "pane"):
return BlockReasonLeaseFailure
default:
return BlockReasonSystem
}
}
type Estimate struct {
Value float64 `json:"value"`
Who string `json:"who"`
Confidence float64 `json:"confidence"`
}
// SessionEvidence is captured by the machine that owns a pane immediately
// before it drops its mapping. It is deliberately observation-only: it never
// claims that a pane is still live after the worker has closed it.
type SessionEvidence struct {
PaneID string `json:"pane_id,omitempty"`
HarnessID string `json:"harness_id,omitempty"`
PaneState string `json:"pane_state,omitempty"`
Source string `json:"source,omitempty"`
CapturedAt time.Time `json:"captured_at,omitempty"`
CheckedAt time.Time `json:"checked_at,omitempty"`
}
type Lease struct {
HarnessID string `json:"harness_id"`
Until time.Time `json:"until"`
@@ -62,17 +120,27 @@ type Task struct {
// HandoffRef survives the queued interval between TaskReleased and the
// next router-owned TaskLeased event; it is the only artifact the worker
// may use for local pickup validation.
HandoffRef string `json:"handoff_ref,omitempty"`
Version int `json:"version"`
Title string `json:"title,omitempty"`
Description string `json:"description,omitempty"`
HandoffRef string `json:"handoff_ref,omitempty"`
// ReleaseTransaction and ReleaseAnchor bind successor pickup to the exact
// durable predecessor checkpoint. They survive queueing and re-lease.
ReleaseTransaction string `json:"release_transaction,omitempty"`
ReleaseAnchor string `json:"release_anchor,omitempty"`
PickupTransaction string `json:"pickup_transaction,omitempty"`
PickupLeaseVersion int `json:"pickup_lease_version,omitempty"`
Version int `json:"version"`
Title string `json:"title,omitempty"`
Description string `json:"description,omitempty"`
Acceptance []string `json:"acceptance,omitempty"`
QualityGate string `json:"quality_gate,omitempty"`
// Block evidence is projected from TaskBlocked so terminal records remain
// diagnosable after the live coordinator mapping is gone.
Blocker string `json:"blocker,omitempty"`
BlockedAt time.Time `json:"blocked_at,omitempty"`
LastPaneID string `json:"last_pane_id,omitempty"`
LastHarness string `json:"last_harness_id,omitempty"`
PaneState string `json:"pane_state,omitempty"` // open, closed, unreachable, unknown
Blocker string `json:"blocker,omitempty"`
BlockReason BlockReason `json:"block_reason,omitempty"`
BlockedAt time.Time `json:"blocked_at,omitempty"`
LastPaneID string `json:"last_pane_id,omitempty"`
LastHarness string `json:"last_harness_id,omitempty"`
PaneState string `json:"pane_state,omitempty"` // open, closed, unreachable, unknown
LastSession SessionEvidence `json:"last_session,omitempty"`
}
type Event struct {
@@ -112,7 +180,7 @@ func ValidateEvent(e Event) error {
if e.SchemaVersion >= 2 && strings.TrimSpace(e.Surface) == "" {
return fmt.Errorf("%w: surface required", ErrInvalid)
}
allowed := map[string]bool{"TaskCreated": true, "TaskLeased": true, "TaskReleased": true, "TaskCompleted": true, "TaskFailed": true, "TaskBlocked": true, "ApprovalRequested": true, "ApprovalGranted": true, "ApprovalDenied": true, "TaskAmended": true, "TaskCorrected": true, "QuotaReported": true, "StandupAdvisory": true}
allowed := map[string]bool{"TaskCreated": true, "TaskLeased": true, "TaskLeaseRenewed": true, "TaskReleased": true, "TaskPickupValidated": true, "TaskCompleted": true, "TaskFailed": true, "TaskBlocked": true, "ApprovalRequested": true, "ApprovalGranted": true, "ApprovalDenied": true, "TaskAmended": true, "TaskCorrected": true, "QuotaReported": true, "StandupAdvisory": true}
if !allowed[e.Type] {
return fmt.Errorf("%w: unknown type %q", ErrInvalid, e.Type)
}
@@ -160,10 +228,36 @@ func ValidatePayload(typ string, p map[string]any) error {
if v, ok := p["expected_version"].(float64); !ok || v < 0 || v != float64(int(v)) {
return fmt.Errorf("%w: expected_version invalid", ErrInvalid)
}
case "TaskLeaseRenewed":
if err := requiredString("harness_id"); err != nil {
return err
}
until, ok := p["until_ns"].(float64)
if !ok || until <= float64(time.Now().UnixNano()) {
return fmt.Errorf("%w: until_ns required", ErrInvalid)
}
if v, ok := p["expected_version"].(float64); !ok || v < 0 || v != float64(int(v)) {
return fmt.Errorf("%w: expected_version invalid", ErrInvalid)
}
case "TaskReleased":
if err := requiredString("handoff_ref"); err != nil && p["reason"] == nil {
return err
}
case "TaskPickupValidated":
for _, key := range []string{"transaction_id", "handoff_ref", "anchor_sha", "harness_id"} {
if err := requiredString(key); err != nil {
return err
}
}
if err := requiredHash(p, "handoff_ref"); err != nil {
return err
}
if v, ok := p["anchor_sha"].(string); !ok || len(v) != 40 {
return fmt.Errorf("%w: anchor_sha invalid", ErrInvalid)
}
if v, ok := p["lease_version"].(float64); !ok || v < 1 || v != float64(int(v)) {
return fmt.Errorf("%w: lease_version invalid", ErrInvalid)
}
if _, ok := p["handoff_ref"]; ok {
if err := requiredHash(p, "handoff_ref"); err != nil {
return err
@@ -183,6 +277,17 @@ func ValidatePayload(typ string, p map[string]any) error {
if receipt, ok := p["receipt"].(map[string]any); !ok || len(receipt) == 0 {
return fmt.Errorf("%w: receipt required", ErrInvalid)
}
if v, ok := p["result_sha"]; ok {
if s, ok := v.(string); !ok || len(s) != 40 {
return fmt.Errorf("%w: result_sha invalid", ErrInvalid)
}
if err := requiredString("branch"); err != nil {
return err
}
if err := requiredString("remote"); err != nil {
return err
}
}
case "TaskFailed":
if err := requiredString("reason"); err != nil {
return err
@@ -191,6 +296,12 @@ func ValidatePayload(typ string, p map[string]any) error {
if err := requiredString("blocker"); err != nil {
return err
}
if v, ok := p["block_reason"]; ok {
s, ok := v.(string)
if !ok || !BlockReason(s).Valid() {
return fmt.Errorf("%w: block_reason invalid", ErrInvalid)
}
}
if _, ok := p["handoff_ref"]; ok {
if err := requiredHash(p, "handoff_ref"); err != nil {
return err
+19 -5
View File
@@ -24,7 +24,7 @@ type Client struct {
}
func (c Client) Register(ctx context.Context, w Worker) error {
b, err := json.Marshal(map[string]any{"id": w.ID, "address": w.Address, "capacity": w.Capacity, "token": c.Token})
b, err := json.Marshal(map[string]any{"id": w.ID, "address": w.Address, "capacity": w.Capacity, "supported_projects": w.SupportedProjects, "build": w.Build, "token": c.Token})
if err != nil {
return err
}
@@ -131,6 +131,13 @@ func (c Client) Heartbeat(ctx context.Context, health WorkerHealth) error {
}
return err
}
func (c Client) Renew(ctx context.Context, taskID string, expectedVersion, ttlSeconds int) error {
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/renew", map[string]any{"task_id": taskID, "expected_version": expectedVersion, "ttl_seconds": ttlSeconds})
if resp != nil {
resp.Body.Close()
}
return err
}
func (c Client) Artifact(ctx context.Context, ref string) ([]byte, error) {
resp, err := c.request(ctx, http.MethodGet, "/v1/artifacts/"+url.PathEscape(ref), nil)
if err != nil {
@@ -169,15 +176,22 @@ func (c Client) PutArtifact(ctx context.Context, b []byte) (string, error) {
}
return out.Ref, nil
}
func (c Client) Release(ctx context.Context, taskID, ref, anchor string) error {
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/handoff", map[string]string{"task_id": taskID, "handoff_ref": ref, "anchor_sha": anchor})
func (c Client) Release(ctx context.Context, taskID, ref, anchor, transactionID string, expectedVersion int, evidence domain.SessionEvidence) error {
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/handoff", map[string]any{"task_id": taskID, "handoff_ref": ref, "anchor_sha": anchor, "transaction_id": transactionID, "expected_version": expectedVersion, "session_evidence": evidence})
if resp != nil {
resp.Body.Close()
}
return err
}
func (c Client) Complete(ctx context.Context, taskID, reportRef string) error {
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/complete", map[string]string{"task_id": taskID, "handoff_ref": reportRef})
func (c Client) Pickup(ctx context.Context, taskID, ref, anchor, transactionID string, leaseVersion int, evidence domain.SessionEvidence) error {
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/pickup", map[string]any{"task_id": taskID, "handoff_ref": ref, "anchor_sha": anchor, "transaction_id": transactionID, "lease_version": leaseVersion, "session_evidence": evidence})
if resp != nil {
resp.Body.Close()
}
return err
}
func (c Client) Complete(ctx context.Context, taskID, reportRef, resultSHA, branch, remote string, expectedVersion int, receipt map[string]any, evidence domain.SessionEvidence) error {
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/complete", map[string]any{"task_id": taskID, "handoff_ref": reportRef, "result_sha": resultSHA, "branch": branch, "remote": remote, "expected_version": expectedVersion, "receipt": receipt, "session_evidence": evidence})
if resp != nil {
resp.Body.Close()
}
+130 -68
View File
@@ -248,80 +248,99 @@ func (a CLIAdapter) NotifyConventionsChanged(ctx context.Context, s Session) err
// handoff is refused rather than guessed at: the caller (Coordinator.rotate)
// leaves the lease intact and retries next tick, giving the agent time to
// finish writing it.
func (a CLIAdapter) Release(ctx context.Context, s Session) (string, error) {
// PreparedRelease is the durable, coordinator-independent half of a release.
// The worker persists it before publishing TaskReleased so a lost HTTP reply
// never requires reconstructing (or deleting) the agent's report.
type PreparedRelease struct {
Ref string
AnchorSHA string
}
// PrepareRelease seals an immutable Git checkpoint and uploads its canonical
// handoff, but deliberately leaves both the pane claim and report in place.
// The caller controls the retryable transaction around coordinator acceptance.
func (a CLIAdapter) PrepareRelease(ctx context.Context, s Session) (PreparedRelease, error) {
if a.CAS == nil {
return "", fmt.Errorf("adapter: CAS store required to upload handoff")
return PreparedRelease{}, fmt.Errorf("adapter: CAS store required to upload handoff")
}
// A federation worker always supplies the immutable task hash and remote.
// The empty-hash case is retained solely for old in-process adapter users;
// it is not reachable from the worker release path.
if s.TaskFileSHA != "" {
if a.Remote == "" {
return PreparedRelease{}, fmt.Errorf("adapter: project remote required for checkpoint")
}
if err := continuity.VerifyTaskFile(s.Worktree, s.TaskFileSHA); err != nil {
return PreparedRelease{}, fmt.Errorf("adapter: verify immutable TASK.md: %w", err)
}
}
path := filepath.Join(s.Worktree, HandoffReportFile)
b, err := os.ReadFile(path)
if err != nil {
return "", fmt.Errorf("adapter: semantic handoff report not written yet (%s): %w", path, err)
return PreparedRelease{}, fmt.Errorf("adapter: semantic handoff report not written yet (%s): %w", path, err)
}
if strings.TrimSpace(string(b)) == "" {
return "", fmt.Errorf("adapter: semantic handoff report is empty")
return PreparedRelease{}, fmt.Errorf("adapter: semantic handoff report is empty")
}
h, err := canonicalHandoff(s, string(b), a.lastObservedCommand(s))
if err != nil {
return "", err
return PreparedRelease{}, err
}
sha, err := HeadSHA(s.Worktree)
// Always checkpoint and push, including already-committed clean work. Git
// is the cross-machine transport, so merely observing a local clean HEAD is
// not a sufficient anchor.
branch := "orchestra/scratch/" + h.Meta.ID
if err := continuity.ScratchCommit(s.Worktree, branch, "orchestra: pre-release WIP snapshot ("+h.Meta.ID+")"); err != nil {
return PreparedRelease{}, fmt.Errorf("adapter: scratch commit: %w", err)
}
anchor, err := HeadSHA(s.Worktree)
if err != nil {
return "", fmt.Errorf("adapter: read worktree HEAD: %w", err)
return PreparedRelease{}, fmt.Errorf("adapter: read checkpoint HEAD: %w", err)
}
_ = sha
for _, d := range h.Anchor.Dirty {
if hex.EncodeToString(sha256sum(filepath.Join(s.Worktree, d.Path))) != d.SHA256 {
return "", fmt.Errorf("adapter: handoff dirty file changed since it was written: %s", d.Path)
if a.Remote != "" {
if err := continuity.ScratchPush(s.Worktree, branch, a.Remote); err != nil {
return PreparedRelease{}, fmt.Errorf("adapter: push checkpoint: %w", err)
}
out, err := exec.CommandContext(ctx, "git", "-C", s.Worktree, "ls-remote", a.Remote, "refs/heads/"+branch).Output()
if err != nil || !strings.HasPrefix(string(out), anchor+"\t") {
return PreparedRelease{}, fmt.Errorf("adapter: verify pushed anchor: got %q: %w", strings.TrimSpace(string(out)), err)
}
}
// The semantic report is transferred in the CAS handoff, not in the
// scratch checkout. Keeping it in the scratch commit makes a successor
// mistake the predecessor's report for a newly requested handoff and can
// cause an immediate release/pickup loop.
dirty := h.Anchor.Dirty[:0]
for _, d := range h.Anchor.Dirty {
if filepath.Clean(d.Path) != HandoffReportFile {
dirty = append(dirty, d)
}
}
h.Anchor.Dirty = dirty
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
return "", fmt.Errorf("adapter: remove transferred semantic report: %w", err)
}
// Atomically commit whatever the handoff described as dirty onto a
// per-task scratch branch (§6.2 step 3) *before* uploading, so the
// successor's pickup validation collapses to a single HEAD compare
// instead of re-hashing every dirty file individually.
if len(h.Anchor.Dirty) > 0 {
branch := "orchestra/scratch/" + h.Meta.ID
if err := continuity.ScratchCommit(s.Worktree, branch, "orchestra: pre-release WIP snapshot ("+h.Meta.ID+")"); err != nil {
return "", fmt.Errorf("adapter: scratch commit: %w", err)
}
if a.Remote != "" {
if err := continuity.ScratchPush(s.Worktree, branch, a.Remote); err != nil {
return "", fmt.Errorf("adapter: push scratch branch: %w", err)
}
}
newSHA, err := HeadSHA(s.Worktree)
if err != nil {
return "", fmt.Errorf("adapter: read scratch HEAD: %w", err)
}
h.Anchor.GitSHA = newSHA
h.Anchor.Branch = branch
h.Anchor.Dirty = nil
}
h.Anchor.GitSHA, h.Anchor.Branch, h.Anchor.Dirty = anchor, branch, nil
ref, err := continuity.Save(h, a.CAS)
if err != nil {
return "", fmt.Errorf("adapter: upload handoff: %w", err)
return PreparedRelease{}, fmt.Errorf("adapter: upload handoff: %w", err)
}
return PreparedRelease{Ref: ref, AnchorSHA: anchor}, nil
}
// ReleaseAgent drops only herdr's harness binding. It does not close the pane:
// a predecessor stays recoverable until the successor has validated pickup.
func (a CLIAdapter) ReleaseAgent(ctx context.Context, s Session) error {
if err := a.Client.Call(ctx, "pane.release_agent", map[string]any{
"pane_id": s.PaneID,
"source": "herdr:" + a.Harness,
"agent": agentForSession(s, a.Harness),
}, nil); err != nil {
return "", fmt.Errorf("adapter: pane.release_agent: %w", err)
return fmt.Errorf("adapter: pane.release_agent: %w", err)
}
return ref, nil
return nil
}
// Release is retained for the coordinator's legacy local path. Federation
// workers use PrepareRelease and ReleaseAgent as separate durable phases.
func (a CLIAdapter) Release(ctx context.Context, s Session) (string, error) {
p, err := a.PrepareRelease(ctx, s)
if err != nil {
return "", err
}
if err := a.ReleaseAgent(ctx, s); err != nil {
return "", err
}
if err := os.Remove(filepath.Join(s.Worktree, HandoffReportFile)); err != nil && !os.IsNotExist(err) {
return "", fmt.Errorf("adapter: remove transferred semantic report: %w", err)
}
return p.Ref, nil
}
// canonicalHandoff keeps Git-derived protocol facts on the worker that owns
@@ -483,16 +502,27 @@ func handoffID(s Session) string {
}
func dirtyFiles(root string) ([]continuity.Dirty, error) {
out, err := exec.Command("git", "-C", root, "status", "--porcelain=v1", "-z").Output()
if err != nil {
return nil, err
}
paths := map[string]bool{}
for _, args := range [][]string{{"diff", "--name-only", "-z"}, {"ls-files", "--others", "--exclude-standard", "-z"}} {
out, err := exec.Command("git", append([]string{"-C", root}, args...)...).Output()
if err != nil {
return nil, err
deleted := map[string]bool{}
parts := strings.Split(string(out), "\x00")
for i := 0; i < len(parts); i++ {
record := parts[i]
if len(record) < 4 {
continue
}
for _, path := range strings.Split(string(out), "\x00") {
if path != "" && path != HandoffFile {
paths[path] = true
}
status, path := record[:2], record[3:]
if path == HandoffFile || path == HandoffReportFile || path == ".orchestra/done" || strings.HasPrefix(path, ".orchestra/") {
continue
}
paths[path] = true
deleted[path] = strings.Contains(status, "D")
// A rename/copy record has the original path as the next NUL item.
if status[0] == 'R' || status[0] == 'C' || status[1] == 'R' || status[1] == 'C' {
i++
}
}
keys := make([]string, 0, len(paths))
@@ -502,11 +532,15 @@ func dirtyFiles(root string) ([]continuity.Dirty, error) {
sort.Strings(keys)
dirty := make([]continuity.Dirty, 0, len(keys))
for _, path := range keys {
sum := sha256sum(filepath.Join(root, path))
if len(sum) == 0 {
return nil, fmt.Errorf("adapter: hash dirty file %s", path)
d := continuity.Dirty{Path: path, Deleted: deleted[path]}
if !d.Deleted {
sum := sha256sum(filepath.Join(root, path))
if len(sum) == 0 {
return nil, fmt.Errorf("adapter: hash dirty file %s", path)
}
d.SHA256 = hex.EncodeToString(sum)
}
dirty = append(dirty, continuity.Dirty{Path: path, SHA256: hex.EncodeToString(sum)})
dirty = append(dirty, d)
}
return dirty, nil
}
@@ -641,6 +675,13 @@ var _ = json.RawMessage{}
// callers (Coordinator.rotate, refreshSessionHealth) surface it instead of
// mistaking "we don't know" for "occupancy is zero".
func (a CLIAdapter) Occupancy(s Session) (float64, error) {
if a.Harness == "opencode" {
u, err := OpenCodeSessionUsage(s.SessionID)
if err != nil {
return 0, err
}
return Fraction(u, a.Window), nil
}
if a.Usage == nil {
return 0, fmt.Errorf("adapter: usage reader required")
}
@@ -659,6 +700,32 @@ func (a CLIAdapter) Occupancy(s Session) (float64, error) {
return Fraction(u, a.Window), nil
}
// ResolveSessionIdentity discovers and returns the harness-native session
// identity. Callers persist the returned Session before relying on occupancy,
// so restart recovery keeps observing the same harness session.
func (a CLIAdapter) ResolveSessionIdentity(s Session) (Session, error) {
if a.Harness == "opencode" {
if s.SessionID != "" {
return s, nil
}
id, err := OpenCodeSessionID(s.Worktree)
if err != nil {
return s, err
}
s.SessionID = id
return s, nil
}
if s.SessionFile != "" {
return s, nil
}
path, err := a.resolveSessionFile(s)
if err != nil {
return s, err
}
s.SessionFile = path
return s, nil
}
func (a CLIAdapter) resolveSessionFile(s Session) (string, error) {
switch a.Harness {
case "claude":
@@ -667,12 +734,7 @@ func (a CLIAdapter) resolveSessionFile(s Session) (string, error) {
_, path, err := CodexActiveUsage("")
return path, err
default:
// opencode's session-file resolution needs the running session id,
// which is only available via the SSE/status API (OpenCodeStatus),
// not derivable from the worktree alone. Per AUDIT.md Phase 1, wiring
// this needs verification against a live opencode instance before it
// can drive rotation — refuse loudly rather than guess a path.
return "", fmt.Errorf("adapter: harness %q has no session-file resolver; verify against a live session first (AUDIT.md Phase 1)", a.Harness)
return "", fmt.Errorf("adapter: harness %q has no session-file resolver", a.Harness)
}
}
+4
View File
@@ -152,6 +152,10 @@ type Session struct {
// CLIAdapter.Occupancy), since the file may not exist yet immediately
// after lease.
SessionFile string `json:"session_file,omitempty"`
// SessionID is the harness-native identity when its usage is stored in a
// database rather than a transcript. OpenCode's SQLite session ID is kept
// here so rotation never guesses "the newest session" after a restart.
SessionID string `json:"session_id,omitempty"`
// TaskFileSHA is the sha256 of the worktree's TASK.md at the time this
// session's lease was created — the immutable-spec hash continuity's
// pickup validation compares against on the next rotation (§6.2).
+56
View File
@@ -10,6 +10,7 @@ import (
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"time"
)
@@ -30,6 +31,7 @@ func Fraction(u Usage, w int64) float64 {
}
return f
}
// ClaudeSessionFile resolves the transcript file for a Claude Code session
// running against worktree, by newest-mtime under Claude Code's encoded
// project directory (~/.claude/projects/<abs-worktree-path-with-/-replaced-
@@ -202,6 +204,60 @@ func OpenCodeUsage(p string) (Usage, error) {
return Usage{x.Tokens.Input, x.Tokens.Cache.Read, x.Tokens.Cache.Write, x.Tokens.Output}, e
}
// OpenCodeSessionID resolves the exact OpenCode session associated with a
// checkout. OpenCode stores usage in its SQLite session table, not in a pane
// transcript. Selecting by directory and persisting the returned id prevents
// a multi-pane worker from attributing another task's newest session to this
// lease.
func OpenCodeSessionID(worktree string) (string, error) {
db := os.Getenv("ORCHESTRA_OPENCODE_DB")
if db == "" {
db = filepath.Join(os.Getenv("HOME"), ".local", "share", "opencode", "opencode.db")
}
abs, err := filepath.Abs(worktree)
if err != nil {
return "", err
}
out, err := exec.Command("sqlite3", "-readonly", "-noheader", db, "select id from session where directory = "+sqliteQuote(abs)+" order by time_updated desc limit 1;").Output()
if err != nil {
return "", fmt.Errorf("opencode session lookup: %w", err)
}
id := strings.TrimSpace(string(out))
if id == "" {
return "", fmt.Errorf("opencode session lookup: no session for %s", abs)
}
return id, nil
}
// OpenCodeSessionUsage reads the token counters for one persisted session.
func OpenCodeSessionUsage(sessionID string) (Usage, error) {
if sessionID == "" {
return Usage{}, fmt.Errorf("opencode session id required")
}
db := os.Getenv("ORCHESTRA_OPENCODE_DB")
if db == "" {
db = filepath.Join(os.Getenv("HOME"), ".local", "share", "opencode", "opencode.db")
}
out, err := exec.Command("sqlite3", "-readonly", "-noheader", "-separator", "|", db, "select tokens_input,tokens_cache_read,tokens_cache_write,tokens_output from session where id = "+sqliteQuote(sessionID)+";").Output()
if err != nil {
return Usage{}, fmt.Errorf("opencode usage lookup: %w", err)
}
parts := strings.Split(strings.TrimSpace(string(out)), "|")
if len(parts) != 4 {
return Usage{}, fmt.Errorf("opencode usage lookup: unknown session %q", sessionID)
}
values := [4]int64{}
for i, part := range parts {
values[i], err = strconv.ParseInt(part, 10, 64)
if err != nil {
return Usage{}, fmt.Errorf("opencode usage lookup: %w", err)
}
}
return Usage{Input: values[0], CacheRead: values[1], CacheWrite: values[2], Output: values[3]}, nil
}
func sqliteQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", "''") + "'" }
// OpenCodeStatus probes the server fast path. Callers can use the returned
// status and fall back to OpenCodeUsage when the SSE/server is unavailable.
func OpenCodeStatus(ctx context.Context, baseURL, sessionID string) (string, error) {
+14 -22
View File
@@ -774,18 +774,21 @@ func (c *Coordinator) TurnDecision(ctx context.Context, taskID string) (string,
if existingReason := handoffReason(session.Worktree); existingReason == "manual" || existingReason == "milestone" || existingReason == "thrash" {
return c.finishRelease(ctx, taskID, task, session, a, existingReason)
}
if trigger, deadEnds := checkActivityTriggers(ctx, a, session, c.Thrash); trigger != "" {
c.requestReasonedHandoff(ctx, taskID, session, a, trigger, deadEnds)
return TurnPrepareHandoff, nil
d := (RotationStateMachine{Soft: c.soft(), Hard: c.Hard, Thrash: c.Thrash}).Evaluate(ctx, a, session)
if d.Degraded != nil {
return "", fmt.Errorf("orchestrator: rotation: %w", d.Degraded)
}
occupancy, err := a.Occupancy(session)
if err != nil {
return "", fmt.Errorf("orchestrator: occupancy: %w", err)
}
if occupancy < c.soft() {
if d.Action == TurnContinue {
return TurnContinue, nil
}
if occupancy < c.Hard {
if d.Action == TurnRefuse {
return TurnRefuse, nil
}
if d.Reason == "milestone" || d.Reason == "thrash" {
c.requestReasonedHandoff(ctx, taskID, session, a, d.Reason, d.DeadEnds)
return TurnPrepareHandoff, nil
}
if d.Action == TurnPrepareHandoff {
// Soft threshold (§5.3): advisory only. Ask the agent to start
// preparing a handoff well before Hard forces one, but don't block
// the turn on a boundary check — the agent is free to keep working.
@@ -805,18 +808,6 @@ func (c *Coordinator) TurnDecision(ctx context.Context, taskID string) (string,
}
return TurnPrepareHandoff, nil
}
if boundary, ok := a.(herdr.TurnBoundary); ok {
atBoundary, boundaryErr := boundary.AtTurnBoundary(ctx, session)
if boundaryErr != nil {
c.recordTurnBoundaryDegraded()
return TurnRefuse, nil
}
if !atBoundary {
return TurnRefuse, nil
}
} else {
c.recordTurnBoundaryDegraded()
}
if requester, ok := a.(herdr.HandoffRequester); ok {
if _, statErr := os.Stat(filepath.Join(session.Worktree, herdr.HandoffReportFile)); statErr != nil {
if !session.HandoffRequested {
@@ -979,11 +970,12 @@ func (c *Coordinator) rememberSession(taskID string, s herdr.Session) error {
}
func (c *Coordinator) block(t domain.Task, reason string) error {
p := map[string]string{"blocker": reason, "pane_state": "unknown"}
p := map[string]any{"blocker": reason, "block_reason": string(domain.InferBlockReason(reason)), "pane_state": "unknown", "session_evidence": domain.SessionEvidence{PaneState: "unknown", Source: "coordinator", CheckedAt: time.Now().UTC()}}
if s, ok := c.Session(t.ID); ok {
p["pane_id"] = s.PaneID
p["harness_id"] = s.HerdrID
p["pane_state"] = "open"
p["session_evidence"] = domain.SessionEvidence{PaneID: s.PaneID, HarnessID: s.HerdrID, PaneState: "open", Source: "coordinator", CheckedAt: time.Now().UTC()}
}
b, _ := json.Marshal(p)
return c.Store.Append(domain.Event{ID: domain.NewID(), Type: "TaskBlocked", TaskID: t.ID, Version: t.Version + 1, Payload: b, Surface: string(authz.System)})
+82
View File
@@ -0,0 +1,82 @@
package orchestrator
import (
"context"
"fmt"
"orchestra/internal/continuity"
"orchestra/internal/herdr"
)
// RotationStateMachine is the shared, side-effect-free rotation policy used
// by both the coordinator's synchronous turn path and federation workers.
// Callers persist request/release side effects themselves, but must never
// replace an unavailable occupancy reading with zero.
type RotationStateMachine struct {
Soft float64
Hard float64
Thrash herdr.ThrashConfig
}
type RotationDecision struct {
Action string // continue, prepare_handoff, rotate_now
Reason string
DeadEnds []continuity.DeadEnd
// ActivityDegraded is advisory (threshold rotation still has a real usage
// source); it is surfaced so a missing milestone/thrash feed cannot be a
// silent no-op.
ActivityDegraded error
Degraded error
}
func (m RotationStateMachine) Evaluate(ctx context.Context, a herdr.Adapter, s herdr.Session) RotationDecision {
soft := m.Soft
if soft <= 0 {
soft = defaultSoft
}
if m.Hard <= 0 || m.Hard <= soft {
return RotationDecision{Degraded: fmt.Errorf("invalid rotation thresholds soft=%v hard=%v", soft, m.Hard)}
}
if reader, ok := a.(herdr.ActivityReader); ok {
calls, err := reader.Activity(ctx, s)
if err == nil {
if thrash, deadEnds := herdr.DetectThrash(calls, m.Thrash); thrash {
return RotationDecision{Action: TurnPrepareHandoff, Reason: "thrash", DeadEnds: deadEnds}
}
if herdr.DetectMilestone(calls) {
return RotationDecision{Action: TurnPrepareHandoff, Reason: "milestone"}
}
} else {
return m.evaluateOccupancy(ctx, a, s, fmt.Errorf("activity unknown: %w", err))
}
}
return m.evaluateOccupancy(ctx, a, s, nil)
}
func (m RotationStateMachine) evaluateOccupancy(ctx context.Context, a herdr.Adapter, s herdr.Session, activityErr error) RotationDecision {
soft := m.Soft
if soft <= 0 {
soft = defaultSoft
}
occupancy, err := a.Occupancy(s)
if err != nil {
return RotationDecision{ActivityDegraded: activityErr, Degraded: fmt.Errorf("occupancy unknown: %w", err)}
}
if occupancy < soft {
return RotationDecision{Action: TurnContinue, ActivityDegraded: activityErr}
}
if occupancy < m.Hard {
return RotationDecision{Action: TurnPrepareHandoff, Reason: "threshold", ActivityDegraded: activityErr}
}
boundary, ok := a.(herdr.TurnBoundary)
if !ok {
return RotationDecision{ActivityDegraded: activityErr, Degraded: fmt.Errorf("turn boundary unknown at hard threshold"), Action: TurnRefuse, Reason: "threshold"}
}
atBoundary, err := boundary.AtTurnBoundary(ctx, s)
if err != nil {
return RotationDecision{ActivityDegraded: activityErr, Degraded: fmt.Errorf("turn boundary unknown: %w", err), Action: TurnRefuse, Reason: "threshold"}
}
if !atBoundary {
return RotationDecision{Action: TurnRefuse, Reason: "threshold", ActivityDegraded: activityErr}
}
return RotationDecision{Action: TurnRotateNow, Reason: "threshold", ActivityDegraded: activityErr}
}
+30 -12
View File
@@ -708,6 +708,7 @@ type activityAdapter struct {
activityErr error
reasonAsked []string
deadEndsSeen []continuity.DeadEnd
observed chan struct{}
}
func (a *activityAdapter) Activity(context.Context, herdr.Session) ([]herdr.ToolCall, error) {
@@ -717,6 +718,12 @@ func (a *activityAdapter) Activity(context.Context, herdr.Session) ([]herdr.Tool
func (a *activityAdapter) RequestHandoffReason(_ context.Context, _ herdr.Session, reason string, deadEnds []continuity.DeadEnd) error {
a.reasonAsked = append(a.reasonAsked, reason)
a.deadEndsSeen = deadEnds
if a.observed != nil {
select {
case a.observed <- struct{}{}:
default:
}
}
return nil
}
@@ -764,7 +771,7 @@ func TestActivityTriggersRequestReasonedHandoffWithoutReleasing(t *testing.T) {
}
t.Run("thrash requests a reasoned handoff and does not release, via TurnDecision", func(t *testing.T) {
a := &activityAdapter{fakeAdapter: fakeAdapter{occupancy: 0, boundary: true}, calls: thrashCalls}
a := &activityAdapter{fakeAdapter: fakeAdapter{occupancy: 0, boundary: true}, calls: thrashCalls, observed: make(chan struct{}, 1)}
c, st, task := newCoordinator(a)
decision, err := c.TurnDecision(context.Background(), task.ID)
if err != nil {
@@ -862,13 +869,15 @@ func TestActivityTriggersRequestReasonedHandoffWithoutReleasing(t *testing.T) {
c, st, task := newCoordinator(a)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go c.Monitor(ctx, .8, time.Millisecond)
done := make(chan error, 1)
go func() { done <- c.Monitor(ctx, .8, time.Millisecond) }()
deadline := time.Now().Add(300 * time.Millisecond)
for time.Now().Before(deadline) && len(a.reasonAsked) == 0 {
time.Sleep(time.Millisecond)
select {
case <-a.observed:
case <-time.After(300 * time.Millisecond):
}
cancel()
<-done
if len(a.reasonAsked) == 0 || a.reasonAsked[0] != "thrash" {
t.Fatalf("reasonAsked=%v want a thrash request from rotate()", a.reasonAsked)
}
@@ -959,10 +968,17 @@ func (w specWorktrees) Spec(domain.Task) (string, string, bool) { re
type conventionsAdapter struct {
fakeAdapter
notifications int
observed chan struct{}
}
func (a *conventionsAdapter) NotifyConventionsChanged(context.Context, herdr.Session) error {
a.notifications++
if a.observed != nil {
select {
case a.observed <- struct{}{}:
default:
}
}
return nil
}
@@ -991,7 +1007,7 @@ func TestConventionsDriftNotifiesActiveSession(t *testing.T) {
t.Fatal(err)
}
task := s.Tasks()[0]
a := &conventionsAdapter{fakeAdapter: fakeAdapter{occupancy: 0}}
a := &conventionsAdapter{fakeAdapter: fakeAdapter{occupancy: 0}, observed: make(chan struct{}, 1)}
c := &orchestrator.Coordinator{Store: s, Worktrees: specWorktrees{wtPath: worktree, repoPath: repo}, Adapters: adapters{a}, StatePath: t.TempDir() + "/sessions.json"}
leaseEvt, err := s.Lease(task.ID, "h1", time.Minute)
@@ -1003,8 +1019,8 @@ func TestConventionsDriftNotifiesActiveSession(t *testing.T) {
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go c.Monitor(ctx, .8, time.Millisecond)
done := make(chan error, 1)
go func() { done <- c.Monitor(ctx, .8, time.Millisecond) }()
time.Sleep(50 * time.Millisecond)
if a.notifications != 0 {
@@ -1015,10 +1031,12 @@ func TestConventionsDriftNotifiesActiveSession(t *testing.T) {
t.Fatal(err)
}
deadline := time.Now().Add(time.Second)
for time.Now().Before(deadline) && a.notifications == 0 {
time.Sleep(time.Millisecond)
select {
case <-a.observed:
case <-time.After(time.Second):
}
cancel()
<-done
if a.notifications == 0 {
t.Fatal("session was never notified of the conventions-doc update")
}
+64
View File
@@ -138,14 +138,34 @@ func (s *Store) apply(e domain.Event) error {
if v, ok := p["description"].(string); ok {
t.Description = v
}
if v, ok := p["acceptance"].([]any); ok {
for _, item := range v {
if text, ok := item.(string); ok {
t.Acceptance = append(t.Acceptance, text)
}
}
}
if v, ok := p["quality_gate"].(string); ok {
t.QualityGate = v
}
s.external[t.Source+"\x00"+t.ExternalID] = t.ID
case "TaskLeased":
t.State = domain.StateLeased
t.Lease = &domain.Lease{HarnessID: p["harness_id"].(string), Until: time.Unix(0, int64(p["until_ns"].(float64)))}
case "TaskLeaseRenewed":
t.Lease = &domain.Lease{HarnessID: p["harness_id"].(string), Until: time.Unix(0, int64(p["until_ns"].(float64)))}
case "TaskReleased":
t.State = domain.StateQueued
t.Lease = nil
t.HandoffRef, _ = p["handoff_ref"].(string)
t.ReleaseTransaction, _ = p["transaction_id"].(string)
t.ReleaseAnchor, _ = p["anchor_sha"].(string)
t.PickupTransaction, t.PickupLeaseVersion = "", 0
case "TaskPickupValidated":
t.PickupTransaction, _ = p["transaction_id"].(string)
if v, ok := p["lease_version"].(float64); ok {
t.PickupLeaseVersion = int(v)
}
case "TaskCompleted":
t.State = domain.StateCompleted
t.Lease = nil
@@ -156,6 +176,10 @@ func (s *Store) apply(e domain.Event) error {
t.State = domain.StateBlocked
t.Lease = nil
t.Blocker, _ = p["blocker"].(string)
t.BlockReason = domain.InferBlockReason(t.Blocker)
if v, ok := p["block_reason"].(string); ok && domain.BlockReason(v).Valid() {
t.BlockReason = domain.BlockReason(v)
}
t.BlockedAt = e.At
t.LastPaneID, _ = p["pane_id"].(string)
t.LastHarness, _ = p["harness_id"].(string)
@@ -200,6 +224,25 @@ func (s *Store) apply(e domain.Event) error {
}
}
}
// Terminal and release events may carry an owner-produced snapshot from
// immediately before a worker/coordinator drops its live session mapping.
// Preserve it independently of the current task state so historical task
// pages never have to imply a pane is live just because its ID is known.
if raw, ok := p["session_evidence"].(map[string]any); ok {
var evidence domain.SessionEvidence
if b, err := json.Marshal(raw); err == nil && json.Unmarshal(b, &evidence) == nil {
t.LastSession = evidence
if evidence.PaneID != "" {
t.LastPaneID = evidence.PaneID
}
if evidence.HarnessID != "" {
t.LastHarness = evidence.HarnessID
}
if evidence.PaneState != "" {
t.PaneState = evidence.PaneState
}
}
}
t.Version = e.Version
s.tasks[e.TaskID] = t
return nil
@@ -410,12 +453,33 @@ func (s *Store) Lease(id, harness string, ttl time.Duration) (domain.Event, erro
payload := map[string]any{"harness_id": harness, "ttl": ttl.Seconds(), "until_ns": time.Now().Add(ttl).UnixNano(), "expected_version": t.Version}
if t.HandoffRef != "" {
payload["handoff_ref"] = t.HandoffRef
payload["transaction_id"] = t.ReleaseTransaction
payload["anchor_sha"] = t.ReleaseAnchor
}
p, _ := json.Marshal(payload)
e := domain.Event{ID: domain.NewID(), Type: "TaskLeased", TaskID: id, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
return e, s.Append(e)
}
// RenewLease atomically extends the current owner's lease. The observed task
// version is part of the request so an old worker can never renew a lease
// after release/reassignment.
func (s *Store) RenewLease(id, harness string, expectedVersion int, ttl time.Duration) (domain.Event, error) {
if ttl <= 0 {
return domain.Event{}, fmt.Errorf("%w: ttl must be positive", domain.ErrInvalid)
}
t, ok := s.Task(id)
if !ok {
return domain.Event{}, domain.ErrNotFound
}
if t.State != domain.StateLeased || t.Lease == nil || t.Lease.HarnessID != harness || t.Version != expectedVersion {
return domain.Event{}, domain.ErrConflict
}
p, _ := json.Marshal(map[string]any{"harness_id": harness, "until_ns": time.Now().Add(ttl).UnixNano(), "expected_version": expectedVersion})
e := domain.Event{ID: domain.NewID(), Type: "TaskLeaseRenewed", TaskID: id, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
return e, s.Append(e)
}
func (s *Store) ExpireLeases(now time.Time) ([]domain.Event, error) {
var out []domain.Event
for _, t := range s.Tasks() {
+113
View File
@@ -5,6 +5,7 @@ import (
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -51,6 +52,80 @@ func TestLeaseCarriesReleasedHandoffRef(t *testing.T) {
}
}
func TestRenewLeaseRequiresCurrentOwnerAndVersion(t *testing.T) {
s, err := Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
if err := s.Append(domain.Event{ID: "create", Type: "TaskCreated", TaskID: "t", Version: 1, Payload: []byte(`{"source":"s","external_id":"renew","project":"p"}`), Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
if _, err := s.Lease("t", "worker-a", time.Minute); err != nil {
t.Fatal(err)
}
before, _ := s.Task("t")
if _, err := s.RenewLease("t", "worker-b", before.Version, time.Hour); !errors.Is(err, domain.ErrConflict) {
t.Fatalf("other worker renewal = %v, want conflict", err)
}
if _, err := s.RenewLease("t", "worker-a", before.Version-1, time.Hour); !errors.Is(err, domain.ErrConflict) {
t.Fatalf("stale renewal = %v, want conflict", err)
}
e, err := s.RenewLease("t", "worker-a", before.Version, time.Hour)
if err != nil {
t.Fatal(err)
}
after, _ := s.Task("t")
if e.Type != "TaskLeaseRenewed" || after.Version != before.Version+1 || after.Lease == nil || !after.Lease.Until.After(before.Lease.Until) {
t.Fatalf("renewal was not projected: before=%+v after=%+v event=%+v", before, after, e)
}
if _, err := s.RenewLease("t", "worker-a", before.Version, time.Hour); !errors.Is(err, domain.ErrConflict) {
t.Fatalf("replayed renewal = %v, want conflict", err)
}
}
func TestBlockedTaskProjectsStructuredDiagnosisAndLegacyFallback(t *testing.T) {
s, err := Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
if err := s.Append(domain.Event{ID: "create", Type: "TaskCreated", TaskID: "t", Version: 1, Payload: []byte(`{"source":"s","external_id":"blocked","project":"p"}`), Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
payload := []byte(`{"blocker":"worker is offline","block_reason":"worker_offline","pane_id":"p1","harness_id":"w1","pane_state":"unreachable"}`)
if err := s.Append(domain.Event{ID: "blocked", Type: "TaskBlocked", TaskID: "t", Version: 2, Payload: payload, Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
task, _ := s.Task("t")
if task.BlockReason != domain.BlockReasonWorkerOffline || task.LastPaneID != "p1" || task.PaneState != "unreachable" {
t.Fatalf("blocked diagnosis was not projected: %+v", task)
}
if got := domain.InferBlockReason("handoff validation failed"); got != domain.BlockReasonHandoffValidation {
t.Fatalf("legacy fallback=%q", got)
}
}
func TestTerminalSessionEvidenceSurvivesSessionCleanup(t *testing.T) {
s, err := Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
if err := s.Append(domain.Event{ID: "create", Type: "TaskCreated", TaskID: "t", Version: 1, Payload: []byte(`{"source":"s","external_id":"evidence","project":"p"}`), Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
ref, err := s.PutArtifact([]byte("report"))
if err != nil {
t.Fatal(err)
}
payload := []byte(`{"report_ref":"` + ref + `","receipt":{"source":"worker"},"session_evidence":{"pane_id":"w:p1","harness_id":"worker-1","pane_state":"open","source":"worker","captured_at":"2026-07-29T12:00:00Z","checked_at":"2026-07-29T12:00:01Z"}}`)
if err := s.Append(domain.Event{ID: "complete", Type: "TaskCompleted", TaskID: "t", Version: 2, Payload: payload, Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
task, _ := s.Task("t")
if task.LastSession.PaneID != "w:p1" || task.LastSession.Source != "worker" || task.LastSession.CapturedAt.IsZero() || task.LastHarness != "worker-1" {
t.Fatalf("terminal session evidence was lost: %+v", task)
}
}
func TestAppendReplayAndDeduplicate(t *testing.T) {
dir := t.TempDir()
s, err := Open(dir)
@@ -314,3 +389,41 @@ func TestExpectedVersionIsCheckedForEveryWriter(t *testing.T) {
t.Fatalf("expected CAS conflict, got %v", err)
}
}
func TestReleaseTransactionSurvivesReLeaseUntilMatchingPickup(t *testing.T) {
s, err := Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
if err := s.Append(created("create")); err != nil {
t.Fatal(err)
}
leased, err := s.Lease("task-1", "predecessor", time.Minute)
if err != nil {
t.Fatal(err)
}
ref, err := s.PutArtifact([]byte("handoff"))
if err != nil {
t.Fatal(err)
}
anchor := strings.Repeat("a", 40)
p, _ := json.Marshal(map[string]any{"handoff_ref": ref, "anchor_sha": anchor, "transaction_id": "tx-1", "expected_version": leased.Version})
if err := s.Append(domain.Event{ID: "release", Type: "TaskReleased", TaskID: "task-1", Version: leased.Version + 1, Payload: p, Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
if _, err := s.Lease("task-1", "successor", time.Minute); err != nil {
t.Fatal(err)
}
task, _ := s.Task("task-1")
if task.ReleaseTransaction != "tx-1" || task.ReleaseAnchor != anchor || task.HandoffRef != ref {
t.Fatalf("re-lease lost transaction: %+v", task)
}
p, _ = json.Marshal(map[string]any{"transaction_id": "tx-1", "handoff_ref": ref, "anchor_sha": anchor, "harness_id": "successor", "lease_version": task.Version, "expected_version": task.Version})
if err := s.Append(domain.Event{ID: "pickup", Type: "TaskPickupValidated", TaskID: "task-1", Version: task.Version + 1, Payload: p, Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
task, _ = s.Task("task-1")
if task.PickupTransaction != "tx-1" || task.PickupLeaseVersion != 4 {
t.Fatalf("pickup not bound to transaction/epoch: %+v", task)
}
}