fix: make worker handoff rotation durable
This commit is contained in:
@@ -40,7 +40,17 @@ func RenderTaskFile(t domain.Task) []byte {
|
||||
if strings.TrimSpace(t.Description) != "" {
|
||||
fmt.Fprintf(&b, "\n## Instructions\n\n%s\n", t.Description)
|
||||
}
|
||||
b.WriteString("\nThis file is immutable for the lifetime of the task (§6.2) — its hash is\ncarried in every handoff and re-verified on every pickup. Do not edit it.\n")
|
||||
if len(t.Acceptance) > 0 {
|
||||
b.WriteString("\n## Acceptance criteria\n")
|
||||
for _, criterion := range t.Acceptance {
|
||||
fmt.Fprintf(&b, "\n- %s", criterion)
|
||||
}
|
||||
b.WriteByte('\n')
|
||||
}
|
||||
if t.QualityGate != "" {
|
||||
fmt.Fprintf(&b, "\n## Quality gate\n\n%s\n", t.QualityGate)
|
||||
}
|
||||
b.WriteString("\n## Completion\n\nRun the configured quality gate. When the task is ready for the worker to verify and deliver, create `.orchestra/done`. Do not write a prose completion report.\n\nThis file is immutable for the lifetime of the task (§6.2) — its hash is\ncarried in every handoff and re-verified on every pickup. Do not edit it.\n")
|
||||
return []byte(b.String())
|
||||
}
|
||||
|
||||
@@ -80,8 +90,9 @@ func ConventionsHash(root string) (string, error) {
|
||||
}
|
||||
|
||||
type Dirty struct {
|
||||
Path string `json:"path"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Path string `json:"path"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Deleted bool `json:"deleted,omitempty"`
|
||||
}
|
||||
type Completed struct {
|
||||
What string `json:"what"`
|
||||
@@ -147,7 +158,7 @@ func (h Handoff) Validate() error {
|
||||
}
|
||||
}
|
||||
for _, d := range h.Anchor.Dirty {
|
||||
if filepath.IsAbs(d.Path) || d.Path == "" || len(d.SHA256) != 64 {
|
||||
if filepath.IsAbs(d.Path) || d.Path == "" || (!d.Deleted && len(d.SHA256) != 64) {
|
||||
return errors.New("invalid dirty anchor")
|
||||
}
|
||||
}
|
||||
@@ -199,6 +210,12 @@ func ValidatePickup(root string, h Handoff, taskFileSHA string) error {
|
||||
return errors.New("handoff anchor HEAD mismatch")
|
||||
}
|
||||
for _, d := range h.Anchor.Dirty {
|
||||
if d.Deleted {
|
||||
if _, e := os.Stat(filepath.Join(root, d.Path)); !errors.Is(e, os.ErrNotExist) {
|
||||
return fmt.Errorf("handoff deleted file restored: %s", d.Path)
|
||||
}
|
||||
continue
|
||||
}
|
||||
b, e := os.ReadFile(filepath.Join(root, d.Path))
|
||||
if e != nil {
|
||||
return e
|
||||
@@ -257,7 +274,12 @@ func Load(ref string, cas CAS) (Handoff, error) {
|
||||
return Decode(b)
|
||||
}
|
||||
|
||||
// ScratchCommit records WIP atomically on a dedicated branch before rotation.
|
||||
// ScratchCommit records every piece of repository work except Orchestra's
|
||||
// ephemeral protocol markers. In particular, git add -A is intentional: it
|
||||
// includes already-staged changes, deletions, renames, and untracked files.
|
||||
// TASK.md is checked before touching the index; it is an immutable input, not
|
||||
// deliverable work. The report/done markers remain local so a successor never
|
||||
// mistakes an old protocol signal for a new one.
|
||||
func ScratchCommit(root, branch, message string) error {
|
||||
if branch == "" || strings.ContainsAny(branch, " \t\n") {
|
||||
return errors.New("invalid scratch branch")
|
||||
@@ -279,14 +301,20 @@ func ScratchCommit(root, branch, message string) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := exec.Command("git", "-C", root, "add", "-A").Run(); err != nil {
|
||||
// A harness may have staged a protocol marker itself. Remove it from the
|
||||
// index before staging the real checkpoint; this does not alter its working
|
||||
// tree contents and makes the exclusion apply to staged state too.
|
||||
for _, marker := range []string{".orchestra", ".orchestra-handoff.json", ".orchestra-handoff-report.md"} {
|
||||
if err := exec.Command("git", "-C", root, "reset", "-q", "HEAD", "--", marker).Run(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := exec.Command("git", "-C", root, "add", "-A", "--", ".", ":(exclude)TASK.md", ":(exclude).orchestra", ":(exclude).orchestra-handoff.json", ":(exclude).orchestra-handoff-report.md").Run(); err != nil {
|
||||
return err
|
||||
}
|
||||
full, err := exec.Command("git", "-C", root, "status", "--porcelain").Output()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(full) == 0 {
|
||||
// Only staged non-protocol work is committed. Remaining marker files are
|
||||
// expected and must not suppress a clean committed-anchor checkpoint.
|
||||
if exec.Command("git", "-C", root, "diff", "--cached", "--quiet").Run() == nil {
|
||||
return nil // nothing to snapshot; branch already reflects the worktree
|
||||
}
|
||||
return exec.Command("git", "-C", root, "commit", "-m", message).Run()
|
||||
|
||||
@@ -12,6 +12,54 @@ import (
|
||||
"orchestra/internal/store"
|
||||
)
|
||||
|
||||
func TestScratchCommitCapturesAllGitStatesExceptProtocolMarkers(t *testing.T) {
|
||||
repo := t.TempDir()
|
||||
run := func(args ...string) {
|
||||
t.Helper()
|
||||
if out, err := exec.Command("git", append([]string{"-C", repo}, args...)...).CombinedOutput(); err != nil {
|
||||
t.Fatalf("git %v: %v: %s", args, err, out)
|
||||
}
|
||||
}
|
||||
run("init")
|
||||
run("config", "user.email", "t@t")
|
||||
run("config", "user.name", "t")
|
||||
for _, name := range []string{"TASK.md", "deleted.txt", "renamed.txt", "staged.txt"} {
|
||||
if err := os.WriteFile(filepath.Join(repo, name), []byte(name), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
run("add", "-A")
|
||||
run("commit", "-m", "base")
|
||||
if err := os.WriteFile(filepath.Join(repo, "staged.txt"), []byte("staged change"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
run("add", "staged.txt")
|
||||
if err := os.Remove(filepath.Join(repo, "deleted.txt")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
run("mv", "renamed.txt", "renamed-new.txt")
|
||||
if err := os.WriteFile(filepath.Join(repo, "untracked.txt"), []byte("new"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(repo, ".orchestra-handoff-report.md"), []byte("protocol"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ScratchCommit(repo, "orchestra/scratch/test", "checkpoint"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{"staged.txt", "renamed-new.txt", "untracked.txt"} {
|
||||
if err := exec.Command("git", "-C", repo, "cat-file", "-e", "HEAD:"+want).Run(); err != nil {
|
||||
t.Fatalf("checkpoint omitted %s: %v", want, err)
|
||||
}
|
||||
}
|
||||
if err := exec.Command("git", "-C", repo, "cat-file", "-e", "HEAD:deleted.txt").Run(); err == nil {
|
||||
t.Fatal("checkpoint retained deleted file")
|
||||
}
|
||||
if err := exec.Command("git", "-C", repo, "cat-file", "-e", "HEAD:.orchestra-handoff-report.md").Run(); err == nil {
|
||||
t.Fatal("checkpoint committed protocol marker")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandoffCASAndPickup(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
run := func(a ...string) {
|
||||
|
||||
+121
-10
@@ -38,11 +38,69 @@ const (
|
||||
StateBlocked TaskState = "blocked"
|
||||
)
|
||||
|
||||
// BlockReason is the machine-readable diagnosis for a TaskBlocked event.
|
||||
// Blocker remains the operator-facing detail; this field lets projections
|
||||
// group attention without repeatedly parsing prose at read time.
|
||||
type BlockReason string
|
||||
|
||||
const (
|
||||
BlockReasonLeaseFailure BlockReason = "lease_failure"
|
||||
BlockReasonWorkerOffline BlockReason = "worker_offline"
|
||||
BlockReasonLeaseExpired BlockReason = "lease_expired"
|
||||
BlockReasonApproval BlockReason = "approval"
|
||||
BlockReasonHandoffValidation BlockReason = "handoff_validation"
|
||||
BlockReasonOperator BlockReason = "operator_block"
|
||||
BlockReasonSystem BlockReason = "system_error"
|
||||
BlockReasonUnknown BlockReason = "unknown"
|
||||
)
|
||||
|
||||
func (r BlockReason) Valid() bool {
|
||||
switch r {
|
||||
case BlockReasonLeaseFailure, BlockReasonWorkerOffline, BlockReasonLeaseExpired,
|
||||
BlockReasonApproval, BlockReasonHandoffValidation, BlockReasonOperator,
|
||||
BlockReasonSystem, BlockReasonUnknown:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// InferBlockReason supplies a stable category for older events which only
|
||||
// recorded a prose blocker. New producers should send block_reason directly.
|
||||
func InferBlockReason(blocker string) BlockReason {
|
||||
v := strings.ToLower(blocker)
|
||||
switch {
|
||||
case strings.Contains(v, "handoff"):
|
||||
return BlockReasonHandoffValidation
|
||||
case strings.Contains(v, "approval") || strings.Contains(v, "permission"):
|
||||
return BlockReasonApproval
|
||||
case strings.Contains(v, "expired") && strings.Contains(v, "lease"):
|
||||
return BlockReasonLeaseExpired
|
||||
case strings.Contains(v, "worker") && (strings.Contains(v, "offline") || strings.Contains(v, "unreachable")):
|
||||
return BlockReasonWorkerOffline
|
||||
case strings.Contains(v, "lease") || strings.Contains(v, "agent.start") || strings.Contains(v, "pane"):
|
||||
return BlockReasonLeaseFailure
|
||||
default:
|
||||
return BlockReasonSystem
|
||||
}
|
||||
}
|
||||
|
||||
type Estimate struct {
|
||||
Value float64 `json:"value"`
|
||||
Who string `json:"who"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
}
|
||||
|
||||
// SessionEvidence is captured by the machine that owns a pane immediately
|
||||
// before it drops its mapping. It is deliberately observation-only: it never
|
||||
// claims that a pane is still live after the worker has closed it.
|
||||
type SessionEvidence struct {
|
||||
PaneID string `json:"pane_id,omitempty"`
|
||||
HarnessID string `json:"harness_id,omitempty"`
|
||||
PaneState string `json:"pane_state,omitempty"`
|
||||
Source string `json:"source,omitempty"`
|
||||
CapturedAt time.Time `json:"captured_at,omitempty"`
|
||||
CheckedAt time.Time `json:"checked_at,omitempty"`
|
||||
}
|
||||
type Lease struct {
|
||||
HarnessID string `json:"harness_id"`
|
||||
Until time.Time `json:"until"`
|
||||
@@ -62,17 +120,27 @@ type Task struct {
|
||||
// HandoffRef survives the queued interval between TaskReleased and the
|
||||
// next router-owned TaskLeased event; it is the only artifact the worker
|
||||
// may use for local pickup validation.
|
||||
HandoffRef string `json:"handoff_ref,omitempty"`
|
||||
Version int `json:"version"`
|
||||
Title string `json:"title,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
HandoffRef string `json:"handoff_ref,omitempty"`
|
||||
// ReleaseTransaction and ReleaseAnchor bind successor pickup to the exact
|
||||
// durable predecessor checkpoint. They survive queueing and re-lease.
|
||||
ReleaseTransaction string `json:"release_transaction,omitempty"`
|
||||
ReleaseAnchor string `json:"release_anchor,omitempty"`
|
||||
PickupTransaction string `json:"pickup_transaction,omitempty"`
|
||||
PickupLeaseVersion int `json:"pickup_lease_version,omitempty"`
|
||||
Version int `json:"version"`
|
||||
Title string `json:"title,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Acceptance []string `json:"acceptance,omitempty"`
|
||||
QualityGate string `json:"quality_gate,omitempty"`
|
||||
// Block evidence is projected from TaskBlocked so terminal records remain
|
||||
// diagnosable after the live coordinator mapping is gone.
|
||||
Blocker string `json:"blocker,omitempty"`
|
||||
BlockedAt time.Time `json:"blocked_at,omitempty"`
|
||||
LastPaneID string `json:"last_pane_id,omitempty"`
|
||||
LastHarness string `json:"last_harness_id,omitempty"`
|
||||
PaneState string `json:"pane_state,omitempty"` // open, closed, unreachable, unknown
|
||||
Blocker string `json:"blocker,omitempty"`
|
||||
BlockReason BlockReason `json:"block_reason,omitempty"`
|
||||
BlockedAt time.Time `json:"blocked_at,omitempty"`
|
||||
LastPaneID string `json:"last_pane_id,omitempty"`
|
||||
LastHarness string `json:"last_harness_id,omitempty"`
|
||||
PaneState string `json:"pane_state,omitempty"` // open, closed, unreachable, unknown
|
||||
LastSession SessionEvidence `json:"last_session,omitempty"`
|
||||
}
|
||||
|
||||
type Event struct {
|
||||
@@ -112,7 +180,7 @@ func ValidateEvent(e Event) error {
|
||||
if e.SchemaVersion >= 2 && strings.TrimSpace(e.Surface) == "" {
|
||||
return fmt.Errorf("%w: surface required", ErrInvalid)
|
||||
}
|
||||
allowed := map[string]bool{"TaskCreated": true, "TaskLeased": true, "TaskReleased": true, "TaskCompleted": true, "TaskFailed": true, "TaskBlocked": true, "ApprovalRequested": true, "ApprovalGranted": true, "ApprovalDenied": true, "TaskAmended": true, "TaskCorrected": true, "QuotaReported": true, "StandupAdvisory": true}
|
||||
allowed := map[string]bool{"TaskCreated": true, "TaskLeased": true, "TaskLeaseRenewed": true, "TaskReleased": true, "TaskPickupValidated": true, "TaskCompleted": true, "TaskFailed": true, "TaskBlocked": true, "ApprovalRequested": true, "ApprovalGranted": true, "ApprovalDenied": true, "TaskAmended": true, "TaskCorrected": true, "QuotaReported": true, "StandupAdvisory": true}
|
||||
if !allowed[e.Type] {
|
||||
return fmt.Errorf("%w: unknown type %q", ErrInvalid, e.Type)
|
||||
}
|
||||
@@ -160,10 +228,36 @@ func ValidatePayload(typ string, p map[string]any) error {
|
||||
if v, ok := p["expected_version"].(float64); !ok || v < 0 || v != float64(int(v)) {
|
||||
return fmt.Errorf("%w: expected_version invalid", ErrInvalid)
|
||||
}
|
||||
case "TaskLeaseRenewed":
|
||||
if err := requiredString("harness_id"); err != nil {
|
||||
return err
|
||||
}
|
||||
until, ok := p["until_ns"].(float64)
|
||||
if !ok || until <= float64(time.Now().UnixNano()) {
|
||||
return fmt.Errorf("%w: until_ns required", ErrInvalid)
|
||||
}
|
||||
if v, ok := p["expected_version"].(float64); !ok || v < 0 || v != float64(int(v)) {
|
||||
return fmt.Errorf("%w: expected_version invalid", ErrInvalid)
|
||||
}
|
||||
case "TaskReleased":
|
||||
if err := requiredString("handoff_ref"); err != nil && p["reason"] == nil {
|
||||
return err
|
||||
}
|
||||
case "TaskPickupValidated":
|
||||
for _, key := range []string{"transaction_id", "handoff_ref", "anchor_sha", "harness_id"} {
|
||||
if err := requiredString(key); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := requiredHash(p, "handoff_ref"); err != nil {
|
||||
return err
|
||||
}
|
||||
if v, ok := p["anchor_sha"].(string); !ok || len(v) != 40 {
|
||||
return fmt.Errorf("%w: anchor_sha invalid", ErrInvalid)
|
||||
}
|
||||
if v, ok := p["lease_version"].(float64); !ok || v < 1 || v != float64(int(v)) {
|
||||
return fmt.Errorf("%w: lease_version invalid", ErrInvalid)
|
||||
}
|
||||
if _, ok := p["handoff_ref"]; ok {
|
||||
if err := requiredHash(p, "handoff_ref"); err != nil {
|
||||
return err
|
||||
@@ -183,6 +277,17 @@ func ValidatePayload(typ string, p map[string]any) error {
|
||||
if receipt, ok := p["receipt"].(map[string]any); !ok || len(receipt) == 0 {
|
||||
return fmt.Errorf("%w: receipt required", ErrInvalid)
|
||||
}
|
||||
if v, ok := p["result_sha"]; ok {
|
||||
if s, ok := v.(string); !ok || len(s) != 40 {
|
||||
return fmt.Errorf("%w: result_sha invalid", ErrInvalid)
|
||||
}
|
||||
if err := requiredString("branch"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := requiredString("remote"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
case "TaskFailed":
|
||||
if err := requiredString("reason"); err != nil {
|
||||
return err
|
||||
@@ -191,6 +296,12 @@ func ValidatePayload(typ string, p map[string]any) error {
|
||||
if err := requiredString("blocker"); err != nil {
|
||||
return err
|
||||
}
|
||||
if v, ok := p["block_reason"]; ok {
|
||||
s, ok := v.(string)
|
||||
if !ok || !BlockReason(s).Valid() {
|
||||
return fmt.Errorf("%w: block_reason invalid", ErrInvalid)
|
||||
}
|
||||
}
|
||||
if _, ok := p["handoff_ref"]; ok {
|
||||
if err := requiredHash(p, "handoff_ref"); err != nil {
|
||||
return err
|
||||
|
||||
@@ -24,7 +24,7 @@ type Client struct {
|
||||
}
|
||||
|
||||
func (c Client) Register(ctx context.Context, w Worker) error {
|
||||
b, err := json.Marshal(map[string]any{"id": w.ID, "address": w.Address, "capacity": w.Capacity, "token": c.Token})
|
||||
b, err := json.Marshal(map[string]any{"id": w.ID, "address": w.Address, "capacity": w.Capacity, "supported_projects": w.SupportedProjects, "build": w.Build, "token": c.Token})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -131,6 +131,13 @@ func (c Client) Heartbeat(ctx context.Context, health WorkerHealth) error {
|
||||
}
|
||||
return err
|
||||
}
|
||||
func (c Client) Renew(ctx context.Context, taskID string, expectedVersion, ttlSeconds int) error {
|
||||
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/renew", map[string]any{"task_id": taskID, "expected_version": expectedVersion, "ttl_seconds": ttlSeconds})
|
||||
if resp != nil {
|
||||
resp.Body.Close()
|
||||
}
|
||||
return err
|
||||
}
|
||||
func (c Client) Artifact(ctx context.Context, ref string) ([]byte, error) {
|
||||
resp, err := c.request(ctx, http.MethodGet, "/v1/artifacts/"+url.PathEscape(ref), nil)
|
||||
if err != nil {
|
||||
@@ -169,15 +176,22 @@ func (c Client) PutArtifact(ctx context.Context, b []byte) (string, error) {
|
||||
}
|
||||
return out.Ref, nil
|
||||
}
|
||||
func (c Client) Release(ctx context.Context, taskID, ref, anchor string) error {
|
||||
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/handoff", map[string]string{"task_id": taskID, "handoff_ref": ref, "anchor_sha": anchor})
|
||||
func (c Client) Release(ctx context.Context, taskID, ref, anchor, transactionID string, expectedVersion int, evidence domain.SessionEvidence) error {
|
||||
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/handoff", map[string]any{"task_id": taskID, "handoff_ref": ref, "anchor_sha": anchor, "transaction_id": transactionID, "expected_version": expectedVersion, "session_evidence": evidence})
|
||||
if resp != nil {
|
||||
resp.Body.Close()
|
||||
}
|
||||
return err
|
||||
}
|
||||
func (c Client) Complete(ctx context.Context, taskID, reportRef string) error {
|
||||
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/complete", map[string]string{"task_id": taskID, "handoff_ref": reportRef})
|
||||
func (c Client) Pickup(ctx context.Context, taskID, ref, anchor, transactionID string, leaseVersion int, evidence domain.SessionEvidence) error {
|
||||
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/pickup", map[string]any{"task_id": taskID, "handoff_ref": ref, "anchor_sha": anchor, "transaction_id": transactionID, "lease_version": leaseVersion, "session_evidence": evidence})
|
||||
if resp != nil {
|
||||
resp.Body.Close()
|
||||
}
|
||||
return err
|
||||
}
|
||||
func (c Client) Complete(ctx context.Context, taskID, reportRef, resultSHA, branch, remote string, expectedVersion int, receipt map[string]any, evidence domain.SessionEvidence) error {
|
||||
resp, err := c.request(ctx, http.MethodPost, "/v1/federation/workers/"+url.PathEscape(c.WorkerID)+"/complete", map[string]any{"task_id": taskID, "handoff_ref": reportRef, "result_sha": resultSHA, "branch": branch, "remote": remote, "expected_version": expectedVersion, "receipt": receipt, "session_evidence": evidence})
|
||||
if resp != nil {
|
||||
resp.Body.Close()
|
||||
}
|
||||
|
||||
+130
-68
@@ -248,80 +248,99 @@ func (a CLIAdapter) NotifyConventionsChanged(ctx context.Context, s Session) err
|
||||
// handoff is refused rather than guessed at: the caller (Coordinator.rotate)
|
||||
// leaves the lease intact and retries next tick, giving the agent time to
|
||||
// finish writing it.
|
||||
func (a CLIAdapter) Release(ctx context.Context, s Session) (string, error) {
|
||||
// PreparedRelease is the durable, coordinator-independent half of a release.
|
||||
// The worker persists it before publishing TaskReleased so a lost HTTP reply
|
||||
// never requires reconstructing (or deleting) the agent's report.
|
||||
type PreparedRelease struct {
|
||||
Ref string
|
||||
AnchorSHA string
|
||||
}
|
||||
|
||||
// PrepareRelease seals an immutable Git checkpoint and uploads its canonical
|
||||
// handoff, but deliberately leaves both the pane claim and report in place.
|
||||
// The caller controls the retryable transaction around coordinator acceptance.
|
||||
func (a CLIAdapter) PrepareRelease(ctx context.Context, s Session) (PreparedRelease, error) {
|
||||
if a.CAS == nil {
|
||||
return "", fmt.Errorf("adapter: CAS store required to upload handoff")
|
||||
return PreparedRelease{}, fmt.Errorf("adapter: CAS store required to upload handoff")
|
||||
}
|
||||
// A federation worker always supplies the immutable task hash and remote.
|
||||
// The empty-hash case is retained solely for old in-process adapter users;
|
||||
// it is not reachable from the worker release path.
|
||||
if s.TaskFileSHA != "" {
|
||||
if a.Remote == "" {
|
||||
return PreparedRelease{}, fmt.Errorf("adapter: project remote required for checkpoint")
|
||||
}
|
||||
if err := continuity.VerifyTaskFile(s.Worktree, s.TaskFileSHA); err != nil {
|
||||
return PreparedRelease{}, fmt.Errorf("adapter: verify immutable TASK.md: %w", err)
|
||||
}
|
||||
}
|
||||
path := filepath.Join(s.Worktree, HandoffReportFile)
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("adapter: semantic handoff report not written yet (%s): %w", path, err)
|
||||
return PreparedRelease{}, fmt.Errorf("adapter: semantic handoff report not written yet (%s): %w", path, err)
|
||||
}
|
||||
if strings.TrimSpace(string(b)) == "" {
|
||||
return "", fmt.Errorf("adapter: semantic handoff report is empty")
|
||||
return PreparedRelease{}, fmt.Errorf("adapter: semantic handoff report is empty")
|
||||
}
|
||||
h, err := canonicalHandoff(s, string(b), a.lastObservedCommand(s))
|
||||
if err != nil {
|
||||
return "", err
|
||||
return PreparedRelease{}, err
|
||||
}
|
||||
sha, err := HeadSHA(s.Worktree)
|
||||
// Always checkpoint and push, including already-committed clean work. Git
|
||||
// is the cross-machine transport, so merely observing a local clean HEAD is
|
||||
// not a sufficient anchor.
|
||||
branch := "orchestra/scratch/" + h.Meta.ID
|
||||
if err := continuity.ScratchCommit(s.Worktree, branch, "orchestra: pre-release WIP snapshot ("+h.Meta.ID+")"); err != nil {
|
||||
return PreparedRelease{}, fmt.Errorf("adapter: scratch commit: %w", err)
|
||||
}
|
||||
anchor, err := HeadSHA(s.Worktree)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("adapter: read worktree HEAD: %w", err)
|
||||
return PreparedRelease{}, fmt.Errorf("adapter: read checkpoint HEAD: %w", err)
|
||||
}
|
||||
_ = sha
|
||||
for _, d := range h.Anchor.Dirty {
|
||||
if hex.EncodeToString(sha256sum(filepath.Join(s.Worktree, d.Path))) != d.SHA256 {
|
||||
return "", fmt.Errorf("adapter: handoff dirty file changed since it was written: %s", d.Path)
|
||||
if a.Remote != "" {
|
||||
if err := continuity.ScratchPush(s.Worktree, branch, a.Remote); err != nil {
|
||||
return PreparedRelease{}, fmt.Errorf("adapter: push checkpoint: %w", err)
|
||||
}
|
||||
out, err := exec.CommandContext(ctx, "git", "-C", s.Worktree, "ls-remote", a.Remote, "refs/heads/"+branch).Output()
|
||||
if err != nil || !strings.HasPrefix(string(out), anchor+"\t") {
|
||||
return PreparedRelease{}, fmt.Errorf("adapter: verify pushed anchor: got %q: %w", strings.TrimSpace(string(out)), err)
|
||||
}
|
||||
}
|
||||
// The semantic report is transferred in the CAS handoff, not in the
|
||||
// scratch checkout. Keeping it in the scratch commit makes a successor
|
||||
// mistake the predecessor's report for a newly requested handoff and can
|
||||
// cause an immediate release/pickup loop.
|
||||
dirty := h.Anchor.Dirty[:0]
|
||||
for _, d := range h.Anchor.Dirty {
|
||||
if filepath.Clean(d.Path) != HandoffReportFile {
|
||||
dirty = append(dirty, d)
|
||||
}
|
||||
}
|
||||
h.Anchor.Dirty = dirty
|
||||
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
|
||||
return "", fmt.Errorf("adapter: remove transferred semantic report: %w", err)
|
||||
}
|
||||
// Atomically commit whatever the handoff described as dirty onto a
|
||||
// per-task scratch branch (§6.2 step 3) *before* uploading, so the
|
||||
// successor's pickup validation collapses to a single HEAD compare
|
||||
// instead of re-hashing every dirty file individually.
|
||||
if len(h.Anchor.Dirty) > 0 {
|
||||
branch := "orchestra/scratch/" + h.Meta.ID
|
||||
if err := continuity.ScratchCommit(s.Worktree, branch, "orchestra: pre-release WIP snapshot ("+h.Meta.ID+")"); err != nil {
|
||||
return "", fmt.Errorf("adapter: scratch commit: %w", err)
|
||||
}
|
||||
if a.Remote != "" {
|
||||
if err := continuity.ScratchPush(s.Worktree, branch, a.Remote); err != nil {
|
||||
return "", fmt.Errorf("adapter: push scratch branch: %w", err)
|
||||
}
|
||||
}
|
||||
newSHA, err := HeadSHA(s.Worktree)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("adapter: read scratch HEAD: %w", err)
|
||||
}
|
||||
h.Anchor.GitSHA = newSHA
|
||||
h.Anchor.Branch = branch
|
||||
h.Anchor.Dirty = nil
|
||||
}
|
||||
h.Anchor.GitSHA, h.Anchor.Branch, h.Anchor.Dirty = anchor, branch, nil
|
||||
ref, err := continuity.Save(h, a.CAS)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("adapter: upload handoff: %w", err)
|
||||
return PreparedRelease{}, fmt.Errorf("adapter: upload handoff: %w", err)
|
||||
}
|
||||
return PreparedRelease{Ref: ref, AnchorSHA: anchor}, nil
|
||||
}
|
||||
|
||||
// ReleaseAgent drops only herdr's harness binding. It does not close the pane:
|
||||
// a predecessor stays recoverable until the successor has validated pickup.
|
||||
func (a CLIAdapter) ReleaseAgent(ctx context.Context, s Session) error {
|
||||
if err := a.Client.Call(ctx, "pane.release_agent", map[string]any{
|
||||
"pane_id": s.PaneID,
|
||||
"source": "herdr:" + a.Harness,
|
||||
"agent": agentForSession(s, a.Harness),
|
||||
}, nil); err != nil {
|
||||
return "", fmt.Errorf("adapter: pane.release_agent: %w", err)
|
||||
return fmt.Errorf("adapter: pane.release_agent: %w", err)
|
||||
}
|
||||
return ref, nil
|
||||
return nil
|
||||
}
|
||||
|
||||
// Release is retained for the coordinator's legacy local path. Federation
|
||||
// workers use PrepareRelease and ReleaseAgent as separate durable phases.
|
||||
func (a CLIAdapter) Release(ctx context.Context, s Session) (string, error) {
|
||||
p, err := a.PrepareRelease(ctx, s)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := a.ReleaseAgent(ctx, s); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := os.Remove(filepath.Join(s.Worktree, HandoffReportFile)); err != nil && !os.IsNotExist(err) {
|
||||
return "", fmt.Errorf("adapter: remove transferred semantic report: %w", err)
|
||||
}
|
||||
return p.Ref, nil
|
||||
}
|
||||
|
||||
// canonicalHandoff keeps Git-derived protocol facts on the worker that owns
|
||||
@@ -483,16 +502,27 @@ func handoffID(s Session) string {
|
||||
}
|
||||
|
||||
func dirtyFiles(root string) ([]continuity.Dirty, error) {
|
||||
out, err := exec.Command("git", "-C", root, "status", "--porcelain=v1", "-z").Output()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
paths := map[string]bool{}
|
||||
for _, args := range [][]string{{"diff", "--name-only", "-z"}, {"ls-files", "--others", "--exclude-standard", "-z"}} {
|
||||
out, err := exec.Command("git", append([]string{"-C", root}, args...)...).Output()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
deleted := map[string]bool{}
|
||||
parts := strings.Split(string(out), "\x00")
|
||||
for i := 0; i < len(parts); i++ {
|
||||
record := parts[i]
|
||||
if len(record) < 4 {
|
||||
continue
|
||||
}
|
||||
for _, path := range strings.Split(string(out), "\x00") {
|
||||
if path != "" && path != HandoffFile {
|
||||
paths[path] = true
|
||||
}
|
||||
status, path := record[:2], record[3:]
|
||||
if path == HandoffFile || path == HandoffReportFile || path == ".orchestra/done" || strings.HasPrefix(path, ".orchestra/") {
|
||||
continue
|
||||
}
|
||||
paths[path] = true
|
||||
deleted[path] = strings.Contains(status, "D")
|
||||
// A rename/copy record has the original path as the next NUL item.
|
||||
if status[0] == 'R' || status[0] == 'C' || status[1] == 'R' || status[1] == 'C' {
|
||||
i++
|
||||
}
|
||||
}
|
||||
keys := make([]string, 0, len(paths))
|
||||
@@ -502,11 +532,15 @@ func dirtyFiles(root string) ([]continuity.Dirty, error) {
|
||||
sort.Strings(keys)
|
||||
dirty := make([]continuity.Dirty, 0, len(keys))
|
||||
for _, path := range keys {
|
||||
sum := sha256sum(filepath.Join(root, path))
|
||||
if len(sum) == 0 {
|
||||
return nil, fmt.Errorf("adapter: hash dirty file %s", path)
|
||||
d := continuity.Dirty{Path: path, Deleted: deleted[path]}
|
||||
if !d.Deleted {
|
||||
sum := sha256sum(filepath.Join(root, path))
|
||||
if len(sum) == 0 {
|
||||
return nil, fmt.Errorf("adapter: hash dirty file %s", path)
|
||||
}
|
||||
d.SHA256 = hex.EncodeToString(sum)
|
||||
}
|
||||
dirty = append(dirty, continuity.Dirty{Path: path, SHA256: hex.EncodeToString(sum)})
|
||||
dirty = append(dirty, d)
|
||||
}
|
||||
return dirty, nil
|
||||
}
|
||||
@@ -641,6 +675,13 @@ var _ = json.RawMessage{}
|
||||
// callers (Coordinator.rotate, refreshSessionHealth) surface it instead of
|
||||
// mistaking "we don't know" for "occupancy is zero".
|
||||
func (a CLIAdapter) Occupancy(s Session) (float64, error) {
|
||||
if a.Harness == "opencode" {
|
||||
u, err := OpenCodeSessionUsage(s.SessionID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return Fraction(u, a.Window), nil
|
||||
}
|
||||
if a.Usage == nil {
|
||||
return 0, fmt.Errorf("adapter: usage reader required")
|
||||
}
|
||||
@@ -659,6 +700,32 @@ func (a CLIAdapter) Occupancy(s Session) (float64, error) {
|
||||
return Fraction(u, a.Window), nil
|
||||
}
|
||||
|
||||
// ResolveSessionIdentity discovers and returns the harness-native session
|
||||
// identity. Callers persist the returned Session before relying on occupancy,
|
||||
// so restart recovery keeps observing the same harness session.
|
||||
func (a CLIAdapter) ResolveSessionIdentity(s Session) (Session, error) {
|
||||
if a.Harness == "opencode" {
|
||||
if s.SessionID != "" {
|
||||
return s, nil
|
||||
}
|
||||
id, err := OpenCodeSessionID(s.Worktree)
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
s.SessionID = id
|
||||
return s, nil
|
||||
}
|
||||
if s.SessionFile != "" {
|
||||
return s, nil
|
||||
}
|
||||
path, err := a.resolveSessionFile(s)
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
s.SessionFile = path
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (a CLIAdapter) resolveSessionFile(s Session) (string, error) {
|
||||
switch a.Harness {
|
||||
case "claude":
|
||||
@@ -667,12 +734,7 @@ func (a CLIAdapter) resolveSessionFile(s Session) (string, error) {
|
||||
_, path, err := CodexActiveUsage("")
|
||||
return path, err
|
||||
default:
|
||||
// opencode's session-file resolution needs the running session id,
|
||||
// which is only available via the SSE/status API (OpenCodeStatus),
|
||||
// not derivable from the worktree alone. Per AUDIT.md Phase 1, wiring
|
||||
// this needs verification against a live opencode instance before it
|
||||
// can drive rotation — refuse loudly rather than guess a path.
|
||||
return "", fmt.Errorf("adapter: harness %q has no session-file resolver; verify against a live session first (AUDIT.md Phase 1)", a.Harness)
|
||||
return "", fmt.Errorf("adapter: harness %q has no session-file resolver", a.Harness)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -152,6 +152,10 @@ type Session struct {
|
||||
// CLIAdapter.Occupancy), since the file may not exist yet immediately
|
||||
// after lease.
|
||||
SessionFile string `json:"session_file,omitempty"`
|
||||
// SessionID is the harness-native identity when its usage is stored in a
|
||||
// database rather than a transcript. OpenCode's SQLite session ID is kept
|
||||
// here so rotation never guesses "the newest session" after a restart.
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
// TaskFileSHA is the sha256 of the worktree's TASK.md at the time this
|
||||
// session's lease was created — the immutable-spec hash continuity's
|
||||
// pickup validation compares against on the next rotation (§6.2).
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
@@ -30,6 +31,7 @@ func Fraction(u Usage, w int64) float64 {
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// ClaudeSessionFile resolves the transcript file for a Claude Code session
|
||||
// running against worktree, by newest-mtime under Claude Code's encoded
|
||||
// project directory (~/.claude/projects/<abs-worktree-path-with-/-replaced-
|
||||
@@ -202,6 +204,60 @@ func OpenCodeUsage(p string) (Usage, error) {
|
||||
return Usage{x.Tokens.Input, x.Tokens.Cache.Read, x.Tokens.Cache.Write, x.Tokens.Output}, e
|
||||
}
|
||||
|
||||
// OpenCodeSessionID resolves the exact OpenCode session associated with a
|
||||
// checkout. OpenCode stores usage in its SQLite session table, not in a pane
|
||||
// transcript. Selecting by directory and persisting the returned id prevents
|
||||
// a multi-pane worker from attributing another task's newest session to this
|
||||
// lease.
|
||||
func OpenCodeSessionID(worktree string) (string, error) {
|
||||
db := os.Getenv("ORCHESTRA_OPENCODE_DB")
|
||||
if db == "" {
|
||||
db = filepath.Join(os.Getenv("HOME"), ".local", "share", "opencode", "opencode.db")
|
||||
}
|
||||
abs, err := filepath.Abs(worktree)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
out, err := exec.Command("sqlite3", "-readonly", "-noheader", db, "select id from session where directory = "+sqliteQuote(abs)+" order by time_updated desc limit 1;").Output()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("opencode session lookup: %w", err)
|
||||
}
|
||||
id := strings.TrimSpace(string(out))
|
||||
if id == "" {
|
||||
return "", fmt.Errorf("opencode session lookup: no session for %s", abs)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// OpenCodeSessionUsage reads the token counters for one persisted session.
|
||||
func OpenCodeSessionUsage(sessionID string) (Usage, error) {
|
||||
if sessionID == "" {
|
||||
return Usage{}, fmt.Errorf("opencode session id required")
|
||||
}
|
||||
db := os.Getenv("ORCHESTRA_OPENCODE_DB")
|
||||
if db == "" {
|
||||
db = filepath.Join(os.Getenv("HOME"), ".local", "share", "opencode", "opencode.db")
|
||||
}
|
||||
out, err := exec.Command("sqlite3", "-readonly", "-noheader", "-separator", "|", db, "select tokens_input,tokens_cache_read,tokens_cache_write,tokens_output from session where id = "+sqliteQuote(sessionID)+";").Output()
|
||||
if err != nil {
|
||||
return Usage{}, fmt.Errorf("opencode usage lookup: %w", err)
|
||||
}
|
||||
parts := strings.Split(strings.TrimSpace(string(out)), "|")
|
||||
if len(parts) != 4 {
|
||||
return Usage{}, fmt.Errorf("opencode usage lookup: unknown session %q", sessionID)
|
||||
}
|
||||
values := [4]int64{}
|
||||
for i, part := range parts {
|
||||
values[i], err = strconv.ParseInt(part, 10, 64)
|
||||
if err != nil {
|
||||
return Usage{}, fmt.Errorf("opencode usage lookup: %w", err)
|
||||
}
|
||||
}
|
||||
return Usage{Input: values[0], CacheRead: values[1], CacheWrite: values[2], Output: values[3]}, nil
|
||||
}
|
||||
|
||||
func sqliteQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", "''") + "'" }
|
||||
|
||||
// OpenCodeStatus probes the server fast path. Callers can use the returned
|
||||
// status and fall back to OpenCodeUsage when the SSE/server is unavailable.
|
||||
func OpenCodeStatus(ctx context.Context, baseURL, sessionID string) (string, error) {
|
||||
|
||||
@@ -774,18 +774,21 @@ func (c *Coordinator) TurnDecision(ctx context.Context, taskID string) (string,
|
||||
if existingReason := handoffReason(session.Worktree); existingReason == "manual" || existingReason == "milestone" || existingReason == "thrash" {
|
||||
return c.finishRelease(ctx, taskID, task, session, a, existingReason)
|
||||
}
|
||||
if trigger, deadEnds := checkActivityTriggers(ctx, a, session, c.Thrash); trigger != "" {
|
||||
c.requestReasonedHandoff(ctx, taskID, session, a, trigger, deadEnds)
|
||||
return TurnPrepareHandoff, nil
|
||||
d := (RotationStateMachine{Soft: c.soft(), Hard: c.Hard, Thrash: c.Thrash}).Evaluate(ctx, a, session)
|
||||
if d.Degraded != nil {
|
||||
return "", fmt.Errorf("orchestrator: rotation: %w", d.Degraded)
|
||||
}
|
||||
occupancy, err := a.Occupancy(session)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("orchestrator: occupancy: %w", err)
|
||||
}
|
||||
if occupancy < c.soft() {
|
||||
if d.Action == TurnContinue {
|
||||
return TurnContinue, nil
|
||||
}
|
||||
if occupancy < c.Hard {
|
||||
if d.Action == TurnRefuse {
|
||||
return TurnRefuse, nil
|
||||
}
|
||||
if d.Reason == "milestone" || d.Reason == "thrash" {
|
||||
c.requestReasonedHandoff(ctx, taskID, session, a, d.Reason, d.DeadEnds)
|
||||
return TurnPrepareHandoff, nil
|
||||
}
|
||||
if d.Action == TurnPrepareHandoff {
|
||||
// Soft threshold (§5.3): advisory only. Ask the agent to start
|
||||
// preparing a handoff well before Hard forces one, but don't block
|
||||
// the turn on a boundary check — the agent is free to keep working.
|
||||
@@ -805,18 +808,6 @@ func (c *Coordinator) TurnDecision(ctx context.Context, taskID string) (string,
|
||||
}
|
||||
return TurnPrepareHandoff, nil
|
||||
}
|
||||
if boundary, ok := a.(herdr.TurnBoundary); ok {
|
||||
atBoundary, boundaryErr := boundary.AtTurnBoundary(ctx, session)
|
||||
if boundaryErr != nil {
|
||||
c.recordTurnBoundaryDegraded()
|
||||
return TurnRefuse, nil
|
||||
}
|
||||
if !atBoundary {
|
||||
return TurnRefuse, nil
|
||||
}
|
||||
} else {
|
||||
c.recordTurnBoundaryDegraded()
|
||||
}
|
||||
if requester, ok := a.(herdr.HandoffRequester); ok {
|
||||
if _, statErr := os.Stat(filepath.Join(session.Worktree, herdr.HandoffReportFile)); statErr != nil {
|
||||
if !session.HandoffRequested {
|
||||
@@ -979,11 +970,12 @@ func (c *Coordinator) rememberSession(taskID string, s herdr.Session) error {
|
||||
}
|
||||
|
||||
func (c *Coordinator) block(t domain.Task, reason string) error {
|
||||
p := map[string]string{"blocker": reason, "pane_state": "unknown"}
|
||||
p := map[string]any{"blocker": reason, "block_reason": string(domain.InferBlockReason(reason)), "pane_state": "unknown", "session_evidence": domain.SessionEvidence{PaneState: "unknown", Source: "coordinator", CheckedAt: time.Now().UTC()}}
|
||||
if s, ok := c.Session(t.ID); ok {
|
||||
p["pane_id"] = s.PaneID
|
||||
p["harness_id"] = s.HerdrID
|
||||
p["pane_state"] = "open"
|
||||
p["session_evidence"] = domain.SessionEvidence{PaneID: s.PaneID, HarnessID: s.HerdrID, PaneState: "open", Source: "coordinator", CheckedAt: time.Now().UTC()}
|
||||
}
|
||||
b, _ := json.Marshal(p)
|
||||
return c.Store.Append(domain.Event{ID: domain.NewID(), Type: "TaskBlocked", TaskID: t.ID, Version: t.Version + 1, Payload: b, Surface: string(authz.System)})
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
package orchestrator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"orchestra/internal/continuity"
|
||||
"orchestra/internal/herdr"
|
||||
)
|
||||
|
||||
// RotationStateMachine is the shared, side-effect-free rotation policy used
|
||||
// by both the coordinator's synchronous turn path and federation workers.
|
||||
// Callers persist request/release side effects themselves, but must never
|
||||
// replace an unavailable occupancy reading with zero.
|
||||
type RotationStateMachine struct {
|
||||
Soft float64
|
||||
Hard float64
|
||||
Thrash herdr.ThrashConfig
|
||||
}
|
||||
|
||||
type RotationDecision struct {
|
||||
Action string // continue, prepare_handoff, rotate_now
|
||||
Reason string
|
||||
DeadEnds []continuity.DeadEnd
|
||||
// ActivityDegraded is advisory (threshold rotation still has a real usage
|
||||
// source); it is surfaced so a missing milestone/thrash feed cannot be a
|
||||
// silent no-op.
|
||||
ActivityDegraded error
|
||||
Degraded error
|
||||
}
|
||||
|
||||
func (m RotationStateMachine) Evaluate(ctx context.Context, a herdr.Adapter, s herdr.Session) RotationDecision {
|
||||
soft := m.Soft
|
||||
if soft <= 0 {
|
||||
soft = defaultSoft
|
||||
}
|
||||
if m.Hard <= 0 || m.Hard <= soft {
|
||||
return RotationDecision{Degraded: fmt.Errorf("invalid rotation thresholds soft=%v hard=%v", soft, m.Hard)}
|
||||
}
|
||||
if reader, ok := a.(herdr.ActivityReader); ok {
|
||||
calls, err := reader.Activity(ctx, s)
|
||||
if err == nil {
|
||||
if thrash, deadEnds := herdr.DetectThrash(calls, m.Thrash); thrash {
|
||||
return RotationDecision{Action: TurnPrepareHandoff, Reason: "thrash", DeadEnds: deadEnds}
|
||||
}
|
||||
if herdr.DetectMilestone(calls) {
|
||||
return RotationDecision{Action: TurnPrepareHandoff, Reason: "milestone"}
|
||||
}
|
||||
} else {
|
||||
return m.evaluateOccupancy(ctx, a, s, fmt.Errorf("activity unknown: %w", err))
|
||||
}
|
||||
}
|
||||
return m.evaluateOccupancy(ctx, a, s, nil)
|
||||
}
|
||||
|
||||
func (m RotationStateMachine) evaluateOccupancy(ctx context.Context, a herdr.Adapter, s herdr.Session, activityErr error) RotationDecision {
|
||||
soft := m.Soft
|
||||
if soft <= 0 {
|
||||
soft = defaultSoft
|
||||
}
|
||||
occupancy, err := a.Occupancy(s)
|
||||
if err != nil {
|
||||
return RotationDecision{ActivityDegraded: activityErr, Degraded: fmt.Errorf("occupancy unknown: %w", err)}
|
||||
}
|
||||
if occupancy < soft {
|
||||
return RotationDecision{Action: TurnContinue, ActivityDegraded: activityErr}
|
||||
}
|
||||
if occupancy < m.Hard {
|
||||
return RotationDecision{Action: TurnPrepareHandoff, Reason: "threshold", ActivityDegraded: activityErr}
|
||||
}
|
||||
boundary, ok := a.(herdr.TurnBoundary)
|
||||
if !ok {
|
||||
return RotationDecision{ActivityDegraded: activityErr, Degraded: fmt.Errorf("turn boundary unknown at hard threshold"), Action: TurnRefuse, Reason: "threshold"}
|
||||
}
|
||||
atBoundary, err := boundary.AtTurnBoundary(ctx, s)
|
||||
if err != nil {
|
||||
return RotationDecision{ActivityDegraded: activityErr, Degraded: fmt.Errorf("turn boundary unknown: %w", err), Action: TurnRefuse, Reason: "threshold"}
|
||||
}
|
||||
if !atBoundary {
|
||||
return RotationDecision{Action: TurnRefuse, Reason: "threshold", ActivityDegraded: activityErr}
|
||||
}
|
||||
return RotationDecision{Action: TurnRotateNow, Reason: "threshold", ActivityDegraded: activityErr}
|
||||
}
|
||||
@@ -708,6 +708,7 @@ type activityAdapter struct {
|
||||
activityErr error
|
||||
reasonAsked []string
|
||||
deadEndsSeen []continuity.DeadEnd
|
||||
observed chan struct{}
|
||||
}
|
||||
|
||||
func (a *activityAdapter) Activity(context.Context, herdr.Session) ([]herdr.ToolCall, error) {
|
||||
@@ -717,6 +718,12 @@ func (a *activityAdapter) Activity(context.Context, herdr.Session) ([]herdr.Tool
|
||||
func (a *activityAdapter) RequestHandoffReason(_ context.Context, _ herdr.Session, reason string, deadEnds []continuity.DeadEnd) error {
|
||||
a.reasonAsked = append(a.reasonAsked, reason)
|
||||
a.deadEndsSeen = deadEnds
|
||||
if a.observed != nil {
|
||||
select {
|
||||
case a.observed <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -764,7 +771,7 @@ func TestActivityTriggersRequestReasonedHandoffWithoutReleasing(t *testing.T) {
|
||||
}
|
||||
|
||||
t.Run("thrash requests a reasoned handoff and does not release, via TurnDecision", func(t *testing.T) {
|
||||
a := &activityAdapter{fakeAdapter: fakeAdapter{occupancy: 0, boundary: true}, calls: thrashCalls}
|
||||
a := &activityAdapter{fakeAdapter: fakeAdapter{occupancy: 0, boundary: true}, calls: thrashCalls, observed: make(chan struct{}, 1)}
|
||||
c, st, task := newCoordinator(a)
|
||||
decision, err := c.TurnDecision(context.Background(), task.ID)
|
||||
if err != nil {
|
||||
@@ -862,13 +869,15 @@ func TestActivityTriggersRequestReasonedHandoffWithoutReleasing(t *testing.T) {
|
||||
c, st, task := newCoordinator(a)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go c.Monitor(ctx, .8, time.Millisecond)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- c.Monitor(ctx, .8, time.Millisecond) }()
|
||||
|
||||
deadline := time.Now().Add(300 * time.Millisecond)
|
||||
for time.Now().Before(deadline) && len(a.reasonAsked) == 0 {
|
||||
time.Sleep(time.Millisecond)
|
||||
select {
|
||||
case <-a.observed:
|
||||
case <-time.After(300 * time.Millisecond):
|
||||
}
|
||||
cancel()
|
||||
<-done
|
||||
if len(a.reasonAsked) == 0 || a.reasonAsked[0] != "thrash" {
|
||||
t.Fatalf("reasonAsked=%v want a thrash request from rotate()", a.reasonAsked)
|
||||
}
|
||||
@@ -959,10 +968,17 @@ func (w specWorktrees) Spec(domain.Task) (string, string, bool) { re
|
||||
type conventionsAdapter struct {
|
||||
fakeAdapter
|
||||
notifications int
|
||||
observed chan struct{}
|
||||
}
|
||||
|
||||
func (a *conventionsAdapter) NotifyConventionsChanged(context.Context, herdr.Session) error {
|
||||
a.notifications++
|
||||
if a.observed != nil {
|
||||
select {
|
||||
case a.observed <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -991,7 +1007,7 @@ func TestConventionsDriftNotifiesActiveSession(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := s.Tasks()[0]
|
||||
a := &conventionsAdapter{fakeAdapter: fakeAdapter{occupancy: 0}}
|
||||
a := &conventionsAdapter{fakeAdapter: fakeAdapter{occupancy: 0}, observed: make(chan struct{}, 1)}
|
||||
c := &orchestrator.Coordinator{Store: s, Worktrees: specWorktrees{wtPath: worktree, repoPath: repo}, Adapters: adapters{a}, StatePath: t.TempDir() + "/sessions.json"}
|
||||
|
||||
leaseEvt, err := s.Lease(task.ID, "h1", time.Minute)
|
||||
@@ -1003,8 +1019,8 @@ func TestConventionsDriftNotifiesActiveSession(t *testing.T) {
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go c.Monitor(ctx, .8, time.Millisecond)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- c.Monitor(ctx, .8, time.Millisecond) }()
|
||||
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if a.notifications != 0 {
|
||||
@@ -1015,10 +1031,12 @@ func TestConventionsDriftNotifiesActiveSession(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for time.Now().Before(deadline) && a.notifications == 0 {
|
||||
time.Sleep(time.Millisecond)
|
||||
select {
|
||||
case <-a.observed:
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
cancel()
|
||||
<-done
|
||||
if a.notifications == 0 {
|
||||
t.Fatal("session was never notified of the conventions-doc update")
|
||||
}
|
||||
|
||||
@@ -138,14 +138,34 @@ func (s *Store) apply(e domain.Event) error {
|
||||
if v, ok := p["description"].(string); ok {
|
||||
t.Description = v
|
||||
}
|
||||
if v, ok := p["acceptance"].([]any); ok {
|
||||
for _, item := range v {
|
||||
if text, ok := item.(string); ok {
|
||||
t.Acceptance = append(t.Acceptance, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
if v, ok := p["quality_gate"].(string); ok {
|
||||
t.QualityGate = v
|
||||
}
|
||||
s.external[t.Source+"\x00"+t.ExternalID] = t.ID
|
||||
case "TaskLeased":
|
||||
t.State = domain.StateLeased
|
||||
t.Lease = &domain.Lease{HarnessID: p["harness_id"].(string), Until: time.Unix(0, int64(p["until_ns"].(float64)))}
|
||||
case "TaskLeaseRenewed":
|
||||
t.Lease = &domain.Lease{HarnessID: p["harness_id"].(string), Until: time.Unix(0, int64(p["until_ns"].(float64)))}
|
||||
case "TaskReleased":
|
||||
t.State = domain.StateQueued
|
||||
t.Lease = nil
|
||||
t.HandoffRef, _ = p["handoff_ref"].(string)
|
||||
t.ReleaseTransaction, _ = p["transaction_id"].(string)
|
||||
t.ReleaseAnchor, _ = p["anchor_sha"].(string)
|
||||
t.PickupTransaction, t.PickupLeaseVersion = "", 0
|
||||
case "TaskPickupValidated":
|
||||
t.PickupTransaction, _ = p["transaction_id"].(string)
|
||||
if v, ok := p["lease_version"].(float64); ok {
|
||||
t.PickupLeaseVersion = int(v)
|
||||
}
|
||||
case "TaskCompleted":
|
||||
t.State = domain.StateCompleted
|
||||
t.Lease = nil
|
||||
@@ -156,6 +176,10 @@ func (s *Store) apply(e domain.Event) error {
|
||||
t.State = domain.StateBlocked
|
||||
t.Lease = nil
|
||||
t.Blocker, _ = p["blocker"].(string)
|
||||
t.BlockReason = domain.InferBlockReason(t.Blocker)
|
||||
if v, ok := p["block_reason"].(string); ok && domain.BlockReason(v).Valid() {
|
||||
t.BlockReason = domain.BlockReason(v)
|
||||
}
|
||||
t.BlockedAt = e.At
|
||||
t.LastPaneID, _ = p["pane_id"].(string)
|
||||
t.LastHarness, _ = p["harness_id"].(string)
|
||||
@@ -200,6 +224,25 @@ func (s *Store) apply(e domain.Event) error {
|
||||
}
|
||||
}
|
||||
}
|
||||
// Terminal and release events may carry an owner-produced snapshot from
|
||||
// immediately before a worker/coordinator drops its live session mapping.
|
||||
// Preserve it independently of the current task state so historical task
|
||||
// pages never have to imply a pane is live just because its ID is known.
|
||||
if raw, ok := p["session_evidence"].(map[string]any); ok {
|
||||
var evidence domain.SessionEvidence
|
||||
if b, err := json.Marshal(raw); err == nil && json.Unmarshal(b, &evidence) == nil {
|
||||
t.LastSession = evidence
|
||||
if evidence.PaneID != "" {
|
||||
t.LastPaneID = evidence.PaneID
|
||||
}
|
||||
if evidence.HarnessID != "" {
|
||||
t.LastHarness = evidence.HarnessID
|
||||
}
|
||||
if evidence.PaneState != "" {
|
||||
t.PaneState = evidence.PaneState
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Version = e.Version
|
||||
s.tasks[e.TaskID] = t
|
||||
return nil
|
||||
@@ -410,12 +453,33 @@ func (s *Store) Lease(id, harness string, ttl time.Duration) (domain.Event, erro
|
||||
payload := map[string]any{"harness_id": harness, "ttl": ttl.Seconds(), "until_ns": time.Now().Add(ttl).UnixNano(), "expected_version": t.Version}
|
||||
if t.HandoffRef != "" {
|
||||
payload["handoff_ref"] = t.HandoffRef
|
||||
payload["transaction_id"] = t.ReleaseTransaction
|
||||
payload["anchor_sha"] = t.ReleaseAnchor
|
||||
}
|
||||
p, _ := json.Marshal(payload)
|
||||
e := domain.Event{ID: domain.NewID(), Type: "TaskLeased", TaskID: id, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
|
||||
return e, s.Append(e)
|
||||
}
|
||||
|
||||
// RenewLease atomically extends the current owner's lease. The observed task
|
||||
// version is part of the request so an old worker can never renew a lease
|
||||
// after release/reassignment.
|
||||
func (s *Store) RenewLease(id, harness string, expectedVersion int, ttl time.Duration) (domain.Event, error) {
|
||||
if ttl <= 0 {
|
||||
return domain.Event{}, fmt.Errorf("%w: ttl must be positive", domain.ErrInvalid)
|
||||
}
|
||||
t, ok := s.Task(id)
|
||||
if !ok {
|
||||
return domain.Event{}, domain.ErrNotFound
|
||||
}
|
||||
if t.State != domain.StateLeased || t.Lease == nil || t.Lease.HarnessID != harness || t.Version != expectedVersion {
|
||||
return domain.Event{}, domain.ErrConflict
|
||||
}
|
||||
p, _ := json.Marshal(map[string]any{"harness_id": harness, "until_ns": time.Now().Add(ttl).UnixNano(), "expected_version": expectedVersion})
|
||||
e := domain.Event{ID: domain.NewID(), Type: "TaskLeaseRenewed", TaskID: id, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
|
||||
return e, s.Append(e)
|
||||
}
|
||||
|
||||
func (s *Store) ExpireLeases(now time.Time) ([]domain.Event, error) {
|
||||
var out []domain.Event
|
||||
for _, t := range s.Tasks() {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -51,6 +52,80 @@ func TestLeaseCarriesReleasedHandoffRef(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenewLeaseRequiresCurrentOwnerAndVersion(t *testing.T) {
|
||||
s, err := Open(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Append(domain.Event{ID: "create", Type: "TaskCreated", TaskID: "t", Version: 1, Payload: []byte(`{"source":"s","external_id":"renew","project":"p"}`), Surface: string(authz.System)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.Lease("t", "worker-a", time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before, _ := s.Task("t")
|
||||
if _, err := s.RenewLease("t", "worker-b", before.Version, time.Hour); !errors.Is(err, domain.ErrConflict) {
|
||||
t.Fatalf("other worker renewal = %v, want conflict", err)
|
||||
}
|
||||
if _, err := s.RenewLease("t", "worker-a", before.Version-1, time.Hour); !errors.Is(err, domain.ErrConflict) {
|
||||
t.Fatalf("stale renewal = %v, want conflict", err)
|
||||
}
|
||||
e, err := s.RenewLease("t", "worker-a", before.Version, time.Hour)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after, _ := s.Task("t")
|
||||
if e.Type != "TaskLeaseRenewed" || after.Version != before.Version+1 || after.Lease == nil || !after.Lease.Until.After(before.Lease.Until) {
|
||||
t.Fatalf("renewal was not projected: before=%+v after=%+v event=%+v", before, after, e)
|
||||
}
|
||||
if _, err := s.RenewLease("t", "worker-a", before.Version, time.Hour); !errors.Is(err, domain.ErrConflict) {
|
||||
t.Fatalf("replayed renewal = %v, want conflict", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlockedTaskProjectsStructuredDiagnosisAndLegacyFallback(t *testing.T) {
|
||||
s, err := Open(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Append(domain.Event{ID: "create", Type: "TaskCreated", TaskID: "t", Version: 1, Payload: []byte(`{"source":"s","external_id":"blocked","project":"p"}`), Surface: string(authz.System)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
payload := []byte(`{"blocker":"worker is offline","block_reason":"worker_offline","pane_id":"p1","harness_id":"w1","pane_state":"unreachable"}`)
|
||||
if err := s.Append(domain.Event{ID: "blocked", Type: "TaskBlocked", TaskID: "t", Version: 2, Payload: payload, Surface: string(authz.System)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task, _ := s.Task("t")
|
||||
if task.BlockReason != domain.BlockReasonWorkerOffline || task.LastPaneID != "p1" || task.PaneState != "unreachable" {
|
||||
t.Fatalf("blocked diagnosis was not projected: %+v", task)
|
||||
}
|
||||
if got := domain.InferBlockReason("handoff validation failed"); got != domain.BlockReasonHandoffValidation {
|
||||
t.Fatalf("legacy fallback=%q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTerminalSessionEvidenceSurvivesSessionCleanup(t *testing.T) {
|
||||
s, err := Open(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Append(domain.Event{ID: "create", Type: "TaskCreated", TaskID: "t", Version: 1, Payload: []byte(`{"source":"s","external_id":"evidence","project":"p"}`), Surface: string(authz.System)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ref, err := s.PutArtifact([]byte("report"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
payload := []byte(`{"report_ref":"` + ref + `","receipt":{"source":"worker"},"session_evidence":{"pane_id":"w:p1","harness_id":"worker-1","pane_state":"open","source":"worker","captured_at":"2026-07-29T12:00:00Z","checked_at":"2026-07-29T12:00:01Z"}}`)
|
||||
if err := s.Append(domain.Event{ID: "complete", Type: "TaskCompleted", TaskID: "t", Version: 2, Payload: payload, Surface: string(authz.System)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task, _ := s.Task("t")
|
||||
if task.LastSession.PaneID != "w:p1" || task.LastSession.Source != "worker" || task.LastSession.CapturedAt.IsZero() || task.LastHarness != "worker-1" {
|
||||
t.Fatalf("terminal session evidence was lost: %+v", task)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppendReplayAndDeduplicate(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
s, err := Open(dir)
|
||||
@@ -314,3 +389,41 @@ func TestExpectedVersionIsCheckedForEveryWriter(t *testing.T) {
|
||||
t.Fatalf("expected CAS conflict, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReleaseTransactionSurvivesReLeaseUntilMatchingPickup(t *testing.T) {
|
||||
s, err := Open(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Append(created("create")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
leased, err := s.Lease("task-1", "predecessor", time.Minute)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ref, err := s.PutArtifact([]byte("handoff"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
anchor := strings.Repeat("a", 40)
|
||||
p, _ := json.Marshal(map[string]any{"handoff_ref": ref, "anchor_sha": anchor, "transaction_id": "tx-1", "expected_version": leased.Version})
|
||||
if err := s.Append(domain.Event{ID: "release", Type: "TaskReleased", TaskID: "task-1", Version: leased.Version + 1, Payload: p, Surface: string(authz.System)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.Lease("task-1", "successor", time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task, _ := s.Task("task-1")
|
||||
if task.ReleaseTransaction != "tx-1" || task.ReleaseAnchor != anchor || task.HandoffRef != ref {
|
||||
t.Fatalf("re-lease lost transaction: %+v", task)
|
||||
}
|
||||
p, _ = json.Marshal(map[string]any{"transaction_id": "tx-1", "handoff_ref": ref, "anchor_sha": anchor, "harness_id": "successor", "lease_version": task.Version, "expected_version": task.Version})
|
||||
if err := s.Append(domain.Event{ID: "pickup", Type: "TaskPickupValidated", TaskID: "task-1", Version: task.Version + 1, Payload: p, Surface: string(authz.System)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task, _ = s.Task("task-1")
|
||||
if task.PickupTransaction != "tx-1" || task.PickupLeaseVersion != 4 {
|
||||
t.Fatalf("pickup not bound to transaction/epoch: %+v", task)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user