kami/orchestra
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Complete autonomous recovery controls

Browse Source
This commit is contained in:
kami
2026-07-30 14:57:25 +04:00
parent 8174400b1a
commit e8fadfc998
18 changed files with 364 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/orchestra-worker/main.go
+71 -9
View File
@@ -87,6 +87,7 @@ type lease struct {
PickupAcknowledged bool `json:"pickup_acknowledged,omitempty"`
Version int `json:"version"`
Until time.Time `json:"until"`
UsageBaseline float64 `json:"usage_baseline,omitempty"`
}
type releaseTransaction struct {
ID string `json:"id"`
@@ -99,10 +100,11 @@ type releaseTransaction struct {
UpdatedAt time.Time `json:"updated_at"`
}
type projectConfig struct {
Repo string `json:"repo"`
Root string `json:"worktree_root"`
Remote string `json:"remote"`
QualityGate string `json:"quality_gate,omitempty"`
Repo string `json:"repo"`
Root string `json:"worktree_root"`
Remote string `json:"remote"`
QualityGate string `json:"quality_gate,omitempty"`
SafeOperations []string `json:"safe_operations,omitempty"`
}
type completionEvidence struct {
TaskID string `json:"task_id"`
@@ -323,6 +325,9 @@ func (w *worker) start(ctx context.Context, t domain.Task, ref string) error {
}
s.TaskFileSHA = taskHash(t)
prompt := "Read TASK.md at the worktree root and execute it."
if len(p.SafeOperations) > 0 {
prompt += " This project's audited no-grant policy permits only worktree-local " + strings.Join(p.SafeOperations, ", ") + ". Network, secrets, destructive actions, and paths outside this worktree still require an explicit operator approval."
}
if ref != "" {
prompt += " A validated handoff exists; inspect local Git history and the recorded checkpoint before continuing."
}
@@ -335,12 +340,35 @@ func (w *worker) start(ctx context.Context, t domain.Task, ref string) error {
if err := w.herdr.Prompt(ctx, s.PaneID, prompt, 0); err != nil {
return err
}
if l, ok := w.leases[t.ID]; ok {
if err := w.api.Start(ctx, t.ID, l.Epoch, l.Version, w.sessionEvidence(ctx, t.ID, s)); err != nil {
return fmt.Errorf("ack start: %w", err)
}
l.Version++
w.leases[t.ID] = l
if err := w.save(); err != nil {
return err
}
}
if ref != "" {
return w.ackPickup(ctx, t.ID, s)
}
return nil
}
func classifyLaunchError(err error, sessionStarted bool) string {
if sessionStarted {
// A prompt response can be lost after herdr accepted it. Never reclaim
// that pane just because its acknowledgement was uncertain.
return "launch_uncertain"
}
text := strings.ToLower(err.Error())
if strings.Contains(text, "handoff") || strings.Contains(text, "pickup") || strings.Contains(text, "task.md") {
return "invalid_handoff"
}
return "launch_transient"
}
func taskHash(t domain.Task) string { b := continuity.RenderTaskFile(t); return domain.Hash(b) }
func (w *worker) releaseReady(ctx context.Context) {
@@ -355,6 +383,17 @@ func (w *worker) releaseReady(ctx context.Context) {
}
}
if _, err := os.Stat(filepath.Join(s.Worktree, ".orchestra", "done")); err == nil {
// A done marker is an intent, not enough on its own: do not race a
// still-running native harness into committing half-written work.
status, statusErr := (herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}).AgentStatus(ctx, s)
if statusErr != nil {
w.recordError(fmt.Errorf("completion identity %s: %w", id, statusErr))
continue
}
if herdr.IsBusy(status) {
w.recordError(fmt.Errorf("completion %s deferred: agent status %s", id, status))
continue
}
evidence, err := w.finalize(ctx, id, s)
if err != nil {
w.recordError(fmt.Errorf("complete %s: %w", id, err))
@@ -368,7 +407,7 @@ func (w *worker) releaseReady(ctx context.Context) {
log.Printf("upload completion %s: %v", id, err)
continue
}
if err = w.api.Complete(ctx, id, ref, evidence.ResultSHA, evidence.Branch, evidence.Remote, w.leases[id].Epoch, w.leases[id].Version, w.usageReceipt(s), w.sessionEvidence(ctx, id, s)); err != nil {
if err = w.api.Complete(ctx, id, ref, evidence.ResultSHA, evidence.Branch, evidence.Remote, w.leases[id].Epoch, w.leases[id].Version, w.usageReceipt(s, w.leases[id]), w.sessionEvidence(ctx, id, s)); err != nil {
w.recordError(fmt.Errorf("complete %s: %w", id, err))
log.Printf("complete %s: %v", id, err)
continue
@@ -576,9 +615,9 @@ func (w *worker) sessionEvidence(ctx context.Context, taskID string, s herdr.Ses
return e
}
func (w *worker) usageReceipt(s herdr.Session) map[string]any {
func (w *worker) usageReceipt(s herdr.Session, l lease) map[string]any {
if s.SessionFile == "" && !(w.harness == "opencode" && s.SessionID != "") {
return map[string]any{"harness_id": w.harnessID, "consumed": 0}
return map[string]any{"harness_id": w.harnessID, "consumed": 0, "known": false, "error": "native usage identity missing"}
}
var usage herdr.Usage
var err error
@@ -591,9 +630,13 @@ func (w *worker) usageReceipt(s herdr.Session) map[string]any {
usage, err = herdr.OpenCodeSessionUsage(s.SessionID)
}
if err != nil {
return map[string]any{"harness_id": w.harnessID, "consumed": 0, "error": err.Error()}
return map[string]any{"harness_id": w.harnessID, "consumed": 0, "known": false, "error": err.Error()}
}
return map[string]any{"harness_id": w.harnessID, "input_tokens": usage.Input, "cache_read_tokens": usage.CacheRead, "cache_write_tokens": usage.CacheWrite, "output_tokens": usage.Output, "consumed": usage.Numerator()}
delta := float64(usage.Numerator()) - l.UsageBaseline
if delta < 0 {
delta = 0
}
return map[string]any{"harness_id": w.harnessID, "input_tokens": usage.Input, "cache_read_tokens": usage.CacheRead, "cache_write_tokens": usage.CacheWrite, "consumed": delta, "lease_usage_delta": delta, "known": true}
}
func git(ctx context.Context, dir string, args ...string) ([]byte, error) {
@@ -814,6 +857,12 @@ func (w *worker) once(ctx context.Context) error {
w.leases[e.TaskID] = l
}
}
if e.Type == "TaskLaunchAcknowledged" {
if l, ok := w.leases[e.TaskID]; ok {
l.Version = e.Version
w.leases[e.TaskID] = l
}
}
if e.Type == "TaskNeedsAttention" {
// The diagnostic event increments the aggregate version but leaves
// ownership intact. Keep our locally persisted expected version in
@@ -914,6 +963,19 @@ func (w *worker) once(ctx context.Context) error {
if t, ok := w.tasks[taskID]; ok {
if err := w.start(ctx, t, l.HandoffRef); err != nil {
log.Printf("lease %s: %v", t.ID, err)
_, started := w.sessions[taskID]
class := classifyLaunchError(err, started)
var evidence domain.SessionEvidence
if session, ok := w.sessions[taskID]; ok {
evidence = w.sessionEvidence(ctx, taskID, session)
}
if nackErr := w.api.NackStart(ctx, taskID, l.Epoch, l.Version, class, err.Error(), evidence); nackErr != nil {
w.recordError(fmt.Errorf("nack launch %s: %w", taskID, nackErr))
continue
}
if class != "launch_uncertain" {
delete(w.leases, taskID)
}
}
}
}
cmd/orchestra-worker/main_test.go
+12
View File
@@ -51,6 +51,18 @@ func TestWorkerRefusesCorruptDurableState(t *testing.T) {
}
}
func TestClassifyLaunchErrorPreservesUncertainLivePane(t *testing.T) {
if got := classifyLaunchError(errors.New("prompt response lost"), true); got != "launch_uncertain" {
t.Fatalf("live pane class=%q", got)
}
if got := classifyLaunchError(errors.New("pickup anchor mismatch"), false); got != "invalid_handoff" {
t.Fatalf("bad handoff class=%q", got)
}
if got := classifyLaunchError(errors.New("temporary herdr outage"), false); got != "launch_transient" {
t.Fatalf("transient class=%q", got)
}
}
func TestInitialReplayDoesNotResurrectReleasedLease(t *testing.T) {
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {