kami/orchestra
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

v3 workflow: intent, phases, review, submission, enforcement, burn-in

Browse Source 
The v3 stack, previously an uncommitted working tree, plus this session's two
units and the burn-in instrument. This commit is the burn-in build identity:
coordinator and worker must both report this revision before a task is created.

Workflow (earlier sessions, uncommitted until now): human decision events and
reduction, source cursors and reconcile-before-launch, turn-boundary
reconciliation, internal/agentctx as the single renderer, ace-fca phases with
sealed artifacts, the trajectory gate, bounded grilling, independent review,
task pr enforcement, and human review reflection.

Capability restrictions at the agent boundary: an authz.Agent surface at
GatedWrite may ask and may not act. It also fixes two bugs the unit exposed --
gated surfaces could not reach the two endpoints written for them, and
RequestHumanDecision would block an unowned task while rejecting a question
from the session that did own it.

Turn-boundary reconcile-failure escalation: a streak of consecutive failures
asks the session to hand off, fenced on the lease epoch, with reconcile_failure
as a real handoff reason. The worker was dropping the coordinator's verdict on
the floor; it now acts on it.

Burn-in: herdr.WriteLaunchContext dumps the exact agentctx.Build result to
<worktree>/.orchestra/launch.md at every launch, local and federated. BURNIN.md
is the runbook. deploy/build.sh stamps both binaries from one commit.

go build, go vet and go test ./... pass, 20 packages.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
kami
2026-08-26 18:31:20 +04:00
parent 97a9c65302
commit 7f12c7fc37
78 changed files with 16417 additions and 352 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/orchestrator/orchestrator.go
+313 -34
View File
@@ -8,11 +8,14 @@ import (
"encoding/json"
"errors"
"fmt"
"orchestra/internal/agentctx"
"orchestra/internal/authz"
"orchestra/internal/continuity"
"orchestra/internal/domain"
"orchestra/internal/herdr"
"orchestra/internal/operations"
"orchestra/internal/store"
"orchestra/internal/workphase"
"os"
"os/exec"
"path/filepath"
@@ -192,9 +195,69 @@ type Coordinator struct {
// defaultSoft), so existing callers that never set this field keep
// working unchanged.
Soft float64
// Reconcile imports newer human input for one task. Store.PreLease covers
// the moment ownership begins; this field covers the other half, a
// correction written while a lease is already live. It runs only at a
// verified turn boundary, so nothing preempts a running tool call.
ReconcileHumanInput func(ctx context.Context, taskID string) error
// Thrash tunes DetectThrash's three circuit breakers (§5.3). Zero-value
// fields fall back to herdr's own defaults, so leaving this unset works.
Thrash herdr.ThrashConfig
// ReconcileFailureHandoff is how many *consecutive* failed turn-boundary
// reconciles escalate to prepare_handoff. One failure is transient and
// continuing is right; a streak means Orchestra can no longer promise
// that the newest human input outranks this session's intent, so the
// honest move is to hand the task to a successor whose Store.PreLease
// reconcile fails closed while the source is down. Zero means the package
// default (defaultReconcileFailureHandoff).
ReconcileFailureHandoff int
// reconcileFailures is the streak per task, fenced on the lease epoch so
// a successor never inherits its predecessor's count and no release path
// needs a cleanup hook. Guarded by healthMu.
reconcileFailures map[string]reconcileStreak
}
type reconcileStreak struct {
Epoch string
N int
}
// defaultReconcileFailureHandoff is used whenever ReconcileFailureHandoff is
// unset. Three consecutive verified boundaries is long enough to ride out a
// restart or a brief network fault, short enough that a stuck source does not
// let a session run indefinitely on intent Orchestra cannot refresh.
const defaultReconcileFailureHandoff = 3
func (c *Coordinator) reconcileFailureThreshold() int {
if c.ReconcileFailureHandoff > 0 {
return c.ReconcileFailureHandoff
}
return defaultReconcileFailureHandoff
}
// noteReconcileResult records one turn boundary's reconcile outcome and reports
// whether this session has reached the escalation threshold. Success resets the
// streak, so two failures followed by a success escalate nothing.
func (c *Coordinator) noteReconcileResult(taskID, epoch string, err error) bool {
c.healthMu.Lock()
defer c.healthMu.Unlock()
if c.reconcileFailures == nil {
c.reconcileFailures = map[string]reconcileStreak{}
}
if err == nil {
delete(c.reconcileFailures, taskID)
return false
}
streak := c.reconcileFailures[taskID]
if streak.Epoch != epoch {
// A different owner: count this session's failures, not the previous
// lease's.
streak = reconcileStreak{Epoch: epoch}
}
streak.N++
c.reconcileFailures[taskID] = streak
c.recordSessionErrorLocked(taskID, fmt.Sprintf("reconcile human input (%d consecutive): %v", streak.N, err))
return streak.N >= c.reconcileFailureThreshold()
}
// defaultSoft is used whenever Coordinator.Soft is unset (zero value).
@@ -340,6 +403,67 @@ func (c *Coordinator) recordTurnBoundaryDegraded() {
c.health.TurnBoundaryDegraded++
}
// recordSessionError keeps a non-fatal failure observable instead of letting
// a bare continue hide it, which is this codebase's recurring bug shape.
func (c *Coordinator) recordSessionError(taskID, msg string) {
c.healthMu.Lock()
defer c.healthMu.Unlock()
c.recordSessionErrorLocked(taskID, msg)
}
// recordSessionErrorLocked is recordSessionError for callers already holding
// healthMu.
func (c *Coordinator) recordSessionErrorLocked(taskID, msg string) {
if c.health.Sessions == nil {
c.health.Sessions = map[string]SessionHealth{}
}
h := c.health.Sessions[taskID]
h.LastError = msg
h.UpdatedAt = time.Now().UTC()
c.health.Sessions[taskID] = h
}
// deliverDecisions sends the human decisions this session has not been shown
// yet. It runs only at a verified turn boundary, and only when the turn
// verdict is continue, so it never interrupts a running tool call and never
// competes with a rotation that is about to hand the task to a successor.
func (c *Coordinator) deliverDecisions(ctx context.Context, taskID string, session herdr.Session, a herdr.Adapter) {
notifier, ok := a.(herdr.DecisionNotifier)
if !ok {
return
}
intent, err := c.Store.EffectiveIntent(taskID)
if err != nil {
c.recordSessionError(taskID, "effective intent: "+err.Error())
return
}
seen := make(map[string]bool, len(session.DeliveredDecisions))
for _, id := range session.DeliveredDecisions {
seen[id] = true
}
var fresh []domain.HumanDecision
for _, d := range intent.Decisions {
if !seen[d.ID] {
fresh = append(fresh, d)
}
}
if len(fresh) == 0 {
return
}
if err := notifier.NotifyDecisions(ctx, session, agentctx.DecisionNotice(fresh)); err != nil {
// Not recorded as delivered, so the next boundary retries.
c.recordSessionError(taskID, "deliver decisions: "+err.Error())
return
}
for _, d := range fresh {
session.DeliveredDecisions = append(session.DeliveredDecisions, d.ID)
}
c.mu.Lock()
c.sessions[taskID] = session
_ = c.saveSessionsLocked()
c.mu.Unlock()
}
func waitingForApproval(status string) bool {
s := strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(status, "-", "_"), " ", "_"))
return s == "waiting_for_approval" || s == "awaiting_approval" || s == "approval_required"
@@ -681,6 +805,18 @@ func handoffReason(worktree string) string {
return h.Meta.Reason
}
// reasonReconcileFailure is the handoff reason for a session released because
// human input could not be reconciled at repeated verified turn boundaries.
const reasonReconcileFailure = "reconcile_failure"
// bypassReason reports whether a handoff already carrying this reason is
// itself the boundary signal, so occupancy and the turn-boundary probe are
// skipped and the session is released immediately. reconcile_failure joins the
// list because Orchestra, not the context window, asked for that handoff.
func bypassReason(r string) bool {
return r == "manual" || r == "milestone" || r == "thrash" || r == reasonReconcileFailure
}
func (c *Coordinator) rotate(ctx context.Context, hard float64) {
c.loadSessions()
c.mu.Lock()
@@ -705,7 +841,7 @@ func (c *Coordinator) rotate(ctx context.Context, hard float64) {
// question has already been answered, so skip occupancy and the
// turn-boundary probe and go straight to release.
existingReason := handoffReason(session.Worktree)
bypass := existingReason == "manual" || existingReason == "milestone" || existingReason == "thrash"
bypass := bypassReason(existingReason)
if bypass {
reason = existingReason
} else {
@@ -805,6 +941,67 @@ func (c *Coordinator) rotate(ctx context.Context, hard float64) {
}
}
// RemoteTurn is the federated half of a turn boundary. A worker owns the pane,
// so it evaluates rotation locally and reports the verdict it reached; the
// coordinator owns authority, so it reconciles human input here and answers
// with the decisions that session has not been shown yet.
//
// The split is deliberate. Duplicating the rotation state machine in the
// worker would give two answers to "should this session stop"; asking the
// coordinator to probe a remote pane would give it a checkout it cannot
// validate. Neither half is authoritative about the other's state.
//
// Decisions are returned only when the verdict is continue, matching the local
// path: a rotating session's successor picks them up at re-lease.
func (c *Coordinator) RemoteTurn(ctx context.Context, taskID, epoch, verdict string, delivered []string) (string, []domain.HumanDecision, error) {
if c.Store == nil {
return "", nil, fmt.Errorf("orchestrator: dependencies required")
}
t, ok := c.Store.Task(taskID)
if !ok {
return "", nil, domain.ErrNotFound
}
if t.State != domain.StateLeased && t.State != domain.StateNeedsAttention {
return "", nil, fmt.Errorf("orchestrator: task %q not leased", taskID)
}
// Fenced like every other worker-driven call: a worker whose lease was
// reassigned must not be handed the current session's decisions.
if t.Lease == nil || epoch == "" || t.Lease.Epoch != epoch {
return "", nil, domain.ErrConflict
}
escalate := false
if c.ReconcileHumanInput != nil {
// Same contract as the local boundary: one failure is observable, not
// fatal, because refusing would freeze a live remote session without
// making its current intent any less stale. A streak escalates, on the
// same threshold the local path uses.
escalate = c.noteReconcileResult(taskID, epoch, c.ReconcileHumanInput(ctx, taskID))
}
if verdict != TurnContinue {
// The worker already wants to stop. Answering with a second reason
// would manufacture a rotation trigger nothing needs.
return verdict, nil, nil
}
if escalate {
return TurnPrepareHandoff, nil, nil
}
intent, err := c.Store.EffectiveIntent(taskID)
if err != nil {
return "", nil, err
}
seen := make(map[string]bool, len(delivered))
for _, id := range delivered {
seen[id] = true
}
var fresh []domain.HumanDecision
for _, d := range intent.Decisions {
if !seen[d.ID] {
fresh = append(fresh, d)
}
}
return verdict, fresh, nil
}
// Turn decision verdicts (spec §5.3, AUDIT.md Phase 2 items 1-2). These are
// the only valid results of TurnDecision and the only values the
// POST /v1/harness/turn endpoint may return.
@@ -839,11 +1036,26 @@ func (c *Coordinator) TurnDecision(ctx context.Context, taskID string) (string,
if err != nil {
return "", fmt.Errorf("orchestrator: adapter: %w", err)
}
// A turn boundary is the one point where the agent is verifiably between
// actions, so it is where newer human input is imported for a live lease.
// The task is re-read afterwards because a recorded decision bumps its
// version, and finishRelease below writes against that version.
escalate := false
if c.ReconcileHumanInput != nil {
// One failure is recorded and the turn continues: blocking would not
// remove stale intent from the running agent, and a source outage
// would freeze every live session. A streak is different, and acts on
// the continue path below.
escalate = c.noteReconcileResult(taskID, task.Lease.Epoch, c.ReconcileHumanInput(ctx, taskID))
if fresh, ok := c.Store.Task(taskID); ok {
task = fresh
}
}
// Agent-initiated ROTATE, and the two orchestrator-detected triggers
// (§5.3: manual / milestone / thrash): a handoff already written with one
// of these reasons is itself the boundary signal — skip occupancy and the
// turn-boundary probe and release immediately.
if existingReason := handoffReason(session.Worktree); existingReason == "manual" || existingReason == "milestone" || existingReason == "thrash" {
if existingReason := handoffReason(session.Worktree); bypassReason(existingReason) {
return c.finishRelease(ctx, taskID, task, session, a, existingReason)
}
d := (RotationStateMachine{Soft: c.soft(), Hard: c.Hard, Thrash: c.Thrash}).Evaluate(ctx, a, session)
@@ -851,6 +1063,18 @@ func (c *Coordinator) TurnDecision(ctx context.Context, taskID string) (string,
return "", fmt.Errorf("orchestrator: rotation: %w", d.Degraded)
}
if d.Action == TurnContinue {
if escalate {
// Rotation has no reason of its own, so this is the one place the
// reconcile streak can act. Ask for a handoff; release runs through
// the ordinary bypass path once the agent writes it, and the
// successor's Store.PreLease reconcile fails closed while the
// source is still down.
c.requestReasonedHandoff(ctx, taskID, session, a, reasonReconcileFailure, nil)
return TurnPrepareHandoff, nil
}
// Only on continue. A rotating session's successor picks the decision
// up through Store.PreLease when it acquires the lease.
c.deliverDecisions(ctx, taskID, session, a)
return TurnContinue, nil
}
if d.Action == TurnRefuse {
@@ -963,7 +1187,66 @@ func (c *Coordinator) Start(ctx context.Context, e domain.Event) error {
return c.block(t, "worktree: "+err.Error())
}
taskFileSHA, _ := continuity.TaskFileHash(w)
prompt := taskLaunchPrompt(t)
// One renderer. The launch instruction is built by agentctx so a decision
// the human recorded while this task was queued is visible to the agent
// from its first turn, above anything it will later read as continuity.
intent, err := c.Store.EffectiveIntent(t.ID)
if err != nil {
return c.block(t, "effective intent: "+err.Error())
}
git := agentctx.GitState{Worktree: w, Branch: "orchestra/" + t.ID}
if sha, shaErr := herdr.HeadSHA(w); shaErr == nil {
git.HeadSHA = sha
}
// §6.2 pickup validation happens before the agent exists, not after: the
// handoff is part of the launch instruction now, so it must be trusted
// before it is rendered. A failure blocks the task without ever starting
// a session (this is the gap AUDIT.md's B6 named as unreached).
var handoff *continuity.Handoff
if p.HandoffRef != "" {
h, loadErr := continuity.Load(p.HandoffRef, c.Store)
if loadErr != nil {
return c.block(t, "handoff: "+loadErr.Error())
}
if err := continuity.ValidatePickup(w, h, taskFileSHA); err != nil {
return c.block(t, "pickup: "+err.Error())
}
handoff = &h
}
in := agentctx.Input{
Task: t, Intent: intent, Handoff: handoff, Git: git,
Phase: t.WorkPhase, RepoRules: agentctx.DiscoverRepoRules(w),
DecisionRequest: t.DecisionRequest,
}
if t.ResearchRef != "" {
r, refErr := c.research(t.ResearchRef)
if refErr != nil {
return c.block(t, "research artifact: "+refErr.Error())
}
in.Research = r
}
if t.PlanRef != "" {
pl, refErr := c.plan(t.PlanRef)
if refErr != nil {
return c.block(t, "plan artifact: "+refErr.Error())
}
in.Plan = pl
}
if t.Review != nil {
r, refErr := operations.TaskReview(c.Store, t)
if refErr != nil {
return c.block(t, "review artifact: "+refErr.Error())
}
in.Review = r
}
built, err := agentctx.Build(in)
if err != nil {
return c.block(t, "context: "+err.Error())
}
prompt := built.System + "\n\n" + built.Task
if writeErr := herdr.WriteLaunchContext(w, prompt); writeErr != nil {
c.recordSessionError(t.ID, "launch context: "+writeErr.Error())
}
var s herdr.Session
if promptLeaser, ok := a.(herdr.PromptLeaser); ok {
s, err = promptLeaser.LeasePrompt(ctx, t.ID, w, prompt)
@@ -982,28 +1265,13 @@ func (c *Coordinator) Start(ctx context.Context, e domain.Event) error {
}
return c.block(t, "lease: "+err.Error())
}
if p.HandoffRef != "" {
// §6.2 pickup validation: never bootstrap a successor onto a handoff
// whose anchor/dirty-file/TASK.md hashes don't match what's actually
// in the worktree. A failure here blocks the task rather than
// silently trusting an unvalidated ref (this is the gap AUDIT.md's
// B6 named as unreached from the live path).
h, err := continuity.Load(p.HandoffRef, c.Store)
if err != nil {
_ = a.Kill(ctx, s)
return c.block(t, "handoff: "+err.Error())
}
if err := continuity.ValidatePickup(w, h, taskFileSHA); err != nil {
_ = a.Kill(ctx, s)
return c.block(t, "pickup: "+err.Error())
}
if err = a.Bootstrap(ctx, s, p.HandoffRef); err != nil {
_ = a.Kill(ctx, s)
return c.block(t, "bootstrap: "+err.Error())
}
}
s.HerdrID = p.HarnessID
s.TaskFileSHA = taskFileSHA
// The launch instruction carried these, so the first turn boundary must
// not re-deliver them as news.
for _, d := range intent.Decisions {
s.DeliveredDecisions = append(s.DeliveredDecisions, d.ID)
}
// Best-effort, same caveat as taskFileSHA above: only meaningful for a
// worktree this process can read locally. Snapshots the shared-docs
// state this session starts trusting; checkConventions notices drift
@@ -1021,19 +1289,30 @@ func (c *Coordinator) Start(ctx context.Context, e domain.Event) error {
return err
}
func taskLaunchPrompt(t domain.Task) string {
var b strings.Builder
fmt.Fprintf(&b, "Begin Orchestra task %s.\n", t.ID)
if t.Title != "" {
fmt.Fprintf(&b, "Title: %s\n", t.Title)
// research and plan read a sealed phase artifact. A stored ref that will not
// decode is a blocked task, not a silently empty context section.
func (c *Coordinator) research(ref string) (*workphase.Research, error) {
b, err := c.Store.Artifact(ref)
if err != nil {
return nil, err
}
if t.Description != "" {
fmt.Fprintf(&b, "Instructions:\n%s\n", t.Description)
} else {
b.WriteString("Inspect the repository, understand the task context, and proceed with the requested work.\n")
r, err := workphase.DecodeResearch(b)
if err != nil {
return nil, err
}
b.WriteString("This is the authoritative task instruction. Work only within this task's worktree. Do not edit TASK.md if it exists.")
return b.String()
return &r, nil
}
func (c *Coordinator) plan(ref string) (*workphase.Plan, error) {
b, err := c.Store.Artifact(ref)
if err != nil {
return nil, err
}
p, err := workphase.DecodePlan(b)
if err != nil {
return nil, err
}
return &p, nil
}
func (c *Coordinator) rememberSession(taskID string, s herdr.Session) error {