fix: make worker handoff rotation durable
This commit is contained in:
kami
+130
-68
@@ -248,80 +248,99 @@ func (a CLIAdapter) NotifyConventionsChanged(ctx context.Context, s Session) err
|
||||
// handoff is refused rather than guessed at: the caller (Coordinator.rotate)
|
||||
// leaves the lease intact and retries next tick, giving the agent time to
|
||||
// finish writing it.
|
||||
func (a CLIAdapter) Release(ctx context.Context, s Session) (string, error) {
|
||||
// PreparedRelease is the durable, coordinator-independent half of a release.
|
||||
// The worker persists it before publishing TaskReleased so a lost HTTP reply
|
||||
// never requires reconstructing (or deleting) the agent's report.
|
||||
type PreparedRelease struct {
|
||||
Ref string
|
||||
AnchorSHA string
|
||||
}
|
||||
|
||||
// PrepareRelease seals an immutable Git checkpoint and uploads its canonical
|
||||
// handoff, but deliberately leaves both the pane claim and report in place.
|
||||
// The caller controls the retryable transaction around coordinator acceptance.
|
||||
func (a CLIAdapter) PrepareRelease(ctx context.Context, s Session) (PreparedRelease, error) {
|
||||
if a.CAS == nil {
|
||||
return "", fmt.Errorf("adapter: CAS store required to upload handoff")
|
||||
return PreparedRelease{}, fmt.Errorf("adapter: CAS store required to upload handoff")
|
||||
}
|
||||
// A federation worker always supplies the immutable task hash and remote.
|
||||
// The empty-hash case is retained solely for old in-process adapter users;
|
||||
// it is not reachable from the worker release path.
|
||||
if s.TaskFileSHA != "" {
|
||||
if a.Remote == "" {
|
||||
return PreparedRelease{}, fmt.Errorf("adapter: project remote required for checkpoint")
|
||||
}
|
||||
if err := continuity.VerifyTaskFile(s.Worktree, s.TaskFileSHA); err != nil {
|
||||
return PreparedRelease{}, fmt.Errorf("adapter: verify immutable TASK.md: %w", err)
|
||||
}
|
||||
}
|
||||
path := filepath.Join(s.Worktree, HandoffReportFile)
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("adapter: semantic handoff report not written yet (%s): %w", path, err)
|
||||
return PreparedRelease{}, fmt.Errorf("adapter: semantic handoff report not written yet (%s): %w", path, err)
|
||||
}
|
||||
if strings.TrimSpace(string(b)) == "" {
|
||||
return "", fmt.Errorf("adapter: semantic handoff report is empty")
|
||||
return PreparedRelease{}, fmt.Errorf("adapter: semantic handoff report is empty")
|
||||
}
|
||||
h, err := canonicalHandoff(s, string(b), a.lastObservedCommand(s))
|
||||
if err != nil {
|
||||
return "", err
|
||||
return PreparedRelease{}, err
|
||||
}
|
||||
sha, err := HeadSHA(s.Worktree)
|
||||
// Always checkpoint and push, including already-committed clean work. Git
|
||||
// is the cross-machine transport, so merely observing a local clean HEAD is
|
||||
// not a sufficient anchor.
|
||||
branch := "orchestra/scratch/" + h.Meta.ID
|
||||
if err := continuity.ScratchCommit(s.Worktree, branch, "orchestra: pre-release WIP snapshot ("+h.Meta.ID+")"); err != nil {
|
||||
return PreparedRelease{}, fmt.Errorf("adapter: scratch commit: %w", err)
|
||||
}
|
||||
anchor, err := HeadSHA(s.Worktree)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("adapter: read worktree HEAD: %w", err)
|
||||
return PreparedRelease{}, fmt.Errorf("adapter: read checkpoint HEAD: %w", err)
|
||||
}
|
||||
_ = sha
|
||||
for _, d := range h.Anchor.Dirty {
|
||||
if hex.EncodeToString(sha256sum(filepath.Join(s.Worktree, d.Path))) != d.SHA256 {
|
||||
return "", fmt.Errorf("adapter: handoff dirty file changed since it was written: %s", d.Path)
|
||||
if a.Remote != "" {
|
||||
if err := continuity.ScratchPush(s.Worktree, branch, a.Remote); err != nil {
|
||||
return PreparedRelease{}, fmt.Errorf("adapter: push checkpoint: %w", err)
|
||||
}
|
||||
out, err := exec.CommandContext(ctx, "git", "-C", s.Worktree, "ls-remote", a.Remote, "refs/heads/"+branch).Output()
|
||||
if err != nil || !strings.HasPrefix(string(out), anchor+"\t") {
|
||||
return PreparedRelease{}, fmt.Errorf("adapter: verify pushed anchor: got %q: %w", strings.TrimSpace(string(out)), err)
|
||||
}
|
||||
}
|
||||
// The semantic report is transferred in the CAS handoff, not in the
|
||||
// scratch checkout. Keeping it in the scratch commit makes a successor
|
||||
// mistake the predecessor's report for a newly requested handoff and can
|
||||
// cause an immediate release/pickup loop.
|
||||
dirty := h.Anchor.Dirty[:0]
|
||||
for _, d := range h.Anchor.Dirty {
|
||||
if filepath.Clean(d.Path) != HandoffReportFile {
|
||||
dirty = append(dirty, d)
|
||||
}
|
||||
}
|
||||
h.Anchor.Dirty = dirty
|
||||
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
|
||||
return "", fmt.Errorf("adapter: remove transferred semantic report: %w", err)
|
||||
}
|
||||
// Atomically commit whatever the handoff described as dirty onto a
|
||||
// per-task scratch branch (§6.2 step 3) *before* uploading, so the
|
||||
// successor's pickup validation collapses to a single HEAD compare
|
||||
// instead of re-hashing every dirty file individually.
|
||||
if len(h.Anchor.Dirty) > 0 {
|
||||
branch := "orchestra/scratch/" + h.Meta.ID
|
||||
if err := continuity.ScratchCommit(s.Worktree, branch, "orchestra: pre-release WIP snapshot ("+h.Meta.ID+")"); err != nil {
|
||||
return "", fmt.Errorf("adapter: scratch commit: %w", err)
|
||||
}
|
||||
if a.Remote != "" {
|
||||
if err := continuity.ScratchPush(s.Worktree, branch, a.Remote); err != nil {
|
||||
return "", fmt.Errorf("adapter: push scratch branch: %w", err)
|
||||
}
|
||||
}
|
||||
newSHA, err := HeadSHA(s.Worktree)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("adapter: read scratch HEAD: %w", err)
|
||||
}
|
||||
h.Anchor.GitSHA = newSHA
|
||||
h.Anchor.Branch = branch
|
||||
h.Anchor.Dirty = nil
|
||||
}
|
||||
h.Anchor.GitSHA, h.Anchor.Branch, h.Anchor.Dirty = anchor, branch, nil
|
||||
ref, err := continuity.Save(h, a.CAS)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("adapter: upload handoff: %w", err)
|
||||
return PreparedRelease{}, fmt.Errorf("adapter: upload handoff: %w", err)
|
||||
}
|
||||
return PreparedRelease{Ref: ref, AnchorSHA: anchor}, nil
|
||||
}
|
||||
|
||||
// ReleaseAgent drops only herdr's harness binding. It does not close the pane:
|
||||
// a predecessor stays recoverable until the successor has validated pickup.
|
||||
func (a CLIAdapter) ReleaseAgent(ctx context.Context, s Session) error {
|
||||
if err := a.Client.Call(ctx, "pane.release_agent", map[string]any{
|
||||
"pane_id": s.PaneID,
|
||||
"source": "herdr:" + a.Harness,
|
||||
"agent": agentForSession(s, a.Harness),
|
||||
}, nil); err != nil {
|
||||
return "", fmt.Errorf("adapter: pane.release_agent: %w", err)
|
||||
return fmt.Errorf("adapter: pane.release_agent: %w", err)
|
||||
}
|
||||
return ref, nil
|
||||
return nil
|
||||
}
|
||||
|
||||
// Release is retained for the coordinator's legacy local path. Federation
|
||||
// workers use PrepareRelease and ReleaseAgent as separate durable phases.
|
||||
func (a CLIAdapter) Release(ctx context.Context, s Session) (string, error) {
|
||||
p, err := a.PrepareRelease(ctx, s)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := a.ReleaseAgent(ctx, s); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := os.Remove(filepath.Join(s.Worktree, HandoffReportFile)); err != nil && !os.IsNotExist(err) {
|
||||
return "", fmt.Errorf("adapter: remove transferred semantic report: %w", err)
|
||||
}
|
||||
return p.Ref, nil
|
||||
}
|
||||
|
||||
// canonicalHandoff keeps Git-derived protocol facts on the worker that owns
|
||||
@@ -483,16 +502,27 @@ func handoffID(s Session) string {
|
||||
}
|
||||
|
||||
func dirtyFiles(root string) ([]continuity.Dirty, error) {
|
||||
out, err := exec.Command("git", "-C", root, "status", "--porcelain=v1", "-z").Output()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
paths := map[string]bool{}
|
||||
for _, args := range [][]string{{"diff", "--name-only", "-z"}, {"ls-files", "--others", "--exclude-standard", "-z"}} {
|
||||
out, err := exec.Command("git", append([]string{"-C", root}, args...)...).Output()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
deleted := map[string]bool{}
|
||||
parts := strings.Split(string(out), "\x00")
|
||||
for i := 0; i < len(parts); i++ {
|
||||
record := parts[i]
|
||||
if len(record) < 4 {
|
||||
continue
|
||||
}
|
||||
for _, path := range strings.Split(string(out), "\x00") {
|
||||
if path != "" && path != HandoffFile {
|
||||
paths[path] = true
|
||||
}
|
||||
status, path := record[:2], record[3:]
|
||||
if path == HandoffFile || path == HandoffReportFile || path == ".orchestra/done" || strings.HasPrefix(path, ".orchestra/") {
|
||||
continue
|
||||
}
|
||||
paths[path] = true
|
||||
deleted[path] = strings.Contains(status, "D")
|
||||
// A rename/copy record has the original path as the next NUL item.
|
||||
if status[0] == 'R' || status[0] == 'C' || status[1] == 'R' || status[1] == 'C' {
|
||||
i++
|
||||
}
|
||||
}
|
||||
keys := make([]string, 0, len(paths))
|
||||
@@ -502,11 +532,15 @@ func dirtyFiles(root string) ([]continuity.Dirty, error) {
|
||||
sort.Strings(keys)
|
||||
dirty := make([]continuity.Dirty, 0, len(keys))
|
||||
for _, path := range keys {
|
||||
sum := sha256sum(filepath.Join(root, path))
|
||||
if len(sum) == 0 {
|
||||
return nil, fmt.Errorf("adapter: hash dirty file %s", path)
|
||||
d := continuity.Dirty{Path: path, Deleted: deleted[path]}
|
||||
if !d.Deleted {
|
||||
sum := sha256sum(filepath.Join(root, path))
|
||||
if len(sum) == 0 {
|
||||
return nil, fmt.Errorf("adapter: hash dirty file %s", path)
|
||||
}
|
||||
d.SHA256 = hex.EncodeToString(sum)
|
||||
}
|
||||
dirty = append(dirty, continuity.Dirty{Path: path, SHA256: hex.EncodeToString(sum)})
|
||||
dirty = append(dirty, d)
|
||||
}
|
||||
return dirty, nil
|
||||
}
|
||||
@@ -641,6 +675,13 @@ var _ = json.RawMessage{}
|
||||
// callers (Coordinator.rotate, refreshSessionHealth) surface it instead of
|
||||
// mistaking "we don't know" for "occupancy is zero".
|
||||
func (a CLIAdapter) Occupancy(s Session) (float64, error) {
|
||||
if a.Harness == "opencode" {
|
||||
u, err := OpenCodeSessionUsage(s.SessionID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return Fraction(u, a.Window), nil
|
||||
}
|
||||
if a.Usage == nil {
|
||||
return 0, fmt.Errorf("adapter: usage reader required")
|
||||
}
|
||||
@@ -659,6 +700,32 @@ func (a CLIAdapter) Occupancy(s Session) (float64, error) {
|
||||
return Fraction(u, a.Window), nil
|
||||
}
|
||||
|
||||
// ResolveSessionIdentity discovers and returns the harness-native session
|
||||
// identity. Callers persist the returned Session before relying on occupancy,
|
||||
// so restart recovery keeps observing the same harness session.
|
||||
func (a CLIAdapter) ResolveSessionIdentity(s Session) (Session, error) {
|
||||
if a.Harness == "opencode" {
|
||||
if s.SessionID != "" {
|
||||
return s, nil
|
||||
}
|
||||
id, err := OpenCodeSessionID(s.Worktree)
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
s.SessionID = id
|
||||
return s, nil
|
||||
}
|
||||
if s.SessionFile != "" {
|
||||
return s, nil
|
||||
}
|
||||
path, err := a.resolveSessionFile(s)
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
s.SessionFile = path
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (a CLIAdapter) resolveSessionFile(s Session) (string, error) {
|
||||
switch a.Harness {
|
||||
case "claude":
|
||||
@@ -667,12 +734,7 @@ func (a CLIAdapter) resolveSessionFile(s Session) (string, error) {
|
||||
_, path, err := CodexActiveUsage("")
|
||||
return path, err
|
||||
default:
|
||||
// opencode's session-file resolution needs the running session id,
|
||||
// which is only available via the SSE/status API (OpenCodeStatus),
|
||||
// not derivable from the worktree alone. Per AUDIT.md Phase 1, wiring
|
||||
// this needs verification against a live opencode instance before it
|
||||
// can drive rotation — refuse loudly rather than guess a path.
|
||||
return "", fmt.Errorf("adapter: harness %q has no session-file resolver; verify against a live session first (AUDIT.md Phase 1)", a.Harness)
|
||||
return "", fmt.Errorf("adapter: harness %q has no session-file resolver", a.Harness)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -152,6 +152,10 @@ type Session struct {
|
||||
// CLIAdapter.Occupancy), since the file may not exist yet immediately
|
||||
// after lease.
|
||||
SessionFile string `json:"session_file,omitempty"`
|
||||
// SessionID is the harness-native identity when its usage is stored in a
|
||||
// database rather than a transcript. OpenCode's SQLite session ID is kept
|
||||
// here so rotation never guesses "the newest session" after a restart.
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
// TaskFileSHA is the sha256 of the worktree's TASK.md at the time this
|
||||
// session's lease was created — the immutable-spec hash continuity's
|
||||
// pickup validation compares against on the next rotation (§6.2).
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
@@ -30,6 +31,7 @@ func Fraction(u Usage, w int64) float64 {
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// ClaudeSessionFile resolves the transcript file for a Claude Code session
|
||||
// running against worktree, by newest-mtime under Claude Code's encoded
|
||||
// project directory (~/.claude/projects/<abs-worktree-path-with-/-replaced-
|
||||
@@ -202,6 +204,60 @@ func OpenCodeUsage(p string) (Usage, error) {
|
||||
return Usage{x.Tokens.Input, x.Tokens.Cache.Read, x.Tokens.Cache.Write, x.Tokens.Output}, e
|
||||
}
|
||||
|
||||
// OpenCodeSessionID resolves the exact OpenCode session associated with a
|
||||
// checkout. OpenCode stores usage in its SQLite session table, not in a pane
|
||||
// transcript. Selecting by directory and persisting the returned id prevents
|
||||
// a multi-pane worker from attributing another task's newest session to this
|
||||
// lease.
|
||||
func OpenCodeSessionID(worktree string) (string, error) {
|
||||
db := os.Getenv("ORCHESTRA_OPENCODE_DB")
|
||||
if db == "" {
|
||||
db = filepath.Join(os.Getenv("HOME"), ".local", "share", "opencode", "opencode.db")
|
||||
}
|
||||
abs, err := filepath.Abs(worktree)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
out, err := exec.Command("sqlite3", "-readonly", "-noheader", db, "select id from session where directory = "+sqliteQuote(abs)+" order by time_updated desc limit 1;").Output()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("opencode session lookup: %w", err)
|
||||
}
|
||||
id := strings.TrimSpace(string(out))
|
||||
if id == "" {
|
||||
return "", fmt.Errorf("opencode session lookup: no session for %s", abs)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// OpenCodeSessionUsage reads the token counters for one persisted session.
|
||||
func OpenCodeSessionUsage(sessionID string) (Usage, error) {
|
||||
if sessionID == "" {
|
||||
return Usage{}, fmt.Errorf("opencode session id required")
|
||||
}
|
||||
db := os.Getenv("ORCHESTRA_OPENCODE_DB")
|
||||
if db == "" {
|
||||
db = filepath.Join(os.Getenv("HOME"), ".local", "share", "opencode", "opencode.db")
|
||||
}
|
||||
out, err := exec.Command("sqlite3", "-readonly", "-noheader", "-separator", "|", db, "select tokens_input,tokens_cache_read,tokens_cache_write,tokens_output from session where id = "+sqliteQuote(sessionID)+";").Output()
|
||||
if err != nil {
|
||||
return Usage{}, fmt.Errorf("opencode usage lookup: %w", err)
|
||||
}
|
||||
parts := strings.Split(strings.TrimSpace(string(out)), "|")
|
||||
if len(parts) != 4 {
|
||||
return Usage{}, fmt.Errorf("opencode usage lookup: unknown session %q", sessionID)
|
||||
}
|
||||
values := [4]int64{}
|
||||
for i, part := range parts {
|
||||
values[i], err = strconv.ParseInt(part, 10, 64)
|
||||
if err != nil {
|
||||
return Usage{}, fmt.Errorf("opencode usage lookup: %w", err)
|
||||
}
|
||||
}
|
||||
return Usage{Input: values[0], CacheRead: values[1], CacheWrite: values[2], Output: values[3]}, nil
|
||||
}
|
||||
|
||||
func sqliteQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", "''") + "'" }
|
||||
|
||||
// OpenCodeStatus probes the server fast path. Callers can use the returned
|
||||
// status and fall back to OpenCodeUsage when the SSE/server is unavailable.
|
||||
func OpenCodeStatus(ctx context.Context, baseURL, sessionID string) (string, error) {
|
||||
|
||||
Reference in New Issue
Block a user