checkpoint: multi-repo Gitea ingestion, per-project repos, rotation anchor_sha fix
Pre-existing uncommitted work found at session start: rotation now emits anchor_sha on TaskReleased (previously silently dropped by store.Append validation), multi-repo Gitea provider support, per-project git worktree roots, and associated test coverage. Committing as a checkpoint before starting remediation work tracked in AUDIT.md.
This commit is contained in:
@@ -7,6 +7,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"orchestra/internal/authz"
|
||||
"orchestra/internal/continuity"
|
||||
"orchestra/internal/domain"
|
||||
"orchestra/internal/herdr"
|
||||
@@ -14,6 +15,7 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
@@ -21,6 +23,12 @@ import (
|
||||
type Worktrees interface {
|
||||
Create(context.Context, domain.Task) (string, error)
|
||||
}
|
||||
type WorktreeSpec interface {
|
||||
Spec(domain.Task) (string, string, bool)
|
||||
}
|
||||
type WorktreeCleaner interface {
|
||||
Remove(context.Context, domain.Task, string) error
|
||||
}
|
||||
type Adapters interface {
|
||||
Adapter(string) (herdr.Adapter, error)
|
||||
}
|
||||
@@ -34,6 +42,10 @@ type GitWorktrees struct {
|
||||
TaskFileSHA string
|
||||
}
|
||||
|
||||
func (w GitWorktrees) Spec(domain.Task) (string, string, bool) {
|
||||
return w.Repo, w.Root, w.Repo != "" && w.Root != ""
|
||||
}
|
||||
|
||||
func (w GitWorktrees) Create(ctx context.Context, t domain.Task) (string, error) {
|
||||
if w.Root == "" || w.Repo == "" {
|
||||
return "", fmt.Errorf("worktree: root and repo required")
|
||||
@@ -63,6 +75,62 @@ func (w GitWorktrees) Create(ctx context.Context, t domain.Task) (string, error)
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func (w GitWorktrees) Remove(ctx context.Context, _ domain.Task, path string) error {
|
||||
if path == "" {
|
||||
return fmt.Errorf("worktree: path required")
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "git", "-C", w.Repo, "worktree", "remove", "--force", path)
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("%s: %w", string(out), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ProjectRepo is the minimal shape PerProjectGitWorktrees needs from a
|
||||
// project's registry entry — kept local (not importing internal/registry)
|
||||
// so orchestrator does not depend on registry's config-loading concerns.
|
||||
type ProjectRepo struct {
|
||||
Repo string
|
||||
WorktreeRoot string
|
||||
}
|
||||
|
||||
// PerProjectGitWorktrees resolves a task's repo/root by its project (spec
|
||||
// §2.2: each project is first-class and may have its own checkout), falling
|
||||
// back to Default for any project not present in Projects — this keeps
|
||||
// single-repo deployments working unchanged.
|
||||
type PerProjectGitWorktrees struct {
|
||||
Projects map[string]ProjectRepo
|
||||
Default GitWorktrees
|
||||
TaskFileSHA string
|
||||
}
|
||||
|
||||
func (w PerProjectGitWorktrees) Spec(t domain.Task) (string, string, bool) {
|
||||
g := w.Default
|
||||
if p, ok := w.Projects[t.Project]; ok && p.Repo != "" && p.WorktreeRoot != "" {
|
||||
g = GitWorktrees{Root: p.WorktreeRoot, Repo: p.Repo}
|
||||
}
|
||||
return g.Spec(t)
|
||||
}
|
||||
|
||||
func (w PerProjectGitWorktrees) Create(ctx context.Context, t domain.Task) (string, error) {
|
||||
g := w.Default
|
||||
if p, ok := w.Projects[t.Project]; ok && p.Repo != "" && p.WorktreeRoot != "" {
|
||||
g = GitWorktrees{Root: p.WorktreeRoot, Repo: p.Repo, TaskFileSHA: w.TaskFileSHA}
|
||||
}
|
||||
if g.TaskFileSHA == "" {
|
||||
g.TaskFileSHA = w.TaskFileSHA
|
||||
}
|
||||
return g.Create(ctx, t)
|
||||
}
|
||||
|
||||
func (w PerProjectGitWorktrees) Remove(ctx context.Context, t domain.Task, path string) error {
|
||||
g := w.Default
|
||||
if p, ok := w.Projects[t.Project]; ok && p.Repo != "" && p.WorktreeRoot != "" {
|
||||
g = GitWorktrees{Root: p.WorktreeRoot, Repo: p.Repo}
|
||||
}
|
||||
return g.Remove(ctx, t, path)
|
||||
}
|
||||
|
||||
type AdapterFactory struct{ Herdrs map[string]herdr.Adapter }
|
||||
|
||||
func (f AdapterFactory) Adapter(id string) (herdr.Adapter, error) {
|
||||
@@ -90,6 +158,21 @@ type MonitorHealth struct {
|
||||
LastRun time.Time `json:"last_run"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
Expired int `json:"expired"`
|
||||
// TurnBoundaryDegraded counts rotation ticks where Face B (spec §5.2,
|
||||
// §5.3 — "the Stop-hook/Face-B decides rotation, not the router") could
|
||||
// not be consulted, so occupancy-only thresholding is standing in. This
|
||||
// must stay observable rather than a silent fallback: an operator (or
|
||||
// the brief) can see when a deployment's rotation safety is degraded.
|
||||
TurnBoundaryDegraded int `json:"turn_boundary_degraded"`
|
||||
Sessions map[string]SessionHealth `json:"sessions,omitempty"`
|
||||
}
|
||||
|
||||
type SessionHealth struct {
|
||||
Status string `json:"status,omitempty"`
|
||||
WaitingForApproval bool `json:"waiting_for_approval"`
|
||||
Blocker string `json:"blocker,omitempty"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
}
|
||||
|
||||
func (c *Coordinator) MonitorHealth() MonitorHealth {
|
||||
@@ -110,6 +193,58 @@ func (c *Coordinator) setMonitorHealth(err error, expired int) {
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Coordinator) recordTurnBoundaryDegraded() {
|
||||
c.healthMu.Lock()
|
||||
defer c.healthMu.Unlock()
|
||||
c.health.TurnBoundaryDegraded++
|
||||
}
|
||||
|
||||
func waitingForApproval(status string) bool {
|
||||
s := strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(status, "-", "_"), " ", "_"))
|
||||
return s == "waiting_for_approval" || s == "awaiting_approval" || s == "approval_required"
|
||||
}
|
||||
|
||||
func (c *Coordinator) refreshSessionHealth(ctx context.Context) {
|
||||
c.loadSessions()
|
||||
c.mu.Lock()
|
||||
sessions := make(map[string]herdr.Session, len(c.sessions))
|
||||
for id, s := range c.sessions {
|
||||
sessions[id] = s
|
||||
}
|
||||
c.mu.Unlock()
|
||||
c.healthMu.Lock()
|
||||
if c.health.Sessions == nil {
|
||||
c.health.Sessions = map[string]SessionHealth{}
|
||||
}
|
||||
c.healthMu.Unlock()
|
||||
for taskID, session := range sessions {
|
||||
adapterID := session.HerdrID
|
||||
if adapterID == "" {
|
||||
if task, ok := c.Store.Task(taskID); ok && task.Lease != nil {
|
||||
adapterID = task.Lease.HarnessID
|
||||
}
|
||||
}
|
||||
a, err := c.Adapters.Adapter(adapterID)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
p, ok := a.(herdr.AgentStatus)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
status, err := p.AgentStatus(ctx, session)
|
||||
h := SessionHealth{Status: status, WaitingForApproval: waitingForApproval(status), UpdatedAt: time.Now().UTC()}
|
||||
if err != nil {
|
||||
h.LastError = err.Error()
|
||||
} else if blocker, ok := a.(herdr.AgentBlocker); ok && strings.EqualFold(status, "blocked") {
|
||||
h.Blocker, _ = blocker.AgentBlocker(ctx, session)
|
||||
}
|
||||
c.healthMu.Lock()
|
||||
c.health.Sessions[taskID] = h
|
||||
c.healthMu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Coordinator) loadSessions() {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
@@ -148,7 +283,6 @@ func (c *Coordinator) saveSessionsLocked() error {
|
||||
func (c *Coordinator) Reconcile(ctx context.Context) error {
|
||||
c.loadSessions()
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
for taskID, session := range c.sessions {
|
||||
t, ok := c.Store.Task(taskID)
|
||||
if ok && t.State == domain.StateLeased {
|
||||
@@ -159,7 +293,16 @@ func (c *Coordinator) Reconcile(ctx context.Context) error {
|
||||
}
|
||||
delete(c.sessions, taskID)
|
||||
}
|
||||
return c.saveSessionsLocked()
|
||||
err := c.saveSessionsLocked()
|
||||
c.mu.Unlock()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Session health is derived state. Rebuild it immediately from the
|
||||
// durable session mappings so a restart does not hide an outstanding
|
||||
// approval until the first periodic monitor tick.
|
||||
c.refreshSessionHealth(ctx)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Monitor performs conservative hard-threshold rotation. The adapter owns
|
||||
@@ -183,6 +326,8 @@ func (c *Coordinator) Monitor(ctx context.Context, hard float64, interval time.D
|
||||
c.healthMu.Unlock()
|
||||
return ctx.Err()
|
||||
case <-t.C:
|
||||
c.refreshSessionHealth(ctx)
|
||||
c.cleanupCompleted(ctx)
|
||||
expired, err := c.expire(ctx)
|
||||
c.setMonitorHealth(err, len(expired))
|
||||
if err != nil {
|
||||
@@ -193,6 +338,34 @@ func (c *Coordinator) Monitor(ctx context.Context, hard float64, interval time.D
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Coordinator) cleanupCompleted(ctx context.Context) {
|
||||
cleaner, ok := c.Worktrees.(WorktreeCleaner)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
c.loadSessions()
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
changed := false
|
||||
for taskID, session := range c.sessions {
|
||||
t, exists := c.Store.Task(taskID)
|
||||
if !exists || t.State != domain.StateCompleted {
|
||||
continue
|
||||
}
|
||||
if err := cleaner.Remove(ctx, t, session.Worktree); err != nil {
|
||||
c.healthMu.Lock()
|
||||
c.health.LastError = "worktree cleanup: " + err.Error()
|
||||
c.healthMu.Unlock()
|
||||
continue
|
||||
}
|
||||
delete(c.sessions, taskID)
|
||||
changed = true
|
||||
}
|
||||
if changed {
|
||||
_ = c.saveSessionsLocked()
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Coordinator) expire(ctx context.Context) ([]domain.Event, error) {
|
||||
// pane.exited is the low-latency path; lease expiry below remains the
|
||||
// authoritative backstop when herdr misses an exit notification.
|
||||
@@ -204,7 +377,7 @@ func (c *Coordinator) expire(ctx context.Context) ([]domain.Event, error) {
|
||||
if p, ok := a.(herdr.PaneExit); ok {
|
||||
if exited, ee := p.PaneExited(ctx, s); ee == nil && exited {
|
||||
b, _ := json.Marshal(map[string]string{"reason": "pane_exited", "harness_id": s.Harness})
|
||||
_ = c.Store.Append(domain.Event{ID: domain.NewID(), Type: "TaskReleased", TaskID: taskID, Version: t.Version + 1, Payload: b})
|
||||
_ = c.Store.Append(domain.Event{ID: domain.NewID(), Type: "TaskReleased", TaskID: taskID, Version: t.Version + 1, Payload: b, Surface: string(authz.System)})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -257,11 +430,25 @@ func (c *Coordinator) rotate(ctx context.Context, hard float64) {
|
||||
if err != nil || (occupancy < hard && reason == "threshold") {
|
||||
continue
|
||||
}
|
||||
// Face B is treated as required, not best-effort (spec §5.2/§5.3):
|
||||
// an adapter that supports the turn-boundary probe but fails to
|
||||
// answer it blocks this tick's release rather than silently
|
||||
// proceeding as if mid-turn interruption were safe. Only an
|
||||
// adapter that genuinely does not implement TurnBoundary at all
|
||||
// falls back to occupancy-only thresholding, and that fallback is
|
||||
// recorded so it is observable (MonitorHealth.TurnBoundaryDegraded)
|
||||
// instead of invisible.
|
||||
if boundary, ok := a.(herdr.TurnBoundary); ok {
|
||||
atBoundary, boundaryErr := boundary.AtTurnBoundary(ctx, session)
|
||||
if boundaryErr == nil && !atBoundary {
|
||||
if boundaryErr != nil {
|
||||
c.recordTurnBoundaryDegraded()
|
||||
continue
|
||||
}
|
||||
if !atBoundary {
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
c.recordTurnBoundaryDegraded()
|
||||
}
|
||||
ref, err := a.Release(ctx, session)
|
||||
if err != nil {
|
||||
@@ -270,8 +457,16 @@ func (c *Coordinator) rotate(ctx context.Context, hard float64) {
|
||||
if ref == "" {
|
||||
continue
|
||||
}
|
||||
b, _ := json.Marshal(map[string]string{"handoff_ref": ref, "reason": reason})
|
||||
e := domain.Event{ID: domain.NewID(), Type: "TaskReleased", TaskID: taskID, Version: task.Version + 1, Payload: b}
|
||||
anchorSHA, err := herdr.HeadSHA(session.Worktree)
|
||||
if err != nil {
|
||||
// Cannot certify the anchor: do not release with an invalid
|
||||
// TaskReleased payload (it would fail validation and strand
|
||||
// the lease/session). Leave the lease intact for the next
|
||||
// tick or TTL expiry to reclaim.
|
||||
continue
|
||||
}
|
||||
b, _ := json.Marshal(map[string]string{"handoff_ref": ref, "reason": reason, "anchor_sha": anchorSHA})
|
||||
e := domain.Event{ID: domain.NewID(), Type: "TaskReleased", TaskID: taskID, Version: task.Version + 1, Payload: b, Surface: string(authz.System)}
|
||||
if c.Store.Append(e) == nil {
|
||||
c.mu.Lock()
|
||||
delete(c.sessions, taskID)
|
||||
@@ -300,14 +495,27 @@ func (c *Coordinator) Start(ctx context.Context, e domain.Event) error {
|
||||
if err := json.Unmarshal(e.Payload, &p); err != nil || p.HarnessID == "" {
|
||||
return fmt.Errorf("orchestrator: invalid lease")
|
||||
}
|
||||
w, err := c.Worktrees.Create(ctx, t)
|
||||
if err != nil {
|
||||
return c.block(t, "worktree: "+err.Error())
|
||||
}
|
||||
a, err := c.Adapters.Adapter(p.HarnessID)
|
||||
if err != nil {
|
||||
return c.block(t, "adapter: "+err.Error())
|
||||
}
|
||||
var w string
|
||||
if creator, ok := a.(herdr.WorktreeCreator); ok {
|
||||
planner, planned := c.Worktrees.(WorktreeSpec)
|
||||
if !planned {
|
||||
return c.block(t, "worktree: repository specification unavailable")
|
||||
}
|
||||
repo, root, valid := planner.Spec(t)
|
||||
if !valid {
|
||||
return c.block(t, "worktree: repository and root required")
|
||||
}
|
||||
w, err = creator.CreateWorktree(ctx, repo, root, t.ID)
|
||||
} else {
|
||||
w, err = c.Worktrees.Create(ctx, t)
|
||||
}
|
||||
if err != nil {
|
||||
return c.block(t, "worktree: "+err.Error())
|
||||
}
|
||||
s, err := a.Lease(ctx, t.ID, w)
|
||||
if err != nil {
|
||||
return c.block(t, "lease: "+err.Error())
|
||||
@@ -318,6 +526,7 @@ func (c *Coordinator) Start(ctx context.Context, e domain.Event) error {
|
||||
return c.block(t, "bootstrap: "+err.Error())
|
||||
}
|
||||
}
|
||||
s.HerdrID = p.HarnessID
|
||||
c.mu.Lock()
|
||||
if c.sessions == nil {
|
||||
c.sessions = map[string]herdr.Session{}
|
||||
@@ -325,12 +534,18 @@ func (c *Coordinator) Start(ctx context.Context, e domain.Event) error {
|
||||
c.sessions[t.ID] = s
|
||||
err = c.saveSessionsLocked()
|
||||
c.mu.Unlock()
|
||||
c.healthMu.Lock()
|
||||
if c.health.Sessions == nil {
|
||||
c.health.Sessions = map[string]SessionHealth{}
|
||||
}
|
||||
c.health.Sessions[t.ID] = SessionHealth{Status: "running", UpdatedAt: time.Now().UTC()}
|
||||
c.healthMu.Unlock()
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Coordinator) block(t domain.Task, reason string) error {
|
||||
b, _ := json.Marshal(map[string]string{"blocker": reason})
|
||||
return c.Store.Append(domain.Event{ID: domain.NewID(), Type: "TaskBlocked", TaskID: t.ID, Version: t.Version + 1, Payload: b})
|
||||
return c.Store.Append(domain.Event{ID: domain.NewID(), Type: "TaskBlocked", TaskID: t.ID, Version: t.Version + 1, Payload: b, Surface: string(authz.System)})
|
||||
}
|
||||
|
||||
func (c *Coordinator) Session(taskID string) (herdr.Session, bool) {
|
||||
@@ -340,3 +555,25 @@ func (c *Coordinator) Session(taskID string) (herdr.Session, bool) {
|
||||
s, ok := c.sessions[taskID]
|
||||
return s, ok
|
||||
}
|
||||
|
||||
func (c *Coordinator) Capture(ctx context.Context, taskID, source string) (string, error) {
|
||||
s, ok := c.Session(taskID)
|
||||
if !ok {
|
||||
return "", domain.ErrNotFound
|
||||
}
|
||||
id := s.HerdrID
|
||||
if id == "" {
|
||||
if t, ok := c.Store.Task(taskID); ok && t.Lease != nil {
|
||||
id = t.Lease.HarnessID
|
||||
}
|
||||
}
|
||||
a, err := c.Adapters.Adapter(id)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
p, ok := a.(herdr.PaneCapture)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("pane capture unsupported")
|
||||
}
|
||||
return p.PaneCapture(ctx, s, source)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,267 @@
|
||||
package orchestrator_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"orchestra/internal/authz"
|
||||
"orchestra/internal/domain"
|
||||
"orchestra/internal/herdr"
|
||||
"orchestra/internal/orchestrator"
|
||||
"orchestra/internal/store"
|
||||
"os/exec"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type fakeAdapter struct {
|
||||
occupancy float64
|
||||
boundary bool
|
||||
ref string
|
||||
releases int
|
||||
}
|
||||
|
||||
func (a *fakeAdapter) Lease(_ context.Context, _ string, worktree string) (herdr.Session, error) {
|
||||
return herdr.Session{Harness: "h1", PaneID: "pane-1", Worktree: worktree}, nil
|
||||
}
|
||||
func (a *fakeAdapter) Bootstrap(context.Context, herdr.Session, string) error { return nil }
|
||||
func (a *fakeAdapter) Release(context.Context, herdr.Session) (string, error) {
|
||||
a.releases++
|
||||
return a.ref, nil
|
||||
}
|
||||
func (a *fakeAdapter) Kill(context.Context, herdr.Session) error { return nil }
|
||||
func (a *fakeAdapter) Occupancy(herdr.Session) (float64, error) { return a.occupancy, nil }
|
||||
func (a *fakeAdapter) AtTurnBoundary(context.Context, herdr.Session) (bool, error) {
|
||||
return a.boundary, nil
|
||||
}
|
||||
|
||||
type worktrees struct{ path string }
|
||||
|
||||
func (w worktrees) Create(context.Context, domain.Task) (string, error) { return w.path, nil }
|
||||
|
||||
type adapters struct{ a herdr.Adapter }
|
||||
|
||||
func (a adapters) Adapter(string) (herdr.Adapter, error) { return a.a, nil }
|
||||
|
||||
func run(t *testing.T, dir string, args ...string) {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", append([]string{"-C", dir}, args...)...)
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
t.Fatalf("git %v: %v: %s", args, err, out)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRotationEmitsValidReleaseWithAnchorSHA guards the highest-priority spec
|
||||
// defect noted in progress.md: automated rotation must emit a TaskReleased
|
||||
// event that satisfies domain.ValidatePayload (handoff_ref + anchor_sha), not
|
||||
// a payload missing anchor_sha that silently fails to append.
|
||||
func TestRotationEmitsValidReleaseWithAnchorSHA(t *testing.T) {
|
||||
repo := t.TempDir()
|
||||
run(t, repo, "init")
|
||||
run(t, repo, "config", "user.email", "t@t")
|
||||
run(t, repo, "config", "user.name", "t")
|
||||
run(t, repo, "commit", "--allow-empty", "-m", "init")
|
||||
head, err := herdr.HeadSHA(repo)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
s, err := store.Open(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Append(domain.Event{ID: domain.NewID(), Type: "TaskCreated", TaskID: "t1", Surface: string(authz.System), Payload: mustJSON(map[string]any{
|
||||
"source": "jsonl", "external_id": "1", "project": "p",
|
||||
})}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := s.Tasks()[0]
|
||||
ref, err := s.PutArtifact([]byte("handoff"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := &fakeAdapter{occupancy: .95, boundary: true, ref: ref}
|
||||
c := &orchestrator.Coordinator{Store: s, Worktrees: worktrees{path: repo}, Adapters: adapters{a}, StatePath: t.TempDir() + "/sessions.json"}
|
||||
|
||||
leaseEvt, err := s.Lease(task.ID, "h1", time.Minute)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Start(context.Background(), leaseEvt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go c.Monitor(ctx, .8, time.Millisecond)
|
||||
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if got, ok := s.Task(task.ID); ok && got.State == domain.StateQueued {
|
||||
break
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
got, ok := s.Task(task.ID)
|
||||
if !ok || got.State != domain.StateQueued {
|
||||
t.Fatalf("rotation did not complete: state=%v ok=%v", got.State, ok)
|
||||
}
|
||||
if a.releases == 0 {
|
||||
t.Fatalf("adapter Release was never invoked")
|
||||
}
|
||||
|
||||
// Walk raw events to confirm the coordinator itself wrote a valid
|
||||
// TaskReleased payload with anchor_sha == the worktree's real HEAD.
|
||||
found := false
|
||||
for _, e := range s.Events(0) {
|
||||
if e.TaskID != task.ID || e.Type != "TaskReleased" {
|
||||
continue
|
||||
}
|
||||
var p map[string]any
|
||||
if err := json.Unmarshal(e.Payload, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := domain.ValidatePayload("TaskReleased", p); err != nil {
|
||||
t.Fatalf("coordinator emitted invalid TaskReleased: %v (%v)", err, p)
|
||||
}
|
||||
if p["anchor_sha"] != head {
|
||||
t.Fatalf("anchor_sha=%v want=%s", p["anchor_sha"], head)
|
||||
}
|
||||
found = true
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("coordinator never emitted a TaskReleased event")
|
||||
}
|
||||
}
|
||||
|
||||
func mustJSON(v any) []byte { b, _ := json.Marshal(v); return b }
|
||||
|
||||
// erroringBoundaryAdapter supports Face B but its probe always fails — this
|
||||
// must block release (never silently treat an unanswerable boundary check
|
||||
// as safe to interrupt), unlike an adapter that doesn't implement the
|
||||
// interface at all.
|
||||
type erroringBoundaryAdapter struct{ fakeAdapter }
|
||||
|
||||
func (a *erroringBoundaryAdapter) AtTurnBoundary(context.Context, herdr.Session) (bool, error) {
|
||||
return false, errors.New("pane.status unsupported")
|
||||
}
|
||||
|
||||
// noBoundaryAdapter never implements herdr.TurnBoundary at all, exercising
|
||||
// the genuine occupancy-only degraded fallback.
|
||||
type noBoundaryAdapter struct {
|
||||
occupancy float64
|
||||
ref string
|
||||
releases int
|
||||
}
|
||||
|
||||
func (a *noBoundaryAdapter) Lease(_ context.Context, _ string, worktree string) (herdr.Session, error) {
|
||||
return herdr.Session{Harness: "h1", PaneID: "pane-1", Worktree: worktree}, nil
|
||||
}
|
||||
func (a *noBoundaryAdapter) Bootstrap(context.Context, herdr.Session, string) error { return nil }
|
||||
func (a *noBoundaryAdapter) Release(context.Context, herdr.Session) (string, error) {
|
||||
a.releases++
|
||||
return a.ref, nil
|
||||
}
|
||||
func (a *noBoundaryAdapter) Kill(context.Context, herdr.Session) error { return nil }
|
||||
func (a *noBoundaryAdapter) Occupancy(herdr.Session) (float64, error) { return a.occupancy, nil }
|
||||
|
||||
func setupRotationTask(t *testing.T, repo string) (*store.Store, string, domain.Task, string) {
|
||||
t.Helper()
|
||||
run(t, repo, "init")
|
||||
run(t, repo, "config", "user.email", "t@t")
|
||||
run(t, repo, "config", "user.name", "t")
|
||||
run(t, repo, "commit", "--allow-empty", "-m", "init")
|
||||
head, err := herdr.HeadSHA(repo)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s, err := store.Open(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Append(domain.Event{ID: domain.NewID(), Type: "TaskCreated", TaskID: "t1", Surface: string(authz.System), Payload: mustJSON(map[string]any{
|
||||
"source": "jsonl", "external_id": "1", "project": "p",
|
||||
})}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := s.Tasks()[0]
|
||||
ref, err := s.PutArtifact([]byte("handoff"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return s, head, task, ref
|
||||
}
|
||||
|
||||
// TestTurnBoundaryErrorBlocksRelease proves an adapter that implements Face B
|
||||
// but cannot currently answer it (a transient herdr error) never falls
|
||||
// through to an unconfirmed release — spec §5.2/§5.3 treats the boundary
|
||||
// check as required, not best-effort.
|
||||
func TestTurnBoundaryErrorBlocksRelease(t *testing.T) {
|
||||
repo := t.TempDir()
|
||||
s, _, task, ref := setupRotationTask(t, repo)
|
||||
a := &erroringBoundaryAdapter{fakeAdapter{occupancy: .95, boundary: true, ref: ref}}
|
||||
c := &orchestrator.Coordinator{Store: s, Worktrees: worktrees{path: repo}, Adapters: adapters{a}, StatePath: t.TempDir() + "/sessions.json"}
|
||||
|
||||
leaseEvt, err := s.Lease(task.ID, "h1", time.Minute)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Start(context.Background(), leaseEvt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go c.Monitor(ctx, .8, time.Millisecond)
|
||||
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
got, _ := s.Task(task.ID)
|
||||
if got.State != domain.StateLeased {
|
||||
t.Fatalf("release proceeded despite an unanswerable turn-boundary check: state=%s", got.State)
|
||||
}
|
||||
if a.releases != 0 {
|
||||
t.Fatalf("adapter.Release was called despite the boundary error, releases=%d", a.releases)
|
||||
}
|
||||
if c.MonitorHealth().TurnBoundaryDegraded == 0 {
|
||||
t.Fatal("turn-boundary degradation was not recorded")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNoTurnBoundarySupportDegradesVisibly proves an adapter that never
|
||||
// implements Face B still falls back to occupancy-only thresholding (so
|
||||
// existing deployments keep working) but the degradation is observable via
|
||||
// MonitorHealth, not silent.
|
||||
func TestNoTurnBoundarySupportDegradesVisibly(t *testing.T) {
|
||||
repo := t.TempDir()
|
||||
s, head, task, ref := setupRotationTask(t, repo)
|
||||
a := &noBoundaryAdapter{occupancy: .95, ref: ref}
|
||||
c := &orchestrator.Coordinator{Store: s, Worktrees: worktrees{path: repo}, Adapters: adapters{a}, StatePath: t.TempDir() + "/sessions.json"}
|
||||
|
||||
leaseEvt, err := s.Lease(task.ID, "h1", time.Minute)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Start(context.Background(), leaseEvt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go c.Monitor(ctx, .8, time.Millisecond)
|
||||
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if got, ok := s.Task(task.ID); ok && got.State == domain.StateQueued {
|
||||
break
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
got, ok := s.Task(task.ID)
|
||||
if !ok || got.State != domain.StateQueued {
|
||||
t.Fatalf("rotation did not complete without Face B support: state=%v ok=%v", got.State, ok)
|
||||
}
|
||||
_ = head
|
||||
if c.MonitorHealth().TurnBoundaryDegraded == 0 {
|
||||
t.Fatal("missing Face B support was not recorded as degraded")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package orchestrator_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"orchestra/internal/domain"
|
||||
"orchestra/internal/orchestrator"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func initRepo(t *testing.T, dir string) {
|
||||
t.Helper()
|
||||
run := func(args ...string) {
|
||||
cmd := exec.Command("git", args...)
|
||||
cmd.Dir = dir
|
||||
cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@t", "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@t")
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
t.Fatalf("git %v: %v: %s", args, err, out)
|
||||
}
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
run("init")
|
||||
if err := os.WriteFile(filepath.Join(dir, "README"), []byte("x"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
run("add", "README")
|
||||
run("commit", "-m", "init")
|
||||
}
|
||||
|
||||
func TestPerProjectGitWorktreesResolvesByProject(t *testing.T) {
|
||||
base := t.TempDir()
|
||||
repoA := filepath.Join(base, "repo-a")
|
||||
repoB := filepath.Join(base, "repo-b")
|
||||
initRepo(t, repoA)
|
||||
initRepo(t, repoB)
|
||||
|
||||
w := orchestrator.PerProjectGitWorktrees{
|
||||
Projects: map[string]orchestrator.ProjectRepo{
|
||||
"proj-a": {Repo: repoA, WorktreeRoot: filepath.Join(base, "wt-a")},
|
||||
},
|
||||
Default: orchestrator.GitWorktrees{Root: filepath.Join(base, "wt-default"), Repo: repoB},
|
||||
}
|
||||
|
||||
pathA, err := w.Create(context.Background(), domain.Task{ID: "t1", Project: "proj-a"})
|
||||
if err != nil {
|
||||
t.Fatalf("create for proj-a: %v", err)
|
||||
}
|
||||
if filepath.Dir(pathA) != filepath.Join(base, "wt-a") {
|
||||
t.Fatalf("expected proj-a worktree under wt-a, got %s", pathA)
|
||||
}
|
||||
|
||||
pathDefault, err := w.Create(context.Background(), domain.Task{ID: "t2", Project: "unconfigured-project"})
|
||||
if err != nil {
|
||||
t.Fatalf("create for unconfigured project: %v", err)
|
||||
}
|
||||
if filepath.Dir(pathDefault) != filepath.Join(base, "wt-default") {
|
||||
t.Fatalf("expected unconfigured project to use default worktree root, got %s", pathDefault)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user