checkpoint: multi-repo Gitea ingestion, per-project repos, rotation anchor_sha fix

Pre-existing uncommitted work found at session start: rotation now emits anchor_sha on TaskReleased (previously silently dropped by store.Append validation), multi-repo Gitea provider support, per-project git worktree roots, and associated test coverage. Committing as a checkpoint before starting remediation work tracked in AUDIT.md.
2026-07-27 18:15:02 +04:00
parent 325c684eb0
commit ce6f02f9e6
31 changed files with 2717 additions and 320 deletions
@@ -7,6 +7,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"orchestra/internal/authz"
 	"orchestra/internal/continuity"
 	"orchestra/internal/domain"
 	"orchestra/internal/herdr"
@@ -14,6 +15,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
 	"sync"
 	"time"
 )
@@ -21,6 +23,12 @@ import (
 type Worktrees interface {
 	Create(context.Context, domain.Task) (string, error)
 }
+type WorktreeSpec interface {
+	Spec(domain.Task) (string, string, bool)
+}
+type WorktreeCleaner interface {
+	Remove(context.Context, domain.Task, string) error
+}
 type Adapters interface {
 	Adapter(string) (herdr.Adapter, error)
 }
@@ -34,6 +42,10 @@ type GitWorktrees struct {
 	TaskFileSHA string
 }

+func (w GitWorktrees) Spec(domain.Task) (string, string, bool) {
+	return w.Repo, w.Root, w.Repo != "" && w.Root != ""
+}
+
 func (w GitWorktrees) Create(ctx context.Context, t domain.Task) (string, error) {
 	if w.Root == "" || w.Repo == "" {
 		return "", fmt.Errorf("worktree: root and repo required")
@@ -63,6 +75,62 @@ func (w GitWorktrees) Create(ctx context.Context, t domain.Task) (string, error)
 	return p, nil
 }

+func (w GitWorktrees) Remove(ctx context.Context, _ domain.Task, path string) error {
+	if path == "" {
+		return fmt.Errorf("worktree: path required")
+	}
+	cmd := exec.CommandContext(ctx, "git", "-C", w.Repo, "worktree", "remove", "--force", path)
+	if out, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("%s: %w", string(out), err)
+	}
+	return nil
+}
+
+// ProjectRepo is the minimal shape PerProjectGitWorktrees needs from a
+// project's registry entry — kept local (not importing internal/registry)
+// so orchestrator does not depend on registry's config-loading concerns.
+type ProjectRepo struct {
+	Repo         string
+	WorktreeRoot string
+}
+
+// PerProjectGitWorktrees resolves a task's repo/root by its project (spec
+// §2.2: each project is first-class and may have its own checkout), falling
+// back to Default for any project not present in Projects — this keeps
+// single-repo deployments working unchanged.
+type PerProjectGitWorktrees struct {
+	Projects    map[string]ProjectRepo
+	Default     GitWorktrees
+	TaskFileSHA string
+}
+
+func (w PerProjectGitWorktrees) Spec(t domain.Task) (string, string, bool) {
+	g := w.Default
+	if p, ok := w.Projects[t.Project]; ok && p.Repo != "" && p.WorktreeRoot != "" {
+		g = GitWorktrees{Root: p.WorktreeRoot, Repo: p.Repo}
+	}
+	return g.Spec(t)
+}
+
+func (w PerProjectGitWorktrees) Create(ctx context.Context, t domain.Task) (string, error) {
+	g := w.Default
+	if p, ok := w.Projects[t.Project]; ok && p.Repo != "" && p.WorktreeRoot != "" {
+		g = GitWorktrees{Root: p.WorktreeRoot, Repo: p.Repo, TaskFileSHA: w.TaskFileSHA}
+	}
+	if g.TaskFileSHA == "" {
+		g.TaskFileSHA = w.TaskFileSHA
+	}
+	return g.Create(ctx, t)
+}
+
+func (w PerProjectGitWorktrees) Remove(ctx context.Context, t domain.Task, path string) error {
+	g := w.Default
+	if p, ok := w.Projects[t.Project]; ok && p.Repo != "" && p.WorktreeRoot != "" {
+		g = GitWorktrees{Root: p.WorktreeRoot, Repo: p.Repo}
+	}
+	return g.Remove(ctx, t, path)
+}
+
 type AdapterFactory struct{ Herdrs map[string]herdr.Adapter }

 func (f AdapterFactory) Adapter(id string) (herdr.Adapter, error) {
@@ -90,6 +158,21 @@ type MonitorHealth struct {
 	LastRun   time.Time `json:"last_run"`
 	LastError string    `json:"last_error,omitempty"`
 	Expired   int       `json:"expired"`
+	// TurnBoundaryDegraded counts rotation ticks where Face B (spec §5.2,
+	// §5.3 — "the Stop-hook/Face-B decides rotation, not the router") could
+	// not be consulted, so occupancy-only thresholding is standing in. This
+	// must stay observable rather than a silent fallback: an operator (or
+	// the brief) can see when a deployment's rotation safety is degraded.
+	TurnBoundaryDegraded int                      `json:"turn_boundary_degraded"`
+	Sessions             map[string]SessionHealth `json:"sessions,omitempty"`
+}
+
+type SessionHealth struct {
+	Status             string    `json:"status,omitempty"`
+	WaitingForApproval bool      `json:"waiting_for_approval"`
+	Blocker            string    `json:"blocker,omitempty"`
+	UpdatedAt          time.Time `json:"updated_at"`
+	LastError          string    `json:"last_error,omitempty"`
 }

 func (c *Coordinator) MonitorHealth() MonitorHealth {
@@ -110,6 +193,58 @@ func (c *Coordinator) setMonitorHealth(err error, expired int) {
 	}
 }

+func (c *Coordinator) recordTurnBoundaryDegraded() {
+	c.healthMu.Lock()
+	defer c.healthMu.Unlock()
+	c.health.TurnBoundaryDegraded++
+}
+
+func waitingForApproval(status string) bool {
+	s := strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(status, "-", "_"), " ", "_"))
+	return s == "waiting_for_approval" || s == "awaiting_approval" || s == "approval_required"
+}
+
+func (c *Coordinator) refreshSessionHealth(ctx context.Context) {
+	c.loadSessions()
+	c.mu.Lock()
+	sessions := make(map[string]herdr.Session, len(c.sessions))
+	for id, s := range c.sessions {
+		sessions[id] = s
+	}
+	c.mu.Unlock()
+	c.healthMu.Lock()
+	if c.health.Sessions == nil {
+		c.health.Sessions = map[string]SessionHealth{}
+	}
+	c.healthMu.Unlock()
+	for taskID, session := range sessions {
+		adapterID := session.HerdrID
+		if adapterID == "" {
+			if task, ok := c.Store.Task(taskID); ok && task.Lease != nil {
+				adapterID = task.Lease.HarnessID
+			}
+		}
+		a, err := c.Adapters.Adapter(adapterID)
+		if err != nil {
+			continue
+		}
+		p, ok := a.(herdr.AgentStatus)
+		if !ok {
+			continue
+		}
+		status, err := p.AgentStatus(ctx, session)
+		h := SessionHealth{Status: status, WaitingForApproval: waitingForApproval(status), UpdatedAt: time.Now().UTC()}
+		if err != nil {
+			h.LastError = err.Error()
+		} else if blocker, ok := a.(herdr.AgentBlocker); ok && strings.EqualFold(status, "blocked") {
+			h.Blocker, _ = blocker.AgentBlocker(ctx, session)
+		}
+		c.healthMu.Lock()
+		c.health.Sessions[taskID] = h
+		c.healthMu.Unlock()
+	}
+}
+
 func (c *Coordinator) loadSessions() {
 	c.mu.Lock()
 	defer c.mu.Unlock()
@@ -148,7 +283,6 @@ func (c *Coordinator) saveSessionsLocked() error {
 func (c *Coordinator) Reconcile(ctx context.Context) error {
 	c.loadSessions()
 	c.mu.Lock()
-	defer c.mu.Unlock()
 	for taskID, session := range c.sessions {
 		t, ok := c.Store.Task(taskID)
 		if ok && t.State == domain.StateLeased {
@@ -159,7 +293,16 @@ func (c *Coordinator) Reconcile(ctx context.Context) error {
 		}
 		delete(c.sessions, taskID)
 	}
-	return c.saveSessionsLocked()
+	err := c.saveSessionsLocked()
+	c.mu.Unlock()
+	if err != nil {
+		return err
+	}
+	// Session health is derived state. Rebuild it immediately from the
+	// durable session mappings so a restart does not hide an outstanding
+	// approval until the first periodic monitor tick.
+	c.refreshSessionHealth(ctx)
+	return nil
 }

 // Monitor performs conservative hard-threshold rotation. The adapter owns
@@ -183,6 +326,8 @@ func (c *Coordinator) Monitor(ctx context.Context, hard float64, interval time.D
 			c.healthMu.Unlock()
 			return ctx.Err()
 		case <-t.C:
+			c.refreshSessionHealth(ctx)
+			c.cleanupCompleted(ctx)
 			expired, err := c.expire(ctx)
 			c.setMonitorHealth(err, len(expired))
 			if err != nil {
@@ -193,6 +338,34 @@ func (c *Coordinator) Monitor(ctx context.Context, hard float64, interval time.D
 	}
 }

+func (c *Coordinator) cleanupCompleted(ctx context.Context) {
+	cleaner, ok := c.Worktrees.(WorktreeCleaner)
+	if !ok {
+		return
+	}
+	c.loadSessions()
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	changed := false
+	for taskID, session := range c.sessions {
+		t, exists := c.Store.Task(taskID)
+		if !exists || t.State != domain.StateCompleted {
+			continue
+		}
+		if err := cleaner.Remove(ctx, t, session.Worktree); err != nil {
+			c.healthMu.Lock()
+			c.health.LastError = "worktree cleanup: " + err.Error()
+			c.healthMu.Unlock()
+			continue
+		}
+		delete(c.sessions, taskID)
+		changed = true
+	}
+	if changed {
+		_ = c.saveSessionsLocked()
+	}
+}
+
 func (c *Coordinator) expire(ctx context.Context) ([]domain.Event, error) {
 	// pane.exited is the low-latency path; lease expiry below remains the
 	// authoritative backstop when herdr misses an exit notification.
@@ -204,7 +377,7 @@ func (c *Coordinator) expire(ctx context.Context) ([]domain.Event, error) {
 				if p, ok := a.(herdr.PaneExit); ok {
 					if exited, ee := p.PaneExited(ctx, s); ee == nil && exited {
 						b, _ := json.Marshal(map[string]string{"reason": "pane_exited", "harness_id": s.Harness})
-						_ = c.Store.Append(domain.Event{ID: domain.NewID(), Type: "TaskReleased", TaskID: taskID, Version: t.Version + 1, Payload: b})
+						_ = c.Store.Append(domain.Event{ID: domain.NewID(), Type: "TaskReleased", TaskID: taskID, Version: t.Version + 1, Payload: b, Surface: string(authz.System)})
 					}
 				}
 			}
@@ -257,11 +430,25 @@ func (c *Coordinator) rotate(ctx context.Context, hard float64) {
 		if err != nil || (occupancy < hard && reason == "threshold") {
 			continue
 		}
+		// Face B is treated as required, not best-effort (spec §5.2/§5.3):
+		// an adapter that supports the turn-boundary probe but fails to
+		// answer it blocks this tick's release rather than silently
+		// proceeding as if mid-turn interruption were safe. Only an
+		// adapter that genuinely does not implement TurnBoundary at all
+		// falls back to occupancy-only thresholding, and that fallback is
+		// recorded so it is observable (MonitorHealth.TurnBoundaryDegraded)
+		// instead of invisible.
 		if boundary, ok := a.(herdr.TurnBoundary); ok {
 			atBoundary, boundaryErr := boundary.AtTurnBoundary(ctx, session)
-			if boundaryErr == nil && !atBoundary {
+			if boundaryErr != nil {
+				c.recordTurnBoundaryDegraded()
 				continue
 			}
+			if !atBoundary {
+				continue
+			}
+		} else {
+			c.recordTurnBoundaryDegraded()
 		}
 		ref, err := a.Release(ctx, session)
 		if err != nil {
@@ -270,8 +457,16 @@ func (c *Coordinator) rotate(ctx context.Context, hard float64) {
 		if ref == "" {
 			continue
 		}
-		b, _ := json.Marshal(map[string]string{"handoff_ref": ref, "reason": reason})
-		e := domain.Event{ID: domain.NewID(), Type: "TaskReleased", TaskID: taskID, Version: task.Version + 1, Payload: b}
+		anchorSHA, err := herdr.HeadSHA(session.Worktree)
+		if err != nil {
+			// Cannot certify the anchor: do not release with an invalid
+			// TaskReleased payload (it would fail validation and strand
+			// the lease/session). Leave the lease intact for the next
+			// tick or TTL expiry to reclaim.
+			continue
+		}
+		b, _ := json.Marshal(map[string]string{"handoff_ref": ref, "reason": reason, "anchor_sha": anchorSHA})
+		e := domain.Event{ID: domain.NewID(), Type: "TaskReleased", TaskID: taskID, Version: task.Version + 1, Payload: b, Surface: string(authz.System)}
 		if c.Store.Append(e) == nil {
 			c.mu.Lock()
 			delete(c.sessions, taskID)
@@ -300,14 +495,27 @@ func (c *Coordinator) Start(ctx context.Context, e domain.Event) error {
 	if err := json.Unmarshal(e.Payload, &p); err != nil || p.HarnessID == "" {
 		return fmt.Errorf("orchestrator: invalid lease")
 	}
-	w, err := c.Worktrees.Create(ctx, t)
-	if err != nil {
-		return c.block(t, "worktree: "+err.Error())
-	}
 	a, err := c.Adapters.Adapter(p.HarnessID)
 	if err != nil {
 		return c.block(t, "adapter: "+err.Error())
 	}
+	var w string
+	if creator, ok := a.(herdr.WorktreeCreator); ok {
+		planner, planned := c.Worktrees.(WorktreeSpec)
+		if !planned {
+			return c.block(t, "worktree: repository specification unavailable")
+		}
+		repo, root, valid := planner.Spec(t)
+		if !valid {
+			return c.block(t, "worktree: repository and root required")
+		}
+		w, err = creator.CreateWorktree(ctx, repo, root, t.ID)
+	} else {
+		w, err = c.Worktrees.Create(ctx, t)
+	}
+	if err != nil {
+		return c.block(t, "worktree: "+err.Error())
+	}
 	s, err := a.Lease(ctx, t.ID, w)
 	if err != nil {
 		return c.block(t, "lease: "+err.Error())
@@ -318,6 +526,7 @@ func (c *Coordinator) Start(ctx context.Context, e domain.Event) error {
 			return c.block(t, "bootstrap: "+err.Error())
 		}
 	}
+	s.HerdrID = p.HarnessID
 	c.mu.Lock()
 	if c.sessions == nil {
 		c.sessions = map[string]herdr.Session{}
@@ -325,12 +534,18 @@ func (c *Coordinator) Start(ctx context.Context, e domain.Event) error {
 	c.sessions[t.ID] = s
 	err = c.saveSessionsLocked()
 	c.mu.Unlock()
+	c.healthMu.Lock()
+	if c.health.Sessions == nil {
+		c.health.Sessions = map[string]SessionHealth{}
+	}
+	c.health.Sessions[t.ID] = SessionHealth{Status: "running", UpdatedAt: time.Now().UTC()}
+	c.healthMu.Unlock()
 	return err
 }

 func (c *Coordinator) block(t domain.Task, reason string) error {
 	b, _ := json.Marshal(map[string]string{"blocker": reason})
-	return c.Store.Append(domain.Event{ID: domain.NewID(), Type: "TaskBlocked", TaskID: t.ID, Version: t.Version + 1, Payload: b})
+	return c.Store.Append(domain.Event{ID: domain.NewID(), Type: "TaskBlocked", TaskID: t.ID, Version: t.Version + 1, Payload: b, Surface: string(authz.System)})
 }

 func (c *Coordinator) Session(taskID string) (herdr.Session, bool) {
@@ -340,3 +555,25 @@ func (c *Coordinator) Session(taskID string) (herdr.Session, bool) {
 	s, ok := c.sessions[taskID]
 	return s, ok
 }
+
+func (c *Coordinator) Capture(ctx context.Context, taskID, source string) (string, error) {
+	s, ok := c.Session(taskID)
+	if !ok {
+		return "", domain.ErrNotFound
+	}
+	id := s.HerdrID
+	if id == "" {
+		if t, ok := c.Store.Task(taskID); ok && t.Lease != nil {
+			id = t.Lease.HarnessID
+		}
+	}
+	a, err := c.Adapters.Adapter(id)
+	if err != nil {
+		return "", err
+	}
+	p, ok := a.(herdr.PaneCapture)
+	if !ok {
+		return "", fmt.Errorf("pane capture unsupported")
+	}
+	return p.PaneCapture(ctx, s, source)
+}