orchestra/cmd/orchestra-worker/main.go

// orchestra-worker consumes router-issued leases for one local herdr. Homesrv
// remains the scheduler and CAS authority; this process owns only local Git
// and pane operations.
package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"orchestra/internal/buildinfo"
	"orchestra/internal/continuity"
	"orchestra/internal/domain"
	"orchestra/internal/federation"
	"orchestra/internal/herdr"
	"orchestra/internal/orchestrator"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"syscall"
	"time"
)

type worker struct {
	api                                    federation.Client
	herdr                                  *herdr.Client
	harnessID, harness, repo, root, remote string
	projects                               map[string]projectConfig
	cursor                                 uint64
	tasks                                  map[string]domain.Task
	sessions                               map[string]herdr.Session
	leases                                 map[string]lease
	releases                               map[string]releaseTransaction
	statePath                              string
	hard                                   float64
	registration                           federation.Worker
	lastError                              string
	lastErrorAt                            time.Time
	soft                                   float64
	window                                 int64
}

func (w *worker) recordError(err error) {
	if err == nil {
		return
	}
	w.lastError = err.Error()
	w.lastErrorAt = time.Now().UTC()
}

func (w *worker) health(ctx context.Context) federation.WorkerHealth {
	h := federation.WorkerHealth{HerdrStatus: "unknown"}
	for taskID, session := range w.sessions {
		// Workers currently advertise capacity one. Pick deterministically so a
		// recovered legacy state with more sessions remains intelligible.
		if h.ActiveTask == "" || taskID < h.ActiveTask {
			h.ActiveTask, h.ActivePane = taskID, session.PaneID
		}
	}
	if w.herdr != nil {
		checkCtx, cancel := context.WithTimeout(ctx, 3*time.Second)
		err := w.herdr.CheckProtocol(checkCtx, "17")
		cancel()
		h.CheckedAt = time.Now().UTC()
		if err == nil {
			h.HerdrStatus = "reachable"
		} else {
			h.HerdrStatus = "unreachable"
			w.recordError(fmt.Errorf("local herdr: %w", err))
		}
	}
	h.LastError, h.ErrorAt = w.lastError, w.lastErrorAt
	return h
}

type lease struct {
	HandoffRef         string    `json:"handoff_ref,omitempty"`
	TransactionID      string    `json:"transaction_id,omitempty"`
	AnchorSHA          string    `json:"anchor_sha,omitempty"`
	PickupAcknowledged bool      `json:"pickup_acknowledged,omitempty"`
	Version            int       `json:"version"`
	Until              time.Time `json:"until"`
}
type releaseTransaction struct {
	ID            string    `json:"id"`
	LeaseVersion  int       `json:"lease_version"`
	Ref           string    `json:"handoff_ref,omitempty"`
	AnchorSHA     string    `json:"anchor_sha,omitempty"`
	Phase         string    `json:"phase"` // prepared, anchor_pushed, event_committed, pickup_validated, predecessor_retired
	AgentReleased bool      `json:"agent_released,omitempty"`
	LastError     string    `json:"last_error,omitempty"`
	UpdatedAt     time.Time `json:"updated_at"`
}
type projectConfig struct {
	Repo        string `json:"repo"`
	Root        string `json:"worktree_root"`
	Remote      string `json:"remote"`
	QualityGate string `json:"quality_gate,omitempty"`
}
type completionEvidence struct {
	TaskID      string    `json:"task_id"`
	Project     string    `json:"project"`
	Worker      string    `json:"worker"`
	Harness     string    `json:"harness"`
	PaneID      string    `json:"pane_id"`
	BaseSHA     string    `json:"base_sha"`
	ResultSHA   string    `json:"result_sha"`
	Branch      string    `json:"branch"`
	Remote      string    `json:"remote"`
	QualityGate string    `json:"quality_gate,omitempty"`
	GateExit    int       `json:"gate_exit"`
	CompletedAt time.Time `json:"completed_at"`
}
type workerState struct {
	Cursor   uint64                        `json:"cursor"`
	Sessions map[string]herdr.Session      `json:"sessions"`
	Tasks    map[string]domain.Task        `json:"tasks"`
	Leases   map[string]lease              `json:"leases"`
	Releases map[string]releaseTransaction `json:"releases"`
}

func (w *worker) load() {
	b, e := os.ReadFile(w.statePath)
	if e == nil {
		var s workerState
		if json.Unmarshal(b, &s) == nil {
			w.cursor = s.Cursor
			w.sessions = s.Sessions
			w.tasks = s.Tasks
			w.leases = s.Leases
			w.releases = s.Releases
		}
	}
	if w.sessions == nil {
		w.sessions = map[string]herdr.Session{}
	}
	if w.tasks == nil {
		w.tasks = map[string]domain.Task{}
	}
	if w.leases == nil {
		w.leases = map[string]lease{}
	}
	if w.releases == nil {
		w.releases = map[string]releaseTransaction{}
	}
}
func (w *worker) save() error {
	b, e := json.Marshal(workerState{Cursor: w.cursor, Sessions: w.sessions, Tasks: w.tasks, Leases: w.leases, Releases: w.releases})
	if e != nil {
		return e
	}
	if e := os.MkdirAll(filepath.Dir(w.statePath), 0700); e != nil {
		return e
	}
	return os.WriteFile(w.statePath, b, 0600)
}

type artifactCAS struct{ api federation.Client }

func (c artifactCAS) PutArtifact(b []byte) (string, error) {
	return c.api.PutArtifact(context.Background(), b)
}
func (c artifactCAS) Artifact(ref string) ([]byte, error) {
	return c.api.Artifact(context.Background(), ref)
}

func created(e domain.Event) (domain.Task, bool) {
	if e.Type != "TaskCreated" {
		return domain.Task{}, false
	}
	var p struct {
		Source      string   `json:"source"`
		ExternalID  string   `json:"external_id"`
		Project     string   `json:"project"`
		Capability  []string `json:"capability"`
		Title       string   `json:"title"`
		Description string   `json:"description"`
		Acceptance  []string `json:"acceptance"`
		QualityGate string   `json:"quality_gate"`
	}
	if json.Unmarshal(e.Payload, &p) != nil || p.Source == "" || p.ExternalID == "" || p.Project == "" {
		return domain.Task{}, false
	}
	return domain.Task{ID: e.TaskID, Source: p.Source, ExternalID: p.ExternalID, Project: p.Project, Capability: p.Capability, Title: p.Title, Description: p.Description, Acceptance: p.Acceptance, QualityGate: p.QualityGate}, true
}

func (w *worker) project(t domain.Task) (projectConfig, error) {
	if w.projects != nil {
		if p, ok := w.projects[t.Project]; ok && p.Repo != "" && p.Root != "" && p.Remote != "" {
			return p, nil
		}
		return projectConfig{}, fmt.Errorf("project %q is not configured on worker", t.Project)
	}
	return projectConfig{Repo: w.repo, Root: w.root, Remote: w.remote}, nil
}

func (w *worker) syncBase(ctx context.Context, p projectConfig) error {
	if out, err := exec.CommandContext(ctx, "git", "-C", p.Repo, "fetch", p.Remote, "--prune").CombinedOutput(); err != nil {
		return fmt.Errorf("fetch base checkout: %s: %w", out, err)
	}
	branch, err := exec.CommandContext(ctx, "git", "-C", p.Repo, "symbolic-ref", "--quiet", "--short", "HEAD").Output()
	if err != nil {
		return fmt.Errorf("identify base branch: %w", err)
	}
	branchName := strings.TrimSpace(string(branch))
	if out, err := exec.CommandContext(ctx, "git", "-C", p.Repo, "merge", "--ff-only", p.Remote+"/"+branchName).CombinedOutput(); err != nil {
		return fmt.Errorf("fast-forward base checkout: %s: %w", out, err)
	}
	return nil
}

func (w *worker) start(ctx context.Context, t domain.Task, ref string) error {
	var wt string
	var h continuity.Handoff
	var err error
	p, err := w.project(t)
	if err != nil {
		return err
	}
	// Synchronize the local base before any worktree operation. A worker never
	// treats a coordinator-side path as truth; the Git remote is the only
	// cross-machine transport.
	if err := w.syncBase(ctx, p); err != nil {
		return err
	}
	if ref != "" {
		b, err := w.api.Artifact(ctx, ref)
		if err != nil {
			return err
		}
		h, err = continuity.Decode(b)
		if err != nil {
			return err
		}
		if out, err := exec.CommandContext(ctx, "git", "-C", p.Repo, "fetch", p.Remote, "--prune").CombinedOutput(); err != nil {
			return fmt.Errorf("fetch pickup anchor: %s: %w", out, err)
		}
		wt = filepath.Join(p.Root, t.ID)
		if _, err := os.Stat(wt); os.IsNotExist(err) {
			if out, err := exec.CommandContext(ctx, "git", "-C", p.Repo, "worktree", "add", "-b", "orchestra/"+t.ID, wt, h.Anchor.GitSHA).CombinedOutput(); err != nil {
				return fmt.Errorf("create pickup worktree: %s: %w", out, err)
			}
		}
		if err = continuity.ValidatePickup(wt, h, taskHash(t)); err != nil {
			return err
		}
	} else {
		wt, err = (orchestrator.GitWorktrees{Repo: p.Repo, Root: p.Root}).Create(ctx, t)
		if err != nil {
			return err
		}
	}
	if _, err = w.herdr.Worktree(ctx, p.Repo, wt, "orchestra/"+t.ID); err != nil {
		return err
	}
	s, err := w.herdr.StartAgent(ctx, wt, wt, "orchestra/"+t.ID, w.harness, t.ID)
	if err != nil {
		return err
	}
	s.TaskFileSHA = taskHash(t)
	prompt := "Read TASK.md at the worktree root and execute it."
	if ref != "" {
		prompt += " A validated handoff exists; inspect local Git history and the recorded checkpoint before continuing."
	}
	w.sessions[t.ID] = s
	if err := w.save(); err != nil {
		return err
	}
	// A prompt response can be lost after herdr accepted it. Persist the
	// session first so the worker can reconcile/release it after restart.
	if err := w.herdr.Prompt(ctx, s.PaneID, prompt, 0); err != nil {
		return err
	}
	if ref != "" {
		return w.ackPickup(ctx, t.ID, s)
	}
	return nil
}

func taskHash(t domain.Task) string { b := continuity.RenderTaskFile(t); return domain.Hash(b) }

func (w *worker) releaseReady(ctx context.Context) {
	for id, s := range w.sessions {
		if l := w.leases[id]; l.HandoffRef != "" && !l.PickupAcknowledged {
			if err := w.ackPickup(ctx, id, s); err != nil {
				w.recordError(err)
				continue
			}
		}
		if _, err := os.Stat(filepath.Join(s.Worktree, ".orchestra", "done")); err == nil {
			evidence, err := w.finalize(ctx, id, s)
			if err != nil {
				w.recordError(fmt.Errorf("complete %s: %w", id, err))
				log.Printf("complete %s: %v", id, err)
				continue
			}
			report, _ := json.Marshal(evidence)
			ref, err := w.api.PutArtifact(ctx, report)
			if err != nil {
				w.recordError(fmt.Errorf("upload completion %s: %w", id, err))
				log.Printf("upload completion %s: %v", id, err)
				continue
			}
			if err = w.api.Complete(ctx, id, ref, evidence.ResultSHA, evidence.Branch, evidence.Remote, w.leases[id].Version, w.usageReceipt(s), w.sessionEvidence(ctx, id, s)); err != nil {
				w.recordError(fmt.Errorf("complete %s: %w", id, err))
				log.Printf("complete %s: %v", id, err)
				continue
			}
			// Completion is durable before closing the exact pane. If close
			// fails, retain the session mapping for a later explicit cleanup.
			a := herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}
			if err := a.Kill(ctx, s); err != nil {
				w.recordError(fmt.Errorf("close completed pane %s: %w", id, err))
				log.Printf("close completed pane %s: %v", id, err)
				continue
			}
			_ = os.Remove(filepath.Join(s.Worktree, ".orchestra", "done"))
			_ = os.Remove(filepath.Join(s.Worktree, ".orchestra"))
			delete(w.sessions, id)
			delete(w.leases, id)
			_ = w.save()
			continue
		}
		if _, err := os.Stat(filepath.Join(s.Worktree, herdr.HandoffReportFile)); err == nil || w.releases[id].ID != "" {
			w.advanceRelease(ctx, id, s)
			continue
		}
		w.rotationTick(ctx, id, s)
	}
}

func (w *worker) adapter(s herdr.Session, remote string) herdr.CLIAdapter {
	a := herdr.CLIAdapter{Client: w.herdr, Harness: w.harness, Window: w.window, CAS: artifactCAS{w.api}, Remote: remote}
	switch w.harness {
	case "claude":
		a.Usage = herdr.ClaudeUsage
	case "codex":
		a.Usage = herdr.CodexUsage
	case "opencode":
		a.Usage = herdr.OpenCodeUsage
	}
	return a
}

// rotationTick is the checkout-owner state machine. Occupancy, tool activity,
// and pane status are all read from the persisted harness session identity;
// any unknown source is recorded and never treated as zero usage.
func (w *worker) rotationTick(ctx context.Context, id string, s herdr.Session) {
	t, ok := w.tasks[id]
	if !ok {
		w.recordError(fmt.Errorf("rotation %s: task cache missing", id))
		return
	}
	p, err := w.project(t)
	if err != nil {
		w.recordError(err)
		return
	}
	a := w.adapter(s, p.Remote)
	resolved, err := a.ResolveSessionIdentity(s)
	if err != nil {
		w.recordError(fmt.Errorf("rotation %s occupancy degraded: %w", id, err))
		return
	}
	if resolved != s {
		w.sessions[id] = resolved
		s = resolved
		_ = w.save()
	}
	d := (orchestrator.RotationStateMachine{Soft: w.soft, Hard: w.hard}).Evaluate(ctx, a, s)
	if d.ActivityDegraded != nil {
		w.recordError(fmt.Errorf("rotation %s activity degraded: %w", id, d.ActivityDegraded))
	}
	if d.Degraded != nil {
		w.recordError(fmt.Errorf("rotation %s degraded: %w", id, d.Degraded))
		if d.Action == orchestrator.TurnContinue || d.Action == "" {
			return
		}
	}
	if d.Action == orchestrator.TurnContinue || d.Action == orchestrator.TurnRefuse || s.HandoffRequested {
		return
	}
	if d.Reason == "milestone" || d.Reason == "thrash" {
		if err := a.RequestHandoffReason(ctx, s, d.Reason, d.DeadEnds); err != nil {
			w.recordError(fmt.Errorf("rotation %s %s prompt: %w", id, d.Reason, err))
			return
		}
	} else if err := a.RequestHandoff(ctx, s); err != nil {
		w.recordError(fmt.Errorf("rotation %s threshold prompt: %w", id, err))
		return
	}
	s.HandoffRequested, s.HandoffReason = true, d.Reason
	w.sessions[id] = s
	_ = w.save()
}

func (w *worker) advanceRelease(ctx context.Context, id string, s herdr.Session) {
	t, ok := w.tasks[id]
	if !ok {
		w.recordError(fmt.Errorf("release %s: task cache missing", id))
		return
	}
	p, err := w.project(t)
	if err != nil {
		w.recordError(fmt.Errorf("release %s: %w", id, err))
		return
	}
	if w.releases == nil {
		w.releases = map[string]releaseTransaction{}
	}
	tx := w.releases[id]
	if tx.ID == "" {
		l, ok := w.leases[id]
		if !ok {
			w.recordError(fmt.Errorf("release %s: lease missing", id))
			return
		}
		tx = releaseTransaction{ID: domain.NewID(), LeaseVersion: l.Version, Phase: "prepared", UpdatedAt: time.Now().UTC()}
		w.releases[id] = tx
		_ = w.save()
	}
	a := w.adapter(s, p.Remote)
	if tx.Phase == "prepared" {
		prepared, err := a.PrepareRelease(ctx, s)
		if err != nil {
			tx.LastError, tx.UpdatedAt = err.Error(), time.Now().UTC()
			w.releases[id] = tx
			_ = w.save()
			w.recordError(fmt.Errorf("release %s prepare: %w", id, err))
			return
		}
		tx.Ref, tx.AnchorSHA, tx.Phase, tx.LastError, tx.UpdatedAt = prepared.Ref, prepared.AnchorSHA, "anchor_pushed", "", time.Now().UTC()
		w.releases[id] = tx
		_ = w.save()
	}
	if tx.Phase == "anchor_pushed" {
		if err := w.api.Release(ctx, id, tx.Ref, tx.AnchorSHA, tx.ID, tx.LeaseVersion, w.sessionEvidence(ctx, id, s)); err != nil {
			tx.LastError, tx.UpdatedAt = err.Error(), time.Now().UTC()
			w.releases[id] = tx
			_ = w.save()
			w.recordError(fmt.Errorf("release %s commit: %w", id, err))
			return
		}
		tx.Phase, tx.LastError, tx.UpdatedAt = "event_committed", "", time.Now().UTC()
		w.releases[id] = tx
		_ = w.save()
	}
	if tx.Phase == "event_committed" && !tx.AgentReleased {
		if err := a.ReleaseAgent(ctx, s); err != nil {
			tx.LastError, tx.UpdatedAt = err.Error(), time.Now().UTC()
			w.releases[id] = tx
			_ = w.save()
			w.recordError(fmt.Errorf("release %s release agent: %w", id, err))
			return
		}
		tx.AgentReleased, tx.LastError, tx.UpdatedAt = true, "", time.Now().UTC()
		w.releases[id] = tx
		_ = w.save()
	}
	if tx.Phase == "pickup_validated" {
		if err := a.Kill(ctx, s); err != nil {
			tx.LastError, tx.UpdatedAt = err.Error(), time.Now().UTC()
			w.releases[id] = tx
			_ = w.save()
			w.recordError(fmt.Errorf("release %s retire predecessor: %w", id, err))
			return
		}
		_ = os.Remove(filepath.Join(s.Worktree, herdr.HandoffReportFile))
		tx.Phase, tx.UpdatedAt = "predecessor_retired", time.Now().UTC()
		w.releases[id] = tx
		_ = w.save()
		delete(w.sessions, id)
		delete(w.releases, id)
		_ = w.save()
	}
}

// ackPickup retries the successor acknowledgement from persisted lease state.
// It is safe after a lost response: the coordinator recognizes the exact
// transaction/lease epoch as an idempotent pickup.
func (w *worker) ackPickup(ctx context.Context, id string, s herdr.Session) error {
	l, ok := w.leases[id]
	if !ok || l.HandoffRef == "" || l.TransactionID == "" || l.AnchorSHA == "" {
		return fmt.Errorf("pickup %s: lease is missing its release transaction", id)
	}
	if l.PickupAcknowledged {
		return nil
	}
	if err := w.api.Pickup(ctx, id, l.HandoffRef, l.AnchorSHA, l.TransactionID, l.Version, w.sessionEvidence(ctx, id, s)); err != nil {
		return fmt.Errorf("pickup %s acknowledgement: %w", id, err)
	}
	l.PickupAcknowledged = true
	l.Version++ // TaskPickupValidated increments the task version.
	w.leases[id] = l
	return w.save()
}

func (w *worker) sessionEvidence(ctx context.Context, taskID string, s herdr.Session) domain.SessionEvidence {
	e := domain.SessionEvidence{PaneID: s.PaneID, HarnessID: w.harnessID, PaneState: "open", Source: "worker", CheckedAt: time.Now().UTC()}
	text, err := (herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}).PaneCapture(ctx, s, "recent")
	if err != nil {
		e.PaneState = "unreachable"
		return e
	}
	if capture, err := w.api.PublishCapture(ctx, federation.Capture{TaskID: taskID, PaneID: s.PaneID, Text: text}); err == nil {
		e.CapturedAt = capture.At
	}
	return e
}

func (w *worker) usageReceipt(s herdr.Session) map[string]any {
	if s.SessionFile == "" && !(w.harness == "opencode" && s.SessionID != "") {
		return map[string]any{"harness_id": w.harnessID, "consumed": 0}
	}
	var usage herdr.Usage
	var err error
	switch w.harness {
	case "claude":
		usage, err = herdr.ClaudeUsage(s.SessionFile)
	case "codex":
		usage, err = herdr.CodexUsage(s.SessionFile)
	case "opencode":
		usage, err = herdr.OpenCodeSessionUsage(s.SessionID)
	}
	if err != nil {
		return map[string]any{"harness_id": w.harnessID, "consumed": 0, "error": err.Error()}
	}
	return map[string]any{"harness_id": w.harnessID, "input_tokens": usage.Input, "cache_read_tokens": usage.CacheRead, "cache_write_tokens": usage.CacheWrite, "output_tokens": usage.Output, "consumed": usage.Numerator()}
}

func git(ctx context.Context, dir string, args ...string) ([]byte, error) {
	return exec.CommandContext(ctx, "git", append([]string{"-C", dir}, args...)...).CombinedOutput()
}

// finalize performs only mechanical delivery work. It never asks the harness
// to narrate Git state, gates, or a report; those are generated from the
// worker-owned checkout and then verified against the configured remote.
func (w *worker) finalize(ctx context.Context, id string, s herdr.Session) (completionEvidence, error) {
	t, ok := w.tasks[id]
	if !ok {
		return completionEvidence{}, fmt.Errorf("task cache missing")
	}
	p, err := w.project(t)
	if err != nil {
		return completionEvidence{}, err
	}
	if s.TaskFileSHA != "" {
		if err := continuity.VerifyTaskFile(s.Worktree, s.TaskFileSHA); err != nil {
			return completionEvidence{}, fmt.Errorf("verify immutable TASK.md: %w", err)
		}
	}
	base, err := git(ctx, s.Worktree, "rev-parse", "HEAD")
	if err != nil {
		return completionEvidence{}, fmt.Errorf("base sha: %s: %w", base, err)
	}
	e := completionEvidence{TaskID: id, Project: t.Project, Worker: w.harnessID, Harness: w.harness, PaneID: s.PaneID, BaseSHA: strings.TrimSpace(string(base)), Remote: p.Remote, QualityGate: t.QualityGate, CompletedAt: time.Now().UTC()}
	gateCommand := t.QualityGate
	if gateCommand == "" {
		gateCommand = p.QualityGate
	}
	e.QualityGate = gateCommand
	if gateCommand != "" {
		gate := exec.CommandContext(ctx, "sh", "-c", gateCommand)
		gate.Dir = s.Worktree
		if out, err := gate.CombinedOutput(); err != nil {
			e.GateExit = 1
			return completionEvidence{}, fmt.Errorf("quality gate %q: %s: %w", gateCommand, out, err)
		}
	}
	if _, err := git(ctx, s.Worktree, "diff", "--quiet", "--", "TASK.md"); err != nil {
		return completionEvidence{}, errors.New("TASK.md was modified")
	}
	if out, err := git(ctx, s.Worktree, "add", "-A", "--", ".", ":!.orchestra/done"); err != nil {
		return completionEvidence{}, fmt.Errorf("stage result: %s: %w", out, err)
	}
	if _, err := git(ctx, s.Worktree, "diff", "--cached", "--quiet"); err != nil {
		if out, err := git(ctx, s.Worktree, "commit", "-m", "orchestra: complete "+id); err != nil {
			return completionEvidence{}, fmt.Errorf("commit result: %s: %w", out, err)
		}
	}
	branch, err := git(ctx, s.Worktree, "branch", "--show-current")
	if err != nil || strings.TrimSpace(string(branch)) == "" {
		return completionEvidence{}, fmt.Errorf("result branch: %s: %w", branch, err)
	}
	e.Branch = strings.TrimSpace(string(branch))
	sha, err := git(ctx, s.Worktree, "rev-parse", "HEAD")
	if err != nil {
		return completionEvidence{}, fmt.Errorf("result sha: %s: %w", sha, err)
	}
	e.ResultSHA = strings.TrimSpace(string(sha))
	if out, err := git(ctx, s.Worktree, "push", p.Remote, "HEAD:refs/heads/"+e.Branch); err != nil {
		return completionEvidence{}, fmt.Errorf("push result: %s: %w", out, err)
	}
	remote, err := git(ctx, s.Worktree, "ls-remote", p.Remote, "refs/heads/"+e.Branch)
	if err != nil || !strings.HasPrefix(string(remote), e.ResultSHA+"\t") {
		return completionEvidence{}, fmt.Errorf("verify pushed sha: got %q: %w", strings.TrimSpace(string(remote)), err)
	}
	return e, nil
}

func (w *worker) renewLeases(ctx context.Context) {
	if w.herdr == nil {
		return
	}
	now := time.Now()
	for taskID, l := range w.leases {
		s, ok := w.sessions[taskID]
		if !ok || l.Version == 0 || l.Until.After(now.Add(10*time.Minute)) {
			continue
		}
		if _, err := (herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}).PaneCapture(ctx, s, "recent"); err != nil {
			w.recordError(fmt.Errorf("validate lease %s: %w", taskID, err))
			continue
		}
		if err := w.api.Renew(ctx, taskID, l.Version, int((30 * time.Minute).Seconds())); err != nil {
			w.recordError(fmt.Errorf("renew lease %s: %w", taskID, err))
			log.Printf("renew lease %s: %v", taskID, err)
		} else {
			// RenewLease appends one event. Retain that epoch locally until its
			// replay arrives so a release transaction uses the same version.
			l.Version++
			l.Until = now.Add(30 * time.Minute)
			w.leases[taskID] = l
			_ = w.save()
		}
	}
}

// publishCaptures makes remote panes observable without allowing the
// coordinator to touch their unix herdr socket.
func (w *worker) publishCaptures(ctx context.Context) {
	for taskID, session := range w.sessions {
		text, err := (herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}).PaneCapture(ctx, session, "recent")
		if err != nil {
			continue
		}
		if _, err := w.api.PublishCapture(ctx, federation.Capture{TaskID: taskID, PaneID: session.PaneID, Text: text}); err != nil {
			log.Printf("publish capture %s: %v", taskID, err)
		}
	}
}

type approvalInput struct {
	Text string
	Keys []string
}

func approvalResponse(text, kind string) (approvalInput, bool) {
	low := strings.ToLower(text)
	// Never invent a keystroke. y/n prompts label both decisions directly.
	if strings.Contains(low, "[y/n]") || strings.Contains(low, "(y/n)") {
		if kind == "grant_approval" {
			return approvalInput{Text: "y\n"}, true
		}
		return approvalInput{Text: "n\n"}, true
	}
	// OpenCode's explicit selector states "Allow once  Allow always  Reject"
	// and "enter confirm". Send a real ENTER key, not a newline through
	// pane.send_text: OpenCode's selector does not treat the latter as input.
	// Enter is consequently a bounded one-time grant;
	// rejection would require unobservable selector navigation, so refuse it.
	if kind == "grant_approval" && strings.Contains(low, "allow once") && strings.Contains(low, "allow always") && strings.Contains(low, "reject") && strings.Contains(low, "enter confirm") {
		return approvalInput{Keys: []string{"ENTER"}}, true
	}
	return approvalInput{}, false
}
func (w *worker) runCommands(ctx context.Context) {
	commands, err := w.api.Commands(ctx)
	if err != nil {
		log.Printf("poll controls: %v", err)
		return
	}
	for _, command := range commands {
		session, ok := w.sessions[command.TaskID]
		if !ok || session.PaneID != command.PaneID {
			_ = w.api.ResolveCommand(ctx, command.ID, "stale", "session or pane changed")
			continue
		}
		text, err := (herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}).PaneCapture(ctx, session, "recent")
		if err != nil {
			_ = w.api.ResolveCommand(ctx, command.ID, "rejected", "capture unavailable: "+err.Error())
			continue
		}
		capture, err := w.api.PublishCapture(ctx, federation.Capture{TaskID: command.TaskID, PaneID: session.PaneID, Text: text})
		if err != nil {
			_ = w.api.ResolveCommand(ctx, command.ID, "rejected", "cannot publish capture: "+err.Error())
			continue
		}
		if capture.Revision != command.CaptureRevision {
			_ = w.api.ResolveCommand(ctx, command.ID, "stale", "capture revision changed")
			continue
		}
		input, ok := approvalResponse(text, command.Kind)
		if !ok {
			_ = w.api.ResolveCommand(ctx, command.ID, "rejected", "prompt does not expose an executable approval control")
			continue
		}
		method, params := "pane.send_text", map[string]any{"pane_id": session.PaneID, "text": input.Text}
		if len(input.Keys) > 0 {
			method, params = "pane.send_keys", map[string]any{"pane_id": session.PaneID, "keys": input.Keys}
		}
		if err := w.herdr.Call(ctx, method, params, nil); err != nil {
			_ = w.api.ResolveCommand(ctx, command.ID, "rejected", "herdr did not acknowledge input: "+err.Error())
			continue
		}
		if err := w.api.ResolveCommand(ctx, command.ID, "acknowledged", ""); err != nil {
			log.Printf("ack command %s: %v", command.ID, err)
		}
	}
}

func (w *worker) once(ctx context.Context) error {
	es, _, err := w.api.Events(ctx, w.cursor)
	if err != nil {
		return err
	}
	for _, e := range es {
		if t, ok := created(e); ok {
			w.tasks[t.ID] = t
		}
		if e.Type == "TaskLeased" {
			var p struct {
				HarnessID     string `json:"harness_id"`
				HandoffRef    string `json:"handoff_ref"`
				TransactionID string `json:"transaction_id"`
				AnchorSHA     string `json:"anchor_sha"`
			}
			if json.Unmarshal(e.Payload, &p) == nil && p.HarnessID == w.harnessID {
				var until struct {
					UntilNS int64 `json:"until_ns"`
				}
				_ = json.Unmarshal(e.Payload, &until)
				w.leases[e.TaskID] = lease{HandoffRef: p.HandoffRef, TransactionID: p.TransactionID, AnchorSHA: p.AnchorSHA, Version: e.Version, Until: time.Unix(0, until.UntilNS)}
			}
		}
		if e.Type == "TaskLeaseRenewed" {
			var p struct {
				HarnessID string `json:"harness_id"`
				UntilNS   int64  `json:"until_ns"`
			}
			if json.Unmarshal(e.Payload, &p) == nil && p.HarnessID == w.harnessID {
				l := w.leases[e.TaskID]
				l.Version, l.Until = e.Version, time.Unix(0, p.UntilNS)
				w.leases[e.TaskID] = l
			}
		}
		if e.Type == "TaskCompleted" {
			delete(w.leases, e.TaskID)
			if session, active := w.sessions[e.TaskID]; active {
				if w.herdr == nil {
					delete(w.sessions, e.TaskID)
				} else if err := (herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}).Kill(ctx, session); err != nil {
					w.recordError(fmt.Errorf("close completed pane %s: %w", e.TaskID, err))
					continue
				} else {
					_ = os.Remove(filepath.Join(session.Worktree, ".orchestra", "done"))
					_ = os.Remove(filepath.Join(session.Worktree, ".orchestra"))
					delete(w.sessions, e.TaskID)
				}
			}
		}
		if e.Type == "TaskPickupValidated" {
			var p struct {
				TransactionID string `json:"transaction_id"`
			}
			if json.Unmarshal(e.Payload, &p) == nil {
				if tx := w.releases[e.TaskID]; tx.ID != "" && tx.ID == p.TransactionID {
					tx.Phase, tx.UpdatedAt = "pickup_validated", time.Now().UTC()
					w.releases[e.TaskID] = tx
				}
			}
			if l, ok := w.leases[e.TaskID]; ok {
				l.Version = e.Version
				w.leases[e.TaskID] = l
			}
		}
		if e.Type == "TaskReleased" || e.Type == "TaskFailed" || e.Type == "TaskBlocked" {
			if e.Type == "TaskReleased" {
				var p struct {
					TransactionID string `json:"transaction_id"`
				}
				if json.Unmarshal(e.Payload, &p) == nil {
					if tx := w.releases[e.TaskID]; tx.ID != "" && tx.ID == p.TransactionID && tx.Phase == "anchor_pushed" {
						tx.Phase, tx.LastError, tx.UpdatedAt = "event_committed", "", time.Now().UTC()
						w.releases[e.TaskID] = tx
					}
				}
			}
			delete(w.leases, e.TaskID)
			// A releasing predecessor remains intentionally recoverable until
			// TaskPickupValidated for its transaction. Do not erase its pane
			// mapping merely because our own release event was replayed.
			if _, releasing := w.releases[e.TaskID]; !releasing {
				delete(w.sessions, e.TaskID)
			}
		}
		if e.Seq > w.cursor {
			w.cursor = e.Seq
		}
	}
	// A coordinator restart can restore its task snapshot without retaining
	// the in-memory event tail. In that state a worker with a persisted cursor
	// receives an empty page even though a lease is currently assigned to it.
	// Reconcile the authoritative projection before treating an empty page as
	// "nothing to do"; otherwise the lease remains invisible until expiry.
	if len(es) == 0 {
		if err := w.reconcileLeases(ctx); err != nil {
			return err
		}
	}
	// State is only a cache. If a lease survived but its TaskCreated event is
	// older than the worker's cursor (or the event has been compacted), hydrate
	// the authoritative task projection before deciding whether to start.
	for taskID := range w.leases {
		if _, ok := w.tasks[taskID]; !ok {
			tasks, err := w.api.Tasks(ctx)
			if err != nil {
				return fmt.Errorf("hydrate leased task %s: %w", taskID, err)
			}
			for _, task := range tasks {
				w.tasks[task.ID] = task
			}
			break
		}
	}
	// Project the whole batch before launching. This prevents a new worker
	// from resurrecting every historical lease during its initial replay.
	for taskID, l := range w.leases {
		if _, started := w.sessions[taskID]; started {
			continue
		}
		if t, ok := w.tasks[taskID]; ok {
			if err := w.start(ctx, t, l.HandoffRef); err != nil {
				log.Printf("lease %s: %v", t.ID, err)
			}
		}
	}
	// Unit/replay-only workers intentionally have no herdr connection. A
	// production worker always does, and only then participates in the live
	// capture/control protocol.
	if w.herdr != nil {
		w.publishCaptures(ctx)
		w.runCommands(ctx)
		w.renewLeases(ctx)
	}
	w.releaseReady(ctx)
	if err := w.save(); err != nil {
		return err
	}
	return w.api.Ack(ctx, w.cursor)
}

func (w *worker) reconcileLeases(ctx context.Context) error {
	tasks, err := w.api.Tasks(ctx)
	if err != nil {
		return fmt.Errorf("reconcile leased tasks: %w", err)
	}
	active := make(map[string]lease)
	for _, task := range tasks {
		w.tasks[task.ID] = task
		if task.State == domain.StateLeased && task.Lease != nil && task.Lease.HarnessID == w.harnessID {
			active[task.ID] = lease{HandoffRef: task.HandoffRef, TransactionID: task.ReleaseTransaction, AnchorSHA: task.ReleaseAnchor, Version: task.Version, Until: task.Lease.Until}
		}
	}
	for taskID := range w.leases {
		if _, ok := active[taskID]; !ok {
			delete(w.leases, taskID)
		}
	}
	for taskID, l := range active {
		w.leases[taskID] = l
	}
	return nil
}

// reRegisterAfterCoordinatorRestart restores the coordinator's in-memory
// worker registry. A worker must survive a server restart without operator
// intervention; its persisted cursor and sessions remain valid.
func (w *worker) reRegisterAfterCoordinatorRestart(ctx context.Context, cause error) bool {
	if cause == nil || !strings.Contains(cause.Error(), "401 Unauthorized: unknown worker") {
		return false
	}
	if err := w.api.Register(ctx, w.registration); err != nil {
		log.Printf("re-register: %v", err)
		return false
	}
	log.Printf("re-registered after coordinator restart")
	return true
}
func required(k string) string {
	v := os.Getenv(k)
	if v == "" {
		log.Fatalf("%s required", k)
	}
	return v
}
func main() {
	id, token := required("ORCHESTRA_WORKER_ID"), required("ORCHESTRA_WORKER_TOKEN")
	hard := .75
	if v, err := strconv.ParseFloat(os.Getenv("ORCHESTRA_OCCUPANCY_HARD"), 64); err == nil && v > 0 && v < 1 {
		hard = v
	}
	projects := map[string]projectConfig{}
	if path := os.Getenv("ORCHESTRA_WORKER_PROJECT_CONFIG_FILE"); path != "" {
		b, err := os.ReadFile(path)
		if err != nil {
			log.Fatalf("read ORCHESTRA_WORKER_PROJECT_CONFIG_FILE: %v", err)
		}
		if err := json.Unmarshal(b, &projects); err != nil {
			log.Fatalf("parse ORCHESTRA_WORKER_PROJECT_CONFIG_FILE: %v", err)
		}
	} else {
		for _, project := range strings.Split(os.Getenv("ORCHESTRA_WORKER_PROJECTS"), ",") {
			if project = strings.TrimSpace(project); project != "" {
				projects[project] = projectConfig{Repo: required("ORCHESTRA_REPO"), Root: required("ORCHESTRA_WORKTREE_ROOT"), Remote: required("ORCHESTRA_GIT_REMOTE")}
			}
		}
	}
	if len(projects) == 0 {
		// Existing deployments may be upgraded before their protected systemd
		// environment is amended. Stay observable and fail closed in that
		// interval: an empty declaration makes this worker ineligible for all
		// new leases instead of turning a configuration rollout into a crash
		// loop or treating its legacy global checkout as every project.
		log.Printf("no ORCHESTRA_WORKER_PROJECT_CONFIG_FILE/ORCHESTRA_WORKER_PROJECTS; registering with no supported projects")
	}
	for project, config := range projects {
		if config.Repo == "" || config.Root == "" || config.Remote == "" {
			log.Fatalf("project %q requires repo, worktree_root, and remote", project)
		}
	}
	supported := make([]string, 0, len(projects))
	for project := range projects {
		supported = append(supported, project)
	}
	sort.Strings(supported)
	repo, root, remote := required("ORCHESTRA_REPO"), required("ORCHESTRA_WORKTREE_ROOT"), required("ORCHESTRA_GIT_REMOTE")
	if len(supported) > 0 {
		first := projects[supported[0]]
		repo, root, remote = first.Repo, first.Root, first.Remote
	}
	soft, window := .55, int64(200000)
	if v, err := strconv.ParseFloat(os.Getenv("ORCHESTRA_OCCUPANCY_SOFT"), 64); err == nil && v > 0 && v < hard {
		soft = v
	}
	if v, err := strconv.ParseInt(os.Getenv("ORCHESTRA_CONTEXT_WINDOW"), 10, 64); err == nil && v > 0 {
		window = v
	}
	w := &worker{api: federation.Client{BaseURL: required("ORCHESTRA_URL"), WorkerID: id, Token: token, AdmitToken: os.Getenv("ORCHESTRA_FEDERATION_ADMIT_TOKEN")}, harnessID: required("ORCHESTRA_WORKER_HERDR_ID"), harness: required("ORCHESTRA_WORKER_HARNESS"), repo: repo, root: root, remote: remote, projects: projects, tasks: map[string]domain.Task{}, sessions: map[string]herdr.Session{}, leases: map[string]lease{}, releases: map[string]releaseTransaction{}, statePath: os.Getenv("ORCHESTRA_WORKER_STATE"), hard: hard, soft: soft, window: window, registration: federation.Worker{ID: id, Address: os.Getenv("ORCHESTRA_WORKER_ADDRESS"), Capacity: 1, SupportedProjects: supported, Build: buildinfo.Current()}}
	if w.statePath == "" {
		w.statePath = filepath.Join(w.root, ".orchestra-worker-state.json")
	}
	w.load()
	w.herdr = herdr.New(required("ORCHESTRA_WORKER_HERDR"))
	if id != w.harnessID {
		log.Fatal("ORCHESTRA_WORKER_ID must equal ORCHESTRA_WORKER_HERDR_ID so leases and offline recovery have one owner")
	}
	if err := w.api.Register(context.Background(), w.registration); err != nil {
		log.Fatal(err)
	}
	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
	defer stop()
	ticker := time.NewTicker(5 * time.Second)
	defer ticker.Stop()
	for {
		if err := w.api.Heartbeat(ctx, w.health(ctx)); err != nil {
			w.recordError(fmt.Errorf("heartbeat: %w", err))
			log.Printf("heartbeat: %v", err)
			if w.reRegisterAfterCoordinatorRestart(ctx, err) {
				continue
			}
		}
		if err := w.once(ctx); err != nil {
			w.recordError(fmt.Errorf("poll: %w", err))
			log.Printf("poll: %v", err)
			w.reRegisterAfterCoordinatorRestart(ctx, err)
		}
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
		}
	}
}