From 24ee81d538f7f9eebdd1146d0add8ea60c90513d Mon Sep 17 00:00:00 2001 From: kami Date: Sun, 26 Jul 2026 18:56:59 +0400 Subject: [PATCH] complete item 1 task substrate --- cmd/orchestra/main.go | 118 ++++++++ go.mod | 3 + internal/domain/domain.go | 141 ++++++++++ internal/provider/provider.go | 45 +++ internal/provider/provider_test.go | 24 ++ internal/store/store.go | 245 ++++++++++++++++ internal/store/store_test.go | 74 +++++ orchestra-spec (1).md | 429 +++++++++++++++++++++++++++++ progress.md | 108 ++++++++ 9 files changed, 1187 insertions(+) create mode 100644 cmd/orchestra/main.go create mode 100644 go.mod create mode 100644 internal/domain/domain.go create mode 100644 internal/provider/provider.go create mode 100644 internal/provider/provider_test.go create mode 100644 internal/store/store.go create mode 100644 internal/store/store_test.go create mode 100644 orchestra-spec (1).md create mode 100644 progress.md diff --git a/cmd/orchestra/main.go b/cmd/orchestra/main.go new file mode 100644 index 0000000..4da15d5 --- /dev/null +++ b/cmd/orchestra/main.go @@ -0,0 +1,118 @@ +package main + +import ( + "encoding/json" + "log" + "net/http" + "orchestra/internal/domain" + "orchestra/internal/store" + "os" + "strconv" + "strings" + "time" +) + +func id() string { return domain.NewID() } +func main() { + dir := os.Getenv("ORCHESTRA_DATA") + if dir == "" { + dir = "./data" + } + s, err := store.Open(dir) + if err != nil { + log.Fatal(err) + } + mux := http.NewServeMux() + mux.HandleFunc("/v1/tasks", func(w http.ResponseWriter, r *http.Request) { + if r.Method == "GET" { + json.NewEncoder(w).Encode(s.Tasks()) + return + } + if r.Method != "POST" { + http.Error(w, "method not allowed", 405) + return + } + var p map[string]any + if json.NewDecoder(r.Body).Decode(&p) != nil { + http.Error(w, "invalid json", 400) + return + } + b, _ := json.Marshal(p) + e := domain.Event{ID: id(), Type: "TaskCreated", TaskID: id(), Version: 1, Payload: b} + if err := s.Append(e); err != nil { + http.Error(w, err.Error(), 400) + return + } + e = s.Events(0)[len(s.Events(0))-1] + w.WriteHeader(201) + json.NewEncoder(w).Encode(e) + }) + mux.HandleFunc("/v1/events", func(w http.ResponseWriter, r *http.Request) { + var n uint64 + if x, err := strconv.ParseUint(r.URL.Query().Get("since"), 10, 64); err == nil { + n = x + } + json.NewEncoder(w).Encode(s.Events(n)) + }) + mux.HandleFunc("/v1/tasks/", func(w http.ResponseWriter, r *http.Request) { + parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/") + if len(parts) != 4 || r.Method != "POST" { + http.Error(w, "not found", http.StatusNotFound) + return + } + taskID, action := parts[2], parts[3] + var e domain.Event + var err error + switch action { + case "lease": + var p struct { + HarnessID string `json:"harness_id"` + TTLSeconds int `json:"ttl_seconds"` + } + if json.NewDecoder(r.Body).Decode(&p) != nil || p.HarnessID == "" { + http.Error(w, "harness_id required", 400) + return + } + if p.TTLSeconds <= 0 { + p.TTLSeconds = 1800 + } + e, err = s.Lease(taskID, p.HarnessID, time.Duration(p.TTLSeconds)*time.Second) + case "release", "complete", "block": + t, ok := s.Task(taskID) + if !ok { + http.Error(w, "task not found", 404) + return + } + types := map[string]string{"release": "TaskReleased", "complete": "TaskCompleted", "block": "TaskBlocked"} + e = domain.Event{ID: id(), Type: types[action], TaskID: taskID, Version: t.Version + 1, Payload: json.RawMessage(`{"source":"api"}`)} + err = s.Append(e) + default: + http.Error(w, "unknown action", 404) + return + } + if err != nil { + http.Error(w, err.Error(), 409) + return + } + json.NewEncoder(w).Encode(e) + }) + mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("ok\n")) }) + port := os.Getenv("ORCHESTRA_PORT") + if port == "" { + port = "9145" + } + log.Println("orchestra listening on :" + port) + log.Fatal(http.ListenAndServe(":"+port, auth(mux))) +} +func auth(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + surface := strings.ToLower(r.Header.Get("X-Orchestra-Surface")) + if surface == "telegram" || surface == "ntfy" { + if r.Method != "GET" { + http.Error(w, "notify-only surface", 403) + return + } + } + next.ServeHTTP(w, r) + }) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..f618340 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module orchestra + +go 1.22 diff --git a/internal/domain/domain.go b/internal/domain/domain.go new file mode 100644 index 0000000..b8120ee --- /dev/null +++ b/internal/domain/domain.go @@ -0,0 +1,141 @@ +package domain + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/base32" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strings" + "time" +) + +var ErrConflict = errors.New("task version conflict") +var ErrNotFound = errors.New("task not found") +var ErrInvalid = errors.New("invalid event") + +type TaskState string + +const ( + StateQueued TaskState = "queued" + StateLeased TaskState = "leased" + StateCompleted TaskState = "completed" + StateFailed TaskState = "failed" + StateBlocked TaskState = "blocked" +) + +type Estimate struct { + Value float64 `json:"value"` + Who string `json:"who"` + Confidence float64 `json:"confidence"` +} +type Lease struct { + HarnessID string `json:"harness_id"` + Until time.Time `json:"until"` +} +type Task struct { + ID string `json:"id"` + Source string `json:"source"` + ExternalID string `json:"external_id"` + Project string `json:"project"` + Capability []string `json:"capability"` + Parent string `json:"parent,omitempty"` + InherentPriority int `json:"inherent_priority"` + Due *time.Time `json:"due,omitempty"` + Estimate *Estimate `json:"estimate,omitempty"` + State TaskState `json:"state"` + Lease *Lease `json:"lease,omitempty"` + Version int `json:"version"` + Title string `json:"title,omitempty"` +} + +type Event struct { + Seq uint64 `json:"seq"` + ID string `json:"id"` + Type string `json:"type"` + TaskID string `json:"task_id"` + Version int `json:"version"` + At time.Time `json:"at"` + Payload json.RawMessage `json:"payload"` +} + +func Hash(v []byte) string { h := sha256.Sum256(v); return hex.EncodeToString(h[:]) } + +// NewID returns a sortable, 128-bit ULID-like identifier using the canonical +// 48-bit millisecond timestamp plus 80 bits of cryptographic randomness. + +var ulidEncoding = base32.NewEncoding("0123456789ABCDEFGHJKMNPQRSTVWXYZ").WithPadding(base32.NoPadding) + +func NewID() string { + b := make([]byte, 16) + binary.BigEndian.PutUint64(b[:8], uint64(time.Now().UnixMilli())<<16) + _, _ = rand.Read(b[6:]) + return ulidEncoding.EncodeToString(b) +} +func ValidateEvent(e Event) error { + if e.Type == "" || e.TaskID == "" || len(e.Payload) == 0 || len(e.Payload) > 64*1024 { + return ErrInvalid + } + allowed := map[string]bool{"TaskCreated": true, "TaskLeased": true, "TaskReleased": true, "TaskCompleted": true, "TaskFailed": true, "TaskBlocked": true, "ApprovalRequested": true, "ApprovalGranted": true, "ApprovalDenied": true, "TaskAmended": true} + if !allowed[e.Type] { + return fmt.Errorf("%w: unknown type %q", ErrInvalid, e.Type) + } + var p map[string]any + if err := json.Unmarshal(e.Payload, &p); err != nil { + return fmt.Errorf("%w: payload is not JSON", ErrInvalid) + } + return ValidatePayload(e.Type, p) +} +func ValidateCreated(p map[string]any) error { + for _, k := range []string{"source", "external_id", "project"} { + if s, ok := p[k].(string); !ok || strings.TrimSpace(s) == "" { + return fmt.Errorf("%w: %s required", ErrInvalid, k) + } + } + return nil +} + +func ValidatePayload(typ string, p map[string]any) error { + requiredString := func(key string) error { + v, ok := p[key].(string) + if !ok || strings.TrimSpace(v) == "" { + return fmt.Errorf("%w: %s required", ErrInvalid, key) + } + return nil + } + switch typ { + case "TaskCreated": + return ValidateCreated(p) + case "TaskLeased": + if err := requiredString("harness_id"); err != nil { + return err + } + if _, ok := p["until_ns"].(float64); !ok { + return fmt.Errorf("%w: until_ns required", ErrInvalid) + } + case "TaskReleased": + if err := requiredString("handoff_ref"); err != nil && p["reason"] == nil { + return err + } + case "TaskCompleted": + if err := requiredString("report_ref"); err != nil { + return err + } + case "TaskFailed": + if err := requiredString("reason"); err != nil { + return err + } + case "TaskBlocked": + if err := requiredString("blocker"); err != nil { + return err + } + case "TaskAmended": + if len(p) == 0 { + return fmt.Errorf("%w: amendment cannot be empty", ErrInvalid) + } + } + return nil +} diff --git a/internal/provider/provider.go b/internal/provider/provider.go new file mode 100644 index 0000000..dd8d97f --- /dev/null +++ b/internal/provider/provider.go @@ -0,0 +1,45 @@ +package provider + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "orchestra/internal/domain" +) + +type Sink interface{ Append(domain.Event) error } +type Provider interface { + Ingest(io.Reader, Sink) (int, error) +} + +// JSONL treats each line as an external task object. Replaying the same input +// is safe because the store deduplicates the stable source/external_id key. +type JSONL struct{} + +func (JSONL) Ingest(r io.Reader, sink Sink) (int, error) { + sc := bufio.NewScanner(r) + count := 0 + line := 0 + for sc.Scan() { + line++ + raw := sc.Bytes() + if len(raw) == 0 { + continue + } + var p map[string]any + if err := json.Unmarshal(raw, &p); err != nil { + return count, fmt.Errorf("line %d: %w", line, err) + } + if err := domain.ValidateCreated(p); err != nil { + return count, fmt.Errorf("line %d: %w", line, err) + } + b, _ := json.Marshal(p) + e := domain.Event{ID: domain.NewID(), Type: "TaskCreated", TaskID: domain.NewID(), Version: 1, Payload: b} + if err := sink.Append(e); err != nil { + return count, fmt.Errorf("line %d: %w", line, err) + } + count++ + } + return count, sc.Err() +} diff --git a/internal/provider/provider_test.go b/internal/provider/provider_test.go new file mode 100644 index 0000000..805410c --- /dev/null +++ b/internal/provider/provider_test.go @@ -0,0 +1,24 @@ +package provider + +import ( + "orchestra/internal/domain" + "strings" + "testing" +) + +type sink struct{ events []domain.Event } + +func (s *sink) Append(e domain.Event) error { s.events = append(s.events, e); return nil } +func TestJSONLIngest(t *testing.T) { + s := &sink{} + n, err := (JSONL{}).Ingest(strings.NewReader("{\"source\":\"local\",\"external_id\":\"1\",\"project\":\"demo\"}\n"), s) + if err != nil || n != 1 || len(s.events) != 1 { + t.Fatalf("n=%d events=%d err=%v", n, len(s.events), err) + } +} +func TestJSONLRejectsMalformedLine(t *testing.T) { + n, err := (JSONL{}).Ingest(strings.NewReader("not-json\n"), &sink{}) + if err == nil || n != 0 { + t.Fatalf("n=%d err=%v", n, err) + } +} diff --git a/internal/store/store.go b/internal/store/store.go new file mode 100644 index 0000000..450aefa --- /dev/null +++ b/internal/store/store.go @@ -0,0 +1,245 @@ +package store + +import ( + "bufio" + "encoding/json" + "errors" + "fmt" + "orchestra/internal/domain" + "os" + "path/filepath" + "sync" + "time" +) + +type Store struct { + mu sync.Mutex + path string + cas string + events []domain.Event + tasks map[string]domain.Task + external map[string]string + snapshot string +} + +func Open(dir string) (*Store, error) { + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, err + } + s := &Store{path: filepath.Join(dir, "events.jsonl"), cas: filepath.Join(dir, "cas"), snapshot: filepath.Join(dir, "snapshot.json"), tasks: map[string]domain.Task{}, external: map[string]string{}} + if err := os.MkdirAll(s.cas, 0755); err != nil { + return nil, err + } + f, err := os.Open(s.path) + if os.IsNotExist(err) { + return s, nil + } + if err != nil { + return nil, err + } + defer f.Close() + sc := bufio.NewScanner(f) + for sc.Scan() { + var e domain.Event + if err := json.Unmarshal(sc.Bytes(), &e); err == nil { + s.events = append(s.events, e) + if err := s.apply(e); err != nil { + return nil, err + } + } else { + return nil, err + } + } + return s, sc.Err() +} +func (s *Store) apply(e domain.Event) error { + var p map[string]any + if err := json.Unmarshal(e.Payload, &p); err != nil { + return err + } + t := s.tasks[e.TaskID] + switch e.Type { + case "TaskCreated": + if err := domain.ValidateCreated(p); err != nil { + return err + } + t = domain.Task{ID: e.TaskID, Source: p["source"].(string), ExternalID: p["external_id"].(string), Project: p["project"].(string), State: domain.StateQueued} + if v, ok := p["capability"].([]any); ok { + for _, x := range v { + if z, ok := x.(string); ok { + t.Capability = append(t.Capability, z) + } + } + } + if v, ok := p["title"].(string); ok { + t.Title = v + } + s.external[t.Source+"\x00"+t.ExternalID] = t.ID + case "TaskLeased": + t.State = domain.StateLeased + t.Lease = &domain.Lease{HarnessID: p["harness_id"].(string), Until: time.Unix(0, int64(p["until_ns"].(float64)))} + case "TaskReleased": + t.State = domain.StateQueued + t.Lease = nil + case "TaskCompleted": + t.State = domain.StateCompleted + t.Lease = nil + case "TaskFailed": + t.State = domain.StateFailed + t.Lease = nil + case "TaskBlocked": + t.State = domain.StateBlocked + t.Lease = nil + case "TaskAmended": + for k, v := range p { + if k == "title" { + t.Title, v = v.(string) + } + } + } + t.Version = e.Version + s.tasks[e.TaskID] = t + return nil +} +func (s *Store) Append(e domain.Event) error { + s.mu.Lock() + defer s.mu.Unlock() + if err := domain.ValidateEvent(e); err != nil { + return err + } + if e.At.IsZero() { + e.At = time.Now().UTC() + } + if e.Seq == 0 { + e.Seq = uint64(len(s.events) + 1) + } + if e.Type == "TaskCreated" { + var p map[string]any + if err := json.Unmarshal(e.Payload, &p); err != nil { + return err + } + if id := s.external[p["source"].(string)+"\x00"+p["external_id"].(string)]; id != "" { + return nil + } + } + if t, ok := s.tasks[e.TaskID]; ok && e.Version != t.Version+1 { + return domain.ErrConflict + } + if _, ok := s.tasks[e.TaskID]; !ok && e.Type != "TaskCreated" { + return domain.ErrNotFound + } + if e.Type != "TaskCreated" && (e.Type == "TaskCompleted" || e.Type == "TaskBlocked" || e.Type == "TaskReleased") { + var p map[string]any + _ = json.Unmarshal(e.Payload, &p) + for _, k := range []string{"handoff_ref", "report_ref"} { + if ref, ok := p[k].(string); ok { + if _, err := os.Stat(filepath.Join(s.cas, ref)); err != nil { + return fmt.Errorf("%w: missing artifact %s", domain.ErrInvalid, ref) + } + } + } + } + if err := s.apply(e); err != nil { + return err + } + f, err := os.OpenFile(s.path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer f.Close() + b, _ := json.Marshal(e) + if _, err = f.Write(append(b, '\n')); err != nil { + return err + } + if err = f.Sync(); err != nil { + return err + } + s.events = append(s.events, e) + if err := s.writeSnapshot(); err != nil { + return err + } + return nil +} +func (s *Store) writeSnapshot() error { + tasks := make([]domain.Task, 0, len(s.tasks)) + for _, t := range s.tasks { + tasks = append(tasks, t) + } + b, err := json.Marshal(struct { + Seq uint64 `json:"seq"` + Tasks []domain.Task `json:"tasks"` + }{uint64(len(s.events)), tasks}) + if err != nil { + return err + } + tmp := s.snapshot + ".tmp" + if err = os.WriteFile(tmp, b, 0644); err != nil { + return err + } + return os.Rename(tmp, s.snapshot) +} +func (s *Store) Tasks() []domain.Task { + s.mu.Lock() + defer s.mu.Unlock() + out := make([]domain.Task, 0, len(s.tasks)) + for _, t := range s.tasks { + out = append(out, t) + } + return out +} +func (s *Store) Events(since uint64) []domain.Event { + s.mu.Lock() + defer s.mu.Unlock() + var out []domain.Event + for _, e := range s.events { + if e.Seq > since { + out = append(out, e) + } + } + return out +} +func (s *Store) PutArtifact(b []byte) (string, error) { + h := domain.Hash(b) + p := filepath.Join(s.cas, h) + if _, err := os.Stat(p); errors.Is(err, os.ErrNotExist) { + if err = os.WriteFile(p, b, 0644); err != nil { + return "", err + } + } + return h, nil +} + +func (s *Store) Task(id string) (domain.Task, bool) { + s.mu.Lock() + defer s.mu.Unlock() + t, ok := s.tasks[id] + return t, ok +} + +func (s *Store) Lease(id, harness string, ttl time.Duration) (domain.Event, error) { + t, ok := s.Task(id) + if !ok { + return domain.Event{}, domain.ErrNotFound + } + if t.State != domain.StateQueued { + return domain.Event{}, domain.ErrConflict + } + p, _ := json.Marshal(map[string]any{"harness_id": harness, "until_ns": time.Now().Add(ttl).UnixNano()}) + e := domain.Event{ID: id, Type: "TaskLeased", TaskID: id, Version: t.Version + 1, Payload: p} + return e, s.Append(e) +} + +func (s *Store) ExpireLeases(now time.Time) ([]domain.Event, error) { + var out []domain.Event + for _, t := range s.Tasks() { + if t.State == domain.StateLeased && t.Lease != nil && !t.Lease.Until.After(now) { + p, _ := json.Marshal(map[string]any{"reason": "lease_expired", "harness_id": t.Lease.HarnessID}) + e := domain.Event{ID: t.ID, Type: "TaskReleased", TaskID: t.ID, Version: t.Version + 1, Payload: p} + if err := s.Append(e); err != nil { + return out, err + } + out = append(out, e) + } + } + return out, nil +} diff --git a/internal/store/store_test.go b/internal/store/store_test.go new file mode 100644 index 0000000..3e6e777 --- /dev/null +++ b/internal/store/store_test.go @@ -0,0 +1,74 @@ +package store + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "orchestra/internal/domain" +) + +func created(id string) domain.Event { + b, _ := json.Marshal(map[string]any{"source": "jsonl", "external_id": "42", "project": "demo", "capability": []string{"mechanical"}}) + return domain.Event{ID: id, Type: "TaskCreated", TaskID: "task-1", Version: 1, Payload: b} +} + +func TestAppendReplayAndDeduplicate(t *testing.T) { + dir := t.TempDir() + s, err := Open(dir) + if err != nil { + t.Fatal(err) + } + if err := s.Append(created("e1")); err != nil { + t.Fatal(err) + } + if err := s.Append(created("e2")); err != nil { + t.Fatal(err) + } + if got := len(s.Events(0)); got != 1 { + t.Fatalf("duplicate ingest appended %d events", got) + } + ref, err := s.PutArtifact([]byte("report")) + if err != nil { + t.Fatal(err) + } + completion, _ := json.Marshal(map[string]string{"report_ref": ref}) + if err := s.Append(domain.Event{Type: "TaskCompleted", TaskID: "task-1", Version: 2, Payload: completion}); err != nil { + t.Fatal(err) + } + if err := s.Append(domain.Event{Type: "TaskReleased", TaskID: "task-1", Version: 2, Payload: json.RawMessage(`{"handoff_ref":"x"}`)}); err != domain.ErrConflict { + t.Fatalf("expected conflict, got %v", err) + } + s2, err := Open(dir) + if err != nil { + t.Fatal(err) + } + if got := s2.Tasks()[0].State; got != domain.StateCompleted { + t.Fatalf("replay state = %s", got) + } + if _, err := os.Stat(filepath.Join(dir, "snapshot.json")); err != nil { + t.Fatal(err) + } +} + +func TestArtifactIsContentAddressed(t *testing.T) { + s, err := Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + h1, err := s.PutArtifact([]byte("proof")) + if err != nil { + t.Fatal(err) + } + h2, err := s.PutArtifact([]byte("proof")) + if err != nil { + t.Fatal(err) + } + if h1 != h2 { + t.Fatal("same artifact received different hashes") + } + if _, err := os.Stat(filepath.Join(s.cas, h1)); err != nil { + t.Fatal(err) + } +} diff --git a/orchestra-spec (1).md b/orchestra-spec (1).md new file mode 100644 index 0000000..95718a6 --- /dev/null +++ b/orchestra-spec (1).md @@ -0,0 +1,429 @@ +# [ORCHESTRA] — Multi-Harness Agent Orchestration + +> Working name; yours to set. This is the binding spec — implementers (agents or you) bind to it, not to prose in a chat log. + +**Status legend** +`[S]` settled — decided, do not re-litigate. +`[D]` default — a value/choice not explicitly ratified; tunable, veto freely. Never treat as fact. +`[O]` open — unresolved. Has a degrade-safe default so a gap never becomes a hallucinated decision. + +--- + +## 0. Orientation + +**What this is.** A control plane for an *orchestra of opaque harnesses* — Claude Code, Codex, opencode(-zen/go), and local models via llama-server. Subscription access is CLI-only: you orchestrate the **process**, not the model. Every harness is a black box with a terminal; herdr is the grip on it. + +**What this is NOT.** `[S]` This is not Correx. Correx drives models you *call* (local inference, direct API). This drives harnesses you *cannot call*. They share **ontology** (task schema, vocab, the `*.md` conventions) so Maven speaks one language to both — they do **not** share execution code. The layers below have nothing in common with Correx's proposal-validator plane except discipline. + +**The one metric.** `[S]` The objective is **maximized unattended runtime** — work completed per stretch you can walk away from — not cost-per-turn. Babysitting is a real cost that never appears in `ccusage`. Every trade-off resolves against this: Opus for correlation work cheaper models can't do, local/free for loud-failure mechanical work, rotation to extend the run before the context wall ends it. + +**Deploy target.** `[S]` Runs on both machines (server, workpc). Single-user, homelab-internal. + +--- + +## 1. Invariants — must hold everywhere + +These are asserted at every boundary, not implemented as a component. If a change violates one, the change is wrong. + +1. **`[S]` The log is append-only, forever.** Corrections are *compensation events* appended on top. There is no rewrite/compaction path — you cannot replay a decision you deleted, and receipts are the point. +2. **`[S]` The plane emits events, not the agent.** The agent only writes code and artifacts. The router, the adapter, and the stop-hook/wrapper emit lifecycle events. The agent is not a participant in the task system — the harness is. (This is what keeps task tooling out of the agent's context.) +3. **`[S]` Events reference artifacts by hash; they never embed them.** Handoffs, reports, diffs → CAS. The event carries the hash. Event payloads are always small. +4. **`[S]` No free-form field is trusted as instruction.** Every event payload and artifact field is typed and schema-validated at write. Knowledge/handoff prose is framed and consumed as **data**. This is the injection boundary — saturated on purpose: assert it everywhere, never build it as one guarded component. +5. **`[S]` State lives in artifacts verifiable against ground truth — never in agent memory or prose.** Pickup-without-rediscovery works only because a fresh agent validates against repo + `TASK.md`, never against a predecessor's narrative. +6. **`[S]` The log is not the transcript.** Liveness and per-turn progress are answered by asking herdr / exit codes, not by replaying events. + +--- + +## 2. Architecture + +Four layers. Each reads from the one below. Conflating them is the trap. + +``` +┌─ Layer 4 · SURFACES ──────────────────────────────────────┐ +│ tui/web (full) · telegram+ntfy (notify) · Maven (gated) │ +│ authorization enforced on the bus, not per-client │ +└───────────────▲───────────────────────────────────────────┘ + │ subscribe / emit (per authz) +┌─ Layer 3 · CONTINUITY ────────────────────────────────────┐ +│ handoff schema · pickup contract · TASK.md · shared *.md │ +└───────────────▲───────────────────────────────────────────┘ + │ artifacts (by hash) +┌─ Layer 2 · HARNESS (execution) ───────────────────────────┐ +│ herdr backbone · harness adapters · router · rotation │ +│ supervises opaque CLI processes; no API in the loop │ +└───────────────▲───────────────────────────────────────────┘ + │ consumes tasks / emits lifecycle +┌─ Layer 1 · SUBSTRATE (the sink) ──────────────────────────┐ +│ canonical event log · provider port · task aggregate │ +│ event catalog = THE contract (§4) │ +└────────────────────────────────────────────────────────────┘ +``` + +**Build order is bottom-up (§8).** Layer 1 is build-ready now; higher layers carry marked open items. + +### 2.1 Deployment topology `[D]` (parse of a federated design — verify) + +`[S]` **Federated, not single-host.** Agents run on **both** homesrv and workpc, on demand — but workpc is **not** always on, so this is a worker-federation, not a symmetric pair. + +Two registries: +- **Machine registry** — hosts (`homesrv` always-on; `workpc` intermittent; +future), each with mesh address + reachability. +- **Herdr registry** — herdr instances, each **bound to a machine-registry entry**, carrying static per-instance config for what it may run. + +`[S]` **homesrv is authoritative.** Event log + router + CAS live there and **only** there — a laptop that sleeps can't hold canonical truth. `[S]` **workpc is a worker + client**, never a source of task truth. + +`[S]` **herdr on both; we drive it, we don't own it.** The router dispatches leases to **any registered + reachable** herdr per its static config. A herdr that's offline (workpc asleep) simply isn't a candidate that cycle — no special-casing. + +`[S]` **Git is the cross-machine transport.** Server holds the remotes; a machine going offline has already pushed, a machine coming online pulls. **This makes the scratch-branch commit (§6.2) one mechanism for three handoffs — cross-rotation, cross-agent, and cross-machine.** State in git + a validated artifact, picked up by whoever/wherever. + +| Component | Host | Notes | +|---|---|---| +| event log · router · CAS | **homesrv only** | canonical; always-on | +| herdr + leased harnesses | homesrv **and** workpc | worker federation; offline = not a candidate | +| git remotes | homesrv | cross-machine transport | +| tui / web control | workpc (and anywhere) | client | +| Maven | homesrv | fully server-side (§2.2) | + +### 2.2 Projects & machine affinity `[S]`(model) `[D]`(examples) + +`[S]` **Tasks bind to a project at creation** — you pick which. `[S]` Projects are **first-class and extensible**: adding more later reshapes nothing (the binding is a field + a config entry, not a schema change). `[S]` A project declares **machine affinity** — which host(s) its harnesses run on — as **static manual config** ("work however it sees fit based on static configuration"). + +Resolution chain: **task → project → affinity → candidate machines → router filters by reachable+configured herdr there.** + +`[D]` Illustrative affinities (yours, not the system's — shown to fix the model): + +| Project | Affinity | +|---|---| +| correx | workpc-primary, git remote on homesrv | +| Maven | homesrv-full; workpc only for CPT / fine-tune / LoRA | +| Kdrive · Muzic · Nexus · Praxis · Hexis | homesrv-exclusive | + +`[O]` **Fork — is affinity hard or soft?** Night queue on homesrv: a workpc-primary project (correx) has its remote on the server, so homesrv *could* run it against the remote — or you may want it to wait for workpc. **Default `[D]`: hard** (never run where not configured; overnight queue = server-runnable projects only) — safer, predictable, matches static config. Soft (prefer workpc, fall back to server overnight) is a one-flag change if you want it. See §9. + +### 2.3 Workflow — end to end `[S]` + +Two walk-away modes, both reducing to the same lifecycle: + +- **Mode A — mid-day handoff.** workpc is on; you queue/assign and walk away; workpc's herdr runs to completion; results commit + push to the homesrv remote; `TaskCompleted` per task. (You were just using workpc, so it's up — no sync-wait.) +- **Mode B — overnight batch.** You queue N tasks for the night on homesrv (server-runnable projects, per affinity above); homesrv's herdr works through them; each commits + pushes; **git-sync means workpc pulls in the morning**; a single **morning brief (§7.4)** rolls up the whole night. This is the payoff of the sync + brief machinery. + +**The whole lifecycle, once:** + +``` +ingest → TaskCreated (provider adapter; §3.2, §4) + → router: project → affinity → machine → lease (§2.2, §5.5) ── offline herdr = skipped + → herdr on that machine: bootstrap · work · gate · TDD (§5, §6) + → rotation as needed (intra-task lease transfer; state via git) (§5.3, §6.2) + → TaskCompleted + report_ref (proofs + receipts) (§4) + │ or → TaskFailed / TaskBlocked → you (needs-human) + → reflect status out · git push · (workpc pulls on wake) (§2.1, §3.2) + → brief rolls up the window → surfaces per authz (§7.4, §7.1) +``` + +`[S]` Everything crossing a machine boundary is git + a validated artifact — never live state over the wire. The event log is the only always-on shared surface; nothing holds a second copy. + +--- + +## 3. Layer 1 — Task Substrate `[S]` (spine settled) + +### 3.1 Canonical event log + +`[S]` The event log is the single source of task truth. Append-only. Global monotonic sequence number = the subscriber cursor. Each task aggregate additionally carries a per-task version for optimistic concurrency (§4). + +`[S]` **Corrections = compensation.** A wrong event is never edited; a compensating event is appended and replay sees both. + +`[S]` **Snapshots** cache "state as of sequence N" so projections don't replay from zero. Raw events are retained regardless — snapshots are an optimization, never a substitute. + +`[D]` **Storage backend.** Append-only log store TBD; CAS blobs can sit on existing MinIO. Choose in §8 spike. + +### 3.2 Provider port — bidirectional adapter + +`[S]` The provider is an **interface, not a default implementation**, and it is **not** the source of task truth. It has two directions: + +- **Ingest:** external task → `TaskCreated` event. +- **Reflect:** task state transitions → external status (e.g. close the Gitea issue on completion). + +`[S]` **Stable external key `(source, external_id)`** per adapter, serving double duty: +- **Idempotency on ingest** — a webhook firing twice, or a poll re-seeing an issue, must not create two `TaskCreated`. First sight creates; repeats no-op or update. +- **Reflection address** — tells the adapter which external record to update outbound. + +Adapters (all implement the same port): +- Gitea issues / CI failures (webhook + poll) +- Pure-local JSONL (trivial baseline — build first) +- (later) anything Maven feeds in + +### 3.3 Task aggregate + +`[S]` **Task is the only aggregate.** Epic / roadmap / sprint are **projections** over parent links, not their own streams (sprint's open/close lifecycle is the one possible exception — deferred to the agile layer). + +Fields: + +| Field | Type | Notes | +|---|---|---| +| `id` | ulid | internal | +| `source`, `external_id` | str | the stable external key | +| `project` | project-id | **bound at creation** (§2.2); resolves to machine affinity. Sink and brief group on this. | +| `capability` | tag set | **static**, set at creation. What the task *needs*. Router matches on this. | +| `lease` | Lease? | **dynamic**. Who holds it *now*. See §3.4. | +| `parent` | task-id? | task→epic→roadmap link | +| `inherent_priority` | enum | raw priority, editable | +| `due` | instant? | no hard deadlines; feeds derived importance | +| `estimate` | `{value, who, confidence}` | provenance-carrying (§3.6) | +| `state` | enum | derived from lifecycle events | +| `version` | int | optimistic-concurrency guard | + +### 3.4 Two axes: capability vs. lease `[S]` + +These are **independent** and must not be merged: + +- **Capability** — a property of the *task*. Static tag set (`{correlation}`, `{mechanical}`, …). Set at creation, matched at pickup. +- **Lease** — a property of a *harness instance*. Dynamic: who holds it, and until when (TTL, §5.4). Router sets it, agent releases it, rotation transfers it. + +Merging them is wrong because it would force recomputing task metadata every time quota shifts — the task didn't change, the harness did. + +`[S]` **Availability ≠ capability.** Whether a given harness *can accept right now* (alive? quota headroom? under concurrency cap?) is a **runtime filter the router applies at lease time**, not a task field. Router **matches** on capability, **filters** by availability. + +`[S]` **Pickup-without-rediscovery** = lease release keeps capability intact; the next matching+available harness picks up from task state. Rotation-handoff and agent-to-agent-handoff are **the same lease transfer over the same task state** — one mechanism. + +### 3.5 Derived importance `[S]` + +Importance is **not stored**. It is a sort key over the sink: `f(inherent_priority, proximity_to_due)`, recomputed as due approaches. `[S]` **Pickup-only** — importance orders the sink when a lease frees; it **never preempts running work**. No preemption event, no interrupt path in the harness. + +### 3.6 Agile structure `[S]` + +- Hierarchy = parent links (§3.3), rendered as projections. +- Sprint = a time-boxed *selection* over the sink. +- Estimates carry `{who, confidence}` — agents give weighted judgment; you keep the last word. (This is the "standup" mechanic reduced to a field; see §7 for the advisory-event form.) + +--- + +## 4. Event Catalog — THE contract + +`[S]` An event earns a **distinct type only when a subscriber reacts in a way a generic "something changed" could not dispatch.** Everything else is one generic amend. + +### Behavioral events (distinct subscriber reactions) + +| Event | Emitted by | Key payload | Reacted to by | Concurrency | +|---|---|---|---|---| +| `TaskCreated` | provider adapter (ingest) | `project`, `capability`, `source`, `external_id`, `parent?`, `inherent_priority`, `due?` | router → evaluate for assignment | dedup on `(source,external_id)` | +| `TaskLeased` | router | `harness_id`, `expected_version`, `ttl` | the matched harness → start work | **CAS on task `version`** — second writer fails, no lock | +| `TaskReleased` | agent/harness (via plane) | `handoff_ref` (hash), `anchor_sha` | router → find next picker | version-guarded | +| `TaskCompleted` | stop-hook/wrapper (via plane) | `report_ref` (hash), `receipt` | fan-out: telegram/Gitea/Maven; reflect status out | terminal | +| `TaskFailed` | router | `reason`, `attempts` | needs-human path; stop retrying | terminal | +| `TaskBlocked` | agent/harness (via plane) | `blocker`, `handoff_ref?` | needs-human path | — | +| `ApprovalRequested` | harness (gate/arbiter) | `subject_ref`, `options` | full+gated surfaces present it; harness blocks | — | +| `ApprovalGranted` / `ApprovalDenied` | full-control surface | `subject_ref`, `by` | harness unblocks; other surfaces learn via this event | resolve-by-event, no timeout | + +### Generic event + +| Event | Emitted by | Payload | Reacted to by | +|---|---|---|---| +| `TaskAmended` | human or agent | `{fields}`: title, due, description, `inherent_priority` | sink re-projects | + +`[S]` `TaskAmended` **never** carries lease or lifecycle. That boundary is the whole thing keeping it from rotting into a CRUD blob. + +### Explicitly NOT in the log + +- **Per-turn progress, liveness** → herdr socket (`pane.agent_status_changed`, `pane.exited`) + exit codes. +- **Token spend** → a **separate projection** keyed by `harness+window`, not by task. Per-task cost lands as a `receipt` field on `TaskCompleted`, **summed across the task's lease intervals** (a rotated task spans several sessions → a sum, not a single delta). + +### The "no not-mine excuses" mechanism `[S]` + +Don't trust the agent to self-report a lint/format/test issue. The **quality gate** (detekt/ktlint, mypy/ruff, shellcheck, schema-validate) catches it mechanically and emits `TaskCreated` for anything it won't block on — deterministic and automatic. The only residue is a logic bug the agent *notices* but no gate catches → one narrow "file-a-task" affordance, kept small. + +--- + +## 5. Layer 2 — Harness (execution) + +### 5.1 herdr backbone `[S]` + +herdr (rust agent multiplexer; v0.7.x, AGPL-3.0/commercial — matters only if linked, not if shelled out to) gives real turn state where tmux gave none: + +- `agent.prompt` with inline `wait{until,timeout_ms}` — bootstrap injection + turn-completion wait in **one request**, no gap. +- `agent.wait` **pins the pane occupant** — a replacement can't satisfy the old wait. +- `events.subscribe` on `pane.agent_status_changed` — no polling. +- `pane.report_metadata` (`tokens` map, TTL) — live occupancy gauge in the sidebar once you have the number. +- `pane.exited` — crash fast-path. `notification.show` — HALT pages you through herdr itself. +- `worktree.create` — one worktree per rotation chain → anchor validation collapses to one `rev-parse`. + +`[S]` Install native integrations (`herdr integration install {claude,codex,opencode}`) so "done" is reported, not inferred. `[S]` Socket at `~/.config/herdr/herdr.sock` is unauthenticated → bind harness fleet to the WireGuard mesh, never `0.0.0.0`; on VPS, reconsider entirely. `[S]` Generate the client off `herdr api schema --json`; check protocol version via `ping` before depending on new behavior. + +`[D]` **Driving herdr — imperative is fine, except two ops.** Poke panes over the socket (send-text / capture-pane-equivalents) for ordinary control and for feeding the TUI/brief — it's a multiplexer you don't own, this is expected. But keep the **two rotation-critical ops** on the semantic primitives: **turn-end** via Face B / `agent_status` (§5.2.1), and **bootstrap injection** via `agent.prompt` with inline `wait` (`agent.wait` pins the occupant so a replacement can't satisfy it). capture-pane polling + send-keys for *those two* reintroduces the exact tmux race — a bootstrap or `/clear` fired into a half-rendered prompt, silently, unattended. Veto only if you've solved that race another way. + +### 5.2 Harness adapter — the second load-bearing contract `[S]` + +A harness adapter has **two faces**, and it is the *same seam* as the Layer-1 provider concept applied to execution: + +``` +interface HarnessAdapter { + // Face A — lifecycle / IO + fun lease(task, worktree): Session // start the opaque CLI in a herdr pane + fun bootstrap(session, handoffRef) // ~200-token prompt: "read handoff, run validate, proceed" + fun release(session): TaskReleased // checkpoint + emit + fun kill(session) + + // Face B — turn-boundary + occupancy detection (per-harness impl; §5.2.1) + fun onTurnBoundary(cb) // cc: Stop hook (exit 2 refuses turn); codex: rollout tail; opencode: SSE session.status + fun occupancy(session): Fraction // all three read native session state — NO proxy required +} +``` + +`[S]` **cc's Face B** = the Stop hook. Exit **2** blocks stop and feeds stderr back → over-threshold-with-no-valid-handoff refuses the turn until the handoff is written and validates, then exit 0. `[S]` **codex/opencode's Face B** read their own native session state (§5.2.1) — no proxy. The trigger is harness-agnostic **above** the adapter; per-harness **below** it. `[S]` The router and rotation logic never know which harness they're driving. + +### 5.2.1 Occupancy sources — verified `[S]`/`[D]` + +`[S]` herdr has no token accounting, but the earlier "proxy for codex/opencode" assumption was wrong: **all three harnesses write usage into local session state; none require an interception proxy.** They differ only in the cleanest access path. Build this measurement and verify it against a live session **before** wiring any trigger — the whole rotation system rests on this number. + +| Harness | Source `[S]` | Path / endpoint | Per-turn shape `[S]` | Occupancy numerator (what's in the window *now*) | +|---|---|---|---|---| +| **Claude Code** | session JSONL, `message.usage` per assistant entry | `~/.claude/projects//.jsonl` (hook gives `transcript_path` on stdin) | **absolute per entry**: `input_tokens`, `cache_creation_input_tokens`, `cache_read_input_tokens`, `output_tokens` | last assistant entry: `input + cache_read + cache_creation` (the context sent). Dedupe by message `uuid`. | +| **Codex** | rollout JSONL `event_msg`, `payload.type=="token_count"` | `~/.codex/sessions/YYYY/MM/DD/rollout-*.jsonl` (or `CODEX_HOME`). Active file authoritative via `state_*.sqlite → threads.rollout_path` — discover, don't hardcode | `info.total_token_usage` is **cumulative** (diff successive); `info.last_token_usage` is the last step | `last_token_usage.last_input (+ last_cached)`. Only present since codex ≥ 2025-09-06; older logs have none. | +| **opencode** | server API (it's already a Hono server on `:4096`) | SSE `GET /event` → `session.status` (idle/busy) + `message.updated`; or `GET /session/{id}/messages`; files at `~/.local/share/opencode/storage/message/` (`OPENCODE_DATA_DIR`) | per-message `tokens = {input, output, reasoning, cache:{read,write}}` (`cost:0` in files) | last message: `tokens.input + tokens.cache.read`. | + +`[S]` **The trap that would silently break rotation: never use the cumulative session total as occupancy.** All three accumulate `cache_read` per turn without bound — one real opencode session reported ~56M "total tokens" against a 200k window; codex burns from 21k→560k cumulative in a startup loop. Cumulative total answers "what did this session cost," not "how full is the context." **Occupancy = *this turn's* input+cache against the model's context window**, i.e. the numerator column above ÷ window (200k Sonnet/Opus-class; the gpt-5.x window for codex — model-dependent constants, keep them in config). + +`[D]` **Proxy is the fallback, not the primary.** A localhost shim on `ANTHROPIC_BASE_URL`/`OPENAI_BASE_URL` reading `usage` off responses still works and is the one *uniform* implementation across all three — worth it only if per-harness parsing gets annoying or you want request-time-synchronous truth. Native sources are simpler and are the default. + +`[D]` **opencode SSE is not rock-solid** — the `/event` stream has known bugs (closes right after `server.connected`; stochastic REST behavior under `opencode serve`). Treat the event stream as the fast path and keep the message-file / `opencode stats --json` read as the backstop, same fast-path-plus-backstop shape as `pane.exited`+TTL. + +### 5.3 Rotation `[S]` + +`[S]` **The Stop-hook/Face-B decides rotation, not the router.** It has the transcript and fires at turn boundaries. The router decides whether rotation is *permitted* (caps, budget) and executes it. + +Triggers: +- `[D]` **soft ~55%** → "prepare handoff at next natural stop" +- `[D]` **hard ~75%** → rotate now; reserve `[D]` ~10–15k tokens to actually write the handoff (do not wait for 90% — compaction beats you) +- `[S]` **milestone** → phase boundary rotates regardless of occupancy (a fresh agent at a boundary beats a 60%-full one; cheap context is a feature) +- `[S]` **thrash** → N failed test runs / same file edited M times with no pass / identical tool calls repeating → rotate with `reason=thrash` + populated `dead_ends` = circuit breaker +- `[S]` **agent-initiated `ROTATE`** → emitted when a coherent unit finishes and the next is independent + +`[S]` **Rotation = intra-task lease transfer** (§3.4). `[S]` **Split-then-close** (cheap): new pane validates the anchor before the old pane closes — atomic, with a real rollback path, and it drops the `/clear`-slash-command dependency that codex/opencode don't share. + +### 5.4 Failure & lease semantics `[S]` + +- `[S]` **Lease carries a TTL.** Reclamation = **lease-expiry**, not crash-detection. A harness that dies without releasing lets its lease go stale → task returns to the sink. You don't need to reliably observe death; you need leases to not be eternal. +- `[S]` `pane.exited` = fast-path to relet sooner; TTL = backstop for the hang `pane.exited` misses. +- `[S]` Policy: **retry N, then `TaskFailed`**; agent continues if nothing more important is queued. `[D]` `N`, `ttl`, retry backoff = config. + +### 5.5 Router `[S]` + +Makes assignment decisions at **exactly two moments**: on `TaskCreated` (ingest) and on lease-free (`TaskReleased`/expiry). **Never against running work** (pickup-only, §3.5). Algorithm: resolve **project → machine affinity** (§2.2) to candidate hosts → **restrict** to registered + **reachable** herdr on those hosts (offline = not a candidate) → **match** by `capability` → **filter** by availability (alive, quota headroom, under concurrency cap) → **order** by derived importance → lease the top via version-guarded `TaskLeased`. `[S]` Machine resolution is first because it's the hard constraint; capability and availability narrow within it. + +### 5.6 Build note `[S]` + +Orchestrator in **bash first** (~150 lines: subscribe, read request, validate, split, prompt, ack, close) — run it on one real task, find where the state machine is wrong (it will be, around thrash + timeout). **Then** port to Kotlin/Spring as a proper long-lived supervised service and slot it into the fleet. Hooks stay bash forever (they run in the agent's process every turn — no runtime, no deps). + +--- + +## 6. Layer 3 — Continuity + +### 6.1 Handoff schema `[S]` + +Every field **verifiable against the repo** — prose is where hallucination lives and it compounds across rotations. `[D]` **Size cap ~4–6k tokens**; if it doesn't fit, task decomposition is wrong, not the schema. Stored in CAS; events carry the hash (Invariant 3). + +```toml +[meta] id, parent, reason = "threshold|milestone|thrash|manual", rotation_index +[anchor] git_sha, branch, dirty = [{path, sha256}] # verifiable state +[objective] goal (one sentence), done_when = [checkable, criteria] +[progress] completed = [{what, evidence = "file:line"}], remaining = [...] +[next] action (single step), command (literal first cmd), files = [...] +[knowledge] invariants = [...], dead_ends = [{tried, why_failed}], open_questions = [...] +[verify] build, test, last_result = {command, exit_code, at_sha} +``` + +`[S]` **`dead_ends` is the highest-value block and the one everyone omits** — negative knowledge is exactly what's lost on rotation; without it the fresh agent re-derives the same failure. `[S]` **`[knowledge]` is framed as data, not instructions** (Invariant 4) — this is the injection-persistence surface; strict schema, no free-form. + +### 6.2 Pickup contract — the make-or-break `[S]` + +`[S]` Agents validate; the orchestra validates. `[S]` The validation is a **concrete, testable procedure** run before the fresh agent continues — not a principle: + +``` +validate-handoff : + 1. schema-check the handoff (types, required fields, no free-form in [knowledge]) + 2. git rev-parse HEAD == anchor.git_sha + 3. for each anchor.dirty: sha256(path) == recorded # or: WIP committed to scratch branch, one sha compare + 4. re-read immutable TASK.md (original spec) + → all pass → proceed + → any fail → do NOT rotate/continue; agent fixes, re-validate +``` + +`[S]` **`TASK.md` is immutable**, holds the original spec, is **re-injected every rotation**, and is **never rewritten**. The fresh agent validates against repo + `TASK.md` — **never against the predecessor's handoff prose.** This is the defense against the telephone game across long autonomous runs. `[S]` **Commit WIP to a scratch branch first** so step 3 is one sha compare, atomic and free. + +### 6.3 Shared memory & vocab `[S]` + +- `[S]` Shared `*.md` (AGENTS.md, CLAUDE.md, vocab) — **all agents contribute.** +- `[S]` On update, the **orchestra injects a notice** to agents whose current task is **adjacent** (staleness handled at the orchestra layer, not by trusting cached agent views — same shape as herdr's metadata TTL, one layer up). +- `[S]` Docs are self-containing, human-readable **and** digestible-at-a-glance, with references where available and mockups at specing stage. The policy SHOULDs ("no not-mine excuses", TDD-by-default, spec-first, cheap-harness-before-manual, QA-mandatory) live **here as convention text + gates**, not as code. + +--- + +## 7. Layer 4 — Surfaces + +### 7.1 Authorization model — enforced on the bus `[S]` + +Not a UX detail — a capability per surface on the event bus. "Who can do what from where" is enforced **once, at the bus**, not per-client. + +| Surface | Capability | Can emit | Notes | +|---|---|---|---| +| **telegram + ntfy** | notify-only | *nothing* | read-side subscriber; pure notification | +| **tui / web-ui** | full control | any event, incl. approvals | the control seats | +| **MCP** `[D]` | read + **gated** write | queries freely; task create/amend via `ApprovalRequested` | programmatic surface for external agents/tools; `[D]` promote to full if it's your own trusted client. (MCP-as-*ingest* — Maven feeding tasks — is the §3.2 provider port, separate from this.) | +| **Maven** | observation + **gated** control | **only** via `ApprovalRequested` → your grant; never direct | north-star assistant, fenced | + +`[S]` **Clients poll; they do not store.** `[S]` An approval granted on **any** full/gated surface resolves by `ApprovalGranted`/`Denied`; others **learn via the event** later. **No approval timeout** (you may be asleep). `[S]` Artifacts crossing into a **control-capable** surface are **re-validated against schema at that boundary** (Invariant 4), never passed through. + +### 7.2 Quota transparency `[O]` + +`[S]` **Quota ≠ occupancy — different numbers, do not conflate.** Occupancy (§5.2.1) is *context-window fill of one session* and drives rotation. Quota here is *subscription-pool consumption across sessions* (the 5-hour rolling + weekly caps) and drives the router's availability filter + 3am safety. Same source files, different accounting: `ccusage blocks --json` gives the 5h-window view; codex even surfaces its own 5h/weekly % in the rollout logs. + +`[S]` Transparent quota is a **projection** (§4, keyed `harness+window`), fed by the same per-harness session state as §5.2.1 (`ccusage`-style) — no separate proxy needed. `[O]` **Accuracy is unresolved** — each pool is reconstructed, none authoritative, and a 3am autonomous run trusting a wrong "you have headroom" hits a wall mid-task. **Degrade-safe default `[D]`:** treat `[D]` 80% reported as full; the router's availability filter (§5.5) uses the conservative number. Name the failure mode in-system; refine after measuring a real run. + +### 7.3 Standups `[S]` + +Scheduled read over the log that emits an **advisory** event (agents provide opinion + weighted judgment). `[S]` **Last word is yours** — the advisory routes through the same approval gate; nothing self-applies. + +### 7.4 The brief `[S]` + +`[S]` A **projection over the event log for a window** (nightly, or on demand) — **not** a new event source, not something an agent authors. It aggregates, for that window: `TaskCompleted` / `TaskFailed` / `TaskBlocked` with their `report_ref` proofs+receipts, what **needs you** (pending `ApprovalRequested`, failures, blocks), quota consumed per harness (§7.2), and the **git sync state** (what pushed, what's on which branch, what workpc still needs to pull). Read-only, **digestible at a glance** (the SHOULD). + +`[S]` **The morning brief is just the overnight window's brief** — the payoff of Mode B (§2.3). Delivered per surface authz (§7.1): telegram gets the headline + ntfy ping, tui/web the full rollup. `[S]` It **surfaces** judgment and attention items; nothing in it self-applies — your call, same gate as everywhere. + +The per-task report (`report_ref`, §4) is one task's receipts; the brief is the window's rollup **over** those reports. Two levels, one is not the other. + +--- + +## 8. Build Sequence `[S]` + +Bottom-up. Each step is a consumer of the one below; do not start a step before its dependency is proven. + +1. **Task schema + provider port** — one interface, one **JSONL adapter**. (§3) +2. **Event log + state projection** — prove the read model; pick the log store + CAS backend here. (§3.1, §4) +3. **`*.md` conventions** — near-zero code; encodes most policy SHOULDs. (§6.3) +4. **herdr harness + rotation** — as a **consumer** of 1–2. Build the **occupancy measurement first** and verify against a live session (§5.2). Orchestrator in **bash**, then port. (§5) +5. **Surfaces, one at a time** — each a subscriber with its bus capability (§7). Telegram/ntfy first (read-only, lowest risk) → tui/web → Maven-gated. + +**Cross-cutting from day one:** Invariants (§1) are asserted as each layer lands — especially append-only (1), plane-emits (2), hash-refs (3), typed-no-freeform (4). TDD by default; QA suites mandatory; cheap harness before any manual debug/test. + +--- + +## 9. Open Questions — with degrade-safe defaults + +Named so the spec is written against a known frontier. Each has a default so a gap never becomes a hallucinated decision. + +| # | Question | Status | Degrade-safe default | +|---|---|---|---| +| 1 | **Quota projection accuracy** across cc/codex/opencode (reconstructed, non-authoritative) | `[O]` | Treat `[D]` 80% reported as full; router uses conservative number. Refine post-measurement. | +| 2 | **Schema-evolution mechanic** — upcast-on-read vs. tolerant-reader | `[O]` (versioned = decided) | Tolerant-reader + versioned envelope until a real migration forces upcasting. | +| 3 | **Log retention / cold-storage aging** as the log grows | `[O]` (deferred) | Snapshot + keep all raw events online; revisit only when size bites. | +| 4 | **Storage backend** for append-only log (+ CAS on MinIO?) | `[O]` | Decide in Step 2 spike; JSONL baseline works meanwhile. | +| 5 | **Cross-surface approval propagation** detail (poll cadence, dedup) | `[O]` (deferrable) | Resolve-by-event, no timeout; clients poll `[D]` on a short interval. | +| 6 | **Capability tag vocabulary** — the actual tag set beyond `{correlation, mechanical}` | `[O]` | Start with those two; grow from real task triage, not up front. | +| 7 | **Machine affinity: hard or soft?** (§2.2) — can a workpc-primary project run on homesrv against the remote overnight, or wait for workpc? | `[O]` | **Hard** — never run where not configured; overnight queue = server-runnable projects only. Soft is a one-flag change. | +| 8 | **Cross-machine lease correctness** (§2.1) — worktree/anchor validation and quota accounting when a lease's git checkout lives on a different host than the router | `[O]` | Git is the only cross-machine transport; validate against the local checkout wherever the harness runs; quota accounted per-host. Prove on the first federated run. | + +**Decided elsewhere (do not reopen):** compensation over compaction; versioned schema; snapshot+replay; per-harness trigger = adapter Face B; retry-then-relet with TTL reclamation; capability-static vs. availability-dynamic; pickup-only importance (no preemption); surface authorization split; injection as an everywhere-invariant; **federated worker topology (homesrv authoritative, workpc intermittent worker); herdr on both, driven-not-owned; git as cross-machine transport; task→project→machine-affinity; the brief as a windowed projection.** + +**Explicitly out of scope — deferred by decision, not omission:** cheap-model / free-inference **delegation** (offloading mechanical sub-work to local llama-server or free providers). Concluded orthogonal to this system — **rotation is the quota lever, delegation is at most a trim**, and the real objective is unattended runtime, not per-turn price (§0). Revisit later as a per-task routing hint carried in task metadata, once the substrate runs and a profiled session shows where tokens actually go. Named here so it stays a choice. + +--- + +*This spec is the contract. Where it says `[O]`, the default holds until you decide — an implementing agent must surface the open item, never silently pick.* diff --git a/progress.md b/progress.md new file mode 100644 index 0000000..db99556 --- /dev/null +++ b/progress.md @@ -0,0 +1,108 @@ +# Orchestra progress + +Updated: 2026-07-26 + +## Server implementation checklist + +This is the implementation-oriented breakdown of the specification. It is a project checklist, not a replacement for the binding spec. + +1. **Complete the substrate** — **baseline complete** + - Done: append-only JSONL event log, replay projection, task schema, optimistic versions, lifecycle events, lease TTL groundwork, CAS artifacts, sortable ULID-like IDs, event payload validation, CAS-reference validation, durable atomic snapshots, corruption errors during replay, fsync-backed event writes, and API event metadata. + - Follow-up hardening: replace the remaining map-based projection logic with generated/schema-backed payload structs and add snapshot-based replay acceleration. + +2. **Provider layer** — **partial** + - Done: Provider/Sink contracts and replay-safe JSONL adapter. + - Remaining: + - JSONL file watcher/ingester + - Gitea issue and CI-failure adapter + - Reflect task state back to external systems + - Webhook authentication and polling + +3. **Projects and machine registry** — **not started** + - Project configuration + - Machine registry + - Herdr registry + - Reachability checks + - Hard machine affinity resolution + +4. **Router and leases** — **partial groundwork** + - Done: manual lease/release/complete/block endpoints and lease-expiry release. + - Remaining: + - Assignment on `TaskCreated` + - Assignment on release/expiry + - Capability matching + - Availability and concurrency filtering + - Importance ordering + - Retry/backoff and eventual `TaskFailed` + +5. **Herdr integration** — **not started** + - Herdr socket client + - Protocol-version check + - Harness adapters for Claude, Codex, and opencode + - Bootstrap prompts + - Native occupancy measurement + - Stop-hook/turn-boundary rotation + - Worktree and anchor validation + - Split-then-close rotation flow + +6. **Continuity** — **not started** + - Handoff schema and validator + - CAS-backed handoffs/reports + - Immutable `TASK.md` handling + - Scratch-branch pickup contract + - Shared Markdown change notifications + +7. **Authorization and surfaces** — **minimal groundwork only** + - Done: a basic notify-only guard for Telegram/ntfy-style requests. + - Remaining: + - Real bus-level authorization + - TUI/web control surface + - Telegram/ntfy read-only subscribers + - Approval request/grant/deny flow + - MCP gated writes + - Maven gated control + +8. **Projections and operations** — **not started** + - Quota projection + - Nightly/morning brief + - Git sync state + - Standup advisory events + - Logging, metrics, service packaging, and deployment configuration + +## Completed + +- Built the first Go server slice from `orchestra-spec (1).md`. +- Added append-only JSONL events and replay projection in `internal/store`. +- Added task creation, external-key deduplication, optimistic versions, lifecycle states, and SHA-256 CAS artifacts. +- Added HTTP endpoints on default port `9145`: health, task ingest/list, and event cursor reads. +- Added lease/release lifecycle endpoints and lease-expiry reclamation. +- Finished the item 1 provider port: `provider.Provider`/`Sink` interfaces and a replay-safe JSONL adapter. +- Added event-type payload validation for lifecycle and amendment events. +- Unit tests pass with `go test ./...`. + +## Current API additions + +- `POST /v1/tasks/{id}/lease` with `{"harness_id":"...","ttl_seconds":1800}` +- `POST /v1/tasks/{id}/release` +- `POST /v1/tasks/{id}/complete` +- `POST /v1/tasks/{id}/block` + +## Item 1 status + +Item 1 (task schema + provider port + JSONL adapter) is implemented as the baseline slice. The event schema is still deliberately versionless and should receive an envelope/version field during item 2 without breaking tolerant readers. + +## Important limitations + +- This is still a Layer 1 prototype. No router, project/machine/herdr registries, harness adapters, herdr socket integration, rotation, handoff validation, provider interface, Gitea adapter, approvals, TUI/web, quota projection, or morning brief exists yet. +- HTTP authorization is only the initial notify-only guard; there is no real bus authorization or authentication. +- Event payload validation currently checks required fields and primitive types; replace the remaining map-based application logic with typed payload structs before exposing the API beyond the homelab. +- Lease expiry currently releases tasks but does not yet implement retry counts/backoff or emit `TaskFailed` after a configured limit. + +## Next agent: recommended order + +1. Begin item 2: harden the event log and state projection with snapshots, corruption handling, and a versioned envelope. +2. Add project, machine, and herdr registries from static TOML/JSON config. +3. Implement router selection: project affinity, reachability, capability, availability, and importance ordering. +4. Add retry policy and a background lease-expiry loop. +5. Implement handoff/report schemas and CAS reference validation. +6. Integrate herdr only after the substrate/router tests are stable.