fix(herdr): retry agent.start/agent.prompt through pane-boot readiness race (B12)
herdr hands a freshly created pane/agent back before it's actually ready,
and rejects the very next call with a range of different transient errors
("not an available shell", "not an active named agent", "target ... not
found") depending on timing. String-matching each wording as it turned up
live proved unwinnable across three live redeploy-and-test rounds, so
StartAgent and Prompt now retry any error for up to 15s (bounded by
wall-clock time, not attempt count) rather than pattern-matching herdr's
error text.
Confirmed live against workpc: a fresh lease (wD:p1) now reaches a real
attached claude session instead of failing before the agent starts.
Live testing also exposed a second, separate defect (B13, documented in
AUDIT.md, not fixed here): agent.start can return success while never
actually starting an agent when two leases land close together, with no
error for a retry to catch. Left three test panes on workpc untouched
(wD:p1, wE:p1, wF:p1) pending manual cleanup, per the standing rule against
destructive herdr calls without asking first.
Also folds in the already-flattened AUDIT.md/progress.md merge that was
staged ahead of this session's changes.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01W1rkJ2hBMybnJctPbcy4tT
This commit is contained in:
kami
+54
-8
@@ -165,12 +165,39 @@ type Session struct {
|
|||||||
ConventionsHash string `json:"conventions_hash,omitempty"`
|
ConventionsHash string `json:"conventions_hash,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// bootDeadline bounds the retry loops below. Freshly created panes/agents
|
||||||
|
// have been observed (live, 2026-07-28) to reject the very next call for a
|
||||||
|
// range of different transient reasons as herdr finishes bringing them up —
|
||||||
|
// "not an available shell", "not an active named agent", "target ... not
|
||||||
|
// found" — a new wording each time a prior one got fixed. There is no other
|
||||||
|
// legitimate reason a call against state orchestra itself just created would
|
||||||
|
// fail immediately, so these loops retry any error rather than pattern-match
|
||||||
|
// an open-ended and apparently still-growing set of herdr wordings, bounded
|
||||||
|
// by wall-clock time rather than attempt count so a slow-booting pane still
|
||||||
|
// gets the same real budget as a fast-failing one.
|
||||||
|
const (
|
||||||
|
bootRetryWindow = 15 * time.Second
|
||||||
|
bootRetryDelay = 500 * time.Millisecond
|
||||||
|
)
|
||||||
|
|
||||||
func (c *Client) Prompt(ctx context.Context, pane, text string, wait time.Duration) error {
|
func (c *Client) Prompt(ctx context.Context, pane, text string, wait time.Duration) error {
|
||||||
p := map[string]any{"target": pane, "text": text}
|
p := map[string]any{"target": pane, "text": text}
|
||||||
if wait > 0 {
|
if wait > 0 {
|
||||||
p["wait"] = map[string]any{"until": []string{"idle"}, "timeout_ms": wait.Milliseconds()}
|
p["wait"] = map[string]any{"until": []string{"idle"}, "timeout_ms": wait.Milliseconds()}
|
||||||
}
|
}
|
||||||
return c.Call(ctx, "agent.prompt", p, nil)
|
deadline := time.Now().Add(bootRetryWindow)
|
||||||
|
var err error
|
||||||
|
for {
|
||||||
|
err = c.Call(ctx, "agent.prompt", p, nil)
|
||||||
|
if err == nil || time.Now().After(deadline) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
case <-time.After(bootRetryDelay):
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
func (c *Client) Worktree(ctx context.Context, cwd, path, branch string) (string, error) {
|
func (c *Client) Worktree(ctx context.Context, cwd, path, branch string) (string, error) {
|
||||||
var r worktreeResponse
|
var r worktreeResponse
|
||||||
@@ -204,15 +231,34 @@ func (c *Client) StartAgent(ctx context.Context, cwd, path, branch, harness, tas
|
|||||||
return Session{}, fmt.Errorf("herdr: no pane recorded for worktree %s", path)
|
return Session{}, fmt.Errorf("herdr: no pane recorded for worktree %s", path)
|
||||||
}
|
}
|
||||||
var s Session
|
var s Session
|
||||||
if err := c.Call(ctx, "agent.start", map[string]any{
|
deadline := time.Now().Add(bootRetryWindow)
|
||||||
"pane_id": paneID,
|
var err error
|
||||||
"kind": harness,
|
for {
|
||||||
"name": harness,
|
s = Session{}
|
||||||
"args": []string{},
|
err = c.Call(ctx, "agent.start", map[string]any{
|
||||||
}, &s); err != nil {
|
"pane_id": paneID,
|
||||||
if !strings.Contains(strings.ToLower(err.Error()), "already") {
|
"kind": harness,
|
||||||
|
"name": harness,
|
||||||
|
"args": []string{},
|
||||||
|
}, &s)
|
||||||
|
if err == nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if strings.Contains(strings.ToLower(err.Error()), "already") {
|
||||||
|
err = nil
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
return Session{}, err
|
return Session{}, err
|
||||||
}
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return Session{}, ctx.Err()
|
||||||
|
case <-time.After(bootRetryDelay):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return Session{}, err
|
||||||
}
|
}
|
||||||
s.PaneID = paneID
|
s.PaneID = paneID
|
||||||
s.Worktree = path
|
s.Worktree = path
|
||||||
|
|||||||
-628
@@ -1,628 +0,0 @@
|
|||||||
# Orchestra progress
|
|
||||||
|
|
||||||
Updated: 2026-07-27
|
|
||||||
|
|
||||||
## AUDIT.md remediation — in progress
|
|
||||||
|
|
||||||
Working through `AUDIT.md`'s blocking/secondary defects in order of the
|
|
||||||
"suggested order of attack." Each item below is landed, tested, and
|
|
||||||
committed individually; see the git log for the exact commits.
|
|
||||||
|
|
||||||
Fixed so far:
|
|
||||||
- **B2** — adapters were looked up by `session.Harness` (the harness kind,
|
|
||||||
e.g. `"claude"`) in `Reconcile`/`expire`/`rotate`, but `AdapterFactory.Herdrs`
|
|
||||||
is keyed by herdr instance id (e.g. `"homesrv-claude"`). Every one of those
|
|
||||||
call sites silently no-opped. Added `Coordinator.adapterFor`, routed all
|
|
||||||
four call sites through it. Regression test registers an adapter under a
|
|
||||||
herdr-id key distinct from the harness kind and asserts rotation fires.
|
|
||||||
- **B1** — `CLIAdapter.Occupancy` called `a.Usage(s.PaneID)`, but the usage
|
|
||||||
readers want a filesystem path to session state, not a herdr pane id.
|
|
||||||
Added `herdr.Session.SessionFile` and per-harness resolution
|
|
||||||
(`ClaudeSessionFile` by newest-mtime under Claude Code's own project
|
|
||||||
directory; codex via the existing `CodexActiveUsage` sqlite discovery;
|
|
||||||
opencode refuses loudly — needs a live session id, not resolvable from the
|
|
||||||
worktree alone). A missing/unreadable session file is now a hard error,
|
|
||||||
surfaced via new `SessionHealth.Occupancy`/`OccupancyError` fields on
|
|
||||||
`GET /v1/tasks/{id}/health`, not a silent zero. **Still needs live
|
|
||||||
verification against a real Claude Code session** (the spec's own
|
|
||||||
acceptance bar for this phase) — not possible from this sandbox.
|
|
||||||
- **B4** — the router counted every `TaskReleased` (including rotation,
|
|
||||||
which *is* a `TaskReleased` carrying a valid `handoff_ref`) against
|
|
||||||
`MaxAttempts`, and double-counted by also incrementing on every
|
|
||||||
subsequent lease. A task that rotated twice hit the default
|
|
||||||
`MaxAttempts=3` and was killed. Now only a release without a
|
|
||||||
`handoff_ref` (expiry/crash) advances the counter.
|
|
||||||
- **B8** — `X-Orchestra-Surface: system` was reachable from an HTTP request
|
|
||||||
header in both `authz.HTTP` and `main.go`'s `surface` closure (the one
|
|
||||||
every handler actually calls). Since no deployment sets
|
|
||||||
`ORCHESTRA_SYSTEM_TOKEN`, this was an unauthenticated full-control bypass
|
|
||||||
reachable from any LAN caller. Both call sites now downgrade `system` to
|
|
||||||
`web` before doing anything else with it.
|
|
||||||
- **S1** — `Brief.From`/`To` and `GitSync.Branch`/`Head`/`Status` all shared
|
|
||||||
one JSON tag each (Go only honors the first `json:"..."` tag on a
|
|
||||||
combined field declaration). `go vet ./...` now passes clean.
|
|
||||||
- **S5** — `Store.Lease`/`ExpireLeases` set `Event.ID` to the task id, so
|
|
||||||
every lease of a task produced colliding event IDs. Now `domain.NewID()`.
|
|
||||||
- **S6** — the ingest dedup path returned `nil` (success) without
|
|
||||||
appending; `main.go` then returned an unrelated event with `201`. Added
|
|
||||||
`domain.ErrDuplicate` and `Store.TaskBySource`; `POST /v1/tasks` now
|
|
||||||
returns the existing task with `200` on a duplicate. Updated every other
|
|
||||||
`Append` caller (Gitea poll/webhook, JSONL ingest) to treat
|
|
||||||
`ErrDuplicate` as expected rather than a failure — without that, Gitea
|
|
||||||
polling would error out of its scan loop on the first already-ingested
|
|
||||||
issue in every batch.
|
|
||||||
|
|
||||||
- **B3 (partial)** — added `POST /v1/harness/complete`, the first automatic
|
|
||||||
`TaskCompleted` producer (previously only a human calling
|
|
||||||
`/v1/tasks/{id}/complete` could ever complete a task). A Claude Code Stop
|
|
||||||
hook (`deploy/hooks/orchestra-stop.sh`) fires on every turn boundary but
|
|
||||||
only reports completion if the agent has written a `.orchestra-report.md`
|
|
||||||
marker at the worktree root first — an ordinary turn boundary is a no-op,
|
|
||||||
so this doesn't fire completion prematurely. The server reads the
|
|
||||||
transcript locally via `herdr.ClaudeUsage` to build the `receipt` itself
|
|
||||||
(input/cache/output token counts) rather than trusting a self-reported
|
|
||||||
number, and uploads the report body to CAS for `report_ref`. Guarded by an
|
|
||||||
optional `ORCHESTRA_HARNESS_TOKEN` bearer check; the event is appended with
|
|
||||||
`Surface: system` set directly in Go (not derived from a request header —
|
|
||||||
consistent with the B8 fix that system must never be header-controlled).
|
|
||||||
**Not done:** Codex/opencode equivalents (Claude-only for now — Codex would
|
|
||||||
need `CodexActiveUsage`, opencode `OpenCodeUsage`/`OpenCodeStatus`, wired
|
|
||||||
the same way), and the turn-boundary decision endpoint
|
|
||||||
(`continue`/`prepare_handoff`/`rotate_now`/`refuse`) from Phase 2 items 1–2
|
|
||||||
is still unbuilt — only the completion half of Phase 2 landed. No test
|
|
||||||
added for the new HTTP handler; `cmd/orchestra/main.go` has zero test
|
|
||||||
coverage for any handler (pre-existing gap, everything lives inline in
|
|
||||||
`main()`) so this follows the existing (untested) pattern rather than
|
|
||||||
introducing a one-off test harness.
|
|
||||||
|
|
||||||
- **B5 (loose end)** — `CLIAdapter.Lease`'s initial prompt used `wait=0`,
|
|
||||||
skipping the inline wait `Bootstrap` already used; the spec (§5.1) requires
|
|
||||||
inline `wait` on `agent.prompt` for bootstrap injection to avoid sending
|
|
||||||
into a half-rendered prompt. Changed to `time.Minute`, matching `Bootstrap`.
|
|
||||||
Small, contained fix — `Release`'s real implementation (needs Phase 4
|
|
||||||
handoff production) is still outstanding from B5.
|
|
||||||
|
|
||||||
- **B6 (partial — Phase 4 items 1 and 4)** — nothing wrote a `TASK.md` into a
|
|
||||||
worktree, so pickup validation had nothing to check and never ran anyway.
|
|
||||||
Fixed both halves: `GitWorktrees.Create` now writes and commits an
|
|
||||||
immutable `TASK.md` (`continuity.RenderTaskFile`) into every freshly
|
|
||||||
created worktree, and `Coordinator.Start` now runs
|
|
||||||
`continuity.ValidatePickup` (loading the handoff from CAS, checking anchor
|
|
||||||
SHA + dirty-file hashes + TASK.md hash) before bootstrapping a successor
|
|
||||||
onto a `handoff_ref` — a failure kills the session and emits `TaskBlocked`
|
|
||||||
instead of trusting an unvalidated ref. Covered by
|
|
||||||
`TestGitWorktreesCommitsTaskFile` and `TestStartBlocksOnInvalidPickup` in
|
|
||||||
`internal/orchestrator`. **Not done:** handoff *production* (nothing yet
|
|
||||||
writes a real §6.1 handoff — `Release` still refuses per B5), wiring
|
|
||||||
`ScratchCommit` before release, and the §6.2 bootstrap-prompt rewrite. See
|
|
||||||
AUDIT.md's "B6 — partial fix" section for the full breakdown, including a
|
|
||||||
named caveat: TASK.md hashing is best-effort and untested for the
|
|
||||||
herdr-hosted (`WorktreeCreator`) worktree path.
|
|
||||||
|
|
||||||
- **B5 (closed)** — `CLIAdapter.Release` previously just refused (no real
|
|
||||||
herdr method existed to call and there was nothing to validate against).
|
|
||||||
Now: reads the agent-authored `.orchestra-handoff.json` from the worktree
|
|
||||||
root, validates it with `continuity.Decode`, cross-checks its anchor SHA
|
|
||||||
against the worktree's real `HeadSHA` (never trusts the agent's self-report
|
|
||||||
outright), uploads it to CAS via `continuity.Save` to mint the
|
|
||||||
`handoff_ref`, and only then calls the real `pane.release_agent({pane_id,
|
|
||||||
source, agent})` to drop herdr's claim — sequenced last so a herdr-side
|
|
||||||
error can't strand an uploaded handoff. Any failure (missing file, invalid
|
|
||||||
schema, anchor mismatch, herdr error) is a refusal, which `rotate` already
|
|
||||||
treats as "retry next tick" rather than stranding the task. `herdr.Claude/
|
|
||||||
Codex/OpenCode` now take a `continuity.CAS` (main.go passes the existing
|
|
||||||
`*store.Store`). New tests in `internal/herdr/adapter_test.go` cover all
|
|
||||||
four paths against a real git worktree and a fake in-process herdr
|
|
||||||
listener. **Not done:** nothing yet makes the agent actually *write*
|
|
||||||
`.orchestra-handoff.json` (needs a stop-hook convention analogous to
|
|
||||||
`.orchestra-report.md`) — that and the rest of Phase 4 (ScratchCommit
|
|
||||||
before release, §6.2 bootstrap-prompt rewrite, `MarkdownChanges`) remain
|
|
||||||
open.
|
|
||||||
|
|
||||||
- **Phase 4 items 3, 5, 6** — `CLIAdapter.Release` now re-verifies every
|
|
||||||
`Anchor.Dirty` file hash (previously only the top-level `Anchor.GitSHA`
|
|
||||||
was checked; a file edited after the handoff was written but before
|
|
||||||
release would have gone through unnoticed), then, if there were dirty
|
|
||||||
entries, snapshots them atomically onto a per-task scratch branch
|
|
||||||
(`continuity.ScratchCommit`, made idempotent so a task can rotate more
|
|
||||||
than once) and rewrites the handoff's anchor to that new commit with
|
|
||||||
`Dirty` cleared before uploading — so the successor's pickup check is a
|
|
||||||
single HEAD compare, not N file rehashes. `CLIAdapter.Bootstrap`'s prompt
|
|
||||||
was rewritten to point the agent at `git log`/the scratch branch instead
|
|
||||||
of a vague "read the handoff" instruction, and deliberately avoids
|
|
||||||
claiming a `GET /v1/artifacts/<ref>` endpoint, since no such route exists
|
|
||||||
(`/v1/artifacts` is POST-only). `continuity.MarkdownChanges` (§6.3
|
|
||||||
adjacent-task notice) had zero callers and zero tests despite being
|
|
||||||
listed as implemented in an earlier snapshot — deleted rather than
|
|
||||||
half-wired, per AUDIT.md's explicit "delete and record the deviation"
|
|
||||||
option. New tests: `TestReleaseScratchCommitsDirtyFilesBeforeUpload`,
|
|
||||||
`TestReleaseRefusesOnStaleDirtyFile` (internal/herdr/adapter_test.go).
|
|
||||||
**§6.3 rewired for real, 2026-07-27 (later same day):** the deleted
|
|
||||||
`MarkdownChanges` above was zero-caller dead code, but the underlying spec
|
|
||||||
requirement ("on update, the orchestra injects a notice to agents whose
|
|
||||||
current task is adjacent") wasn't abandoned — rebuilt independently.
|
|
||||||
`continuity.ConventionsHash(root)` hashes whichever of
|
|
||||||
`AGENTS.md`/`CLAUDE.md`/`VOCAB.md` exist at a path; `herdr.Session` gained
|
|
||||||
`ConventionsHash`, snapshotted from the fresh worktree at
|
|
||||||
`Coordinator.Start`; a new `Coordinator.checkConventions`, run every
|
|
||||||
`Monitor` tick, recomputes the hash of the project's *base repo* (via
|
|
||||||
`WorktreeSpec.Spec` — "adjacent" = same project) for every leased session
|
|
||||||
and compares it against that session's stored snapshot. A mismatch calls a
|
|
||||||
new optional `herdr.ConventionsNotifier` capability
|
|
||||||
(`CLIAdapter.NotifyConventionsChanged`, an in-pane `agent.prompt` telling
|
|
||||||
the agent to re-read the docs) and updates the stored hash so the notice
|
|
||||||
fires once per drift, not every tick. Covered by
|
|
||||||
`TestConventionsDriftNotifiesActiveSession`
|
|
||||||
(internal/orchestrator/rotation_test.go): asserts no notification while
|
|
||||||
the base repo is unchanged, then one once it diverges.
|
|
||||||
**Was still open:** Phase 4 item 2 — nothing drove *any* harness to write
|
|
||||||
`.orchestra-handoff.json`, since Release only validated a file whose
|
|
||||||
existence was never solicited. **Closed 2026-07-27:** `rotate()` now checks
|
|
||||||
for the adapter's optional `herdr.HandoffRequester` capability; when
|
|
||||||
`HandoffFile` is missing at the worktree root, it prompts the agent once
|
|
||||||
(`CLIAdapter.RequestHandoff`, mirroring the `.orchestra-report.md`/B3
|
|
||||||
convention — the plane asks for a handoff, it never invents one) and skips
|
|
||||||
Release that tick, retrying every subsequent tick until the file appears.
|
|
||||||
`herdr.Session.HandoffRequested` avoids re-prompting every tick. Covered by
|
|
||||||
`TestRotationRequestsHandoffBeforeReleasing`
|
|
||||||
(`internal/orchestrator/rotation_test.go`), which asserts Release is never
|
|
||||||
called before the file exists and fires once it does. Codex/opencode still
|
|
||||||
share this same path (no harness-specific gap remains); the only leftover
|
|
||||||
question is whether each harness's own Stop-equivalent hook honors the
|
|
||||||
in-pane prompt to write the file before exiting, which is a live-deployment
|
|
||||||
fact, not something provable from source.
|
|
||||||
|
|
||||||
- **B7 (post-hoc producer) + Phase 2 turn-decision endpoint** — landed
|
|
||||||
together, since both are new `QuotaReported`/turn-boundary paths off the
|
|
||||||
same completion/turn events. `POST /v1/harness/complete` now appends a
|
|
||||||
`QuotaReported` event (`harness_id` from the closing lease, `consumed`
|
|
||||||
from the same `usage.Numerator()` used for the receipt), so the router's
|
|
||||||
5h/weekly availability filter and the brief's `quota_consumed` stop
|
|
||||||
evaluating against a permanent zero. New `Coordinator.TurnDecision`
|
|
||||||
(`internal/orchestrator/orchestrator.go`) mirrors `rotate()`'s per-task
|
|
||||||
logic (occupancy → turn-boundary → handoff-file → release) but runs
|
|
||||||
synchronously once per turn instead of waiting for `Monitor`'s ticker,
|
|
||||||
returning one of `continue`/`prepare_handoff`/`rotate_now`/`refuse` via the
|
|
||||||
new `POST /v1/harness/turn`. The Claude Stop hook
|
|
||||||
(`deploy/hooks/orchestra-stop.sh`) now calls this endpoint on every
|
|
||||||
ordinary turn boundary (report marker absent) instead of no-op'ing, and
|
|
||||||
exits 2 on `refuse` to stop the harness from finishing an unsafe turn.
|
|
||||||
Covered by `TestTurnDecision` (`internal/orchestrator/rotation_test.go`):
|
|
||||||
continue-below-threshold, refuse-when-not-at-boundary, and
|
|
||||||
rotate_now-releases-and-emits-a-valid-TaskReleased cases.
|
|
||||||
**Not done:** live per-harness *push* producers (Claude statusline,
|
|
||||||
Codex rollout tail) that would give B7 a second, continuous producer
|
|
||||||
independent of task completion — recorded as a design investigation in
|
|
||||||
AUDIT.md ("Real harness quota sources") but not implemented; Codex/
|
|
||||||
opencode's own equivalents of the Claude Stop hook (whether their
|
|
||||||
turn-boundary mechanism actually calls `/v1/harness/turn`) also remain
|
|
||||||
unbuilt, same caveat as Phase 2 item 4 already named for `/complete`.
|
|
||||||
|
|
||||||
- **S4** — `delivery.Fanout.Run` used to `return` on the first sender error,
|
|
||||||
permanently killing the notification goroutine (a single ntfy hiccup meant
|
|
||||||
no notifications for the rest of the process's lifetime, since nothing
|
|
||||||
restarts it). Failed sends now go through an `OnError` hook instead of
|
|
||||||
aborting the loop. Cursor is also persisted now (`SaveCursor` → a
|
|
||||||
`delivery-cursor` file next to `ORCHESTRA_DATA`, loaded on startup), so a
|
|
||||||
restart resumes from the last delivered event instead of re-notifying the
|
|
||||||
entire log from seq 0. `internal/delivery` previously had zero tests;
|
|
||||||
added `TestFanoutContinuesAfterSendError`.
|
|
||||||
|
|
||||||
- **S2 + S3** — `/v1/brief`'s git state used to come from `ORCHESTRA_DATA`
|
|
||||||
(the event-log directory, not a git checkout — always reported
|
|
||||||
`"git unavailable"`), and completions were only counted, never surfaced
|
|
||||||
with proof. `operations.Brief.Git` is now `map[string]GitSync` keyed by
|
|
||||||
project ID, built in `main.go` from each `registry.Project.Repo` (falling
|
|
||||||
back to a single `"default"` entry off `ORCHESTRA_REPO` for deployments
|
|
||||||
without per-project repos); `GitSync` gained `Ahead`/`Behind` vs upstream.
|
|
||||||
New `Brief.Receipts []operations.CompletionReceipt` pulls `report_ref`/
|
|
||||||
`receipt` straight out of each `TaskCompleted` event's existing payload.
|
|
||||||
Covered by an updated `TestBuildBrief`.
|
|
||||||
|
|
||||||
- **S7** — `TaskAmended` only ever applied `title` from the amendment
|
|
||||||
payload; `due`/`description`/`inherent_priority` amendments were accepted
|
|
||||||
and logged but silently dropped by the projection. Also added the missing
|
|
||||||
`Task.Description` field (it didn't exist at all, so `TaskCreated` dropped
|
|
||||||
it too). Both `TaskCreated` and `TaskAmended` now populate all four fields.
|
|
||||||
Covered by `TestTaskAmendedAppliesAllFields` (`internal/store`).
|
|
||||||
|
|
||||||
- **Codex/opencode completion producers** — `/v1/harness/complete` was
|
|
||||||
Claude-only (hardcoded `herdr.ClaudeUsage`). Added an optional `harness`
|
|
||||||
field to the request body (`""`/`"claude"` unchanged default); `"codex"`
|
|
||||||
dispatches to `herdr.CodexUsage`, `"opencode"` to `herdr.OpenCodeUsage`,
|
|
||||||
anything else is a 400. Both readers already existed and were tested in
|
|
||||||
`internal/herdr` but had zero callers outside tests — same "written, not
|
|
||||||
wired" pattern the rest of this audit keeps finding. The Claude stop hook
|
|
||||||
(`deploy/hooks/orchestra-stop.sh`) now sends `"harness":"claude"`
|
|
||||||
explicitly for symmetry, no behavior change. **Not done:** an actual
|
|
||||||
Codex/opencode Stop-hook-equivalent script — Codex has no native stop hook
|
|
||||||
(would need a rollout-tail poller deciding when to call this), and
|
|
||||||
opencode's own turn-boundary mechanism is unverified from source (same
|
|
||||||
caveat AUDIT.md already names for Phase 2 item 4). This change unblocks
|
|
||||||
the server side; the harness-side wrapper for either is still open. No
|
|
||||||
new test: `cmd/orchestra` has zero handler test coverage of any kind
|
|
||||||
(pre-existing, confirmed by grep before writing this), so this follows the
|
|
||||||
existing pattern rather than introducing a one-off test harness for one
|
|
||||||
handler.
|
|
||||||
|
|
||||||
- **S9** — the coordinator's `Monitor` loop (30s ticker, `Coordinator.expire`)
|
|
||||||
and `main.go`'s own 1s reclaim ticker both called `Store.ExpireLeases`
|
|
||||||
independently. AUDIT.md filed this as "harmless — CAS rejects the loser,"
|
|
||||||
but the real effect was worse: the coordinator's `expire()` is the *only*
|
|
||||||
place that kills the herdr session/pane for an expired lease, and since the
|
|
||||||
1s ticker ran 30x more often it almost always expired the lease first,
|
|
||||||
leaving the coordinator's own `ExpireLeases` call with nothing left to
|
|
||||||
expire — so its pane-kill path silently never ran, orphaning herdr panes
|
|
||||||
past their TTL whenever a coordinator was configured. Fixed by having
|
|
||||||
`main.go`'s ticker skip `ExpireLeases` entirely when `coordinator != nil`
|
|
||||||
and defer reclaim to the coordinator's loop, keeping only `AssignPending`
|
|
||||||
as a periodic retry. No coordinator (e.g. no herdrs configured) still uses
|
|
||||||
the direct `ExpireLeases` path, since nothing else would reclaim leases in
|
|
||||||
that case.
|
|
||||||
|
|
||||||
- **S10** — `federation.Registry.Register` accepted a self-declared `id` and
|
|
||||||
self-chosen `token` from any caller with no admission control, and
|
|
||||||
silently let a second caller re-register an existing worker id with a
|
|
||||||
*different* token, hijacking that worker's identity/capacity out from
|
|
||||||
under it. Added `Registry.AdmitToken` (a pre-shared secret, wired from
|
|
||||||
`ORCHESTRA_FEDERATION_ADMIT_TOKEN`, checked against the registration
|
|
||||||
request's `Authorization: Bearer` header in `main.go`) and a same-ID
|
|
||||||
re-registration now requires presenting the existing worker's own token.
|
|
||||||
Covered by `TestRegisterRequiresAdmitTokenAndOwnToken`
|
|
||||||
(`internal/federation/federation_test.go`): wrong admit token rejected,
|
|
||||||
correct admit token accepted, same-id-different-token rejected as a
|
|
||||||
hijack, same-id-same-token (legitimate restart) still succeeds.
|
|
||||||
|
|
||||||
- **S11 (partial — soft threshold only)** — added `Coordinator.Soft`
|
|
||||||
(default 0.55, `ORCHESTRA_OCCUPANCY_SOFT` override). Both `rotate()` and
|
|
||||||
`TurnDecision` now request a handoff once occupancy crosses `Soft`, well
|
|
||||||
before `Hard` forces one; `TurnDecision` returns `prepare_handoff` for this
|
|
||||||
advisory case without requiring a turn boundary, leaving the task leased.
|
|
||||||
Covered by a new `TestTurnDecision` subtest. **Not done:** milestone
|
|
||||||
rotation, thrash detection, agent-initiated `ROTATE` — see AUDIT.md's S11
|
|
||||||
section for why those are bigger than a threshold check.
|
|
||||||
|
|
||||||
- **S11 (agent-initiated ROTATE)** — the schema already accepted
|
|
||||||
`meta.reason: "manual"` (since B5) but nothing checked for it. New
|
|
||||||
`handoffReason(worktree)` reads a written `.orchestra-handoff.json` and, if
|
|
||||||
its reason is `"manual"`, both `rotate()` and `TurnDecision` skip occupancy
|
|
||||||
and the turn-boundary probe entirely and release immediately — the agent's
|
|
||||||
own handoff *is* the boundary signal per §5.3 ("a coherent unit finished and
|
|
||||||
the next is independent"). Extracted the shared release-and-certify tail
|
|
||||||
into `Coordinator.finishRelease` so the manual path reaches the same
|
|
||||||
anchor-safety guarantee as the threshold path. Covered by a new
|
|
||||||
`TestTurnDecision` subtest (occupancy=0, boundary=false — every other path
|
|
||||||
would refuse or continue — asserting rotation happens anyway once a
|
|
||||||
`reason=manual` handoff exists).
|
|
||||||
|
|
||||||
- **S8** — no compensation-event mechanism existed for §3.1's own invariant
|
|
||||||
("a wrong event is never edited; a compensating event is appended and
|
|
||||||
replay sees both"). Added `TaskCorrected`: payload requires `corrects`
|
|
||||||
(the id of the event it repairs) plus at least one change (`state`, or the
|
|
||||||
existing amend-style metadata fields). `Store.Append` rejects a `corrects`
|
|
||||||
that doesn't name a real prior event on the same task; `store.apply`
|
|
||||||
applies the state/field changes and clears `Lease` like every other
|
|
||||||
terminal-state branch. Covered by `TestTaskCorrected`
|
|
||||||
(`internal/store/store_test.go`): a mistaken `TaskFailed` reverted to
|
|
||||||
`queued`, an unknown-`corrects` rejection, and both events surviving a
|
|
||||||
snapshot+replay reopen.
|
|
||||||
|
|
||||||
- **Codex/opencode Stop-hook-equivalent scripts** — the server side
|
|
||||||
(`/v1/harness/turn`, `/v1/harness/complete` dispatching by `harness`) has
|
|
||||||
existed since the turn-decision endpoint and the codex/opencode dispatch
|
|
||||||
commit, but nothing called it for either harness: both lack a native Stop
|
|
||||||
hook, so `orchestra-stop.sh`'s per-turn-boundary call never had an
|
|
||||||
equivalent. Added `deploy/hooks/orchestra-codex-poll.sh` and
|
|
||||||
`deploy/hooks/orchestra-opencode-poll.sh` — background poll loops (default
|
|
||||||
60s, `ORCHESTRA_POLL_INTERVAL`) meant to run alongside the harness process
|
|
||||||
in its pane. Each tick: find the newest session-state file (codex: newest
|
|
||||||
`rollout-*.jsonl` under `~/.codex/sessions`, matching `CodexActiveUsage`'s
|
|
||||||
own "most recently touched" heuristic; opencode: newest file under
|
|
||||||
`~/.local/share/opencode/storage/message/`, the same backstop
|
|
||||||
`OpenCodeUsage` reads, since a session id for the SSE fast path isn't
|
|
||||||
reliably available outside the opencode process itself); if
|
|
||||||
`.orchestra-report.md` exists, POST it plus the discovered path to
|
|
||||||
`/v1/harness/complete` with the right `harness` value and remove the
|
|
||||||
marker; otherwise POST task id to `/v1/harness/turn` and log (not act on)
|
|
||||||
`refuse`/`rotate_now` — unlike the Claude Stop hook's `exit 2`, there's no
|
|
||||||
turn boundary to refuse *at* from outside the harness process for either
|
|
||||||
of these, so this is advisory-only until real app-server/SSE integration
|
|
||||||
exists. No Go changes; nothing new to build/vet/test.
|
|
||||||
|
|
||||||
- **S11 (milestone + thrash detection) — closed, 2026-07-28.** The last two
|
|
||||||
of S11's four rotation triggers, previously deferred as needing "transcript/
|
|
||||||
tool-call introspection this repo doesn't have a source for" — that source
|
|
||||||
now exists. New `internal/herdr/activity.go`:
|
|
||||||
- `ToolCall{Name, Kind, Key, Success, IsTest}` normalizes one tool/function
|
|
||||||
call across harnesses (`Kind` is `"file"` or `"command"`, `Key` the path
|
|
||||||
or shell command — the same two field names, `file_path`/`command`, both
|
|
||||||
Claude's `tool_use.input` and Codex's function-call `arguments` use).
|
|
||||||
- `ClaudeActivity` parses the same transcript `ClaudeUsage`/`ClaudeSessionFile`
|
|
||||||
already open, pairing `tool_use`/`tool_result` blocks by `tool_use_id`
|
|
||||||
(an unresolved tool_use — mid-turn — is dropped, not reported).
|
|
||||||
- `CodexActivity` parses the rollout's `function_call`/`function_call_output`
|
|
||||||
payload pairs, mirroring `CodexUsage`'s existing `payload.type` wrapper —
|
|
||||||
**explicitly marked best-effort/unverified** in its doc comment, same bar
|
|
||||||
AUDIT.md's Phase 0 set for herdr methods: this hasn't been checked against
|
|
||||||
a live rollout, only against `CodexUsage`'s already-confirmed shape.
|
|
||||||
- `OpenCodeActivity` **refuses outright** rather than guess: opencode's
|
|
||||||
on-disk message storage is only confirmed to carry aggregate token counts
|
|
||||||
(what `OpenCodeUsage` already reads), not per-tool-call records, so
|
|
||||||
fabricating a parser against an unconfirmed shape would repeat the exact
|
|
||||||
mistake this repo's own audit exists to catch.
|
|
||||||
- `DetectThrash(calls, ThrashConfig)` implements all three §5.3 rules — N
|
|
||||||
consecutive failed test runs (regex-matched shell commands: go/pytest/npm/
|
|
||||||
yarn/cargo/make/jest/mvn/rspec/ctest), the same file edited M times
|
|
||||||
(`Edit`/`Write`/`MultiEdit`/`NotebookEdit`, explicitly *not* `Read` —
|
|
||||||
caught by a test that initially failed because rule 3 didn't exclude
|
|
||||||
reads), and the identical tool call repeated K times back-to-back
|
|
||||||
(explicitly excluding test re-runs and file reads, since re-running the
|
|
||||||
same test after a fix attempt is expected behavior, not thrashing — caught
|
|
||||||
by another initially-failing test). Each rule that trips populates a
|
|
||||||
`continuity.DeadEnd`, handed straight to the new prompt below rather than
|
|
||||||
left for the agent to invent. Defaults: 3/5/4, overridable via
|
|
||||||
`Coordinator.Thrash`.
|
|
||||||
- `DetectMilestone(calls)` — deliberately narrow: only "the most recent
|
|
||||||
call was a successful `git commit`". Spec-fuzzier definitions (a passing
|
|
||||||
test suite, a finished subtask) were **not** guessed at; same restraint
|
|
||||||
the rest of this audit has applied to unverified behavior.
|
|
||||||
- `CLIAdapter.Activity` (adapter.go) resolves the session file the same way
|
|
||||||
`Occupancy` does and dispatches to the right parser; `ActivityReader` is
|
|
||||||
an optional capability like every other Face-B interface in this package.
|
|
||||||
- New `ReasonedHandoffRequester`/`CLIAdapter.RequestHandoffReason` — like
|
|
||||||
`RequestHandoff` but names *why* (thrash's specific dead ends, or the
|
|
||||||
milestone reasoning) instead of the generic "context budget reached"
|
|
||||||
framing, and asks the agent to write `meta.reason` accordingly.
|
|
||||||
- `Coordinator.rotate()` and `TurnDecision()`: the existing "reason=manual
|
|
||||||
bypasses occupancy" shortcut generalized to `manual`/`milestone`/`thrash`
|
|
||||||
alike — once a handoff exists carrying one of these reasons, that **is**
|
|
||||||
the boundary signal, same treatment as agent-initiated ROTATE already
|
|
||||||
got. Before falling through to occupancy/soft/hard, both now call new
|
|
||||||
`checkActivityTriggers` (adapter implements `ActivityReader`? read
|
|
||||||
calls, thrash takes priority over milestone) and, on a hit,
|
|
||||||
`requestReasonedHandoff` (same `HandoffRequested`-guarded ask-once pattern
|
|
||||||
the occupancy path already uses) — request only, never release, exactly
|
|
||||||
like the soft-threshold path's `prepare_handoff`.
|
|
||||||
- Tests: `internal/herdr/activity_test.go` (parser + all three detector
|
|
||||||
rules, including the two cases that caught real bugs above) and
|
|
||||||
`internal/orchestrator/rotation_test.go`'s new
|
|
||||||
`TestActivityTriggersRequestReasonedHandoffWithoutReleasing` (thrash and
|
|
||||||
milestone each request-without-releasing via `TurnDecision`, a
|
|
||||||
thrash-reasoned handoff already on disk bypasses occupancy and releases,
|
|
||||||
and `rotate()`'s periodic path does the same request-without-release).
|
|
||||||
- Codex/opencode Stop-hook-equivalent poll scripts
|
|
||||||
(`deploy/hooks/orchestra-codex-poll.sh`,
|
|
||||||
`orchestra-opencode-poll.sh`, added earlier this session) already call
|
|
||||||
`/v1/harness/turn`, so `TurnDecision`'s new thrash/milestone checks reach
|
|
||||||
those harnesses for free wherever `Activity` is implemented (Codex, once
|
|
||||||
its unverified parser is checked; opencode not until a real per-tool-call
|
|
||||||
source is found).
|
|
||||||
|
|
||||||
- **S11 (CodexActivity verified against a live rollout) — closed, 2026-07-28.**
|
|
||||||
The previously-unverified `function_call`/`function_call_output` shape
|
|
||||||
turned out not to exist in any real Codex rollout on this machine. Rewrote
|
|
||||||
`CodexActivity` against the real shape found in `~/.codex/sessions`:
|
|
||||||
`event_msg`/`patch_apply_end` for file edits (has `changes`+`success`
|
|
||||||
directly, no pairing needed), and `response_item`/`custom_tool_call` named
|
|
||||||
`"exec"` (a freeform JS-scripted tool, not flat arguments — commands are
|
|
||||||
extracted from an embedded `cmd:"..."` via regex) paired with
|
|
||||||
`custom_tool_call_output`, whose failure signal is the observed literal
|
|
||||||
prefix `"Script error:"` in the output text. New tests
|
|
||||||
(`TestCodexActivityParsesRealRolloutShape`,
|
|
||||||
`TestCodexActivityMarksScriptErrorAsFailure`) use fixtures built from the
|
|
||||||
confirmed shape; also manually re-ran the parser against a real multi-
|
|
||||||
hundred-line rollout file and spot-checked the output by eye. See
|
|
||||||
AUDIT.md's S11 section for the full writeup and the one known remaining
|
|
||||||
limitation (only the first command in a chained exec script is extracted,
|
|
||||||
so `DetectMilestone` can miss a commit that isn't the first call in its
|
|
||||||
script).
|
|
||||||
|
|
||||||
- **Live status check, 2026-07-28** — the stuck task named throughout this
|
|
||||||
file (`06FT6CKD9Y98AZRX6X8K3QXFZG`) is no longer "stuck" from Orchestra's
|
|
||||||
own point of view: it now reads `state: "failed"` (retries exhausted,
|
|
||||||
`MaxAttempts` hit it). But the underlying herdr pane (`wA:p1`, opencode)
|
|
||||||
is still live and `agent_status: "blocked"` — confirmed via a direct
|
|
||||||
`agent.get` probe against `192.168.1.105:9245` — meaning the orphaned-pane
|
|
||||||
prediction in AUDIT.md's B2/B5 sections was correct: the router gave up
|
|
||||||
and moved on, but nothing ever released or killed the actual agent. Left
|
|
||||||
untouched deliberately (no `pane.close`/`release_agent` call made) — user
|
|
||||||
was asked and chose to leave it for now rather than have it cleaned up in
|
|
||||||
this session.
|
|
||||||
|
|
||||||
Not yet started: live verification of Phase 1 occupancy against a real
|
|
||||||
session, and the two-machine federation run. See `AUDIT.md` for the full
|
|
||||||
plan.
|
|
||||||
|
|
||||||
**Phase 0 done (2026-07-27):** this box has live TCP reachability to the real
|
|
||||||
herdr instance at `192.168.1.105:9245` — verified by hand (raw JSON-RPC
|
|
||||||
probes, no `herdr` CLI available locally). Real method list captured in
|
|
||||||
`deploy/herdr-schema.json`. Confirmed `pane.release`/`pane.kill`/
|
|
||||||
`pane.rotation_signal` are invented, as AUDIT.md's B5 suspected.
|
|
||||||
`pane.kill`→`pane.close` fixed as a drop-in. `pane.rotation_signal`/
|
|
||||||
`RotationSignal` deleted (no replacement exists). `Release` now refuses
|
|
||||||
loudly instead of calling a nonexistent method — its real implementation
|
|
||||||
needs Phase 4 (handoff production) first, since even the real
|
|
||||||
`pane.release_agent` can't return a `handoff_ref` (herdr doesn't write
|
|
||||||
handoffs, the agent does). See AUDIT.md's new "Phase 0 — done" section for
|
|
||||||
full detail. **Also found: a real task is currently stuck live** — workspace
|
|
||||||
`wA`, task `06FT6CKD9Y98AZRX6X8K3QXFZG`, opencode, pane `wA:p1`, blocked —
|
|
||||||
deliberately not touched from this session.
|
|
||||||
|
|
||||||
## Current state
|
|
||||||
|
|
||||||
This is a working Go implementation of `orchestra-spec (1).md`'s Layer 1–3
|
|
||||||
(substrate, harness/rotation, continuity) plus a first cut of Layer 4
|
|
||||||
(surfaces). `go build ./...` and `go test ./...` both pass. The codebase is
|
|
||||||
small (~4.6k lines across `internal/{domain,store,provider,registry,router,
|
|
||||||
herdr,orchestrator,continuity,federation,delivery,authz,operations,admin}`
|
|
||||||
and `cmd/orchestra/main.go`).
|
|
||||||
|
|
||||||
Earlier revisions of this file accumulated a long, self-contradictory
|
|
||||||
chronological log — gaps were listed as open in one section and then claimed
|
|
||||||
closed in a later section, sometimes inaccurately. This revision replaces
|
|
||||||
that log with one audited snapshot. Treat prior git history of this file as
|
|
||||||
session notes, not as ground truth.
|
|
||||||
|
|
||||||
### Verified fixed this pass
|
|
||||||
|
|
||||||
- **Rotation emitted an invalid `TaskReleased` (the previously reported
|
|
||||||
highest-priority defect) — now fixed.** `internal/orchestrator.Coordinator.rotate`
|
|
||||||
built the release payload as `{"handoff_ref","reason"}`, omitting the
|
|
||||||
`anchor_sha` the spec (§4, §6.2) and `domain.ValidatePayload` require
|
|
||||||
whenever `handoff_ref` is present. `store.Append` would reject it, the
|
|
||||||
error was discarded (`if c.Store.Append(e) == nil`), and the lease/session
|
|
||||||
silently never rotated — the coordinator would just retry next tick with
|
|
||||||
no visible failure. Fixed by adding `herdr.HeadSHA(worktree)` and having
|
|
||||||
`rotate` populate `anchor_sha` from the real worktree HEAD before
|
|
||||||
appending; if the anchor can't be read, rotation now correctly skips that
|
|
||||||
tick (leaving the lease intact for TTL/next-tick reclaim) instead of
|
|
||||||
emitting a payload guaranteed to fail validation.
|
|
||||||
Covered by `internal/orchestrator/rotation_test.go`
|
|
||||||
(`TestRotationEmitsValidReleaseWithAnchorSHA`), which drives the real
|
|
||||||
`Coordinator.Monitor` loop against an actual git worktree and asserts the
|
|
||||||
emitted event passes `domain.ValidatePayload` with the correct SHA — the
|
|
||||||
previous end-to-end test masked this bug by manually crafting a
|
|
||||||
replacement `TaskReleased` event after observing the (silently failed)
|
|
||||||
adapter-side release.
|
|
||||||
- **The federation worker release endpoint had the same gap.** The
|
|
||||||
`/v1/federation/workers/{id}/release` handler (cmd/orchestra/main.go)
|
|
||||||
built `TaskReleased` from a request body with only `handoff_ref`, no
|
|
||||||
`anchor_sha`. Since a remote worker is the only party with the actual
|
|
||||||
checkout (§2.1: "validate against the local checkout wherever the harness
|
|
||||||
runs"), the endpoint now requires and forwards a 40-hex-char `anchor_sha`
|
|
||||||
in the request body, rejecting the call with 400 otherwise.
|
|
||||||
|
|
||||||
### Multi-repo Gitea ingestion (new)
|
|
||||||
|
|
||||||
- `provider.Gitea` gained an optional `Project` field and `SourceName()`
|
|
||||||
(`"gitea"` if unset, `"gitea:<project>"` if set) — the namespaced source
|
|
||||||
doubles as the `(source,external_id)` dedup key, so issue #7 in two
|
|
||||||
different repos never collides, and as the reflection dispatch key.
|
|
||||||
- New `provider.MultiGitea{Sources map[string]Gitea}` implements
|
|
||||||
`TaskReflector` by looking up `task.Source` and forwarding to the matching
|
|
||||||
Gitea instance — lets several Gitea repos (one per project) share one
|
|
||||||
`ReflectingSink`.
|
|
||||||
- New `provider.GiteaSourceConfig` + `LoadGiteaConfigs(path)` load a JSON
|
|
||||||
array of `{project,base_url,owner,repo,token,webhook_secret}`.
|
|
||||||
`main.go` reads this from `ORCHESTRA_GITEA_CONFIG` if set; each source
|
|
||||||
gets its own poll supervisor (`gitea:<project>`) and webhook path
|
|
||||||
(`/v1/providers/gitea/webhook/<project>`).
|
|
||||||
- The legacy single-repo env vars (`ORCHESTRA_GITEA_URL/TOKEN/OWNER/REPO/
|
|
||||||
WEBHOOK_SECRET`) still work unchanged when `ORCHESTRA_GITEA_CONFIG` is
|
|
||||||
unset — same unprefixed webhook path, same `project = ORCHESTRA_GITEA_REPO`
|
|
||||||
tagging, same dedup source `"gitea"` — so existing deployments and
|
|
||||||
already-configured Gitea webhooks need no changes.
|
|
||||||
- Added `internal/provider/gitea_test.go` — previously **there were zero
|
|
||||||
tests exercising the Gitea provider at all** despite progress.md's prior
|
|
||||||
claim of Gitea webhook/poll test coverage; that claim was not accurate.
|
|
||||||
New tests cover source-name namespacing, webhook signature
|
|
||||||
verification/rejection, project tagging, `MultiGitea` dispatch-by-source
|
|
||||||
(via two `httptest.Server`s, asserting only the right one is hit), and
|
|
||||||
`LoadGiteaConfigs` validation/duplicate-project rejection.
|
|
||||||
|
|
||||||
### Per-project repos (new)
|
|
||||||
|
|
||||||
- `registry.Project` gained optional `repo`/`worktree_root` fields. Each
|
|
||||||
project can now resolve its own git checkout rather than every project
|
|
||||||
sharing one global `ORCHESTRA_REPO`/`ORCHESTRA_WORKTREE_ROOT` — matches
|
|
||||||
spec §2.2 ("projects are first-class and extensible... the binding is a
|
|
||||||
field + a config entry, not a schema change"). `main.go` builds a
|
|
||||||
`orchestrator.PerProjectGitWorktrees` from the registry, falling back to
|
|
||||||
the global default for any project that omits these fields, so
|
|
||||||
single-repo deployments are unaffected. Covered by
|
|
||||||
`internal/orchestrator/worktrees_test.go`.
|
|
||||||
|
|
||||||
### Closed this pass (were open gaps as of the last snapshot)
|
|
||||||
|
|
||||||
- **Bus-level authorization.** `authz.AuthorizeEvent` is now enforced inside
|
|
||||||
`store.Append` itself — the single choke point every event passes through
|
|
||||||
(HTTP handlers, router, coordinator/rotation, providers, federation relay)
|
|
||||||
— not just at HTTP handlers. Event schema bumped to v2, which requires
|
|
||||||
every event to declare a `Surface`; a new `authz.System` surface (full
|
|
||||||
control) covers internal emitters (router leases/failures, coordinator
|
|
||||||
releases/blocks, standup advisory/apply). Schema v1 events on disk still
|
|
||||||
replay (tolerant reader). Covered by `internal/store/store_test.go` and
|
|
||||||
`internal/router/router_test.go` additions asserting a non-HTTP append
|
|
||||||
with no/wrong surface is rejected.
|
|
||||||
- **Dual quota windows.** `router.QuotaAvailability` now tracks a 5-hour
|
|
||||||
rolling window and a 7-day weekly window independently per harness
|
|
||||||
(`QuotaWindowLimits{FiveHour, Weekly}`), applying the conservative 80%
|
|
||||||
rule to each separately — a harness over threshold on either window is
|
|
||||||
unavailable. Replaces the old single-`Window` field. Covered by new
|
|
||||||
`router_test.go` cases for weekly-only and 5h-only exhaustion.
|
|
||||||
- **Turn-boundary detection made observable, not silently optional.**
|
|
||||||
Rotation still can't force a harness adapter to implement `TurnBoundary`
|
|
||||||
Face B, but an adapter that fails to answer it now blocks that tick's
|
|
||||||
release (never treats a failed check as "safe to proceed"), and any
|
|
||||||
adapter without the capability — or one whose check errors — increments
|
|
||||||
`MonitorHealth.TurnBoundaryDegraded`, exposed via the coordinator's health
|
|
||||||
endpoint so degraded-safety operation is visible, not silent.
|
|
||||||
- **Cross-machine lease correctness has a real test.**
|
|
||||||
`internal/integration/federation_lease_test.go`
|
|
||||||
(`TestCrossMachineLeaseAnchorAndQuotaArePerHost`) exercises a lease
|
|
||||||
claimed through the federation worker HTTP API, validates the anchor
|
|
||||||
against that worker's own local checkout (not the router's), and asserts
|
|
||||||
quota is accounted per-host. Spec §9 item 8 said "prove on the first
|
|
||||||
federated run" — this is that proof for the primitives that exist today
|
|
||||||
(registration, heartbeat, lease-claim); it does not yet run against two
|
|
||||||
real physical machines.
|
|
||||||
- **Fuzz coverage for lifecycle payload validation.**
|
|
||||||
`internal/domain/fuzz_test.go` adds `FuzzValidatePayload` and
|
|
||||||
`FuzzValidateEvent` covering all event types (including malformed nested
|
|
||||||
`receipt`/`knowledge` shapes) — asserts no panic and always a typed error
|
|
||||||
on adversarial input.
|
|
||||||
|
|
||||||
### Believed accurate from prior sessions (spot-checked, not exhaustively re-verified)
|
|
||||||
|
|
||||||
- Event log: append-only JSONL, versioned envelope (schema v1), snapshot
|
|
||||||
load/replay, CAS with content-hash verification at append.
|
|
||||||
- `domain.ValidatePayload` enforces required fields per event type,
|
|
||||||
including `expected_version`/`ttl` on `TaskLeased`, `anchor_sha` on
|
|
||||||
`TaskReleased` (now correctly emitted, see above), `report_ref`+`receipt`
|
|
||||||
on `TaskCompleted`, and `blocker` on `TaskBlocked`.
|
|
||||||
- Router: project→affinity→machine resolution, capability match, quota
|
|
||||||
availability at conservative 80% threshold, derived-importance ordering,
|
|
||||||
retry-then-`TaskFailed`.
|
|
||||||
- herdr adapters (Claude/Codex/opencode) with native occupancy readers,
|
|
||||||
optional `TurnBoundary`/`RotationSignal`/`PaneExit` capability interfaces,
|
|
||||||
bootstrap/lease/release/kill.
|
|
||||||
- Continuity: strict handoff schema/validation, CAS save/load, pickup
|
|
||||||
validation (HEAD match, dirty-file hashes, immutable `TASK.md` hash),
|
|
||||||
scratch-branch commit/push/pull helpers.
|
|
||||||
- Provider layer: JSONL watcher, Gitea webhook+poll with HMAC auth,
|
|
||||||
idempotent `(source,external_id)` dedup, terminal-state reflection,
|
|
||||||
supervised restart with backoff.
|
|
||||||
- Federation: worker registration, heartbeat/TTL offline detection, event
|
|
||||||
cursor polling/ack, lease claim endpoint.
|
|
||||||
- Authorization: bus-level capability table (notify-only / full / gated) is
|
|
||||||
applied to lifecycle and approval writes via `AuthorizeEvent`.
|
|
||||||
- Delivery: Telegram/ntfy fan-out for completion/failure/block/approval
|
|
||||||
events.
|
|
||||||
- `/readyz`, `/v1/brief`, `/v1/providers/health`, `/v1/standup` exist and
|
|
||||||
return real state (not stubs).
|
|
||||||
|
|
||||||
## Known open gaps (named, not silently assumed done)
|
|
||||||
|
|
||||||
- **Cross-machine lease correctness is proven at the primitive level, not on
|
|
||||||
real hardware.** `TestCrossMachineLeaseAnchorAndQuotaArePerHost` exercises
|
|
||||||
the federation worker HTTP API (registration, lease-claim, anchor
|
|
||||||
validation against the worker's own checkout, per-host quota) inside one
|
|
||||||
test process. Spec §9 item 8 says "prove on the first federated run" —
|
|
||||||
that means an actual homesrv/workpc pair over the real mesh, which this
|
|
||||||
repo cannot exercise by itself. Named here as the one item that needs a
|
|
||||||
live two-machine run to fully close, not more code.
|
|
||||||
- **Turn-boundary Face B still degrades to occupancy-only for adapters that
|
|
||||||
don't implement it**, by design — the spec's Face B is per-harness native
|
|
||||||
session state (Stop hook / rollout tail / SSE), which this repo can only
|
|
||||||
wire against a real running herdr+harness pair. The degradation is now
|
|
||||||
observable (`MonitorHealth.TurnBoundaryDegraded`) and blocks-on-failure
|
|
||||||
rather than silently proceeding, but whether Claude/Codex/opencode's
|
|
||||||
native hooks are wired in a live deployment is a deployment-config fact,
|
|
||||||
not something provable from source alone.
|
|
||||||
|
|
||||||
Everything else named as open in the previous snapshot (bus-level
|
|
||||||
authorization, dual 5h/weekly quota windows, fuzz coverage of lifecycle
|
|
||||||
payload validation) is now closed — see "Closed this pass" above. Broader
|
|
||||||
areas (provider layer, continuity, router matching, delivery, federation
|
|
||||||
registration) were spot-checked against the code and their tests and
|
|
||||||
matched their described behavior.
|
|
||||||
Reference in New Issue
Block a user