e0601296e0
Slice B, second half. OperatorInterventionRecorded is the one command for saying "I fixed this by hand": a manual repair happens outside Orchestra by definition, so the only honest way to have the evidence is for the person who made it to state it. Inferring "an operator probably intervened" from a gap would put guesses into the record the ledger is built from. The debt projection now consumes both new kinds. A closed incident is one observation carrying its repeat count as intensity, so recurrence stays a count of independent incidents: 301 repeats on one lease and 2 on another is a recurrence of two with an intensity of 303, not a recurrence of 303. Both kinds were previously reported as holes in the system. They are ordinary evidence now, so their absence from a history is a fact about that history, and the gap list says so. The worker also stamps a per-process incarnation on registration and every heartbeat. Nothing else on the wire distinguishes a restarted worker from a running one, and an incident cannot outlive the process that reported it. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CVbaKucEYBjMqVeUgJUsc1
156 lines
6.1 KiB
Go
156 lines
6.1 KiB
Go
package store
|
|
|
|
import (
|
|
"encoding/json"
|
|
"testing"
|
|
"time"
|
|
|
|
"orchestra/internal/domain"
|
|
)
|
|
|
|
func debtEvent(id, typ, task string, payload map[string]any) domain.Event {
|
|
b, _ := json.Marshal(payload)
|
|
return domain.Event{ID: id, Type: typ, TaskID: task, Payload: b, At: time.Now().UTC()}
|
|
}
|
|
|
|
// The opencode shape from run 14: one harness failing the same way across
|
|
// several tasks. That is the recurrence the ledger exists to notice.
|
|
func TestProjectDebtGroupsOneFailureShapeAcrossTasks(t *testing.T) {
|
|
events := []domain.Event{
|
|
debtEvent("e1", "TaskFailed", "t1", map[string]any{"failure_class": "retry_limit", "harness_id": "workpc-opencode"}),
|
|
debtEvent("e2", "TaskFailed", "t2", map[string]any{"failure_class": "retry_limit", "harness_id": "workpc-opencode"}),
|
|
debtEvent("e3", "TaskFailed", "t3", map[string]any{"failure_class": "retry_limit", "harness_id": "workpc-opencode"}),
|
|
debtEvent("e4", "TaskFailed", "t4", map[string]any{"failure_class": "retry_limit", "harness_id": "workpc-claude"}),
|
|
// A handoff release is ordinary progress and must not become debt.
|
|
debtEvent("e5", "TaskReleased", "t5", map[string]any{"handoff_ref": "sha256:abc", "harness_id": "workpc-claude"}),
|
|
}
|
|
ledger := ProjectDebt(events)
|
|
if len(ledger.Items) != 2 {
|
|
t.Fatalf("want one item per harness, got %d: %+v", len(ledger.Items), ledger.Items)
|
|
}
|
|
top := ledger.Items[0]
|
|
if top.Recurrence() != 3 || top.AffectedTasks() != 3 {
|
|
t.Fatalf("recurrence %d across %d tasks, want 3 and 3", top.Recurrence(), top.AffectedTasks())
|
|
}
|
|
if top.Class != domain.DebtOperational {
|
|
t.Fatalf("class %q, want operational", top.Class)
|
|
}
|
|
if got := top.ID; got != "v1:operational:retry_limit:workpc-opencode:lease" {
|
|
t.Fatalf("signature %q", got)
|
|
}
|
|
for _, o := range top.Observations {
|
|
if o.EventID == "" || o.LegacyRef != "" {
|
|
t.Fatalf("observation lost its event provenance: %+v", o)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A lifecycle stop is the system working. Waiting for a human is not debt, and
|
|
// counting it would drown the real signal.
|
|
func TestProjectDebtIgnoresOrdinaryLifecycleStops(t *testing.T) {
|
|
events := []domain.Event{
|
|
debtEvent("e1", "TaskBlocked", "t1", map[string]any{"block_reason": "human_decision", "blocker": "which route"}),
|
|
debtEvent("e2", "TaskBlocked", "t2", map[string]any{"block_reason": "trajectory_gate", "blocker": "confirm plan"}),
|
|
debtEvent("e3", "TaskBlocked", "t3", map[string]any{"block_reason": "lease_expired", "blocker": "lease expired"}),
|
|
}
|
|
ledger := ProjectDebt(events)
|
|
if len(ledger.Items) != 1 || ledger.Items[0].Class != domain.DebtOperational {
|
|
t.Fatalf("want only the lease_expired item, got %+v", ledger.Items)
|
|
}
|
|
}
|
|
|
|
// Incompleteness is part of the result. A ledger that stays silent about what
|
|
// it cannot record reads as "no operator cost" when it means "operator cost is
|
|
// not recorded anywhere".
|
|
func TestProjectDebtReportsWhatItCannotSee(t *testing.T) {
|
|
ledger := ProjectDebt(nil)
|
|
for _, g := range ledger.Gaps {
|
|
if g.Reason == "" {
|
|
t.Fatalf("gap %q has no reason", g.Kind)
|
|
}
|
|
// Slice B closed the two holes this ledger used to report about
|
|
// itself. Every silence is now a fact about one history, never a kind
|
|
// of evidence the system cannot record at all.
|
|
if !g.Durable {
|
|
t.Fatalf("gap %q is reported as unrecordable: %+v", g.Kind, g)
|
|
}
|
|
}
|
|
var worker, manual bool
|
|
for _, g := range ledger.Gaps {
|
|
switch g.Kind {
|
|
case domain.ObservationWorkerFailure:
|
|
worker = true
|
|
case domain.ObservationManualIntervention:
|
|
manual = true
|
|
}
|
|
}
|
|
if !worker || !manual {
|
|
t.Fatalf("an empty history should still name both kinds as absent: %+v", ledger.Gaps)
|
|
}
|
|
}
|
|
|
|
// The whole point of incidents. One worker stuck in a retry loop must not
|
|
// manufacture recurrence, while its intensity is still on the record.
|
|
func TestRecurrenceCountsIncidentsAndKeepsIntensitySeparate(t *testing.T) {
|
|
closed := func(id, worker, task, epoch string, repeats int) domain.Event {
|
|
b, _ := json.Marshal(domain.ObservationIncident{
|
|
ID: id, WorkerID: worker, TaskID: task, LeaseEpoch: epoch,
|
|
Signature: "lease <id> not renewed: agent status idle and pane unchanged",
|
|
Detail: "lease " + task + " not renewed: agent status idle and pane unchanged",
|
|
RepeatCount: repeats, CloseReason: domain.ObservationCloseEpochChange,
|
|
})
|
|
return domain.Event{ID: id, Type: domain.EventObservationIncidentClosed, TaskID: "system", Payload: b}
|
|
}
|
|
ledger := ProjectDebt([]domain.Event{
|
|
closed("i1", "workpc-claude", "task-a", "e1", 301),
|
|
closed("i2", "workpc-claude", "task-b", "e2", 2),
|
|
})
|
|
if len(ledger.Items) != 1 {
|
|
t.Fatalf("one kind of failure produced %d items", len(ledger.Items))
|
|
}
|
|
item := ledger.Items[0]
|
|
if len(item.Observations) != 2 {
|
|
t.Fatalf("recurrence = %d, want one per incident", len(item.Observations))
|
|
}
|
|
intensity := 0
|
|
for _, o := range item.Observations {
|
|
if o.Kind != domain.ObservationWorkerFailure {
|
|
t.Fatalf("observation kind = %q", o.Kind)
|
|
}
|
|
intensity += o.Repeats
|
|
}
|
|
if intensity != 303 {
|
|
t.Fatalf("intensity = %d, want 303 carried alongside a recurrence of 2", intensity)
|
|
}
|
|
tasks := map[string]bool{}
|
|
for _, o := range item.Observations {
|
|
tasks[o.TaskID] = true
|
|
}
|
|
if len(tasks) != 2 {
|
|
t.Fatalf("the two incidents are not attributed to their tasks: %+v", item.Observations)
|
|
}
|
|
}
|
|
|
|
// A repair the operator made by hand is evidence like any other, once they say
|
|
// it happened.
|
|
func TestAnOperatorRepairBecomesDebtEvidence(t *testing.T) {
|
|
b, _ := json.Marshal(domain.OperatorIntervention{
|
|
WorkerID: "workpc-opencode", Kind: domain.InterventionTransactionClean,
|
|
Reason: "deleted a release transaction stuck at prepared so the pane could be reused",
|
|
})
|
|
ledger := ProjectDebt([]domain.Event{{
|
|
ID: "i1", Type: domain.EventOperatorInterventionRecorded, TaskID: "system", Payload: b,
|
|
}})
|
|
if len(ledger.Items) != 1 || len(ledger.Items[0].Observations) != 1 {
|
|
t.Fatalf("the repair produced no debt evidence: %+v", ledger.Items)
|
|
}
|
|
if got := ledger.Items[0].Observations[0].Kind; got != domain.ObservationManualIntervention {
|
|
t.Fatalf("kind = %q", got)
|
|
}
|
|
for _, g := range ledger.Gaps {
|
|
if g.Kind == domain.ObservationManualIntervention {
|
|
t.Fatal("manual intervention is still reported as missing from a history that contains one")
|
|
}
|
|
}
|
|
}
|