kami/orchestra
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Record what an operator repaired, and let debt count incidents

Browse Source 
Slice B, second half. OperatorInterventionRecorded is the one command for
saying "I fixed this by hand": a manual repair happens outside Orchestra by
definition, so the only honest way to have the evidence is for the person
who made it to state it. Inferring "an operator probably intervened" from a
gap would put guesses into the record the ledger is built from.

The debt projection now consumes both new kinds. A closed incident is one
observation carrying its repeat count as intensity, so recurrence stays a
count of independent incidents: 301 repeats on one lease and 2 on another
is a recurrence of two with an intensity of 303, not a recurrence of 303.

Both kinds were previously reported as holes in the system. They are
ordinary evidence now, so their absence from a history is a fact about that
history, and the gap list says so.

The worker also stamps a per-process incarnation on registration and every
heartbeat. Nothing else on the wire distinguishes a restarted worker from a
running one, and an incident cannot outlive the process that reported it.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CVbaKucEYBjMqVeUgJUsc1
This commit is contained in:
kami
2026-08-30 06:53:26 +04:00
parent 438c1d6df3
commit e0601296e0
9 changed files with 337 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/store/debt_projection.go
+36 -7
View File
@@ -102,6 +102,36 @@ func projectDebt(events []domain.Event, readArtifact func(string) ([]byte, error
o.Detail = str(p["last_error"])
add(class, domain.DebtSignature(class, failure, harness, "lease"), o,
"tasks end in "+failure, review.Important)
case domain.EventObservationIncidentClosed:
// One incident, whatever it repeated. The debt class comes from
// the signature's shape rather than a failure class, because a
// worker observation is a symptom the worker described, not a
// lifecycle outcome Orchestra decided.
var inc domain.ObservationIncident
if json.Unmarshal(e.Payload, &inc) != nil || inc.Signature == "" {
continue
}
o := base
o.Kind = domain.ObservationWorkerFailure
o.TaskID = inc.TaskID
o.Detail = inc.Detail
o.Repeats = inc.RepeatCount
add(domain.DebtOperational,
domain.DebtSignature(domain.DebtOperational, inc.Signature, inc.WorkerID, "worker"),
o, "workers report "+inc.Signature, review.Important)
case domain.EventOperatorInterventionRecorded:
var in domain.OperatorIntervention
if json.Unmarshal(e.Payload, &in) != nil || !in.Kind.Valid() {
continue
}
o := base
o.Kind = domain.ObservationManualIntervention
o.TaskID = in.TaskID
o.Detail = in.Reason
o.Paths = in.Components
add(domain.DebtOperational,
domain.DebtSignature(domain.DebtOperational, string(in.Kind), in.WorkerID, "manual"),
o, "an operator repairs this by hand ("+string(in.Kind)+")", review.Important)
case domain.EventPlanMismatchRecorded:
o := base
o.Kind = domain.ObservationPlanMismatch
@@ -167,16 +197,15 @@ func projectDebt(events []domain.Event, readArtifact func(string) ([]byte, error
// hole in the system. A kind the log could carry and does not is a fact about
// this history.
func debtGaps(seen map[domain.ObservationKind]bool) []domain.EvidenceGap {
gaps := []domain.EvidenceGap{
{Kind: domain.ObservationManualIntervention, Durable: false,
Reason: "no event type records an operator repair, so every manual recovery is invisible to this ledger"},
{Kind: domain.ObservationWorkerFailure, Durable: false,
Reason: "worker observations live in worker memory and reach the coordinator only inside WorkerHealth, which is not persisted"},
}
// Both of these were once permanent holes in the system. They are ordinary
// evidence now, so their absence is a fact about this history rather than
// about Orchestra.
var gaps []domain.EvidenceGap
for _, k := range []domain.ObservationKind{
domain.ObservationBlockReason, domain.ObservationFailureClass,
domain.ObservationReviewFinding, domain.ObservationPlanMismatch,
domain.ObservationDeferredFinding,
domain.ObservationDeferredFinding, domain.ObservationWorkerFailure,
domain.ObservationManualIntervention,
} {
if !seen[k] {
gaps = append(gaps, domain.EvidenceGap{Kind: k, Durable: true,
internal/store/debt_projection_test.go
+83 -15
View File
@@ -64,24 +64,92 @@ func TestProjectDebtIgnoresOrdinaryLifecycleStops(t *testing.T) {
// not recorded anywhere".
func TestProjectDebtReportsWhatItCannotSee(t *testing.T) {
ledger := ProjectDebt(nil)
var manual, worker bool
for _, g := range ledger.Gaps {
if g.Durable {
continue
}
switch g.Kind {
case domain.ObservationManualIntervention:
manual = true
case domain.ObservationWorkerFailure:
worker = true
}
}
if !manual || !worker {
t.Fatalf("the two known holes must always be reported: %+v", ledger.Gaps)
}
for _, g := range ledger.Gaps {
if g.Reason == "" {
t.Fatalf("gap %q has no reason", g.Kind)
}
// Slice B closed the two holes this ledger used to report about
// itself. Every silence is now a fact about one history, never a kind
// of evidence the system cannot record at all.
if !g.Durable {
t.Fatalf("gap %q is reported as unrecordable: %+v", g.Kind, g)
}
}
var worker, manual bool
for _, g := range ledger.Gaps {
switch g.Kind {
case domain.ObservationWorkerFailure:
worker = true
case domain.ObservationManualIntervention:
manual = true
}
}
if !worker || !manual {
t.Fatalf("an empty history should still name both kinds as absent: %+v", ledger.Gaps)
}
}
// The whole point of incidents. One worker stuck in a retry loop must not
// manufacture recurrence, while its intensity is still on the record.
func TestRecurrenceCountsIncidentsAndKeepsIntensitySeparate(t *testing.T) {
closed := func(id, worker, task, epoch string, repeats int) domain.Event {
b, _ := json.Marshal(domain.ObservationIncident{
ID: id, WorkerID: worker, TaskID: task, LeaseEpoch: epoch,
Signature: "lease <id> not renewed: agent status idle and pane unchanged",
Detail: "lease " + task + " not renewed: agent status idle and pane unchanged",
RepeatCount: repeats, CloseReason: domain.ObservationCloseEpochChange,
})
return domain.Event{ID: id, Type: domain.EventObservationIncidentClosed, TaskID: "system", Payload: b}
}
ledger := ProjectDebt([]domain.Event{
closed("i1", "workpc-claude", "task-a", "e1", 301),
closed("i2", "workpc-claude", "task-b", "e2", 2),
})
if len(ledger.Items) != 1 {
t.Fatalf("one kind of failure produced %d items", len(ledger.Items))
}
item := ledger.Items[0]
if len(item.Observations) != 2 {
t.Fatalf("recurrence = %d, want one per incident", len(item.Observations))
}
intensity := 0
for _, o := range item.Observations {
if o.Kind != domain.ObservationWorkerFailure {
t.Fatalf("observation kind = %q", o.Kind)
}
intensity += o.Repeats
}
if intensity != 303 {
t.Fatalf("intensity = %d, want 303 carried alongside a recurrence of 2", intensity)
}
tasks := map[string]bool{}
for _, o := range item.Observations {
tasks[o.TaskID] = true
}
if len(tasks) != 2 {
t.Fatalf("the two incidents are not attributed to their tasks: %+v", item.Observations)
}
}
// A repair the operator made by hand is evidence like any other, once they say
// it happened.
func TestAnOperatorRepairBecomesDebtEvidence(t *testing.T) {
b, _ := json.Marshal(domain.OperatorIntervention{
WorkerID: "workpc-opencode", Kind: domain.InterventionTransactionClean,
Reason: "deleted a release transaction stuck at prepared so the pane could be reused",
})
ledger := ProjectDebt([]domain.Event{{
ID: "i1", Type: domain.EventOperatorInterventionRecorded, TaskID: "system", Payload: b,
}})
if len(ledger.Items) != 1 || len(ledger.Items[0].Observations) != 1 {
t.Fatalf("the repair produced no debt evidence: %+v", ledger.Items)
}
if got := ledger.Items[0].Observations[0].Kind; got != domain.ObservationManualIntervention {
t.Fatalf("kind = %q", got)
}
for _, g := range ledger.Gaps {
if g.Kind == domain.ObservationManualIntervention {
t.Fatal("manual intervention is still reported as missing from a history that contains one")
}
}
}