Record what an operator repaired, and let debt count incidents
Slice B, second half. OperatorInterventionRecorded is the one command for saying "I fixed this by hand": a manual repair happens outside Orchestra by definition, so the only honest way to have the evidence is for the person who made it to state it. Inferring "an operator probably intervened" from a gap would put guesses into the record the ledger is built from. The debt projection now consumes both new kinds. A closed incident is one observation carrying its repeat count as intensity, so recurrence stays a count of independent incidents: 301 repeats on one lease and 2 on another is a recurrence of two with an intensity of 303, not a recurrence of 303. Both kinds were previously reported as holes in the system. They are ordinary evidence now, so their absence from a history is a fact about that history, and the gap list says so. The worker also stamps a per-process incarnation on registration and every heartbeat. Nothing else on the wire distinguishes a restarted worker from a running one, and an incident cannot outlive the process that reported it. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CVbaKucEYBjMqVeUgJUsc1
This commit is contained in:
@@ -113,6 +113,12 @@ type DebtObservation struct {
|
||||
Detail string `json:"detail,omitempty"`
|
||||
Paths []string `json:"paths,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
// Repeats is how many times this one incident recurred. It is intensity,
|
||||
// never recurrence: one worker stuck in a five-second retry loop produced
|
||||
// 301 repeats of a single failure, and counting those as 301 pieces of
|
||||
// evidence would make one broken worker look like chronic, system-wide
|
||||
// debt. Recurrence is the number of independent observations.
|
||||
Repeats int `json:"repeats,omitempty"`
|
||||
}
|
||||
|
||||
func (o DebtObservation) Validate() error {
|
||||
|
||||
@@ -305,6 +305,8 @@ func EventWithoutTask(typ string) bool {
|
||||
switch typ {
|
||||
case "QuotaReported", "StandupAdvisory", "ApprovalGranted", "ApprovalDenied",
|
||||
EventObservationIncidentOpened, EventObservationIncidentClosed:
|
||||
// An intervention is deliberately absent: it names a task when it
|
||||
// repaired one, and that task must exist.
|
||||
return true
|
||||
}
|
||||
return false
|
||||
@@ -317,7 +319,7 @@ func ValidateEvent(e Event) error {
|
||||
if e.SchemaVersion >= 2 && strings.TrimSpace(e.Surface) == "" {
|
||||
return fmt.Errorf("%w: surface required", ErrInvalid)
|
||||
}
|
||||
allowed := map[string]bool{"TaskCreated": true, "TaskLeased": true, "TaskLeaseRenewed": true, "TaskReleased": true, "TaskLaunchAcknowledged": true, "TaskPickupValidated": true, "TaskCompleted": true, "TaskFailed": true, "TaskBlocked": true, "TaskNeedsAttention": true, "ApprovalRequested": true, "ApprovalGranted": true, "ApprovalDenied": true, "TaskAmended": true, "TaskCorrected": true, "QuotaReported": true, "StandupAdvisory": true, EventHumanDecisionRecorded: true, EventHumanDecisionSuperseded: true, EventWorkPhaseChanged: true, EventDeferredFindingRecorded: true, EventReviewRecorded: true, EventTaskSubmitted: true, EventTaskChangesRequested: true, EventPlanPhaseVerified: true, EventPlanMismatchRecorded: true, EventObservationIncidentOpened: true, EventObservationIncidentClosed: true}
|
||||
allowed := map[string]bool{"TaskCreated": true, "TaskLeased": true, "TaskLeaseRenewed": true, "TaskReleased": true, "TaskLaunchAcknowledged": true, "TaskPickupValidated": true, "TaskCompleted": true, "TaskFailed": true, "TaskBlocked": true, "TaskNeedsAttention": true, "ApprovalRequested": true, "ApprovalGranted": true, "ApprovalDenied": true, "TaskAmended": true, "TaskCorrected": true, "QuotaReported": true, "StandupAdvisory": true, EventHumanDecisionRecorded: true, EventHumanDecisionSuperseded: true, EventWorkPhaseChanged: true, EventDeferredFindingRecorded: true, EventReviewRecorded: true, EventTaskSubmitted: true, EventTaskChangesRequested: true, EventPlanPhaseVerified: true, EventPlanMismatchRecorded: true, EventObservationIncidentOpened: true, EventObservationIncidentClosed: true, EventOperatorInterventionRecorded: true}
|
||||
if !allowed[e.Type] {
|
||||
return fmt.Errorf("%w: unknown type %q", ErrInvalid, e.Type)
|
||||
}
|
||||
@@ -603,6 +605,8 @@ func ValidatePayload(typ string, p map[string]any) error {
|
||||
return ValidateObservationIncidentOpened(p)
|
||||
case EventObservationIncidentClosed:
|
||||
return ValidateObservationIncidentClosed(p)
|
||||
case EventOperatorInterventionRecorded:
|
||||
return ValidateOperatorInterventionRecorded(p)
|
||||
case EventReviewRecorded:
|
||||
if err := requiredHash(p, "artifact_ref"); err != nil {
|
||||
return err
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// EventOperatorInterventionRecorded is a human saying what they repaired by
|
||||
// hand. Nothing infers it: a manual fix happens outside Orchestra by
|
||||
// definition, so the only honest way to have the evidence is for the operator
|
||||
// to state it. Without this the debt ledger reported manual recovery as a
|
||||
// permanent gap, and every repair that kept the system running was invisible
|
||||
// to the record of how much the system costs to run.
|
||||
const EventOperatorInterventionRecorded = "OperatorInterventionRecorded"
|
||||
|
||||
// InterventionKind is what the operator did. The list is closed so the ledger
|
||||
// can group repairs; an unrecognised kind is refused rather than guessed at.
|
||||
type InterventionKind string
|
||||
|
||||
const (
|
||||
InterventionWorkerRestart InterventionKind = "worker_restart"
|
||||
InterventionTransactionClean InterventionKind = "transaction_cleanup"
|
||||
InterventionForcedRelease InterventionKind = "forced_release"
|
||||
InterventionStateRepair InterventionKind = "state_repair"
|
||||
InterventionManualRequeue InterventionKind = "manual_requeue"
|
||||
InterventionPhaseRecovery InterventionKind = "manual_phase_recovery"
|
||||
)
|
||||
|
||||
func (k InterventionKind) Valid() bool {
|
||||
switch k {
|
||||
case InterventionWorkerRestart, InterventionTransactionClean, InterventionForcedRelease,
|
||||
InterventionStateRepair, InterventionManualRequeue, InterventionPhaseRecovery:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// OperatorIntervention is one recorded manual repair.
|
||||
type OperatorIntervention struct {
|
||||
TaskID string `json:"task_id,omitempty"`
|
||||
WorkerID string `json:"worker_id,omitempty"`
|
||||
Kind InterventionKind `json:"kind"`
|
||||
// Reason is the operator's own account of why it was needed. It is the
|
||||
// part a later reader cannot reconstruct from anything else.
|
||||
Reason string `json:"reason"`
|
||||
// RelatedEventID and RelatedTransactionID point at what was repaired, so a
|
||||
// reader can find the failure this answered rather than infer it.
|
||||
RelatedEventID string `json:"related_event_id,omitempty"`
|
||||
RelatedTransactionID string `json:"related_transaction_id,omitempty"`
|
||||
Components []string `json:"components,omitempty"`
|
||||
At time.Time `json:"at,omitempty"`
|
||||
}
|
||||
|
||||
const maxInterventionReason = 1000
|
||||
|
||||
func (i OperatorIntervention) Validate() error {
|
||||
if !i.Kind.Valid() {
|
||||
return fmt.Errorf("%w: %q is not an intervention kind", ErrInvalid, i.Kind)
|
||||
}
|
||||
if strings.TrimSpace(i.Reason) == "" {
|
||||
return fmt.Errorf("%w: an intervention states why it was needed", ErrInvalid)
|
||||
}
|
||||
if len(i.Reason) > maxInterventionReason {
|
||||
return fmt.Errorf("%w: reason exceeds %d characters", ErrInvalid, maxInterventionReason)
|
||||
}
|
||||
if i.TaskID == "" && i.WorkerID == "" {
|
||||
return fmt.Errorf("%w: an intervention names the task or the worker it repaired", ErrInvalid)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ValidateOperatorInterventionRecorded(p map[string]any) error {
|
||||
kind, _ := p["kind"].(string)
|
||||
reason, _ := p["reason"].(string)
|
||||
task, _ := p["task_id"].(string)
|
||||
worker, _ := p["worker_id"].(string)
|
||||
return OperatorIntervention{
|
||||
Kind: InterventionKind(kind), Reason: reason, TaskID: task, WorkerID: worker,
|
||||
}.Validate()
|
||||
}
|
||||
Reference in New Issue
Block a user