package domain import ( "fmt" "strings" "time" ) // EventOperatorInterventionRecorded is a human saying what they repaired by // hand. Nothing infers it: a manual fix happens outside Orchestra by // definition, so the only honest way to have the evidence is for the operator // to state it. Without this the debt ledger reported manual recovery as a // permanent gap, and every repair that kept the system running was invisible // to the record of how much the system costs to run. const EventOperatorInterventionRecorded = "OperatorInterventionRecorded" // InterventionKind is what the operator did. The list is closed so the ledger // can group repairs; an unrecognised kind is refused rather than guessed at. type InterventionKind string const ( InterventionWorkerRestart InterventionKind = "worker_restart" InterventionTransactionClean InterventionKind = "transaction_cleanup" InterventionForcedRelease InterventionKind = "forced_release" InterventionStateRepair InterventionKind = "state_repair" InterventionManualRequeue InterventionKind = "manual_requeue" InterventionPhaseRecovery InterventionKind = "manual_phase_recovery" ) func (k InterventionKind) Valid() bool { switch k { case InterventionWorkerRestart, InterventionTransactionClean, InterventionForcedRelease, InterventionStateRepair, InterventionManualRequeue, InterventionPhaseRecovery: return true } return false } // OperatorIntervention is one recorded manual repair. type OperatorIntervention struct { TaskID string `json:"task_id,omitempty"` WorkerID string `json:"worker_id,omitempty"` Kind InterventionKind `json:"kind"` // Reason is the operator's own account of why it was needed. It is the // part a later reader cannot reconstruct from anything else. Reason string `json:"reason"` // RelatedEventID and RelatedTransactionID point at what was repaired, so a // reader can find the failure this answered rather than infer it. RelatedEventID string `json:"related_event_id,omitempty"` RelatedTransactionID string `json:"related_transaction_id,omitempty"` Components []string `json:"components,omitempty"` At time.Time `json:"at,omitempty"` } const maxInterventionReason = 1000 func (i OperatorIntervention) Validate() error { if !i.Kind.Valid() { return fmt.Errorf("%w: %q is not an intervention kind", ErrInvalid, i.Kind) } if strings.TrimSpace(i.Reason) == "" { return fmt.Errorf("%w: an intervention states why it was needed", ErrInvalid) } if len(i.Reason) > maxInterventionReason { return fmt.Errorf("%w: reason exceeds %d characters", ErrInvalid, maxInterventionReason) } if i.TaskID == "" && i.WorkerID == "" { return fmt.Errorf("%w: an intervention names the task or the worker it repaired", ErrInvalid) } return nil } func ValidateOperatorInterventionRecorded(p map[string]any) error { kind, _ := p["kind"].(string) reason, _ := p["reason"].(string) task, _ := p["task_id"].(string) worker, _ := p["worker_id"].(string) return OperatorIntervention{ Kind: InterventionKind(kind), Reason: reason, TaskID: task, WorkerID: worker, }.Validate() }