Record what an operator repaired, and let debt count incidents

Slice B, second half. OperatorInterventionRecorded is the one command for
saying "I fixed this by hand": a manual repair happens outside Orchestra by
definition, so the only honest way to have the evidence is for the person
who made it to state it. Inferring "an operator probably intervened" from a
gap would put guesses into the record the ledger is built from.

The debt projection now consumes both new kinds. A closed incident is one
observation carrying its repeat count as intensity, so recurrence stays a
count of independent incidents: 301 repeats on one lease and 2 on another
is a recurrence of two with an intensity of 303, not a recurrence of 303.

Both kinds were previously reported as holes in the system. They are
ordinary evidence now, so their absence from a history is a fact about that
history, and the gap list says so.

The worker also stamps a per-process incarnation on registration and every
heartbeat. Nothing else on the wire distinguishes a restarted worker from a
running one, and an incident cannot outlive the process that reported it.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CVbaKucEYBjMqVeUgJUsc1

cmd/orchestra-worker/main.go

@@ -103,8 +103,13 @@ func (w *worker) recordError(err error) {
 	w.observations = append(w.observations, federation.Observation{Message: msg, Count: 1, First: now, Last: now})
 }
 
+// workerIncarnation identifies this process. A restarted worker cannot
+// continue the previous process's failures, and nothing else on the wire says
+// a restart happened: build revision and worker id both survive it.
+var workerIncarnation = domain.NewID()
+
 func (w *worker) health(ctx context.Context) federation.WorkerHealth {
-	h := federation.WorkerHealth{HerdrStatus: "unknown"}
+	h := federation.WorkerHealth{HerdrStatus: "unknown", Incarnation: workerIncarnation}
 	if backend := w.executionBackend(); backend != nil {
 		h.Backend = backend.Kind()
 	}
@@ -1884,7 +1889,7 @@ func main() {
 			window:      window,
 			// Capacity stays one per identity because a herdr's declared
 			// concurrency is one. Serving N harnesses gives the process N slots.
-			registration: federation.Worker{ID: spec.ID, Address: spec.Address, Capacity: 1, SupportedProjects: supported, Build: buildinfo.Current()},
+			registration: federation.Worker{ID: spec.ID, Address: spec.Address, Capacity: 1, SupportedProjects: supported, Build: buildinfo.Current(), Incarnation: workerIncarnation},
 		}
 		if w.registration.Address == "" {
 			w.registration.Address = os.Getenv("ORCHESTRA_WORKER_ADDRESS")

cmd/orchestra/main.go

@@ -228,6 +228,9 @@ func main() {
 	// Pull-request readers by source name, for reflecting submitted work.
 	pullRequests := map[string]human.PullRequestSource{}
 	localMachine := os.Getenv("ORCHESTRA_MACHINE_ID")
+	// One tracker for the process: it holds the per-incident accumulation that
+	// makes an evicted and recreated ring entry add up instead of restarting.
+	observations := &operations.ObservationTracker{Store: s}
 	workers := &federation.Registry{AdmitToken: os.Getenv("ORCHESTRA_FEDERATION_ADMIT_TOKEN"), StatePath: filepath.Join(dir, "federation-state.json")}
 	if err := workers.Load(); err != nil {
 		log.Fatalf("load federation state: %v", err)
@@ -445,6 +448,49 @@ func main() {
 		}
 		json.NewEncoder(w).Encode(s.Events(n))
 	})
+	mux.HandleFunc("/v1/interventions", func(w http.ResponseWriter, r *http.Request) {
+		// The one command an operator has for saying "I fixed this by hand".
+		// Everything Orchestra does to itself is already an event; a manual
+		// repair is the only kind of recovery that leaves no trace unless the
+		// person who made it says so.
+		if r.Method == http.MethodGet {
+			out := make([]domain.Event, 0)
+			for _, e := range s.Events(0) {
+				if e.Type == domain.EventOperatorInterventionRecorded {
+					out = append(out, e)
+				}
+			}
+			json.NewEncoder(w).Encode(out)
+			return
+		}
+		if r.Method != http.MethodPost {
+			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+			return
+		}
+		if err := authz.AuthorizeEvent(surface(r), domain.EventOperatorInterventionRecorded); err != nil {
+			http.Error(w, err.Error(), http.StatusForbidden)
+			return
+		}
+		var in domain.OperatorIntervention
+		if json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<16)).Decode(&in) != nil {
+			http.Error(w, "invalid json", http.StatusBadRequest)
+			return
+		}
+		e, err := operations.RecordIntervention(s, surface(r), in)
+		switch {
+		case errors.Is(err, domain.ErrNotFound):
+			http.Error(w, err.Error(), http.StatusNotFound)
+			return
+		case errors.Is(err, domain.ErrInvalid):
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		case err != nil:
+			http.Error(w, err.Error(), http.StatusConflict)
+			return
+		}
+		w.WriteHeader(http.StatusCreated)
+		json.NewEncoder(w).Encode(e)
+	})
 	mux.HandleFunc("/v1/debt", func(w http.ResponseWriter, r *http.Request) {
 		// Read-only, and deliberately so. This projection is evidence about
 		// history, not a new kind of truth: nothing here writes an event,
@@ -1391,6 +1437,29 @@ func main() {
 				http.Error(w, err.Error(), 404)
 				return
 			}
+			// The ring carries messages and nothing else, so the coordinator
+			// attributes them: the worker's active task, and that task's
+			// current lease epoch, are what bind an incident to the work it
+			// happened during. A failed fold is logged rather than failing the
+			// heartbeat, because losing evidence about a worker is not a reason
+			// to also stop believing the worker is alive.
+			report := operations.WorkerReport{
+				WorkerID: parts[3], Incarnation: health.Incarnation,
+				TaskID: health.ActiveTask, At: time.Now().UTC(),
+			}
+			if report.TaskID != "" {
+				if t, ok := s.Task(report.TaskID); ok && t.Lease != nil {
+					report.LeaseEpoch = t.Lease.Epoch
+				}
+			}
+			for _, o := range health.Observations {
+				report.Observations = append(report.Observations, domain.WorkerObservation{
+					Message: o.Message, Count: o.Count, First: o.First, Last: o.Last,
+				})
+			}
+			if _, err := observations.Ingest(report); err != nil {
+				log.Printf("observation incidents for %s: %v", parts[3], err)
+			}
 			w.WriteHeader(http.StatusNoContent)
 			return
 		}