Make worker observations durable as incidents, not as symptoms

Slice B, first half. The F18 ring is bounded, lossy and local, so the debt
ledger reported it as a gap about itself. Two events make it durable:
ObservationIncidentOpened at first sight, appended immediately so a
coordinator that dies mid-incident still leaves the fact that it existed,
and ObservationIncidentClosed carrying the aggregate.

The rules are what matter. Repeats update the aggregate and append nothing,
because run 11's 409 loop was one incident with an intensity of 301 rather
than 301 pieces of evidence. Absence from the ring closes nothing, since a
bounded history evicts as easily as it recovers. An incident is scoped to
its lease and closes on lease end, epoch change, worker restart, or, for
observations with no lease to bound them, on last_seen going stale. A ring
entry that is evicted and recreated accumulates: 34 then 3 is 37.

Also collapses three drifted copies of the list of events that carry no
task into one predicate. Adding a type to two of them left it rejected by
the third, which is how the first version of this failed.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CVbaKucEYBjMqVeUgJUsc1

internal/domain/domain.go

@@ -241,7 +241,7 @@ type Task struct {
 	// is answered, because after either it is history rather than a live
 	// instruction.
 	PlanMismatch *PlanMismatch `json:"plan_mismatch,omitempty"`
-	LastError   string   `json:"last_error,omitempty"`
+	LastError    string        `json:"last_error,omitempty"`
 }
 
 // ReviewRef binds a sealed review artifact to one commit.
@@ -293,6 +293,23 @@ func NewID() string {
 	_, _ = rand.Read(b[6:])
 	return ulidEncoding.EncodeToString(b)
 }
+
+// EventWithoutTask reports whether an event records something about the
+// system rather than about one task's lifecycle. These carry the "system"
+// aggregate id, so no task projection has to exist for them.
+//
+// One list, because there were three: the replay guard, the append guard and
+// the transition check each kept their own copy, and adding an event type to
+// two of them left it rejected by the third.
+func EventWithoutTask(typ string) bool {
+	switch typ {
+	case "QuotaReported", "StandupAdvisory", "ApprovalGranted", "ApprovalDenied",
+		EventObservationIncidentOpened, EventObservationIncidentClosed:
+		return true
+	}
+	return false
+}
+
 func ValidateEvent(e Event) error {
 	if e.SchemaVersion > CurrentEventSchema || e.Type == "" || e.TaskID == "" || len(e.Payload) == 0 || len(e.Payload) > 64*1024 {
 		return ErrInvalid
@@ -300,7 +317,7 @@ func ValidateEvent(e Event) error {
 	if e.SchemaVersion >= 2 && strings.TrimSpace(e.Surface) == "" {
 		return fmt.Errorf("%w: surface required", ErrInvalid)
 	}
-	allowed := map[string]bool{"TaskCreated": true, "TaskLeased": true, "TaskLeaseRenewed": true, "TaskReleased": true, "TaskLaunchAcknowledged": true, "TaskPickupValidated": true, "TaskCompleted": true, "TaskFailed": true, "TaskBlocked": true, "TaskNeedsAttention": true, "ApprovalRequested": true, "ApprovalGranted": true, "ApprovalDenied": true, "TaskAmended": true, "TaskCorrected": true, "QuotaReported": true, "StandupAdvisory": true, EventHumanDecisionRecorded: true, EventHumanDecisionSuperseded: true, EventWorkPhaseChanged: true, EventDeferredFindingRecorded: true, EventReviewRecorded: true, EventTaskSubmitted: true, EventTaskChangesRequested: true, EventPlanPhaseVerified: true, EventPlanMismatchRecorded: true}
+	allowed := map[string]bool{"TaskCreated": true, "TaskLeased": true, "TaskLeaseRenewed": true, "TaskReleased": true, "TaskLaunchAcknowledged": true, "TaskPickupValidated": true, "TaskCompleted": true, "TaskFailed": true, "TaskBlocked": true, "TaskNeedsAttention": true, "ApprovalRequested": true, "ApprovalGranted": true, "ApprovalDenied": true, "TaskAmended": true, "TaskCorrected": true, "QuotaReported": true, "StandupAdvisory": true, EventHumanDecisionRecorded: true, EventHumanDecisionSuperseded: true, EventWorkPhaseChanged: true, EventDeferredFindingRecorded: true, EventReviewRecorded: true, EventTaskSubmitted: true, EventTaskChangesRequested: true, EventPlanPhaseVerified: true, EventPlanMismatchRecorded: true, EventObservationIncidentOpened: true, EventObservationIncidentClosed: true}
 	if !allowed[e.Type] {
 		return fmt.Errorf("%w: unknown type %q", ErrInvalid, e.Type)
 	}
@@ -582,6 +599,10 @@ func ValidatePayload(typ string, p map[string]any) error {
 		return ValidatePlanPhaseVerified(p)
 	case EventPlanMismatchRecorded:
 		return ValidatePlanMismatchRecorded(p)
+	case EventObservationIncidentOpened:
+		return ValidateObservationIncidentOpened(p)
+	case EventObservationIncidentClosed:
+		return ValidateObservationIncidentClosed(p)
 	case EventReviewRecorded:
 		if err := requiredHash(p, "artifact_ref"); err != nil {
 			return err

internal/domain/observation.go

@@ -0,0 +1,142 @@
+package domain
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+	"time"
+)
+
+// A worker's observation ring is bounded, lossy, and local: it holds distinct
+// failure messages with repeat counts and nothing else, and it disappears when
+// the process does. The debt ledger reported that gap about itself, because no
+// event carried any of it.
+//
+// These two events make it durable as incidents rather than as symptoms. Run
+// 11 saw the same 409 refusal 301 times; that is one incident with an
+// intensity of 301, not 301 pieces of evidence. Recurrence has to mean "this
+// happened on four independent leases", or one stuck loop makes everything
+// look chronic.
+const (
+	EventObservationIncidentOpened = "ObservationIncidentOpened"
+	EventObservationIncidentClosed = "ObservationIncidentClosed"
+)
+
+// ObservationCloseReason is why Orchestra finalized an incident. None of them
+// is "the message stopped appearing in the ring": the ring is a bounded
+// history, so absence proves eviction as easily as recovery.
+type ObservationCloseReason string
+
+const (
+	// ObservationCloseLeaseEnd and ObservationCloseEpochChange are the natural
+	// boundaries of a lease-scoped incident. The work it was about is over.
+	ObservationCloseLeaseEnd    ObservationCloseReason = "lease_end"
+	ObservationCloseEpochChange ObservationCloseReason = "epoch_change"
+	// ObservationCloseWorkerRestart ends every incident of an incarnation. A
+	// new process cannot continue the old one's symptom.
+	ObservationCloseWorkerRestart ObservationCloseReason = "worker_restart"
+	// ObservationCloseQuietTimeout is the only closer for an observation with
+	// no lease to bound it, and it fires on last_seen going stale rather than
+	// on the entry vanishing.
+	ObservationCloseQuietTimeout ObservationCloseReason = "quiet_timeout"
+)
+
+func (r ObservationCloseReason) Valid() bool {
+	switch r {
+	case ObservationCloseLeaseEnd, ObservationCloseEpochChange, ObservationCloseWorkerRestart, ObservationCloseQuietTimeout:
+		return true
+	}
+	return false
+}
+
+// WorkerObservation is one entry of a worker's ring as reported on a
+// heartbeat. It is the input to the incident projection, never a stored event.
+type WorkerObservation struct {
+	Message string    `json:"message"`
+	Count   int       `json:"count"`
+	First   time.Time `json:"first"`
+	Last    time.Time `json:"last"`
+}
+
+// ObservationIncident is one durable incident: a signature seen by one worker,
+// on one lease when there is one, from its first occurrence to the boundary
+// that ended it.
+type ObservationIncident struct {
+	ID          string `json:"observation_id"`
+	WorkerID    string `json:"worker_id"`
+	Incarnation string `json:"incarnation,omitempty"`
+	TaskID      string `json:"task_id,omitempty"`
+	LeaseEpoch  string `json:"lease_epoch,omitempty"`
+	// Signature is the message with its task ids, commit shas, paths and
+	// durations replaced, so the same failure on two tasks shares it. Grouping
+	// on the raw message would make every task its own kind of problem.
+	Signature string    `json:"signature"`
+	Detail    string    `json:"detail,omitempty"`
+	FirstSeen time.Time `json:"first_seen"`
+	// LastSeen is the last actual occurrence. ClosedAt is when Orchestra
+	// finalized the incident, which is later and often much later: an incident
+	// stays open until its lease ends, and open means "not yet final evidence"
+	// rather than "happening right now".
+	LastSeen    time.Time              `json:"last_seen,omitempty"`
+	ClosedAt    time.Time              `json:"closed_at,omitempty"`
+	RepeatCount int                    `json:"repeat_count,omitempty"`
+	CloseReason ObservationCloseReason `json:"close_reason,omitempty"`
+}
+
+// Key identifies an incident. Two workers reporting the same failure are two
+// incidents, and so are two leases of one task.
+func (i ObservationIncident) Key() string {
+	return strings.Join([]string{i.WorkerID, i.TaskID, i.LeaseEpoch, i.Signature}, "\x00")
+}
+
+var (
+	observationID       = regexp.MustCompile(`\b[0-9A-HJKMNP-TV-Z]{26}\b`)
+	observationSHA      = regexp.MustCompile(`\b[0-9a-f]{7,64}\b`)
+	observationDuration = regexp.MustCompile(`\b\d+(\.\d+)?(ns|us|µs|ms|s|m|h)(\d+(\.\d+)?(ns|us|µs|ms|s|m|h))*\b`)
+	observationNumber   = regexp.MustCompile(`\b\d+\b`)
+	observationPath     = regexp.MustCompile(`(/[\w.-]+){2,}`)
+)
+
+// ObservationSignature collapses one message to the kind of failure it is.
+// "lease A not renewed" and "lease B not renewed" are the same problem seen
+// twice, which is the whole basis of counting recurrence across tasks.
+func ObservationSignature(message string) string {
+	s := strings.TrimSpace(message)
+	s = observationID.ReplaceAllString(s, "<id>")
+	s = observationPath.ReplaceAllString(s, "<path>")
+	s = observationDuration.ReplaceAllString(s, "<dur>")
+	s = observationSHA.ReplaceAllString(s, "<sha>")
+	s = observationNumber.ReplaceAllString(s, "<n>")
+	s = strings.Join(strings.Fields(s), " ")
+	if len(s) > 200 {
+		s = s[:200]
+	}
+	return s
+}
+
+func ValidateObservationIncidentOpened(p map[string]any) error {
+	if id, _ := p["observation_id"].(string); strings.TrimSpace(id) == "" {
+		return fmt.Errorf("%w: observation_id required", ErrInvalid)
+	}
+	if w, _ := p["worker_id"].(string); strings.TrimSpace(w) == "" {
+		return fmt.Errorf("%w: worker_id required", ErrInvalid)
+	}
+	if sig, _ := p["signature"].(string); strings.TrimSpace(sig) == "" {
+		return fmt.Errorf("%w: signature required", ErrInvalid)
+	}
+	return nil
+}
+
+func ValidateObservationIncidentClosed(p map[string]any) error {
+	if id, _ := p["observation_id"].(string); strings.TrimSpace(id) == "" {
+		return fmt.Errorf("%w: observation_id required", ErrInvalid)
+	}
+	reason, _ := p["close_reason"].(string)
+	if !ObservationCloseReason(reason).Valid() {
+		return fmt.Errorf("%w: close_reason %q is not a close reason", ErrInvalid, reason)
+	}
+	if c, ok := p["repeat_count"].(float64); ok && c < 0 {
+		return fmt.Errorf("%w: repeat_count cannot be negative", ErrInvalid)
+	}
+	return nil
+}

internal/operations/observations.go

@@ -0,0 +1,201 @@
+package operations
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"orchestra/internal/authz"
+	"orchestra/internal/domain"
+	"orchestra/internal/store"
+)
+
+// QuietTimeout bounds an incident that has no lease to bound it. A worker-level
+// observation has no terminal boundary of its own, so staleness of its last
+// actual occurrence is the only honest closer.
+const QuietTimeout = 10 * time.Minute
+
+// ObservationTracker turns a worker's bounded, lossy ring into durable
+// incidents. It is deliberately not a copy of the ring.
+//
+// The rules that matter, and why:
+//
+//   - An incident is opened at first sight and appended immediately, so a
+//     coordinator that dies mid-incident still leaves the fact that it existed.
+//   - A repeat updates the aggregate and appends nothing. Run 11's 409 loop
+//     repeated 301 times; appending each would have been 301 pieces of evidence
+//     for one problem, and would have made every debt item eligible at once.
+//   - Absence from the ring closes nothing. The ring is a bounded history, so a
+//     message can vanish because it was evicted rather than because it stopped.
+//   - The lease is the scope. The same signature going quiet and returning
+//     inside one epoch is one incident, not two recurrences.
+type ObservationTracker struct {
+	Store *store.Store
+	// counts is the last count this tracker saw for an open incident, so a
+	// ring entry that is evicted and recreated accumulates rather than
+	// restarting. Reported 34, evicted, reported 3 again means 37 occurrences,
+	// not 3. In-memory: a restart loses the accumulation, never the incident.
+	counts map[string]int
+	// incarnations is the last incarnation seen per worker, which is what makes
+	// a restart detectable at all.
+	incarnations map[string]string
+}
+
+// WorkerReport is one heartbeat's worth of attributed observations. The
+// coordinator attributes them, because the ring carries only messages: the
+// worker's active task and that task's current lease epoch are what bind an
+// incident to the work it happened during.
+type WorkerReport struct {
+	WorkerID     string
+	Incarnation  string
+	TaskID       string
+	LeaseEpoch   string
+	Observations []domain.WorkerObservation
+	At           time.Time
+}
+
+// Ingest folds one heartbeat into the durable incidents and returns the events
+// it appended. Every close it decides is one of the four boundaries; none of
+// them is "the message is no longer in the ring".
+func (t *ObservationTracker) Ingest(r WorkerReport) ([]domain.Event, error) {
+	if t == nil || t.Store == nil || r.WorkerID == "" {
+		return nil, nil
+	}
+	if t.counts == nil {
+		t.counts, t.incarnations = map[string]int{}, map[string]string{}
+	}
+	at := r.At
+	if at.IsZero() {
+		at = time.Now().UTC()
+	}
+	var appended []domain.Event
+
+	// A new process cannot continue the previous one's symptom, so its
+	// incidents are finalized before anything this heartbeat says is folded in.
+	if previous, seen := t.incarnations[r.WorkerID]; r.Incarnation != "" && seen && previous != r.Incarnation {
+		closed, err := t.closeWhere(at, domain.ObservationCloseWorkerRestart, func(inc domain.ObservationIncident) bool {
+			return inc.WorkerID == r.WorkerID
+		})
+		appended = append(appended, closed...)
+		if err != nil {
+			return appended, err
+		}
+	}
+	if r.Incarnation != "" {
+		t.incarnations[r.WorkerID] = r.Incarnation
+	}
+
+	open := map[string]domain.ObservationIncident{}
+	for _, inc := range t.Store.OpenObservations() {
+		if inc.WorkerID == r.WorkerID {
+			open[inc.Key()] = inc
+		}
+	}
+	for _, o := range r.Observations {
+		signature := domain.ObservationSignature(o.Message)
+		if signature == "" {
+			continue
+		}
+		candidate := domain.ObservationIncident{
+			WorkerID: r.WorkerID, Incarnation: r.Incarnation, Signature: signature,
+			TaskID: r.TaskID, LeaseEpoch: r.LeaseEpoch,
+		}
+		existing, isOpen := open[candidate.Key()]
+		if !isOpen {
+			candidate.ID = domain.NewID()
+			candidate.Detail = o.Message
+			candidate.FirstSeen = firstOr(o.First, at)
+			candidate.LastSeen = firstOr(o.Last, at)
+			e, err := t.append(domain.EventObservationIncidentOpened, candidate)
+			if err != nil {
+				return appended, err
+			}
+			appended = append(appended, e)
+			t.counts[candidate.ID] = o.Count
+			continue
+		}
+		// Open already: accumulate, append nothing. A count lower than the last
+		// one means the ring evicted the entry and started it again.
+		delta := o.Count - t.counts[existing.ID]
+		if delta < 0 {
+			delta = o.Count
+		}
+		t.counts[existing.ID] += delta
+		if last := firstOr(o.Last, at); last.After(existing.LastSeen) {
+			existing.LastSeen = last
+			t.Store.NoteObservation(existing) // last_seen is durable at close
+		}
+	}
+
+	// The boundaries. A lease that ended, an epoch that changed, and a
+	// worker-level incident whose last occurrence has gone stale.
+	closed, err := t.closeWhere(at, "", func(inc domain.ObservationIncident) bool {
+		if inc.WorkerID != r.WorkerID {
+			return false
+		}
+		if inc.TaskID == "" {
+			return at.Sub(inc.LastSeen) > QuietTimeout
+		}
+		return inc.TaskID != r.TaskID || inc.LeaseEpoch != r.LeaseEpoch
+	})
+	appended = append(appended, closed...)
+	return appended, err
+}
+
+// closeWhere finalizes every open incident the predicate selects. A reason of
+// "" is resolved per incident, which is what lets one sweep close a lease that
+// ended and a worker-level incident that went quiet.
+func (t *ObservationTracker) closeWhere(at time.Time, reason domain.ObservationCloseReason, match func(domain.ObservationIncident) bool) ([]domain.Event, error) {
+	var out []domain.Event
+	for _, inc := range t.Store.OpenObservations() {
+		if !match(inc) {
+			continue
+		}
+		inc.ClosedAt = at
+		inc.RepeatCount = t.counts[inc.ID]
+		inc.CloseReason = reason
+		if inc.CloseReason == "" {
+			switch {
+			case inc.TaskID == "":
+				inc.CloseReason = domain.ObservationCloseQuietTimeout
+			case inc.LeaseEpoch != "":
+				inc.CloseReason = domain.ObservationCloseEpochChange
+			default:
+				inc.CloseReason = domain.ObservationCloseLeaseEnd
+			}
+		}
+		e, err := t.append(domain.EventObservationIncidentClosed, inc)
+		if err != nil {
+			return out, err
+		}
+		delete(t.counts, inc.ID)
+		out = append(out, e)
+	}
+	return out, nil
+}
+
+// append writes the incident as a worker-scoped event. The task it happened
+// during is carried in the payload rather than in Event.TaskID on purpose: an
+// incident is evidence about a worker, and binding it to the task aggregate
+// would bump that task's version from a path the lease knows nothing about.
+func (t *ObservationTracker) append(typ string, inc domain.ObservationIncident) (domain.Event, error) {
+	b, err := json.Marshal(inc)
+	if err != nil {
+		return domain.Event{}, err
+	}
+	// "system" is the same aggregate QuotaReported uses for worker-scoped
+	// facts: every event needs a task id, and this evidence belongs to a
+	// worker rather than to any one task.
+	e := domain.Event{ID: domain.NewID(), Type: typ, TaskID: "system", Payload: b, Surface: string(authz.System)}
+	if err := t.Store.Append(e); err != nil {
+		return domain.Event{}, fmt.Errorf("record observation incident: %w", err)
+	}
+	return e, nil
+}
+
+func firstOr(t, fallback time.Time) time.Time {
+	if t.IsZero() {
+		return fallback
+	}
+	return t
+}

internal/operations/observations_test.go

@@ -0,0 +1,209 @@
+package operations
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	"orchestra/internal/domain"
+	"orchestra/internal/store"
+)
+
+func tracker(t *testing.T) (*ObservationTracker, *store.Store) {
+	t.Helper()
+	s, err := store.Open(t.TempDir())
+	if err != nil {
+		t.Fatal(err)
+	}
+	return &ObservationTracker{Store: s}, s
+}
+
+func ring(message string, count int, last time.Time) []domain.WorkerObservation {
+	return []domain.WorkerObservation{{Message: message, Count: count, First: last.Add(-time.Minute), Last: last}}
+}
+
+func closedIncident(t *testing.T, s *store.Store) domain.ObservationIncident {
+	t.Helper()
+	var out domain.ObservationIncident
+	found := 0
+	for _, e := range s.Events(0) {
+		if e.Type != domain.EventObservationIncidentClosed {
+			continue
+		}
+		found++
+		if err := json.Unmarshal(e.Payload, &out); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if found != 1 {
+		t.Fatalf("closed incidents = %d, want 1", found)
+	}
+	return out
+}
+
+// 301 repeats of one refusal are one incident with an intensity of 301, not
+// 301 pieces of evidence. Appending each would spam the log and make one stuck
+// loop look like chronic, recurring debt.
+func TestRepeatsAreOneIncident(t *testing.T) {
+	tr, s := tracker(t)
+	at := time.Unix(1700000000, 0).UTC()
+	report := func(count int, when time.Time) {
+		if _, err := tr.Ingest(WorkerReport{
+			WorkerID: "workpc-claude", Incarnation: "boot-1", TaskID: "task-a", LeaseEpoch: "epoch-1",
+			Observations: ring("release task-a commit: 409 superseded", count, when), At: when,
+		}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	report(1, at)
+	report(40, at.Add(time.Minute))
+	report(301, at.Add(2*time.Minute))
+
+	opened := 0
+	for _, e := range s.Events(0) {
+		if e.Type == domain.EventObservationIncidentOpened {
+			opened++
+		}
+		if e.Type == domain.EventObservationIncidentClosed {
+			t.Fatal("an incident was closed while its lease was still running")
+		}
+	}
+	if opened != 1 {
+		t.Fatalf("opened %d incidents for one repeating failure", opened)
+	}
+	if open := s.OpenObservations(); len(open) != 1 || open[0].TaskID != "task-a" {
+		t.Fatalf("open incidents = %+v", open)
+	}
+}
+
+// The ring is a bounded history, so an entry that disappears may have been
+// evicted rather than resolved. Absence must not close anything, and a
+// recreated entry must accumulate rather than restart its count.
+func TestEvictionNeitherClosesNorRestartsTheCount(t *testing.T) {
+	tr, s := tracker(t)
+	at := time.Unix(1700000000, 0).UTC()
+	send := func(obs []domain.WorkerObservation, when time.Time) {
+		if _, err := tr.Ingest(WorkerReport{
+			WorkerID: "w", Incarnation: "boot-1", TaskID: "task-a", LeaseEpoch: "epoch-1",
+			Observations: obs, At: when,
+		}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	send(ring("lease task-a not renewed: agent idle", 34, at), at)
+	// Evicted: the message is simply gone from this heartbeat.
+	send(nil, at.Add(time.Minute))
+	if len(s.OpenObservations()) != 1 {
+		t.Fatal("an incident was closed because its message left a bounded ring")
+	}
+	// Recreated, counting from scratch on the worker side.
+	send(ring("lease task-a not renewed: agent idle", 3, at.Add(2*time.Minute)), at.Add(2*time.Minute))
+
+	// The lease ends, which is a real boundary.
+	if _, err := tr.Ingest(WorkerReport{WorkerID: "w", Incarnation: "boot-1", At: at.Add(3 * time.Minute)}); err != nil {
+		t.Fatal(err)
+	}
+	inc := closedIncident(t, s)
+	if inc.RepeatCount != 37 {
+		t.Fatalf("repeat_count = %d, want 37 (34 before eviction plus 3 after)", inc.RepeatCount)
+	}
+	if inc.CloseReason != domain.ObservationCloseEpochChange {
+		t.Fatalf("close_reason = %q", inc.CloseReason)
+	}
+	if !inc.LastSeen.Equal(at.Add(2 * time.Minute)) {
+		t.Fatalf("last_seen = %s, want the last actual occurrence", inc.LastSeen)
+	}
+	if !inc.ClosedAt.After(inc.LastSeen) {
+		t.Fatal("closed_at must be when Orchestra finalized it, not when the failure last happened")
+	}
+}
+
+// Recurrence is independent incidents. The same signature on two tasks is two,
+// which is the evidence that means something; repeats inside one are intensity.
+func TestTheSameSignatureOnAnotherTaskIsASecondIncident(t *testing.T) {
+	tr, s := tracker(t)
+	at := time.Unix(1700000000, 0).UTC()
+	if _, err := tr.Ingest(WorkerReport{
+		WorkerID: "w", Incarnation: "boot-1", TaskID: "task-a", LeaseEpoch: "e1",
+		Observations: ring("lease task-a not renewed: agent idle", 5, at), At: at,
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := tr.Ingest(WorkerReport{
+		WorkerID: "w", Incarnation: "boot-1", TaskID: "task-b", LeaseEpoch: "e2",
+		Observations: ring("lease task-b not renewed: agent idle", 2, at.Add(time.Minute)), At: at.Add(time.Minute),
+	}); err != nil {
+		t.Fatal(err)
+	}
+	opened, closed := 0, 0
+	for _, e := range s.Events(0) {
+		switch e.Type {
+		case domain.EventObservationIncidentOpened:
+			opened++
+		case domain.EventObservationIncidentClosed:
+			closed++
+		}
+	}
+	if opened != 2 {
+		t.Fatalf("opened = %d, want one incident per lease", opened)
+	}
+	if closed != 1 {
+		t.Fatalf("closed = %d, want the first lease finalized when the second began", closed)
+	}
+}
+
+// A restart cannot continue the previous process's symptom.
+func TestAWorkerRestartClosesItsIncidents(t *testing.T) {
+	tr, s := tracker(t)
+	at := time.Unix(1700000000, 0).UTC()
+	if _, err := tr.Ingest(WorkerReport{
+		WorkerID: "w", Incarnation: "boot-1", TaskID: "task-a", LeaseEpoch: "e1",
+		Observations: ring("herdr unreachable", 9, at), At: at,
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := tr.Ingest(WorkerReport{WorkerID: "w", Incarnation: "boot-2", At: at.Add(time.Minute)}); err != nil {
+		t.Fatal(err)
+	}
+	inc := closedIncident(t, s)
+	if inc.CloseReason != domain.ObservationCloseWorkerRestart || inc.RepeatCount != 9 {
+		t.Fatalf("incident = %+v", inc)
+	}
+}
+
+// An observation with no lease has no terminal boundary, so staleness of its
+// last actual occurrence is what ends it.
+func TestAWorkerLevelIncidentClosesOnQuietTimeout(t *testing.T) {
+	tr, s := tracker(t)
+	at := time.Unix(1700000000, 0).UTC()
+	if _, err := tr.Ingest(WorkerReport{
+		WorkerID: "w", Incarnation: "boot-1",
+		Observations: ring("heartbeat: connection refused", 4, at), At: at,
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := tr.Ingest(WorkerReport{WorkerID: "w", Incarnation: "boot-1", At: at.Add(time.Minute)}); err != nil {
+		t.Fatal(err)
+	}
+	if len(s.OpenObservations()) != 1 {
+		t.Fatal("a worker-level incident closed before its quiet timeout")
+	}
+	if _, err := tr.Ingest(WorkerReport{WorkerID: "w", Incarnation: "boot-1", At: at.Add(QuietTimeout + time.Minute)}); err != nil {
+		t.Fatal(err)
+	}
+	if inc := closedIncident(t, s); inc.CloseReason != domain.ObservationCloseQuietTimeout {
+		t.Fatalf("close_reason = %q", inc.CloseReason)
+	}
+}
+
+// The signature is what makes recurrence countable across tasks.
+func TestSignatureCollapsesIdsAndCounts(t *testing.T) {
+	a := domain.ObservationSignature("lease 06G4WJ9T4F35NZC4Z8QQXM9Z6G not renewed: agent status idle and pane unchanged")
+	b := domain.ObservationSignature("lease 06G4VF5HZW7Q4JBM3TTY7W1Y64 not renewed: agent status idle and pane unchanged")
+	if a != b {
+		t.Fatalf("the same failure on two tasks has two signatures:\n%s\n%s", a, b)
+	}
+	if c := domain.ObservationSignature("release 06G4WJ9T4F35NZC4Z8QQXM9Z6G commit: 409 superseded"); c == a {
+		t.Fatal("two different failures collapsed to one signature")
+	}
+}

internal/store/store.go

@@ -49,6 +49,9 @@ type Store struct {
 	cursors        map[string]string
 	cursorPath     string
 	decisionSource map[string]string
+	// openObservations are the incidents opened and not yet closed, by id.
+	// Derived from the log, so a restart finds them again.
+	openObservations map[string]domain.ObservationIncident
 	// PreLease runs immediately before a lease is minted, which is the single
 	// point where ownership of a task begins. Reconciliation of newer human
 	// input belongs here rather than in any individual launch path, because a
@@ -96,7 +99,7 @@ func Open(dir string) (*Store, error) {
 			if t, ok := s.tasks[e.TaskID]; ok && e.Version != t.Version+1 {
 				return nil, domain.ErrConflict
 			}
-			global := e.Type == "QuotaReported" || e.Type == "StandupAdvisory" || e.Type == "ApprovalGranted" || e.Type == "ApprovalDenied"
+			global := domain.EventWithoutTask(e.Type)
 			if _, ok := s.tasks[e.TaskID]; !ok && e.Type != "TaskCreated" && !global {
 				return nil, domain.ErrNotFound
 			}
@@ -490,6 +493,26 @@ func (s *Store) apply(e domain.Event) error {
 		t.Version = e.Version
 		s.replaceTask(e.TaskID, t)
 		return nil
+	case domain.EventObservationIncidentOpened:
+		// Open incidents are projected so a coordinator restart resumes them
+		// instead of orphaning them half-recorded. The event is what makes an
+		// incident durable at first sight; this is how it is found again.
+		var inc domain.ObservationIncident
+		if err := json.Unmarshal(e.Payload, &inc); err != nil {
+			return err
+		}
+		if s.openObservations == nil {
+			s.openObservations = map[string]domain.ObservationIncident{}
+		}
+		s.openObservations[inc.ID] = inc
+		return nil
+	case domain.EventObservationIncidentClosed:
+		var inc domain.ObservationIncident
+		if err := json.Unmarshal(e.Payload, &inc); err != nil {
+			return err
+		}
+		delete(s.openObservations, inc.ID)
+		return nil
 	case domain.EventPlanMismatchRecorded:
 		// Projected so the phase this reopens can be told what reopened it.
 		// The event is the record; this is the live instruction derived from
@@ -847,7 +870,7 @@ func (s *Store) Append(e domain.Event) error {
 			return fmt.Errorf("%w: corrects references unknown event %q for this task", domain.ErrInvalid, corrects)
 		}
 	}
-	global := e.Type == "QuotaReported" || e.Type == "StandupAdvisory" || e.Type == "ApprovalGranted" || e.Type == "ApprovalDenied"
+	global := domain.EventWithoutTask(e.Type)
 	if !taskExists && e.Type != "TaskCreated" && !global {
 		return domain.ErrNotFound
 	}
@@ -909,7 +932,7 @@ func (s *Store) Append(e domain.Event) error {
 // for every lifecycle mutation.
 func (s *Store) validateTransition(e domain.Event, t domain.Task, exists bool, p map[string]any) error {
 	if !exists {
-		if e.Type != "TaskCreated" && e.Type != "QuotaReported" && e.Type != "StandupAdvisory" && e.Type != "ApprovalGranted" && e.Type != "ApprovalDenied" {
+		if e.Type != "TaskCreated" && !domain.EventWithoutTask(e.Type) {
 			return domain.ErrNotFound
 		}
 		return nil
@@ -1113,6 +1136,31 @@ func (s *Store) Artifact(ref string) ([]byte, error) {
 	return b, nil
 }
 
+// NoteObservation updates an open incident's last actual occurrence. It writes
+// no event: the aggregate is durable when the incident is finalized, and
+// appending one per heartbeat is exactly the spam this design exists to avoid.
+func (s *Store) NoteObservation(inc domain.ObservationIncident) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if _, ok := s.openObservations[inc.ID]; ok {
+		s.openObservations[inc.ID] = inc
+	}
+}
+
+// OpenObservations returns the incidents that are open, newest first by first
+// sight. Open means not yet finalized as durable evidence, never "the failure
+// is happening right now".
+func (s *Store) OpenObservations() []domain.ObservationIncident {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	out := make([]domain.ObservationIncident, 0, len(s.openObservations))
+	for _, inc := range s.openObservations {
+		out = append(out, inc)
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].FirstSeen.After(out[j].FirstSeen) })
+	return out
+}
+
 func (s *Store) Task(id string) (domain.Task, bool) {
 	s.mu.Lock()
 	defer s.mu.Unlock()