feat(server,cli): event-sourced health checks — disk watermark probe (observability §4)

Health states are events. A HealthMonitor polls registered HealthProbes on an
interval and appends HealthDegraded/HealthRestored only on a status transition
(hysteresis), so health is observable, replayable, and renderable like every
other fact in the log. The probe reads nondeterministic environment state and
records it at observe-time (Invariant #9); HealthProjection rebuilds current
health by replay, never re-probing (Invariant #8). Monitor is live-only, started
in ServerModule.start alongside narration; seeded from the projection so a
restart does not re-emit a degraded the log already records.

First concrete probe: DiskWatermarkProbe — event-log + CAS size and growth rate,
warning before the 2am surprise. Read surface mirrors `correx stats`:
GET /health/checks replays the system session and `correx health` renders it.
Health events ride the reserved __system__ session, filtered from operator
session listings.

LLAMA_SERVER (liveness + tokens/sec trend) and EVENT_STORE (append/fsync latency)
probes plug into the same HealthProbe framework as follow-ups.

core/config/src/main/kotlin/com/correx/core/config/CorrexConfig.kt

@@ -16,6 +16,21 @@ data class CorrexConfig(
     val project: ProjectConfig = ProjectConfig(),
     val personalization: PersonalizationConfig = PersonalizationConfig(),
     val orchestration: OrchestrationKnobs = OrchestrationKnobs(),
+    val health: HealthConfig = HealthConfig(),
+)
+
+/**
+ * Continuous health watch (observability-spec §4). When [enabled], a background monitor polls
+ * the on-disk footprint every [intervalMs] and records a degraded/restored event on the edge.
+ * [diskWarnBytes] is the absolute event-log + CAS size that trips a warning; [diskWarnGrowthBytesPerMin]
+ * is the growth rate that warns log compaction is becoming non-theoretical. Defaults suit a homelab.
+ */
+@Serializable
+data class HealthConfig(
+    val enabled: Boolean = true,
+    val intervalMs: Long = 30_000,
+    val diskWarnBytes: Long = 5_000_000_000,
+    val diskWarnGrowthBytesPerMin: Long = 50_000_000,
 )
 
 /**

core/events/src/main/kotlin/com/correx/core/events/events/HealthEvents.kt

@@ -0,0 +1,56 @@
+package com.correx.core.events.events
+
+import com.correx.core.events.types.SessionId
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+
+/**
+ * The organs the health monitor watches (observability-spec §4). Each is small, paranoid, and
+ * continuous; a state change for any one is recorded as a [HealthDegradedEvent] / [HealthRestoredEvent]
+ * so health is observable, replayable, and renderable like every other fact in the log.
+ */
+@Serializable
+enum class HealthSubject {
+    /** Local inference process: liveness + tokens/sec degradation (KV-cache pressure, thermal throttle). */
+    LLAMA_SERVER,
+
+    /** The append-only event log: append latency + fsync health. The organ that must never lie. */
+    EVENT_STORE,
+
+    /** CAS + log size and growth rate. Append-only means monotonic growth; warn before the 2am surprise. */
+    DISK,
+}
+
+/**
+ * A monitored [subject] crossed from healthy into degraded. Emitted only on the edge (the monitor
+ * holds the last status per subject), so the log records transitions, not every poll. The
+ * environment observation that triggered it is captured here at observe-time (Hard Invariant #9):
+ * replay reads this recorded fact and never re-probes the filesystem/process/store.
+ *
+ * [metric] names the dimension that crossed (e.g. `total_bytes`, `growth_bytes_per_min`);
+ * [observedValue] is its value at the edge; [threshold] is the watermark it crossed.
+ */
+@Serializable
+@SerialName("HealthDegraded")
+data class HealthDegradedEvent(
+    val sessionId: SessionId,
+    val subject: HealthSubject,
+    val metric: String,
+    val observedValue: Long,
+    val threshold: Long,
+    val detail: String,
+) : EventPayload
+
+/**
+ * A monitored [subject] recovered from degraded back to healthy. Edge-triggered counterpart to
+ * [HealthDegradedEvent]; [observedValue] is the [metric] value that brought it back under the line.
+ */
+@Serializable
+@SerialName("HealthRestored")
+data class HealthRestoredEvent(
+    val sessionId: SessionId,
+    val subject: HealthSubject,
+    val metric: String,
+    val observedValue: Long,
+    val detail: String,
+) : EventPayload

core/events/src/main/kotlin/com/correx/core/events/serialization/Serialization.kt

@@ -19,6 +19,8 @@ import com.correx.core.events.events.ContextTruncatedEvent
 import com.correx.core.events.events.EventPayload
 import com.correx.core.events.events.ExecutionPlanLockedEvent
 import com.correx.core.events.events.ExecutionPlanRejectedEvent
+import com.correx.core.events.events.HealthDegradedEvent
+import com.correx.core.events.events.HealthRestoredEvent
 import com.correx.core.events.events.JournalCompactedEvent
 import com.correx.core.events.events.FileWrittenEvent
 import com.correx.core.events.events.L3MemoryRetrievedEvent
@@ -113,6 +115,8 @@ val eventModule = SerializersModule {
         subclass(ExecutionPlanLockedEvent::class)
         subclass(ExecutionPlanRejectedEvent::class)
         subclass(JournalCompactedEvent::class)
+        subclass(HealthDegradedEvent::class)
+        subclass(HealthRestoredEvent::class)
     }
 }
 

core/events/src/test/kotlin/com/correx/core/events/serialization/HealthEventsSerializationTest.kt

@@ -0,0 +1,62 @@
+package com.correx.core.events.serialization
+
+import com.correx.core.events.events.EventPayload
+import com.correx.core.events.events.HealthDegradedEvent
+import com.correx.core.events.events.HealthRestoredEvent
+import com.correx.core.events.events.HealthSubject
+import com.correx.core.events.types.SessionId
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertTrue
+
+class HealthEventsSerializationTest {
+
+    @Test
+    fun `HealthDegradedEvent round-trips as polymorphic EventPayload`() {
+        val sample: EventPayload = HealthDegradedEvent(
+            sessionId = SessionId("__system__"),
+            subject = HealthSubject.DISK,
+            metric = "total_bytes",
+            observedValue = 6_000_000_000,
+            threshold = 5_000_000_000,
+            detail = "event log + CAS at 5.6 GiB (warn ≥ 4.7 GiB)",
+        )
+        val encoded = eventJson.encodeToString(EventPayload.serializer(), sample)
+        assertTrue(encoded.contains("\"type\":\"HealthDegraded\""), "SerialName must be present: $encoded")
+        assertEquals(sample, eventJson.decodeFromString(EventPayload.serializer(), encoded))
+    }
+
+    @Test
+    fun `HealthRestoredEvent round-trips as polymorphic EventPayload`() {
+        val sample: EventPayload = HealthRestoredEvent(
+            sessionId = SessionId("__system__"),
+            subject = HealthSubject.DISK,
+            metric = "total_bytes",
+            observedValue = 1_000_000_000,
+            detail = "event log + CAS at 953.7 MiB",
+        )
+        val encoded = eventJson.encodeToString(EventPayload.serializer(), sample)
+        assertTrue(encoded.contains("\"type\":\"HealthRestored\""), "SerialName must be present: $encoded")
+        assertEquals(sample, eventJson.decodeFromString(EventPayload.serializer(), encoded))
+    }
+
+    @Test
+    fun `every HealthSubject round-trips`() {
+        for (subject in HealthSubject.entries) {
+            val sample: EventPayload = HealthRestoredEvent(
+                sessionId = SessionId("__system__"),
+                subject = subject,
+                metric = "m",
+                observedValue = 0,
+                detail = "ok",
+            )
+            assertEquals(
+                sample,
+                eventJson.decodeFromString(
+                    EventPayload.serializer(),
+                    eventJson.encodeToString(EventPayload.serializer(), sample),
+                ),
+            )
+        }
+    }
+}