feat: correx-managed model lifecycle slice 4 — resource telemetry

Vendor-agnostic ResourceProbe (commons): NvidiaResourceProbe reads nvidia-smi (injectable runner) for whole-device VRAM/util and /proc/<pid>/status for the managed llama-server RSS, both fail-soft to null; UnavailableProbe for non-GPU hosts / the static path. DefaultModelManager.currentPid() + LlamaProcess.pid expose the managed pid. ServerMessage.ResourceStatus (resource.status, NonEventMessage, all-nullable) pushed every 2.5s on the global stream plus one in the initial snapshot. Live gauge only — never event-sourced (feeds no core decision), so invariants #8/#9 hold. Main wires the NVIDIA probe on the managed path when nvidia-smi is present, else UnavailableProbe. Tests: csv parsing (single/multi/malformed), gpu+rss combine, fallbacks. Plan: docs/plans/2026-05-31-model-lifecycle-management.md (slice 4 of 5).
2026-06-01 12:36:38 +04:00
parent 7341ab578e
commit 7b1df95627
10 changed files with 273 additions and 0 deletions
@@ -16,6 +16,7 @@ dependencies {
    implementation "io.ktor:ktor-client-cio:$ktor_version"
    implementation "io.ktor:ktor-client-content-negotiation:$ktor_version"
    implementation "io.ktor:ktor-serialization-kotlinx-json:$ktor_version"
+    implementation "org.slf4j:slf4j-api:2.0.16"
 }

 tasks.named("koverVerify").configure { enabled = false }
@@ -0,0 +1,80 @@
+package com.correx.infrastructure.inference.commons
+
+import org.slf4j.LoggerFactory
+import java.io.File
+
+private val log = LoggerFactory.getLogger("NvidiaResourceProbe")
+
+/**
+ * [ResourceProbe] backed by `nvidia-smi` for GPU stats and `/proc/<pid>/status` for the managed
+ * model process RSS. Both reads are isolated and fail soft to null, so a missing `nvidia-smi`,
+ * a non-NVIDIA host, or a dead pid never throws.
+ *
+ * The `nvidia-smi` invocation and RSS read are injectable so the parsing is unit-testable without
+ * a GPU or a live process.
+ *
+ * @param pidSupplier returns the managed `llama-server` pid, or null when no model is resident.
+ * @param nvidiaSmiRunner returns raw `nvidia-smi` CSV stdout, or null when unavailable.
+ * @param rssReader returns the RSS in bytes for a pid, or null when unreadable.
+ */
+class NvidiaResourceProbe(
+    private val pidSupplier: () -> Long?,
+    private val nvidiaSmiRunner: () -> String? = ::runNvidiaSmi,
+    private val rssReader: (Long) -> Long? = ::readRssBytes,
+) : ResourceProbe {
+
+    override fun probe(): ResourceSnapshot {
+        val gpu = nvidiaSmiRunner()?.let { parseGpu(it) }
+        val rss = pidSupplier()?.let { rssReader(it) }
+        return ResourceSnapshot(gpu = gpu, processRssBytes = rss)
+    }
+
+    companion object {
+        private const val EXPECTED_FIELDS = 3
+        private val QUERY = listOf(
+            "nvidia-smi",
+            "--query-gpu=memory.used,memory.total,utilization.gpu",
+            "--format=csv,noheader,nounits",
+        )
+
+        /**
+         * Parses the first GPU row of `nvidia-smi --format=csv,noheader,nounits` output
+         * (`memory.used, memory.total, utilization.gpu`, all in MiB / percent). Returns null on any
+         * shape mismatch so the caller surfaces "unavailable" instead of partial data.
+         */
+        fun parseGpu(csv: String): GpuStatus? {
+            val row = csv.lineSequence().map { it.trim() }.firstOrNull { it.isNotEmpty() } ?: return null
+            val fields = row.split(',').map { it.trim() }
+            if (fields.size < EXPECTED_FIELDS) return null
+            val used = fields[0].toLongOrNull() ?: return null
+            val total = fields[1].toLongOrNull() ?: return null
+            val util = fields[2].toIntOrNull() ?: return null
+            return GpuStatus(memoryUsedMb = used, memoryTotalMb = total, utilizationPct = util)
+        }
+
+        /** Returns true if `nvidia-smi -L` runs successfully — used to pick the probe at boot. */
+        fun isAvailable(): Boolean = runCatching {
+            val proc = ProcessBuilder("nvidia-smi", "-L")
+                .redirectErrorStream(true)
+                .start()
+            proc.waitFor() == 0
+        }.getOrElse { false }
+
+        private fun runNvidiaSmi(): String? = runCatching {
+            val proc = ProcessBuilder(QUERY).redirectErrorStream(true).start()
+            val out = proc.inputStream.bufferedReader().readText()
+            if (proc.waitFor() == 0) out else null
+        }.onFailure { log.debug("nvidia-smi query failed: {}", it.message) }.getOrNull()
+
+        private fun readRssBytes(pid: Long): Long? = runCatching {
+            val statusFile = File("/proc/$pid/status")
+            if (!statusFile.exists()) return null
+            val vmRss = statusFile.readLines().firstOrNull { it.startsWith("VmRSS:") } ?: return null
+            // Format: "VmRSS:\t   12345 kB"
+            val kb = vmRss.removePrefix("VmRSS:").trim().substringBefore(' ').toLongOrNull() ?: return null
+            kb * BYTES_PER_KB
+        }.onFailure { log.debug("RSS read failed for pid {}: {}", pid, it.message) }.getOrNull()
+
+        private const val BYTES_PER_KB = 1024L
+    }
+}
@@ -0,0 +1,40 @@
+package com.correx.infrastructure.inference.commons
+
+/**
+ * Whole-device GPU usage at an instant. All fields are device-wide (not per-process) —
+ * per-model VRAM attribution is an explicit non-goal.
+ */
+data class GpuStatus(
+    val memoryUsedMb: Long,
+    val memoryTotalMb: Long,
+    val utilizationPct: Int,
+)
+
+/**
+ * A point-in-time reading of local-model resource usage. Both fields are nullable so that
+ * "unavailable" is representable without exceptions: [gpu] is null when no GPU probe is present
+ * (non-NVIDIA box / `nvidia-smi` absent), [processRssBytes] is null when no model is resident.
+ */
+data class ResourceSnapshot(
+    val gpu: GpuStatus?,
+    val processRssBytes: Long?,
+) {
+    companion object {
+        val UNAVAILABLE = ResourceSnapshot(gpu = null, processRssBytes = null)
+    }
+}
+
+/**
+ * Vendor-agnostic live gauge of GPU/process resource usage. This is a **live gauge, never recorded
+ * as events** — it feeds no core decision, so invariant #9 does not require event-sourcing it, and
+ * replay (invariant #8) never reads it. Implementations must degrade gracefully (return
+ * [ResourceSnapshot.UNAVAILABLE]-shaped data) rather than throw.
+ */
+fun interface ResourceProbe {
+    fun probe(): ResourceSnapshot
+}
+
+/** Always-unavailable probe — used on non-GPU boxes and the static (unmanaged) provider path. */
+object UnavailableProbe : ResourceProbe {
+    override fun probe(): ResourceSnapshot = ResourceSnapshot.UNAVAILABLE
+}
@@ -129,6 +129,13 @@ class DefaultModelManager(

    override fun currentModel(): ModelDescriptor? = currentDescriptor

+    /**
+     * The managed `llama-server` OS pid, or null when no model is resident. Read without the mutex
+     * for the live resource gauge — a stale/null read for one probe tick during a swap is acceptable
+     * (telemetry only, feeds no core decision).
+     */
+    fun currentPid(): Long? = currentProcess?.pid
+
    override fun healthCheck(): ProviderHealth {
        val process = currentProcess ?: return ProviderHealth.Unavailable("No model loaded")
        return if (process.isAlive) {
@@ -16,6 +16,10 @@ class LlamaProcess(
    val isAlive: Boolean
        get() = process?.isAlive ?: false

+    /** The OS pid of the running process, or null when not started / not alive. */
+    val pid: Long?
+        get() = process?.takeIf { it.isAlive }?.pid()
+
    suspend fun start() = withContext(Dispatchers.IO) {
        if (isAlive) return@withContext