fix(kernel,server,tui,inference): generic resource gauge + F-002/F-003/F-004

- generic SystemResourceProbe: owner-agnostic /proc/meminfo system RAM
  overlaid on the vendor GPU probe, wired on both managed and static paths
  so the VRAM/GPU/RAM gauge renders with an external llama-server (was dormant)
- F-002: classify provider 4xx (e.g. llama.cpp 400) as terminal, not retryable
  (except 408/429); stops retrying deterministic request failures to exhaustion
- F-003: FileSystemPromptLoader expands leading ~ / ~/ to user home
- F-004: map InferenceFailedEvent + RetryAttemptedEvent to new
  ServerMessage.InferenceFailed / RetryAttempted, decoded+rendered in tui-go;
  ArtifactContentStoredEvent explicitly mapped to null (internal, no warn)
- tests: tilde expansion, SystemResourceProbe (static/managed/fail-soft)
This commit is contained in:
2026-06-10 11:07:08 +04:00
parent a92b5a3531
commit ee24b7786a
14 changed files with 254 additions and 24 deletions
@@ -74,6 +74,7 @@ import com.correx.infrastructure.inference.commons.ManagedInferenceRouter
import com.correx.infrastructure.inference.commons.AmdResourceProbe
import com.correx.infrastructure.inference.commons.NvidiaResourceProbe
import com.correx.infrastructure.inference.commons.ResourceProbe
import com.correx.infrastructure.inference.commons.SystemResourceProbe
import com.correx.infrastructure.inference.commons.UnavailableProbe
import com.correx.core.inference.InferenceProvider
import com.correx.infrastructure.inference.llama.cpp.LlamaCppInferenceProvider
@@ -119,8 +120,10 @@ fun main() {
val inferenceRouter: InferenceRouter
// Non-null only on the managed path; backs manual model swap/pin from clients.
var modelSwapper: ManagedInferenceRouter? = null
// Live GPU/RAM gauge; NVIDIA-backed on the managed path when nvidia-smi is present, else unavailable.
var resourceProbe: ResourceProbe = UnavailableProbe
// Live GPU/RAM gauge. The managed pid lets the GPU probe also report the model's process RSS;
// it stays null on the static path. System RAM (overlaid below) is owner-agnostic, so the gauge
// renders on both paths regardless of whether correx manages the model process.
var managedPidSupplier: () -> Long? = { null }
if (correxConfig.models.isNotEmpty()) {
val settings = correxConfig.modelsSettings
@@ -146,13 +149,7 @@ fun main() {
val managedRouter = ManagedInferenceRouter(modelManager, descriptors, targetModelConfig.id)
inferenceRouter = managedRouter
modelSwapper = managedRouter
resourceProbe = when {
NvidiaResourceProbe.isAvailable() ->
NvidiaResourceProbe(pidSupplier = { modelManager.currentPid() })
AmdResourceProbe.isAvailable() ->
AmdResourceProbe(pidSupplier = { modelManager.currentPid() })
else -> UnavailableProbe
}
managedPidSupplier = { modelManager.currentPid() }
// Shutdown hook to kill the managed llama-server
Runtime.getRuntime().addShutdownHook(Thread {
log.info("Shutdown: unloading managed model '{}'", targetModelConfig.id)
@@ -169,6 +166,15 @@ fun main() {
inferenceRouter = DefaultInferenceRouter(infraRegistry, FirstAvailableRoutingStrategy())
}
// Build the resource gauge for whichever GPU vendor is present (pid-less GPU/VRAM reads work on
// both paths), then overlay owner-agnostic system RAM so the gauge always renders (F-001 gauge).
val gpuProbe: ResourceProbe = when {
NvidiaResourceProbe.isAvailable() -> NvidiaResourceProbe(pidSupplier = managedPidSupplier)
AmdResourceProbe.isAvailable() -> AmdResourceProbe(pidSupplier = managedPidSupplier)
else -> UnavailableProbe
}
val resourceProbe: ResourceProbe = SystemResourceProbe(gpuProbe)
val modelId = firstProvider.id.value
logModelInfo(modelId, infraRegistry)
@@ -8,14 +8,17 @@ import com.correx.apps.server.protocol.toDto
import com.correx.core.artifactstore.ArtifactStore
import com.correx.core.events.events.ApprovalDecisionResolvedEvent
import com.correx.core.events.events.ApprovalRequestedEvent
import com.correx.core.events.events.ArtifactContentStoredEvent
import com.correx.core.events.events.ArtifactCreatedEvent
import com.correx.core.events.events.ChatSessionStartedEvent
import com.correx.core.events.events.ChatTurnEvent
import com.correx.core.events.events.InferenceCompletedEvent
import com.correx.core.events.events.SessionWorkspaceBoundEvent
import com.correx.core.events.events.WorkflowStartedEvent
import com.correx.core.events.events.InferenceFailedEvent
import com.correx.core.events.events.InferenceStartedEvent
import com.correx.core.events.events.InferenceTimeoutEvent
import com.correx.core.events.events.RetryAttemptedEvent
import com.correx.core.events.events.ModelLoadedEvent
import com.correx.core.events.events.ModelUnloadedEvent
import com.correx.core.events.events.OrchestrationPausedEvent
@@ -149,6 +152,22 @@ suspend fun domainEventToServerMessage(
sequence = seq,
sessionSequence = sessionSequence,
)
is InferenceFailedEvent -> ServerMessage.InferenceFailed(
sessionId = p.sessionId,
stageId = p.stageId,
reason = p.reason,
sequence = seq,
sessionSequence = sessionSequence,
)
is RetryAttemptedEvent -> ServerMessage.RetryAttempted(
sessionId = p.sessionId,
stageId = p.stageId,
attemptNumber = p.attemptNumber,
maxAttempts = p.maxAttempts,
failureReason = p.failureReason,
sequence = seq,
sessionSequence = sessionSequence,
)
is ToolInvocationRequestedEvent -> ServerMessage.ToolStarted(
sessionId = p.sessionId,
@@ -232,6 +251,9 @@ suspend fun domainEventToServerMessage(
providerId = p.providerId.value,
loaded = false,
)
// Internal slot→CAS-hash bookkeeping (F-007 durable bridge); no operator-facing surface.
is ArtifactContentStoredEvent -> null
else -> {
log.debug(
"DomainEventMapper: unmapped payload type={} sessionId={} sequence={}",
@@ -191,6 +191,28 @@ sealed interface ServerMessage {
override val sessionSequence: Long,
) : ServerMessage, SessionMessage
@Serializable
@SerialName("inference.failed")
data class InferenceFailed(
val sessionId: SessionId,
val stageId: StageId,
val reason: String,
override val sequence: Long,
override val sessionSequence: Long,
) : ServerMessage, SessionMessage
@Serializable
@SerialName("inference.retry")
data class RetryAttempted(
val sessionId: SessionId,
val stageId: StageId,
val attemptNumber: Int,
val maxAttempts: Int,
val failureReason: String,
override val sequence: Long,
override val sessionSequence: Long,
) : ServerMessage, SessionMessage
// -- Tool execution --
@Serializable
@@ -336,6 +358,8 @@ sealed interface ServerMessage {
val gpuMemoryTotalMb: Long?,
val gpuUtilizationPct: Int?,
val processRssMb: Long?,
val systemRamUsedMb: Long? = null,
val systemRamTotalMb: Long? = null,
override val sequence: Long? = null,
override val sessionSequence: Long? = null,
) : ServerMessage, NonEventMessage
@@ -61,6 +61,8 @@ private fun ResourceSnapshot.toResourceStatus(): ServerMessage.ResourceStatus =
gpuMemoryTotalMb = gpu?.memoryTotalMb,
gpuUtilizationPct = gpu?.utilizationPct,
processRssMb = processRssBytes?.let { it / BYTES_PER_MB },
systemRamUsedMb = systemRamUsedBytes?.let { it / BYTES_PER_MB },
systemRamTotalMb = systemRamTotalBytes?.let { it / BYTES_PER_MB },
)
class GlobalStreamHandler(private val module: ServerModule) {