kami/correx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(kernel,server,tui,inference): generic resource gauge + F-002/F-003/F-004

Browse Source 
- generic SystemResourceProbe: owner-agnostic /proc/meminfo system RAM
  overlaid on the vendor GPU probe, wired on both managed and static paths
  so the VRAM/GPU/RAM gauge renders with an external llama-server (was dormant)
- F-002: classify provider 4xx (e.g. llama.cpp 400) as terminal, not retryable
  (except 408/429); stops retrying deterministic request failures to exhaustion
- F-003: FileSystemPromptLoader expands leading ~ / ~/ to user home
- F-004: map InferenceFailedEvent + RetryAttemptedEvent to new
  ServerMessage.InferenceFailed / RetryAttempted, decoded+rendered in tui-go;
  ArtifactContentStoredEvent explicitly mapped to null (internal, no warn)
- tests: tilde expansion, SystemResourceProbe (static/managed/fail-soft)
This commit is contained in:
kami
2026-06-10 11:07:08 +04:00
parent a92b5a3531
commit ee24b7786a
14 changed files with 254 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/server/src/main/kotlin/com/correx/apps/server/Main.kt
+15 -9
View File
@@ -74,6 +74,7 @@ import com.correx.infrastructure.inference.commons.ManagedInferenceRouter
import com.correx.infrastructure.inference.commons.AmdResourceProbe
import com.correx.infrastructure.inference.commons.NvidiaResourceProbe
import com.correx.infrastructure.inference.commons.ResourceProbe
import com.correx.infrastructure.inference.commons.SystemResourceProbe
import com.correx.infrastructure.inference.commons.UnavailableProbe
import com.correx.core.inference.InferenceProvider
import com.correx.infrastructure.inference.llama.cpp.LlamaCppInferenceProvider
@@ -119,8 +120,10 @@ fun main() {
val inferenceRouter: InferenceRouter
// Non-null only on the managed path; backs manual model swap/pin from clients.
var modelSwapper: ManagedInferenceRouter? = null
// Live GPU/RAM gauge; NVIDIA-backed on the managed path when nvidia-smi is present, else unavailable.
var resourceProbe: ResourceProbe = UnavailableProbe
// Live GPU/RAM gauge. The managed pid lets the GPU probe also report the model's process RSS;
// it stays null on the static path. System RAM (overlaid below) is owner-agnostic, so the gauge
// renders on both paths regardless of whether correx manages the model process.
var managedPidSupplier: () -> Long? = { null }
if (correxConfig.models.isNotEmpty()) {
val settings = correxConfig.modelsSettings
@@ -146,13 +149,7 @@ fun main() {
val managedRouter = ManagedInferenceRouter(modelManager, descriptors, targetModelConfig.id)
inferenceRouter = managedRouter
modelSwapper = managedRouter
resourceProbe = when {
NvidiaResourceProbe.isAvailable() ->
NvidiaResourceProbe(pidSupplier = { modelManager.currentPid() })
AmdResourceProbe.isAvailable() ->
AmdResourceProbe(pidSupplier = { modelManager.currentPid() })
else -> UnavailableProbe
}
managedPidSupplier = { modelManager.currentPid() }
// Shutdown hook to kill the managed llama-server
Runtime.getRuntime().addShutdownHook(Thread {
log.info("Shutdown: unloading managed model '{}'", targetModelConfig.id)
@@ -169,6 +166,15 @@ fun main() {
inferenceRouter = DefaultInferenceRouter(infraRegistry, FirstAvailableRoutingStrategy())
}
// Build the resource gauge for whichever GPU vendor is present (pid-less GPU/VRAM reads work on
// both paths), then overlay owner-agnostic system RAM so the gauge always renders (F-001 gauge).
val gpuProbe: ResourceProbe = when {
NvidiaResourceProbe.isAvailable() -> NvidiaResourceProbe(pidSupplier = managedPidSupplier)
AmdResourceProbe.isAvailable() -> AmdResourceProbe(pidSupplier = managedPidSupplier)
else -> UnavailableProbe
}
val resourceProbe: ResourceProbe = SystemResourceProbe(gpuProbe)
val modelId = firstProvider.id.value
logModelInfo(modelId, infraRegistry)
apps/server/src/main/kotlin/com/correx/apps/server/bridge/DomainEventMapper.kt
+22
View File
@@ -8,14 +8,17 @@ import com.correx.apps.server.protocol.toDto
import com.correx.core.artifactstore.ArtifactStore
import com.correx.core.events.events.ApprovalDecisionResolvedEvent
import com.correx.core.events.events.ApprovalRequestedEvent
import com.correx.core.events.events.ArtifactContentStoredEvent
import com.correx.core.events.events.ArtifactCreatedEvent
import com.correx.core.events.events.ChatSessionStartedEvent
import com.correx.core.events.events.ChatTurnEvent
import com.correx.core.events.events.InferenceCompletedEvent
import com.correx.core.events.events.SessionWorkspaceBoundEvent
import com.correx.core.events.events.WorkflowStartedEvent
import com.correx.core.events.events.InferenceFailedEvent
import com.correx.core.events.events.InferenceStartedEvent
import com.correx.core.events.events.InferenceTimeoutEvent
import com.correx.core.events.events.RetryAttemptedEvent
import com.correx.core.events.events.ModelLoadedEvent
import com.correx.core.events.events.ModelUnloadedEvent
import com.correx.core.events.events.OrchestrationPausedEvent
@@ -149,6 +152,22 @@ suspend fun domainEventToServerMessage(
sequence = seq,
sessionSequence = sessionSequence,
)
is InferenceFailedEvent -> ServerMessage.InferenceFailed(
sessionId = p.sessionId,
stageId = p.stageId,
reason = p.reason,
sequence = seq,
sessionSequence = sessionSequence,
)
is RetryAttemptedEvent -> ServerMessage.RetryAttempted(
sessionId = p.sessionId,
stageId = p.stageId,
attemptNumber = p.attemptNumber,
maxAttempts = p.maxAttempts,
failureReason = p.failureReason,
sequence = seq,
sessionSequence = sessionSequence,
)
is ToolInvocationRequestedEvent -> ServerMessage.ToolStarted(
sessionId = p.sessionId,
@@ -232,6 +251,9 @@ suspend fun domainEventToServerMessage(
providerId = p.providerId.value,
loaded = false,
)
// Internal slot→CAS-hash bookkeeping (F-007 durable bridge); no operator-facing surface.
is ArtifactContentStoredEvent -> null
else -> {
log.debug(
"DomainEventMapper: unmapped payload type={} sessionId={} sequence={}",
apps/server/src/main/kotlin/com/correx/apps/server/protocol/ServerMessage.kt
+24
View File
@@ -191,6 +191,28 @@ sealed interface ServerMessage {
override val sessionSequence: Long,
) : ServerMessage, SessionMessage
@Serializable
@SerialName("inference.failed")
data class InferenceFailed(
val sessionId: SessionId,
val stageId: StageId,
val reason: String,
override val sequence: Long,
override val sessionSequence: Long,
) : ServerMessage, SessionMessage
@Serializable
@SerialName("inference.retry")
data class RetryAttempted(
val sessionId: SessionId,
val stageId: StageId,
val attemptNumber: Int,
val maxAttempts: Int,
val failureReason: String,
override val sequence: Long,
override val sessionSequence: Long,
) : ServerMessage, SessionMessage
// -- Tool execution --
@Serializable
@@ -336,6 +358,8 @@ sealed interface ServerMessage {
val gpuMemoryTotalMb: Long?,
val gpuUtilizationPct: Int?,
val processRssMb: Long?,
val systemRamUsedMb: Long? = null,
val systemRamTotalMb: Long? = null,
override val sequence: Long? = null,
override val sessionSequence: Long? = null,
) : ServerMessage, NonEventMessage
apps/server/src/main/kotlin/com/correx/apps/server/ws/GlobalStreamHandler.kt
+2
View File
@@ -61,6 +61,8 @@ private fun ResourceSnapshot.toResourceStatus(): ServerMessage.ResourceStatus =
gpuMemoryTotalMb = gpu?.memoryTotalMb,
gpuUtilizationPct = gpu?.utilizationPct,
processRssMb = processRssBytes?.let { it / BYTES_PER_MB },
systemRamUsedMb = systemRamUsedBytes?.let { it / BYTES_PER_MB },
systemRamTotalMb = systemRamTotalBytes?.let { it / BYTES_PER_MB },
)
class GlobalStreamHandler(private val module: ServerModule) {
apps/tui-go/internal/app/model.go
+2
View File
@@ -183,6 +183,8 @@ type Model struct {
gpuTotalMB *int64
gpuUtil *int
ramMB *int64
sysRamUsedMB *int64
sysRamTotalMB *int64
// diff / overlay
overlay OverlayKind
apps/tui-go/internal/app/server.go
+13
View File
@@ -3,6 +3,7 @@ package app
import (
"crypto/rand"
"encoding/hex"
"fmt"
"strings"
"time"
@@ -172,6 +173,16 @@ func (m *Model) applyServer(msg protocol.ServerMessage) {
s.Active = false
s.addEvent(nowMillis(), "InferenceTimedOut", msg.StageID)
}
case protocol.TypeInferenceFailed:
if s := m.session(msg.SessionID); s != nil {
s.Active = false
s.addEvent(nowMillis(), "InferenceFailed", msg.StageID+": "+msg.Reason)
}
case protocol.TypeInferenceRetry:
if s := m.session(msg.SessionID); s != nil {
s.addEvent(nowMillis(), "RetryAttempted",
fmt.Sprintf("%s (%d/%d): %s", msg.StageID, msg.AttemptNumber, msg.MaxAttempts, msg.FailureReason))
}
case protocol.TypeToolStarted:
if s := m.session(msg.SessionID); s != nil {
s.Active = true
@@ -274,6 +285,8 @@ func (m *Model) applyServer(msg protocol.ServerMessage) {
m.gpuTotalMB = msg.GpuMemoryTotalMb
m.gpuUtil = msg.GpuUtilizationPct
m.ramMB = msg.ProcessRssMb
m.sysRamUsedMB = msg.SystemRamUsedMb
m.sysRamTotalMB = msg.SystemRamTotalMb
case protocol.TypeArtifactList:
m.artifacts = msg.Artifacts
m.artifactsFor = msg.SessionID
apps/tui-go/internal/app/view.go
+4 -1
View File
@@ -117,8 +117,11 @@ func (m Model) gaugeText() string {
if m.gpuUtil != nil {
parts = append(parts, "GPU "+itoa(*m.gpuUtil)+"%")
}
if m.sysRamUsedMB != nil && m.sysRamTotalMB != nil {
parts = append(parts, "RAM "+itoa(int(*m.sysRamUsedMB))+"/"+itoa(int(*m.sysRamTotalMB))+"M")
}
if m.ramMB != nil {
parts = append(parts, "RAM "+itoa(int(*m.ramMB))+"M")
parts = append(parts, "proc "+itoa(int(*m.ramMB))+"M")
}
return strings.Join(parts, " ")
}
apps/tui-go/internal/protocol/protocol.go
+10
View File
@@ -26,6 +26,8 @@ const (
TypeInferenceStarted = "inference.started"
TypeInferenceDone = "inference.completed"
TypeInferenceTimeout = "inference.timed_out"
TypeInferenceFailed = "inference.failed"
TypeInferenceRetry = "inference.retry"
TypeToolStarted = "tool.started"
TypeToolCompleted = "tool.completed"
TypeToolFailed = "tool.failed"
@@ -77,6 +79,11 @@ type ServerMessage struct {
Outcome string `json:"outcome"` // approval.resolved: APPROVED | REJECTED | AUTO_APPROVED
WorkspaceRoot string `json:"workspaceRoot"` // session.workspace_bound: the bound cwd
// inference.retry
AttemptNumber int `json:"attemptNumber"`
MaxAttempts int `json:"maxAttempts"`
FailureReason string `json:"failureReason"`
SteeringEmitted bool `json:"steeringEmitted"`
OccurredAt int64 `json:"occurredAt"`
ElapsedMs int64 `json:"elapsedMs"`
@@ -99,6 +106,8 @@ type ServerMessage struct {
GpuMemoryTotalMb *int64 `json:"gpuMemoryTotalMb"`
GpuUtilizationPct *int `json:"gpuUtilizationPct"`
ProcessRssMb *int64 `json:"processRssMb"`
SystemRamUsedMb *int64 `json:"systemRamUsedMb"`
SystemRamTotalMb *int64 `json:"systemRamTotalMb"`
State *SessionStateDto `json:"state"`
RiskSummary *RiskSummaryDto `json:"riskSummary"`
@@ -209,6 +218,7 @@ func (m ServerMessage) IsEventBearing() bool {
TypeSessionCompleted, TypeSessionFailed,
TypeStageStarted, TypeStageCompleted, TypeStageFailed,
TypeInferenceStarted, TypeInferenceDone, TypeInferenceTimeout,
TypeInferenceFailed, TypeInferenceRetry,
TypeToolStarted, TypeToolCompleted, TypeToolFailed, TypeToolRejected,
TypeToolAssessed,
TypeApprovalRequired, TypeApprovalResolved, TypeWorkspaceBound, TypeArtifactCreated,