kami/correx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(qa): remote NIM provider + headless-QA robustness

Browse Source 
Enable autonomous QA through a remote OpenAI-compatible provider (NVIDIA NIM)
and harden the tool/approval path so unattended multi-stage runs complete.

- inference: add openai_compat provider (Bearer chat-completions for NIM/OpenAI),
  dispatched by provider type "nim"/"openai"; key via api_key/api_key_env.
- server: bind configured [server] host/port instead of a hardcoded 8080;
  POST /sessions accepts an optional `intent` (WS parity) for intent-driven workflows.
- kernel: thread the bound operator profile's approval_mode into per-tool gating so
  auto/yolo enable unattended approval (engine still consulted; policy/plane-2 BLOCK
  stays terminal); on a recoverable tool failure feed the tool's arg-schema back into
  context so the model self-corrects instead of repeating a malformed call.
- tools: split deletion out of file_write into a separate, explicitly-named file_delete
  tool — a model can no longer delete a file by getting a write-mode parameter wrong.
- server: add GET /metrics/tool-reliability — per-model tool-call validity from the
  event log (measurement groundwork for capability-aware routing).
- docs: update AGENTS.md across kernel, tools, server, inference.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
kami
2026-06-29 10:50:16 +00:00
parent d26f20c316
commit 238d353653
27 changed files with 924 additions and 268 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/server/AGENTS.md
+2 -1
View File
@@ -11,13 +11,14 @@ All sources under `apps/server/src/`.
## Local Contracts
### HTTP REST routes
- `GET/POST /sessions` — session browse and start
- `GET/POST /sessions` — session browse and start (`POST` accepts an optional `intent` brief, parity with the WS `StartSession`)
- `POST /sessions/{id}/resume` — resume a session after restart
- `GET/POST /tasks` — task listing and management
- `GET/POST /providers` — provider configuration
- `GET/POST /workflows` — workflow management
- `GET /health` — health report (probes: event-store, llama-server, disk watermark)
- `GET /stats` — metrics report (MetricsProjection)
- `GET /metrics/tool-reliability` — per-model tool-call validity across the event log (`ToolReliabilityInspectionService`); groundwork for capability-aware routing
### WebSocket protocol (`/ws`)
- **ServerMessage** (server → client): sealed hierarchy — `SessionMessage` (event-derived, carries `sequence` + `sessionSequence`) and `NonEventMessage` (control/infra). Variants include session lifecycle, approval requests, clarification requests, narration, proposed workflows, health/metrics pushes.
apps/server/build.gradle
+1
View File
@@ -36,6 +36,7 @@ dependencies {
implementation project(':infrastructure:persistence')
implementation project(':infrastructure:inference')
implementation project(':infrastructure:inference:llama_cpp')
implementation project(':infrastructure:inference:openai_compat')
implementation project(':infrastructure:inference:commons')
implementation project(':core:router')
implementation project(':core:tools')
apps/server/src/main/kotlin/com/correx/apps/server/Application.kt
+8
View File
@@ -1,6 +1,7 @@
package com.correx.apps.server
import com.correx.apps.server.health.HealthInspectionService
import com.correx.apps.server.metrics.ToolReliabilityInspectionService
import com.correx.apps.server.routes.providerRoutes
import com.correx.apps.server.routes.sessionRoutes
import com.correx.apps.server.routes.taskRoutes
@@ -38,6 +39,7 @@ fun Application.configureServer(module: ServerModule) {
val globalStreamHandler = GlobalStreamHandler(module)
val healthInspection = HealthInspectionService(module.eventStore)
val toolReliability = ToolReliabilityInspectionService(module.eventStore)
routing {
get("/health") {
@@ -51,6 +53,12 @@ fun Application.configureServer(module: ServerModule) {
call.respond(healthInspection.inspect())
}
// Per-model tool-call reliability across the whole event log (valid vs. failed tool calls
// per provider). Measurement groundwork for capability-aware routing of tool-heavy stages.
get("/metrics/tool-reliability") {
call.respond(toolReliability.inspect())
}
webSocket("/stream") {
globalStreamHandler.handle(this)
}
apps/server/src/main/kotlin/com/correx/apps/server/Main.kt
+49 -25
View File
@@ -84,7 +84,6 @@ import com.correx.infrastructure.inference.commons.ResourceProbe
import com.correx.infrastructure.inference.commons.SystemResourceProbe
import com.correx.infrastructure.inference.commons.UnavailableProbe
import com.correx.core.inference.InferenceProvider
import com.correx.infrastructure.inference.llama.cpp.LlamaCppInferenceProvider
import com.correx.infrastructure.tools.DispatchingToolExecutor
import com.correx.infrastructure.tools.task.TaskTools
import com.correx.infrastructure.tools.FileEditConfig
@@ -148,12 +147,9 @@ fun main() {
firstProvider = managedProvider
infraRegistry = InfrastructureModule.createProviderRegistry(listOf(managedProvider))
// Also register any static providers from [[providers]] (non-managed remotes)
correxConfig.providers.mapNotNull { providerConfig ->
when (providerConfig.type.lowercase()) {
"llamacpp" -> buildProviderFromConfig(providerConfig)
else -> { log.error("Unknown provider type: {}", providerConfig.type); null }
}
}.forEach { infraRegistry.register(it) }
correxConfig.providers
.mapNotNull { providerConfig -> buildProviderFromConfig(providerConfig) }
.forEach { infraRegistry.register(it) }
// Managed router owns per-stage model selection: it ensures the resolved model is resident
// before routing (stage.modelId > capability match > default), evicting the prior model.
val managedRouter = ManagedInferenceRouter(modelManager, descriptors, targetModelConfig.id)
@@ -555,7 +551,9 @@ fun main() {
module.start()
log.info("==============================")
embeddedServer(Netty, port = 8080) { configureServer(module) }.start(wait = true)
embeddedServer(Netty, port = correxConfig.server.port, host = correxConfig.server.host) {
configureServer(module)
}.start(wait = true)
}
fun renderProjectProfileText(profile: com.correx.core.sessions.BoundProjectProfile?): String? {
@@ -640,18 +638,10 @@ private fun resolveConfigRelativePath(raw: String, configDir: Path?): Path {
}
}
private fun buildProviders(config: CorrexConfig): List<LlamaCppInferenceProvider> {
private fun buildProviders(config: CorrexConfig): List<InferenceProvider> {
return if (config.providers.isNotEmpty()) {
log.info("Loading {} provider(s) from config", config.providers.size)
config.providers.mapNotNull { providerConfig ->
when (providerConfig.type.lowercase()) {
"llamacpp" -> buildProviderFromConfig(providerConfig)
else -> {
log.error("Unknown provider type: {}", providerConfig.type)
null
}
}
}
config.providers.mapNotNull { providerConfig -> buildProviderFromConfig(providerConfig) }
} else {
log.info("No providers in config; using env var fallback")
listOf(
@@ -664,14 +654,48 @@ private fun buildProviders(config: CorrexConfig): List<LlamaCppInferenceProvider
}
}
private fun buildProviderFromConfig(config: ProviderConfig): LlamaCppInferenceProvider {
// Single dispatch from a [[providers]] entry to a concrete InferenceProvider.
private fun buildProviderFromConfig(config: ProviderConfig): InferenceProvider? {
val capabilities = parseCapabilitiesFromConfig(config.capabilities)
return InfrastructureModule.createLlamaCppProvider(
modelId = config.modelId,
modelPath = config.modelPath,
baseUrl = config.url,
capabilities = capabilities,
)
return when (config.type.lowercase()) {
"llamacpp" -> InfrastructureModule.createLlamaCppProvider(
modelId = config.modelId,
modelPath = config.modelPath,
baseUrl = config.url,
capabilities = capabilities,
)
"nim", "openai" -> {
val apiKey = resolveApiKey(config)
if (apiKey.isBlank()) {
log.error(
"Provider '{}' (type {}) has no API key (set api_key or api_key_env); skipping",
config.id, config.type,
)
null
} else {
InfrastructureModule.createOpenAiCompatProvider(
modelId = config.modelId,
baseUrl = config.url,
apiKey = apiKey,
idPrefix = config.type.lowercase(),
capabilities = capabilities,
)
}
}
else -> {
log.error("Unknown provider type: {}", config.type)
null
}
}
}
// Prefer the literal api_key; otherwise read the named env var. Returns "" when neither resolves.
private fun resolveApiKey(config: ProviderConfig): String = when {
config.apiKey.isNotBlank() -> config.apiKey
config.apiKeyEnv.isNotBlank() -> System.getenv(config.apiKeyEnv).orEmpty()
else -> ""
}
private fun parseCapabilitiesFromConfig(capsMap: Map<String, Double>): Set<CapabilityScore> {
apps/server/src/main/kotlin/com/correx/apps/server/metrics/ToolReliabilityInspectionService.kt
+107
View File
@@ -0,0 +1,107 @@
package com.correx.apps.server.metrics
import com.correx.core.events.events.InferenceCompletedEvent
import com.correx.core.events.events.ToolExecutionCompletedEvent
import com.correx.core.events.events.ToolExecutionFailedEvent
import com.correx.core.events.stores.EventStore
import kotlinx.serialization.Serializable
private const val PERCENT = 100.0
private const val REASON_BUCKET_MAX = 80
private const val TOP_REASONS = 5
@Serializable
data class ReasonCount(val reason: String, val count: Long)
@Serializable
data class ToolReliabilityRow(
val toolName: String,
val calls: Long,
val failures: Long,
val validityPct: Double,
)
@Serializable
data class ProviderToolReliability(
val provider: String,
val calls: Long,
val failures: Long,
val validityPct: Double,
val byTool: List<ToolReliabilityRow>,
val topFailureReasons: List<ReasonCount>,
)
@Serializable
data class ToolReliabilityReport(val providers: List<ProviderToolReliability>)
/**
* Per-model tool-call reliability, derived by replaying the whole event log: for each provider, how
* often did the tool calls it produced actually execute vs. fail (a failed `ToolExecutionFailedEvent`
* is the deterministic signal of a malformed/invalid call — wrong args, missing required field, …).
*
* Tool events don't carry a providerId, so each tool outcome is attributed to the provider of the
* most recent `InferenceCompletedEvent` in the same session (one model drives a session's inference
* under the default routing). This is the measurement groundwork for capability-aware routing —
* steering tool-heavy stages to models with a high observed validity rate.
*/
class ToolReliabilityInspectionService(private val eventStore: EventStore) {
private class Agg {
var calls: Long = 0
var failures: Long = 0
val reasons: MutableMap<String, Long> = linkedMapOf()
}
fun inspect(): ToolReliabilityReport {
val sessionProvider = mutableMapOf<String, String>()
val byProvider = linkedMapOf<String, Agg>()
val byProviderTool = linkedMapOf<Pair<String, String>, Agg>()
fun record(provider: String, tool: String, failureReason: String?) {
val p = byProvider.getOrPut(provider) { Agg() }
val pt = byProviderTool.getOrPut(provider to tool) { Agg() }
p.calls++; pt.calls++
if (failureReason != null) {
p.failures++; pt.failures++
val bucket = failureReason.lineSequence().firstOrNull().orEmpty().take(REASON_BUCKET_MAX)
p.reasons[bucket] = (p.reasons[bucket] ?: 0) + 1
}
}
eventStore.allEvents().forEach { stored ->
when (val payload = stored.payload) {
is InferenceCompletedEvent -> sessionProvider[payload.sessionId.value] = payload.providerId.value
is ToolExecutionCompletedEvent ->
record(sessionProvider[payload.sessionId.value] ?: "unknown", payload.toolName, null)
is ToolExecutionFailedEvent ->
record(sessionProvider[payload.sessionId.value] ?: "unknown", payload.toolName, payload.reason)
else -> Unit
}
}
val providers = byProvider.entries
.sortedByDescending { it.value.calls }
.map { (provider, agg) ->
ProviderToolReliability(
provider = provider,
calls = agg.calls,
failures = agg.failures,
validityPct = validity(agg.calls, agg.failures),
byTool = byProviderTool.entries
.filter { it.key.first == provider }
.sortedByDescending { it.value.calls }
.map { (key, a) ->
ToolReliabilityRow(key.second, a.calls, a.failures, validity(a.calls, a.failures))
},
topFailureReasons = agg.reasons.entries
.sortedByDescending { it.value }
.take(TOP_REASONS)
.map { ReasonCount(it.key, it.value) },
)
}
return ToolReliabilityReport(providers)
}
private fun validity(calls: Long, failures: Long): Double =
if (calls == 0L) PERCENT else (calls - failures).toDouble() / calls * PERCENT
}
apps/server/src/main/kotlin/com/correx/apps/server/routes/SessionRoutes.kt
+11 -1
View File
@@ -9,6 +9,7 @@ import com.correx.apps.server.replay.ReplayInspectionService
import com.correx.apps.server.serialization.payloadDiscriminator
import com.correx.apps.server.ws.SessionStreamHandler
import com.correx.core.events.events.EventPayload
import com.correx.core.events.events.InitialIntentEvent
import com.correx.core.events.serialization.eventJson
import com.correx.core.events.types.SessionId
import com.correx.core.utils.TypeId
@@ -39,7 +40,13 @@ data class EventRow(
)
@Serializable
data class StartSessionRequest(val workflowId: String, val config: SessionConfigDto? = null)
data class StartSessionRequest(
val workflowId: String,
val config: SessionConfigDto? = null,
// Optional freeform brief for intent-driven workflows (REST parity with the WS StartSession).
// Seeded as an InitialIntentEvent before the run so it lands in the decision journal.
val intent: String? = null,
)
@Serializable
data class SessionSummaryResponse(
@@ -108,6 +115,9 @@ private fun Route.startSessionRoute(module: ServerModule) {
val graph = module.workflowRegistry.find(body.workflowId)
?: return@post call.respond(HttpStatusCode.BadRequest, "Unknown workflowId: ${body.workflowId}")
val sessionId: SessionId = TypeId(UUID.randomUUID().toString())
body.intent?.takeIf { it.isNotBlank() }?.let { intent ->
EventDispatcher(module.eventStore).emit(InitialIntentEvent(sessionId, intent), sessionId)
}
module.launchSessionRun(sessionId, graph)
call.respond(HttpStatusCode.Accepted, StartSessionResponse(sessionId.value))
}