Merge remote-tracking branch 'origin/feat/session-robustness-and-dox' into feat/session-robustness-and-dox

apps/server/AGENTS.md

@@ -11,13 +11,14 @@ All sources under `apps/server/src/`.
 ## Local Contracts
 
 ### HTTP REST routes
-- `GET/POST /sessions` — session browse and start
+- `GET/POST /sessions` — session browse and start (`POST` accepts an optional `intent` brief, parity with the WS `StartSession`)
 - `POST /sessions/{id}/resume` — resume a session after restart
 - `GET/POST /tasks` — task listing and management
 - `GET/POST /providers` — provider configuration
 - `GET/POST /workflows` — workflow management
 - `GET /health` — health report (probes: event-store, llama-server, disk watermark)
 - `GET /stats` — metrics report (MetricsProjection)
+- `GET /metrics/tool-reliability` — per-model tool-call validity across the event log (`ToolReliabilityInspectionService`); groundwork for capability-aware routing
 
 ### WebSocket protocol (`/ws`)
 - **ServerMessage** (server → client): sealed hierarchy — `SessionMessage` (event-derived, carries `sequence` + `sessionSequence`) and `NonEventMessage` (control/infra). Variants include session lifecycle, approval requests, clarification requests, narration, proposed workflows, health/metrics pushes.

apps/server/build.gradle

@@ -36,6 +36,7 @@ dependencies {
     implementation project(':infrastructure:persistence')
     implementation project(':infrastructure:inference')
     implementation project(':infrastructure:inference:llama_cpp')
+    implementation project(':infrastructure:inference:openai_compat')
     implementation project(':infrastructure:inference:commons')
     implementation project(':core:router')
     implementation project(':core:tools')

apps/server/src/main/kotlin/com/correx/apps/server/Application.kt

@@ -1,6 +1,7 @@
 package com.correx.apps.server
 
 import com.correx.apps.server.health.HealthInspectionService
+import com.correx.apps.server.metrics.ToolReliabilityInspectionService
 import com.correx.apps.server.routes.providerRoutes
 import com.correx.apps.server.routes.sessionRoutes
 import com.correx.apps.server.routes.taskRoutes
@@ -38,6 +39,7 @@ fun Application.configureServer(module: ServerModule) {
 
     val globalStreamHandler = GlobalStreamHandler(module)
     val healthInspection = HealthInspectionService(module.eventStore)
+    val toolReliability = ToolReliabilityInspectionService(module.eventStore)
 
     routing {
         get("/health") {
@@ -51,6 +53,12 @@ fun Application.configureServer(module: ServerModule) {
             call.respond(healthInspection.inspect())
         }
 
+        // Per-model tool-call reliability across the whole event log (valid vs. failed tool calls
+        // per provider). Measurement groundwork for capability-aware routing of tool-heavy stages.
+        get("/metrics/tool-reliability") {
+            call.respond(toolReliability.inspect())
+        }
+
         webSocket("/stream") {
             globalStreamHandler.handle(this)
         }

apps/server/src/main/kotlin/com/correx/apps/server/Main.kt

@@ -76,7 +76,7 @@ import com.correx.core.events.types.SessionId
 import com.correx.infrastructure.InfrastructureModule
 import com.correx.infrastructure.inference.DefaultProviderRegistry
 import com.correx.infrastructure.workflow.ExecutionPlanCompiler
-import com.correx.infrastructure.inference.FirstAvailableRoutingStrategy
+import com.correx.infrastructure.inference.CapabilityAwareRoutingStrategy
 import com.correx.infrastructure.inference.commons.ManagedInferenceRouter
 import com.correx.infrastructure.inference.commons.AmdResourceProbe
 import com.correx.infrastructure.inference.commons.NvidiaResourceProbe
@@ -84,7 +84,6 @@ import com.correx.infrastructure.inference.commons.ResourceProbe
 import com.correx.infrastructure.inference.commons.SystemResourceProbe
 import com.correx.infrastructure.inference.commons.UnavailableProbe
 import com.correx.core.inference.InferenceProvider
-import com.correx.infrastructure.inference.llama.cpp.LlamaCppInferenceProvider
 import com.correx.infrastructure.tools.DispatchingToolExecutor
 import com.correx.infrastructure.tools.task.TaskTools
 import com.correx.infrastructure.tools.FileEditConfig
@@ -148,12 +147,9 @@ fun main() {
         firstProvider = managedProvider
         infraRegistry = InfrastructureModule.createProviderRegistry(listOf(managedProvider))
         // Also register any static providers from [[providers]] (non-managed remotes)
-        correxConfig.providers.mapNotNull { providerConfig ->
-            when (providerConfig.type.lowercase()) {
-                "llamacpp" -> buildProviderFromConfig(providerConfig)
-                else -> { log.error("Unknown provider type: {}", providerConfig.type); null }
-            }
-        }.forEach { infraRegistry.register(it) }
+        correxConfig.providers
+            .mapNotNull { providerConfig -> buildProviderFromConfig(providerConfig) }
+            .forEach { infraRegistry.register(it) }
         // Managed router owns per-stage model selection: it ensures the resolved model is resident
         // before routing (stage.modelId > capability match > default), evicting the prior model.
         val managedRouter = ManagedInferenceRouter(modelManager, descriptors, targetModelConfig.id)
@@ -174,7 +170,7 @@ fun main() {
         require(staticProviders.isNotEmpty()) { "At least one provider must be configured" }
         firstProvider = staticProviders.first()
         infraRegistry = InfrastructureModule.createProviderRegistry(staticProviders)
-        inferenceRouter = DefaultInferenceRouter(infraRegistry, FirstAvailableRoutingStrategy())
+        inferenceRouter = DefaultInferenceRouter(infraRegistry, CapabilityAwareRoutingStrategy())
         llamaBaseUrl = correxConfig.providers.firstOrNull()?.url
             ?: System.getenv("CORREX_LLAMA_URL")
             ?: "http://127.0.0.1:10000"
@@ -566,7 +562,9 @@ fun main() {
     module.start()
     log.info("==============================")
 
-    embeddedServer(Netty, port = 8080) { configureServer(module) }.start(wait = true)
+    embeddedServer(Netty, port = correxConfig.server.port, host = correxConfig.server.host) {
+        configureServer(module)
+    }.start(wait = true)
 }
 
 fun renderProjectProfileText(profile: com.correx.core.sessions.BoundProjectProfile?): String? {
@@ -651,18 +649,10 @@ private fun resolveConfigRelativePath(raw: String, configDir: Path?): Path {
     }
 }
 
-private fun buildProviders(config: CorrexConfig): List<LlamaCppInferenceProvider> {
+private fun buildProviders(config: CorrexConfig): List<InferenceProvider> {
     return if (config.providers.isNotEmpty()) {
         log.info("Loading {} provider(s) from config", config.providers.size)
-        config.providers.mapNotNull { providerConfig ->
-            when (providerConfig.type.lowercase()) {
-                "llamacpp" -> buildProviderFromConfig(providerConfig)
-                else -> {
-                    log.error("Unknown provider type: {}", providerConfig.type)
-                    null
-                }
-            }
-        }
+        config.providers.mapNotNull { providerConfig -> buildProviderFromConfig(providerConfig) }
     } else {
         log.info("No providers in config; using env var fallback")
         listOf(
@@ -675,14 +665,48 @@ private fun buildProviders(config: CorrexConfig): List<LlamaCppInferenceProvider
     }
 }
 
-private fun buildProviderFromConfig(config: ProviderConfig): LlamaCppInferenceProvider {
+// Single dispatch from a [[providers]] entry to a concrete InferenceProvider.
+private fun buildProviderFromConfig(config: ProviderConfig): InferenceProvider? {
     val capabilities = parseCapabilitiesFromConfig(config.capabilities)
-    return InfrastructureModule.createLlamaCppProvider(
-        modelId = config.modelId,
-        modelPath = config.modelPath,
-        baseUrl = config.url,
-        capabilities = capabilities,
-    )
+    return when (config.type.lowercase()) {
+        "llamacpp" -> InfrastructureModule.createLlamaCppProvider(
+            modelId = config.modelId,
+            modelPath = config.modelPath,
+            baseUrl = config.url,
+            capabilities = capabilities,
+        )
+
+        "nim", "openai" -> {
+            val apiKey = resolveApiKey(config)
+            if (apiKey.isBlank()) {
+                log.error(
+                    "Provider '{}' (type {}) has no API key (set api_key or api_key_env); skipping",
+                    config.id, config.type,
+                )
+                null
+            } else {
+                InfrastructureModule.createOpenAiCompatProvider(
+                    modelId = config.modelId,
+                    baseUrl = config.url,
+                    apiKey = apiKey,
+                    idPrefix = config.type.lowercase(),
+                    capabilities = capabilities,
+                )
+            }
+        }
+
+        else -> {
+            log.error("Unknown provider type: {}", config.type)
+            null
+        }
+    }
+}
+
+// Prefer the literal api_key; otherwise read the named env var. Returns "" when neither resolves.
+private fun resolveApiKey(config: ProviderConfig): String = when {
+    config.apiKey.isNotBlank() -> config.apiKey
+    config.apiKeyEnv.isNotBlank() -> System.getenv(config.apiKeyEnv).orEmpty()
+    else -> ""
 }
 
 private fun parseCapabilitiesFromConfig(capsMap: Map<String, Double>): Set<CapabilityScore> {
@@ -740,6 +764,7 @@ private fun buildToolConfig(
         fileRead = FileReadConfig(
             enabled = toolsConfig.fileReadEnabled,
             allowedPaths = allowed,
+            workingDir = workingDir,
         ),
         fileWrite = FileWriteConfig(
             allowedPaths = allowed,

apps/server/src/main/kotlin/com/correx/apps/server/metrics/ToolReliabilityInspectionService.kt

@@ -0,0 +1,107 @@
+package com.correx.apps.server.metrics
+
+import com.correx.core.events.events.InferenceCompletedEvent
+import com.correx.core.events.events.ToolExecutionCompletedEvent
+import com.correx.core.events.events.ToolExecutionFailedEvent
+import com.correx.core.events.stores.EventStore
+import kotlinx.serialization.Serializable
+
+private const val PERCENT = 100.0
+private const val REASON_BUCKET_MAX = 80
+private const val TOP_REASONS = 5
+
+@Serializable
+data class ReasonCount(val reason: String, val count: Long)
+
+@Serializable
+data class ToolReliabilityRow(
+    val toolName: String,
+    val calls: Long,
+    val failures: Long,
+    val validityPct: Double,
+)
+
+@Serializable
+data class ProviderToolReliability(
+    val provider: String,
+    val calls: Long,
+    val failures: Long,
+    val validityPct: Double,
+    val byTool: List<ToolReliabilityRow>,
+    val topFailureReasons: List<ReasonCount>,
+)
+
+@Serializable
+data class ToolReliabilityReport(val providers: List<ProviderToolReliability>)
+
+/**
+ * Per-model tool-call reliability, derived by replaying the whole event log: for each provider, how
+ * often did the tool calls it produced actually execute vs. fail (a failed `ToolExecutionFailedEvent`
+ * is the deterministic signal of a malformed/invalid call — wrong args, missing required field, …).
+ *
+ * Tool events don't carry a providerId, so each tool outcome is attributed to the provider of the
+ * most recent `InferenceCompletedEvent` in the same session (one model drives a session's inference
+ * under the default routing). This is the measurement groundwork for capability-aware routing —
+ * steering tool-heavy stages to models with a high observed validity rate.
+ */
+class ToolReliabilityInspectionService(private val eventStore: EventStore) {
+
+    private class Agg {
+        var calls: Long = 0
+        var failures: Long = 0
+        val reasons: MutableMap<String, Long> = linkedMapOf()
+    }
+
+    fun inspect(): ToolReliabilityReport {
+        val sessionProvider = mutableMapOf<String, String>()
+        val byProvider = linkedMapOf<String, Agg>()
+        val byProviderTool = linkedMapOf<Pair<String, String>, Agg>()
+
+        fun record(provider: String, tool: String, failureReason: String?) {
+            val p = byProvider.getOrPut(provider) { Agg() }
+            val pt = byProviderTool.getOrPut(provider to tool) { Agg() }
+            p.calls++; pt.calls++
+            if (failureReason != null) {
+                p.failures++; pt.failures++
+                val bucket = failureReason.lineSequence().firstOrNull().orEmpty().take(REASON_BUCKET_MAX)
+                p.reasons[bucket] = (p.reasons[bucket] ?: 0) + 1
+            }
+        }
+
+        eventStore.allEvents().forEach { stored ->
+            when (val payload = stored.payload) {
+                is InferenceCompletedEvent -> sessionProvider[payload.sessionId.value] = payload.providerId.value
+                is ToolExecutionCompletedEvent ->
+                    record(sessionProvider[payload.sessionId.value] ?: "unknown", payload.toolName, null)
+                is ToolExecutionFailedEvent ->
+                    record(sessionProvider[payload.sessionId.value] ?: "unknown", payload.toolName, payload.reason)
+                else -> Unit
+            }
+        }
+
+        val providers = byProvider.entries
+            .sortedByDescending { it.value.calls }
+            .map { (provider, agg) ->
+                ProviderToolReliability(
+                    provider = provider,
+                    calls = agg.calls,
+                    failures = agg.failures,
+                    validityPct = validity(agg.calls, agg.failures),
+                    byTool = byProviderTool.entries
+                        .filter { it.key.first == provider }
+                        .sortedByDescending { it.value.calls }
+                        .map { (key, a) ->
+                            ToolReliabilityRow(key.second, a.calls, a.failures, validity(a.calls, a.failures))
+                        },
+                    topFailureReasons = agg.reasons.entries
+                        .sortedByDescending { it.value }
+                        .take(TOP_REASONS)
+                        .map { ReasonCount(it.key, it.value) },
+                )
+            }
+        return ToolReliabilityReport(providers)
+    }
+
+    private fun validity(calls: Long, failures: Long): Double =
+        if (calls == 0L) PERCENT else (calls - failures).toDouble() / calls * PERCENT
+}

apps/server/src/main/kotlin/com/correx/apps/server/routes/SessionRoutes.kt

@@ -9,6 +9,7 @@ import com.correx.apps.server.replay.ReplayInspectionService
 import com.correx.apps.server.serialization.payloadDiscriminator
 import com.correx.apps.server.ws.SessionStreamHandler
 import com.correx.core.events.events.EventPayload
+import com.correx.core.events.events.InitialIntentEvent
 import com.correx.core.events.serialization.eventJson
 import com.correx.core.events.types.SessionId
 import com.correx.core.utils.TypeId
@@ -39,7 +40,13 @@ data class EventRow(
 )
 
 @Serializable
-data class StartSessionRequest(val workflowId: String, val config: SessionConfigDto? = null)
+data class StartSessionRequest(
+    val workflowId: String,
+    val config: SessionConfigDto? = null,
+    // Optional freeform brief for intent-driven workflows (REST parity with the WS StartSession).
+    // Seeded as an InitialIntentEvent before the run so it lands in the decision journal.
+    val intent: String? = null,
+)
 
 @Serializable
 data class SessionSummaryResponse(
@@ -108,6 +115,9 @@ private fun Route.startSessionRoute(module: ServerModule) {
         val graph = module.workflowRegistry.find(body.workflowId)
             ?: return@post call.respond(HttpStatusCode.BadRequest, "Unknown workflowId: ${body.workflowId}")
         val sessionId: SessionId = TypeId(UUID.randomUUID().toString())
+        body.intent?.takeIf { it.isNotBlank() }?.let { intent ->
+            EventDispatcher(module.eventStore).emit(InitialIntentEvent(sessionId, intent), sessionId)
+        }
         module.launchSessionRun(sessionId, graph)
         call.respond(HttpStatusCode.Accepted, StartSessionResponse(sessionId.value))
     }