diff --git a/apps/server/src/main/kotlin/com/correx/apps/server/Main.kt b/apps/server/src/main/kotlin/com/correx/apps/server/Main.kt index d61a367a..4566a94d 100644 --- a/apps/server/src/main/kotlin/com/correx/apps/server/Main.kt +++ b/apps/server/src/main/kotlin/com/correx/apps/server/Main.kt @@ -260,6 +260,9 @@ fun main() { com.correx.apps.server.tasks.EventStoreSessionFactRecorder(eventStore), com.correx.apps.server.tasks.EventStoreSessionWrites(eventStore), ) + // Retrieves full tool output the kernel spilled to CAS on truncation (ref shown in-context). + val toolOutputTool = com.correx.infrastructure.tools.ToolOutputTool(artifactStore) + val extraTools = taskTools + toolOutputTool val toolRegistry = InfrastructureModule.createToolRegistry( buildToolConfig( workspaceRoot, @@ -268,7 +271,7 @@ fun main() { toolsConfig, researchToolConfig, ), - extraTools = taskTools, + extraTools = extraTools, ) val toolExecutor = InfrastructureModule.createToolExecutor( registry = toolRegistry, @@ -301,7 +304,7 @@ fun main() { val wsToolRegistryProvider = WorkspaceToolRegistryProvider { workspace -> val wsRegistry = InfrastructureModule.createToolRegistry( buildToolConfigForWorkspace(workspace, shellAllowedExecutables, toolsConfig, researchToolConfig), - extraTools = taskTools, + extraTools = extraTools, ) val wsExecutor = DispatchingToolExecutor(wsRegistry) WorkspaceTools(registry = wsRegistry, executor = wsExecutor) diff --git a/core/events/src/main/kotlin/com/correx/core/events/events/ToolReceipt.kt b/core/events/src/main/kotlin/com/correx/core/events/events/ToolReceipt.kt index 35698795..42dfe1ad 100644 --- a/core/events/src/main/kotlin/com/correx/core/events/events/ToolReceipt.kt +++ b/core/events/src/main/kotlin/com/correx/core/events/events/ToolReceipt.kt @@ -19,4 +19,8 @@ data class ToolReceipt( val tier: Tier, val timestamp: Instant, val diff: String? = null, + // Artifact-store hash of the FULL tool output when it was truncated for model context (the + // outputSummary above is a bounded preview). Null when nothing was truncated. Lets an agent + // retrieve everything via the tool_output tool without bloating the event log with raw output. + val fullOutputHash: String? = null, ) diff --git a/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/SessionOrchestrator.kt b/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/SessionOrchestrator.kt index 40e796d7..0f881618 100644 --- a/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/SessionOrchestrator.kt +++ b/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/SessionOrchestrator.kt @@ -203,6 +203,31 @@ private const val SCOPE_PROPOSAL_TOOL = "propose_scope" private const val READ_BEFORE_WRITE_CODE = "READ_BEFORE_WRITE" private const val REFERENCE_EXISTS_CODE = "REFERENCE_EXISTS" private const val OUTPUT_SUMMARY_LIMIT = 500 + +// Global floor on tool-result text entering model context, applied beneath each tool's own +// outputCompressor. A tool without a compressor (or one whose output survives compression) can still +// flood a small model's window, so any Success body over this many chars is shown head+tail with a +// marker and the FULL raw output spilled to the artifact store for retrieval via tool_output. +// ponytail: fixed constants; lift to OrchestrationTuning if operators need to tune per-deployment. +private const val TOOL_RESULT_MAX_CHARS = 8_000 +private const val TOOL_RESULT_HEAD_LINES = 60 +private const val TOOL_RESULT_TAIL_LINES = 60 +private const val TOOL_OUTPUT_TOOL = "tool_output" + +/** + * Frame an over-cap tool output as `header` + head lines + a truncation marker (naming the + * [tool_output] ref that retrieves the full text) + tail lines. Head and tail are each char-capped + * so a single pathological long line can't defeat the line-count bound. Pure — the caller spills the + * full output and supplies its [ref]. + */ +internal fun frameTruncatedToolResult(header: String, compressed: String, ref: String): String { + val lines = compressed.split("\n") + val head = lines.take(TOOL_RESULT_HEAD_LINES).joinToString("\n").take(TOOL_RESULT_MAX_CHARS / 2) + val tail = lines.takeLast(TOOL_RESULT_TAIL_LINES).joinToString("\n").takeLast(TOOL_RESULT_MAX_CHARS / 2) + val marker = "… output truncated (${lines.size} lines); " + + "call $TOOL_OUTPUT_TOOL(ref=\"$ref\") for the full output …" + return "$header\n$head\n$marker\n$tail" +} // Read-on-demand doc catalog: the top-N docs (by repo-map recency score) surfaced as // `path — descriptor` so a stage learns which docs exist and can file_read one when relevant, // instead of docs being force-fed (or excluded outright). One line each, hard-capped, so it can @@ -1101,6 +1126,38 @@ abstract class SessionOrchestrator( "\nAccepted parameters for '${it.name}': ${it.parametersSchema}" }.orEmpty() + private data class RenderedToolResult(val content: String, val fullOutputHash: String?) + + /** + * Render a tool result into the consistently-framed, bounded text the model sees. Success output + * is run through the tool's [ToolOutputCompressor], then framed with a uniform `[tool exit=N]` + * header. If it still exceeds [TOOL_RESULT_MAX_CHARS] the FULL raw output is spilled to the + * artifact store (CAS — durable, its hash recorded on the ToolReceipt in the event log) and only a + * head+tail preview is shown, with a marker telling the model to call `tool_output(ref=…)` for the + * rest. Failures keep their `ERROR:`/`FATAL:` sentinels unchanged — the all-rejected loop breaker + * keys on those prefixes. + */ + private suspend fun renderToolResult(toolName: String, tool: Tool?, result: ToolResult): RenderedToolResult = + when (result) { + is ToolResult.Failure -> RenderedToolResult( + if (!result.recoverable) "FATAL: ${result.reason}" else "ERROR: ${result.reason}${toolArgsHint(tool)}", + null, + ) + is ToolResult.Success -> { + val compressed = tool?.outputCompressor?.compress(result.output, ToolOutputContext(result.exitCode)) + ?: result.output + val header = "[$toolName exit=${result.exitCode}]" + if (compressed.length <= TOOL_RESULT_MAX_CHARS) { + RenderedToolResult("$header\n$compressed", null) + } else { + // Spill the full RAW output (most complete) so retrieval returns everything, not the + // already-compressed preview the model saw. + val ref = artifactStore.put(result.output.toByteArray()).value + RenderedToolResult(frameTruncatedToolResult(header, compressed, ref), ref) + } + } + } + private suspend fun dispatchToolCalls( sessionId: SessionId, stageId: StageId, @@ -1410,6 +1467,9 @@ abstract class SessionOrchestrator( ?.let { request.copy(grantedPaths = grantedOutside + it.toString()) } ?: request val result = executor.execute(executedRequest) + // Frame + bound the result once; on truncation this spills the full output to CAS and + // returns its hash, which we record on the receipt so the event log points at the full text. + val rendered = renderToolResult(toolCall.function.name, tool, result) // Store ProcessResult artifact for every shell execution outcome for (slot in processResultSlots) { @@ -1452,7 +1512,7 @@ abstract class SessionOrchestrator( // artifact carries the diff as evidence of the change. recordToolExecution( sessionId, stageId, toolCall, invocationId, tier, result, - tool as? FileAffectingTool, request, fileWrittenSlots, + tool as? FileAffectingTool, request, fileWrittenSlots, rendered.fullOutputHash, ) val sourceId = toolCall.id ?: invocationId.value @@ -1466,22 +1526,13 @@ abstract class SessionOrchestrator( role = EntryRole.ASSISTANT, reasoning = toolCallReasoning, ) - val resultContent = when (result) { - is ToolResult.Success -> - tool?.outputCompressor?.compress(result.output, ToolOutputContext(result.exitCode)) - ?: result.output - is ToolResult.Failure -> { - if (!result.recoverable) "FATAL: ${result.reason}" - else "ERROR: ${result.reason}${toolArgsHint(tool)}" - } - } val resultEntry = ContextEntry( id = ContextEntryId(UUID.randomUUID().toString()), layer = ContextLayer.L2, sourceType = "toolResult", sourceId = sourceId, - content = resultContent, - tokenEstimate = estimateTokens(resultContent), + content = rendered.content, + tokenEstimate = estimateTokens(rendered.content), role = EntryRole.TOOL, ) val steeringEntry = approvalNote?.takeIf { it.isNotBlank() }?.let { @@ -2098,6 +2149,7 @@ abstract class SessionOrchestrator( fileTool: FileAffectingTool?, request: ToolRequest, fileWrittenSlots: List, + fullOutputHash: String? = null, ) { // Invariant #5: every tool side effect is captured. This is the single authoritative // ToolExecutionCompleted/Failed record and the source the read-before-write gate replays. @@ -2136,6 +2188,7 @@ abstract class SessionOrchestrator( tier = tier, timestamp = Clock.System.now(), diff = diff, + fullOutputHash = fullOutputHash, ), ), ) diff --git a/core/kernel/src/test/kotlin/com/correx/core/kernel/orchestration/FrameTruncatedToolResultTest.kt b/core/kernel/src/test/kotlin/com/correx/core/kernel/orchestration/FrameTruncatedToolResultTest.kt new file mode 100644 index 00000000..af1c182f --- /dev/null +++ b/core/kernel/src/test/kotlin/com/correx/core/kernel/orchestration/FrameTruncatedToolResultTest.kt @@ -0,0 +1,30 @@ +package com.correx.core.kernel.orchestration + +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class FrameTruncatedToolResultTest { + + @Test + fun `keeps header, head, tail and a marker naming the retrieval ref`() { + val body = (1..300).joinToString("\n") { "line-$it" } + val out = frameTruncatedToolResult("[grep exit=0]", body, "abc123") + + assertTrue(out.startsWith("[grep exit=0]"), out) + assertTrue(out.contains("line-1\n"), "head lines kept") + assertTrue(out.contains("line-300"), "tail lines kept") + assertTrue(out.contains("output truncated (300 lines)"), out) + assertTrue(out.contains("tool_output(ref=\"abc123\")"), out) + // Middle is dropped — a line well inside the elided span must be absent. + assertTrue(!out.contains("line-150"), "middle elided") + } + + @Test + fun `char-caps a single pathological long line so the bound is not defeated`() { + val giant = "x".repeat(1_000_000) + val out = frameTruncatedToolResult("[shell exit=0]", giant, "ref") + // One line means head==tail==giant; each is char-capped, so output can't be ~2MB. + assertTrue(out.length < 20_000, "length was ${out.length}") + assertTrue(out.contains("tool_output(ref=\"ref\")"), out) + } +} diff --git a/core/transitions/src/main/kotlin/com/correx/core/transitions/graph/StageConfig.kt b/core/transitions/src/main/kotlin/com/correx/core/transitions/graph/StageConfig.kt index a4bfef33..b178d1fd 100644 --- a/core/transitions/src/main/kotlin/com/correx/core/transitions/graph/StageConfig.kt +++ b/core/transitions/src/main/kotlin/com/correx/core/transitions/graph/StageConfig.kt @@ -66,6 +66,7 @@ data class StageConfig( companion object { /** Read-only tools every tool-granting stage may call regardless of its declared set. */ - val ALWAYS_AVAILABLE_READ_TOOLS: Set = setOf("file_read", "list_dir", "glob", "grep") + val ALWAYS_AVAILABLE_READ_TOOLS: Set = + setOf("file_read", "list_dir", "glob", "grep", "tool_output") } } diff --git a/infrastructure/tools/src/main/kotlin/com/correx/infrastructure/tools/ToolOutputTool.kt b/infrastructure/tools/src/main/kotlin/com/correx/infrastructure/tools/ToolOutputTool.kt new file mode 100644 index 00000000..918a00c6 --- /dev/null +++ b/infrastructure/tools/src/main/kotlin/com/correx/infrastructure/tools/ToolOutputTool.kt @@ -0,0 +1,68 @@ +package com.correx.infrastructure.tools + +import com.correx.core.approvals.Tier +import com.correx.core.artifactstore.ArtifactStore +import com.correx.core.events.events.ToolRequest +import com.correx.core.events.types.ArtifactId +import com.correx.core.tools.contract.Tool +import com.correx.core.tools.contract.ToolCapability +import com.correx.core.tools.contract.ToolExecutor +import com.correx.core.tools.contract.ToolResult +import com.correx.core.tools.contract.ValidationResult +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.buildJsonArray +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.put +import kotlinx.serialization.json.putJsonObject + +/** + * Retrieves the full text of an earlier tool result that was truncated for model context. When a + * Success output exceeds the context floor, the kernel spills the full raw output to the artifact + * store and shows a marker containing its `ref` (the content hash, also recorded on the + * ToolExecutionCompleted receipt in the event log). This tool resolves that `ref` back to the full + * content. Read-only (Tier T1, no filesystem capability — it reads the content-addressed store by a + * hash the model was explicitly handed, never arbitrary paths). + */ +class ToolOutputTool(private val artifactStore: ArtifactStore) : Tool, ToolExecutor { + + override val name: String = "tool_output" + override val description: String = + "Retrieve the FULL output of an earlier tool call that was truncated. Pass the ref shown in " + + "the truncation marker (e.g. tool_output(ref=\"\"))." + override val parametersSchema: JsonObject = buildJsonObject { + put("type", "object") + putJsonObject("properties") { + putJsonObject("ref") { + put("type", "string") + put("description", "The ref/hash from a '… output truncated …' marker.") + } + } + put("required", buildJsonArray { add(JsonPrimitive("ref")) }) + } + override val tier: Tier = Tier.T1 + override val requiredCapabilities: Set = emptySet() + + override fun validateRequest(request: ToolRequest): ValidationResult = + if ((request.parameters["ref"] as? String).isNullOrBlank()) { + ValidationResult.Invalid("tool_output requires a non-empty 'ref'") + } else { + ValidationResult.Valid + } + + override suspend fun execute(request: ToolRequest): ToolResult { + val ref = (request.parameters["ref"] as? String)?.takeIf { it.isNotBlank() } + ?: return ToolResult.Failure( + request.invocationId, + "tool_output requires a non-empty 'ref'", + recoverable = true, + ) + return artifactStore.get(ArtifactId(ref)) + ?.let { ToolResult.Success(request.invocationId, output = it.toString(Charsets.UTF_8)) } + ?: ToolResult.Failure( + request.invocationId, + "No stored output for ref '$ref' — it may have expired or the ref is wrong.", + recoverable = true, + ) + } +} diff --git a/infrastructure/tools/src/test/kotlin/com/correx/infrastructure/tools/ToolOutputToolTest.kt b/infrastructure/tools/src/test/kotlin/com/correx/infrastructure/tools/ToolOutputToolTest.kt new file mode 100644 index 00000000..d003dc16 --- /dev/null +++ b/infrastructure/tools/src/test/kotlin/com/correx/infrastructure/tools/ToolOutputToolTest.kt @@ -0,0 +1,52 @@ +package com.correx.infrastructure.tools + +import com.correx.core.artifactstore.ArtifactStore +import com.correx.core.events.events.ToolRequest +import com.correx.core.events.types.ArtifactId +import com.correx.core.events.types.SessionId +import com.correx.core.events.types.StageId +import com.correx.core.events.types.ToolInvocationId +import com.correx.core.tools.contract.ToolResult +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.util.* + +class ToolOutputToolTest { + + private class FakeStore(private val entries: Map) : ArtifactStore { + override suspend fun put(bytes: ByteArray): ArtifactId = error("unused") + override suspend fun get(id: ArtifactId): ByteArray? = entries[id.value] + override suspend fun flushBefore(commit: suspend () -> Unit) = commit() + } + + private fun request(ref: String?) = ToolRequest( + invocationId = ToolInvocationId(UUID.randomUUID().toString()), + sessionId = SessionId(UUID.randomUUID().toString()), + stageId = StageId(UUID.randomUUID().toString()), + toolName = "tool_output", + parameters = ref?.let { mapOf("ref" to it) } ?: emptyMap(), + ) + + @Test + fun `returns the full stored output for a known ref`(): Unit = runBlocking { + val tool = ToolOutputTool(FakeStore(mapOf("hash1" to "the full output".toByteArray()))) + val out = (tool.execute(request("hash1")) as ToolResult.Success).output + assertEquals("the full output", out) + } + + @Test + fun `unknown ref fails recoverably`(): Unit = runBlocking { + val tool = ToolOutputTool(FakeStore(emptyMap())) + val result = tool.execute(request("missing")) + assertTrue(result is ToolResult.Failure && result.recoverable, result.toString()) + } + + @Test + fun `missing ref fails recoverably`(): Unit = runBlocking { + val tool = ToolOutputTool(FakeStore(emptyMap())) + val result = tool.execute(request(null)) + assertTrue(result is ToolResult.Failure && result.recoverable, result.toString()) + } +}