feat(context): reasoning-thread continuity, L3 doc filtering, and gate hardening

Threads reasoning_content across inference calls and journal renders, filters
markdown noise out of L3 repo-knowledge retrieval, adds PlanLinter H3 checks,
and tightens filesystem tool output/dir-listing behavior surfaced by prior
live QA (see project_readloop_campaign memory).
This commit is contained in:
2026-07-10 11:13:43 +04:00
parent f51a8dada4
commit 3d5e05c1fb
32 changed files with 742 additions and 85 deletions
@@ -94,6 +94,10 @@ data class ProjectConfig(
companion object {
val DEFAULT_IGNORES: List<String> = listOf(
".git", "node_modules", "build", "target", ".gradle", "dist", ".idea", ".opencode",
// Role-prompt templates: loaded by workflow config as LLM system prompts, not project
// docs — indexing them lets a stage `file_read` another role's prompt as if it were
// documentation.
"examples/workflows/prompts",
)
}
}
@@ -26,6 +26,9 @@ import com.correx.core.events.types.ContextEntryId
import com.correx.core.events.types.ContextPackId
import com.correx.core.events.types.SessionId
import com.correx.core.events.types.StageId
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.jsonObject
import kotlinx.serialization.json.jsonPrimitive
/**
* Runs the compression pipeline (docs/plans/correx-compression-pipeline.md) in fixed pipeline
@@ -80,13 +83,21 @@ class DefaultContextPackBuilder(
// reorder freely and PromptRenderer can restore true turn order from the ordinal.
val stamped = entries.mapIndexed { index, entry -> entry.copy(ordinal = index) }
// A stalled stage re-issuing the exact same call otherwise burns budget on N identical
// (tool, arguments) pairs that all say the same thing — collapse those to the most recent
// occurrence. Fingerprint is (name, arguments) verbatim, so a call whose arguments genuinely
// differ (different start_line/end_line, different path) is a distinct read and untouched.
// Must run before FORMAT_COMPRESS below: that stage reencodes assistantToolCall content from
// raw ToolCallRequest JSON into a dotted-line format the fingerprint parser can't read.
val deduped = dedupeRepeatedToolCalls(stamped)
// Stage 1 (FORMAT_COMPRESS): lossless json→dotted-line compaction of structured entries.
val formatted = if (policy.enabled(CompressionStage.FORMAT_COMPRESS)) {
stamped.map { entry ->
deduped.map { entry ->
if (classifier.classify(entry) == ContextClass.STRUCTURED) reencode(entry, formatCompressor.compress(entry.content))
else entry
}
} else stamped
} else deduped
// Stage 3 (TOKEN_PRUNE): prune freeform prose, preserving protected spans. When TIER_SPLIT
// is on, the newest TIER0_TURNS freeform turns are left full-fidelity (tier 0).
@@ -151,7 +162,7 @@ class DefaultContextPackBuilder(
result
}
val retained = (pinned + compressed).sortedBy { it.ordinal }
val retained = reconcileToolPairs((pinned + compressed).sortedBy { it.ordinal })
val layers = retained.groupBy { it.layer }
val droppedCount = entries.size - retained.count { it.sourceType != "factSheet" }
val truncatedLayers = if (droppedCount > 0) {
@@ -235,6 +246,65 @@ class DefaultContextPackBuilder(
return rest + ranked
}
/**
* Collapses assistantToolCall/toolResult pairs that repeat an earlier call's exact (name,
* arguments) — keeping only the most recent occurrence — so a stalled stage repeating the same
* read doesn't show the model N copies of an identical question-and-answer. Content is the
* `ToolCallRequest` JSON built in SessionOrchestrator; parsed generically here (not via the
* `core:inference` type) to avoid a cross-core dependency. Calls whose arguments differ (e.g.
* different start_line/end_line, a different path) fingerprint differently and are untouched —
* that's genuine incremental progress, not repetition.
*/
private fun dedupeRepeatedToolCalls(entries: List<ContextEntry>): List<ContextEntry> {
val calls = entries.filter { it.sourceType == "assistantToolCall" }
val callsById = calls.associateBy { it.sourceId }
val fingerprints = calls.mapNotNull { entry ->
toolCallFingerprint(entry.content)?.let { entry.sourceId to it }
}
val dropIds = fingerprints.groupBy({ it.second }, { it.first })
.values
.filter { it.size > 1 }
.flatMapTo(mutableSetOf()) { sourceIds ->
val keepId = sourceIds.maxWith(compareBy(
{ id: String -> if (callsById.getValue(id).reasoning != null) 1 else 0 },
{ id: String -> callsById.getValue(id).ordinal }
))
sourceIds.filterNot { it == keepId }
}
if (dropIds.isEmpty()) return entries
return entries.filterNot {
(it.sourceType == "assistantToolCall" || it.sourceType == "toolResult") && it.sourceId in dropIds
}
}
private fun toolCallFingerprint(content: String): String? = runCatching {
val obj = Json.parseToJsonElement(content).jsonObject
val fn = obj["function"]?.jsonObject ?: return null
val name = fn["name"]?.jsonPrimitive?.content ?: return null
val arguments = fn["arguments"]?.jsonPrimitive?.content ?: ""
"$name:$arguments"
}.getOrNull()
/**
* A completed tool call is represented as two independently-IDed, independently-compressed
* entries — assistantToolCall + toolResult — linked only by [ContextEntry.sourceId]. Because
* `compressed` groups and trims by sourceType (see the `grouped` fold above), one side of a
* pair can survive its group's budget while the other is trimmed away. An orphaned
* assistantToolCall with no matching result renders as "I called this tool" with nothing after
* it — indistinguishable, to the model, from the call silently failing — which drove a live
* read-loop repro (2026-07-08: discovery stage re-issued the same file_read repeatedly once its
* result vanished from two consecutive rounds). Drop whichever side lost its pair so every
* tool-call entry that survives has its result, and vice versa.
*/
private fun reconcileToolPairs(entries: List<ContextEntry>): List<ContextEntry> {
val callIds = entries.filter { it.sourceType == "assistantToolCall" }.mapTo(mutableSetOf()) { it.sourceId }
val resultIds = entries.filter { it.sourceType == "toolResult" }.mapTo(mutableSetOf()) { it.sourceId }
return entries.filterNot {
(it.sourceType == "assistantToolCall" && it.sourceId !in resultIds) ||
(it.sourceType == "toolResult" && it.sourceId !in callIds)
}
}
private fun reencode(entry: ContextEntry, content: String): ContextEntry =
if (content == entry.content) entry else entry.copy(content = content, tokenEstimate = estimateTokens(content))
@@ -32,4 +32,10 @@ data class ContextEntry(
// Default OPTIONAL keeps prior behavior for callers that don't stamp buckets (router chat);
// the orchestrator stamps REQUIRED on task/constraint/input/failure entries.
val bucket: ContextBucket = ContextBucket.OPTIONAL,
// Chain-of-thought that preceded this ASSISTANT turn (assistantToolCall entries only), as
// captured on InferenceResponse.reasoning. Carried as a side channel, not JSON-encoded into
// `content`, so FORMAT_COMPRESS's dotted-line flatten of the tool-call JSON never touches it.
// Threaded back into the request as ChatMessage.reasoning so models that expect their own
// prior reasoning ahead of a tool-call turn (e.g. Gemma) don't degenerate into re-deriving it.
val reasoning: String? = null,
)
@@ -25,3 +25,18 @@ data class RepoKnowledgeRetrievedEvent(
val query: String,
val hits: List<RepoKnowledgeHit>,
) : EventPayload
/**
* Candidates that scored above the L3 index's own top-k cutoff but below the retriever's
* minimum-similarity floor, so were never rendered to the agent as "relevant." Without this,
* a hit below the floor is indistinguishable from one that was never a candidate — recorded
* per invariant #9 since the embedding-similarity score is itself an environment observation.
*/
@Serializable
@SerialName("RepoKnowledgeHitsFiltered")
data class RepoKnowledgeHitsFilteredEvent(
val sessionId: SessionId,
val query: String,
val threshold: Float,
val dropped: List<RepoKnowledgeHit>,
) : EventPayload
@@ -57,6 +57,7 @@ import com.correx.core.events.events.ModelUnloadedEvent
import com.correx.core.events.events.OrchestrationPausedEvent
import com.correx.core.events.events.OrchestrationResumedEvent
import com.correx.core.events.events.RefinementIterationEvent
import com.correx.core.events.events.RepoKnowledgeHitsFilteredEvent
import com.correx.core.events.events.RepoKnowledgeRetrievedEvent
import com.correx.core.events.events.RepoMapComputedEvent
import com.correx.core.events.events.RetryAttemptedEvent
@@ -154,6 +155,7 @@ val eventModule = SerializersModule {
subclass(RepoMapComputedEvent::class)
subclass(WorkspaceStateObservedEvent::class)
subclass(RepoKnowledgeRetrievedEvent::class)
subclass(RepoKnowledgeHitsFilteredEvent::class)
subclass(BriefGroundingCheckedEvent::class)
subclass(BriefEchoMismatchEvent::class)
subclass(StaticAnalysisCompletedEvent::class)
@@ -10,6 +10,14 @@ import kotlinx.serialization.Serializable
data class ChatMessage(
val role: String,
val content: String,
// Carries ContextEntry.reasoning forward for assistant tool-call turns — see that field's
// doc for why this exists (Gemma-family looping when its own prior reasoning is missing).
val reasoning: String? = null,
// Optional tool-call-id that links a role:tool response to the assistant's tool_calls entry
// (sourceId in the event store). Required by the Gemma4 Jinja template to render tool
// response blocks with the correct function name; without it every tool response is
// rendered as "unknown" and the thought/reasoning channel logic can break.
val toolCallId: String? = null,
)
object PromptRenderer {
@@ -61,5 +69,7 @@ object PromptRenderer {
EntryRole.USER -> "user"
},
content = content,
reasoning = reasoning,
toolCallId = if (role == EntryRole.TOOL) sourceId else null,
)
}
@@ -7,7 +7,7 @@ import com.correx.core.journal.model.DecisionRecord
class DecisionJournalRenderer(private val keepLast: Int = 40) {
fun render(state: DecisionJournalState, summaryText: String? = null): String {
if (state.records.isEmpty() && summaryText == null) return ""
val visible = foldResolvedRetries(state.records)
val visible = foldResolvedRetries(state.records).filter { it.kind in AGENT_RELEVANT_KINDS }
return buildString {
appendLine("## Decision history (chronological — ground truth)")
if (summaryText != null) {
@@ -16,7 +16,7 @@ class DecisionJournalRenderer(private val keepLast: Int = 40) {
appendLine("+${state.lowSalienceOmittedCount} routine transitions/retries omitted")
}
}
visible.takeLast(keepLast).forEach { appendLine("- [${it.kind}] ${it.summary}") }
visible.takeLast(keepLast).forEach { appendLine("- ${it.summary}") }
}.trimEnd()
}
@@ -46,4 +46,15 @@ class DecisionJournalRenderer(private val keepLast: Int = 40) {
record.copy(summary = "$stage: $count retr${if (count == 1) "y" else "ies"}, resolved — stage completed")
}
}
companion object {
/**
* What actually shapes an agent's next decision: the goal, human steering, and failures
* worth not repeating. APPROVAL/ARTIFACT/TRANSITION are bookkeeping for the audit trail —
* agents don't act on "Approval APPROVED (tier T2)" or "Advanced x → y", so they're
* dropped from the render (still present in [DecisionJournalState.records] for the journal).
*/
private val AGENT_RELEVANT_KINDS =
setOf(DecisionKind.INTENT, DecisionKind.STEERING, DecisionKind.PREEMPT, DecisionKind.FAILURE, DecisionKind.RETRY)
}
}
@@ -27,11 +27,28 @@ class DecisionJournalRendererTest {
records = listOf(record(1), record(2)),
)
val result = renderer.render(state)
assertTrue(result.contains("- [INTENT] summary-1"))
assertTrue(result.contains("- [INTENT] summary-2"))
assertTrue(result.contains("- summary-1"))
assertTrue(result.contains("- summary-2"))
assertFalse(result.contains("omitted"))
}
@Test
fun `approval, artifact and transition records are excluded from the render`() {
val state = DecisionJournalState(
records = listOf(
record(1, DecisionKind.APPROVAL, "Approval APPROVED (tier T2)"),
record(2, DecisionKind.ARTIFACT, "Validated artifact x at stage y"),
record(3, DecisionKind.TRANSITION, "Advanced a → b (t1)").copy(stageId = "b", fromStageId = "a"),
record(4, DecisionKind.FAILURE, "Stage b failed: boom"),
),
)
val result = renderer.render(state)
assertFalse(result.contains("Approval APPROVED"))
assertFalse(result.contains("Validated artifact"))
assertFalse(result.contains("Advanced a"))
assertTrue(result.contains("- Stage b failed: boom"))
}
@Test
fun `summaryText appears before records`() {
val state = DecisionJournalState(
@@ -42,7 +59,7 @@ class DecisionJournalRendererTest {
val lines = result.lines()
val headerIdx = lines.indexOfFirst { it.startsWith("## Decision history") }
val summaryIdx = lines.indexOfFirst { it == "Compaction summary here" }
val recordIdx = lines.indexOfFirst { it.startsWith("- [INTENT]") }
val recordIdx = lines.indexOfFirst { it.startsWith("- summary") }
assertTrue(headerIdx >= 0)
assertTrue(summaryIdx > headerIdx)
assertTrue(recordIdx > summaryIdx)
@@ -75,7 +92,7 @@ class DecisionJournalRendererTest {
assertTrue(result.contains("## Decision history"))
assertTrue(result.contains("Compacted summary"))
assertTrue(result.contains("+2 routine transitions/retries omitted"))
assertFalse(result.contains("- ["))
assertFalse(result.contains("- "))
}
@Test
@@ -90,8 +107,8 @@ class DecisionJournalRendererTest {
)
val result = renderer.render(state)
assertFalse(result.contains("did not produce"), "resolved retry detail must not render")
assertTrue(result.contains("- [RETRY] scaffold: 2 retries, resolved — stage completed"))
assertTrue(result.contains("- [TRANSITION] Advanced scaffold → hook"))
assertTrue(result.contains("- scaffold: 2 retries, resolved — stage completed"))
assertFalse(result.contains("Advanced scaffold"), "TRANSITION records are audit-only, not agent-relevant")
}
@Test
@@ -102,7 +119,7 @@ class DecisionJournalRendererTest {
),
)
val result = renderer.render(state)
assertTrue(result.contains("- [RETRY] Retry 1/3 of hook: did not produce"))
assertTrue(result.contains("- Retry 1/3 of hook: did not produce"))
}
@Test
@@ -115,6 +132,6 @@ class DecisionJournalRendererTest {
),
)
val result = renderer.render(state)
assertTrue(result.contains("- [RETRY] Retry 1/3 of scaffold: regression"))
assertTrue(result.contains("- Retry 1/3 of scaffold: regression"))
}
}
@@ -62,6 +62,7 @@ import com.correx.core.events.events.EventMetadata
import com.correx.core.events.events.EventPayload
import com.correx.core.events.events.InferenceCompletedEvent
import com.correx.core.events.events.RepoKnowledgeHit
import com.correx.core.events.events.InitialIntentEvent
import com.correx.core.events.events.RepoKnowledgeRetrievedEvent
import com.correx.core.events.events.RepoMapComputedEvent
import com.correx.core.events.events.InferenceFailedEvent
@@ -582,6 +583,11 @@ abstract class SessionOrchestrator(
var toolRounds = 0
var consecutiveReadOnlyRounds = 0
var consecutiveRejectedRounds = 0
// Fingerprints ("tool:arguments") of every read-only call made during the current
// no-write streak. A round only counts against the read-loop breaker if it repeats calls
// already seen — a stage reading N distinct files before writing is legitimate context
// gathering, not a loop, and shouldn't trip the same counter as re-reading the same file.
val seenReadFingerprints = mutableSetOf<String>()
// Set when the model produces its artifact via the emit_artifact tool instead of a final
// JSON message; overrides the post-loop capture of the (then-empty) assistant text.
var llmArtifactOverride: String? = null
@@ -697,7 +703,8 @@ abstract class SessionOrchestrator(
}
val toolEntries = dispatchToolCalls(sessionId, stageId,
inferenceResult.response.toolCalls, stageConfig, effectives,
approvalModeFor(session.state.boundProfile?.approvalMode))
approvalModeFor(session.state.boundProfile?.approvalMode),
inferenceResult.response.reasoning)
val fatalEntry = toolEntries.firstOrNull { it.content.startsWith("FATAL:") }
if (fatalEntry != null) {
emitProcessResultEvents(sessionId, stageId, stageConfig)
@@ -718,14 +725,23 @@ abstract class SessionOrchestrator(
if (owesFileWrite() &&
inferenceResult.response.toolCalls.none { it.function.name in WRITE_TOOL_NAMES }
) {
consecutiveReadOnlyRounds++
if (consecutiveReadOnlyRounds >= READ_LOOP_NUDGE_THRESHOLD) {
val roundFingerprints = inferenceResult.response.toolCalls
.map { "${it.function.name}:${it.function.arguments}" }
val madeProgress = roundFingerprints.any { it !in seenReadFingerprints }
seenReadFingerprints += roundFingerprints
if (madeProgress) {
consecutiveReadOnlyRounds = 0
inferenceResult = pushBack(readLoopNudge, forceWriteOnly = true)
continue
} else {
consecutiveReadOnlyRounds++
if (consecutiveReadOnlyRounds >= READ_LOOP_NUDGE_THRESHOLD) {
consecutiveReadOnlyRounds = 0
inferenceResult = pushBack(readLoopNudge, forceWriteOnly = true)
continue
}
}
} else {
consecutiveReadOnlyRounds = 0
seenReadFingerprints.clear()
}
// Rejection-loop breaker (stage-type agnostic): every tool result this round was a
// rejection (plane-2 BLOCKED or a stage/policy ERROR) and nothing succeeded. A stage that
@@ -1028,10 +1044,14 @@ abstract class SessionOrchestrator(
// On a recoverable tool failure, append the tool's argument schema to the error fed back to the
// model so it can self-correct its next attempt (it already sees its own malformed call in the
// assistant entry above) rather than repeating the same mistake. Kept compact to bound context.
// NOTE: the "corrected arguments" phrasing was removed because it actively drives loops: a model
// that called list_dir frontend/ and got "directory not found" interprets "re-issue with corrected
// arguments" as "retry the same call" rather than "try a different path" — the arguments were
// valid, the path simply doesn't exist. The schema reference stays so malformed calls can still
// self-correct; the model must infer the tactic from the tool's error text.
private fun toolArgsHint(tool: Tool?): String =
tool?.let {
"\nThe '${it.name}' tool requires arguments matching this JSON schema — re-issue the " +
"call with corrected arguments: ${it.parametersSchema}"
"\nAccepted parameters for '${it.name}': ${it.parametersSchema}"
}.orEmpty()
private suspend fun dispatchToolCalls(
@@ -1041,11 +1061,18 @@ abstract class SessionOrchestrator(
stageConfig: StageConfig,
effectives: RunEffectives,
approvalMode: ApprovalMode,
// The reasoning that preceded this whole batch of tool calls (they all came from one
// inference response). Attached only to the first call's assistant entry — one thought
// block per model turn, matching how the response actually arrived.
reasoning: String? = null,
): List<ContextEntry> {
val executor = effectives.executor ?: return emptyList()
val processResultSlots = stageConfig.produces.filter { it.kind.id == "process_result" }
val fileWrittenSlots = stageConfig.produces.filter { it.kind.id == "file_written" }
return toolCalls.flatMap { toolCall ->
return toolCalls.withIndex().flatMap { (toolCallIndex, toolCall) ->
// Only the first call's assistant entry carries the reasoning — the whole batch came
// from one model turn/one thought block, so later calls in the same batch get none.
val toolCallReasoning = reasoning.takeIf { toolCallIndex == 0 }
val invocationId = ToolInvocationId(UUID.randomUUID().toString())
val parameters = parseToolArguments(toolCall.function.arguments)
val request = ToolRequest(
@@ -1099,6 +1126,7 @@ abstract class SessionOrchestrator(
content = Json.encodeToString(ToolCallRequest.serializer(), toolCall),
tokenEstimate = estimateTokens(toolCall.function.arguments),
role = EntryRole.ASSISTANT,
reasoning = toolCallReasoning,
),
ContextEntry(
id = ContextEntryId(UUID.randomUUID().toString()),
@@ -1155,6 +1183,7 @@ abstract class SessionOrchestrator(
content = Json.encodeToString(ToolCallRequest.serializer(), toolCall),
tokenEstimate = estimateTokens(toolCall.function.arguments),
role = EntryRole.ASSISTANT,
reasoning = toolCallReasoning,
),
ContextEntry(
id = ContextEntryId(UUID.randomUUID().toString()),
@@ -1223,6 +1252,7 @@ abstract class SessionOrchestrator(
content = Json.encodeToString(ToolCallRequest.serializer(), toolCall),
tokenEstimate = estimateTokens(toolCall.function.arguments),
role = EntryRole.ASSISTANT,
reasoning = toolCallReasoning,
),
ContextEntry(
id = ContextEntryId(UUID.randomUUID().toString()),
@@ -1282,6 +1312,7 @@ abstract class SessionOrchestrator(
content = Json.encodeToString(ToolCallRequest.serializer(), toolCall),
tokenEstimate = estimateTokens(toolCall.function.arguments),
role = EntryRole.ASSISTANT,
reasoning = toolCallReasoning,
),
ContextEntry(
id = ContextEntryId(UUID.randomUUID().toString()),
@@ -1353,6 +1384,7 @@ abstract class SessionOrchestrator(
content = Json.encodeToString(ToolCallRequest.serializer(), toolCall),
tokenEstimate = estimateTokens(toolCall.function.arguments),
role = EntryRole.ASSISTANT,
reasoning = toolCallReasoning,
)
val resultContent = when (result) {
is ToolResult.Success ->
@@ -1702,13 +1734,47 @@ abstract class SessionOrchestrator(
return closestPathByFilename(guessed, map.entries.map { it.path })
}
/**
* Resolves the actual prompt text a stage will see, for use as the repo-knowledge query.
* `metadata["prompt"]` holds a *path* to the prompt file (resolved into context separately,
* around line 431) — using that path string itself as the embedding query means the query the
* retriever compares against is a filesystem path, not the stage's real instructions, so
* similarity scores never reflect genuine relevance (2026-07-08 finding: no file scored a
* clear match, everything clustered in the embedder's baseline noise band for short strings).
*/
private fun resolveStagePromptText(stageId: StageId, stageConfig: StageConfig): String {
stageConfig.metadata["promptInline"]?.trim()?.takeIf { it.isNotBlank() }?.let { return it }
stageConfig.metadata["prompt"]?.let { path ->
runCatching { promptResolver.resolve(path) }.getOrNull()
?.trim()?.takeIf { it.isNotBlank() }
?.let { return it }
}
return stageId.value
}
/**
* The role's own prompt text (see [resolveStagePromptText]) is identical across every session
* that runs the stage — it carries zero session-specific signal, so similarity against it can
* only ever reflect generic lexical overlap with the role's boilerplate ("file_read", "ls",
* "grep"...), never what the user actually asked for. The one thing that IS session-specific
* is the initial user intent, so it's prepended here as the real anchor for the query (2026-07-08
* finding: without it, the query matched a tool script's own symbol names over the files the
* user's request actually named).
*/
private fun repoKnowledgeQuery(sessionId: SessionId, stageId: StageId, stageConfig: StageConfig): String {
val intent = eventStore.read(sessionId)
.mapNotNull { it.payload as? InitialIntentEvent }
.firstOrNull()?.intent?.trim()
val stagePrompt = resolveStagePromptText(stageId, stageConfig)
return if (intent.isNullOrBlank()) stagePrompt else "$intent\n\n$stagePrompt"
}
private suspend fun buildContextualRepoEntries(
sessionId: SessionId,
stageId: StageId,
stageConfig: StageConfig,
): List<ContextEntry> {
val stagePrompt = (stageConfig.metadata["promptInline"] ?: stageConfig.metadata["prompt"] ?: stageId.value)
.trim().ifBlank { stageId.value }
val stagePrompt = repoKnowledgeQuery(sessionId, stageId, stageConfig)
val recorded = eventStore.read(sessionId)
.mapNotNull { it.payload as? RepoKnowledgeRetrievedEvent }
.lastOrNull { it.stageId == stageId }