feat(context): reasoning-thread continuity, L3 doc filtering, and gate hardening

Threads reasoning_content across inference calls and journal renders, filters markdown noise out of L3 repo-knowledge retrieval, adds PlanLinter H3 checks, and tightens filesystem tool output/dir-listing behavior surfaced by prior live QA (see project_readloop_campaign memory).
2026-07-10 11:13:43 +04:00
parent f51a8dada4
commit 3d5e05c1fb
32 changed files with 742 additions and 85 deletions
@@ -26,6 +26,9 @@ import com.correx.core.events.types.ContextEntryId
 import com.correx.core.events.types.ContextPackId
 import com.correx.core.events.types.SessionId
 import com.correx.core.events.types.StageId
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.jsonObject
+import kotlinx.serialization.json.jsonPrimitive

 /**
 * Runs the compression pipeline (docs/plans/correx-compression-pipeline.md) in fixed pipeline
@@ -80,13 +83,21 @@ class DefaultContextPackBuilder(
        // reorder freely and PromptRenderer can restore true turn order from the ordinal.
        val stamped = entries.mapIndexed { index, entry -> entry.copy(ordinal = index) }

+        // A stalled stage re-issuing the exact same call otherwise burns budget on N identical
+        // (tool, arguments) pairs that all say the same thing — collapse those to the most recent
+        // occurrence. Fingerprint is (name, arguments) verbatim, so a call whose arguments genuinely
+        // differ (different start_line/end_line, different path) is a distinct read and untouched.
+        // Must run before FORMAT_COMPRESS below: that stage reencodes assistantToolCall content from
+        // raw ToolCallRequest JSON into a dotted-line format the fingerprint parser can't read.
+        val deduped = dedupeRepeatedToolCalls(stamped)
+
        // Stage 1 (FORMAT_COMPRESS): lossless json→dotted-line compaction of structured entries.
        val formatted = if (policy.enabled(CompressionStage.FORMAT_COMPRESS)) {
-            stamped.map { entry ->
+            deduped.map { entry ->
                if (classifier.classify(entry) == ContextClass.STRUCTURED) reencode(entry, formatCompressor.compress(entry.content))
                else entry
            }
-        } else stamped
+        } else deduped

        // Stage 3 (TOKEN_PRUNE): prune freeform prose, preserving protected spans. When TIER_SPLIT
        // is on, the newest TIER0_TURNS freeform turns are left full-fidelity (tier 0).
@@ -151,7 +162,7 @@ class DefaultContextPackBuilder(
            result
        }

-        val retained = (pinned + compressed).sortedBy { it.ordinal }
+        val retained = reconcileToolPairs((pinned + compressed).sortedBy { it.ordinal })
        val layers = retained.groupBy { it.layer }
        val droppedCount = entries.size - retained.count { it.sourceType != "factSheet" }
        val truncatedLayers = if (droppedCount > 0) {
@@ -235,6 +246,65 @@ class DefaultContextPackBuilder(
        return rest + ranked
    }

+    /**
+     * Collapses assistantToolCall/toolResult pairs that repeat an earlier call's exact (name,
+     * arguments) — keeping only the most recent occurrence — so a stalled stage repeating the same
+     * read doesn't show the model N copies of an identical question-and-answer. Content is the
+     * `ToolCallRequest` JSON built in SessionOrchestrator; parsed generically here (not via the
+     * `core:inference` type) to avoid a cross-core dependency. Calls whose arguments differ (e.g.
+     * different start_line/end_line, a different path) fingerprint differently and are untouched —
+     * that's genuine incremental progress, not repetition.
+     */
+    private fun dedupeRepeatedToolCalls(entries: List<ContextEntry>): List<ContextEntry> {
+        val calls = entries.filter { it.sourceType == "assistantToolCall" }
+        val callsById = calls.associateBy { it.sourceId }
+        val fingerprints = calls.mapNotNull { entry ->
+            toolCallFingerprint(entry.content)?.let { entry.sourceId to it }
+        }
+        val dropIds = fingerprints.groupBy({ it.second }, { it.first })
+            .values
+            .filter { it.size > 1 }
+            .flatMapTo(mutableSetOf()) { sourceIds ->
+                val keepId = sourceIds.maxWith(compareBy(
+                    { id: String -> if (callsById.getValue(id).reasoning != null) 1 else 0 },
+                    { id: String -> callsById.getValue(id).ordinal }
+                ))
+                sourceIds.filterNot { it == keepId }
+            }
+        if (dropIds.isEmpty()) return entries
+        return entries.filterNot {
+            (it.sourceType == "assistantToolCall" || it.sourceType == "toolResult") && it.sourceId in dropIds
+        }
+    }
+
+    private fun toolCallFingerprint(content: String): String? = runCatching {
+        val obj = Json.parseToJsonElement(content).jsonObject
+        val fn = obj["function"]?.jsonObject ?: return null
+        val name = fn["name"]?.jsonPrimitive?.content ?: return null
+        val arguments = fn["arguments"]?.jsonPrimitive?.content ?: ""
+        "$name:$arguments"
+    }.getOrNull()
+
+    /**
+     * A completed tool call is represented as two independently-IDed, independently-compressed
+     * entries — assistantToolCall + toolResult — linked only by [ContextEntry.sourceId]. Because
+     * `compressed` groups and trims by sourceType (see the `grouped` fold above), one side of a
+     * pair can survive its group's budget while the other is trimmed away. An orphaned
+     * assistantToolCall with no matching result renders as "I called this tool" with nothing after
+     * it — indistinguishable, to the model, from the call silently failing — which drove a live
+     * read-loop repro (2026-07-08: discovery stage re-issued the same file_read repeatedly once its
+     * result vanished from two consecutive rounds). Drop whichever side lost its pair so every
+     * tool-call entry that survives has its result, and vice versa.
+     */
+    private fun reconcileToolPairs(entries: List<ContextEntry>): List<ContextEntry> {
+        val callIds = entries.filter { it.sourceType == "assistantToolCall" }.mapTo(mutableSetOf()) { it.sourceId }
+        val resultIds = entries.filter { it.sourceType == "toolResult" }.mapTo(mutableSetOf()) { it.sourceId }
+        return entries.filterNot {
+            (it.sourceType == "assistantToolCall" && it.sourceId !in resultIds) ||
+                (it.sourceType == "toolResult" && it.sourceId !in callIds)
+        }
+    }
+
    private fun reencode(entry: ContextEntry, content: String): ContextEntry =
        if (content == entry.content) entry else entry.copy(content = content, tokenEstimate = estimateTokens(content))

@@ -32,4 +32,10 @@ data class ContextEntry(
    // Default OPTIONAL keeps prior behavior for callers that don't stamp buckets (router chat);
    // the orchestrator stamps REQUIRED on task/constraint/input/failure entries.
    val bucket: ContextBucket = ContextBucket.OPTIONAL,
+    // Chain-of-thought that preceded this ASSISTANT turn (assistantToolCall entries only), as
+    // captured on InferenceResponse.reasoning. Carried as a side channel, not JSON-encoded into
+    // `content`, so FORMAT_COMPRESS's dotted-line flatten of the tool-call JSON never touches it.
+    // Threaded back into the request as ChatMessage.reasoning so models that expect their own
+    // prior reasoning ahead of a tool-call turn (e.g. Gemma) don't degenerate into re-deriving it.
+    val reasoning: String? = null,
 )