fix(kernel,workflow): five fixes from the 954da1a9 post-mortem (#705,#706,#709,#710,#712)
The run died on a build gate running the wrong toolchain and burned 43% of its tool calls on repeats, rejections and failures. Five fixes, each traceable to a measured cost in docs/audits/2026-08-11-session-954da1a9-postmortem.md. #705 build gate toolchain scope. The gate is armed session-scoped but read the toolchain stage-scoped, so a reviewer stage that wrote nothing fell back to the flat `build` alias and ran ./gradlew assemble on an all-frontend session. It now falls back to the session's own manifest first. 32.5 min, 33% of that run. #706 action ledger. L2 keeps ten conversation entries, so a stage past round five has no memory of what it tried; 29% of tool calls were byte-identical repeats. One pinned line per call (tool, target, outcome) with repeats collapsed to a count, ~3k tokens for a whole run. #709 plan-compile lint for blocked runners. Plans prescribed `npx tailwindcss init -p`, rejected by the shell denylist at every attempt. Rejected at compile time now, where the architect can still rewrite the step. #710 near-greedy sampling on tool-call rounds. temperature 1.0 on argv emitted `./gradlew_`, `npm_prefix=frontend`, `create_vite@latest`. Prose rounds keep the operator's sampling. #712 auto-approve manifest-contained writes. 94 approvals, all APPROVED, no steering, 19 min. A write inside the declared manifest already proved its containment by getting past ManifestContainmentRule. DENY mode still denies. Tests: core:kernel 133, infrastructure:workflow 99, testing:integration 176, testing:deterministic 79, all green. detekt clean (no new findings). Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013dVqqci5H5b3s6xzv6Lojq
This commit is contained in:
@@ -0,0 +1,100 @@
|
||||
package com.correx.core.kernel.orchestration
|
||||
|
||||
import com.correx.core.context.model.ContextEntry
|
||||
import com.correx.core.context.model.ContextLayer
|
||||
import com.correx.core.context.model.EntryRole
|
||||
import com.correx.core.events.types.ContextEntryId
|
||||
import com.correx.core.inference.ToolCallRequest
|
||||
import java.util.UUID
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
import kotlinx.serialization.json.jsonObject
|
||||
|
||||
/**
|
||||
* Deterministic action ledger (#706, post-mortem of run 954da1a9). L2 holds the last ten
|
||||
* conversation entries — five tool call/result pairs — so a stage past round five has no memory of
|
||||
* what it already tried. 29% of that run's 258 tool calls were byte-identical repeats: `list_dir
|
||||
* frontend` nine times, `./gradlew assemble` four times, each one failing the same way.
|
||||
*
|
||||
* The ledger is one line per call — tool, target, outcome — pinned so it never evicts. A whole run
|
||||
* is roughly 3k tokens, cheaper than the duplicates it removes, and it carries the "already tried,
|
||||
* same failure" signal no window size provides. Repeats collapse to a count, so a thrashing loop
|
||||
* reads as `shell ./gradlew assemble -> exit 1 (x4)` rather than four separate lines.
|
||||
*/
|
||||
private const val LEDGER_MAX_LINES = 80
|
||||
private const val LEDGER_OUTCOME_CHARS = 90
|
||||
private const val LEDGER_TARGET_CHARS = 70
|
||||
|
||||
private val ledgerJson = Json { ignoreUnknownKeys = true }
|
||||
|
||||
/**
|
||||
* Folds this round's tool entries into `tool target -> outcome` lines. Pairs the `assistantToolCall`
|
||||
* entry with its `toolResult` by sourceId (both are stamped with it in dispatchToolCalls), so this
|
||||
* reads only what the loop already has — no event-store re-read.
|
||||
*/
|
||||
internal fun ledgerLinesFrom(entries: List<ContextEntry>): List<String> {
|
||||
val results = entries.filter { it.sourceType == "toolResult" }.associateBy { it.sourceId }
|
||||
return entries.filter { it.sourceType == "assistantToolCall" }.mapNotNull { call ->
|
||||
val request = runCatching {
|
||||
ledgerJson.decodeFromString(ToolCallRequest.serializer(), call.content)
|
||||
}.getOrNull() ?: return@mapNotNull null
|
||||
val target = ledgerTarget(request.function.arguments)
|
||||
val outcome = ledgerOutcome(results[call.sourceId]?.content)
|
||||
listOfNotNull(request.function.name, target).joinToString(" ") + " -> " + outcome
|
||||
}
|
||||
}
|
||||
|
||||
/** The one argument worth showing: the path/command a call acted on, else the first string value. */
|
||||
private fun ledgerTarget(arguments: String): String? {
|
||||
val obj = runCatching { ledgerJson.parseToJsonElement(arguments).jsonObject }.getOrNull() ?: return null
|
||||
val strings = obj.mapValues { (_, v) -> (v as? JsonPrimitive)?.takeIf { it.isString }?.content }
|
||||
val picked = listOf("path", "command", "file_path", "query", "pattern")
|
||||
.firstNotNullOfOrNull { strings[it] }
|
||||
?: strings.values.filterNotNull().firstOrNull()
|
||||
return picked?.trim()?.take(LEDGER_TARGET_CHARS)
|
||||
}
|
||||
|
||||
private fun ledgerOutcome(result: String?): String = when {
|
||||
result == null -> "no result"
|
||||
result.startsWith("ERROR:") || result.startsWith("BLOCKED:") ->
|
||||
result.lineSequence().first().take(LEDGER_OUTCOME_CHARS)
|
||||
else -> "ok"
|
||||
}
|
||||
|
||||
/**
|
||||
* Renders the pinned ledger entry. Identical lines collapse to one with a repeat count — that count
|
||||
* IS the signal, so it must not be lost to dedup. Keeps the most recent [LEDGER_MAX_LINES] distinct
|
||||
* lines and says how many it dropped, rather than silently truncating.
|
||||
*/
|
||||
internal fun buildActionLedgerEntry(lines: List<String>): ContextEntry? {
|
||||
if (lines.isEmpty()) return null
|
||||
val counted = LinkedHashMap<String, Int>()
|
||||
lines.forEach { counted[it] = (counted[it] ?: 0) + 1 }
|
||||
val dropped = (counted.size - LEDGER_MAX_LINES).coerceAtLeast(0)
|
||||
val content = buildString {
|
||||
append("## Already done this stage\n")
|
||||
append(
|
||||
"Every tool call you have made in this stage, in order, with its outcome. Do NOT repeat " +
|
||||
"a call listed here: it will return the same thing. A line marked (xN) is a call you " +
|
||||
"have already retried N times without the result changing — try something different " +
|
||||
"or move on.\n",
|
||||
)
|
||||
if (dropped > 0) append("- ... $dropped earlier calls omitted\n")
|
||||
counted.entries.drop(dropped).forEach { (line, count) ->
|
||||
append("- ").append(line)
|
||||
if (count > 1) append(" (x").append(count).append(")")
|
||||
append("\n")
|
||||
}
|
||||
}.trimEnd()
|
||||
return ContextEntry(
|
||||
id = ContextEntryId(UUID.randomUUID().toString()),
|
||||
layer = ContextLayer.L1,
|
||||
content = content,
|
||||
sourceType = "actionLedger",
|
||||
sourceId = "action-ledger",
|
||||
tokenEstimate = content.length / 4,
|
||||
// Rebuilt every round, like remainingDelta — USER, so it never invalidates the cached
|
||||
// system prefix and never competes with the stage's own instructions.
|
||||
role = EntryRole.USER,
|
||||
)
|
||||
}
|
||||
+10
@@ -14,6 +14,16 @@ internal fun SessionOrchestrator.stageProducedToolchain(
|
||||
stageId: StageId,
|
||||
): KindContractTable.Toolchain? = toolchainForPaths(stageWrittenPaths(sessionId, stageId))
|
||||
|
||||
/**
|
||||
* The toolchain the execution gate runs for a stage: what this stage wrote, else what the session
|
||||
* wrote. The gate is armed session-scoped, so a stage that wrote nothing (a reviewer) must not fall
|
||||
* through to the profile's flat `build` alias and run some other stack's build (#705).
|
||||
*/
|
||||
internal fun resolveGateToolchain(
|
||||
stagePaths: List<String>,
|
||||
sessionPaths: List<String>,
|
||||
): KindContractTable.Toolchain? = toolchainForPaths(stagePaths) ?: toolchainForPaths(sessionPaths)
|
||||
|
||||
internal fun toolchainForPaths(paths: List<String>): KindContractTable.Toolchain? =
|
||||
paths.asReversed().firstNotNullOfOrNull { path ->
|
||||
KindInference.kindFor(path)?.let(KindContractTable::toolchainFor)
|
||||
|
||||
+17
-2
@@ -106,6 +106,12 @@ internal const val TOOL_RESULT_HEAD_LINES = 60
|
||||
internal const val TOOL_RESULT_TAIL_LINES = 60
|
||||
internal const val TOOL_OUTPUT_TOOL = "tool_output"
|
||||
|
||||
// ponytail: near-greedy, not greedy (temperature 0) — a hard 0 makes a stuck model repeat the same
|
||||
// failing call forever, and the repeat rate is already the problem (#706). Fixed constants, not
|
||||
// config: they describe the field type (argv/path), not an operator preference.
|
||||
internal const val TOOL_CALL_TEMPERATURE = 0.15
|
||||
internal const val TOOL_CALL_TOP_P = 0.9
|
||||
|
||||
/**
|
||||
* Frame an over-cap tool output as `header` + head lines + a truncation marker (naming the
|
||||
* [tool_output] ref that retrieves the full text) + tail lines. Head and tail are each char-capped
|
||||
@@ -332,8 +338,9 @@ abstract class SessionOrchestrator(
|
||||
// A stage that grants tools needs a tool-calling model, so request ToolCalling on top of any
|
||||
// declared capabilities — the capability-aware strategy then ranks eligible providers by their
|
||||
// ToolCalling score and routes the stage to the best tool-caller.
|
||||
val toolCallRound = withTools && stageConfig.allowedTools.isNotEmpty()
|
||||
val requiredCapabilities = stageConfig.requiredCapabilities +
|
||||
if (withTools && stageConfig.allowedTools.isNotEmpty()) setOf(ModelCapability.ToolCalling) else emptySet()
|
||||
if (toolCallRound) setOf(ModelCapability.ToolCalling) else emptySet()
|
||||
// Routing itself can fail transiently (provider mid-crash-recovery) — it must be retryable
|
||||
// like any other inference failure, not escape and kill the whole session (see #299).
|
||||
val provider = try {
|
||||
@@ -353,7 +360,15 @@ abstract class SessionOrchestrator(
|
||||
sessionId = sessionId,
|
||||
stageId = stageId,
|
||||
contextPack = contextPack,
|
||||
generationConfig = stageConfig.generationConfig,
|
||||
// A round that carries tools is answered with argv, paths and flags — fields where
|
||||
// exactly one string is correct. Chat-temperature sampling there emits `./gradlew_`,
|
||||
// `npm_prefix=frontend`, `create_vite@latest` (#710, run 954da1a9). Go near-greedy for
|
||||
// those rounds and keep the operator's sampling for prose rounds (artifact + review).
|
||||
generationConfig = if (toolCallRound) {
|
||||
stageConfig.generationConfig.copy(temperature = TOOL_CALL_TEMPERATURE, topP = TOOL_CALL_TOP_P)
|
||||
} else {
|
||||
stageConfig.generationConfig
|
||||
},
|
||||
responseFormat = responseFormat,
|
||||
tools = if (!withTools) {
|
||||
emptyList()
|
||||
|
||||
+10
@@ -318,6 +318,10 @@ internal suspend fun SessionOrchestrator.executeStage(
|
||||
// already seen — a stage reading N distinct files before writing is legitimate context
|
||||
// gathering, not a loop, and shouldn't trip the same counter as re-reading the same file.
|
||||
val seenReadFingerprints = mutableSetOf<String>()
|
||||
// #706: every tool call this stage made, one line each, pinned so it outlives L2 eviction.
|
||||
// Without it the model's memory is the last five call/result pairs and it re-issues calls it
|
||||
// already made (29% of run 954da1a9's tool calls were byte-identical repeats).
|
||||
val ledgerLines = mutableListOf<String>()
|
||||
// Set when the model produces its artifact via the emit_artifact tool instead of a final
|
||||
// JSON message; overrides the post-loop capture of the (then-empty) assistant text.
|
||||
var llmArtifactOverride: String? = null
|
||||
@@ -473,6 +477,12 @@ internal suspend fun SessionOrchestrator.executeStage(
|
||||
// loop as tool-result context so the model can see the error and adapt (bounded by
|
||||
// MAX_TOOL_ROUNDS). Only FATAL: failures (handled above) abort the stage.
|
||||
accumulatedEntries = accumulatedEntries + toolEntries
|
||||
// #706: fold this round into the pinned action ledger before any pushBack rebuilds the pack,
|
||||
// so a nudged round already carries the "you have tried this N times" line.
|
||||
ledgerLines += ledgerLinesFrom(toolEntries)
|
||||
buildActionLedgerEntry(ledgerLines)?.let { ledger ->
|
||||
accumulatedEntries = accumulatedEntries.filterNot { it.sourceType == "actionLedger" } + ledger
|
||||
}
|
||||
// Read-loop breaker: this round called only read-only tools yet the stage still owes a
|
||||
// file_written artifact. Left alone the model keeps reading until MAX_TOOL_ROUNDS and
|
||||
// never writes (F-018 nudges only cover a prose turn or a premature stage_complete, not
|
||||
|
||||
+9
-1
@@ -174,7 +174,15 @@ internal suspend fun SessionOrchestrator.runExecutionGate(
|
||||
val runner = staticAnalysisRunner
|
||||
val workspaceRoot = effectives.policy?.workspaceRoot
|
||||
if (runner == null || workspaceRoot == null) return StageExecutionResult.Success(emptyList())
|
||||
val toolchain = stageProducedToolchain(sessionId, stageId)?.profileKey
|
||||
// The gate is turned on session-scoped (sessionProducedBuildTarget) but the toolchain was read
|
||||
// stage-scoped, so a stage that wrote nothing (a reviewer) resolved null and fell back to the
|
||||
// flat `build` alias — run 954da1a9 ran `./gradlew assemble` on an all-`frontend/**` session and
|
||||
// died on it (#705). Two lookups deciding one command must share a scope: fall back to the
|
||||
// session's own manifest, the same one that armed the gate, before the flat alias.
|
||||
val toolchain = resolveGateToolchain(
|
||||
stageWrittenPaths(sessionId, stageId),
|
||||
sessionWrittenPaths(sessionId),
|
||||
)?.profileKey
|
||||
val command = expectation.commandFor(profileCommands, toolchain)
|
||||
if (command.isNullOrBlank()) {
|
||||
log.warn(
|
||||
|
||||
+13
-3
@@ -305,12 +305,22 @@ internal suspend fun SessionOrchestrator.dispatchToolCalls(
|
||||
assessment
|
||||
}
|
||||
val plane2Prompts = plane2Risk?.recommendedAction == RiskAction.PROMPT_USER
|
||||
// #712: a write that lands inside the stage's declared manifest (or the claimed task's
|
||||
// affected_paths) needs no interrupt — the operator already approved that path set when the
|
||||
// plan was approved, and ManifestContainmentRule BLOCKs anything outside it above, so
|
||||
// reaching here with a clean plane-2 verdict IS the containment proof. Run 954da1a9 spent
|
||||
// 19 min over 94 prompts, every one APPROVED with no steering. DENY mode still denies.
|
||||
val writeInsideManifest = plane2Risk?.recommendedAction == RiskAction.PROCEED &&
|
||||
approvalMode != ApprovalMode.DENY &&
|
||||
effectiveManifest.isNotEmpty() &&
|
||||
tool?.requiredCapabilities?.contains(ToolCapability.FILE_WRITE) == true
|
||||
// A steering note attached to a human approval is captured here and injected after the
|
||||
// tool result, so the same-stage loop re-infers with it and the model acts on the note.
|
||||
var approvalNote: String? = null
|
||||
if ((tier.isAtMost(Tier.T1) && !plane2Prompts) || alreadyGranted) {
|
||||
// no approval needed — either within the auto-approve tier, or this out-of-workspace
|
||||
// read path was already approved earlier this session (this-path-this-session).
|
||||
if ((tier.isAtMost(Tier.T1) && !plane2Prompts) || alreadyGranted || writeInsideManifest) {
|
||||
// no approval needed — within the auto-approve tier, an out-of-workspace read path
|
||||
// already approved earlier this session (this-path-this-session), or a write contained
|
||||
// by the stage's declared manifest.
|
||||
} else {
|
||||
// Grants in effect = this session's own (SESSION/STAGE) unioned with the
|
||||
// cross-session ledger (PROJECT/GLOBAL). projectId is derived from the bound
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
package com.correx.core.kernel.orchestration
|
||||
|
||||
import com.correx.core.context.model.ContextEntry
|
||||
import com.correx.core.context.model.ContextLayer
|
||||
import com.correx.core.context.model.EntryRole
|
||||
import com.correx.core.events.types.ContextEntryId
|
||||
import com.correx.core.inference.ToolCallFunction
|
||||
import com.correx.core.inference.ToolCallRequest
|
||||
import kotlinx.serialization.json.Json
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Assertions.assertFalse
|
||||
import org.junit.jupiter.api.Assertions.assertNull
|
||||
import org.junit.jupiter.api.Assertions.assertTrue
|
||||
import org.junit.jupiter.api.Test
|
||||
|
||||
class ActionLedgerTest {
|
||||
|
||||
private fun round(id: String, tool: String, args: String, result: String): List<ContextEntry> = listOf(
|
||||
entry(id, "assistantToolCall", Json.encodeToString(
|
||||
ToolCallRequest.serializer(),
|
||||
ToolCallRequest(id = id, function = ToolCallFunction(tool, args)),
|
||||
), EntryRole.ASSISTANT),
|
||||
entry(id, "toolResult", result, EntryRole.TOOL),
|
||||
)
|
||||
|
||||
private fun entry(sourceId: String, sourceType: String, content: String, role: EntryRole) = ContextEntry(
|
||||
id = ContextEntryId(sourceId + sourceType),
|
||||
layer = ContextLayer.L2,
|
||||
content = content,
|
||||
sourceType = sourceType,
|
||||
sourceId = sourceId,
|
||||
tokenEstimate = content.length / 4,
|
||||
role = role,
|
||||
)
|
||||
|
||||
@Test
|
||||
fun `a call folds to one tool-target-outcome line`() {
|
||||
val lines = ledgerLinesFrom(round("1", "file_read", """{"path":"frontend/package.json"}""", "{...}"))
|
||||
assertEquals(listOf("file_read frontend/package.json -> ok"), lines)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a failed call keeps its first error line`() {
|
||||
val lines = ledgerLinesFrom(
|
||||
round("1", "shell", """{"command":"./gradlew assemble"}""", "ERROR: exit 1\nCould not resolve io.ktor"),
|
||||
)
|
||||
assertEquals(listOf("shell ./gradlew assemble -> ERROR: exit 1"), lines)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `repeats collapse to a count instead of N lines`() {
|
||||
val line = "list_dir frontend -> ok"
|
||||
val content = buildActionLedgerEntry(List(9) { line })!!.content
|
||||
assertTrue(content.contains("$line (x9)"), content)
|
||||
assertEquals(1, content.lines().count { it.contains("list_dir") }, content)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `the ledger is empty until a call is made`() {
|
||||
assertNull(buildActionLedgerEntry(emptyList()))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `an overlong ledger drops the oldest lines and says so`() {
|
||||
val content = buildActionLedgerEntry((1..100).map { "file_read f$it.kt -> ok" })!!.content
|
||||
assertTrue(content.contains("20 earlier calls omitted"), content)
|
||||
assertFalse(content.contains("f1.kt"), content)
|
||||
assertTrue(content.contains("f100.kt"), content)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a call whose result never landed is still recorded`() {
|
||||
val call = round("1", "file_write", """{"path":"a.kt","content":"x"}""", "ok").first()
|
||||
assertEquals(listOf("file_write a.kt -> no result"), ledgerLinesFrom(listOf(call)))
|
||||
}
|
||||
}
|
||||
+12
@@ -24,6 +24,18 @@ class BuildPrerequisiteDecisionTest {
|
||||
assertEquals(KindContractTable.Toolchain.JVM, toolchainForPaths(listOf("core/kernel/FooService.kt")))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a stage that wrote nothing falls back to the session toolchain (#705)`() {
|
||||
val session = listOf("frontend/package.json", "frontend/src/App.tsx", "README.md")
|
||||
assertEquals(KindContractTable.Toolchain.NODE, resolveGateToolchain(emptyList(), session))
|
||||
// The stage's own writes still win when it has any.
|
||||
assertEquals(
|
||||
KindContractTable.Toolchain.JVM,
|
||||
resolveGateToolchain(listOf("core/kernel/FooService.kt"), session),
|
||||
)
|
||||
assertNull(resolveGateToolchain(emptyList(), listOf("README.md")))
|
||||
}
|
||||
|
||||
private val reason =
|
||||
"stage impl repeatedly referenced missing build prerequisite 'frontend/package.json' " +
|
||||
"(3 blocked attempts). Create or repair the project setup before continuing."
|
||||
|
||||
Reference in New Issue
Block a user