feat(kernel): stage-global repeated-failure loop breaker (#78)
Existing read-loop/rejection-loop breakers key on CONSECUTIVE rounds, so a single interleaved success resets them — the hole that let the 2026-07-16 audition run-3 thrash npm install against a hallucinated package across 57 inferences. Add repeatedToolFailureLoop: cumulative count per normalized tool-failure signature within a stage; once a signature hits stageFailureLoopLimit (default 6) the stage fails with STAGE_LOOP_BREAK_GATE and the step handler routes straight to recovery (never retried in place). Pure fold over recorded events (replay-safe, invariants #8/#9); ToolExecutionFailedEvent lacks stageId so failures correlate via invocationId -> ToolInvocationRequestedEvent. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
+5
@@ -69,6 +69,11 @@ internal val EVIDENCE_PATH_RE = Regex("""[\w./@-]+\.[A-Za-z0-9]+""")
|
||||
// (bounded bootstrap turn / recovery routing) is triggered off this id before the normal retry path.
|
||||
internal const val WORKSPACE_PRECONDITION_GATE = "workspace_precondition"
|
||||
|
||||
// Gate id for a provably-stuck stage failure loop (Vikunja #78). Detected by repeatedToolFailureLoop
|
||||
// when the SAME tool-failure signature recurs past the tuning limit; routed straight to recovery
|
||||
// (never retried in place) by the step handler before the normal retry path.
|
||||
internal const val STAGE_LOOP_BREAK_GATE = "stage_loop_break"
|
||||
|
||||
internal val GATE_REQUIRED_CAPABILITY: Map<String, String> = mapOf(
|
||||
"execution" to "file_write",
|
||||
"contract" to "file_write",
|
||||
|
||||
+9
@@ -264,6 +264,15 @@ internal suspend fun DefaultSessionOrchestrator.enterStage(
|
||||
is PrerequisiteOutcome.Done -> return outcome.result
|
||||
}
|
||||
}
|
||||
// A proven same-signature failure loop (Vikunja #78) must never be retried in place —
|
||||
// retrying is what got us here. Route straight to recovery regardless of capability;
|
||||
// terminal if the graph offers no recovery route.
|
||||
if (result.gate == STAGE_LOOP_BREAK_GATE) {
|
||||
return routeToRecovery(ctx, stageId, result.gate, "file_write", result.reason, refreshedState)
|
||||
?: StepResult.Terminal(
|
||||
failWorkflow(ctx.sessionId, stageId, result.reason, retryExhausted = true),
|
||||
)
|
||||
}
|
||||
// Retry-agency invariant: a gate this stage lacks the capability to fix must not
|
||||
// be retried in place (futile). Route to a recovery stage that holds the
|
||||
// capability, if one exists and the route budget remains.
|
||||
|
||||
+6
@@ -41,4 +41,10 @@ data class OrchestrationTuning(
|
||||
val recoveryRouteBudget: Int = 2,
|
||||
/** Budget for tier-2 intent-holder arbiter re-routing. */
|
||||
val intentRouteBudget: Int = 2,
|
||||
/**
|
||||
* Cumulative same-signature tool failures within a stage before it's broken out of the loop and
|
||||
* routed to recovery (Vikunja #78). Unlike the consecutive read/rejection nudges, this survives
|
||||
* interleaved successes — it counts a normalized failure signature across the whole stage.
|
||||
*/
|
||||
val stageFailureLoopLimit: Int = 6,
|
||||
)
|
||||
|
||||
+3
@@ -417,6 +417,9 @@ internal suspend fun SessionOrchestrator.executeStage(
|
||||
repeatedBuildCriticalReferenceBlock(sessionId, stageId)?.let { reason ->
|
||||
return StageExecutionResult.Failure(reason, retryable = true, gate = WORKSPACE_PRECONDITION_GATE)
|
||||
}
|
||||
repeatedToolFailureLoop(sessionId, stageId, tuning.stageFailureLoopLimit)?.let { reason ->
|
||||
return StageExecutionResult.Failure(reason, retryable = true, gate = STAGE_LOOP_BREAK_GATE)
|
||||
}
|
||||
val fatalEntry = toolEntries.firstOrNull { it.content.startsWith("FATAL:") }
|
||||
if (fatalEntry != null) {
|
||||
emitProcessResultEvents(sessionId, stageId, stageConfig)
|
||||
|
||||
+56
@@ -9,6 +9,7 @@ import com.correx.core.events.events.OrchestrationPausedEvent
|
||||
import com.correx.core.events.events.OrchestrationResumedEvent
|
||||
import com.correx.core.events.events.StoredEvent
|
||||
import com.correx.core.events.events.ToolCallAssessedEvent
|
||||
import com.correx.core.events.events.ToolExecutionFailedEvent
|
||||
import com.correx.core.events.events.ToolInvocationRequestedEvent
|
||||
import com.correx.core.events.orchestration.OrchestrationState
|
||||
import com.correx.core.events.risk.RiskAction
|
||||
@@ -59,6 +60,60 @@ internal fun SessionOrchestrator.repeatedBuildCriticalReferenceBlock(
|
||||
"(${repeated.value} blocked attempts). Create or repair the project setup before continuing."
|
||||
}
|
||||
|
||||
/**
|
||||
* Global backstop for a stage stuck retrying the *same* failing action (Vikunja #78). The existing
|
||||
* read-loop and rejection-loop breakers key on CONSECUTIVE rounds, so a single interleaved success
|
||||
* resets their counters — exactly what let the 2026-07-16 audition run-3 thrash `npm install` against
|
||||
* a hallucinated package across 57 inferences. This counts CUMULATIVELY per normalized failure
|
||||
* signature within the stage: once one signature has failed [limit] times, retrying it is provably
|
||||
* pointless, so return a retryable gate failure that the step handler routes straight to recovery.
|
||||
*
|
||||
* Pure fold over recorded events (replay-safe, invariants #8/#9). [ToolExecutionFailedEvent] carries
|
||||
* no stageId, so failures are correlated to the stage via their invocationId → the emitting
|
||||
* [ToolInvocationRequestedEvent] (which does).
|
||||
*
|
||||
* ponytail: cumulative over the whole stage, not windowed per re-entry — a break escalates to
|
||||
* recovery under a bounded budget, so an unfixable loop terminates rather than re-tripping forever.
|
||||
* Add a per-re-entry window only if a legitimate later attempt gets cut short.
|
||||
*/
|
||||
internal fun SessionOrchestrator.repeatedToolFailureLoop(
|
||||
sessionId: SessionId,
|
||||
stageId: StageId,
|
||||
limit: Int,
|
||||
): String? = detectRepeatedToolFailure(eventStore.read(sessionId), stageId, limit)
|
||||
|
||||
/** Pure core of [repeatedToolFailureLoop] — a fold over recorded events (unit-tested in isolation). */
|
||||
internal fun detectRepeatedToolFailure(
|
||||
events: List<StoredEvent>,
|
||||
stageId: StageId,
|
||||
limit: Int,
|
||||
): String? {
|
||||
val stageInvocations = events
|
||||
.mapNotNull { it.payload as? ToolInvocationRequestedEvent }
|
||||
.filter { it.stageId == stageId }
|
||||
.map { it.invocationId }
|
||||
.toSet()
|
||||
val repeated = events
|
||||
.mapNotNull { it.payload as? ToolExecutionFailedEvent }
|
||||
.filter { it.invocationId in stageInvocations }
|
||||
// Collapse to a stable signature so equivalent retries group together: drop digits (package
|
||||
// versions, ports, counts) and punctuation/paths, keep the tool name and alphabetic core.
|
||||
// `npm install @types/vite@4.0.0` 404 and `@types/vite@5` 404 map to the same signature.
|
||||
.groupingBy { failure ->
|
||||
failure.toolName + "|" + failure.reason.lowercase()
|
||||
.replace(Regex("[0-9]+"), " ")
|
||||
.replace(Regex("[^a-z ]+"), " ")
|
||||
.replace(Regex("\\s+"), " ")
|
||||
.trim()
|
||||
.take(FAILURE_SIGNATURE_PREFIX)
|
||||
}
|
||||
.eachCount().entries
|
||||
.firstOrNull { it.value >= limit }
|
||||
?: return null
|
||||
return "stage ${stageId.value} is stuck: the same tool failure recurred ${repeated.value} times " +
|
||||
"(${repeated.key}). Retrying it keeps failing — a materially different action is required."
|
||||
}
|
||||
|
||||
/**
|
||||
* Which bounded precondition-resolution path a repeated build-prerequisite block takes (design #170).
|
||||
* Pure over the three observable facts — deterministic/replay-safe, unit-tested in isolation.
|
||||
@@ -230,6 +285,7 @@ private fun isBuildCriticalPath(path: String): Boolean {
|
||||
private const val BOOTSTRAP_BUDGET = 1
|
||||
private val PREREQUISITE_PATH_RE = Regex("""missing build prerequisite '([^']+)'""")
|
||||
private const val REFERENCE_EXISTS_REPEAT_LIMIT = 3
|
||||
private const val FAILURE_SIGNATURE_PREFIX = 80
|
||||
private val BUILD_PREREQUISITE_FILENAMES = setOf(
|
||||
"package.json", "tsconfig.json", "vite.config.ts", "build.gradle", "build.gradle.kts", "pom.xml",
|
||||
)
|
||||
|
||||
+91
@@ -0,0 +1,91 @@
|
||||
package com.correx.core.kernel.orchestration
|
||||
|
||||
import com.correx.core.approvals.Tier
|
||||
import com.correx.core.events.events.EventMetadata
|
||||
import com.correx.core.events.events.EventPayload
|
||||
import com.correx.core.events.events.StoredEvent
|
||||
import com.correx.core.events.events.ToolExecutionFailedEvent
|
||||
import com.correx.core.events.events.ToolInvocationRequestedEvent
|
||||
import com.correx.core.events.events.ToolRequest
|
||||
import com.correx.core.events.types.EventId
|
||||
import com.correx.core.events.types.SessionId
|
||||
import com.correx.core.events.types.StageId
|
||||
import com.correx.core.events.types.ToolInvocationId
|
||||
import kotlinx.datetime.Instant
|
||||
import org.junit.jupiter.api.Assertions.assertNotNull
|
||||
import org.junit.jupiter.api.Assertions.assertNull
|
||||
import org.junit.jupiter.api.Assertions.assertTrue
|
||||
import org.junit.jupiter.api.Test
|
||||
|
||||
class RepeatedToolFailureLoopTest {
|
||||
|
||||
private val stage = StageId("install_dependencies")
|
||||
|
||||
@Test
|
||||
fun `same-signature failures trip the limit even when interleaved with successes`() {
|
||||
// The rejection-loop breaker would reset on the interleaved success; this must not.
|
||||
val events = buildList {
|
||||
repeat(3) { addAll(failure("npm install @types/vite@4.0.0 -> registry 404 not found")) }
|
||||
add(success()) // resets any CONSECUTIVE counter
|
||||
repeat(3) { addAll(failure("npm install @types/vite@5.1.2 -> registry 404 not found")) }
|
||||
}
|
||||
val reason = detectRepeatedToolFailure(events, stage, limit = 6)
|
||||
assertNotNull(reason)
|
||||
assertTrue(reason!!.contains("install_dependencies"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `distinct failures below the limit do not trip`() {
|
||||
val events = listOf(
|
||||
failure("npm install a -> 404"),
|
||||
failure("tsc -> type error in App.tsx"),
|
||||
failure("eslint -> no config found"),
|
||||
).flatten()
|
||||
assertNull(detectRepeatedToolFailure(events, stage, limit = 6))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `failures from other stages are not counted`() {
|
||||
val other = StageId("scaffold")
|
||||
val events = (1..8).flatMap { failure("npm install x -> 404", stageOf = other) }
|
||||
assertNull(detectRepeatedToolFailure(events, stage, limit = 6))
|
||||
}
|
||||
|
||||
private var seq = 0L
|
||||
private val session = SessionId("s1")
|
||||
|
||||
private fun ev(payload: EventPayload) = StoredEvent(
|
||||
metadata = EventMetadata(
|
||||
eventId = EventId("e${seq++}"),
|
||||
sessionId = session,
|
||||
timestamp = Instant.parse("2026-01-01T00:00:00Z"),
|
||||
schemaVersion = 1,
|
||||
causationId = null,
|
||||
correlationId = null,
|
||||
),
|
||||
sequence = seq,
|
||||
sessionSequence = seq,
|
||||
payload = payload,
|
||||
)
|
||||
|
||||
// A failure = the request (carries stageId) + the failure (carries only invocationId), correlated by id.
|
||||
private fun failure(reason: String, stageOf: StageId = stage): List<StoredEvent> {
|
||||
val id = ToolInvocationId("inv-${seq}")
|
||||
val req = ev(
|
||||
ToolInvocationRequestedEvent(
|
||||
invocationId = id, sessionId = session, stageId = stageOf,
|
||||
toolName = "shell", tier = Tier.T1,
|
||||
request = ToolRequest(id, session, stageOf, "shell"),
|
||||
),
|
||||
)
|
||||
return listOf(req, ev(ToolExecutionFailedEvent(id, session, "shell", reason)))
|
||||
}
|
||||
|
||||
private fun success() = ev(
|
||||
ToolInvocationRequestedEvent(
|
||||
invocationId = ToolInvocationId("ok-${seq}"), sessionId = session, stageId = stage,
|
||||
toolName = "shell", tier = Tier.T1,
|
||||
request = ToolRequest(ToolInvocationId("ok-$seq"), session, stage, "shell"),
|
||||
),
|
||||
)
|
||||
}
|
||||
Reference in New Issue
Block a user