feat(kernel): stage-global repeated-failure loop breaker (#78)

Existing read-loop/rejection-loop breakers key on CONSECUTIVE rounds, so a
single interleaved success resets them — the hole that let the 2026-07-16
audition run-3 thrash npm install against a hallucinated package across 57
inferences. Add repeatedToolFailureLoop: cumulative count per normalized
tool-failure signature within a stage; once a signature hits
stageFailureLoopLimit (default 6) the stage fails with STAGE_LOOP_BREAK_GATE
and the step handler routes straight to recovery (never retried in place).

Pure fold over recorded events (replay-safe, invariants #8/#9);
ToolExecutionFailedEvent lacks stageId so failures correlate via
invocationId -> ToolInvocationRequestedEvent.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-16 18:50:11 +04:00
parent a2b97221d5
commit 59a0ad3c2b
6 changed files with 170 additions and 0 deletions
@@ -69,6 +69,11 @@ internal val EVIDENCE_PATH_RE = Regex("""[\w./@-]+\.[A-Za-z0-9]+""")
// (bounded bootstrap turn / recovery routing) is triggered off this id before the normal retry path. // (bounded bootstrap turn / recovery routing) is triggered off this id before the normal retry path.
internal const val WORKSPACE_PRECONDITION_GATE = "workspace_precondition" internal const val WORKSPACE_PRECONDITION_GATE = "workspace_precondition"
// Gate id for a provably-stuck stage failure loop (Vikunja #78). Detected by repeatedToolFailureLoop
// when the SAME tool-failure signature recurs past the tuning limit; routed straight to recovery
// (never retried in place) by the step handler before the normal retry path.
internal const val STAGE_LOOP_BREAK_GATE = "stage_loop_break"
internal val GATE_REQUIRED_CAPABILITY: Map<String, String> = mapOf( internal val GATE_REQUIRED_CAPABILITY: Map<String, String> = mapOf(
"execution" to "file_write", "execution" to "file_write",
"contract" to "file_write", "contract" to "file_write",
@@ -264,6 +264,15 @@ internal suspend fun DefaultSessionOrchestrator.enterStage(
is PrerequisiteOutcome.Done -> return outcome.result is PrerequisiteOutcome.Done -> return outcome.result
} }
} }
// A proven same-signature failure loop (Vikunja #78) must never be retried in place —
// retrying is what got us here. Route straight to recovery regardless of capability;
// terminal if the graph offers no recovery route.
if (result.gate == STAGE_LOOP_BREAK_GATE) {
return routeToRecovery(ctx, stageId, result.gate, "file_write", result.reason, refreshedState)
?: StepResult.Terminal(
failWorkflow(ctx.sessionId, stageId, result.reason, retryExhausted = true),
)
}
// Retry-agency invariant: a gate this stage lacks the capability to fix must not // Retry-agency invariant: a gate this stage lacks the capability to fix must not
// be retried in place (futile). Route to a recovery stage that holds the // be retried in place (futile). Route to a recovery stage that holds the
// capability, if one exists and the route budget remains. // capability, if one exists and the route budget remains.
@@ -41,4 +41,10 @@ data class OrchestrationTuning(
val recoveryRouteBudget: Int = 2, val recoveryRouteBudget: Int = 2,
/** Budget for tier-2 intent-holder arbiter re-routing. */ /** Budget for tier-2 intent-holder arbiter re-routing. */
val intentRouteBudget: Int = 2, val intentRouteBudget: Int = 2,
/**
* Cumulative same-signature tool failures within a stage before it's broken out of the loop and
* routed to recovery (Vikunja #78). Unlike the consecutive read/rejection nudges, this survives
* interleaved successes — it counts a normalized failure signature across the whole stage.
*/
val stageFailureLoopLimit: Int = 6,
) )
@@ -417,6 +417,9 @@ internal suspend fun SessionOrchestrator.executeStage(
repeatedBuildCriticalReferenceBlock(sessionId, stageId)?.let { reason -> repeatedBuildCriticalReferenceBlock(sessionId, stageId)?.let { reason ->
return StageExecutionResult.Failure(reason, retryable = true, gate = WORKSPACE_PRECONDITION_GATE) return StageExecutionResult.Failure(reason, retryable = true, gate = WORKSPACE_PRECONDITION_GATE)
} }
repeatedToolFailureLoop(sessionId, stageId, tuning.stageFailureLoopLimit)?.let { reason ->
return StageExecutionResult.Failure(reason, retryable = true, gate = STAGE_LOOP_BREAK_GATE)
}
val fatalEntry = toolEntries.firstOrNull { it.content.startsWith("FATAL:") } val fatalEntry = toolEntries.firstOrNull { it.content.startsWith("FATAL:") }
if (fatalEntry != null) { if (fatalEntry != null) {
emitProcessResultEvents(sessionId, stageId, stageConfig) emitProcessResultEvents(sessionId, stageId, stageConfig)
@@ -9,6 +9,7 @@ import com.correx.core.events.events.OrchestrationPausedEvent
import com.correx.core.events.events.OrchestrationResumedEvent import com.correx.core.events.events.OrchestrationResumedEvent
import com.correx.core.events.events.StoredEvent import com.correx.core.events.events.StoredEvent
import com.correx.core.events.events.ToolCallAssessedEvent import com.correx.core.events.events.ToolCallAssessedEvent
import com.correx.core.events.events.ToolExecutionFailedEvent
import com.correx.core.events.events.ToolInvocationRequestedEvent import com.correx.core.events.events.ToolInvocationRequestedEvent
import com.correx.core.events.orchestration.OrchestrationState import com.correx.core.events.orchestration.OrchestrationState
import com.correx.core.events.risk.RiskAction import com.correx.core.events.risk.RiskAction
@@ -59,6 +60,60 @@ internal fun SessionOrchestrator.repeatedBuildCriticalReferenceBlock(
"(${repeated.value} blocked attempts). Create or repair the project setup before continuing." "(${repeated.value} blocked attempts). Create or repair the project setup before continuing."
} }
/**
* Global backstop for a stage stuck retrying the *same* failing action (Vikunja #78). The existing
* read-loop and rejection-loop breakers key on CONSECUTIVE rounds, so a single interleaved success
* resets their counters — exactly what let the 2026-07-16 audition run-3 thrash `npm install` against
* a hallucinated package across 57 inferences. This counts CUMULATIVELY per normalized failure
* signature within the stage: once one signature has failed [limit] times, retrying it is provably
* pointless, so return a retryable gate failure that the step handler routes straight to recovery.
*
* Pure fold over recorded events (replay-safe, invariants #8/#9). [ToolExecutionFailedEvent] carries
* no stageId, so failures are correlated to the stage via their invocationId → the emitting
* [ToolInvocationRequestedEvent] (which does).
*
* ponytail: cumulative over the whole stage, not windowed per re-entry — a break escalates to
* recovery under a bounded budget, so an unfixable loop terminates rather than re-tripping forever.
* Add a per-re-entry window only if a legitimate later attempt gets cut short.
*/
internal fun SessionOrchestrator.repeatedToolFailureLoop(
sessionId: SessionId,
stageId: StageId,
limit: Int,
): String? = detectRepeatedToolFailure(eventStore.read(sessionId), stageId, limit)
/** Pure core of [repeatedToolFailureLoop] — a fold over recorded events (unit-tested in isolation). */
internal fun detectRepeatedToolFailure(
events: List<StoredEvent>,
stageId: StageId,
limit: Int,
): String? {
val stageInvocations = events
.mapNotNull { it.payload as? ToolInvocationRequestedEvent }
.filter { it.stageId == stageId }
.map { it.invocationId }
.toSet()
val repeated = events
.mapNotNull { it.payload as? ToolExecutionFailedEvent }
.filter { it.invocationId in stageInvocations }
// Collapse to a stable signature so equivalent retries group together: drop digits (package
// versions, ports, counts) and punctuation/paths, keep the tool name and alphabetic core.
// `npm install @types/vite@4.0.0` 404 and `@types/vite@5` 404 map to the same signature.
.groupingBy { failure ->
failure.toolName + "|" + failure.reason.lowercase()
.replace(Regex("[0-9]+"), " ")
.replace(Regex("[^a-z ]+"), " ")
.replace(Regex("\\s+"), " ")
.trim()
.take(FAILURE_SIGNATURE_PREFIX)
}
.eachCount().entries
.firstOrNull { it.value >= limit }
?: return null
return "stage ${stageId.value} is stuck: the same tool failure recurred ${repeated.value} times " +
"(${repeated.key}). Retrying it keeps failing — a materially different action is required."
}
/** /**
* Which bounded precondition-resolution path a repeated build-prerequisite block takes (design #170). * Which bounded precondition-resolution path a repeated build-prerequisite block takes (design #170).
* Pure over the three observable facts — deterministic/replay-safe, unit-tested in isolation. * Pure over the three observable facts — deterministic/replay-safe, unit-tested in isolation.
@@ -230,6 +285,7 @@ private fun isBuildCriticalPath(path: String): Boolean {
private const val BOOTSTRAP_BUDGET = 1 private const val BOOTSTRAP_BUDGET = 1
private val PREREQUISITE_PATH_RE = Regex("""missing build prerequisite '([^']+)'""") private val PREREQUISITE_PATH_RE = Regex("""missing build prerequisite '([^']+)'""")
private const val REFERENCE_EXISTS_REPEAT_LIMIT = 3 private const val REFERENCE_EXISTS_REPEAT_LIMIT = 3
private const val FAILURE_SIGNATURE_PREFIX = 80
private val BUILD_PREREQUISITE_FILENAMES = setOf( private val BUILD_PREREQUISITE_FILENAMES = setOf(
"package.json", "tsconfig.json", "vite.config.ts", "build.gradle", "build.gradle.kts", "pom.xml", "package.json", "tsconfig.json", "vite.config.ts", "build.gradle", "build.gradle.kts", "pom.xml",
) )
@@ -0,0 +1,91 @@
package com.correx.core.kernel.orchestration
import com.correx.core.approvals.Tier
import com.correx.core.events.events.EventMetadata
import com.correx.core.events.events.EventPayload
import com.correx.core.events.events.StoredEvent
import com.correx.core.events.events.ToolExecutionFailedEvent
import com.correx.core.events.events.ToolInvocationRequestedEvent
import com.correx.core.events.events.ToolRequest
import com.correx.core.events.types.EventId
import com.correx.core.events.types.SessionId
import com.correx.core.events.types.StageId
import com.correx.core.events.types.ToolInvocationId
import kotlinx.datetime.Instant
import org.junit.jupiter.api.Assertions.assertNotNull
import org.junit.jupiter.api.Assertions.assertNull
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.Test
class RepeatedToolFailureLoopTest {
private val stage = StageId("install_dependencies")
@Test
fun `same-signature failures trip the limit even when interleaved with successes`() {
// The rejection-loop breaker would reset on the interleaved success; this must not.
val events = buildList {
repeat(3) { addAll(failure("npm install @types/vite@4.0.0 -> registry 404 not found")) }
add(success()) // resets any CONSECUTIVE counter
repeat(3) { addAll(failure("npm install @types/vite@5.1.2 -> registry 404 not found")) }
}
val reason = detectRepeatedToolFailure(events, stage, limit = 6)
assertNotNull(reason)
assertTrue(reason!!.contains("install_dependencies"))
}
@Test
fun `distinct failures below the limit do not trip`() {
val events = listOf(
failure("npm install a -> 404"),
failure("tsc -> type error in App.tsx"),
failure("eslint -> no config found"),
).flatten()
assertNull(detectRepeatedToolFailure(events, stage, limit = 6))
}
@Test
fun `failures from other stages are not counted`() {
val other = StageId("scaffold")
val events = (1..8).flatMap { failure("npm install x -> 404", stageOf = other) }
assertNull(detectRepeatedToolFailure(events, stage, limit = 6))
}
private var seq = 0L
private val session = SessionId("s1")
private fun ev(payload: EventPayload) = StoredEvent(
metadata = EventMetadata(
eventId = EventId("e${seq++}"),
sessionId = session,
timestamp = Instant.parse("2026-01-01T00:00:00Z"),
schemaVersion = 1,
causationId = null,
correlationId = null,
),
sequence = seq,
sessionSequence = seq,
payload = payload,
)
// A failure = the request (carries stageId) + the failure (carries only invocationId), correlated by id.
private fun failure(reason: String, stageOf: StageId = stage): List<StoredEvent> {
val id = ToolInvocationId("inv-${seq}")
val req = ev(
ToolInvocationRequestedEvent(
invocationId = id, sessionId = session, stageId = stageOf,
toolName = "shell", tier = Tier.T1,
request = ToolRequest(id, session, stageOf, "shell"),
),
)
return listOf(req, ev(ToolExecutionFailedEvent(id, session, "shell", reason)))
}
private fun success() = ev(
ToolInvocationRequestedEvent(
invocationId = ToolInvocationId("ok-${seq}"), sessionId = session, stageId = stage,
toolName = "shell", tier = Tier.T1,
request = ToolRequest(ToolInvocationId("ok-$seq"), session, stage, "shell"),
),
)
}