feat(kernel): stage-global repeated-failure loop breaker (#78)
Existing read-loop/rejection-loop breakers key on CONSECUTIVE rounds, so a single interleaved success resets them — the hole that let the 2026-07-16 audition run-3 thrash npm install against a hallucinated package across 57 inferences. Add repeatedToolFailureLoop: cumulative count per normalized tool-failure signature within a stage; once a signature hits stageFailureLoopLimit (default 6) the stage fails with STAGE_LOOP_BREAK_GATE and the step handler routes straight to recovery (never retried in place). Pure fold over recorded events (replay-safe, invariants #8/#9); ToolExecutionFailedEvent lacks stageId so failures correlate via invocationId -> ToolInvocationRequestedEvent. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
+5
@@ -69,6 +69,11 @@ internal val EVIDENCE_PATH_RE = Regex("""[\w./@-]+\.[A-Za-z0-9]+""")
|
|||||||
// (bounded bootstrap turn / recovery routing) is triggered off this id before the normal retry path.
|
// (bounded bootstrap turn / recovery routing) is triggered off this id before the normal retry path.
|
||||||
internal const val WORKSPACE_PRECONDITION_GATE = "workspace_precondition"
|
internal const val WORKSPACE_PRECONDITION_GATE = "workspace_precondition"
|
||||||
|
|
||||||
|
// Gate id for a provably-stuck stage failure loop (Vikunja #78). Detected by repeatedToolFailureLoop
|
||||||
|
// when the SAME tool-failure signature recurs past the tuning limit; routed straight to recovery
|
||||||
|
// (never retried in place) by the step handler before the normal retry path.
|
||||||
|
internal const val STAGE_LOOP_BREAK_GATE = "stage_loop_break"
|
||||||
|
|
||||||
internal val GATE_REQUIRED_CAPABILITY: Map<String, String> = mapOf(
|
internal val GATE_REQUIRED_CAPABILITY: Map<String, String> = mapOf(
|
||||||
"execution" to "file_write",
|
"execution" to "file_write",
|
||||||
"contract" to "file_write",
|
"contract" to "file_write",
|
||||||
|
|||||||
+9
@@ -264,6 +264,15 @@ internal suspend fun DefaultSessionOrchestrator.enterStage(
|
|||||||
is PrerequisiteOutcome.Done -> return outcome.result
|
is PrerequisiteOutcome.Done -> return outcome.result
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// A proven same-signature failure loop (Vikunja #78) must never be retried in place —
|
||||||
|
// retrying is what got us here. Route straight to recovery regardless of capability;
|
||||||
|
// terminal if the graph offers no recovery route.
|
||||||
|
if (result.gate == STAGE_LOOP_BREAK_GATE) {
|
||||||
|
return routeToRecovery(ctx, stageId, result.gate, "file_write", result.reason, refreshedState)
|
||||||
|
?: StepResult.Terminal(
|
||||||
|
failWorkflow(ctx.sessionId, stageId, result.reason, retryExhausted = true),
|
||||||
|
)
|
||||||
|
}
|
||||||
// Retry-agency invariant: a gate this stage lacks the capability to fix must not
|
// Retry-agency invariant: a gate this stage lacks the capability to fix must not
|
||||||
// be retried in place (futile). Route to a recovery stage that holds the
|
// be retried in place (futile). Route to a recovery stage that holds the
|
||||||
// capability, if one exists and the route budget remains.
|
// capability, if one exists and the route budget remains.
|
||||||
|
|||||||
+6
@@ -41,4 +41,10 @@ data class OrchestrationTuning(
|
|||||||
val recoveryRouteBudget: Int = 2,
|
val recoveryRouteBudget: Int = 2,
|
||||||
/** Budget for tier-2 intent-holder arbiter re-routing. */
|
/** Budget for tier-2 intent-holder arbiter re-routing. */
|
||||||
val intentRouteBudget: Int = 2,
|
val intentRouteBudget: Int = 2,
|
||||||
|
/**
|
||||||
|
* Cumulative same-signature tool failures within a stage before it's broken out of the loop and
|
||||||
|
* routed to recovery (Vikunja #78). Unlike the consecutive read/rejection nudges, this survives
|
||||||
|
* interleaved successes — it counts a normalized failure signature across the whole stage.
|
||||||
|
*/
|
||||||
|
val stageFailureLoopLimit: Int = 6,
|
||||||
)
|
)
|
||||||
|
|||||||
+3
@@ -417,6 +417,9 @@ internal suspend fun SessionOrchestrator.executeStage(
|
|||||||
repeatedBuildCriticalReferenceBlock(sessionId, stageId)?.let { reason ->
|
repeatedBuildCriticalReferenceBlock(sessionId, stageId)?.let { reason ->
|
||||||
return StageExecutionResult.Failure(reason, retryable = true, gate = WORKSPACE_PRECONDITION_GATE)
|
return StageExecutionResult.Failure(reason, retryable = true, gate = WORKSPACE_PRECONDITION_GATE)
|
||||||
}
|
}
|
||||||
|
repeatedToolFailureLoop(sessionId, stageId, tuning.stageFailureLoopLimit)?.let { reason ->
|
||||||
|
return StageExecutionResult.Failure(reason, retryable = true, gate = STAGE_LOOP_BREAK_GATE)
|
||||||
|
}
|
||||||
val fatalEntry = toolEntries.firstOrNull { it.content.startsWith("FATAL:") }
|
val fatalEntry = toolEntries.firstOrNull { it.content.startsWith("FATAL:") }
|
||||||
if (fatalEntry != null) {
|
if (fatalEntry != null) {
|
||||||
emitProcessResultEvents(sessionId, stageId, stageConfig)
|
emitProcessResultEvents(sessionId, stageId, stageConfig)
|
||||||
|
|||||||
+56
@@ -9,6 +9,7 @@ import com.correx.core.events.events.OrchestrationPausedEvent
|
|||||||
import com.correx.core.events.events.OrchestrationResumedEvent
|
import com.correx.core.events.events.OrchestrationResumedEvent
|
||||||
import com.correx.core.events.events.StoredEvent
|
import com.correx.core.events.events.StoredEvent
|
||||||
import com.correx.core.events.events.ToolCallAssessedEvent
|
import com.correx.core.events.events.ToolCallAssessedEvent
|
||||||
|
import com.correx.core.events.events.ToolExecutionFailedEvent
|
||||||
import com.correx.core.events.events.ToolInvocationRequestedEvent
|
import com.correx.core.events.events.ToolInvocationRequestedEvent
|
||||||
import com.correx.core.events.orchestration.OrchestrationState
|
import com.correx.core.events.orchestration.OrchestrationState
|
||||||
import com.correx.core.events.risk.RiskAction
|
import com.correx.core.events.risk.RiskAction
|
||||||
@@ -59,6 +60,60 @@ internal fun SessionOrchestrator.repeatedBuildCriticalReferenceBlock(
|
|||||||
"(${repeated.value} blocked attempts). Create or repair the project setup before continuing."
|
"(${repeated.value} blocked attempts). Create or repair the project setup before continuing."
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Global backstop for a stage stuck retrying the *same* failing action (Vikunja #78). The existing
|
||||||
|
* read-loop and rejection-loop breakers key on CONSECUTIVE rounds, so a single interleaved success
|
||||||
|
* resets their counters — exactly what let the 2026-07-16 audition run-3 thrash `npm install` against
|
||||||
|
* a hallucinated package across 57 inferences. This counts CUMULATIVELY per normalized failure
|
||||||
|
* signature within the stage: once one signature has failed [limit] times, retrying it is provably
|
||||||
|
* pointless, so return a retryable gate failure that the step handler routes straight to recovery.
|
||||||
|
*
|
||||||
|
* Pure fold over recorded events (replay-safe, invariants #8/#9). [ToolExecutionFailedEvent] carries
|
||||||
|
* no stageId, so failures are correlated to the stage via their invocationId → the emitting
|
||||||
|
* [ToolInvocationRequestedEvent] (which does).
|
||||||
|
*
|
||||||
|
* ponytail: cumulative over the whole stage, not windowed per re-entry — a break escalates to
|
||||||
|
* recovery under a bounded budget, so an unfixable loop terminates rather than re-tripping forever.
|
||||||
|
* Add a per-re-entry window only if a legitimate later attempt gets cut short.
|
||||||
|
*/
|
||||||
|
internal fun SessionOrchestrator.repeatedToolFailureLoop(
|
||||||
|
sessionId: SessionId,
|
||||||
|
stageId: StageId,
|
||||||
|
limit: Int,
|
||||||
|
): String? = detectRepeatedToolFailure(eventStore.read(sessionId), stageId, limit)
|
||||||
|
|
||||||
|
/** Pure core of [repeatedToolFailureLoop] — a fold over recorded events (unit-tested in isolation). */
|
||||||
|
internal fun detectRepeatedToolFailure(
|
||||||
|
events: List<StoredEvent>,
|
||||||
|
stageId: StageId,
|
||||||
|
limit: Int,
|
||||||
|
): String? {
|
||||||
|
val stageInvocations = events
|
||||||
|
.mapNotNull { it.payload as? ToolInvocationRequestedEvent }
|
||||||
|
.filter { it.stageId == stageId }
|
||||||
|
.map { it.invocationId }
|
||||||
|
.toSet()
|
||||||
|
val repeated = events
|
||||||
|
.mapNotNull { it.payload as? ToolExecutionFailedEvent }
|
||||||
|
.filter { it.invocationId in stageInvocations }
|
||||||
|
// Collapse to a stable signature so equivalent retries group together: drop digits (package
|
||||||
|
// versions, ports, counts) and punctuation/paths, keep the tool name and alphabetic core.
|
||||||
|
// `npm install @types/vite@4.0.0` 404 and `@types/vite@5` 404 map to the same signature.
|
||||||
|
.groupingBy { failure ->
|
||||||
|
failure.toolName + "|" + failure.reason.lowercase()
|
||||||
|
.replace(Regex("[0-9]+"), " ")
|
||||||
|
.replace(Regex("[^a-z ]+"), " ")
|
||||||
|
.replace(Regex("\\s+"), " ")
|
||||||
|
.trim()
|
||||||
|
.take(FAILURE_SIGNATURE_PREFIX)
|
||||||
|
}
|
||||||
|
.eachCount().entries
|
||||||
|
.firstOrNull { it.value >= limit }
|
||||||
|
?: return null
|
||||||
|
return "stage ${stageId.value} is stuck: the same tool failure recurred ${repeated.value} times " +
|
||||||
|
"(${repeated.key}). Retrying it keeps failing — a materially different action is required."
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Which bounded precondition-resolution path a repeated build-prerequisite block takes (design #170).
|
* Which bounded precondition-resolution path a repeated build-prerequisite block takes (design #170).
|
||||||
* Pure over the three observable facts — deterministic/replay-safe, unit-tested in isolation.
|
* Pure over the three observable facts — deterministic/replay-safe, unit-tested in isolation.
|
||||||
@@ -230,6 +285,7 @@ private fun isBuildCriticalPath(path: String): Boolean {
|
|||||||
private const val BOOTSTRAP_BUDGET = 1
|
private const val BOOTSTRAP_BUDGET = 1
|
||||||
private val PREREQUISITE_PATH_RE = Regex("""missing build prerequisite '([^']+)'""")
|
private val PREREQUISITE_PATH_RE = Regex("""missing build prerequisite '([^']+)'""")
|
||||||
private const val REFERENCE_EXISTS_REPEAT_LIMIT = 3
|
private const val REFERENCE_EXISTS_REPEAT_LIMIT = 3
|
||||||
|
private const val FAILURE_SIGNATURE_PREFIX = 80
|
||||||
private val BUILD_PREREQUISITE_FILENAMES = setOf(
|
private val BUILD_PREREQUISITE_FILENAMES = setOf(
|
||||||
"package.json", "tsconfig.json", "vite.config.ts", "build.gradle", "build.gradle.kts", "pom.xml",
|
"package.json", "tsconfig.json", "vite.config.ts", "build.gradle", "build.gradle.kts", "pom.xml",
|
||||||
)
|
)
|
||||||
|
|||||||
+91
@@ -0,0 +1,91 @@
|
|||||||
|
package com.correx.core.kernel.orchestration
|
||||||
|
|
||||||
|
import com.correx.core.approvals.Tier
|
||||||
|
import com.correx.core.events.events.EventMetadata
|
||||||
|
import com.correx.core.events.events.EventPayload
|
||||||
|
import com.correx.core.events.events.StoredEvent
|
||||||
|
import com.correx.core.events.events.ToolExecutionFailedEvent
|
||||||
|
import com.correx.core.events.events.ToolInvocationRequestedEvent
|
||||||
|
import com.correx.core.events.events.ToolRequest
|
||||||
|
import com.correx.core.events.types.EventId
|
||||||
|
import com.correx.core.events.types.SessionId
|
||||||
|
import com.correx.core.events.types.StageId
|
||||||
|
import com.correx.core.events.types.ToolInvocationId
|
||||||
|
import kotlinx.datetime.Instant
|
||||||
|
import org.junit.jupiter.api.Assertions.assertNotNull
|
||||||
|
import org.junit.jupiter.api.Assertions.assertNull
|
||||||
|
import org.junit.jupiter.api.Assertions.assertTrue
|
||||||
|
import org.junit.jupiter.api.Test
|
||||||
|
|
||||||
|
class RepeatedToolFailureLoopTest {
|
||||||
|
|
||||||
|
private val stage = StageId("install_dependencies")
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `same-signature failures trip the limit even when interleaved with successes`() {
|
||||||
|
// The rejection-loop breaker would reset on the interleaved success; this must not.
|
||||||
|
val events = buildList {
|
||||||
|
repeat(3) { addAll(failure("npm install @types/vite@4.0.0 -> registry 404 not found")) }
|
||||||
|
add(success()) // resets any CONSECUTIVE counter
|
||||||
|
repeat(3) { addAll(failure("npm install @types/vite@5.1.2 -> registry 404 not found")) }
|
||||||
|
}
|
||||||
|
val reason = detectRepeatedToolFailure(events, stage, limit = 6)
|
||||||
|
assertNotNull(reason)
|
||||||
|
assertTrue(reason!!.contains("install_dependencies"))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `distinct failures below the limit do not trip`() {
|
||||||
|
val events = listOf(
|
||||||
|
failure("npm install a -> 404"),
|
||||||
|
failure("tsc -> type error in App.tsx"),
|
||||||
|
failure("eslint -> no config found"),
|
||||||
|
).flatten()
|
||||||
|
assertNull(detectRepeatedToolFailure(events, stage, limit = 6))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `failures from other stages are not counted`() {
|
||||||
|
val other = StageId("scaffold")
|
||||||
|
val events = (1..8).flatMap { failure("npm install x -> 404", stageOf = other) }
|
||||||
|
assertNull(detectRepeatedToolFailure(events, stage, limit = 6))
|
||||||
|
}
|
||||||
|
|
||||||
|
private var seq = 0L
|
||||||
|
private val session = SessionId("s1")
|
||||||
|
|
||||||
|
private fun ev(payload: EventPayload) = StoredEvent(
|
||||||
|
metadata = EventMetadata(
|
||||||
|
eventId = EventId("e${seq++}"),
|
||||||
|
sessionId = session,
|
||||||
|
timestamp = Instant.parse("2026-01-01T00:00:00Z"),
|
||||||
|
schemaVersion = 1,
|
||||||
|
causationId = null,
|
||||||
|
correlationId = null,
|
||||||
|
),
|
||||||
|
sequence = seq,
|
||||||
|
sessionSequence = seq,
|
||||||
|
payload = payload,
|
||||||
|
)
|
||||||
|
|
||||||
|
// A failure = the request (carries stageId) + the failure (carries only invocationId), correlated by id.
|
||||||
|
private fun failure(reason: String, stageOf: StageId = stage): List<StoredEvent> {
|
||||||
|
val id = ToolInvocationId("inv-${seq}")
|
||||||
|
val req = ev(
|
||||||
|
ToolInvocationRequestedEvent(
|
||||||
|
invocationId = id, sessionId = session, stageId = stageOf,
|
||||||
|
toolName = "shell", tier = Tier.T1,
|
||||||
|
request = ToolRequest(id, session, stageOf, "shell"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return listOf(req, ev(ToolExecutionFailedEvent(id, session, "shell", reason)))
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun success() = ev(
|
||||||
|
ToolInvocationRequestedEvent(
|
||||||
|
invocationId = ToolInvocationId("ok-${seq}"), sessionId = session, stageId = stage,
|
||||||
|
toolName = "shell", tier = Tier.T1,
|
||||||
|
request = ToolRequest(ToolInvocationId("ok-$seq"), session, stage, "shell"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user