fix(kernel): reset the stage_loop_break window on recovery entry (#304)
Recovery was dying on a stale failure cap, not on anything recovery itself did. detectRepeatedToolFailure (stage_loop_break, #78) counted same-signature ToolExecutionFailedEvents cumulatively across the whole stage. A stage routed to recovery, retried by the ladder, and routed again would re-trip the SAME pre-recovery count on its very first post-recovery round — recovery would burn a full expensive turn and still fail on retryExhausted=true from stale evidence. Fix: window the fold to events after the most recent FailureTicketOpenedEvent naming the stage (mirrors the existing BuildPrerequisiteBootstrapAttemptedEvent windowing pattern already used by repeatedBuildCriticalReferenceBlock). A stage returning from a genuine repair attempt now starts the count clean; it must accumulate stageFailureLoopLimit (default 6) NEW failures post-recovery before re-tripping. This does not weaken RECOVERY_ROUTE_BUDGET/INTENT_ROUTE_BUDGET: those are charged directly off FailureTicketOpenedEvent by the reducer (OrchestrationState.recoveryRoutes/recoveryFailureFingerprints), independent of this fold, and are deliberately left untouched by TransitionExecutedEvent already. A full ladder round-trip therefore stays bounded at RECOVERY_ROUTE_BUDGET(2) + INTENT_ROUTE_BUDGET(2) route-in/route-out cycles, each itself requiring a fresh stageFailureLoopLimit(6) failures to re-trip. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HgDL1v3GuQ9RZnYR6fDT95
This commit is contained in:
+16
-3
@@ -5,6 +5,7 @@ import com.correx.core.events.events.ClarificationQuestion
|
|||||||
import com.correx.core.events.events.ClarificationRequestedEvent
|
import com.correx.core.events.events.ClarificationRequestedEvent
|
||||||
import com.correx.core.events.events.ClarificationAnswer
|
import com.correx.core.events.events.ClarificationAnswer
|
||||||
import com.correx.core.events.events.ClarificationAnsweredEvent
|
import com.correx.core.events.events.ClarificationAnsweredEvent
|
||||||
|
import com.correx.core.events.events.FailureTicketOpenedEvent
|
||||||
import com.correx.core.events.events.OrchestrationPausedEvent
|
import com.correx.core.events.events.OrchestrationPausedEvent
|
||||||
import com.correx.core.events.events.OrchestrationResumedEvent
|
import com.correx.core.events.events.OrchestrationResumedEvent
|
||||||
import com.correx.core.events.events.StoredEvent
|
import com.correx.core.events.events.StoredEvent
|
||||||
@@ -74,7 +75,15 @@ internal fun SessionOrchestrator.repeatedBuildCriticalReferenceBlock(
|
|||||||
*
|
*
|
||||||
* ponytail: cumulative over the whole stage, not windowed per re-entry — a break escalates to
|
* ponytail: cumulative over the whole stage, not windowed per re-entry — a break escalates to
|
||||||
* recovery under a bounded budget, so an unfixable loop terminates rather than re-tripping forever.
|
* recovery under a bounded budget, so an unfixable loop terminates rather than re-tripping forever.
|
||||||
* Add a per-re-entry window only if a legitimate later attempt gets cut short.
|
*
|
||||||
|
* #304: the window IS reset once the stage has been routed to recovery (a [FailureTicketOpenedEvent]
|
||||||
|
* naming it), so a stage returning from a genuine repair attempt gets a clean count instead of
|
||||||
|
* re-tripping this gate on its very first post-recovery execution off stale, pre-recovery failures —
|
||||||
|
* which is exactly what turned recovery into an expensive predetermined dead end (session
|
||||||
|
* 67ef4b3f-9ce9-4436-9870-543feb0ca450). The recovery ROUTE budget ([RECOVERY_ROUTE_BUDGET] /
|
||||||
|
* `recoveryRoutes`) is untouched by this reset — it is charged by the reducer off the ticket event
|
||||||
|
* itself, independent of this fold — so a stage that keeps failing after recovery still terminates
|
||||||
|
* once that budget is spent.
|
||||||
*/
|
*/
|
||||||
internal fun SessionOrchestrator.repeatedToolFailureLoop(
|
internal fun SessionOrchestrator.repeatedToolFailureLoop(
|
||||||
sessionId: SessionId,
|
sessionId: SessionId,
|
||||||
@@ -88,12 +97,16 @@ internal fun detectRepeatedToolFailure(
|
|||||||
stageId: StageId,
|
stageId: StageId,
|
||||||
limit: Int,
|
limit: Int,
|
||||||
): String? {
|
): String? {
|
||||||
val stageInvocations = events
|
val windowStart = events
|
||||||
|
.filter { (it.payload as? FailureTicketOpenedEvent)?.stageId == stageId }
|
||||||
|
.maxOfOrNull { it.sequence } ?: Long.MIN_VALUE
|
||||||
|
val windowed = events.filter { it.sequence > windowStart }
|
||||||
|
val stageInvocations = windowed
|
||||||
.mapNotNull { it.payload as? ToolInvocationRequestedEvent }
|
.mapNotNull { it.payload as? ToolInvocationRequestedEvent }
|
||||||
.filter { it.stageId == stageId }
|
.filter { it.stageId == stageId }
|
||||||
.map { it.invocationId }
|
.map { it.invocationId }
|
||||||
.toSet()
|
.toSet()
|
||||||
val repeated = events
|
val repeated = windowed
|
||||||
.mapNotNull { it.payload as? ToolExecutionFailedEvent }
|
.mapNotNull { it.payload as? ToolExecutionFailedEvent }
|
||||||
.filter { it.invocationId in stageInvocations }
|
.filter { it.invocationId in stageInvocations }
|
||||||
// Collapse to a stable signature so equivalent retries group together: drop digits (package
|
// Collapse to a stable signature so equivalent retries group together: drop digits (package
|
||||||
|
|||||||
+32
@@ -3,6 +3,7 @@ package com.correx.core.kernel.orchestration
|
|||||||
import com.correx.core.approvals.Tier
|
import com.correx.core.approvals.Tier
|
||||||
import com.correx.core.events.events.EventMetadata
|
import com.correx.core.events.events.EventMetadata
|
||||||
import com.correx.core.events.events.EventPayload
|
import com.correx.core.events.events.EventPayload
|
||||||
|
import com.correx.core.events.events.FailureTicketOpenedEvent
|
||||||
import com.correx.core.events.events.StoredEvent
|
import com.correx.core.events.events.StoredEvent
|
||||||
import com.correx.core.events.events.ToolExecutionFailedEvent
|
import com.correx.core.events.events.ToolExecutionFailedEvent
|
||||||
import com.correx.core.events.events.ToolInvocationRequestedEvent
|
import com.correx.core.events.events.ToolInvocationRequestedEvent
|
||||||
@@ -44,6 +45,37 @@ class RepeatedToolFailureLoopTest {
|
|||||||
assertNull(detectRepeatedToolFailure(events, stage, limit = 6))
|
assertNull(detectRepeatedToolFailure(events, stage, limit = 6))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `#304 - a FailureTicketOpenedEvent for the stage resets the window so stale failures don't re-trip`() {
|
||||||
|
// 5 pre-recovery failures (below limit 6) + a ticket routing to recovery + 5 MORE post-recovery
|
||||||
|
// failures of the SAME signature must not sum to 10 and trip the gate — recovery gets a clean
|
||||||
|
// count, so this must stay null until 6 NEW failures accumulate after the ticket.
|
||||||
|
val events = buildList {
|
||||||
|
repeat(5) { addAll(failure("build gate: queries.ts(39,3): error TS1005: '}' expected")) }
|
||||||
|
add(
|
||||||
|
ev(
|
||||||
|
FailureTicketOpenedEvent(
|
||||||
|
sessionId = session,
|
||||||
|
stageId = stage,
|
||||||
|
gate = "stage_loop_break",
|
||||||
|
category = "implementation",
|
||||||
|
requiredCapability = "file_write",
|
||||||
|
routeTo = StageId("recovery"),
|
||||||
|
evidence = "stuck",
|
||||||
|
routeAttempt = 1,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
repeat(5) { addAll(failure("build gate: queries.ts(39,3): error TS1005: '}' expected")) }
|
||||||
|
}
|
||||||
|
assertNull(detectRepeatedToolFailure(events, stage, limit = 6))
|
||||||
|
|
||||||
|
// A 6th post-ticket failure of the same signature DOES trip it — the reset only clears stale
|
||||||
|
// pre-recovery count, it does not disable the breaker going forward.
|
||||||
|
val tripped = events + failure("build gate: queries.ts(39,3): error TS1005: '}' expected")
|
||||||
|
assertNotNull(detectRepeatedToolFailure(tripped, stage, limit = 6))
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
fun `failures from other stages are not counted`() {
|
fun `failures from other stages are not counted`() {
|
||||||
val other = StageId("scaffold")
|
val other = StageId("scaffold")
|
||||||
|
|||||||
Reference in New Issue
Block a user