diff --git a/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/DefaultSessionOrchestrator.kt b/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/DefaultSessionOrchestrator.kt index f7610edd..2ad993e8 100644 --- a/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/DefaultSessionOrchestrator.kt +++ b/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/DefaultSessionOrchestrator.kt @@ -69,6 +69,11 @@ internal val EVIDENCE_PATH_RE = Regex("""[\w./@-]+\.[A-Za-z0-9]+""") // (bounded bootstrap turn / recovery routing) is triggered off this id before the normal retry path. internal const val WORKSPACE_PRECONDITION_GATE = "workspace_precondition" +// Gate id for a provably-stuck stage failure loop (Vikunja #78). Detected by repeatedToolFailureLoop +// when the SAME tool-failure signature recurs past the tuning limit; routed straight to recovery +// (never retried in place) by the step handler before the normal retry path. +internal const val STAGE_LOOP_BREAK_GATE = "stage_loop_break" + internal val GATE_REQUIRED_CAPABILITY: Map = mapOf( "execution" to "file_write", "contract" to "file_write", diff --git a/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/DefaultSessionOrchestratorStep.kt b/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/DefaultSessionOrchestratorStep.kt index 40872e01..263c6bb9 100644 --- a/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/DefaultSessionOrchestratorStep.kt +++ b/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/DefaultSessionOrchestratorStep.kt @@ -264,6 +264,15 @@ internal suspend fun DefaultSessionOrchestrator.enterStage( is PrerequisiteOutcome.Done -> return outcome.result } } + // A proven same-signature failure loop (Vikunja #78) must never be retried in place — + // retrying is what got us here. Route straight to recovery regardless of capability; + // terminal if the graph offers no recovery route. + if (result.gate == STAGE_LOOP_BREAK_GATE) { + return routeToRecovery(ctx, stageId, result.gate, "file_write", result.reason, refreshedState) + ?: StepResult.Terminal( + failWorkflow(ctx.sessionId, stageId, result.reason, retryExhausted = true), + ) + } // Retry-agency invariant: a gate this stage lacks the capability to fix must not // be retried in place (futile). Route to a recovery stage that holds the // capability, if one exists and the route budget remains. diff --git a/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/OrchestrationTuning.kt b/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/OrchestrationTuning.kt index 891dab48..173c2a88 100644 --- a/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/OrchestrationTuning.kt +++ b/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/OrchestrationTuning.kt @@ -41,4 +41,10 @@ data class OrchestrationTuning( val recoveryRouteBudget: Int = 2, /** Budget for tier-2 intent-holder arbiter re-routing. */ val intentRouteBudget: Int = 2, + /** + * Cumulative same-signature tool failures within a stage before it's broken out of the loop and + * routed to recovery (Vikunja #78). Unlike the consecutive read/rejection nudges, this survives + * interleaved successes — it counts a normalized failure signature across the whole stage. + */ + val stageFailureLoopLimit: Int = 6, ) diff --git a/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/SessionOrchestratorExecution.kt b/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/SessionOrchestratorExecution.kt index ab4338a8..902c5966 100644 --- a/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/SessionOrchestratorExecution.kt +++ b/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/SessionOrchestratorExecution.kt @@ -417,6 +417,9 @@ internal suspend fun SessionOrchestrator.executeStage( repeatedBuildCriticalReferenceBlock(sessionId, stageId)?.let { reason -> return StageExecutionResult.Failure(reason, retryable = true, gate = WORKSPACE_PRECONDITION_GATE) } + repeatedToolFailureLoop(sessionId, stageId, tuning.stageFailureLoopLimit)?.let { reason -> + return StageExecutionResult.Failure(reason, retryable = true, gate = STAGE_LOOP_BREAK_GATE) + } val fatalEntry = toolEntries.firstOrNull { it.content.startsWith("FATAL:") } if (fatalEntry != null) { emitProcessResultEvents(sessionId, stageId, stageConfig) diff --git a/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/SessionOrchestratorPreconditions.kt b/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/SessionOrchestratorPreconditions.kt index 7e5388a9..53802301 100644 --- a/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/SessionOrchestratorPreconditions.kt +++ b/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/SessionOrchestratorPreconditions.kt @@ -9,6 +9,7 @@ import com.correx.core.events.events.OrchestrationPausedEvent import com.correx.core.events.events.OrchestrationResumedEvent import com.correx.core.events.events.StoredEvent import com.correx.core.events.events.ToolCallAssessedEvent +import com.correx.core.events.events.ToolExecutionFailedEvent import com.correx.core.events.events.ToolInvocationRequestedEvent import com.correx.core.events.orchestration.OrchestrationState import com.correx.core.events.risk.RiskAction @@ -59,6 +60,60 @@ internal fun SessionOrchestrator.repeatedBuildCriticalReferenceBlock( "(${repeated.value} blocked attempts). Create or repair the project setup before continuing." } +/** + * Global backstop for a stage stuck retrying the *same* failing action (Vikunja #78). The existing + * read-loop and rejection-loop breakers key on CONSECUTIVE rounds, so a single interleaved success + * resets their counters — exactly what let the 2026-07-16 audition run-3 thrash `npm install` against + * a hallucinated package across 57 inferences. This counts CUMULATIVELY per normalized failure + * signature within the stage: once one signature has failed [limit] times, retrying it is provably + * pointless, so return a retryable gate failure that the step handler routes straight to recovery. + * + * Pure fold over recorded events (replay-safe, invariants #8/#9). [ToolExecutionFailedEvent] carries + * no stageId, so failures are correlated to the stage via their invocationId → the emitting + * [ToolInvocationRequestedEvent] (which does). + * + * ponytail: cumulative over the whole stage, not windowed per re-entry — a break escalates to + * recovery under a bounded budget, so an unfixable loop terminates rather than re-tripping forever. + * Add a per-re-entry window only if a legitimate later attempt gets cut short. + */ +internal fun SessionOrchestrator.repeatedToolFailureLoop( + sessionId: SessionId, + stageId: StageId, + limit: Int, +): String? = detectRepeatedToolFailure(eventStore.read(sessionId), stageId, limit) + +/** Pure core of [repeatedToolFailureLoop] — a fold over recorded events (unit-tested in isolation). */ +internal fun detectRepeatedToolFailure( + events: List, + stageId: StageId, + limit: Int, +): String? { + val stageInvocations = events + .mapNotNull { it.payload as? ToolInvocationRequestedEvent } + .filter { it.stageId == stageId } + .map { it.invocationId } + .toSet() + val repeated = events + .mapNotNull { it.payload as? ToolExecutionFailedEvent } + .filter { it.invocationId in stageInvocations } + // Collapse to a stable signature so equivalent retries group together: drop digits (package + // versions, ports, counts) and punctuation/paths, keep the tool name and alphabetic core. + // `npm install @types/vite@4.0.0` 404 and `@types/vite@5` 404 map to the same signature. + .groupingBy { failure -> + failure.toolName + "|" + failure.reason.lowercase() + .replace(Regex("[0-9]+"), " ") + .replace(Regex("[^a-z ]+"), " ") + .replace(Regex("\\s+"), " ") + .trim() + .take(FAILURE_SIGNATURE_PREFIX) + } + .eachCount().entries + .firstOrNull { it.value >= limit } + ?: return null + return "stage ${stageId.value} is stuck: the same tool failure recurred ${repeated.value} times " + + "(${repeated.key}). Retrying it keeps failing — a materially different action is required." +} + /** * Which bounded precondition-resolution path a repeated build-prerequisite block takes (design #170). * Pure over the three observable facts — deterministic/replay-safe, unit-tested in isolation. @@ -230,6 +285,7 @@ private fun isBuildCriticalPath(path: String): Boolean { private const val BOOTSTRAP_BUDGET = 1 private val PREREQUISITE_PATH_RE = Regex("""missing build prerequisite '([^']+)'""") private const val REFERENCE_EXISTS_REPEAT_LIMIT = 3 +private const val FAILURE_SIGNATURE_PREFIX = 80 private val BUILD_PREREQUISITE_FILENAMES = setOf( "package.json", "tsconfig.json", "vite.config.ts", "build.gradle", "build.gradle.kts", "pom.xml", ) diff --git a/core/kernel/src/test/kotlin/com/correx/core/kernel/orchestration/RepeatedToolFailureLoopTest.kt b/core/kernel/src/test/kotlin/com/correx/core/kernel/orchestration/RepeatedToolFailureLoopTest.kt new file mode 100644 index 00000000..ed147068 --- /dev/null +++ b/core/kernel/src/test/kotlin/com/correx/core/kernel/orchestration/RepeatedToolFailureLoopTest.kt @@ -0,0 +1,91 @@ +package com.correx.core.kernel.orchestration + +import com.correx.core.approvals.Tier +import com.correx.core.events.events.EventMetadata +import com.correx.core.events.events.EventPayload +import com.correx.core.events.events.StoredEvent +import com.correx.core.events.events.ToolExecutionFailedEvent +import com.correx.core.events.events.ToolInvocationRequestedEvent +import com.correx.core.events.events.ToolRequest +import com.correx.core.events.types.EventId +import com.correx.core.events.types.SessionId +import com.correx.core.events.types.StageId +import com.correx.core.events.types.ToolInvocationId +import kotlinx.datetime.Instant +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class RepeatedToolFailureLoopTest { + + private val stage = StageId("install_dependencies") + + @Test + fun `same-signature failures trip the limit even when interleaved with successes`() { + // The rejection-loop breaker would reset on the interleaved success; this must not. + val events = buildList { + repeat(3) { addAll(failure("npm install @types/vite@4.0.0 -> registry 404 not found")) } + add(success()) // resets any CONSECUTIVE counter + repeat(3) { addAll(failure("npm install @types/vite@5.1.2 -> registry 404 not found")) } + } + val reason = detectRepeatedToolFailure(events, stage, limit = 6) + assertNotNull(reason) + assertTrue(reason!!.contains("install_dependencies")) + } + + @Test + fun `distinct failures below the limit do not trip`() { + val events = listOf( + failure("npm install a -> 404"), + failure("tsc -> type error in App.tsx"), + failure("eslint -> no config found"), + ).flatten() + assertNull(detectRepeatedToolFailure(events, stage, limit = 6)) + } + + @Test + fun `failures from other stages are not counted`() { + val other = StageId("scaffold") + val events = (1..8).flatMap { failure("npm install x -> 404", stageOf = other) } + assertNull(detectRepeatedToolFailure(events, stage, limit = 6)) + } + + private var seq = 0L + private val session = SessionId("s1") + + private fun ev(payload: EventPayload) = StoredEvent( + metadata = EventMetadata( + eventId = EventId("e${seq++}"), + sessionId = session, + timestamp = Instant.parse("2026-01-01T00:00:00Z"), + schemaVersion = 1, + causationId = null, + correlationId = null, + ), + sequence = seq, + sessionSequence = seq, + payload = payload, + ) + + // A failure = the request (carries stageId) + the failure (carries only invocationId), correlated by id. + private fun failure(reason: String, stageOf: StageId = stage): List { + val id = ToolInvocationId("inv-${seq}") + val req = ev( + ToolInvocationRequestedEvent( + invocationId = id, sessionId = session, stageId = stageOf, + toolName = "shell", tier = Tier.T1, + request = ToolRequest(id, session, stageOf, "shell"), + ), + ) + return listOf(req, ev(ToolExecutionFailedEvent(id, session, "shell", reason))) + } + + private fun success() = ev( + ToolInvocationRequestedEvent( + invocationId = ToolInvocationId("ok-${seq}"), sessionId = session, stageId = stage, + toolName = "shell", tier = Tier.T1, + request = ToolRequest(ToolInvocationId("ok-$seq"), session, stage, "shell"), + ), + ) +}