feat(recovery): route failed write-less stages to recovery instead of futile retry

Adds the failure-ticket + recovery-routing mechanism: a deterministic
gate->capability table gates whether a stage has agency to fix its own
failure, opens a FailureTicketOpenedEvent when it doesn't, and routes to
a metadata role=recovery stage (per-stage budget, cap 2, not reset by
TransitionExecuted) instead of retrying in place. Extends salvage
decisions with a RECOVER option so the review-gate judge can also route
to recovery, unifies deterministic-gate and review-gate routing through
routeToRecovery(), and has ExecutionPlanCompiler synthesize a
write-capable recovery stage + edge for freestyle plans. Read-only tools
(file_read, list_dir) are now always available on any tool-granting
stage so a recovery stage can inspect the write-less stage's failure
without flooding context via shell ls -R.
This commit is contained in:
2026-07-08 10:28:53 +04:00
parent d6bada6f10
commit f51a8dada4
21 changed files with 325 additions and 29 deletions
@@ -126,7 +126,10 @@ class RecoveryRoutingTest {
override fun all(): List<Tool> = listOf(tool)
}
private fun buildOrchestrator(): Pair<DefaultSessionOrchestrator, InMemoryEventStore> {
private fun buildOrchestrator(
staticAnalysis: StaticAnalysisRunner =
StaticAnalysisRunner { _, _ -> StaticAnalysisRunResult(exitCode = 1, output = "TS5023: bad option") },
): Pair<DefaultSessionOrchestrator, InMemoryEventStore> {
val eventStore = InMemoryEventStore()
val shell = ShellTool(allowedExecutables = setOf("true"), workingDir = workspace)
val toolRegistry = SingleToolRegistry(shell)
@@ -141,9 +144,6 @@ class RecoveryRoutingTest {
val inferenceRouter = object : InferenceRouter {
override suspend fun route(stageId: StageId, requiredCapabilities: Set<ModelCapability>) = provider
}
val failingStaticAnalysis =
StaticAnalysisRunner { _, _ -> StaticAnalysisRunResult(exitCode = 1, output = "TS5023: bad option") }
val repositories = OrchestratorRepositories(
eventStore = eventStore,
inferenceRepository = InferenceRepository(object : EventReplayer<InferenceState> {
@@ -172,7 +172,7 @@ class RecoveryRoutingTest {
toolExecutor = executor,
toolRegistry = toolRegistry,
workspacePolicy = WorkspacePolicy(workspace),
staticAnalysisRunner = failingStaticAnalysis,
staticAnalysisRunner = staticAnalysis,
approvalEngine = object : ApprovalEngine {
override fun evaluate(
request: DomainApprovalRequest,
@@ -254,4 +254,37 @@ class RecoveryRoutingTest {
assertEquals(listOf(1, 2), tickets.map { it.routeAttempt })
assertTrue(events.any { it.payload is WorkflowFailedEvent })
}
@Test
fun `recovery whose failure keeps changing is not charged the route budget (progress-aware)`(): Unit = runBlocking {
// Each verify run surfaces a DIFFERENT failure (distinct non-digit words so FailureFingerprint,
// which collapses digit runs, actually sees a changed fingerprint). This models a genuine
// multi-cause repair: recovery fixes one cause, the gate now fails on the next. Progress must
// NOT charge the small route budget — so the run routes to recovery MORE than
// RECOVERY_ROUTE_BUDGET times and is instead bounded by the recovery->verify refinement guard.
val variants = listOf("cannot resolve module alpha", "cannot resolve module beta", "cannot resolve module gamma", "cannot resolve module delta", "cannot resolve module epsilon")
var call = 0
val changingStaticAnalysis = StaticAnalysisRunner { _, _ ->
StaticAnalysisRunResult(exitCode = 1, output = variants[minOf(call++, variants.lastIndex)])
}
val (orchestrator, eventStore) = buildOrchestrator(changingStaticAnalysis)
val sessionId = SessionId("recovery-progress")
val config = OrchestrationConfig(retryPolicy = RetryPolicy(maxAttempts = 10, backoffMs = 0))
val result = orchestrator.run(sessionId, recoveryGraph(), config)
assertTrue(result is WorkflowResult.Failed, "still terminates — via the refinement guard, not the route budget")
val tickets = eventStore.read(sessionId).mapNotNull { it.payload as? FailureTicketOpenedEvent }
assertTrue(tickets.size > RECOVERY_ROUTE_BUDGET_TEST, "progress routes are free, so more than the budget of routes happen: got ${tickets.size}")
// Every route progressed (changed fingerprint) except the first, so the charged count never
// climbs past its initial 1 — the old unconditional charging would have read [1, 2] then died.
assertEquals(1, tickets.map { it.routeAttempt }.max(), "no-progress charge count must never exceed 1 when every round progresses")
assertTrue(tickets.map { it.fingerprint }.toSet().size == tickets.size, "each route recorded a distinct fingerprint")
}
private companion object {
// Mirror of DefaultSessionOrchestrator.RECOVERY_ROUTE_BUDGET (private there); kept in sync by the
// `[1, 2]` assertion in the exhaustion test above.
const val RECOVERY_ROUTE_BUDGET_TEST = 2
}
}