feat(recovery): route failed write-less stages to recovery instead of futile retry
Adds the failure-ticket + recovery-routing mechanism: a deterministic gate->capability table gates whether a stage has agency to fix its own failure, opens a FailureTicketOpenedEvent when it doesn't, and routes to a metadata role=recovery stage (per-stage budget, cap 2, not reset by TransitionExecuted) instead of retrying in place. Extends salvage decisions with a RECOVER option so the review-gate judge can also route to recovery, unifies deterministic-gate and review-gate routing through routeToRecovery(), and has ExecutionPlanCompiler synthesize a write-capable recovery stage + edge for freestyle plans. Read-only tools (file_read, list_dir) are now always available on any tool-granting stage so a recovery stage can inspect the write-less stage's failure without flooding context via shell ls -R.
This commit is contained in:
@@ -126,7 +126,10 @@ class RecoveryRoutingTest {
|
||||
override fun all(): List<Tool> = listOf(tool)
|
||||
}
|
||||
|
||||
private fun buildOrchestrator(): Pair<DefaultSessionOrchestrator, InMemoryEventStore> {
|
||||
private fun buildOrchestrator(
|
||||
staticAnalysis: StaticAnalysisRunner =
|
||||
StaticAnalysisRunner { _, _ -> StaticAnalysisRunResult(exitCode = 1, output = "TS5023: bad option") },
|
||||
): Pair<DefaultSessionOrchestrator, InMemoryEventStore> {
|
||||
val eventStore = InMemoryEventStore()
|
||||
val shell = ShellTool(allowedExecutables = setOf("true"), workingDir = workspace)
|
||||
val toolRegistry = SingleToolRegistry(shell)
|
||||
@@ -141,9 +144,6 @@ class RecoveryRoutingTest {
|
||||
val inferenceRouter = object : InferenceRouter {
|
||||
override suspend fun route(stageId: StageId, requiredCapabilities: Set<ModelCapability>) = provider
|
||||
}
|
||||
val failingStaticAnalysis =
|
||||
StaticAnalysisRunner { _, _ -> StaticAnalysisRunResult(exitCode = 1, output = "TS5023: bad option") }
|
||||
|
||||
val repositories = OrchestratorRepositories(
|
||||
eventStore = eventStore,
|
||||
inferenceRepository = InferenceRepository(object : EventReplayer<InferenceState> {
|
||||
@@ -172,7 +172,7 @@ class RecoveryRoutingTest {
|
||||
toolExecutor = executor,
|
||||
toolRegistry = toolRegistry,
|
||||
workspacePolicy = WorkspacePolicy(workspace),
|
||||
staticAnalysisRunner = failingStaticAnalysis,
|
||||
staticAnalysisRunner = staticAnalysis,
|
||||
approvalEngine = object : ApprovalEngine {
|
||||
override fun evaluate(
|
||||
request: DomainApprovalRequest,
|
||||
@@ -254,4 +254,37 @@ class RecoveryRoutingTest {
|
||||
assertEquals(listOf(1, 2), tickets.map { it.routeAttempt })
|
||||
assertTrue(events.any { it.payload is WorkflowFailedEvent })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `recovery whose failure keeps changing is not charged the route budget (progress-aware)`(): Unit = runBlocking {
|
||||
// Each verify run surfaces a DIFFERENT failure (distinct non-digit words so FailureFingerprint,
|
||||
// which collapses digit runs, actually sees a changed fingerprint). This models a genuine
|
||||
// multi-cause repair: recovery fixes one cause, the gate now fails on the next. Progress must
|
||||
// NOT charge the small route budget — so the run routes to recovery MORE than
|
||||
// RECOVERY_ROUTE_BUDGET times and is instead bounded by the recovery->verify refinement guard.
|
||||
val variants = listOf("cannot resolve module alpha", "cannot resolve module beta", "cannot resolve module gamma", "cannot resolve module delta", "cannot resolve module epsilon")
|
||||
var call = 0
|
||||
val changingStaticAnalysis = StaticAnalysisRunner { _, _ ->
|
||||
StaticAnalysisRunResult(exitCode = 1, output = variants[minOf(call++, variants.lastIndex)])
|
||||
}
|
||||
val (orchestrator, eventStore) = buildOrchestrator(changingStaticAnalysis)
|
||||
val sessionId = SessionId("recovery-progress")
|
||||
val config = OrchestrationConfig(retryPolicy = RetryPolicy(maxAttempts = 10, backoffMs = 0))
|
||||
|
||||
val result = orchestrator.run(sessionId, recoveryGraph(), config)
|
||||
|
||||
assertTrue(result is WorkflowResult.Failed, "still terminates — via the refinement guard, not the route budget")
|
||||
val tickets = eventStore.read(sessionId).mapNotNull { it.payload as? FailureTicketOpenedEvent }
|
||||
assertTrue(tickets.size > RECOVERY_ROUTE_BUDGET_TEST, "progress routes are free, so more than the budget of routes happen: got ${tickets.size}")
|
||||
// Every route progressed (changed fingerprint) except the first, so the charged count never
|
||||
// climbs past its initial 1 — the old unconditional charging would have read [1, 2] then died.
|
||||
assertEquals(1, tickets.map { it.routeAttempt }.max(), "no-progress charge count must never exceed 1 when every round progresses")
|
||||
assertTrue(tickets.map { it.fingerprint }.toSet().size == tickets.size, "each route recorded a distinct fingerprint")
|
||||
}
|
||||
|
||||
private companion object {
|
||||
// Mirror of DefaultSessionOrchestrator.RECOVERY_ROUTE_BUDGET (private there); kept in sync by the
|
||||
// `[1, 2]` assertion in the exhaustion test above.
|
||||
const val RECOVERY_ROUTE_BUDGET_TEST = 2
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user