feat(recovery): route failed write-less stages to recovery instead of futile retry
Adds the failure-ticket + recovery-routing mechanism: a deterministic gate->capability table gates whether a stage has agency to fix its own failure, opens a FailureTicketOpenedEvent when it doesn't, and routes to a metadata role=recovery stage (per-stage budget, cap 2, not reset by TransitionExecuted) instead of retrying in place. Extends salvage decisions with a RECOVER option so the review-gate judge can also route to recovery, unifies deterministic-gate and review-gate routing through routeToRecovery(), and has ExecutionPlanCompiler synthesize a write-capable recovery stage + edge for freestyle plans. Read-only tools (file_read, list_dir) are now always available on any tool-granting stage so a recovery stage can inspect the write-less stage's failure without flooding context via shell ls -R.
This commit is contained in:
+4
-2
@@ -95,10 +95,12 @@ class DefaultOrchestrationReducer : OrchestrationReducer {
|
||||
refinementIterations = state.refinementIterations + (p.cycleKey to p.iteration),
|
||||
)
|
||||
|
||||
// Charge the failing stage's recovery-route budget. routeAttempt is the 1-based count the
|
||||
// orchestrator computed, so fold it directly (idempotent under replay).
|
||||
// Charge the failing stage's recovery-route budget. routeAttempt is the no-progress count the
|
||||
// orchestrator computed (progress-aware), so fold it directly (idempotent under replay), and
|
||||
// record the failure fingerprint the next route compares against.
|
||||
is FailureTicketOpenedEvent -> state.copy(
|
||||
recoveryRoutes = state.recoveryRoutes + (p.stageId.value to p.routeAttempt),
|
||||
recoveryFailureFingerprints = state.recoveryFailureFingerprints + (p.stageId.value to p.fingerprint),
|
||||
)
|
||||
|
||||
else -> state
|
||||
|
||||
+19
-4
@@ -36,6 +36,7 @@ import com.correx.core.kernel.execution.WorkflowResult
|
||||
import com.correx.core.kernel.orchestration.subagent.InSessionSubagentRunner
|
||||
import com.correx.core.kernel.orchestration.subagent.SubagentRunRequest
|
||||
import com.correx.core.kernel.orchestration.subagent.SubagentRunner
|
||||
import com.correx.core.kernel.retry.FailureFingerprint
|
||||
import com.correx.core.kernel.retry.RetryCoordinator
|
||||
import com.correx.core.kernel.retry.RetryDecision
|
||||
import com.correx.core.sessions.Session
|
||||
@@ -298,8 +299,19 @@ class DefaultSessionOrchestrator(
|
||||
state: OrchestrationState,
|
||||
): StepResult? {
|
||||
val recoveryId = findRecoveryStage(ctx.graph, stageId) ?: return null // no recovery stage: legacy path
|
||||
// Progress-aware charging (mirrors the per-gate retry budget, design §5): compare this
|
||||
// failure's fingerprint to the previous route's. A changed fingerprint means the last
|
||||
// recovery round moved the needle — fixed one cause and surfaced a different one — so the
|
||||
// route is FREE (budget unchanged) and only the recorded fingerprint advances. Same
|
||||
// fingerprint = the round changed nothing, so it charges the small route budget. Terminal
|
||||
// only once the no-progress count is spent; the recovery→origin back-edge refinement guard
|
||||
// (executeMove, cap = origin stage maxRetries) remains the ultimate loop bound for the
|
||||
// pathological all-progress case.
|
||||
val fingerprint = FailureFingerprint.of(reason)
|
||||
val prev = state.recoveryFailureFingerprints[stageId.value]
|
||||
val progressed = prev != null && prev != fingerprint
|
||||
val used = state.recoveryRoutes[stageId.value] ?: 0
|
||||
if (used >= RECOVERY_ROUTE_BUDGET) {
|
||||
if (!progressed && used >= RECOVERY_ROUTE_BUDGET) {
|
||||
return StepResult.Terminal(
|
||||
failWorkflow(
|
||||
ctx.sessionId,
|
||||
@@ -309,6 +321,7 @@ class DefaultSessionOrchestrator(
|
||||
),
|
||||
)
|
||||
}
|
||||
val routeAttempt = if (progressed) used else used + 1
|
||||
emit(
|
||||
ctx.sessionId,
|
||||
FailureTicketOpenedEvent(
|
||||
@@ -319,13 +332,15 @@ class DefaultSessionOrchestrator(
|
||||
requiredCapability = requiredCapability,
|
||||
routeTo = recoveryId,
|
||||
evidence = reason,
|
||||
routeAttempt = used + 1,
|
||||
routeAttempt = routeAttempt,
|
||||
fingerprint = fingerprint,
|
||||
),
|
||||
)
|
||||
log.info(
|
||||
"[Orchestrator] failure ticket session={} stage={} gate={} lacks={} -> route to recovery={} ({}/{})",
|
||||
"[Orchestrator] failure ticket session={} stage={} gate={} lacks={} -> recovery={} " +
|
||||
"(charged={}/{}, progressed={})",
|
||||
ctx.sessionId.value, stageId.value, gate, requiredCapability,
|
||||
recoveryId.value, used + 1, RECOVERY_ROUTE_BUDGET,
|
||||
recoveryId.value, routeAttempt, RECOVERY_ROUTE_BUDGET, progressed,
|
||||
)
|
||||
val advancedTo = advanceStage(
|
||||
ctx.sessionId,
|
||||
|
||||
+6
-1
@@ -1915,6 +1915,11 @@ abstract class SessionOrchestrator(
|
||||
is ToolResult.Success -> {
|
||||
val diff = result.metadata["diff"]?.takeIf { it.isNotBlank() }
|
||||
val affected = fileTool?.affectedPaths(request).orEmpty()
|
||||
// Read-only tools (file_read, list_dir) have no FileAffectingTool, so `affected`
|
||||
// is empty — fall back to the raw `path` argument so the receipt still names
|
||||
// the file/dir the tool looked at (surfaced in the TUI's output row).
|
||||
val affectedNames = affected.map { it.toString() }
|
||||
.ifEmpty { listOfNotNull(request.parameters["path"] as? String) }
|
||||
emit(
|
||||
sessionId,
|
||||
ToolExecutionCompletedEvent(
|
||||
@@ -1929,7 +1934,7 @@ abstract class SessionOrchestrator(
|
||||
// Carry the tool's structured metadata (e.g. file_read's contentHash) so
|
||||
// session projections can read it.
|
||||
structuredOutput = result.metadata,
|
||||
affectedEntities = affected.map { it.toString() },
|
||||
affectedEntities = affectedNames,
|
||||
durationMs = 0,
|
||||
tier = tier,
|
||||
timestamp = Clock.System.now(),
|
||||
|
||||
Reference in New Issue
Block a user