feat(recovery): route failed write-less stages to recovery instead of futile retry

Adds the failure-ticket + recovery-routing mechanism: a deterministic
gate->capability table gates whether a stage has agency to fix its own
failure, opens a FailureTicketOpenedEvent when it doesn't, and routes to
a metadata role=recovery stage (per-stage budget, cap 2, not reset by
TransitionExecuted) instead of retrying in place. Extends salvage
decisions with a RECOVER option so the review-gate judge can also route
to recovery, unifies deterministic-gate and review-gate routing through
routeToRecovery(), and has ExecutionPlanCompiler synthesize a
write-capable recovery stage + edge for freestyle plans. Read-only tools
(file_read, list_dir) are now always available on any tool-granting
stage so a recovery stage can inspect the write-less stage's failure
without flooding context via shell ls -R.
This commit is contained in:
2026-07-08 10:28:53 +04:00
parent d6bada6f10
commit f51a8dada4
21 changed files with 325 additions and 29 deletions
@@ -145,6 +145,13 @@ object KindContractTable {
val overrides = projectOverrides[kindId].orEmpty()
.filter { ov -> base.none { it.id == ov.id } }
.map { it.copy(target = path) }
return base + overrides
// A dotfile (basename begins with '.') may legitimately be empty: .gitkeep/.keep directory
// placeholders, .nojekyll markers, an empty .gitignore. Asserting file_nonempty on one makes
// the contract unsatisfiable by construction — a real .gitkeep IS empty — which live-locks the
// producing stage rewriting the same placeholder every retry (observed: scaffold_frontend
// looping on four src/*/.gitkeep files). file_exists still applies, so the file must exist.
return (base + overrides).filterNot { it.id == "file_nonempty" && isDotfile(path) }
}
private fun isDotfile(path: String): Boolean = path.substringAfterLast('/').startsWith(".")
}
@@ -61,6 +61,32 @@ class KindContractTableTest {
assertTrue("registered_in_serialization" in ids)
}
@Test
fun `dotfiles are exempt from file_nonempty but still must exist`() {
// Regression: a .gitkeep placeholder is empty by definition, so a file_nonempty floor made
// its contract unsatisfiable and live-locked scaffold_frontend rewriting src/*/.gitkeep every
// retry. Any dotfile (basename begins with '.') may legitimately be empty.
val gitkeep = KindContractTable.assertionsFor(
KindInference.kindFor("frontend/src/hooks/.gitkeep") ?: "",
"frontend/src/hooks/.gitkeep",
).map { it.id }
assertTrue("file_exists" in gitkeep, "dotfile must still be required to exist")
assertTrue("file_nonempty" !in gitkeep, "an empty .gitkeep must not fail file_nonempty")
// A non-dotfile with the same unknown kind keeps the full floor.
val regular = KindContractTable.assertionsFor("", "frontend/src/hooks/placeholder.ts").map { it.id }
assertTrue("file_nonempty" in regular, "non-dotfiles keep the nonempty floor")
}
@Test
fun `dotfile exemption also strips a would-be file_nonempty project override`() {
KindContractTable.projectOverrides = mapOf(
"" to listOf(ContractAssertion("file_nonempty", "", AssertionEvaluator.FS)),
)
val ids = KindContractTable.assertionsFor("", "config/.nojekyll").map { it.id }
assertTrue("file_nonempty" !in ids, "dotfile exemption is final, even against overrides")
}
@Test
fun `project override is additive and cannot remove checked-in assertions`() {
KindContractTable.projectOverrides = mapOf(
@@ -105,8 +105,16 @@ data class FailureTicketOpenedEvent(
val routeTo: StageId,
// The gate's captured failure output (command/exit/stderr/findings) that recovery must resolve.
val evidence: String,
// 1-based count of times this stage has been routed to recovery (own, small route budget).
// Count of NO-PROGRESS routes charged against this stage's recovery budget so far. A route whose
// failure fingerprint changed from the previous route (recovery made progress — a different error
// now) is FREE and leaves this flat; only a route that reproduces the same failure charges it. So
// a multi-cause failure repaired one layer at a time is not penalised for progressing. Folded
// directly by the reducer (idempotent under replay). Defaulted for backward-compatible replay of
// pre-progress-aware tickets.
val routeAttempt: Int,
// Fingerprint of [evidence] (whitespace/digit-normalised, see FailureFingerprint) at route time.
// The next route compares against this to decide progress vs. no-progress. Empty on old events.
val fingerprint: String = "",
) : EventPayload
/**
@@ -31,4 +31,11 @@ data class OrchestrationState(
// verification→recovery→verification loop must stay bounded across those transitions, exactly as
// refinementIterations does for back-edge loops.
val recoveryRoutes: Map<String, Int> = emptyMap(),
// Last-seen failure fingerprint per recovering stage id — the recovery-routing analogue of
// gateFailureFingerprints. Lets routeToRecovery tell a no-progress route (same fingerprint,
// charged against recoveryRoutes) from a genuine-progress one (changed fingerprint, free), so a
// recovery that fixes one cause and surfaces the next is not killed by the small route budget.
// Like recoveryRoutes, NOT reset by TransitionExecutedEvent — it must survive the
// verification→recovery→verification round-trip.
val recoveryFailureFingerprints: Map<String, String> = emptyMap(),
)
@@ -95,10 +95,12 @@ class DefaultOrchestrationReducer : OrchestrationReducer {
refinementIterations = state.refinementIterations + (p.cycleKey to p.iteration),
)
// Charge the failing stage's recovery-route budget. routeAttempt is the 1-based count the
// orchestrator computed, so fold it directly (idempotent under replay).
// Charge the failing stage's recovery-route budget. routeAttempt is the no-progress count the
// orchestrator computed (progress-aware), so fold it directly (idempotent under replay), and
// record the failure fingerprint the next route compares against.
is FailureTicketOpenedEvent -> state.copy(
recoveryRoutes = state.recoveryRoutes + (p.stageId.value to p.routeAttempt),
recoveryFailureFingerprints = state.recoveryFailureFingerprints + (p.stageId.value to p.fingerprint),
)
else -> state
@@ -36,6 +36,7 @@ import com.correx.core.kernel.execution.WorkflowResult
import com.correx.core.kernel.orchestration.subagent.InSessionSubagentRunner
import com.correx.core.kernel.orchestration.subagent.SubagentRunRequest
import com.correx.core.kernel.orchestration.subagent.SubagentRunner
import com.correx.core.kernel.retry.FailureFingerprint
import com.correx.core.kernel.retry.RetryCoordinator
import com.correx.core.kernel.retry.RetryDecision
import com.correx.core.sessions.Session
@@ -298,8 +299,19 @@ class DefaultSessionOrchestrator(
state: OrchestrationState,
): StepResult? {
val recoveryId = findRecoveryStage(ctx.graph, stageId) ?: return null // no recovery stage: legacy path
// Progress-aware charging (mirrors the per-gate retry budget, design §5): compare this
// failure's fingerprint to the previous route's. A changed fingerprint means the last
// recovery round moved the needle — fixed one cause and surfaced a different one — so the
// route is FREE (budget unchanged) and only the recorded fingerprint advances. Same
// fingerprint = the round changed nothing, so it charges the small route budget. Terminal
// only once the no-progress count is spent; the recovery→origin back-edge refinement guard
// (executeMove, cap = origin stage maxRetries) remains the ultimate loop bound for the
// pathological all-progress case.
val fingerprint = FailureFingerprint.of(reason)
val prev = state.recoveryFailureFingerprints[stageId.value]
val progressed = prev != null && prev != fingerprint
val used = state.recoveryRoutes[stageId.value] ?: 0
if (used >= RECOVERY_ROUTE_BUDGET) {
if (!progressed && used >= RECOVERY_ROUTE_BUDGET) {
return StepResult.Terminal(
failWorkflow(
ctx.sessionId,
@@ -309,6 +321,7 @@ class DefaultSessionOrchestrator(
),
)
}
val routeAttempt = if (progressed) used else used + 1
emit(
ctx.sessionId,
FailureTicketOpenedEvent(
@@ -319,13 +332,15 @@ class DefaultSessionOrchestrator(
requiredCapability = requiredCapability,
routeTo = recoveryId,
evidence = reason,
routeAttempt = used + 1,
routeAttempt = routeAttempt,
fingerprint = fingerprint,
),
)
log.info(
"[Orchestrator] failure ticket session={} stage={} gate={} lacks={} -> route to recovery={} ({}/{})",
"[Orchestrator] failure ticket session={} stage={} gate={} lacks={} -> recovery={} " +
"(charged={}/{}, progressed={})",
ctx.sessionId.value, stageId.value, gate, requiredCapability,
recoveryId.value, used + 1, RECOVERY_ROUTE_BUDGET,
recoveryId.value, routeAttempt, RECOVERY_ROUTE_BUDGET, progressed,
)
val advancedTo = advanceStage(
ctx.sessionId,
@@ -1915,6 +1915,11 @@ abstract class SessionOrchestrator(
is ToolResult.Success -> {
val diff = result.metadata["diff"]?.takeIf { it.isNotBlank() }
val affected = fileTool?.affectedPaths(request).orEmpty()
// Read-only tools (file_read, list_dir) have no FileAffectingTool, so `affected`
// is empty — fall back to the raw `path` argument so the receipt still names
// the file/dir the tool looked at (surfaced in the TUI's output row).
val affectedNames = affected.map { it.toString() }
.ifEmpty { listOfNotNull(request.parameters["path"] as? String) }
emit(
sessionId,
ToolExecutionCompletedEvent(
@@ -1929,7 +1934,7 @@ abstract class SessionOrchestrator(
// Carry the tool's structured metadata (e.g. file_read's contentHash) so
// session projections can read it.
structuredOutput = result.metadata,
affectedEntities = affected.map { it.toString() },
affectedEntities = affectedNames,
durationMs = 0,
tier = tier,
timestamp = Clock.System.now(),