feat(recovery): one bounded post-failure diagnostic before terminal FAILED (#294)

At the terminal boundary (repair ladder spent, or a non-recoverable gate
exhaustion), run exactly one tool-free diagnostic inference per terminal
failure fingerprint over recorded facts only. A validated — materially new,
confident, recovery-stage-available — RecoveryProposal routes once into the
existing recovery stage via the ticket machinery, bypassing the spent route
budget but bounded by a one-diagnosis-per-fingerprint dedupe so no loop is
possible. Otherwise the run stays terminal FAILED (safe degrade when no
diagnoser is wired).

- New PostFailureDiagnosedEvent + nested RecoveryProposal (registered in
  eventModule); every observation/proposal/decision/route recorded for replay.
- PostFailureDiagnoser seam (nullable, mirrors SalvageJudge) + DiagnosisInput
  built from the event log only (no fresh workspace observation).
- diagnosisMinConfidence tuning knob.
- Hooked at both terminal boundaries: routeToRecovery ladder-exhausted and
  decideGateExhaustion.

Tests: RecoveryRoutingTest (route-once-then-bounded, low-confidence stays
terminal), EventsTest serialization round-trip (proposal + null).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-20 18:51:18 +04:00
parent 3bf4dd7379
commit 4da1653017
10 changed files with 307 additions and 9 deletions
@@ -169,6 +169,47 @@ data class RetrySalvageDecidedEvent(
val rationale: String,
) : EventPayload
/**
* A structured, untrusted recovery proposal produced by the one-shot post-failure diagnostic
* (design task #294). The diagnostic inference is tool-free and reads only recorded facts; this is
* its proposal. [expectedFingerprint] is the failure fingerprint the proposed [recoveryAction] is
* predicted to change the run to — the kernel routes only when it is materially different from the
* current terminal fingerprint (i.e. a genuinely new path, not the same dead end). [noRecovery]
* lets the model explicitly decline; [confidence] is thresholded by the kernel. LLM-proposed and
* therefore untrusted (invariant #7): validated deterministically before it can affect routing.
*/
@Serializable
data class RecoveryProposal(
val diagnosis: String,
val citedEvidence: String,
val recoveryAction: String,
val expectedFingerprint: String,
val confidence: Double,
val noRecovery: Boolean = false,
)
/**
* Records the one-shot post-failure diagnostic (design task #294): when a run is about to become
* terminal, exactly one tool-free diagnostic inference runs per terminal-failure [fingerprint]. The
* nondeterministic [proposal] (LLM-backed, null when none/unparseable), the kernel's deterministic
* validation [decision], and whether it [routed] into the existing recovery stage are all recorded
* here so replay reproduces the decision without re-invoking the diagnoser (invariants #7/#9). The
* per-fingerprint dedupe that bounds this to one attempt keys off this event.
*/
@Serializable
@SerialName("PostFailureDiagnosed")
data class PostFailureDiagnosedEvent(
val sessionId: SessionId,
val stageId: StageId,
val gate: String,
val fingerprint: String,
val proposal: RecoveryProposal?,
// ROUTE | TERMINAL_NO_PROPOSAL | TERMINAL_NO_RECOVERY | TERMINAL_LOW_CONFIDENCE |
// TERMINAL_NOT_MATERIAL | TERMINAL_NO_ROUTE
val decision: String,
val routed: Boolean,
) : EventPayload
/**
* A stage repeatedly blocked on a missing build prerequisite (see repeatedBuildCriticalReferenceBlock,
* design 2026-07-15 §#170) AND the stage both holds file_write and declares the path in-scope, so the
@@ -65,6 +65,7 @@ import com.correx.core.events.events.RepoKnowledgeRetrievedEvent
import com.correx.core.events.events.RepoMapComputedEvent
import com.correx.core.events.events.RetryAttemptedEvent
import com.correx.core.events.events.RetrySalvageDecidedEvent
import com.correx.core.events.events.PostFailureDiagnosedEvent
import com.correx.core.events.events.FailureTicketOpenedEvent
import com.correx.core.events.events.BuildPrerequisiteBootstrapAttemptedEvent
import com.correx.core.events.events.WorkspaceVerificationObservedEvent
@@ -161,6 +162,7 @@ val eventModule = SerializersModule {
subclass(WorkflowCompletedEvent::class)
subclass(RetryAttemptedEvent::class)
subclass(RetrySalvageDecidedEvent::class)
subclass(PostFailureDiagnosedEvent::class)
subclass(FailureTicketOpenedEvent::class)
subclass(BuildPrerequisiteBootstrapAttemptedEvent::class)
subclass(WorkspaceVerificationObservedEvent::class)
@@ -121,6 +121,10 @@ class DefaultSessionOrchestrator(
// decideGateExhaustion) so the feature degrades safely without inference.
internal val salvageJudge: SalvageJudge? = engines.salvageJudge
// One-shot post-failure diagnostic (#294): consulted once per terminal-failure fingerprint just
// before a run goes terminal. Null = deterministic degrade (fail terminally, see terminalOrDiagnose).
internal val postFailureDiagnoser: PostFailureDiagnoser? = engines.postFailureDiagnoser
override val subagentRunner: SubagentRunner = InSessionSubagentRunner(
executeStage = { sid, stg, graph, session, cfg ->
executeStage(sid, stg, graph, session, cfg, effectivesFor(cfg))
@@ -1,5 +1,8 @@
package com.correx.core.kernel.orchestration
import com.correx.core.events.events.FileWrittenEvent
import com.correx.core.events.events.InitialIntentEvent
import com.correx.core.events.events.PostFailureDiagnosedEvent
import com.correx.core.events.events.RetryAttemptedEvent
import com.correx.core.events.events.ToolInvocationRequestedEvent
import com.correx.core.events.events.FailureTicketOpenedEvent
import com.correx.core.events.events.StoredEvent
@@ -109,13 +112,14 @@ internal suspend fun DefaultSessionOrchestrator.routeToRecovery(
}
if (route == null) {
if (owner == null && arbiter == null) return null // nothing to route to: legacy retry path
return StepResult.Terminal(
failWorkflow(
ctx.sessionId,
stageId,
"repair ladder exhausted for stage ${stageId.value} (gate=$gate): $reason",
retryExhausted = true,
),
// Terminal boundary: the repair ladder is spent. Give the run one bounded post-failure
// diagnostic (#294) before FAILED — it may find a materially-new route the budget accounting lacked.
return terminalOrDiagnose(
ctx,
stageId,
gate,
"repair ladder exhausted for stage ${stageId.value} (gate=$gate): $reason",
state,
)
}
@@ -153,6 +157,92 @@ internal suspend fun DefaultSessionOrchestrator.routeToRecovery(
return enterStage(ctx.copy(currentStageId = advancedTo), route.target)
}
// How many recent tool actions to hand the diagnostic as "what was already tried".
private const val DIAGNOSIS_ACTION_TAIL = 10
/**
* One-shot post-failure diagnostic (design task #294). Called at the terminal boundary — when a run
* is about to become terminal FAILED. Runs exactly one tool-free diagnostic inference per terminal
* failure [fingerprint] (deduped on the recorded [PostFailureDiagnosedEvent], so no loop is possible)
* over recorded facts only. If the untrusted proposal is validated as materially new, confident, and
* a recovery stage exists, routes once into that stage via the existing ticket machinery — bypassing
* the already-spent route budget, since the fresh proposal is evidence the budget accounting lacked.
* Otherwise, or when no diagnoser is wired, returns the terminal failure unchanged (safe degrade).
* Every observation, proposal, validation decision and route is recorded (invariants #7/#9), so
* replay reproduces the decision without re-invoking the diagnoser.
*/
@Suppress("ReturnCount") // guard-clause ladder over the validation decision — flattest form
internal suspend fun DefaultSessionOrchestrator.terminalOrDiagnose(
ctx: EnrichedExecutionContext,
stageId: StageId,
gate: String,
reason: String,
state: OrchestrationState,
): StepResult {
val terminal: suspend () -> StepResult =
{ StepResult.Terminal(failWorkflow(ctx.sessionId, stageId, reason, retryExhausted = true)) }
val diagnoser = postFailureDiagnoser ?: return terminal()
val fingerprint = FailureFingerprint.of(reason)
val events = repositories.eventStore.read(ctx.sessionId)
// Acceptance #1/#6: at most one diagnosis per terminal fingerprint — this is what bounds the loop.
if (events.any { (it.payload as? PostFailureDiagnosedEvent)?.fingerprint == fingerprint }) return terminal()
val recoveryStage = findRecoveryStage(ctx.graph, stageId)
// Acceptance #2: recorded facts only, no fresh workspace observation.
val input = DiagnosisInput(
intent = events.mapNotNull { it.payload as? InitialIntentEvent }.lastOrNull()?.intent.orEmpty(),
gate = gate,
reason = reason,
fingerprint = fingerprint,
retryHistory = events.mapNotNull { it.payload as? RetryAttemptedEvent }
.map { "${it.gate} attempt=${it.attemptNumber} fp=${it.fingerprint}" },
attemptedActions = events.mapNotNull { it.payload as? ToolInvocationRequestedEvent }
.takeLast(DIAGNOSIS_ACTION_TAIL).map { it.toolName },
recoveryAvailable = recoveryStage != null,
)
val proposal = runCatching { diagnoser.diagnose(input) }.getOrNull()
val decision = when {
proposal == null -> "TERMINAL_NO_PROPOSAL"
proposal.noRecovery -> "TERMINAL_NO_RECOVERY"
proposal.confidence < tuning.diagnosisMinConfidence -> "TERMINAL_LOW_CONFIDENCE"
proposal.expectedFingerprint.isBlank() || proposal.expectedFingerprint == fingerprint -> "TERMINAL_NOT_MATERIAL"
recoveryStage == null -> "TERMINAL_NO_ROUTE"
else -> "ROUTE"
}
val routed = decision == "ROUTE"
emit(
ctx.sessionId,
PostFailureDiagnosedEvent(ctx.sessionId, stageId, gate, fingerprint, proposal, decision, routed),
)
log.info(
"[Orchestrator] post-failure diagnosis session={} stage={} gate={} decision={} routed={}",
ctx.sessionId.value, stageId.value, gate, decision, routed,
)
if (!routed || recoveryStage == null) return terminal()
// One validated, materially-new route into the existing recovery stage. Reuses the ticket
// machinery so buildRecoveryTicketEntry feeds the narrow repair bundle and ticketReturnMove
// re-runs the origin gate. Bounded by the per-fingerprint dedupe above, not the spent budget.
val used = state.recoveryRoutes[stageId.value + INTENT_BUDGET_SUFFIX] ?: 0
emit(
ctx.sessionId,
FailureTicketOpenedEvent(
sessionId = ctx.sessionId,
stageId = stageId,
gate = gate,
category = ticketCategory(gate),
requiredCapability = GATE_REQUIRED_CAPABILITY[gate] ?: "file_write",
routeTo = recoveryStage,
evidence = reason,
routeAttempt = used + 1,
fingerprint = fingerprint,
escalated = true,
),
)
val advancedTo = advanceStage(ctx.sessionId, stageId, TransitionDecision.Move(TICKET_ROUTE, recoveryStage))
return enterStage(ctx.copy(currentStageId = advancedTo), recoveryStage)
}
/** A chosen rung of the repair ladder: where to route, which budget it charges, and its tier. */
private data class RouteTier(
val target: StageId,
@@ -374,7 +374,8 @@ internal suspend fun DefaultSessionOrchestrator.decideGateExhaustion(
): StepResult? {
val sessionId = ctx.sessionId
if (gate != "review" || state.gateSalvageUsed.contains(gate)) {
return StepResult.Terminal(failWorkflow(sessionId, stageId, reason, retryExhausted = true))
// Terminal boundary: give the run one bounded post-failure diagnostic (#294) before FAILED.
return terminalOrDiagnose(ctx, stageId, gate, reason, state)
}
// No judge wired: degrade safely with a deterministic allow-one-reset-then-fail policy —
// the gateSalvageUsed check above already ensures this fires at most once per gate.
@@ -386,7 +387,7 @@ internal suspend fun DefaultSessionOrchestrator.decideGateExhaustion(
emit(sessionId, RetrySalvageDecidedEvent(sessionId, stageId, gate, judgment.decision, judgment.rationale))
return when (judgment.decision) {
SalvageDecision.CONTINUE -> null
SalvageDecision.FAIL -> StepResult.Terminal(failWorkflow(sessionId, stageId, reason, retryExhausted = true))
SalvageDecision.FAIL -> terminalOrDiagnose(ctx, stageId, gate, reason, state)
// The judge chose recovery: route to the recovery stage (file_write is the capability it
// provides). Degrade to terminal if the graph declares no recovery stage.
SalvageDecision.RECOVER ->
@@ -49,4 +49,6 @@ data class OrchestrationTuning(
* interleaved successes — it counts a normalized failure signature across the whole stage.
*/
val stageFailureLoopLimit: Int = 6,
/** Minimum confidence for a post-failure diagnostic proposal (#294) to be routed into recovery. */
val diagnosisMinConfidence: Double = 0.5,
)
@@ -48,4 +48,8 @@ data class OrchestratorEngines(
// only when the "review" gate exhausts its retry budget. Null = no LLM judge wired; the
// orchestrator then falls back to a deterministic allow-one-reset-then-fail policy.
val salvageJudge: SalvageJudge? = null,
// One-shot post-failure diagnostic (design task #294), consulted once per terminal-failure
// fingerprint just before a run becomes terminal. Null = no diagnostic; the run fails terminally
// as before (deterministic degrade).
val postFailureDiagnoser: PostFailureDiagnoser? = null,
)
@@ -0,0 +1,35 @@
package com.correx.core.kernel.orchestration
import com.correx.core.events.events.RecoveryProposal
/**
* The recorded facts handed to the one-shot post-failure diagnostic (design task #294). Assembled
* by the kernel from the event log only — no fresh workspace observation (acceptance #2) — so the
* diagnostic reasons over the same evidence replay will see.
*/
data class DiagnosisInput(
val intent: String,
val gate: String,
val reason: String,
val fingerprint: String,
val retryHistory: List<String>,
val attemptedActions: List<String>,
val recoveryAvailable: Boolean,
)
/**
* Seam for the one-shot post-failure diagnostic (design task #294). When a run is about to become
* terminal, the kernel consults this once per terminal-failure fingerprint to get an untrusted
* [RecoveryProposal] for a materially different recovery route. Like the other inference seams
* ([SalvageJudge], [SemanticReviewer]), the implementation is injected so the deterministic core
* never runs inference itself; the call is tool-free and must not alter model temperature.
*
* The proposal is nondeterministic (LLM-backed), so invariant #9 requires the caller to record it —
* and the kernel's validation decision — as a
* [com.correx.core.events.events.PostFailureDiagnosedEvent]; replay reads that back and never
* re-invokes the diagnoser. When none is wired (`null` in [OrchestratorEngines]), the run fails
* terminally as before, so the feature degrades safely without inference.
*/
fun interface PostFailureDiagnoser {
suspend fun diagnose(input: DiagnosisInput): RecoveryProposal?
}
@@ -3,6 +3,8 @@ package com.correx.testing.contracts.model.orchestration
import com.correx.core.events.events.EventPayload
import com.correx.core.events.events.OrchestrationPausedEvent
import com.correx.core.events.events.OrchestrationResumedEvent
import com.correx.core.events.events.PostFailureDiagnosedEvent
import com.correx.core.events.events.RecoveryProposal
import com.correx.core.events.events.RetryAttemptedEvent
import com.correx.core.events.events.WorkflowCompletedEvent
import com.correx.core.events.events.WorkflowFailedEvent
@@ -45,6 +47,50 @@ class EventsTest {
assertEquals(event, decoded)
}
@Test
fun `PostFailureDiagnosedEvent survives round-trip with a proposal`() {
// Guards the #294 event AND its nested RecoveryProposal against the silent-registration trap.
val event: EventPayload = PostFailureDiagnosedEvent(
sessionId = SessionId("s1"),
stageId = StageId("stage-a"),
gate = "execution",
fingerprint = "fp-abc",
proposal = RecoveryProposal(
diagnosis = "type mismatch",
citedEvidence = "App.tsx: TS2322",
recoveryAction = "narrow the return type",
expectedFingerprint = "fp-xyz",
confidence = 0.9,
noRecovery = false,
),
decision = "ROUTE",
routed = true,
)
val json = eventJson.encodeToString(EventPayload.serializer(), event)
val decoded = eventJson.decodeFromString(EventPayload.serializer(), json)
assertEquals(event, decoded)
}
@Test
fun `PostFailureDiagnosedEvent survives round-trip with a null proposal`() {
val event: EventPayload = PostFailureDiagnosedEvent(
sessionId = SessionId("s1"),
stageId = StageId("stage-a"),
gate = "execution",
fingerprint = "fp-abc",
proposal = null,
decision = "TERMINAL_NO_PROPOSAL",
routed = false,
)
val json = eventJson.encodeToString(EventPayload.serializer(), event)
val decoded = eventJson.decodeFromString(EventPayload.serializer(), json)
assertEquals(event, decoded)
}
@Test
fun `WorkflowFailedEvent survives round-trip`() {
val event: EventPayload = WorkflowFailedEvent(
@@ -11,6 +11,8 @@ import com.correx.core.approvals.model.DomainApprovalRequest
import com.correx.core.artifacts.DefaultArtifactReducer
import com.correx.core.events.EventDispatcher
import com.correx.core.events.events.FailureTicketOpenedEvent
import com.correx.core.events.events.PostFailureDiagnosedEvent
import com.correx.core.events.events.RecoveryProposal
import com.correx.core.events.events.StaticAnalysisCompletedEvent
import com.correx.core.events.events.TransitionExecutedEvent
import com.correx.core.events.events.WorkflowCompletedEvent
@@ -46,6 +48,7 @@ import com.correx.core.kernel.orchestration.OrchestrationRepository
import com.correx.core.kernel.orchestration.OrchestratorEngines
import com.correx.core.kernel.orchestration.OrchestratorRepositories
import com.correx.core.kernel.orchestration.StaticAnalysisRunResult
import com.correx.core.kernel.orchestration.PostFailureDiagnoser
import com.correx.core.kernel.orchestration.StaticAnalysisRunner
import com.correx.core.kernel.retry.DefaultRetryCoordinator
import com.correx.core.risk.DefaultRiskAssessor
@@ -77,6 +80,7 @@ import kotlinx.datetime.Instant
import java.nio.file.Files
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertFalse
import kotlin.test.assertTrue
/**
@@ -428,6 +432,7 @@ class RecoveryRoutingTest {
staticOutput: String = "App.tsx: TS2322 type mismatch",
provider: InferenceProvider = StageRoutingProvider(),
staticExitCode: Int = 1,
postFailureDiagnoser: PostFailureDiagnoser? = null,
): Pair<DefaultSessionOrchestrator, InMemoryEventStore> {
val eventStore = InMemoryEventStore()
val shell = ShellTool(allowedExecutables = setOf("true"), workingDir = workspace)
@@ -477,6 +482,7 @@ class RecoveryRoutingTest {
staticAnalysisRunner = StaticAnalysisRunner { _, _ ->
StaticAnalysisRunResult(exitCode = staticExitCode, output = staticOutput)
},
postFailureDiagnoser = postFailureDiagnoser,
approvalEngine = object : ApprovalEngine {
override fun evaluate(
request: DomainApprovalRequest,
@@ -650,6 +656,73 @@ class RecoveryRoutingTest {
}
}
@Test
fun `post-failure diagnosis routes one validated proposal into recovery then stays bounded`(): Unit = runBlocking {
// #294: after the recovery ladder is fully spent on an unchanged fingerprint, the one-shot
// diagnostic proposes a materially-new, confident route. It routes once more into recovery,
// then — the fingerprint still unchanged — the per-fingerprint dedupe forces terminal. No loop.
var diagnosisCalls = 0
val diagnoser = PostFailureDiagnoser { input ->
diagnosisCalls++
RecoveryProposal(
diagnosis = "impl keeps emitting the same type error",
citedEvidence = input.reason,
recoveryAction = "narrow the return type in App.tsx",
expectedFingerprint = "materially-different-fingerprint",
confidence = 0.9,
)
}
val sid = SessionId("diag-route")
val (orchestrator, eventStore) = buildMultiToolOrchestrator(
staticOutput = "App.tsx: TS2322 type mismatch",
postFailureDiagnoser = diagnoser,
)
val config = OrchestrationConfig(
retryPolicy = RetryPolicy(maxAttempts = 10, backoffMs = 0, perGateMaxAttempts = mapOf("static_analysis" to 1)),
)
val result = orchestrator.run(sid, capableButStuckGraph(), config)
assertTrue(result is WorkflowResult.Failed, "bounded: still terminates after the one diagnostic route")
val diagnoses = eventStore.read(sid).mapNotNull { it.payload as? PostFailureDiagnosedEvent }
assertEquals(1, diagnoses.size, "exactly one diagnosis per terminal fingerprint (#1/#6)")
assertEquals("ROUTE", diagnoses.single().decision)
assertTrue(diagnoses.single().routed, "a validated materially-new proposal routes into recovery (#3)")
assertEquals(1, diagnosisCalls, "the diagnoser is consulted at most once for the fingerprint")
// Acceptance #2: the diagnostic saw recorded gate evidence, not a fresh observation.
assertTrue(
diagnoses.single().proposal?.citedEvidence?.contains("TS2322") == true,
"diagnosis input carried the recorded gate evidence",
)
}
@Test
fun `low-confidence post-failure proposal leaves the session terminal`(): Unit = runBlocking {
// #294 acceptance #4: an invalid (here: low-confidence) proposal is recorded but never routed.
val diagnoser = PostFailureDiagnoser {
RecoveryProposal(
diagnosis = "unsure",
citedEvidence = it.reason,
recoveryAction = "maybe try again",
expectedFingerprint = "different-but-unconfident",
confidence = 0.1,
)
}
val sid = SessionId("diag-terminal")
val (orchestrator, eventStore) = buildMultiToolOrchestrator(postFailureDiagnoser = diagnoser)
val config = OrchestrationConfig(
retryPolicy = RetryPolicy(maxAttempts = 10, backoffMs = 0, perGateMaxAttempts = mapOf("static_analysis" to 1)),
)
val result = orchestrator.run(sid, capableButStuckGraph(), config)
assertTrue(result is WorkflowResult.Failed)
val diagnoses = eventStore.read(sid).mapNotNull { it.payload as? PostFailureDiagnosedEvent }
assertEquals(1, diagnoses.size, "still one diagnosis attempt, but not routed")
assertEquals("TERMINAL_LOW_CONFIDENCE", diagnoses.single().decision)
assertFalse(diagnoses.single().routed)
}
// impl carries a generative "scaffold" mandate as its own prompt. On the FORWARD visit it applies;
// when the ticket routes impl back to repair its own file, that mandate must be suppressed (else it
// re-scaffolds and stubs the real file). Same topology as ownerGraph otherwise.