feat(recovery): one bounded post-failure diagnostic before terminal FAILED (#294)

At the terminal boundary (repair ladder spent, or a non-recoverable gate
exhaustion), run exactly one tool-free diagnostic inference per terminal
failure fingerprint over recorded facts only. A validated — materially new,
confident, recovery-stage-available — RecoveryProposal routes once into the
existing recovery stage via the ticket machinery, bypassing the spent route
budget but bounded by a one-diagnosis-per-fingerprint dedupe so no loop is
possible. Otherwise the run stays terminal FAILED (safe degrade when no
diagnoser is wired).

- New PostFailureDiagnosedEvent + nested RecoveryProposal (registered in
  eventModule); every observation/proposal/decision/route recorded for replay.
- PostFailureDiagnoser seam (nullable, mirrors SalvageJudge) + DiagnosisInput
  built from the event log only (no fresh workspace observation).
- diagnosisMinConfidence tuning knob.
- Hooked at both terminal boundaries: routeToRecovery ladder-exhausted and
  decideGateExhaustion.

Tests: RecoveryRoutingTest (route-once-then-bounded, low-confidence stays
terminal), EventsTest serialization round-trip (proposal + null).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-20 18:51:18 +04:00
parent 3bf4dd7379
commit 4da1653017
10 changed files with 307 additions and 9 deletions
@@ -121,6 +121,10 @@ class DefaultSessionOrchestrator(
// decideGateExhaustion) so the feature degrades safely without inference.
internal val salvageJudge: SalvageJudge? = engines.salvageJudge
// One-shot post-failure diagnostic (#294): consulted once per terminal-failure fingerprint just
// before a run goes terminal. Null = deterministic degrade (fail terminally, see terminalOrDiagnose).
internal val postFailureDiagnoser: PostFailureDiagnoser? = engines.postFailureDiagnoser
override val subagentRunner: SubagentRunner = InSessionSubagentRunner(
executeStage = { sid, stg, graph, session, cfg ->
executeStage(sid, stg, graph, session, cfg, effectivesFor(cfg))
@@ -1,5 +1,8 @@
package com.correx.core.kernel.orchestration
import com.correx.core.events.events.FileWrittenEvent
import com.correx.core.events.events.InitialIntentEvent
import com.correx.core.events.events.PostFailureDiagnosedEvent
import com.correx.core.events.events.RetryAttemptedEvent
import com.correx.core.events.events.ToolInvocationRequestedEvent
import com.correx.core.events.events.FailureTicketOpenedEvent
import com.correx.core.events.events.StoredEvent
@@ -109,13 +112,14 @@ internal suspend fun DefaultSessionOrchestrator.routeToRecovery(
}
if (route == null) {
if (owner == null && arbiter == null) return null // nothing to route to: legacy retry path
return StepResult.Terminal(
failWorkflow(
ctx.sessionId,
stageId,
"repair ladder exhausted for stage ${stageId.value} (gate=$gate): $reason",
retryExhausted = true,
),
// Terminal boundary: the repair ladder is spent. Give the run one bounded post-failure
// diagnostic (#294) before FAILED — it may find a materially-new route the budget accounting lacked.
return terminalOrDiagnose(
ctx,
stageId,
gate,
"repair ladder exhausted for stage ${stageId.value} (gate=$gate): $reason",
state,
)
}
@@ -153,6 +157,92 @@ internal suspend fun DefaultSessionOrchestrator.routeToRecovery(
return enterStage(ctx.copy(currentStageId = advancedTo), route.target)
}
// How many recent tool actions to hand the diagnostic as "what was already tried".
private const val DIAGNOSIS_ACTION_TAIL = 10
/**
* One-shot post-failure diagnostic (design task #294). Called at the terminal boundary — when a run
* is about to become terminal FAILED. Runs exactly one tool-free diagnostic inference per terminal
* failure [fingerprint] (deduped on the recorded [PostFailureDiagnosedEvent], so no loop is possible)
* over recorded facts only. If the untrusted proposal is validated as materially new, confident, and
* a recovery stage exists, routes once into that stage via the existing ticket machinery — bypassing
* the already-spent route budget, since the fresh proposal is evidence the budget accounting lacked.
* Otherwise, or when no diagnoser is wired, returns the terminal failure unchanged (safe degrade).
* Every observation, proposal, validation decision and route is recorded (invariants #7/#9), so
* replay reproduces the decision without re-invoking the diagnoser.
*/
@Suppress("ReturnCount") // guard-clause ladder over the validation decision — flattest form
internal suspend fun DefaultSessionOrchestrator.terminalOrDiagnose(
ctx: EnrichedExecutionContext,
stageId: StageId,
gate: String,
reason: String,
state: OrchestrationState,
): StepResult {
val terminal: suspend () -> StepResult =
{ StepResult.Terminal(failWorkflow(ctx.sessionId, stageId, reason, retryExhausted = true)) }
val diagnoser = postFailureDiagnoser ?: return terminal()
val fingerprint = FailureFingerprint.of(reason)
val events = repositories.eventStore.read(ctx.sessionId)
// Acceptance #1/#6: at most one diagnosis per terminal fingerprint — this is what bounds the loop.
if (events.any { (it.payload as? PostFailureDiagnosedEvent)?.fingerprint == fingerprint }) return terminal()
val recoveryStage = findRecoveryStage(ctx.graph, stageId)
// Acceptance #2: recorded facts only, no fresh workspace observation.
val input = DiagnosisInput(
intent = events.mapNotNull { it.payload as? InitialIntentEvent }.lastOrNull()?.intent.orEmpty(),
gate = gate,
reason = reason,
fingerprint = fingerprint,
retryHistory = events.mapNotNull { it.payload as? RetryAttemptedEvent }
.map { "${it.gate} attempt=${it.attemptNumber} fp=${it.fingerprint}" },
attemptedActions = events.mapNotNull { it.payload as? ToolInvocationRequestedEvent }
.takeLast(DIAGNOSIS_ACTION_TAIL).map { it.toolName },
recoveryAvailable = recoveryStage != null,
)
val proposal = runCatching { diagnoser.diagnose(input) }.getOrNull()
val decision = when {
proposal == null -> "TERMINAL_NO_PROPOSAL"
proposal.noRecovery -> "TERMINAL_NO_RECOVERY"
proposal.confidence < tuning.diagnosisMinConfidence -> "TERMINAL_LOW_CONFIDENCE"
proposal.expectedFingerprint.isBlank() || proposal.expectedFingerprint == fingerprint -> "TERMINAL_NOT_MATERIAL"
recoveryStage == null -> "TERMINAL_NO_ROUTE"
else -> "ROUTE"
}
val routed = decision == "ROUTE"
emit(
ctx.sessionId,
PostFailureDiagnosedEvent(ctx.sessionId, stageId, gate, fingerprint, proposal, decision, routed),
)
log.info(
"[Orchestrator] post-failure diagnosis session={} stage={} gate={} decision={} routed={}",
ctx.sessionId.value, stageId.value, gate, decision, routed,
)
if (!routed || recoveryStage == null) return terminal()
// One validated, materially-new route into the existing recovery stage. Reuses the ticket
// machinery so buildRecoveryTicketEntry feeds the narrow repair bundle and ticketReturnMove
// re-runs the origin gate. Bounded by the per-fingerprint dedupe above, not the spent budget.
val used = state.recoveryRoutes[stageId.value + INTENT_BUDGET_SUFFIX] ?: 0
emit(
ctx.sessionId,
FailureTicketOpenedEvent(
sessionId = ctx.sessionId,
stageId = stageId,
gate = gate,
category = ticketCategory(gate),
requiredCapability = GATE_REQUIRED_CAPABILITY[gate] ?: "file_write",
routeTo = recoveryStage,
evidence = reason,
routeAttempt = used + 1,
fingerprint = fingerprint,
escalated = true,
),
)
val advancedTo = advanceStage(ctx.sessionId, stageId, TransitionDecision.Move(TICKET_ROUTE, recoveryStage))
return enterStage(ctx.copy(currentStageId = advancedTo), recoveryStage)
}
/** A chosen rung of the repair ladder: where to route, which budget it charges, and its tier. */
private data class RouteTier(
val target: StageId,
@@ -374,7 +374,8 @@ internal suspend fun DefaultSessionOrchestrator.decideGateExhaustion(
): StepResult? {
val sessionId = ctx.sessionId
if (gate != "review" || state.gateSalvageUsed.contains(gate)) {
return StepResult.Terminal(failWorkflow(sessionId, stageId, reason, retryExhausted = true))
// Terminal boundary: give the run one bounded post-failure diagnostic (#294) before FAILED.
return terminalOrDiagnose(ctx, stageId, gate, reason, state)
}
// No judge wired: degrade safely with a deterministic allow-one-reset-then-fail policy —
// the gateSalvageUsed check above already ensures this fires at most once per gate.
@@ -386,7 +387,7 @@ internal suspend fun DefaultSessionOrchestrator.decideGateExhaustion(
emit(sessionId, RetrySalvageDecidedEvent(sessionId, stageId, gate, judgment.decision, judgment.rationale))
return when (judgment.decision) {
SalvageDecision.CONTINUE -> null
SalvageDecision.FAIL -> StepResult.Terminal(failWorkflow(sessionId, stageId, reason, retryExhausted = true))
SalvageDecision.FAIL -> terminalOrDiagnose(ctx, stageId, gate, reason, state)
// The judge chose recovery: route to the recovery stage (file_write is the capability it
// provides). Degrade to terminal if the graph declares no recovery stage.
SalvageDecision.RECOVER ->
@@ -49,4 +49,6 @@ data class OrchestrationTuning(
* interleaved successes — it counts a normalized failure signature across the whole stage.
*/
val stageFailureLoopLimit: Int = 6,
/** Minimum confidence for a post-failure diagnostic proposal (#294) to be routed into recovery. */
val diagnosisMinConfidence: Double = 0.5,
)
@@ -48,4 +48,8 @@ data class OrchestratorEngines(
// only when the "review" gate exhausts its retry budget. Null = no LLM judge wired; the
// orchestrator then falls back to a deterministic allow-one-reset-then-fail policy.
val salvageJudge: SalvageJudge? = null,
// One-shot post-failure diagnostic (design task #294), consulted once per terminal-failure
// fingerprint just before a run becomes terminal. Null = no diagnostic; the run fails terminally
// as before (deterministic degrade).
val postFailureDiagnoser: PostFailureDiagnoser? = null,
)
@@ -0,0 +1,35 @@
package com.correx.core.kernel.orchestration
import com.correx.core.events.events.RecoveryProposal
/**
* The recorded facts handed to the one-shot post-failure diagnostic (design task #294). Assembled
* by the kernel from the event log only — no fresh workspace observation (acceptance #2) — so the
* diagnostic reasons over the same evidence replay will see.
*/
data class DiagnosisInput(
val intent: String,
val gate: String,
val reason: String,
val fingerprint: String,
val retryHistory: List<String>,
val attemptedActions: List<String>,
val recoveryAvailable: Boolean,
)
/**
* Seam for the one-shot post-failure diagnostic (design task #294). When a run is about to become
* terminal, the kernel consults this once per terminal-failure fingerprint to get an untrusted
* [RecoveryProposal] for a materially different recovery route. Like the other inference seams
* ([SalvageJudge], [SemanticReviewer]), the implementation is injected so the deterministic core
* never runs inference itself; the call is tool-free and must not alter model temperature.
*
* The proposal is nondeterministic (LLM-backed), so invariant #9 requires the caller to record it —
* and the kernel's validation decision — as a
* [com.correx.core.events.events.PostFailureDiagnosedEvent]; replay reads that back and never
* re-invokes the diagnoser. When none is wired (`null` in [OrchestratorEngines]), the run fails
* terminally as before, so the feature degrades safely without inference.
*/
fun interface PostFailureDiagnoser {
suspend fun diagnose(input: DiagnosisInput): RecoveryProposal?
}