feat(recovery): one bounded post-failure diagnostic before terminal FAILED (#294)
At the terminal boundary (repair ladder spent, or a non-recoverable gate exhaustion), run exactly one tool-free diagnostic inference per terminal failure fingerprint over recorded facts only. A validated — materially new, confident, recovery-stage-available — RecoveryProposal routes once into the existing recovery stage via the ticket machinery, bypassing the spent route budget but bounded by a one-diagnosis-per-fingerprint dedupe so no loop is possible. Otherwise the run stays terminal FAILED (safe degrade when no diagnoser is wired). - New PostFailureDiagnosedEvent + nested RecoveryProposal (registered in eventModule); every observation/proposal/decision/route recorded for replay. - PostFailureDiagnoser seam (nullable, mirrors SalvageJudge) + DiagnosisInput built from the event log only (no fresh workspace observation). - diagnosisMinConfidence tuning knob. - Hooked at both terminal boundaries: routeToRecovery ladder-exhausted and decideGateExhaustion. Tests: RecoveryRoutingTest (route-once-then-bounded, low-confidence stays terminal), EventsTest serialization round-trip (proposal + null). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
+4
@@ -121,6 +121,10 @@ class DefaultSessionOrchestrator(
|
||||
// decideGateExhaustion) so the feature degrades safely without inference.
|
||||
internal val salvageJudge: SalvageJudge? = engines.salvageJudge
|
||||
|
||||
// One-shot post-failure diagnostic (#294): consulted once per terminal-failure fingerprint just
|
||||
// before a run goes terminal. Null = deterministic degrade (fail terminally, see terminalOrDiagnose).
|
||||
internal val postFailureDiagnoser: PostFailureDiagnoser? = engines.postFailureDiagnoser
|
||||
|
||||
override val subagentRunner: SubagentRunner = InSessionSubagentRunner(
|
||||
executeStage = { sid, stg, graph, session, cfg ->
|
||||
executeStage(sid, stg, graph, session, cfg, effectivesFor(cfg))
|
||||
|
||||
+97
-7
@@ -1,5 +1,8 @@
|
||||
package com.correx.core.kernel.orchestration
|
||||
import com.correx.core.events.events.FileWrittenEvent
|
||||
import com.correx.core.events.events.InitialIntentEvent
|
||||
import com.correx.core.events.events.PostFailureDiagnosedEvent
|
||||
import com.correx.core.events.events.RetryAttemptedEvent
|
||||
import com.correx.core.events.events.ToolInvocationRequestedEvent
|
||||
import com.correx.core.events.events.FailureTicketOpenedEvent
|
||||
import com.correx.core.events.events.StoredEvent
|
||||
@@ -109,13 +112,14 @@ internal suspend fun DefaultSessionOrchestrator.routeToRecovery(
|
||||
}
|
||||
if (route == null) {
|
||||
if (owner == null && arbiter == null) return null // nothing to route to: legacy retry path
|
||||
return StepResult.Terminal(
|
||||
failWorkflow(
|
||||
ctx.sessionId,
|
||||
stageId,
|
||||
"repair ladder exhausted for stage ${stageId.value} (gate=$gate): $reason",
|
||||
retryExhausted = true,
|
||||
),
|
||||
// Terminal boundary: the repair ladder is spent. Give the run one bounded post-failure
|
||||
// diagnostic (#294) before FAILED — it may find a materially-new route the budget accounting lacked.
|
||||
return terminalOrDiagnose(
|
||||
ctx,
|
||||
stageId,
|
||||
gate,
|
||||
"repair ladder exhausted for stage ${stageId.value} (gate=$gate): $reason",
|
||||
state,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -153,6 +157,92 @@ internal suspend fun DefaultSessionOrchestrator.routeToRecovery(
|
||||
return enterStage(ctx.copy(currentStageId = advancedTo), route.target)
|
||||
}
|
||||
|
||||
// How many recent tool actions to hand the diagnostic as "what was already tried".
|
||||
private const val DIAGNOSIS_ACTION_TAIL = 10
|
||||
|
||||
/**
|
||||
* One-shot post-failure diagnostic (design task #294). Called at the terminal boundary — when a run
|
||||
* is about to become terminal FAILED. Runs exactly one tool-free diagnostic inference per terminal
|
||||
* failure [fingerprint] (deduped on the recorded [PostFailureDiagnosedEvent], so no loop is possible)
|
||||
* over recorded facts only. If the untrusted proposal is validated as materially new, confident, and
|
||||
* a recovery stage exists, routes once into that stage via the existing ticket machinery — bypassing
|
||||
* the already-spent route budget, since the fresh proposal is evidence the budget accounting lacked.
|
||||
* Otherwise, or when no diagnoser is wired, returns the terminal failure unchanged (safe degrade).
|
||||
* Every observation, proposal, validation decision and route is recorded (invariants #7/#9), so
|
||||
* replay reproduces the decision without re-invoking the diagnoser.
|
||||
*/
|
||||
@Suppress("ReturnCount") // guard-clause ladder over the validation decision — flattest form
|
||||
internal suspend fun DefaultSessionOrchestrator.terminalOrDiagnose(
|
||||
ctx: EnrichedExecutionContext,
|
||||
stageId: StageId,
|
||||
gate: String,
|
||||
reason: String,
|
||||
state: OrchestrationState,
|
||||
): StepResult {
|
||||
val terminal: suspend () -> StepResult =
|
||||
{ StepResult.Terminal(failWorkflow(ctx.sessionId, stageId, reason, retryExhausted = true)) }
|
||||
val diagnoser = postFailureDiagnoser ?: return terminal()
|
||||
val fingerprint = FailureFingerprint.of(reason)
|
||||
val events = repositories.eventStore.read(ctx.sessionId)
|
||||
// Acceptance #1/#6: at most one diagnosis per terminal fingerprint — this is what bounds the loop.
|
||||
if (events.any { (it.payload as? PostFailureDiagnosedEvent)?.fingerprint == fingerprint }) return terminal()
|
||||
|
||||
val recoveryStage = findRecoveryStage(ctx.graph, stageId)
|
||||
// Acceptance #2: recorded facts only, no fresh workspace observation.
|
||||
val input = DiagnosisInput(
|
||||
intent = events.mapNotNull { it.payload as? InitialIntentEvent }.lastOrNull()?.intent.orEmpty(),
|
||||
gate = gate,
|
||||
reason = reason,
|
||||
fingerprint = fingerprint,
|
||||
retryHistory = events.mapNotNull { it.payload as? RetryAttemptedEvent }
|
||||
.map { "${it.gate} attempt=${it.attemptNumber} fp=${it.fingerprint}" },
|
||||
attemptedActions = events.mapNotNull { it.payload as? ToolInvocationRequestedEvent }
|
||||
.takeLast(DIAGNOSIS_ACTION_TAIL).map { it.toolName },
|
||||
recoveryAvailable = recoveryStage != null,
|
||||
)
|
||||
val proposal = runCatching { diagnoser.diagnose(input) }.getOrNull()
|
||||
val decision = when {
|
||||
proposal == null -> "TERMINAL_NO_PROPOSAL"
|
||||
proposal.noRecovery -> "TERMINAL_NO_RECOVERY"
|
||||
proposal.confidence < tuning.diagnosisMinConfidence -> "TERMINAL_LOW_CONFIDENCE"
|
||||
proposal.expectedFingerprint.isBlank() || proposal.expectedFingerprint == fingerprint -> "TERMINAL_NOT_MATERIAL"
|
||||
recoveryStage == null -> "TERMINAL_NO_ROUTE"
|
||||
else -> "ROUTE"
|
||||
}
|
||||
val routed = decision == "ROUTE"
|
||||
emit(
|
||||
ctx.sessionId,
|
||||
PostFailureDiagnosedEvent(ctx.sessionId, stageId, gate, fingerprint, proposal, decision, routed),
|
||||
)
|
||||
log.info(
|
||||
"[Orchestrator] post-failure diagnosis session={} stage={} gate={} decision={} routed={}",
|
||||
ctx.sessionId.value, stageId.value, gate, decision, routed,
|
||||
)
|
||||
if (!routed || recoveryStage == null) return terminal()
|
||||
|
||||
// One validated, materially-new route into the existing recovery stage. Reuses the ticket
|
||||
// machinery so buildRecoveryTicketEntry feeds the narrow repair bundle and ticketReturnMove
|
||||
// re-runs the origin gate. Bounded by the per-fingerprint dedupe above, not the spent budget.
|
||||
val used = state.recoveryRoutes[stageId.value + INTENT_BUDGET_SUFFIX] ?: 0
|
||||
emit(
|
||||
ctx.sessionId,
|
||||
FailureTicketOpenedEvent(
|
||||
sessionId = ctx.sessionId,
|
||||
stageId = stageId,
|
||||
gate = gate,
|
||||
category = ticketCategory(gate),
|
||||
requiredCapability = GATE_REQUIRED_CAPABILITY[gate] ?: "file_write",
|
||||
routeTo = recoveryStage,
|
||||
evidence = reason,
|
||||
routeAttempt = used + 1,
|
||||
fingerprint = fingerprint,
|
||||
escalated = true,
|
||||
),
|
||||
)
|
||||
val advancedTo = advanceStage(ctx.sessionId, stageId, TransitionDecision.Move(TICKET_ROUTE, recoveryStage))
|
||||
return enterStage(ctx.copy(currentStageId = advancedTo), recoveryStage)
|
||||
}
|
||||
|
||||
/** A chosen rung of the repair ladder: where to route, which budget it charges, and its tier. */
|
||||
private data class RouteTier(
|
||||
val target: StageId,
|
||||
|
||||
+3
-2
@@ -374,7 +374,8 @@ internal suspend fun DefaultSessionOrchestrator.decideGateExhaustion(
|
||||
): StepResult? {
|
||||
val sessionId = ctx.sessionId
|
||||
if (gate != "review" || state.gateSalvageUsed.contains(gate)) {
|
||||
return StepResult.Terminal(failWorkflow(sessionId, stageId, reason, retryExhausted = true))
|
||||
// Terminal boundary: give the run one bounded post-failure diagnostic (#294) before FAILED.
|
||||
return terminalOrDiagnose(ctx, stageId, gate, reason, state)
|
||||
}
|
||||
// No judge wired: degrade safely with a deterministic allow-one-reset-then-fail policy —
|
||||
// the gateSalvageUsed check above already ensures this fires at most once per gate.
|
||||
@@ -386,7 +387,7 @@ internal suspend fun DefaultSessionOrchestrator.decideGateExhaustion(
|
||||
emit(sessionId, RetrySalvageDecidedEvent(sessionId, stageId, gate, judgment.decision, judgment.rationale))
|
||||
return when (judgment.decision) {
|
||||
SalvageDecision.CONTINUE -> null
|
||||
SalvageDecision.FAIL -> StepResult.Terminal(failWorkflow(sessionId, stageId, reason, retryExhausted = true))
|
||||
SalvageDecision.FAIL -> terminalOrDiagnose(ctx, stageId, gate, reason, state)
|
||||
// The judge chose recovery: route to the recovery stage (file_write is the capability it
|
||||
// provides). Degrade to terminal if the graph declares no recovery stage.
|
||||
SalvageDecision.RECOVER ->
|
||||
|
||||
+2
@@ -49,4 +49,6 @@ data class OrchestrationTuning(
|
||||
* interleaved successes — it counts a normalized failure signature across the whole stage.
|
||||
*/
|
||||
val stageFailureLoopLimit: Int = 6,
|
||||
/** Minimum confidence for a post-failure diagnostic proposal (#294) to be routed into recovery. */
|
||||
val diagnosisMinConfidence: Double = 0.5,
|
||||
)
|
||||
|
||||
+4
@@ -48,4 +48,8 @@ data class OrchestratorEngines(
|
||||
// only when the "review" gate exhausts its retry budget. Null = no LLM judge wired; the
|
||||
// orchestrator then falls back to a deterministic allow-one-reset-then-fail policy.
|
||||
val salvageJudge: SalvageJudge? = null,
|
||||
// One-shot post-failure diagnostic (design task #294), consulted once per terminal-failure
|
||||
// fingerprint just before a run becomes terminal. Null = no diagnostic; the run fails terminally
|
||||
// as before (deterministic degrade).
|
||||
val postFailureDiagnoser: PostFailureDiagnoser? = null,
|
||||
)
|
||||
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
package com.correx.core.kernel.orchestration
|
||||
|
||||
import com.correx.core.events.events.RecoveryProposal
|
||||
|
||||
/**
|
||||
* The recorded facts handed to the one-shot post-failure diagnostic (design task #294). Assembled
|
||||
* by the kernel from the event log only — no fresh workspace observation (acceptance #2) — so the
|
||||
* diagnostic reasons over the same evidence replay will see.
|
||||
*/
|
||||
data class DiagnosisInput(
|
||||
val intent: String,
|
||||
val gate: String,
|
||||
val reason: String,
|
||||
val fingerprint: String,
|
||||
val retryHistory: List<String>,
|
||||
val attemptedActions: List<String>,
|
||||
val recoveryAvailable: Boolean,
|
||||
)
|
||||
|
||||
/**
|
||||
* Seam for the one-shot post-failure diagnostic (design task #294). When a run is about to become
|
||||
* terminal, the kernel consults this once per terminal-failure fingerprint to get an untrusted
|
||||
* [RecoveryProposal] for a materially different recovery route. Like the other inference seams
|
||||
* ([SalvageJudge], [SemanticReviewer]), the implementation is injected so the deterministic core
|
||||
* never runs inference itself; the call is tool-free and must not alter model temperature.
|
||||
*
|
||||
* The proposal is nondeterministic (LLM-backed), so invariant #9 requires the caller to record it —
|
||||
* and the kernel's validation decision — as a
|
||||
* [com.correx.core.events.events.PostFailureDiagnosedEvent]; replay reads that back and never
|
||||
* re-invokes the diagnoser. When none is wired (`null` in [OrchestratorEngines]), the run fails
|
||||
* terminally as before, so the feature degrades safely without inference.
|
||||
*/
|
||||
fun interface PostFailureDiagnoser {
|
||||
suspend fun diagnose(input: DiagnosisInput): RecoveryProposal?
|
||||
}
|
||||
Reference in New Issue
Block a user