feat(recovery): one bounded post-failure diagnostic before terminal FAILED (#294)
At the terminal boundary (repair ladder spent, or a non-recoverable gate exhaustion), run exactly one tool-free diagnostic inference per terminal failure fingerprint over recorded facts only. A validated — materially new, confident, recovery-stage-available — RecoveryProposal routes once into the existing recovery stage via the ticket machinery, bypassing the spent route budget but bounded by a one-diagnosis-per-fingerprint dedupe so no loop is possible. Otherwise the run stays terminal FAILED (safe degrade when no diagnoser is wired). - New PostFailureDiagnosedEvent + nested RecoveryProposal (registered in eventModule); every observation/proposal/decision/route recorded for replay. - PostFailureDiagnoser seam (nullable, mirrors SalvageJudge) + DiagnosisInput built from the event log only (no fresh workspace observation). - diagnosisMinConfidence tuning knob. - Hooked at both terminal boundaries: routeToRecovery ladder-exhausted and decideGateExhaustion. Tests: RecoveryRoutingTest (route-once-then-bounded, low-confidence stays terminal), EventsTest serialization round-trip (proposal + null). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -169,6 +169,47 @@ data class RetrySalvageDecidedEvent(
|
|||||||
val rationale: String,
|
val rationale: String,
|
||||||
) : EventPayload
|
) : EventPayload
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A structured, untrusted recovery proposal produced by the one-shot post-failure diagnostic
|
||||||
|
* (design task #294). The diagnostic inference is tool-free and reads only recorded facts; this is
|
||||||
|
* its proposal. [expectedFingerprint] is the failure fingerprint the proposed [recoveryAction] is
|
||||||
|
* predicted to change the run to — the kernel routes only when it is materially different from the
|
||||||
|
* current terminal fingerprint (i.e. a genuinely new path, not the same dead end). [noRecovery]
|
||||||
|
* lets the model explicitly decline; [confidence] is thresholded by the kernel. LLM-proposed and
|
||||||
|
* therefore untrusted (invariant #7): validated deterministically before it can affect routing.
|
||||||
|
*/
|
||||||
|
@Serializable
|
||||||
|
data class RecoveryProposal(
|
||||||
|
val diagnosis: String,
|
||||||
|
val citedEvidence: String,
|
||||||
|
val recoveryAction: String,
|
||||||
|
val expectedFingerprint: String,
|
||||||
|
val confidence: Double,
|
||||||
|
val noRecovery: Boolean = false,
|
||||||
|
)
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Records the one-shot post-failure diagnostic (design task #294): when a run is about to become
|
||||||
|
* terminal, exactly one tool-free diagnostic inference runs per terminal-failure [fingerprint]. The
|
||||||
|
* nondeterministic [proposal] (LLM-backed, null when none/unparseable), the kernel's deterministic
|
||||||
|
* validation [decision], and whether it [routed] into the existing recovery stage are all recorded
|
||||||
|
* here so replay reproduces the decision without re-invoking the diagnoser (invariants #7/#9). The
|
||||||
|
* per-fingerprint dedupe that bounds this to one attempt keys off this event.
|
||||||
|
*/
|
||||||
|
@Serializable
|
||||||
|
@SerialName("PostFailureDiagnosed")
|
||||||
|
data class PostFailureDiagnosedEvent(
|
||||||
|
val sessionId: SessionId,
|
||||||
|
val stageId: StageId,
|
||||||
|
val gate: String,
|
||||||
|
val fingerprint: String,
|
||||||
|
val proposal: RecoveryProposal?,
|
||||||
|
// ROUTE | TERMINAL_NO_PROPOSAL | TERMINAL_NO_RECOVERY | TERMINAL_LOW_CONFIDENCE |
|
||||||
|
// TERMINAL_NOT_MATERIAL | TERMINAL_NO_ROUTE
|
||||||
|
val decision: String,
|
||||||
|
val routed: Boolean,
|
||||||
|
) : EventPayload
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A stage repeatedly blocked on a missing build prerequisite (see repeatedBuildCriticalReferenceBlock,
|
* A stage repeatedly blocked on a missing build prerequisite (see repeatedBuildCriticalReferenceBlock,
|
||||||
* design 2026-07-15 §#170) AND the stage both holds file_write and declares the path in-scope, so the
|
* design 2026-07-15 §#170) AND the stage both holds file_write and declares the path in-scope, so the
|
||||||
|
|||||||
@@ -65,6 +65,7 @@ import com.correx.core.events.events.RepoKnowledgeRetrievedEvent
|
|||||||
import com.correx.core.events.events.RepoMapComputedEvent
|
import com.correx.core.events.events.RepoMapComputedEvent
|
||||||
import com.correx.core.events.events.RetryAttemptedEvent
|
import com.correx.core.events.events.RetryAttemptedEvent
|
||||||
import com.correx.core.events.events.RetrySalvageDecidedEvent
|
import com.correx.core.events.events.RetrySalvageDecidedEvent
|
||||||
|
import com.correx.core.events.events.PostFailureDiagnosedEvent
|
||||||
import com.correx.core.events.events.FailureTicketOpenedEvent
|
import com.correx.core.events.events.FailureTicketOpenedEvent
|
||||||
import com.correx.core.events.events.BuildPrerequisiteBootstrapAttemptedEvent
|
import com.correx.core.events.events.BuildPrerequisiteBootstrapAttemptedEvent
|
||||||
import com.correx.core.events.events.WorkspaceVerificationObservedEvent
|
import com.correx.core.events.events.WorkspaceVerificationObservedEvent
|
||||||
@@ -161,6 +162,7 @@ val eventModule = SerializersModule {
|
|||||||
subclass(WorkflowCompletedEvent::class)
|
subclass(WorkflowCompletedEvent::class)
|
||||||
subclass(RetryAttemptedEvent::class)
|
subclass(RetryAttemptedEvent::class)
|
||||||
subclass(RetrySalvageDecidedEvent::class)
|
subclass(RetrySalvageDecidedEvent::class)
|
||||||
|
subclass(PostFailureDiagnosedEvent::class)
|
||||||
subclass(FailureTicketOpenedEvent::class)
|
subclass(FailureTicketOpenedEvent::class)
|
||||||
subclass(BuildPrerequisiteBootstrapAttemptedEvent::class)
|
subclass(BuildPrerequisiteBootstrapAttemptedEvent::class)
|
||||||
subclass(WorkspaceVerificationObservedEvent::class)
|
subclass(WorkspaceVerificationObservedEvent::class)
|
||||||
|
|||||||
+4
@@ -121,6 +121,10 @@ class DefaultSessionOrchestrator(
|
|||||||
// decideGateExhaustion) so the feature degrades safely without inference.
|
// decideGateExhaustion) so the feature degrades safely without inference.
|
||||||
internal val salvageJudge: SalvageJudge? = engines.salvageJudge
|
internal val salvageJudge: SalvageJudge? = engines.salvageJudge
|
||||||
|
|
||||||
|
// One-shot post-failure diagnostic (#294): consulted once per terminal-failure fingerprint just
|
||||||
|
// before a run goes terminal. Null = deterministic degrade (fail terminally, see terminalOrDiagnose).
|
||||||
|
internal val postFailureDiagnoser: PostFailureDiagnoser? = engines.postFailureDiagnoser
|
||||||
|
|
||||||
override val subagentRunner: SubagentRunner = InSessionSubagentRunner(
|
override val subagentRunner: SubagentRunner = InSessionSubagentRunner(
|
||||||
executeStage = { sid, stg, graph, session, cfg ->
|
executeStage = { sid, stg, graph, session, cfg ->
|
||||||
executeStage(sid, stg, graph, session, cfg, effectivesFor(cfg))
|
executeStage(sid, stg, graph, session, cfg, effectivesFor(cfg))
|
||||||
|
|||||||
+95
-5
@@ -1,5 +1,8 @@
|
|||||||
package com.correx.core.kernel.orchestration
|
package com.correx.core.kernel.orchestration
|
||||||
import com.correx.core.events.events.FileWrittenEvent
|
import com.correx.core.events.events.FileWrittenEvent
|
||||||
|
import com.correx.core.events.events.InitialIntentEvent
|
||||||
|
import com.correx.core.events.events.PostFailureDiagnosedEvent
|
||||||
|
import com.correx.core.events.events.RetryAttemptedEvent
|
||||||
import com.correx.core.events.events.ToolInvocationRequestedEvent
|
import com.correx.core.events.events.ToolInvocationRequestedEvent
|
||||||
import com.correx.core.events.events.FailureTicketOpenedEvent
|
import com.correx.core.events.events.FailureTicketOpenedEvent
|
||||||
import com.correx.core.events.events.StoredEvent
|
import com.correx.core.events.events.StoredEvent
|
||||||
@@ -109,13 +112,14 @@ internal suspend fun DefaultSessionOrchestrator.routeToRecovery(
|
|||||||
}
|
}
|
||||||
if (route == null) {
|
if (route == null) {
|
||||||
if (owner == null && arbiter == null) return null // nothing to route to: legacy retry path
|
if (owner == null && arbiter == null) return null // nothing to route to: legacy retry path
|
||||||
return StepResult.Terminal(
|
// Terminal boundary: the repair ladder is spent. Give the run one bounded post-failure
|
||||||
failWorkflow(
|
// diagnostic (#294) before FAILED — it may find a materially-new route the budget accounting lacked.
|
||||||
ctx.sessionId,
|
return terminalOrDiagnose(
|
||||||
|
ctx,
|
||||||
stageId,
|
stageId,
|
||||||
|
gate,
|
||||||
"repair ladder exhausted for stage ${stageId.value} (gate=$gate): $reason",
|
"repair ladder exhausted for stage ${stageId.value} (gate=$gate): $reason",
|
||||||
retryExhausted = true,
|
state,
|
||||||
),
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -153,6 +157,92 @@ internal suspend fun DefaultSessionOrchestrator.routeToRecovery(
|
|||||||
return enterStage(ctx.copy(currentStageId = advancedTo), route.target)
|
return enterStage(ctx.copy(currentStageId = advancedTo), route.target)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// How many recent tool actions to hand the diagnostic as "what was already tried".
|
||||||
|
private const val DIAGNOSIS_ACTION_TAIL = 10
|
||||||
|
|
||||||
|
/**
|
||||||
|
* One-shot post-failure diagnostic (design task #294). Called at the terminal boundary — when a run
|
||||||
|
* is about to become terminal FAILED. Runs exactly one tool-free diagnostic inference per terminal
|
||||||
|
* failure [fingerprint] (deduped on the recorded [PostFailureDiagnosedEvent], so no loop is possible)
|
||||||
|
* over recorded facts only. If the untrusted proposal is validated as materially new, confident, and
|
||||||
|
* a recovery stage exists, routes once into that stage via the existing ticket machinery — bypassing
|
||||||
|
* the already-spent route budget, since the fresh proposal is evidence the budget accounting lacked.
|
||||||
|
* Otherwise, or when no diagnoser is wired, returns the terminal failure unchanged (safe degrade).
|
||||||
|
* Every observation, proposal, validation decision and route is recorded (invariants #7/#9), so
|
||||||
|
* replay reproduces the decision without re-invoking the diagnoser.
|
||||||
|
*/
|
||||||
|
@Suppress("ReturnCount") // guard-clause ladder over the validation decision — flattest form
|
||||||
|
internal suspend fun DefaultSessionOrchestrator.terminalOrDiagnose(
|
||||||
|
ctx: EnrichedExecutionContext,
|
||||||
|
stageId: StageId,
|
||||||
|
gate: String,
|
||||||
|
reason: String,
|
||||||
|
state: OrchestrationState,
|
||||||
|
): StepResult {
|
||||||
|
val terminal: suspend () -> StepResult =
|
||||||
|
{ StepResult.Terminal(failWorkflow(ctx.sessionId, stageId, reason, retryExhausted = true)) }
|
||||||
|
val diagnoser = postFailureDiagnoser ?: return terminal()
|
||||||
|
val fingerprint = FailureFingerprint.of(reason)
|
||||||
|
val events = repositories.eventStore.read(ctx.sessionId)
|
||||||
|
// Acceptance #1/#6: at most one diagnosis per terminal fingerprint — this is what bounds the loop.
|
||||||
|
if (events.any { (it.payload as? PostFailureDiagnosedEvent)?.fingerprint == fingerprint }) return terminal()
|
||||||
|
|
||||||
|
val recoveryStage = findRecoveryStage(ctx.graph, stageId)
|
||||||
|
// Acceptance #2: recorded facts only, no fresh workspace observation.
|
||||||
|
val input = DiagnosisInput(
|
||||||
|
intent = events.mapNotNull { it.payload as? InitialIntentEvent }.lastOrNull()?.intent.orEmpty(),
|
||||||
|
gate = gate,
|
||||||
|
reason = reason,
|
||||||
|
fingerprint = fingerprint,
|
||||||
|
retryHistory = events.mapNotNull { it.payload as? RetryAttemptedEvent }
|
||||||
|
.map { "${it.gate} attempt=${it.attemptNumber} fp=${it.fingerprint}" },
|
||||||
|
attemptedActions = events.mapNotNull { it.payload as? ToolInvocationRequestedEvent }
|
||||||
|
.takeLast(DIAGNOSIS_ACTION_TAIL).map { it.toolName },
|
||||||
|
recoveryAvailable = recoveryStage != null,
|
||||||
|
)
|
||||||
|
val proposal = runCatching { diagnoser.diagnose(input) }.getOrNull()
|
||||||
|
val decision = when {
|
||||||
|
proposal == null -> "TERMINAL_NO_PROPOSAL"
|
||||||
|
proposal.noRecovery -> "TERMINAL_NO_RECOVERY"
|
||||||
|
proposal.confidence < tuning.diagnosisMinConfidence -> "TERMINAL_LOW_CONFIDENCE"
|
||||||
|
proposal.expectedFingerprint.isBlank() || proposal.expectedFingerprint == fingerprint -> "TERMINAL_NOT_MATERIAL"
|
||||||
|
recoveryStage == null -> "TERMINAL_NO_ROUTE"
|
||||||
|
else -> "ROUTE"
|
||||||
|
}
|
||||||
|
val routed = decision == "ROUTE"
|
||||||
|
emit(
|
||||||
|
ctx.sessionId,
|
||||||
|
PostFailureDiagnosedEvent(ctx.sessionId, stageId, gate, fingerprint, proposal, decision, routed),
|
||||||
|
)
|
||||||
|
log.info(
|
||||||
|
"[Orchestrator] post-failure diagnosis session={} stage={} gate={} decision={} routed={}",
|
||||||
|
ctx.sessionId.value, stageId.value, gate, decision, routed,
|
||||||
|
)
|
||||||
|
if (!routed || recoveryStage == null) return terminal()
|
||||||
|
|
||||||
|
// One validated, materially-new route into the existing recovery stage. Reuses the ticket
|
||||||
|
// machinery so buildRecoveryTicketEntry feeds the narrow repair bundle and ticketReturnMove
|
||||||
|
// re-runs the origin gate. Bounded by the per-fingerprint dedupe above, not the spent budget.
|
||||||
|
val used = state.recoveryRoutes[stageId.value + INTENT_BUDGET_SUFFIX] ?: 0
|
||||||
|
emit(
|
||||||
|
ctx.sessionId,
|
||||||
|
FailureTicketOpenedEvent(
|
||||||
|
sessionId = ctx.sessionId,
|
||||||
|
stageId = stageId,
|
||||||
|
gate = gate,
|
||||||
|
category = ticketCategory(gate),
|
||||||
|
requiredCapability = GATE_REQUIRED_CAPABILITY[gate] ?: "file_write",
|
||||||
|
routeTo = recoveryStage,
|
||||||
|
evidence = reason,
|
||||||
|
routeAttempt = used + 1,
|
||||||
|
fingerprint = fingerprint,
|
||||||
|
escalated = true,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
val advancedTo = advanceStage(ctx.sessionId, stageId, TransitionDecision.Move(TICKET_ROUTE, recoveryStage))
|
||||||
|
return enterStage(ctx.copy(currentStageId = advancedTo), recoveryStage)
|
||||||
|
}
|
||||||
|
|
||||||
/** A chosen rung of the repair ladder: where to route, which budget it charges, and its tier. */
|
/** A chosen rung of the repair ladder: where to route, which budget it charges, and its tier. */
|
||||||
private data class RouteTier(
|
private data class RouteTier(
|
||||||
val target: StageId,
|
val target: StageId,
|
||||||
|
|||||||
+3
-2
@@ -374,7 +374,8 @@ internal suspend fun DefaultSessionOrchestrator.decideGateExhaustion(
|
|||||||
): StepResult? {
|
): StepResult? {
|
||||||
val sessionId = ctx.sessionId
|
val sessionId = ctx.sessionId
|
||||||
if (gate != "review" || state.gateSalvageUsed.contains(gate)) {
|
if (gate != "review" || state.gateSalvageUsed.contains(gate)) {
|
||||||
return StepResult.Terminal(failWorkflow(sessionId, stageId, reason, retryExhausted = true))
|
// Terminal boundary: give the run one bounded post-failure diagnostic (#294) before FAILED.
|
||||||
|
return terminalOrDiagnose(ctx, stageId, gate, reason, state)
|
||||||
}
|
}
|
||||||
// No judge wired: degrade safely with a deterministic allow-one-reset-then-fail policy —
|
// No judge wired: degrade safely with a deterministic allow-one-reset-then-fail policy —
|
||||||
// the gateSalvageUsed check above already ensures this fires at most once per gate.
|
// the gateSalvageUsed check above already ensures this fires at most once per gate.
|
||||||
@@ -386,7 +387,7 @@ internal suspend fun DefaultSessionOrchestrator.decideGateExhaustion(
|
|||||||
emit(sessionId, RetrySalvageDecidedEvent(sessionId, stageId, gate, judgment.decision, judgment.rationale))
|
emit(sessionId, RetrySalvageDecidedEvent(sessionId, stageId, gate, judgment.decision, judgment.rationale))
|
||||||
return when (judgment.decision) {
|
return when (judgment.decision) {
|
||||||
SalvageDecision.CONTINUE -> null
|
SalvageDecision.CONTINUE -> null
|
||||||
SalvageDecision.FAIL -> StepResult.Terminal(failWorkflow(sessionId, stageId, reason, retryExhausted = true))
|
SalvageDecision.FAIL -> terminalOrDiagnose(ctx, stageId, gate, reason, state)
|
||||||
// The judge chose recovery: route to the recovery stage (file_write is the capability it
|
// The judge chose recovery: route to the recovery stage (file_write is the capability it
|
||||||
// provides). Degrade to terminal if the graph declares no recovery stage.
|
// provides). Degrade to terminal if the graph declares no recovery stage.
|
||||||
SalvageDecision.RECOVER ->
|
SalvageDecision.RECOVER ->
|
||||||
|
|||||||
+2
@@ -49,4 +49,6 @@ data class OrchestrationTuning(
|
|||||||
* interleaved successes — it counts a normalized failure signature across the whole stage.
|
* interleaved successes — it counts a normalized failure signature across the whole stage.
|
||||||
*/
|
*/
|
||||||
val stageFailureLoopLimit: Int = 6,
|
val stageFailureLoopLimit: Int = 6,
|
||||||
|
/** Minimum confidence for a post-failure diagnostic proposal (#294) to be routed into recovery. */
|
||||||
|
val diagnosisMinConfidence: Double = 0.5,
|
||||||
)
|
)
|
||||||
|
|||||||
+4
@@ -48,4 +48,8 @@ data class OrchestratorEngines(
|
|||||||
// only when the "review" gate exhausts its retry budget. Null = no LLM judge wired; the
|
// only when the "review" gate exhausts its retry budget. Null = no LLM judge wired; the
|
||||||
// orchestrator then falls back to a deterministic allow-one-reset-then-fail policy.
|
// orchestrator then falls back to a deterministic allow-one-reset-then-fail policy.
|
||||||
val salvageJudge: SalvageJudge? = null,
|
val salvageJudge: SalvageJudge? = null,
|
||||||
|
// One-shot post-failure diagnostic (design task #294), consulted once per terminal-failure
|
||||||
|
// fingerprint just before a run becomes terminal. Null = no diagnostic; the run fails terminally
|
||||||
|
// as before (deterministic degrade).
|
||||||
|
val postFailureDiagnoser: PostFailureDiagnoser? = null,
|
||||||
)
|
)
|
||||||
|
|||||||
+35
@@ -0,0 +1,35 @@
|
|||||||
|
package com.correx.core.kernel.orchestration
|
||||||
|
|
||||||
|
import com.correx.core.events.events.RecoveryProposal
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The recorded facts handed to the one-shot post-failure diagnostic (design task #294). Assembled
|
||||||
|
* by the kernel from the event log only — no fresh workspace observation (acceptance #2) — so the
|
||||||
|
* diagnostic reasons over the same evidence replay will see.
|
||||||
|
*/
|
||||||
|
data class DiagnosisInput(
|
||||||
|
val intent: String,
|
||||||
|
val gate: String,
|
||||||
|
val reason: String,
|
||||||
|
val fingerprint: String,
|
||||||
|
val retryHistory: List<String>,
|
||||||
|
val attemptedActions: List<String>,
|
||||||
|
val recoveryAvailable: Boolean,
|
||||||
|
)
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Seam for the one-shot post-failure diagnostic (design task #294). When a run is about to become
|
||||||
|
* terminal, the kernel consults this once per terminal-failure fingerprint to get an untrusted
|
||||||
|
* [RecoveryProposal] for a materially different recovery route. Like the other inference seams
|
||||||
|
* ([SalvageJudge], [SemanticReviewer]), the implementation is injected so the deterministic core
|
||||||
|
* never runs inference itself; the call is tool-free and must not alter model temperature.
|
||||||
|
*
|
||||||
|
* The proposal is nondeterministic (LLM-backed), so invariant #9 requires the caller to record it —
|
||||||
|
* and the kernel's validation decision — as a
|
||||||
|
* [com.correx.core.events.events.PostFailureDiagnosedEvent]; replay reads that back and never
|
||||||
|
* re-invokes the diagnoser. When none is wired (`null` in [OrchestratorEngines]), the run fails
|
||||||
|
* terminally as before, so the feature degrades safely without inference.
|
||||||
|
*/
|
||||||
|
fun interface PostFailureDiagnoser {
|
||||||
|
suspend fun diagnose(input: DiagnosisInput): RecoveryProposal?
|
||||||
|
}
|
||||||
+46
@@ -3,6 +3,8 @@ package com.correx.testing.contracts.model.orchestration
|
|||||||
import com.correx.core.events.events.EventPayload
|
import com.correx.core.events.events.EventPayload
|
||||||
import com.correx.core.events.events.OrchestrationPausedEvent
|
import com.correx.core.events.events.OrchestrationPausedEvent
|
||||||
import com.correx.core.events.events.OrchestrationResumedEvent
|
import com.correx.core.events.events.OrchestrationResumedEvent
|
||||||
|
import com.correx.core.events.events.PostFailureDiagnosedEvent
|
||||||
|
import com.correx.core.events.events.RecoveryProposal
|
||||||
import com.correx.core.events.events.RetryAttemptedEvent
|
import com.correx.core.events.events.RetryAttemptedEvent
|
||||||
import com.correx.core.events.events.WorkflowCompletedEvent
|
import com.correx.core.events.events.WorkflowCompletedEvent
|
||||||
import com.correx.core.events.events.WorkflowFailedEvent
|
import com.correx.core.events.events.WorkflowFailedEvent
|
||||||
@@ -45,6 +47,50 @@ class EventsTest {
|
|||||||
assertEquals(event, decoded)
|
assertEquals(event, decoded)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `PostFailureDiagnosedEvent survives round-trip with a proposal`() {
|
||||||
|
// Guards the #294 event AND its nested RecoveryProposal against the silent-registration trap.
|
||||||
|
val event: EventPayload = PostFailureDiagnosedEvent(
|
||||||
|
sessionId = SessionId("s1"),
|
||||||
|
stageId = StageId("stage-a"),
|
||||||
|
gate = "execution",
|
||||||
|
fingerprint = "fp-abc",
|
||||||
|
proposal = RecoveryProposal(
|
||||||
|
diagnosis = "type mismatch",
|
||||||
|
citedEvidence = "App.tsx: TS2322",
|
||||||
|
recoveryAction = "narrow the return type",
|
||||||
|
expectedFingerprint = "fp-xyz",
|
||||||
|
confidence = 0.9,
|
||||||
|
noRecovery = false,
|
||||||
|
),
|
||||||
|
decision = "ROUTE",
|
||||||
|
routed = true,
|
||||||
|
)
|
||||||
|
|
||||||
|
val json = eventJson.encodeToString(EventPayload.serializer(), event)
|
||||||
|
val decoded = eventJson.decodeFromString(EventPayload.serializer(), json)
|
||||||
|
|
||||||
|
assertEquals(event, decoded)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `PostFailureDiagnosedEvent survives round-trip with a null proposal`() {
|
||||||
|
val event: EventPayload = PostFailureDiagnosedEvent(
|
||||||
|
sessionId = SessionId("s1"),
|
||||||
|
stageId = StageId("stage-a"),
|
||||||
|
gate = "execution",
|
||||||
|
fingerprint = "fp-abc",
|
||||||
|
proposal = null,
|
||||||
|
decision = "TERMINAL_NO_PROPOSAL",
|
||||||
|
routed = false,
|
||||||
|
)
|
||||||
|
|
||||||
|
val json = eventJson.encodeToString(EventPayload.serializer(), event)
|
||||||
|
val decoded = eventJson.decodeFromString(EventPayload.serializer(), json)
|
||||||
|
|
||||||
|
assertEquals(event, decoded)
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
fun `WorkflowFailedEvent survives round-trip`() {
|
fun `WorkflowFailedEvent survives round-trip`() {
|
||||||
val event: EventPayload = WorkflowFailedEvent(
|
val event: EventPayload = WorkflowFailedEvent(
|
||||||
|
|||||||
@@ -11,6 +11,8 @@ import com.correx.core.approvals.model.DomainApprovalRequest
|
|||||||
import com.correx.core.artifacts.DefaultArtifactReducer
|
import com.correx.core.artifacts.DefaultArtifactReducer
|
||||||
import com.correx.core.events.EventDispatcher
|
import com.correx.core.events.EventDispatcher
|
||||||
import com.correx.core.events.events.FailureTicketOpenedEvent
|
import com.correx.core.events.events.FailureTicketOpenedEvent
|
||||||
|
import com.correx.core.events.events.PostFailureDiagnosedEvent
|
||||||
|
import com.correx.core.events.events.RecoveryProposal
|
||||||
import com.correx.core.events.events.StaticAnalysisCompletedEvent
|
import com.correx.core.events.events.StaticAnalysisCompletedEvent
|
||||||
import com.correx.core.events.events.TransitionExecutedEvent
|
import com.correx.core.events.events.TransitionExecutedEvent
|
||||||
import com.correx.core.events.events.WorkflowCompletedEvent
|
import com.correx.core.events.events.WorkflowCompletedEvent
|
||||||
@@ -46,6 +48,7 @@ import com.correx.core.kernel.orchestration.OrchestrationRepository
|
|||||||
import com.correx.core.kernel.orchestration.OrchestratorEngines
|
import com.correx.core.kernel.orchestration.OrchestratorEngines
|
||||||
import com.correx.core.kernel.orchestration.OrchestratorRepositories
|
import com.correx.core.kernel.orchestration.OrchestratorRepositories
|
||||||
import com.correx.core.kernel.orchestration.StaticAnalysisRunResult
|
import com.correx.core.kernel.orchestration.StaticAnalysisRunResult
|
||||||
|
import com.correx.core.kernel.orchestration.PostFailureDiagnoser
|
||||||
import com.correx.core.kernel.orchestration.StaticAnalysisRunner
|
import com.correx.core.kernel.orchestration.StaticAnalysisRunner
|
||||||
import com.correx.core.kernel.retry.DefaultRetryCoordinator
|
import com.correx.core.kernel.retry.DefaultRetryCoordinator
|
||||||
import com.correx.core.risk.DefaultRiskAssessor
|
import com.correx.core.risk.DefaultRiskAssessor
|
||||||
@@ -77,6 +80,7 @@ import kotlinx.datetime.Instant
|
|||||||
import java.nio.file.Files
|
import java.nio.file.Files
|
||||||
import kotlin.test.Test
|
import kotlin.test.Test
|
||||||
import kotlin.test.assertEquals
|
import kotlin.test.assertEquals
|
||||||
|
import kotlin.test.assertFalse
|
||||||
import kotlin.test.assertTrue
|
import kotlin.test.assertTrue
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -428,6 +432,7 @@ class RecoveryRoutingTest {
|
|||||||
staticOutput: String = "App.tsx: TS2322 type mismatch",
|
staticOutput: String = "App.tsx: TS2322 type mismatch",
|
||||||
provider: InferenceProvider = StageRoutingProvider(),
|
provider: InferenceProvider = StageRoutingProvider(),
|
||||||
staticExitCode: Int = 1,
|
staticExitCode: Int = 1,
|
||||||
|
postFailureDiagnoser: PostFailureDiagnoser? = null,
|
||||||
): Pair<DefaultSessionOrchestrator, InMemoryEventStore> {
|
): Pair<DefaultSessionOrchestrator, InMemoryEventStore> {
|
||||||
val eventStore = InMemoryEventStore()
|
val eventStore = InMemoryEventStore()
|
||||||
val shell = ShellTool(allowedExecutables = setOf("true"), workingDir = workspace)
|
val shell = ShellTool(allowedExecutables = setOf("true"), workingDir = workspace)
|
||||||
@@ -477,6 +482,7 @@ class RecoveryRoutingTest {
|
|||||||
staticAnalysisRunner = StaticAnalysisRunner { _, _ ->
|
staticAnalysisRunner = StaticAnalysisRunner { _, _ ->
|
||||||
StaticAnalysisRunResult(exitCode = staticExitCode, output = staticOutput)
|
StaticAnalysisRunResult(exitCode = staticExitCode, output = staticOutput)
|
||||||
},
|
},
|
||||||
|
postFailureDiagnoser = postFailureDiagnoser,
|
||||||
approvalEngine = object : ApprovalEngine {
|
approvalEngine = object : ApprovalEngine {
|
||||||
override fun evaluate(
|
override fun evaluate(
|
||||||
request: DomainApprovalRequest,
|
request: DomainApprovalRequest,
|
||||||
@@ -650,6 +656,73 @@ class RecoveryRoutingTest {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `post-failure diagnosis routes one validated proposal into recovery then stays bounded`(): Unit = runBlocking {
|
||||||
|
// #294: after the recovery ladder is fully spent on an unchanged fingerprint, the one-shot
|
||||||
|
// diagnostic proposes a materially-new, confident route. It routes once more into recovery,
|
||||||
|
// then — the fingerprint still unchanged — the per-fingerprint dedupe forces terminal. No loop.
|
||||||
|
var diagnosisCalls = 0
|
||||||
|
val diagnoser = PostFailureDiagnoser { input ->
|
||||||
|
diagnosisCalls++
|
||||||
|
RecoveryProposal(
|
||||||
|
diagnosis = "impl keeps emitting the same type error",
|
||||||
|
citedEvidence = input.reason,
|
||||||
|
recoveryAction = "narrow the return type in App.tsx",
|
||||||
|
expectedFingerprint = "materially-different-fingerprint",
|
||||||
|
confidence = 0.9,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
val sid = SessionId("diag-route")
|
||||||
|
val (orchestrator, eventStore) = buildMultiToolOrchestrator(
|
||||||
|
staticOutput = "App.tsx: TS2322 type mismatch",
|
||||||
|
postFailureDiagnoser = diagnoser,
|
||||||
|
)
|
||||||
|
val config = OrchestrationConfig(
|
||||||
|
retryPolicy = RetryPolicy(maxAttempts = 10, backoffMs = 0, perGateMaxAttempts = mapOf("static_analysis" to 1)),
|
||||||
|
)
|
||||||
|
|
||||||
|
val result = orchestrator.run(sid, capableButStuckGraph(), config)
|
||||||
|
|
||||||
|
assertTrue(result is WorkflowResult.Failed, "bounded: still terminates after the one diagnostic route")
|
||||||
|
val diagnoses = eventStore.read(sid).mapNotNull { it.payload as? PostFailureDiagnosedEvent }
|
||||||
|
assertEquals(1, diagnoses.size, "exactly one diagnosis per terminal fingerprint (#1/#6)")
|
||||||
|
assertEquals("ROUTE", diagnoses.single().decision)
|
||||||
|
assertTrue(diagnoses.single().routed, "a validated materially-new proposal routes into recovery (#3)")
|
||||||
|
assertEquals(1, diagnosisCalls, "the diagnoser is consulted at most once for the fingerprint")
|
||||||
|
// Acceptance #2: the diagnostic saw recorded gate evidence, not a fresh observation.
|
||||||
|
assertTrue(
|
||||||
|
diagnoses.single().proposal?.citedEvidence?.contains("TS2322") == true,
|
||||||
|
"diagnosis input carried the recorded gate evidence",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `low-confidence post-failure proposal leaves the session terminal`(): Unit = runBlocking {
|
||||||
|
// #294 acceptance #4: an invalid (here: low-confidence) proposal is recorded but never routed.
|
||||||
|
val diagnoser = PostFailureDiagnoser {
|
||||||
|
RecoveryProposal(
|
||||||
|
diagnosis = "unsure",
|
||||||
|
citedEvidence = it.reason,
|
||||||
|
recoveryAction = "maybe try again",
|
||||||
|
expectedFingerprint = "different-but-unconfident",
|
||||||
|
confidence = 0.1,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
val sid = SessionId("diag-terminal")
|
||||||
|
val (orchestrator, eventStore) = buildMultiToolOrchestrator(postFailureDiagnoser = diagnoser)
|
||||||
|
val config = OrchestrationConfig(
|
||||||
|
retryPolicy = RetryPolicy(maxAttempts = 10, backoffMs = 0, perGateMaxAttempts = mapOf("static_analysis" to 1)),
|
||||||
|
)
|
||||||
|
|
||||||
|
val result = orchestrator.run(sid, capableButStuckGraph(), config)
|
||||||
|
|
||||||
|
assertTrue(result is WorkflowResult.Failed)
|
||||||
|
val diagnoses = eventStore.read(sid).mapNotNull { it.payload as? PostFailureDiagnosedEvent }
|
||||||
|
assertEquals(1, diagnoses.size, "still one diagnosis attempt, but not routed")
|
||||||
|
assertEquals("TERMINAL_LOW_CONFIDENCE", diagnoses.single().decision)
|
||||||
|
assertFalse(diagnoses.single().routed)
|
||||||
|
}
|
||||||
|
|
||||||
// impl carries a generative "scaffold" mandate as its own prompt. On the FORWARD visit it applies;
|
// impl carries a generative "scaffold" mandate as its own prompt. On the FORWARD visit it applies;
|
||||||
// when the ticket routes impl back to repair its own file, that mandate must be suppressed (else it
|
// when the ticket routes impl back to repair its own file, that mandate must be suppressed (else it
|
||||||
// re-scaffolds and stubs the real file). Same topology as ownerGraph otherwise.
|
// re-scaffolds and stubs the real file). Same topology as ownerGraph otherwise.
|
||||||
|
|||||||
Reference in New Issue
Block a user