feat(retry): per-gate retry budgets + progress-aware charging + hybrid salvage
Replaces the single shared per-stage retryCount (reset on TransitionExecuted, shared across all post-stage gates) with a per-gate budget. Motivated by run 2e468f9f, where a promising freestyle run was terminally FAILED because the contract gate burned all 3 shared retries on one trivial miss before the semantic-review gate ever ran. - StageExecutionResult.Failure gains a `gate` id; each gate tags its failures. - OrchestrationState gains gateRetryBudgets / gateFailureFingerprints / gateSalvageUsed (all rebuilt from events, reset per-stage on TransitionExecuted). - FailureFingerprint normalizes+hashes the failure reason; RetryCoordinator.decide charges a gate's budget only when the fingerprint is unchanged (no progress), so a moving run is never penalised. Returns Retry | Exhausted. - Hybrid exhaustion: deterministic gates fail; the review gate consults a SalvageJudge (LLM, or a deterministic allow-one-reset fallback), recorded as RetrySalvageDecidedEvent (invariant #9). CONTINUE resets the gate budget once; a second exhaustion is terminal. - Review gate is now the sole authority on review-retry termination; the old REVIEW_BLOCK_RETRY_CAP is repurposed as a high absolute backstop only. - ServerModule escaped-exception path now records a truthful terminal WorkflowFailed (real stage/reason/retryExhausted) via recordUnhandledFailure. Tests: DefaultRetryCoordinatorTest (fingerprint charging) + GateRetryBudgetExhaustionTest (deterministic exhaust / review CONTINUE-then- terminal / review FAIL / null-judge fallback) + reducer coverage.
This commit is contained in:
@@ -348,33 +348,53 @@ class ServerModule(
|
||||
.onFailure { log.warn("Profile adaptation failed: {}", it.message) }
|
||||
}
|
||||
}
|
||||
}.onFailure { ex ->
|
||||
log.error("runSession: session={} failed: {}", sessionId.value, ex.message, ex)
|
||||
eventStore.append(
|
||||
NewEvent(
|
||||
metadata = EventMetadata(
|
||||
eventId = EventId(java.util.UUID.randomUUID().toString()),
|
||||
sessionId = sessionId,
|
||||
timestamp = Clock.System.now(),
|
||||
schemaVersion = 1,
|
||||
causationId = null,
|
||||
correlationId = null,
|
||||
),
|
||||
payload = WorkflowFailedEvent(
|
||||
sessionId = sessionId,
|
||||
stageId = graph.start,
|
||||
reason = ex.message ?: "unexpected orchestrator failure",
|
||||
retryExhausted = false,
|
||||
),
|
||||
),
|
||||
)
|
||||
}
|
||||
}.onFailure { ex -> recordUnhandledFailure(sessionId, graph, sessionConfig, ex) }
|
||||
activeSessionJobs.remove(sessionId)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An exception escaped the orchestrator (the run never reached its own terminal WorkflowFailed).
|
||||
// Record a truthful terminal event: the real failing stage (not graph.start), the real message,
|
||||
// and a computed retryExhausted — so a headless client sees an honest failure, not a placeholder.
|
||||
private suspend fun recordUnhandledFailure(
|
||||
sessionId: SessionId,
|
||||
graph: com.correx.core.transitions.graph.WorkflowGraph,
|
||||
sessionConfig: OrchestrationConfig,
|
||||
ex: Throwable,
|
||||
) {
|
||||
log.error("runSession: session={} failed: {}", sessionId.value, ex.message, ex)
|
||||
val orchState = orchestrationRepository.getState(sessionId)
|
||||
val failingStageId = orchState.currentStageId ?: graph.start
|
||||
val reason = ex.message
|
||||
?.trim()
|
||||
?.takeIf { it.isNotEmpty() }
|
||||
?: "unexpected orchestrator failure: unhandled exception with no message " +
|
||||
"during stage '${failingStageId.value}'"
|
||||
val maxAttempts = orchState.retryPolicy?.maxAttempts
|
||||
?: sessionConfig.retryPolicy.maxAttempts
|
||||
val retryExhausted = orchState.retryCount >= maxAttempts
|
||||
eventStore.append(
|
||||
NewEvent(
|
||||
metadata = EventMetadata(
|
||||
eventId = EventId(java.util.UUID.randomUUID().toString()),
|
||||
sessionId = sessionId,
|
||||
timestamp = Clock.System.now(),
|
||||
schemaVersion = 1,
|
||||
causationId = null,
|
||||
correlationId = null,
|
||||
),
|
||||
payload = WorkflowFailedEvent(
|
||||
sessionId = sessionId,
|
||||
stageId = failingStageId,
|
||||
reason = reason,
|
||||
retryExhausted = retryExhausted,
|
||||
),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the workflow graph for resuming a freestyle phase-2 session. Phase-2 graphs
|
||||
* (`freestyle-<sessionId>`) are compiled from the locked plan, never registered in the
|
||||
|
||||
Reference in New Issue
Block a user