feat(retry): per-gate retry budgets + progress-aware charging + hybrid salvage
Replaces the single shared per-stage retryCount (reset on TransitionExecuted, shared across all post-stage gates) with a per-gate budget. Motivated by run 2e468f9f, where a promising freestyle run was terminally FAILED because the contract gate burned all 3 shared retries on one trivial miss before the semantic-review gate ever ran. - StageExecutionResult.Failure gains a `gate` id; each gate tags its failures. - OrchestrationState gains gateRetryBudgets / gateFailureFingerprints / gateSalvageUsed (all rebuilt from events, reset per-stage on TransitionExecuted). - FailureFingerprint normalizes+hashes the failure reason; RetryCoordinator.decide charges a gate's budget only when the fingerprint is unchanged (no progress), so a moving run is never penalised. Returns Retry | Exhausted. - Hybrid exhaustion: deterministic gates fail; the review gate consults a SalvageJudge (LLM, or a deterministic allow-one-reset fallback), recorded as RetrySalvageDecidedEvent (invariant #9). CONTINUE resets the gate budget once; a second exhaustion is terminal. - Review gate is now the sole authority on review-retry termination; the old REVIEW_BLOCK_RETRY_CAP is repurposed as a high absolute backstop only. - ServerModule escaped-exception path now records a truthful terminal WorkflowFailed (real stage/reason/retryExhausted) via recordUnhandledFailure. Tests: DefaultRetryCoordinatorTest (fingerprint charging) + GateRetryBudgetExhaustionTest (deterministic exhaust / review CONTINUE-then- terminal / review FAIL / null-judge fallback) + reducer coverage.
This commit is contained in:
@@ -59,4 +59,32 @@ data class RetryAttemptedEvent(
|
||||
val attemptNumber: Int,
|
||||
val maxAttempts: Int,
|
||||
val failureReason: String,
|
||||
// Per-gate retry budgets (design 2026-07-06-per-gate-retry-budgets.md): budget key this attempt
|
||||
// is charged against. Defaulted to the legacy shared "stage" bucket for schema back-compat.
|
||||
val gate: String = "stage",
|
||||
// Stable hash of the failure's normalised reason, used to detect progress across attempts.
|
||||
val fingerprint: String = "",
|
||||
// Whether this attempt consumed the gate's budget (false when the fingerprint changed from the
|
||||
// previous attempt — i.e. the run is moving, so the retry is free).
|
||||
val charged: Boolean = true,
|
||||
) : EventPayload
|
||||
|
||||
/** Outcome of the hybrid-exhaustion salvage judgement for the semantic review gate. */
|
||||
@Serializable
|
||||
enum class SalvageDecision { CONTINUE, FAIL }
|
||||
|
||||
/**
|
||||
* Recorded when a gate exhausts its retry budget and (per design decision 3) the salvage judge is
|
||||
* consulted — currently only the "review" gate. The judgement is nondeterministic (LLM-backed, or a
|
||||
* deterministic fallback when no judge is wired), so invariant #9 requires it be recorded here;
|
||||
* replay reads this event back and never re-invokes the judge.
|
||||
*/
|
||||
@Serializable
|
||||
@SerialName("RetrySalvageDecided")
|
||||
data class RetrySalvageDecidedEvent(
|
||||
val sessionId: SessionId,
|
||||
val stageId: StageId,
|
||||
val gate: String,
|
||||
val decision: SalvageDecision,
|
||||
val rationale: String,
|
||||
) : EventPayload
|
||||
|
||||
@@ -6,8 +6,15 @@ import kotlinx.serialization.Serializable
|
||||
data class RetryPolicy(
|
||||
val maxAttempts: Int,
|
||||
val backoffMs: Long = 1_000L,
|
||||
// Per-gate override (design 2026-07-06-per-gate-retry-budgets.md §8): a gate id present here uses
|
||||
// its own budget instead of [maxAttempts]. Empty by default — every gate then independently gets
|
||||
// [maxAttempts] (un-shared, per T2/T5), preserving today's numbers with no TOML changes required.
|
||||
val perGateMaxAttempts: Map<String, Int> = emptyMap(),
|
||||
) {
|
||||
init {
|
||||
require(maxAttempts >= 1) { "maxAttempts must be >= 1" }
|
||||
require(perGateMaxAttempts.values.all { it >= 1 }) { "perGateMaxAttempts values must be >= 1" }
|
||||
}
|
||||
|
||||
fun maxAttemptsFor(gate: String): Int = perGateMaxAttempts[gate] ?: maxAttempts
|
||||
}
|
||||
|
||||
+10
@@ -15,4 +15,14 @@ data class OrchestrationState(
|
||||
val failureReason: String? = null,
|
||||
val retryPolicy: RetryPolicy? = null,
|
||||
val refinementIterations: Map<String, Int> = emptyMap(),
|
||||
// Per-gate retry budget (design 2026-07-06-per-gate-retry-budgets.md): each gate id (produces,
|
||||
// brief_grounding, brief_echo, contract, plan_compile, static_analysis, execution, review, ...)
|
||||
// exhausts its own budget rather than sharing retryCount session-wide. Attempts charged so far.
|
||||
val gateRetryBudgets: Map<String, Int> = emptyMap(),
|
||||
// Last-seen failure fingerprint per gate, used to tell a no-progress retry (same fingerprint,
|
||||
// charged) from a genuine-progress retry (changed fingerprint, free).
|
||||
val gateFailureFingerprints: Map<String, String> = emptyMap(),
|
||||
// Gates that have already spent their one hybrid-exhaustion salvage reset (review gate only,
|
||||
// see RetrySalvageDecidedEvent) — a second exhaustion for that gate is terminal.
|
||||
val gateSalvageUsed: Set<String> = emptySet(),
|
||||
)
|
||||
|
||||
@@ -58,6 +58,7 @@ import com.correx.core.events.events.RefinementIterationEvent
|
||||
import com.correx.core.events.events.RepoKnowledgeRetrievedEvent
|
||||
import com.correx.core.events.events.RepoMapComputedEvent
|
||||
import com.correx.core.events.events.RetryAttemptedEvent
|
||||
import com.correx.core.events.events.RetrySalvageDecidedEvent
|
||||
import com.correx.core.events.events.WorkspaceStateObservedEvent
|
||||
import com.correx.core.events.events.RiskAssessedEvent
|
||||
import com.correx.core.events.events.SourceFetchedEvent
|
||||
@@ -144,6 +145,7 @@ val eventModule = SerializersModule {
|
||||
subclass(WorkflowFailedEvent::class)
|
||||
subclass(WorkflowCompletedEvent::class)
|
||||
subclass(RetryAttemptedEvent::class)
|
||||
subclass(RetrySalvageDecidedEvent::class)
|
||||
subclass(RefinementIterationEvent::class)
|
||||
subclass(RepoMapComputedEvent::class)
|
||||
subclass(WorkspaceStateObservedEvent::class)
|
||||
|
||||
Reference in New Issue
Block a user