feat(retry): per-gate retry budgets + progress-aware charging + hybrid salvage

Replaces the single shared per-stage retryCount (reset on TransitionExecuted,
shared across all post-stage gates) with a per-gate budget. Motivated by run
2e468f9f, where a promising freestyle run was terminally FAILED because the
contract gate burned all 3 shared retries on one trivial miss before the
semantic-review gate ever ran.

- StageExecutionResult.Failure gains a `gate` id; each gate tags its failures.
- OrchestrationState gains gateRetryBudgets / gateFailureFingerprints /
  gateSalvageUsed (all rebuilt from events, reset per-stage on TransitionExecuted).
- FailureFingerprint normalizes+hashes the failure reason; RetryCoordinator.decide
  charges a gate's budget only when the fingerprint is unchanged (no progress),
  so a moving run is never penalised. Returns Retry | Exhausted.
- Hybrid exhaustion: deterministic gates fail; the review gate consults a
  SalvageJudge (LLM, or a deterministic allow-one-reset fallback), recorded as
  RetrySalvageDecidedEvent (invariant #9). CONTINUE resets the gate budget once;
  a second exhaustion is terminal.
- Review gate is now the sole authority on review-retry termination; the old
  REVIEW_BLOCK_RETRY_CAP is repurposed as a high absolute backstop only.
- ServerModule escaped-exception path now records a truthful terminal
  WorkflowFailed (real stage/reason/retryExhausted) via recordUnhandledFailure.

Tests: DefaultRetryCoordinatorTest (fingerprint charging) +
GateRetryBudgetExhaustionTest (deterministic exhaust / review CONTINUE-then-
terminal / review FAIL / null-judge fallback) + reducer coverage.
This commit is contained in:
2026-07-06 19:20:46 +04:00
parent 9f12c87bb1
commit 79e2c38a88
17 changed files with 1085 additions and 54 deletions
@@ -59,4 +59,32 @@ data class RetryAttemptedEvent(
val attemptNumber: Int,
val maxAttempts: Int,
val failureReason: String,
// Per-gate retry budgets (design 2026-07-06-per-gate-retry-budgets.md): budget key this attempt
// is charged against. Defaulted to the legacy shared "stage" bucket for schema back-compat.
val gate: String = "stage",
// Stable hash of the failure's normalised reason, used to detect progress across attempts.
val fingerprint: String = "",
// Whether this attempt consumed the gate's budget (false when the fingerprint changed from the
// previous attempt — i.e. the run is moving, so the retry is free).
val charged: Boolean = true,
) : EventPayload
/** Outcome of the hybrid-exhaustion salvage judgement for the semantic review gate. */
@Serializable
enum class SalvageDecision { CONTINUE, FAIL }
/**
* Recorded when a gate exhausts its retry budget and (per design decision 3) the salvage judge is
* consulted — currently only the "review" gate. The judgement is nondeterministic (LLM-backed, or a
* deterministic fallback when no judge is wired), so invariant #9 requires it be recorded here;
* replay reads this event back and never re-invokes the judge.
*/
@Serializable
@SerialName("RetrySalvageDecided")
data class RetrySalvageDecidedEvent(
val sessionId: SessionId,
val stageId: StageId,
val gate: String,
val decision: SalvageDecision,
val rationale: String,
) : EventPayload
@@ -6,8 +6,15 @@ import kotlinx.serialization.Serializable
data class RetryPolicy(
val maxAttempts: Int,
val backoffMs: Long = 1_000L,
// Per-gate override (design 2026-07-06-per-gate-retry-budgets.md §8): a gate id present here uses
// its own budget instead of [maxAttempts]. Empty by default — every gate then independently gets
// [maxAttempts] (un-shared, per T2/T5), preserving today's numbers with no TOML changes required.
val perGateMaxAttempts: Map<String, Int> = emptyMap(),
) {
init {
require(maxAttempts >= 1) { "maxAttempts must be >= 1" }
require(perGateMaxAttempts.values.all { it >= 1 }) { "perGateMaxAttempts values must be >= 1" }
}
fun maxAttemptsFor(gate: String): Int = perGateMaxAttempts[gate] ?: maxAttempts
}
@@ -15,4 +15,14 @@ data class OrchestrationState(
val failureReason: String? = null,
val retryPolicy: RetryPolicy? = null,
val refinementIterations: Map<String, Int> = emptyMap(),
// Per-gate retry budget (design 2026-07-06-per-gate-retry-budgets.md): each gate id (produces,
// brief_grounding, brief_echo, contract, plan_compile, static_analysis, execution, review, ...)
// exhausts its own budget rather than sharing retryCount session-wide. Attempts charged so far.
val gateRetryBudgets: Map<String, Int> = emptyMap(),
// Last-seen failure fingerprint per gate, used to tell a no-progress retry (same fingerprint,
// charged) from a genuine-progress retry (changed fingerprint, free).
val gateFailureFingerprints: Map<String, String> = emptyMap(),
// Gates that have already spent their one hybrid-exhaustion salvage reset (review gate only,
// see RetrySalvageDecidedEvent) — a second exhaustion for that gate is terminal.
val gateSalvageUsed: Set<String> = emptySet(),
)
@@ -58,6 +58,7 @@ import com.correx.core.events.events.RefinementIterationEvent
import com.correx.core.events.events.RepoKnowledgeRetrievedEvent
import com.correx.core.events.events.RepoMapComputedEvent
import com.correx.core.events.events.RetryAttemptedEvent
import com.correx.core.events.events.RetrySalvageDecidedEvent
import com.correx.core.events.events.WorkspaceStateObservedEvent
import com.correx.core.events.events.RiskAssessedEvent
import com.correx.core.events.events.SourceFetchedEvent
@@ -144,6 +145,7 @@ val eventModule = SerializersModule {
subclass(WorkflowFailedEvent::class)
subclass(WorkflowCompletedEvent::class)
subclass(RetryAttemptedEvent::class)
subclass(RetrySalvageDecidedEvent::class)
subclass(RefinementIterationEvent::class)
subclass(RepoMapComputedEvent::class)
subclass(WorkspaceStateObservedEvent::class)