feat(retry): per-gate retry budgets + progress-aware charging + hybrid salvage

Replaces the single shared per-stage retryCount (reset on TransitionExecuted,
shared across all post-stage gates) with a per-gate budget. Motivated by run
2e468f9f, where a promising freestyle run was terminally FAILED because the
contract gate burned all 3 shared retries on one trivial miss before the
semantic-review gate ever ran.

- StageExecutionResult.Failure gains a `gate` id; each gate tags its failures.
- OrchestrationState gains gateRetryBudgets / gateFailureFingerprints /
  gateSalvageUsed (all rebuilt from events, reset per-stage on TransitionExecuted).
- FailureFingerprint normalizes+hashes the failure reason; RetryCoordinator.decide
  charges a gate's budget only when the fingerprint is unchanged (no progress),
  so a moving run is never penalised. Returns Retry | Exhausted.
- Hybrid exhaustion: deterministic gates fail; the review gate consults a
  SalvageJudge (LLM, or a deterministic allow-one-reset fallback), recorded as
  RetrySalvageDecidedEvent (invariant #9). CONTINUE resets the gate budget once;
  a second exhaustion is terminal.
- Review gate is now the sole authority on review-retry termination; the old
  REVIEW_BLOCK_RETRY_CAP is repurposed as a high absolute backstop only.
- ServerModule escaped-exception path now records a truthful terminal
  WorkflowFailed (real stage/reason/retryExhausted) via recordUnhandledFailure.

Tests: DefaultRetryCoordinatorTest (fingerprint charging) +
GateRetryBudgetExhaustionTest (deterministic exhaust / review CONTINUE-then-
terminal / review FAIL / null-judge fallback) + reducer coverage.
This commit is contained in:
2026-07-06 19:20:46 +04:00
parent 9f12c87bb1
commit 79e2c38a88
17 changed files with 1085 additions and 54 deletions
@@ -1,9 +1,12 @@
import com.correx.core.events.events.RetryAttemptedEvent
import com.correx.core.events.execution.RetryPolicy
import com.correx.core.events.orchestration.OrchestrationState
import com.correx.core.events.stores.EventStore
import com.correx.core.events.types.SessionId
import com.correx.core.events.types.StageId
import com.correx.core.kernel.retry.DefaultRetryCoordinator
import com.correx.core.kernel.retry.FailureFingerprint
import com.correx.core.kernel.retry.RetryDecision
import com.correx.infrastructure.persistence.InMemoryEventStore
import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.test.currentTime
@@ -165,4 +168,95 @@ class DefaultRetryCoordinatorTest {
// virtual time should have advanced by backoffMs
assertEquals(1000L, currentTime)
}
// --- decide(): per-gate, progress-aware retry decision ---
private val sessionId = SessionId(UUID.randomUUID().toString())
private val stageId = StageId("stage1")
@Test
fun `decide charges the gate budget when the fingerprint is unchanged from the previous attempt`() = runTest {
val policy = RetryPolicy(maxAttempts = 3, backoffMs = 0L)
val state = OrchestrationState(
gateRetryBudgets = mapOf("contract" to 1),
gateFailureFingerprints = mapOf("contract" to FailureFingerprint.of("same-reason")),
)
val decision = retryCoordinator.decide(
sessionId, stageId, gate = "contract", failureReason = "same-reason", state = state, policy = policy,
)
assertEquals(RetryDecision.Retry, decision)
val event = eventStore.read(sessionId).single().payload as RetryAttemptedEvent
assertEquals(true, event.charged)
assertEquals(2, event.attemptNumber)
assertEquals("contract", event.gate)
}
@Test
fun `decide is free when the fingerprint changed — progress on the run`() = runTest {
val policy = RetryPolicy(maxAttempts = 1, backoffMs = 0L)
// Budget already at maxAttempts for this gate — a charged retry would be Exhausted, but a
// changed fingerprint (progress) must still be free and retry.
val state = OrchestrationState(
gateRetryBudgets = mapOf("contract" to 1),
gateFailureFingerprints = mapOf("contract" to "old-reason"),
)
val decision = retryCoordinator.decide(
sessionId, stageId, gate = "contract", failureReason = "new-reason", state = state, policy = policy,
)
assertEquals(RetryDecision.Retry, decision)
val event = eventStore.read(sessionId).single().payload as RetryAttemptedEvent
assertEquals(false, event.charged)
assertEquals(1, event.attemptNumber) // unchanged — this retry wasn't charged
}
@Test
fun `decide returns Exhausted without emitting an event once the gate budget is spent`() = runTest {
val policy = RetryPolicy(maxAttempts = 2, backoffMs = 0L)
val state = OrchestrationState(
gateRetryBudgets = mapOf("contract" to 2),
gateFailureFingerprints = mapOf("contract" to FailureFingerprint.of("same-reason")),
)
val decision = retryCoordinator.decide(
sessionId, stageId, gate = "contract", failureReason = "same-reason", state = state, policy = policy,
)
assertEquals(RetryDecision.Exhausted, decision)
Assertions.assertTrue(eventStore.read(sessionId).isEmpty())
}
@Test
fun `decide keys the budget by gate id — a different gate has its own fresh budget`() = runTest {
val policy = RetryPolicy(maxAttempts = 1, backoffMs = 0L)
val state = OrchestrationState(
gateRetryBudgets = mapOf("contract" to 1),
gateFailureFingerprints = mapOf("contract" to FailureFingerprint.of("same-reason")),
)
// "contract" is already exhausted at maxAttempts=1, but "review" has never charged.
val decision = retryCoordinator.decide(
sessionId, stageId, gate = "review", failureReason = "some finding", state = state, policy = policy,
)
assertEquals(RetryDecision.Retry, decision)
}
@Test
fun `decide honours perGateMaxAttempts override`() = runTest {
val policy = RetryPolicy(maxAttempts = 1, backoffMs = 0L, perGateMaxAttempts = mapOf("review" to 5))
val state = OrchestrationState(
gateRetryBudgets = mapOf("review" to 4),
gateFailureFingerprints = mapOf("review" to "same-reason"),
)
val decision = retryCoordinator.decide(
sessionId, stageId, gate = "review", failureReason = "same-reason", state = state, policy = policy,
)
assertEquals(RetryDecision.Retry, decision)
}
}