feat(retry): per-gate retry budgets + progress-aware charging + hybrid salvage
Replaces the single shared per-stage retryCount (reset on TransitionExecuted, shared across all post-stage gates) with a per-gate budget. Motivated by run 2e468f9f, where a promising freestyle run was terminally FAILED because the contract gate burned all 3 shared retries on one trivial miss before the semantic-review gate ever ran. - StageExecutionResult.Failure gains a `gate` id; each gate tags its failures. - OrchestrationState gains gateRetryBudgets / gateFailureFingerprints / gateSalvageUsed (all rebuilt from events, reset per-stage on TransitionExecuted). - FailureFingerprint normalizes+hashes the failure reason; RetryCoordinator.decide charges a gate's budget only when the fingerprint is unchanged (no progress), so a moving run is never penalised. Returns Retry | Exhausted. - Hybrid exhaustion: deterministic gates fail; the review gate consults a SalvageJudge (LLM, or a deterministic allow-one-reset fallback), recorded as RetrySalvageDecidedEvent (invariant #9). CONTINUE resets the gate budget once; a second exhaustion is terminal. - Review gate is now the sole authority on review-retry termination; the old REVIEW_BLOCK_RETRY_CAP is repurposed as a high absolute backstop only. - ServerModule escaped-exception path now records a truthful terminal WorkflowFailed (real stage/reason/retryExhausted) via recordUnhandledFailure. Tests: DefaultRetryCoordinatorTest (fingerprint charging) + GateRetryBudgetExhaustionTest (deterministic exhaust / review CONTINUE-then- terminal / review FAIL / null-judge fallback) + reducer coverage.
This commit is contained in:
@@ -1,9 +1,12 @@
|
||||
import com.correx.core.events.events.RetryAttemptedEvent
|
||||
import com.correx.core.events.execution.RetryPolicy
|
||||
import com.correx.core.events.orchestration.OrchestrationState
|
||||
import com.correx.core.events.stores.EventStore
|
||||
import com.correx.core.events.types.SessionId
|
||||
import com.correx.core.events.types.StageId
|
||||
import com.correx.core.kernel.retry.DefaultRetryCoordinator
|
||||
import com.correx.core.kernel.retry.FailureFingerprint
|
||||
import com.correx.core.kernel.retry.RetryDecision
|
||||
import com.correx.infrastructure.persistence.InMemoryEventStore
|
||||
import kotlinx.coroutines.ExperimentalCoroutinesApi
|
||||
import kotlinx.coroutines.test.currentTime
|
||||
@@ -165,4 +168,95 @@ class DefaultRetryCoordinatorTest {
|
||||
// virtual time should have advanced by backoffMs
|
||||
assertEquals(1000L, currentTime)
|
||||
}
|
||||
|
||||
// --- decide(): per-gate, progress-aware retry decision ---
|
||||
|
||||
private val sessionId = SessionId(UUID.randomUUID().toString())
|
||||
private val stageId = StageId("stage1")
|
||||
|
||||
@Test
|
||||
fun `decide charges the gate budget when the fingerprint is unchanged from the previous attempt`() = runTest {
|
||||
val policy = RetryPolicy(maxAttempts = 3, backoffMs = 0L)
|
||||
val state = OrchestrationState(
|
||||
gateRetryBudgets = mapOf("contract" to 1),
|
||||
gateFailureFingerprints = mapOf("contract" to FailureFingerprint.of("same-reason")),
|
||||
)
|
||||
|
||||
val decision = retryCoordinator.decide(
|
||||
sessionId, stageId, gate = "contract", failureReason = "same-reason", state = state, policy = policy,
|
||||
)
|
||||
|
||||
assertEquals(RetryDecision.Retry, decision)
|
||||
val event = eventStore.read(sessionId).single().payload as RetryAttemptedEvent
|
||||
assertEquals(true, event.charged)
|
||||
assertEquals(2, event.attemptNumber)
|
||||
assertEquals("contract", event.gate)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `decide is free when the fingerprint changed — progress on the run`() = runTest {
|
||||
val policy = RetryPolicy(maxAttempts = 1, backoffMs = 0L)
|
||||
// Budget already at maxAttempts for this gate — a charged retry would be Exhausted, but a
|
||||
// changed fingerprint (progress) must still be free and retry.
|
||||
val state = OrchestrationState(
|
||||
gateRetryBudgets = mapOf("contract" to 1),
|
||||
gateFailureFingerprints = mapOf("contract" to "old-reason"),
|
||||
)
|
||||
|
||||
val decision = retryCoordinator.decide(
|
||||
sessionId, stageId, gate = "contract", failureReason = "new-reason", state = state, policy = policy,
|
||||
)
|
||||
|
||||
assertEquals(RetryDecision.Retry, decision)
|
||||
val event = eventStore.read(sessionId).single().payload as RetryAttemptedEvent
|
||||
assertEquals(false, event.charged)
|
||||
assertEquals(1, event.attemptNumber) // unchanged — this retry wasn't charged
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `decide returns Exhausted without emitting an event once the gate budget is spent`() = runTest {
|
||||
val policy = RetryPolicy(maxAttempts = 2, backoffMs = 0L)
|
||||
val state = OrchestrationState(
|
||||
gateRetryBudgets = mapOf("contract" to 2),
|
||||
gateFailureFingerprints = mapOf("contract" to FailureFingerprint.of("same-reason")),
|
||||
)
|
||||
|
||||
val decision = retryCoordinator.decide(
|
||||
sessionId, stageId, gate = "contract", failureReason = "same-reason", state = state, policy = policy,
|
||||
)
|
||||
|
||||
assertEquals(RetryDecision.Exhausted, decision)
|
||||
Assertions.assertTrue(eventStore.read(sessionId).isEmpty())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `decide keys the budget by gate id — a different gate has its own fresh budget`() = runTest {
|
||||
val policy = RetryPolicy(maxAttempts = 1, backoffMs = 0L)
|
||||
val state = OrchestrationState(
|
||||
gateRetryBudgets = mapOf("contract" to 1),
|
||||
gateFailureFingerprints = mapOf("contract" to FailureFingerprint.of("same-reason")),
|
||||
)
|
||||
|
||||
// "contract" is already exhausted at maxAttempts=1, but "review" has never charged.
|
||||
val decision = retryCoordinator.decide(
|
||||
sessionId, stageId, gate = "review", failureReason = "some finding", state = state, policy = policy,
|
||||
)
|
||||
|
||||
assertEquals(RetryDecision.Retry, decision)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `decide honours perGateMaxAttempts override`() = runTest {
|
||||
val policy = RetryPolicy(maxAttempts = 1, backoffMs = 0L, perGateMaxAttempts = mapOf("review" to 5))
|
||||
val state = OrchestrationState(
|
||||
gateRetryBudgets = mapOf("review" to 4),
|
||||
gateFailureFingerprints = mapOf("review" to "same-reason"),
|
||||
)
|
||||
|
||||
val decision = retryCoordinator.decide(
|
||||
sessionId, stageId, gate = "review", failureReason = "same-reason", state = state, policy = policy,
|
||||
)
|
||||
|
||||
assertEquals(RetryDecision.Retry, decision)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user