feat(retry): per-gate retry budgets + progress-aware charging + hybrid salvage
Replaces the single shared per-stage retryCount (reset on TransitionExecuted, shared across all post-stage gates) with a per-gate budget. Motivated by run 2e468f9f, where a promising freestyle run was terminally FAILED because the contract gate burned all 3 shared retries on one trivial miss before the semantic-review gate ever ran. - StageExecutionResult.Failure gains a `gate` id; each gate tags its failures. - OrchestrationState gains gateRetryBudgets / gateFailureFingerprints / gateSalvageUsed (all rebuilt from events, reset per-stage on TransitionExecuted). - FailureFingerprint normalizes+hashes the failure reason; RetryCoordinator.decide charges a gate's budget only when the fingerprint is unchanged (no progress), so a moving run is never penalised. Returns Retry | Exhausted. - Hybrid exhaustion: deterministic gates fail; the review gate consults a SalvageJudge (LLM, or a deterministic allow-one-reset fallback), recorded as RetrySalvageDecidedEvent (invariant #9). CONTINUE resets the gate budget once; a second exhaustion is terminal. - Review gate is now the sole authority on review-retry termination; the old REVIEW_BLOCK_RETRY_CAP is repurposed as a high absolute backstop only. - ServerModule escaped-exception path now records a truthful terminal WorkflowFailed (real stage/reason/retryExhausted) via recordUnhandledFailure. Tests: DefaultRetryCoordinatorTest (fingerprint charging) + GateRetryBudgetExhaustionTest (deterministic exhaust / review CONTINUE-then- terminal / review FAIL / null-judge fallback) + reducer coverage.
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
import com.correx.core.events.events.OrchestrationPausedEvent
|
||||
import com.correx.core.events.events.OrchestrationResumedEvent
|
||||
import com.correx.core.events.events.RetryAttemptedEvent
|
||||
import com.correx.core.events.events.RetrySalvageDecidedEvent
|
||||
import com.correx.core.events.events.SalvageDecision
|
||||
import com.correx.core.events.events.TransitionExecutedEvent
|
||||
import com.correx.core.events.events.WorkflowCompletedEvent
|
||||
import com.correx.core.events.events.WorkflowFailedEvent
|
||||
@@ -153,6 +155,141 @@ class OrchestrationReducerTest {
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
fun `RetryAttemptedEvent with charged=true bumps the gate's per-gate budget and fingerprint`() {
|
||||
val retried = reducer.reduce(
|
||||
state,
|
||||
stored(
|
||||
payload = RetryAttemptedEvent(
|
||||
sessionId, stageId, 1, 3, "boom",
|
||||
gate = "contract", fingerprint = "fp1", charged = true,
|
||||
),
|
||||
),
|
||||
)
|
||||
assertEquals(1, retried.gateRetryBudgets["contract"])
|
||||
assertEquals("fp1", retried.gateFailureFingerprints["contract"])
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `RetryAttemptedEvent with charged=false leaves the gate's budget untouched but records the fingerprint`() {
|
||||
val charged = reducer.reduce(
|
||||
state,
|
||||
stored(
|
||||
payload = RetryAttemptedEvent(
|
||||
sessionId, stageId, 1, 3, "boom",
|
||||
gate = "contract", fingerprint = "fp1", charged = true,
|
||||
),
|
||||
),
|
||||
)
|
||||
val free = reducer.reduce(
|
||||
charged,
|
||||
stored(
|
||||
payload = RetryAttemptedEvent(
|
||||
sessionId, stageId, 1, 3, "different problem",
|
||||
gate = "contract", fingerprint = "fp2", charged = false,
|
||||
),
|
||||
),
|
||||
)
|
||||
assertEquals(1, free.gateRetryBudgets["contract"])
|
||||
assertEquals("fp2", free.gateFailureFingerprints["contract"])
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `gate budgets are independent per gate id`() {
|
||||
val contractCharged = reducer.reduce(
|
||||
state,
|
||||
stored(
|
||||
payload = RetryAttemptedEvent(
|
||||
sessionId, stageId, 1, 3, "boom",
|
||||
gate = "contract", fingerprint = "fp1", charged = true,
|
||||
),
|
||||
),
|
||||
)
|
||||
val reviewCharged = reducer.reduce(
|
||||
contractCharged,
|
||||
stored(
|
||||
payload = RetryAttemptedEvent(
|
||||
sessionId, stageId, 1, 3, "boom2",
|
||||
gate = "review", fingerprint = "fpr1", charged = true,
|
||||
),
|
||||
),
|
||||
)
|
||||
assertEquals(1, reviewCharged.gateRetryBudgets["contract"])
|
||||
assertEquals(1, reviewCharged.gateRetryBudgets["review"])
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `TransitionExecutedEvent resets per-gate budgets, fingerprints, and salvage-used set`() {
|
||||
val charged = reducer.reduce(
|
||||
state,
|
||||
stored(
|
||||
payload = RetryAttemptedEvent(
|
||||
sessionId, stageId, 1, 3, "boom",
|
||||
gate = "review", fingerprint = "fp1", charged = true,
|
||||
),
|
||||
),
|
||||
)
|
||||
val salvaged = reducer.reduce(
|
||||
charged,
|
||||
stored(
|
||||
payload = RetrySalvageDecidedEvent(sessionId, stageId, "review", SalvageDecision.CONTINUE, "fixable"),
|
||||
),
|
||||
)
|
||||
assertTrue(salvaged.gateSalvageUsed.contains("review"))
|
||||
|
||||
val nextStage = StageId("stage-2")
|
||||
val advanced = reducer.reduce(
|
||||
salvaged,
|
||||
stored(payload = TransitionExecutedEvent(sessionId, stageId, nextStage, TransitionId("t1"))),
|
||||
)
|
||||
assertTrue(advanced.gateRetryBudgets.isEmpty())
|
||||
assertTrue(advanced.gateFailureFingerprints.isEmpty())
|
||||
assertTrue(advanced.gateSalvageUsed.isEmpty())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `RetrySalvageDecidedEvent CONTINUE resets the gate's budget and marks salvage used`() {
|
||||
val charged = reducer.reduce(
|
||||
state,
|
||||
stored(
|
||||
payload = RetryAttemptedEvent(
|
||||
sessionId, stageId, 1, 3, "boom",
|
||||
gate = "review", fingerprint = "fp1", charged = true,
|
||||
),
|
||||
),
|
||||
)
|
||||
assertEquals(1, charged.gateRetryBudgets["review"])
|
||||
|
||||
val salvaged = reducer.reduce(
|
||||
charged,
|
||||
stored(
|
||||
payload = RetrySalvageDecidedEvent(sessionId, stageId, "review", SalvageDecision.CONTINUE, "fixable"),
|
||||
),
|
||||
)
|
||||
assertEquals(0, salvaged.gateRetryBudgets["review"])
|
||||
assertTrue(salvaged.gateSalvageUsed.contains("review"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `RetrySalvageDecidedEvent FAIL does not touch state`() {
|
||||
val charged = reducer.reduce(
|
||||
state,
|
||||
stored(
|
||||
payload = RetryAttemptedEvent(
|
||||
sessionId, stageId, 1, 3, "boom",
|
||||
gate = "review", fingerprint = "fp1", charged = true,
|
||||
),
|
||||
),
|
||||
)
|
||||
val decided = reducer.reduce(
|
||||
charged,
|
||||
stored(
|
||||
payload = RetrySalvageDecidedEvent(sessionId, stageId, "review", SalvageDecision.FAIL, "not fixable"),
|
||||
),
|
||||
)
|
||||
assertEquals(charged, decided)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `unrelated event does nothing`() {
|
||||
val started = reducer.reduce(
|
||||
|
||||
Reference in New Issue
Block a user