Files
correx/testing/projections/src/test/kotlin/OrchestrationReducerTest.kt
T
kami 79e2c38a88 feat(retry): per-gate retry budgets + progress-aware charging + hybrid salvage
Replaces the single shared per-stage retryCount (reset on TransitionExecuted,
shared across all post-stage gates) with a per-gate budget. Motivated by run
2e468f9f, where a promising freestyle run was terminally FAILED because the
contract gate burned all 3 shared retries on one trivial miss before the
semantic-review gate ever ran.

- StageExecutionResult.Failure gains a `gate` id; each gate tags its failures.
- OrchestrationState gains gateRetryBudgets / gateFailureFingerprints /
  gateSalvageUsed (all rebuilt from events, reset per-stage on TransitionExecuted).
- FailureFingerprint normalizes+hashes the failure reason; RetryCoordinator.decide
  charges a gate's budget only when the fingerprint is unchanged (no progress),
  so a moving run is never penalised. Returns Retry | Exhausted.
- Hybrid exhaustion: deterministic gates fail; the review gate consults a
  SalvageJudge (LLM, or a deterministic allow-one-reset fallback), recorded as
  RetrySalvageDecidedEvent (invariant #9). CONTINUE resets the gate budget once;
  a second exhaustion is terminal.
- Review gate is now the sole authority on review-retry termination; the old
  REVIEW_BLOCK_RETRY_CAP is repurposed as a high absolute backstop only.
- ServerModule escaped-exception path now records a truthful terminal
  WorkflowFailed (real stage/reason/retryExhausted) via recordUnhandledFailure.

Tests: DefaultRetryCoordinatorTest (fingerprint charging) +
GateRetryBudgetExhaustionTest (deterministic exhaust / review CONTINUE-then-
terminal / review FAIL / null-judge fallback) + reducer coverage.
2026-07-06 19:20:46 +04:00

304 lines
11 KiB
Kotlin

import com.correx.core.events.events.OrchestrationPausedEvent
import com.correx.core.events.events.OrchestrationResumedEvent
import com.correx.core.events.events.RetryAttemptedEvent
import com.correx.core.events.events.RetrySalvageDecidedEvent
import com.correx.core.events.events.SalvageDecision
import com.correx.core.events.events.TransitionExecutedEvent
import com.correx.core.events.events.WorkflowCompletedEvent
import com.correx.core.events.events.WorkflowFailedEvent
import com.correx.core.events.events.WorkflowStartedEvent
import com.correx.core.events.orchestration.OrchestrationState
import com.correx.core.events.orchestration.OrchestrationStatus
import com.correx.core.events.types.SessionId
import com.correx.core.events.types.StageId
import com.correx.core.events.types.TransitionId
import com.correx.core.kernel.orchestration.DefaultOrchestrationReducer
import com.correx.testing.fixtures.EventFixtures.stored
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertFalse
import org.junit.jupiter.api.Assertions.assertNull
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.Test
class OrchestrationReducerTest {
private val reducer = DefaultOrchestrationReducer()
private val sessionId = SessionId("s1")
private val stageId = StageId("stage-1")
private val workflowId = "workflow-test"
private val state = OrchestrationState(workflowId, stageId)
@Test
fun `WorkflowStartedEvent sets status to RUNNING`() {
val state = reducer.reduce(
state,
stored(payload = WorkflowStartedEvent(sessionId, workflowId, stageId)),
)
assertEquals(OrchestrationStatus.RUNNING, state.status)
assertEquals(stageId, state.currentStageId)
}
@Test
fun `WorkflowFailedEvent sets status to FAILED and failure reason`() {
val started = reducer.reduce(
state,
stored(payload = WorkflowStartedEvent(sessionId, workflowId, stageId)),
)
val failed = reducer.reduce(
started,
stored(payload = WorkflowFailedEvent(sessionId, stageId, "Something went wrong", false)),
)
assertFalse(failed.failureReason.isNullOrBlank())
assertEquals(OrchestrationStatus.FAILED, failed.status)
}
@Test
fun `WorkflowCompletedEvent sets status to COMPLETED`() {
val started = reducer.reduce(
state,
stored(payload = WorkflowStartedEvent(sessionId, workflowId, stageId)),
)
val completed = reducer.reduce(
started,
stored(payload = WorkflowCompletedEvent(sessionId, stageId, 1)),
)
assertTrue(completed.failureReason.isNullOrBlank())
assertEquals(OrchestrationStatus.COMPLETED, completed.status)
assertEquals(stageId, completed.currentStageId)
}
@Test
fun `OrchestrationPausedEvent sets status to PAUSED with reason and pending approval`() {
val started = reducer.reduce(
state,
stored(payload = WorkflowStartedEvent(sessionId, workflowId, stageId)),
)
val paused = reducer.reduce(
started,
stored(payload = OrchestrationPausedEvent(sessionId, stageId, "Requires user approval")),
)
assertFalse(paused.pauseReason.isNullOrBlank())
assertEquals(OrchestrationStatus.PAUSED, paused.status)
assertTrue(paused.pendingApproval)
}
@Test
fun `OrchestrationResumedEvent sets status to RUNNING with null reason and no pending approval`() {
val started = reducer.reduce(
state,
stored(payload = WorkflowStartedEvent(sessionId, workflowId, stageId)),
)
val paused = reducer.reduce(
started,
stored(payload = OrchestrationPausedEvent(sessionId, stageId, "Requires user approval")),
)
val resumed = reducer.reduce(
paused,
stored(payload = OrchestrationResumedEvent(sessionId, stageId)),
)
assertTrue(resumed.pauseReason.isNullOrBlank())
assertEquals(OrchestrationStatus.RUNNING, resumed.status)
assertFalse(resumed.pendingApproval)
}
@Test
fun `RetryAttemptedEvent sets status to RUNNING with null reason and no pending approval`() {
val started = reducer.reduce(
state,
stored(payload = WorkflowStartedEvent(sessionId, workflowId, stageId)),
)
val failed = reducer.reduce(
started,
stored(payload = WorkflowFailedEvent(sessionId, stageId, "Something went wrong", false)),
)
val retrying1 = reducer.reduce(
failed,
stored(payload = RetryAttemptedEvent(sessionId, stageId, 1, 3, "Something went wrong")),
)
val retrying2 = reducer.reduce(
retrying1,
stored(payload = RetryAttemptedEvent(sessionId, stageId, 2, 3, "Something went wrong")),
)
val retrying3 = reducer.reduce(
retrying2,
stored(payload = RetryAttemptedEvent(sessionId, stageId, 3, 3, "Something went wrong")),
)
assertTrue(retrying1.retryCount < retrying2.retryCount && retrying2.retryCount < retrying3.retryCount)
assertEquals(OrchestrationStatus.RUNNING, retrying1.status)
assertNull(retrying1.failureReason)
}
@Test
fun `TransitionExecutedEvent resets retryCount so each stage gets its own retry budget`() {
val exhausted = reducer.reduce(
state,
stored(payload = RetryAttemptedEvent(sessionId, stageId, 3, 3, "boom")),
)
assertEquals(3, exhausted.retryCount)
val nextStage = StageId("stage-2")
val advanced = reducer.reduce(
exhausted,
stored(payload = TransitionExecutedEvent(sessionId, stageId, nextStage, TransitionId("t1"))),
)
assertEquals(nextStage, advanced.currentStageId)
assertEquals(0, advanced.retryCount)
}
@Test
fun `RetryAttemptedEvent with charged=true bumps the gate's per-gate budget and fingerprint`() {
val retried = reducer.reduce(
state,
stored(
payload = RetryAttemptedEvent(
sessionId, stageId, 1, 3, "boom",
gate = "contract", fingerprint = "fp1", charged = true,
),
),
)
assertEquals(1, retried.gateRetryBudgets["contract"])
assertEquals("fp1", retried.gateFailureFingerprints["contract"])
}
@Test
fun `RetryAttemptedEvent with charged=false leaves the gate's budget untouched but records the fingerprint`() {
val charged = reducer.reduce(
state,
stored(
payload = RetryAttemptedEvent(
sessionId, stageId, 1, 3, "boom",
gate = "contract", fingerprint = "fp1", charged = true,
),
),
)
val free = reducer.reduce(
charged,
stored(
payload = RetryAttemptedEvent(
sessionId, stageId, 1, 3, "different problem",
gate = "contract", fingerprint = "fp2", charged = false,
),
),
)
assertEquals(1, free.gateRetryBudgets["contract"])
assertEquals("fp2", free.gateFailureFingerprints["contract"])
}
@Test
fun `gate budgets are independent per gate id`() {
val contractCharged = reducer.reduce(
state,
stored(
payload = RetryAttemptedEvent(
sessionId, stageId, 1, 3, "boom",
gate = "contract", fingerprint = "fp1", charged = true,
),
),
)
val reviewCharged = reducer.reduce(
contractCharged,
stored(
payload = RetryAttemptedEvent(
sessionId, stageId, 1, 3, "boom2",
gate = "review", fingerprint = "fpr1", charged = true,
),
),
)
assertEquals(1, reviewCharged.gateRetryBudgets["contract"])
assertEquals(1, reviewCharged.gateRetryBudgets["review"])
}
@Test
fun `TransitionExecutedEvent resets per-gate budgets, fingerprints, and salvage-used set`() {
val charged = reducer.reduce(
state,
stored(
payload = RetryAttemptedEvent(
sessionId, stageId, 1, 3, "boom",
gate = "review", fingerprint = "fp1", charged = true,
),
),
)
val salvaged = reducer.reduce(
charged,
stored(
payload = RetrySalvageDecidedEvent(sessionId, stageId, "review", SalvageDecision.CONTINUE, "fixable"),
),
)
assertTrue(salvaged.gateSalvageUsed.contains("review"))
val nextStage = StageId("stage-2")
val advanced = reducer.reduce(
salvaged,
stored(payload = TransitionExecutedEvent(sessionId, stageId, nextStage, TransitionId("t1"))),
)
assertTrue(advanced.gateRetryBudgets.isEmpty())
assertTrue(advanced.gateFailureFingerprints.isEmpty())
assertTrue(advanced.gateSalvageUsed.isEmpty())
}
@Test
fun `RetrySalvageDecidedEvent CONTINUE resets the gate's budget and marks salvage used`() {
val charged = reducer.reduce(
state,
stored(
payload = RetryAttemptedEvent(
sessionId, stageId, 1, 3, "boom",
gate = "review", fingerprint = "fp1", charged = true,
),
),
)
assertEquals(1, charged.gateRetryBudgets["review"])
val salvaged = reducer.reduce(
charged,
stored(
payload = RetrySalvageDecidedEvent(sessionId, stageId, "review", SalvageDecision.CONTINUE, "fixable"),
),
)
assertEquals(0, salvaged.gateRetryBudgets["review"])
assertTrue(salvaged.gateSalvageUsed.contains("review"))
}
@Test
fun `RetrySalvageDecidedEvent FAIL does not touch state`() {
val charged = reducer.reduce(
state,
stored(
payload = RetryAttemptedEvent(
sessionId, stageId, 1, 3, "boom",
gate = "review", fingerprint = "fp1", charged = true,
),
),
)
val decided = reducer.reduce(
charged,
stored(
payload = RetrySalvageDecidedEvent(sessionId, stageId, "review", SalvageDecision.FAIL, "not fixable"),
),
)
assertEquals(charged, decided)
}
@Test
fun `unrelated event does nothing`() {
val started = reducer.reduce(
state,
stored(payload = WorkflowStartedEvent(sessionId, workflowId, stageId)),
)
val unrelated = reducer.reduce(started, stored())
assertEquals(started, unrelated)
}
}