79e2c38a88
Replaces the single shared per-stage retryCount (reset on TransitionExecuted, shared across all post-stage gates) with a per-gate budget. Motivated by run 2e468f9f, where a promising freestyle run was terminally FAILED because the contract gate burned all 3 shared retries on one trivial miss before the semantic-review gate ever ran. - StageExecutionResult.Failure gains a `gate` id; each gate tags its failures. - OrchestrationState gains gateRetryBudgets / gateFailureFingerprints / gateSalvageUsed (all rebuilt from events, reset per-stage on TransitionExecuted). - FailureFingerprint normalizes+hashes the failure reason; RetryCoordinator.decide charges a gate's budget only when the fingerprint is unchanged (no progress), so a moving run is never penalised. Returns Retry | Exhausted. - Hybrid exhaustion: deterministic gates fail; the review gate consults a SalvageJudge (LLM, or a deterministic allow-one-reset fallback), recorded as RetrySalvageDecidedEvent (invariant #9). CONTINUE resets the gate budget once; a second exhaustion is terminal. - Review gate is now the sole authority on review-retry termination; the old REVIEW_BLOCK_RETRY_CAP is repurposed as a high absolute backstop only. - ServerModule escaped-exception path now records a truthful terminal WorkflowFailed (real stage/reason/retryExhausted) via recordUnhandledFailure. Tests: DefaultRetryCoordinatorTest (fingerprint charging) + GateRetryBudgetExhaustionTest (deterministic exhaust / review CONTINUE-then- terminal / review FAIL / null-judge fallback) + reducer coverage.
304 lines
11 KiB
Kotlin
304 lines
11 KiB
Kotlin
import com.correx.core.events.events.OrchestrationPausedEvent
|
|
import com.correx.core.events.events.OrchestrationResumedEvent
|
|
import com.correx.core.events.events.RetryAttemptedEvent
|
|
import com.correx.core.events.events.RetrySalvageDecidedEvent
|
|
import com.correx.core.events.events.SalvageDecision
|
|
import com.correx.core.events.events.TransitionExecutedEvent
|
|
import com.correx.core.events.events.WorkflowCompletedEvent
|
|
import com.correx.core.events.events.WorkflowFailedEvent
|
|
import com.correx.core.events.events.WorkflowStartedEvent
|
|
import com.correx.core.events.orchestration.OrchestrationState
|
|
import com.correx.core.events.orchestration.OrchestrationStatus
|
|
import com.correx.core.events.types.SessionId
|
|
import com.correx.core.events.types.StageId
|
|
import com.correx.core.events.types.TransitionId
|
|
import com.correx.core.kernel.orchestration.DefaultOrchestrationReducer
|
|
import com.correx.testing.fixtures.EventFixtures.stored
|
|
import org.junit.jupiter.api.Assertions.assertEquals
|
|
import org.junit.jupiter.api.Assertions.assertFalse
|
|
import org.junit.jupiter.api.Assertions.assertNull
|
|
import org.junit.jupiter.api.Assertions.assertTrue
|
|
import org.junit.jupiter.api.Test
|
|
|
|
class OrchestrationReducerTest {
|
|
|
|
private val reducer = DefaultOrchestrationReducer()
|
|
private val sessionId = SessionId("s1")
|
|
private val stageId = StageId("stage-1")
|
|
private val workflowId = "workflow-test"
|
|
private val state = OrchestrationState(workflowId, stageId)
|
|
|
|
@Test
|
|
fun `WorkflowStartedEvent sets status to RUNNING`() {
|
|
val state = reducer.reduce(
|
|
state,
|
|
stored(payload = WorkflowStartedEvent(sessionId, workflowId, stageId)),
|
|
)
|
|
assertEquals(OrchestrationStatus.RUNNING, state.status)
|
|
assertEquals(stageId, state.currentStageId)
|
|
}
|
|
|
|
@Test
|
|
fun `WorkflowFailedEvent sets status to FAILED and failure reason`() {
|
|
val started = reducer.reduce(
|
|
state,
|
|
stored(payload = WorkflowStartedEvent(sessionId, workflowId, stageId)),
|
|
)
|
|
|
|
val failed = reducer.reduce(
|
|
started,
|
|
stored(payload = WorkflowFailedEvent(sessionId, stageId, "Something went wrong", false)),
|
|
)
|
|
assertFalse(failed.failureReason.isNullOrBlank())
|
|
assertEquals(OrchestrationStatus.FAILED, failed.status)
|
|
}
|
|
|
|
@Test
|
|
fun `WorkflowCompletedEvent sets status to COMPLETED`() {
|
|
val started = reducer.reduce(
|
|
state,
|
|
stored(payload = WorkflowStartedEvent(sessionId, workflowId, stageId)),
|
|
)
|
|
|
|
val completed = reducer.reduce(
|
|
started,
|
|
stored(payload = WorkflowCompletedEvent(sessionId, stageId, 1)),
|
|
)
|
|
assertTrue(completed.failureReason.isNullOrBlank())
|
|
assertEquals(OrchestrationStatus.COMPLETED, completed.status)
|
|
assertEquals(stageId, completed.currentStageId)
|
|
}
|
|
|
|
@Test
|
|
fun `OrchestrationPausedEvent sets status to PAUSED with reason and pending approval`() {
|
|
val started = reducer.reduce(
|
|
state,
|
|
stored(payload = WorkflowStartedEvent(sessionId, workflowId, stageId)),
|
|
)
|
|
|
|
val paused = reducer.reduce(
|
|
started,
|
|
stored(payload = OrchestrationPausedEvent(sessionId, stageId, "Requires user approval")),
|
|
)
|
|
assertFalse(paused.pauseReason.isNullOrBlank())
|
|
assertEquals(OrchestrationStatus.PAUSED, paused.status)
|
|
assertTrue(paused.pendingApproval)
|
|
}
|
|
|
|
@Test
|
|
fun `OrchestrationResumedEvent sets status to RUNNING with null reason and no pending approval`() {
|
|
val started = reducer.reduce(
|
|
state,
|
|
stored(payload = WorkflowStartedEvent(sessionId, workflowId, stageId)),
|
|
)
|
|
|
|
val paused = reducer.reduce(
|
|
started,
|
|
stored(payload = OrchestrationPausedEvent(sessionId, stageId, "Requires user approval")),
|
|
)
|
|
|
|
val resumed = reducer.reduce(
|
|
paused,
|
|
stored(payload = OrchestrationResumedEvent(sessionId, stageId)),
|
|
)
|
|
assertTrue(resumed.pauseReason.isNullOrBlank())
|
|
assertEquals(OrchestrationStatus.RUNNING, resumed.status)
|
|
assertFalse(resumed.pendingApproval)
|
|
}
|
|
|
|
@Test
|
|
fun `RetryAttemptedEvent sets status to RUNNING with null reason and no pending approval`() {
|
|
val started = reducer.reduce(
|
|
state,
|
|
stored(payload = WorkflowStartedEvent(sessionId, workflowId, stageId)),
|
|
)
|
|
|
|
val failed = reducer.reduce(
|
|
started,
|
|
stored(payload = WorkflowFailedEvent(sessionId, stageId, "Something went wrong", false)),
|
|
)
|
|
|
|
val retrying1 = reducer.reduce(
|
|
failed,
|
|
stored(payload = RetryAttemptedEvent(sessionId, stageId, 1, 3, "Something went wrong")),
|
|
)
|
|
|
|
val retrying2 = reducer.reduce(
|
|
retrying1,
|
|
stored(payload = RetryAttemptedEvent(sessionId, stageId, 2, 3, "Something went wrong")),
|
|
)
|
|
|
|
val retrying3 = reducer.reduce(
|
|
retrying2,
|
|
stored(payload = RetryAttemptedEvent(sessionId, stageId, 3, 3, "Something went wrong")),
|
|
)
|
|
assertTrue(retrying1.retryCount < retrying2.retryCount && retrying2.retryCount < retrying3.retryCount)
|
|
assertEquals(OrchestrationStatus.RUNNING, retrying1.status)
|
|
assertNull(retrying1.failureReason)
|
|
}
|
|
|
|
@Test
|
|
fun `TransitionExecutedEvent resets retryCount so each stage gets its own retry budget`() {
|
|
val exhausted = reducer.reduce(
|
|
state,
|
|
stored(payload = RetryAttemptedEvent(sessionId, stageId, 3, 3, "boom")),
|
|
)
|
|
assertEquals(3, exhausted.retryCount)
|
|
|
|
val nextStage = StageId("stage-2")
|
|
val advanced = reducer.reduce(
|
|
exhausted,
|
|
stored(payload = TransitionExecutedEvent(sessionId, stageId, nextStage, TransitionId("t1"))),
|
|
)
|
|
assertEquals(nextStage, advanced.currentStageId)
|
|
assertEquals(0, advanced.retryCount)
|
|
}
|
|
|
|
|
|
@Test
|
|
fun `RetryAttemptedEvent with charged=true bumps the gate's per-gate budget and fingerprint`() {
|
|
val retried = reducer.reduce(
|
|
state,
|
|
stored(
|
|
payload = RetryAttemptedEvent(
|
|
sessionId, stageId, 1, 3, "boom",
|
|
gate = "contract", fingerprint = "fp1", charged = true,
|
|
),
|
|
),
|
|
)
|
|
assertEquals(1, retried.gateRetryBudgets["contract"])
|
|
assertEquals("fp1", retried.gateFailureFingerprints["contract"])
|
|
}
|
|
|
|
@Test
|
|
fun `RetryAttemptedEvent with charged=false leaves the gate's budget untouched but records the fingerprint`() {
|
|
val charged = reducer.reduce(
|
|
state,
|
|
stored(
|
|
payload = RetryAttemptedEvent(
|
|
sessionId, stageId, 1, 3, "boom",
|
|
gate = "contract", fingerprint = "fp1", charged = true,
|
|
),
|
|
),
|
|
)
|
|
val free = reducer.reduce(
|
|
charged,
|
|
stored(
|
|
payload = RetryAttemptedEvent(
|
|
sessionId, stageId, 1, 3, "different problem",
|
|
gate = "contract", fingerprint = "fp2", charged = false,
|
|
),
|
|
),
|
|
)
|
|
assertEquals(1, free.gateRetryBudgets["contract"])
|
|
assertEquals("fp2", free.gateFailureFingerprints["contract"])
|
|
}
|
|
|
|
@Test
|
|
fun `gate budgets are independent per gate id`() {
|
|
val contractCharged = reducer.reduce(
|
|
state,
|
|
stored(
|
|
payload = RetryAttemptedEvent(
|
|
sessionId, stageId, 1, 3, "boom",
|
|
gate = "contract", fingerprint = "fp1", charged = true,
|
|
),
|
|
),
|
|
)
|
|
val reviewCharged = reducer.reduce(
|
|
contractCharged,
|
|
stored(
|
|
payload = RetryAttemptedEvent(
|
|
sessionId, stageId, 1, 3, "boom2",
|
|
gate = "review", fingerprint = "fpr1", charged = true,
|
|
),
|
|
),
|
|
)
|
|
assertEquals(1, reviewCharged.gateRetryBudgets["contract"])
|
|
assertEquals(1, reviewCharged.gateRetryBudgets["review"])
|
|
}
|
|
|
|
@Test
|
|
fun `TransitionExecutedEvent resets per-gate budgets, fingerprints, and salvage-used set`() {
|
|
val charged = reducer.reduce(
|
|
state,
|
|
stored(
|
|
payload = RetryAttemptedEvent(
|
|
sessionId, stageId, 1, 3, "boom",
|
|
gate = "review", fingerprint = "fp1", charged = true,
|
|
),
|
|
),
|
|
)
|
|
val salvaged = reducer.reduce(
|
|
charged,
|
|
stored(
|
|
payload = RetrySalvageDecidedEvent(sessionId, stageId, "review", SalvageDecision.CONTINUE, "fixable"),
|
|
),
|
|
)
|
|
assertTrue(salvaged.gateSalvageUsed.contains("review"))
|
|
|
|
val nextStage = StageId("stage-2")
|
|
val advanced = reducer.reduce(
|
|
salvaged,
|
|
stored(payload = TransitionExecutedEvent(sessionId, stageId, nextStage, TransitionId("t1"))),
|
|
)
|
|
assertTrue(advanced.gateRetryBudgets.isEmpty())
|
|
assertTrue(advanced.gateFailureFingerprints.isEmpty())
|
|
assertTrue(advanced.gateSalvageUsed.isEmpty())
|
|
}
|
|
|
|
@Test
|
|
fun `RetrySalvageDecidedEvent CONTINUE resets the gate's budget and marks salvage used`() {
|
|
val charged = reducer.reduce(
|
|
state,
|
|
stored(
|
|
payload = RetryAttemptedEvent(
|
|
sessionId, stageId, 1, 3, "boom",
|
|
gate = "review", fingerprint = "fp1", charged = true,
|
|
),
|
|
),
|
|
)
|
|
assertEquals(1, charged.gateRetryBudgets["review"])
|
|
|
|
val salvaged = reducer.reduce(
|
|
charged,
|
|
stored(
|
|
payload = RetrySalvageDecidedEvent(sessionId, stageId, "review", SalvageDecision.CONTINUE, "fixable"),
|
|
),
|
|
)
|
|
assertEquals(0, salvaged.gateRetryBudgets["review"])
|
|
assertTrue(salvaged.gateSalvageUsed.contains("review"))
|
|
}
|
|
|
|
@Test
|
|
fun `RetrySalvageDecidedEvent FAIL does not touch state`() {
|
|
val charged = reducer.reduce(
|
|
state,
|
|
stored(
|
|
payload = RetryAttemptedEvent(
|
|
sessionId, stageId, 1, 3, "boom",
|
|
gate = "review", fingerprint = "fp1", charged = true,
|
|
),
|
|
),
|
|
)
|
|
val decided = reducer.reduce(
|
|
charged,
|
|
stored(
|
|
payload = RetrySalvageDecidedEvent(sessionId, stageId, "review", SalvageDecision.FAIL, "not fixable"),
|
|
),
|
|
)
|
|
assertEquals(charged, decided)
|
|
}
|
|
|
|
@Test
|
|
fun `unrelated event does nothing`() {
|
|
val started = reducer.reduce(
|
|
state,
|
|
stored(payload = WorkflowStartedEvent(sessionId, workflowId, stageId)),
|
|
)
|
|
|
|
val unrelated = reducer.reduce(started, stored())
|
|
assertEquals(started, unrelated)
|
|
}
|
|
}
|