feat(retry): per-gate retry budgets + progress-aware charging + hybrid salvage

Replaces the single shared per-stage retryCount (reset on TransitionExecuted,
shared across all post-stage gates) with a per-gate budget. Motivated by run
2e468f9f, where a promising freestyle run was terminally FAILED because the
contract gate burned all 3 shared retries on one trivial miss before the
semantic-review gate ever ran.

- StageExecutionResult.Failure gains a `gate` id; each gate tags its failures.
- OrchestrationState gains gateRetryBudgets / gateFailureFingerprints /
  gateSalvageUsed (all rebuilt from events, reset per-stage on TransitionExecuted).
- FailureFingerprint normalizes+hashes the failure reason; RetryCoordinator.decide
  charges a gate's budget only when the fingerprint is unchanged (no progress),
  so a moving run is never penalised. Returns Retry | Exhausted.
- Hybrid exhaustion: deterministic gates fail; the review gate consults a
  SalvageJudge (LLM, or a deterministic allow-one-reset fallback), recorded as
  RetrySalvageDecidedEvent (invariant #9). CONTINUE resets the gate budget once;
  a second exhaustion is terminal.
- Review gate is now the sole authority on review-retry termination; the old
  REVIEW_BLOCK_RETRY_CAP is repurposed as a high absolute backstop only.
- ServerModule escaped-exception path now records a truthful terminal
  WorkflowFailed (real stage/reason/retryExhausted) via recordUnhandledFailure.

Tests: DefaultRetryCoordinatorTest (fingerprint charging) +
GateRetryBudgetExhaustionTest (deterministic exhaust / review CONTINUE-then-
terminal / review FAIL / null-judge fallback) + reducer coverage.
This commit is contained in:
2026-07-06 19:20:46 +04:00
parent 9f12c87bb1
commit 79e2c38a88
17 changed files with 1085 additions and 54 deletions
@@ -348,33 +348,53 @@ class ServerModule(
.onFailure { log.warn("Profile adaptation failed: {}", it.message) }
}
}
}.onFailure { ex ->
log.error("runSession: session={} failed: {}", sessionId.value, ex.message, ex)
eventStore.append(
NewEvent(
metadata = EventMetadata(
eventId = EventId(java.util.UUID.randomUUID().toString()),
sessionId = sessionId,
timestamp = Clock.System.now(),
schemaVersion = 1,
causationId = null,
correlationId = null,
),
payload = WorkflowFailedEvent(
sessionId = sessionId,
stageId = graph.start,
reason = ex.message ?: "unexpected orchestrator failure",
retryExhausted = false,
),
),
)
}
}.onFailure { ex -> recordUnhandledFailure(sessionId, graph, sessionConfig, ex) }
activeSessionJobs.remove(sessionId)
}
}
}
}
// An exception escaped the orchestrator (the run never reached its own terminal WorkflowFailed).
// Record a truthful terminal event: the real failing stage (not graph.start), the real message,
// and a computed retryExhausted — so a headless client sees an honest failure, not a placeholder.
private suspend fun recordUnhandledFailure(
sessionId: SessionId,
graph: com.correx.core.transitions.graph.WorkflowGraph,
sessionConfig: OrchestrationConfig,
ex: Throwable,
) {
log.error("runSession: session={} failed: {}", sessionId.value, ex.message, ex)
val orchState = orchestrationRepository.getState(sessionId)
val failingStageId = orchState.currentStageId ?: graph.start
val reason = ex.message
?.trim()
?.takeIf { it.isNotEmpty() }
?: "unexpected orchestrator failure: unhandled exception with no message " +
"during stage '${failingStageId.value}'"
val maxAttempts = orchState.retryPolicy?.maxAttempts
?: sessionConfig.retryPolicy.maxAttempts
val retryExhausted = orchState.retryCount >= maxAttempts
eventStore.append(
NewEvent(
metadata = EventMetadata(
eventId = EventId(java.util.UUID.randomUUID().toString()),
sessionId = sessionId,
timestamp = Clock.System.now(),
schemaVersion = 1,
causationId = null,
correlationId = null,
),
payload = WorkflowFailedEvent(
sessionId = sessionId,
stageId = failingStageId,
reason = reason,
retryExhausted = retryExhausted,
),
),
)
}
/**
* Resolves the workflow graph for resuming a freestyle phase-2 session. Phase-2 graphs
* (`freestyle-<sessionId>`) are compiled from the locked plan, never registered in the