fix(inference): tolerate a briefly-absent provider instead of killing the session (#299)
A transient provider connection drop (e.g. OOM-killed llama.cpp mid-request) collapsed into a hard NoEligibleProviderException that escaped runInference's try/catch (the route() call sat outside it), propagated past the stage retry loop, and landed in ServerModule's session-level catch — failing the WHOLE session even though the failure was retryable and the provider recovered seconds later. - SessionOrchestrator.runInference: wrap router.route() in try/catch so routing failures become InferenceResult.Failed and flow through the normal retryable/backoff/exhaustion machinery instead of escaping. - DefaultInferenceRouter.route: distinguish "capability never configured on any provider" (fail fast, waiting can't help) from "configured but currently unhealthy" (bounded wait/backoff — default 3 attempts x 2s — re-checking health before declaring NoEligibleProvider terminal). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01GeyGFXczJb8RUWGBKmkm6G
This commit is contained in:
@@ -46,6 +46,7 @@ import com.correx.core.inference.InferenceResponse
|
||||
import com.correx.core.inference.InferenceRouter
|
||||
import com.correx.core.inference.InferenceState
|
||||
import com.correx.core.inference.ModelCapability
|
||||
import com.correx.core.inference.NoEligibleProviderException
|
||||
import com.correx.core.inference.ProviderHealth
|
||||
import com.correx.core.inference.Token
|
||||
import com.correx.core.inference.TokenUsage
|
||||
@@ -216,6 +217,42 @@ class SessionOrchestratorIntegrationTest {
|
||||
assertTrue(failed.retryExhausted)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `transient NoEligibleProviderException on a retry is tolerated, not a session-killing crash (#299)`(): Unit =
|
||||
runBlocking {
|
||||
val sessionId = SessionId("s2b")
|
||||
val config = OrchestrationConfig(
|
||||
retryPolicy = RetryPolicy(maxAttempts = 3, backoffMs = 0),
|
||||
)
|
||||
val graph = threeStageGraph()
|
||||
var routeCalls = 0
|
||||
|
||||
val recoveringOrchestrator = DefaultSessionOrchestrator(
|
||||
repositories = repositories,
|
||||
engines = engines.copy(
|
||||
inferenceRouter = object : InferenceRouter {
|
||||
override suspend fun route(stageId: StageId, requiredCapabilities: Set<ModelCapability>): InferenceProvider {
|
||||
routeCalls++
|
||||
// First attempt hits the provider mid-crash — subsequent retries find it back.
|
||||
if (routeCalls == 1) throw NoEligibleProviderException(stageId, requiredCapabilities)
|
||||
return MockInferenceProvider()
|
||||
}
|
||||
},
|
||||
),
|
||||
retryCoordinator = retryCoordinator,
|
||||
artifactStore = artifactStore,
|
||||
decisionJournalRepository = decisionJournalRepository,
|
||||
)
|
||||
recoveringOrchestrator.run(sessionId, graph, config)
|
||||
|
||||
val events = eventStore.read(sessionId)
|
||||
// A crash would have skipped straight to an unhandled failure with no WorkflowCompletedEvent
|
||||
// and no orderly retry — assert the session survived and made it through via the retry path.
|
||||
assertTrue(routeCalls > 1) { "expected routing to be retried, only called $routeCalls time(s)" }
|
||||
assertNotNull(events.find { it.payload is WorkflowCompletedEvent })
|
||||
assertTrue(events.none { it.payload is WorkflowFailedEvent })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `stage that matches no transition retries through the budget before failing`(): Unit = runBlocking {
|
||||
val sessionId = SessionId("s-no-transition-retry")
|
||||
|
||||
Reference in New Issue
Block a user