fix(inference): tolerate a briefly-absent provider instead of killing the session (#299)

A transient provider connection drop (e.g. OOM-killed llama.cpp mid-request)
collapsed into a hard NoEligibleProviderException that escaped runInference's
try/catch (the route() call sat outside it), propagated past the stage retry
loop, and landed in ServerModule's session-level catch — failing the WHOLE
session even though the failure was retryable and the provider recovered
seconds later.

- SessionOrchestrator.runInference: wrap router.route() in try/catch so
  routing failures become InferenceResult.Failed and flow through the normal
  retryable/backoff/exhaustion machinery instead of escaping.
- DefaultInferenceRouter.route: distinguish "capability never configured on
  any provider" (fail fast, waiting can't help) from "configured but
  currently unhealthy" (bounded wait/backoff — default 3 attempts x 2s —
  re-checking health before declaring NoEligibleProvider terminal).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01GeyGFXczJb8RUWGBKmkm6G
This commit is contained in:
kami
2026-07-21 01:57:04 +04:00
parent d69cb12ce9
commit c5289420a1
4 changed files with 137 additions and 2 deletions
@@ -13,8 +13,10 @@ import com.correx.core.inference.RoutingStrategy
import com.correx.testing.fixtures.inference.MockInferenceProvider
import kotlinx.coroutines.runBlocking
import org.junit.jupiter.api.Assertions.assertSame
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.assertThrows
import kotlin.time.Duration.Companion.milliseconds
class DefaultInferenceRouterTest {
@@ -157,4 +159,58 @@ class DefaultInferenceRouterTest {
val result = router.route(stage, setOf(ModelCapability.General), "llama-cpp:sick")
assertSame(healthy, result)
}
// ── bounded wait for a briefly-absent provider (#299) ─────────────────────
@Test
fun `waits for the sole capable provider to recover instead of failing immediately`(): Unit = runBlocking {
var healthChecks = 0
val recovering = object : InferenceProvider by provider("a", ModelCapability.ToolCalling) {
override suspend fun healthCheck(): ProviderHealth {
healthChecks++
return if (healthChecks < 3) ProviderHealth.Unavailable("connection dropped") else ProviderHealth.Healthy
}
}
val router = DefaultInferenceRouter(
registryOf(recovering),
firstStrategy(),
unavailableRetryAttempts = 5,
unavailableRetryDelay = 5.milliseconds,
)
val result = router.route(stage, setOf(ModelCapability.ToolCalling))
assertSame(recovering, result)
assertTrue(healthChecks >= 3) { "expected at least 3 health checks, got $healthChecks" }
}
@Test
fun `still throws NoEligibleProviderException if the sole provider never recovers within the bound`() {
val neverRecovers = MockInferenceProvider(
id = ProviderId("a"),
declaredCapabilities = setOf(CapabilityScore(ModelCapability.ToolCalling, 1.0)),
health = ProviderHealth.Unavailable("still down"),
)
val router = DefaultInferenceRouter(
registryOf(neverRecovers),
throwingStrategy(),
unavailableRetryAttempts = 2,
unavailableRetryDelay = 5.milliseconds,
)
assertThrows<NoEligibleProviderException> {
runBlocking { router.route(stage, setOf(ModelCapability.ToolCalling)) }
}
}
@Test
fun `fails fast without waiting when the capability was never configured on any provider`(): Unit = runBlocking {
val p = provider("a", ModelCapability.General) // does not declare ToolCalling
val router = DefaultInferenceRouter(
registryOf(p),
throwingStrategy(),
unavailableRetryAttempts = 5,
unavailableRetryDelay = 10_000.milliseconds, // would time the test out if the wait loop ran
)
assertThrows<NoEligibleProviderException> {
router.route(stage, setOf(ModelCapability.ToolCalling))
}
}
}