feat(health): gate routing event-driven on connection drop, not just periodic poll (#300)

HealthMonitor's periodic poll took ~18s to notice a dead provider — long
after a retry had already re-selected it and died. Health needs to GATE
routing, not just log reactively.

- InferenceRouter.reportFailure(providerId, reason): new default-no-op
  method; DefaultInferenceRouter implements it by writing Unavailable
  straight into its health cache, bypassing healthCheck()/TTL entirely.
- SessionOrchestrator.runInference: on a connection-level exception
  (ConnectException/SocketException/IOException or a message matching
  "prematurely closed"/"connection refused"/"connection reset"), call
  reportFailure immediately so the very next route() call — the retry driven
  by #299 — sees the provider as down right away instead of re-selecting it
  and dying again.

Pairs with #299: routing now (1) reacts to a connection drop instantly and
(2) waits/backs off for the capability to recover before declaring terminal.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01GeyGFXczJb8RUWGBKmkm6G
This commit is contained in:
kami
2026-07-21 02:01:35 +04:00
parent c5289420a1
commit bf36252736
5 changed files with 107 additions and 0 deletions
@@ -59,6 +59,16 @@ class DefaultInferenceRouter(
fresh
}
// Bypasses healthCheck()/TTL entirely — writes Unavailable straight into the cache so the very
// next route() call gates on it, closing the ~18s reactive-poll lag (#300). A later cache-TTL
// expiry or the bounded-wait re-check in route() will naturally pick the provider back up once
// its own healthCheck() reports healthy again.
override suspend fun reportFailure(providerId: ProviderId, reason: String) {
lockFor(providerId).withLock {
mapMutex.withLock { cache[providerId] = HealthEntry(ProviderHealth.Unavailable(reason), timeSource.markNow()) }
}
}
override suspend fun route(stageId: StageId, requiredCapabilities: Set<ModelCapability>): InferenceProvider {
val candidates = requiredCapabilities
.flatMap { registry.resolve(it) }
@@ -39,6 +39,14 @@ interface InferenceRouter {
requiredCapabilities: Set<ModelCapability>,
modelId: String?,
): InferenceProvider = route(stageId, requiredCapabilities)
/**
* Event-driven health gate: called the moment a connection-level failure is observed on
* [providerId] (e.g. mid-request connection drop), so the NEXT route() call sees it as
* unavailable immediately instead of waiting for the next periodic health poll/cache TTL to
* catch up. Default no-op for routers that don't cache health.
*/
suspend fun reportFailure(providerId: com.correx.core.events.types.ProviderId, reason: String) = Unit
}
class NoEligibleProviderException(
@@ -409,6 +409,13 @@ abstract class SessionOrchestrator(
} catch (e: CancellationException) {
throw e // never swallow
} catch (e: Exception) {
if (isConnectionLevelFailure(e)) {
// Mark it down NOW instead of waiting for the next periodic health poll (~18s lag,
// see #300) — the retry's route() call must see this provider as unavailable
// immediately so it gates/waits (#299) rather than instantly re-selecting the dead
// provider again.
inferenceRouter.reportFailure(provider.id, e.message ?: "connection failure")
}
emit(
sessionId,
InferenceFailedEvent(
@@ -423,6 +430,19 @@ abstract class SessionOrchestrator(
}
}
// Connection-level failures (provider crashed/restarting mid-request) should gate routing
// immediately; other failures (bad request, model error, HTTP 4xx) should not mark the
// provider down since the provider itself is still reachable.
private fun isConnectionLevelFailure(e: Exception): Boolean {
val message = e.message.orEmpty()
return e is java.net.ConnectException ||
e is java.net.SocketException ||
e is java.io.IOException || // covers ktor/CIO's IOException, which extends java.io.IOException on the JVM
message.contains("prematurely closed", ignoreCase = true) ||
message.contains("connection refused", ignoreCase = true) ||
message.contains("connection reset", ignoreCase = true)
}
// --- token estimation ---
internal open suspend fun estimateTokens(content: String): Int {