fix(inference): single ContentNegotiation config + firstOrNull on empty choices
This commit is contained in:
+6
-12
@@ -42,22 +42,14 @@ private fun defaultHttpClient(): HttpClient = HttpClient(CIO) {
|
|||||||
Json {
|
Json {
|
||||||
ignoreUnknownKeys = true
|
ignoreUnknownKeys = true
|
||||||
isLenient = true
|
isLenient = true
|
||||||
encodeDefaults = false
|
explicitNulls = false
|
||||||
|
encodeDefaults = true
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
install(HttpTimeout) {
|
install(HttpTimeout) {
|
||||||
requestTimeoutMillis = DEFAULT_REQUEST_TIMEOUT_MS
|
requestTimeoutMillis = DEFAULT_REQUEST_TIMEOUT_MS
|
||||||
}
|
}
|
||||||
install(ContentNegotiation) {
|
|
||||||
json(
|
|
||||||
Json {
|
|
||||||
ignoreUnknownKeys = true
|
|
||||||
explicitNulls = false
|
|
||||||
encodeDefaults = true
|
|
||||||
},
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
private val json = Json {
|
private val json = Json {
|
||||||
ignoreUnknownKeys = true
|
ignoreUnknownKeys = true
|
||||||
@@ -199,7 +191,9 @@ class LlamaCppInferenceProvider(
|
|||||||
|
|
||||||
log.debug("got response from llm: {}", response)
|
log.debug("got response from llm: {}", response)
|
||||||
|
|
||||||
val message = response.choices.first().message
|
val choice = response.choices.firstOrNull()
|
||||||
|
?: error("llama-server returned no choices in the completion response")
|
||||||
|
val message = choice.message
|
||||||
// Some local models emit the tool call as a JSON blob in `content` instead of the native
|
// Some local models emit the tool call as a JSON blob in `content` instead of the native
|
||||||
// tool_calls array; salvage it so the orchestrator gets a real call rather than treating
|
// tool_calls array; salvage it so the orchestrator gets a real call rather than treating
|
||||||
// the blob as a (failing) artifact and retry-looping to exhaustion.
|
// the blob as a (failing) artifact and retry-looping to exhaustion.
|
||||||
@@ -207,7 +201,7 @@ class LlamaCppInferenceProvider(
|
|||||||
val toolCalls = message.toolCalls.ifEmpty { salvaged }
|
val toolCalls = message.toolCalls.ifEmpty { salvaged }
|
||||||
val finishReason = when {
|
val finishReason = when {
|
||||||
toolCalls.isNotEmpty() -> FinishReason.ToolCall
|
toolCalls.isNotEmpty() -> FinishReason.ToolCall
|
||||||
response.choices.first().finishReason.lowercase() == "length" -> FinishReason.Length
|
choice.finishReason.lowercase() == "length" -> FinishReason.Length
|
||||||
else -> FinishReason.Stop
|
else -> FinishReason.Stop
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user