From 201b599472a9eeb69741b5b06be2f89c1640e395 Mon Sep 17 00:00:00 2001 From: kami Date: Sun, 28 Jun 2026 20:43:00 +0400 Subject: [PATCH] fix(inference): recover tool calls emitted as JSON in message content Some local models write the tool call as a JSON blob in `content` instead of the native tool_calls array (toolCalls arrives empty). The orchestrator then treated the blob as a failing artifact and retry-looped to exhaustion. salvageToolCalls parses content as a ToolCallRequest (object or array) when tool_calls is empty; real artifact JSON lacks a `function` field so it's left untouched. --- .../llama/cpp/LlamaCppInferenceProvider.kt | 28 +++++++++++--- .../llama/cpp/SalvageToolCallsTest.kt | 38 +++++++++++++++++++ 2 files changed, 61 insertions(+), 5 deletions(-) create mode 100644 infrastructure/inference/llama_cpp/src/test/kotlin/com/correx/infrastructure/inference/llama/cpp/SalvageToolCallsTest.kt diff --git a/infrastructure/inference/llama_cpp/src/main/kotlin/com/correx/infrastructure/inference/llama/cpp/LlamaCppInferenceProvider.kt b/infrastructure/inference/llama_cpp/src/main/kotlin/com/correx/infrastructure/inference/llama/cpp/LlamaCppInferenceProvider.kt index c18258a4..7518b70d 100644 --- a/infrastructure/inference/llama_cpp/src/main/kotlin/com/correx/infrastructure/inference/llama/cpp/LlamaCppInferenceProvider.kt +++ b/infrastructure/inference/llama_cpp/src/main/kotlin/com/correx/infrastructure/inference/llama/cpp/LlamaCppInferenceProvider.kt @@ -11,6 +11,7 @@ import com.correx.core.inference.ProviderHealth import com.correx.core.inference.ResponseFormat import com.correx.core.inference.TokenUsage import com.correx.core.inference.Tokenizer +import com.correx.core.inference.ToolCallRequest import com.correx.infrastructure.inference.commons.ModelDescriptor import io.ktor.client.HttpClient import io.ktor.client.call.body @@ -62,6 +63,18 @@ private val json = Json { private val log = LoggerFactory.getLogger(LlamaCppInferenceProvider::class.java) +// salvageToolCalls parses a tool call the model wrote into `content` as raw JSON (single object +// or array) instead of the native tool_calls array. Real artifact JSON has no `function` field, +// so it fails to decode and is left alone — only genuine tool-call shapes are recovered. +internal fun salvageToolCalls(content: String?): List { + val s = content?.trim().orEmpty() + if (s.isEmpty() || (s.first() != '{' && s.first() != '[')) return emptyList() + return runCatching { json.decodeFromString>(s) } + .recoverCatching { listOf(json.decodeFromString(s)) } + .getOrDefault(emptyList()) + .filter { it.function.name.isNotBlank() } +} + @Suppress("TooGenericExceptionCaught", "MagicNumber") class LlamaCppInferenceProvider( private val descriptor: ModelDescriptor, @@ -132,22 +145,27 @@ class LlamaCppInferenceProvider( log.debug("got response from llm: {}", response) val message = response.choices.first().message - val finishReason = when (response.choices.first().finishReason.lowercase()) { - "length" -> FinishReason.Length - "tool_calls" -> FinishReason.ToolCall + // Some local models emit the tool call as a JSON blob in `content` instead of the native + // tool_calls array; salvage it so the orchestrator gets a real call rather than treating + // the blob as a (failing) artifact and retry-looping to exhaustion. + val salvaged = if (message.toolCalls.isEmpty()) salvageToolCalls(message.content) else emptyList() + val toolCalls = message.toolCalls.ifEmpty { salvaged } + val finishReason = when { + toolCalls.isNotEmpty() -> FinishReason.ToolCall + response.choices.first().finishReason.lowercase() == "length" -> FinishReason.Length else -> FinishReason.Stop } return InferenceResponse( requestId = request.requestId, - text = message.content ?: "", + text = if (salvaged.isNotEmpty()) "" else message.content ?: "", finishReason = finishReason, tokensUsed = TokenUsage( promptTokens = response.usage.promptTokens, completionTokens = response.usage.completionTokens, ), latencyMs = System.currentTimeMillis() - startTime, - toolCalls = message.toolCalls, + toolCalls = toolCalls, ) } diff --git a/infrastructure/inference/llama_cpp/src/test/kotlin/com/correx/infrastructure/inference/llama/cpp/SalvageToolCallsTest.kt b/infrastructure/inference/llama_cpp/src/test/kotlin/com/correx/infrastructure/inference/llama/cpp/SalvageToolCallsTest.kt new file mode 100644 index 00000000..c227deb8 --- /dev/null +++ b/infrastructure/inference/llama_cpp/src/test/kotlin/com/correx/infrastructure/inference/llama/cpp/SalvageToolCallsTest.kt @@ -0,0 +1,38 @@ +package com.correx.infrastructure.inference.llama.cpp + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +class SalvageToolCallsTest { + + @Test + fun `recovers a tool call emitted as a JSON object in content`() { + val args = """{\"path\":\"apps/server/src\"}""" + val content = """{"id":"abc","function":{"name":"file_read","arguments":"$args"}}""" + val calls = salvageToolCalls(content) + assertEquals(1, calls.size) + assertEquals("file_read", calls.first().function.name) + } + + @Test + fun `recovers a JSON array of tool calls`() { + val one = """{"function":{"name":"file_read","arguments":"{}"}}""" + val two = """{"function":{"name":"ls","arguments":"{}"}}""" + val calls = salvageToolCalls("[$one,$two]") + assertEquals(listOf("file_read", "ls"), calls.map { it.function.name }) + } + + @Test + fun `leaves a real artifact JSON alone`() { + val artifact = """{"summary":"add a web ui","requirements":["one","two"]}""" + assertTrue(salvageToolCalls(artifact).isEmpty()) + } + + @Test + fun `ignores empty, prose, and null content`() { + assertTrue(salvageToolCalls(null).isEmpty()) + assertTrue(salvageToolCalls(" ").isEmpty()) + assertTrue(salvageToolCalls("Let me explore the frontend directory.").isEmpty()) + } +}