fix(inference): tokenize endpoint sent wrong field — every token estimate was 0
LlamaCppTokenizer posted {"tokens":[text]} but llama.cpp's /tokenize expects
{"content":text}; the server silently answered {"tokens":[]}, so countTokens
returned 0 for every string. Every context entry carried tokenEstimate=0 and
the whole budget/trim pipeline was blind — live QA saw a 34k-token prompt
sail through a 16384 stage budget (three raw file_read results of plan docs),
crater t/s, degrade output, and fail the stage on validation 3x.
Also guard estimateTokens: a zero count for non-blank content falls back to
the chars/4 heuristic so a lying tokenizer can never blind budgeting again.
This commit is contained in:
+7
-1
@@ -1542,7 +1542,13 @@ abstract class SessionOrchestrator(
|
|||||||
protected open suspend fun estimateTokens(content: String): Int {
|
protected open suspend fun estimateTokens(content: String): Int {
|
||||||
val t = tokenizer
|
val t = tokenizer
|
||||||
if (t != null) {
|
if (t != null) {
|
||||||
return runCatching { t.countTokens(content) }.getOrElse { fallbackTokenEstimate(content) }
|
// A zero count for non-blank content means the tokenizer endpoint is lying
|
||||||
|
// (e.g. a malformed request silently answered with an empty token list) —
|
||||||
|
// trust the heuristic instead, or every budget check goes blind.
|
||||||
|
return runCatching { t.countTokens(content) }
|
||||||
|
.getOrElse { fallbackTokenEstimate(content) }
|
||||||
|
.takeIf { it > 0 || content.isBlank() }
|
||||||
|
?: fallbackTokenEstimate(content)
|
||||||
}
|
}
|
||||||
return fallbackTokenEstimate(content)
|
return fallbackTokenEstimate(content)
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -49,7 +49,7 @@ data class Usage(
|
|||||||
)
|
)
|
||||||
|
|
||||||
@Serializable
|
@Serializable
|
||||||
data class TokenizeRequest(val tokens: List<String>)
|
data class TokenizeRequest(val content: String)
|
||||||
|
|
||||||
@Serializable
|
@Serializable
|
||||||
data class TokenizeResponse(val tokens: List<Int>)
|
data class TokenizeResponse(val tokens: List<Int>)
|
||||||
|
|||||||
+1
-1
@@ -19,7 +19,7 @@ class LlamaCppTokenizer(
|
|||||||
httpClient.post("$baseUrl/tokenize") {
|
httpClient.post("$baseUrl/tokenize") {
|
||||||
contentType(ContentType.Application.Json)
|
contentType(ContentType.Application.Json)
|
||||||
accept(ContentType.Application.Json)
|
accept(ContentType.Application.Json)
|
||||||
setBody(TokenizeRequest(tokens = listOf(text)))
|
setBody(TokenizeRequest(content = text))
|
||||||
}.body<TokenizeResponse>().tokens.map { Token(it) }
|
}.body<TokenizeResponse>().tokens.map { Token(it) }
|
||||||
|
|
||||||
override suspend fun countTokens(text: String): Int = tokenize(text).size
|
override suspend fun countTokens(text: String): Int = tokenize(text).size
|
||||||
|
|||||||
@@ -31,6 +31,36 @@ class DefaultContextPackBuilderTest {
|
|||||||
tokenEstimate = tokens
|
tokenEstimate = tokens
|
||||||
)
|
)
|
||||||
|
|
||||||
|
private fun typedEntry(id: String, layer: ContextLayer, tokens: Int, sourceType: String) = ContextEntry(
|
||||||
|
id = ContextEntryId(id),
|
||||||
|
layer = layer,
|
||||||
|
content = "content-$id",
|
||||||
|
sourceType = sourceType,
|
||||||
|
sourceId = id,
|
||||||
|
tokenEstimate = tokens
|
||||||
|
)
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `oversized tool results from a stage tool loop are trimmed to budget`() {
|
||||||
|
// Live repro (2026-06-11): analyst stage with three file_read results of ~5.6k/11.9k/10.7k
|
||||||
|
// tokens sailed through a 16384 budget untrimmed and produced a 34k-token prompt.
|
||||||
|
val entries = listOf(
|
||||||
|
typedEntry("sys", ContextLayer.L0, 3500, "systemPrompt"),
|
||||||
|
typedEntry("task", ContextLayer.L1, 150, "agentPrompt"),
|
||||||
|
typedEntry("call1", ContextLayer.L2, 40, "assistantToolCall"),
|
||||||
|
typedEntry("res1", ContextLayer.L2, 5600, "toolResult"),
|
||||||
|
typedEntry("call2", ContextLayer.L2, 40, "assistantToolCall"),
|
||||||
|
typedEntry("res2", ContextLayer.L2, 11900, "toolResult"),
|
||||||
|
typedEntry("call3", ContextLayer.L2, 40, "assistantToolCall"),
|
||||||
|
typedEntry("res3", ContextLayer.L2, 10700, "toolResult"),
|
||||||
|
)
|
||||||
|
val pack = builder.build(packId, sessionId, stageId, entries, TokenBudget(limit = 16384))
|
||||||
|
assertTrue(
|
||||||
|
pack.budgetUsed <= 16384,
|
||||||
|
"budgetUsed ${pack.budgetUsed} must not exceed the 16384 stage budget",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
fun `build groups entries by layer`() {
|
fun `build groups entries by layer`() {
|
||||||
val entries = listOf(
|
val entries = listOf(
|
||||||
|
|||||||
Reference in New Issue
Block a user