fix(inference): tokenize endpoint sent wrong field — every token estimate was 0
LlamaCppTokenizer posted {"tokens":[text]} but llama.cpp's /tokenize expects
{"content":text}; the server silently answered {"tokens":[]}, so countTokens
returned 0 for every string. Every context entry carried tokenEstimate=0 and
the whole budget/trim pipeline was blind — live QA saw a 34k-token prompt
sail through a 16384 stage budget (three raw file_read results of plan docs),
crater t/s, degrade output, and fail the stage on validation 3x.
Also guard estimateTokens: a zero count for non-blank content falls back to
the chars/4 heuristic so a lying tokenizer can never blind budgeting again.
This commit is contained in:
+1
-1
@@ -49,7 +49,7 @@ data class Usage(
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class TokenizeRequest(val tokens: List<String>)
|
||||
data class TokenizeRequest(val content: String)
|
||||
|
||||
@Serializable
|
||||
data class TokenizeResponse(val tokens: List<Int>)
|
||||
|
||||
+1
-1
@@ -19,7 +19,7 @@ class LlamaCppTokenizer(
|
||||
httpClient.post("$baseUrl/tokenize") {
|
||||
contentType(ContentType.Application.Json)
|
||||
accept(ContentType.Application.Json)
|
||||
setBody(TokenizeRequest(tokens = listOf(text)))
|
||||
setBody(TokenizeRequest(content = text))
|
||||
}.body<TokenizeResponse>().tokens.map { Token(it) }
|
||||
|
||||
override suspend fun countTokens(text: String): Int = tokenize(text).size
|
||||
|
||||
Reference in New Issue
Block a user