feat(inference): operator-tunable sampling knobs (top_k/min_p/repeat_penalty) for stage requests

GenerationConfig only carried temperature/top_p/max_tokens/stop/seed. Added nullable
topK/minP/repeatPenalty, serialized to the llama.cpp and OpenAI-compat request bodies
via @EncodeDefault(NEVER) so an unset knob is omitted (the model keeps its own default)
and behavior is unchanged unless the operator opts in.

Surfaced as a new [sampling] config section feeding the default stage GenerationConfig
(the main agentic loop) through TomlWorkflowLoader + ExecutionPlanCompiler; the former
hardcoded temperature=0.7/topP=1.0 stage defaults now come from config. Talkie
chat/narration keep their own generation settings.

Vikunja #46 (task 76) — sampling half.
This commit is contained in:
2026-07-12 17:54:25 +04:00
parent b2c7bbe401
commit d89d4e32a9
13 changed files with 125 additions and 16 deletions
@@ -3,6 +3,7 @@ package com.correx.infrastructure.inference.llama.cpp
import com.correx.core.inference.ChatMessage
import com.correx.core.inference.ToolCallRequest
import com.correx.core.inference.ToolDefinition
import kotlinx.serialization.EncodeDefault
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
@@ -15,6 +16,12 @@ data class ChatCompletionRequest(
@SerialName("max_tokens") val maxTokens: Int,
@SerialName("stop") val stopSequences: List<String> = emptyList(),
val seed: Long? = null,
// EncodeDefault.NEVER overrides the class-level encodeDefaults=true so an unset (null) sampling
// knob is omitted from the JSON entirely, letting the model keep its own default (rather than
// sending "top_k": null, which llama.cpp may reject or misread).
@EncodeDefault(EncodeDefault.Mode.NEVER) @SerialName("top_k") val topK: Int? = null,
@EncodeDefault(EncodeDefault.Mode.NEVER) @SerialName("min_p") val minP: Double? = null,
@EncodeDefault(EncodeDefault.Mode.NEVER) @SerialName("repeat_penalty") val repeatPenalty: Double? = null,
val stream: Boolean = false,
val grammar: String? = null,
val tools: List<ToolDefinition>? = null,
@@ -170,6 +170,9 @@ class LlamaCppInferenceProvider(
maxTokens = request.generationConfig.maxTokens,
stopSequences = request.generationConfig.stopSequences,
seed = request.generationConfig.seed,
topK = request.generationConfig.topK,
minP = request.generationConfig.minP,
repeatPenalty = request.generationConfig.repeatPenalty,
stream = false,
grammar = grammar,
tools = tools,
@@ -0,0 +1,37 @@
package com.correx.infrastructure.inference.llama.cpp
import kotlinx.serialization.encodeToString
import kotlinx.serialization.json.Json
import kotlin.test.Test
import kotlin.test.assertFalse
import kotlin.test.assertTrue
// Guards the EncodeDefault.NEVER behavior on the sampling knobs: with encodeDefaults=true (matching
// the provider's Json), an unset (null) top_k/min_p/repeat_penalty must be OMITTED from the body so
// the model keeps its own default; a set value must appear.
class SamplingRequestSerializationTest {
private val json = Json { encodeDefaults = true }
@Test
fun `unset sampling knobs are omitted from the request body`() {
val body = ChatCompletionRequest(
model = "m", messages = emptyList(), temperature = 0.7, topP = 1.0, maxTokens = 16,
)
val out = json.encodeToString(body)
assertFalse("top_k" in out, out)
assertFalse("min_p" in out, out)
assertFalse("repeat_penalty" in out, out)
}
@Test
fun `set sampling knobs are serialized`() {
val body = ChatCompletionRequest(
model = "m", messages = emptyList(), temperature = 0.7, topP = 1.0, maxTokens = 16,
topK = 40, minP = 0.05, repeatPenalty = 1.1,
)
val out = json.encodeToString(body)
assertTrue("\"top_k\":40" in out, out)
assertTrue("\"min_p\":0.05" in out, out)
assertTrue("\"repeat_penalty\":1.1" in out, out)
}
}