feat(inference): operator-tunable sampling knobs (top_k/min_p/repeat_penalty) for stage requests
GenerationConfig only carried temperature/top_p/max_tokens/stop/seed. Added nullable topK/minP/repeatPenalty, serialized to the llama.cpp and OpenAI-compat request bodies via @EncodeDefault(NEVER) so an unset knob is omitted (the model keeps its own default) and behavior is unchanged unless the operator opts in. Surfaced as a new [sampling] config section feeding the default stage GenerationConfig (the main agentic loop) through TomlWorkflowLoader + ExecutionPlanCompiler; the former hardcoded temperature=0.7/topP=1.0 stage defaults now come from config. Talkie chat/narration keep their own generation settings. Vikunja #46 (task 76) — sampling half.
This commit is contained in:
+6
@@ -2,6 +2,7 @@ package com.correx.infrastructure.inference.openai
|
||||
|
||||
import com.correx.core.inference.ToolCallRequest
|
||||
import com.correx.core.inference.ToolDefinition
|
||||
import kotlinx.serialization.EncodeDefault
|
||||
import kotlinx.serialization.SerialName
|
||||
import kotlinx.serialization.Serializable
|
||||
|
||||
@@ -20,6 +21,11 @@ data class OpenAiChatCompletionRequest(
|
||||
@SerialName("max_tokens") val maxTokens: Int,
|
||||
@SerialName("stop") val stopSequences: List<String>? = null,
|
||||
val seed: Long? = null,
|
||||
// Non-standard OpenAI params accepted by local backends (vLLM, llama.cpp --api). EncodeDefault.NEVER
|
||||
// omits them when unset so a strict endpoint never sees an unknown key unless the operator opts in.
|
||||
@EncodeDefault(EncodeDefault.Mode.NEVER) @SerialName("top_k") val topK: Int? = null,
|
||||
@EncodeDefault(EncodeDefault.Mode.NEVER) @SerialName("min_p") val minP: Double? = null,
|
||||
@EncodeDefault(EncodeDefault.Mode.NEVER) @SerialName("repeat_penalty") val repeatPenalty: Double? = null,
|
||||
val stream: Boolean = false,
|
||||
val tools: List<ToolDefinition>? = null,
|
||||
)
|
||||
|
||||
+3
@@ -104,6 +104,9 @@ class OpenAiCompatInferenceProvider(
|
||||
maxTokens = request.generationConfig.maxTokens,
|
||||
stopSequences = request.generationConfig.stopSequences.ifEmpty { null },
|
||||
seed = request.generationConfig.seed,
|
||||
topK = request.generationConfig.topK,
|
||||
minP = request.generationConfig.minP,
|
||||
repeatPenalty = request.generationConfig.repeatPenalty,
|
||||
stream = false,
|
||||
tools = tools,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user