feat(inference): operator-tunable sampling knobs (top_k/min_p/repeat_penalty) for stage requests
GenerationConfig only carried temperature/top_p/max_tokens/stop/seed. Added nullable topK/minP/repeatPenalty, serialized to the llama.cpp and OpenAI-compat request bodies via @EncodeDefault(NEVER) so an unset knob is omitted (the model keeps its own default) and behavior is unchanged unless the operator opts in. Surfaced as a new [sampling] config section feeding the default stage GenerationConfig (the main agentic loop) through TomlWorkflowLoader + ExecutionPlanCompiler; the former hardcoded temperature=0.7/topP=1.0 stage defaults now come from config. Talkie chat/narration keep their own generation settings. Vikunja #46 (task 76) — sampling half.
This commit is contained in:
+6
-8
@@ -59,12 +59,6 @@ private const val DEFAULT_STAGE_TOKEN_BUDGET = 16384
|
||||
// default, or the model is truncated (finishReason=length) mid-artifact — and a degenerating local
|
||||
// model burns the whole 2048 on garbage (e.g. a `<|channel>thought` repetition loop) before it can
|
||||
// stop. Mirror the static TomlWorkflowLoader path: pin the completion cap to the stage token budget.
|
||||
private val DEFAULT_STAGE_GENERATION = GenerationConfig(
|
||||
temperature = 0.7,
|
||||
topP = 1.0,
|
||||
maxTokens = DEFAULT_STAGE_TOKEN_BUDGET,
|
||||
)
|
||||
|
||||
class ExecutionPlanCompiler(
|
||||
private val registry: ArtifactKindRegistry,
|
||||
// Names of every registered tool. A stage that references a tool the runtime can't resolve
|
||||
@@ -79,7 +73,11 @@ class ExecutionPlanCompiler(
|
||||
// retries (Vikunja #41). Off by default so the compiler's own unit tests see only plan stages;
|
||||
// the server's freestyle path (Main.kt) turns it on.
|
||||
private val injectRecovery: Boolean = false,
|
||||
// Operator sampling defaults for freestyle-compiled stages; maxTokens pinned to the stage budget.
|
||||
// Default reproduces the former hardcoded temperature=0.7/topP=1.0.
|
||||
private val samplingDefaults: GenerationConfig = GenerationConfig(temperature = 0.7, topP = 1.0, maxTokens = 0),
|
||||
) {
|
||||
private val defaultStageGeneration = samplingDefaults.copy(maxTokens = DEFAULT_STAGE_TOKEN_BUDGET)
|
||||
private val mapper = JsonMapper.builder()
|
||||
.addModule(kotlinModule())
|
||||
.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
|
||||
@@ -150,7 +148,7 @@ class ExecutionPlanCompiler(
|
||||
autoBuildGate = s.id == autoGateStageId,
|
||||
semanticReview = s.semanticReview,
|
||||
tokenBudget = DEFAULT_STAGE_TOKEN_BUDGET,
|
||||
generationConfig = DEFAULT_STAGE_GENERATION,
|
||||
generationConfig = defaultStageGeneration,
|
||||
metadata = mapOf("promptInline" to s.prompt),
|
||||
)
|
||||
}
|
||||
@@ -168,7 +166,7 @@ class ExecutionPlanCompiler(
|
||||
StageId(RECOVERY_STAGE) to StageConfig(
|
||||
allowedTools = knownTools.ifEmpty { setOf("file_write", "file_edit", "shell") },
|
||||
tokenBudget = DEFAULT_STAGE_TOKEN_BUDGET,
|
||||
generationConfig = DEFAULT_STAGE_GENERATION,
|
||||
generationConfig = defaultStageGeneration,
|
||||
metadata = mapOf("role" to "recovery", "promptInline" to RECOVERY_PROMPT),
|
||||
)
|
||||
}
|
||||
|
||||
+4
-5
@@ -76,6 +76,9 @@ private val mapper = TomlMapper.builder()
|
||||
|
||||
class TomlWorkflowLoader(
|
||||
private val registry: ArtifactKindRegistry = DefaultArtifactKindRegistry(),
|
||||
// Operator sampling defaults applied to every stage's inference request (maxTokens is still pinned
|
||||
// per-stage to the token budget). Defaults reproduce the former hardcoded temperature=0.7/topP=1.0.
|
||||
private val samplingDefaults: GenerationConfig = GenerationConfig(temperature = 0.7, topP = 1.0, maxTokens = 0),
|
||||
) : WorkflowLoader {
|
||||
override fun load(path: Path): WorkflowGraph {
|
||||
val raw = path.readText()
|
||||
@@ -116,11 +119,7 @@ class TomlWorkflowLoader(
|
||||
// Propagate the declared token budget to the inference completion cap.
|
||||
// Without this the StageConfig default (maxTokens=2048) is used, truncating
|
||||
// larger artifacts (finishReason=length) → invalid JSON → validation failure.
|
||||
generationConfig = GenerationConfig(
|
||||
temperature = 0.7,
|
||||
topP = 1.0,
|
||||
maxTokens = s.tokenBudget,
|
||||
),
|
||||
generationConfig = samplingDefaults.copy(maxTokens = s.tokenBudget),
|
||||
maxRetries = s.maxRetries,
|
||||
metadata = s.toMetadata(workflowDir),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user