feat(inference): operator-tunable sampling knobs (top_k/min_p/repeat_penalty) for stage requests
GenerationConfig only carried temperature/top_p/max_tokens/stop/seed. Added nullable topK/minP/repeatPenalty, serialized to the llama.cpp and OpenAI-compat request bodies via @EncodeDefault(NEVER) so an unset knob is omitted (the model keeps its own default) and behavior is unchanged unless the operator opts in. Surfaced as a new [sampling] config section feeding the default stage GenerationConfig (the main agentic loop) through TomlWorkflowLoader + ExecutionPlanCompiler; the former hardcoded temperature=0.7/topP=1.0 stage defaults now come from config. Talkie chat/narration keep their own generation settings. Vikunja #46 (task 76) — sampling half.
This commit is contained in:
@@ -662,6 +662,15 @@ object ConfigLoader {
|
||||
)
|
||||
}
|
||||
|
||||
val samplingSection = sections["sampling"] ?: emptyMap()
|
||||
val sampling = SamplingConfig(
|
||||
temperature = asDouble(samplingSection["temperature"], 0.7),
|
||||
topP = asDouble(samplingSection["top_p"], 1.0),
|
||||
topK = samplingSection["top_k"]?.let { asInt(it) },
|
||||
minP = samplingSection["min_p"]?.let { asDouble(it) },
|
||||
repeatPenalty = samplingSection["repeat_penalty"]?.let { asDouble(it) },
|
||||
)
|
||||
|
||||
return CorrexConfig(
|
||||
server = server,
|
||||
tui = tui,
|
||||
@@ -675,6 +684,7 @@ object ConfigLoader {
|
||||
project = project,
|
||||
personalization = personalization,
|
||||
orchestration = orchestration,
|
||||
sampling = sampling,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -16,9 +16,26 @@ data class CorrexConfig(
|
||||
val project: ProjectConfig = ProjectConfig(),
|
||||
val personalization: PersonalizationConfig = PersonalizationConfig(),
|
||||
val orchestration: OrchestrationKnobs = OrchestrationKnobs(),
|
||||
val sampling: SamplingConfig = SamplingConfig(),
|
||||
val health: HealthConfig = HealthConfig(),
|
||||
)
|
||||
|
||||
/**
|
||||
* Sampling knobs sent to the llama-server on every *stage* inference request (the main agentic
|
||||
* loop). [temperature] and [topP] default to the former hardcoded stage values. [topK], [minP] and
|
||||
* [repeatPenalty] are null by default, meaning the request omits them and the model keeps its own
|
||||
* default — set them to tighten a rambling local model. maxTokens is not here: it is pinned per-stage
|
||||
* to the token budget. Talkie chat/narration have their own [GenerationSettings]/[NarrationSettings].
|
||||
*/
|
||||
@Serializable
|
||||
data class SamplingConfig(
|
||||
val temperature: Double = 0.7,
|
||||
val topP: Double = 1.0,
|
||||
val topK: Int? = null,
|
||||
val minP: Double? = null,
|
||||
val repeatPenalty: Double? = null,
|
||||
)
|
||||
|
||||
/**
|
||||
* Continuous health watch (observability-spec §4). When [enabled], a background monitor polls
|
||||
* the on-disk footprint every [intervalMs] and records a degraded/restored event on the edge.
|
||||
|
||||
@@ -102,6 +102,13 @@ object CorrexConfigWriter {
|
||||
b.kv("recovery_route_budget", cfg.orchestration.recoveryRouteBudget)
|
||||
b.kv("intent_route_budget", cfg.orchestration.intentRouteBudget)
|
||||
|
||||
b.section("sampling")
|
||||
b.kv("temperature", cfg.sampling.temperature)
|
||||
b.kv("top_p", cfg.sampling.topP)
|
||||
cfg.sampling.topK?.let { b.kv("top_k", it) }
|
||||
cfg.sampling.minP?.let { b.kv("min_p", it) }
|
||||
cfg.sampling.repeatPenalty?.let { b.kv("repeat_penalty", it) }
|
||||
|
||||
b.section("personalization")
|
||||
b.kv("enabled", cfg.personalization.enabled)
|
||||
b.kv("learn", cfg.personalization.learn)
|
||||
|
||||
@@ -13,4 +13,9 @@ data class GenerationConfig(
|
||||
val maxTokens: Int,
|
||||
val stopSequences: List<String> = emptyList(),
|
||||
val seed: Long? = null, // null = non-deterministic; set for replay
|
||||
// Sampling knobs. null = omit from the request so the provider/model keeps its own default,
|
||||
// preserving prior behavior. Serialized only when set (top_k / min_p / repeat_penalty).
|
||||
val topK: Int? = null,
|
||||
val minP: Double? = null,
|
||||
val repeatPenalty: Double? = null,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user