feat(inference): pass [[models]] params through to llama-server (enables draft-MTP) (#243)
ModelConfig.params was loaded from config but never consumed — the spawn
command in DefaultModelManager was hardcoded. Thread it through: params (a
flag->value map) flattens to token order on ModelDescriptor.extraArgs and
appends to the llama-server argv. This engages Multi-Token Prediction
speculative decoding purely from config:
[[models]]
params = { "-md" = "/path/draft.gguf", "-ngld" = "99", "--spec-type" = "draft-mtp", "--spec-draft-n-max" = "4" }
Each entry stays a distinct argv token (no shell), so paths with spaces are safe.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
+3
@@ -12,4 +12,7 @@ data class ModelDescriptor(
|
||||
val idleTimeoutMs: Long = 60_000L,
|
||||
val contextSize: Int = 24_576,
|
||||
val capabilities: Set<CapabilityScore> = emptySet(),
|
||||
/** Extra llama-server CLI args (already flattened to token order), e.g. draft-MTP speculative
|
||||
* decoding: `-md <draft.gguf> -ngld 99 --spec-type draft-mtp --spec-draft-n-max 4`. */
|
||||
val extraArgs: List<String> = emptyList(),
|
||||
)
|
||||
|
||||
+1
-1
@@ -66,7 +66,7 @@ class DefaultModelManager(
|
||||
"--ctx-size", descriptor.contextSize.toString(),
|
||||
"--host", host,
|
||||
"--port", port.toString(),
|
||||
)
|
||||
) + descriptor.extraArgs
|
||||
val lp = LlamaProcess(command, logFile)
|
||||
lp.start()
|
||||
lp
|
||||
|
||||
Reference in New Issue
Block a user