feat(inference): pass [[models]] params through to llama-server (enables draft-MTP) (#243)
ModelConfig.params was loaded from config but never consumed — the spawn
command in DefaultModelManager was hardcoded. Thread it through: params (a
flag->value map) flattens to token order on ModelDescriptor.extraArgs and
appends to the llama-server argv. This engages Multi-Token Prediction
speculative decoding purely from config:
[[models]]
params = { "-md" = "/path/draft.gguf", "-ngld" = "99", "--spec-type" = "draft-mtp", "--spec-draft-n-max" = "4" }
Each entry stays a distinct argv token (no shell), so paths with spaces are safe.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -170,6 +170,10 @@ object InfrastructureModule {
|
||||
residencyMode = ResidencyMode.PERSISTENT,
|
||||
contextSize = config.contextSize,
|
||||
capabilities = parseCapabilitiesFromConfig(config.capabilities),
|
||||
// [[models]] `params` (a flag->value map) passes straight through to llama-server, e.g.
|
||||
// draft-MTP speculative decoding. Flatten to token order; each stays a distinct argv entry
|
||||
// (no shell), so paths with spaces are safe.
|
||||
extraArgs = config.params.flatMap { (flag, value) -> listOf(flag, value) },
|
||||
)
|
||||
|
||||
private fun parseCapabilitiesFromConfig(capsMap: Map<String, Double>): Set<CapabilityScore> {
|
||||
|
||||
+21
@@ -42,6 +42,27 @@ class InfrastructureModuleModelTest {
|
||||
assertEquals(0.8, codingScore!!.score)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `modelConfigToDescriptor flattens params into extraArgs for llama-server`() {
|
||||
val config = ModelConfig(
|
||||
id = "mtp-model",
|
||||
modelPath = "/models/main.gguf",
|
||||
params = linkedMapOf(
|
||||
"-md" to "/models/draft.gguf",
|
||||
"-ngld" to "99",
|
||||
"--spec-type" to "draft-mtp",
|
||||
"--spec-draft-n-max" to "4",
|
||||
),
|
||||
)
|
||||
|
||||
val descriptor = InfrastructureModule.modelConfigToDescriptor(config)
|
||||
|
||||
assertEquals(
|
||||
listOf("-md", "/models/draft.gguf", "-ngld", "99", "--spec-type", "draft-mtp", "--spec-draft-n-max", "4"),
|
||||
descriptor.extraArgs,
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `modelConfigToDescriptor handles empty capabilities`() {
|
||||
val config = ModelConfig(
|
||||
|
||||
Reference in New Issue
Block a user