claude
1456336652
serve.py lived only on workpc, which was fine while systemd launched it and is
not fine now that mavgpud does. Two endpoints and no framework: /health answers
503 until the model is loaded, /transcribe takes raw PCM and returns
{"text","confidence"}.
The unit carries CW2_TOKEN through EnvironmentFile and the child inherits it,
so the token is never a flag value.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013ptwopxyo3Z2kwFckHkLvN
46 lines
1.3 KiB
JSON
46 lines
1.3 KiB
JSON
{
|
|
"listen": ":8080",
|
|
"llama_addr": "127.0.0.1:10000",
|
|
"llama_bin": "llama-server",
|
|
"llama_args": [
|
|
"-m", "/mnt/D/AI/gemma4/gemma-4-12B-it-qat-UD-Q4_K_XL.gguf",
|
|
"-md", "/mnt/D/AI/gemma4/mtp-gemma-4-12B-it-BF16.gguf",
|
|
"-ngl", "99",
|
|
"-fa", "on",
|
|
"-np", "1",
|
|
"--host", "127.0.0.1",
|
|
"--port", "10000",
|
|
"--ctx-size", "32768",
|
|
"--threads", "6",
|
|
"--batch-size", "2048",
|
|
"--ubatch-size", "512",
|
|
"--jinja",
|
|
"--chat-template-kwargs", "{\"enable_thinking\":false}",
|
|
"--spec-type", "draft-mtp",
|
|
"--spec-draft-n-max", "2"
|
|
],
|
|
|
|
"//stt": [
|
|
"CrisperWhisper 2.0 turbo, which Maven reaches directly on port 8081.",
|
|
"mavgpud runs it because it is a ROCm process on this card: under its own",
|
|
"systemd unit it registered on the KFD and the supervisor evicted",
|
|
"llama-server every few seconds. CW2_TOKEN comes from the unit's",
|
|
"EnvironmentFile and is never a flag value."
|
|
],
|
|
"stt": {
|
|
"addr": "127.0.0.1:8081",
|
|
"bin": "/home/kami/Programs/cw2-eval/.venv/bin/python",
|
|
"args": ["/home/kami/Programs/cw2-service/serve.py"]
|
|
},
|
|
|
|
"kfd_root": "/sys/class/kfd/kfd/proc",
|
|
"drm_device": "/sys/class/drm/card1/device",
|
|
|
|
"poll": "1s",
|
|
"idle_timeout": "15m",
|
|
"stop_grace": "20s",
|
|
"min_free_vram_bytes": 10737418240,
|
|
"evict_after_polls": 2,
|
|
"start_after_polls": 5
|
|
}
|