phraser: gate every llm.Client call on the swap drain
The drain counted only the phrasing paths in internal/phraser. The router, the replier, the mail extractor and the memory evaluator reach llama-server through llm.Client, so quiesce could report zero requests in flight while the router was mid-generation, and the old server was killed under it. The turn then finished on the new model, which is the split turn the swap exists to prevent. llm.Client now enters an optional Gate before every completion and LLMPhraser implements it, so one counter covers every holder of the base URL. A total failure also reported itself as a rollback. Swap set RolledBack on the path where the rollback failed too, so the page rendered "rolled back to — she is still answering, with the old model" over an empty model name and a daemon with no model at all. The total failure has its own flag now, LiveModel stops naming a gguf that is not loaded, and the log says another attempt can recover without a restart, which is true. The swap also ran on the connection every other page shares. ipc.Client holds its mutex for a whole roundtrip with no read deadline on either side, so a load froze /dash, /history and /notifications for minutes. mavweb dials a second connection for /models alone. POST /models joins the route table, and the load settings no longer come off a form that renders no input for them. Found in review of #68.
This commit is contained in:
+23
-7
@@ -36,6 +36,7 @@ var modelsTmpl = template.Must(template.New("models").Funcs(shellFuncs()).Parse(
|
||||
const modelsHTML = `{{template "shellTop" "models"}}
|
||||
<h1>Resident model</h1>
|
||||
<p class=hint>swapping requires step-up — <a href=/auth/passkey>assert a passkey</a> first. The old model is unloaded before the new one is loaded (one model fits the iGPU at a time), so turns during the load are refused and fall back to the classifier.</p>
|
||||
<p class=hint>a swap is not remembered. Nothing writes it down, so the next restart of the daemon — including the one <code>mavupdate</code> does — comes back on <code>phraser.model_path</code> from the config. Make it stick by editing that.</p>
|
||||
{{if .Msg}}<div class="msg msg-ok">{{.Msg}}</div>{{end}}
|
||||
{{if .Err}}<div class="msg msg-err">{{.Err}}</div>{{end}}
|
||||
{{if .Off}}
|
||||
@@ -80,12 +81,22 @@ type modelsPage struct {
|
||||
// A failed swap is reported as a failure with the model that is still serving
|
||||
// named, because that is the state the operator needs: the daemon rolled back
|
||||
// and is answering turns, it just is not answering them with what he asked for.
|
||||
func handleModels(w http.ResponseWriter, r *http.Request, core ipc.CoreAPI, session *webauthn.PasskeySession, requireStepUp bool) {
|
||||
// swapConn, when non-nil, is a SECOND connection to the same core, used for
|
||||
// nothing but this page. ipc.Client holds its mutex for a whole roundtrip and
|
||||
// neither side sets a read deadline, so a swap on the shared connection blocks
|
||||
// /dash, /history, /notifications and everything else for as long as the load
|
||||
// takes: a 90s drain plus a 60s launch plus a 30s probe, doubled if it rolls
|
||||
// back. No browser timeout frees them, because the server side keeps reading
|
||||
// the reply. On its own connection the swap only blocks the swap.
|
||||
func handleModels(w http.ResponseWriter, r *http.Request, core ipc.CoreAPI, swapConn modelController, session *webauthn.PasskeySession, requireStepUp bool) {
|
||||
if core == nil {
|
||||
http.Error(w, "models disabled (no -core)", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
mc, ok := core.(modelController)
|
||||
mc, ok := swapConn, swapConn != nil
|
||||
if !ok {
|
||||
mc, ok = core.(modelController)
|
||||
}
|
||||
if !ok {
|
||||
http.Error(w, "models unavailable: core connection does not support model swap", http.StatusServiceUnavailable)
|
||||
return
|
||||
@@ -103,11 +114,13 @@ func handleModels(w http.ResponseWriter, r *http.Request, core ipc.CoreAPI, sess
|
||||
http.Error(w, "model_path required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
req := ipc.SwapModelReq{ModelPath: path}
|
||||
if v, err := strconv.Atoi(r.FormValue("n_ctx")); err == nil {
|
||||
req.NCtx = v
|
||||
}
|
||||
res, err := mc.SwapModel(ctx, req)
|
||||
// Only the path comes off the form. n_ctx and n_gpu_layers are load
|
||||
// settings the daemon keeps from what is live, and the resident model is
|
||||
// a Thinking variant whose 4096-token window is sized for reasoning
|
||||
// tokens (CLAUDE.md). A field nothing renders, that a hand-crafted POST
|
||||
// could use to shrink the window under the router, is not worth having.
|
||||
// Changing them is a config edit and a restart.
|
||||
res, err := mc.SwapModel(ctx, ipc.SwapModelReq{ModelPath: path})
|
||||
switch {
|
||||
case err == nil:
|
||||
page.Msg = "loaded " + res.Model + " (" + strconv.FormatInt(res.TookMs, 10) + "ms)"
|
||||
@@ -118,6 +131,9 @@ func handleModels(w http.ResponseWriter, r *http.Request, core ipc.CoreAPI, sess
|
||||
case errors.Is(err, ipc.ErrUnknownMethod):
|
||||
http.Error(w, "swap not configured on this core", http.StatusServiceUnavailable)
|
||||
return
|
||||
case res.NoBackend:
|
||||
page.Err = "swap failed AND the rollback failed — no model is loaded. She is answering from templates and routing on the classifier. Try loading a model again; a restart is not needed."
|
||||
log.Printf("models: swap to %s failed and the rollback failed, no model loaded: %v", path, err)
|
||||
case res.RolledBack:
|
||||
page.Err = "swap failed, rolled back to " + res.Model + " — she is still answering, with the old model"
|
||||
log.Printf("models: swap to %s failed, rolled back: %v", path, err)
|
||||
|
||||
Reference in New Issue
Block a user