kami
810076451f
On the deployment deploy/README.md documents, source_dir and install_dir are
the same tree and the restart command rebuilds the image from it. The
Dockerfile builds from cmd/ and internal/ and .dockerignore keeps the host
binaries out, so restoring the snapshotted binaries restored bytes nothing
reads. A bad commit therefore cost two health timeouts and two image builds
and ended in ErrRollbackFailed with an instruction to copy files back by hand,
which would not have helped either.
A deployment that rebuilds from source now has to say how the source is put
back. source_rollback "git" records the commit before the update and checks it
back out before the rollback restart. It refuses a dirty tree, because the
recorded commit does not describe one and a forced checkout would delete his
work. A build-from-source config that says nothing is refused by Validate, at
startup, rather than at the one rollback that mattered.
Also in this change, all from the same review:
- MethodPing, the one method a locked daemon answers. Preflight passed on an
unlocked daemon and the post-restart Presence read failed on a locked one,
so a good update read as SHE IS PROBABLY DOWN once the env key is gone.
- A dial failure is reported apart from a read failure. The documented
socket is under /var/lib/docker, which a non-root operator cannot
traverse, and "she is not answering" was the wrong diagnosis.
- Verify refuses to run as root over a tree owned by someone else. It runs
make build and make test in place, and root-owned artifacts break his next
ordinary make.
- A rollback no longer reverts config_files. That undid every config edit
since the last apply, phraser.model_path among them.
- The verify-failure path no longer reports rolled_back for a compile error.
- waitHealthy caps each attempt at the remaining budget, so a 90s timeout
cannot run to 99s.
- tail cuts on a rune boundary. Russian test names showed the seam.
- The claim that mavend does not import internal/update is replaced with
what is enforced: mavend constructs no Updater and nothing can call Apply.
- snapshot_dir inside source_dir is refused. It landed in the build context.
Found in review of #69.
101 lines
3.7 KiB
Go
101 lines
3.7 KiB
Go
package update
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/kami/maven/internal/ipc"
|
|
)
|
|
|
|
// The health check is the whole basis for rolling back, so it has to mean
|
|
// something. "The process is running" does not: mavend can be up with a dead
|
|
// store, a socket it never bound, or a config it failed to parse. What is
|
|
// checked instead is that she answers a real read over the real IPC socket —
|
|
// which exercises the socket, the dispatch table and the store in one call.
|
|
//
|
|
// Presence is the method used because it is read-only (safe to retry), needs no
|
|
// arguments, and touches the store. It cannot write anything, so a health check
|
|
// never leaves a trace in her memory.
|
|
//
|
|
// A locked daemon is healthy. With a passkey enrolled and no env key mavend
|
|
// boots locked and refuses every CoreAPI method until an assertion arrives, so
|
|
// a Presence read there fails for a daemon that came up perfectly. Rolling back
|
|
// on that would turn a good update into the manual-recovery case, and the
|
|
// rollback would boot locked too. MethodPing reaches no store and answers in
|
|
// locked mode, so it is asked first: an answer of "locked" is proof of life and
|
|
// is where the check stops.
|
|
|
|
// ErrHealthDial — the socket could not be opened at all. Kept apart from a
|
|
// failed read because the two have different causes and different fixes: on the
|
|
// docker deployment the socket lives under /var/lib/docker, which is
|
|
// drwx--x--- root root, so a non-root operator gets EACCES before reaching
|
|
// mavend. "She is not answering" would be the wrong thing to tell him.
|
|
var ErrHealthDial = errors.New("update: cannot open the health socket")
|
|
|
|
// DialHealth connects to the mavend socket and proves someone is serving it.
|
|
func DialHealth(ctx context.Context, socket string) error {
|
|
c, err := ipc.Dial(socket)
|
|
if err != nil {
|
|
return fmt.Errorf("%w %s: %v", ErrHealthDial, socket, err)
|
|
}
|
|
defer c.Close()
|
|
// Liveness first, because it is the only question a locked daemon can
|
|
// answer. ErrUnknownMethod means an older mavend on the other end, which is
|
|
// exactly the case during a rollback to a build from before ping existed —
|
|
// fall through to the store read rather than calling that a failure.
|
|
switch p, perr := c.Ping(ctx); {
|
|
case perr == nil && p.Locked:
|
|
return nil
|
|
case perr != nil && !errors.Is(perr, ipc.ErrUnknownMethod):
|
|
return fmt.Errorf("update: health ping: %w", perr)
|
|
}
|
|
if _, err := c.Presence(ctx); err != nil {
|
|
return fmt.Errorf("update: health read: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// waitHealthy retries the health check until it passes or the timeout elapses.
|
|
// A restart is not instantaneous — she loads a 1.7B on boot — so the first few
|
|
// failures are expected and are not a reason to roll back.
|
|
func (u *Updater) waitHealthy(ctx context.Context, timeout time.Duration) error {
|
|
deadline := u.now().Add(timeout)
|
|
delay := 500 * time.Millisecond
|
|
var last error
|
|
for {
|
|
// Cap the attempt at whatever is left of the budget, not a flat 10s: an
|
|
// attempt starting at 89s of a 90s timeout would otherwise run to 99s,
|
|
// and the caller asked for 90.
|
|
attempt := 10 * time.Second
|
|
if left := deadline.Sub(u.now()); left < attempt {
|
|
attempt = left
|
|
}
|
|
if attempt <= 0 {
|
|
if last == nil {
|
|
last = context.DeadlineExceeded
|
|
}
|
|
return fmt.Errorf("update: not healthy after %s: %w", timeout, last)
|
|
}
|
|
attemptCtx, cancel := context.WithTimeout(ctx, attempt)
|
|
err := u.health(attemptCtx, u.cfg.HealthSocket)
|
|
cancel()
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
last = err
|
|
if u.now().After(deadline) {
|
|
return fmt.Errorf("update: not healthy after %s: %w", timeout, last)
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-time.After(delay):
|
|
}
|
|
if delay < 5*time.Second {
|
|
delay *= 2
|
|
}
|
|
}
|
|
}
|