package update import ( "context" "errors" "fmt" "time" "github.com/kami/maven/internal/ipc" ) // The health check is the whole basis for rolling back, so it has to mean // something. "The process is running" does not: mavend can be up with a dead // store, a socket it never bound, or a config it failed to parse. What is // checked instead is that she answers a real read over the real IPC socket — // which exercises the socket, the dispatch table and the store in one call. // // Presence is the method used because it is read-only (safe to retry), needs no // arguments, and touches the store. It cannot write anything, so a health check // never leaves a trace in her memory. // // A locked daemon is healthy. With a passkey enrolled and no env key mavend // boots locked and refuses every CoreAPI method until an assertion arrives, so // a Presence read there fails for a daemon that came up perfectly. Rolling back // on that would turn a good update into the manual-recovery case, and the // rollback would boot locked too. MethodPing reaches no store and answers in // locked mode, so it is asked first: an answer of "locked" is proof of life and // is where the check stops. // ErrHealthDial — the socket could not be opened at all. Kept apart from a // failed read because the two have different causes and different fixes: on the // docker deployment the socket lives under /var/lib/docker, which is // drwx--x--- root root, so a non-root operator gets EACCES before reaching // mavend. "She is not answering" would be the wrong thing to tell him. var ErrHealthDial = errors.New("update: cannot open the health socket") // DialHealth connects to the mavend socket and proves someone is serving it. func DialHealth(ctx context.Context, socket string) error { c, err := ipc.Dial(socket) if err != nil { return fmt.Errorf("%w %s: %v", ErrHealthDial, socket, err) } defer c.Close() // Liveness first, because it is the only question a locked daemon can // answer. ErrUnknownMethod means an older mavend on the other end, which is // exactly the case during a rollback to a build from before ping existed — // fall through to the store read rather than calling that a failure. switch p, perr := c.Ping(ctx); { case perr == nil && p.Locked: return nil case perr != nil && !errors.Is(perr, ipc.ErrUnknownMethod): return fmt.Errorf("update: health ping: %w", perr) } if _, err := c.Presence(ctx); err != nil { return fmt.Errorf("update: health read: %w", err) } return nil } // waitHealthy retries the health check until it passes or the timeout elapses. // A restart is not instantaneous — she loads a 1.7B on boot — so the first few // failures are expected and are not a reason to roll back. func (u *Updater) waitHealthy(ctx context.Context, timeout time.Duration) error { deadline := u.now().Add(timeout) delay := 500 * time.Millisecond var last error for { // Cap the attempt at whatever is left of the budget, not a flat 10s: an // attempt starting at 89s of a 90s timeout would otherwise run to 99s, // and the caller asked for 90. attempt := 10 * time.Second if left := deadline.Sub(u.now()); left < attempt { attempt = left } if attempt <= 0 { if last == nil { last = context.DeadlineExceeded } return fmt.Errorf("update: not healthy after %s: %w", timeout, last) } attemptCtx, cancel := context.WithTimeout(ctx, attempt) err := u.health(attemptCtx, u.cfg.HealthSocket) cancel() if err == nil { return nil } last = err if u.now().After(deadline) { return fmt.Errorf("update: not healthy after %s: %w", timeout, last) } select { case <-ctx.Done(): return ctx.Err() case <-time.After(delay): } if delay < 5*time.Second { delay *= 2 } } }