package update import ( "context" "errors" "fmt" "os" "path/filepath" "strings" "time" ) // Result — the full account of one Apply. Every field is filled in on the // failure paths too, because "what state is my box in" is the only question that // matters after a failed update. type Result struct { Verified bool SnapshotID string // the rollback target; named even when the rollback failed Installed []string Restarted bool Healthy bool RolledBack bool // RollbackHealthy — whether she answered again after the restore. False with // RolledBack true is the manual-recovery case. RollbackHealthy bool Steps []Step Took time.Duration } // Apply is the whole update, in the only order that is safe. // // It is called by a human running cmd/mavupdate on the box. Nothing else calls // it: no timer, no IPC method, no web route, no act. See the package comment. func (u *Updater) Apply(ctx context.Context) (Result, error) { start := u.now() res := Result{} defer func() { res.Took = u.now().Sub(start) }() // 0. She has to be answering before we start. Otherwise a failed update and // a box that was already broken look identical afterwards, and the rollback // has no baseline to prove itself against. u.log("preflight: checking the running daemon") if err := u.health(ctx, u.cfg.HealthSocket); err != nil { return res, fmt.Errorf("%w: %v", ErrUnhealthyBefore, err) } // 1. Snapshot what is deployed now, BEFORE the build. // // The order matters and it is not the obvious one. `make build` writes its // binaries into the working tree, and on the docker deployment the working // tree IS the install dir — so snapshotting after the build would snapshot // the new artifacts and leave nothing to roll back to. The snapshot is the // only thing standing between a bad build and a box that needs a screwdriver, // so it is taken first, while the deployed bytes are still the old ones. names := append(append([]string{}, u.cfg.Binaries...), u.cfg.ConfigFiles...) snap, err := u.store.Save(u.cfg.InstallDir, names, u.gitHead(ctx), "pre-update") if err != nil { return res, err } res.SnapshotID = snap.ID u.log("snapshot: %s (%d files) in %s", snap.ID, len(snap.Files), snap.Dir()) // 2. Build and test before anything is deployed. A broken tree costs time // and nothing else — but `make build` has already overwritten the binaries in // the tree, so restore them: otherwise a later restart by hand would deploy // code that failed its own tests. Nothing has been restarted, so this is a // file restore with no restart and no health check. steps, err := u.Verify(ctx) res.Steps = append(res.Steps, steps...) if err != nil { if rerr := snap.Restore(u.cfg.InstallDir); rerr != nil { u.log("verify failed and the artifacts could not be put back: %v — the previous ones are in %s", rerr, snap.Dir()) } else { res.RolledBack = true u.log("verify failed; the previously deployed artifacts are back in place, she was never restarted") } return res, err } res.Verified = true // 3. Install. Per-file temp+rename, so an interruption leaves whole files. // Config is snapshotted but never overwritten — an update does not get to // replace the operator's config. installed, err := u.install() res.Installed = installed if err != nil { // Files may be half-swapped across the set, so restore before returning // even though nothing has been restarted yet. u.log("install failed: %v — restoring", err) return u.rollback(ctx, snap, res, err) } u.log("install: %d artifact(s) into %s", len(installed), u.cfg.InstallDir) // 4. Restart, then 5. prove she answers. if err := u.restart(ctx, &res); err != nil { return u.rollback(ctx, snap, res, err) } u.log("restart: ok, waiting for her to answer (up to %s)", u.cfg.healthTimeout()) if err := u.waitHealthy(ctx, u.cfg.healthTimeout()); err != nil { return u.rollback(ctx, snap, res, err) } res.Healthy = true u.log("health: she answers on %s — update committed", u.cfg.HealthSocket) if err := u.store.Prune(u.cfg.KeepSnapshots); err != nil { u.log("prune: %v (harmless)", err) } return res, nil } // Rollback restores a snapshot by id (empty = the newest) and restarts. Exposed // separately so the operator can undo an update that verified, restarted and // answered a Presence call but is wrong in a way no health check can see. func (u *Updater) Rollback(ctx context.Context, id string) (Result, error) { var snap Snapshot var err error if id == "" { snaps, lerr := u.store.List() if lerr != nil { return Result{}, lerr } if len(snaps) == 0 { return Result{}, errors.New("update: no snapshots to roll back to") } snap = snaps[0] } else if snap, err = u.store.Load(id); err != nil { return Result{}, err } res := Result{SnapshotID: snap.ID} return u.rollback(ctx, snap, res, errors.New("operator asked for a rollback")) } // rollback restores the snapshot and restarts, then reports whether that worked. // It depends on nothing that the update changed: file copies out of the snapshot // dir and the same restart command. No build, no migration, no cooperation from // the code being replaced. func (u *Updater) rollback(ctx context.Context, snap Snapshot, res Result, cause error) (Result, error) { // A rollback interrupted halfway is the one outcome worse than the failure // that triggered it, so it does not inherit the caller's cancellation: a // Ctrl-C during the health wait must not abandon the restore mid-restart. ctx = context.WithoutCancel(ctx) res.RolledBack = true u.log("rollback: restoring snapshot %s over %s", snap.ID, u.cfg.InstallDir) if err := snap.Restore(u.cfg.InstallDir); err != nil { u.log("rollback: RESTORE FAILED: %v", err) return res, fmt.Errorf("%w: %v (after %v); the previous artifacts are in %s — copy them back by hand", ErrRollbackFailed, err, cause, snap.Dir()) } // A restore with no restart leaves the failed process running, so a failed // restart here is still the manual-recovery case. if err := u.restart(ctx, &res); err != nil { u.log("rollback: RESTART FAILED: %v", err) return res, fmt.Errorf("%w: restored %s but the restart failed: %v (after %v)", ErrRollbackFailed, snap.ID, err, cause) } if err := u.waitHealthy(ctx, u.cfg.healthTimeout()); err != nil { u.log("rollback: she still does not answer: %v", err) return res, fmt.Errorf("%w: restored %s and restarted but she does not answer: %v (after %v)", ErrRollbackFailed, snap.ID, err, cause) } res.RollbackHealthy = true u.log("rollback: she answers again on the previous build (%s)", snap.ID) return res, fmt.Errorf("%w to %s: %v", ErrRolledBack, snap.ID, cause) } // install copies the freshly built binaries from SourceDir into InstallDir. // // When the two are the same directory — the docker deployment builds the image // from the working tree — this is a no-op by design rather than by accident: the // artifacts are already where they belong and the restart command rebuilds the // image from them. func (u *Updater) install() ([]string, error) { if filepath.Clean(u.cfg.SourceDir) == filepath.Clean(u.cfg.InstallDir) { return u.cfg.Binaries, nil } var done []string for _, name := range u.cfg.Binaries { src := filepath.Join(u.cfg.SourceDir, name) fi, err := os.Stat(src) if err != nil { return done, fmt.Errorf("update: install %s: %w (did `make build` produce it?)", name, err) } if _, err := copyFile(src, filepath.Join(u.cfg.InstallDir, name), fi.Mode().Perm()); err != nil { return done, fmt.Errorf("update: install %s: %w", name, err) } done = append(done, name) } return done, nil } func (u *Updater) restart(ctx context.Context, res *Result) error { u.log("restart: %v", u.cfg.RestartCmd) out, err := u.run(ctx, u.cfg.SourceDir, u.cfg.RestartCmd) if err != nil { res.Steps = append(res.Steps, Step{Name: "restart", Argv: u.cfg.RestartCmd, Err: err, Output: tail(out, 4000)}) return fmt.Errorf("update: restart %v: %w", u.cfg.RestartCmd, err) } res.Restarted = true res.Steps = append(res.Steps, Step{Name: "restart", Argv: u.cfg.RestartCmd}) return nil } // gitHead records which commit produced a snapshot, for the operator's benefit. // Best-effort: a tree without git is not a reason to refuse to snapshot. func (u *Updater) gitHead(ctx context.Context) string { out, err := u.run(ctx, u.cfg.SourceDir, []string{"git", "rev-parse", "HEAD"}) if err != nil { return "" } return strings.TrimSpace(out) }