Maven/internal/update/apply.go

package update

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"time"
)

// Result — the full account of one Apply. Every field is filled in on the
// failure paths too, because "what state is my box in" is the only question that
// matters after a failed update.
type Result struct {
	Verified   bool
	SnapshotID string // the rollback target; named even when the rollback failed
	Installed  []string
	Restarted  bool
	Healthy    bool
	RolledBack bool
	// RollbackHealthy — whether she answered again after the restore. False with
	// RolledBack true is the manual-recovery case.
	RollbackHealthy bool
	Steps           []Step
	Took            time.Duration
}

// Apply is the whole update, in the only order that is safe.
//
// It is called by a human running cmd/mavupdate on the box. Nothing else calls
// it: no timer, no IPC method, no web route, no act. See the package comment.
func (u *Updater) Apply(ctx context.Context) (Result, error) {
	start := u.now()
	res := Result{}
	defer func() { res.Took = u.now().Sub(start) }()

	// 0. She has to be answering before we start. Otherwise a failed update and
	// a box that was already broken look identical afterwards, and the rollback
	// has no baseline to prove itself against.
	u.log("preflight: checking the running daemon")
	if err := u.health(ctx, u.cfg.HealthSocket); err != nil {
		return res, fmt.Errorf("%w: %v", ErrUnhealthyBefore, err)
	}

	// 1. Snapshot what is deployed now, BEFORE the build.
	//
	// The order matters and it is not the obvious one. `make build` writes its
	// binaries into the working tree, and on the docker deployment the working
	// tree IS the install dir — so snapshotting after the build would snapshot
	// the new artifacts and leave nothing to roll back to. The snapshot is the
	// only thing standing between a bad build and a box that needs a screwdriver,
	// so it is taken first, while the deployed bytes are still the old ones.
	names := append(append([]string{}, u.cfg.Binaries...), u.cfg.ConfigFiles...)
	snap, err := u.store.Save(u.cfg.InstallDir, names, u.gitHead(ctx), "pre-update")
	if err != nil {
		return res, err
	}
	res.SnapshotID = snap.ID
	u.log("snapshot: %s (%d files) in %s", snap.ID, len(snap.Files), snap.Dir())

	// 2. Build and test before anything is deployed. A broken tree costs time
	// and nothing else — but `make build` has already overwritten the binaries in
	// the tree, so restore them: otherwise a later restart by hand would deploy
	// code that failed its own tests. Nothing has been restarted, so this is a
	// file restore with no restart and no health check.
	steps, err := u.Verify(ctx)
	res.Steps = append(res.Steps, steps...)
	if err != nil {
		if rerr := snap.Restore(u.cfg.InstallDir); rerr != nil {
			u.log("verify failed and the artifacts could not be put back: %v — the previous ones are in %s", rerr, snap.Dir())
		} else {
			res.RolledBack = true
			u.log("verify failed; the previously deployed artifacts are back in place, she was never restarted")
		}
		return res, err
	}
	res.Verified = true

	// 3. Install. Per-file temp+rename, so an interruption leaves whole files.
	// Config is snapshotted but never overwritten — an update does not get to
	// replace the operator's config.
	installed, err := u.install()
	res.Installed = installed
	if err != nil {
		// Files may be half-swapped across the set, so restore before returning
		// even though nothing has been restarted yet.
		u.log("install failed: %v — restoring", err)
		return u.rollback(ctx, snap, res, err)
	}
	u.log("install: %d artifact(s) into %s", len(installed), u.cfg.InstallDir)

	// 4. Restart, then 5. prove she answers.
	if err := u.restart(ctx, &res); err != nil {
		return u.rollback(ctx, snap, res, err)
	}
	u.log("restart: ok, waiting for her to answer (up to %s)", u.cfg.healthTimeout())
	if err := u.waitHealthy(ctx, u.cfg.healthTimeout()); err != nil {
		return u.rollback(ctx, snap, res, err)
	}
	res.Healthy = true
	u.log("health: she answers on %s — update committed", u.cfg.HealthSocket)

	if err := u.store.Prune(u.cfg.KeepSnapshots); err != nil {
		u.log("prune: %v (harmless)", err)
	}
	return res, nil
}

// Rollback restores a snapshot by id (empty = the newest) and restarts. Exposed
// separately so the operator can undo an update that verified, restarted and
// answered a Presence call but is wrong in a way no health check can see.
func (u *Updater) Rollback(ctx context.Context, id string) (Result, error) {
	var snap Snapshot
	var err error
	if id == "" {
		snaps, lerr := u.store.List()
		if lerr != nil {
			return Result{}, lerr
		}
		if len(snaps) == 0 {
			return Result{}, errors.New("update: no snapshots to roll back to")
		}
		snap = snaps[0]
	} else if snap, err = u.store.Load(id); err != nil {
		return Result{}, err
	}
	res := Result{SnapshotID: snap.ID}
	return u.rollback(ctx, snap, res, errors.New("operator asked for a rollback"))
}

// rollback restores the snapshot and restarts, then reports whether that worked.
// It depends on nothing that the update changed: file copies out of the snapshot
// dir and the same restart command. No build, no migration, no cooperation from
// the code being replaced.
func (u *Updater) rollback(ctx context.Context, snap Snapshot, res Result, cause error) (Result, error) {
	// A rollback interrupted halfway is the one outcome worse than the failure
	// that triggered it, so it does not inherit the caller's cancellation: a
	// Ctrl-C during the health wait must not abandon the restore mid-restart.
	ctx = context.WithoutCancel(ctx)
	res.RolledBack = true
	u.log("rollback: restoring snapshot %s over %s", snap.ID, u.cfg.InstallDir)
	if err := snap.Restore(u.cfg.InstallDir); err != nil {
		u.log("rollback: RESTORE FAILED: %v", err)
		return res, fmt.Errorf("%w: %v (after %v); the previous artifacts are in %s — copy them back by hand", ErrRollbackFailed, err, cause, snap.Dir())
	}
	// A restore with no restart leaves the failed process running, so a failed
	// restart here is still the manual-recovery case.
	if err := u.restart(ctx, &res); err != nil {
		u.log("rollback: RESTART FAILED: %v", err)
		return res, fmt.Errorf("%w: restored %s but the restart failed: %v (after %v)", ErrRollbackFailed, snap.ID, err, cause)
	}
	if err := u.waitHealthy(ctx, u.cfg.healthTimeout()); err != nil {
		u.log("rollback: she still does not answer: %v", err)
		return res, fmt.Errorf("%w: restored %s and restarted but she does not answer: %v (after %v)", ErrRollbackFailed, snap.ID, err, cause)
	}
	res.RollbackHealthy = true
	u.log("rollback: she answers again on the previous build (%s)", snap.ID)
	return res, fmt.Errorf("%w to %s: %v", ErrRolledBack, snap.ID, cause)
}

// install copies the freshly built binaries from SourceDir into InstallDir.
//
// When the two are the same directory — the docker deployment builds the image
// from the working tree — this is a no-op by design rather than by accident: the
// artifacts are already where they belong and the restart command rebuilds the
// image from them.
func (u *Updater) install() ([]string, error) {
	if filepath.Clean(u.cfg.SourceDir) == filepath.Clean(u.cfg.InstallDir) {
		return u.cfg.Binaries, nil
	}
	var done []string
	for _, name := range u.cfg.Binaries {
		src := filepath.Join(u.cfg.SourceDir, name)
		fi, err := os.Stat(src)
		if err != nil {
			return done, fmt.Errorf("update: install %s: %w (did `make build` produce it?)", name, err)
		}
		if _, err := copyFile(src, filepath.Join(u.cfg.InstallDir, name), fi.Mode().Perm()); err != nil {
			return done, fmt.Errorf("update: install %s: %w", name, err)
		}
		done = append(done, name)
	}
	return done, nil
}

func (u *Updater) restart(ctx context.Context, res *Result) error {
	u.log("restart: %v", u.cfg.RestartCmd)
	out, err := u.run(ctx, u.cfg.SourceDir, u.cfg.RestartCmd)
	if err != nil {
		res.Steps = append(res.Steps, Step{Name: "restart", Argv: u.cfg.RestartCmd, Err: err, Output: tail(out, 4000)})
		return fmt.Errorf("update: restart %v: %w", u.cfg.RestartCmd, err)
	}
	res.Restarted = true
	res.Steps = append(res.Steps, Step{Name: "restart", Argv: u.cfg.RestartCmd})
	return nil
}

// gitHead records which commit produced a snapshot, for the operator's benefit.
// Best-effort: a tree without git is not a reason to refuse to snapshot.
func (u *Updater) gitHead(ctx context.Context) string {
	out, err := u.run(ctx, u.cfg.SourceDir, []string{"git", "rev-parse", "HEAD"})
	if err != nil {
		return ""
	}
	return strings.TrimSpace(out)
}