Maven/internal/update/apply.go

package update

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"time"
)

// Result — the full account of one Apply. Every field is filled in on the
// failure paths too, because "what state is my box in" is the only question that
// matters after a failed update.
type Result struct {
	Verified   bool
	SnapshotID string // the rollback target; named even when the rollback failed
	Installed  []string
	Restarted  bool
	Healthy    bool
	RolledBack bool
	// RollbackHealthy — whether she answered again after the restore. False with
	// RolledBack true is the manual-recovery case.
	RollbackHealthy bool
	Steps           []Step
	Took            time.Duration
}

// Apply is the whole update, in the only order that is safe.
//
// It is called by a human running cmd/mavupdate on the box. Nothing else calls
// it: no timer, no IPC method, no web route, no act. See the package comment.
func (u *Updater) Apply(ctx context.Context) (Result, error) {
	start := u.now()
	res := Result{}
	defer func() { res.Took = u.now().Sub(start) }()

	// 0. She has to be answering before we start. Otherwise a failed update and
	// a box that was already broken look identical afterwards, and the rollback
	// has no baseline to prove itself against.
	u.log("preflight: checking the running daemon")
	if err := u.health(ctx, u.cfg.HealthSocket); err != nil {
		if errors.Is(err, ErrHealthDial) {
			// Not the daemon's fault and not fixed by fixing the daemon. The
			// usual cause is the socket path: under a docker volume it sits in
			// /var/lib/docker, which the operator's account cannot traverse.
			return res, err
		}
		return res, fmt.Errorf("%w: %v", ErrUnhealthyBefore, err)
	}

	// 0b. If the source is part of what a rollback has to put back, it must be
	// in a state that can be described and restored. A dirty tree is neither:
	// the commit recorded in the snapshot does not say what is deployed, and a
	// forced checkout on the way back would delete his uncommitted work.
	commit := u.gitHead(ctx)
	if u.cfg.SourceRollback == "git" {
		if commit == "" {
			return res, fmt.Errorf("%w: source_rollback is \"git\" but %s has no readable git HEAD", ErrSourceRollback, u.cfg.SourceDir)
		}
		if dirty, err := u.gitDirty(ctx); err != nil {
			return res, fmt.Errorf("%w: %v", ErrDirtyTree, err)
		} else if dirty {
			return res, fmt.Errorf("%w: %s", ErrDirtyTree, u.cfg.SourceDir)
		}
	}

	// 1. Snapshot what is deployed now, BEFORE the build.
	//
	// The order matters and it is not the obvious one. `make build` writes its
	// binaries into the working tree, and on the docker deployment the working
	// tree IS the install dir — so snapshotting after the build would snapshot
	// the new artifacts and leave nothing to roll back to. The snapshot is the
	// only thing standing between a bad build and a box that needs a screwdriver,
	// so it is taken first, while the deployed bytes are still the old ones.
	names := append(append([]string{}, u.cfg.Binaries...), u.cfg.ConfigFiles...)
	snap, err := u.store.Save(u.cfg.InstallDir, names, commit, "pre-update")
	if err != nil {
		return res, err
	}
	res.SnapshotID = snap.ID
	u.log("snapshot: %s (%d files) in %s", snap.ID, len(snap.Files), snap.Dir())

	// 2. Build and test before anything is deployed. A broken tree costs time
	// and nothing else — but `make build` has already overwritten the binaries in
	// the tree, so restore them: otherwise a later restart by hand would deploy
	// code that failed its own tests. Nothing has been restarted, so this is a
	// file restore with no restart and no health check.
	steps, err := u.Verify(ctx)
	res.Steps = append(res.Steps, steps...)
	if err != nil {
		// RolledBack stays false here on purpose. Nothing was installed and
		// nothing was restarted, so there is no rollback to report; a compile
		// error printing rolled_back=true sends the operator looking for a
		// restart that never happened. The log line carries what was done.
		if rerr := u.restoreBinaries(snap); rerr != nil {
			u.log("verify failed and the artifacts could not be put back: %v — the previous ones are in %s", rerr, snap.Dir())
		} else {
			u.log("verify failed; the previously deployed artifacts are back in place, she was never restarted")
		}
		return res, err
	}
	res.Verified = true

	// 3. Install. Per-file temp+rename, so an interruption leaves whole files.
	// Config is snapshotted but never overwritten — an update does not get to
	// replace the operator's config.
	installed, err := u.install()
	res.Installed = installed
	if err != nil {
		// Files may be half-swapped across the set, so restore before returning
		// even though nothing has been restarted yet.
		u.log("install failed: %v — restoring", err)
		return u.rollback(ctx, snap, res, err)
	}
	u.log("install: %d artifact(s) into %s", len(installed), u.cfg.InstallDir)

	// 4. Restart, then 5. prove she answers.
	if err := u.restart(ctx, &res); err != nil {
		return u.rollback(ctx, snap, res, err)
	}
	u.log("restart: ok, waiting for her to answer (up to %s)", u.cfg.healthTimeout())
	if err := u.waitHealthy(ctx, u.cfg.healthTimeout()); err != nil {
		return u.rollback(ctx, snap, res, err)
	}
	res.Healthy = true
	u.log("health: she answers on %s — update committed", u.cfg.HealthSocket)

	if err := u.store.Prune(u.cfg.KeepSnapshots); err != nil {
		u.log("prune: %v (harmless)", err)
	}
	return res, nil
}

// Rollback restores a snapshot by id (empty = the newest) and restarts. Exposed
// separately so the operator can undo an update that verified, restarted and
// answered a Presence call but is wrong in a way no health check can see.
func (u *Updater) Rollback(ctx context.Context, id string) (Result, error) {
	var snap Snapshot
	var err error
	if id == "" {
		snaps, lerr := u.store.List()
		if lerr != nil {
			return Result{}, lerr
		}
		if len(snaps) == 0 {
			return Result{}, errors.New("update: no snapshots to roll back to")
		}
		snap = snaps[0]
	} else if snap, err = u.store.Load(id); err != nil {
		return Result{}, err
	}
	res := Result{SnapshotID: snap.ID}
	return u.rollback(ctx, snap, res, errors.New("operator asked for a rollback"))
}

// rollback restores the snapshot and restarts, then reports whether that worked.
// It depends on nothing that the update changed: file copies out of the snapshot
// dir and the same restart command. No build, no migration, no cooperation from
// the code being replaced.
func (u *Updater) rollback(ctx context.Context, snap Snapshot, res Result, cause error) (Result, error) {
	// A rollback interrupted halfway is the one outcome worse than the failure
	// that triggered it, so it does not inherit the caller's cancellation: a
	// Ctrl-C during the health wait must not abandon the restore mid-restart.
	ctx = context.WithoutCancel(ctx)
	res.RolledBack = true
	u.log("rollback: restoring snapshot %s over %s", snap.ID, u.cfg.InstallDir)
	if err := u.restoreBinaries(snap); err != nil {
		u.log("rollback: RESTORE FAILED: %v", err)
		return res, fmt.Errorf("%w: %v (after %v); the previous artifacts are in %s — copy them back by hand", ErrRollbackFailed, err, cause, snap.Dir())
	}
	// On a deployment that rebuilds from source, putting the binaries back is
	// the part that changes nothing. The source is what the restart deploys, so
	// it goes back too, and it goes back before the restart that reads it.
	if err := u.restoreSource(ctx, snap); err != nil {
		u.log("rollback: SOURCE CHECKOUT FAILED: %v", err)
		return res, fmt.Errorf("%w: %v (after %v); the tree is still on the new commit, so a restart would redeploy it — `git -C %s checkout --force %s` by hand", ErrRollbackFailed, err, cause, u.cfg.SourceDir, snap.Commit)
	}
	// A restore with no restart leaves the failed process running, so a failed
	// restart here is still the manual-recovery case.
	if err := u.restart(ctx, &res); err != nil {
		u.log("rollback: RESTART FAILED: %v", err)
		return res, fmt.Errorf("%w: restored %s but the restart failed: %v (after %v)", ErrRollbackFailed, snap.ID, err, cause)
	}
	if err := u.waitHealthy(ctx, u.cfg.healthTimeout()); err != nil {
		u.log("rollback: she still does not answer: %v", err)
		return res, fmt.Errorf("%w: restored %s and restarted but she does not answer: %v (after %v)", ErrRollbackFailed, snap.ID, err, cause)
	}
	res.RollbackHealthy = true
	u.log("rollback: she answers again on the previous build (%s)", snap.ID)
	return res, fmt.Errorf("%w to %s: %v", ErrRolledBack, snap.ID, cause)
}

// install copies the freshly built binaries from SourceDir into InstallDir.
//
// When the two are the same directory — the docker deployment builds the image
// from the working tree — this is a no-op by design rather than by accident: the
// artifacts are already where they belong and the restart command rebuilds the
// image from them.
func (u *Updater) install() ([]string, error) {
	if filepath.Clean(u.cfg.SourceDir) == filepath.Clean(u.cfg.InstallDir) {
		return u.cfg.Binaries, nil
	}
	var done []string
	for _, name := range u.cfg.Binaries {
		src := filepath.Join(u.cfg.SourceDir, name)
		fi, err := os.Stat(src)
		if err != nil {
			return done, fmt.Errorf("update: install %s: %w (did `make build` produce it?)", name, err)
		}
		if _, err := copyFile(src, filepath.Join(u.cfg.InstallDir, name), fi.Mode().Perm()); err != nil {
			return done, fmt.Errorf("update: install %s: %w", name, err)
		}
		done = append(done, name)
	}
	return done, nil
}

func (u *Updater) restart(ctx context.Context, res *Result) error {
	u.log("restart: %v", u.cfg.RestartCmd)
	out, err := u.run(ctx, u.cfg.SourceDir, u.cfg.RestartCmd)
	if err != nil {
		res.Steps = append(res.Steps, Step{Name: "restart", Argv: u.cfg.RestartCmd, Err: err, Output: tail(out, 4000)})
		return fmt.Errorf("update: restart %v: %w", u.cfg.RestartCmd, err)
	}
	res.Restarted = true
	res.Steps = append(res.Steps, Step{Name: "restart", Argv: u.cfg.RestartCmd})
	return nil
}

// restoreBinaries puts back the built artifacts and nothing else.
//
// ConfigFiles are snapshotted and deliberately not restored. The Config doc
// says an update never replaces the operator's config, and a rollback that
// quietly reverted deploy/mavend.json would undo edits made since the last
// apply — a phraser.model_path change among them, which is how the resident
// model gets swapped. The copies stay in the snapshot dir for him to take by
// hand if the config is what he wants back.
func (u *Updater) restoreBinaries(snap Snapshot) error {
	return snap.RestoreOnly(u.cfg.InstallDir, u.cfg.Binaries)
}

// restoreSource puts the working tree back on the commit the snapshot was taken
// at, for the deployments where that is what the restart command deploys. A
// no-op for every other shape.
func (u *Updater) restoreSource(ctx context.Context, snap Snapshot) error {
	if u.cfg.SourceRollback != "git" {
		return nil
	}
	if snap.Commit == "" {
		return fmt.Errorf("snapshot %s records no commit, so there is nothing to check out", snap.ID)
	}
	u.log("rollback: checking %s back out to %s", u.cfg.SourceDir, snap.Commit)
	// --force because the failed build left artifacts in the tree. Safe only
	// because Apply refused to start on a dirty tree, so nothing uncommitted of
	// his is in reach.
	out, err := u.run(ctx, u.cfg.SourceDir, []string{"git", "checkout", "--force", snap.Commit})
	if err != nil {
		return fmt.Errorf("git checkout %s: %v: %s", snap.Commit, err, tail(out, 1000))
	}
	return nil
}

// gitDirty reports whether the working tree has uncommitted changes.
func (u *Updater) gitDirty(ctx context.Context) (bool, error) {
	out, err := u.run(ctx, u.cfg.SourceDir, []string{"git", "status", "--porcelain"})
	if err != nil {
		return false, fmt.Errorf("git status in %s: %v", u.cfg.SourceDir, err)
	}
	return strings.TrimSpace(out) != "", nil
}

// gitHead records which commit produced a snapshot, for the operator's benefit.
// Best-effort: a tree without git is not a reason to refuse to snapshot.
func (u *Updater) gitHead(ctx context.Context) string {
	out, err := u.run(ctx, u.cfg.SourceDir, []string{"git", "rev-parse", "HEAD"})
	if err != nil {
		return ""
	}
	return strings.TrimSpace(out)
}