update: roll back what the restart actually deploys

On the deployment deploy/README.md documents, source_dir and install_dir are
the same tree and the restart command rebuilds the image from it. The
Dockerfile builds from cmd/ and internal/ and .dockerignore keeps the host
binaries out, so restoring the snapshotted binaries restored bytes nothing
reads. A bad commit therefore cost two health timeouts and two image builds
and ended in ErrRollbackFailed with an instruction to copy files back by hand,
which would not have helped either.

A deployment that rebuilds from source now has to say how the source is put
back. source_rollback "git" records the commit before the update and checks it
back out before the rollback restart. It refuses a dirty tree, because the
recorded commit does not describe one and a forced checkout would delete his
work. A build-from-source config that says nothing is refused by Validate, at
startup, rather than at the one rollback that mattered.

Also in this change, all from the same review:

  - MethodPing, the one method a locked daemon answers. Preflight passed on an
    unlocked daemon and the post-restart Presence read failed on a locked one,
    so a good update read as SHE IS PROBABLY DOWN once the env key is gone.
  - A dial failure is reported apart from a read failure. The documented
    socket is under /var/lib/docker, which a non-root operator cannot
    traverse, and "she is not answering" was the wrong diagnosis.
  - Verify refuses to run as root over a tree owned by someone else. It runs
    make build and make test in place, and root-owned artifacts break his next
    ordinary make.
  - A rollback no longer reverts config_files. That undid every config edit
    since the last apply, phraser.model_path among them.
  - The verify-failure path no longer reports rolled_back for a compile error.
  - waitHealthy caps each attempt at the remaining budget, so a 90s timeout
    cannot run to 99s.
  - tail cuts on a rune boundary. Russian test names showed the seam.
  - The claim that mavend does not import internal/update is replaced with
    what is enforced: mavend constructs no Updater and nothing can call Apply.
  - snapshot_dir inside source_dir is refused. It landed in the build context.

Found in review of #69.

internal/update/apply.go

@@ -41,9 +41,31 @@ func (u *Updater) Apply(ctx context.Context) (Result, error) {
 	// has no baseline to prove itself against.
 	u.log("preflight: checking the running daemon")
 	if err := u.health(ctx, u.cfg.HealthSocket); err != nil {
+		if errors.Is(err, ErrHealthDial) {
+			// Not the daemon's fault and not fixed by fixing the daemon. The
+			// usual cause is the socket path: under a docker volume it sits in
+			// /var/lib/docker, which the operator's account cannot traverse.
+			return res, err
+		}
 		return res, fmt.Errorf("%w: %v", ErrUnhealthyBefore, err)
 	}
 
+	// 0b. If the source is part of what a rollback has to put back, it must be
+	// in a state that can be described and restored. A dirty tree is neither:
+	// the commit recorded in the snapshot does not say what is deployed, and a
+	// forced checkout on the way back would delete his uncommitted work.
+	commit := u.gitHead(ctx)
+	if u.cfg.SourceRollback == "git" {
+		if commit == "" {
+			return res, fmt.Errorf("%w: source_rollback is \"git\" but %s has no readable git HEAD", ErrSourceRollback, u.cfg.SourceDir)
+		}
+		if dirty, err := u.gitDirty(ctx); err != nil {
+			return res, fmt.Errorf("%w: %v", ErrDirtyTree, err)
+		} else if dirty {
+			return res, fmt.Errorf("%w: %s", ErrDirtyTree, u.cfg.SourceDir)
+		}
+	}
+
 	// 1. Snapshot what is deployed now, BEFORE the build.
 	//
 	// The order matters and it is not the obvious one. `make build` writes its
@@ -53,7 +75,7 @@ func (u *Updater) Apply(ctx context.Context) (Result, error) {
 	// only thing standing between a bad build and a box that needs a screwdriver,
 	// so it is taken first, while the deployed bytes are still the old ones.
 	names := append(append([]string{}, u.cfg.Binaries...), u.cfg.ConfigFiles...)
-	snap, err := u.store.Save(u.cfg.InstallDir, names, u.gitHead(ctx), "pre-update")
+	snap, err := u.store.Save(u.cfg.InstallDir, names, commit, "pre-update")
 	if err != nil {
 		return res, err
 	}
@@ -68,10 +90,13 @@ func (u *Updater) Apply(ctx context.Context) (Result, error) {
 	steps, err := u.Verify(ctx)
 	res.Steps = append(res.Steps, steps...)
 	if err != nil {
-		if rerr := snap.Restore(u.cfg.InstallDir); rerr != nil {
+		// RolledBack stays false here on purpose. Nothing was installed and
+		// nothing was restarted, so there is no rollback to report; a compile
+		// error printing rolled_back=true sends the operator looking for a
+		// restart that never happened. The log line carries what was done.
+		if rerr := u.restoreBinaries(snap); rerr != nil {
 			u.log("verify failed and the artifacts could not be put back: %v — the previous ones are in %s", rerr, snap.Dir())
 		} else {
-			res.RolledBack = true
 			u.log("verify failed; the previously deployed artifacts are back in place, she was never restarted")
 		}
 		return res, err
@@ -141,10 +166,17 @@ func (u *Updater) rollback(ctx context.Context, snap Snapshot, res Result, cause
 	ctx = context.WithoutCancel(ctx)
 	res.RolledBack = true
 	u.log("rollback: restoring snapshot %s over %s", snap.ID, u.cfg.InstallDir)
-	if err := snap.Restore(u.cfg.InstallDir); err != nil {
+	if err := u.restoreBinaries(snap); err != nil {
 		u.log("rollback: RESTORE FAILED: %v", err)
 		return res, fmt.Errorf("%w: %v (after %v); the previous artifacts are in %s — copy them back by hand", ErrRollbackFailed, err, cause, snap.Dir())
 	}
+	// On a deployment that rebuilds from source, putting the binaries back is
+	// the part that changes nothing. The source is what the restart deploys, so
+	// it goes back too, and it goes back before the restart that reads it.
+	if err := u.restoreSource(ctx, snap); err != nil {
+		u.log("rollback: SOURCE CHECKOUT FAILED: %v", err)
+		return res, fmt.Errorf("%w: %v (after %v); the tree is still on the new commit, so a restart would redeploy it — `git -C %s checkout --force %s` by hand", ErrRollbackFailed, err, cause, u.cfg.SourceDir, snap.Commit)
+	}
 	// A restore with no restart leaves the failed process running, so a failed
 	// restart here is still the manual-recovery case.
 	if err := u.restart(ctx, &res); err != nil {
@@ -197,6 +229,48 @@ func (u *Updater) restart(ctx context.Context, res *Result) error {
 	return nil
 }
 
+// restoreBinaries puts back the built artifacts and nothing else.
+//
+// ConfigFiles are snapshotted and deliberately not restored. The Config doc
+// says an update never replaces the operator's config, and a rollback that
+// quietly reverted deploy/mavend.json would undo edits made since the last
+// apply — a phraser.model_path change among them, which is how the resident
+// model gets swapped. The copies stay in the snapshot dir for him to take by
+// hand if the config is what he wants back.
+func (u *Updater) restoreBinaries(snap Snapshot) error {
+	return snap.RestoreOnly(u.cfg.InstallDir, u.cfg.Binaries)
+}
+
+// restoreSource puts the working tree back on the commit the snapshot was taken
+// at, for the deployments where that is what the restart command deploys. A
+// no-op for every other shape.
+func (u *Updater) restoreSource(ctx context.Context, snap Snapshot) error {
+	if u.cfg.SourceRollback != "git" {
+		return nil
+	}
+	if snap.Commit == "" {
+		return fmt.Errorf("snapshot %s records no commit, so there is nothing to check out", snap.ID)
+	}
+	u.log("rollback: checking %s back out to %s", u.cfg.SourceDir, snap.Commit)
+	// --force because the failed build left artifacts in the tree. Safe only
+	// because Apply refused to start on a dirty tree, so nothing uncommitted of
+	// his is in reach.
+	out, err := u.run(ctx, u.cfg.SourceDir, []string{"git", "checkout", "--force", snap.Commit})
+	if err != nil {
+		return fmt.Errorf("git checkout %s: %v: %s", snap.Commit, err, tail(out, 1000))
+	}
+	return nil
+}
+
+// gitDirty reports whether the working tree has uncommitted changes.
+func (u *Updater) gitDirty(ctx context.Context) (bool, error) {
+	out, err := u.run(ctx, u.cfg.SourceDir, []string{"git", "status", "--porcelain"})
+	if err != nil {
+		return false, fmt.Errorf("git status in %s: %v", u.cfg.SourceDir, err)
+	}
+	return strings.TrimSpace(out) != "", nil
+}
+
 // gitHead records which commit produced a snapshot, for the operator's benefit.
 // Best-effort: a tree without git is not a reason to refuse to snapshot.
 func (u *Updater) gitHead(ctx context.Context) string {