Files
Maven/internal/update/update_test.go
T
kami 810076451f update: roll back what the restart actually deploys
On the deployment deploy/README.md documents, source_dir and install_dir are
the same tree and the restart command rebuilds the image from it. The
Dockerfile builds from cmd/ and internal/ and .dockerignore keeps the host
binaries out, so restoring the snapshotted binaries restored bytes nothing
reads. A bad commit therefore cost two health timeouts and two image builds
and ended in ErrRollbackFailed with an instruction to copy files back by hand,
which would not have helped either.

A deployment that rebuilds from source now has to say how the source is put
back. source_rollback "git" records the commit before the update and checks it
back out before the rollback restart. It refuses a dirty tree, because the
recorded commit does not describe one and a forced checkout would delete his
work. A build-from-source config that says nothing is refused by Validate, at
startup, rather than at the one rollback that mattered.

Also in this change, all from the same review:

  - MethodPing, the one method a locked daemon answers. Preflight passed on an
    unlocked daemon and the post-restart Presence read failed on a locked one,
    so a good update read as SHE IS PROBABLY DOWN once the env key is gone.
  - A dial failure is reported apart from a read failure. The documented
    socket is under /var/lib/docker, which a non-root operator cannot
    traverse, and "she is not answering" was the wrong diagnosis.
  - Verify refuses to run as root over a tree owned by someone else. It runs
    make build and make test in place, and root-owned artifacts break his next
    ordinary make.
  - A rollback no longer reverts config_files. That undid every config edit
    since the last apply, phraser.model_path among them.
  - The verify-failure path no longer reports rolled_back for a compile error.
  - waitHealthy caps each attempt at the remaining budget, so a 90s timeout
    cannot run to 99s.
  - tail cuts on a rune boundary. Russian test names showed the seam.
  - The claim that mavend does not import internal/update is replaced with
    what is enforced: mavend constructs no Updater and nothing can call Apply.
  - snapshot_dir inside source_dir is refused. It landed in the build context.

Found in review of #69.
2026-08-01 14:06:00 +04:00

477 lines
16 KiB
Go

package update
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
// The tests drive the whole orchestration against a fake box: a directory tree
// standing in for the install dir, an injected Runner standing in for
// make/git/docker, and an injected HealthCheck standing in for mavend. That is
// what makes the failure paths — the ones that matter — testable at all: you
// cannot ask a real deployment to fail its health check on demand, and the
// rollback path is exactly the path nobody exercises by hand.
type fakeBox struct {
t *testing.T
root string
// what the fake `make build` writes into the source tree
newBytes string
// scripted failures
buildErr error
testErr error
restartErr error
// health: fails until the Nth call, then follows healthy
healthErrs int // remaining failures to serve
healthy bool
healthChecks int
// healthFn overrides the scripted behaviour entirely, for tests whose
// verdict depends on what the last restart actually deployed.
healthFn func() error
// deployedAtRestart records the installed bytes each time restart runs, so a
// test can prove the rollback put the old bytes back BEFORE restarting.
deployedAtRestart []string
// builtFromSource records the source the restart would have built an image
// from, each time it runs.
builtFromSource []string
ran []string
// The git half, for the deployment whose restart rebuilds from the tree.
// commit is HEAD; byCommit is what each commit's source says; dirty makes
// `git status --porcelain` report uncommitted work.
commit string
byCommit map[string]string
dirty bool
}
func newFakeBox(t *testing.T) *fakeBox {
t.Helper()
root := t.TempDir()
for _, d := range []string{"src", "install", "snapshots"} {
if err := os.MkdirAll(filepath.Join(root, d), 0o755); err != nil {
t.Fatal(err)
}
}
// The currently deployed build, and a config file next to it.
write(t, filepath.Join(root, "install", "mavend"), "OLD-BUILD")
write(t, filepath.Join(root, "install", "mavend.json"), `{"tick_interval":"60s"}`)
// The source tree already contains a stale binary; `make build` overwrites it.
write(t, filepath.Join(root, "src", "mavend"), "STALE")
return &fakeBox{
t: t, root: root, newBytes: "NEW-BUILD", healthy: true,
commit: "cafebabecafebabecafebabecafebabecafebabe",
byCommit: map[string]string{},
}
}
func (b *fakeBox) cfg() Config {
return Config{
SourceDir: filepath.Join(b.root, "src"),
InstallDir: filepath.Join(b.root, "install"),
SnapshotDir: filepath.Join(b.root, "snapshots"),
Binaries: []string{"mavend"},
ConfigFiles: []string{"mavend.json"},
RestartCmd: []string{"restart-the-thing"},
HealthSocket: filepath.Join(b.root, "mavend.sock"),
HealthTimeoutSec: 1,
KeepSnapshots: 3,
}
}
func (b *fakeBox) run(ctx context.Context, dir string, argv []string) (string, error) {
b.ran = append(b.ran, strings.Join(argv, " "))
switch strings.Join(argv, " ") {
case "make build":
if b.buildErr != nil {
return "ld: undefined reference to everything", b.buildErr
}
// A real build writes its artifacts into the working tree — the behaviour
// the snapshot-before-build ordering exists to survive.
write(b.t, filepath.Join(b.root, "src", "mavend"), b.newBytes)
return "built", nil
case "make test":
if b.testErr != nil {
return "--- FAIL: TestSomething", b.testErr
}
return "ok", nil
case "git rev-parse HEAD":
return b.commit + "\n", nil
case "git status --porcelain":
if b.dirty {
return " M internal/router/router.go\n", nil
}
return "", nil
case "restart-the-thing":
b.deployedAtRestart = append(b.deployedAtRestart, read(b.t, filepath.Join(b.root, "install", "mavend")))
// The docker shape: the restart rebuilds the image from the tree, so
// what it deploys is the source, not any binary on the host.
if src, err := os.ReadFile(filepath.Join(b.root, "src", "source.go")); err == nil {
b.builtFromSource = append(b.builtFromSource, string(src))
}
if b.restartErr != nil {
return "no such container", b.restartErr
}
return "restarted", nil
}
if len(argv) == 4 && argv[0] == "git" && argv[1] == "checkout" && argv[2] == "--force" {
content, ok := b.byCommit[argv[3]]
if !ok {
return "error: pathspec did not match", errors.New("exit status 1")
}
b.commit = argv[3]
write(b.t, filepath.Join(b.root, "src", "source.go"), content)
return "HEAD is now at " + argv[3], nil
}
return "", errors.New("unexpected command: " + strings.Join(argv, " "))
}
func (b *fakeBox) health(ctx context.Context, socket string) error {
b.healthChecks++
if b.healthFn != nil {
return b.healthFn()
}
if b.healthErrs > 0 {
b.healthErrs--
return errors.New("connection refused")
}
if !b.healthy {
return errors.New("she does not answer")
}
return nil
}
func (b *fakeBox) updater(t *testing.T, extra ...Option) *Updater {
t.Helper()
opts := append([]Option{WithRunner(b.run), WithHealth(b.health)}, extra...)
u, err := New(b.cfg(), opts...)
if err != nil {
t.Fatal(err)
}
return u
}
func (b *fakeBox) deployed() string { return read(b.t, filepath.Join(b.root, "install", "mavend")) }
func write(t *testing.T, path, content string) {
t.Helper()
if err := os.WriteFile(path, []byte(content), 0o755); err != nil {
t.Fatal(err)
}
}
func read(t *testing.T, path string) string {
t.Helper()
b, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
return string(b)
}
func TestApply_HappyPath(t *testing.T) {
b := newFakeBox(t)
res, err := b.updater(t).Apply(context.Background())
if err != nil {
t.Fatalf("Apply: %v", err)
}
if !res.Verified || !res.Restarted || !res.Healthy || res.RolledBack {
t.Fatalf("result = %+v; want verified+restarted+healthy and no rollback", res)
}
if got := b.deployed(); got != "NEW-BUILD" {
t.Errorf("deployed binary = %q; want the new build", got)
}
// The order is the property: health, snapshot, build, test, install, restart.
want := []string{"git rev-parse HEAD", "make build", "make test", "restart-the-thing"}
if strings.Join(b.ran, "|") != strings.Join(want, "|") {
t.Errorf("commands ran = %v; want %v", b.ran, want)
}
if res.SnapshotID == "" {
t.Error("no snapshot was taken")
}
}
func TestApply_RefusesWhenSheIsAlreadyDown(t *testing.T) {
// A box that is already broken has no baseline for the rollback to prove
// itself against, so the update never starts.
b := newFakeBox(t)
b.healthy = false
res, err := b.updater(t).Apply(context.Background())
if !errors.Is(err, ErrUnhealthyBefore) {
t.Fatalf("Apply on an unhealthy box = %v; want ErrUnhealthyBefore", err)
}
if len(b.ran) != 0 {
t.Errorf("a refused update still ran %v", b.ran)
}
if res.SnapshotID != "" {
t.Error("a refused update still took a snapshot")
}
}
func TestApply_TestFailureDeploysNothingAndPutsTheTreeBack(t *testing.T) {
b := newFakeBox(t)
b.testErr = errors.New("exit status 1")
res, err := b.updater(t).Apply(context.Background())
if !errors.Is(err, ErrVerifyFailed) {
t.Fatalf("Apply with failing tests = %v; want ErrVerifyFailed", err)
}
if res.Verified {
t.Error("result claims verified after a failing test suite")
}
for _, c := range b.ran {
if c == "restart-the-thing" {
t.Fatal("a failed verification restarted the daemon")
}
}
if got := b.deployed(); got != "OLD-BUILD" {
t.Errorf("deployed binary = %q; want the old build untouched", got)
}
// The failing output is kept so the operator can see why.
var found bool
for _, s := range res.Steps {
if s.Name == "test" && strings.Contains(s.Output, "FAIL") {
found = true
}
}
if !found {
t.Error("the failing test output was not retained")
}
}
func TestApply_BuildFailureIsCaughtBeforeTheTests(t *testing.T) {
b := newFakeBox(t)
b.buildErr = errors.New("exit status 2")
if _, err := b.updater(t).Apply(context.Background()); !errors.Is(err, ErrVerifyFailed) {
t.Fatalf("Apply with a failing build = %v; want ErrVerifyFailed", err)
}
for _, c := range b.ran {
if c == "make test" {
t.Error("ran the test suite after the build failed")
}
}
}
func TestApply_UnhealthyAfterRestartRollsBackToTheOldBytes(t *testing.T) {
// The case the package exists for: everything verifies, the new build
// installs, and then she does not come up.
b := newFakeBox(t)
b.healthErrs = 1 // the preflight check passes, then she stops answering
b.healthy = false
u := b.updater(t)
// Once the rollback restores the old build, she answers again.
restored := false
u.health = func(ctx context.Context, socket string) error {
b.healthChecks++
if b.deployed() == "OLD-BUILD" && restored {
return nil
}
if b.healthChecks == 1 {
return nil // preflight: the old build is up
}
if b.deployed() == "OLD-BUILD" {
restored = true
return nil
}
return errors.New("she does not answer on the new build")
}
res, err := u.Apply(context.Background())
if !errors.Is(err, ErrRolledBack) {
t.Fatalf("Apply with a dead new build = %v; want ErrRolledBack", err)
}
if !res.RolledBack || !res.RollbackHealthy || res.Healthy {
t.Fatalf("result = %+v; want rolled back and healthy again on the old build", res)
}
if got := b.deployed(); got != "OLD-BUILD" {
t.Errorf("deployed binary after the rollback = %q; want OLD-BUILD", got)
}
// And the restore happened BEFORE the second restart, not after it.
if len(b.deployedAtRestart) != 2 {
t.Fatalf("restarts = %v; want two (the update and the rollback)", b.deployedAtRestart)
}
if b.deployedAtRestart[0] != "NEW-BUILD" || b.deployedAtRestart[1] != "OLD-BUILD" {
t.Errorf("bytes in place at each restart = %v; want [NEW-BUILD OLD-BUILD]", b.deployedAtRestart)
}
}
func TestApply_RestartFailureRollsBack(t *testing.T) {
b := newFakeBox(t)
b.restartErr = errors.New("exit status 1")
res, err := b.updater(t).Apply(context.Background())
// The rollback's own restart fails too, so this is the manual-recovery case —
// and it says so instead of reporting a tidy rollback.
if !errors.Is(err, ErrRollbackFailed) {
t.Fatalf("Apply with a broken restart command = %v; want ErrRollbackFailed", err)
}
if !res.RolledBack || res.RollbackHealthy {
t.Fatalf("result = %+v; want rolled back but not healthy", res)
}
if got := b.deployed(); got != "OLD-BUILD" {
t.Errorf("deployed binary = %q; want the old bytes restored even so", got)
}
}
func TestApply_RollbackNeedsNoBuildAndNoNewCode(t *testing.T) {
// The rollback must not depend on the toolchain, the source tree, or the
// code it is replacing. Prove it: delete the source tree's binary and make
// every command except the restart fail, then roll back.
b := newFakeBox(t)
if _, err := b.updater(t).Apply(context.Background()); err != nil {
t.Fatalf("setup Apply: %v", err)
}
if b.deployed() != "NEW-BUILD" {
t.Fatal("setup did not deploy")
}
os.RemoveAll(filepath.Join(b.root, "src"))
if err := os.MkdirAll(filepath.Join(b.root, "src"), 0o755); err != nil {
t.Fatal(err)
}
b.buildErr = errors.New("no toolchain here")
b.testErr = errors.New("no toolchain here")
b.ran = nil
res, err := b.updater(t).Rollback(context.Background(), "")
if err != nil && !errors.Is(err, ErrRolledBack) {
t.Fatalf("Rollback: %v", err)
}
if !res.RollbackHealthy {
t.Fatalf("result = %+v; want a healthy rollback", res)
}
if got := b.deployed(); got != "OLD-BUILD" {
t.Errorf("deployed binary = %q; want OLD-BUILD", got)
}
for _, c := range b.ran {
if strings.HasPrefix(c, "make") {
t.Errorf("the rollback ran %q — it must not need a build", c)
}
}
}
func TestApply_ConfigIsSnapshottedButNeverOverwritten(t *testing.T) {
b := newFakeBox(t)
// A config in the source tree must not be deployed over the operator's.
write(t, filepath.Join(b.root, "src", "mavend.json"), `{"tick_interval":"1s"}`)
if _, err := b.updater(t).Apply(context.Background()); err != nil {
t.Fatalf("Apply: %v", err)
}
if got := read(t, filepath.Join(b.root, "install", "mavend.json")); !strings.Contains(got, "60s") {
t.Errorf("installed config = %q; an update must not replace his config", got)
}
snaps, err := b.updater(t).Snapshots()
if err != nil || len(snaps) == 0 {
t.Fatalf("Snapshots: %v %v", snaps, err)
}
var names []string
for _, f := range snaps[0].Files {
names = append(names, f.Name)
}
if len(names) != 2 {
t.Errorf("snapshot files = %v; want the binary and the config", names)
}
if snaps[0].Commit == "" {
t.Error("the snapshot did not record which commit produced it")
}
}
func TestRollback_CorruptSnapshotIsRefusedNotRestored(t *testing.T) {
b := newFakeBox(t)
if _, err := b.updater(t).Apply(context.Background()); err != nil {
t.Fatalf("setup Apply: %v", err)
}
snaps, _ := b.updater(t).Snapshots()
// Something ate the snapshot. Restoring it would deploy garbage.
write(t, filepath.Join(snaps[0].Dir(), "mavend"), "CORRUPT")
_, err := b.updater(t).Rollback(context.Background(), snaps[0].ID)
if !errors.Is(err, ErrRollbackFailed) || !strings.Contains(err.Error(), "corrupt") {
t.Fatalf("Rollback of a corrupt snapshot = %v; want a refusal naming the corruption", err)
}
if got := b.deployed(); got != "NEW-BUILD" {
t.Errorf("deployed binary = %q; a refused restore must change nothing", got)
}
}
func TestRollback_NoSnapshots(t *testing.T) {
b := newFakeBox(t)
if _, err := b.updater(t).Rollback(context.Background(), ""); err == nil {
t.Error("Rollback with no snapshots succeeded; want an error")
}
}
func TestPrune_KeepsTheNewestAsTheRollbackTarget(t *testing.T) {
b := newFakeBox(t)
st := &Store{Dir: filepath.Join(b.root, "snapshots")}
base := time.Date(2026, 8, 1, 3, 0, 0, 0, time.UTC)
for i := 0; i < 4; i++ {
i := i
st.now = func() time.Time { return base.Add(time.Duration(i) * time.Minute) }
if _, err := st.Save(filepath.Join(b.root, "install"), []string{"mavend"}, "", ""); err != nil {
t.Fatal(err)
}
}
if err := st.Prune(0); err != nil { // 0 is clamped to 1, never to zero
t.Fatal(err)
}
snaps, err := st.List()
if err != nil {
t.Fatal(err)
}
if len(snaps) != 1 {
t.Fatalf("kept %d snapshots; want 1", len(snaps))
}
if snaps[0].ID != "20260801-030300" {
t.Errorf("kept %s; want the newest", snaps[0].ID)
}
}
func TestList_IgnoresSnapshotsWithNoManifest(t *testing.T) {
// An interrupted snapshot has files but no manifest. It must never be offered
// as a rollback target — restoring a half-copied binary is the worst outcome
// in the package.
b := newFakeBox(t)
dir := filepath.Join(b.root, "snapshots", "20260801-000000")
if err := os.MkdirAll(dir, 0o700); err != nil {
t.Fatal(err)
}
write(t, filepath.Join(dir, "mavend"), "HALF")
snaps, err := (&Store{Dir: filepath.Join(b.root, "snapshots")}).List()
if err != nil {
t.Fatal(err)
}
if len(snaps) != 0 {
t.Errorf("List returned %d snapshots; want none (no manifest)", len(snaps))
}
}
func TestConfigValidate(t *testing.T) {
ok := (&fakeBox{root: t.TempDir()}).cfg()
if err := ok.Validate(); err != nil {
t.Fatalf("valid config rejected: %v", err)
}
bad := map[string]func(c Config) Config{
"relative source": func(c Config) Config { c.SourceDir = "src"; return c },
"no restart command": func(c Config) Config { c.RestartCmd = nil; return c },
"no health socket": func(c Config) Config { c.HealthSocket = ""; return c },
"no binaries": func(c Config) Config { c.Binaries = nil; return c },
"escaping artifact name": func(c Config) Config { c.Binaries = []string{"../../etc/passwd"}; return c },
"absolute artifact name": func(c Config) Config { c.Binaries = []string{"/usr/bin/mavend"}; return c },
"snapshots inside install": func(c Config) Config { c.SnapshotDir = filepath.Join(c.InstallDir, "snaps"); return c },
}
for name, mutate := range bad {
if err := mutate(ok).Validate(); err == nil {
t.Errorf("%s was accepted; want a startup failure", name)
}
}
// And New refuses an invalid config outright rather than half-configuring.
if _, err := New(mutate(ok, "no health socket", bad)); err == nil {
t.Error("New accepted a config with no health socket")
}
}
func mutate(c Config, key string, m map[string]func(Config) Config) Config { return m[key](c) }