810076451f
On the deployment deploy/README.md documents, source_dir and install_dir are
the same tree and the restart command rebuilds the image from it. The
Dockerfile builds from cmd/ and internal/ and .dockerignore keeps the host
binaries out, so restoring the snapshotted binaries restored bytes nothing
reads. A bad commit therefore cost two health timeouts and two image builds
and ended in ErrRollbackFailed with an instruction to copy files back by hand,
which would not have helped either.
A deployment that rebuilds from source now has to say how the source is put
back. source_rollback "git" records the commit before the update and checks it
back out before the rollback restart. It refuses a dirty tree, because the
recorded commit does not describe one and a forced checkout would delete his
work. A build-from-source config that says nothing is refused by Validate, at
startup, rather than at the one rollback that mattered.
Also in this change, all from the same review:
- MethodPing, the one method a locked daemon answers. Preflight passed on an
unlocked daemon and the post-restart Presence read failed on a locked one,
so a good update read as SHE IS PROBABLY DOWN once the env key is gone.
- A dial failure is reported apart from a read failure. The documented
socket is under /var/lib/docker, which a non-root operator cannot
traverse, and "she is not answering" was the wrong diagnosis.
- Verify refuses to run as root over a tree owned by someone else. It runs
make build and make test in place, and root-owned artifacts break his next
ordinary make.
- A rollback no longer reverts config_files. That undid every config edit
since the last apply, phraser.model_path among them.
- The verify-failure path no longer reports rolled_back for a compile error.
- waitHealthy caps each attempt at the remaining budget, so a 90s timeout
cannot run to 99s.
- tail cuts on a rune boundary. Russian test names showed the seam.
- The claim that mavend does not import internal/update is replaced with
what is enforced: mavend constructs no Updater and nothing can call Apply.
- snapshot_dir inside source_dir is refused. It landed in the build context.
Found in review of #69.
477 lines
16 KiB
Go
477 lines
16 KiB
Go
package update
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// The tests drive the whole orchestration against a fake box: a directory tree
|
|
// standing in for the install dir, an injected Runner standing in for
|
|
// make/git/docker, and an injected HealthCheck standing in for mavend. That is
|
|
// what makes the failure paths — the ones that matter — testable at all: you
|
|
// cannot ask a real deployment to fail its health check on demand, and the
|
|
// rollback path is exactly the path nobody exercises by hand.
|
|
|
|
type fakeBox struct {
|
|
t *testing.T
|
|
root string
|
|
|
|
// what the fake `make build` writes into the source tree
|
|
newBytes string
|
|
// scripted failures
|
|
buildErr error
|
|
testErr error
|
|
restartErr error
|
|
|
|
// health: fails until the Nth call, then follows healthy
|
|
healthErrs int // remaining failures to serve
|
|
healthy bool
|
|
healthChecks int
|
|
// healthFn overrides the scripted behaviour entirely, for tests whose
|
|
// verdict depends on what the last restart actually deployed.
|
|
healthFn func() error
|
|
// deployedAtRestart records the installed bytes each time restart runs, so a
|
|
// test can prove the rollback put the old bytes back BEFORE restarting.
|
|
deployedAtRestart []string
|
|
// builtFromSource records the source the restart would have built an image
|
|
// from, each time it runs.
|
|
builtFromSource []string
|
|
|
|
ran []string
|
|
|
|
// The git half, for the deployment whose restart rebuilds from the tree.
|
|
// commit is HEAD; byCommit is what each commit's source says; dirty makes
|
|
// `git status --porcelain` report uncommitted work.
|
|
commit string
|
|
byCommit map[string]string
|
|
dirty bool
|
|
}
|
|
|
|
func newFakeBox(t *testing.T) *fakeBox {
|
|
t.Helper()
|
|
root := t.TempDir()
|
|
for _, d := range []string{"src", "install", "snapshots"} {
|
|
if err := os.MkdirAll(filepath.Join(root, d), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
// The currently deployed build, and a config file next to it.
|
|
write(t, filepath.Join(root, "install", "mavend"), "OLD-BUILD")
|
|
write(t, filepath.Join(root, "install", "mavend.json"), `{"tick_interval":"60s"}`)
|
|
// The source tree already contains a stale binary; `make build` overwrites it.
|
|
write(t, filepath.Join(root, "src", "mavend"), "STALE")
|
|
return &fakeBox{
|
|
t: t, root: root, newBytes: "NEW-BUILD", healthy: true,
|
|
commit: "cafebabecafebabecafebabecafebabecafebabe",
|
|
byCommit: map[string]string{},
|
|
}
|
|
}
|
|
|
|
func (b *fakeBox) cfg() Config {
|
|
return Config{
|
|
SourceDir: filepath.Join(b.root, "src"),
|
|
InstallDir: filepath.Join(b.root, "install"),
|
|
SnapshotDir: filepath.Join(b.root, "snapshots"),
|
|
Binaries: []string{"mavend"},
|
|
ConfigFiles: []string{"mavend.json"},
|
|
RestartCmd: []string{"restart-the-thing"},
|
|
HealthSocket: filepath.Join(b.root, "mavend.sock"),
|
|
HealthTimeoutSec: 1,
|
|
KeepSnapshots: 3,
|
|
}
|
|
}
|
|
|
|
func (b *fakeBox) run(ctx context.Context, dir string, argv []string) (string, error) {
|
|
b.ran = append(b.ran, strings.Join(argv, " "))
|
|
switch strings.Join(argv, " ") {
|
|
case "make build":
|
|
if b.buildErr != nil {
|
|
return "ld: undefined reference to everything", b.buildErr
|
|
}
|
|
// A real build writes its artifacts into the working tree — the behaviour
|
|
// the snapshot-before-build ordering exists to survive.
|
|
write(b.t, filepath.Join(b.root, "src", "mavend"), b.newBytes)
|
|
return "built", nil
|
|
case "make test":
|
|
if b.testErr != nil {
|
|
return "--- FAIL: TestSomething", b.testErr
|
|
}
|
|
return "ok", nil
|
|
case "git rev-parse HEAD":
|
|
return b.commit + "\n", nil
|
|
case "git status --porcelain":
|
|
if b.dirty {
|
|
return " M internal/router/router.go\n", nil
|
|
}
|
|
return "", nil
|
|
case "restart-the-thing":
|
|
b.deployedAtRestart = append(b.deployedAtRestart, read(b.t, filepath.Join(b.root, "install", "mavend")))
|
|
// The docker shape: the restart rebuilds the image from the tree, so
|
|
// what it deploys is the source, not any binary on the host.
|
|
if src, err := os.ReadFile(filepath.Join(b.root, "src", "source.go")); err == nil {
|
|
b.builtFromSource = append(b.builtFromSource, string(src))
|
|
}
|
|
if b.restartErr != nil {
|
|
return "no such container", b.restartErr
|
|
}
|
|
return "restarted", nil
|
|
}
|
|
if len(argv) == 4 && argv[0] == "git" && argv[1] == "checkout" && argv[2] == "--force" {
|
|
content, ok := b.byCommit[argv[3]]
|
|
if !ok {
|
|
return "error: pathspec did not match", errors.New("exit status 1")
|
|
}
|
|
b.commit = argv[3]
|
|
write(b.t, filepath.Join(b.root, "src", "source.go"), content)
|
|
return "HEAD is now at " + argv[3], nil
|
|
}
|
|
return "", errors.New("unexpected command: " + strings.Join(argv, " "))
|
|
}
|
|
|
|
func (b *fakeBox) health(ctx context.Context, socket string) error {
|
|
b.healthChecks++
|
|
if b.healthFn != nil {
|
|
return b.healthFn()
|
|
}
|
|
if b.healthErrs > 0 {
|
|
b.healthErrs--
|
|
return errors.New("connection refused")
|
|
}
|
|
if !b.healthy {
|
|
return errors.New("she does not answer")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (b *fakeBox) updater(t *testing.T, extra ...Option) *Updater {
|
|
t.Helper()
|
|
opts := append([]Option{WithRunner(b.run), WithHealth(b.health)}, extra...)
|
|
u, err := New(b.cfg(), opts...)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return u
|
|
}
|
|
|
|
func (b *fakeBox) deployed() string { return read(b.t, filepath.Join(b.root, "install", "mavend")) }
|
|
|
|
func write(t *testing.T, path, content string) {
|
|
t.Helper()
|
|
if err := os.WriteFile(path, []byte(content), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func read(t *testing.T, path string) string {
|
|
t.Helper()
|
|
b, err := os.ReadFile(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return string(b)
|
|
}
|
|
|
|
func TestApply_HappyPath(t *testing.T) {
|
|
b := newFakeBox(t)
|
|
res, err := b.updater(t).Apply(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("Apply: %v", err)
|
|
}
|
|
if !res.Verified || !res.Restarted || !res.Healthy || res.RolledBack {
|
|
t.Fatalf("result = %+v; want verified+restarted+healthy and no rollback", res)
|
|
}
|
|
if got := b.deployed(); got != "NEW-BUILD" {
|
|
t.Errorf("deployed binary = %q; want the new build", got)
|
|
}
|
|
// The order is the property: health, snapshot, build, test, install, restart.
|
|
want := []string{"git rev-parse HEAD", "make build", "make test", "restart-the-thing"}
|
|
if strings.Join(b.ran, "|") != strings.Join(want, "|") {
|
|
t.Errorf("commands ran = %v; want %v", b.ran, want)
|
|
}
|
|
if res.SnapshotID == "" {
|
|
t.Error("no snapshot was taken")
|
|
}
|
|
}
|
|
|
|
func TestApply_RefusesWhenSheIsAlreadyDown(t *testing.T) {
|
|
// A box that is already broken has no baseline for the rollback to prove
|
|
// itself against, so the update never starts.
|
|
b := newFakeBox(t)
|
|
b.healthy = false
|
|
res, err := b.updater(t).Apply(context.Background())
|
|
if !errors.Is(err, ErrUnhealthyBefore) {
|
|
t.Fatalf("Apply on an unhealthy box = %v; want ErrUnhealthyBefore", err)
|
|
}
|
|
if len(b.ran) != 0 {
|
|
t.Errorf("a refused update still ran %v", b.ran)
|
|
}
|
|
if res.SnapshotID != "" {
|
|
t.Error("a refused update still took a snapshot")
|
|
}
|
|
}
|
|
|
|
func TestApply_TestFailureDeploysNothingAndPutsTheTreeBack(t *testing.T) {
|
|
b := newFakeBox(t)
|
|
b.testErr = errors.New("exit status 1")
|
|
res, err := b.updater(t).Apply(context.Background())
|
|
if !errors.Is(err, ErrVerifyFailed) {
|
|
t.Fatalf("Apply with failing tests = %v; want ErrVerifyFailed", err)
|
|
}
|
|
if res.Verified {
|
|
t.Error("result claims verified after a failing test suite")
|
|
}
|
|
for _, c := range b.ran {
|
|
if c == "restart-the-thing" {
|
|
t.Fatal("a failed verification restarted the daemon")
|
|
}
|
|
}
|
|
if got := b.deployed(); got != "OLD-BUILD" {
|
|
t.Errorf("deployed binary = %q; want the old build untouched", got)
|
|
}
|
|
// The failing output is kept so the operator can see why.
|
|
var found bool
|
|
for _, s := range res.Steps {
|
|
if s.Name == "test" && strings.Contains(s.Output, "FAIL") {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
t.Error("the failing test output was not retained")
|
|
}
|
|
}
|
|
|
|
func TestApply_BuildFailureIsCaughtBeforeTheTests(t *testing.T) {
|
|
b := newFakeBox(t)
|
|
b.buildErr = errors.New("exit status 2")
|
|
if _, err := b.updater(t).Apply(context.Background()); !errors.Is(err, ErrVerifyFailed) {
|
|
t.Fatalf("Apply with a failing build = %v; want ErrVerifyFailed", err)
|
|
}
|
|
for _, c := range b.ran {
|
|
if c == "make test" {
|
|
t.Error("ran the test suite after the build failed")
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestApply_UnhealthyAfterRestartRollsBackToTheOldBytes(t *testing.T) {
|
|
// The case the package exists for: everything verifies, the new build
|
|
// installs, and then she does not come up.
|
|
b := newFakeBox(t)
|
|
b.healthErrs = 1 // the preflight check passes, then she stops answering
|
|
b.healthy = false
|
|
u := b.updater(t)
|
|
// Once the rollback restores the old build, she answers again.
|
|
restored := false
|
|
u.health = func(ctx context.Context, socket string) error {
|
|
b.healthChecks++
|
|
if b.deployed() == "OLD-BUILD" && restored {
|
|
return nil
|
|
}
|
|
if b.healthChecks == 1 {
|
|
return nil // preflight: the old build is up
|
|
}
|
|
if b.deployed() == "OLD-BUILD" {
|
|
restored = true
|
|
return nil
|
|
}
|
|
return errors.New("she does not answer on the new build")
|
|
}
|
|
res, err := u.Apply(context.Background())
|
|
if !errors.Is(err, ErrRolledBack) {
|
|
t.Fatalf("Apply with a dead new build = %v; want ErrRolledBack", err)
|
|
}
|
|
if !res.RolledBack || !res.RollbackHealthy || res.Healthy {
|
|
t.Fatalf("result = %+v; want rolled back and healthy again on the old build", res)
|
|
}
|
|
if got := b.deployed(); got != "OLD-BUILD" {
|
|
t.Errorf("deployed binary after the rollback = %q; want OLD-BUILD", got)
|
|
}
|
|
// And the restore happened BEFORE the second restart, not after it.
|
|
if len(b.deployedAtRestart) != 2 {
|
|
t.Fatalf("restarts = %v; want two (the update and the rollback)", b.deployedAtRestart)
|
|
}
|
|
if b.deployedAtRestart[0] != "NEW-BUILD" || b.deployedAtRestart[1] != "OLD-BUILD" {
|
|
t.Errorf("bytes in place at each restart = %v; want [NEW-BUILD OLD-BUILD]", b.deployedAtRestart)
|
|
}
|
|
}
|
|
|
|
func TestApply_RestartFailureRollsBack(t *testing.T) {
|
|
b := newFakeBox(t)
|
|
b.restartErr = errors.New("exit status 1")
|
|
res, err := b.updater(t).Apply(context.Background())
|
|
// The rollback's own restart fails too, so this is the manual-recovery case —
|
|
// and it says so instead of reporting a tidy rollback.
|
|
if !errors.Is(err, ErrRollbackFailed) {
|
|
t.Fatalf("Apply with a broken restart command = %v; want ErrRollbackFailed", err)
|
|
}
|
|
if !res.RolledBack || res.RollbackHealthy {
|
|
t.Fatalf("result = %+v; want rolled back but not healthy", res)
|
|
}
|
|
if got := b.deployed(); got != "OLD-BUILD" {
|
|
t.Errorf("deployed binary = %q; want the old bytes restored even so", got)
|
|
}
|
|
}
|
|
|
|
func TestApply_RollbackNeedsNoBuildAndNoNewCode(t *testing.T) {
|
|
// The rollback must not depend on the toolchain, the source tree, or the
|
|
// code it is replacing. Prove it: delete the source tree's binary and make
|
|
// every command except the restart fail, then roll back.
|
|
b := newFakeBox(t)
|
|
if _, err := b.updater(t).Apply(context.Background()); err != nil {
|
|
t.Fatalf("setup Apply: %v", err)
|
|
}
|
|
if b.deployed() != "NEW-BUILD" {
|
|
t.Fatal("setup did not deploy")
|
|
}
|
|
os.RemoveAll(filepath.Join(b.root, "src"))
|
|
if err := os.MkdirAll(filepath.Join(b.root, "src"), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
b.buildErr = errors.New("no toolchain here")
|
|
b.testErr = errors.New("no toolchain here")
|
|
b.ran = nil
|
|
|
|
res, err := b.updater(t).Rollback(context.Background(), "")
|
|
if err != nil && !errors.Is(err, ErrRolledBack) {
|
|
t.Fatalf("Rollback: %v", err)
|
|
}
|
|
if !res.RollbackHealthy {
|
|
t.Fatalf("result = %+v; want a healthy rollback", res)
|
|
}
|
|
if got := b.deployed(); got != "OLD-BUILD" {
|
|
t.Errorf("deployed binary = %q; want OLD-BUILD", got)
|
|
}
|
|
for _, c := range b.ran {
|
|
if strings.HasPrefix(c, "make") {
|
|
t.Errorf("the rollback ran %q — it must not need a build", c)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestApply_ConfigIsSnapshottedButNeverOverwritten(t *testing.T) {
|
|
b := newFakeBox(t)
|
|
// A config in the source tree must not be deployed over the operator's.
|
|
write(t, filepath.Join(b.root, "src", "mavend.json"), `{"tick_interval":"1s"}`)
|
|
if _, err := b.updater(t).Apply(context.Background()); err != nil {
|
|
t.Fatalf("Apply: %v", err)
|
|
}
|
|
if got := read(t, filepath.Join(b.root, "install", "mavend.json")); !strings.Contains(got, "60s") {
|
|
t.Errorf("installed config = %q; an update must not replace his config", got)
|
|
}
|
|
snaps, err := b.updater(t).Snapshots()
|
|
if err != nil || len(snaps) == 0 {
|
|
t.Fatalf("Snapshots: %v %v", snaps, err)
|
|
}
|
|
var names []string
|
|
for _, f := range snaps[0].Files {
|
|
names = append(names, f.Name)
|
|
}
|
|
if len(names) != 2 {
|
|
t.Errorf("snapshot files = %v; want the binary and the config", names)
|
|
}
|
|
if snaps[0].Commit == "" {
|
|
t.Error("the snapshot did not record which commit produced it")
|
|
}
|
|
}
|
|
|
|
func TestRollback_CorruptSnapshotIsRefusedNotRestored(t *testing.T) {
|
|
b := newFakeBox(t)
|
|
if _, err := b.updater(t).Apply(context.Background()); err != nil {
|
|
t.Fatalf("setup Apply: %v", err)
|
|
}
|
|
snaps, _ := b.updater(t).Snapshots()
|
|
// Something ate the snapshot. Restoring it would deploy garbage.
|
|
write(t, filepath.Join(snaps[0].Dir(), "mavend"), "CORRUPT")
|
|
_, err := b.updater(t).Rollback(context.Background(), snaps[0].ID)
|
|
if !errors.Is(err, ErrRollbackFailed) || !strings.Contains(err.Error(), "corrupt") {
|
|
t.Fatalf("Rollback of a corrupt snapshot = %v; want a refusal naming the corruption", err)
|
|
}
|
|
if got := b.deployed(); got != "NEW-BUILD" {
|
|
t.Errorf("deployed binary = %q; a refused restore must change nothing", got)
|
|
}
|
|
}
|
|
|
|
func TestRollback_NoSnapshots(t *testing.T) {
|
|
b := newFakeBox(t)
|
|
if _, err := b.updater(t).Rollback(context.Background(), ""); err == nil {
|
|
t.Error("Rollback with no snapshots succeeded; want an error")
|
|
}
|
|
}
|
|
|
|
func TestPrune_KeepsTheNewestAsTheRollbackTarget(t *testing.T) {
|
|
b := newFakeBox(t)
|
|
st := &Store{Dir: filepath.Join(b.root, "snapshots")}
|
|
base := time.Date(2026, 8, 1, 3, 0, 0, 0, time.UTC)
|
|
for i := 0; i < 4; i++ {
|
|
i := i
|
|
st.now = func() time.Time { return base.Add(time.Duration(i) * time.Minute) }
|
|
if _, err := st.Save(filepath.Join(b.root, "install"), []string{"mavend"}, "", ""); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := st.Prune(0); err != nil { // 0 is clamped to 1, never to zero
|
|
t.Fatal(err)
|
|
}
|
|
snaps, err := st.List()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(snaps) != 1 {
|
|
t.Fatalf("kept %d snapshots; want 1", len(snaps))
|
|
}
|
|
if snaps[0].ID != "20260801-030300" {
|
|
t.Errorf("kept %s; want the newest", snaps[0].ID)
|
|
}
|
|
}
|
|
|
|
func TestList_IgnoresSnapshotsWithNoManifest(t *testing.T) {
|
|
// An interrupted snapshot has files but no manifest. It must never be offered
|
|
// as a rollback target — restoring a half-copied binary is the worst outcome
|
|
// in the package.
|
|
b := newFakeBox(t)
|
|
dir := filepath.Join(b.root, "snapshots", "20260801-000000")
|
|
if err := os.MkdirAll(dir, 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
write(t, filepath.Join(dir, "mavend"), "HALF")
|
|
snaps, err := (&Store{Dir: filepath.Join(b.root, "snapshots")}).List()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(snaps) != 0 {
|
|
t.Errorf("List returned %d snapshots; want none (no manifest)", len(snaps))
|
|
}
|
|
}
|
|
|
|
func TestConfigValidate(t *testing.T) {
|
|
ok := (&fakeBox{root: t.TempDir()}).cfg()
|
|
if err := ok.Validate(); err != nil {
|
|
t.Fatalf("valid config rejected: %v", err)
|
|
}
|
|
bad := map[string]func(c Config) Config{
|
|
"relative source": func(c Config) Config { c.SourceDir = "src"; return c },
|
|
"no restart command": func(c Config) Config { c.RestartCmd = nil; return c },
|
|
"no health socket": func(c Config) Config { c.HealthSocket = ""; return c },
|
|
"no binaries": func(c Config) Config { c.Binaries = nil; return c },
|
|
"escaping artifact name": func(c Config) Config { c.Binaries = []string{"../../etc/passwd"}; return c },
|
|
"absolute artifact name": func(c Config) Config { c.Binaries = []string{"/usr/bin/mavend"}; return c },
|
|
"snapshots inside install": func(c Config) Config { c.SnapshotDir = filepath.Join(c.InstallDir, "snaps"); return c },
|
|
}
|
|
for name, mutate := range bad {
|
|
if err := mutate(ok).Validate(); err == nil {
|
|
t.Errorf("%s was accepted; want a startup failure", name)
|
|
}
|
|
}
|
|
// And New refuses an invalid config outright rather than half-configuring.
|
|
if _, err := New(mutate(ok, "no health socket", bad)); err == nil {
|
|
t.Error("New accepted a config with no health socket")
|
|
}
|
|
}
|
|
|
|
func mutate(c Config, key string, m map[string]func(Config) Config) Config { return m[key](c) }
|