Files
Maven/internal/update/update_test.go
kami be066a4b04 Deploy a new build with verification and automatic rollback (#249)
internal/update applies a new build of Maven to the box she runs on and
undoes it when the new build does not come up. cmd/mavupdate is the only
trigger: a CLI the owner runs on the host.

Apply is health-check the running daemon, snapshot the deployed artifacts,
make build, make test, install, restart, health-check — and restore the
snapshot on any failure. The order is load-bearing:

  - The preflight health check refuses to update a daemon that is already
    not answering. Without a working baseline, a failed update and a box
    that was already broken are indistinguishable, and the rollback has
    nothing to prove itself against.
  - The snapshot is taken BEFORE the build, because make build writes its
    binaries into the working tree and on the docker deployment the tree
    is the install dir — snapshotting afterwards would snapshot the new
    artifacts and leave nothing to roll back to.
  - Verification is make build plus make test, before anything is
    deployed, so a broken tree costs time and nothing else. A failed
    verify also puts the tree's artifacts back, so a later restart by
    hand cannot deploy code that failed its own tests.
  - The rollback depends on nothing that just changed: byte-for-byte
    copies out of the snapshot dir, sha256-verified on the way in, and
    the same restart command. No build, no migration, no cooperation from
    the code being replaced. It also runs on an uncancellable context —
    a rollback interrupted halfway is worse than the failure that caused
    it. When the restore itself fails it says so and names the directory
    to copy back by hand rather than reporting a tidy rollback.

Off unless configured, and the refusals are code, not documentation. The
daemon does not import this package: there is no IPC method, no web route,
no timer and no act that can start an update, so nothing Maven says or
routes reaches it. Nothing fetches code — the new version is whatever the
owner pulled into the tree. The plan's release checker, auto-update
channel and in-process crash-loop supervisor are deliberately absent; a
process cannot reliably notice that it keeps dying, and restart-on-crash
belongs to compose or systemd. The database is never snapshotted or rolled
back; schema compatibility stays store.Migrate's job.

The config is refused at load without a health socket, since an update
that cannot check its own result cannot roll back, and refused when the
snapshot dir is inside the install dir, since a restore must not read from
what the install writes.

Vikunja #249
2026-08-01 04:09:30 +04:00

438 lines
14 KiB
Go

package update
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
// The tests drive the whole orchestration against a fake box: a directory tree
// standing in for the install dir, an injected Runner standing in for
// make/git/docker, and an injected HealthCheck standing in for mavend. That is
// what makes the failure paths — the ones that matter — testable at all: you
// cannot ask a real deployment to fail its health check on demand, and the
// rollback path is exactly the path nobody exercises by hand.
type fakeBox struct {
t *testing.T
root string
// what the fake `make build` writes into the source tree
newBytes string
// scripted failures
buildErr error
testErr error
restartErr error
// health: fails until the Nth call, then follows healthy
healthErrs int // remaining failures to serve
healthy bool
healthChecks int
// deployedAtRestart records the installed bytes each time restart runs, so a
// test can prove the rollback put the old bytes back BEFORE restarting.
deployedAtRestart []string
ran []string
}
func newFakeBox(t *testing.T) *fakeBox {
t.Helper()
root := t.TempDir()
for _, d := range []string{"src", "install", "snapshots"} {
if err := os.MkdirAll(filepath.Join(root, d), 0o755); err != nil {
t.Fatal(err)
}
}
// The currently deployed build, and a config file next to it.
write(t, filepath.Join(root, "install", "mavend"), "OLD-BUILD")
write(t, filepath.Join(root, "install", "mavend.json"), `{"tick_interval":"60s"}`)
// The source tree already contains a stale binary; `make build` overwrites it.
write(t, filepath.Join(root, "src", "mavend"), "STALE")
return &fakeBox{t: t, root: root, newBytes: "NEW-BUILD", healthy: true}
}
func (b *fakeBox) cfg() Config {
return Config{
SourceDir: filepath.Join(b.root, "src"),
InstallDir: filepath.Join(b.root, "install"),
SnapshotDir: filepath.Join(b.root, "snapshots"),
Binaries: []string{"mavend"},
ConfigFiles: []string{"mavend.json"},
RestartCmd: []string{"restart-the-thing"},
HealthSocket: filepath.Join(b.root, "mavend.sock"),
HealthTimeoutSec: 1,
KeepSnapshots: 3,
}
}
func (b *fakeBox) run(ctx context.Context, dir string, argv []string) (string, error) {
b.ran = append(b.ran, strings.Join(argv, " "))
switch strings.Join(argv, " ") {
case "make build":
if b.buildErr != nil {
return "ld: undefined reference to everything", b.buildErr
}
// A real build writes its artifacts into the working tree — the behaviour
// the snapshot-before-build ordering exists to survive.
write(b.t, filepath.Join(b.root, "src", "mavend"), b.newBytes)
return "built", nil
case "make test":
if b.testErr != nil {
return "--- FAIL: TestSomething", b.testErr
}
return "ok", nil
case "git rev-parse HEAD":
return "cafebabecafebabecafebabecafebabecafebabe\n", nil
case "restart-the-thing":
b.deployedAtRestart = append(b.deployedAtRestart, read(b.t, filepath.Join(b.root, "install", "mavend")))
if b.restartErr != nil {
return "no such container", b.restartErr
}
return "restarted", nil
}
return "", errors.New("unexpected command: " + strings.Join(argv, " "))
}
func (b *fakeBox) health(ctx context.Context, socket string) error {
b.healthChecks++
if b.healthErrs > 0 {
b.healthErrs--
return errors.New("connection refused")
}
if !b.healthy {
return errors.New("she does not answer")
}
return nil
}
func (b *fakeBox) updater(t *testing.T, extra ...Option) *Updater {
t.Helper()
opts := append([]Option{WithRunner(b.run), WithHealth(b.health)}, extra...)
u, err := New(b.cfg(), opts...)
if err != nil {
t.Fatal(err)
}
return u
}
func (b *fakeBox) deployed() string { return read(b.t, filepath.Join(b.root, "install", "mavend")) }
func write(t *testing.T, path, content string) {
t.Helper()
if err := os.WriteFile(path, []byte(content), 0o755); err != nil {
t.Fatal(err)
}
}
func read(t *testing.T, path string) string {
t.Helper()
b, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
return string(b)
}
func TestApply_HappyPath(t *testing.T) {
b := newFakeBox(t)
res, err := b.updater(t).Apply(context.Background())
if err != nil {
t.Fatalf("Apply: %v", err)
}
if !res.Verified || !res.Restarted || !res.Healthy || res.RolledBack {
t.Fatalf("result = %+v; want verified+restarted+healthy and no rollback", res)
}
if got := b.deployed(); got != "NEW-BUILD" {
t.Errorf("deployed binary = %q; want the new build", got)
}
// The order is the property: health, snapshot, build, test, install, restart.
want := []string{"git rev-parse HEAD", "make build", "make test", "restart-the-thing"}
if strings.Join(b.ran, "|") != strings.Join(want, "|") {
t.Errorf("commands ran = %v; want %v", b.ran, want)
}
if res.SnapshotID == "" {
t.Error("no snapshot was taken")
}
}
func TestApply_RefusesWhenSheIsAlreadyDown(t *testing.T) {
// A box that is already broken has no baseline for the rollback to prove
// itself against, so the update never starts.
b := newFakeBox(t)
b.healthy = false
res, err := b.updater(t).Apply(context.Background())
if !errors.Is(err, ErrUnhealthyBefore) {
t.Fatalf("Apply on an unhealthy box = %v; want ErrUnhealthyBefore", err)
}
if len(b.ran) != 0 {
t.Errorf("a refused update still ran %v", b.ran)
}
if res.SnapshotID != "" {
t.Error("a refused update still took a snapshot")
}
}
func TestApply_TestFailureDeploysNothingAndPutsTheTreeBack(t *testing.T) {
b := newFakeBox(t)
b.testErr = errors.New("exit status 1")
res, err := b.updater(t).Apply(context.Background())
if !errors.Is(err, ErrVerifyFailed) {
t.Fatalf("Apply with failing tests = %v; want ErrVerifyFailed", err)
}
if res.Verified {
t.Error("result claims verified after a failing test suite")
}
for _, c := range b.ran {
if c == "restart-the-thing" {
t.Fatal("a failed verification restarted the daemon")
}
}
if got := b.deployed(); got != "OLD-BUILD" {
t.Errorf("deployed binary = %q; want the old build untouched", got)
}
// The failing output is kept so the operator can see why.
var found bool
for _, s := range res.Steps {
if s.Name == "test" && strings.Contains(s.Output, "FAIL") {
found = true
}
}
if !found {
t.Error("the failing test output was not retained")
}
}
func TestApply_BuildFailureIsCaughtBeforeTheTests(t *testing.T) {
b := newFakeBox(t)
b.buildErr = errors.New("exit status 2")
if _, err := b.updater(t).Apply(context.Background()); !errors.Is(err, ErrVerifyFailed) {
t.Fatalf("Apply with a failing build = %v; want ErrVerifyFailed", err)
}
for _, c := range b.ran {
if c == "make test" {
t.Error("ran the test suite after the build failed")
}
}
}
func TestApply_UnhealthyAfterRestartRollsBackToTheOldBytes(t *testing.T) {
// The case the package exists for: everything verifies, the new build
// installs, and then she does not come up.
b := newFakeBox(t)
b.healthErrs = 1 // the preflight check passes, then she stops answering
b.healthy = false
u := b.updater(t)
// Once the rollback restores the old build, she answers again.
restored := false
u.health = func(ctx context.Context, socket string) error {
b.healthChecks++
if b.deployed() == "OLD-BUILD" && restored {
return nil
}
if b.healthChecks == 1 {
return nil // preflight: the old build is up
}
if b.deployed() == "OLD-BUILD" {
restored = true
return nil
}
return errors.New("she does not answer on the new build")
}
res, err := u.Apply(context.Background())
if !errors.Is(err, ErrRolledBack) {
t.Fatalf("Apply with a dead new build = %v; want ErrRolledBack", err)
}
if !res.RolledBack || !res.RollbackHealthy || res.Healthy {
t.Fatalf("result = %+v; want rolled back and healthy again on the old build", res)
}
if got := b.deployed(); got != "OLD-BUILD" {
t.Errorf("deployed binary after the rollback = %q; want OLD-BUILD", got)
}
// And the restore happened BEFORE the second restart, not after it.
if len(b.deployedAtRestart) != 2 {
t.Fatalf("restarts = %v; want two (the update and the rollback)", b.deployedAtRestart)
}
if b.deployedAtRestart[0] != "NEW-BUILD" || b.deployedAtRestart[1] != "OLD-BUILD" {
t.Errorf("bytes in place at each restart = %v; want [NEW-BUILD OLD-BUILD]", b.deployedAtRestart)
}
}
func TestApply_RestartFailureRollsBack(t *testing.T) {
b := newFakeBox(t)
b.restartErr = errors.New("exit status 1")
res, err := b.updater(t).Apply(context.Background())
// The rollback's own restart fails too, so this is the manual-recovery case —
// and it says so instead of reporting a tidy rollback.
if !errors.Is(err, ErrRollbackFailed) {
t.Fatalf("Apply with a broken restart command = %v; want ErrRollbackFailed", err)
}
if !res.RolledBack || res.RollbackHealthy {
t.Fatalf("result = %+v; want rolled back but not healthy", res)
}
if got := b.deployed(); got != "OLD-BUILD" {
t.Errorf("deployed binary = %q; want the old bytes restored even so", got)
}
}
func TestApply_RollbackNeedsNoBuildAndNoNewCode(t *testing.T) {
// The rollback must not depend on the toolchain, the source tree, or the
// code it is replacing. Prove it: delete the source tree's binary and make
// every command except the restart fail, then roll back.
b := newFakeBox(t)
if _, err := b.updater(t).Apply(context.Background()); err != nil {
t.Fatalf("setup Apply: %v", err)
}
if b.deployed() != "NEW-BUILD" {
t.Fatal("setup did not deploy")
}
os.RemoveAll(filepath.Join(b.root, "src"))
if err := os.MkdirAll(filepath.Join(b.root, "src"), 0o755); err != nil {
t.Fatal(err)
}
b.buildErr = errors.New("no toolchain here")
b.testErr = errors.New("no toolchain here")
b.ran = nil
res, err := b.updater(t).Rollback(context.Background(), "")
if err != nil && !errors.Is(err, ErrRolledBack) {
t.Fatalf("Rollback: %v", err)
}
if !res.RollbackHealthy {
t.Fatalf("result = %+v; want a healthy rollback", res)
}
if got := b.deployed(); got != "OLD-BUILD" {
t.Errorf("deployed binary = %q; want OLD-BUILD", got)
}
for _, c := range b.ran {
if strings.HasPrefix(c, "make") {
t.Errorf("the rollback ran %q — it must not need a build", c)
}
}
}
func TestApply_ConfigIsSnapshottedButNeverOverwritten(t *testing.T) {
b := newFakeBox(t)
// A config in the source tree must not be deployed over the operator's.
write(t, filepath.Join(b.root, "src", "mavend.json"), `{"tick_interval":"1s"}`)
if _, err := b.updater(t).Apply(context.Background()); err != nil {
t.Fatalf("Apply: %v", err)
}
if got := read(t, filepath.Join(b.root, "install", "mavend.json")); !strings.Contains(got, "60s") {
t.Errorf("installed config = %q; an update must not replace his config", got)
}
snaps, err := b.updater(t).Snapshots()
if err != nil || len(snaps) == 0 {
t.Fatalf("Snapshots: %v %v", snaps, err)
}
var names []string
for _, f := range snaps[0].Files {
names = append(names, f.Name)
}
if len(names) != 2 {
t.Errorf("snapshot files = %v; want the binary and the config", names)
}
if snaps[0].Commit == "" {
t.Error("the snapshot did not record which commit produced it")
}
}
func TestRollback_CorruptSnapshotIsRefusedNotRestored(t *testing.T) {
b := newFakeBox(t)
if _, err := b.updater(t).Apply(context.Background()); err != nil {
t.Fatalf("setup Apply: %v", err)
}
snaps, _ := b.updater(t).Snapshots()
// Something ate the snapshot. Restoring it would deploy garbage.
write(t, filepath.Join(snaps[0].Dir(), "mavend"), "CORRUPT")
_, err := b.updater(t).Rollback(context.Background(), snaps[0].ID)
if !errors.Is(err, ErrRollbackFailed) || !strings.Contains(err.Error(), "corrupt") {
t.Fatalf("Rollback of a corrupt snapshot = %v; want a refusal naming the corruption", err)
}
if got := b.deployed(); got != "NEW-BUILD" {
t.Errorf("deployed binary = %q; a refused restore must change nothing", got)
}
}
func TestRollback_NoSnapshots(t *testing.T) {
b := newFakeBox(t)
if _, err := b.updater(t).Rollback(context.Background(), ""); err == nil {
t.Error("Rollback with no snapshots succeeded; want an error")
}
}
func TestPrune_KeepsTheNewestAsTheRollbackTarget(t *testing.T) {
b := newFakeBox(t)
st := &Store{Dir: filepath.Join(b.root, "snapshots")}
base := time.Date(2026, 8, 1, 3, 0, 0, 0, time.UTC)
for i := 0; i < 4; i++ {
i := i
st.now = func() time.Time { return base.Add(time.Duration(i) * time.Minute) }
if _, err := st.Save(filepath.Join(b.root, "install"), []string{"mavend"}, "", ""); err != nil {
t.Fatal(err)
}
}
if err := st.Prune(0); err != nil { // 0 is clamped to 1, never to zero
t.Fatal(err)
}
snaps, err := st.List()
if err != nil {
t.Fatal(err)
}
if len(snaps) != 1 {
t.Fatalf("kept %d snapshots; want 1", len(snaps))
}
if snaps[0].ID != "20260801-030300" {
t.Errorf("kept %s; want the newest", snaps[0].ID)
}
}
func TestList_IgnoresSnapshotsWithNoManifest(t *testing.T) {
// An interrupted snapshot has files but no manifest. It must never be offered
// as a rollback target — restoring a half-copied binary is the worst outcome
// in the package.
b := newFakeBox(t)
dir := filepath.Join(b.root, "snapshots", "20260801-000000")
if err := os.MkdirAll(dir, 0o700); err != nil {
t.Fatal(err)
}
write(t, filepath.Join(dir, "mavend"), "HALF")
snaps, err := (&Store{Dir: filepath.Join(b.root, "snapshots")}).List()
if err != nil {
t.Fatal(err)
}
if len(snaps) != 0 {
t.Errorf("List returned %d snapshots; want none (no manifest)", len(snaps))
}
}
func TestConfigValidate(t *testing.T) {
ok := (&fakeBox{root: t.TempDir()}).cfg()
if err := ok.Validate(); err != nil {
t.Fatalf("valid config rejected: %v", err)
}
bad := map[string]func(c Config) Config{
"relative source": func(c Config) Config { c.SourceDir = "src"; return c },
"no restart command": func(c Config) Config { c.RestartCmd = nil; return c },
"no health socket": func(c Config) Config { c.HealthSocket = ""; return c },
"no binaries": func(c Config) Config { c.Binaries = nil; return c },
"escaping artifact name": func(c Config) Config { c.Binaries = []string{"../../etc/passwd"}; return c },
"absolute artifact name": func(c Config) Config { c.Binaries = []string{"/usr/bin/mavend"}; return c },
"snapshots inside install": func(c Config) Config { c.SnapshotDir = filepath.Join(c.InstallDir, "snaps"); return c },
}
for name, mutate := range bad {
if err := mutate(ok).Validate(); err == nil {
t.Errorf("%s was accepted; want a startup failure", name)
}
}
// And New refuses an invalid config outright rather than half-configuring.
if _, err := New(mutate(ok, "no health socket", bad)); err == nil {
t.Error("New accepted a config with no health socket")
}
}
func mutate(c Config, key string, m map[string]func(Config) Config) Config { return m[key](c) }