package update import ( "context" "errors" "os" "path/filepath" "strings" "testing" "time" ) // The tests drive the whole orchestration against a fake box: a directory tree // standing in for the install dir, an injected Runner standing in for // make/git/docker, and an injected HealthCheck standing in for mavend. That is // what makes the failure paths — the ones that matter — testable at all: you // cannot ask a real deployment to fail its health check on demand, and the // rollback path is exactly the path nobody exercises by hand. type fakeBox struct { t *testing.T root string // what the fake `make build` writes into the source tree newBytes string // scripted failures buildErr error testErr error restartErr error // health: fails until the Nth call, then follows healthy healthErrs int // remaining failures to serve healthy bool healthChecks int // healthFn overrides the scripted behaviour entirely, for tests whose // verdict depends on what the last restart actually deployed. healthFn func() error // deployedAtRestart records the installed bytes each time restart runs, so a // test can prove the rollback put the old bytes back BEFORE restarting. deployedAtRestart []string // builtFromSource records the source the restart would have built an image // from, each time it runs. builtFromSource []string ran []string // The git half, for the deployment whose restart rebuilds from the tree. // commit is HEAD; byCommit is what each commit's source says; dirty makes // `git status --porcelain` report uncommitted work. commit string byCommit map[string]string dirty bool } func newFakeBox(t *testing.T) *fakeBox { t.Helper() root := t.TempDir() for _, d := range []string{"src", "install", "snapshots"} { if err := os.MkdirAll(filepath.Join(root, d), 0o755); err != nil { t.Fatal(err) } } // The currently deployed build, and a config file next to it. write(t, filepath.Join(root, "install", "mavend"), "OLD-BUILD") write(t, filepath.Join(root, "install", "mavend.json"), `{"tick_interval":"60s"}`) // The source tree already contains a stale binary; `make build` overwrites it. write(t, filepath.Join(root, "src", "mavend"), "STALE") return &fakeBox{ t: t, root: root, newBytes: "NEW-BUILD", healthy: true, commit: "cafebabecafebabecafebabecafebabecafebabe", byCommit: map[string]string{}, } } func (b *fakeBox) cfg() Config { return Config{ SourceDir: filepath.Join(b.root, "src"), InstallDir: filepath.Join(b.root, "install"), SnapshotDir: filepath.Join(b.root, "snapshots"), Binaries: []string{"mavend"}, ConfigFiles: []string{"mavend.json"}, RestartCmd: []string{"restart-the-thing"}, HealthSocket: filepath.Join(b.root, "mavend.sock"), HealthTimeoutSec: 1, KeepSnapshots: 3, } } func (b *fakeBox) run(ctx context.Context, dir string, argv []string) (string, error) { b.ran = append(b.ran, strings.Join(argv, " ")) switch strings.Join(argv, " ") { case "make build": if b.buildErr != nil { return "ld: undefined reference to everything", b.buildErr } // A real build writes its artifacts into the working tree — the behaviour // the snapshot-before-build ordering exists to survive. write(b.t, filepath.Join(b.root, "src", "mavend"), b.newBytes) return "built", nil case "make test": if b.testErr != nil { return "--- FAIL: TestSomething", b.testErr } return "ok", nil case "git rev-parse HEAD": return b.commit + "\n", nil case "git status --porcelain": if b.dirty { return " M internal/router/router.go\n", nil } return "", nil case "restart-the-thing": b.deployedAtRestart = append(b.deployedAtRestart, read(b.t, filepath.Join(b.root, "install", "mavend"))) // The docker shape: the restart rebuilds the image from the tree, so // what it deploys is the source, not any binary on the host. if src, err := os.ReadFile(filepath.Join(b.root, "src", "source.go")); err == nil { b.builtFromSource = append(b.builtFromSource, string(src)) } if b.restartErr != nil { return "no such container", b.restartErr } return "restarted", nil } if len(argv) == 4 && argv[0] == "git" && argv[1] == "checkout" && argv[2] == "--force" { content, ok := b.byCommit[argv[3]] if !ok { return "error: pathspec did not match", errors.New("exit status 1") } b.commit = argv[3] write(b.t, filepath.Join(b.root, "src", "source.go"), content) return "HEAD is now at " + argv[3], nil } return "", errors.New("unexpected command: " + strings.Join(argv, " ")) } func (b *fakeBox) health(ctx context.Context, socket string) error { b.healthChecks++ if b.healthFn != nil { return b.healthFn() } if b.healthErrs > 0 { b.healthErrs-- return errors.New("connection refused") } if !b.healthy { return errors.New("she does not answer") } return nil } func (b *fakeBox) updater(t *testing.T, extra ...Option) *Updater { t.Helper() opts := append([]Option{WithRunner(b.run), WithHealth(b.health)}, extra...) u, err := New(b.cfg(), opts...) if err != nil { t.Fatal(err) } return u } func (b *fakeBox) deployed() string { return read(b.t, filepath.Join(b.root, "install", "mavend")) } func write(t *testing.T, path, content string) { t.Helper() if err := os.WriteFile(path, []byte(content), 0o755); err != nil { t.Fatal(err) } } func read(t *testing.T, path string) string { t.Helper() b, err := os.ReadFile(path) if err != nil { t.Fatal(err) } return string(b) } func TestApply_HappyPath(t *testing.T) { b := newFakeBox(t) res, err := b.updater(t).Apply(context.Background()) if err != nil { t.Fatalf("Apply: %v", err) } if !res.Verified || !res.Restarted || !res.Healthy || res.RolledBack { t.Fatalf("result = %+v; want verified+restarted+healthy and no rollback", res) } if got := b.deployed(); got != "NEW-BUILD" { t.Errorf("deployed binary = %q; want the new build", got) } // The order is the property: health, snapshot, build, test, install, restart. want := []string{"git rev-parse HEAD", "make build", "make test", "restart-the-thing"} if strings.Join(b.ran, "|") != strings.Join(want, "|") { t.Errorf("commands ran = %v; want %v", b.ran, want) } if res.SnapshotID == "" { t.Error("no snapshot was taken") } } func TestApply_RefusesWhenSheIsAlreadyDown(t *testing.T) { // A box that is already broken has no baseline for the rollback to prove // itself against, so the update never starts. b := newFakeBox(t) b.healthy = false res, err := b.updater(t).Apply(context.Background()) if !errors.Is(err, ErrUnhealthyBefore) { t.Fatalf("Apply on an unhealthy box = %v; want ErrUnhealthyBefore", err) } if len(b.ran) != 0 { t.Errorf("a refused update still ran %v", b.ran) } if res.SnapshotID != "" { t.Error("a refused update still took a snapshot") } } func TestApply_TestFailureDeploysNothingAndPutsTheTreeBack(t *testing.T) { b := newFakeBox(t) b.testErr = errors.New("exit status 1") res, err := b.updater(t).Apply(context.Background()) if !errors.Is(err, ErrVerifyFailed) { t.Fatalf("Apply with failing tests = %v; want ErrVerifyFailed", err) } if res.Verified { t.Error("result claims verified after a failing test suite") } for _, c := range b.ran { if c == "restart-the-thing" { t.Fatal("a failed verification restarted the daemon") } } if got := b.deployed(); got != "OLD-BUILD" { t.Errorf("deployed binary = %q; want the old build untouched", got) } // The failing output is kept so the operator can see why. var found bool for _, s := range res.Steps { if s.Name == "test" && strings.Contains(s.Output, "FAIL") { found = true } } if !found { t.Error("the failing test output was not retained") } } func TestApply_BuildFailureIsCaughtBeforeTheTests(t *testing.T) { b := newFakeBox(t) b.buildErr = errors.New("exit status 2") if _, err := b.updater(t).Apply(context.Background()); !errors.Is(err, ErrVerifyFailed) { t.Fatalf("Apply with a failing build = %v; want ErrVerifyFailed", err) } for _, c := range b.ran { if c == "make test" { t.Error("ran the test suite after the build failed") } } } func TestApply_UnhealthyAfterRestartRollsBackToTheOldBytes(t *testing.T) { // The case the package exists for: everything verifies, the new build // installs, and then she does not come up. b := newFakeBox(t) b.healthErrs = 1 // the preflight check passes, then she stops answering b.healthy = false u := b.updater(t) // Once the rollback restores the old build, she answers again. restored := false u.health = func(ctx context.Context, socket string) error { b.healthChecks++ if b.deployed() == "OLD-BUILD" && restored { return nil } if b.healthChecks == 1 { return nil // preflight: the old build is up } if b.deployed() == "OLD-BUILD" { restored = true return nil } return errors.New("she does not answer on the new build") } res, err := u.Apply(context.Background()) if !errors.Is(err, ErrRolledBack) { t.Fatalf("Apply with a dead new build = %v; want ErrRolledBack", err) } if !res.RolledBack || !res.RollbackHealthy || res.Healthy { t.Fatalf("result = %+v; want rolled back and healthy again on the old build", res) } if got := b.deployed(); got != "OLD-BUILD" { t.Errorf("deployed binary after the rollback = %q; want OLD-BUILD", got) } // And the restore happened BEFORE the second restart, not after it. if len(b.deployedAtRestart) != 2 { t.Fatalf("restarts = %v; want two (the update and the rollback)", b.deployedAtRestart) } if b.deployedAtRestart[0] != "NEW-BUILD" || b.deployedAtRestart[1] != "OLD-BUILD" { t.Errorf("bytes in place at each restart = %v; want [NEW-BUILD OLD-BUILD]", b.deployedAtRestart) } } func TestApply_RestartFailureRollsBack(t *testing.T) { b := newFakeBox(t) b.restartErr = errors.New("exit status 1") res, err := b.updater(t).Apply(context.Background()) // The rollback's own restart fails too, so this is the manual-recovery case — // and it says so instead of reporting a tidy rollback. if !errors.Is(err, ErrRollbackFailed) { t.Fatalf("Apply with a broken restart command = %v; want ErrRollbackFailed", err) } if !res.RolledBack || res.RollbackHealthy { t.Fatalf("result = %+v; want rolled back but not healthy", res) } if got := b.deployed(); got != "OLD-BUILD" { t.Errorf("deployed binary = %q; want the old bytes restored even so", got) } } func TestApply_RollbackNeedsNoBuildAndNoNewCode(t *testing.T) { // The rollback must not depend on the toolchain, the source tree, or the // code it is replacing. Prove it: delete the source tree's binary and make // every command except the restart fail, then roll back. b := newFakeBox(t) if _, err := b.updater(t).Apply(context.Background()); err != nil { t.Fatalf("setup Apply: %v", err) } if b.deployed() != "NEW-BUILD" { t.Fatal("setup did not deploy") } os.RemoveAll(filepath.Join(b.root, "src")) if err := os.MkdirAll(filepath.Join(b.root, "src"), 0o755); err != nil { t.Fatal(err) } b.buildErr = errors.New("no toolchain here") b.testErr = errors.New("no toolchain here") b.ran = nil res, err := b.updater(t).Rollback(context.Background(), "") if err != nil && !errors.Is(err, ErrRolledBack) { t.Fatalf("Rollback: %v", err) } if !res.RollbackHealthy { t.Fatalf("result = %+v; want a healthy rollback", res) } if got := b.deployed(); got != "OLD-BUILD" { t.Errorf("deployed binary = %q; want OLD-BUILD", got) } for _, c := range b.ran { if strings.HasPrefix(c, "make") { t.Errorf("the rollback ran %q — it must not need a build", c) } } } func TestApply_ConfigIsSnapshottedButNeverOverwritten(t *testing.T) { b := newFakeBox(t) // A config in the source tree must not be deployed over the operator's. write(t, filepath.Join(b.root, "src", "mavend.json"), `{"tick_interval":"1s"}`) if _, err := b.updater(t).Apply(context.Background()); err != nil { t.Fatalf("Apply: %v", err) } if got := read(t, filepath.Join(b.root, "install", "mavend.json")); !strings.Contains(got, "60s") { t.Errorf("installed config = %q; an update must not replace his config", got) } snaps, err := b.updater(t).Snapshots() if err != nil || len(snaps) == 0 { t.Fatalf("Snapshots: %v %v", snaps, err) } var names []string for _, f := range snaps[0].Files { names = append(names, f.Name) } if len(names) != 2 { t.Errorf("snapshot files = %v; want the binary and the config", names) } if snaps[0].Commit == "" { t.Error("the snapshot did not record which commit produced it") } } func TestRollback_CorruptSnapshotIsRefusedNotRestored(t *testing.T) { b := newFakeBox(t) if _, err := b.updater(t).Apply(context.Background()); err != nil { t.Fatalf("setup Apply: %v", err) } snaps, _ := b.updater(t).Snapshots() // Something ate the snapshot. Restoring it would deploy garbage. write(t, filepath.Join(snaps[0].Dir(), "mavend"), "CORRUPT") _, err := b.updater(t).Rollback(context.Background(), snaps[0].ID) if !errors.Is(err, ErrRollbackFailed) || !strings.Contains(err.Error(), "corrupt") { t.Fatalf("Rollback of a corrupt snapshot = %v; want a refusal naming the corruption", err) } if got := b.deployed(); got != "NEW-BUILD" { t.Errorf("deployed binary = %q; a refused restore must change nothing", got) } } func TestRollback_NoSnapshots(t *testing.T) { b := newFakeBox(t) if _, err := b.updater(t).Rollback(context.Background(), ""); err == nil { t.Error("Rollback with no snapshots succeeded; want an error") } } func TestPrune_KeepsTheNewestAsTheRollbackTarget(t *testing.T) { b := newFakeBox(t) st := &Store{Dir: filepath.Join(b.root, "snapshots")} base := time.Date(2026, 8, 1, 3, 0, 0, 0, time.UTC) for i := 0; i < 4; i++ { i := i st.now = func() time.Time { return base.Add(time.Duration(i) * time.Minute) } if _, err := st.Save(filepath.Join(b.root, "install"), []string{"mavend"}, "", ""); err != nil { t.Fatal(err) } } if err := st.Prune(0); err != nil { // 0 is clamped to 1, never to zero t.Fatal(err) } snaps, err := st.List() if err != nil { t.Fatal(err) } if len(snaps) != 1 { t.Fatalf("kept %d snapshots; want 1", len(snaps)) } if snaps[0].ID != "20260801-030300" { t.Errorf("kept %s; want the newest", snaps[0].ID) } } func TestList_IgnoresSnapshotsWithNoManifest(t *testing.T) { // An interrupted snapshot has files but no manifest. It must never be offered // as a rollback target — restoring a half-copied binary is the worst outcome // in the package. b := newFakeBox(t) dir := filepath.Join(b.root, "snapshots", "20260801-000000") if err := os.MkdirAll(dir, 0o700); err != nil { t.Fatal(err) } write(t, filepath.Join(dir, "mavend"), "HALF") snaps, err := (&Store{Dir: filepath.Join(b.root, "snapshots")}).List() if err != nil { t.Fatal(err) } if len(snaps) != 0 { t.Errorf("List returned %d snapshots; want none (no manifest)", len(snaps)) } } func TestConfigValidate(t *testing.T) { ok := (&fakeBox{root: t.TempDir()}).cfg() if err := ok.Validate(); err != nil { t.Fatalf("valid config rejected: %v", err) } bad := map[string]func(c Config) Config{ "relative source": func(c Config) Config { c.SourceDir = "src"; return c }, "no restart command": func(c Config) Config { c.RestartCmd = nil; return c }, "no health socket": func(c Config) Config { c.HealthSocket = ""; return c }, "no binaries": func(c Config) Config { c.Binaries = nil; return c }, "escaping artifact name": func(c Config) Config { c.Binaries = []string{"../../etc/passwd"}; return c }, "absolute artifact name": func(c Config) Config { c.Binaries = []string{"/usr/bin/mavend"}; return c }, "snapshots inside install": func(c Config) Config { c.SnapshotDir = filepath.Join(c.InstallDir, "snaps"); return c }, } for name, mutate := range bad { if err := mutate(ok).Validate(); err == nil { t.Errorf("%s was accepted; want a startup failure", name) } } // And New refuses an invalid config outright rather than half-configuring. if _, err := New(mutate(ok, "no health socket", bad)); err == nil { t.Error("New accepted a config with no health socket") } } func mutate(c Config, key string, m map[string]func(Config) Config) Config { return m[key](c) }