Files
Maven/internal/media/store_test.go
T
kami 4b052fb9d2 media: make retention and the disk budget true
Put wrote the blob and then the sidecar. A full disk or a crash between
the two left bytes on disk with no sidecar, and List walks sidecars, so
Prune could never see them: Put returned an error and an image nobody
knew about became permanent. The sidecar goes first, a failed write is
rolled back, and Prune also collects blob files that have no readable
sidecar and are past retention, which picks up whatever an older build
leaked.

The per-blob cap bounds one call and nothing bounded their sum. Content
addressing does not help, because one flipped pixel is a different
digest, so 64 MiB per call and an unlimited number of calls fills the
disk mavend's database lives on. The store now carries a whole-store
budget, seeded from disk at open so a restart does not begin at zero.

Found in review of #72.
2026-08-01 14:21:29 +04:00

343 lines
10 KiB
Go

package media
import (
"crypto/sha256"
"encoding/hex"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func testStore(t *testing.T) *Store {
t.Helper()
s, err := Open(t.TempDir(), 0, 0)
if err != nil {
t.Fatalf("open: %v", err)
}
return s
}
func TestPutAndRead(t *testing.T) {
s := testStore(t)
b, err := s.Put(KindImage, "image/png", "web:upload", []byte("pretend png"))
if err != nil {
t.Fatalf("put: %v", err)
}
if len(b.ID) != 64 {
t.Fatalf("id is not a sha256 hex digest: %q", b.ID)
}
if b.Size != int64(len("pretend png")) {
t.Errorf("size = %d", b.Size)
}
if !strings.HasSuffix(b.Path, ".png") {
t.Errorf("extension not taken from mime: %s", b.Path)
}
got, data, err := s.Read(b.ID)
if err != nil {
t.Fatalf("read: %v", err)
}
if string(data) != "pretend png" {
t.Errorf("data = %q", data)
}
if got.Source != "web:upload" || got.Kind != KindImage {
t.Errorf("metadata not round-tripped: %+v", got)
}
}
// The same bytes twice must be one file, and must NOT get a fresh creation
// time — otherwise re-sending a photo keeps it alive past retention forever.
func TestPutIsIdempotentAndKeepsFirstSeenTime(t *testing.T) {
s := testStore(t)
base := time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC)
s.now = func() time.Time { return base }
first, err := s.Put(KindAudio, "audio/wav", "capture:meeting", []byte("pcm"))
if err != nil {
t.Fatalf("put: %v", err)
}
s.now = func() time.Time { return base.Add(72 * time.Hour) }
second, err := s.Put(KindAudio, "audio/wav", "capture:meeting", []byte("pcm"))
if err != nil {
t.Fatalf("re-put: %v", err)
}
if first.ID != second.ID {
t.Fatalf("same bytes produced two ids")
}
if !second.Created.Equal(base) {
t.Errorf("re-put moved created time to %v, want %v", second.Created, base)
}
list, err := s.List(KindAudio)
if err != nil {
t.Fatalf("list: %v", err)
}
if len(list) != 1 {
t.Errorf("got %d blobs, want 1", len(list))
}
}
func TestPutRejects(t *testing.T) {
s, err := Open(t.TempDir(), 8, 0)
if err != nil {
t.Fatal(err)
}
if _, err := s.Put(KindImage, "image/png", "x", nil); !errors.Is(err, ErrEmpty) {
t.Errorf("empty payload: %v", err)
}
if _, err := s.Put("video", "video/mp4", "x", []byte("ab")); !errors.Is(err, ErrBadKind) {
t.Errorf("bad kind: %v", err)
}
if _, err := s.Put(KindImage, "image/png", "x", []byte("way too many bytes")); !errors.Is(err, ErrTooLarge) {
t.Errorf("over cap: %v", err)
}
}
// A caller-supplied id becomes a path, so a traversal attempt must be refused
// before it touches the filesystem rather than escaping the store root.
func TestMalformedIDIsRefused(t *testing.T) {
s := testStore(t)
for _, id := range []string{"", "../../etc/passwd", strings.Repeat("z", 64), strings.Repeat("a", 63)} {
if _, err := s.Get(id); !errors.Is(err, ErrBadID) && !errors.Is(err, ErrNotFound) {
t.Errorf("Get(%q) = %v, want a refusal", id, err)
}
if _, _, err := s.Read(id); err == nil {
t.Errorf("Read(%q) succeeded", id)
}
if err := s.Delete(id); err == nil && id != "" {
// Delete of a well-formed but absent id is fine; these are not
// well-formed.
t.Errorf("Delete(%q) succeeded", id)
}
}
}
func TestGetMissingIsNotFound(t *testing.T) {
s := testStore(t)
if _, err := s.Get(strings.Repeat("a", 64)); !errors.Is(err, ErrNotFound) {
t.Errorf("got %v, want ErrNotFound", err)
}
}
func TestPruneEnforcesRetention(t *testing.T) {
s, err := Open(t.TempDir(), 0, 48*time.Hour)
if err != nil {
t.Fatal(err)
}
now := time.Date(2026, 8, 1, 0, 0, 0, 0, time.UTC)
s.now = func() time.Time { return now.Add(-96 * time.Hour) }
old, _ := s.Put(KindAudio, "audio/wav", "capture:meeting", []byte("old meeting"))
s.now = func() time.Time { return now.Add(-1 * time.Hour) }
fresh, _ := s.Put(KindImage, "image/png", "telegram", []byte("recent photo"))
s.now = func() time.Time { return now }
n, err := s.Prune()
if err != nil {
t.Fatalf("prune: %v", err)
}
if n != 1 {
t.Errorf("pruned %d, want 1", n)
}
if _, err := s.Get(old.ID); !errors.Is(err, ErrNotFound) {
t.Errorf("stale blob survived prune: %v", err)
}
if _, err := s.Get(fresh.ID); err != nil {
t.Errorf("fresh blob was pruned: %v", err)
}
}
func TestListIsNewestFirstAcrossKinds(t *testing.T) {
s := testStore(t)
base := time.Date(2026, 8, 1, 0, 0, 0, 0, time.UTC)
s.now = func() time.Time { return base }
_, _ = s.Put(KindImage, "image/png", "telegram", []byte("one"))
s.now = func() time.Time { return base.Add(time.Hour) }
newest, _ := s.Put(KindAudio, "audio/wav", "capture:meeting", []byte("two"))
all, err := s.List("")
if err != nil {
t.Fatalf("list: %v", err)
}
if len(all) != 2 {
t.Fatalf("got %d, want 2", len(all))
}
if all[0].ID != newest.ID {
t.Errorf("list is not newest-first")
}
}
// Recordings of people are 0700/0600 and nothing else.
func TestPermissionsAreOwnerOnly(t *testing.T) {
dir := t.TempDir()
s, err := Open(filepath.Join(dir, "blobs"), 0, 0)
if err != nil {
t.Fatal(err)
}
b, err := s.Put(KindAudio, "audio/wav", "capture:meeting", []byte("pcm"))
if err != nil {
t.Fatal(err)
}
di, err := os.Stat(s.Dir())
if err != nil {
t.Fatal(err)
}
if di.Mode().Perm() != 0o700 {
t.Errorf("store dir mode = %o, want 700", di.Mode().Perm())
}
fi, err := os.Stat(b.Path)
if err != nil {
t.Fatal(err)
}
if fi.Mode().Perm() != 0o600 {
t.Errorf("blob mode = %o, want 600", fi.Mode().Perm())
}
}
func TestDeleteRemovesBytesAndSidecar(t *testing.T) {
s := testStore(t)
b, _ := s.Put(KindImage, "image/png", "telegram", []byte("bytes"))
if err := s.Delete(b.ID); err != nil {
t.Fatalf("delete: %v", err)
}
if _, err := os.Stat(b.Path); !os.IsNotExist(err) {
t.Errorf("bytes survived delete")
}
if _, err := s.Get(b.ID); !errors.Is(err, ErrNotFound) {
t.Errorf("sidecar survived delete: %v", err)
}
}
func TestOpenRejectsEmptyDir(t *testing.T) {
if _, err := Open(" ", 0, 0); err == nil {
t.Error("empty dir accepted")
}
}
// A blob whose sidecar is missing was invisible to List, so Prune never saw it
// and the bytes stayed on disk forever. Put produced exactly that state, by
// writing the blob first and the sidecar second.
func TestPruneCollectsASidecarlessBlob(t *testing.T) {
s := testStore(t)
b, err := s.Put(KindImage, "image/png", "web:upload", []byte("orphan"))
if err != nil {
t.Fatal(err)
}
meta := filepath.Join(s.dir, string(KindImage), b.ID[:2], b.ID+".json")
if err := os.Remove(meta); err != nil {
t.Fatal(err)
}
// Age the file past retention, the same way a real orphan gets there.
old := time.Now().Add(-2 * DefaultRetention)
if err := os.Chtimes(b.Path, old, old); err != nil {
t.Fatal(err)
}
n, err := s.Prune()
if err != nil {
t.Fatal(err)
}
if n != 1 {
t.Fatalf("pruned %d, want the orphan collected", n)
}
if _, err := os.Stat(b.Path); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("the orphaned bytes are still on disk: %v", err)
}
}
// A young orphan is left alone: a Put racing a Prune must not lose its bytes.
func TestPruneLeavesAYoungOrphan(t *testing.T) {
s := testStore(t)
b, err := s.Put(KindImage, "image/png", "web:upload", []byte("fresh"))
if err != nil {
t.Fatal(err)
}
if err := os.Remove(filepath.Join(s.dir, string(KindImage), b.ID[:2], b.ID+".json")); err != nil {
t.Fatal(err)
}
if n, err := s.Prune(); err != nil || n != 0 {
t.Fatalf("prune = %d, %v; want the fresh orphan kept", n, err)
}
}
// Put writes the sidecar first, so a failure writing the bytes leaves nothing
// at all rather than an uncollectable blob.
func TestPutLeavesNothingWhenTheBytesCannotBeWritten(t *testing.T) {
s := testStore(t)
data := []byte("will not land")
sum := sha256.Sum256(data)
id := hex.EncodeToString(sum[:])
bucket := filepath.Join(s.dir, string(KindImage), id[:2])
if err := os.MkdirAll(bucket, 0o700); err != nil {
t.Fatal(err)
}
// A directory where the blob file needs to be: rename onto it fails, which
// is the same shape as a full disk one step later.
if err := os.Mkdir(filepath.Join(bucket, id+".png"), 0o700); err != nil {
t.Fatal(err)
}
if _, err := s.Put(KindImage, "image/png", "web:upload", data); err == nil {
t.Fatal("put must fail")
}
if _, err := os.Stat(filepath.Join(bucket, id+".json")); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("a sidecar was left behind claiming a blob that does not exist: %v", err)
}
if s.Total() != 0 {
t.Errorf("total = %d, want the failed put not counted", s.Total())
}
}
// The per-blob cap bounds one call and nothing bounded their sum. 64 MiB per
// call times unlimited calls inside a seven-day window fills the disk mavend's
// database lives on.
func TestPutRefusesPastTheStoreBudget(t *testing.T) {
s, err := OpenWithBudget(t.TempDir(), 16, 48, 0)
if err != nil {
t.Fatal(err)
}
for i, want := range []bool{true, true, true, false} {
data := []byte(strings.Repeat(string(rune('a'+i)), 16))
_, err := s.Put(KindImage, "image/png", "web:upload", data)
if ok := err == nil; ok != want {
t.Fatalf("put %d: err = %v, want ok=%v", i, err, want)
}
if !want && !errors.Is(err, ErrStoreFull) {
t.Fatalf("put %d: err = %v, want ErrStoreFull", i, err)
}
}
// The same bytes again cost nothing, so they are not refused.
if _, err := s.Put(KindImage, "image/png", "web:upload", []byte(strings.Repeat("a", 16))); err != nil {
t.Fatalf("a re-send of stored bytes was refused: %v", err)
}
// Deleting frees the budget again.
blobs, err := s.List(KindImage)
if err != nil {
t.Fatal(err)
}
if err := s.Delete(blobs[0].ID); err != nil {
t.Fatal(err)
}
if _, err := s.Put(KindImage, "image/png", "web:upload", []byte(strings.Repeat("z", 16))); err != nil {
t.Fatalf("budget was not released on delete: %v", err)
}
}
// A restart must not start the budget over at zero.
func TestOpenSeedsTheBudgetFromDisk(t *testing.T) {
dir := t.TempDir()
s, err := OpenWithBudget(dir, 16, 48, 0)
if err != nil {
t.Fatal(err)
}
if _, err := s.Put(KindImage, "image/png", "web:upload", []byte(strings.Repeat("a", 16))); err != nil {
t.Fatal(err)
}
again, err := OpenWithBudget(dir, 16, 48, 0)
if err != nil {
t.Fatal(err)
}
if again.Total() != 16 {
t.Fatalf("total after reopen = %d, want 16", again.Total())
}
}