0b994ff1c3
Put and PutFile added the blob size to s.total before writing, and only the writeFile and os.Rename failure paths released it. A writeMeta failure in either, and a chmod failure on the spool in PutFile, kept the size, so a store that hit a full disk over-counted itself and could answer ErrStoreFull while the disk had room until the next Open re-measured. One defer per function now owns the release, disarmed on the success return, so a future early return cannot reintroduce the leak. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
410 lines
12 KiB
Go
410 lines
12 KiB
Go
package media
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func testStore(t *testing.T) *Store {
|
|
t.Helper()
|
|
s, err := Open(t.TempDir(), 0, 0)
|
|
if err != nil {
|
|
t.Fatalf("open: %v", err)
|
|
}
|
|
return s
|
|
}
|
|
|
|
func TestPutAndRead(t *testing.T) {
|
|
s := testStore(t)
|
|
b, err := s.Put(KindImage, "image/png", "web:upload", []byte("pretend png"))
|
|
if err != nil {
|
|
t.Fatalf("put: %v", err)
|
|
}
|
|
if len(b.ID) != 64 {
|
|
t.Fatalf("id is not a sha256 hex digest: %q", b.ID)
|
|
}
|
|
if b.Size != int64(len("pretend png")) {
|
|
t.Errorf("size = %d", b.Size)
|
|
}
|
|
if !strings.HasSuffix(b.Path, ".png") {
|
|
t.Errorf("extension not taken from mime: %s", b.Path)
|
|
}
|
|
got, data, err := s.Read(b.ID)
|
|
if err != nil {
|
|
t.Fatalf("read: %v", err)
|
|
}
|
|
if string(data) != "pretend png" {
|
|
t.Errorf("data = %q", data)
|
|
}
|
|
if got.Source != "web:upload" || got.Kind != KindImage {
|
|
t.Errorf("metadata not round-tripped: %+v", got)
|
|
}
|
|
}
|
|
|
|
// The same bytes twice must be one file, and must NOT get a fresh creation
|
|
// time — otherwise re-sending a photo keeps it alive past retention forever.
|
|
func TestPutIsIdempotentAndKeepsFirstSeenTime(t *testing.T) {
|
|
s := testStore(t)
|
|
base := time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC)
|
|
s.now = func() time.Time { return base }
|
|
|
|
first, err := s.Put(KindAudio, "audio/wav", "capture:meeting", []byte("pcm"))
|
|
if err != nil {
|
|
t.Fatalf("put: %v", err)
|
|
}
|
|
s.now = func() time.Time { return base.Add(72 * time.Hour) }
|
|
second, err := s.Put(KindAudio, "audio/wav", "capture:meeting", []byte("pcm"))
|
|
if err != nil {
|
|
t.Fatalf("re-put: %v", err)
|
|
}
|
|
if first.ID != second.ID {
|
|
t.Fatalf("same bytes produced two ids")
|
|
}
|
|
if !second.Created.Equal(base) {
|
|
t.Errorf("re-put moved created time to %v, want %v", second.Created, base)
|
|
}
|
|
list, err := s.List(KindAudio)
|
|
if err != nil {
|
|
t.Fatalf("list: %v", err)
|
|
}
|
|
if len(list) != 1 {
|
|
t.Errorf("got %d blobs, want 1", len(list))
|
|
}
|
|
}
|
|
|
|
func TestPutRejects(t *testing.T) {
|
|
s, err := Open(t.TempDir(), 8, 0)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := s.Put(KindImage, "image/png", "x", nil); !errors.Is(err, ErrEmpty) {
|
|
t.Errorf("empty payload: %v", err)
|
|
}
|
|
if _, err := s.Put("video", "video/mp4", "x", []byte("ab")); !errors.Is(err, ErrBadKind) {
|
|
t.Errorf("bad kind: %v", err)
|
|
}
|
|
if _, err := s.Put(KindImage, "image/png", "x", []byte("way too many bytes")); !errors.Is(err, ErrTooLarge) {
|
|
t.Errorf("over cap: %v", err)
|
|
}
|
|
}
|
|
|
|
// A caller-supplied id becomes a path, so a traversal attempt must be refused
|
|
// before it touches the filesystem rather than escaping the store root.
|
|
func TestMalformedIDIsRefused(t *testing.T) {
|
|
s := testStore(t)
|
|
for _, id := range []string{"", "../../etc/passwd", strings.Repeat("z", 64), strings.Repeat("a", 63)} {
|
|
if _, err := s.Get(id); !errors.Is(err, ErrBadID) && !errors.Is(err, ErrNotFound) {
|
|
t.Errorf("Get(%q) = %v, want a refusal", id, err)
|
|
}
|
|
if _, _, err := s.Read(id); err == nil {
|
|
t.Errorf("Read(%q) succeeded", id)
|
|
}
|
|
if err := s.Delete(id); err == nil && id != "" {
|
|
// Delete of a well-formed but absent id is fine; these are not
|
|
// well-formed.
|
|
t.Errorf("Delete(%q) succeeded", id)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestGetMissingIsNotFound(t *testing.T) {
|
|
s := testStore(t)
|
|
if _, err := s.Get(strings.Repeat("a", 64)); !errors.Is(err, ErrNotFound) {
|
|
t.Errorf("got %v, want ErrNotFound", err)
|
|
}
|
|
}
|
|
|
|
func TestPruneEnforcesRetention(t *testing.T) {
|
|
s, err := Open(t.TempDir(), 0, 48*time.Hour)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
now := time.Date(2026, 8, 1, 0, 0, 0, 0, time.UTC)
|
|
|
|
s.now = func() time.Time { return now.Add(-96 * time.Hour) }
|
|
old, _ := s.Put(KindAudio, "audio/wav", "capture:meeting", []byte("old meeting"))
|
|
s.now = func() time.Time { return now.Add(-1 * time.Hour) }
|
|
fresh, _ := s.Put(KindImage, "image/png", "telegram", []byte("recent photo"))
|
|
|
|
s.now = func() time.Time { return now }
|
|
n, err := s.Prune()
|
|
if err != nil {
|
|
t.Fatalf("prune: %v", err)
|
|
}
|
|
if n != 1 {
|
|
t.Errorf("pruned %d, want 1", n)
|
|
}
|
|
if _, err := s.Get(old.ID); !errors.Is(err, ErrNotFound) {
|
|
t.Errorf("stale blob survived prune: %v", err)
|
|
}
|
|
if _, err := s.Get(fresh.ID); err != nil {
|
|
t.Errorf("fresh blob was pruned: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestListIsNewestFirstAcrossKinds(t *testing.T) {
|
|
s := testStore(t)
|
|
base := time.Date(2026, 8, 1, 0, 0, 0, 0, time.UTC)
|
|
s.now = func() time.Time { return base }
|
|
_, _ = s.Put(KindImage, "image/png", "telegram", []byte("one"))
|
|
s.now = func() time.Time { return base.Add(time.Hour) }
|
|
newest, _ := s.Put(KindAudio, "audio/wav", "capture:meeting", []byte("two"))
|
|
|
|
all, err := s.List("")
|
|
if err != nil {
|
|
t.Fatalf("list: %v", err)
|
|
}
|
|
if len(all) != 2 {
|
|
t.Fatalf("got %d, want 2", len(all))
|
|
}
|
|
if all[0].ID != newest.ID {
|
|
t.Errorf("list is not newest-first")
|
|
}
|
|
}
|
|
|
|
// Recordings of people are 0700/0600 and nothing else.
|
|
func TestPermissionsAreOwnerOnly(t *testing.T) {
|
|
dir := t.TempDir()
|
|
s, err := Open(filepath.Join(dir, "blobs"), 0, 0)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
b, err := s.Put(KindAudio, "audio/wav", "capture:meeting", []byte("pcm"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
di, err := os.Stat(s.Dir())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if di.Mode().Perm() != 0o700 {
|
|
t.Errorf("store dir mode = %o, want 700", di.Mode().Perm())
|
|
}
|
|
fi, err := os.Stat(b.Path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if fi.Mode().Perm() != 0o600 {
|
|
t.Errorf("blob mode = %o, want 600", fi.Mode().Perm())
|
|
}
|
|
}
|
|
|
|
func TestDeleteRemovesBytesAndSidecar(t *testing.T) {
|
|
s := testStore(t)
|
|
b, _ := s.Put(KindImage, "image/png", "telegram", []byte("bytes"))
|
|
if err := s.Delete(b.ID); err != nil {
|
|
t.Fatalf("delete: %v", err)
|
|
}
|
|
if _, err := os.Stat(b.Path); !os.IsNotExist(err) {
|
|
t.Errorf("bytes survived delete")
|
|
}
|
|
if _, err := s.Get(b.ID); !errors.Is(err, ErrNotFound) {
|
|
t.Errorf("sidecar survived delete: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestOpenRejectsEmptyDir(t *testing.T) {
|
|
if _, err := Open(" ", 0, 0); err == nil {
|
|
t.Error("empty dir accepted")
|
|
}
|
|
}
|
|
|
|
// A blob whose sidecar is missing was invisible to List, so Prune never saw it
|
|
// and the bytes stayed on disk forever. Put produced exactly that state, by
|
|
// writing the blob first and the sidecar second.
|
|
func TestPruneCollectsASidecarlessBlob(t *testing.T) {
|
|
s := testStore(t)
|
|
b, err := s.Put(KindImage, "image/png", "web:upload", []byte("orphan"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
meta := filepath.Join(s.dir, string(KindImage), b.ID[:2], b.ID+".json")
|
|
if err := os.Remove(meta); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Age the file past retention, the same way a real orphan gets there.
|
|
old := time.Now().Add(-2 * DefaultRetention)
|
|
if err := os.Chtimes(b.Path, old, old); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
n, err := s.Prune()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n != 1 {
|
|
t.Fatalf("pruned %d, want the orphan collected", n)
|
|
}
|
|
if _, err := os.Stat(b.Path); !errors.Is(err, os.ErrNotExist) {
|
|
t.Fatalf("the orphaned bytes are still on disk: %v", err)
|
|
}
|
|
}
|
|
|
|
// A young orphan is left alone: a Put racing a Prune must not lose its bytes.
|
|
func TestPruneLeavesAYoungOrphan(t *testing.T) {
|
|
s := testStore(t)
|
|
b, err := s.Put(KindImage, "image/png", "web:upload", []byte("fresh"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.Remove(filepath.Join(s.dir, string(KindImage), b.ID[:2], b.ID+".json")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n, err := s.Prune(); err != nil || n != 0 {
|
|
t.Fatalf("prune = %d, %v; want the fresh orphan kept", n, err)
|
|
}
|
|
}
|
|
|
|
// Put writes the sidecar first, so a failure writing the bytes leaves nothing
|
|
// at all rather than an uncollectable blob.
|
|
func TestPutLeavesNothingWhenTheBytesCannotBeWritten(t *testing.T) {
|
|
s := testStore(t)
|
|
data := []byte("will not land")
|
|
sum := sha256.Sum256(data)
|
|
id := hex.EncodeToString(sum[:])
|
|
bucket := filepath.Join(s.dir, string(KindImage), id[:2])
|
|
if err := os.MkdirAll(bucket, 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// A directory where the blob file needs to be: rename onto it fails, which
|
|
// is the same shape as a full disk one step later.
|
|
if err := os.Mkdir(filepath.Join(bucket, id+".png"), 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := s.Put(KindImage, "image/png", "web:upload", data); err == nil {
|
|
t.Fatal("put must fail")
|
|
}
|
|
if _, err := os.Stat(filepath.Join(bucket, id+".json")); !errors.Is(err, os.ErrNotExist) {
|
|
t.Fatalf("a sidecar was left behind claiming a blob that does not exist: %v", err)
|
|
}
|
|
if s.Total() != 0 {
|
|
t.Errorf("total = %d, want the failed put not counted", s.Total())
|
|
}
|
|
}
|
|
|
|
// An over-counted store answers ErrStoreFull while the disk has room, and only
|
|
// the next Open corrects it. So every failed write has to give its reservation
|
|
// back, not just the one that remembered to.
|
|
func TestPutReleasesTheBudgetWhenTheSidecarCannotBeWritten(t *testing.T) {
|
|
s := testStore(t)
|
|
data := []byte("no sidecar for this")
|
|
blockSidecar(t, s, KindImage, data)
|
|
if _, err := s.Put(KindImage, "image/png", "web:upload", data); err == nil {
|
|
t.Fatal("put must fail")
|
|
}
|
|
if s.Total() != 0 {
|
|
t.Errorf("total = %d, want the failed put not counted", s.Total())
|
|
}
|
|
}
|
|
|
|
func TestPutFileReleasesTheBudgetWhenTheSidecarCannotBeWritten(t *testing.T) {
|
|
s := testStore(t)
|
|
data := []byte("no sidecar for this either")
|
|
blockSidecar(t, s, KindAudio, data)
|
|
src := filepath.Join(t.TempDir(), "capture.wav")
|
|
if err := os.WriteFile(src, data, 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := s.PutFile(KindAudio, "audio/wav", "meeting", src); err == nil {
|
|
t.Fatal("put file must fail")
|
|
}
|
|
if s.Total() != 0 {
|
|
t.Errorf("total = %d, want the failed put not counted", s.Total())
|
|
}
|
|
}
|
|
|
|
// The chmod arm is PutFile's alone: Put never touches a spool file.
|
|
func TestPutFileReleasesTheBudgetWhenTheSpoolCannotBeChmodded(t *testing.T) {
|
|
if os.Geteuid() == 0 {
|
|
t.Skip("root can chmod a file it does not own")
|
|
}
|
|
// A symlink to a file owned by somebody else. Stat and the hash follow it
|
|
// and succeed; chmod follows it too and is refused.
|
|
src := filepath.Join(t.TempDir(), "capture.wav")
|
|
if err := os.Symlink("/etc/hosts", src); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
info, err := os.Stat(src)
|
|
if err != nil || info.Size() == 0 {
|
|
t.Skip("no readable /etc/hosts to point at")
|
|
}
|
|
s := testStore(t)
|
|
if _, err := s.PutFile(KindAudio, "audio/wav", "meeting", src); err == nil {
|
|
t.Fatal("put file must fail")
|
|
}
|
|
if s.Total() != 0 {
|
|
t.Errorf("total = %d, want the failed put not counted", s.Total())
|
|
}
|
|
}
|
|
|
|
// blockSidecar puts a directory where the sidecar for data has to go, so
|
|
// writeMeta fails while the blob path is still free.
|
|
func blockSidecar(t *testing.T, s *Store, kind Kind, data []byte) {
|
|
t.Helper()
|
|
sum := sha256.Sum256(data)
|
|
id := hex.EncodeToString(sum[:])
|
|
bucket := filepath.Join(s.dir, string(kind), id[:2])
|
|
if err := os.MkdirAll(filepath.Join(bucket, id+".json"), 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// The per-blob cap bounds one call and nothing bounded their sum. 64 MiB per
|
|
// call times unlimited calls inside a seven-day window fills the disk mavend's
|
|
// database lives on.
|
|
func TestPutRefusesPastTheStoreBudget(t *testing.T) {
|
|
s, err := OpenWithBudget(t.TempDir(), 16, 48, 0)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for i, want := range []bool{true, true, true, false} {
|
|
data := []byte(strings.Repeat(string(rune('a'+i)), 16))
|
|
_, err := s.Put(KindImage, "image/png", "web:upload", data)
|
|
if ok := err == nil; ok != want {
|
|
t.Fatalf("put %d: err = %v, want ok=%v", i, err, want)
|
|
}
|
|
if !want && !errors.Is(err, ErrStoreFull) {
|
|
t.Fatalf("put %d: err = %v, want ErrStoreFull", i, err)
|
|
}
|
|
}
|
|
// The same bytes again cost nothing, so they are not refused.
|
|
if _, err := s.Put(KindImage, "image/png", "web:upload", []byte(strings.Repeat("a", 16))); err != nil {
|
|
t.Fatalf("a re-send of stored bytes was refused: %v", err)
|
|
}
|
|
// Deleting frees the budget again.
|
|
blobs, err := s.List(KindImage)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.Delete(blobs[0].ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := s.Put(KindImage, "image/png", "web:upload", []byte(strings.Repeat("z", 16))); err != nil {
|
|
t.Fatalf("budget was not released on delete: %v", err)
|
|
}
|
|
}
|
|
|
|
// A restart must not start the budget over at zero.
|
|
func TestOpenSeedsTheBudgetFromDisk(t *testing.T) {
|
|
dir := t.TempDir()
|
|
s, err := OpenWithBudget(dir, 16, 48, 0)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := s.Put(KindImage, "image/png", "web:upload", []byte(strings.Repeat("a", 16))); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
again, err := OpenWithBudget(dir, 16, 48, 0)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if again.Total() != 16 {
|
|
t.Fatalf("total after reopen = %d, want 16", again.Total())
|
|
}
|
|
}
|