From 7e7aadc6cb255b6790cc4682923694c145717a32 Mon Sep 17 00:00:00 2001 From: Ingo Oppermann Date: Fri, 17 Mar 2023 15:56:15 +0100 Subject: [PATCH] Allow to bulk delete files based on ListOptions --- app/api/api.go | 2 +- io/fs/disk.go | 57 ++++++++++++++++++++++++++++++++++++++++-- io/fs/fs.go | 6 ++--- io/fs/fs_test.go | 37 ++++++++++++++++++++------- io/fs/mem.go | 53 ++++++++++++++++++++++++++++++++++++--- io/fs/readonly.go | 2 +- io/fs/readonly_test.go | 2 +- io/fs/s3.go | 48 +++++++++++++++++++++++++++++++++-- 8 files changed, 184 insertions(+), 23 deletions(-) diff --git a/app/api/api.go b/app/api/api.go index a0821eb9..23033a5b 100644 --- a/app/api/api.go +++ b/app/api/api.go @@ -1446,7 +1446,7 @@ func (a *api) Destroy() { // Free the MemFS if a.memfs != nil { - a.memfs.RemoveAll() + a.memfs.RemoveList("/", fs.ListOptions{}) a.memfs = nil } } diff --git a/io/fs/disk.go b/io/fs/disk.go index 0b1d75d7..96ebda1a 100644 --- a/io/fs/disk.go +++ b/io/fs/disk.go @@ -519,8 +519,61 @@ func (fs *diskFilesystem) Remove(path string) int64 { return size } -func (fs *diskFilesystem) RemoveAll() int64 { - return 0 +func (fs *diskFilesystem) RemoveList(path string, options ListOptions) int64 { + path = fs.cleanPath(path) + + var size int64 = 0 + + fs.walk(path, func(path string, info os.FileInfo) { + if path == fs.root { + return + } + + name := strings.TrimPrefix(path, fs.root) + if name[0] != os.PathSeparator { + name = string(os.PathSeparator) + name + } + + if info.IsDir() { + return + } + + if len(options.Pattern) != 0 { + if ok, _ := glob.Match(options.Pattern, name, '/'); !ok { + return + } + } + + if options.ModifiedStart != nil { + if info.ModTime().Before(*options.ModifiedStart) { + return + } + } + + if options.ModifiedEnd != nil { + if info.ModTime().After(*options.ModifiedEnd) { + return + } + } + + if options.SizeMin > 0 { + if info.Size() < options.SizeMin { + return + } + } + + if options.SizeMax > 0 { + if info.Size() > options.SizeMax { + return + } + } + + if err := os.Remove(path); err == nil { + size += info.Size() + } + }) + + return size } func (fs *diskFilesystem) List(path string, options ListOptions) []FileInfo { diff --git a/io/fs/fs.go b/io/fs/fs.go index 0f804eb2..7c0245a9 100644 --- a/io/fs/fs.go +++ b/io/fs/fs.go @@ -117,12 +117,12 @@ type WriteFilesystem interface { Copy(src, dst string) error // Remove removes a file at the given path from the filesystem. Returns the size of - // the remove file in bytes. The size is negative if the file doesn't exist. + // the removed file in bytes. The size is negative if the file doesn't exist. Remove(path string) int64 - // RemoveAll removes all files from the filesystem. Returns the size of the + // RemoveList removes all files from the filesystem. Returns the size of the // removed files in bytes. - RemoveAll() int64 + RemoveList(path string, options ListOptions) int64 } // Filesystem is an interface that provides access to a filesystem. diff --git a/io/fs/fs_test.go b/io/fs/fs_test.go index 5e178df4..764adecd 100644 --- a/io/fs/fs_test.go +++ b/io/fs/fs_test.go @@ -88,14 +88,15 @@ func TestFilesystem(t *testing.T) { "writeFileSafe": testWriteFileSafe, "writeFileReader": testWriteFileReader, "writeFileDir": testWriteFileDir, - "delete": testDelete, + "remove": testRemove, "files": testFiles, "replace": testReplace, "list": testList, "listGlob": testListGlob, "listSize": testListSize, "listModified": testListModified, - "deleteAll": testDeleteAll, + "removeAll": testRemoveAll, + "removeList": testRemoveList, "data": testData, "statDir": testStatDir, "mkdirAll": testMkdirAll, @@ -224,7 +225,7 @@ func testOpen(t *testing.T, fs Filesystem) { require.Equal(t, false, stat.IsDir()) } -func testDelete(t *testing.T, fs Filesystem) { +func testRemove(t *testing.T, fs Filesystem) { size := fs.Remove("/foobar") require.Equal(t, int64(-1), size) @@ -461,11 +462,7 @@ func testListModified(t *testing.T, fs Filesystem) { require.ElementsMatch(t, []string{"/b", "/c"}, files) } -func testDeleteAll(t *testing.T, fs Filesystem) { - if _, ok := fs.(*diskFilesystem); ok { - return - } - +func testRemoveAll(t *testing.T, fs Filesystem) { fs.WriteFileReader("/foobar1", strings.NewReader("abc")) fs.WriteFileReader("/path/foobar2", strings.NewReader("abc")) fs.WriteFileReader("/path/to/foobar3", strings.NewReader("abc")) @@ -475,7 +472,9 @@ func testDeleteAll(t *testing.T, fs Filesystem) { require.Equal(t, int64(4), cur) - size := fs.RemoveAll() + size := fs.RemoveList("/", ListOptions{ + Pattern: "", + }) require.Equal(t, int64(12), size) cur = fs.Files() @@ -483,6 +482,26 @@ func testDeleteAll(t *testing.T, fs Filesystem) { require.Equal(t, int64(0), cur) } +func testRemoveList(t *testing.T, fs Filesystem) { + fs.WriteFileReader("/foobar1", strings.NewReader("abc")) + fs.WriteFileReader("/path/foobar2", strings.NewReader("abc")) + fs.WriteFileReader("/path/to/foobar3", strings.NewReader("abc")) + fs.WriteFileReader("/foobar4", strings.NewReader("abc")) + + cur := fs.Files() + + require.Equal(t, int64(4), cur) + + size := fs.RemoveList("/", ListOptions{ + Pattern: "/path/**", + }) + require.Equal(t, int64(6), size) + + cur = fs.Files() + + require.Equal(t, int64(2), cur) +} + func testData(t *testing.T, fs Filesystem) { file := fs.Open("/foobar") require.Nil(t, file) diff --git a/io/fs/mem.go b/io/fs/mem.go index 1ee852de..54b45d64 100644 --- a/io/fs/mem.go +++ b/io/fs/mem.go @@ -651,6 +651,10 @@ func (fs *memFilesystem) Remove(path string) int64 { fs.filesLock.Lock() defer fs.filesLock.Unlock() + return fs.remove(path) +} + +func (fs *memFilesystem) remove(path string) int64 { file, ok := fs.files[path] if ok { delete(fs.files, path) @@ -671,14 +675,55 @@ func (fs *memFilesystem) Remove(path string) int64 { return file.size } -func (fs *memFilesystem) RemoveAll() int64 { +func (fs *memFilesystem) RemoveList(path string, options ListOptions) int64 { + path = fs.cleanPath(path) + fs.filesLock.Lock() defer fs.filesLock.Unlock() - size := fs.currentSize + var size int64 = 0 - fs.files = make(map[string]*internalMemFile) - fs.currentSize = 0 + for _, file := range fs.files { + if !strings.HasPrefix(file.name, path) { + continue + } + + if len(options.Pattern) != 0 { + if ok, _ := glob.Match(options.Pattern, file.name, '/'); !ok { + continue + } + } + + if options.ModifiedStart != nil { + if file.lastMod.Before(*options.ModifiedStart) { + continue + } + } + + if options.ModifiedEnd != nil { + if file.lastMod.After(*options.ModifiedEnd) { + continue + } + } + + if options.SizeMin > 0 { + if file.size < options.SizeMin { + continue + } + } + + if options.SizeMax > 0 { + if file.size > options.SizeMax { + continue + } + } + + if file.dir { + continue + } + + size += fs.remove(file.name) + } return size } diff --git a/io/fs/readonly.go b/io/fs/readonly.go index 889672a4..00eb7782 100644 --- a/io/fs/readonly.go +++ b/io/fs/readonly.go @@ -41,7 +41,7 @@ func (r *readOnlyFilesystem) Remove(path string) int64 { return -1 } -func (r *readOnlyFilesystem) RemoveAll() int64 { +func (r *readOnlyFilesystem) RemoveList(path string, options ListOptions) int64 { return 0 } diff --git a/io/fs/readonly_test.go b/io/fs/readonly_test.go index 13360b47..66dd2380 100644 --- a/io/fs/readonly_test.go +++ b/io/fs/readonly_test.go @@ -32,7 +32,7 @@ func TestReadOnly(t *testing.T) { res := ro.Remove("/readonly.go") require.Equal(t, int64(-1), res) - res = ro.RemoveAll() + res = ro.RemoveList("/", ListOptions{}) require.Equal(t, int64(0), res) rop, ok := ro.(PurgeFilesystem) diff --git a/io/fs/s3.go b/io/fs/s3.go index 766695e0..8dc02161 100644 --- a/io/fs/s3.go +++ b/io/fs/s3.go @@ -428,7 +428,9 @@ func (fs *s3Filesystem) Remove(path string) int64 { return stat.Size } -func (fs *s3Filesystem) RemoveAll() int64 { +func (fs *s3Filesystem) RemoveList(path string, options ListOptions) int64 { + path = fs.cleanPath(path) + ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -441,12 +443,54 @@ func (fs *s3Filesystem) RemoveAll() int64 { defer close(objectsCh) for object := range fs.client.ListObjects(ctx, fs.bucket, minio.ListObjectsOptions{ - Recursive: true, + WithVersions: false, + WithMetadata: false, + Prefix: path, + Recursive: true, + MaxKeys: 0, + StartAfter: "", + UseV1: false, }) { if object.Err != nil { fs.logger.WithError(object.Err).Log("Listing object failed") continue } + key := "/" + object.Key + if strings.HasSuffix(key, "/"+fakeDirEntry) { + // filter out fake directory entries (see MkdirAll) + continue + } + + if len(options.Pattern) != 0 { + if ok, _ := glob.Match(options.Pattern, key, '/'); !ok { + continue + } + } + + if options.ModifiedStart != nil { + if object.LastModified.Before(*options.ModifiedStart) { + continue + } + } + + if options.ModifiedEnd != nil { + if object.LastModified.After(*options.ModifiedEnd) { + continue + } + } + + if options.SizeMin > 0 { + if object.Size < options.SizeMin { + continue + } + } + + if options.SizeMax > 0 { + if object.Size > options.SizeMax { + continue + } + } + totalSize += object.Size objectsCh <- object }