neoris/core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Allow to bulk delete files based on ListOptions

Browse Source
This commit is contained in:
Ingo Oppermann 2023-03-17 15:56:15 +01:00
parent ecfbbe3857
commit 7e7aadc6cb
No known key found for this signature in database
GPG Key ID: 2AB32426E9DD229E
8 changed files with 184 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/api/api.go
View File

@ -1446,7 +1446,7 @@ func (a *api) Destroy() {
// Free the MemFS
if a.memfs != nil {
a.memfs.RemoveAll()
a.memfs.RemoveList("/", fs.ListOptions{})
a.memfs = nil
}
}

57
io/fs/disk.go
View File

@ -519,8 +519,61 @@ func (fs *diskFilesystem) Remove(path string) int64 {
return size
}
func (fs *diskFilesystem) RemoveAll() int64 {
return 0
func (fs *diskFilesystem) RemoveList(path string, options ListOptions) int64 {
path = fs.cleanPath(path)
var size int64 = 0
fs.walk(path, func(path string, info os.FileInfo) {
if path == fs.root {
return
}
name := strings.TrimPrefix(path, fs.root)
if name[0] != os.PathSeparator {
name = string(os.PathSeparator) + name
}
if info.IsDir() {
return
}
if len(options.Pattern) != 0 {
if ok, _ := glob.Match(options.Pattern, name, '/'); !ok {
return
}
}
if options.ModifiedStart != nil {
if info.ModTime().Before(*options.ModifiedStart) {
return
}
}
if options.ModifiedEnd != nil {
if info.ModTime().After(*options.ModifiedEnd) {
return
}
}
if options.SizeMin > 0 {
if info.Size() < options.SizeMin {
return
}
}
if options.SizeMax > 0 {
if info.Size() > options.SizeMax {
return
}
}
if err := os.Remove(path); err == nil {
size += info.Size()
}
})
return size
}
func (fs *diskFilesystem) List(path string, options ListOptions) []FileInfo {

6
io/fs/fs.go
View File

@ -117,12 +117,12 @@ type WriteFilesystem interface {
Copy(src, dst string) error
// Remove removes a file at the given path from the filesystem. Returns the size of
// the remove file in bytes. The size is negative if the file doesn't exist.
// the removed file in bytes. The size is negative if the file doesn't exist.
Remove(path string) int64
// RemoveAll removes all files from the filesystem. Returns the size of the
// RemoveList removes all files from the filesystem. Returns the size of the
// removed files in bytes.
RemoveAll() int64
RemoveList(path string, options ListOptions) int64
}
// Filesystem is an interface that provides access to a filesystem.

37
io/fs/fs_test.go
View File

@ -88,14 +88,15 @@ func TestFilesystem(t *testing.T) {
"writeFileSafe": testWriteFileSafe,
"writeFileReader": testWriteFileReader,
"writeFileDir": testWriteFileDir,
"delete": testDelete,
"remove": testRemove,
"files": testFiles,
"replace": testReplace,
"list": testList,
"listGlob": testListGlob,
"listSize": testListSize,
"listModified": testListModified,
"deleteAll": testDeleteAll,
"removeAll": testRemoveAll,
"removeList": testRemoveList,
"data": testData,
"statDir": testStatDir,
"mkdirAll": testMkdirAll,
@ -224,7 +225,7 @@ func testOpen(t *testing.T, fs Filesystem) {
require.Equal(t, false, stat.IsDir())
}
func testDelete(t *testing.T, fs Filesystem) {
func testRemove(t *testing.T, fs Filesystem) {
size := fs.Remove("/foobar")
require.Equal(t, int64(-1), size)
@ -461,11 +462,7 @@ func testListModified(t *testing.T, fs Filesystem) {
require.ElementsMatch(t, []string{"/b", "/c"}, files)
}
func testDeleteAll(t *testing.T, fs Filesystem) {
if _, ok := fs.(*diskFilesystem); ok {
return
}
func testRemoveAll(t *testing.T, fs Filesystem) {
fs.WriteFileReader("/foobar1", strings.NewReader("abc"))
fs.WriteFileReader("/path/foobar2", strings.NewReader("abc"))
fs.WriteFileReader("/path/to/foobar3", strings.NewReader("abc"))
@ -475,7 +472,9 @@ func testDeleteAll(t *testing.T, fs Filesystem) {
require.Equal(t, int64(4), cur)
size := fs.RemoveAll()
size := fs.RemoveList("/", ListOptions{
Pattern: "",
})
require.Equal(t, int64(12), size)
cur = fs.Files()
@ -483,6 +482,26 @@ func testDeleteAll(t *testing.T, fs Filesystem) {
require.Equal(t, int64(0), cur)
}
func testRemoveList(t *testing.T, fs Filesystem) {
fs.WriteFileReader("/foobar1", strings.NewReader("abc"))
fs.WriteFileReader("/path/foobar2", strings.NewReader("abc"))
fs.WriteFileReader("/path/to/foobar3", strings.NewReader("abc"))
fs.WriteFileReader("/foobar4", strings.NewReader("abc"))
cur := fs.Files()
require.Equal(t, int64(4), cur)
size := fs.RemoveList("/", ListOptions{
Pattern: "/path/**",
})
require.Equal(t, int64(6), size)
cur = fs.Files()
require.Equal(t, int64(2), cur)
}
func testData(t *testing.T, fs Filesystem) {
file := fs.Open("/foobar")
require.Nil(t, file)

53
io/fs/mem.go
View File

@ -651,6 +651,10 @@ func (fs *memFilesystem) Remove(path string) int64 {
fs.filesLock.Lock()
defer fs.filesLock.Unlock()
return fs.remove(path)
}
func (fs *memFilesystem) remove(path string) int64 {
file, ok := fs.files[path]
if ok {
delete(fs.files, path)
@ -671,14 +675,55 @@ func (fs *memFilesystem) Remove(path string) int64 {
return file.size
}
func (fs *memFilesystem) RemoveAll() int64 {
func (fs *memFilesystem) RemoveList(path string, options ListOptions) int64 {
path = fs.cleanPath(path)
fs.filesLock.Lock()
defer fs.filesLock.Unlock()
size := fs.currentSize
var size int64 = 0
fs.files = make(map[string]*internalMemFile)
fs.currentSize = 0
for _, file := range fs.files {
if !strings.HasPrefix(file.name, path) {
continue
}
if len(options.Pattern) != 0 {
if ok, _ := glob.Match(options.Pattern, file.name, '/'); !ok {
continue
}
}
if options.ModifiedStart != nil {
if file.lastMod.Before(*options.ModifiedStart) {
continue
}
}
if options.ModifiedEnd != nil {
if file.lastMod.After(*options.ModifiedEnd) {
continue
}
}
if options.SizeMin > 0 {
if file.size < options.SizeMin {
continue
}
}
if options.SizeMax > 0 {
if file.size > options.SizeMax {
continue
}
}
if file.dir {
continue
}
size += fs.remove(file.name)
}
return size
}

2
io/fs/readonly.go
View File

@ -41,7 +41,7 @@ func (r *readOnlyFilesystem) Remove(path string) int64 {
return -1
}
func (r *readOnlyFilesystem) RemoveAll() int64 {
func (r *readOnlyFilesystem) RemoveList(path string, options ListOptions) int64 {
return 0
}

2
io/fs/readonly_test.go
View File

@ -32,7 +32,7 @@ func TestReadOnly(t *testing.T) {
res := ro.Remove("/readonly.go")
require.Equal(t, int64(-1), res)
res = ro.RemoveAll()
res = ro.RemoveList("/", ListOptions{})
require.Equal(t, int64(0), res)
rop, ok := ro.(PurgeFilesystem)

48
io/fs/s3.go
View File

@ -428,7 +428,9 @@ func (fs *s3Filesystem) Remove(path string) int64 {
return stat.Size
}
func (fs *s3Filesystem) RemoveAll() int64 {
func (fs *s3Filesystem) RemoveList(path string, options ListOptions) int64 {
path = fs.cleanPath(path)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@ -441,12 +443,54 @@ func (fs *s3Filesystem) RemoveAll() int64 {
defer close(objectsCh)
for object := range fs.client.ListObjects(ctx, fs.bucket, minio.ListObjectsOptions{
Recursive: true,
WithVersions: false,
WithMetadata: false,
Prefix: path,
Recursive: true,
MaxKeys: 0,
StartAfter: "",
UseV1: false,
}) {
if object.Err != nil {
fs.logger.WithError(object.Err).Log("Listing object failed")
continue
}
key := "/" + object.Key
if strings.HasSuffix(key, "/"+fakeDirEntry) {
// filter out fake directory entries (see MkdirAll)
continue
}
if len(options.Pattern) != 0 {
if ok, _ := glob.Match(options.Pattern, key, '/'); !ok {
continue
}
}
if options.ModifiedStart != nil {
if object.LastModified.Before(*options.ModifiedStart) {
continue
}
}
if options.ModifiedEnd != nil {
if object.LastModified.After(*options.ModifiedEnd) {
continue
}
}
if options.SizeMin > 0 {
if object.Size < options.SizeMin {
continue
}
}
if options.SizeMax > 0 {
if object.Size > options.SizeMax {
continue
}
}
totalSize += object.Size
objectsCh <- object
}