diff --git a/docs/docs.go b/docs/docs.go index 472d7b11..e478638a 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -467,6 +467,72 @@ const docTemplate = `{ } } } + }, + "delete": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Delete all files on a filesystem based on patterns", + "produces": [ + "application/json" + ], + "tags": [ + "v16.?.?" + ], + "summary": "Delete all files on a filesystem based on patterns", + "operationId": "filesystem-3-delete-files", + "parameters": [ + { + "type": "string", + "description": "Name of the filesystem", + "name": "storage", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "glob pattern for file names", + "name": "glob", + "in": "query" + }, + { + "type": "integer", + "description": "minimal size of files", + "name": "size_min", + "in": "query" + }, + { + "type": "integer", + "description": "maximal size of files", + "name": "size_max", + "in": "query" + }, + { + "type": "integer", + "description": "minimal last modification time", + "name": "lastmod_start", + "in": "query" + }, + { + "type": "integer", + "description": "maximal last modification time", + "name": "lastmod_end", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + } } }, "/api/v3/fs/{storage}/{filepath}": { @@ -605,7 +671,7 @@ const docTemplate = `{ "v16.7.2" ], "summary": "Remove a file from a filesystem", - "operationId": "filesystem-3-delete-file", + "operationId": "filesystem-3-delete-files", "parameters": [ { "type": "string", diff --git a/docs/swagger.json b/docs/swagger.json index bd0c4f1d..1c9a0a9d 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -460,6 +460,72 @@ } } } + }, + "delete": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Delete all files on a filesystem based on patterns", + "produces": [ + "application/json" + ], + "tags": [ + "v16.?.?" + ], + "summary": "Delete all files on a filesystem based on patterns", + "operationId": "filesystem-3-delete-files", + "parameters": [ + { + "type": "string", + "description": "Name of the filesystem", + "name": "storage", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "glob pattern for file names", + "name": "glob", + "in": "query" + }, + { + "type": "integer", + "description": "minimal size of files", + "name": "size_min", + "in": "query" + }, + { + "type": "integer", + "description": "maximal size of files", + "name": "size_max", + "in": "query" + }, + { + "type": "integer", + "description": "minimal last modification time", + "name": "lastmod_start", + "in": "query" + }, + { + "type": "integer", + "description": "maximal last modification time", + "name": "lastmod_end", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + } } }, "/api/v3/fs/{storage}/{filepath}": { @@ -598,7 +664,7 @@ "v16.7.2" ], "summary": "Remove a file from a filesystem", - "operationId": "filesystem-3-delete-file", + "operationId": "filesystem-3-delete-files", "parameters": [ { "type": "string", diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 79a3946b..41c84e81 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -2205,6 +2205,49 @@ paths: tags: - v16.?.? /api/v3/fs/{storage}: + delete: + description: Delete all files on a filesystem based on patterns + operationId: filesystem-3-delete-files + parameters: + - description: Name of the filesystem + in: path + name: storage + required: true + type: string + - description: glob pattern for file names + in: query + name: glob + type: string + - description: minimal size of files + in: query + name: size_min + type: integer + - description: maximal size of files + in: query + name: size_max + type: integer + - description: minimal last modification time + in: query + name: lastmod_start + type: integer + - description: maximal last modification time + in: query + name: lastmod_end + type: integer + produces: + - application/json + responses: + "200": + description: OK + schema: + items: + type: string + type: array + security: + - ApiKeyAuth: [] + summary: Delete all files on a filesystem based on patterns + tags: + - v16.?.? get: description: List all files on a filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order. @@ -2260,7 +2303,7 @@ paths: /api/v3/fs/{storage}/{filepath}: delete: description: Remove a file from a filesystem - operationId: filesystem-3-delete-file + operationId: filesystem-3-delete-files parameters: - description: Name of the filesystem in: path diff --git a/http/handler/api/filesystems.go b/http/handler/api/filesystems.go index 7e0ddec2..1e0c1988 100644 --- a/http/handler/api/filesystems.go +++ b/http/handler/api/filesystems.go @@ -29,7 +29,7 @@ func NewFS(filesystems map[string]FSConfig) *FSHandler { } } -// GetFileAPI returns the file at the given path +// GetFile returns the file at the given path // @Summary Fetch a file from a filesystem // @Description Fetch a file from a filesystem // @Tags v16.7.2 @@ -44,7 +44,7 @@ func NewFS(filesystems map[string]FSConfig) *FSHandler { // @Security ApiKeyAuth // @Router /api/v3/fs/{storage}/{filepath} [get] func (h *FSHandler) GetFile(c echo.Context) error { - name := util.PathParam(c, "name") + name := util.PathParam(c, "storage") config, ok := h.filesystems[name] if !ok { @@ -54,7 +54,7 @@ func (h *FSHandler) GetFile(c echo.Context) error { return config.Handler.GetFile(c) } -// PutFileAPI adds or overwrites a file at the given path +// PutFile adds or overwrites a file at the given path // @Summary Add a file to a filesystem // @Description Writes or overwrites a file on a filesystem // @Tags v16.7.2 @@ -71,7 +71,7 @@ func (h *FSHandler) GetFile(c echo.Context) error { // @Security ApiKeyAuth // @Router /api/v3/fs/{storage}/{filepath} [put] func (h *FSHandler) PutFile(c echo.Context) error { - name := util.PathParam(c, "name") + name := util.PathParam(c, "storage") config, ok := h.filesystems[name] if !ok { @@ -81,11 +81,11 @@ func (h *FSHandler) PutFile(c echo.Context) error { return config.Handler.PutFile(c) } -// DeleteFileAPI removes a file from a filesystem +// DeleteFile removes a file from a filesystem // @Summary Remove a file from a filesystem // @Description Remove a file from a filesystem // @Tags v16.7.2 -// @ID filesystem-3-delete-file +// @ID filesystem-3-delete-files // @Produce text/plain // @Param storage path string true "Name of the filesystem" // @Param filepath path string true "Path to file" @@ -94,7 +94,7 @@ func (h *FSHandler) PutFile(c echo.Context) error { // @Security ApiKeyAuth // @Router /api/v3/fs/{storage}/{filepath} [delete] func (h *FSHandler) DeleteFile(c echo.Context) error { - name := util.PathParam(c, "name") + name := util.PathParam(c, "storage") config, ok := h.filesystems[name] if !ok { @@ -104,6 +104,32 @@ func (h *FSHandler) DeleteFile(c echo.Context) error { return config.Handler.DeleteFile(c) } +// DeleteFiles deletes all files on a filesystem based on patterns +// @Summary Delete all files on a filesystem based on patterns +// @Description Delete all files on a filesystem based on patterns +// @Tags v16.?.? +// @ID filesystem-3-delete-files +// @Produce json +// @Param storage path string true "Name of the filesystem" +// @Param glob query string false "glob pattern for file names" +// @Param size_min query int64 false "minimal size of files" +// @Param size_max query int64 false "maximal size of files" +// @Param lastmod_start query int64 false "minimal last modification time" +// @Param lastmod_end query int64 false "maximal last modification time" +// @Success 200 {array} string +// @Security ApiKeyAuth +// @Router /api/v3/fs/{storage} [delete] +func (h *FSHandler) DeleteFiles(c echo.Context) error { + name := util.PathParam(c, "storage") + + config, ok := h.filesystems[name] + if !ok { + return api.Err(http.StatusNotFound, "File not found", "unknown filesystem: %s", name) + } + + return config.Handler.DeleteFiles(c) +} + // ListFiles lists all files on a filesystem // @Summary List all files on a filesystem // @Description List all files on a filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order. @@ -122,7 +148,7 @@ func (h *FSHandler) DeleteFile(c echo.Context) error { // @Security ApiKeyAuth // @Router /api/v3/fs/{storage} [get] func (h *FSHandler) ListFiles(c echo.Context) error { - name := util.PathParam(c, "name") + name := util.PathParam(c, "storage") config, ok := h.filesystems[name] if !ok { diff --git a/http/handler/api/filesystems_test.go b/http/handler/api/filesystems_test.go index 9f2f5739..462cf34c 100644 --- a/http/handler/api/filesystems_test.go +++ b/http/handler/api/filesystems_test.go @@ -43,10 +43,11 @@ func getDummyFilesystemsRouter(filesystems []httpfs.FS) (*echo.Echo, error) { return nil, err } - router.GET("/:name/*", handler.GetFile) - router.PUT("/:name/*", handler.PutFile) - router.DELETE("/:name/*", handler.DeleteFile) - router.GET("/:name", handler.ListFiles) + router.GET("/:storage/*", handler.GetFile) + router.PUT("/:storage/*", handler.PutFile) + router.DELETE("/:storage", handler.DeleteFiles) + router.DELETE("/:storage/*", handler.DeleteFile) + router.GET("/:storage", handler.ListFiles) router.GET("/", handler.List) router.PUT("/", handler.FileOperation) @@ -206,11 +207,11 @@ func TestFilesystemsListLastmod(t *testing.T) { require.NoError(t, err) memfs.WriteFileReader("/a", strings.NewReader("a")) - time.Sleep(500 * time.Millisecond) + time.Sleep(1 * time.Second) memfs.WriteFileReader("/b", strings.NewReader("b")) - time.Sleep(500 * time.Millisecond) + time.Sleep(1 * time.Second) memfs.WriteFileReader("/c", strings.NewReader("c")) - time.Sleep(500 * time.Millisecond) + time.Sleep(1 * time.Second) memfs.WriteFileReader("/d", strings.NewReader("d")) var a, b, c, d time.Time @@ -268,6 +269,129 @@ func TestFilesystemsListLastmod(t *testing.T) { require.ElementsMatch(t, []string{"/b", "/c"}, files) } +func TestFilesystemsDeleteFiles(t *testing.T) { + memfs, err := fs.NewMemFilesystem(fs.MemConfig{}) + require.NoError(t, err) + + memfs.WriteFileReader("/a", strings.NewReader("a")) + memfs.WriteFileReader("/aa", strings.NewReader("aa")) + memfs.WriteFileReader("/aaa", strings.NewReader("aaa")) + memfs.WriteFileReader("/aaaa", strings.NewReader("aaaa")) + + filesystems := []httpfs.FS{ + { + Name: "foo", + Mountpoint: "/foo", + AllowWrite: true, + Filesystem: memfs, + }, + } + + router, err := getDummyFilesystemsRouter(filesystems) + require.NoError(t, err) + + mock.Request(t, http.StatusBadRequest, router, "DELETE", "/foo", nil) + + getNames := func(r *mock.Response) []string { + files := []string{} + err := json.Unmarshal(r.Raw, &files) + require.NoError(t, err) + + return files + } + + files := getNames(mock.Request(t, http.StatusOK, router, "DELETE", "/foo?glob=/**", nil)) + require.Equal(t, 4, len(files)) + require.ElementsMatch(t, []string{"/a", "/aa", "/aaa", "/aaaa"}, files) + + require.Equal(t, int64(0), memfs.Files()) +} + +func TestFilesystemsDeleteFilesSize(t *testing.T) { + memfs, err := fs.NewMemFilesystem(fs.MemConfig{}) + require.NoError(t, err) + + memfs.WriteFileReader("/a", strings.NewReader("a")) + memfs.WriteFileReader("/aa", strings.NewReader("aa")) + memfs.WriteFileReader("/aaa", strings.NewReader("aaa")) + memfs.WriteFileReader("/aaaa", strings.NewReader("aaaa")) + + filesystems := []httpfs.FS{ + { + Name: "foo", + Mountpoint: "/foo", + AllowWrite: true, + Filesystem: memfs, + }, + } + + router, err := getDummyFilesystemsRouter(filesystems) + require.NoError(t, err) + + getNames := func(r *mock.Response) []string { + files := []string{} + err := json.Unmarshal(r.Raw, &files) + require.NoError(t, err) + + return files + } + + files := getNames(mock.Request(t, http.StatusOK, router, "DELETE", "/foo?glob=/**&size_min=2&size_max=3", nil)) + require.Equal(t, 2, len(files)) + require.ElementsMatch(t, []string{"/aa", "/aaa"}, files) + + require.Equal(t, int64(2), memfs.Files()) +} + +func TestFilesystemsDeleteFilesLastmod(t *testing.T) { + memfs, err := fs.NewMemFilesystem(fs.MemConfig{}) + require.NoError(t, err) + + memfs.WriteFileReader("/a", strings.NewReader("a")) + time.Sleep(1 * time.Second) + memfs.WriteFileReader("/b", strings.NewReader("b")) + time.Sleep(1 * time.Second) + memfs.WriteFileReader("/c", strings.NewReader("c")) + time.Sleep(1 * time.Second) + memfs.WriteFileReader("/d", strings.NewReader("d")) + + var b, c time.Time + + for _, f := range memfs.List("/", fs.ListOptions{}) { + if f.Name() == "/b" { + b = f.ModTime() + } else if f.Name() == "/c" { + c = f.ModTime() + } + } + + filesystems := []httpfs.FS{ + { + Name: "foo", + Mountpoint: "/foo", + AllowWrite: true, + Filesystem: memfs, + }, + } + + router, err := getDummyFilesystemsRouter(filesystems) + require.NoError(t, err) + + getNames := func(r *mock.Response) []string { + files := []string{} + err := json.Unmarshal(r.Raw, &files) + require.NoError(t, err) + + return files + } + + files := getNames(mock.Request(t, http.StatusOK, router, "DELETE", "/foo?glob=/**&lastmod_start="+strconv.FormatInt(b.Unix(), 10)+"&lastmod_end="+strconv.FormatInt(c.Unix(), 10), nil)) + require.Equal(t, 2, len(files)) + require.ElementsMatch(t, []string{"/b", "/c"}, files) + + require.Equal(t, int64(2), memfs.Files()) +} + func TestFileOperation(t *testing.T) { memfs1, err := fs.NewMemFilesystem(fs.MemConfig{}) require.NoError(t, err) diff --git a/http/handler/filesystem.go b/http/handler/filesystem.go index 2cfafa4a..106cab7a 100644 --- a/http/handler/filesystem.go +++ b/http/handler/filesystem.go @@ -170,6 +170,62 @@ func (h *FSHandler) DeleteFile(c echo.Context) error { return c.String(http.StatusOK, "Deleted: "+path) } +func (h *FSHandler) DeleteFiles(c echo.Context) error { + pattern := util.DefaultQuery(c, "glob", "") + sizeMin := util.DefaultQuery(c, "size_min", "0") + sizeMax := util.DefaultQuery(c, "size_max", "0") + modifiedStart := util.DefaultQuery(c, "lastmod_start", "") + modifiedEnd := util.DefaultQuery(c, "lastmod_end", "") + + if len(pattern) == 0 { + return api.Err(http.StatusBadRequest, "Bad request", "A glob pattern is required") + } + + options := fs.ListOptions{ + Pattern: pattern, + } + + if x, err := strconv.ParseInt(sizeMin, 10, 64); err != nil { + return api.Err(http.StatusBadRequest, "Bad request", "%s", err) + } else { + options.SizeMin = x + } + + if x, err := strconv.ParseInt(sizeMax, 10, 64); err != nil { + return api.Err(http.StatusBadRequest, "Bad request", "%s", err) + } else { + options.SizeMax = x + } + + if len(modifiedStart) != 0 { + if x, err := strconv.ParseInt(modifiedStart, 10, 64); err != nil { + return api.Err(http.StatusBadRequest, "Bad request", "%s", err) + } else { + t := time.Unix(x, 0) + options.ModifiedStart = &t + } + } + + if len(modifiedEnd) != 0 { + if x, err := strconv.ParseInt(modifiedEnd, 10, 64); err != nil { + return api.Err(http.StatusBadRequest, "Bad request", "%s", err) + } else { + t := time.Unix(x+1, 0) + options.ModifiedEnd = &t + } + } + + paths, _ := h.FS.Filesystem.RemoveList("/", options) + + if h.FS.Cache != nil { + for _, path := range paths { + h.FS.Cache.Delete(path) + } + } + + return c.JSON(http.StatusOK, paths) +} + func (h *FSHandler) ListFiles(c echo.Context) error { pattern := util.DefaultQuery(c, "glob", "") sizeMin := util.DefaultQuery(c, "size_min", "0") diff --git a/http/server.go b/http/server.go index 620ab80f..033d42f9 100644 --- a/http/server.go +++ b/http/server.go @@ -607,19 +607,20 @@ func (s *server) setRoutesV3(v3 *echo.Group) { v3.GET("/fs", handler.List) v3.PUT("/fs", handler.FileOperation) - v3.GET("/fs/:name", handler.ListFiles) - v3.GET("/fs/:name/*", handler.GetFile, mwmime.NewWithConfig(mwmime.Config{ + v3.GET("/fs/:storage", handler.ListFiles) + v3.GET("/fs/:storage/*", handler.GetFile, mwmime.NewWithConfig(mwmime.Config{ MimeTypesFile: s.mimeTypesFile, DefaultContentType: "application/data", })) - v3.HEAD("/fs/:name/*", handler.GetFile, mwmime.NewWithConfig(mwmime.Config{ + v3.HEAD("/fs/:storage/*", handler.GetFile, mwmime.NewWithConfig(mwmime.Config{ MimeTypesFile: s.mimeTypesFile, DefaultContentType: "application/data", })) if !s.readOnly { - v3.PUT("/fs/:name/*", handler.PutFile) - v3.DELETE("/fs/:name/*", handler.DeleteFile) + v3.PUT("/fs/:storage/*", handler.PutFile) + v3.DELETE("/fs/:storage", handler.DeleteFiles) + v3.DELETE("/fs/:storage/*", handler.DeleteFile) } // v3 RTMP diff --git a/io/fs/disk.go b/io/fs/disk.go index 96ebda1a..f9a9aea5 100644 --- a/io/fs/disk.go +++ b/io/fs/disk.go @@ -519,10 +519,11 @@ func (fs *diskFilesystem) Remove(path string) int64 { return size } -func (fs *diskFilesystem) RemoveList(path string, options ListOptions) int64 { +func (fs *diskFilesystem) RemoveList(path string, options ListOptions) ([]string, int64) { path = fs.cleanPath(path) var size int64 = 0 + files := []string{} fs.walk(path, func(path string, info os.FileInfo) { if path == fs.root { @@ -569,11 +570,12 @@ func (fs *diskFilesystem) RemoveList(path string, options ListOptions) int64 { } if err := os.Remove(path); err == nil { + files = append(files, name) size += info.Size() } }) - return size + return files, size } func (fs *diskFilesystem) List(path string, options ListOptions) []FileInfo { diff --git a/io/fs/fs.go b/io/fs/fs.go index 7c0245a9..82ed5117 100644 --- a/io/fs/fs.go +++ b/io/fs/fs.go @@ -122,7 +122,7 @@ type WriteFilesystem interface { // RemoveList removes all files from the filesystem. Returns the size of the // removed files in bytes. - RemoveList(path string, options ListOptions) int64 + RemoveList(path string, options ListOptions) ([]string, int64) } // Filesystem is an interface that provides access to a filesystem. diff --git a/io/fs/fs_test.go b/io/fs/fs_test.go index 764adecd..48820429 100644 --- a/io/fs/fs_test.go +++ b/io/fs/fs_test.go @@ -472,7 +472,7 @@ func testRemoveAll(t *testing.T, fs Filesystem) { require.Equal(t, int64(4), cur) - size := fs.RemoveList("/", ListOptions{ + _, size := fs.RemoveList("/", ListOptions{ Pattern: "", }) require.Equal(t, int64(12), size) @@ -492,7 +492,7 @@ func testRemoveList(t *testing.T, fs Filesystem) { require.Equal(t, int64(4), cur) - size := fs.RemoveList("/", ListOptions{ + _, size := fs.RemoveList("/", ListOptions{ Pattern: "/path/**", }) require.Equal(t, int64(6), size) diff --git a/io/fs/mem.go b/io/fs/mem.go index 54b45d64..44295d78 100644 --- a/io/fs/mem.go +++ b/io/fs/mem.go @@ -675,13 +675,14 @@ func (fs *memFilesystem) remove(path string) int64 { return file.size } -func (fs *memFilesystem) RemoveList(path string, options ListOptions) int64 { +func (fs *memFilesystem) RemoveList(path string, options ListOptions) ([]string, int64) { path = fs.cleanPath(path) fs.filesLock.Lock() defer fs.filesLock.Unlock() var size int64 = 0 + files := []string{} for _, file := range fs.files { if !strings.HasPrefix(file.name, path) { @@ -723,9 +724,11 @@ func (fs *memFilesystem) RemoveList(path string, options ListOptions) int64 { } size += fs.remove(file.name) + + files = append(files, file.name) } - return size + return files, size } func (fs *memFilesystem) List(path string, options ListOptions) []FileInfo { diff --git a/io/fs/readonly.go b/io/fs/readonly.go index 00eb7782..2991944c 100644 --- a/io/fs/readonly.go +++ b/io/fs/readonly.go @@ -41,8 +41,8 @@ func (r *readOnlyFilesystem) Remove(path string) int64 { return -1 } -func (r *readOnlyFilesystem) RemoveList(path string, options ListOptions) int64 { - return 0 +func (r *readOnlyFilesystem) RemoveList(path string, options ListOptions) ([]string, int64) { + return nil, 0 } func (r *readOnlyFilesystem) Purge(size int64) int64 { diff --git a/io/fs/readonly_test.go b/io/fs/readonly_test.go index 66dd2380..ffcf8727 100644 --- a/io/fs/readonly_test.go +++ b/io/fs/readonly_test.go @@ -32,7 +32,7 @@ func TestReadOnly(t *testing.T) { res := ro.Remove("/readonly.go") require.Equal(t, int64(-1), res) - res = ro.RemoveList("/", ListOptions{}) + _, res = ro.RemoveList("/", ListOptions{}) require.Equal(t, int64(0), res) rop, ok := ro.(PurgeFilesystem) diff --git a/io/fs/s3.go b/io/fs/s3.go index 8dc02161..41000088 100644 --- a/io/fs/s3.go +++ b/io/fs/s3.go @@ -428,13 +428,14 @@ func (fs *s3Filesystem) Remove(path string) int64 { return stat.Size } -func (fs *s3Filesystem) RemoveList(path string, options ListOptions) int64 { +func (fs *s3Filesystem) RemoveList(path string, options ListOptions) ([]string, int64) { path = fs.cleanPath(path) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - totalSize := int64(0) + var totalSize int64 = 0 + files := []string{} objectsCh := make(chan minio.ObjectInfo) @@ -493,6 +494,8 @@ func (fs *s3Filesystem) RemoveList(path string, options ListOptions) int64 { totalSize += object.Size objectsCh <- object + + files = append(files, key) } }() @@ -504,7 +507,7 @@ func (fs *s3Filesystem) RemoveList(path string, options ListOptions) int64 { fs.logger.Debug().Log("Deleted all files") - return totalSize + return files, totalSize } func (fs *s3Filesystem) List(path string, options ListOptions) []FileInfo {