From ecfbbe38574819c3431b1b9846cd5239df9b942d Mon Sep 17 00:00:00 2001 From: Ingo Oppermann Date: Fri, 17 Mar 2023 15:15:20 +0100 Subject: [PATCH] Allow to list files by ranges of size and/or lastmod The listing options are implemented by the query parameters size_min, size_max, lastmod_start, and lastmod_end, or by the new ListOptions type. size_min and size_max expect a number of bytes, lastmod_start and lastmod_end expect a unix timestamp. All values are inclusive. --- docs/docs.go | 29 +++++- docs/swagger.json | 29 +++++- docs/swagger.yaml | 21 ++++- http/handler/api/filesystems.go | 4 + http/handler/api/filesystems_test.go | 132 ++++++++++++++++++++++++++- http/handler/filesystem.go | 47 +++++++++- io/fs/disk.go | 30 +++++- io/fs/fs.go | 10 +- io/fs/fs_test.go | 103 +++++++++++++++++++-- io/fs/mem.go | 30 +++++- io/fs/mem_test.go | 2 +- io/fs/s3.go | 32 ++++++- restream/fs/fs.go | 4 +- restream/fs/fs_test.go | 8 +- 14 files changed, 435 insertions(+), 46 deletions(-) diff --git a/docs/docs.go b/docs/docs.go index bf6a0a52..472d7b11 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -322,7 +322,6 @@ const docTemplate = `{ "application/json" ], "tags": [ - "v16.12.0", "v16.12.0" ], "summary": "List all registered filesystems", @@ -402,7 +401,6 @@ const docTemplate = `{ "application/json" ], "tags": [ - "v16.7.2", "v16.7.2" ], "summary": "List all files on a filesystem", @@ -421,6 +419,30 @@ const docTemplate = `{ "name": "glob", "in": "query" }, + { + "type": "integer", + "description": "minimal size of files", + "name": "size_min", + "in": "query" + }, + { + "type": "integer", + "description": "maximal size of files", + "name": "size_max", + "in": "query" + }, + { + "type": "integer", + "description": "minimal last modification time", + "name": "lastmod_start", + "in": "query" + }, + { + "type": "integer", + "description": "maximal last modification time", + "name": "lastmod_end", + "in": "query" + }, { "type": "string", "description": "none, name, size, lastmod", @@ -460,7 +482,6 @@ const docTemplate = `{ "application/json" ], "tags": [ - "v16.7.2", "v16.7.2" ], "summary": "Fetch a file from a filesystem", @@ -517,7 +538,6 @@ const docTemplate = `{ "application/json" ], "tags": [ - "v16.7.2", "v16.7.2" ], "summary": "Add a file to a filesystem", @@ -582,7 +602,6 @@ const docTemplate = `{ "text/plain" ], "tags": [ - "v16.7.2", "v16.7.2" ], "summary": "Remove a file from a filesystem", diff --git a/docs/swagger.json b/docs/swagger.json index 451abadb..bd0c4f1d 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -315,7 +315,6 @@ "application/json" ], "tags": [ - "v16.12.0", "v16.12.0" ], "summary": "List all registered filesystems", @@ -395,7 +394,6 @@ "application/json" ], "tags": [ - "v16.7.2", "v16.7.2" ], "summary": "List all files on a filesystem", @@ -414,6 +412,30 @@ "name": "glob", "in": "query" }, + { + "type": "integer", + "description": "minimal size of files", + "name": "size_min", + "in": "query" + }, + { + "type": "integer", + "description": "maximal size of files", + "name": "size_max", + "in": "query" + }, + { + "type": "integer", + "description": "minimal last modification time", + "name": "lastmod_start", + "in": "query" + }, + { + "type": "integer", + "description": "maximal last modification time", + "name": "lastmod_end", + "in": "query" + }, { "type": "string", "description": "none, name, size, lastmod", @@ -453,7 +475,6 @@ "application/json" ], "tags": [ - "v16.7.2", "v16.7.2" ], "summary": "Fetch a file from a filesystem", @@ -510,7 +531,6 @@ "application/json" ], "tags": [ - "v16.7.2", "v16.7.2" ], "summary": "Add a file to a filesystem", @@ -575,7 +595,6 @@ "text/plain" ], "tags": [ - "v16.7.2", "v16.7.2" ], "summary": "Remove a file from a filesystem", diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 3156530c..79a3946b 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -2172,7 +2172,6 @@ paths: summary: List all registered filesystems tags: - v16.12.0 - - v16.12.0 put: consumes: - application/json @@ -2220,6 +2219,22 @@ paths: in: query name: glob type: string + - description: minimal size of files + in: query + name: size_min + type: integer + - description: maximal size of files + in: query + name: size_max + type: integer + - description: minimal last modification time + in: query + name: lastmod_start + type: integer + - description: maximal last modification time + in: query + name: lastmod_end + type: integer - description: none, name, size, lastmod in: query name: sort @@ -2242,7 +2257,6 @@ paths: summary: List all files on a filesystem tags: - v16.7.2 - - v16.7.2 /api/v3/fs/{storage}/{filepath}: delete: description: Remove a file from a filesystem @@ -2274,7 +2288,6 @@ paths: summary: Remove a file from a filesystem tags: - v16.7.2 - - v16.7.2 get: description: Fetch a file from a filesystem operationId: filesystem-3-get-file @@ -2310,7 +2323,6 @@ paths: summary: Fetch a file from a filesystem tags: - v16.7.2 - - v16.7.2 put: consumes: - application/data @@ -2356,7 +2368,6 @@ paths: summary: Add a file to a filesystem tags: - v16.7.2 - - v16.7.2 /api/v3/log: get: description: Get the last log lines of the Restreamer application diff --git a/http/handler/api/filesystems.go b/http/handler/api/filesystems.go index a19edd9b..7e0ddec2 100644 --- a/http/handler/api/filesystems.go +++ b/http/handler/api/filesystems.go @@ -112,6 +112,10 @@ func (h *FSHandler) DeleteFile(c echo.Context) error { // @Produce json // @Param storage path string true "Name of the filesystem" // @Param glob query string false "glob pattern for file names" +// @Param size_min query int64 false "minimal size of files" +// @Param size_max query int64 false "maximal size of files" +// @Param lastmod_start query int64 false "minimal last modification time" +// @Param lastmod_end query int64 false "maximal last modification time" // @Param sort query string false "none, name, size, lastmod" // @Param order query string false "asc, desc" // @Success 200 {array} api.FileInfo diff --git a/http/handler/api/filesystems_test.go b/http/handler/api/filesystems_test.go index 4d33dd24..9f2f5739 100644 --- a/http/handler/api/filesystems_test.go +++ b/http/handler/api/filesystems_test.go @@ -5,7 +5,10 @@ import ( "encoding/json" "io" "net/http" + "strconv" + "strings" "testing" + "time" "github.com/datarhei/core/v16/http/api" httpfs "github.com/datarhei/core/v16/http/fs" @@ -102,7 +105,7 @@ func TestFilesystems(t *testing.T) { mock.Request(t, http.StatusNoContent, router, "PUT", "/foo/file", data) - require.Equal(t, 1, len(memfs.List("/", ""))) + require.Equal(t, 1, len(memfs.List("/", fs.ListOptions{}))) response = mock.Request(t, http.StatusOK, router, "GET", "/foo", nil) @@ -127,7 +130,7 @@ func TestFilesystems(t *testing.T) { mock.Request(t, http.StatusOK, router, "DELETE", "/foo/file", nil) mock.Request(t, http.StatusNotFound, router, "GET", "/foo/file", nil) - require.Equal(t, 0, len(memfs.List("/", ""))) + require.Equal(t, 0, len(memfs.List("/", fs.ListOptions{}))) response = mock.Request(t, http.StatusOK, router, "GET", "/foo", nil) @@ -140,6 +143,131 @@ func TestFilesystems(t *testing.T) { require.Equal(t, 0, len(l)) } +func TestFilesystemsListSize(t *testing.T) { + memfs, err := fs.NewMemFilesystem(fs.MemConfig{}) + require.NoError(t, err) + + memfs.WriteFileReader("/a", strings.NewReader("a")) + memfs.WriteFileReader("/aa", strings.NewReader("aa")) + memfs.WriteFileReader("/aaa", strings.NewReader("aaa")) + memfs.WriteFileReader("/aaaa", strings.NewReader("aaaa")) + + filesystems := []httpfs.FS{ + { + Name: "foo", + Mountpoint: "/foo", + AllowWrite: true, + Filesystem: memfs, + }, + } + + router, err := getDummyFilesystemsRouter(filesystems) + require.NoError(t, err) + + response := mock.Request(t, http.StatusOK, router, "GET", "/foo", nil) + + f := []api.FilesystemInfo{} + err = json.Unmarshal(response.Raw, &f) + require.NoError(t, err) + + require.Equal(t, 4, len(f)) + + getNames := func(r *mock.Response) []string { + files := []api.FilesystemInfo{} + err := json.Unmarshal(r.Raw, &files) + require.NoError(t, err) + + names := []string{} + for _, f := range files { + names = append(names, f.Name) + } + return names + } + + files := getNames(mock.Request(t, http.StatusOK, router, "GET", "/foo?size_min=1", nil)) + require.Equal(t, 4, len(files)) + require.ElementsMatch(t, []string{"/a", "/aa", "/aaa", "/aaaa"}, files) + + files = getNames(mock.Request(t, http.StatusOK, router, "GET", "/foo?size_min=2", nil)) + require.Equal(t, 3, len(files)) + require.ElementsMatch(t, []string{"/aa", "/aaa", "/aaaa"}, files) + + files = getNames(mock.Request(t, http.StatusOK, router, "GET", "/foo?size_max=4", nil)) + require.Equal(t, 4, len(files)) + require.ElementsMatch(t, []string{"/a", "/aa", "/aaa", "/aaaa"}, files) + + files = getNames(mock.Request(t, http.StatusOK, router, "GET", "/foo?size_min=2&size_max=3", nil)) + require.Equal(t, 2, len(files)) + require.ElementsMatch(t, []string{"/aa", "/aaa"}, files) +} + +func TestFilesystemsListLastmod(t *testing.T) { + memfs, err := fs.NewMemFilesystem(fs.MemConfig{}) + require.NoError(t, err) + + memfs.WriteFileReader("/a", strings.NewReader("a")) + time.Sleep(500 * time.Millisecond) + memfs.WriteFileReader("/b", strings.NewReader("b")) + time.Sleep(500 * time.Millisecond) + memfs.WriteFileReader("/c", strings.NewReader("c")) + time.Sleep(500 * time.Millisecond) + memfs.WriteFileReader("/d", strings.NewReader("d")) + + var a, b, c, d time.Time + + for _, f := range memfs.List("/", fs.ListOptions{}) { + if f.Name() == "/a" { + a = f.ModTime() + } else if f.Name() == "/b" { + b = f.ModTime() + } else if f.Name() == "/c" { + c = f.ModTime() + } else if f.Name() == "/d" { + d = f.ModTime() + } + } + + filesystems := []httpfs.FS{ + { + Name: "foo", + Mountpoint: "/foo", + AllowWrite: true, + Filesystem: memfs, + }, + } + + router, err := getDummyFilesystemsRouter(filesystems) + require.NoError(t, err) + + getNames := func(r *mock.Response) []string { + files := []api.FilesystemInfo{} + err := json.Unmarshal(r.Raw, &files) + require.NoError(t, err) + + names := []string{} + for _, f := range files { + names = append(names, f.Name) + } + return names + } + + files := getNames(mock.Request(t, http.StatusOK, router, "GET", "/foo?lastmod_start="+strconv.FormatInt(a.Unix(), 10), nil)) + require.Equal(t, 4, len(files)) + require.ElementsMatch(t, []string{"/a", "/b", "/c", "/d"}, files) + + files = getNames(mock.Request(t, http.StatusOK, router, "GET", "/foo?lastmod_start="+strconv.FormatInt(b.Unix(), 10), nil)) + require.Equal(t, 3, len(files)) + require.ElementsMatch(t, []string{"/b", "/c", "/d"}, files) + + files = getNames(mock.Request(t, http.StatusOK, router, "GET", "/foo?lastmod_end="+strconv.FormatInt(d.Unix(), 10), nil)) + require.Equal(t, 4, len(files)) + require.ElementsMatch(t, []string{"/a", "/b", "/c", "/d"}, files) + + files = getNames(mock.Request(t, http.StatusOK, router, "GET", "/foo?lastmod_start="+strconv.FormatInt(b.Unix(), 10)+"&lastmod_end="+strconv.FormatInt(c.Unix(), 10), nil)) + require.Equal(t, 2, len(files)) + require.ElementsMatch(t, []string{"/b", "/c"}, files) +} + func TestFileOperation(t *testing.T) { memfs1, err := fs.NewMemFilesystem(fs.MemConfig{}) require.NoError(t, err) diff --git a/http/handler/filesystem.go b/http/handler/filesystem.go index 3e0b91d0..2cfafa4a 100644 --- a/http/handler/filesystem.go +++ b/http/handler/filesystem.go @@ -13,19 +13,20 @@ import ( "time" "github.com/datarhei/core/v16/http/api" - "github.com/datarhei/core/v16/http/fs" + httpfs "github.com/datarhei/core/v16/http/fs" "github.com/datarhei/core/v16/http/handler/util" + "github.com/datarhei/core/v16/io/fs" "github.com/labstack/echo/v4" ) // The FSHandler type provides handlers for manipulating a filesystem type FSHandler struct { - FS fs.FS + FS httpfs.FS } // NewFS return a new FSHandler type. You have to provide a filesystem to act on. -func NewFS(fs fs.FS) *FSHandler { +func NewFS(fs httpfs.FS) *FSHandler { return &FSHandler{ FS: fs, } @@ -171,10 +172,48 @@ func (h *FSHandler) DeleteFile(c echo.Context) error { func (h *FSHandler) ListFiles(c echo.Context) error { pattern := util.DefaultQuery(c, "glob", "") + sizeMin := util.DefaultQuery(c, "size_min", "0") + sizeMax := util.DefaultQuery(c, "size_max", "0") + modifiedStart := util.DefaultQuery(c, "lastmod_start", "") + modifiedEnd := util.DefaultQuery(c, "lastmod_end", "") sortby := util.DefaultQuery(c, "sort", "none") order := util.DefaultQuery(c, "order", "asc") - files := h.FS.Filesystem.List("/", pattern) + options := fs.ListOptions{ + Pattern: pattern, + } + + if x, err := strconv.ParseInt(sizeMin, 10, 64); err != nil { + return api.Err(http.StatusBadRequest, "Bad request", "%s", err) + } else { + options.SizeMin = x + } + + if x, err := strconv.ParseInt(sizeMax, 10, 64); err != nil { + return api.Err(http.StatusBadRequest, "Bad request", "%s", err) + } else { + options.SizeMax = x + } + + if len(modifiedStart) != 0 { + if x, err := strconv.ParseInt(modifiedStart, 10, 64); err != nil { + return api.Err(http.StatusBadRequest, "Bad request", "%s", err) + } else { + t := time.Unix(x, 0) + options.ModifiedStart = &t + } + } + + if len(modifiedEnd) != 0 { + if x, err := strconv.ParseInt(modifiedEnd, 10, 64); err != nil { + return api.Err(http.StatusBadRequest, "Bad request", "%s", err) + } else { + t := time.Unix(x+1, 0) + options.ModifiedEnd = &t + } + } + + files := h.FS.Filesystem.List("/", options) var sortFunc func(i, j int) bool diff --git a/io/fs/disk.go b/io/fs/disk.go index b6d352d1..0b1d75d7 100644 --- a/io/fs/disk.go +++ b/io/fs/disk.go @@ -523,7 +523,7 @@ func (fs *diskFilesystem) RemoveAll() int64 { return 0 } -func (fs *diskFilesystem) List(path, pattern string) []FileInfo { +func (fs *diskFilesystem) List(path string, options ListOptions) []FileInfo { path = fs.cleanPath(path) files := []FileInfo{} @@ -541,8 +541,32 @@ func (fs *diskFilesystem) List(path, pattern string) []FileInfo { return } - if len(pattern) != 0 { - if ok, _ := glob.Match(pattern, name, '/'); !ok { + if len(options.Pattern) != 0 { + if ok, _ := glob.Match(options.Pattern, name, '/'); !ok { + return + } + } + + if options.ModifiedStart != nil { + if info.ModTime().Before(*options.ModifiedStart) { + return + } + } + + if options.ModifiedEnd != nil { + if info.ModTime().After(*options.ModifiedEnd) { + return + } + } + + if options.SizeMin > 0 { + if info.Size() < options.SizeMin { + return + } + } + + if options.SizeMax > 0 { + if info.Size() > options.SizeMax { return } } diff --git a/io/fs/fs.go b/io/fs/fs.go index d7102783..0f804eb2 100644 --- a/io/fs/fs.go +++ b/io/fs/fs.go @@ -42,6 +42,14 @@ type File interface { Stat() (FileInfo, error) } +type ListOptions struct { + Pattern string + ModifiedStart *time.Time + ModifiedEnd *time.Time + SizeMin int64 + SizeMax int64 +} + type ReadFilesystem interface { // Size returns the consumed size and capacity of the filesystem in bytes. The // capacity is zero or negative if the filesystem can consume as much space as it wants. @@ -65,7 +73,7 @@ type ReadFilesystem interface { Stat(path string) (FileInfo, error) // List lists all files that are currently on the filesystem. - List(path, pattern string) []FileInfo + List(path string, options ListOptions) []FileInfo // LookPath searches for an executable named file in the directories named by the PATH environment // variable. If file contains a slash, it is tried directly and the PATH is not consulted. Otherwise, diff --git a/io/fs/fs_test.go b/io/fs/fs_test.go index 9ef10080..5e178df4 100644 --- a/io/fs/fs_test.go +++ b/io/fs/fs_test.go @@ -93,6 +93,8 @@ func TestFilesystem(t *testing.T) { "replace": testReplace, "list": testList, "listGlob": testListGlob, + "listSize": testListSize, + "listModified": testListModified, "deleteAll": testDeleteAll, "data": testData, "statDir": testStatDir, @@ -322,12 +324,12 @@ func testList(t *testing.T, fs Filesystem) { return names } - files := fs.List("/", "") + files := fs.List("/", ListOptions{}) require.Equal(t, 6, len(files)) require.ElementsMatch(t, []string{"/foobar1", "/foobar2", "/foobar3", "/foobar4", "/path/foobar3", "/path/to/foobar4"}, getNames(files)) - files = fs.List("/path", "") + files = fs.List("/path", ListOptions{}) require.Equal(t, 2, len(files)) require.ElementsMatch(t, []string{"/path/foobar3", "/path/to/foobar4"}, getNames(files)) @@ -351,27 +353,114 @@ func testListGlob(t *testing.T, fs Filesystem) { return names } - files := getNames(fs.List("/", "/foo*")) + files := getNames(fs.List("/", ListOptions{Pattern: "/foo*"})) require.Equal(t, 2, len(files)) require.ElementsMatch(t, []string{"/foobar1", "/foobar4"}, files) - files = getNames(fs.List("/", "/*bar?")) + files = getNames(fs.List("/", ListOptions{Pattern: "/*bar?"})) require.Equal(t, 2, len(files)) require.ElementsMatch(t, []string{"/foobar1", "/foobar4"}, files) - files = getNames(fs.List("/", "/path/*")) + files = getNames(fs.List("/", ListOptions{Pattern: "/path/*"})) require.Equal(t, 1, len(files)) require.ElementsMatch(t, []string{"/path/foobar2"}, files) - files = getNames(fs.List("/", "/path/**")) + files = getNames(fs.List("/", ListOptions{Pattern: "/path/**"})) require.Equal(t, 2, len(files)) require.ElementsMatch(t, []string{"/path/foobar2", "/path/to/foobar3"}, files) - files = getNames(fs.List("/path", "/**")) + files = getNames(fs.List("/path", ListOptions{Pattern: "/**"})) require.Equal(t, 2, len(files)) require.ElementsMatch(t, []string{"/path/foobar2", "/path/to/foobar3"}, files) } +func testListSize(t *testing.T, fs Filesystem) { + fs.WriteFileReader("/a", strings.NewReader("a")) + fs.WriteFileReader("/aa", strings.NewReader("aa")) + fs.WriteFileReader("/aaa", strings.NewReader("aaa")) + fs.WriteFileReader("/aaaa", strings.NewReader("aaaa")) + + cur := fs.Files() + + require.Equal(t, int64(4), cur) + + getNames := func(files []FileInfo) []string { + names := []string{} + for _, f := range files { + names = append(names, f.Name()) + } + return names + } + + files := getNames(fs.List("/", ListOptions{SizeMin: 1})) + require.Equal(t, 4, len(files)) + require.ElementsMatch(t, []string{"/a", "/aa", "/aaa", "/aaaa"}, files) + + files = getNames(fs.List("/", ListOptions{SizeMin: 2})) + require.Equal(t, 3, len(files)) + require.ElementsMatch(t, []string{"/aa", "/aaa", "/aaaa"}, files) + + files = getNames(fs.List("/", ListOptions{SizeMax: 4})) + require.Equal(t, 4, len(files)) + require.ElementsMatch(t, []string{"/a", "/aa", "/aaa", "/aaaa"}, files) + + files = getNames(fs.List("/", ListOptions{SizeMin: 2, SizeMax: 3})) + require.Equal(t, 2, len(files)) + require.ElementsMatch(t, []string{"/aa", "/aaa"}, files) +} + +func testListModified(t *testing.T, fs Filesystem) { + fs.WriteFileReader("/a", strings.NewReader("a")) + time.Sleep(500 * time.Millisecond) + fs.WriteFileReader("/b", strings.NewReader("b")) + time.Sleep(500 * time.Millisecond) + fs.WriteFileReader("/c", strings.NewReader("c")) + time.Sleep(500 * time.Millisecond) + fs.WriteFileReader("/d", strings.NewReader("d")) + + cur := fs.Files() + + require.Equal(t, int64(4), cur) + + getNames := func(files []FileInfo) []string { + names := []string{} + for _, f := range files { + names = append(names, f.Name()) + } + return names + } + + var a, b, c, d time.Time + + for _, f := range fs.List("/", ListOptions{}) { + if f.Name() == "/a" { + a = f.ModTime() + } else if f.Name() == "/b" { + b = f.ModTime() + } else if f.Name() == "/c" { + c = f.ModTime() + } else if f.Name() == "/d" { + d = f.ModTime() + } + } + + files := getNames(fs.List("/", ListOptions{ModifiedStart: &a})) + require.Equal(t, 4, len(files)) + require.ElementsMatch(t, []string{"/a", "/b", "/c", "/d"}, files) + + files = getNames(fs.List("/", ListOptions{ModifiedStart: &b})) + require.Equal(t, 3, len(files)) + require.ElementsMatch(t, []string{"/b", "/c", "/d"}, files) + + files = getNames(fs.List("/", ListOptions{ModifiedEnd: &d})) + require.Equal(t, 4, len(files)) + require.ElementsMatch(t, []string{"/a", "/b", "/c", "/d"}, files) + + files = getNames(fs.List("/", ListOptions{ModifiedStart: &b, ModifiedEnd: &c})) + require.Equal(t, 2, len(files)) + require.ElementsMatch(t, []string{"/b", "/c"}, files) +} + func testDeleteAll(t *testing.T, fs Filesystem) { if _, ok := fs.(*diskFilesystem); ok { return diff --git a/io/fs/mem.go b/io/fs/mem.go index 56629d75..1ee852de 100644 --- a/io/fs/mem.go +++ b/io/fs/mem.go @@ -683,7 +683,7 @@ func (fs *memFilesystem) RemoveAll() int64 { return size } -func (fs *memFilesystem) List(path, pattern string) []FileInfo { +func (fs *memFilesystem) List(path string, options ListOptions) []FileInfo { path = fs.cleanPath(path) files := []FileInfo{} @@ -695,8 +695,32 @@ func (fs *memFilesystem) List(path, pattern string) []FileInfo { continue } - if len(pattern) != 0 { - if ok, _ := glob.Match(pattern, file.name, '/'); !ok { + if len(options.Pattern) != 0 { + if ok, _ := glob.Match(options.Pattern, file.name, '/'); !ok { + continue + } + } + + if options.ModifiedStart != nil { + if file.lastMod.Before(*options.ModifiedStart) { + continue + } + } + + if options.ModifiedEnd != nil { + if file.lastMod.After(*options.ModifiedEnd) { + continue + } + } + + if options.SizeMin > 0 { + if file.size < options.SizeMin { + continue + } + } + + if options.SizeMax > 0 { + if file.size > options.SizeMax { continue } } diff --git a/io/fs/mem_test.go b/io/fs/mem_test.go index d28a0d92..431d23b4 100644 --- a/io/fs/mem_test.go +++ b/io/fs/mem_test.go @@ -11,7 +11,7 @@ func TestMemFromDir(t *testing.T) { require.NoError(t, err) names := []string{} - for _, f := range mem.List("/", "/*.go") { + for _, f := range mem.List("/", ListOptions{Pattern: "/*.go"}) { names = append(names, f.Name()) } diff --git a/io/fs/s3.go b/io/fs/s3.go index c155d916..766695e0 100644 --- a/io/fs/s3.go +++ b/io/fs/s3.go @@ -140,7 +140,7 @@ func (fs *s3Filesystem) SetMetadata(key, data string) { func (fs *s3Filesystem) Size() (int64, int64) { size := int64(0) - files := fs.List("/", "") + files := fs.List("/", ListOptions{}) for _, file := range files { size += file.Size() @@ -463,7 +463,7 @@ func (fs *s3Filesystem) RemoveAll() int64 { return totalSize } -func (fs *s3Filesystem) List(path, pattern string) []FileInfo { +func (fs *s3Filesystem) List(path string, options ListOptions) []FileInfo { path = fs.cleanPath(path) ctx, cancel := context.WithCancel(context.Background()) @@ -493,8 +493,32 @@ func (fs *s3Filesystem) List(path, pattern string) []FileInfo { continue } - if len(pattern) != 0 { - if ok, _ := glob.Match(pattern, key, '/'); !ok { + if len(options.Pattern) != 0 { + if ok, _ := glob.Match(options.Pattern, key, '/'); !ok { + continue + } + } + + if options.ModifiedStart != nil { + if object.LastModified.Before(*options.ModifiedStart) { + continue + } + } + + if options.ModifiedEnd != nil { + if object.LastModified.After(*options.ModifiedEnd) { + continue + } + } + + if options.SizeMin > 0 { + if object.Size < options.SizeMin { + continue + } + } + + if options.SizeMax > 0 { + if object.Size > options.SizeMax { continue } } diff --git a/restream/fs/fs.go b/restream/fs/fs.go index 0e676da9..6ee7285b 100644 --- a/restream/fs/fs.go +++ b/restream/fs/fs.go @@ -135,7 +135,7 @@ func (rfs *filesystem) cleanup() { for _, patterns := range rfs.cleanupPatterns { for _, pattern := range patterns { - filesAndDirs := rfs.Filesystem.List("/", pattern.Pattern) + filesAndDirs := rfs.Filesystem.List("/", fs.ListOptions{Pattern: pattern.Pattern}) files := []fs.FileInfo{} for _, f := range filesAndDirs { @@ -175,7 +175,7 @@ func (rfs *filesystem) purge(patterns []Pattern) (nfiles uint64) { continue } - files := rfs.Filesystem.List("/", pattern.Pattern) + files := rfs.Filesystem.List("/", fs.ListOptions{Pattern: pattern.Pattern}) sort.Slice(files, func(i, j int) bool { return len(files[i].Name()) > len(files[j].Name()) }) for _, f := range files { rfs.logger.Debug().WithField("path", f.Name()).Log("Purging file") diff --git a/restream/fs/fs_test.go b/restream/fs/fs_test.go index 0162be1d..c95f38c9 100644 --- a/restream/fs/fs_test.go +++ b/restream/fs/fs_test.go @@ -43,7 +43,7 @@ func TestMaxFiles(t *testing.T) { names := []string{} - for _, f := range cleanfs.List("/", "/*.ts") { + for _, f := range cleanfs.List("/", fs.ListOptions{Pattern: "/*.ts"}) { names = append(names, f.Name()) } @@ -89,7 +89,7 @@ func TestMaxAge(t *testing.T) { names := []string{} - for _, f := range cleanfs.List("/", "/*.ts") { + for _, f := range cleanfs.List("/", fs.ListOptions{Pattern: "/*.ts"}) { names = append(names, f.Name()) } @@ -135,7 +135,7 @@ func TestUnsetCleanup(t *testing.T) { names := []string{} - for _, f := range cleanfs.List("/", "/*.ts") { + for _, f := range cleanfs.List("/", fs.ListOptions{Pattern: "/*.ts"}) { names = append(names, f.Name()) } @@ -155,7 +155,7 @@ func TestUnsetCleanup(t *testing.T) { names := []string{} - for _, f := range cleanfs.List("/", "/*.ts") { + for _, f := range cleanfs.List("/", fs.ListOptions{Pattern: "/*.ts"}) { names = append(names, f.Name()) }