From 562b7aed922159ff5462aac81355533a5776b9b6 Mon Sep 17 00:00:00 2001 From: Ingo Oppermann Date: Fri, 17 Mar 2023 13:55:19 +0100 Subject: [PATCH 1/6] Use better naming for storage endpoint documentation --- docs/docs.go | 36 +++++++++++++++++++++++---------- docs/swagger.json | 33 +++++++++++++++++++++--------- docs/swagger.yaml | 28 ++++++++++++++++--------- http/handler/api/filesystems.go | 27 +++++++++++++++---------- 4 files changed, 84 insertions(+), 40 deletions(-) diff --git a/docs/docs.go b/docs/docs.go index 353c8b58..484bd9ca 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -1,5 +1,4 @@ -// Package docs GENERATED BY SWAG; DO NOT EDIT -// This file was generated by swaggo/swag +// Code generated by swaggo/swag. DO NOT EDIT package docs import "github.com/swaggo/swag" @@ -322,6 +321,9 @@ const docTemplate = `{ "produces": [ "application/json" ], + "tags": [ + "v16.12.0" + ], "summary": "List all registered filesystems", "operationId": "filesystem-3-list", "responses": { @@ -337,7 +339,7 @@ const docTemplate = `{ } } }, - "/api/v3/fs/{name}": { + "/api/v3/fs/{storage}": { "get": { "security": [ { @@ -348,13 +350,16 @@ const docTemplate = `{ "produces": [ "application/json" ], + "tags": [ + "v16.7.2" + ], "summary": "List all files on a filesystem", "operationId": "filesystem-3-list-files", "parameters": [ { "type": "string", "description": "Name of the filesystem", - "name": "name", + "name": "storage", "in": "path", "required": true }, @@ -390,7 +395,7 @@ const docTemplate = `{ } } }, - "/api/v3/fs/{name}/{path}": { + "/api/v3/fs/{storage}/{filepath}": { "get": { "security": [ { @@ -402,20 +407,23 @@ const docTemplate = `{ "application/data", "application/json" ], + "tags": [ + "v16.7.2" + ], "summary": "Fetch a file from a filesystem", "operationId": "filesystem-3-get-file", "parameters": [ { "type": "string", "description": "Name of the filesystem", - "name": "name", + "name": "storage", "in": "path", "required": true }, { "type": "string", "description": "Path to file", - "name": "path", + "name": "filepath", "in": "path", "required": true } @@ -455,20 +463,23 @@ const docTemplate = `{ "text/plain", "application/json" ], + "tags": [ + "v16.7.2" + ], "summary": "Add a file to a filesystem", "operationId": "filesystem-3-put-file", "parameters": [ { "type": "string", "description": "Name of the filesystem", - "name": "name", + "name": "storage", "in": "path", "required": true }, { "type": "string", "description": "Path to file", - "name": "path", + "name": "filepath", "in": "path", "required": true }, @@ -516,20 +527,23 @@ const docTemplate = `{ "produces": [ "text/plain" ], + "tags": [ + "v16.7.2" + ], "summary": "Remove a file from a filesystem", "operationId": "filesystem-3-delete-file", "parameters": [ { "type": "string", "description": "Name of the filesystem", - "name": "name", + "name": "storage", "in": "path", "required": true }, { "type": "string", "description": "Path to file", - "name": "path", + "name": "filepath", "in": "path", "required": true } diff --git a/docs/swagger.json b/docs/swagger.json index 75d15a44..a76838f6 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -314,6 +314,9 @@ "produces": [ "application/json" ], + "tags": [ + "v16.12.0" + ], "summary": "List all registered filesystems", "operationId": "filesystem-3-list", "responses": { @@ -329,7 +332,7 @@ } } }, - "/api/v3/fs/{name}": { + "/api/v3/fs/{storage}": { "get": { "security": [ { @@ -340,13 +343,16 @@ "produces": [ "application/json" ], + "tags": [ + "v16.7.2" + ], "summary": "List all files on a filesystem", "operationId": "filesystem-3-list-files", "parameters": [ { "type": "string", "description": "Name of the filesystem", - "name": "name", + "name": "storage", "in": "path", "required": true }, @@ -382,7 +388,7 @@ } } }, - "/api/v3/fs/{name}/{path}": { + "/api/v3/fs/{storage}/{filepath}": { "get": { "security": [ { @@ -394,20 +400,23 @@ "application/data", "application/json" ], + "tags": [ + "v16.7.2" + ], "summary": "Fetch a file from a filesystem", "operationId": "filesystem-3-get-file", "parameters": [ { "type": "string", "description": "Name of the filesystem", - "name": "name", + "name": "storage", "in": "path", "required": true }, { "type": "string", "description": "Path to file", - "name": "path", + "name": "filepath", "in": "path", "required": true } @@ -447,20 +456,23 @@ "text/plain", "application/json" ], + "tags": [ + "v16.7.2" + ], "summary": "Add a file to a filesystem", "operationId": "filesystem-3-put-file", "parameters": [ { "type": "string", "description": "Name of the filesystem", - "name": "name", + "name": "storage", "in": "path", "required": true }, { "type": "string", "description": "Path to file", - "name": "path", + "name": "filepath", "in": "path", "required": true }, @@ -508,20 +520,23 @@ "produces": [ "text/plain" ], + "tags": [ + "v16.7.2" + ], "summary": "Remove a file from a filesystem", "operationId": "filesystem-3-delete-file", "parameters": [ { "type": "string", "description": "Name of the filesystem", - "name": "name", + "name": "storage", "in": "path", "required": true }, { "type": "string", "description": "Path to file", - "name": "path", + "name": "filepath", "in": "path", "required": true } diff --git a/docs/swagger.yaml b/docs/swagger.yaml index d735b7c8..43e9479f 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -2126,7 +2126,9 @@ paths: security: - ApiKeyAuth: [] summary: List all registered filesystems - /api/v3/fs/{name}: + tags: + - v16.12.0 + /api/v3/fs/{storage}: get: description: List all files on a filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order. @@ -2134,7 +2136,7 @@ paths: parameters: - description: Name of the filesystem in: path - name: name + name: storage required: true type: string - description: glob pattern for file names @@ -2161,19 +2163,21 @@ paths: security: - ApiKeyAuth: [] summary: List all files on a filesystem - /api/v3/fs/{name}/{path}: + tags: + - v16.7.2 + /api/v3/fs/{storage}/{filepath}: delete: description: Remove a file from a filesystem operationId: filesystem-3-delete-file parameters: - description: Name of the filesystem in: path - name: name + name: storage required: true type: string - description: Path to file in: path - name: path + name: filepath required: true type: string produces: @@ -2190,18 +2194,20 @@ paths: security: - ApiKeyAuth: [] summary: Remove a file from a filesystem + tags: + - v16.7.2 get: description: Fetch a file from a filesystem operationId: filesystem-3-get-file parameters: - description: Name of the filesystem in: path - name: name + name: storage required: true type: string - description: Path to file in: path - name: path + name: filepath required: true type: string produces: @@ -2223,6 +2229,8 @@ paths: security: - ApiKeyAuth: [] summary: Fetch a file from a filesystem + tags: + - v16.7.2 put: consumes: - application/data @@ -2231,12 +2239,12 @@ paths: parameters: - description: Name of the filesystem in: path - name: name + name: storage required: true type: string - description: Path to file in: path - name: path + name: filepath required: true type: string - description: File data @@ -2266,6 +2274,8 @@ paths: security: - ApiKeyAuth: [] summary: Add a file to a filesystem + tags: + - v16.7.2 /api/v3/log: get: description: Get the last log lines of the Restreamer application diff --git a/http/handler/api/filesystems.go b/http/handler/api/filesystems.go index ce93812b..4b42a56c 100644 --- a/http/handler/api/filesystems.go +++ b/http/handler/api/filesystems.go @@ -31,16 +31,17 @@ func NewFS(filesystems map[string]FSConfig) *FSHandler { // GetFileAPI returns the file at the given path // @Summary Fetch a file from a filesystem // @Description Fetch a file from a filesystem +// @Tags v16.7.2 // @ID filesystem-3-get-file // @Produce application/data // @Produce json -// @Param name path string true "Name of the filesystem" -// @Param path path string true "Path to file" +// @Param storage path string true "Name of the filesystem" +// @Param filepath path string true "Path to file" // @Success 200 {file} byte // @Success 301 {string} string // @Failure 404 {object} api.Error // @Security ApiKeyAuth -// @Router /api/v3/fs/{name}/{path} [get] +// @Router /api/v3/fs/{storage}/{filepath} [get] func (h *FSHandler) GetFile(c echo.Context) error { name := util.PathParam(c, "name") @@ -55,18 +56,19 @@ func (h *FSHandler) GetFile(c echo.Context) error { // PutFileAPI adds or overwrites a file at the given path // @Summary Add a file to a filesystem // @Description Writes or overwrites a file on a filesystem +// @Tags v16.7.2 // @ID filesystem-3-put-file // @Accept application/data // @Produce text/plain // @Produce json -// @Param name path string true "Name of the filesystem" -// @Param path path string true "Path to file" +// @Param storage path string true "Name of the filesystem" +// @Param filepath path string true "Path to file" // @Param data body []byte true "File data" // @Success 201 {string} string // @Success 204 {string} string // @Failure 507 {object} api.Error // @Security ApiKeyAuth -// @Router /api/v3/fs/{name}/{path} [put] +// @Router /api/v3/fs/{storage}/{filepath} [put] func (h *FSHandler) PutFile(c echo.Context) error { name := util.PathParam(c, "name") @@ -81,14 +83,15 @@ func (h *FSHandler) PutFile(c echo.Context) error { // DeleteFileAPI removes a file from a filesystem // @Summary Remove a file from a filesystem // @Description Remove a file from a filesystem +// @Tags v16.7.2 // @ID filesystem-3-delete-file // @Produce text/plain -// @Param name path string true "Name of the filesystem" -// @Param path path string true "Path to file" +// @Param storage path string true "Name of the filesystem" +// @Param filepath path string true "Path to file" // @Success 200 {string} string // @Failure 404 {object} api.Error // @Security ApiKeyAuth -// @Router /api/v3/fs/{name}/{path} [delete] +// @Router /api/v3/fs/{storage}/{filepath} [delete] func (h *FSHandler) DeleteFile(c echo.Context) error { name := util.PathParam(c, "name") @@ -103,15 +106,16 @@ func (h *FSHandler) DeleteFile(c echo.Context) error { // ListFiles lists all files on a filesystem // @Summary List all files on a filesystem // @Description List all files on a filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order. +// @Tags v16.7.2 // @ID filesystem-3-list-files // @Produce json -// @Param name path string true "Name of the filesystem" +// @Param storage path string true "Name of the filesystem" // @Param glob query string false "glob pattern for file names" // @Param sort query string false "none, name, size, lastmod" // @Param order query string false "asc, desc" // @Success 200 {array} api.FileInfo // @Security ApiKeyAuth -// @Router /api/v3/fs/{name} [get] +// @Router /api/v3/fs/{storage} [get] func (h *FSHandler) ListFiles(c echo.Context) error { name := util.PathParam(c, "name") @@ -126,6 +130,7 @@ func (h *FSHandler) ListFiles(c echo.Context) error { // List lists all registered filesystems // @Summary List all registered filesystems // @Description Listall registered filesystems +// @Tags v16.12.0 // @ID filesystem-3-list // @Produce json // @Success 200 {array} api.FilesystemInfo From 3149572a648ab6ba499076e469bff38bfc5d217d Mon Sep 17 00:00:00 2001 From: Ingo Oppermann Date: Fri, 17 Mar 2023 18:40:20 +0100 Subject: [PATCH 2/6] Fix freeing up S3 mounts --- app/api/api.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/api/api.go b/app/api/api.go index f82846f5..695b7f40 100644 --- a/app/api/api.go +++ b/app/api/api.go @@ -1342,6 +1342,9 @@ func (a *api) stop() { a.cache = nil } + // Free the S3 mounts + a.s3fs = map[string]fs.Filesystem{} + // Stop the SRT server if a.srtserver != nil { a.log.logger.srt.Info().Log("Stopping ...") From de207b02a1dfd04333e449bcc52a413ae7d8ece9 Mon Sep 17 00:00:00 2001 From: Ingo Oppermann Date: Wed, 22 Mar 2023 15:49:52 +0100 Subject: [PATCH 3/6] Fix URL validation if the path contains FFmpeg specific placeholders --- net/url/url.go | 98 +++++++++++++++++++++++++++----- net/url/url_test.go | 132 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 216 insertions(+), 14 deletions(-) diff --git a/net/url/url.go b/net/url/url.go index d99b9ebd..1f89b240 100644 --- a/net/url/url.go +++ b/net/url/url.go @@ -4,25 +4,99 @@ import ( "net" "net/url" "regexp" + "strings" ) -var reScheme = regexp.MustCompile(`(?i)^([a-z][a-z0-9.+-:]*)://`) +type URL struct { + Scheme string + Opaque string // encoded opaque data + User *url.Userinfo // username and password information + Host string // host or host:port + RawPath string // path (relative paths may omit leading slash) + RawQuery string // encoded query values, without '?' + RawFragment string // fragment for references, without '#' +} -// Validate checks whether the given address is a valid URL +func (u *URL) Hostname() string { + if !strings.Contains(u.Host, ":") { + return u.Host + } + + hostname, _, _ := net.SplitHostPort(u.Host) + + return hostname +} + +func (u *URL) Port() string { + if !strings.Contains(u.Host, ":") { + return "" + } + + _, port, _ := net.SplitHostPort(u.Host) + + return port +} + +var reScheme = regexp.MustCompile(`(?i)^([a-z][a-z0-9.+-:]*):/{1,3}`) + +// Validate checks whether the given address is a valid URL, based on the +// relaxed version of Parse in this package. func Validate(address string) error { _, err := Parse(address) return err } -// Parse parses an URL into its components. Returns a net/url.URL or -// an error if the URL couldn't be parsed. -func Parse(address string) (*url.URL, error) { - address = reScheme.ReplaceAllString(address, "//") +// Parse parses an URL into its components. It is a more relaxed version of +// url.Parse as it's not checking the escaping of the path, query, and fragment. +func Parse(address string) (*URL, error) { + address, frag, _ := strings.Cut(address, "#") - u, err := url.Parse(address) + u := &URL{ + RawFragment: frag, + } - return u, err + matches := reScheme.FindStringSubmatch(address) + if matches != nil { + u.Scheme = matches[1] + address = strings.Replace(address, u.Scheme+":", "", 1) + } + + address, query, _ := strings.Cut(address, "?") + u.RawQuery = query + + if strings.HasPrefix(address, "///") { + u.RawPath = strings.TrimPrefix(address, "//") + return u, nil + } + + if strings.HasPrefix(address, "//") { + host, path, _ := strings.Cut(address[2:], "/") + u.RawPath = "/" + path + + parsedHost, err := url.Parse("//" + host) + if err != nil { + return nil, err + } + + u.User = parsedHost.User + u.Host = parsedHost.Host + + return u, nil + } + + if strings.HasPrefix(address, "/") { + u.RawPath = address + + return u, nil + } + + scheme, address, _ := strings.Cut(address, ":") + + u.Scheme = scheme + u.Opaque = address + + return u, nil } // HasScheme returns whether the address has an URL scheme prefix @@ -48,15 +122,11 @@ func Lookup(address string) (string, error) { return "", err } - if len(u.Host) == 0 { + host := u.Hostname() + if len(host) == 0 { return "", nil } - host, _, err := net.SplitHostPort(u.Host) - if err != nil { - host = u.Host - } - addrs, err := net.LookupHost(host) if err != nil { return "", err diff --git a/net/url/url_test.go b/net/url/url_test.go index 460663e7..dff373b8 100644 --- a/net/url/url_test.go +++ b/net/url/url_test.go @@ -36,6 +36,12 @@ func TestValidate(t *testing.T) { err = Validate("foobar") require.NoError(t, err) + + err = Validate("http://localhost/foobar_%25v") + require.NoError(t, err) + + err = Validate("http://localhost/foobar_%v") + require.NoError(t, err) } func TestScheme(t *testing.T) { @@ -48,3 +54,129 @@ func TestScheme(t *testing.T) { r = HasScheme("//localhost/foobar") require.False(t, r) } + +func TestPars(t *testing.T) { + u, err := Parse("http://localhost/foobar") + require.NoError(t, err) + require.Equal(t, &URL{ + Scheme: "http", + Opaque: "", + User: nil, + Host: "localhost", + RawPath: "/foobar", + RawQuery: "", + RawFragment: "", + }, u) + + u, err = Parse("iueriherfd://localhost/foobar") + require.NoError(t, err) + require.Equal(t, &URL{ + Scheme: "iueriherfd", + Opaque: "", + User: nil, + Host: "localhost", + RawPath: "/foobar", + RawQuery: "", + RawFragment: "", + }, u) + + u, err = Parse("//localhost/foobar") + require.NoError(t, err) + require.Equal(t, &URL{ + Scheme: "", + Opaque: "", + User: nil, + Host: "localhost", + RawPath: "/foobar", + RawQuery: "", + RawFragment: "", + }, u) + + u, err = Parse("http://localhost/foobar_%v?foo=bar#foobar") + require.NoError(t, err) + require.Equal(t, &URL{ + Scheme: "http", + Opaque: "", + User: nil, + Host: "localhost", + RawPath: "/foobar_%v", + RawQuery: "foo=bar", + RawFragment: "foobar", + }, u) + + u, err = Parse("http:localhost/foobar_%v?foo=bar#foobar") + require.NoError(t, err) + require.Equal(t, &URL{ + Scheme: "http", + Opaque: "localhost/foobar_%v", + User: nil, + Host: "", + RawPath: "", + RawQuery: "foo=bar", + RawFragment: "foobar", + }, u) + + u, err = Parse("http:/localhost/foobar_%v?foo=bar#foobar") + require.NoError(t, err) + require.Equal(t, &URL{ + Scheme: "http", + Opaque: "", + User: nil, + Host: "", + RawPath: "/localhost/foobar_%v", + RawQuery: "foo=bar", + RawFragment: "foobar", + }, u) + + u, err = Parse("http:///localhost/foobar_%v?foo=bar#foobar") + require.NoError(t, err) + require.Equal(t, &URL{ + Scheme: "http", + Opaque: "", + User: nil, + Host: "", + RawPath: "/localhost/foobar_%v", + RawQuery: "foo=bar", + RawFragment: "foobar", + }, u) + + u, err = Parse("foo:bar://localhost/foobar_%v?foo=bar#foobar") + require.NoError(t, err) + require.Equal(t, &URL{ + Scheme: "foo:bar", + Opaque: "", + User: nil, + Host: "localhost", + RawPath: "/foobar_%v", + RawQuery: "foo=bar", + RawFragment: "foobar", + }, u) + + u, err = Parse("http://localhost:8080/foobar") + require.NoError(t, err) + require.Equal(t, &URL{ + Scheme: "http", + Opaque: "", + User: nil, + Host: "localhost:8080", + RawPath: "/foobar", + RawQuery: "", + RawFragment: "", + }, u) + require.Equal(t, "localhost", u.Hostname()) + require.Equal(t, "8080", u.Port()) + + u, err = Parse("https://www.google.com") + require.NoError(t, err) + require.Equal(t, &URL{ + Scheme: "https", + Opaque: "", + User: nil, + Host: "www.google.com", + RawPath: "/", + RawQuery: "", + RawFragment: "", + }, u) + require.Equal(t, "www.google.com", u.Hostname()) + require.Equal(t, "", u.Port()) +} From 52df872198ced4070fdef6c587e2fe25438827c2 Mon Sep 17 00:00:00 2001 From: Ingo Oppermann Date: Wed, 22 Mar 2023 15:52:06 +0100 Subject: [PATCH 4/6] Update changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21042f0a..c7f4285d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Core +### Core v16.12.0 > v16.?.? + +- Fix better naming for storage endpoint documentation +- Fix freeing up S3 mounts +- Fix URL validation if the path contains FFmpeg specific placeholders + ### Core v16.11.0 > v16.12.0 - Add S3 storage support From 48678fb4c60a3beeeede03191333809029dac5e6 Mon Sep 17 00:00:00 2001 From: Ingo Oppermann Date: Thu, 23 Mar 2023 11:17:13 +0100 Subject: [PATCH 5/6] Fix purging default file from HTTP cache --- CHANGELOG.md | 1 + http/handler/filesystem.go | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7f4285d..bdd254d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - Fix better naming for storage endpoint documentation - Fix freeing up S3 mounts - Fix URL validation if the path contains FFmpeg specific placeholders +- Fix purging default file from HTTP cache ### Core v16.11.0 > v16.12.0 diff --git a/http/handler/filesystem.go b/http/handler/filesystem.go index a8277e7c..54435238 100644 --- a/http/handler/filesystem.go +++ b/http/handler/filesystem.go @@ -4,6 +4,7 @@ import ( "net/http" "path/filepath" "sort" + "strings" "github.com/datarhei/core/v16/http/api" "github.com/datarhei/core/v16/http/fs" @@ -107,12 +108,19 @@ func (h *FSHandler) DeleteFile(c echo.Context) error { size := h.fs.Filesystem.Remove(path) - if size < 0 { - return api.Err(http.StatusNotFound, "File not found", path) - } - if h.fs.Cache != nil { h.fs.Cache.Delete(path) + + if len(h.fs.DefaultFile) != 0 { + if strings.HasSuffix(path, "/"+h.fs.DefaultFile) { + path := strings.TrimSuffix(path, h.fs.DefaultFile) + h.fs.Cache.Delete(path) + } + } + } + + if size < 0 { + return api.Err(http.StatusNotFound, "File not found", path) } return c.String(http.StatusOK, "Deleted: "+path) From 6eefa5ca2b2dfc4a6d55974691a464eafafedc22 Mon Sep 17 00:00:00 2001 From: Ingo Oppermann Date: Mon, 3 Apr 2023 10:27:04 +0200 Subject: [PATCH 6/6] Fix purging default file from HTTP cache --- http/handler/filesystem.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/http/handler/filesystem.go b/http/handler/filesystem.go index 54435238..9e00e37c 100644 --- a/http/handler/filesystem.go +++ b/http/handler/filesystem.go @@ -90,6 +90,13 @@ func (h *FSHandler) PutFile(c echo.Context) error { if h.fs.Cache != nil { h.fs.Cache.Delete(path) + + if len(h.fs.DefaultFile) != 0 { + if strings.HasSuffix(path, "/"+h.fs.DefaultFile) { + path := strings.TrimSuffix(path, h.fs.DefaultFile) + h.fs.Cache.Delete(path) + } + } } c.Response().Header().Set("Content-Location", req.URL.RequestURI())