diff --git a/CHANGELOG.md b/CHANGELOG.md index 21042f0a..bdd254d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Core +### Core v16.12.0 > v16.?.? + +- Fix better naming for storage endpoint documentation +- Fix freeing up S3 mounts +- Fix URL validation if the path contains FFmpeg specific placeholders +- Fix purging default file from HTTP cache + ### Core v16.11.0 > v16.12.0 - Add S3 storage support diff --git a/app/api/api.go b/app/api/api.go index 8d5d3c98..68a46961 100644 --- a/app/api/api.go +++ b/app/api/api.go @@ -1447,6 +1447,9 @@ func (a *api) stop() { a.cache = nil } + // Free the S3 mounts + a.s3fs = map[string]fs.Filesystem{} + // Stop the SRT server if a.srtserver != nil { a.log.logger.srt.Info().Log("Stopping ...") diff --git a/docs/docs.go b/docs/docs.go index e287b3b9..fab47d0f 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -240,6 +240,9 @@ const docTemplate = `{ "produces": [ "application/json" ], + "tags": [ + "v16.12.0" + ], "summary": "List all registered filesystems", "operationId": "filesystem-3-list", "responses": { @@ -255,7 +258,7 @@ const docTemplate = `{ } } }, - "/api/v3/fs/{name}": { + "/api/v3/fs/{storage}": { "get": { "security": [ { @@ -266,13 +269,16 @@ const docTemplate = `{ "produces": [ "application/json" ], + "tags": [ + "v16.7.2" + ], "summary": "List all files on a filesystem", "operationId": "filesystem-3-list-files", "parameters": [ { "type": "string", "description": "Name of the filesystem", - "name": "name", + "name": "storage", "in": "path", "required": true }, @@ -308,7 +314,7 @@ const docTemplate = `{ } } }, - "/api/v3/fs/{name}/{path}": { + "/api/v3/fs/{storage}/{filepath}": { "get": { "security": [ { @@ -320,20 +326,23 @@ const docTemplate = `{ "application/data", "application/json" ], + "tags": [ + "v16.7.2" + ], "summary": "Fetch a file from a filesystem", "operationId": "filesystem-3-get-file", "parameters": [ { "type": "string", "description": "Name of the filesystem", - "name": "name", + "name": "storage", "in": "path", "required": true }, { "type": "string", "description": "Path to file", - "name": "path", + "name": "filepath", "in": "path", "required": true } @@ -373,20 +382,23 @@ const docTemplate = `{ "text/plain", "application/json" ], + "tags": [ + "v16.7.2" + ], "summary": "Add a file to a filesystem", "operationId": "filesystem-3-put-file", "parameters": [ { "type": "string", "description": "Name of the filesystem", - "name": "name", + "name": "storage", "in": "path", "required": true }, { "type": "string", "description": "Path to file", - "name": "path", + "name": "filepath", "in": "path", "required": true }, @@ -434,20 +446,23 @@ const docTemplate = `{ "produces": [ "text/plain" ], + "tags": [ + "v16.7.2" + ], "summary": "Remove a file from a filesystem", "operationId": "filesystem-3-delete-file", "parameters": [ { "type": "string", "description": "Name of the filesystem", - "name": "name", + "name": "storage", "in": "path", "required": true }, { "type": "string", "description": "Path to file", - "name": "path", + "name": "filepath", "in": "path", "required": true } diff --git a/docs/swagger.json b/docs/swagger.json index f9ae935f..242e5ad9 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -233,6 +233,9 @@ "produces": [ "application/json" ], + "tags": [ + "v16.12.0" + ], "summary": "List all registered filesystems", "operationId": "filesystem-3-list", "responses": { @@ -248,7 +251,7 @@ } } }, - "/api/v3/fs/{name}": { + "/api/v3/fs/{storage}": { "get": { "security": [ { @@ -259,13 +262,16 @@ "produces": [ "application/json" ], + "tags": [ + "v16.7.2" + ], "summary": "List all files on a filesystem", "operationId": "filesystem-3-list-files", "parameters": [ { "type": "string", "description": "Name of the filesystem", - "name": "name", + "name": "storage", "in": "path", "required": true }, @@ -301,7 +307,7 @@ } } }, - "/api/v3/fs/{name}/{path}": { + "/api/v3/fs/{storage}/{filepath}": { "get": { "security": [ { @@ -313,20 +319,23 @@ "application/data", "application/json" ], + "tags": [ + "v16.7.2" + ], "summary": "Fetch a file from a filesystem", "operationId": "filesystem-3-get-file", "parameters": [ { "type": "string", "description": "Name of the filesystem", - "name": "name", + "name": "storage", "in": "path", "required": true }, { "type": "string", "description": "Path to file", - "name": "path", + "name": "filepath", "in": "path", "required": true } @@ -366,20 +375,23 @@ "text/plain", "application/json" ], + "tags": [ + "v16.7.2" + ], "summary": "Add a file to a filesystem", "operationId": "filesystem-3-put-file", "parameters": [ { "type": "string", "description": "Name of the filesystem", - "name": "name", + "name": "storage", "in": "path", "required": true }, { "type": "string", "description": "Path to file", - "name": "path", + "name": "filepath", "in": "path", "required": true }, @@ -427,20 +439,23 @@ "produces": [ "text/plain" ], + "tags": [ + "v16.7.2" + ], "summary": "Remove a file from a filesystem", "operationId": "filesystem-3-delete-file", "parameters": [ { "type": "string", "description": "Name of the filesystem", - "name": "name", + "name": "storage", "in": "path", "required": true }, { "type": "string", "description": "Path to file", - "name": "path", + "name": "filepath", "in": "path", "required": true } diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 44c487a3..f24e4b82 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -2146,7 +2146,9 @@ paths: security: - ApiKeyAuth: [] summary: List all registered filesystems - /api/v3/fs/{name}: + tags: + - v16.12.0 + /api/v3/fs/{storage}: get: description: List all files on a filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order. @@ -2154,7 +2156,7 @@ paths: parameters: - description: Name of the filesystem in: path - name: name + name: storage required: true type: string - description: glob pattern for file names @@ -2181,19 +2183,21 @@ paths: security: - ApiKeyAuth: [] summary: List all files on a filesystem - /api/v3/fs/{name}/{path}: + tags: + - v16.7.2 + /api/v3/fs/{storage}/{filepath}: delete: description: Remove a file from a filesystem operationId: filesystem-3-delete-file parameters: - description: Name of the filesystem in: path - name: name + name: storage required: true type: string - description: Path to file in: path - name: path + name: filepath required: true type: string produces: @@ -2210,18 +2214,20 @@ paths: security: - ApiKeyAuth: [] summary: Remove a file from a filesystem + tags: + - v16.7.2 get: description: Fetch a file from a filesystem operationId: filesystem-3-get-file parameters: - description: Name of the filesystem in: path - name: name + name: storage required: true type: string - description: Path to file in: path - name: path + name: filepath required: true type: string produces: @@ -2243,6 +2249,8 @@ paths: security: - ApiKeyAuth: [] summary: Fetch a file from a filesystem + tags: + - v16.7.2 put: consumes: - application/data @@ -2251,12 +2259,12 @@ paths: parameters: - description: Name of the filesystem in: path - name: name + name: storage required: true type: string - description: Path to file in: path - name: path + name: filepath required: true type: string - description: File data @@ -2286,6 +2294,8 @@ paths: security: - ApiKeyAuth: [] summary: Add a file to a filesystem + tags: + - v16.7.2 /api/v3/iam/group: get: description: List all groups diff --git a/http/handler/api/filesystems.go b/http/handler/api/filesystems.go index ce93812b..4b42a56c 100644 --- a/http/handler/api/filesystems.go +++ b/http/handler/api/filesystems.go @@ -31,16 +31,17 @@ func NewFS(filesystems map[string]FSConfig) *FSHandler { // GetFileAPI returns the file at the given path // @Summary Fetch a file from a filesystem // @Description Fetch a file from a filesystem +// @Tags v16.7.2 // @ID filesystem-3-get-file // @Produce application/data // @Produce json -// @Param name path string true "Name of the filesystem" -// @Param path path string true "Path to file" +// @Param storage path string true "Name of the filesystem" +// @Param filepath path string true "Path to file" // @Success 200 {file} byte // @Success 301 {string} string // @Failure 404 {object} api.Error // @Security ApiKeyAuth -// @Router /api/v3/fs/{name}/{path} [get] +// @Router /api/v3/fs/{storage}/{filepath} [get] func (h *FSHandler) GetFile(c echo.Context) error { name := util.PathParam(c, "name") @@ -55,18 +56,19 @@ func (h *FSHandler) GetFile(c echo.Context) error { // PutFileAPI adds or overwrites a file at the given path // @Summary Add a file to a filesystem // @Description Writes or overwrites a file on a filesystem +// @Tags v16.7.2 // @ID filesystem-3-put-file // @Accept application/data // @Produce text/plain // @Produce json -// @Param name path string true "Name of the filesystem" -// @Param path path string true "Path to file" +// @Param storage path string true "Name of the filesystem" +// @Param filepath path string true "Path to file" // @Param data body []byte true "File data" // @Success 201 {string} string // @Success 204 {string} string // @Failure 507 {object} api.Error // @Security ApiKeyAuth -// @Router /api/v3/fs/{name}/{path} [put] +// @Router /api/v3/fs/{storage}/{filepath} [put] func (h *FSHandler) PutFile(c echo.Context) error { name := util.PathParam(c, "name") @@ -81,14 +83,15 @@ func (h *FSHandler) PutFile(c echo.Context) error { // DeleteFileAPI removes a file from a filesystem // @Summary Remove a file from a filesystem // @Description Remove a file from a filesystem +// @Tags v16.7.2 // @ID filesystem-3-delete-file // @Produce text/plain -// @Param name path string true "Name of the filesystem" -// @Param path path string true "Path to file" +// @Param storage path string true "Name of the filesystem" +// @Param filepath path string true "Path to file" // @Success 200 {string} string // @Failure 404 {object} api.Error // @Security ApiKeyAuth -// @Router /api/v3/fs/{name}/{path} [delete] +// @Router /api/v3/fs/{storage}/{filepath} [delete] func (h *FSHandler) DeleteFile(c echo.Context) error { name := util.PathParam(c, "name") @@ -103,15 +106,16 @@ func (h *FSHandler) DeleteFile(c echo.Context) error { // ListFiles lists all files on a filesystem // @Summary List all files on a filesystem // @Description List all files on a filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order. +// @Tags v16.7.2 // @ID filesystem-3-list-files // @Produce json -// @Param name path string true "Name of the filesystem" +// @Param storage path string true "Name of the filesystem" // @Param glob query string false "glob pattern for file names" // @Param sort query string false "none, name, size, lastmod" // @Param order query string false "asc, desc" // @Success 200 {array} api.FileInfo // @Security ApiKeyAuth -// @Router /api/v3/fs/{name} [get] +// @Router /api/v3/fs/{storage} [get] func (h *FSHandler) ListFiles(c echo.Context) error { name := util.PathParam(c, "name") @@ -126,6 +130,7 @@ func (h *FSHandler) ListFiles(c echo.Context) error { // List lists all registered filesystems // @Summary List all registered filesystems // @Description Listall registered filesystems +// @Tags v16.12.0 // @ID filesystem-3-list // @Produce json // @Success 200 {array} api.FilesystemInfo diff --git a/http/handler/filesystem.go b/http/handler/filesystem.go index a8277e7c..9e00e37c 100644 --- a/http/handler/filesystem.go +++ b/http/handler/filesystem.go @@ -4,6 +4,7 @@ import ( "net/http" "path/filepath" "sort" + "strings" "github.com/datarhei/core/v16/http/api" "github.com/datarhei/core/v16/http/fs" @@ -89,6 +90,13 @@ func (h *FSHandler) PutFile(c echo.Context) error { if h.fs.Cache != nil { h.fs.Cache.Delete(path) + + if len(h.fs.DefaultFile) != 0 { + if strings.HasSuffix(path, "/"+h.fs.DefaultFile) { + path := strings.TrimSuffix(path, h.fs.DefaultFile) + h.fs.Cache.Delete(path) + } + } } c.Response().Header().Set("Content-Location", req.URL.RequestURI()) @@ -107,12 +115,19 @@ func (h *FSHandler) DeleteFile(c echo.Context) error { size := h.fs.Filesystem.Remove(path) - if size < 0 { - return api.Err(http.StatusNotFound, "File not found", path) - } - if h.fs.Cache != nil { h.fs.Cache.Delete(path) + + if len(h.fs.DefaultFile) != 0 { + if strings.HasSuffix(path, "/"+h.fs.DefaultFile) { + path := strings.TrimSuffix(path, h.fs.DefaultFile) + h.fs.Cache.Delete(path) + } + } + } + + if size < 0 { + return api.Err(http.StatusNotFound, "File not found", path) } return c.String(http.StatusOK, "Deleted: "+path) diff --git a/net/url/url.go b/net/url/url.go index 1d13e38c..1f89b240 100644 --- a/net/url/url.go +++ b/net/url/url.go @@ -4,23 +4,99 @@ import ( "net" "net/url" "regexp" + "strings" ) -var reScheme = regexp.MustCompile(`(?i)^([a-z][a-z0-9.+-:]*)://`) +type URL struct { + Scheme string + Opaque string // encoded opaque data + User *url.Userinfo // username and password information + Host string // host or host:port + RawPath string // path (relative paths may omit leading slash) + RawQuery string // encoded query values, without '?' + RawFragment string // fragment for references, without '#' +} -// Validate checks whether the given address is a valid URL +func (u *URL) Hostname() string { + if !strings.Contains(u.Host, ":") { + return u.Host + } + + hostname, _, _ := net.SplitHostPort(u.Host) + + return hostname +} + +func (u *URL) Port() string { + if !strings.Contains(u.Host, ":") { + return "" + } + + _, port, _ := net.SplitHostPort(u.Host) + + return port +} + +var reScheme = regexp.MustCompile(`(?i)^([a-z][a-z0-9.+-:]*):/{1,3}`) + +// Validate checks whether the given address is a valid URL, based on the +// relaxed version of Parse in this package. func Validate(address string) error { _, err := Parse(address) return err } -// Parse parses an URL into its components. Returns a net/url.URL or -// an error if the URL couldn't be parsed. -func Parse(address string) (*url.URL, error) { - u, err := url.Parse(address) +// Parse parses an URL into its components. It is a more relaxed version of +// url.Parse as it's not checking the escaping of the path, query, and fragment. +func Parse(address string) (*URL, error) { + address, frag, _ := strings.Cut(address, "#") - return u, err + u := &URL{ + RawFragment: frag, + } + + matches := reScheme.FindStringSubmatch(address) + if matches != nil { + u.Scheme = matches[1] + address = strings.Replace(address, u.Scheme+":", "", 1) + } + + address, query, _ := strings.Cut(address, "?") + u.RawQuery = query + + if strings.HasPrefix(address, "///") { + u.RawPath = strings.TrimPrefix(address, "//") + return u, nil + } + + if strings.HasPrefix(address, "//") { + host, path, _ := strings.Cut(address[2:], "/") + u.RawPath = "/" + path + + parsedHost, err := url.Parse("//" + host) + if err != nil { + return nil, err + } + + u.User = parsedHost.User + u.Host = parsedHost.Host + + return u, nil + } + + if strings.HasPrefix(address, "/") { + u.RawPath = address + + return u, nil + } + + scheme, address, _ := strings.Cut(address, ":") + + u.Scheme = scheme + u.Opaque = address + + return u, nil } // HasScheme returns whether the address has an URL scheme prefix @@ -46,15 +122,11 @@ func Lookup(address string) (string, error) { return "", err } - if len(u.Host) == 0 { + host := u.Hostname() + if len(host) == 0 { return "", nil } - host, _, err := net.SplitHostPort(u.Host) - if err != nil { - host = u.Host - } - addrs, err := net.LookupHost(host) if err != nil { return "", err diff --git a/net/url/url_test.go b/net/url/url_test.go index 460663e7..dff373b8 100644 --- a/net/url/url_test.go +++ b/net/url/url_test.go @@ -36,6 +36,12 @@ func TestValidate(t *testing.T) { err = Validate("foobar") require.NoError(t, err) + + err = Validate("http://localhost/foobar_%25v") + require.NoError(t, err) + + err = Validate("http://localhost/foobar_%v") + require.NoError(t, err) } func TestScheme(t *testing.T) { @@ -48,3 +54,129 @@ func TestScheme(t *testing.T) { r = HasScheme("//localhost/foobar") require.False(t, r) } + +func TestPars(t *testing.T) { + u, err := Parse("http://localhost/foobar") + require.NoError(t, err) + require.Equal(t, &URL{ + Scheme: "http", + Opaque: "", + User: nil, + Host: "localhost", + RawPath: "/foobar", + RawQuery: "", + RawFragment: "", + }, u) + + u, err = Parse("iueriherfd://localhost/foobar") + require.NoError(t, err) + require.Equal(t, &URL{ + Scheme: "iueriherfd", + Opaque: "", + User: nil, + Host: "localhost", + RawPath: "/foobar", + RawQuery: "", + RawFragment: "", + }, u) + + u, err = Parse("//localhost/foobar") + require.NoError(t, err) + require.Equal(t, &URL{ + Scheme: "", + Opaque: "", + User: nil, + Host: "localhost", + RawPath: "/foobar", + RawQuery: "", + RawFragment: "", + }, u) + + u, err = Parse("http://localhost/foobar_%v?foo=bar#foobar") + require.NoError(t, err) + require.Equal(t, &URL{ + Scheme: "http", + Opaque: "", + User: nil, + Host: "localhost", + RawPath: "/foobar_%v", + RawQuery: "foo=bar", + RawFragment: "foobar", + }, u) + + u, err = Parse("http:localhost/foobar_%v?foo=bar#foobar") + require.NoError(t, err) + require.Equal(t, &URL{ + Scheme: "http", + Opaque: "localhost/foobar_%v", + User: nil, + Host: "", + RawPath: "", + RawQuery: "foo=bar", + RawFragment: "foobar", + }, u) + + u, err = Parse("http:/localhost/foobar_%v?foo=bar#foobar") + require.NoError(t, err) + require.Equal(t, &URL{ + Scheme: "http", + Opaque: "", + User: nil, + Host: "", + RawPath: "/localhost/foobar_%v", + RawQuery: "foo=bar", + RawFragment: "foobar", + }, u) + + u, err = Parse("http:///localhost/foobar_%v?foo=bar#foobar") + require.NoError(t, err) + require.Equal(t, &URL{ + Scheme: "http", + Opaque: "", + User: nil, + Host: "", + RawPath: "/localhost/foobar_%v", + RawQuery: "foo=bar", + RawFragment: "foobar", + }, u) + + u, err = Parse("foo:bar://localhost/foobar_%v?foo=bar#foobar") + require.NoError(t, err) + require.Equal(t, &URL{ + Scheme: "foo:bar", + Opaque: "", + User: nil, + Host: "localhost", + RawPath: "/foobar_%v", + RawQuery: "foo=bar", + RawFragment: "foobar", + }, u) + + u, err = Parse("http://localhost:8080/foobar") + require.NoError(t, err) + require.Equal(t, &URL{ + Scheme: "http", + Opaque: "", + User: nil, + Host: "localhost:8080", + RawPath: "/foobar", + RawQuery: "", + RawFragment: "", + }, u) + require.Equal(t, "localhost", u.Hostname()) + require.Equal(t, "8080", u.Port()) + + u, err = Parse("https://www.google.com") + require.NoError(t, err) + require.Equal(t, &URL{ + Scheme: "https", + Opaque: "", + User: nil, + Host: "www.google.com", + RawPath: "/", + RawQuery: "", + RawFragment: "", + }, u) + require.Equal(t, "www.google.com", u.Hostname()) + require.Equal(t, "", u.Port()) +} diff --git a/restream/restream.go b/restream/restream.go index 68016bf8..edcbc927 100644 --- a/restream/restream.go +++ b/restream/restream.go @@ -947,7 +947,7 @@ func (r *restream) resolveAddress(tasks map[string]*task, id, address string) (s } if matches["source"] == "hls" { - if (u.Scheme == "http" || u.Scheme == "https") && strings.HasSuffix(u.Path, ".m3u8") { + if (u.Scheme == "http" || u.Scheme == "https") && strings.HasSuffix(u.RawPath, ".m3u8") { return r.rewrite.RewriteAddress(a, identity, rewrite.READ), nil } } else if matches["source"] == "rtmp" {