neoris/core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'dev' into iam

Browse Source
This commit is contained in:
Ingo Oppermann 2023-04-03 15:44:16 +02:00
commit 84817f137a
No known key found for this signature in database
GPG Key ID: 2AB32426E9DD229E
10 changed files with 330 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
CHANGELOG.md
View File

@ -1,5 +1,12 @@
# Core
### Core v16.12.0 > v16.?.?
- Fix better naming for storage endpoint documentation
- Fix freeing up S3 mounts
- Fix URL validation if the path contains FFmpeg specific placeholders
- Fix purging default file from HTTP cache
### Core v16.11.0 > v16.12.0
- Add S3 storage support

3
app/api/api.go
View File

@ -1447,6 +1447,9 @@ func (a *api) stop() {
a.cache = nil
}
// Free the S3 mounts
a.s3fs = map[string]fs.Filesystem{}
// Stop the SRT server
if a.srtserver != nil {
a.log.logger.srt.Info().Log("Stopping ...")

33
docs/docs.go
View File

@ -240,6 +240,9 @@ const docTemplate = `{
"produces": [
"application/json"
],
"tags": [
"v16.12.0"
],
"summary": "List all registered filesystems",
"operationId": "filesystem-3-list",
"responses": {
@ -255,7 +258,7 @@ const docTemplate = `{
}
}
},
"/api/v3/fs/{name}": {
"/api/v3/fs/{storage}": {
"get": {
"security": [
{
@ -266,13 +269,16 @@ const docTemplate = `{
"produces": [
"application/json"
],
"tags": [
"v16.7.2"
],
"summary": "List all files on a filesystem",
"operationId": "filesystem-3-list-files",
"parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"name": "storage",
"in": "path",
"required": true
},
@ -308,7 +314,7 @@ const docTemplate = `{
}
}
},
"/api/v3/fs/{name}/{path}": {
"/api/v3/fs/{storage}/{filepath}": {
"get": {
"security": [
{
@ -320,20 +326,23 @@ const docTemplate = `{
"application/data",
"application/json"
],
"tags": [
"v16.7.2"
],
"summary": "Fetch a file from a filesystem",
"operationId": "filesystem-3-get-file",
"parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"name": "storage",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Path to file",
"name": "path",
"name": "filepath",
"in": "path",
"required": true
}
@ -373,20 +382,23 @@ const docTemplate = `{
"text/plain",
"application/json"
],
"tags": [
"v16.7.2"
],
"summary": "Add a file to a filesystem",
"operationId": "filesystem-3-put-file",
"parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"name": "storage",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Path to file",
"name": "path",
"name": "filepath",
"in": "path",
"required": true
},
@ -434,20 +446,23 @@ const docTemplate = `{
"produces": [
"text/plain"
],
"tags": [
"v16.7.2"
],
"summary": "Remove a file from a filesystem",
"operationId": "filesystem-3-delete-file",
"parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"name": "storage",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Path to file",
"name": "path",
"name": "filepath",
"in": "path",
"required": true
}

33
docs/swagger.json
View File

@ -233,6 +233,9 @@
"produces": [
"application/json"
],
"tags": [
"v16.12.0"
],
"summary": "List all registered filesystems",
"operationId": "filesystem-3-list",
"responses": {
@ -248,7 +251,7 @@
}
}
},
"/api/v3/fs/{name}": {
"/api/v3/fs/{storage}": {
"get": {
"security": [
{
@ -259,13 +262,16 @@
"produces": [
"application/json"
],
"tags": [
"v16.7.2"
],
"summary": "List all files on a filesystem",
"operationId": "filesystem-3-list-files",
"parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"name": "storage",
"in": "path",
"required": true
},
@ -301,7 +307,7 @@
}
}
},
"/api/v3/fs/{name}/{path}": {
"/api/v3/fs/{storage}/{filepath}": {
"get": {
"security": [
{
@ -313,20 +319,23 @@
"application/data",
"application/json"
],
"tags": [
"v16.7.2"
],
"summary": "Fetch a file from a filesystem",
"operationId": "filesystem-3-get-file",
"parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"name": "storage",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Path to file",
"name": "path",
"name": "filepath",
"in": "path",
"required": true
}
@ -366,20 +375,23 @@
"text/plain",
"application/json"
],
"tags": [
"v16.7.2"
],
"summary": "Add a file to a filesystem",
"operationId": "filesystem-3-put-file",
"parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"name": "storage",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Path to file",
"name": "path",
"name": "filepath",
"in": "path",
"required": true
},
@ -427,20 +439,23 @@
"produces": [
"text/plain"
],
"tags": [
"v16.7.2"
],
"summary": "Remove a file from a filesystem",
"operationId": "filesystem-3-delete-file",
"parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"name": "storage",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Path to file",
"name": "path",
"name": "filepath",
"in": "path",
"required": true
}

28
docs/swagger.yaml
View File

@ -2146,7 +2146,9 @@ paths:
security:
- ApiKeyAuth: []
summary: List all registered filesystems
/api/v3/fs/{name}:
tags:
- v16.12.0
/api/v3/fs/{storage}:
get:
description: List all files on a filesystem. The listing can be ordered by name,
size, or date of last modification in ascending or descending order.
@ -2154,7 +2156,7 @@ paths:
parameters:
- description: Name of the filesystem
in: path
name: name
name: storage
required: true
type: string
- description: glob pattern for file names
@ -2181,19 +2183,21 @@ paths:
security:
- ApiKeyAuth: []
summary: List all files on a filesystem
/api/v3/fs/{name}/{path}:
tags:
- v16.7.2
/api/v3/fs/{storage}/{filepath}:
delete:
description: Remove a file from a filesystem
operationId: filesystem-3-delete-file
parameters:
- description: Name of the filesystem
in: path
name: name
name: storage
required: true
type: string
- description: Path to file
in: path
name: path
name: filepath
required: true
type: string
produces:
@ -2210,18 +2214,20 @@ paths:
security:
- ApiKeyAuth: []
summary: Remove a file from a filesystem
tags:
- v16.7.2
get:
description: Fetch a file from a filesystem
operationId: filesystem-3-get-file
parameters:
- description: Name of the filesystem
in: path
name: name
name: storage
required: true
type: string
- description: Path to file
in: path
name: path
name: filepath
required: true
type: string
produces:
@ -2243,6 +2249,8 @@ paths:
security:
- ApiKeyAuth: []
summary: Fetch a file from a filesystem
tags:
- v16.7.2
put:
consumes:
- application/data
@ -2251,12 +2259,12 @@ paths:
parameters:
- description: Name of the filesystem
in: path
name: name
name: storage
required: true
type: string
- description: Path to file
in: path
name: path
name: filepath
required: true
type: string
- description: File data
@ -2286,6 +2294,8 @@ paths:
security:
- ApiKeyAuth: []
summary: Add a file to a filesystem
tags:
- v16.7.2
/api/v3/iam/group:
get:
description: List all groups

27
http/handler/api/filesystems.go
View File

@ -31,16 +31,17 @@ func NewFS(filesystems map[string]FSConfig) *FSHandler {
// GetFileAPI returns the file at the given path
// @Summary Fetch a file from a filesystem
// @Description Fetch a file from a filesystem
// @Tags v16.7.2
// @ID filesystem-3-get-file
// @Produce application/data
// @Produce json
// @Param name path string true "Name of the filesystem"
// @Param path path string true "Path to file"
// @Param storage path string true "Name of the filesystem"
// @Param filepath path string true "Path to file"
// @Success 200 {file} byte
// @Success 301 {string} string
// @Failure 404 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/{name}/{path} [get]
// @Router /api/v3/fs/{storage}/{filepath} [get]
func (h *FSHandler) GetFile(c echo.Context) error {
name := util.PathParam(c, "name")
@ -55,18 +56,19 @@ func (h *FSHandler) GetFile(c echo.Context) error {
// PutFileAPI adds or overwrites a file at the given path
// @Summary Add a file to a filesystem
// @Description Writes or overwrites a file on a filesystem
// @Tags v16.7.2
// @ID filesystem-3-put-file
// @Accept application/data
// @Produce text/plain
// @Produce json
// @Param name path string true "Name of the filesystem"
// @Param path path string true "Path to file"
// @Param storage path string true "Name of the filesystem"
// @Param filepath path string true "Path to file"
// @Param data body []byte true "File data"
// @Success 201 {string} string
// @Success 204 {string} string
// @Failure 507 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/{name}/{path} [put]
// @Router /api/v3/fs/{storage}/{filepath} [put]
func (h *FSHandler) PutFile(c echo.Context) error {
name := util.PathParam(c, "name")
@ -81,14 +83,15 @@ func (h *FSHandler) PutFile(c echo.Context) error {
// DeleteFileAPI removes a file from a filesystem
// @Summary Remove a file from a filesystem
// @Description Remove a file from a filesystem
// @Tags v16.7.2
// @ID filesystem-3-delete-file
// @Produce text/plain
// @Param name path string true "Name of the filesystem"
// @Param path path string true "Path to file"
// @Param storage path string true "Name of the filesystem"
// @Param filepath path string true "Path to file"
// @Success 200 {string} string
// @Failure 404 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/{name}/{path} [delete]
// @Router /api/v3/fs/{storage}/{filepath} [delete]
func (h *FSHandler) DeleteFile(c echo.Context) error {
name := util.PathParam(c, "name")
@ -103,15 +106,16 @@ func (h *FSHandler) DeleteFile(c echo.Context) error {
// ListFiles lists all files on a filesystem
// @Summary List all files on a filesystem
// @Description List all files on a filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.
// @Tags v16.7.2
// @ID filesystem-3-list-files
// @Produce json
// @Param name path string true "Name of the filesystem"
// @Param storage path string true "Name of the filesystem"
// @Param glob query string false "glob pattern for file names"
// @Param sort query string false "none, name, size, lastmod"
// @Param order query string false "asc, desc"
// @Success 200 {array} api.FileInfo
// @Security ApiKeyAuth
// @Router /api/v3/fs/{name} [get]
// @Router /api/v3/fs/{storage} [get]
func (h *FSHandler) ListFiles(c echo.Context) error {
name := util.PathParam(c, "name")
@ -126,6 +130,7 @@ func (h *FSHandler) ListFiles(c echo.Context) error {
// List lists all registered filesystems
// @Summary List all registered filesystems
// @Description Listall registered filesystems
// @Tags v16.12.0
// @ID filesystem-3-list
// @Produce json
// @Success 200 {array} api.FilesystemInfo

23
http/handler/filesystem.go
View File

@ -4,6 +4,7 @@ import (
"net/http"
"path/filepath"
"sort"
"strings"
"github.com/datarhei/core/v16/http/api"
"github.com/datarhei/core/v16/http/fs"
@ -89,6 +90,13 @@ func (h *FSHandler) PutFile(c echo.Context) error {
if h.fs.Cache != nil {
h.fs.Cache.Delete(path)
if len(h.fs.DefaultFile) != 0 {
if strings.HasSuffix(path, "/"+h.fs.DefaultFile) {
path := strings.TrimSuffix(path, h.fs.DefaultFile)
h.fs.Cache.Delete(path)
}
}
}
c.Response().Header().Set("Content-Location", req.URL.RequestURI())
@ -107,12 +115,19 @@ func (h *FSHandler) DeleteFile(c echo.Context) error {
size := h.fs.Filesystem.Remove(path)
if size < 0 {
return api.Err(http.StatusNotFound, "File not found", path)
}
if h.fs.Cache != nil {
h.fs.Cache.Delete(path)
if len(h.fs.DefaultFile) != 0 {
if strings.HasSuffix(path, "/"+h.fs.DefaultFile) {
path := strings.TrimSuffix(path, h.fs.DefaultFile)
h.fs.Cache.Delete(path)
}
}
}
if size < 0 {
return api.Err(http.StatusNotFound, "File not found", path)
}
return c.String(http.StatusOK, "Deleted: "+path)

98
net/url/url.go
View File

@ -4,23 +4,99 @@ import (
"net"
"net/url"
"regexp"
"strings"
)
var reScheme = regexp.MustCompile(`(?i)^([a-z][a-z0-9.+-:]*)://`)
type URL struct {
Scheme string
Opaque string // encoded opaque data
User *url.Userinfo // username and password information
Host string // host or host:port
RawPath string // path (relative paths may omit leading slash)
RawQuery string // encoded query values, without '?'
RawFragment string // fragment for references, without '#'
}
// Validate checks whether the given address is a valid URL
func (u *URL) Hostname() string {
if !strings.Contains(u.Host, ":") {
return u.Host
}
hostname, _, _ := net.SplitHostPort(u.Host)
return hostname
}
func (u *URL) Port() string {
if !strings.Contains(u.Host, ":") {
return ""
}
_, port, _ := net.SplitHostPort(u.Host)
return port
}
var reScheme = regexp.MustCompile(`(?i)^([a-z][a-z0-9.+-:]*):/{1,3}`)
// Validate checks whether the given address is a valid URL, based on the
// relaxed version of Parse in this package.
func Validate(address string) error {
_, err := Parse(address)
return err
}
// Parse parses an URL into its components. Returns a net/url.URL or
// an error if the URL couldn't be parsed.
func Parse(address string) (*url.URL, error) {
u, err := url.Parse(address)
// Parse parses an URL into its components. It is a more relaxed version of
// url.Parse as it's not checking the escaping of the path, query, and fragment.
func Parse(address string) (*URL, error) {
address, frag, _ := strings.Cut(address, "#")
return u, err
u := &URL{
RawFragment: frag,
}
matches := reScheme.FindStringSubmatch(address)
if matches != nil {
u.Scheme = matches[1]
address = strings.Replace(address, u.Scheme+":", "", 1)
}
address, query, _ := strings.Cut(address, "?")
u.RawQuery = query
if strings.HasPrefix(address, "///") {
u.RawPath = strings.TrimPrefix(address, "//")
return u, nil
}
if strings.HasPrefix(address, "//") {
host, path, _ := strings.Cut(address[2:], "/")
u.RawPath = "/" + path
parsedHost, err := url.Parse("//" + host)
if err != nil {
return nil, err
}
u.User = parsedHost.User
u.Host = parsedHost.Host
return u, nil
}
if strings.HasPrefix(address, "/") {
u.RawPath = address
return u, nil
}
scheme, address, _ := strings.Cut(address, ":")
u.Scheme = scheme
u.Opaque = address
return u, nil
}
// HasScheme returns whether the address has an URL scheme prefix
@ -46,15 +122,11 @@ func Lookup(address string) (string, error) {
return "", err
}
if len(u.Host) == 0 {
host := u.Hostname()
if len(host) == 0 {
return "", nil
}
host, _, err := net.SplitHostPort(u.Host)
if err != nil {
host = u.Host
}
addrs, err := net.LookupHost(host)
if err != nil {
return "", err

132
net/url/url_test.go
View File

@ -36,6 +36,12 @@ func TestValidate(t *testing.T) {
err = Validate("foobar")
require.NoError(t, err)
err = Validate("http://localhost/foobar_%25v")
require.NoError(t, err)
err = Validate("http://localhost/foobar_%v")
require.NoError(t, err)
}
func TestScheme(t *testing.T) {
@ -48,3 +54,129 @@ func TestScheme(t *testing.T) {
r = HasScheme("//localhost/foobar")
require.False(t, r)
}
func TestPars(t *testing.T) {
u, err := Parse("http://localhost/foobar")
require.NoError(t, err)
require.Equal(t, &URL{
Scheme: "http",
Opaque: "",
User: nil,
Host: "localhost",
RawPath: "/foobar",
RawQuery: "",
RawFragment: "",
}, u)
u, err = Parse("iueriherfd://localhost/foobar")
require.NoError(t, err)
require.Equal(t, &URL{
Scheme: "iueriherfd",
Opaque: "",
User: nil,
Host: "localhost",
RawPath: "/foobar",
RawQuery: "",
RawFragment: "",
}, u)
u, err = Parse("//localhost/foobar")
require.NoError(t, err)
require.Equal(t, &URL{
Scheme: "",
Opaque: "",
User: nil,
Host: "localhost",
RawPath: "/foobar",
RawQuery: "",
RawFragment: "",
}, u)
u, err = Parse("http://localhost/foobar_%v?foo=bar#foobar")
require.NoError(t, err)
require.Equal(t, &URL{
Scheme: "http",
Opaque: "",
User: nil,
Host: "localhost",
RawPath: "/foobar_%v",
RawQuery: "foo=bar",
RawFragment: "foobar",
}, u)
u, err = Parse("http:localhost/foobar_%v?foo=bar#foobar")
require.NoError(t, err)
require.Equal(t, &URL{
Scheme: "http",
Opaque: "localhost/foobar_%v",
User: nil,
Host: "",
RawPath: "",
RawQuery: "foo=bar",
RawFragment: "foobar",
}, u)
u, err = Parse("http:/localhost/foobar_%v?foo=bar#foobar")
require.NoError(t, err)
require.Equal(t, &URL{
Scheme: "http",
Opaque: "",
User: nil,
Host: "",
RawPath: "/localhost/foobar_%v",
RawQuery: "foo=bar",
RawFragment: "foobar",
}, u)
u, err = Parse("http:///localhost/foobar_%v?foo=bar#foobar")
require.NoError(t, err)
require.Equal(t, &URL{
Scheme: "http",
Opaque: "",
User: nil,
Host: "",
RawPath: "/localhost/foobar_%v",
RawQuery: "foo=bar",
RawFragment: "foobar",
}, u)
u, err = Parse("foo:bar://localhost/foobar_%v?foo=bar#foobar")
require.NoError(t, err)
require.Equal(t, &URL{
Scheme: "foo:bar",
Opaque: "",
User: nil,
Host: "localhost",
RawPath: "/foobar_%v",
RawQuery: "foo=bar",
RawFragment: "foobar",
}, u)
u, err = Parse("http://localhost:8080/foobar")
require.NoError(t, err)
require.Equal(t, &URL{
Scheme: "http",
Opaque: "",
User: nil,
Host: "localhost:8080",
RawPath: "/foobar",
RawQuery: "",
RawFragment: "",
}, u)
require.Equal(t, "localhost", u.Hostname())
require.Equal(t, "8080", u.Port())
u, err = Parse("https://www.google.com")
require.NoError(t, err)
require.Equal(t, &URL{
Scheme: "https",
Opaque: "",
User: nil,
Host: "www.google.com",
RawPath: "/",
RawQuery: "",
RawFragment: "",
}, u)
require.Equal(t, "www.google.com", u.Hostname())
require.Equal(t, "", u.Port())
}

2
restream/restream.go
View File

@ -947,7 +947,7 @@ func (r *restream) resolveAddress(tasks map[string]*task, id, address string) (s
}
if matches["source"] == "hls" {
if (u.Scheme == "http" || u.Scheme == "https") && strings.HasSuffix(u.Path, ".m3u8") {
if (u.Scheme == "http" || u.Scheme == "https") && strings.HasSuffix(u.RawPath, ".m3u8") {
return r.rewrite.RewriteAddress(a, identity, rewrite.READ), nil
}
} else if matches["source"] == "rtmp" {