neoris/core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix using prefix syntax for filesystem operations

Browse Source
This commit is contained in:
Ingo Oppermann 2023-03-03 16:18:46 +01:00
parent a3ff16ef30
commit 175cfc2324
No known key found for this signature in database
GPG Key ID: 2AB32426E9DD229E
5 changed files with 114 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
config/value/s3.go
View File

@ -3,6 +3,7 @@ package value
import (
"fmt"
"net/url"
"regexp"
"strings"
"golang.org/x/net/publicsuffix"
@ -12,19 +13,21 @@ import (
// https://access_key_id:secret_access_id@region.endpoint/bucket?name=aaa&mount=/abc&username=xxx&password=yyy
type S3Storage struct {
Name string `json:"name"`
Mountpoint string `json:"mountpoint"`
Auth struct {
Enable bool `json:"enable"`
Username string `json:"username"`
Password string `json:"password"`
} `json:"auth"`
Endpoint string `json:"endpoint"`
AccessKeyID string `json:"access_key_id"`
SecretAccessKey string `json:"secret_access_key"`
Bucket string `json:"bucket"`
Region string `json:"region"`
UseSSL bool `json:"use_ssl"`
Name string `json:"name"`
Mountpoint string `json:"mountpoint"`
Auth S3StorageAuth `json:"auth"`
Endpoint string `json:"endpoint"`
AccessKeyID string `json:"access_key_id"`
SecretAccessKey string `json:"secret_access_key"`
Bucket string `json:"bucket"`
Region string `json:"region"`
UseSSL bool `json:"use_ssl"`
}
type S3StorageAuth struct {
Enable bool `json:"enable"`
Username string `json:"username"`
Password string `json:"password"`
}
func (t *S3Storage) String() string {
@ -50,7 +53,7 @@ func (t *S3Storage) String() string {
v := url.Values{}
v.Set("name", t.Name)
v.Set("mountpoint", t.Mountpoint)
v.Set("mount", t.Mountpoint)
if t.Auth.Enable {
if len(t.Auth.Username) != 0 {
@ -70,12 +73,14 @@ func (t *S3Storage) String() string {
type s3StorageListValue struct {
p *[]S3Storage
separator string
reName *regexp.Regexp
}
func NewS3StorageListValue(p *[]S3Storage, val []S3Storage, separator string) *s3StorageListValue {
v := &s3StorageListValue{
p: p,
separator: separator,
reName: regexp.MustCompile(`^[A-Za-z0-9_-]+$`),
}
*p = val
@ -93,10 +98,13 @@ func (s *s3StorageListValue) Set(val string) error {
t := S3Storage{
Name: u.Query().Get("name"),
Mountpoint: u.Query().Get("mountpoint"),
Mountpoint: u.Query().Get("mount"),
AccessKeyID: u.User.Username(),
}
password, _ := u.User.Password()
t.SecretAccessKey = password
hostname := u.Hostname()
port := u.Port()
@ -129,7 +137,7 @@ func (s *s3StorageListValue) Set(val string) error {
if u.Query().Has("username") || u.Query().Has("password") {
t.Auth.Enable = true
t.Auth.Username = u.Query().Get("username")
t.Auth.Username = u.Query().Get("password")
t.Auth.Password = u.Query().Get("password")
}
list = append(list, t)
@ -160,6 +168,10 @@ func (s *s3StorageListValue) Validate() error {
return fmt.Errorf("the name for s3 storage %d is missing", i)
}
if !s.reName.MatchString(t.Name) {
return fmt.Errorf("the name for s3 storage must match the pattern %s", s.reName.String())
}
if len(t.Mountpoint) == 0 {
return fmt.Errorf("the mountpoint for s3 storage %d is missing", i)
}

53
config/value/s3_test.go Normal file
View File

@ -0,0 +1,53 @@
package value
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestS3Value(t *testing.T) {
filesystems := []S3Storage{}
v := NewS3StorageListValue(&filesystems, nil, " ")
require.Equal(t, "(empty)", v.String())
v.Set("https://access_key_id1:secret_access_id1@region1.endpoint1.com/bucket1?name=aaa1&mount=/abc1&username=xxx1&password=yyy1 http://access_key_id2:secret_access_id2@region2.endpoint2.com/bucket2?name=aaa2&mount=/abc2&username=xxx2&password=yyy2")
require.Equal(t, []S3Storage{
{
Name: "aaa1",
Mountpoint: "/abc1",
Auth: S3StorageAuth{
Enable: true,
Username: "xxx1",
Password: "yyy1",
},
Endpoint: "endpoint1.com",
AccessKeyID: "access_key_id1",
SecretAccessKey: "secret_access_id1",
Bucket: "bucket1",
Region: "region1",
UseSSL: true,
},
{
Name: "aaa2",
Mountpoint: "/abc2",
Auth: S3StorageAuth{
Enable: true,
Username: "xxx2",
Password: "yyy2",
},
Endpoint: "endpoint2.com",
AccessKeyID: "access_key_id2",
SecretAccessKey: "secret_access_id2",
Bucket: "bucket2",
Region: "region2",
UseSSL: false,
},
}, filesystems)
require.Equal(t, "https://access_key_id1:---@region1.endpoint1.com/bucket1?mount=%2Fabc1&name=aaa1&password=---&username=xxx1 http://access_key_id2:---@region2.endpoint2.com/bucket2?mount=%2Fabc2&name=aaa2&password=---&username=xxx2", v.String())
require.NoError(t, v.Validate())
v.Set("https://access_key_id1:secret_access_id1@region1.endpoint1.com/bucket1?name=djk*;..&mount=/abc1&username=xxx1&password=yyy1")
require.Error(t, v.Validate())
}

27
http/handler/api/filesystems.go
View File

@ -2,8 +2,7 @@ package api
import (
"net/http"
"path/filepath"
"strings"
"regexp"
"github.com/datarhei/core/v16/http/api"
"github.com/datarhei/core/v16/http/handler"
@ -176,12 +175,15 @@ func (h *FSHandler) FileOperation(c echo.Context) error {
return api.Err(http.StatusBadRequest, "Invalid operation", "%s", operation.Operation)
}
from := strings.Split(filepath.Join("/", operation.From), "/")
if len(from) < 2 {
return api.Err(http.StatusBadRequest, "Invalid source path", "%s", operation.From)
rePrefix := regexp.MustCompile(`^(.+):`)
matches := rePrefix.FindStringSubmatch(operation.From)
if matches == nil {
return api.Err(http.StatusBadRequest, "Missing source filesystem prefix")
}
fromFSName := from[1]
fromPath := strings.Join(from[2:], "/")
fromFSName := matches[1]
fromPath := rePrefix.ReplaceAllString(operation.From, "")
fromFS, ok := h.filesystems[fromFSName]
if !ok {
return api.Err(http.StatusBadRequest, "Source filesystem not found", "%s", fromFSName)
@ -191,12 +193,13 @@ func (h *FSHandler) FileOperation(c echo.Context) error {
return c.JSON(http.StatusOK, "OK")
}
to := strings.Split(filepath.Join("/", operation.To), "/")
if len(to) < 2 {
return api.Err(http.StatusBadRequest, "Invalid target path", "%s", operation.To)
matches = rePrefix.FindStringSubmatch(operation.To)
if matches == nil {
return api.Err(http.StatusBadRequest, "Missing target filesystem prefix")
}
toFSName := to[1]
toPath := strings.Join(to[2:], "/")
toFSName := matches[1]
toPath := rePrefix.ReplaceAllString(operation.To, "")
toFS, ok := h.filesystems[toFSName]
if !ok {
return api.Err(http.StatusBadRequest, "Target filesystem not found", "%s", toFSName)

34
http/handler/api/filesystems_test.go
View File

@ -193,7 +193,7 @@ func TestFileOperation(t *testing.T) {
op = api.FilesystemOperation{
Operation: "copy",
From: "foo/elif",
From: "foo:/elif",
}
jsondata, err = json.Marshal(op)
@ -203,18 +203,7 @@ func TestFileOperation(t *testing.T) {
op = api.FilesystemOperation{
Operation: "copy",
From: "foo/elif",
To: "/bar",
}
jsondata, err = json.Marshal(op)
require.NoError(t, err)
mock.Request(t, http.StatusNotFound, router, "PUT", "/", bytes.NewReader(jsondata))
op = api.FilesystemOperation{
Operation: "copy",
From: "foo/file",
From: "foo:/elif",
To: "/bar",
}
@ -225,8 +214,19 @@ func TestFileOperation(t *testing.T) {
op = api.FilesystemOperation{
Operation: "copy",
From: "foo/file",
To: "/bar/file",
From: "foo:/file",
To: "/bar",
}
jsondata, err = json.Marshal(op)
require.NoError(t, err)
mock.Request(t, http.StatusBadRequest, router, "PUT", "/", bytes.NewReader(jsondata))
op = api.FilesystemOperation{
Operation: "copy",
From: "foo:file",
To: "bar:/file",
}
jsondata, err = json.Marshal(op)
@ -244,8 +244,8 @@ func TestFileOperation(t *testing.T) {
op = api.FilesystemOperation{
Operation: "move",
From: "foo/file",
To: "/bar/file",
From: "foo:file",
To: "bar:/file",
}
jsondata, err = json.Marshal(op)

2
restream/restream.go
View File

@ -541,7 +541,7 @@ func (r *restream) onArgs(cfg *app.Config) func([]string) []string {
}
func (r *restream) setCleanup(id string, config *app.Config) {
rePrefix := regexp.MustCompile(`^([a-z]+):`)
rePrefix := regexp.MustCompile(`^(.+):`)
for _, output := range config.Output {
for _, c := range output.Cleanup {